From 55f7b42411886153328ea24f4644b18b37cf5e74 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:04:14 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m1/ffi-speed/codspeed.md Quest-Claim: e63c358e5262464b From 501b81cd3cfec9055f3e7056bce07ad168e9efe3 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:50:46 -0800 Subject: [PATCH 2/8] feat(bench): the binding benchmark on CodSpeed's CPU simulation Each program calls instrument-hooks through moq/bench/codspeed.h around each single-threaded case, and a workflow runs them under CodSpeed on main and on PRs that change what just ffi bench reads. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/codspeed.yml | 76 ++++++++++++++++++ bench/ffi/README.md | 33 +++++++- ffi/justfile | 92 +++++++++++++++++----- ffi/moq/bench/codspeed.c | 101 ++++++++++++++++++++++++ ffi/moq/bench/codspeed.h | 28 +++++++ ffi/moq/bench/export.c | 104 +++++++++++++++--------- ffi/moq/bench/export.cpp | 115 ++++++++++++++++----------- ffi/moq/bench/export.dart | 103 ++++++++++++++++++------ ffi/moq/bench/export.go | 139 ++++++++++++++++++++------------- ffi/moq/bench/export.kt | 125 +++++++++++++++++++---------- ffi/moq/bench/export.py | 99 +++++++++++++++-------- ffi/moq/bench/export.rs | 94 +++++++++++++++------- ffi/moq/bench/export.swift | 97 ++++++++++++++--------- ffi/moq/bench/moq-c.c | 14 +++- flake.lock | 18 +++++ flake.nix | 9 +++ quest/m1/ffi-speed/README.md | 8 +- quest/m2/wasm/backend.md | 4 +- 18 files changed, 932 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 ffi/moq/bench/codspeed.c create mode 100644 ffi/moq/bench/codspeed.h diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..1f2c39b19 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,76 @@ +name: CodSpeed + +# The binding benchmark under CodSpeed's CPU simulation (bench/ffi/README.md): each push to +# main is its baseline, and a PR that changes what `just ffi bench` reads gets CodSpeed's +# comment. Informational: no threshold fails it, and no ruleset requires it. + +permissions: + contents: read + +on: + push: + branches: [main] + paths: + - ffi/** + - bench/ffi/** + - flake.nix + - flake.lock + - .github/workflows/codspeed.yml + - .github/actions/nix/** + # What ffi/ takes from ts/: the moq pin ts/justfile reads, and the nightly that + # ffi/rust-toolchain.toml links to. + - ts/justfile + - ts/fixtures/model/Cargo.toml + - ts/rust-toolchain.toml + - "!**/*.md" + pull_request: + paths: + - ffi/** + - bench/ffi/** + - flake.nix + - flake.lock + - .github/workflows/codspeed.yml + - .github/actions/nix/** + - ts/justfile + - ts/fixtures/model/Cargo.toml + - ts/rust-toolchain.toml + - "!**/*.md" + workflow_dispatch: + +# A PR's newer push supersedes its run; each of main's runs is a baseline, so none is cancelled. +concurrency: + group: codspeed-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + codspeed: + name: CodSpeed + # Hosted, which costs a public repository nothing, as does CodSpeed. + runs-on: ubuntu-24.04 + timeout-minutes: 180 + permissions: + contents: read + # CodSpeed's upload authenticates through GitHub's OIDC token. + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/nix + + - name: Build + run: nix develop .#bench --command just ffi bench-build + + # The bench shell's environment as a script the run sources, so Nix itself never runs + # under the simulation's Valgrind, which traces every process the run starts. + - name: Environment + run: nix print-dev-env .#bench > "$RUNNER_TEMP/bench.sh" + + - name: Run + uses: CodSpeedHQ/action@af75f77f17a054769abefaa42fe8dc2975469b5f # v5.6.3 + with: + mode: simulation + run: . "$RUNNER_TEMP/bench.sh" && just ffi codspeed diff --git a/bench/ffi/README.md b/bench/ffi/README.md index 906eed7e3..273d4358f 100644 --- a/bench/ffi/README.md +++ b/bench/ffi/README.md @@ -19,7 +19,8 @@ from an emptied target and without a compiler cache, the export (`rust2-ffi`, wh runs `cargo check` of moq under its driver) beside moq's own `cargo check` of the same crates, and the generated crate's release build beside moq's own release build, each with its peak RSS, through [`measure.py`](measure.py). Nightly runs it on arm64 Linux into -the run's summary; nothing fails on a number. +the run's summary; nothing fails on a number. [CodSpeed](#codspeed) counts each +single-threaded case on every push to main and on each PR that can move one. | case | what it times | BoltFFI's nearest | |---|---|---| @@ -73,6 +74,36 @@ must not be ranked against the managed `loop` row. C, C++, and Rust's existing ` cases already manually poll in one loop; they remain the native floor, without printing the same measurement twice under another name. +## CodSpeed + +[CodSpeed](https://app.codspeed.io/kixel-ai/rust2) keeps main's history of each +single-threaded case, and comments on each PR with its change per case. +[`codspeed.yml`](../../.github/workflows/codspeed.yml) runs on every push to main, its +baseline, and on each PR that changes what `just ffi bench` reads: it builds the programs in +the bench shell, then runs `just ffi codspeed` under CodSpeed's CPU simulation in that +shell's environment, so Nix itself never runs under Valgrind. It is informational: no +threshold fails it and no ruleset requires it. + +```bash +nix develop .#bench --command just ffi bench-build +nix develop .#bench --command codspeed run --mode simulation -- just ffi codspeed # CodSpeed's CLI, on Ubuntu or Debian +``` + +Each program calls CodSpeed's +[instrument-hooks](https://github.com/CodSpeedHQ/instrument-hooks), which the flake pins, +through [`codspeed.h`](../../ffi/moq/bench/codspeed.h) and its own FFI around each case: +one benchmark per language and case, such as `kotlin::write` or `c::throughput[1024]`. +Under CodSpeed a program runs its round once, without the warm-up round, and skips the +`shared:*`, `own:*`, and Kotlin's `objects` cases: the simulation runs one thread at a +time, so it would count a contention that never happens. It counts instructions and +simulated cache misses, not time, so a system call or a wait costs nothing in it: Swift's +and Go's woken reads and reader loops count their threads' work, not their wake. The +tables below keep wall and CPU time. + +A language joins only if its numbers hold still: the same commit, run twice, must give +each of its cases the same count. `codspeed_langs` in [`ffi/justfile`](../../ffi/justfile) +lists those that do. + ## Concurrent calls Measured 2026-11-13 on **nixos**, an AMD Ryzen 9 5900XT (16 cores in two 8-core dies, 32 diff --git a/ffi/justfile b/ffi/justfile index 8e31a847a..952e4232b 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -17,6 +17,12 @@ moq_rev := "ffcb7cd9748422daa30e935ec464108b5a8ebb71" # swift-collections 1.7.5, whose `UniqueArray` the generated Swift imports. swift_collections_rev := "f96b909e1a9e469620bc5037c1e24e00153f376f" +# The binding benchmark's programs, each a language, as bench-program runs them. +bench_langs := "rust c cpp moq-c kotlin python swift go dart" + +# Those whose cases CodSpeed counts the same on every run of one commit, as bench/ffi/README.md's CodSpeed section says. +codspeed_langs := "rust c cpp moq-c kotlin python swift go dart" + default: @just --list @@ -62,13 +68,14 @@ census checkout="": (standin checkout) bench checkout="": (bench-build checkout) bench-run # Build the binding benchmark's programs: Rust's floor, C, C++, Kotlin, Python, Swift, Go, and Dart through the export, and C through moq's hand-written rs/moq-c. -bench-build checkout="": (moq-export checkout) +bench-build checkout="": (moq-export checkout) codspeed-hooks #!/usr/bin/env bash set -euo pipefail [ -n "${LIBCLANG_PATH:-}" ] || { echo "LIBCLANG_PATH is unset: run this in the bench shell, nix develop .#bench" >&2; exit 1; } src="{{ checkout }}" [ -n "$src" ] || src=target/moq out=target/export/moq + hooks=$PWD/target/codspeed # The exporter bench-run times, built as a consumer installs it. cargo build --locked --quiet --release # The floor: the stand-in's API called from Rust, resolved as the export is. @@ -76,22 +83,47 @@ bench-build checkout="": (moq-export checkout) mkdir -p target/bench-rust printf '[package]\nname = "bench"\nversion = "0.0.0"\nedition = "2024"\npublish = false\n\n[[bin]]\nname = "bench"\npath = "%s"\n\n[dependencies]\nlibc = "0.2"\nmoq = { package = "moq-ffi", path = "../moq-ffi" }\n\n[workspace]\n' "$PWD/moq/bench/export.rs" > target/bench-rust/Cargo.toml cp target/moq-ffi/Cargo.lock target/bench-rust/Cargo.lock - cargo build --quiet --release --manifest-path target/bench-rust/Cargo.toml + # The hooks' search path goes to the program alone, so moq builds as bench-run's export does. + cargo rustc --quiet --release --manifest-path target/bench-rust/Cargo.toml -- -L "native=$hooks" # moq-c's header is written to its build script's output directory. moqc=$(cd "$src" && cargo build --locked --quiet --release --package moq-c --message-format=json \ | jq -r 'select(.reason == "build-script-executed" and (.package_id | contains("moq-c#"))) | .out_dir') - cc -O2 -std=c11 -Wall -Wextra -Werror -I "$out/include" moq/bench/export.c "$out/target/release/libmoq.a" -lpthread -ldl -lm -o target/bench-export - cc -O2 -std=c11 -Wall -Wextra -Werror -I "$moqc/include" moq/bench/moq-c.c "$src/target/release/libmoq.a" -lpthread -ldl -lm -o target/bench-moq-c - c++ -O2 -std=c++17 -Wall -Wextra -Werror -I "$out/include" moq/bench/export.cpp "$out/target/release/libmoq.a" -lpthread -ldl -lm -o target/bench-export-cpp + cc -O2 -std=c11 -Wall -Wextra -Werror -I "$out/include" -I "$hooks" moq/bench/export.c "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export + cc -O2 -std=c11 -Wall -Wextra -Werror -I "$moqc/include" -I "$hooks" moq/bench/moq-c.c "$src/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-moq-c + c++ -O2 -std=c++17 -Wall -Wextra -Werror -I "$out/include" -I "$hooks" moq/bench/export.cpp "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export-cpp {{ quote(just_executable()) }} go-module bench-go moq/bench/export.go read -ra native < target/moq-native - export CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a ${native[*]}" GOFLAGS=-buildvcs=false + export CGO_CFLAGS="-I$hooks" CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a $hooks/libcodspeed.a ${native[*]}" GOFLAGS=-buildvcs=false (cd target/bench-go && go build -o bench .) {{ quote(just_executable()) }} dart-program bench-dart moq/bench/export.dart rm -rf target/bench-kt kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/bench/export.kt -d target/bench-kt {{ quote(just_executable()) }} swift-module -O - swiftc -O -warnings-as-errors $({{ quote(just_executable()) }} swift-target 15) -I target/swift -I target/swift-collections/build -I "$out/swift" moq/bench/export.swift target/swift/libMoq.a -L target/swift-collections/build -lBasicContainers -lInternalCollectionsUtilities -lSpanPreview "$out/target/release/libmoq.a" -lpthread -ldl -lm -o target/bench-export-swift + swiftc -O -warnings-as-errors $({{ quote(just_executable()) }} swift-target 15) -I target/swift -I target/swift-collections/build -I "$out/swift" -I "$hooks" moq/bench/export.swift target/swift/libMoq.a -L target/swift-collections/build -lBasicContainers -lInternalCollectionsUtilities -lSpanPreview "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export-swift + +# Build moq/bench/codspeed.c over the bench shell's instrument-hooks into target/codspeed: a static library the compiled programs link, a shared one with the JNI functions that Kotlin, Python, and Dart load, and a module map Swift imports it by. +[private] +codspeed-hooks: + #!/usr/bin/env bash + set -euo pipefail + [ -n "${CODSPEED_INSTRUMENT_HOOKS:-}" ] || { echo "CODSPEED_INSTRUMENT_HOOKS is unset: run this in the bench shell, nix develop .#bench" >&2; exit 1; } + src=$CODSPEED_INSTRUMENT_HOOKS + dir=target/codspeed + rm -rf "$dir" + mkdir -p "$dir" + case "{{ os() }}" in + macos) lib=libcodspeed.dylib jni=darwin ;; + *) lib=libcodspeed.so jni=linux ;; + esac + # instrument-hooks' generated C warns, harmlessly, its guide says, and puts C23's + # attributes where C23 refuses them. + cc -O2 -fPIC -std=gnu11 -w -I "$src/includes" -c "$src/dist/core.c" -o "$dir/core.o" + flags=(-O2 -fPIC -std=c11 -Wall -Wextra -Werror -isystem "$src/includes") + cc "${flags[@]}" -c moq/bench/codspeed.c -o "$dir/codspeed.o" + ar rcs "$dir/libcodspeed.a" "$dir/codspeed.o" "$dir/core.o" + cc "${flags[@]}" -shared -DCODSPEED_JNI -isystem "${JAVA_HOME:?}/include" -isystem "$JAVA_HOME/include/$jni" moq/bench/codspeed.c "$dir/core.o" -o "$dir/$lib" + cp moq/bench/codspeed.h "$dir/" + printf 'module CodSpeed {\n header "codspeed.h"\n export *\n}\n' > "$dir/module.modulemap" # Run what bench-build built: each program `rounds` times into target/bench/raw.tsv, and the export and its crate's build, beside moq's, each round into target/bench/build.tsv, with the 1-minute load average over each run, and their medians and the export's sizes into target/bench/report.md. bench-run rounds="3": @@ -113,9 +145,8 @@ bench-run rounds="3": } run() { local lang=$1 before after - shift before=$(load) - "$@" > target/bench/run.tsv + {{ quote(just_executable()) }} bench-program "$lang" > target/bench/run.tsv after=$(load) awk -v lang="$lang" -v round="$r" -v loadavg="$(echo "$before $after" | awk '{ print ($1 + $2) / 2 }')" -F '\t' \ 'NF == 3 { print lang "\t" round "\t" $1 "\t" $2 "\t" $3 "\t" loadavg; next } { print "bench: " lang ": unexpected line: " $0 > "/dev/stderr"; exit 1 }' \ @@ -150,19 +181,44 @@ bench-run rounds="3": } for r in $(seq {{ rounds }}); do build - run rust target/bench-rust/target/release/bench - run c target/bench-export - run cpp target/bench-export-cpp - run moq-c target/bench-moq-c - run kotlin kotlin -Djava.library.path="$out/target/release" -cp "target/bench-kt:$KOTLINX_COROUTINES" ExportKt - PYTHONPATH="$out/python" run python python3 -P moq/bench/export.py - run swift target/bench-export-swift - run go target/bench-go/bench - run dart dart target/bench-dart/main.dill + for lang in {{ bench_langs }}; do + run "$lang" + done done python3 -I ../bench/ffi/report.py "$raw" "$builds" "$out" > target/bench/report.md cat target/bench/report.md +# Run what bench-build built once each, under `codspeed run` in the bench shell's environment, so CodSpeed's CPU simulation counts each single-threaded case: the CodSpeed workflow's run. +codspeed: + #!/usr/bin/env bash + set -euo pipefail + for lang in {{ codspeed_langs }}; do + {{ quote(just_executable()) }} bench-program "$lang" + done + +# Run what bench-build built for one of bench_langs. +[private] +bench-program lang: + #!/usr/bin/env bash + set -euo pipefail + out=target/export/moq + case "{{ os() }}" in + macos) hooks=$PWD/target/codspeed/libcodspeed.dylib ;; + *) hooks=$PWD/target/codspeed/libcodspeed.so ;; + esac + case "{{ lang }}" in + rust) exec target/bench-rust/target/release/bench ;; + c) exec target/bench-export ;; + cpp) exec target/bench-export-cpp ;; + moq-c) exec target/bench-moq-c ;; + kotlin) exec kotlin -Djava.library.path="$out/target/release:$PWD/target/codspeed" -cp "target/bench-kt:$KOTLINX_COROUTINES" ExportKt ;; + python) PYTHONPATH="$out/python" exec python3 -P moq/bench/export.py "$hooks" ;; + swift) exec target/bench-export-swift ;; + go) exec target/bench-go/bench ;; + dart) exec dart target/bench-dart/main.dill "$hooks" ;; + *) echo "bench-program: no language {{ lang }}" >&2; exit 1 ;; + esac + # Export moq's crates' C ABI, build it, and run moq/moq.c, moq/moq.cpp, moq/moq.kt, moq/moq.py, moq/moq.swift, moq/moq.go, and moq/moq.dart against it, from the pinned moq revision or a checkout. moq checkout="": (moq-cpp checkout) #!/usr/bin/env bash diff --git a/ffi/moq/bench/codspeed.c b/ffi/moq/bench/codspeed.c new file mode 100644 index 000000000..d3d11d64e --- /dev/null +++ b/ffi/moq/bench/codspeed.c @@ -0,0 +1,101 @@ +/* codspeed.h over CodSpeed's instrument-hooks. Built with CODSPEED_JNI, it also holds the + * JNI functions export.kt's `Codspeed` object declares. */ + +#define _POSIX_C_SOURCE 200112L + +#include "codspeed.h" + +#include +#include +#include + +#include "core.h" + +static InstrumentHooks *hooks; +static bool measured; +static char prefix[256]; + +static void fail(const char *what) { + fprintf(stderr, "codspeed: %s failed\n", what); + exit(1); +} + +bool codspeed_init(const char *bench) { + hooks = instrument_hooks_init(); + if (hooks == NULL) { + fail("instrument_hooks_init"); + } + if ((size_t)snprintf(prefix, sizeof prefix, "%s", bench) >= sizeof prefix) { + fail("a bench name this long"); + } + measured = instrument_hooks_is_instrumented(hooks); + if (!measured) { + return false; + } + /* A case is one run, which only the simulation counts exactly; walltime would time one. */ + if (!RUNNING_ON_VALGRIND) { + fail("a CodSpeed instrument but its CPU simulation"); + } + if (instrument_hooks_set_integration(hooks, "rust2-ffi-bench", "0.1.0") != 0) { + fail("instrument_hooks_set_integration"); + } + return true; +} + +void codspeed_start(void) { + if (measured && instrument_hooks_start_benchmark(hooks) != 0) { + fail("instrument_hooks_start_benchmark"); + } +} + +void codspeed_stop(const char *name) { + if (!measured) { + return; + } + if (instrument_hooks_stop_benchmark(hooks) != 0) { + fail("instrument_hooks_stop_benchmark"); + } + char uri[512]; + if ((size_t)snprintf(uri, sizeof uri, "%s::%s", prefix, name) >= sizeof uri) { + fail("a case name this long"); + } + if (instrument_hooks_set_executed_benchmark(hooks, (int32_t)getpid(), uri) != 0) { + fail("instrument_hooks_set_executed_benchmark"); + } +} + +#ifdef CODSPEED_JNI + +#include + +/* A Java string's modified UTF-8, which is plain UTF-8 for the ASCII names here. */ +static const char *chars(JNIEnv *env, jstring s) { + const char *c = (*env)->GetStringUTFChars(env, s, NULL); + if (c == NULL) { + fail("GetStringUTFChars"); + } + return c; +} + +JNIEXPORT jboolean JNICALL Java_Codspeed_init(JNIEnv *env, jobject self, jstring bench) { + (void)self; + const char *c = chars(env, bench); + bool on = codspeed_init(c); + (*env)->ReleaseStringUTFChars(env, bench, c); + return on ? JNI_TRUE : JNI_FALSE; +} + +JNIEXPORT void JNICALL Java_Codspeed_start(JNIEnv *env, jobject self) { + (void)env; + (void)self; + codspeed_start(); +} + +JNIEXPORT void JNICALL Java_Codspeed_stop(JNIEnv *env, jobject self, jstring name) { + (void)self; + const char *c = chars(env, name); + codspeed_stop(c); + (*env)->ReleaseStringUTFChars(env, name, c); +} + +#endif diff --git a/ffi/moq/bench/codspeed.h b/ffi/moq/bench/codspeed.h new file mode 100644 index 000000000..c3ad33f5d --- /dev/null +++ b/ffi/moq/bench/codspeed.h @@ -0,0 +1,28 @@ +/* The binding benchmark's hooks into CodSpeed's CPU simulation, through its instrument-hooks + * library: each program calls them through its own FFI around each case it times, so + * CodSpeed counts that case alone. Outside CodSpeed they do nothing. */ + +#ifndef CODSPEED_H +#define CODSPEED_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Whether CodSpeed measures this process, naming its cases `::`, such as + * `ffi/moq/bench/export.kt::kotlin`: the file, then the language. Call it once, first. */ +bool codspeed_init(const char *bench); + +/* Starts counting a case. */ +void codspeed_start(void); + +/* Stops counting, and names what was counted since codspeed_start. */ +void codspeed_stop(const char *name); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ffi/moq/bench/export.c b/ffi/moq/bench/export.c index fcd361ebb..81b8cab3e 100644 --- a/ffi/moq/bench/export.c +++ b/ffi/moq/bench/export.c @@ -1,7 +1,8 @@ /* Every case bench/ffi/README.md lists, through the exported C ABI: the floor every other * language's binding builds on. bench/export.rs times the same in Rust, and * bench/export.cpp through the C++ header. Futures are polled by hand with a null waker. - * Each case prints its wall time and the process's CPU time. */ + * Each case prints its wall time and the process's CPU time, or under CodSpeed, runs once + * for its CPU simulation to count. */ #define _POSIX_C_SOURCE 200112L @@ -13,6 +14,7 @@ #include #include +#include "codspeed.h" #include "moq.h" #define GROUPS 10000 @@ -43,6 +45,22 @@ static span per(span start, double n) { return (span){(end.wall - start.wall) / n, (end.cpu - start.cpu) / n}; } +/* Whether CodSpeed counts this run, which then skips the warm-up and the threaded cases. */ +static bool measured; + +/* A case's start: now, and CodSpeed counting from here. */ +static span begin(void) { + span start = now(); + codspeed_start(); + return start; +} + +/* The case `name`'s nanoseconds per call since `start`, over `n` calls. */ +static span end(span start, double n, const char *name) { + codspeed_stop(name); + return per(start, n); +} + static moq_buf str(const char *s) { return (moq_buf){.ptr = (const uint8_t *)s, .len = strlen(s)}; } @@ -110,7 +128,9 @@ static span throughput(moq_track_Producer *track, moq_track_Subscriber *subscrib groups = groups < 3 ? 3 : groups > 2000 ? 2000 : groups; uint8_t *payload = calloc(size, 1); moq_buf frame = {.ptr = payload, .len = size}; - span start = now(); + char name[32]; + snprintf(name, sizeof name, "throughput[%zu]", size); + span start = begin(); for (size_t g = 0; g < groups; g++) { moq_group_Producer *group = append_group(track); moq_group_Consumer *reader = recv_group(subscriber); @@ -125,7 +145,7 @@ static span throughput(moq_track_Producer *track, moq_track_Subscriber *subscrib finish(group); moq_group_Consumer_free(reader); } - span ns = per(start, (double)(groups * FRAMES * size)); + span ns = end(start, (double)(groups * FRAMES * size), name); free(payload); return (span){1e3 / ns.wall, 1e3 / ns.cpu}; } @@ -209,55 +229,59 @@ static void round_(int print) { unsigned sum = 0; span start; - start = now(); + start = begin(); for (int i = 0; i < N; i++) { sum += moq_track_Producer_is_closed(track); } - span call = per(start, N); + span call = end(start, N, "call"); moq_group_Producer *demanded = append_group(track); moq_group_Demand *demand = moq_group_Producer_demand(demanded); - start = now(); + start = begin(); for (int i = 0; i < N; i++) { sum += (unsigned)moq_group_Demand_sequence(demand); } - span getter = per(start, N); + span getter = end(start, N, "getter"); /* The handle call and the getter from THREADS threads: on one shared handle, then on a - * clone each, a handle of its own to the same value. */ - const void *shared_track[THREADS], *shared_demand[THREADS], *own_track[THREADS], *own_demand[THREADS]; - for (int i = 0; i < THREADS; i++) { - shared_track[i] = track; - shared_demand[i] = demand; - own_track[i] = moq_track_Producer_clone(track); - own_demand[i] = moq_group_Demand_clone(demand); - } - span shared_call = concurrent(shared_track, 0, &sum); - span own_call = concurrent(own_track, 0, &sum); - span shared_getter = concurrent(shared_demand, 1, &sum); - span own_getter = concurrent(own_demand, 1, &sum); - for (int i = 0; i < THREADS; i++) { - moq_track_Producer_free((moq_track_Producer *)own_track[i]); - moq_group_Demand_free((moq_group_Demand *)own_demand[i]); + * clone each, a handle of its own to the same value. CodSpeed's simulation runs one + * thread at a time, which contends for nothing, so it skips them. */ + span shared_call = {0}, own_call = {0}, shared_getter = {0}, own_getter = {0}; + if (!measured) { + const void *shared_track[THREADS], *shared_demand[THREADS], *own_track[THREADS], *own_demand[THREADS]; + for (int i = 0; i < THREADS; i++) { + shared_track[i] = track; + shared_demand[i] = demand; + own_track[i] = moq_track_Producer_clone(track); + own_demand[i] = moq_group_Demand_clone(demand); + } + shared_call = concurrent(shared_track, 0, &sum); + own_call = concurrent(own_track, 0, &sum); + shared_getter = concurrent(shared_demand, 1, &sum); + own_getter = concurrent(own_demand, 1, &sum); + for (int i = 0; i < THREADS; i++) { + moq_track_Producer_free((moq_track_Producer *)own_track[i]); + moq_group_Demand_free((moq_group_Demand *)own_demand[i]); + } } moq_group_Demand_free(demand); finish(demanded); - start = now(); + start = begin(); for (int i = 0; i < N; i++) { sum += moq_track_Producer_largest(track).some; } - span record = per(start, N); + span record = end(start, N, "record"); - start = now(); + start = begin(); for (int i = 0; i < N; i++) { moq_track_Info_free(moq_track_Producer_info(track)); } - span object = per(start, N); + span object = end(start, N, "object"); /* Each frame stamped through a fresh timestamp handle, as a caller holding a u64 must. */ uint64_t micros = 0; - start = now(); + start = begin(); for (int g = 0; g < GROUPS; g++) { moq_group_Producer *group = append_group(track); for (int f = 0; f < FRAMES; f++) { @@ -271,14 +295,14 @@ static void round_(int print) { } finish(group); } - span stamped = per(start, N); + span stamped = end(start, N, "stamped"); moq_Timestamp *at = NULL; moq_TimeOverflow *overflow = NULL; if (!moq_Timestamp_from_micros(micros, &at, &overflow)) { fail("from_micros"); } - start = now(); + start = begin(); for (int g = 0; g < GROUPS; g++) { moq_group_Producer *group = append_group(track); for (int f = 0; f < FRAMES; f++) { @@ -286,7 +310,7 @@ static void round_(int print) { } finish(group); } - span write = per(start, N); + span write = end(start, N, "write"); moq_track_Consumer *consumer = moq_track_Producer_consume(track); moq_track_Consumer_subscribe_future *subscribing = moq_track_Consumer_subscribe(consumer, NULL); @@ -299,7 +323,7 @@ static void round_(int print) { /* Each frame written, then read through a new future ready at its first poll. */ moq_frame_Frame *last = NULL; - start = now(); + start = begin(); for (int g = 0; g < GROUPS; g++) { moq_group_Producer *group = append_group(track); moq_group_Consumer *reader = recv_group(subscriber); @@ -313,10 +337,10 @@ static void round_(int print) { finish(group); moq_group_Consumer_free(reader); } - span ready = per(start, N); + span ready = end(start, N, "ready"); /* Each read polled before its frame is written, so it pends, then polled again. */ - start = now(); + start = begin(); for (int g = 0; g < GROUPS; g++) { moq_group_Producer *group = append_group(track); moq_group_Consumer *reader = recv_group(subscriber); @@ -336,16 +360,16 @@ static void round_(int print) { finish(group); moq_group_Consumer_free(reader); } - span woken = per(start, N); + span woken = end(start, N, "woken"); /* The last frame's 100 bytes, owned without a copy, then freed. */ - start = now(); + start = begin(); for (int i = 0; i < N; i++) { moq_buf bytes = moq_frame_Frame_payload_get(last); sum += bytes.ptr[0]; moq_buf_free(bytes); } - span payload_ = per(start, N); + span payload_ = end(start, N, "payload"); moq_frame_Frame_free(last); span mbps[sizeof sizes / sizeof sizes[0]]; @@ -373,7 +397,11 @@ static void round_(int print) { } int main(void) { - round_(0); - round_(1); + /* Under CodSpeed, one round, which prints nothing: its simulation counts each case. */ + measured = codspeed_init("ffi/moq/bench/export.c::c"); + if (!measured) { + round_(0); + } + round_(!measured); return 0; } diff --git a/ffi/moq/bench/export.cpp b/ffi/moq/bench/export.cpp index 17bd93f2c..82f22a5b8 100644 --- a/ffi/moq/bench/export.cpp +++ b/ffi/moq/bench/export.cpp @@ -1,6 +1,7 @@ // bench/export.c's cases through the generated C++ header, so the two differ by what the // C++ wrapper adds. Futures are polled by hand with a waker that does nothing. Each case -// prints its wall time and the process's CPU time. +// prints its wall time and the process's CPU time, or under CodSpeed, runs once for its CPU +// simulation to count. #include #include @@ -9,9 +10,11 @@ #include #include #include +#include #include #include +#include "codspeed.h" #include "moq.hpp" namespace { @@ -46,6 +49,22 @@ span per(span start, double calls = n) { return {(end.wall - start.wall) / calls, (end.cpu - start.cpu) / calls}; } +// Whether CodSpeed counts this run, which then skips the warm-up and the threaded cases. +bool measured; + +// A case's start: now, and CodSpeed counting from here. +span begin() { + span start = now(); + codspeed_start(); + return start; +} + +// The case `name`'s nanoseconds per call since `start`, over `calls` calls. +span end(span start, const char *name, double calls = n) { + codspeed_stop(name); + return per(start, calls); +} + template T must(std::optional value, const char *what) { if (!value) { @@ -122,7 +141,8 @@ span concurrent(const T *const (&handles)[threads], F call, unsigned &sum) { span throughput(moq::track_Producer &track, moq::track_Subscriber &subscriber, const moq::Timestamp &at, std::size_t size, unsigned &sum) { std::size_t count = std::clamp((64u << 20) / (size * frames), 3, 2000); std::vector payload(size); - auto start = now(); + std::string name = "throughput[" + std::to_string(size) + "]"; + auto start = begin(); for (std::size_t g = 0; g < count; g++) { auto group = must(track.append_group(), "append_group"); auto reader = ready(subscriber.recv_group(), "recv_group"); @@ -133,7 +153,7 @@ span throughput(moq::track_Producer &track, moq::track_Subscriber &subscriber, c } must(std::move(group).finish(), "finish"); } - span ns = per(start, static_cast(count * frames * size)); + span ns = end(start, name.c_str(), static_cast(count * frames * size)); return {1e3 / ns.wall, 1e3 / ns.cpu}; } @@ -146,62 +166,65 @@ void round(bool print) { moq::view frame(bytes, sizeof bytes); unsigned sum = 0; - auto start = now(); + auto start = begin(); for (int i = 0; i < n; i++) { sum += track.is_closed(); } - span call = per(start); + span call = end(start, "call"); - span getter, shared_call, own_call, shared_getter, own_getter; + span getter, shared_call{}, own_call{}, shared_getter{}, own_getter{}; { auto demanded = must(track.append_group(), "append_group"); auto demand = demanded.demand(); - start = now(); + start = begin(); for (int i = 0; i < n; i++) { sum += static_cast(demand.sequence()); } - getter = per(start); + getter = end(start, "getter"); // The handle call and the getter from `threads` threads: on one shared handle, then - // on a clone each, a handle of its own to the same value. - auto is_closed = [](const moq::track_Producer &t) { return static_cast(t.is_closed()); }; - auto sequence = [](const moq::group_Demand &d) { return static_cast(d.sequence()); }; - std::vector tracks; - std::vector demands; - for (int i = 0; i < threads; i++) { - tracks.push_back(track.clone()); - demands.push_back(demand.clone()); - } - const moq::track_Producer *shared_tracks[threads], *own_tracks[threads]; - const moq::group_Demand *shared_demands[threads], *own_demands[threads]; - for (int i = 0; i < threads; i++) { - shared_tracks[i] = &track; - own_tracks[i] = &tracks[i]; - shared_demands[i] = &demand; - own_demands[i] = &demands[i]; + // on a clone each, a handle of its own to the same value. CodSpeed's simulation runs + // one thread at a time, which contends for nothing, so it skips them. + if (!measured) { + auto is_closed = [](const moq::track_Producer &t) { return static_cast(t.is_closed()); }; + auto sequence = [](const moq::group_Demand &d) { return static_cast(d.sequence()); }; + std::vector tracks; + std::vector demands; + for (int i = 0; i < threads; i++) { + tracks.push_back(track.clone()); + demands.push_back(demand.clone()); + } + const moq::track_Producer *shared_tracks[threads], *own_tracks[threads]; + const moq::group_Demand *shared_demands[threads], *own_demands[threads]; + for (int i = 0; i < threads; i++) { + shared_tracks[i] = &track; + own_tracks[i] = &tracks[i]; + shared_demands[i] = &demand; + own_demands[i] = &demands[i]; + } + shared_call = concurrent(shared_tracks, is_closed, sum); + own_call = concurrent(own_tracks, is_closed, sum); + shared_getter = concurrent(shared_demands, sequence, sum); + own_getter = concurrent(own_demands, sequence, sum); } - shared_call = concurrent(shared_tracks, is_closed, sum); - own_call = concurrent(own_tracks, is_closed, sum); - shared_getter = concurrent(shared_demands, sequence, sum); - own_getter = concurrent(own_demands, sequence, sum); must(std::move(demanded).finish(), "finish"); } - start = now(); + start = begin(); for (int i = 0; i < n; i++) { sum += track.largest().has_value(); } - span record = per(start); + span record = end(start, "record"); - start = now(); + start = begin(); for (int i = 0; i < n; i++) { (void)track.info(); } - span object = per(start); + span object = end(start, "object"); // Each frame stamped through a fresh timestamp handle, as a caller holding a u64 must. std::uint64_t micros = 0; - start = now(); + start = begin(); for (int g = 0; g < groups; g++) { auto group = must(track.append_group(), "append_group"); for (int f = 0; f < frames; f++) { @@ -210,10 +233,10 @@ void round(bool print) { } must(std::move(group).finish(), "finish"); } - span stamped = per(start); + span stamped = end(start, "stamped"); auto at = must(moq::Timestamp::from_micros(micros), "from_micros"); - start = now(); + start = begin(); for (int g = 0; g < groups; g++) { auto group = must(track.append_group(), "append_group"); for (int f = 0; f < frames; f++) { @@ -221,13 +244,13 @@ void round(bool print) { } must(std::move(group).finish(), "finish"); } - span write = per(start); + span write = end(start, "write"); auto subscriber = must(must(track.consume().subscribe(std::nullopt).poll(noop()), "subscribe"), "subscribe"); // Each frame written, then read through a new future ready at its first poll. std::optional last; - start = now(); + start = begin(); for (int g = 0; g < groups; g++) { auto group = must(track.append_group(), "append_group"); auto reader = ready(subscriber.recv_group(), "recv_group"); @@ -237,10 +260,10 @@ void round(bool print) { } must(std::move(group).finish(), "finish"); } - span read = per(start); + span read = end(start, "ready"); // Each read polled before its frame is written, so it pends, then polled again. - start = now(); + start = begin(); for (int g = 0; g < groups; g++) { auto group = must(track.append_group(), "append_group"); auto reader = ready(subscriber.recv_group(), "recv_group"); @@ -255,14 +278,14 @@ void round(bool print) { } must(std::move(group).finish(), "finish"); } - span woken = per(start); + span woken = end(start, "woken"); // The last frame's 100 bytes, owned without a copy, then freed. - start = now(); + start = begin(); for (int i = 0; i < n; i++) { sum += last->payload_get().data()[0]; } - span payload = per(start); + span payload = end(start, "payload"); span mbps[std::size(sizes)]; for (std::size_t i = 0; i < std::size(sizes); i++) { @@ -285,7 +308,11 @@ void round(bool print) { } // namespace int main() { - round(false); - round(true); + // Under CodSpeed, one round, which prints nothing: its simulation counts each case. + measured = codspeed_init("ffi/moq/bench/export.cpp::cpp"); + if (!measured) { + round(false); + } + round(!measured); return 0; } diff --git a/ffi/moq/bench/export.dart b/ffi/moq/bench/export.dart index 0b9d85185..e07ab469d 100644 --- a/ffi/moq/bench/export.dart +++ b/ffi/moq/bench/export.dart @@ -1,6 +1,8 @@ // bench/export.c's cases through the generated Dart, so the two differ by what dart:ffi and // the Dart wrapper add. Each round runs twice, the first to warm the JIT. Each case prints -// its wall time and the process's CPU time, the VM's own threads included. +// its wall time and the process's CPU time, the VM's own threads included, or under +// CodSpeed, runs once for its CPU simulation to count. Its argument is +// moq/bench/codspeed.h's shared library. import 'dart:async'; import 'dart:ffi' as ffi; @@ -45,6 +47,42 @@ Span per(Span start, [num calls = n]) { return Span((end.wall - start.wall) / calls, (end.cpu - start.cpu) / calls); } +/// moq/bench/codspeed.h's functions, from the shared library main opens. +late final bool Function(ffi.Pointer) _codspeedInit; +late final void Function() _codspeedStart; +late final void Function(ffi.Pointer) _codspeedStop; + +/// A case's start: now, and CodSpeed counting from here. +Span begin() { + final start = now(); + _codspeedStart(); + return start; +} + +/// The case `name`'s nanoseconds per call since `start`, over `calls` calls. +Span end(Span start, String name, [num calls = n]) { + final c = _cString(name); + _codspeedStop(c); + _free(c.cast()); + return per(start, calls); +} + +final _malloc = _process + .lookupFunction Function(ffi.Size), ffi.Pointer Function(int)>('malloc'); +final _free = _process.lookupFunction), void Function(ffi.Pointer)>( + 'free', +); + +/// `s`, an ASCII name, as a C string to free. +ffi.Pointer _cString(String s) { + final c = _malloc(s.length + 1).cast(); + for (var i = 0; i < s.length; i++) { + c[i] = s.codeUnitAt(i); + } + c[s.length] = 0; + return c.cast(); +} + /// What the loops sum, kept so the compiler cannot drop the calls. var sink = 0; @@ -53,7 +91,8 @@ var sink = 0; Future throughput(moq.TrackProducer track, moq.TrackSubscriber subscriber, moq.Timestamp at, int size) async { final count = ((64 << 20) ~/ (size * frames)).clamp(3, 2000); final frame = Uint8List(size); - final start = now(); + final name = 'throughput[$size]'; + final start = begin(); for (var g = 0; g < count; g++) { final group = track.appendGroup(); final reader = (await subscriber.recvGroup())!; @@ -67,7 +106,7 @@ Future throughput(moq.TrackProducer track, moq.TrackSubscriber subscriber, group.finish(); group.close(); } - final ns = per(start, count * frames * size); + final ns = end(start, name, count * frames * size); return Span(1e3 / ns.wall, 1e3 / ns.cpu); } @@ -79,38 +118,38 @@ Future> round() async { micro.close(); final frame = Uint8List(100); - var start = now(); + var start = begin(); for (var i = 0; i < n; i++) { if (track.isClosed()) sink++; } - out['call'] = per(start); + out['call'] = end(start, 'call'); final demanded = track.appendGroup(); final demand = demanded.demand(); - start = now(); + start = begin(); for (var i = 0; i < n; i++) { sink += demand.sequence(); } - out['getter'] = per(start); + out['getter'] = end(start, 'getter'); demand.close(); demanded.finish(); demanded.close(); - start = now(); + start = begin(); for (var i = 0; i < n; i++) { if (track.largest() != null) sink++; } - out['record'] = per(start); + out['record'] = end(start, 'record'); - start = now(); + start = begin(); for (var i = 0; i < n; i++) { track.info().close(); } - out['object'] = per(start); + out['object'] = end(start, 'object'); // Each frame stamped through a fresh timestamp object, as a caller holding an int must. var micros = 0; - start = now(); + start = begin(); for (var g = 0; g < groups; g++) { final group = track.appendGroup(); for (var f = 0; f < frames; f++) { @@ -121,10 +160,10 @@ Future> round() async { group.finish(); group.close(); } - out['stamped'] = per(start); + out['stamped'] = end(start, 'stamped'); final at = moq.Timestamp.fromMicros(micros); - start = now(); + start = begin(); for (var g = 0; g < groups; g++) { final group = track.appendGroup(); for (var f = 0; f < frames; f++) { @@ -133,7 +172,7 @@ Future> round() async { group.finish(); group.close(); } - out['write'] = per(start); + out['write'] = end(start, 'write'); final consumer = track.consume(); final subscriber = await consumer.subscribe(null); @@ -142,7 +181,7 @@ Future> round() async { // Each frame written, then read through a call whose future is ready at its first poll, // so it never waits for a wake. moq.FrameFrame? last; - start = now(); + start = begin(); for (var g = 0; g < groups; g++) { final group = track.appendGroup(); final reader = (await subscriber.recvGroup())!; @@ -155,11 +194,11 @@ Future> round() async { group.finish(); group.close(); } - out['ready'] = per(start); + out['ready'] = end(start, 'ready'); // Each read called before its frame is written, so it pends, and the write's wake // reaches the isolate's port, whose event loop polls it again. - start = now(); + start = begin(); for (var g = 0; g < groups; g++) { final group = track.appendGroup(); final reader = (await subscriber.recvGroup())!; @@ -172,10 +211,10 @@ Future> round() async { group.finish(); group.close(); } - out['woken'] = per(start); + out['woken'] = end(start, 'woken'); // One writer future and one reader; one outstanding permit bounds the stream. - start = now(); + start = begin(); final permits = StreamController(); Future writeLoop() async { final next = StreamIterator(permits.stream); @@ -203,7 +242,7 @@ Future> round() async { } await writer; await permits.close(); - out['loop'] = per(start); + out['loop'] = end(start, 'loop'); for (final size in sizes) { out['mbps:$size'] = await throughput(track, subscriber, at, size); @@ -211,11 +250,11 @@ Future> round() async { subscriber.close(); // The last frame's 100 bytes, copied into a list, and one of them read. - start = now(); + start = begin(); for (var i = 0; i < n; i++) { sink += last!.payloadGet[0]; } - out['payload'] = per(start); + out['payload'] = end(start, 'payload'); last!.close(); at.close(); track.close(); @@ -223,7 +262,23 @@ Future> round() async { return out; } -Future main() async { +Future main(List args) async { + final codspeed = ffi.DynamicLibrary.open(args.single); + _codspeedInit = codspeed.lookupFunction), bool Function(ffi.Pointer)>( + 'codspeed_init', + ); + _codspeedStart = codspeed.lookupFunction('codspeed_start'); + _codspeedStop = codspeed.lookupFunction), void Function(ffi.Pointer)>( + 'codspeed_stop', + ); + final bench = _cString('ffi/moq/bench/export.dart::dart'); + final measured = _codspeedInit(bench); + _free(bench.cast()); + // Under CodSpeed, one round, which prints nothing: its simulation counts each case. + if (measured) { + await round(); + return; + } await round(); for (final MapEntry(:key, :value) in (await round()).entries) { print('$key\t${value.wall}\t${value.cpu}'); diff --git a/ffi/moq/bench/export.go b/ffi/moq/bench/export.go index 582c3a7a0..20d702690 100644 --- a/ffi/moq/bench/export.go +++ b/ffi/moq/bench/export.go @@ -1,8 +1,13 @@ // bench/export.c's cases through the generated Go, so the two differ by what cgo and the // Go wrapper add. Each round runs twice, the first to warm up. Each case prints its wall -// time and the process's CPU time, the GC's threads included. +// time and the process's CPU time, the GC's threads included, or under CodSpeed, runs once +// for its CPU simulation to count. package main +// #include +// #include "codspeed.h" +import "C" + import ( "context" "fmt" @@ -10,6 +15,7 @@ import ( "sync" "syscall" "time" + "unsafe" "moq/net" "moq/rt" @@ -47,6 +53,21 @@ func per(start span, calls float64) span { return span{(end.wall - start.wall) / calls, (end.cpu - start.cpu) / calls} } +// begin is a case's start: now, and CodSpeed counting from here. +func begin() span { + start := now() + C.codspeed_start() + return start +} + +// end is the case name's nanoseconds per call since start, over calls calls. +func end(start span, name string, calls float64) span { + c := C.CString(name) + C.codspeed_stop(c) + C.free(unsafe.Pointer(c)) + return per(start, calls) +} + // must is v, failing the bench on err. func must[T any](v T, err error) T { if err != nil { @@ -71,7 +92,8 @@ var sink int func throughput(ctx context.Context, track *net.TrackProducer, subscriber *net.TrackSubscriber, at *net.Timestamp, size int) span { count := min(max((64<<20)/(size*frames), 3), 2000) frame := rt.NewBytes(make([]byte, size)) - start := now() + name := fmt.Sprintf("throughput[%d]", size) + start := begin() for range count { group := must(track.AppendGroup()) reader := must(subscriber.RecvGroup(ctx)) @@ -85,7 +107,7 @@ func throughput(ctx context.Context, track *net.TrackProducer, subscriber *net.T check(group.Finish()) group.Close() } - ns := per(start, float64(count*frames*size)) + ns := end(start, name, float64(count*frames*size)) return span{1e3 / ns.wall, 1e3 / ns.cpu} } @@ -140,7 +162,7 @@ type result struct { value span } -func round() []result { +func round(measured bool) []result { var out []result broadcast := must(net.NewBroadcastInfo().Produce()) micro := net.TimescaleMICRO() @@ -148,74 +170,77 @@ func round() []result { micro.Close() frame := rt.NewBytes(make([]byte, 100)) - start := now() + start := begin() for range n { if must(track.IsClosed()) { sink++ } } - out = append(out, result{"call", per(start, n)}) + out = append(out, result{"call", end(start, "call", n)}) demanded := must(track.AppendGroup()) demand := must(demanded.Demand()) - start = now() + start = begin() for range n { sink += int(must(demand.Sequence())) } - out = append(out, result{"getter", per(start, n)}) + out = append(out, result{"getter", end(start, "getter", n)}) // The handle call and the getter from threads goroutines: on one shared handle, then on - // a clone each, a handle of its own to the same value. - isClosed := func(t *net.TrackProducer) func() int { - return func() int { - if must(t.IsClosed()) { - return 1 + // a clone each, a handle of its own to the same value. CodSpeed's simulation runs one + // thread at a time, which contends for nothing, so it skips them. + if !measured { + isClosed := func(t *net.TrackProducer) func() int { + return func() int { + if must(t.IsClosed()) { + return 1 + } + return 0 } - return 0 } - } - sequence := func(d *net.GroupDemand) func() int { - return func() int { return int(must(d.Sequence())) } - } - var sharedCalls, ownCalls, sharedGetters, ownGetters []func() int - var clones []interface{ Close() } - for range threads { - t := must(track.Clone()) - d := must(demand.Clone()) - clones = append(clones, t, d) - sharedCalls = append(sharedCalls, isClosed(track)) - ownCalls = append(ownCalls, isClosed(t)) - sharedGetters = append(sharedGetters, sequence(demand)) - ownGetters = append(ownGetters, sequence(d)) - } - out = append(out, result{"shared:call", concurrent(sharedCalls)}) - out = append(out, result{"own:call", concurrent(ownCalls)}) - out = append(out, result{"shared:getter", concurrent(sharedGetters)}) - out = append(out, result{"own:getter", concurrent(ownGetters)}) - for _, clone := range clones { - clone.Close() + sequence := func(d *net.GroupDemand) func() int { + return func() int { return int(must(d.Sequence())) } + } + var sharedCalls, ownCalls, sharedGetters, ownGetters []func() int + var clones []interface{ Close() } + for range threads { + t := must(track.Clone()) + d := must(demand.Clone()) + clones = append(clones, t, d) + sharedCalls = append(sharedCalls, isClosed(track)) + ownCalls = append(ownCalls, isClosed(t)) + sharedGetters = append(sharedGetters, sequence(demand)) + ownGetters = append(ownGetters, sequence(d)) + } + out = append(out, result{"shared:call", concurrent(sharedCalls)}) + out = append(out, result{"own:call", concurrent(ownCalls)}) + out = append(out, result{"shared:getter", concurrent(sharedGetters)}) + out = append(out, result{"own:getter", concurrent(ownGetters)}) + for _, clone := range clones { + clone.Close() + } } demand.Close() check(demanded.Finish()) demanded.Close() - start = now() + start = begin() for range n { if must(track.Largest()) != nil { sink++ } } - out = append(out, result{"record", per(start, n)}) + out = append(out, result{"record", end(start, "record", n)}) - start = now() + start = begin() for range n { must(track.Info()).Close() } - out = append(out, result{"object", per(start, n)}) + out = append(out, result{"object", end(start, "object", n)}) // Each frame stamped through a fresh timestamp handle, as a caller holding a u64 must. var micros uint64 - start = now() + start = begin() for range groups { group := must(track.AppendGroup()) for range frames { @@ -227,10 +252,10 @@ func round() []result { check(group.Finish()) group.Close() } - out = append(out, result{"stamped", per(start, n)}) + out = append(out, result{"stamped", end(start, "stamped", n)}) at := must(net.TimestampFromMicros(micros)) - start = now() + start = begin() for range groups { group := must(track.AppendGroup()) for range frames { @@ -239,7 +264,7 @@ func round() []result { check(group.Finish()) group.Close() } - out = append(out, result{"write", per(start, n)}) + out = append(out, result{"write", end(start, "write", n)}) ctx := context.Background() consumer := must(track.Consume()) @@ -249,7 +274,7 @@ func round() []result { // Each frame written, then read through a call whose future is ready at its first poll, // so it never waits. var last *net.FrameFrame - start = now() + start = begin() for range groups { group := must(track.AppendGroup()) reader := must(subscriber.RecvGroup(ctx)) @@ -264,11 +289,11 @@ func round() []result { check(group.Finish()) group.Close() } - out = append(out, result{"ready", per(start, n)}) + out = append(out, result{"ready", end(start, "ready", n)}) // Each read started on a goroutine before its frame is written, which the yield lets // block, and the write's wake resumes it. - start = now() + start = begin() read := make(chan *net.FrameFrame) for range groups { group := must(track.AppendGroup()) @@ -283,11 +308,11 @@ func round() []result { check(group.Finish()) group.Close() } - out = append(out, result{"woken", per(start, n)}) + out = append(out, result{"woken", end(start, "woken", n)}) // One writer and the calling reader live for the whole case. Each write // waits until the read has pended with its waker registered. - start = now() + start = begin() permits := make(chan struct{}) pending := &pendingContext{Context: ctx, parked: permits} done := make(chan struct{}) @@ -316,7 +341,7 @@ func round() []result { if pending.waits != n { panic("each loop read must pend exactly once") } - out = append(out, result{"loop", per(start, n)}) + out = append(out, result{"loop", end(start, "loop", n)}) for _, size := range sizes { out = append(out, result{fmt.Sprintf("mbps:%d", size), throughput(ctx, track, subscriber, at, size)}) @@ -324,11 +349,11 @@ func round() []result { subscriber.Close() // The last frame's 100 bytes, as a Bytes, and one of them read. - start = now() + start = begin() for range n { sink += int(must(last.PayloadGet()).At(0)) } - out = append(out, result{"payload", per(start, n)}) + out = append(out, result{"payload", end(start, "payload", n)}) last.Close() at.Close() track.Close() @@ -337,8 +362,16 @@ func round() []result { } func main() { - round() - for _, r := range round() { + bench := C.CString("ffi/moq/bench/export.go::go") + measured := bool(C.codspeed_init(bench)) + C.free(unsafe.Pointer(bench)) + // Under CodSpeed, one round, which prints nothing: its simulation counts each case. + if measured { + round(true) + return + } + round(false) + for _, r := range round(false) { fmt.Printf("%s\t%f\t%f\n", r.name, r.value.wall, r.value.cpu) } } diff --git a/ffi/moq/bench/export.kt b/ffi/moq/bench/export.kt index 19afb1815..05febb6dc 100644 --- a/ffi/moq/bench/export.kt +++ b/ffi/moq/bench/export.kt @@ -1,6 +1,7 @@ // bench/export.c's cases through the generated Kotlin, so the two differ by what JNI and // the Kotlin wrapper add. Each round runs twice, the first to warm the JIT. Each case prints -// its wall time and the process's CPU time, the JIT's and GC's threads included. +// its wall time and the process's CPU time, the JIT's and GC's threads included, or under +// CodSpeed, runs once for its CPU simulation to count. import java.lang.management.ManagementFactory import java.util.concurrent.CountDownLatch @@ -30,6 +31,32 @@ fun per(start: Span, n: Double = N.toDouble()): Span { return Span((end.wall - start.wall) / n, (end.cpu - start.cpu) / n) } +// moq/bench/codspeed.h, through the JNI functions in the shared library bench-build makes. +object Codspeed { + init { + System.loadLibrary("codspeed") + } + + external fun init(bench: String): Boolean + + external fun start() + + external fun stop(name: String) +} + +// A case's start: now, and CodSpeed counting from here. +fun begin(): Span { + val start = now() + Codspeed.start() + return start +} + +// The case `name`'s nanoseconds per call since `start`, over `n` calls. +fun end(start: Span, name: String, n: Double = N.toDouble()): Span { + Codspeed.stop(name) + return per(start, n) +} + // What the loops sum, kept so the JIT cannot drop the calls. @Volatile var sink = 0 @@ -66,8 +93,9 @@ fun concurrent(handles: List, call: (T) -> Int): Span { suspend fun throughput(track: TrackProducer, subscriber: TrackSubscriber, at: Timestamp, size: Int): Span { val groups = ((64 shl 20) / (size * FRAMES)).coerceIn(3, 2000) val frame = Bytes(ByteArray(size)) + val name = "throughput[$size]" var sum = 0 - val start = now() + val start = begin() repeat(groups) { track.appendGroup().use { group -> subscriber.recvGroup()!!.use { reader -> @@ -80,11 +108,11 @@ suspend fun throughput(track: TrackProducer, subscriber: TrackSubscriber, at: Ti } } sink += sum - val ns = per(start, groups.toDouble() * FRAMES * size) + val ns = end(start, name, groups.toDouble() * FRAMES * size) return Span(1e3 / ns.wall, 1e3 / ns.cpu) } -fun round(): Map { +fun round(measured: Boolean): Map { val out = linkedMapOf() val broadcast = BroadcastInfo.new().produce() val track = Timescale.MICRO.use { micro -> @@ -93,52 +121,58 @@ fun round(): Map { val frame = Bytes(ByteArray(100)) var sum = 0 - var start = now() + var start = begin() repeat(N) { if (track.isClosed()) sum++ } - out["call"] = per(start) + out["call"] = end(start, "call") track.appendGroup().use { group -> group.demand().use { demand -> - start = now() + start = begin() repeat(N) { sum += demand.sequence().toInt() } - out["getter"] = per(start) + out["getter"] = end(start, "getter") // The handle call and the getter from THREADS threads: on one shared handle, then - // on a clone each, a handle of its own to the same value. - val isClosed = { t: TrackProducer -> if (t.isClosed()) 1 else 0 } - val sequence = { d: GroupDemand -> d.sequence().toInt() } - val tracks = List(THREADS) { track.clone() } - val demands = List(THREADS) { demand.clone() } - out["shared:call"] = concurrent(List(THREADS) { track }, isClosed) - out["own:call"] = concurrent(tracks, isClosed) - out["shared:getter"] = concurrent(List(THREADS) { demand }, sequence) - out["own:getter"] = concurrent(demands, sequence) - tracks.forEach { it.close() } - demands.forEach { it.close() } + // on a clone each, a handle of its own to the same value. CodSpeed's simulation runs + // one thread at a time, which contends for nothing, so it skips them. + if (!measured) { + val isClosed = { t: TrackProducer -> if (t.isClosed()) 1 else 0 } + val sequence = { d: GroupDemand -> d.sequence().toInt() } + val tracks = List(THREADS) { track.clone() } + val demands = List(THREADS) { demand.clone() } + out["shared:call"] = concurrent(List(THREADS) { track }, isClosed) + out["own:call"] = concurrent(tracks, isClosed) + out["shared:getter"] = concurrent(List(THREADS) { demand }, sequence) + out["own:getter"] = concurrent(demands, sequence) + tracks.forEach { it.close() } + demands.forEach { it.close() } + } } group.finish() } - start = now() + start = begin() repeat(N) { if (track.largest() != null) sum++ } - out["record"] = per(start) + out["record"] = end(start, "record") - start = now() + start = begin() repeat(N) { track.info().close() } - out["object"] = per(start) + out["object"] = end(start, "object") // The same from THREADS threads at once, so objects made and freed together contend // for whatever the binding shares between them. Kotlin's alone: the others share none. - start = now() - val failures = java.util.concurrent.ConcurrentLinkedQueue() - List(THREADS) { java.lang.Thread { try { repeat(N / THREADS) { track.info().close() } } catch (e: Throwable) { failures.add(e) } }.apply { start() } }.forEach { it.join() } - // A worker that died early would otherwise report a time for fewer than N calls. - failures.peek()?.let { throw it } - out["objects"] = per(start) + // CodSpeed skips it as it does the other threaded cases. + if (!measured) { + val began = now() + val failures = java.util.concurrent.ConcurrentLinkedQueue() + List(THREADS) { java.lang.Thread { try { repeat(N / THREADS) { track.info().close() } } catch (e: Throwable) { failures.add(e) } }.apply { start() } }.forEach { it.join() } + // A worker that died early would otherwise report a time for fewer than N calls. + failures.peek()?.let { throw it } + out["objects"] = per(began) + } // Each frame stamped through a fresh timestamp handle, as a caller holding a u64 must. var micros = 0uL - start = now() + start = begin() repeat(GROUPS) { track.appendGroup().use { group -> repeat(FRAMES) { @@ -147,17 +181,17 @@ fun round(): Map { group.finish() } } - out["stamped"] = per(start) + out["stamped"] = end(start, "stamped") val at = Timestamp.fromMicros(micros) - start = now() + start = begin() repeat(GROUPS) { track.appendGroup().use { group -> repeat(FRAMES) { group.writeFrame(at, frame) } group.finish() } } - out["write"] = per(start) + out["write"] = end(start, "write") var last: FrameFrame? = null runBlocking { @@ -165,7 +199,7 @@ fun round(): Map { // Each frame written, then read through a suspend fun whose future is ready at its // first poll, so it never suspends. - var t0 = now() + var t0 = begin() repeat(GROUPS) { track.appendGroup().use { group -> subscriber.recvGroup()!!.use { reader -> @@ -178,11 +212,11 @@ fun round(): Map { group.finish() } } - out["ready"] = per(t0) + out["ready"] = end(t0, "ready") // Each read started before its frame is written, so it suspends, and the write's wake // resumes it through runBlocking's event loop. - t0 = now() + t0 = begin() repeat(GROUPS) { track.appendGroup().use { group -> subscriber.recvGroup()!!.use { reader -> @@ -195,10 +229,10 @@ fun round(): Map { group.finish() } } - out["woken"] = per(t0) + out["woken"] = end(t0, "woken") // One writer coroutine and one reader, with at most one frame permitted. - t0 = now() + t0 = begin() val permits = Channel() val writer = launch { repeat(GROUPS) { @@ -223,18 +257,18 @@ fun round(): Map { } writer.join() permits.close() - out["loop"] = per(t0) + out["loop"] = end(t0, "loop") for (size in SIZES) out["mbps:$size"] = throughput(track, subscriber, at, size) subscriber.close() } // The last frame's 100 bytes, copied across, and one of them read. - start = now() + start = begin() last!!.use { read -> repeat(N) { sum += read.payloadGet[0] } } - out["payload"] = per(start) + out["payload"] = end(start, "payload") at.close() track.close() broadcast.close() @@ -243,6 +277,11 @@ fun round(): Map { } fun main() { - round() - for ((case, value) in round()) println("$case\t${value.wall}\t${value.cpu}") + // Under CodSpeed, one round, which prints nothing: its simulation counts each case. + if (Codspeed.init("ffi/moq/bench/export.kt::kotlin")) { + round(true) + return + } + round(false) + for ((case, value) in round(false)) println("$case\t${value.wall}\t${value.cpu}") } diff --git a/ffi/moq/bench/export.py b/ffi/moq/bench/export.py index 466b1dd38..370115cfe 100644 --- a/ffi/moq/bench/export.py +++ b/ffi/moq/bench/export.py @@ -1,8 +1,11 @@ # bench/export.c's cases through the generated Python, so the two differ by what CPython # and the Python wrapper add. Each round runs twice, the first to warm CPython's caches. -# Each case prints its wall time and the process's CPU time. +# Each case prints its wall time and the process's CPU time, or under CodSpeed, runs once +# for its CPU simulation to count. Its argument is moq/bench/codspeed.h's shared library. import asyncio +import ctypes +import sys import threading import time @@ -26,6 +29,30 @@ def per(start, n=N): return (wall - start[0]) / n, (cpu - start[1]) / n +codspeed = ctypes.CDLL(sys.argv[1]) +codspeed.codspeed_init.argtypes = [ctypes.c_char_p] +codspeed.codspeed_init.restype = ctypes.c_bool +codspeed.codspeed_start.argtypes = [] +codspeed.codspeed_start.restype = None +codspeed.codspeed_stop.argtypes = [ctypes.c_char_p] +codspeed.codspeed_stop.restype = None +start_case = codspeed.codspeed_start +stop_case = codspeed.codspeed_stop + + +def begin(): + """A case's start: now, and CodSpeed counting from here.""" + start = now() + start_case() + return start + + +def end(start, name, n=N): + """The case `name`'s nanoseconds per call since `start`, over `n` calls.""" + stop_case(name) + return per(start, n) + + def concurrent(calls): """Each of `calls`, a bound method, made N times on a thread of its own, all at once: wall nanoseconds per call on one thread, and CPU nanoseconds per call.""" @@ -63,8 +90,9 @@ async def throughput(track, subscriber, at, size): second.""" groups = min(max((64 << 20) // (size * FRAMES), 3), 2000) frame = bytes(size) + name = f"throughput[{size}]".encode() total = 0 - start = now() + start = begin() for _ in range(groups): group = track.append_group() reader = await subscriber.recv_group() @@ -74,20 +102,20 @@ async def throughput(track, subscriber, at, size): with read.payload_get as payload: total += payload[0] group.finish() - wall, cpu = per(start, groups * FRAMES * size) + wall, cpu = end(start, name, groups * FRAMES * size) return 1e3 / wall, 1e3 / cpu -def round(loop): +def round(loop, measured): broadcast = moq.BroadcastInfo.new().produce() track = broadcast.create_track("bench", moq.TrackInfo.default().with_timescale(moq.Timescale.MICRO)) frame = bytes(100) out = {} - start = now() + start = begin() for _ in range(N): track.is_closed() - out["call"] = per(start) + out["call"] = end(start, b"call") # A bound method called in a loop, as bench/ffi/README.md timed BoltFFI's # `group.sequence()`. BoltFFI holds the GIL through the call, where the export releases @@ -95,48 +123,50 @@ def round(loop): demanded = track.append_group() demand = demanded.demand() sequence = demand.sequence - start = now() + start = begin() for _ in range(N): sequence() - out["getter"] = per(start) + out["getter"] = end(start, b"getter") # The handle call and the getter from THREADS threads: on one shared handle, then on a - # clone each, a handle of its own to the same value. - out["shared:call"] = concurrent([track.is_closed] * THREADS) - out["own:call"] = concurrent([track.clone().is_closed for _ in range(THREADS)]) - out["shared:getter"] = concurrent([sequence] * THREADS) - out["own:getter"] = concurrent([demand.clone().sequence for _ in range(THREADS)]) + # clone each, a handle of its own to the same value. CodSpeed's simulation runs one thread + # at a time, which contends for nothing, so it skips them. + if not measured: + out["shared:call"] = concurrent([track.is_closed] * THREADS) + out["own:call"] = concurrent([track.clone().is_closed for _ in range(THREADS)]) + out["shared:getter"] = concurrent([sequence] * THREADS) + out["own:getter"] = concurrent([demand.clone().sequence for _ in range(THREADS)]) demanded.finish() - start = now() + start = begin() for _ in range(N): track.largest() - out["record"] = per(start) + out["record"] = end(start, b"record") - start = now() + start = begin() for _ in range(N): track.info() - out["object"] = per(start) + out["object"] = end(start, b"object") # Each frame stamped through a fresh timestamp object, as a caller holding an int must. micros = 0 - start = now() + start = begin() for _ in range(GROUPS): group = track.append_group() for _ in range(FRAMES): group.write_frame(moq.Timestamp.from_micros(micros), frame) micros += 1 group.finish() - out["stamped"] = per(start) + out["stamped"] = end(start, b"stamped") at = moq.Timestamp.from_micros(micros) - start = now() + start = begin() for _ in range(GROUPS): group = track.append_group() for _ in range(FRAMES): group.write_frame(at, frame) group.finish() - out["write"] = per(start) + out["write"] = end(start, b"write") async def reads(): subscriber = await track.consume().subscribe(None) @@ -144,7 +174,7 @@ async def reads(): # Each frame written, then read through an awaitable ready at its first poll, so it # never suspends. last = None - start = now() + start = begin() for _ in range(GROUPS): group = track.append_group() reader = await subscriber.recv_group() @@ -152,11 +182,11 @@ async def reads(): group.write_frame(at, frame) last = await reader.read_frame() group.finish() - out["ready"] = per(start) + out["ready"] = end(start, b"ready") # Each read awaited before its frame is written, so it pends, and the write's wake # resumes it through the loop. - start = now() + start = begin() for _ in range(GROUPS): group = track.append_group() reader = await subscriber.recv_group() @@ -166,10 +196,10 @@ async def reads(): group.write_frame(at, frame) await pending group.finish() - out["woken"] = per(start) + out["woken"] = end(start, b"woken") # One writer task, with one outstanding permit, for the whole reader loop. - start = now() + start = begin() permits = asyncio.Queue(maxsize=1) async def write_loop(): @@ -189,7 +219,7 @@ async def write_loop(): await reader.read_frame() permits.put_nowait(None) await writer - out["loop"] = per(start) + out["loop"] = end(start, b"loop") for size in SIZES: out[f"mbps:{size}"] = await throughput(track, subscriber, at, size) @@ -198,16 +228,21 @@ async def write_loop(): last = loop.run_until_complete(reads()) # The last frame's 100 bytes, lent without a copy, then released. - start = now() + start = begin() for _ in range(N): with last.payload_get as payload: payload[0] - out["payload"] = per(start) + out["payload"] = end(start, b"payload") return out +# Under CodSpeed, one round, which prints nothing: its simulation counts each case. +measured = codspeed.codspeed_init(b"ffi/moq/bench/export.py::python") loop = asyncio.new_event_loop() -round(loop) -for case, (wall, cpu) in round(loop).items(): - print(f"{case}\t{wall}\t{cpu}") +if measured: + round(loop, True) +else: + round(loop, False) + for case, (wall, cpu) in round(loop, False).items(): + print(f"{case}\t{wall}\t{cpu}") loop.close() diff --git a/ffi/moq/bench/export.rs b/ffi/moq/bench/export.rs index 22104e6a2..6b40b8597 100644 --- a/ffi/moq/bench/export.rs +++ b/ffi/moq/bench/export.rs @@ -1,8 +1,10 @@ // The Rust floor: every case the bindings time, called on the stand-in's API directly, as a // Rust caller holding a byte slice would. Futures are polled by hand with a no-op waker, // so a woken read costs a re-poll and nothing of an executor. Each case prints its wall -// time and the process's CPU time. +// time and the process's CPU time, or under CodSpeed, runs once for its CPU simulation to +// count. +use std::ffi::{CStr, c_char}; use std::future::Future; use std::hint::black_box; use std::pin::pin; @@ -29,10 +31,33 @@ fn cpu() -> f64 { ts.tv_sec as f64 * 1e9 + ts.tv_nsec as f64 } -fn begin() -> Start { +fn now() -> Start { Start { wall: Instant::now(), cpu: cpu() } } +// moq/bench/codspeed.h, which bench-build links. +#[link(name = "codspeed", kind = "static")] +unsafe extern "C" { + fn codspeed_init(bench: *const c_char) -> bool; + fn codspeed_start(); + fn codspeed_stop(name: *const c_char); +} + +/// A case's start: now, and CodSpeed counting from here. +fn begin() -> Start { + let start = now(); + // SAFETY: codspeed_init ran first, in main. + unsafe { codspeed_start() }; + start +} + +/// The case `name`'s wall and CPU nanoseconds per call since `start`, over `n` calls. +fn end(start: Start, n: u64, name: &CStr) -> (f64, f64) { + // SAFETY: `name` is a C string, and codspeed_init ran first, in main. + unsafe { codspeed_stop(name.as_ptr()) }; + per(start, n) +} + /// Wall and CPU nanoseconds since `start`. fn elapsed(start: Start) -> (f64, f64) { (start.wall.elapsed().as_nanos() as f64, cpu() - start.cpu) @@ -62,7 +87,7 @@ fn concurrent(handles: &[&T], n: u64, call: impl Fn(&T) + Sync) -> (f64 }); } ready.wait(); - let begin = begin(); + let begin = now(); go.wait(); begin }); @@ -83,6 +108,7 @@ fn ready(future: F) -> F::Output { fn throughput(track: &moq::track::Producer, subscriber: &mut moq::track::Subscriber, at: moq::Timestamp, size: usize) -> (f64, f64) { let groups = ((64 << 20) / (size as u64 * FRAMES)).clamp(3, 2000); let frame = vec![0u8; size]; + let name = std::ffi::CString::new(format!("throughput[{size}]")).unwrap(); let mut sum = 0u64; let start = begin(); for _ in 0..groups { @@ -95,13 +121,13 @@ fn throughput(track: &moq::track::Producer, subscriber: &mut moq::track::Subscri } group.finish().unwrap(); } + let (wall, cpu) = end(start, 1, &name); black_box(sum); let bytes = (groups * FRAMES * size as u64) as f64; - let (wall, cpu) = elapsed(start); (bytes * 1e3 / wall, bytes * 1e3 / cpu) } -fn round() -> Vec<(String, (f64, f64))> { +fn round(measured: bool) -> Vec<(String, (f64, f64))> { let broadcast = moq::broadcast::Info::new().produce(); let info = moq::track::Info::default().with_timescale(moq::Timescale::MICRO); let track = broadcast.create_track("bench", info).unwrap(); @@ -113,42 +139,45 @@ fn round() -> Vec<(String, (f64, f64))> { for _ in 0..n { black_box(black_box(&track).is_closed()); } - out.push(("call".into(), per(start, n))); + out.push(("call".into(), end(start, n, c"call"))); let demand = track.append_group().unwrap().demand(); let start = begin(); for _ in 0..n { black_box(black_box(&demand).sequence()); } - out.push(("getter".into(), per(start, n))); + out.push(("getter".into(), end(start, n, c"getter"))); // The handle call and the getter from THREADS threads: on one shared value, then on a - // clone each, as the bindings' own-handle cases do. - let is_closed = |track: &moq::track::Producer| { - black_box(track.is_closed()); - }; - let sequence = |demand: &moq::group::Demand| { - black_box(demand.sequence()); - }; - let tracks: Vec<_> = (0..THREADS).map(|_| track.clone()).collect(); - let demands: Vec<_> = (0..THREADS).map(|_| demand.clone()).collect(); - out.push(("shared:call".into(), concurrent(&[&track; THREADS], n, is_closed))); - out.push(("own:call".into(), concurrent(&tracks.iter().collect::>(), n, is_closed))); - out.push(("shared:getter".into(), concurrent(&[&demand; THREADS], n, sequence))); - out.push(("own:getter".into(), concurrent(&demands.iter().collect::>(), n, sequence))); + // clone each, as the bindings' own-handle cases do. CodSpeed's simulation runs one thread + // at a time, which contends for nothing, so it skips them. + if !measured { + let is_closed = |track: &moq::track::Producer| { + black_box(track.is_closed()); + }; + let sequence = |demand: &moq::group::Demand| { + black_box(demand.sequence()); + }; + let tracks: Vec<_> = (0..THREADS).map(|_| track.clone()).collect(); + let demands: Vec<_> = (0..THREADS).map(|_| demand.clone()).collect(); + out.push(("shared:call".into(), concurrent(&[&track; THREADS], n, is_closed))); + out.push(("own:call".into(), concurrent(&tracks.iter().collect::>(), n, is_closed))); + out.push(("shared:getter".into(), concurrent(&[&demand; THREADS], n, sequence))); + out.push(("own:getter".into(), concurrent(&demands.iter().collect::>(), n, sequence))); + } let start = begin(); for _ in 0..n { black_box(black_box(&track).largest()); } - out.push(("record".into(), per(start, n))); + out.push(("record".into(), end(start, n, c"record"))); // A borrowed `&Info` cloned, as the bindings return an owned copy. let start = begin(); for _ in 0..n { black_box(black_box(&track).info().clone()); } - out.push(("object".into(), per(start, n))); + out.push(("object".into(), end(start, n, c"object"))); let mut micros = 0; let start = begin(); @@ -161,7 +190,7 @@ fn round() -> Vec<(String, (f64, f64))> { } group.finish().unwrap(); } - out.push(("stamped".into(), per(start, n))); + out.push(("stamped".into(), end(start, n, c"stamped"))); let at = moq::Timestamp::from_micros(micros).unwrap(); let start = begin(); @@ -172,7 +201,7 @@ fn round() -> Vec<(String, (f64, f64))> { } group.finish().unwrap(); } - out.push(("write".into(), per(start, n))); + out.push(("write".into(), end(start, n, c"write"))); let mut subscriber = ready(track.consume().subscribe(None)).unwrap(); let mut last = None; @@ -186,7 +215,7 @@ fn round() -> Vec<(String, (f64, f64))> { } group.finish().unwrap(); } - out.push(("ready".into(), per(start, n))); + out.push(("ready".into(), end(start, n, c"ready"))); let start = begin(); for _ in 0..GROUPS { @@ -202,7 +231,7 @@ fn round() -> Vec<(String, (f64, f64))> { } group.finish().unwrap(); } - out.push(("woken".into(), per(start, n))); + out.push(("woken".into(), end(start, n, c"woken"))); // The payload as a binding hands it over: a new owner of the frame's bytes. let last = last.unwrap(); @@ -210,7 +239,7 @@ fn round() -> Vec<(String, (f64, f64))> { for _ in 0..n { black_box(black_box(&last).payload.clone()); } - out.push(("payload".into(), per(start, n))); + out.push(("payload".into(), end(start, n, c"payload"))); for size in SIZES { out.push((format!("mbps:{size}"), throughput(&track, &mut subscriber, at, size))); @@ -219,8 +248,15 @@ fn round() -> Vec<(String, (f64, f64))> { } fn main() { - round(); - for (case, (wall, cpu)) in round() { + // SAFETY: the name is a C string, and this is the first call into the hooks. + let measured = unsafe { codspeed_init(c"ffi/moq/bench/export.rs::rust".as_ptr()) }; + // Under CodSpeed, one round, which prints nothing: its simulation counts each case. + if measured { + round(true); + return; + } + round(false); + for (case, (wall, cpu)) in round(false) { println!("{case}\t{wall}\t{cpu}"); } } diff --git a/ffi/moq/bench/export.swift b/ffi/moq/bench/export.swift index 6181606d1..72d709041 100644 --- a/ffi/moq/bench/export.swift +++ b/ffi/moq/bench/export.swift @@ -1,7 +1,9 @@ // bench/export.c's cases through the generated Swift, so the two differ by what the Swift // wrapper adds. Each round runs twice, the first to warm up. Each case prints its wall time -// and the process's CPU time, the global executor's threads included. +// and the process's CPU time, the global executor's threads included, or under CodSpeed, +// runs once for its CPU simulation to count. +import CodSpeed import Foundation import Moq @@ -34,6 +36,19 @@ func per(_ start: Span, _ calls: Double = Double(n)) -> Span { return Span(wall: (end.wall - start.wall) / calls, cpu: (end.cpu - start.cpu) / calls) } +/// A case's start: now, and CodSpeed counting from here. +func begin() -> Span { + let start = now() + codspeed_start() + return start +} + +/// The case `name`'s nanoseconds per call since `start`, over `calls` calls. +func end(_ start: Span, _ name: String, _ calls: Double = Double(n)) -> Span { + codspeed_stop(name) + return per(start, calls) +} + /// A value a task sets and this thread takes once the task signals. final class Cell: @unchecked Sendable { var value: T? @@ -136,8 +151,9 @@ struct Reads: ~Copyable { func throughput(_ track: TrackProducer, _ subscriber: inout TrackSubscriber, _ at: Timestamp, _ size: Int) async throws -> (Span, Int) { let count = min(max((64 << 20) / (size * frames), 3), 2000) let frame = Bytes(Data(count: size)) + let name = "throughput[\(size)]" var sum = 0 - let start = now() + let start = begin() for _ in 0.. ([(String, Span)], Int) { +func round(_ measured: Bool) throws -> ([(String, Span)], Int) { var out: [(String, Span)] = [] let broadcast = BroadcastInfo.new().produce() let track = try broadcast.createTrack("bench", TrackInfo.default().withTimescale(Timescale.MICRO)) let frame = Bytes(Data(count: 100)) var sum = 0 - var start = now() + var start = begin() for _ in 0.. @Sendable () -> Int in { t.isClosed() ? 1 : 0 } } - let sequence = { (d: Shared) -> @Sendable () -> Int in { Int(truncatingIfNeeded: d.value.sequence()) } } - let shared = Shared(demand.clone()) - out.append(("shared:call", concurrent((0.. @Sendable () -> Int in { t.isClosed() ? 1 : 0 } } + let sequence = { (d: Shared) -> @Sendable () -> Int in { Int(truncatingIfNeeded: d.value.sequence()) } } + let shared = Shared(demand.clone()) + out.append(("shared:call", concurrent((0.. ([(String, Span)], Int) { } try group.finish() } - out.append(("stamped", per(start))) + out.append(("stamped", end(start, "stamped"))) let at = try Timestamp.fromMicros(micros) - start = now() + start = begin() for _ in 0.. ([(String, Span)], Int) { } try group.finish() } - out.append(("write", per(start))) + out.append(("write", end(start, "write"))) let reads = block { var out: [(String, Span)] = [] @@ -226,7 +245,7 @@ func round() throws -> ([(String, Span)], Int) { // Each frame written, then read through an async func whose future is ready at its // first poll, so it never suspends. - var start = now() + var start = begin() var last: FrameFrame? for _ in 0.. ([(String, Span)], Int) { } try group.finish() } - out.append(("ready", per(start))) + out.append(("ready", end(start, "ready"))) // Each read started in a task before its frame is written, which the yield lets // suspend, and the write's wake resumes it on the global executor. - start = now() + start = begin() for _ in 0.. ([(String, Span)], Int) { } try group.finish() } - out.append(("woken", per(start))) + out.append(("woken", end(start, "woken"))) // A single writer task serves the calling reader for the whole case. // Only one permit is outstanding. The write can race the first poll. - start = now() + start = begin() let (permits, send) = AsyncStream.makeStream() let writer = Task { var next = permits.makeAsyncIterator() @@ -283,7 +302,7 @@ func round() throws -> ([(String, Span)], Int) { } send.finish() try await writer.value - out.append(("loop", per(start))) + out.append(("loop", end(start, "loop"))) var sum = 0 for size in sizes { @@ -305,7 +324,7 @@ func round() throws -> ([(String, Span)], Int) { consumed.value = 0 let done = Cell() done.value = false - start = now() + start = begin() Task(executorPreference: queue) { var subscriber = try! await wakeTrack.consume().subscribe(nil) for _ in 0.. ([(String, Span)], Int) { try group.finish() } precondition(done.value == true) - out.append(("drained", per(start))) + out.append(("drained", end(start, "drained"))) // The last frame's 100 bytes, returned as Bytes, and one of them read. - start = now() + start = begin() for _ in 0.. #include #include #include +#include "codspeed.h" #include "moq.h" #define GROUPS 10000 @@ -28,12 +31,14 @@ int main(void) { return 1; } - /* Two rounds, printing the second, as bench/export.c does. */ + /* Two rounds, printing the second, as bench/export.c does; under CodSpeed, one. */ + bool measured = codspeed_init("ffi/moq/bench/moq-c.c::moq-c"); uint8_t payload[100] = {0}; uint64_t micros = 0; - for (int round = 0; round < 2; round++) { + for (int round = measured; round < 2; round++) { double wall = clock_ns(CLOCK_MONOTONIC); double cpu = clock_ns(CLOCK_PROCESS_CPUTIME_ID); + codspeed_start(); for (int g = 0; g < GROUPS; g++) { int32_t group = moq_publish_track_group((uint32_t)track); for (int f = 0; f < FRAMES; f++) { @@ -44,10 +49,11 @@ int main(void) { } moq_publish_group_finish((uint32_t)group); } + codspeed_stop("stamped"); wall = (clock_ns(CLOCK_MONOTONIC) - wall) / (GROUPS * FRAMES); cpu = (clock_ns(CLOCK_PROCESS_CPUTIME_ID) - cpu) / (GROUPS * FRAMES); - if (round == 1) { + if (round == 1 && !measured) { printf("stamped\t%f\t%f\n", wall, cpu); } } diff --git a/flake.lock b/flake.lock index 7f391b61a..3d93ac3f4 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,23 @@ "type": "github" } }, + "instrument-hooks": { + "flake": false, + "locked": { + "lastModified": 1793015678, + "narHash": "sha256-d86nCCmv++8BsuCCDbmaw4xdifbn22ic7R7dMWxfxR8=", + "owner": "CodSpeedHQ", + "repo": "instrument-hooks", + "rev": "8fd5b878499f819561b33219cdafaa6f8c28b4e4", + "type": "github" + }, + "original": { + "owner": "CodSpeedHQ", + "repo": "instrument-hooks", + "rev": "8fd5b878499f819561b33219cdafaa6f8c28b4e4", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1794158174, @@ -122,6 +139,7 @@ "root": { "inputs": { "flake-utils": "flake-utils", + "instrument-hooks": "instrument-hooks", "nixpkgs": "nixpkgs", "quest": "quest", "ra-jit": "ra-jit", diff --git a/flake.nix b/flake.nix index f4f9d27a2..605cf19ce 100644 --- a/flake.nix +++ b/flake.nix @@ -23,6 +23,12 @@ inputs.rust-overlay.follows = "rust-overlay"; inputs.flake-utils.follows = "flake-utils"; }; + # CodSpeed's instrument-hooks, which the binding benchmark's programs call around each + # case its CPU simulation counts. + instrument-hooks = { + url = "github:CodSpeedHQ/instrument-hooks/8fd5b878499f819561b33219cdafaa6f8c28b4e4"; + flake = false; + }; }; outputs = @@ -33,6 +39,7 @@ rust-overlay, quest, ra-jit, + instrument-hooks, ... }: flake-utils.lib.eachSystem @@ -265,6 +272,8 @@ ]; # `just ffi bench` builds moq's rs/moq-c, whose VAAPI bindings run bindgen. LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + # ... and its programs over instrument-hooks, for CodSpeed. + CODSPEED_INSTRUMENT_HOOKS = "${instrument-hooks}"; }); # Clang beside the default shell's GCC, which the nightly builds the exporter's C++ with. devShells.clang = self.devShells.${system}.default.overrideAttrs (old: { diff --git a/quest/m1/ffi-speed/README.md b/quest/m1/ffi-speed/README.md index 32f3aa11b..dc346c521 100644 --- a/quest/m1/ffi-speed/README.md +++ b/quest/m1/ffi-speed/README.md @@ -17,11 +17,13 @@ ABI or a generated API. Kotlin records, moved here from it. Rejected: keeping them there, where three quests that change no API held the pin open. -Each child reports its numbers before and after in its PR, on the benchmark where it has a -row. +Each child's PR shows its change on the benchmark where it has a row through CodSpeed's +comment on the PR, as [the benchmark's README](/bench/ffi/README.md#codspeed) says, and +copies numbers by hand only for what CodSpeed does not count: wall and CPU time, the +contention cases, a language that left CodSpeed, and a measurement of its own. Decided with the maintainer on 2026-11-12, in `/quest-plan`: ✅ the benchmark's own -measurement work joins this epic, as [CodSpeed](/quest/m1/ffi-speed/codspeed.md), +measurement work joins this epic, as CodSpeed, the export's cost (since done), and concurrent calls, since they serve its PRs. Rejected: standalone m1 quests; holding the last two in m3. diff --git a/quest/m2/wasm/backend.md b/quest/m2/wasm/backend.md index 712b2fbae..f3b06c418 100644 --- a/quest/m2/wasm/backend.md +++ b/quest/m2/wasm/backend.md @@ -63,7 +63,9 @@ Decided with the maintainer on 2026-11-12, in the quest audit methods, as this quest first planned and the other languages' bindings have. - ✅ This quest adds a TypeScript column to [the binding benchmark](/bench/ffi/README.md), in Node and a browser, as every generator is measured. Its stand-in must build for - wasm32, which moq-tokio does not: export the crates of it that do, and say which. + wasm32, which moq-tokio does not: export the crates of it that do, and say which. The + column joins CodSpeed under the A/A rule of + [the benchmark's README](/bench/ffi/README.md#codspeed). Open, for the maintainer once the implementer has the facts: From 63d72107f7d364d22aa1df993bc5d4a56240d26f Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:46:57 -0800 Subject: [PATCH 3/8] fix(bench): Go's reader loop writes with a Timestamp's pointer #742 made WriteFrame take *Timestamp; the loop #755 added in parallel still passed the value, so the Go program no longer built. Co-Authored-By: Claude Opus 5.5 --- ffi/moq/bench/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/moq/bench/export.go b/ffi/moq/bench/export.go index 38a965968..ed9602185 100644 --- a/ffi/moq/bench/export.go +++ b/ffi/moq/bench/export.go @@ -320,7 +320,7 @@ func round(measured bool) []result { group := must(track.AppendGroup()) for range frames { <-permits - check(group.WriteFrame(at, frame)) + check(group.WriteFrame(&at, frame)) } <-permits // Keep this group until its reader has consumed every frame. check(group.Finish()) From 776c755896aca8f39e667b75a9db54c538790fd5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:15:25 -0800 Subject: [PATCH 4/8] feat(bench): CodSpeed counts the languages that pass A/A, a tenth of the calls Each program runs a tenth of the groups under CodSpeed, whose simulation counts every call exactly, so the run fits a hosted runner. Swift, Go, Kotlin, and Dart leave codspeed_langs: a local A/A pair moved a case past CodSpeed's 10% threshold or the runtime could not run under Valgrind, as bench/ffi/README.md's CodSpeed section records. kotlinc gets a 2 GiB heap, since moq's Kotlin outgrew the launcher's default. The quest's file goes with it. Co-Authored-By: Claude Opus 5.5 --- bench/ffi/README.md | 35 ++++++++++++++----- ffi/justfile | 9 ++--- ffi/moq/bench/export.c | 7 ++-- ffi/moq/bench/export.cpp | 10 ++++-- ffi/moq/bench/export.dart | 27 ++++++++------- ffi/moq/bench/export.go | 19 +++++++---- ffi/moq/bench/export.kt | 6 ++-- ffi/moq/bench/export.py | 26 +++++++-------- ffi/moq/bench/export.rs | 12 ++++--- ffi/moq/bench/export.swift | 6 ++-- ffi/moq/bench/moq-c.c | 6 +++- quest/m1/ffi-speed/codspeed.md | 61 ---------------------------------- 12 files changed, 104 insertions(+), 120 deletions(-) delete mode 100644 quest/m1/ffi-speed/codspeed.md diff --git a/bench/ffi/README.md b/bench/ffi/README.md index 6550f8d86..6890773e9 100644 --- a/bench/ffi/README.md +++ b/bench/ffi/README.md @@ -92,17 +92,34 @@ nix develop .#bench --command codspeed run --mode simulation -- just ffi codspee Each program calls CodSpeed's [instrument-hooks](https://github.com/CodSpeedHQ/instrument-hooks), which the flake pins, through [`codspeed.h`](../../ffi/moq/bench/codspeed.h) and its own FFI around each case: -one benchmark per language and case, such as `kotlin::write` or `c::throughput[1024]`. -Under CodSpeed a program runs its round once, without the warm-up round, and skips the +one benchmark per language and case, such as `python::write` or `c::throughput[1024]`. +Under CodSpeed a program runs its round once, without the warm-up round, and with a tenth +of the groups (so 100,000 calls a case, Python's 10,000), since the simulation counts +every call exactly and costs 50 to 100 times the program's own time. It skips the `shared:*`, `own:*`, and Kotlin's `objects` cases: the simulation runs one thread at a time, so it would count a contention that never happens. It counts instructions and -simulated cache misses, not time, so a system call or a wait costs nothing in it: Swift's -and Go's woken reads and reader loops count their threads' work, not their wake. The -tables below keep wall and CPU time. - -A language joins only if its numbers hold still: the same commit, run twice, must give -each of its cases the same count. `codspeed_langs` in [`ffi/justfile`](../../ffi/justfile) -lists those that do. +simulated cache misses, not time, so a system call or a wait costs nothing in it. The +tables below keep wall and CPU time, and a case's absolute count is for its whole loop, +not a call: read CodSpeed for each case's change. + +A language joins only if its counts hold still: the same build, run twice, must move no +case far enough for CodSpeed to report a change, which its default 10% regression +threshold decides. `codspeed_langs` in [`ffi/justfile`](../../ffi/justfile) lists those +that pass. Measured 2026-11-14 on **nixos** with stock Valgrind 3.27.1 and the flags +CodSpeed's runner passes, without its fork's cycle estimation, in pairs of runs of one +build, the largest move of any case between two runs: + +| language | largest move | | +|---|---|---| +| Rust, C, C++ | none: every count identical | joins | +| moq-c | 0.36% (`stamped`, with all the calls), 0.1% with a tenth | joins: moq's runtime thread runs beside it | +| Python | 0.21% (`throughput[1024]`) | joins: asyncio's poll of its selector depends on time | +| Swift | 52% (`throughput[16384]`), 8.8% in another pair | leaves: the global executor's threads interleave by time | +| Go | 362% (`stamped`), 330% (`record`), 84% (`loop`) | leaves: its collector and scheduler run inside each case, and its preemption signal crashes Callgrind unless `GODEBUG=asyncpreemptoff=1` | +| Kotlin | 172% (`call`), 108% (`getter`) | leaves: without the warm-up, the JIT's and collector's threads compile and collect inside each case, and the JVM's C2 code crashes under Valgrind unless its implicit null checks are off | +| Dart | does not run | leaves: the VM maps its code at an address hint off a page boundary, which Valgrind refuses | + +Each leaver keeps its hooks, so it rejoins by `codspeed_langs` once a pair of runs holds. ## Concurrent calls diff --git a/ffi/justfile b/ffi/justfile index 952e4232b..b127fefff 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -20,8 +20,8 @@ swift_collections_rev := "f96b909e1a9e469620bc5037c1e24e00153f376f" # The binding benchmark's programs, each a language, as bench-program runs them. bench_langs := "rust c cpp moq-c kotlin python swift go dart" -# Those whose cases CodSpeed counts the same on every run of one commit, as bench/ffi/README.md's CodSpeed section says. -codspeed_langs := "rust c cpp moq-c kotlin python swift go dart" +# Those whose cases CodSpeed counts the same on every run of one commit: bench/ffi/README.md's CodSpeed section says why Swift, Kotlin, Go, and Dart do not. +codspeed_langs := "rust c cpp moq-c python" default: @just --list @@ -97,7 +97,8 @@ bench-build checkout="": (moq-export checkout) codspeed-hooks (cd target/bench-go && go build -o bench .) {{ quote(just_executable()) }} dart-program bench-dart moq/bench/export.dart rm -rf target/bench-kt - kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/bench/export.kt -d target/bench-kt + # moq's Kotlin outgrows the 512 MiB heap kotlinc's launcher defaults to. + JAVA_OPTS="-Xmx2g -Xms128M" kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/bench/export.kt -d target/bench-kt {{ quote(just_executable()) }} swift-module -O swiftc -O -warnings-as-errors $({{ quote(just_executable()) }} swift-target 15) -I target/swift -I target/swift-collections/build -I "$out/swift" -I "$hooks" moq/bench/export.swift target/swift/libMoq.a -L target/swift-collections/build -lBasicContainers -lInternalCollectionsUtilities -lSpanPreview "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export-swift @@ -229,7 +230,7 @@ moq checkout="": (moq-cpp checkout) target/moq-test # The Kotlin, on the JVM, through the JNI functions the kotlin feature builds. rm -rf target/moq-kt - kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/moq.kt -d target/moq-kt + JAVA_OPTS="-Xmx2g -Xms128M" kotlinc -Werror -cp "$KOTLINX_COROUTINES" "$out/kotlin/moq.kt" moq/moq.kt -d target/moq-kt kotlin -Djava.library.path="$out/target/release" -cp "target/moq-kt:$KOTLINX_COROUTINES" MoqKt # The Python, through the CPython extension the python feature builds, and its types # through mypy and pyright, as `api_py_types` checks the api fixture's. diff --git a/ffi/moq/bench/export.c b/ffi/moq/bench/export.c index 1974a4ee5..6e7ad7bfd 100644 --- a/ffi/moq/bench/export.c +++ b/ffi/moq/bench/export.c @@ -17,7 +17,8 @@ #include "codspeed.h" #include "moq.h" -#define GROUPS 10000 +/* A tenth under CodSpeed, whose simulation counts every call exactly. */ +static int GROUPS = 10000; #define FRAMES 100 #define N (GROUPS * FRAMES) #define THREADS 4 @@ -396,7 +397,9 @@ static void round_(int print) { int main(void) { /* Under CodSpeed, one round, which prints nothing: its simulation counts each case. */ measured = codspeed_init("ffi/moq/bench/export.c::c"); - if (!measured) { + if (measured) { + GROUPS /= 10; + } else { round_(0); } round_(!measured); diff --git a/ffi/moq/bench/export.cpp b/ffi/moq/bench/export.cpp index 2a6cd3f76..9871cd29a 100644 --- a/ffi/moq/bench/export.cpp +++ b/ffi/moq/bench/export.cpp @@ -19,9 +19,10 @@ namespace { -constexpr int groups = 10000; +// A tenth under CodSpeed, whose simulation counts every call exactly. +int groups = 10000; constexpr int frames = 100; -constexpr int n = groups * frames; +int n = groups * frames; constexpr int threads = 4; constexpr std::size_t sizes[] = {100, 1024, 16384, 65536, 1048576}; @@ -310,7 +311,10 @@ void round(bool print) { int main() { // Under CodSpeed, one round, which prints nothing: its simulation counts each case. measured = codspeed_init("ffi/moq/bench/export.cpp::cpp"); - if (!measured) { + if (measured) { + groups /= 10; + n = groups * frames; + } else { round(false); } round(!measured); diff --git a/ffi/moq/bench/export.dart b/ffi/moq/bench/export.dart index c7daeb881..e0a16f1ff 100644 --- a/ffi/moq/bench/export.dart +++ b/ffi/moq/bench/export.dart @@ -11,9 +11,10 @@ import 'dart:typed_data'; import 'package:moq/moq.dart' as moq; -const groups = 10000; +/// A tenth under CodSpeed, whose simulation counts every call exactly. +var groups = 10000; const frames = 100; -const n = groups * frames; +int get n => groups * frames; const sizes = [100, 1024, 16384, 65536, 1048576]; /// Wall and CPU nanoseconds: a point, or a span per call. @@ -42,8 +43,9 @@ Span now() { } /// Nanoseconds per call since `start`, over `calls` calls. -Span per(Span start, [num calls = n]) { +Span per(Span start, [num? calls]) { final end = now(); + calls ??= n; return Span((end.wall - start.wall) / calls, (end.cpu - start.cpu) / calls); } @@ -60,15 +62,16 @@ Span begin() { } /// The case `name`'s nanoseconds per call since `start`, over `calls` calls. -Span end(Span start, String name, [num calls = n]) { +Span end(Span start, String name, [num? calls]) { final c = _cString(name); _codspeedStop(c); _free(c.cast()); return per(start, calls); } -final _malloc = _process - .lookupFunction Function(ffi.Size), ffi.Pointer Function(int)>('malloc'); +final _malloc = _process.lookupFunction Function(ffi.Size), ffi.Pointer Function(int)>( + 'malloc', +); final _free = _process.lookupFunction), void Function(ffi.Pointer)>( 'free', ); @@ -226,6 +229,7 @@ Future> round() async { } await next.cancel(); } + final writer = writeLoop(); for (var g = 0; g < groups; g++) { final reader = (await subscriber.recvGroup())!; @@ -259,18 +263,17 @@ Future> round() async { Future main(List args) async { final codspeed = ffi.DynamicLibrary.open(args.single); - _codspeedInit = codspeed.lookupFunction), bool Function(ffi.Pointer)>( - 'codspeed_init', - ); + _codspeedInit = codspeed + .lookupFunction), bool Function(ffi.Pointer)>('codspeed_init'); _codspeedStart = codspeed.lookupFunction('codspeed_start'); - _codspeedStop = codspeed.lookupFunction), void Function(ffi.Pointer)>( - 'codspeed_stop', - ); + _codspeedStop = codspeed + .lookupFunction), void Function(ffi.Pointer)>('codspeed_stop'); final bench = _cString('ffi/moq/bench/export.dart::dart'); final measured = _codspeedInit(bench); _free(bench.cast()); // Under CodSpeed, one round, which prints nothing: its simulation counts each case. if (measured) { + groups ~/= 10; await round(); return; } diff --git a/ffi/moq/bench/export.go b/ffi/moq/bench/export.go index ed9602185..fc0bf0b44 100644 --- a/ffi/moq/bench/export.go +++ b/ffi/moq/bench/export.go @@ -22,12 +22,17 @@ import ( ) const ( - groups = 10000 frames = 100 - n = groups * frames threads = 4 ) +// groups is a tenth under CodSpeed, whose simulation counts every call exactly, and n the +// calls a case makes. +var ( + groups = 10000 + n = groups * frames +) + var sizes = []int{100, 1024, 16384, 65536, 1048576} // span is wall and CPU nanoseconds: a point, or a span per call. @@ -61,11 +66,11 @@ func begin() span { } // end is the case name's nanoseconds per call since start, over calls calls. -func end(start span, name string, calls float64) span { +func end(start span, name string, calls int) span { c := C.CString(name) C.codspeed_stop(c) C.free(unsafe.Pointer(c)) - return per(start, calls) + return per(start, float64(calls)) } // must is v, failing the bench on err. @@ -107,7 +112,7 @@ func throughput(ctx context.Context, track *net.TrackProducer, subscriber *net.T check(group.Finish()) group.Close() } - ns := end(start, name, float64(count*frames*size)) + ns := end(start, name, count*frames*size) return span{1e3 / ns.wall, 1e3 / ns.cpu} } @@ -150,7 +155,7 @@ func concurrent(calls []func() int) span { start := now() close(gate) done.Wait() - ns := per(start, n) + ns := per(start, float64(n)) for _, sum := range sums { sink += sum } @@ -364,6 +369,8 @@ func main() { C.free(unsafe.Pointer(bench)) // Under CodSpeed, one round, which prints nothing: its simulation counts each case. if measured { + groups /= 10 + n = groups * frames round(true) return } diff --git a/ffi/moq/bench/export.kt b/ffi/moq/bench/export.kt index c7e38c4d7..c2c2d2460 100644 --- a/ffi/moq/bench/export.kt +++ b/ffi/moq/bench/export.kt @@ -12,9 +12,10 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.runBlocking import moq.* -const val GROUPS = 10000 +// A tenth under CodSpeed, whose simulation counts every call exactly. +var GROUPS = 10000 const val FRAMES = 100 -const val N = GROUPS * FRAMES +val N get() = GROUPS * FRAMES const val THREADS = 4 val SIZES = listOf(100, 1024, 16384, 65536, 1048576) @@ -276,6 +277,7 @@ fun round(measured: Boolean): Map { fun main() { // Under CodSpeed, one round, which prints nothing: its simulation counts each case. if (Codspeed.init("ffi/moq/bench/export.kt::kotlin")) { + GROUPS /= 10 round(true) return } diff --git a/ffi/moq/bench/export.py b/ffi/moq/bench/export.py index b2aa8f201..160aede1b 100644 --- a/ffi/moq/bench/export.py +++ b/ffi/moq/bench/export.py @@ -11,7 +11,19 @@ import moq -GROUPS = 1000 +codspeed = ctypes.CDLL(sys.argv[1]) +codspeed.codspeed_init.argtypes = [ctypes.c_char_p] +codspeed.codspeed_init.restype = ctypes.c_bool +codspeed.codspeed_start.argtypes = [] +codspeed.codspeed_start.restype = None +codspeed.codspeed_stop.argtypes = [ctypes.c_char_p] +codspeed.codspeed_stop.restype = None +start_case = codspeed.codspeed_start +stop_case = codspeed.codspeed_stop +measured = codspeed.codspeed_init(b"ffi/moq/bench/export.py::python") + +# A tenth under CodSpeed, whose simulation counts every call exactly. +GROUPS = 100 if measured else 1000 FRAMES = 100 N = GROUPS * FRAMES THREADS = 4 @@ -29,17 +41,6 @@ def per(start, n=N): return (wall - start[0]) / n, (cpu - start[1]) / n -codspeed = ctypes.CDLL(sys.argv[1]) -codspeed.codspeed_init.argtypes = [ctypes.c_char_p] -codspeed.codspeed_init.restype = ctypes.c_bool -codspeed.codspeed_start.argtypes = [] -codspeed.codspeed_start.restype = None -codspeed.codspeed_stop.argtypes = [ctypes.c_char_p] -codspeed.codspeed_stop.restype = None -start_case = codspeed.codspeed_start -stop_case = codspeed.codspeed_stop - - def begin(): """A case's start: now, and CodSpeed counting from here.""" start = now() @@ -237,7 +238,6 @@ async def write_loop(): # Under CodSpeed, one round, which prints nothing: its simulation counts each case. -measured = codspeed.codspeed_init(b"ffi/moq/bench/export.py::python") loop = asyncio.new_event_loop() if measured: round(loop, True) diff --git a/ffi/moq/bench/export.rs b/ffi/moq/bench/export.rs index 6b40b8597..488eee17f 100644 --- a/ffi/moq/bench/export.rs +++ b/ffi/moq/bench/export.rs @@ -12,6 +12,7 @@ use std::sync::Barrier; use std::task::{Context, Poll, Waker}; use std::time::Instant; +/// A tenth under CodSpeed, whose simulation counts every call exactly. const GROUPS: u64 = 10000; const FRAMES: u64 = 100; const THREADS: usize = 4; @@ -132,7 +133,8 @@ fn round(measured: bool) -> Vec<(String, (f64, f64))> { let info = moq::track::Info::default().with_timescale(moq::Timescale::MICRO); let track = broadcast.create_track("bench", info).unwrap(); let frame = [0u8; 100]; - let n = GROUPS * FRAMES; + let groups = if measured { GROUPS / 10 } else { GROUPS }; + let n = groups * FRAMES; let mut out = Vec::new(); let start = begin(); @@ -181,7 +183,7 @@ fn round(measured: bool) -> Vec<(String, (f64, f64))> { let mut micros = 0; let start = begin(); - for _ in 0..GROUPS { + for _ in 0..groups { let mut group = track.append_group().unwrap(); for _ in 0..FRAMES { let at = moq::Timestamp::from_micros(micros).unwrap(); @@ -194,7 +196,7 @@ fn round(measured: bool) -> Vec<(String, (f64, f64))> { let at = moq::Timestamp::from_micros(micros).unwrap(); let start = begin(); - for _ in 0..GROUPS { + for _ in 0..groups { let mut group = track.append_group().unwrap(); for _ in 0..FRAMES { group.write_frame(at, frame.as_slice()).unwrap(); @@ -206,7 +208,7 @@ fn round(measured: bool) -> Vec<(String, (f64, f64))> { let mut subscriber = ready(track.consume().subscribe(None)).unwrap(); let mut last = None; let start = begin(); - for _ in 0..GROUPS { + for _ in 0..groups { let mut group = track.append_group().unwrap(); let mut reader = ready(subscriber.recv_group()).unwrap().unwrap(); for _ in 0..FRAMES { @@ -218,7 +220,7 @@ fn round(measured: bool) -> Vec<(String, (f64, f64))> { out.push(("ready".into(), end(start, n, c"ready"))); let start = begin(); - for _ in 0..GROUPS { + for _ in 0..groups { let mut group = track.append_group().unwrap(); let mut reader = ready(subscriber.recv_group()).unwrap().unwrap(); for _ in 0..FRAMES { diff --git a/ffi/moq/bench/export.swift b/ffi/moq/bench/export.swift index e1e48fd20..4cedd5cf4 100644 --- a/ffi/moq/bench/export.swift +++ b/ffi/moq/bench/export.swift @@ -7,7 +7,10 @@ import CodSpeed import Foundation import Moq -let groups = 10000 +/// Whether CodSpeed counts this run, which then skips the warm-up and the threaded cases. +let measured = codspeed_init("ffi/moq/bench/export.swift::swift") +/// A tenth under CodSpeed, whose simulation counts every call exactly. +let groups = measured ? 1000 : 10000 let frames = 100 let n = groups * frames let threads = 4 @@ -361,7 +364,6 @@ func round(_ measured: Bool) throws -> ([(String, Span)], Int) { } // Under CodSpeed, one round, which prints nothing: its simulation counts each case. -let measured = codspeed_init("ffi/moq/bench/export.swift::swift") var sink = 0 if !measured { sink += try round(false).1 diff --git a/ffi/moq/bench/moq-c.c b/ffi/moq/bench/moq-c.c index 18aa123a5..6f6fb0f14 100644 --- a/ffi/moq/bench/moq-c.c +++ b/ffi/moq/bench/moq-c.c @@ -12,7 +12,8 @@ #include "codspeed.h" #include "moq.h" -#define GROUPS 10000 +/* A tenth under CodSpeed, as bench/export.c. */ +static int GROUPS = 10000; #define FRAMES 100 static double clock_ns(clockid_t clock) { @@ -33,6 +34,9 @@ int main(void) { /* Two rounds, printing the second, as bench/export.c does; under CodSpeed, one. */ bool measured = codspeed_init("ffi/moq/bench/moq-c.c::moq-c"); + if (measured) { + GROUPS /= 10; + } uint8_t payload[100] = {0}; uint64_t micros = 0; for (int round = measured; round < 2; round++) { diff --git a/quest/m1/ffi-speed/codspeed.md b/quest/m1/ffi-speed/codspeed.md deleted file mode 100644 index 70872ae7f..000000000 --- a/quest/m1/ffi-speed/codspeed.md +++ /dev/null @@ -1,61 +0,0 @@ -# [L] The binding benchmark on CodSpeed - -## Goal - -Every single-threaded case of [the binding benchmark](/bench/ffi/README.md), in every language, -reports to -[CodSpeed](https://app.codspeed.io/kixel-ai) under its CPU simulation, so main keeps a history -and each PR that can move a binding's cost gets a comment with its change per case. Today the -numbers exist only in the nightly's summary, on a loaded host, and each -[ffi-speed](/quest/m1/ffi-speed/README.md) child copies its before and after by hand. - -## Plan - -Decided with the maintainer on 2026-11-12, in `/quest-plan`: - -- ✅ Informational (rejected: a required merge gate, nightly history only). CodSpeed's check is - not required and no threshold blocks a merge, as no rust2 benchmark sets one. -- ✅ The binding benchmark only (rejected: the build benchmark, whose runs take hours and need a - reset between rounds that `codspeed exec` lacks; the ts benchmark, since CodSpeed drives no - browser and its Node simulation turns off V8's optimizing tiers; the parked go benchmark). -- ✅ CPU simulation on hosted `ubuntu-24.04` (rejected: walltime on CodSpeed's macro runners; - walltime on `nixos`, which CodSpeed's tools don't install on, and which hosts the queue - runner; adding the memory instrument). The repository is public, so hosted runners and - CodSpeed cost nothing. Simulation counts no system calls, so the woken read's wake is - undercounted; the nightly report keeps wall and CPU time. -- ✅ Every language, A/A-checked (rejected: compiled languages and Python only; the Rust side - only, through divan on the C ABI). Each program, Rust's floor and moq-c included, calls - CodSpeed's [instrument-hooks](https://github.com/CodSpeedHQ/instrument-hooks) through its - own FFI around each case, one benchmark per language and case, such as `kotlin::write`. - Run the same commit twice: a language whose numbers move between the two leaves CodSpeed, - with both runs' numbers in the PR and the reason in the README. JVM, Go, and Dart runtimes - under Valgrind are the likely movers. -- ✅ Runs on pushes to main, its baseline, and on PRs that change what `just ffi bench` - reads (rejected: every PR; main and dispatch only): `ffi/`, `bench/ffi/`, the flake, the - workflow itself, and what `ffi/` takes from `ts/`, the moq pin `ts/justfile` reads from - `ts/fixtures/model/Cargo.toml` and `ts/rust-toolchain.toml`, which - `ffi/rust-toolchain.toml` links to. It stays out of the merge gate, in its own workflow, - with `id-token: write` for CodSpeed's OIDC upload scoped to that job. - -Guidance: - -- The CodSpeed GitHub App is installed on `kixel-ai`; enable rust2 at - [app.codspeed.io/kixel-ai](https://app.codspeed.io/kixel-ai) if it isn't, with - `agent-browser`, and set PR comments to post on change. -- A case is one run under simulation, so drop the program's warm-up repeat only where the - hooks wrap it, and keep the nightly's wall and CPU tables as they are. -- Running `nix develop` itself under Valgrind costs minutes: enter the bench shell first and - run CodSpeed's CLI inside it, or show the action's wrapping costs nothing. -- The contention cases, `shared:*` and `own:*`, stay out: simulation runs one thread at a - time, so it would report a contention number that measures no contention. -- A column added later, such as [the TypeScript one](/quest/m2/wasm/backend.md), joins under - the same A/A rule. -- Update [the benchmark's README](/bench/ffi/README.md) and the - [epic's Plan](/quest/m1/ffi-speed/README.md), whose children then point at the CodSpeed - comment instead of copying numbers. - -Public API: none. - -## Hosts - -- linux - CodSpeed's simulation runs only on Linux From 11a94ff135540318f5d5d771b706828c9c44441d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:08:30 -0800 Subject: [PATCH 5/8] fix(bench): cgo keeps its default optimization for the Go program The hooks' include path went in CGO_CFLAGS, which replaced cgo's default -O2 -g, so the generated C trampolines built unoptimized. It goes in CGO_CPPFLAGS instead. Co-Authored-By: Claude Opus 5.5 --- ffi/justfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/justfile b/ffi/justfile index b127fefff..53b58b072 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -93,7 +93,8 @@ bench-build checkout="": (moq-export checkout) codspeed-hooks c++ -O2 -std=c++17 -Wall -Wextra -Werror -I "$out/include" -I "$hooks" moq/bench/export.cpp "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export-cpp {{ quote(just_executable()) }} go-module bench-go moq/bench/export.go read -ra native < target/moq-native - export CGO_CFLAGS="-I$hooks" CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a $hooks/libcodspeed.a ${native[*]}" GOFLAGS=-buildvcs=false + # The include path goes in CGO_CPPFLAGS, so cgo keeps its default -O2 -g in CGO_CFLAGS. + export CGO_CPPFLAGS="-I$hooks" CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a $hooks/libcodspeed.a ${native[*]}" GOFLAGS=-buildvcs=false (cd target/bench-go && go build -o bench .) {{ quote(just_executable()) }} dart-program bench-dart moq/bench/export.dart rm -rf target/bench-kt From 4a71f3c0e65ee43207fca138267f3cce04ae3e73 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:06:13 -0800 Subject: [PATCH 6/8] docs(bench): CodSpeed's CI A/A pair, and its comment only on a change Two runs of one commit on CI left all 58 benchmarks untouched, Python's throughput[1024] moving most at 2.4%. CodSpeed's project now comments on a PR only when a case changes, and its status check is informational, as the workflow is. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/codspeed.yml | 3 ++- bench/ffi/README.md | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1f2c39b19..2d8df31c2 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -2,7 +2,8 @@ name: CodSpeed # The binding benchmark under CodSpeed's CPU simulation (bench/ffi/README.md): each push to # main is its baseline, and a PR that changes what `just ffi bench` reads gets CodSpeed's -# comment. Informational: no threshold fails it, and no ruleset requires it. +# comment once a case changes. Informational: no threshold fails it, and no ruleset +# requires it. permissions: contents: read diff --git a/bench/ffi/README.md b/bench/ffi/README.md index 6890773e9..787089b29 100644 --- a/bench/ffi/README.md +++ b/bench/ffi/README.md @@ -77,7 +77,8 @@ the same measurement twice under another name. ## CodSpeed [CodSpeed](https://app.codspeed.io/kixel-ai/rust2) keeps main's history of each -single-threaded case, and comments on each PR with its change per case. +single-threaded case, and comments on a PR whose run changes a case, with its change per +case. [`codspeed.yml`](../../.github/workflows/codspeed.yml) runs on every push to main, its baseline, and on each PR that changes what `just ffi bench` reads: it builds the programs in the bench shell, then runs `just ffi codspeed` under CodSpeed's CPU simulation in that @@ -119,6 +120,11 @@ build, the largest move of any case between two runs: | Kotlin | 172% (`call`), 108% (`getter`) | leaves: without the warm-up, the JIT's and collector's threads compile and collect inside each case, and the JVM's C2 code crashes under Valgrind unless its implicit null checks are off | | Dart | does not run | leaves: the VM maps its code at an address hint off a page boundary, which Valgrind refuses | +On CI, two runs of one commit (kixel-ai/rust2#805's 11a94ff, a `pull_request` run and a +`workflow_dispatch` one) left all 58 benchmarks untouched in CodSpeed's comparison: the +largest move was Python's `throughput[1024]`, 2.4%, then its `throughput[100]` and +`ready`, under 2%, and every one of Rust's, C's, C++'s, and moq-c's rounded to 0%. + Each leaver keeps its hooks, so it rejoins by `codspeed_langs` once a pair of runs holds. ## Concurrent calls From 58fab649d28461b6c13ee092cc039e415ccd319d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:00:49 -0800 Subject: [PATCH 7/8] fix(ci): give each CodSpeed baseline run its own group, and list the recipe Co-Authored-By: Claude Opus 5.5 --- .github/workflows/codspeed.yml | 5 +++-- ffi/README.md | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 2d8df31c2..5e4159d09 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -38,9 +38,10 @@ on: - "!**/*.md" workflow_dispatch: -# A PR's newer push supersedes its run; each of main's runs is a baseline, so none is cancelled. +# A PR's newer push supersedes its run. Each of main's runs is a baseline, so each takes its own +# group: a shared one would still replace a pending run with a newer one. concurrency: - group: codspeed-${{ github.ref }} + group: codspeed-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: diff --git a/ffi/README.md b/ffi/README.md index d46152560..0b530bdd1 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -1045,6 +1045,7 @@ just ffi census # census each into target/census just ffi shapes # round-trip every sampled type shape in every language but KNOWN's, as the nightly does nix develop .#android --command just ffi android # the Kotlin test, shrunk by R8, on the Android device adb reaches nix develop .#bench --command just ffi bench # the binding benchmark: bench/ffi/README.md's cases from Rust, and through the export from C, C++, Kotlin, Python, Swift, Go, and Dart +nix develop .#bench --command just ffi codspeed # after bench-build, each single-threaded case once under CodSpeed's CPU simulation, as the CodSpeed workflow runs it nix develop .#sanitize --command just ffi sanitize # api's C and C++ scripts under ASan and TSan, and its C ABI under Miri ``` From b5b117f0e984368ca323bbde0d78219f5148c1ca Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:01:06 -0800 Subject: [PATCH 8/8] fix(ci): run CodSpeed when the driver changes Co-Authored-By: Claude Opus 5.5 --- .github/workflows/codspeed.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 5e4159d09..b21a4b551 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -13,6 +13,8 @@ on: branches: [main] paths: - ffi/** + # rust2-ffi builds over the driver by path. + - driver/** - bench/ffi/** - flake.nix - flake.lock @@ -27,6 +29,7 @@ on: pull_request: paths: - ffi/** + - driver/** - bench/ffi/** - flake.nix - flake.lock