From d1ff69388e3a56d14ced9dbf33219a6ce777d912 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 22:35:21 -0700 Subject: [PATCH 1/4] ci(bench): run the archive producer benchmark nightly Co-Authored-By: GPT-6 (cherry picked from commit dfc70437470a374370fb46540ae5cfdaf2bf31a7) --- sh/gh/nightly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 909b9b3268c..9365455d9b0 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -92,7 +92,7 @@ case "$1" in # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \ --bench protect --bench codec --bench table --bench allocations --bench window \ - --bench writer --bench play_buffer --bench jitter + --bench writer --bench producer --bench play_buffer --bench jitter ;; rust-uring) # One run per target makes an unsupported/skipped ring fail report From 4d0c4872d28e66039e30d0e4e30685886dea1543 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 08:48:58 -0700 Subject: [PATCH 2/4] fix(ci): include compression measurements in nightly reports Co-Authored-By: GPT-6 --- js/net/bench/compression.ts | 18 ++++++------------ sh/gh/bench-report.py | 10 ++++++---- sh/gh/bench-report.test.py | 13 +++++++++++++ sh/gh/nightly.sh | 2 +- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/js/net/bench/compression.ts b/js/net/bench/compression.ts index ef0ad91dc4c..75c4ba2d195 100644 --- a/js/net/bench/compression.ts +++ b/js/net/bench/compression.ts @@ -7,7 +7,7 @@ import { Producer as Track } from "../src/track.ts"; const payload = new TextEncoder().encode('{"value":123,"status":"unchanged"}\n'.repeat(64)); const frames = 32; const rounds = 10; -console.log("publishers,readers,publish_us,read_us,stored_bytes,inflated_bytes"); +console.log("publishers,readers,phase,us,size_bytes"); for (const publishers of [1, 4, 16]) { for (const readers of [1, 4, 16]) { let publish = 0; @@ -45,16 +45,8 @@ for (const publishers of [1, 4, 16]) { if (inflated !== publishers * frames * payload.length) throw new Error("inflated bytes scale with readers"); for (const track of tracks) track.close(); } - console.log( - [ - publishers, - readers, - ((publish * 1000) / rounds).toFixed(1), - ((read * 1000) / rounds).toFixed(1), - stored, - inflated, - ].join(","), - ); + console.log([publishers, readers, "publish", ((publish * 1000) / rounds).toFixed(1), stored].join(",")); + console.log([publishers, readers, "read", ((read * 1000) / rounds).toFixed(1), inflated].join(",")); } } @@ -67,4 +59,6 @@ const bundle = await Bun.build({ }); if (!bundle.success) throw new AggregateError(bundle.logs, "consumer bundle failed"); const bytes = new Uint8Array(await bundle.outputs[0].arrayBuffer()); -console.log(`consumer_bundle_bytes=${bytes.length},gzip_bytes=${gzipSync(bytes).length}`); +console.log("measurement,size_bytes"); +console.log(`consumer_bundle,${bytes.length}`); +console.log(`consumer_bundle_gzip,${gzipSync(bytes).length}`); diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index bf6e7da1fdd..6c32c45d721 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -77,7 +77,9 @@ def rust(directory): return checked(entries) -def timing_unit(column): +def measurement_unit(column): + if column == 'size_bytes': + return 'bytes' for unit in ('ns', 'us', 'ms'): if column == unit or column.endswith(f'_{unit}') or column.startswith(f'{unit}_per_'): return unit @@ -95,7 +97,7 @@ def javascript_file(path): 'unit': 'us', 'value': value(float(elapsed))}) continue row = [field.strip() for field in next(csv.reader([line]))] - if len(row) > 1 and any(timing_unit(field) for field in row): + if len(row) > 1 and any(measurement_unit(field) for field in row): if len(set(row)) != len(row): raise ValueError(f'duplicate CSV headers: {path}:{line_number}') headers = row @@ -105,9 +107,9 @@ def javascript_file(path): if len(row) != len(headers): raise ValueError(f'malformed benchmark CSV: {path}:{line_number}') dimensions = '/'.join(f'{key}={quote(field, safe="-_.~")}' for key, field in zip(headers, row) - if timing_unit(key) is None) + if measurement_unit(key) is None) for key, field in zip(headers, row): - if unit := timing_unit(key): + if unit := measurement_unit(key): entries.append({'name': f'js/{path.stem}/{dimensions}/{key}', 'unit': unit, 'value': value(float(field))}) if not entries: diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index b4967bac0e3..9f511e8c52c 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -48,6 +48,19 @@ def test_js_multiple_schemas_and_interests(self): with self.assertRaises(ValueError): report.javascript_file(source) + def test_js_byte_measurements_do_not_become_dimensions(self): + source = self.root / 'compression.csv' + source.write_text('bytes,us,size_bytes\n4,1.5,64\nmeasurement,size_bytes\nbundle,512\n') + entries = report.javascript_file(source) + self.assertEqual([(entry['name'], entry['unit'], entry['value']) for entry in entries], [ + ('js/compression/bytes=4/us', 'us', 1.5), + ('js/compression/bytes=4/size_bytes', 'bytes', 64), + ('js/compression/measurement=bundle/size_bytes', 'bytes', 512), + ]) + source.write_text('bytes,us,size_bytes\n4,2.5,128\nmeasurement,size_bytes\nbundle,1024\n') + self.assertEqual([entry['name'] for entry in report.javascript_file(source)], + [entry['name'] for entry in entries]) + def test_criterion_requires_fresh_measured_estimates(self): run = self.root / 'runs' / 'one' run.mkdir(parents=True) diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 9365455d9b0..e33caab696e 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -102,7 +102,7 @@ case "$1" in done ;; js) - for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track resume interests priority subscribe-drop refresh observed; do + for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track resume interests priority subscribe-drop refresh observed compression; do javascript "$benchmark" just js bench "$benchmark" done javascript catalog just js catalog-bench From a190ae49ff11c0c0de865d344ad5c212b5bf3daf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 11:31:20 -0700 Subject: [PATCH 3/4] fix(ci): drop the removed archive writer bench from nightly Cargo rejects the whole rust-extra invocation while it names `writer`, which #6160 renamed to `producer`. The coverage test now fails on any named target that no selected package declares. Co-Authored-By: Claude Opus 5.5 --- sh/gh/nightly.sh | 2 +- sh/gh/nightly.test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index e33caab696e..a16ce9c60cb 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -92,7 +92,7 @@ case "$1" in # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \ --bench protect --bench codec --bench table --bench allocations --bench window \ - --bench writer --bench producer --bench play_buffer --bench jitter + --bench producer --bench play_buffer --bench jitter ;; rust-uring) # One run per target makes an unsupported/skipped ring fail report diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index a4d09095843..fc48ef9642d 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -73,11 +73,15 @@ def test_every_cargo_bench_target_runs_nightly(self): continue packages = [args[i + 1] for i, arg in enumerate(args) if arg == "-p"] targets = [args[i + 1] for i, arg in enumerate(args) if arg == "--bench"] + names = set() for package in packages: manifest = ROOT / "rs" / package / "Cargo.toml" for bench in tomllib.loads(manifest.read_text()).get("bench", []): + names.add(bench["name"]) if "*" in targets or bench["name"] in targets: covered.add((package, bench["name"])) + # Cargo rejects the whole invocation when any named target is missing. + self.assertEqual(set(targets) - names - {"*"}, set(), call) declared = {(manifest.parent.name, bench["name"]) for manifest in (ROOT / "rs").glob("*/Cargo.toml") for bench in tomllib.loads(manifest.read_text()).get("bench", [])} From 70d7ed178d32550b2667a8ff9a6b86040137930e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 12:07:28 -0700 Subject: [PATCH 4/4] docs(bench): list size_bytes as a nightly JS measurement column Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/README.md b/bench/README.md index bc7e21663f5..0783d096181 100644 --- a/bench/README.md +++ b/bench/README.md @@ -381,7 +381,8 @@ The table includes these measurements: and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates trade precision for nightly runtime; use full paired runs to assess a change. - JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation - variants), retaining every workload dimension in each series name. + variants) and `size_bytes` columns, retaining every other column as a workload + dimension in each series name. Hosted runners vary across nights, so a flag prompts a paired local run rather than proving a regression. Binding ratios compare languages within one run.