Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,8 @@ The table includes these measurements:
and Criterion's quick mode. Per-group benchmark settings can override the default warmup. Quick estimates
trade precision for nightly runtime; use full paired runs to assess a change.
- JS benchmark CSV time columns (`ns`, `us`, `ms`, and their per-operation
variants), retaining every workload dimension in each series name.
variants) and `size_bytes` columns, retaining every other column as a workload
dimension in each series name.

Hosted runners vary across nights, so a flag prompts a paired local run rather
than proving a regression. Binding ratios compare languages within one run.
Expand Down
18 changes: 6 additions & 12 deletions js/net/bench/compression.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { Producer as Track } from "../src/track.ts";
const payload = new TextEncoder().encode('{"value":123,"status":"unchanged"}\n'.repeat(64));
const frames = 32;
const rounds = 10;
console.log("publishers,readers,publish_us,read_us,stored_bytes,inflated_bytes");
console.log("publishers,readers,phase,us,size_bytes");
for (const publishers of [1, 4, 16]) {
for (const readers of [1, 4, 16]) {
let publish = 0;
Expand Down Expand Up @@ -45,16 +45,8 @@ for (const publishers of [1, 4, 16]) {
if (inflated !== publishers * frames * payload.length) throw new Error("inflated bytes scale with readers");
for (const track of tracks) track.close();
}
console.log(
[
publishers,
readers,
((publish * 1000) / rounds).toFixed(1),
((read * 1000) / rounds).toFixed(1),
stored,
inflated,
].join(","),
);
console.log([publishers, readers, "publish", ((publish * 1000) / rounds).toFixed(1), stored].join(","));
console.log([publishers, readers, "read", ((read * 1000) / rounds).toFixed(1), inflated].join(","));
}
}

Expand All @@ -67,4 +59,6 @@ const bundle = await Bun.build({
});
if (!bundle.success) throw new AggregateError(bundle.logs, "consumer bundle failed");
const bytes = new Uint8Array(await bundle.outputs[0].arrayBuffer());
console.log(`consumer_bundle_bytes=${bytes.length},gzip_bytes=${gzipSync(bytes).length}`);
console.log("measurement,size_bytes");
console.log(`consumer_bundle,${bytes.length}`);
console.log(`consumer_bundle_gzip,${gzipSync(bytes).length}`);
10 changes: 6 additions & 4 deletions sh/gh/bench-report.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ def rust(directory):
return checked(entries)


def timing_unit(column):
def measurement_unit(column):
if column == 'size_bytes':
return 'bytes'
Comment thread
kixelated marked this conversation as resolved.
Resolved
for unit in ('ns', 'us', 'ms'):
if column == unit or column.endswith(f'_{unit}') or column.startswith(f'{unit}_per_'):
return unit
Expand All @@ -95,7 +97,7 @@ def javascript_file(path):
'unit': 'us', 'value': value(float(elapsed))})
continue
row = [field.strip() for field in next(csv.reader([line]))]
if len(row) > 1 and any(timing_unit(field) for field in row):
if len(row) > 1 and any(measurement_unit(field) for field in row):
if len(set(row)) != len(row):
raise ValueError(f'duplicate CSV headers: {path}:{line_number}')
headers = row
Expand All @@ -105,9 +107,9 @@ def javascript_file(path):
if len(row) != len(headers):
raise ValueError(f'malformed benchmark CSV: {path}:{line_number}')
dimensions = '/'.join(f'{key}={quote(field, safe="-_.~")}' for key, field in zip(headers, row)
if timing_unit(key) is None)
if measurement_unit(key) is None)
for key, field in zip(headers, row):
if unit := timing_unit(key):
if unit := measurement_unit(key):
entries.append({'name': f'js/{path.stem}/{dimensions}/{key}',
'unit': unit, 'value': value(float(field))})
if not entries:
Expand Down
13 changes: 13 additions & 0 deletions sh/gh/bench-report.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,19 @@ def test_js_multiple_schemas_and_interests(self):
with self.assertRaises(ValueError):
report.javascript_file(source)

def test_js_byte_measurements_do_not_become_dimensions(self):
source = self.root / 'compression.csv'
source.write_text('bytes,us,size_bytes\n4,1.5,64\nmeasurement,size_bytes\nbundle,512\n')
entries = report.javascript_file(source)
self.assertEqual([(entry['name'], entry['unit'], entry['value']) for entry in entries], [
('js/compression/bytes=4/us', 'us', 1.5),
('js/compression/bytes=4/size_bytes', 'bytes', 64),
('js/compression/measurement=bundle/size_bytes', 'bytes', 512),
])
source.write_text('bytes,us,size_bytes\n4,2.5,128\nmeasurement,size_bytes\nbundle,1024\n')
self.assertEqual([entry['name'] for entry in report.javascript_file(source)],
[entry['name'] for entry in entries])

def test_criterion_requires_fresh_measured_estimates(self):
run = self.root / 'runs' / 'one'
run.mkdir(parents=True)
Expand Down
4 changes: 2 additions & 2 deletions sh/gh/nightly.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ case "$1" in
# only Criterion estimates enter the trend report.
criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \
--bench protect --bench codec --bench table --bench allocations --bench window \
--bench writer --bench play_buffer --bench jitter
--bench producer --bench play_buffer --bench jitter
;;
rust-uring)
# One run per target makes an unsupported/skipped ring fail report
Expand All @@ -102,7 +102,7 @@ case "$1" in
done
;;
js)
for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track resume interests priority subscribe-drop refresh observed; do
for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track resume interests priority subscribe-drop refresh observed compression; do
javascript "$benchmark" just js bench "$benchmark"
done
javascript catalog just js catalog-bench
Expand Down
4 changes: 4 additions & 0 deletions sh/gh/nightly.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,15 @@ def test_every_cargo_bench_target_runs_nightly(self):
continue
packages = [args[i + 1] for i, arg in enumerate(args) if arg == "-p"]
targets = [args[i + 1] for i, arg in enumerate(args) if arg == "--bench"]
names = set()
for package in packages:
manifest = ROOT / "rs" / package / "Cargo.toml"
for bench in tomllib.loads(manifest.read_text()).get("bench", []):
names.add(bench["name"])
if "*" in targets or bench["name"] in targets:
covered.add((package, bench["name"]))
# Cargo rejects the whole invocation when any named target is missing.
self.assertEqual(set(targets) - names - {"*"}, set(), call)
declared = {(manifest.parent.name, bench["name"])
for manifest in (ROOT / "rs").glob("*/Cargo.toml")
for bench in tomllib.loads(manifest.read_text()).get("bench", [])}
Expand Down
Loading