Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -590,11 +590,13 @@ jobs:
access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }}
secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }}
- name: Collect and compare sizes
id: sizes
run: nix develop --command just gh size-nightly size-report "$GITHUB_RUN_ID"
env:
GH_TOKEN: ${{ github.token }}
# Only after the comparison ran, so a setup failure stays the job's only error.
- name: Preserve sizes and attribution
if: ${{ !cancelled() }}
if: ${{ !cancelled() && steps.sizes.outcome != 'skipped' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: nightly-size-history
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/platform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,12 @@ jobs:
if: needs.scope.outputs.swift == 'true'
run: just swift check
- name: Swift binding benchmark
id: swift-bench
if: inputs.benchmarks
run: just bench-ffi swift
# Only after the benchmark ran, so an earlier failure stays the job's only error.
- name: Preserve Swift benchmark
if: inputs.benchmarks && !cancelled()
if: ${{ !cancelled() && steps.swift-bench.outcome != 'skipped' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: bench-ffi-swift
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/x11.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,13 @@ jobs:
access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }}
secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }}
- name: Capture comparison
id: capture
run: nix develop --command just rs x11-bench-ci
env:
MOQ_X11_BENCH_LOG: ${{ runner.temp }}/x11-capture-benchmark.log
# Only after the comparison ran, so a setup failure stays the job's only error.
- name: Keep capture measurements
if: ${{ !cancelled() }}
if: ${{ !cancelled() && steps.capture.outcome != 'skipped' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: x11-capture-benchmark
Expand Down
24 changes: 18 additions & 6 deletions bench/browser/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* traced and counted passes stay separate, so tracing and copy counting never inflate a timing.
* Correctness fails the run; timing is reported, never judged.
*/
import { appendFile } from "node:fs/promises";
import { cpus, freemem, loadavg, platform, totalmem } from "node:os";
import { join } from "node:path";
import { parseArgs } from "node:util";
Expand Down Expand Up @@ -200,7 +201,7 @@ try {
await save();
}
console.log(`Retained ${report.samples.length} samples: ${output}`);
printComparisons();
await printComparisons();

type Pass = "timed" | "traced" | "counted" | "probed";

Expand Down Expand Up @@ -651,21 +652,32 @@ function log(row: Row) {
console.log(`${row.key} ${row.variant} ${row.pass}${row.round >= 0 ? ` r${row.round}` : ""}: ${shown}`);
}

function printComparisons() {
console.log("\nPaired change, variant against reference (median of per-round deltas [range]; host load varies):");
async function printComparisons() {
const lines: string[] = [];
const line = (text: string) => {
console.log(text);
lines.push(text);
};
line("\nPaired change, variant against reference (median of per-round deltas [range]; host load varies):");
for (const entry of report.comparisons as ReturnType<typeof compare>) {
if (!HEADLINE.has(entry.metric)) continue;
const sign = (value: number) => `${value >= 0 ? "+" : ""}${value.toFixed(1)}`;
console.log(
line(
` ${entry.key} ${entry.variant} vs ${entry.reference} ${entry.metric}: ${sign(entry.median_pct)}% [${sign(entry.min_pct)}, ${sign(entry.max_pct)}] n=${entry.pairs}${entry.consistent ? "" : " (sign varies)"}`,
);
}
console.log("\nInstrumented passes (one sample each):");
line("\nInstrumented passes (one sample each):");
for (const row of report.samples) {
if (row.pass === "timed") continue;
const shown = Object.entries(row.metrics)
.filter(([name]) => INSTRUMENTED.test(name))
.map(([name, value]) => `${name}=${Number.isInteger(value) ? value : value.toFixed(3)}`);
console.log(` ${row.key} ${row.variant} ${row.pass}: ${shown.join(" ")}`);
line(` ${row.key} ${row.variant} ${row.pass}: ${shown.join(" ")}`);
}
if (process.env.GITHUB_STEP_SUMMARY) {
await appendFile(
process.env.GITHUB_STEP_SUMMARY,
`## Browser benchmark\n\n\`\`\`text\n${lines.join("\n")}\n\`\`\`\n`,
);
}
}
2 changes: 1 addition & 1 deletion bench/ffi-gen/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ just bench-ffi-gen build bench # or some steps: build | bench | size
`target/ffi-gen`, then builds the slice with each generator, its Python, Kotlin, and
Swift bindings, both `.wasm` builds, and BoltFFI's web package. `bench` runs
`MOQ_BENCH_ROUNDS` rounds (default 3), alternating which generator goes first, and
writes `target/ffi-gen/results/report.md`. Each case gets 3 warmup steps and a
writes `target/ffi-gen/results/report.md` and appends the table to the Actions job summary. Each case gets 3 warmup steps and a
`MOQ_BENCH_SECONDS` budget (default 1). Throughput mirrors `just bench-ffi`:
`clamp(8 MiB / size, 1, 64)` frames per group, every byte checked, and CPU from
`getrusage`. `size` builds moq-ffi and moq-wasm for reference, then prints the size
Expand Down
4 changes: 4 additions & 0 deletions bench/ffi-gen/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"""

import json
import os
import statistics
import sys
from collections import defaultdict
Expand Down Expand Up @@ -62,6 +63,9 @@ def main() -> None:
table = "\n".join(lines)
(directory / "report.md").write_text(table + "\n")
print(table)
if summary := os.environ.get("GITHUB_STEP_SUMMARY"):
with Path(summary).open("a") as output:
output.write("## Binding generator comparison\n\n" + table + "\n")


if __name__ == "__main__":
Expand Down
2 changes: 1 addition & 1 deletion js/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ test-changed list:
coverage package="":
python3 ../sh/js/coverage.py '{{ package }}'

# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, subscribe-drop, or compression.
# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, subscribe-drop, subscribe-tracks, refresh, or compression.
bench name:
bun install --frozen-lockfile
bun net/bench/{{ name }}.ts
Expand Down
2 changes: 1 addition & 1 deletion quest/m0/ci/queue-allgreen.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,4 @@ Public API: none. Wire: none.

## Related

- [CI workflow audit](/quest/m1/audit-ci.md) - owns per-job cost; this quest only changes the gate
- [CI workflow audit](/quest/m2/audit-ci.md) - owns per-job cost; this quest only changes the gate
1 change: 0 additions & 1 deletion quest/m1/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ tracks, so per-group stats stop costing cluster announces.
- [One Annex-B exporter](/quest/m1/annexb-export.md) - H.264 and H.265 Annex-B export share one private lifecycle, which also follows a rebound rendition
- [One CLI duration type](/quest/m1/cli-duration.md) - moq-cli, moq-bench and moq-relay reuse moq-tokio's public `cli::Duration` instead of their own copies
- [Shared watch and publish UI](/quest/m1/js-ui.md) - a new `@moq/ui` package holds the graph, buttons, and support banner that watch and publish duplicate
- [CI workflow audit](/quest/m1/audit-ci.md) - every workflow and job gives a signal someone sees, at a cost that signal justifies, or is fixed, moved, or deleted
- [Simplification audit](/quest/m1/audit-simplify.md) - repeat the simplification audit, starting with the m0 leftovers, before the next major release
- [Artifact size audit](/quest/m1/audit-size.md) - binary, FFI, WASM, and bundle growth since the m0 run is explained, fixed, or planned
- [Compile time audit](/quest/m1/audit-compile-time.md) - build-time hotspots are measured against the m0 baselines and the worst are fixed or planned
Expand Down
117 changes: 0 additions & 117 deletions quest/m1/audit-ci.md

This file was deleted.

2 changes: 1 addition & 1 deletion quest/m1/audit-compile-time.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ independently landable and too large to review alongside the rest becomes a
follow-up through `/quest-plan`.

Scope: this audit owns per-crate build cost in one build (dev, release, and
incremental). The [CI workflow audit](/quest/m1/audit-ci.md) owns job-level CI
incremental). The [CI workflow audit](/quest/m2/audit-ci.md) owns job-level CI
cost and caching; the [dependency audit](/quest/m2/audit-deps.md) owns
deduplicating and upgrading dependencies.

Expand Down
2 changes: 1 addition & 1 deletion quest/m1/ci-housekeeping.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@ Public API: none. Wire: none.

## Related

- [CI workflow audit](/quest/m1/audit-ci.md) - the broader per-job review
- [CI workflow audit](/quest/m2/audit-ci.md) - the broader per-job review
1 change: 1 addition & 0 deletions quest/m2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,3 +227,4 @@ ranked just below deterministic TS legs, which it requires.
- [Test cost audit](/quest/m2/audit-test-cost.md) - the slowest, flakiest, and least valuable tests are fixed, deleted, or planned once the release ships

- [Quest tree audit](/quest/m2/audit-quests.md) - repeat the whole-tree audit after the release ships, against the m1 baseline
- [CI workflow audit](/quest/m2/audit-ci.md) - repeat the per-workflow and per-job review after the release ships, against the m1 baselines
Loading
Loading