diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 66698181cc6..edca5a31179 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -590,11 +590,13 @@ jobs: access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} - name: Collect and compare sizes + id: sizes run: nix develop --command just gh size-nightly size-report "$GITHUB_RUN_ID" env: GH_TOKEN: ${{ github.token }} + # Only after the comparison ran, so a setup failure stays the job's only error. - name: Preserve sizes and attribution - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.sizes.outcome != 'skipped' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: nightly-size-history diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index 845f33f8bd5..1ef63cb0b26 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -166,10 +166,12 @@ jobs: if: needs.scope.outputs.swift == 'true' run: just swift check - name: Swift binding benchmark + id: swift-bench if: inputs.benchmarks run: just bench-ffi swift + # Only after the benchmark ran, so an earlier failure stays the job's only error. - name: Preserve Swift benchmark - if: inputs.benchmarks && !cancelled() + if: ${{ !cancelled() && steps.swift-bench.outcome != 'skipped' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: bench-ffi-swift diff --git a/.github/workflows/x11.yml b/.github/workflows/x11.yml index 319e8b02bef..7d01aaa79e4 100644 --- a/.github/workflows/x11.yml +++ b/.github/workflows/x11.yml @@ -31,11 +31,13 @@ jobs: access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} - name: Capture comparison + id: capture run: nix develop --command just rs x11-bench-ci env: MOQ_X11_BENCH_LOG: ${{ runner.temp }}/x11-capture-benchmark.log + # Only after the comparison ran, so a setup failure stays the job's only error. - name: Keep capture measurements - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.capture.outcome != 'skipped' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: x11-capture-benchmark diff --git a/bench/browser/driver.ts b/bench/browser/driver.ts index 5153b652cf6..174910381db 100644 --- a/bench/browser/driver.ts +++ b/bench/browser/driver.ts @@ -5,6 +5,7 @@ * traced and counted passes stay separate, so tracing and copy counting never inflate a timing. * Correctness fails the run; timing is reported, never judged. */ +import { appendFile } from "node:fs/promises"; import { cpus, freemem, loadavg, platform, totalmem } from "node:os"; import { join } from "node:path"; import { parseArgs } from "node:util"; @@ -200,7 +201,7 @@ try { await save(); } console.log(`Retained ${report.samples.length} samples: ${output}`); -printComparisons(); +await printComparisons(); type Pass = "timed" | "traced" | "counted" | "probed"; @@ -651,21 +652,32 @@ function log(row: Row) { console.log(`${row.key} ${row.variant} ${row.pass}${row.round >= 0 ? ` r${row.round}` : ""}: ${shown}`); } -function printComparisons() { - console.log("\nPaired change, variant against reference (median of per-round deltas [range]; host load varies):"); +async function printComparisons() { + const lines: string[] = []; + const line = (text: string) => { + console.log(text); + lines.push(text); + }; + line("\nPaired change, variant against reference (median of per-round deltas [range]; host load varies):"); for (const entry of report.comparisons as ReturnType) { if (!HEADLINE.has(entry.metric)) continue; const sign = (value: number) => `${value >= 0 ? "+" : ""}${value.toFixed(1)}`; - console.log( + line( ` ${entry.key} ${entry.variant} vs ${entry.reference} ${entry.metric}: ${sign(entry.median_pct)}% [${sign(entry.min_pct)}, ${sign(entry.max_pct)}] n=${entry.pairs}${entry.consistent ? "" : " (sign varies)"}`, ); } - console.log("\nInstrumented passes (one sample each):"); + line("\nInstrumented passes (one sample each):"); for (const row of report.samples) { if (row.pass === "timed") continue; const shown = Object.entries(row.metrics) .filter(([name]) => INSTRUMENTED.test(name)) .map(([name, value]) => `${name}=${Number.isInteger(value) ? value : value.toFixed(3)}`); - console.log(` ${row.key} ${row.variant} ${row.pass}: ${shown.join(" ")}`); + line(` ${row.key} ${row.variant} ${row.pass}: ${shown.join(" ")}`); + } + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `## Browser benchmark\n\n\`\`\`text\n${lines.join("\n")}\n\`\`\`\n`, + ); } } diff --git a/bench/ffi-gen/README.md b/bench/ffi-gen/README.md index 33bc31b8b65..c4b49b09b99 100644 --- a/bench/ffi-gen/README.md +++ b/bench/ffi-gen/README.md @@ -36,7 +36,7 @@ just bench-ffi-gen build bench # or some steps: build | bench | size `target/ffi-gen`, then builds the slice with each generator, its Python, Kotlin, and Swift bindings, both `.wasm` builds, and BoltFFI's web package. `bench` runs `MOQ_BENCH_ROUNDS` rounds (default 3), alternating which generator goes first, and -writes `target/ffi-gen/results/report.md`. Each case gets 3 warmup steps and a +writes `target/ffi-gen/results/report.md` and appends the table to the Actions job summary. Each case gets 3 warmup steps and a `MOQ_BENCH_SECONDS` budget (default 1). Throughput mirrors `just bench-ffi`: `clamp(8 MiB / size, 1, 64)` frames per group, every byte checked, and CPU from `getrusage`. `size` builds moq-ffi and moq-wasm for reference, then prints the size diff --git a/bench/ffi-gen/report.py b/bench/ffi-gen/report.py index 15983b57f81..04b8d3c90f8 100644 --- a/bench/ffi-gen/report.py +++ b/bench/ffi-gen/report.py @@ -7,6 +7,7 @@ """ import json +import os import statistics import sys from collections import defaultdict @@ -62,6 +63,9 @@ def main() -> None: table = "\n".join(lines) (directory / "report.md").write_text(table + "\n") print(table) + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with Path(summary).open("a") as output: + output.write("## Binding generator comparison\n\n" + table + "\n") if __name__ == "__main__": diff --git a/js/justfile b/js/justfile index 6c15e6f05d9..f9b21757c2c 100644 --- a/js/justfile +++ b/js/justfile @@ -55,7 +55,7 @@ test-changed list: coverage package="": python3 ../sh/js/coverage.py '{{ package }}' -# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, subscribe-drop, or compression. +# Run one net benchmark: broadcasts, forward, requests, reader, frames, track, resume, interests, observed, varint, lite-varint, priority, request-ids, subscribe-drop, subscribe-tracks, refresh, or compression. bench name: bun install --frozen-lockfile bun net/bench/{{ name }}.ts diff --git a/quest/m0/ci/queue-allgreen.md b/quest/m0/ci/queue-allgreen.md index 749d55efb0a..44b76cb4bf9 100644 --- a/quest/m0/ci/queue-allgreen.md +++ b/quest/m0/ci/queue-allgreen.md @@ -35,4 +35,4 @@ Public API: none. Wire: none. ## Related -- [CI workflow audit](/quest/m1/audit-ci.md) - owns per-job cost; this quest only changes the gate +- [CI workflow audit](/quest/m2/audit-ci.md) - owns per-job cost; this quest only changes the gate diff --git a/quest/m1/README.md b/quest/m1/README.md index e4586ae079e..01d60ffd8a8 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -166,7 +166,6 @@ tracks, so per-group stats stop costing cluster announces. - [One Annex-B exporter](/quest/m1/annexb-export.md) - H.264 and H.265 Annex-B export share one private lifecycle, which also follows a rebound rendition - [One CLI duration type](/quest/m1/cli-duration.md) - moq-cli, moq-bench and moq-relay reuse moq-tokio's public `cli::Duration` instead of their own copies - [Shared watch and publish UI](/quest/m1/js-ui.md) - a new `@moq/ui` package holds the graph, buttons, and support banner that watch and publish duplicate -- [CI workflow audit](/quest/m1/audit-ci.md) - every workflow and job gives a signal someone sees, at a cost that signal justifies, or is fixed, moved, or deleted - [Simplification audit](/quest/m1/audit-simplify.md) - repeat the simplification audit, starting with the m0 leftovers, before the next major release - [Artifact size audit](/quest/m1/audit-size.md) - binary, FFI, WASM, and bundle growth since the m0 run is explained, fixed, or planned - [Compile time audit](/quest/m1/audit-compile-time.md) - build-time hotspots are measured against the m0 baselines and the worst are fixed or planned diff --git a/quest/m1/audit-ci.md b/quest/m1/audit-ci.md deleted file mode 100644 index 4c70b366d97..00000000000 --- a/quest/m1/audit-ci.md +++ /dev/null @@ -1,117 +0,0 @@ -# [M] CI workflow audit - -## Goal - -Every CI workflow and job earns its place: its results reach someone (a gate, -a Discord alert, or a published report) at a cost proportional to that signal. -Each one that doesn't is fixed, moved, or deleted. Individual tests stay with -the [test cost audit](/quest/m2/audit-test-cost.md), and runner and cache -infrastructure with [CI cost and latency](/quest/m0/ci/README.md). - -## Plan - -Recurs once per major release (decided 2026-10-20). The PR that finishes -this run also adds the next one with the same file name at the end of the next -milestone's `Required` (m0 to m1, m1 to m2), never m3, the deferred bucket -that does not shift with releases. The successor keeps the Goal and this -paragraph; the rest of its Plan is rewritten with what this run learned: the -per-workflow table, baselines, and what to check first. A successor placed in m2 lists the -[release cut](/quest/m2/release/cut.md) under `Required`, so it waits for the -release in flight to ship. The parent milestone lists this quest under -`Required`. Only `/quest-delete` ends the recurrence. Fixes land in the run's -PR, one commit each, rather than as follow-ups. Only a change that needs a -maintainer decision (dropping a release target, scope) or is independently -landable and too large to review alongside the rest becomes a follow-up -through `/quest-plan`. - -Split from the test cost audit on 2026-10-20: this audit owns workflows and -jobs, that one owns tests. It also owns job-level build cost, which the -one-time m0 CI pipeline audit handed over on 2026-10-21 after measuring the -one-pass queue. - -Each run tables every workflow in `.github/workflows/` and every Nightly job -with three verdicts: - -- **Signal:** what regression it catches that nothing cheaper catches, and - when it last caught one. -- **Visibility:** who sees a failure or a result. A gate blocks merging, a - scheduled or publishing workflow posts to Discord `#alerts` through - [alert.yml](/.github/workflows/alert.yml) (`just gh check` enforces its - coverage), and a report lands on the - [Pages site](https://git.moq.dev/) or in a job summary. A result - only in a run artifact or log is invisible: publish it, summarize it, or - stop producing it. Each report on the Pages site owns one `gh-pages` - directory: its workflow commits only that directory, deploys the whole - branch, and joins the `benchmark-pages` concurrency group with - `queue: max`, so no deploy drops another report (decided 2026-10-20; the - benchmark, relay, and interop reports do, through `sh/gh/pages-publish.sh`). -- **Cost:** runner-minutes per month by runner type, from the org billing API - and job timings, against that signal. macOS is the scarce one (a 5-job - concurrency cap). - -Check first (from the test cost audit's m0 run, 2026-10-19): - -- Classify every failed job of the latest Nightly, scheduled Interop, and - Platform runs, not just the slowest. On 2026-10-19, 28 Nightly jobs had - failed for eight independent causes, some for days. A red Nightly hides the - next regression: the send-depth gate and all six fuzz targets broke on - 2026-10-18 and stayed broken unnoticed until that audit. Nightly was still - red on 2026-10-20 (fuzz, media players, audit, features, release builds), so - an alert that always fires is not visibility either. -- Platform (macOS, Windows) runs only in the merge queue and is not required, - so a Windows-only compile error in moq-video stayed red for two days across - dozens of queue runs until - [#5966](https://github.com/moq-dev/moq/pull/5966). Most Platform failures - in the sample were that error, a choco `nasm` install outage, or another - queued PR's breakage. Rank Platform by failure cause, not count. -- A queue failure often belongs to another PR in the same batch: a stale - fixture in the entry itself, or a semantic conflict with a PR merged after it - was queued. Check the entry's head branch and what merged around it before - calling it a flake. -- Owned elsewhere: Rust advisories by - [the dependency audit](/quest/m2/audit-deps.md). The release toolchain now - links. -- Re-running a single job in place cannot redeploy Pages: the earlier - attempt's `github-pages` artifact remains and `deploy-pages` refuses two. - Delete the stale artifact first, or wait for the next Nightly. - -Check first (from the m0 CI pipeline audit, 2026-10-21): - -- The queue runs one `just ci` job on the persistent `nix/runner` VM. Over 32 - entries it took a median 5.5 minutes, p90 11.9, at most 24.2. Its job - summary lists seconds per module, and the `cargo-timings` artifact is the - per-crate drill-down behind that table. Re-measure from those, not from log - timestamps. -- An `rs/moq-ffi` change selects every binding, and the bindings were about - half of the slowest entries: C++ with the OBS plugin about 4.5 minutes, - Dart 45 to 160 seconds, Kotlin 30 to 110, Go about a minute. - [moq-dev/moq#6247](https://github.com/moq-dev/moq/pull/6247) compiles - `moq.cpp` twice instead of six times and keeps the Gradle daemon warm - across queue jobs; re-measure both. The dispatcher runs modules one after - another, and most binding steps use one core of eight. -- A core Rust change (moq-net, moq-quic, moq-tokio) spends 3 to 4.5 minutes - compiling the test build and 2 to 2.5 running it, plus separate builds for - `cargo doc` (25 to 60 s), the io_uring clippy (20 to 50 s), the wasm32 - clippy (12 s), and the capture (30 to 36 s) and play (40 to 48 s) tests. -- Deferred to the maintainer (coverage trade-off): fold the capture and play - test builds into the main test build, as the platform build already does - (about 75 s per core Rust entry; the non-feature arms of moq-video, - moq-audio and moq-cli would compile only in Nightly's Features job, and - openh264 in `dev` tests needs measuring first), or move both to Nightly - (the same saving, but the gate stops testing the device features). The - m0 audit recommended folding; the queue keeps both builds until decided. -- Cold builds (release PRs, Nightly) still compile both rustls backends and - aws-lc-sys once per environment; the queue VM keeps those built. -- Nightly on 2026-10-20 took about 1,450 runner-minutes (320 macOS) and was - red for ten causes. Features' clippy was fixed by - [#6117](https://github.com/moq-dev/moq/pull/6117) (its 2026-10-21 failure - is in tests). `gh test` failed again on 2026-10-21 after - [#6086](https://github.com/moq-dev/moq/pull/6086), then - [#6154](https://github.com/moq-dev/moq/pull/6154) removed the assertion. - `rs audio-devices` failed all five playback tests on the hosted ARM runner - on 2026-10-20 and passed on the 2026-10-21 Nightly; it is owned by - [Audio devices on ARM](/quest/m1/audio-devices-arm.md). Owned elsewhere - besides the above: the Rust and JS benchmark legs by the - [performance audit](/quest/m1/audit-perf.md), Audio quality's budgets by - the [audio jitter target](/quest/m1/audio-jitter-target/README.md), and the - bun advisories with the Rust ones. diff --git a/quest/m1/audit-compile-time.md b/quest/m1/audit-compile-time.md index 0b6a9475a87..aecb99b11f8 100644 --- a/quest/m1/audit-compile-time.md +++ b/quest/m1/audit-compile-time.md @@ -20,7 +20,7 @@ independently landable and too large to review alongside the rest becomes a follow-up through `/quest-plan`. Scope: this audit owns per-crate build cost in one build (dev, release, and -incremental). The [CI workflow audit](/quest/m1/audit-ci.md) owns job-level CI +incremental). The [CI workflow audit](/quest/m2/audit-ci.md) owns job-level CI cost and caching; the [dependency audit](/quest/m2/audit-deps.md) owns deduplicating and upgrading dependencies. diff --git a/quest/m1/ci-housekeeping.md b/quest/m1/ci-housekeeping.md index 89f66216c9d..28edcf9ab71 100644 --- a/quest/m1/ci-housekeeping.md +++ b/quest/m1/ci-housekeeping.md @@ -17,4 +17,4 @@ Public API: none. Wire: none. ## Related -- [CI workflow audit](/quest/m1/audit-ci.md) - the broader per-job review +- [CI workflow audit](/quest/m2/audit-ci.md) - the broader per-job review diff --git a/quest/m2/README.md b/quest/m2/README.md index c3d434b002d..5608b7f99c0 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -227,3 +227,4 @@ ranked just below deterministic TS legs, which it requires. - [Test cost audit](/quest/m2/audit-test-cost.md) - the slowest, flakiest, and least valuable tests are fixed, deleted, or planned once the release ships - [Quest tree audit](/quest/m2/audit-quests.md) - repeat the whole-tree audit after the release ships, against the m1 baseline +- [CI workflow audit](/quest/m2/audit-ci.md) - repeat the per-workflow and per-job review after the release ships, against the m1 baselines diff --git a/quest/m2/audit-ci.md b/quest/m2/audit-ci.md new file mode 100644 index 00000000000..9888cf20aa9 --- /dev/null +++ b/quest/m2/audit-ci.md @@ -0,0 +1,212 @@ +# [M] CI workflow audit + +## Goal + +Every CI workflow and job earns its place: its results reach someone (a gate, +a Discord alert, or a published report) at a cost proportional to that signal. +Each one that doesn't is fixed, moved, or deleted. Individual tests stay with +the [test cost audit](/quest/m2/audit-test-cost.md), and runner and cache +infrastructure with [CI cost and latency](/quest/m0/ci/README.md). + +## Plan + +Recurs once per major release (decided 2026-10-20). The PR that finishes +this run also adds the next one with the same file name at the end of the next +milestone's `Required` (m0 to m1, m1 to m2), never m3, the deferred bucket +that does not shift with releases. The successor keeps the Goal and this +paragraph; the rest of its Plan is rewritten with what this run learned: the +per-workflow table, baselines, and what to check first. A successor placed in m2 lists the +[release cut](/quest/m2/release/cut.md) under `Required`, so it waits for the +release in flight to ship. The parent milestone lists this quest under +`Required`. Only `/quest-delete` ends the recurrence. Fixes land in the run's +PR, one commit each, rather than as follow-ups. Only a change that needs a +maintainer decision (dropping a release target, scope) or is independently +landable and too large to review alongside the rest becomes a follow-up +through `/quest-plan`. + +This audit owns workflows and jobs, including job-level build cost; the test +cost audit owns individual tests. Each run tables every workflow in +`.github/workflows/` and every job of the latest scheduled Nightly with three +verdicts: + +- **Signal:** what regression it catches that nothing cheaper catches, and + when it last caught one. +- **Visibility:** who sees a failure or a result. A gate blocks merging, a + scheduled or publishing workflow posts to Discord `#alerts` through + [alert.yml](/.github/workflows/alert.yml) (`just gh check` enforces its + coverage), and a report lands on the [Pages site](https://git.moq.dev/) or + in a job summary. A result only in a run artifact or log is invisible: + publish it, summarize it, or stop producing it. Each Pages report owns one + `gh-pages` directory, commits only that directory, deploys the whole branch, + and joins the `benchmark-pages` concurrency group with `queue: max` + (decided 2026-10-20; `sh/gh/pages-publish.sh`). +- **Cost:** runner-minutes by runner type, from the org billing API and job + timings, against that signal. macOS is the scarce one (a 5-job concurrency + cap). + +### Check first (from the m1 run, 2026-10-22) + +- Classify every failed job of the latest scheduled Nightly, Interop, and TS + runs. `gh run view --log-failed` fetches them in one call; job timings + come from `gh api repos/moq-dev/moq/actions/runs//jobs`. A fix merged + after a night is confirmed only by a later night. +- Nightly was red every night of the m1 run. The 2026-10-21 night failed 18 + jobs and the 2026-10-22 night 19 plus a cancelled Loom job. By the end of the + run, merged fixes covered all but these, which had no owner: + - `just rs loom` on main `f42b336f0e`: `a_front_sees_a_reader_return` + exceeds Loom's branch limit, and the two + `a_*_reader_sees_the_front_end_a_refused_track` models run until the job's + 60-minute timeout cancels it. + - Windows only: `reconnect::a_protocol_violation_ends_reconnect`, added by + [moq-dev/moq#6162](https://github.com/moq-dev/moq/pull/6162), redials + after a protocol violation over TCP. + - WASM and the browser backend baseline: on the 2026-10-22 night, an + uncaught `createUnidirectionalStream` rejection on a closed WebTransport + session failed the harness although every case passed. Two local runs on + main `f42b336f0e` did not reproduce it. +- `just test wasm` on main `f42b336f0e` fails `ietf: reads a published + broadcast` (`remote error: code=0`). Bisected to + [moq-dev/moq#6348](https://github.com/moq-dev/moq/pull/6348): the harness + finishes its track before the pre-draft-20 joining FETCH arrives, which + [join after a quick finish](/quest/m2/ietf-join-after-finish.md) owns. The + merge queue does not run the browser harness, so only Nightly sees it. + - Audio quality: two Chromium Opus `wide` rows exceed their `target_ms_p95` + budget. The [audio jitter target](/quest/m1/audio-jitter-target/README.md) + owns the budgets. +- Fixed after their night, awaiting a green night: Linux release links + ([#6090](https://github.com/moq-dev/moq/pull/6090)), Firefox audio + ([#6321](https://github.com/moq-dev/moq/pull/6321)), macOS Swift SIGBUS + ([#6323](https://github.com/moq-dev/moq/pull/6323)), runner VM user + namespaces ([#6324](https://github.com/moq-dev/moq/pull/6324)), minimal + feature build ([#6312](https://github.com/moq-dev/moq/pull/6312)), CMAF + decoder tests ([#6313](https://github.com/moq-dev/moq/pull/6313)), relay + drain ([#6322](https://github.com/moq-dev/moq/pull/6322)), the impaired + cluster drill ([#6314](https://github.com/moq-dev/moq/pull/6314)), the + browser benchmark build ([#6316](https://github.com/moq-dev/moq/pull/6316)), + the supplemental Criterion list + ([#6326](https://github.com/moq-dev/moq/pull/6326)), setup and feature + fixtures ([#6325](https://github.com/moq-dev/moq/pull/6325)), the + connection fuzz crash ([#6279](https://github.com/moq-dev/moq/pull/6279)), + and the JS and Rust advisories + ([#6315](https://github.com/moq-dev/moq/pull/6315), + [#6317](https://github.com/moq-dev/moq/pull/6317)). The moq-mux large-enum + and moq-net minimal-feature lints pass on main `f42b336f0e`. +- Interop's finite-track tails fail before draft 20 (`Error: old`, a JS + `RESET_STREAM`); [join after a quick finish](/quest/m2/ietf-join-after-finish.md) + owns them. TS fails T-STD overflow and late audio starts; the TS reliability + quests own them. The relay-log `FAIL` lines in Interop's self-test are + negative controls, not failures. +- Infrastructure flakes in the sample: DNS failures installing Nix + (`cli-export`) and a curl exit 56 downloading BBB. Count them across nights + before acting; one night is not a pattern. +- `just gh test` ran only in Nightly during the m1 run, so a PR that added a + bench without registering it, or changed a workflow a fixture asserts, + merged green and reddened the next night. That happened three times in two + days (the setup fixture, the removed `writer` bench, and the unregistered + jitter and SUBSCRIBE_TRACKS benches the m1 run registered). + [Tooling tests gate every merge](/quest/m1/gh-test-gate.md) moves it into + every Check; confirm it landed. +- Platform (macOS, Windows) now runs only in Nightly and on release PRs, not + in the merge queue. A Windows- or macOS-only regression surfaces first in + Nightly, as the reconnect test did. +- An upload that requires files must wait for its producing step, or a setup + failure adds a misleading "No files were found" error. The m1 run gated the + Platform, x11, and size uploads; `failure()`-gated diagnostics still assume + their run directory exists. +- A queue failure often belongs to another PR in the same batch. Check the + entry's head and what merged around it before calling it a flake. +- Re-running a single job cannot redeploy Pages: the earlier attempt's + `github-pages` artifact remains and `deploy-pages` refuses two. Delete the + stale artifact first, or wait for the next Nightly. +- Deferred to the maintainer since m0 (coverage trade-off): fold the capture + and play test builds into the main test build (about 75 s per core Rust + queue entry) or move both to Nightly. The queue keeps both builds until + decided. +- Owned elsewhere: Rust and JS advisories by the + [dependency audit](/quest/m2/audit-deps.md); benchmark legs by the + performance audit; binding build cost, local admission, and Pages artifact + cleanup by their own quests. + +### Baselines + +- Org billing for this repository, October through 2026-10-22 UTC: Linux + 116,552 minutes, Linux ARM 221,937, macOS 54,829, Windows 52,651, all at + zero net charge. The per-workflow and per-run billing endpoints return + nothing for public runs, so attribute cost from job timings, never by + multiplying invocation counts by one sample. +- Nightly elapsed runner-minutes: 1,372 on 2026-10-21 (ARM 636, Linux 260, + macOS 290, Windows 186; 166 jobs, much of it cut short by early failures) and + 2,091 on 2026-10-22 (ARM 1,328, Linux 335, macOS 273, Windows 156; 143 jobs + ran). The longest 2026-10-22 jobs: Audio quality 62 minutes, Loom 60 + (cancelled), binding benchmark 52, Rust bench targets 48, relay profiles 45, + Rust benchmark tests 44, the gate suite 40. +- The merge queue runs one `just ci` job on the persistent `nix/runner` VM: + median 5.5 minutes, p90 11.9, at most 24.2 over 32 entries (m0, 2026-10-21). + Its job summary lists seconds per module, and the `cargo-timings` artifact + is the per-crate drill-down. +- Job summaries now carry the binding generator comparison, Rust and JS + coverage, the browser backend baseline, and browser benchmark comparisons + (m1). Relay-profile traces stay artifact-only diagnosis; native audio rows + wait on the audio jitter target's reporting decision. + +### Every workflow (m1, 2026-10-22) + +Invocations count October runs, including skipped ones, capped by GitHub at +2,500. Minutes are the latest direct run's jobs, by runner. A success is +coverage evidence, not a regression caught. + +| Workflow | Signal | Visibility | October invocations | Latest direct job minutes by runner | Last recorded failure | +| --- | --- | --- | ---: | --- | --- | +| [alert](/.github/workflows/alert.yml) | Scheduled/publishing failure delivery; delivery and coverage guards | Discord failure notice; coverage checked locally | at least 2,500 | no executed job sample | [2026-10-05](https://github.com/moq-dev/moq/actions/runs/37369105130) | +| [android](/.github/workflows/android.yml) | Android cross-compilation and packaging beyond the native gate | Caller gate and Nightly alert | at least 2,500 | Linux 1.9 | [2026-10-19](https://github.com/moq-dev/moq/actions/runs/39343452442) | +| [apt-repo](/.github/workflows/apt-repo.yml) | APT repository metadata, signatures and publication | Published repository/installer and alert | 23 | Linux 1.5 | [2026-05-24](https://github.com/moq-dev/moq/actions/runs/26349324650) | +| [audio-quality](/.github/workflows/audio-quality.yml) | Deterministic audio replay correctness; real-time quality is in Nightly | Caller gate and Nightly alert | 1342 | ARM 16.4 | [2026-10-14](https://github.com/moq-dev/moq/actions/runs/38530745049) | +| [check](/.github/workflows/check.yml) | Required diff-scoped merge admission; module timing summary | Required Check and module timing summary | at least 2,500 | no executed job sample | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39721515505) | +| [cpp](/.github/workflows/cpp.yml) | C++ compiler and ABI consumer builds; not only Rust compilation | Caller gate and Nightly alert | 811 | no executed job sample | [2026-10-19](https://github.com/moq-dev/moq/actions/runs/39343452453) | +| [dependabot](/.github/workflows/dependabot.yml) | Dependency PR policy and maintenance automation | Dependency PR state | at least 2,500 | no executed job sample | No retained standalone failure | +| [docker](/.github/workflows/docker.yml) | Container image build and multi-architecture publication | Caller gate/alert and release artifacts | 16 | ARM 22.6, Linux 29.7 | [2026-10-05](https://github.com/moq-dev/moq/actions/runs/37382213733) | +| [features](/.github/workflows/features.yml) | Feature isolation and missing dependency gates | Caller gate and Nightly alert | 0 | no executed job sample | No retained standalone failure | +| [interop](/.github/workflows/interop.yml) | Cross-language protocol/media behavior | Discord and Pages interop report | 743 | Linux 21.6 | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39787655240) | +| [moq-c](/.github/workflows/moq-c.yml) | C ABI packaging, fixtures and consumer links | Caller gate/alert and release artifacts | 0 | no executed job sample | No retained standalone failure | +| [moq-cli](/.github/workflows/moq-cli.yml) | CLI release archives and target-specific links | Caller gate/alert and release artifacts | 8 | ARM 6.5, Linux 6.2, Windows 17.1, macOS 2.6 | [2026-10-16](https://github.com/moq-dev/moq/actions/runs/38854842213) | +| [moq-gst](/.github/workflows/moq-gst.yml) | GStreamer plugin packaging and dynamic links | Caller gate/alert and release artifacts | 11 | ARM 9.4, Linux 10.3, macOS 10.8 | [2026-10-17](https://github.com/moq-dev/moq/actions/runs/39000627149) | +| [moq-relay](/.github/workflows/moq-relay.yml) | Relay release archives and target-specific links | Caller gate/alert and release artifacts | 8 | ARM 5.1, Linux 5.3, Windows 9.3, macOS 44.1 | [2026-09-30](https://github.com/moq-dev/moq/actions/runs/36747636505) | +| [moq-sh](/.github/workflows/moq-sh.yml) | Installer publication, fixtures and installed artifact smoke | Published repository/installer and alert | 13 | Linux 0.2 | No retained standalone failure | +| [nightly](/.github/workflows/nightly.yml) | Diff-independent regression suite and release dry-runs | Discord, Pages benchmarks, job summaries | 26 | see Baselines | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39779729461) | +| [obs](/.github/workflows/obs.yml) | OBS consumer build and plugin packaging | Caller gate and Nightly alert | 2103 | ARM 22.2, Windows 12.9, macOS 10.9 | [2026-10-19](https://github.com/moq-dev/moq/actions/runs/39343452410) | +| [oss-scanner](/.github/workflows/oss-scanner.yml) | OSS Scanner container builds | Caller gate and Nightly alert | 1 | ARM 9.7 | No retained standalone failure | +| [platform](/.github/workflows/platform.yml) | macOS/Windows compile, bindings and native tests | Nightly and release-gate alert | at least 2,500 | Windows 12.3, macOS 7.0 | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39779729461) | +| [quest](/.github/workflows/quest.yml) | Unscoped quest-tree validation after direct pushes | Quest check and push-failure alert | at least 2,500 | ARM 0.5 | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39727308653) | +| [release-binary](/.github/workflows/release-binary.yml) | Shared target-specific binary packaging | Calling release workflow; not independently triggered | 0 | no executed job sample | No retained standalone failure | +| [release-brew](/.github/workflows/release-brew.yml) | Homebrew publication | Caller gate/alert and release artifacts | 27 | Linux 0.2 | No retained standalone failure | +| [release-cpp](/.github/workflows/release-cpp.yml) | C++ release artifacts and consumers | Caller gate/alert and release artifacts | 2 | ARM 4.7, Linux 26.4, Windows 32.7, macOS 21.4 | No retained standalone failure | +| [release-dart-ffi](/.github/workflows/release-dart-ffi.yml) | Dart native FFI artifacts | Caller gate/alert and release artifacts | 293 | Linux 45.0 | [2026-10-10](https://github.com/moq-dev/moq/actions/runs/38088792983) | +| [release-dart](/.github/workflows/release-dart.yml) | Dart package publication | Caller gate/alert and release artifacts | 198 | Linux 0.3 | [2026-10-16](https://github.com/moq-dev/moq/actions/runs/38973399435) | +| [release-ffi](/.github/workflows/release-ffi.yml) | Shared FFI target builder | Calling release workflow; not independently triggered | 0 | no executed job sample | No retained standalone failure | +| [release-gate](/.github/workflows/release-gate.yml) | Complete release PR admission, including callable platform lanes | Caller gate/alert and release artifacts | 0 | no executed job sample | No retained standalone failure | +| [release-go-ffi](/.github/workflows/release-go-ffi.yml) | Go native FFI artifacts | Caller gate/alert and release artifacts | 58 | Linux 33.5, Windows 17.4, macOS 14.0 | [2026-10-18](https://github.com/moq-dev/moq/actions/runs/39135382582) | +| [release-go](/.github/workflows/release-go.yml) | Go wrapper publication | Caller gate/alert and release artifacts | 179 | Linux 1.3 | [2026-10-14](https://github.com/moq-dev/moq/actions/runs/38638664735) | +| [release-js](/.github/workflows/release-js.yml) | npm/JSR packages and exports | Caller gate/alert and release artifacts | 1391 | Linux 16.9 | [2026-10-19](https://github.com/moq-dev/moq/actions/runs/39304286923) | +| [release-kt-ffi](/.github/workflows/release-kt-ffi.yml) | Kotlin native FFI artifacts | Caller gate/alert and release artifacts | 8 | Linux 40.4, Windows 16.8, macOS 13.4 | [2026-09-14](https://github.com/moq-dev/moq/actions/runs/34897543024) | +| [release-kt-lib](/.github/workflows/release-kt-lib.yml) | Kotlin wrapper artifacts | Caller gate/alert and release artifacts | 281 | Linux 14.8 | [2026-10-16](https://github.com/moq-dev/moq/actions/runs/38828705077) | +| [release-nix](/.github/workflows/release-nix.yml) | Nix release evaluation/publication | Caller gate/alert and release artifacts | 0 | no executed job sample | No retained standalone failure | +| [release-py-ffi](/.github/workflows/release-py-ffi.yml) | Python native wheels | Caller gate/alert and release artifacts | 8 | ARM 7.0, Linux 8.0, Windows 14.4, macOS 6.9 | [2026-09-14](https://github.com/moq-dev/moq/actions/runs/34881733830) | +| [release-py](/.github/workflows/release-py.yml) | Python wrapper package | Caller gate/alert and release artifacts | 8 | Linux 17.0 | [2026-05-20](https://github.com/moq-dev/moq/actions/runs/26167963152) | +| [release-rs](/.github/workflows/release-rs.yml) | Rust package release policy | Caller gate/alert and release artifacts | 86 | Linux 21.9 | [2026-10-06](https://github.com/moq-dev/moq/actions/runs/37396594590) | +| [release-shared-ffi](/.github/workflows/release-shared-ffi.yml) | Shared native FFI archives | Caller gate/alert and release artifacts | 0 | no executed job sample | No retained standalone failure | +| [release-swift-ffi](/.github/workflows/release-swift-ffi.yml) | Swift native FFI artifacts | Caller gate/alert and release artifacts | 8 | Linux 6.4, macOS 27.6 | [2026-08-08](https://github.com/moq-dev/moq/actions/runs/31232064868) | +| [release-swift-lib](/.github/workflows/release-swift-lib.yml) | Swift wrapper publication | Caller gate/alert and release artifacts | 97 | Linux 0.1, macOS 0.8 | [2026-09-22](https://github.com/moq-dev/moq/actions/runs/35742256991) | +| [release-winget](/.github/workflows/release-winget.yml) | Winget manifests and publication | Caller gate/alert and release artifacts | 16 | Windows 0.3 | [2026-10-17](https://github.com/moq-dev/moq/actions/runs/39066762564) | +| [rpm-repo](/.github/workflows/rpm-repo.yml) | RPM repository metadata/signatures/publication | Published repository/installer and alert | 23 | Linux 2.3 | [2026-05-25](https://github.com/moq-dev/moq/actions/runs/26377814471) | +| [runner-vm](/.github/workflows/runner-vm.yml) | Persistent runner VM configuration and boot/image checks | Caller gate and Nightly alert | 4 | Linux 3.2 | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39779729461) | +| [smoke-toolchain](/.github/workflows/smoke-toolchain.yml) | Coupled smoke tooling and Playwright update validation | Update PR and scheduled failure alert | 0 | no executed job sample | No retained standalone failure | +| [smoke](/.github/workflows/smoke.yml) | Published distribution and external-relay behavior | Discord and Pages relay report | 21 | Linux 95.6 | [2026-09-25](https://github.com/moq-dev/moq/actions/runs/36076458401) | +| [ts](/.github/workflows/ts.yml) | TS container compliance and round-trip media integrity | Scheduled failure alert and release gate | 1493 | Linux 34.9 | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39786183107) | +| [uring](/.github/workflows/uring.yml) | io_uring compile and runtime/kernel contract | Caller gate and Nightly alert | 1711 | ARM 7.7 | [2026-10-19](https://github.com/moq-dev/moq/actions/runs/39344299025) | +| [video](/.github/workflows/video.yml) | Browser video renderer behavior | Scheduled failure alert and release gate | 413 | Linux 1.9 | No retained standalone failure | +| [wasm](/.github/workflows/wasm.yml) | Native/wasm cross-language bindings and browser behavior | Nightly and release-gate alert | at least 2,500 | ARM 23.8 | [2026-10-22](https://github.com/moq-dev/moq/actions/runs/39779729461) | +| [x11](/.github/workflows/x11.yml) | SHM and GetImage native capture paths | Caller gate and Nightly alert | 680 | ARM 5.2 | [2026-10-17](https://github.com/moq-dev/moq/actions/runs/39002028489) | + +## Required + +- [Release cut](/quest/m2/release/cut.md) - the release in flight ships before this run starts diff --git a/quest/m2/audit-test-cost.md b/quest/m2/audit-test-cost.md index 1c8c6beb698..64cbf804997 100644 --- a/quest/m2/audit-test-cost.md +++ b/quest/m2/audit-test-cost.md @@ -4,7 +4,7 @@ The slowest, flakiest, and least valuable tests are sped up, fixed, deleted, or planned, so the suite costs what it earns. Workflows and jobs belong to the -[CI workflow audit](/quest/m1/audit-ci.md). +[CI workflow audit](/quest/m2/audit-ci.md). ## Plan diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 10208438da2..fb8af010de2 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -103,7 +103,7 @@ case "$1" in done ;; js) - for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track resume interests priority subscribe-drop refresh observed compression; do + for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track resume interests priority subscribe-drop subscribe-tracks refresh observed compression; do javascript "$benchmark" just js bench "$benchmark" done javascript catalog just js catalog-bench @@ -115,6 +115,7 @@ case "$1" in javascript json-codec just js json-codec-bench javascript json-parse just js json-parse-bench javascript slow-group just js slow-group-bench + javascript jitter just js jitter-bench javascript audio-warmup just js audio-warmup-bench javascript clock just js clock-bench ;; diff --git a/sh/js/coverage.py b/sh/js/coverage.py index fa696fd25ad..73e24669749 100644 --- a/sh/js/coverage.py +++ b/sh/js/coverage.py @@ -2,6 +2,7 @@ """Measure Bun unit tests in separate package processes, as the merge gate does.""" import json +import os from pathlib import Path import subprocess import sys @@ -71,9 +72,16 @@ def main(): result.check_returncode() summary = report(ROOT, selected) (ROOT / "target/coverage/js/summary.json").write_text(json.dumps(summary, indent=2) + "\n") + rows = ["## JS unit coverage", "", "| Package | Covered lines | Measured lines | Unloaded sources |", + "| --- | ---: | ---: | ---: |"] for name, entry in summary.items(): + rows.append(f"| {name} | {entry['covered']} | {entry['lines']} | {len(entry['unloaded'])} |") print(f"{name}: {entry['covered']}/{entry['lines']} measured lines, {len(entry['unloaded'])} unloaded sources") + if destination := os.environ.get("GITHUB_STEP_SUMMARY"): + with Path(destination).open("a") as output: + output.write("\n".join(rows) + "\n") + if __name__ == "__main__": main() diff --git a/sh/rs/coverage.sh b/sh/rs/coverage.sh index 9ee03939c60..e5e5dcd898c 100644 --- a/sh/rs/coverage.sh +++ b/sh/rs/coverage.sh @@ -22,7 +22,15 @@ rm -rf "$out" cargo llvm-cov clean --workspace bash sh/rs/nextest.sh cargo llvm-cov nextest --locked "${packages[@]}" --tests --no-report --test-threads "${NEXTEST_TEST_THREADS:-4}" mkdir -p "$out" -cargo llvm-cov report "${report[@]}" +cargo llvm-cov report "${report[@]}" | tee "$out/summary.txt" cargo llvm-cov report "${report[@]}" --json --summary-only --output-path "$out/summary.json" cargo llvm-cov report "${report[@]}" --lcov --output-path "$out/lcov.info" cargo llvm-cov report "${report[@]}" --html --output-dir "$out" + +if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + { + printf '\n## Rust unit coverage\n\n```text\n' + cat "$out/summary.txt" + printf '\n```\n' + } >>"$GITHUB_STEP_SUMMARY" +fi diff --git a/test/wasm/baseline/driver.ts b/test/wasm/baseline/driver.ts index c6815ea4930..63bdb1f843f 100644 --- a/test/wasm/baseline/driver.ts +++ b/test/wasm/baseline/driver.ts @@ -1,6 +1,6 @@ /** Retain raw backend samples; correctness fails, timing variability is reported. */ import { createHash } from "node:crypto"; -import { readdir } from "node:fs/promises"; +import { appendFile, readdir } from "node:fs/promises"; import { cpus, freemem, loadavg, platform, totalmem } from "node:os"; import { dirname, join } from "node:path"; import { parseArgs } from "node:util"; @@ -189,8 +189,24 @@ try { } finally { await browser.close(); server.stop(true); - report.summary = summaries(); + const measured = summaries(); + report.summary = measured; await Bun.write(output, JSON.stringify(report, null, 2)); + if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + "## Browser backend baseline", + "", + "Medians across rounds; milliseconds. CPU and elapsed time measure different work.", + "", + "| Workload | Rounds | Module | First frame | CPU/frame | Elapsed |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ...measured.map( + (row) => + `| ${JSON.stringify(row.workload).replaceAll("|", "\\|")} | ${row.rounds} | ${row.startup.module_ms.median.toFixed(2)} | ${row.startup.first_frame_ms.median.toFixed(2)} | ${row.cpu.per_frame_ms.median.toFixed(2)} | ${row.frames.elapsed_ms.median.toFixed(2)} |`, + ), + ]; + await appendFile(process.env.GITHUB_STEP_SUMMARY, `${lines.join("\n")}\n`); + } } console.log(`Retained ${report.samples.length} raw backend samples: ${output}`);