diff --git a/.github/justfile b/.github/justfile index 7d0cd69d092..5bc869ed44c 100644 --- a/.github/justfile +++ b/.github/justfile @@ -8,6 +8,9 @@ check: # Tests of repository tooling; nightly, not on every pull request. test: + python3 sh/size/report.test.py + python3 sh/size/release.test.py + python3 sh/size/nightly.test.py python3 bench/compare.test.py python3 bench/summary.test.py python3 bench/relay.test.py @@ -45,6 +48,19 @@ install-formula *args: trigger-repo-publish *args: sh/gh/trigger-repo-publish.sh "$@" +# Measure a workflow run's release uploads, the minimal FFI build, and bundled first-load JS entries. +size-collect output run="": + sh/size/collect.sh '{{ output }}' '{{ run }}' + +# Compare a size report with its retained nightly history. +[positional-arguments] +size-report *args: + python3 sh/size/report.py "$@" + +# Collect and compare nightly shipping sizes through the existing alert path. +size-nightly output run: + sh/size/nightly.sh '{{ output }}' '{{ run }}' + # Install the rendered GStreamer formula against the current macOS tarball. [positional-arguments] gst-formula *args: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a98a76d8594..612efd3f07c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -447,6 +447,45 @@ jobs: permissions: contents: read + sizes: + name: Shipping sizes + # Measures what these release jobs upload rather than rebuilding it. + needs: [release-py-ffi, release-kt-ffi, release-swift-ffi, release-go-ffi, release-moq-c, release-moq-relay, release-moq-cli] + runs-on: ubuntu-24.04-arm + timeout-minutes: 120 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - uses: ./.github/actions/rust-cache + - name: Collect and compare sizes + run: nix develop --command just gh size-nightly size-report "$GITHUB_RUN_ID" + env: + GH_TOKEN: ${{ github.token }} + - name: Preserve sizes and attribution + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: nightly-size-history + path: | + size-report/history.json + size-report/sizes.json + size-report/summary.md + size-report/ffi-bloat.txt + size-report/js/bytes.json + size-report/js/*.metafile.json + retention-days: 45 + if-no-files-found: error + fuzz: name: Fuzz (${{ matrix.target }}) runs-on: ubuntu-24.04-arm diff --git a/flake.nix b/flake.nix index ec537648443..694c7d3d75a 100644 --- a/flake.nix +++ b/flake.nix @@ -133,6 +133,7 @@ cargo-edit cargo-semver-checks cargo-deny + cargo-bloat # Per-crate feature matrices (`just rs features`), where workspace # unification would otherwise hide a broken single-crate build. cargo-hack diff --git a/quest/m1/README.md b/quest/m1/README.md index e55b407d71c..d8fad3d22b4 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -172,7 +172,6 @@ session outcomes, and embedder refusals inside session outcomes. - [moqsrc A/V alignment](/quest/m1/gst-src-av-sync.md) - every `moqsrc` pad of a run shares one timestamp reference and segment base - [TS passthrough export](/quest/m1/ts-passthrough-export.md) - `export ts --passthrough` writes the `m2ts` track back byte-identical (less late drops) on a fixed delay - [FLV and MKV export delay](/quest/m1/export-delay.md) - FLV and MKV interleave through the shared jitter buffer on a fixed delay, breaking the CLI once -- [Size report](/quest/m1/size-report.md) - a nightly job reports every shipped artifact's size, native and JS, and alerts when one grows - [Bindings size profile](/quest/m1/ffi-size-profile.md) - a benchmark decides whether the moq-ffi builds ship at opt-level "s", which halves the dylib - [Go mirror delivery](/quest/m1/go-mirror-delivery.md) - the Go binding's staticlibs stop growing git history by ~210 MiB per release - [Dart on iOS](/quest/m1/dart-ios.md) - prove the shipped iOS native asset actually loads on a device, which no CI can diff --git a/quest/m1/size-report.md b/quest/m1/size-report.md deleted file mode 100644 index e370bfa2a67..00000000000 --- a/quest/m1/size-report.md +++ /dev/null @@ -1,64 +0,0 @@ -# [M] Nightly size report - -## Goal - -A nightly job reports the size of what we ship, built the way the release -builds it, and alerts when any of it grows. Size regressions show -up within a day instead of in an app store review or a user's bundle -analyzer. - -## Plan - -Decided in planning: - -- Nightly, not per PR. Size moves only on dependency bumps, feature flips, or - profile changes, so a per-PR comment would say "no change" almost every - time. -- No hard budgets in CI. The job keeps a rolling history and fails when an - artifact grows past a threshold against the previous run (start at 5%) or - against a longer window (for example 10% over 30 days). The window catches - slow drift, which night-to-night checks miss. The existing `alert.yml` path - turns the failure into a Discord alert. Add the workflow to its list if - needed. -- Scope: - - moq-ffi, default and `--no-default-features`, as cdylib and staticlib - - moq-c, moq-relay, and moq-cli - - the consumer cost of the JS entries: `@moq/net`, `@moq/watch/element` - with and without `/ui`, and `@moq/publish/element`. Measure them the way - a consumer sees them: bundled and minified from the built packages, - first-load chunks only, gzip and brotli. - - the packaged outputs nightly already builds. `nightly.yml` runs the - Python, Kotlin, and Swift release builds, so read their wheel, AAR, and - xcframework sizes instead of rebuilding them. -- Out of scope: moq-wasm, which [Remove moq-wasm](/quest/m2/rs2ts-remove-wasm.md) - deletes (decided 2026-10-08), Docker images, and per-target release assets - nightly doesn't build. The Docker quest measures its images once. Add - others only if one of them regresses unnoticed. -- The job summary also carries `cargo bloat --crates` for the ffi build and a - metafile breakdown for the publish element, so the cause of a jump is - visible without a local rebuild. `cargo bloat` needs symbols, so build that - pass with `CARGO_PROFILE_RELEASE_STRIP=none` and report stripped sizes - separately. -- The work lives in a `sh/` script behind a `just` recipe, and `nightly.yml` - calls the recipe. -- Cover Linux x86_64 and aarch64, and record each artifact's release build - time alongside its size. The unified release profile (fat LTO, one codegen - unit, stripped; #5274) landed without a Linux aarch64 artifact matrix or a - controlled before/after build-cost comparison, so this job's first runs - supply them. Check the Go mirror's staticlibs stay under its 100 MB limit. - -Considered and declined (do not re-ask): - -- Shrinking npm install size by dropping sourcemaps or `inlineSources`. Maps - are 60-77% of the unpacked packages, but they never reach a browser and - they give consumers real stack traces. -- Splitting the lite and IETF implementations out of `@moq/net`: version - negotiation needs both at connect time. -- Feature gates in moq-tokio for reqwest, tracing-subscriber, usage-rs/toml, - and tokio "full" in the bindings. -- Replacing the AV1 codec-string regex in hang. tracing-subscriber's env - filter keeps regex linked anyway. - -## Related - -- [Benchmark regressions in CI](/quest/m1/bench-ci.md) - the same nightly-trend shape for Criterion diff --git a/sh/size/README.md b/sh/size/README.md new file mode 100644 index 00000000000..a84f86caa03 --- /dev/null +++ b/sh/size/README.md @@ -0,0 +1,23 @@ +# Nightly artifact sizes + +`just gh size-collect OUTPUT [RUN]` measures what a workflow run's release jobs +uploaded, plus the builds no release job makes: the minimal-feature FFI library +and real browser consumer bundles (raw, gzip and Brotli). `just gh size-report +OUTPUT [--history HISTORY]` compares the report with its previous report (5% +growth) and the newest report at least 30 days old (10% growth). + +Nightly passes its own run, so the sizes are the shipping artifacts for every +release target, Linux `x86_64` and `aarch64` included, without rebuilding them. Any +retained run works locally too. Each release job's `Build` step durations are +recorded and summarized but never compared, since hosted build time is noisy. +The Go mirror's 100 MiB file limit is enforced by the Go FFI package step; +the report tracks those staticlibs. + +Keys name the artifact (which carries the target) and the filename without its +release version, so a version bump keeps the history comparable. Duplicate keys +and missing release artifacts are refused. An artifact-set change is refused +too: review the new set, then start a fresh history by reporting without one. + +A regression fails Nightly and reaches its existing alert workflow. The job +summary carries the byte counts, build times and FFI crate attribution, the +latter from the symbol-preserving `profiling` profile. diff --git a/sh/size/bundles.ts b/sh/size/bundles.ts new file mode 100644 index 00000000000..9e94d0824c9 --- /dev/null +++ b/sh/size/bundles.ts @@ -0,0 +1,71 @@ +// Measure consumer bundles from published dist entry points, including only static chunks. + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { brotliCompressSync, gzipSync } from "node:zlib"; +import { build } from "esbuild"; + +const root = process.cwd(); +if (!process.argv[2]) throw new Error("usage: bundles.ts OUTPUT"); +const out = resolve(process.argv[2]); +mkdirSync(out, { recursive: true }); +const sizes: Record = {}; +for (const [name, code] of Object.entries({ + net: 'export * from "@moq/net";', + watch: 'import "@moq/watch/element";', + "watch-ui": 'import "@moq/watch/element"; import "@moq/watch/ui";', + publish: 'import "@moq/publish/element";', +})) { + const result = await build({ + stdin: { contents: code, resolveDir: root, sourcefile: `${name}.js` }, + bundle: true, + minify: true, + splitting: true, + format: "esm", + platform: "browser", + metafile: true, + write: false, + outdir: join(out, name), + plugins: [ + { + name: "built-packages", + setup(b) { + b.onResolve({ filter: /^@moq\// }, ({ path }) => { + const [, pkg, ...subpath] = path.split("/"); + const dir = join(root, "js", pkg, "dist"); + // Only packages outside this workspace fall through to node_modules. + if (!existsSync(join(root, "js", pkg, "package.json"))) return; + const manifest: { exports: Record } = JSON.parse( + readFileSync(join(dir, "package.json"), "utf8"), + ); + const entry = manifest.exports[subpath.length ? `./${subpath.join("/")}` : "."]; + if (!entry) throw new Error(`missing built export: ${path}`); + return { path: resolve(dir, typeof entry === "string" ? entry : entry.default) }; + }); + }, + }, + ], + }); + if (!result.metafile || !result.outputFiles) throw new Error(`bundle outputs missing: ${name}`); + writeFileSync(join(out, `${name}.metafile.json`), JSON.stringify(result.metafile, null, 2)); + const outputs = result.metafile.outputs; + const first = new Set(); + const visit = (path: string) => { + if (first.has(path)) return; + first.add(path); + for (const dep of outputs[path].imports) if (!dep.external && dep.kind !== "dynamic-import") visit(dep.path); + }; + for (const [path, output] of Object.entries(outputs)) if (output.entryPoint === `${name}.js`) visit(path); + if (!first.size) throw new Error(`entry output missing: ${name}`); + for (const file of result.outputFiles) { + if (![...first].some((path) => resolve(root, path) === file.path)) continue; + for (const [format, bytes] of Object.entries({ + raw: file.contents.length, + gzip: gzipSync(file.contents).length, + brotli: brotliCompressSync(file.contents).length, + })) + sizes[`js/${name}/${format}`] = (sizes[`js/${name}/${format}`] ?? 0) + bytes; + } + if (!sizes[`js/${name}/raw`]) throw new Error(`no first-load bytes: ${name}`); +} +writeFileSync(join(out, "bytes.json"), JSON.stringify(sizes, null, 2)); diff --git a/sh/size/collect.sh b/sh/size/collect.sh new file mode 100755 index 00000000000..42ddc433494 --- /dev/null +++ b/sh/size/collect.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Measure a workflow run's release uploads and the builds no release job makes, retaining diagnostics. +set -euo pipefail +out=$(realpath -m "${1:?usage: collect.sh OUTPUT [RUN]}") +run=${2:-} +mkdir -p "$out/native" +target=$(cargo metadata --locked --format-version 1 --no-deps | jq -r .target_directory) +case $(uname -s) in Linux) dylib=so ;; Darwin) dylib=dylib ;; *) + echo 'native size capture requires Linux or macOS' >&2 + exit 1 + ;; +esac +# Release jobs ship default features only; this shows what the feature gates save. +cargo build --locked --release -p moq-ffi --no-default-features +cp "$target/release/libmoq_ffi.$dylib" "$out/native/libmoq_ffi-minimal.$dylib" +cp "$target/release/libmoq_ffi.a" "$out/native/libmoq_ffi-minimal.a" +bun install --frozen-lockfile +just js build +bun sh/size/bundles.ts "$out/js" +# Symbols are necessary only for attribution; they never replace the measured shipping sizes. +cargo bloat --locked --profile profiling -p moq-ffi --crates --lib >"$out/ffi-bloat.txt" +python3 - "$out" "$run" <<'PY' +import datetime, json, pathlib, subprocess, sys +sys.path.insert(0, "sh/size") +import release +out, run = pathlib.Path(sys.argv[1]), sys.argv[2] +host = next(line.split(': ', 1)[1] for line in subprocess.check_output(['rustc', '-vV'], text=True).splitlines() if line.startswith('host: ')) +sizes = {f'native/{host}/{p.name}': p.stat().st_size for p in (out / 'native').iterdir()} +sizes.update(json.loads((out / 'js/bytes.json').read_text())) +report = {'version': 1, 'date': datetime.date.today().isoformat(), + 'revision': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), + 'bytes': sizes} +if run: + release.download(run, out / 'release') + sizes.update(release.sizes(out / 'release')) + report['seconds'] = release.seconds(run) +(out / 'sizes.json').write_text(json.dumps(report, indent=2) + '\n') +PY diff --git a/sh/size/nightly.sh b/sh/size/nightly.sh new file mode 100755 index 00000000000..df32ecd04f6 --- /dev/null +++ b/sh/size/nightly.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Load the latest completed history, then retain the new report even if growth fails. +set -euo pipefail +out=${1:?usage: nightly.sh OUTPUT RUN} +run=${2:?usage: nightly.sh OUTPUT RUN} +mkdir -p "$out" +history=() +# A packaging or collector failure may leave no history artifact in the newest run. +# Search retained completed runs rather than silently discarding the older baseline. +runs=$(gh run list --workflow nightly.yml --branch main --limit 45 --status completed --json databaseId --jq '.[].databaseId') +while IFS= read -r prior; do + [[ -n "$prior" ]] || continue + artifact=$(gh api "repos/{owner}/{repo}/actions/runs/$prior/artifacts" --jq '.artifacts[] | select(.name == "nightly-size-history" and .expired == false) | .id') + if [[ -n "$artifact" ]]; then + previous="$out/previous/$prior" + gh run download "$prior" --name nightly-size-history --dir "$previous" + # Failed collectors can upload diagnostics without producing a history. + if [[ -f "$previous/history.json" ]]; then + history=(--history "$previous/history.json") + break + fi + fi +done <<<"$runs" +sh/size/collect.sh "$out" "$run" +status=0 +python3 sh/size/report.py "$out" "${history[@]}" || status=$? +{ + cat "$out/summary.md" + printf '\n
FFI crate attribution\n\n```text\n' + cat "$out/ffi-bloat.txt" + printf '\n```\n
\n' +} >>"${GITHUB_STEP_SUMMARY:-/dev/stdout}" +exit "$status" diff --git a/sh/size/nightly.test.py b/sh/size/nightly.test.py new file mode 100644 index 00000000000..a41b8db74a6 --- /dev/null +++ b/sh/size/nightly.test.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Exercise history recovery without network access or native builds.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +SCRIPT = Path(__file__).with_name("nightly.sh").resolve() +REPORT = Path(__file__).with_name("report.py").resolve() + + +class History(unittest.TestCase): + def test_skips_retained_artifact_without_history(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "bin").mkdir() + (root / "sh/size").mkdir(parents=True) + (root / "sh/size/report.py").symlink_to(REPORT) + gh = root / "bin/gh" + gh.write_text('''#!/usr/bin/env python3 +import json, pathlib, sys +args = sys.argv[1:] +if args[:2] == ['run', 'list']: + print('2\\n1') +elif args[0] == 'api': + print('artifact') +elif args[:2] == ['run', 'download']: + out = pathlib.Path(args[args.index('--dir') + 1]) + out.mkdir(parents=True) + if args[2] == '1': + (out / 'history.json').write_text(json.dumps([{'version': 1, 'date': '2026-10-09', 'bytes': {'a': 100}}])) + else: + (out / 'sizes.json').write_text('{}') +else: + raise SystemExit(args) +''') + gh.chmod(0o755) + collect = root / "sh/size/collect.sh" + collect.write_text('''#!/usr/bin/env bash +set -eu +printf '%s' '{"version":1,"date":"2026-10-10","bytes":{"a":100}}' >"$1/sizes.json" +printf 'attribution' >"$1/ffi-bloat.txt" +''') + collect.chmod(0o755) + env = dict(os.environ, PATH=f"{root / 'bin'}:{os.environ['PATH']}", + GITHUB_STEP_SUMMARY=str(root / "summary")) + result = subprocess.run(["bash", str(SCRIPT), "out", "3"], cwd=root, + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + history = json.loads((root / "out/history.json").read_text()) + self.assertEqual([row['date'] for row in history], ['2026-10-09', '2026-10-10']) + + +unittest.main() diff --git a/sh/size/release.py b/sh/size/release.py new file mode 100644 index 00000000000..e82ac4251a1 --- /dev/null +++ b/sh/size/release.py @@ -0,0 +1,51 @@ +"""Measure what a workflow run's release jobs uploaded, independently of release versions.""" +import datetime as dt +import json +import re +import subprocess +from pathlib import Path + +# Each pattern must yield a shipping file, so a renamed or skipped release job fails loudly. +ARTIFACTS = ["go-lib-*", "moq-c-*", "moq-cli-*", "moq-relay-*", "python-*", "kotlin-package", "swift-ffi-package"] +# What a consumer downloads: archives, packages and the Go mirror's staticlibs. +# Bare binaries, .deb and .rpm repeat the archives; Go's .so and .dll ship inside the packages. +SHIPPING = (".tar.gz", ".zip", ".whl", ".aar", ".jar", ".a", ".lib") +VERSION = re.compile(r"[-_]v?\d+\.\d+\.\d+") + + +def download(run, root): + patterns = [arg for pattern in ARTIFACTS for arg in ("--pattern", pattern)] + subprocess.run(["gh", "run", "download", str(run), "--dir", str(root), *patterns], check=True) + + +def sizes(root): + """Bytes per shipping file, keyed by artifact (which names the target) and versionless filename.""" + result = {} + for pattern in ARTIFACTS: + found = False + for path in sorted(Path(root).glob(f"{pattern}/**/*")): + if not path.is_file() or not path.name.endswith(SHIPPING) or path.name.endswith(("-sources.jar", "-javadoc.jar")): + continue + artifact = path.relative_to(root).parts[0] + key = f"release/{artifact}/{VERSION.sub('', path.name)}" + if key in result: + raise ValueError(f"duplicate release artifact: {key}") + result[key] = path.stat().st_size + found = True + if not found: + raise ValueError(f"no shipping files for {pattern} under {root}") + return result + + +def seconds(run): + """Seconds each successful job spent in its Build steps; too noisy to alert on, so only recorded.""" + lines = subprocess.check_output(["gh", "api", "--paginate", f"repos/{{owner}}/{{repo}}/actions/runs/{run}/jobs?per_page=100", + "--jq", ".jobs[]"], text=True).splitlines() + result = {} + for job in map(json.loads, lines): + steps = [step for step in job["steps"] if step["name"].startswith("Build") and step["conclusion"] == "success"] + if job["conclusion"] == "success" and steps: + result[job["name"]] = sum(round((dt.datetime.fromisoformat(step["completed_at"]) + - dt.datetime.fromisoformat(step["started_at"])).total_seconds()) + for step in steps) + return result diff --git a/sh/size/release.test.py b/sh/size/release.test.py new file mode 100644 index 00000000000..fb68ca1afab --- /dev/null +++ b/sh/size/release.test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import tempfile +import unittest +from pathlib import Path +from release import sizes + + +def fixture(root, version): + """The layout `gh run download` leaves for a nightly run's release uploads.""" + for name in ["go-lib-x86_64-unknown-linux-gnu/libmoq_ffi.a", + "go-lib-x86_64-unknown-linux-gnu/libmoq_ffi.so", + "go-lib-x86_64-pc-windows-msvc/moq_ffi.lib", + f"moq-c-aarch64-unknown-linux-gnu/moq-c-{version}-aarch64-unknown-linux-gnu.tar.gz", + f"moq-cli-x86_64-unknown-linux-gnu/moq-cli-v{version}-x86_64-unknown-linux-gnu.tar.gz", + f"moq-cli-x86_64-unknown-linux-gnu/moq-cli-v{version}-x86_64-unknown-linux-gnu", + f"moq-cli-x86_64-unknown-linux-gnu/moq_{version}_amd64.deb", + f"moq-relay-x86_64-pc-windows-msvc/moq-relay-v{version}-x86_64-pc-windows-msvc.zip", + f"moq-relay-x86_64-unknown-linux-gnu/moq-relay-{version}-1.x86_64.rpm", + f"python-aarch64-unknown-linux-gnu/moq_ffi-{version}-py3-none-manylinux_2_28_aarch64.whl", + f"kotlin-package/maven-local/dev/moq/moq-ffi-android/{version}/moq-ffi-android-{version}.aar", + f"kotlin-package/maven-local/dev/moq/moq-ffi-jvm/{version}/moq-ffi-jvm-{version}.jar", + f"kotlin-package/maven-local/dev/moq/moq-ffi-jvm/{version}/moq-ffi-jvm-{version}-sources.jar", + f"kotlin-package/maven-local/dev/moq/moq-ffi-jvm/{version}/moq-ffi-jvm-{version}.pom", + "swift-ffi-package/MoqFFI.xcframework.zip"]: + path = Path(root) / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"package") + + +class Release(unittest.TestCase): + def test_version_changes_keep_keys(self): + with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: + fixture(a, "0.4.12") + fixture(b, "0.5.0") + self.assertEqual(sizes(a), sizes(b)) + self.assertEqual(sorted(sizes(a)), [ + "release/go-lib-x86_64-pc-windows-msvc/moq_ffi.lib", + "release/go-lib-x86_64-unknown-linux-gnu/libmoq_ffi.a", + "release/kotlin-package/moq-ffi-android.aar", + "release/kotlin-package/moq-ffi-jvm.jar", + "release/moq-c-aarch64-unknown-linux-gnu/moq-c-aarch64-unknown-linux-gnu.tar.gz", + "release/moq-cli-x86_64-unknown-linux-gnu/moq-cli-x86_64-unknown-linux-gnu.tar.gz", + "release/moq-relay-x86_64-pc-windows-msvc/moq-relay-x86_64-pc-windows-msvc.zip", + "release/python-aarch64-unknown-linux-gnu/moq_ffi-py3-none-manylinux_2_28_aarch64.whl", + "release/swift-ffi-package/MoqFFI.xcframework.zip", + ]) + + def test_duplicate_and_missing_artifacts_refused(self): + with tempfile.TemporaryDirectory() as root: + fixture(root, "0.4.12") + for path in Path(root, "moq-relay-x86_64-pc-windows-msvc").iterdir(): + path.unlink() + with self.assertRaisesRegex(ValueError, "moq-relay"): + sizes(root) + with tempfile.TemporaryDirectory() as root: + fixture(root, "0.4.12") + fixture(root, "0.4.13") + with self.assertRaisesRegex(ValueError, "duplicate"): + sizes(root) + + +unittest.main() diff --git a/sh/size/report.py b/sh/size/report.py new file mode 100644 index 00000000000..1e166ea3732 --- /dev/null +++ b/sh/size/report.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Compare byte counts with the previous report and a thirty-day anchor.""" +import argparse +import datetime as dt +import json +from pathlib import Path + + +def compare(current, history): + """Return regressions without comparing different artifact sets or formats.""" + def validate(row): + if row.get("version") != 1 or not row.get("bytes"): + raise ValueError("expected a nonempty version 1 size report") + date = dt.date.fromisoformat(row["date"]) + for name, size in row["bytes"].items(): + if not isinstance(size, int) or isinstance(size, bool) or size <= 0: + raise ValueError(f"invalid byte count for {name}: {size}") + return date + + now = validate(current) + for row in history: + if validate(row) > now: + raise ValueError("history contains a future report") + if row["bytes"].keys() != current["bytes"].keys(): + raise ValueError("artifact set changed; review and explicitly start a new history") + prior = sorted(history, key=lambda row: row["date"]) + if not prior: + return [] + anchors = [row for row in prior if dt.date.fromisoformat(row["date"]) <= now - dt.timedelta(days=30)] + comparisons = [("previous", prior[-1], 0.05)] + if anchors: + comparisons.append(("30 days", anchors[-1], 0.10)) + return [f"{name}: {size} > {row['bytes'][name]} bytes + {limit:.0%} ({label})" + for label, row, limit in comparisons for name, size in current["bytes"].items() + if size > row["bytes"][name] * (1 + limit)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--history", type=Path) + args = parser.parse_args() + current = json.loads((args.directory / "sizes.json").read_text()) + history = json.loads(args.history.read_text()) if args.history else [] + failures = compare(current, history) + # Keep the oldest available thirty-day anchor and every newer report. + cutoff = dt.date.fromisoformat(current["date"]) - dt.timedelta(days=30) + anchors = [row for row in history if dt.date.fromisoformat(row["date"]) <= cutoff] + kept = ([max(anchors, key=lambda row: row["date"])] if anchors else []) + kept += [row for row in history if dt.date.fromisoformat(row["date"]) > cutoff] + (args.directory / "history.json").write_text(json.dumps(kept + [current], indent=2) + "\n") + summary = "| Artifact | Bytes |\n|---|---:|\n" + summary += "".join(f"| {name} | {size:,} |\n" for name, size in sorted(current["bytes"].items())) + summary += "\n" + ("\n".join(failures) if failures else "No size regression; a new artifact history starts with one report.") + "\n" + if current.get("seconds"): + summary += "\n| Release build | Seconds |\n|---|---:|\n" + summary += "".join(f"| {job} | {seconds:,} |\n" for job, seconds in sorted(current["seconds"].items())) + (args.directory / "summary.md").write_text(summary) + print(summary) + if failures: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/sh/size/report.test.py b/sh/size/report.test.py new file mode 100644 index 00000000000..7f1734b5727 --- /dev/null +++ b/sh/size/report.test.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import importlib.util +import unittest +from pathlib import Path +spec = importlib.util.spec_from_file_location("report", Path(__file__).with_name("report.py")) +report = importlib.util.module_from_spec(spec) +spec.loader.exec_module(report) + + +def row(date, size=100): + return {"version": 1, "date": date, "bytes": {"artifact": size}} + + +class Trends(unittest.TestCase): + def test_first_report(self): + self.assertEqual(report.compare(row("2026-10-10"), []), []) + + def test_daily_threshold(self): + self.assertEqual(report.compare(row("2026-10-10", 105), [row("2026-10-09")]), []) + self.assertEqual(len(report.compare(row("2026-10-10", 106), [row("2026-10-09")])), 1) + + def test_slow_drift(self): + failures = report.compare(row("2026-10-10", 111), [row("2026-09-10"), row("2026-10-09", 109)]) + self.assertEqual(len(failures), 1) + self.assertIn("30 days", failures[0]) + + def test_refuses_missing_and_invalid_artifacts(self): + for invalid in [0, -1, None, True, float("nan")]: + with self.assertRaises(ValueError): + report.compare(row("2026-10-10", invalid), []) + with self.assertRaises(ValueError): + report.compare(row("2026-10-10"), [{"date": "2026-10-09", "bytes": {}}]) + + def test_validates_every_history_row(self): + cases = [ + {"version": 2, "date": "2026-09-01", "bytes": {"artifact": 100}}, + {"version": 1, "date": "2026-09-01", "bytes": {"other": 100}}, + row("2026-09-01", 0), + row("2026-10-11"), + ] + for invalid in cases: + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + report.compare(row("2026-10-10"), [invalid, row("2026-10-09")]) + + def test_validates_first_report_date(self): + with self.assertRaises(ValueError): + report.compare(row("invalid"), []) + + +unittest.main()