diff --git a/.github/justfile b/.github/justfile index 0d2dde81ca3..659b4853603 100644 --- a/.github/justfile +++ b/.github/justfile @@ -4,6 +4,10 @@ set working-directory := '..' cache: bash sh/gh/cache.sh +# Publish Smoke's runtime packages and their build artifacts to nix.moq.dev. +cache-runtime: + bash sh/gh/cache-runtime.sh + # Build the runner image with a temporary policy on a disposable hosted runner. runner-vm-ci: #!/usr/bin/env bash diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 8a63655fb9b..13dcb685fbd 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -1,6 +1,6 @@ name: Cache -# Publish development environments independently of the landing worker's local caches. +# Publish dev shells and Smoke's runtime packages independently of the landing worker's local caches. permissions: contents: read @@ -33,3 +33,30 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.NIX_R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.NIX_R2_SECRET_ACCESS_KEY }} run: just gh cache + + runtime: + name: Runtime (ubuntu-24.04) + if: github.ref == 'refs/heads/main' + # Smoke's only current nix platform. Keep its Rust builds independent of + # the dev-shell matrix and use Smoke's cold-build budget. + runs-on: ubuntu-24.04 + timeout-minutes: 75 + environment: cache-write + # Finish the active publication; new pushes replace only the pending run. + # Canceling active builds could starve cold dependency uploads on busy main. + concurrency: + group: cache-runtime-${{ github.ref }} + cancel-in-progress: false + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + - uses: ./.github/actions/just + - name: Cache runtime packages and dependencies + env: + NIX_SIGNING_KEY: ${{ secrets.NIX_SIGNING_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.NIX_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.NIX_R2_SECRET_ACCESS_KEY }} + run: just gh cache-runtime diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index a6af6efe653..d40964896a2 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -41,8 +41,8 @@ jobs: runs-on: ${{ matrix.os }} # The matrix runs in real time (ffmpeg -re pacing + per-subscriber timeouts); # this is just a safety net against a hung relay/client. The nix channel - # gets longer: nix.moq.dev serves the toolchain but not moq's packages at - # HEAD, and compiling them has taken 43 minutes. + # gets longer for a cold build when Cache has not finished publishing + # HEAD's packages yet; compiling them has taken 43 minutes. timeout-minutes: ${{ matrix.channel == 'nix' && 75 || 45 }} # smoke.sh resolves the latest libmoq release via the GitHub API. Without a @@ -74,7 +74,8 @@ jobs: - os: ubuntu-latest channel: docker # macOS cells temporarily disabled to cut CI spend (over budget this - # month). Re-enable by uncommenting; nothing else depends on them. + # month). Re-enable by uncommenting; the nix cell also needs its + # platform added to cache.yml's runtime job. # - os: macos-latest # channel: cargo # - os: macos-latest diff --git a/flake.nix b/flake.nix index 2d0545d981e..33e3283af6e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,16 +1,16 @@ { description = "MoQ - Media over QUIC"; - # Pre-built binaries live in our binary cache at nix.moq.dev. Of the - # packages, only tagged releases are pushed (moq-relay-v*, moq-cli-v*, etc.), - # so pin the flake ref to a recent tag to get a hit. The default branch HEAD - # builds from source, though its dev shell is cached: - # nix run github:moq-dev/moq/moq-relay-v0.12.4#moq-relay --accept-flake-config + # nix.moq.dev caches the default branch's dev shell, along with moq-relay + # and moq on x86 Linux, with a 30-day expiry: + # nix run github:moq-dev/moq#moq-relay --accept-flake-config + # Release Nix retains tagged packages permanently once a release cut + # carries that workflow from main; earlier tags still build from source. # # --accept-flake-config opts into the nixConfig below for one command. To # trust the cache permanently instead, add these settings to nix.conf. # - # The dev shell paths expire 30 days after upload, possibly before a path + # The CI paths expire 30 days after upload, possibly before a path # that depends on them; `fallback` builds whatever is missing instead of # failing the substitution. nixConfig = { diff --git a/infra/README.md b/infra/README.md index ff4d6a36191..0451046f908 100644 --- a/infra/README.md +++ b/infra/README.md @@ -270,7 +270,14 @@ expires. `nix-ci-moq-dev` holds the dev shells `cache.yml` pushes after every merge to `main`, for x86 Linux, ARM Linux, and ARM macOS. It publishes the full build closure from `nix develop --profile`, uploading only missing paths, plus `moq-libav`, its wasm build, and its downloaded sources, which `bun install` -builds outside the dev shell. +builds outside the dev shell. An independent x86 Linux job builds and pushes +the `moq-relay` and `moq` runtime closures for Smoke's nix channel, which +reads them without write keys. Their existing Crane `cargoArtifacts` outputs +are published first so later builds can substitute them when their inputs +are unchanged; those inputs include the patched `kio` sources. The other +platforms have no Smoke nix lane. Runtime warms serialize: the active run +finishes and new pushes replace only the pending run, avoiding duplicate +builds without starving publication or delaying the dev-shell matrix. Its lifecycle rule (`infra/nix/lifecycle.json`) deletes each object 30 days after upload; the next warm re-uploads what it still needs. `flake.nix` sets `fallback = true` because that expiry can remove a path before one that diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 9920ef5064a..5e844f8f831 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -177,7 +177,7 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Read-only pull-request CI](/quest/m0/ci/read-only-secrets.md) - write secrets and publisher identities reach only protected branches and tags, never a pushed branch - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check`, test phases included, finishes within a worker's 5-minute prompt cache, admission wait included - [Verify the R2 Rust cache](/quest/m0/ci/mbx-r2-verify.md) - confirm the hosted Nightly jobs reuse the previous Nightly, then measure restore time and expiry -- [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - confirm releases use `nix.moq.dev`, check its bucket sizes, and retire Cachix +- [Verify the Nix runtime cache](/quest/m0/ci/nix-cache-verify.md) - hosted Cache uploads `main`'s relay and CLI, and Smoke's nix channel substitutes them - [Record hosted cargo-hakari evidence](/quest/m0/ci/hakari.md) - compare scheduled macOS and Windows cache hits and estimated compiler work saved before and after the workspace-hack - [Drop main from the publish Environment](/quest/m0/ci/publish-release-only.md) - the maintainer narrows `publish` to `release` and release tags once no job naming it runs as `main` - [Binary release publishers run on their tag](/quest/m0/ci/release-tag-publishers.md) - brew, Go and Swift releases stop running through `workflow_run` on main diff --git a/quest/m0/ci/mbx-r2-verify.md b/quest/m0/ci/mbx-r2-verify.md index a1cbe7eeda2..1d5945d59fa 100644 --- a/quest/m0/ci/mbx-r2-verify.md +++ b/quest/m0/ci/mbx-r2-verify.md @@ -142,4 +142,4 @@ feature unification, so changed compiler inputs remain a separate cause. ## Related -- [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - the same R2 and Environment bootstrap for Nix +- [Verify the Nix runtime cache](/quest/m0/ci/nix-cache-verify.md) - the same R2 and Environment bootstrap for Nix diff --git a/quest/m0/ci/nix-cache-verify.md b/quest/m0/ci/nix-cache-verify.md index 3b67f7bc579..d86532056a9 100644 --- a/quest/m0/ci/nix-cache-verify.md +++ b/quest/m0/ci/nix-cache-verify.md @@ -1,61 +1,51 @@ -# [S] Verify the Nix cache +# [XS] Verify the Nix runtime cache ## Goal -`nix.moq.dev` serves releases: release tags substitute on a clean machine, -its bucket sizes hold under the 30-day expiry, and Cachix is retired. +Hosted CI proves `main`'s `moq-relay` and `moq` runtime closures reach +`nix.moq.dev`: a successful Cache run uploads both, and a later Smoke nix run +substitutes them instead of compiling. ## Plan [#5693](https://github.com/moq-dev/moq/pull/5693) wired `nix.moq.dev` into -CI. The maintainer ran the bootstrap in `infra/README.md` on 2026-10-19 and 2026-10-20 (the -release tag ruleset, both buckets and the Worker, the `cache-write` -Environment's tag policies and secrets, and the signing key). Check it with -`curl https://nix.moq.dev/nix-cache-info`. +CI. The maintainer ran the bootstrap in `infra/README.md` on 2026-10-19 and +2026-10-20. Check it with `curl https://nix.moq.dev/nix-cache-info`. [#6588](https://github.com/moq-dev/moq/pull/6588) restored `cache.yml`: every merge to `main` pushes the dev shell's build closure for x86 Linux, ARM -Linux, and ARM macOS to `nix-ci-moq-dev` (`sh/gh/cache.sh`), and -`release-nix.yml` pushes on release tags. Hosted CI entering the dev shell is -verified: Nightly run 40506938673 (2026-10-27) substituted it in about a -minute with nothing compiled, against 12-14 minutes of source builds in the -2026-10-16 baseline. - -Then confirm the rest, and record the numbers in the PR that closes this: - -- The first `moq-relay-v*` or `moq-cli-v*` tag cut after `release` carries - `release-nix.yml` pushes from all three legs, and `nix run github:moq-dev/moq/#moq-relay - --accept-flake-config` substitutes on a clean machine. - Tags are cut from `release`, which still has `cachix.yml` and `cache.yml` - and no `release-nix.yml`, so every tag so far, through `moq-relay-v0.17.9` - and `moq-cli-v0.14.9` on 2026-10-28, ran Cachix push and never reached - `nix.moq.dev`. Check with `gh run list --workflow release-nix.yml`; it stays - empty until a release cut merges `main` into `release`. -- Both bucket sizes after a month, against the 30-day expiry in - `infra/nix/lifecycle.json`, so no earlier than 2026-11-24: the first - `cache.yml` push to `nix-ci-moq-dev` ran 2026-10-24. Sizes need a - `wrangler login` to the Cloudflare account. A path expires 30 days after - its upload even while CI still reads it, and `nix copy` re-uploads it on the - next merge. Look for Nightly jobs that fell back to source builds once - paths expired (`building '/nix/store/...uniffi-bindgen` in a job log), and - refresh on a schedule or switch to a closure-aware collector (niks3) if - they recur. -- Retire Cachix once the first release cut from `main` shows - `release-nix.yml` succeeding on all three legs and the clean-machine - `nix run` above substitutes (decided 2026-10-29): that is the first release - a downstream user could fetch from `nix.moq.dev`. Then delete the - `CACHIX_AUTH_TOKEN` secret and the `kixelated` Cachix cache. - [moq-dev/moq#7431](https://github.com/moq-dev/moq/pull/7431) already - dropped Cachix from `main` (decided 2026-11-02, in place of moving its token - into `cache-write`): Smoke's nix channel substitutes from `nix.moq.dev` and - no longer pushes. `release` keeps `cachix.yml` until that cut merges `main` - into it. -- Also push the `moq-relay` and `moq` runtime closures to `nix-ci-moq-dev` - (decided 2026-10-29). The smoke nix channel builds `main` HEAD's - `moq-relay` and `moq` packages, which nothing caches since Smoke stopped - pushing to Cachix, so each new HEAD compiles for 20-43 minutes there. - `cache.yml` already holds the write credentials on every merge, so smoke - needs no secret. +Linux, and ARM macOS to `nix-ci-moq-dev` (`sh/gh/cache.sh`). Hosted CI +entering the dev shell is verified: Nightly run 40506938673 (2026-10-27) +substituted it in about a minute with nothing compiled, against 12-14 +minutes of source builds in the 2026-10-16 baseline. + +[moq-dev/moq#7662](https://github.com/moq-dev/moq/pull/7662) adds a Cache +`runtime` job (`sh/gh/cache-runtime.sh`) that pushes the `moq-relay` and `moq` +runtime closures, after their Crane dependency artifacts, on x86 Linux, +Smoke's only current nix platform (runtime caching decided 2026-10-29). +[moq-dev/moq#7431](https://github.com/moq-dev/moq/pull/7431) dropped Cachix +from `main` (decided 2026-11-02), so Smoke reads `nix.moq.dev` without write +credentials. + +Decided 2026-11-05 (maintainer, /quest-audit, +[moq-dev/moq#7975](https://github.com/moq-dev/moq/pull/7975)): ✅ keep only +this `main` evidence in m0 and move the release-tag proof, Cachix retirement, +and bucket sizes to +[Release tags substitute from nix.moq.dev](/quest/m2/release-nix-cache.md) +behind the release cut. Not chosen: moving the whole quest to m2; merging +`main` into `release` now. + +Use the scheduled and push runs; never dispatch Nightly for this evidence. + +## Waiting + +- moq-dev/moq: Once + [moq-dev/moq#7662](https://github.com/moq-dev/moq/pull/7662) merges, record the first successful Cache run's + `Runtime (ubuntu-24.04)` job uploading both closures + (`gh run list --workflow cache.yml --branch main`), then the next Smoke run + on that head or later (the 08:00 UTC schedule) whose + `ubuntu-latest / nix` job copies `moq-relay` and `moq` from + `https://nix.moq.dev` instead of compiling them. ## Related diff --git a/quest/m0/ci/publish-release-only.md b/quest/m0/ci/publish-release-only.md index e7143de7624..ed1dabc46f8 100644 --- a/quest/m0/ci/publish-release-only.md +++ b/quest/m0/ci/publish-release-only.md @@ -25,7 +25,7 @@ Environment and `CACHIX_AUTH_TOKEN` needs no new home. The alternatives were moving the token into `cache-write`, or deleting Cachix everywhere now with a `release` backport. `release` keeps its Cachix workflows until the next cut merges `main` into it, and -[nix-cache-verify](/quest/m0/ci/nix-cache-verify.md) deletes the Cachix +[Release tags substitute from nix.moq.dev](/quest/m2/release-nix-cache.md) deletes the Cachix cache and token once a release tag substitutes from `nix.moq.dev`. What remains is the maintainer's, in order: diff --git a/quest/m2/README.md b/quest/m2/README.md index 84e83de8cb9..154d2ecb49c 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -35,6 +35,7 @@ lite-07; bitrate enforcement is not part of this release. - [JS announced coalescing](/quest/m2/js-announced-coalesce.md) - `@moq/net`'s `Announced` keeps one pending event per path, like Rust - [An audio track's final flush joins its open group](/quest/m2/audio-flush-group.md) - a zero-`delay.max` subscriber gets the tail, not `Old` - [Release execution](/quest/m2/release/README.md) - publication, installed-artifact proof, cleanup, and downstream adoption after m1 +- [Release tags substitute from nix.moq.dev](/quest/m2/release-nix-cache.md) - the first tag after the cut substitutes on a clean machine, bucket sizes hold, and Cachix is retired - [One policy for oversized container metadata](/quest/m2/import-metadata-policy.md) - fMP4 and MKV refuse or stream huge metadata by one documented rule - [Wire fuzz coverage](/quest/m2/wire-fuzz-coverage.md) - the cmsg walks in moq-sock and moq-uring have invariant fuzz targets, and the unsafe peer input reaches runs under Miri - [VA-API renderer tests](/quest/m2/vaapi-render-tests.md) - the VA-API renderer, decode, and transcode tests run in `just rs gpu` on AMD and Intel and fail instead of skipping, and hosted CI reports them ignored diff --git a/quest/m2/release-nix-cache.md b/quest/m2/release-nix-cache.md new file mode 100644 index 00000000000..cd0cca0cc3c --- /dev/null +++ b/quest/m2/release-nix-cache.md @@ -0,0 +1,59 @@ +# [S] Release tags substitute from nix.moq.dev + +## Goal + +The first release tag cut from `main` substitutes from `nix.moq.dev` on a +clean machine, both bucket sizes hold under the CI bucket's 30-day expiry, and +Cachix is retired. + +## Plan + +Decided 2026-11-05 (maintainer, /quest-audit, +[moq-dev/moq#7975](https://github.com/moq-dev/moq/pull/7975)): ✅ split from +[Verify the Nix runtime cache](/quest/m0/ci/nix-cache-verify.md) and moved to +m2 behind [the cut](/quest/m2/release/cut.md), which brings `main`'s +`release-nix.yml` to `release` when it merges `main` into it. Not chosen: +moving the whole verification quest; merging `main` into `release` now, +which reverses the 2026-11-04 decision. + +Tags run `release`'s copies of the workflows. `release` still has +`cachix.yml` and `cache.yml` and no `release-nix.yml`, so every tag so far, +through `moq-relay-v0.17.9` and `moq-cli-v0.14.9` on 2026-10-28, pushed to +Cachix and never reached `nix.moq.dev`; +`gh run list --workflow release-nix.yml` stays empty until the cut. + +Once the cut lands: + +1. Record the first `moq-relay-v*` or `moq-cli-v*` tag's `release-nix.yml` + pushes from all three legs. +2. Record `nix run github:moq-dev/moq/#moq-relay --accept-flake-config` + substituting on a clean machine (`#moq` for a CLI tag), such as a fresh + container with only Nix installed. +3. Retire Cachix (decided 2026-10-29): that tag is the first release a + downstream user could fetch from `nix.moq.dev`. The cut also brings + `main`'s removal of `cachix.yml` to `release`. + +Public API: none. Wire: none. + +## Waiting + +- Maintainer: Record both bucket sizes no earlier than 2026-11-24, a month + after the first `cache.yml` push to `nix-ci-moq-dev` on 2026-10-24, against + the 30-day expiry in `infra/nix/lifecycle.json`. Sizes need a + `wrangler login` to the Cloudflare account, which agents lack. A path + expires 30 days after upload even while CI still reads it, and `nix copy` + re-uploads it on the next merge. Look for Nightly jobs that fell back to + source builds once paths expired (`building '/nix/store/...uniffi-bindgen` + in a job log), and refresh on a schedule or switch to a closure-aware + collector (niks3) if they recur. +- Maintainer: After step 2, delete the `CACHIX_AUTH_TOKEN` secret and the + `kixelated` Cachix cache; the Cachix account is the maintainer's, and + deleting the cache is irreversible. + +## Required + +- [Cut and publish the prepared breaking release](/quest/m2/release/cut.md) - brings `release-nix.yml` to `release`, where tags run it + +## Related + +- [Verify the Nix runtime cache](/quest/m0/ci/nix-cache-verify.md) - the same cache's `main` evidence diff --git a/sh/gh/cache-runtime.sh b/sh/gh/cache-runtime.sh new file mode 100755 index 00000000000..cc235ba9578 --- /dev/null +++ b/sh/gh/cache-runtime.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Publish Smoke's runtime packages and their reusable dependency artifacts. +set -euo pipefail + +: "${NIX_SIGNING_KEY:?not set; the job needs the cache-write Environment}" +: "${AWS_ACCESS_KEY_ID:?not set}" +: "${AWS_SECRET_ACCESS_KEY:?not set}" + +roots=$(mktemp -d) +trap 'rm -rf "$roots"' EXIT + +# Publish Crane's existing dependency outputs first so later package builds +# can substitute them. Their inputs include the patched kio sources. +nix build .#moq-relay.cargoArtifacts .#moq.cargoArtifacts --out-link "$roots/deps" --print-build-logs +just infra nix push nix-ci-moq-dev "$roots"/deps* + +nix build .#moq-relay .#moq --out-link "$roots/runtime" --print-build-logs +just infra nix push nix-ci-moq-dev "$roots"/runtime*