diff --git a/.github/justfile b/.github/justfile index 21238a2f744..131de93d722 100644 --- a/.github/justfile +++ b/.github/justfile @@ -82,6 +82,7 @@ test: python3 sh/rs/select.test.py python3 sh/rs/config.test.py python3 sh/rs/hakari.test.py + python3 sh/js/install.test.py python3 sh/js/workspace.test.py python3 sh/js/audit.test.py python3 sh/rs/test-lints.test.py diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 0547c599c56..a09496895ee 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -55,7 +55,7 @@ jobs: - name: Workspace dependencies id: setup - run: nix develop --command bun install --frozen-lockfile + run: nix develop --command just js install frozen shell: bash -leo pipefail {0} # Each suite reports independently after setup, while a failed step still fails the job. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e66080f43de..5ae318cdb34 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -425,7 +425,7 @@ jobs: - name: Install publisher and player engines # Firefox loads libpulse at runtime; Playwright's engine dependencies omit it. run: | - nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium firefox" + nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" bash -c "just js install frozen && bunx playwright install --with-deps chromium firefox" sudo apt-get install -y libpulse0 - name: Media and negative controls run: nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" just test media --media-engines firefox diff --git a/.github/workflows/release-js.yml b/.github/workflows/release-js.yml index a8d315dd2f5..76cf4762d05 100644 --- a/.github/workflows/release-js.yml +++ b/.github/workflows/release-js.yml @@ -48,7 +48,7 @@ jobs: package-manager-cache: false - name: Install dependencies - run: nix develop --command bun install --frozen-lockfile + run: nix develop --command just js install frozen - name: Release packages env: diff --git a/bench/browser/run.sh b/bench/browser/run.sh index bd95a6ceb7f..e36a6bfc965 100755 --- a/bench/browser/run.sh +++ b/bench/browser/run.sh @@ -88,7 +88,7 @@ if [[ -z $RELAY ]]; then fi cd "$WORKSPACE" -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen # BROWSER_BIN overrides the Chromium provided by the root devShell. cd "$HERE" diff --git a/demo/boy/justfile b/demo/boy/justfile index b5a3d25d6bd..55e16cf7dbe 100644 --- a/demo/boy/justfile +++ b/demo/boy/justfile @@ -2,7 +2,7 @@ set fallback # Run the GB demo: relay + emulator publisher + web viewer. default: - bun install + uv run --no-project python ../../sh/js/install.py update bun run concurrently --kill-others --prefix-colors auto \ "just relay" \ @@ -57,12 +57,12 @@ web url='http://localhost:4443': # Deploy the rom.moq.dev worker deploy: - bun install + uv run --no-project python ../../sh/js/install.py update bun wrangler deploy # Upload a single ROM to R2 upload file: - bun install + uv run --no-project python ../../sh/js/install.py update bun wrangler r2 object put "rom-moq-dev/{{ file }}" --file "rom/{{ file }}" --remote # Sync all ROMs to R2 @@ -72,5 +72,5 @@ sync dir="rom": # List files in the R2 bucket list: - bun install + uv run --no-project python ../../sh/js/install.py update bun wrangler r2 object list "rom-moq-dev" diff --git a/demo/carla/justfile b/demo/carla/justfile index 39b8696c8e5..1d48b946c73 100644 --- a/demo/carla/justfile +++ b/demo/carla/justfile @@ -10,7 +10,7 @@ set fallback # # Run a local relay, both trucks, and the web player. default: - bun install + uv run --no-project python ../../sh/js/install.py update bun run concurrently --kill-others --names relay,trucks,web --prefix-colors auto \ "just relay" \ "just wait && just wait-port 127.0.0.1 2000 && just worker" \ diff --git a/demo/justfile b/demo/justfile index 08b359d9ffa..9c431de0936 100644 --- a/demo/justfile +++ b/demo/justfile @@ -5,7 +5,7 @@ set fallback # Picks the first free port at/after 4443 so multiple worktrees can run # `just dev` without colliding; see sh/demo/run.sh. default: - bun install + uv run --no-project python ../sh/js/install.py update ../sh/demo/run.sh # Wait for the localhost relay to be ready by repeatedly requesting the cert diff --git a/demo/relay/justfile b/demo/relay/justfile index db04682e600..793d7504ecc 100644 --- a/demo/relay/justfile +++ b/demo/relay/justfile @@ -10,7 +10,7 @@ default: # Cluster peers still authenticate to each other via mTLS (the cert/ca recipes). cluster: - bun install + uv run --no-project python ../../sh/js/install.py update bun run concurrently --kill-others --names root,leaf0,leaf1,bbb,tos,web --prefix-colors auto \ "just root" \ diff --git a/demo/web/justfile b/demo/web/justfile index 29d6637263c..8c4dd42402a 100644 --- a/demo/web/justfile +++ b/demo/web/justfile @@ -13,4 +13,4 @@ serve url='http://localhost:4443': install # Install the workspace deps that vite and the demo's imports need. install: - bun install + uv run --no-project python ../../sh/js/install.py update diff --git a/js/justfile b/js/justfile index 78ba8fd396b..32a229b4320 100644 --- a/js/justfile +++ b/js/justfile @@ -10,6 +10,10 @@ set working-directory := '.' default: just check +# Install root workspace dependencies: frozen for validation, update for lockfile changes. +install mode: + uv run --no-project python ../sh/js/install.py {{ mode }} + # `build` runs here rather than as its own gate: the per-package `check` script # is `tsc --noEmit`, which never runs declaration emit, so the errors that only # surface when emitting a `.d.ts` (TS4023 and friends) pass `check` and then @@ -46,7 +50,7 @@ fix: # Rebuild changed workspace packages (or `--all`) to update their API reports. fix-changed list: - bun install + uv run --no-project python ../sh/js/install.py update MOQ_API_REPORT=update python3 ../sh/js/workspace.py build '{{ list }}' # Run all unit tests. @@ -63,44 +67,44 @@ coverage package="": # Run one net benchmark: broadcasts, forward, requests, fills, fetch, reader, frames, track, resume, observed, drift, ledger, bandwidth, replay, varint, lite-varint, priority, subscribe-drop, refresh, compression, or datagrams. bench name: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun net/bench/{{ name }}.ts # Sweep browser publisher fairness over backlog x viewers using the devShell's Chromium. serve-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/serve-browser.ts # Sweep catalog estimate publishes and subscribe updates over tracks x viewers. catalog-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun publish/bench/catalog.ts # Sweep QMux record decoding over record size x frames per record. qmux-record-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun qmux/bench/record.ts # Sweep QMux datagram receive-queue draining over backlog size (median of 7 runs). qmux-datagrams-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun qmux/bench/datagrams.ts # Sweep CMAF fragment decode over mdat bytes x declared sample count, including a hostile count. cmaf-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun hang/bench/cmaf.ts # Time Effect create and close, which pays for the creation stack its diagnostics print. signals-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun signals/bench/effect.ts # Bundle the worklets with Vite, webpack, esbuild, and Bun, and load them in Chromium, with and # without a strict CSP. Nightly, against the bundlers pinned in test/worklet. worklet-matrix: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun ../test/worklet/matrix.ts # Build all packages. @@ -113,87 +117,87 @@ dist-imports: # Compare eager and cold lazy WebSocket fallback setup in Chromium. qmux-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/qmux.ts # Sweep open arrival handles against their unread backlog. arrivals-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/arrivals.ts # Reduce and compare the worst known peer grant shapes, swept over AUTH_OK size. pattern-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project pattern/bench/tsconfig.json bun pattern/bench/algebra.ts # Sweep manual-clock armed timers x due timers, with independent and shared abort scopes. clock-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun clock/bench/timers.ts # Sweep buffered JSON stream reads over record count, payload size, and compression. json-stream-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/stream.ts # Sweep snapshot codec cost over document size and changed keys. json-codec-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/codec.ts # Compare the duplicate-refusing JSON frame parse against bare JSON.parse. json-parse-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/parse.ts # Sweep a slow hang consumer's skipped groups against its warning traffic and catch-up time. slow-group-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun hang/bench/slow-group.ts # Sweep a hang consumer's buffered groups, frames per group, publishers and players across arrival orders. buffered-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project hang/bench/tsconfig.json bun hang/bench/buffered.ts # Sweep the audio jitter estimator's held history against in-order and reordered frames. jitter-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun hang/bench/jitter.ts # Sweep players against the main-thread CPU and allocation of feeding the audio ring, in Chromium. ring-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/ring.ts # Sweep audio rendition count against repeated mute/unmute catalog updates. audio-support-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/support.ts # Sweep decoder count against PCM callback size during audio warmup. audio-warmup-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/warmup.ts # Sweep data-channel qmux stream count and payload size. p2p-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun p2p/bench/transport.ts # Sweep E2EE frame size against writes in flight through the WebCrypto pump. e2ee-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bun crypto/bench/pump.ts # Sweep audio renditions x raw retention through native WebCodecs in Chromium. raw-audio-bench: - bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project publish/bench/tsconfig.json bun publish/bench/audio.ts diff --git a/justfile b/justfile index c3b1e5dd2ca..59e7d4528ec 100644 --- a/justfile +++ b/justfile @@ -94,7 +94,7 @@ bench-browser *args: # Install repo-wide tooling. Per-language deps install on first check. install: - bun install + uv run --no-project python sh/js/install.py update cargo install --locked cargo-shear cargo-sort cargo-semver-checks release-plz # Lints, compiles, and tests only the packages the branch changed plus @@ -139,12 +139,12 @@ ci-select-platform $BASE="": # Check every supported source file; exclusions live in biome.jsonc. biome *args: - bun install --frozen-lockfile + uv run --no-project python sh/js/install.py frozen bun biome check --error-on-warnings {{ args }} . # Apply Biome's fixes; installs without a frozen lockfile, like the other fixers. biome-fix: - bun install + uv run --no-project python sh/js/install.py update bun biome check --error-on-warnings --write . # Auto-fix lint and formatting for what the branch changed since BASE. diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index b9f9cb1d409..3fd6d9b9424 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -123,7 +123,6 @@ Public API: none. Wire: none. - [Fallback playback](/quest/m0/test-flakes-2/ietf22-fallback.md) - a browser viewer follows a fallback within the media lane's bound without dropping its session - [dispatch-query's terminate test survives load](/quest/m0/test-flakes-2/dispatch-query-terminate.md) - `test_terminating_the_run_stops_the_running_phase` passes reliably under host load, re-enabled once fixed -- [Test PulseAudio servers stay off the desktop's D-Bus](/quest/m0/test-flakes-2/pulse-dbus.md) - concurrent media runs on one desktop never collide on `org.PulseAudio1` - [The Python audio playout-delay test reads its frames](/quest/m0/test-flakes-2/audio-playout-delay.md) - `test_audio_consumer_reports_its_playout_delay` stops timing out on its first read in the merge queue - [The dev host's locked-memory limit fits parallel sessions](/quest/m0/test-flakes-2/uring-memlock.md) - the maintainer's host raises the user's `RLIMIT_MEMLOCK`, which Node.js rings from parallel agent sessions fill before io_uring tests start - [The FFI concurrent-ready test wakes both calls under load](/quest/m0/test-flakes-2/ffi-concurrent-ready.md) - `concurrent_ready_calls_all_wake` stops stalling its reader drain under load and is re-enabled diff --git a/quest/m0/test-flakes-2/pulse-dbus.md b/quest/m0/test-flakes-2/pulse-dbus.md deleted file mode 100644 index 8c66fe3e416..00000000000 --- a/quest/m0/test-flakes-2/pulse-dbus.md +++ /dev/null @@ -1,23 +0,0 @@ -# [S] Test PulseAudio servers stay off the desktop's D-Bus - -## Goal - -Every private PulseAudio server the test suites start stays off the session -bus, so two media runs on one desktop never collide on the -`org.PulseAudio1` name. Concurrent `just test media` runs on a desktop host -pass where today one can fail with `D-Bus name org.PulseAudio1 already taken`. - -## Plan - -Planned 2026-11-05 (maintainer, /quest-plan of the follow-ups from the -2026-11-05 /quest-complete run), from -[moq-dev/moq#7716](https://github.com/moq-dev/moq/pull/7716). Milestone: ✅ m0. - -`test/lib/pulse.py` already drops `DBUS_SESSION_BUS_ADDRESS` from its own -daemon's environment, so another launch path still reaches the bus: find it -(the Playwright media runner, `test/native-play`, `test/audio-quality`, or a -module PulseAudio loads by default) and close it at the source, for example by -starting the daemon without its D-Bus modules. Reproduce with two overlapping -media runs first. - -Public API: none. Wire: none. diff --git a/setup.bat b/setup.bat index 0d8b07c94d1..e0c23d1b4f8 100644 --- a/setup.bat +++ b/setup.bat @@ -41,7 +41,7 @@ echo --- FFmpeg ^(CLI, for the `just dev` demo media^) --- %WG% --id Gyan.FFmpeg echo --- GitHub CLI --- %WG% --id GitHub.cli -echo --- uv ^(Python, for the py/ workspace^) --- +echo --- uv ^(Python for dependency setup and the py/ workspace^) --- %WG% --id astral-sh.uv REM MSVC linker + C++ headers, required for any Rust MSVC build. winget skips @@ -56,7 +56,7 @@ where rustup >nul 2>nul && ( rustup default stable & rustup update stable ) || e echo. echo === Installing JS dependencies ^(bun install^) === -where bun >nul 2>nul && bun install || echo NOTE: bun not on PATH yet. Reopen your terminal, then run: bun install +where bun >nul 2>nul && where uv >nul 2>nul && uv run --no-project python sh/js/install.py update || echo NOTE: bun or uv not on PATH yet. Reopen your terminal, then run: just js install update echo. echo === Done === diff --git a/sh/api-diff/run.sh b/sh/api-diff/run.sh index cbfb61054a8..93ce238356d 100755 --- a/sh/api-diff/run.sh +++ b/sh/api-diff/run.sh @@ -213,8 +213,8 @@ js() { } local manifest dir name version file entry surface # The renderer imports TypeDoc from this checkout's install, which a fresh runner lacks. The - # install scripts build assets, such as @moq/libav, that neither checkout needs here. - bun install --frozen-lockfile --ignore-scripts >&2 + # current checkout uses the coordinated full install shared with other readers. + uv run --no-project python sh/js/install.py frozen >&2 # Not local: the trap runs after a failure has unwound this function's locals. checkout=$out/js/checkout git worktree add --detach "$checkout" "$base" >&2 diff --git a/sh/coverage.test.py b/sh/coverage.test.py index 5aae214c404..f07df6c0ae9 100644 --- a/sh/coverage.test.py +++ b/sh/coverage.test.py @@ -17,16 +17,22 @@ def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) self.root = Path(self.directory.name) - for script in ("sh/js/coverage.py", "sh/rs/coverage.sh", "sh/rs/nextest.sh", "sh/local.sh"): + for script in ("sh/js/install.py", "sh/js/coverage.py", "sh/rs/coverage.sh", "sh/rs/nextest.sh", "sh/local.sh"): target = self.root / script target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ROOT / script, target) + subprocess.run(["git", "init", "-q", str(self.root)], check=True) self.bin = self.root / "bin" self.bin.mkdir() stub = """#!/usr/bin/env python3 import json, os from pathlib import Path import sys +if sys.argv[1:] == ["--version"]: + print("fixture") + sys.exit(0) +if "install" in sys.argv: + Path("node_modules").mkdir(exist_ok=True) with open(os.environ["COVERAGE_CALLS"], "a") as log: log.write(json.dumps([Path(sys.argv[0]).name, str(Path.cwd()), *sys.argv[1:]]) + "\\n") if os.environ.get("COVERAGE_FAIL") and ("nextest" in sys.argv or "test" in sys.argv): diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 5e231e0ece9..7844a7290d7 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -129,18 +129,18 @@ module cpp '^(\.cargo/config\.toml$|cpp/moq/|cpp/justfile$|sh/cpp/|sh/clang-form # the libobs headers it compiles against. module obs_compile '^(\.cargo/config\.toml$|cpp/obs/|cpp/moq/|sh/obs/|rs/moq-ffi/|rs/moq-c/native-libs/|flake\.nix$)' \ tools='cargo cmake pkg-config uniffi-bindgen-cpp' compiles=check lane=cpp -# The bun lane exists because `bun install` relinks the root workspace's -# packages even when nothing changed, so a concurrent install or import of -# them fails. Biome checks the whole repository. It registers before js so +# The bun lane serializes fixers and JS builds that write shared outputs. +# Dependency setup separately uses sh/js/install.py across every entry point. +# Biome checks the whole repository. It registers before js so # that fix runs it first, since Biome can fix sources that the package build # rejects, such as a type-only export. module biome '' \ - tools='bun' lane=bun fix + tools='bun uv' lane=bun fix # The JS tests also read Rust's shared JSON vectors; sh/js/workspace.py # fails a check that reads a cross-tree input this pattern misses. # Root Nix inputs own the browser version every npm Playwright pin matches. module js '^(js/|bench/browser/|doc/|drafts/|demo/(boy|web|carla/web)/|test/|rs/.*\.json$|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$|flake\.(nix|lock)$)' \ - tools='bun python3' lane=bun fix + tools='bun python3 uv' lane=bun fix module kt '^(\.cargo/config\.toml$|kt/|sh/kt/|rs/moq-ffi/|rs/uniffi-bindgen/|doc/lib/kt/|doc/lib/samples\.sh$)' \ tools='cargo gradle java ktfmt' compiles=check fix # Rust source is compiled below. Nix reads Cargo metadata, packaging inputs, @@ -161,11 +161,11 @@ module swift '^(\.cargo/config\.toml$|swift/|sh/swift/|rs/moq-ffi/|rs/uniffi-bin module go '^(\.cargo/config\.toml$|go/|sh/go/|rs/moq-ffi/|doc/lib/go/|doc/lib/samples\.sh$)' \ tools='cargo go uniffi-bindgen-go' compiles=check fix module drafts '^(drafts/|sh/drafts/|doc/\.vitepress/drafts\.ts$)' \ - tools='bun kramdown-rfc xml2rfc' lane=bun + tools='bun uv kramdown-rfc xml2rfc' lane=bun module markdown '' \ - tools='bun' lane=bun fix + tools='bun uv' lane=bun fix module gh '' \ - tools='actionlint zizmor bun python3 cargo release-plz clippy-driver dpkg-deb nfpm rpm' lane=bun + tools='actionlint zizmor bun python3 uv cargo release-plz clippy-driver dpkg-deb nfpm rpm' lane=bun # The drill mutations patch Rust source, so a Rust change can move the code # they target. Only nightly runs the drills; this catches a stale patch. module drill '^(rs/|test/drill/)' @@ -372,7 +372,7 @@ step() { check:ts_paths) python3 sh/gh/ts-paths.py ;; check:biome) just biome ;; check:markdown) - bun install --frozen-lockfile + uv run --no-project python sh/js/install.py frozen sh/markdown.sh check ;; check:large_files) sh/large-files.sh "$large_base" ;; @@ -392,7 +392,7 @@ step() { fix:js) just js fix-changed "$js_list" ;; fix:biome) just biome-fix ;; fix:markdown) - bun install + uv run --no-project python sh/js/install.py update sh/markdown.sh fix ;; fix:python) diff --git a/sh/drafts/check.sh b/sh/drafts/check.sh index e1f852cba4c..ada82b1aa66 100755 --- a/sh/drafts/check.sh +++ b/sh/drafts/check.sh @@ -12,7 +12,7 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen cd drafts if grep -n "$(printf '\t')" draft-*.md; then diff --git a/sh/js/README.md b/sh/js/README.md index 8d87bd23c07..8aaa93f744b 100644 --- a/sh/js/README.md +++ b/sh/js/README.md @@ -34,9 +34,18 @@ The root devShell supplies Chromium and exports its Playwright version. Checks require every npm `playwright` dependency outside `test/smoke` to pin that exact version. A root `flake.nix` or `flake.lock` change selects JS checks too: when nixpkgs moves Playwright, update the reported manifests and regenerate `bun.lock` -with `bun install` in the new shell. Smoke has its own flake and freshness check. +with `uv run --no-project python sh/js/install.py update` in the new shell. Smoke has its own flake and freshness check. Run `python3 sh/js/workspace.test.py` for graph and execution regressions. The same suite runs in `just gh test` on every check. To inspect work without executing it, pass a file containing relevant paths, one per line, to `python3 sh/js/workspace.py plan LISTFILE`, or pass `--all`. + +All root workspace installers use `uv run --no-project python sh/js/install.py frozen` for checks +and tests, or `update` when lockfile changes are allowed. Both reuse an unchanged +installation under one checkout lock, including libav output. Separate Worker, +smoke-client, and disposable baseline workspaces keep their own installs. + +The concurrency guarantee applies to unchanged dependency inputs. Stop active +consumers before changing dependencies or postinstall inputs. Unix installers share an OS lock. Native Windows runs one recipe at a time, +matching its existing development support; uv supplies Python on both setup paths. diff --git a/sh/js/coverage.py b/sh/js/coverage.py index 3d04bc3f57f..6b59fb1494f 100644 --- a/sh/js/coverage.py +++ b/sh/js/coverage.py @@ -66,7 +66,9 @@ def main(): (ROOT / "target/coverage/js/summary.json").unlink(missing_ok=True) for directory in selected: (ROOT / "target/coverage/js" / directory.name / "lcov.info").unlink(missing_ok=True) - subprocess.run(["bun", "install", "--frozen-lockfile"], cwd=ROOT, check=True) + subprocess.run( + ["uv", "run", "--no-project", "python", str(ROOT / "sh/js/install.py"), "frozen"], cwd=ROOT, check=True + ) for directory in selected: out = ROOT / "target/coverage/js" / directory.name out.mkdir(parents=True, exist_ok=True) diff --git a/sh/js/install.py b/sh/js/install.py new file mode 100644 index 00000000000..778f8c77bee --- /dev/null +++ b/sh/js/install.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Install the workspace once per dependency configuration, under a checkout lock.""" + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def fingerprint(): + digest = hashlib.sha256(subprocess.check_output(["bun", "--version"])) + digest.update(str(shutil.which("nix") is not None).encode()) + # Include the postinstall build inputs as well as dependency manifests. + manifest = json.loads((ROOT / "package.json").read_text()) + manifests = {b"package.json"} + for workspace in manifest["workspaces"]: + matches = list(ROOT.glob(workspace)) + # Exact workspace paths still contribute when their manifest was deleted. + for directory in matches or [ROOT / workspace]: + manifests.add(str((directory / "package.json").relative_to(ROOT)).encode()) + paths = subprocess.check_output( + [ + "git", + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + "bun.lock", + "flake.nix", + "flake.lock", + "nix/overlay.nix", + "js/libav", + "patches", + ], + cwd=ROOT, + ).split(b"\0") + paths = set(paths) | manifests + for name in sorted(filter(None, paths)): + digest.update(name) + path = ROOT / name.decode() + if path.exists(): + digest.update(path.read_bytes()) + else: + digest.update(b"missing") + return digest.hexdigest() + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("mode", choices=("frozen", "update")) +args = parser.parse_args() + +(ROOT / ".tmp").mkdir(exist_ok=True) +with (ROOT / ".tmp/bun-install.lock").open("ab") as lock: + # Native Windows supports sequential recipes; Unix coordinates parallel runs. + if os.name != "nt": + import fcntl + + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("waiting for another workspace dependency install", file=sys.stderr, flush=True) + fcntl.flock(lock, fcntl.LOCK_EX) + current = fingerprint() + stamp = ROOT / "node_modules/.moq-install" + # Identical inputs reuse the tree while other invocations consume it. + # Dependency edits invalidate it; stop consumers before changing inputs. + if ( + not stamp.exists() + or stamp.read_text() != current + or not (ROOT / "js/libav/dist/libav-opus-aac-af.mjs").exists() + ): + stamp.unlink(missing_ok=True) + subprocess.run( + ["bun", "install", *(["--frozen-lockfile"] if args.mode == "frozen" else [])], cwd=ROOT, check=True + ) + stamp.write_text(fingerprint()) diff --git a/sh/js/install.test.py b/sh/js/install.test.py new file mode 100644 index 00000000000..7e76e09070c --- /dev/null +++ b/sh/js/install.test.py @@ -0,0 +1,87 @@ +"""Concurrent root recipes and media entry points share one installation.""" + +import os +import shutil +import signal +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class Install(unittest.TestCase): + def test_overlap(self): + (ROOT / ".tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: + root = Path(directory) + (root / "sh/js").mkdir(parents=True) + shutil.copy(ROOT / "sh/js/install.py", root / "sh/js/install.py") + (root / "test").mkdir() + # Extract the setup commands from actual root and test recipes, + # leaving lint and playback to their own suites. + recipe_setup = (ROOT / "justfile").read_text().split("biome-fix:\n")[1].splitlines()[0] + test_setup = (ROOT / "test/justfile").read_text().splitlines() + test_setup = next(line for line in test_setup if "install.py" in line) + (root / "justfile").write_text("biome:\n" + recipe_setup + "\n") + (root / "test/justfile").write_text("media:\n" + test_setup + "\n") + (root / "package.json").write_text('{"workspaces": []}') + (root / "bun.lock").write_text("before") + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "add", "."], check=True) + bin_dir = root / "bin" + bin_dir.mkdir() + bun = bin_dir / "bun" + bun.write_text("""#!/usr/bin/env python3 +from pathlib import Path +import sys +r = Path.cwd() +if sys.argv[1] == '--version': + print('fixture') +else: + with (r / 'calls').open('a') as f: + f.write('install\\n') + # Signal the parent while holding the install lock, then block on a pipe. + print('ready', flush=True) + with open(r / 'continue') as f: + f.read() + (r / 'node_modules').mkdir(exist_ok=True) + (r / 'js/libav/dist').mkdir(parents=True, exist_ok=True) + (r / 'js/libav/dist/libav-opus-aac-af.mjs').touch() + (r / 'bun.lock').write_text('after') +""") + bun.chmod(0o755) + os.mkfifo(root / "continue") + env = {**os.environ, "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"]} + recipe = subprocess.Popen( + ["just", "biome"], cwd=root, env=env, stdout=subprocess.PIPE, text=True, start_new_session=True + ) + self.addCleanup(recipe.stdout.close) + self.addCleanup(lambda: os.killpg(recipe.pid, signal.SIGKILL) if recipe.poll() is None else None) + self.assertEqual(recipe.stdout.readline(), "ready\n") + test = subprocess.Popen( + ["just", "--justfile", "test/justfile", "media"], + cwd=root, + env=env, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + text=True, + start_new_session=True, + ) + self.addCleanup(test.stdout.close) + self.addCleanup(lambda: os.killpg(test.pid, signal.SIGKILL) if test.poll() is None else None) + self.assertIn("install.py frozen", test.stdout.readline()) + self.assertEqual(test.stdout.readline(), "waiting for another workspace dependency install\n") + with (root / "continue").open("w"): + pass + self.assertEqual(recipe.wait(), 0) + self.assertEqual(test.wait(), 0) + self.assertEqual((root / "calls").read_text(), "install\n") + # Frozen readers reuse the post-update fingerprint as well. + subprocess.run(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env, check=True) + self.assertEqual((root / "calls").read_text(), "install\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/sh/js/workspace.py b/sh/js/workspace.py index b7ea3979d64..71cb530ed90 100644 --- a/sh/js/workspace.py +++ b/sh/js/workspace.py @@ -148,7 +148,7 @@ def reports(): if directories: command("bun", "js/common/api.ts", "check", *directories) - command("bun", "install", "--frozen-lockfile") + command("uv", "run", "--no-project", "python", str(ROOT / "sh/js/install.py"), "frozen") if action == "test": command("python3", "sh/js/playwright.test.py") command( diff --git a/sh/js/workspace.test.py b/sh/js/workspace.test.py index 9f9ad2bac50..dce937f7524 100644 --- a/sh/js/workspace.test.py +++ b/sh/js/workspace.test.py @@ -54,6 +54,8 @@ def test_dispatcher_passes_only_js_impact_paths(self): sh = self.root / "sh" shutil.copy(selector.ROOT / "sh/dispatch.sh", sh / "dispatch.sh") shutil.copy(selector.ROOT / "sh/quest.sh", sh / "quest.sh") + (sh / "js").mkdir() + (sh / "js/install.py").write_text("# Dependency setup is outside this selector fixture.\n") changed = sh / "changed.sh" changed.write_text("#!/bin/sh\nprintf '%s\\n' js/auth/src/token.ts demo/pub/worker.ts quest/unrelated.md\n") changed.chmod(0o755) diff --git a/sh/size/collect.sh b/sh/size/collect.sh index ebe54660b4d..96955f13365 100755 --- a/sh/size/collect.sh +++ b/sh/size/collect.sh @@ -15,7 +15,7 @@ esac cargo build --locked --release -p moq-ffi --no-default-features --target-dir "$target/minimal" cp "$target/minimal/release/libmoq_ffi.$dylib" "$out/native/libmoq_ffi-minimal.$dylib" cp "$target/minimal/release/libmoq_ffi.a" "$out/native/libmoq_ffi-minimal.a" -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen just js build bun sh/size/bundles.ts "$out/js" # Symbols are necessary only for attribution; they never replace the measured shipping sizes. diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index 54666f88bf7..3d6d7c10021 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -26,7 +26,7 @@ for tool in python3 pulseaudio; do } done -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen # The native listener is the shipped player, built with its speaker. cargo build --locked --features moq-cli/play -p moq-relay -p moq-cli TARGET_BASE="${CARGO_TARGET_DIR:-$ROOT/target}" diff --git a/test/audio-quality/record.sh b/test/audio-quality/record.sh index 49fc8fd74d2..cb3c682668f 100755 --- a/test/audio-quality/record.sh +++ b/test/audio-quality/record.sh @@ -63,7 +63,7 @@ harness_begin audio-quality-record "$RERUN" echo "building the pages..." ( cd "$CLIENT" - bun install --frozen-lockfile + uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir ) diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index 82fd717f83a..f1cfd36deed 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -241,7 +241,7 @@ done harness_begin audio-quality "$RERUN" echo "installing the workspace..." -(cd "$CLIENT" && bun install --frozen-lockfile) +(cd "$CLIENT" && uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen) failed=0 diff --git a/test/drain/run.sh b/test/drain/run.sh index a4d823ccac4..1faa685bab2 100755 --- a/test/drain/run.sh +++ b/test/drain/run.sh @@ -61,7 +61,7 @@ flag=() TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" [[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" -(cd "$WORKSPACE" && bun install --frozen-lockfile) +uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen # ── relays ────────────────────────────────────────────────────────────────── # Start one relay on a reserved port: `start [extra relay args...]`. diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh index a5fc3b3e5d2..387071eb488 100755 --- a/test/ietf-conformance/run.sh +++ b/test/ietf-conformance/run.sh @@ -30,7 +30,7 @@ relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" # Cargo accepts an absolute target directory too. [[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" -bun install --frozen-lockfile +uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen harness_begin ietf-conformance "$(harness_env HARNESS_RESULTS)just test ietf-conformance$(harness_argv "$@")" python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" # LibreSSL in the dev shell has no system-wide openssl.cnf. diff --git a/test/interop/compat.sh b/test/interop/compat.sh index 73e158e1369..25c715557e8 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -31,7 +31,7 @@ case "$*" in ;; esac cd "$WORKSPACE" -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 7aba8db51a4..62eb2137a72 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -709,7 +709,7 @@ prepare_js() { return fi echo "installing js clients (workspace @moq/* via bun)..." - if ! (cd "$WORKSPACE" && bun install --frozen-lockfile) >"$HARNESS_RUN/js-install.log" 2>&1; then + if ! uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen >"$HARNESS_RUN/js-install.log" 2>&1; then for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done sed 's/^/ /' "$HARNESS_RUN/js-install.log" >&2 || true return diff --git a/test/json-properties.sh b/test/json-properties.sh index ca33fbeda72..810694e1ba1 100644 --- a/test/json-properties.sh +++ b/test/json-properties.sh @@ -7,7 +7,7 @@ MOQ_JSON_SEED=${MOQ_JSON_SEED:-$(python3 -c 'import secrets; print(secrets.randb MOQ_JSON_CASES=${MOQ_JSON_CASES:-4096} export MOQ_JSON_SEED MOQ_JSON_CASES echo "JSON codec properties: MOQ_JSON_SEED=$MOQ_JSON_SEED MOQ_JSON_CASES=$MOQ_JSON_CASES" -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen just rs test -p moq-json --lib -E 'test(snapshot_codec_property) | test(window_codec_property)' bun test ./js/json/src/snapshot/properties.test.ts ./js/json/src/window/properties.test.ts bunx tsc --noEmit --project js/json/bench/tsconfig.json diff --git a/test/justfile b/test/justfile index 060ce9fda5b..60f42295606 100644 --- a/test/justfile +++ b/test/justfile @@ -52,7 +52,7 @@ harness: python3 interop/report.test.py python3 interop/tone.test.py cd .. && bun test {{ replace(harness_compat_tests, "\n", " ") }} - cd .. && bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen cd interop/clients/js && bash ../../../../sh/local.sh bun test {{ replace(harness_browser_tests, "\n", " ") }} # Cross-language media interop test, built from this checkout. Stands up a @@ -66,7 +66,7 @@ harness: # Both runs first check the Rust/JS wire and session interop suites, in both directions. [positional-arguments] interop *args: - cd .. && bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen just rs test -p moq-net -p moq-crypto --lib \ auth_unsupported_interop \ ietf_compression_fallback_interop \ @@ -250,7 +250,7 @@ drain *args: # The @moq/video renderer in headless Chromium: WebGPU, a lost device, and the Canvas2D fallback. video: - cd .. && bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen # A desktop display makes software Vulkan request unsupported window surfaces. cd video && bash ../../sh/local.sh env -u DISPLAY -u WAYLAND_DISPLAY bun test ./renderer.browser.ts @@ -290,10 +290,10 @@ resume-model: # TypeScript-to-Rust qmux interop. qmux: cargo build --locked --quiet --manifest-path ../rs/qmux/Cargo.toml --example interop-server --example interop-client - cd .. && bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen cd .. && QMUX_INTEROP_BIN={{ quote(qmux_examples) }} bun test ./test/qmux/interop.test.ts # Complete MoQ SETUP and transfer a frame over a real browser data-channel pair. p2p: - cd .. && bun install --frozen-lockfile + uv run --no-project python ../sh/js/install.py frozen cd p2p && bash ../../sh/local.sh bun test ./transport.browser.ts diff --git a/test/portrait/run.sh b/test/portrait/run.sh index a2144e8a7c7..c14fe333fca 100755 --- a/test/portrait/run.sh +++ b/test/portrait/run.sh @@ -12,7 +12,7 @@ source "$DIR/../lib/harness.sh" harness_begin portrait 'just test portrait' cd "$WORKSPACE" cargo build --locked -p moq-relay -p moq-cli --features moq-cli/transcode,moq-cli/openh264 -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen (cd test/interop/clients/js && bunx vite build --outDir "$HARNESS_RUN/js-dist") harness_port relay URL="http://127.0.0.1:$HARNESS_PORT" diff --git a/test/resume-model.sh b/test/resume-model.sh index 3523f9b753a..b91a8ed5821 100755 --- a/test/resume-model.sh +++ b/test/resume-model.sh @@ -8,5 +8,5 @@ MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} export MOQ_RESUME_SEED MOQ_RESUME_CASES echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" just rs test -p moq-net --features async --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' -bun install --frozen-lockfile +uv run --no-project python sh/js/install.py frozen bun test ./js/net/src/resume.model.test.ts