From 75b3490a2df0a10850d47518f8ecba3d9b454e33 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 23:11:15 -0800 Subject: [PATCH 01/25] chore(quest): claim quest/m0/test-flakes-2/pulse-dbus.md Quest-Claim: eb53a5a92675d9d8 From f00d57e88cae383bc857f60c6161f9dd04970347 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 23:18:25 -0800 Subject: [PATCH 02/25] fix(test): keep private PulseAudio off the session bus Co-Authored-By: GPT-6.1 Sol --- quest/m0/test-flakes-2/README.md | 1 - quest/m0/test-flakes-2/pulse-dbus.md | 27 --------------------------- 2 files changed, 28 deletions(-) delete mode 100644 quest/m0/test-flakes-2/pulse-dbus.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 875eb5e3ece..4c225277949 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -97,6 +97,5 @@ Public API: none. Wire: none. - [moq-ffi's panicked-origin supervision test survives load](/quest/m0/test-flakes-2/ffi-origin-panic.md) - a read the origin driver's panic ends always reports `MoqError::Task` - [The scheduled macOS WebSocket version test completes reliably](/quest/m0/test-flakes-2/macos-websocket.md) - fix the scheduled timeout while preserving newest-version negotiation assertions - [@moq/net's lite subscribe tests wait for the whole request](/quest/m0/test-flakes-2/js-subscribe-fixture.md) - the three lite subscribe-with-track cases pass under forced yields -- [Concurrent PulseAudio harness runs don't share a D-Bus name](/quest/m0/test-flakes-2/pulse-dbus.md) - parallel media runs no longer collide on `org.PulseAudio1` - [@moq/net's uni-stream flood test survives reruns](/quest/m0/test-flakes-2/js-stream-flood.md) - "a flood of uni streams yields" passes under `--rerun-each` and load - [The JS receive-credit qmux test never completes an oversized frame](/quest/m0/test-flakes-2/receive-credit-qmux.md) - `javascript_partial_frame_gives_way_qmux` stops completing a frame larger than the window on moq-transport-18 diff --git a/quest/m0/test-flakes-2/pulse-dbus.md b/quest/m0/test-flakes-2/pulse-dbus.md deleted file mode 100644 index 3b9db6d1777..00000000000 --- a/quest/m0/test-flakes-2/pulse-dbus.md +++ /dev/null @@ -1,27 +0,0 @@ -# [XS] Concurrent PulseAudio harness runs don't share a D-Bus name - -## Goal - -Two media harness runs on one host each start a private PulseAudio without -`D-Bus name org.PulseAudio1 already taken`, so a concurrent run never fails -before its browser launches. - -## Plan - -Decided 2026-11-03 (maintainer, /quest-plan of the follow-ups from -[moq-dev/moq#7716](https://github.com/moq-dev/moq/pull/7716)): milestone m0, -a child of the test-flakes epic, since parallel agent runs hit it. - -- `test/lib/pulse.py` (lines 44–53) isolates `XDG_RUNTIME_DIR`, `PULSE_*`, - `ALSA_CONFIG_PATH`, and `DISPLAY`. It passes `DBUS_SESSION_BUS_ADDRESS` - through, so every private daemon tries to claim the same session-bus name. - Callers: `test/interop/interop.sh:2118`, `test/audio-quality/run.sh:349`, - and native-play. -- Give each daemon no session bus. Unset the address or disable PulseAudio's - D-Bus module in the private config, after checking that the daemon starts - cleanly with no bus. If it doesn't, run it under its own - `dbus-run-session`. -- Add a test that starts two `pulse.py` instances at once and plays through - each. - -Public API: none. Wire: none. From 42b4384377d0908623bbe9dacd3bde63551fac33 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 23:18:39 -0800 Subject: [PATCH 03/25] test: isolate and exercise concurrent private audio servers Co-Authored-By: GPT-6.1 Sol --- flake.nix | 2 + test/justfile | 1 + test/lib/pulse.py | 7 ++- test/lib/pulse.test.py | 117 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 test/lib/pulse.test.py diff --git a/flake.nix b/flake.nix index 09c3ccfd3dd..b009bce7853 100644 --- a/flake.nix +++ b/flake.nix @@ -188,6 +188,8 @@ pkgs.alsa-plugins # A private clocked null device for native playback tests. (pkgs.lib.getBin pkgs.pulseaudio) + # A shared session bus for the concurrent private audio server test. + pkgs.dbus # moq-video's `pipewire` screen-capture feature: the pipewire crate # links libpipewire-0.3 via pkg-config and generates bindings at build # time (bindgenHook above provides libclang). Linux-only; macOS uses diff --git a/test/justfile b/test/justfile index 467497e2ef5..5df4c862273 100644 --- a/test/justfile +++ b/test/justfile @@ -45,6 +45,7 @@ default: # from plain `bun test` so unit tests do not launch a browser. harness: ./lib/harness.test.sh + python3 lib/pulse.test.py ./interop/relay-log.test.sh python3 interop/staging.test.py python3 ../rs/moq-gst/smoke.test.py diff --git a/test/lib/pulse.py b/test/lib/pulse.py index 4ddd3eb0ede..9f48abf17cd 100644 --- a/test/lib/pulse.py +++ b/test/lib/pulse.py @@ -51,6 +51,11 @@ def run(): "DISPLAY": "", "WAYLAND_DISPLAY": "", } + # The daemon claims org.PulseAudio1 even without a D-Bus module. With no + # session address (and no DISPLAY to autolaunch one), it uses only our socket. + # Keep the wrapped command's session bus available for its other services. + daemon_env = env.copy() + daemon_env.pop("DBUS_SESSION_BUS_ADDRESS", None) # By default the null sink renders 2 s ahead and sleeps through it, and a new stream does not # rewind that, so a stream's first callback waits up to 2 s: Firefox reports its AudioContext @@ -68,7 +73,7 @@ def run(): f"--load=module-native-protocol-unix socket={socket} auth-anonymous=1", "--load=module-null-sink sink_name=moq-null rate=48000 channels=2 norewinds=1", ], - env=env, + env=daemon_env, stderr=subprocess.PIPE, ) try: diff --git a/test/lib/pulse.test.py b/test/lib/pulse.test.py new file mode 100644 index 00000000000..abf62b26e52 --- /dev/null +++ b/test/lib/pulse.test.py @@ -0,0 +1,117 @@ +"""Private audio servers start together and play on their own sinks under one session bus.""" + +import array +import json +import math +import os +import signal +import subprocess +import sys +import tempfile +import unittest +import wave +from contextlib import ExitStack +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] + + +def play(tone): + assert os.environ["DBUS_SESSION_BUS_ADDRESS"] + info = json.loads(subprocess.check_output(["pactl", "--format=json", "info"])) + assert info["server_string"] == os.environ["PULSE_SERVER"] + assert info["default_sink_name"] == "moq-null" + # Hold both daemons alive before either playback can finish and release its D-Bus name. + print("ready", flush=True) + assert sys.stdin.readline() == "play\n" + subprocess.run(["paplay", "--device=moq-null", str(tone)], check=True) + + +def parallel(root): + with ExitStack() as children: + runs = [ + children.enter_context( + subprocess.Popen( + [ + sys.executable, + str(HERE / "pulse.py"), + "--dir", + str(root / name), + "--", + sys.executable, + str(Path(__file__).resolve()), + "--play", + str(root / "tone.wav"), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + ) + for name in ("one", "two") + ] + for child in runs: + assert child.stdout.readline() == b"ready\n", "private server exited before playback" + for child in runs: + child.stdin.write(b"play\n") + child.stdin.flush() + for child in runs: + output, _ = child.communicate() + assert child.returncode == 0, output.decode() + + +@unittest.skipUnless(sys.platform == "linux", "private PulseAudio is Linux-only") +class Pulse(unittest.TestCase): + def test_parallel_playback(self): + scratch = ROOT / ".tmp" + scratch.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=scratch, prefix="pulse-") as directory: + root = Path(directory) + bus_config = root / "bus.conf" + bus_config.write_text( + f"sessionunix:path={root}/bEXTERNAL" + '' + '' + ) + with wave.open(str(root / "tone.wav"), "wb") as tone: + tone.setparams((1, 2, 48000, 0, "NONE", "not compressed")) + samples = array.array("h", (int(8000 * math.sin(2 * math.pi * 440 * i / 48000)) for i in range(12000))) + if sys.byteorder != "little": + samples.byteswap() + tone.writeframes(samples.tobytes()) + # The shared bus is private to this test, never the desktop's session bus. + with subprocess.Popen( + [ + "dbus-run-session", + f"--config-file={bus_config}", + "--", + sys.executable, + str(Path(__file__).resolve()), + "--parallel", + str(root), + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + ) as session: + try: + output, _ = session.communicate(timeout=30) + finally: + # Reap the whole test tree even if startup or a playback client hangs. + try: + os.killpg(session.pid, signal.SIGKILL) + except ProcessLookupError: + pass + session.wait() + logs = "\n".join(f"{log}:\n{log.read_text()}" for log in root.glob("*/pulse.log")) + self.assertEqual(session.returncode, 0, output.decode() + logs) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--play"]: + play(Path(sys.argv[2])) + elif sys.argv[1:2] == ["--parallel"]: + parallel(Path(sys.argv[2])) + else: + unittest.main() From 1ef8eb067e9b14b49cab195f42fe3f48ff002877 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 23:35:53 -0800 Subject: [PATCH 04/25] test: include private audio logs when the concurrency check hangs Co-Authored-By: GPT-6.1 Sol --- test/lib/pulse.test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/lib/pulse.test.py b/test/lib/pulse.test.py index abf62b26e52..ddc0b30ada5 100644 --- a/test/lib/pulse.test.py +++ b/test/lib/pulse.test.py @@ -95,17 +95,22 @@ def test_parallel_playback(self): stderr=subprocess.STDOUT, start_new_session=True, ) as session: + timed_out = False try: - output, _ = session.communicate(timeout=30) + session.communicate(timeout=30) + except subprocess.TimeoutExpired: + timed_out = True finally: # Reap the whole test tree even if startup or a playback client hangs. try: os.killpg(session.pid, signal.SIGKILL) except ProcessLookupError: pass - session.wait() + output, _ = session.communicate() logs = "\n".join(f"{log}:\n{log.read_text()}" for log in root.glob("*/pulse.log")) - self.assertEqual(session.returncode, 0, output.decode() + logs) + diagnostics = output.decode() + logs + self.assertFalse(timed_out, "private audio test timed out\n" + diagnostics) + self.assertEqual(session.returncode, 0, diagnostics) if __name__ == "__main__": From 88bb7e5d91bb95d7d83e7b654af734e62c2f737b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 12:42:20 -0800 Subject: [PATCH 05/25] chore(quest): claim quest/m0/test-flakes-2/pulse-dbus.md Quest-Claim: c8e26c3b75275cd5 From 15ad9101f234c995c8346e24d442210fbe671b03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 12:45:15 -0800 Subject: [PATCH 06/25] chore(quest): record PulseAudio evidence and media validation blocker --- quest/m0/test-flakes-2/pulse-dbus.md | 29 ++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/quest/m0/test-flakes-2/pulse-dbus.md b/quest/m0/test-flakes-2/pulse-dbus.md index 8c66fe3e416..24bc3724637 100644 --- a/quest/m0/test-flakes-2/pulse-dbus.md +++ b/quest/m0/test-flakes-2/pulse-dbus.md @@ -13,11 +13,28 @@ Planned 2026-11-05 (maintainer, /quest-plan of the follow-ups from the 2026-11-05 /quest-complete run), from [moq-dev/moq#7716](https://github.com/moq-dev/moq/pull/7716). Milestone: ✅ m0. -`test/lib/pulse.py` already drops `DBUS_SESSION_BUS_ADDRESS` from its own -daemon's environment, so another launch path still reaches the bus: find it -(the Playwright media runner, `test/native-play`, `test/audio-quality`, or a -module PulseAudio loads by default) and close it at the source, for example by -starting the daemon without its D-Bus modules. Reproduce with two overlapping -media runs first. +The daemon-level name registration is independent of D-Bus modules +([PulseAudio 17 startup](https://github.com/pulseaudio/pulseaudio/blob/v17.0/src/daemon/main.c#L1202-L1211)). +[moq-dev/moq#7831](https://github.com/moq-dev/moq/pull/7831) already isolated +that launch path before this quest was recreated. All explicit media, +native-play, and native audio-quality launches use `test/lib/pulse.py`. + +2026-11-05 verification: `python3 test/lib/pulse.test.py` passes on current +main. A scratch copy restoring the daemon's session-bus inheritance fails +with `D-Bus name org.PulseAudio1 already taken` under the test's private bus. +No desktop services were modified. No remaining launch path has been reproduced. + +Two overlapping `just test media --media-cases none` runs in the Nix shell +both stop before browser startup: the relay rejects `MOQ_QUEST`, which the +shell exports. Concurrent Bun installs also collide in the shared +`node_modules` (`EEXIST`/`ENOENT`). Do not infer an audio result from these runs. + +After the shell fix lands, finish dependency installation before starting +the overlapping runs. If those pass, retire this duplicate quest; otherwise +trace the failing daemon from its run's `pulse.log` and isolate that source. Public API: none. Wire: none. + +## Waiting + +- moq-dev/moq: land [the dev-shell environment fix](https://github.com/moq-dev/moq/pull/7979), then rerun overlapping media cases without altering the harness environment. Recommendation: retain the existing isolation unless a remaining audio collision is reproduced. From 9b480acb2d6240d8066d699c92b4dde424e640ad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 22:55:24 -0800 Subject: [PATCH 07/25] docs(quest): record overlapping media validation and daemon logs --- quest/m0/test-flakes-2/pulse-dbus.md | 35 +++++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/quest/m0/test-flakes-2/pulse-dbus.md b/quest/m0/test-flakes-2/pulse-dbus.md index 24bc3724637..3be90fc993f 100644 --- a/quest/m0/test-flakes-2/pulse-dbus.md +++ b/quest/m0/test-flakes-2/pulse-dbus.md @@ -24,17 +24,34 @@ main. A scratch copy restoring the daemon's session-bus inheritance fails with `D-Bus name org.PulseAudio1 already taken` under the test's private bus. No desktop services were modified. No remaining launch path has been reproduced. -Two overlapping `just test media --media-cases none` runs in the Nix shell -both stop before browser startup: the relay rejects `MOQ_QUEST`, which the -shell exports. Concurrent Bun installs also collide in the shared -`node_modules` (`EEXIST`/`ENOENT`). Do not infer an audio result from these runs. - -After the shell fix lands, finish dependency installation before starting -the overlapping runs. If those pass, retire this duplicate quest; otherwise -trace the failing daemon from its run's `pulse.log` and isolate that source. +2026-11-06 validation: main includes the merged +[dev-shell environment fix](https://github.com/moq-dev/moq/pull/7979). +`nix develop --command quest gate bun install --frozen-lockfile` and +`nix develop --command python3 test/lib/pulse.test.py` pass. + +The previous `--media-cases none` command now selects an empty focused case +set, which the browser runner rejects. Validate two overlapping +`just test media --media-cases pause` runs instead, without changing the +harness environment. Dependency installation is complete before overlap. + +The overlapping pause runs both failed after reaching playback (2026-11-06 +06:53 UTC): run `interop-JSI7CQaU` failed `resume takes effect` with the +presented frame stuck at 31; run `interop-om6rAC4x` failed `audio tone` after +resume (63/75 samples). The latter recorded host idle time of 0.1–2.9%, +which suggests contention but does not establish the cause. + +Both runs' `pulse-1/pulse.log` show `Daemon startup complete.`, independent +runtime directories and normal SIGTERM teardown. Neither reports a D-Bus +name collision. Both warn that D-Bus autolaunch is unavailable without +`DISPLAY`, then successfully start. No failing daemon launch was found. +The relay environment rejection and Bun install collisions did not recur. + +These runs do not meet the requested passing-overlap condition for retirement. +Keep the existing isolation. Investigate the resume failures separately before +claiming an overlap pass; do not infer a PulseAudio defect from these failures. Public API: none. Wire: none. ## Waiting -- moq-dev/moq: land [the dev-shell environment fix](https://github.com/moq-dev/moq/pull/7979), then rerun overlapping media cases without altering the harness environment. Recommendation: retain the existing isolation unless a remaining audio collision is reproduced. +- Maintainer: choose whether to retain this quest pending a passing overlap or retire it based on the existing daemon-isolation test and successful daemon startups. Recommend retaining it pending validation of the resume failures; the requested passing-overlap condition was not met. An agent cannot authorize deletion. From dddf35560d42e017999043b81093cfc5185e8a1e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 15:51:44 -0800 Subject: [PATCH 08/25] quest: Safari and WebSocket publishers wait for WebKit's send estimate Co-Authored-By: Claude Opus 5.5 --- quest/m2/probe/browser-send-estimate.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/quest/m2/probe/browser-send-estimate.md b/quest/m2/probe/browser-send-estimate.md index d9c58207d42..10d9bce0f03 100644 --- a/quest/m2/probe/browser-send-estimate.md +++ b/quest/m2/probe/browser-send-estimate.md @@ -24,15 +24,22 @@ Reason: one rule decided at setup fixes the RTT, the polling, and the counters. What remains once the waits clear: - In stable Chrome (no flag) and Firefox, verify through a shaped link that a publisher's bitrate follows a capacity drop. Reuse the [probe epic](/quest/m2/probe/README.md)'s shaped-uplink harness if it has landed. -- The WebSocket fallback has no estimate whatever the browsers ship, so its publishers still hold their rate; the maintainer decision below covers it. +- The WebSocket fallback has no estimate whatever the browsers ship, so its publishers still hold their rate, as decided below. -Public API: none expected. Wire: none, unless the decision below picks a receiver report. +Decided 2026-11-07 (maintainer, /quest-unblock): Safari publishers and the WebSocket fallback get no estimate before WebKit reports one. + +- ✅ (a) Wait for WebKit, as today. +- (b) A lite-07 receiver report, where the relay sends back the rate it receives from us, which can only lower the estimate while app-limited. +- (c) Estimate locally from how fast the transport drains our writes, which depends on each browser's buffering. + +Reason: Chrome and Firefox need nothing, and (b) and (c) each build an app-layer controller above the browser's own that a later WebKit release makes redundant. + +Public API: none expected. Wire: none. ## Waiting - Chrome: ships `WebTransport.getStats()` with `estimatedSendRate` enabled for all users in 159, per the 2026-11-05 Intent to Ship. Check that stable Chrome's `getStats()` reports a non-null `estimatedSendRate` on a connection that has been sending media for a few seconds. - WebKit: Safari's `getStats()` reports a non-null `estimatedSendRate`. Check the `return_object_property_estimatedSendRate` entry under `api.WebTransport.getStats` in browser-compat-data, or a Safari publisher's `Reservation.rate`. -- Maintainer: whether Safari publishers (and the WebSocket fallback) get an estimate before WebKit reports one. An agent can't pick this because each option changes the wire or adds a congestion estimator above the browser's own. Options: (a) wait for WebKit, as today; (b) a lite-07 receiver report, where the relay sends back the rate it receives from us, which can only lower the estimate while app-limited; (c) estimate locally from how fast the transport drains our writes, which depends on each browser's buffering. Recommendation: (a), since Chrome and Firefox need nothing, and (b) and (c) each build an app-layer controller that a later WebKit release makes redundant. ## Related From b15846d553765b216155794dee1f33c5537377bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 15:54:11 -0800 Subject: [PATCH 09/25] docs: record overlapping media install collision Co-Authored-By: GPT-6.1 Sol --- quest/m0/test-flakes-2/pulse-dbus.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/quest/m0/test-flakes-2/pulse-dbus.md b/quest/m0/test-flakes-2/pulse-dbus.md index 3be90fc993f..3108ecd0de7 100644 --- a/quest/m0/test-flakes-2/pulse-dbus.md +++ b/quest/m0/test-flakes-2/pulse-dbus.md @@ -52,6 +52,21 @@ claiming an overlap pass; do not infer a PulseAudio defect from these failures. Public API: none. Wire: none. +2026-11-06 23:53 UTC rerun after merging current main: dependency installation +passed before overlap, and `quest gate just ci` passed (112 seconds), including +`test gh`. Both overlapping `just test media --media-cases pause` runs exited 1 +before browser startup. Runs `interop-iPWHd34k` and `interop-wdFesAVD` failed +inside the harness's own concurrent Bun installs: `Failed to link semver: +EEXIST` (and concurrently, vite, yaml, and @babel/parser). Both reported +`media checks (browser client unavailable)`. Neither created a PulseAudio +server or `pulse.log`; this attempt gives no new daemon-isolation evidence. +Outputs and full run directories are retained in the implementation checkout's +`.tmp/`. Recommend serializing the harness dependency installation while keeping +the two playback runs concurrent, then repeating the overlap unchanged. + +Decided 2026-11-06 (maintainer, /quest-complete): ✅ retain this quest until +overlapping media runs pass. Retiring it based on clean daemon logs was not chosen. + ## Waiting -- Maintainer: choose whether to retain this quest pending a passing overlap or retire it based on the existing daemon-isolation test and successful daemon startups. Recommend retaining it pending validation of the resume failures; the requested passing-overlap condition was not met. An agent cannot authorize deletion. +- moq-dev/moq: resolve concurrent Bun install link collisions in the media harness, then demonstrate two overlapping `just test media --media-cases pause` runs pass. Preinstalling dependencies alone did not prevent the internal installs from colliding. Retain existing daemon isolation and this quest pending that validation. From 228bc9a4b4dcc1b38b4b86a6debcfd0fee32bb38 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:05:25 -0800 Subject: [PATCH 10/25] fix(test): serialize concurrent interop dependency installs Co-Authored-By: GPT-6.1 Sol --- quest/m0/test-flakes-2/README.md | 1 - quest/m0/test-flakes-2/pulse-dbus.md | 72 ---------------------------- test/interop/interop.sh | 5 +- 3 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 quest/m0/test-flakes-2/pulse-dbus.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index a27864fb548..4fa9fc23a4b 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -117,7 +117,6 @@ Public API: none. Wire: none. - [Fallback playback](/quest/m0/test-flakes-2/ietf22-fallback.md) - a browser viewer follows a fallback within the media lane's bound without dropping its session - [reload.test.ts's close-drain tests pass under load](/quest/m0/test-flakes-2/reload-close-drain.md) - the drain cases in `js/net`'s reload tests pass in loaded multi-file runs - [dispatch-query's terminate test survives load](/quest/m0/test-flakes-2/dispatch-query-terminate.md) - `test_terminating_the_run_stops_the_running_phase` passes reliably under host load, re-enabled once fixed -- [Test PulseAudio servers stay off the desktop's D-Bus](/quest/m0/test-flakes-2/pulse-dbus.md) - concurrent media runs on one desktop never collide on `org.PulseAudio1` - [The Python audio playout-delay test reads its frames](/quest/m0/test-flakes-2/audio-playout-delay.md) - `test_audio_consumer_reports_its_playout_delay` stops timing out on its first read in the merge queue - [io_uring tests fit the shared locked-memory budget](/quest/m0/test-flakes-2/uring-memlock.md) - concurrent moq-relay and moq-uring test runs stop exhausting the user's `RLIMIT_MEMLOCK`, fixed at the cause rather than by raising the limit - [The FFI concurrent-ready test wakes both calls under load](/quest/m0/test-flakes-2/ffi-concurrent-ready.md) - `concurrent_ready_calls_all_wake` stops stalling its reader drain under load and is re-enabled diff --git a/quest/m0/test-flakes-2/pulse-dbus.md b/quest/m0/test-flakes-2/pulse-dbus.md deleted file mode 100644 index 3108ecd0de7..00000000000 --- a/quest/m0/test-flakes-2/pulse-dbus.md +++ /dev/null @@ -1,72 +0,0 @@ -# [S] Test PulseAudio servers stay off the desktop's D-Bus - -## Goal - -Every private PulseAudio server the test suites start stays off the session -bus, so two media runs on one desktop never collide on the -`org.PulseAudio1` name. Concurrent `just test media` runs on a desktop host -pass where today one can fail with `D-Bus name org.PulseAudio1 already taken`. - -## Plan - -Planned 2026-11-05 (maintainer, /quest-plan of the follow-ups from the -2026-11-05 /quest-complete run), from -[moq-dev/moq#7716](https://github.com/moq-dev/moq/pull/7716). Milestone: ✅ m0. - -The daemon-level name registration is independent of D-Bus modules -([PulseAudio 17 startup](https://github.com/pulseaudio/pulseaudio/blob/v17.0/src/daemon/main.c#L1202-L1211)). -[moq-dev/moq#7831](https://github.com/moq-dev/moq/pull/7831) already isolated -that launch path before this quest was recreated. All explicit media, -native-play, and native audio-quality launches use `test/lib/pulse.py`. - -2026-11-05 verification: `python3 test/lib/pulse.test.py` passes on current -main. A scratch copy restoring the daemon's session-bus inheritance fails -with `D-Bus name org.PulseAudio1 already taken` under the test's private bus. -No desktop services were modified. No remaining launch path has been reproduced. - -2026-11-06 validation: main includes the merged -[dev-shell environment fix](https://github.com/moq-dev/moq/pull/7979). -`nix develop --command quest gate bun install --frozen-lockfile` and -`nix develop --command python3 test/lib/pulse.test.py` pass. - -The previous `--media-cases none` command now selects an empty focused case -set, which the browser runner rejects. Validate two overlapping -`just test media --media-cases pause` runs instead, without changing the -harness environment. Dependency installation is complete before overlap. - -The overlapping pause runs both failed after reaching playback (2026-11-06 -06:53 UTC): run `interop-JSI7CQaU` failed `resume takes effect` with the -presented frame stuck at 31; run `interop-om6rAC4x` failed `audio tone` after -resume (63/75 samples). The latter recorded host idle time of 0.1–2.9%, -which suggests contention but does not establish the cause. - -Both runs' `pulse-1/pulse.log` show `Daemon startup complete.`, independent -runtime directories and normal SIGTERM teardown. Neither reports a D-Bus -name collision. Both warn that D-Bus autolaunch is unavailable without -`DISPLAY`, then successfully start. No failing daemon launch was found. -The relay environment rejection and Bun install collisions did not recur. - -These runs do not meet the requested passing-overlap condition for retirement. -Keep the existing isolation. Investigate the resume failures separately before -claiming an overlap pass; do not infer a PulseAudio defect from these failures. - -Public API: none. Wire: none. - -2026-11-06 23:53 UTC rerun after merging current main: dependency installation -passed before overlap, and `quest gate just ci` passed (112 seconds), including -`test gh`. Both overlapping `just test media --media-cases pause` runs exited 1 -before browser startup. Runs `interop-iPWHd34k` and `interop-wdFesAVD` failed -inside the harness's own concurrent Bun installs: `Failed to link semver: -EEXIST` (and concurrently, vite, yaml, and @babel/parser). Both reported -`media checks (browser client unavailable)`. Neither created a PulseAudio -server or `pulse.log`; this attempt gives no new daemon-isolation evidence. -Outputs and full run directories are retained in the implementation checkout's -`.tmp/`. Recommend serializing the harness dependency installation while keeping -the two playback runs concurrent, then repeating the overlap unchanged. - -Decided 2026-11-06 (maintainer, /quest-complete): ✅ retain this quest until -overlapping media runs pass. Retiring it based on clean daemon logs was not chosen. - -## Waiting - -- moq-dev/moq: resolve concurrent Bun install link collisions in the media harness, then demonstrate two overlapping `just test media --media-cases pause` runs pass. Preinstalling dependencies alone did not prevent the internal installs from colliding. Retain existing daemon isolation and this quest pending that validation. diff --git a/test/interop/interop.sh b/test/interop/interop.sh index a2e22c223c5..bba73c2ba4a 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -709,7 +709,10 @@ prepare_js() { return fi echo "installing js clients (workspace @moq/* via bun)..." - if ! (cd "$WORKSPACE" && bun install --frozen-lockfile) >"$HARNESS_RUN/js-install.log" 2>&1; then + # Bun relinks shared node_modules even with a frozen lockfile. Lock only + # the install; each run builds its own page and plays concurrently afterward. + mkdir -p "$WORKSPACE/.tmp" + if ! (cd "$WORKSPACE" && harness_locked "$WORKSPACE/.tmp/bun-install.lock" bun install --frozen-lockfile) >"$HARNESS_RUN/js-install.log" 2>&1; then for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done sed 's/^/ /' "$HARNESS_RUN/js-install.log" >&2 || true return From 841baa88170cd47cde9009d26eec24938c05f605 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:11:50 -0800 Subject: [PATCH 11/25] chore: keep unrelated local quest planning off the PR Co-Authored-By: GPT-6.1 Sol --- quest/m2/probe/browser-send-estimate.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/quest/m2/probe/browser-send-estimate.md b/quest/m2/probe/browser-send-estimate.md index 10d9bce0f03..d9c58207d42 100644 --- a/quest/m2/probe/browser-send-estimate.md +++ b/quest/m2/probe/browser-send-estimate.md @@ -24,22 +24,15 @@ Reason: one rule decided at setup fixes the RTT, the polling, and the counters. What remains once the waits clear: - In stable Chrome (no flag) and Firefox, verify through a shaped link that a publisher's bitrate follows a capacity drop. Reuse the [probe epic](/quest/m2/probe/README.md)'s shaped-uplink harness if it has landed. -- The WebSocket fallback has no estimate whatever the browsers ship, so its publishers still hold their rate, as decided below. +- The WebSocket fallback has no estimate whatever the browsers ship, so its publishers still hold their rate; the maintainer decision below covers it. -Decided 2026-11-07 (maintainer, /quest-unblock): Safari publishers and the WebSocket fallback get no estimate before WebKit reports one. - -- ✅ (a) Wait for WebKit, as today. -- (b) A lite-07 receiver report, where the relay sends back the rate it receives from us, which can only lower the estimate while app-limited. -- (c) Estimate locally from how fast the transport drains our writes, which depends on each browser's buffering. - -Reason: Chrome and Firefox need nothing, and (b) and (c) each build an app-layer controller above the browser's own that a later WebKit release makes redundant. - -Public API: none expected. Wire: none. +Public API: none expected. Wire: none, unless the decision below picks a receiver report. ## Waiting - Chrome: ships `WebTransport.getStats()` with `estimatedSendRate` enabled for all users in 159, per the 2026-11-05 Intent to Ship. Check that stable Chrome's `getStats()` reports a non-null `estimatedSendRate` on a connection that has been sending media for a few seconds. - WebKit: Safari's `getStats()` reports a non-null `estimatedSendRate`. Check the `return_object_property_estimatedSendRate` entry under `api.WebTransport.getStats` in browser-compat-data, or a Safari publisher's `Reservation.rate`. +- Maintainer: whether Safari publishers (and the WebSocket fallback) get an estimate before WebKit reports one. An agent can't pick this because each option changes the wire or adds a congestion estimator above the browser's own. Options: (a) wait for WebKit, as today; (b) a lite-07 receiver report, where the relay sends back the rate it receives from us, which can only lower the estimate while app-limited; (c) estimate locally from how fast the transport drains our writes, which depends on each browser's buffering. Recommendation: (a), since Chrome and Firefox need nothing, and (b) and (c) each build an app-layer controller that a later WebKit release makes redundant. ## Related From d5f16bec398de073f898a93b9c80fdc560925a0c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:14:59 -0800 Subject: [PATCH 12/25] fix(test): reuse installed dependencies across overlapping runs Co-Authored-By: GPT-6.1 Sol --- test/drain/run.sh | 2 +- test/ietf-conformance/run.sh | 2 +- test/interop/compat.sh | 2 +- test/interop/interop.sh | 5 +---- test/json-properties.sh | 2 +- test/justfile | 10 +++++----- test/lib/install.py | 29 +++++++++++++++++++++++++++++ test/portrait/run.sh | 2 +- test/resume-model.sh | 2 +- 9 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 test/lib/install.py diff --git a/test/drain/run.sh b/test/drain/run.sh index a4d823ccac4..c7dd2c047aa 100755 --- a/test/drain/run.sh +++ b/test/drain/run.sh @@ -61,7 +61,7 @@ flag=() TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" [[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" -(cd "$WORKSPACE" && bun install --frozen-lockfile) +python3 "$WORKSPACE/test/lib/install.py" # ── relays ────────────────────────────────────────────────────────────────── # Start one relay on a reserved port: `start [extra relay args...]`. diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh index a5fc3b3e5d2..cc87f6e459c 100755 --- a/test/ietf-conformance/run.sh +++ b/test/ietf-conformance/run.sh @@ -30,7 +30,7 @@ relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" # Cargo accepts an absolute target directory too. [[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" -bun install --frozen-lockfile +python3 test/lib/install.py harness_begin ietf-conformance "$(harness_env HARNESS_RESULTS)just test ietf-conformance$(harness_argv "$@")" python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" # LibreSSL in the dev shell has no system-wide openssl.cnf. diff --git a/test/interop/compat.sh b/test/interop/compat.sh index 73e158e1369..7884411c35b 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -31,7 +31,7 @@ case "$*" in ;; esac cd "$WORKSPACE" -bun install --frozen-lockfile +python3 test/lib/install.py bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index bba73c2ba4a..545967060c2 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -709,10 +709,7 @@ prepare_js() { return fi echo "installing js clients (workspace @moq/* via bun)..." - # Bun relinks shared node_modules even with a frozen lockfile. Lock only - # the install; each run builds its own page and plays concurrently afterward. - mkdir -p "$WORKSPACE/.tmp" - if ! (cd "$WORKSPACE" && harness_locked "$WORKSPACE/.tmp/bun-install.lock" bun install --frozen-lockfile) >"$HARNESS_RUN/js-install.log" 2>&1; then + if ! python3 "$WORKSPACE/test/lib/install.py" >"$HARNESS_RUN/js-install.log" 2>&1; then for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done sed 's/^/ /' "$HARNESS_RUN/js-install.log" >&2 || true return diff --git a/test/json-properties.sh b/test/json-properties.sh index ca33fbeda72..3716418d543 100644 --- a/test/json-properties.sh +++ b/test/json-properties.sh @@ -7,7 +7,7 @@ MOQ_JSON_SEED=${MOQ_JSON_SEED:-$(python3 -c 'import secrets; print(secrets.randb MOQ_JSON_CASES=${MOQ_JSON_CASES:-4096} export MOQ_JSON_SEED MOQ_JSON_CASES echo "JSON codec properties: MOQ_JSON_SEED=$MOQ_JSON_SEED MOQ_JSON_CASES=$MOQ_JSON_CASES" -bun install --frozen-lockfile +python3 test/lib/install.py just rs test -p moq-json --lib -E 'test(snapshot_codec_property) | test(window_codec_property)' bun test ./js/json/src/snapshot/properties.test.ts ./js/json/src/window/properties.test.ts bunx tsc --noEmit --project js/json/bench/tsconfig.json diff --git a/test/justfile b/test/justfile index bd9db4d79e4..93674a912d3 100644 --- a/test/justfile +++ b/test/justfile @@ -52,7 +52,7 @@ harness: python3 interop/report.test.py python3 interop/tone.test.py cd .. && bun test {{ replace(harness_compat_tests, "\n", " ") }} - cd .. && bun install --frozen-lockfile + python3 lib/install.py cd interop/clients/js && bash ../../../../sh/local.sh bun test {{ replace(harness_browser_tests, "\n", " ") }} # Cross-language media interop test, built from this checkout. Stands up a @@ -66,7 +66,7 @@ harness: # Both runs first check the Rust/JS wire and session interop suites, in both directions. [positional-arguments] interop *args: - cd .. && bun install --frozen-lockfile + python3 lib/install.py just rs test -p moq-net -p moq-crypto --lib \ auth_unsupported_interop \ ietf_compression_fallback_interop \ @@ -250,7 +250,7 @@ drain *args: # The @moq/video renderer in headless Chromium: WebGPU, a lost device, and the Canvas2D fallback. video: - cd .. && bun install --frozen-lockfile + python3 lib/install.py # A desktop display makes software Vulkan request unsupported window surfaces. cd video && bash ../../sh/local.sh env -u DISPLAY -u WAYLAND_DISPLAY bun test ./renderer.browser.ts @@ -290,10 +290,10 @@ resume-model: # TypeScript-to-Rust qmux interop. qmux: cargo build --locked --quiet --manifest-path ../rs/qmux/Cargo.toml --example interop-server --example interop-client - cd .. && bun install --frozen-lockfile + python3 lib/install.py cd .. && QMUX_INTEROP_BIN={{ quote(qmux_examples) }} bun test ./test/qmux/interop.test.ts # Complete MoQ SETUP and transfer a frame over a real browser data-channel pair. p2p: - cd .. && bun install --frozen-lockfile + python3 lib/install.py cd p2p && bash ../../sh/local.sh bun test ./transport.browser.ts diff --git a/test/lib/install.py b/test/lib/install.py new file mode 100644 index 00000000000..b1e2de8b3d3 --- /dev/null +++ b/test/lib/install.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Install the workspace once per dependency configuration, under a checkout lock.""" + +import fcntl +import hashlib +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +(ROOT / ".tmp").mkdir(exist_ok=True) +with (ROOT / ".tmp/bun-install.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + digest = hashlib.sha256(subprocess.check_output(["bun", "--version"])) + # Include the postinstall build inputs as well as dependency manifests. + paths = subprocess.check_output( + ["git", "ls-files", "-z", "--", "*package.json", "bun.lock", + "flake.nix", "flake.lock", "nix", "js/libav"], cwd=ROOT, + ).split(b"\0") + for name in sorted(filter(None, paths)): + digest.update(name) + digest.update((ROOT / name.decode()).read_bytes()) + fingerprint = digest.hexdigest() + stamp = ROOT / "node_modules/.moq-install" + # Another invocation must not relink dependencies or rebuild libav while + # the first invocation is importing them or playing its bundled page. + if not stamp.exists() or stamp.read_text() != fingerprint: + stamp.unlink(missing_ok=True) + subprocess.run(["bun", "install", "--frozen-lockfile"], cwd=ROOT, check=True) + stamp.write_text(fingerprint) diff --git a/test/portrait/run.sh b/test/portrait/run.sh index a2144e8a7c7..ed04c80346e 100755 --- a/test/portrait/run.sh +++ b/test/portrait/run.sh @@ -12,7 +12,7 @@ source "$DIR/../lib/harness.sh" harness_begin portrait 'just test portrait' cd "$WORKSPACE" cargo build --locked -p moq-relay -p moq-cli --features moq-cli/transcode,moq-cli/openh264 -bun install --frozen-lockfile +python3 test/lib/install.py (cd test/interop/clients/js && bunx vite build --outDir "$HARNESS_RUN/js-dist") harness_port relay URL="http://127.0.0.1:$HARNESS_PORT" diff --git a/test/resume-model.sh b/test/resume-model.sh index 3523f9b753a..ee518199dab 100755 --- a/test/resume-model.sh +++ b/test/resume-model.sh @@ -8,5 +8,5 @@ MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} export MOQ_RESUME_SEED MOQ_RESUME_CASES echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" just rs test -p moq-net --features async --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' -bun install --frozen-lockfile +python3 test/lib/install.py bun test ./js/net/src/resume.model.test.ts From 8b2834a863bc3c6fe9a27a98fe41c8636cbcc08b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:20:17 -0800 Subject: [PATCH 13/25] style(test): sort installer imports Co-Authored-By: GPT-6.1 Sol --- test/lib/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/install.py b/test/lib/install.py index b1e2de8b3d3..83017c2e5ef 100644 --- a/test/lib/install.py +++ b/test/lib/install.py @@ -3,8 +3,8 @@ import fcntl import hashlib -from pathlib import Path import subprocess +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] (ROOT / ".tmp").mkdir(exist_ok=True) From 6e732473612efe7a2fb561b112ab4733b33f46b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:28:00 -0800 Subject: [PATCH 14/25] fix(test): resolve installer paths and track postinstall inputs Co-Authored-By: GPT-6.1 Sol --- test/ietf-conformance/run.sh | 2 +- test/lib/install.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh index cc87f6e459c..220053c3b7f 100755 --- a/test/ietf-conformance/run.sh +++ b/test/ietf-conformance/run.sh @@ -30,7 +30,7 @@ relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" # Cargo accepts an absolute target directory too. [[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" -python3 test/lib/install.py +python3 "$WORKSPACE/test/lib/install.py" harness_begin ietf-conformance "$(harness_env HARNESS_RESULTS)just test ietf-conformance$(harness_argv "$@")" python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" # LibreSSL in the dev shell has no system-wide openssl.cnf. diff --git a/test/lib/install.py b/test/lib/install.py index 83017c2e5ef..61e138a5a36 100644 --- a/test/lib/install.py +++ b/test/lib/install.py @@ -3,6 +3,7 @@ import fcntl import hashlib +import shutil import subprocess from pathlib import Path @@ -11,19 +12,40 @@ with (ROOT / ".tmp/bun-install.lock").open("w") as lock: fcntl.flock(lock, fcntl.LOCK_EX) digest = hashlib.sha256(subprocess.check_output(["bun", "--version"])) + digest.update(str(shutil.which("nix")).encode()) # Include the postinstall build inputs as well as dependency manifests. paths = subprocess.check_output( - ["git", "ls-files", "-z", "--", "*package.json", "bun.lock", - "flake.nix", "flake.lock", "nix", "js/libav"], cwd=ROOT, + [ + "git", + "ls-files", + "-z", + "--", + "*package.json", + "bun.lock", + "flake.nix", + "flake.lock", + "nix", + "js/libav", + "patches", + ], + cwd=ROOT, ).split(b"\0") for name in sorted(filter(None, paths)): digest.update(name) - digest.update((ROOT / name.decode()).read_bytes()) + path = ROOT / name.decode() + if path.exists(): + digest.update(path.read_bytes()) + else: + digest.update(b"missing") fingerprint = digest.hexdigest() stamp = ROOT / "node_modules/.moq-install" # Another invocation must not relink dependencies or rebuild libav while # the first invocation is importing them or playing its bundled page. - if not stamp.exists() or stamp.read_text() != fingerprint: + if ( + not stamp.exists() + or stamp.read_text() != fingerprint + or not (ROOT / "js/libav/dist/libav-opus-aac-af.mjs").exists() + ): stamp.unlink(missing_ok=True) subprocess.run(["bun", "install", "--frozen-lockfile"], cwd=ROOT, check=True) stamp.write_text(fingerprint) From c66645079478a3ec663d5abd7976f3f8a24837ee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:54:20 -0800 Subject: [PATCH 15/25] fix: coordinate dependency installation across the root workspace Co-Authored-By: GPT-6.1 Sol --- .github/justfile | 1 + .github/workflows/interop.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release-js.yml | 2 +- bench/browser/run.sh | 2 +- demo/boy/justfile | 8 ++-- demo/carla/justfile | 2 +- demo/justfile | 2 +- demo/relay/justfile | 2 +- demo/web/justfile | 2 +- js/justfile | 50 +++++++++++----------- justfile | 6 +-- sh/api-diff/run.sh | 4 +- sh/dispatch.sh | 10 ++--- sh/drafts/check.sh | 2 +- sh/js/README.md | 7 +++- sh/js/coverage.py | 2 +- {test/lib => sh/js}/install.py | 27 ++++++++---- sh/js/install.test.py | 72 ++++++++++++++++++++++++++++++++ sh/js/workspace.py | 2 +- sh/size/collect.sh | 2 +- test/audio-quality/ptt.sh | 2 +- test/audio-quality/record.sh | 2 +- test/audio-quality/run.sh | 2 +- test/drain/run.sh | 2 +- test/ietf-conformance/run.sh | 2 +- test/interop/compat.sh | 2 +- test/interop/interop.sh | 2 +- test/json-properties.sh | 2 +- test/justfile | 10 ++--- test/portrait/run.sh | 2 +- test/resume-model.sh | 2 +- 32 files changed, 165 insertions(+), 74 deletions(-) rename {test/lib => sh/js}/install.py (76%) create mode 100644 sh/js/install.test.py diff --git a/.github/justfile b/.github/justfile index 21238a2f744..131de93d722 100644 --- a/.github/justfile +++ b/.github/justfile @@ -82,6 +82,7 @@ test: python3 sh/rs/select.test.py python3 sh/rs/config.test.py python3 sh/rs/hakari.test.py + python3 sh/js/install.test.py python3 sh/js/workspace.test.py python3 sh/js/audit.test.py python3 sh/rs/test-lints.test.py diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 0547c599c56..a467867c50d 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -55,7 +55,7 @@ jobs: - name: Workspace dependencies id: setup - run: nix develop --command bun install --frozen-lockfile + run: nix develop --command python3 sh/js/install.py frozen shell: bash -leo pipefail {0} # Each suite reports independently after setup, while a failed step still fails the job. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e66080f43de..989aa4d638a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -425,7 +425,7 @@ jobs: - name: Install publisher and player engines # Firefox loads libpulse at runtime; Playwright's engine dependencies omit it. run: | - nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium firefox" + nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" bash -c "python3 sh/js/install.py frozen && bunx playwright install --with-deps chromium firefox" sudo apt-get install -y libpulse0 - name: Media and negative controls run: nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" just test media --media-engines firefox diff --git a/.github/workflows/release-js.yml b/.github/workflows/release-js.yml index a8d315dd2f5..bbae5f20d7c 100644 --- a/.github/workflows/release-js.yml +++ b/.github/workflows/release-js.yml @@ -48,7 +48,7 @@ jobs: package-manager-cache: false - name: Install dependencies - run: nix develop --command bun install --frozen-lockfile + run: nix develop --command python3 sh/js/install.py frozen - name: Release packages env: diff --git a/bench/browser/run.sh b/bench/browser/run.sh index bd95a6ceb7f..106c9ae6404 100755 --- a/bench/browser/run.sh +++ b/bench/browser/run.sh @@ -88,7 +88,7 @@ if [[ -z $RELAY ]]; then fi cd "$WORKSPACE" -bun install --frozen-lockfile +python3 sh/js/install.py frozen # BROWSER_BIN overrides the Chromium provided by the root devShell. cd "$HERE" diff --git a/demo/boy/justfile b/demo/boy/justfile index b5a3d25d6bd..c7a0404774d 100644 --- a/demo/boy/justfile +++ b/demo/boy/justfile @@ -2,7 +2,7 @@ set fallback # Run the GB demo: relay + emulator publisher + web viewer. default: - bun install + python3 ../../sh/js/install.py update bun run concurrently --kill-others --prefix-colors auto \ "just relay" \ @@ -57,12 +57,12 @@ web url='http://localhost:4443': # Deploy the rom.moq.dev worker deploy: - bun install + python3 ../../sh/js/install.py update bun wrangler deploy # Upload a single ROM to R2 upload file: - bun install + python3 ../../sh/js/install.py update bun wrangler r2 object put "rom-moq-dev/{{ file }}" --file "rom/{{ file }}" --remote # Sync all ROMs to R2 @@ -72,5 +72,5 @@ sync dir="rom": # List files in the R2 bucket list: - bun install + python3 ../../sh/js/install.py update bun wrangler r2 object list "rom-moq-dev" diff --git a/demo/carla/justfile b/demo/carla/justfile index 39b8696c8e5..dd44da25e87 100644 --- a/demo/carla/justfile +++ b/demo/carla/justfile @@ -10,7 +10,7 @@ set fallback # # Run a local relay, both trucks, and the web player. default: - bun install + python3 ../../sh/js/install.py update bun run concurrently --kill-others --names relay,trucks,web --prefix-colors auto \ "just relay" \ "just wait && just wait-port 127.0.0.1 2000 && just worker" \ diff --git a/demo/justfile b/demo/justfile index 08b359d9ffa..b568b511bc6 100644 --- a/demo/justfile +++ b/demo/justfile @@ -5,7 +5,7 @@ set fallback # Picks the first free port at/after 4443 so multiple worktrees can run # `just dev` without colliding; see sh/demo/run.sh. default: - bun install + python3 ../sh/js/install.py update ../sh/demo/run.sh # Wait for the localhost relay to be ready by repeatedly requesting the cert diff --git a/demo/relay/justfile b/demo/relay/justfile index db04682e600..c861ea818ef 100644 --- a/demo/relay/justfile +++ b/demo/relay/justfile @@ -10,7 +10,7 @@ default: # Cluster peers still authenticate to each other via mTLS (the cert/ca recipes). cluster: - bun install + python3 ../../sh/js/install.py update bun run concurrently --kill-others --names root,leaf0,leaf1,bbb,tos,web --prefix-colors auto \ "just root" \ diff --git a/demo/web/justfile b/demo/web/justfile index 29d6637263c..b9bf0df3d2d 100644 --- a/demo/web/justfile +++ b/demo/web/justfile @@ -13,4 +13,4 @@ serve url='http://localhost:4443': install # Install the workspace deps that vite and the demo's imports need. install: - bun install + python3 ../../sh/js/install.py update diff --git a/js/justfile b/js/justfile index 78ba8fd396b..7520733c60f 100644 --- a/js/justfile +++ b/js/justfile @@ -46,7 +46,7 @@ fix: # Rebuild changed workspace packages (or `--all`) to update their API reports. fix-changed list: - bun install + python3 ../sh/js/install.py update MOQ_API_REPORT=update python3 ../sh/js/workspace.py build '{{ list }}' # Run all unit tests. @@ -63,44 +63,44 @@ coverage package="": # Run one net benchmark: broadcasts, forward, requests, fills, fetch, reader, frames, track, resume, observed, drift, ledger, bandwidth, replay, varint, lite-varint, priority, subscribe-drop, refresh, compression, or datagrams. bench name: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun net/bench/{{ name }}.ts # Sweep browser publisher fairness over backlog x viewers using the devShell's Chromium. serve-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/serve-browser.ts # Sweep catalog estimate publishes and subscribe updates over tracks x viewers. catalog-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun publish/bench/catalog.ts # Sweep QMux record decoding over record size x frames per record. qmux-record-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun qmux/bench/record.ts # Sweep QMux datagram receive-queue draining over backlog size (median of 7 runs). qmux-datagrams-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun qmux/bench/datagrams.ts # Sweep CMAF fragment decode over mdat bytes x declared sample count, including a hostile count. cmaf-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun hang/bench/cmaf.ts # Time Effect create and close, which pays for the creation stack its diagnostics print. signals-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun signals/bench/effect.ts # Bundle the worklets with Vite, webpack, esbuild, and Bun, and load them in Chromium, with and # without a strict CSP. Nightly, against the bundlers pinned in test/worklet. worklet-matrix: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun ../test/worklet/matrix.ts # Build all packages. @@ -113,87 +113,87 @@ dist-imports: # Compare eager and cold lazy WebSocket fallback setup in Chromium. qmux-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/qmux.ts # Sweep open arrival handles against their unread backlog. arrivals-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun watch/bench/arrivals.ts # Reduce and compare the worst known peer grant shapes, swept over AUTH_OK size. pattern-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project pattern/bench/tsconfig.json bun pattern/bench/algebra.ts # Sweep manual-clock armed timers x due timers, with independent and shared abort scopes. clock-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun clock/bench/timers.ts # Sweep buffered JSON stream reads over record count, payload size, and compression. json-stream-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/stream.ts # Sweep snapshot codec cost over document size and changed keys. json-codec-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/codec.ts # Compare the duplicate-refusing JSON frame parse against bare JSON.parse. json-parse-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/parse.ts # Sweep a slow hang consumer's skipped groups against its warning traffic and catch-up time. slow-group-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun hang/bench/slow-group.ts # Sweep a hang consumer's buffered groups, frames per group, publishers and players across arrival orders. buffered-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project hang/bench/tsconfig.json bun hang/bench/buffered.ts # Sweep the audio jitter estimator's held history against in-order and reordered frames. jitter-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun hang/bench/jitter.ts # Sweep players against the main-thread CPU and allocation of feeding the audio ring, in Chromium. ring-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun watch/bench/ring.ts # Sweep audio rendition count against repeated mute/unmute catalog updates. audio-support-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun watch/bench/support.ts # Sweep decoder count against PCM callback size during audio warmup. audio-warmup-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun watch/bench/warmup.ts # Sweep data-channel qmux stream count and payload size. p2p-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun p2p/bench/transport.ts # Sweep E2EE frame size against writes in flight through the WebCrypto pump. e2ee-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bun crypto/bench/pump.ts # Sweep audio renditions x raw retention through native WebCodecs in Chromium. raw-audio-bench: - bun install --frozen-lockfile + python3 ../sh/js/install.py frozen bunx tsc --noEmit --project publish/bench/tsconfig.json bun publish/bench/audio.ts diff --git a/justfile b/justfile index c3b1e5dd2ca..08a1270928b 100644 --- a/justfile +++ b/justfile @@ -94,7 +94,7 @@ bench-browser *args: # Install repo-wide tooling. Per-language deps install on first check. install: - bun install + python3 sh/js/install.py update cargo install --locked cargo-shear cargo-sort cargo-semver-checks release-plz # Lints, compiles, and tests only the packages the branch changed plus @@ -139,12 +139,12 @@ ci-select-platform $BASE="": # Check every supported source file; exclusions live in biome.jsonc. biome *args: - bun install --frozen-lockfile + python3 sh/js/install.py frozen bun biome check --error-on-warnings {{ args }} . # Apply Biome's fixes; installs without a frozen lockfile, like the other fixers. biome-fix: - bun install + python3 sh/js/install.py update bun biome check --error-on-warnings --write . # Auto-fix lint and formatting for what the branch changed since BASE. diff --git a/sh/api-diff/run.sh b/sh/api-diff/run.sh index cbfb61054a8..3330f12b3da 100755 --- a/sh/api-diff/run.sh +++ b/sh/api-diff/run.sh @@ -213,8 +213,8 @@ js() { } local manifest dir name version file entry surface # The renderer imports TypeDoc from this checkout's install, which a fresh runner lacks. The - # install scripts build assets, such as @moq/libav, that neither checkout needs here. - bun install --frozen-lockfile --ignore-scripts >&2 + # current checkout uses the coordinated full install shared with other readers. + python3 sh/js/install.py frozen >&2 # Not local: the trap runs after a failure has unwound this function's locals. checkout=$out/js/checkout git worktree add --detach "$checkout" "$base" >&2 diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 5e231e0ece9..6cfc2f0bc9b 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -129,9 +129,9 @@ module cpp '^(\.cargo/config\.toml$|cpp/moq/|cpp/justfile$|sh/cpp/|sh/clang-form # the libobs headers it compiles against. module obs_compile '^(\.cargo/config\.toml$|cpp/obs/|cpp/moq/|sh/obs/|rs/moq-ffi/|rs/moq-c/native-libs/|flake\.nix$)' \ tools='cargo cmake pkg-config uniffi-bindgen-cpp' compiles=check lane=cpp -# The bun lane exists because `bun install` relinks the root workspace's -# packages even when nothing changed, so a concurrent install or import of -# them fails. Biome checks the whole repository. It registers before js so +# The bun lane serializes fixers and JS builds that write shared outputs. +# Dependency setup separately uses sh/js/install.py across every entry point. +# Biome checks the whole repository. It registers before js so # that fix runs it first, since Biome can fix sources that the package build # rejects, such as a type-only export. module biome '' \ @@ -372,7 +372,7 @@ step() { check:ts_paths) python3 sh/gh/ts-paths.py ;; check:biome) just biome ;; check:markdown) - bun install --frozen-lockfile + python3 sh/js/install.py frozen sh/markdown.sh check ;; check:large_files) sh/large-files.sh "$large_base" ;; @@ -392,7 +392,7 @@ step() { fix:js) just js fix-changed "$js_list" ;; fix:biome) just biome-fix ;; fix:markdown) - bun install + python3 sh/js/install.py update sh/markdown.sh fix ;; fix:python) diff --git a/sh/drafts/check.sh b/sh/drafts/check.sh index e1f852cba4c..f227a0738f2 100755 --- a/sh/drafts/check.sh +++ b/sh/drafts/check.sh @@ -12,7 +12,7 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" -bun install --frozen-lockfile +python3 sh/js/install.py frozen cd drafts if grep -n "$(printf '\t')" draft-*.md; then diff --git a/sh/js/README.md b/sh/js/README.md index 8d87bd23c07..eb94f765fac 100644 --- a/sh/js/README.md +++ b/sh/js/README.md @@ -34,9 +34,14 @@ The root devShell supplies Chromium and exports its Playwright version. Checks require every npm `playwright` dependency outside `test/smoke` to pin that exact version. A root `flake.nix` or `flake.lock` change selects JS checks too: when nixpkgs moves Playwright, update the reported manifests and regenerate `bun.lock` -with `bun install` in the new shell. Smoke has its own flake and freshness check. +with `python3 sh/js/install.py update` in the new shell. Smoke has its own flake and freshness check. Run `python3 sh/js/workspace.test.py` for graph and execution regressions. The same suite runs in `just gh test` on every check. To inspect work without executing it, pass a file containing relevant paths, one per line, to `python3 sh/js/workspace.py plan LISTFILE`, or pass `--all`. + +All root workspace installers use `python3 sh/js/install.py frozen` for checks +and tests, or `update` when lockfile changes are allowed. Both reuse an unchanged +installation under one checkout lock, including libav output. Separate Worker, +smoke-client, and disposable baseline workspaces keep their own installs. diff --git a/sh/js/coverage.py b/sh/js/coverage.py index 3d04bc3f57f..f2d877541c8 100644 --- a/sh/js/coverage.py +++ b/sh/js/coverage.py @@ -66,7 +66,7 @@ def main(): (ROOT / "target/coverage/js/summary.json").unlink(missing_ok=True) for directory in selected: (ROOT / "target/coverage/js" / directory.name / "lcov.info").unlink(missing_ok=True) - subprocess.run(["bun", "install", "--frozen-lockfile"], cwd=ROOT, check=True) + subprocess.run(["python3", str(ROOT / "sh/js/install.py"), "frozen"], cwd=ROOT, check=True) for directory in selected: out = ROOT / "target/coverage/js" / directory.name out.mkdir(parents=True, exist_ok=True) diff --git a/test/lib/install.py b/sh/js/install.py similarity index 76% rename from test/lib/install.py rename to sh/js/install.py index 61e138a5a36..ad39166ab88 100644 --- a/test/lib/install.py +++ b/sh/js/install.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Install the workspace once per dependency configuration, under a checkout lock.""" +import argparse import fcntl import hashlib import shutil @@ -8,9 +9,9 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -(ROOT / ".tmp").mkdir(exist_ok=True) -with (ROOT / ".tmp/bun-install.lock").open("w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) + + +def fingerprint(): digest = hashlib.sha256(subprocess.check_output(["bun", "--version"])) digest.update(str(shutil.which("nix")).encode()) # Include the postinstall build inputs as well as dependency manifests. @@ -37,15 +38,27 @@ digest.update(path.read_bytes()) else: digest.update(b"missing") - fingerprint = digest.hexdigest() + return digest.hexdigest() + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("mode", choices=("frozen", "update")) +args = parser.parse_args() + +(ROOT / ".tmp").mkdir(exist_ok=True) +with (ROOT / ".tmp/bun-install.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + current = fingerprint() stamp = ROOT / "node_modules/.moq-install" # Another invocation must not relink dependencies or rebuild libav while # the first invocation is importing them or playing its bundled page. if ( not stamp.exists() - or stamp.read_text() != fingerprint + or stamp.read_text() != current or not (ROOT / "js/libav/dist/libav-opus-aac-af.mjs").exists() ): stamp.unlink(missing_ok=True) - subprocess.run(["bun", "install", "--frozen-lockfile"], cwd=ROOT, check=True) - stamp.write_text(fingerprint) + subprocess.run( + ["bun", "install", *(["--frozen-lockfile"] if args.mode == "frozen" else [])], cwd=ROOT, check=True + ) + stamp.write_text(fingerprint()) diff --git a/sh/js/install.test.py b/sh/js/install.test.py new file mode 100644 index 00000000000..c14eedbae08 --- /dev/null +++ b/sh/js/install.test.py @@ -0,0 +1,72 @@ +"""Concurrent root recipes and media entry points share one installation.""" + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class Install(unittest.TestCase): + def test_overlap(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: + root = Path(directory) + (root / "sh/js").mkdir(parents=True) + shutil.copy(ROOT / "sh/js/install.py", root / "sh/js/install.py") + (root / "test").mkdir() + # Extract the setup commands from actual root and test recipes, + # leaving lint and playback to their own suites. + recipe_setup = (ROOT / "justfile").read_text().split("biome-fix:\n")[1].splitlines()[0] + test_setup = (ROOT / "test/justfile").read_text().splitlines() + test_setup = next(line for line in test_setup if "install.py" in line) + (root / "justfile").write_text("biome:\n" + recipe_setup + "\n") + (root / "test/justfile").write_text("media:\n" + test_setup + "\n") + (root / "package.json").write_text("{}") + (root / "bun.lock").write_text("before") + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "add", "."], check=True) + bin_dir = root / "bin" + bin_dir.mkdir() + bun = bin_dir / "bun" + bun.write_text("""#!/usr/bin/env python3 +import os +import sys +r = Path.cwd() +if sys.argv[1] == '--version': + print('fixture') +else: + with (r / 'calls').open('a') as f: + f.write('install\\n') + # Signal the parent while holding the install lock, then block on a pipe. + with open(r / 'started', 'w') as f: + f.write('ready') + with open(r / 'continue') as f: + f.read() + (r / 'node_modules').mkdir(exist_ok=True) + (r / 'js/libav/dist').mkdir(parents=True, exist_ok=True) + (r / 'js/libav/dist/libav-opus-aac-af.mjs').touch() + (r / 'bun.lock').write_text('after') +""") + bun.chmod(0o755) + for name in ("started", "continue"): + os.mkfifo(root / name) + env = {**os.environ, "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"]} + recipe = subprocess.Popen(["just", "biome"], cwd=root, env=env) + with (root / "started").open() as started: + self.assertEqual(started.read(), "ready") + test = subprocess.Popen(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env) + with (root / "continue").open("w"): + pass + self.assertEqual(recipe.wait(), 0) + self.assertEqual(test.wait(), 0) + self.assertEqual((root / "calls").read_text(), "install\n") + # Frozen readers reuse the post-update fingerprint as well. + subprocess.run(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env, check=True) + self.assertEqual((root / "calls").read_text(), "install\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/sh/js/workspace.py b/sh/js/workspace.py index b7ea3979d64..124049de059 100644 --- a/sh/js/workspace.py +++ b/sh/js/workspace.py @@ -148,7 +148,7 @@ def reports(): if directories: command("bun", "js/common/api.ts", "check", *directories) - command("bun", "install", "--frozen-lockfile") + command("python3", str(ROOT / "sh/js/install.py"), "frozen") if action == "test": command("python3", "sh/js/playwright.test.py") command( diff --git a/sh/size/collect.sh b/sh/size/collect.sh index ebe54660b4d..4534946fefe 100755 --- a/sh/size/collect.sh +++ b/sh/size/collect.sh @@ -15,7 +15,7 @@ esac cargo build --locked --release -p moq-ffi --no-default-features --target-dir "$target/minimal" cp "$target/minimal/release/libmoq_ffi.$dylib" "$out/native/libmoq_ffi-minimal.$dylib" cp "$target/minimal/release/libmoq_ffi.a" "$out/native/libmoq_ffi-minimal.a" -bun install --frozen-lockfile +python3 sh/js/install.py frozen just js build bun sh/size/bundles.ts "$out/js" # Symbols are necessary only for attribution; they never replace the measured shipping sizes. diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index f799045e4ba..f04179b2c19 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -19,7 +19,7 @@ export HARNESS_KEEP="${HARNESS_KEEP:-1}" exit 2 } -bun install --frozen-lockfile +python3 sh/js/install.py frozen cargo build --locked -p moq-relay TARGET_BASE="${CARGO_TARGET_DIR:-$ROOT/target}" harness_port relay diff --git a/test/audio-quality/record.sh b/test/audio-quality/record.sh index 49fc8fd74d2..4a4a046204d 100755 --- a/test/audio-quality/record.sh +++ b/test/audio-quality/record.sh @@ -63,7 +63,7 @@ harness_begin audio-quality-record "$RERUN" echo "building the pages..." ( cd "$CLIENT" - bun install --frozen-lockfile + python3 "$WORKSPACE/sh/js/install.py" frozen bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir ) diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index 82fd717f83a..497bd85db8e 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -241,7 +241,7 @@ done harness_begin audio-quality "$RERUN" echo "installing the workspace..." -(cd "$CLIENT" && bun install --frozen-lockfile) +(cd "$CLIENT" && python3 "$WORKSPACE/sh/js/install.py" frozen) failed=0 diff --git a/test/drain/run.sh b/test/drain/run.sh index c7dd2c047aa..796a9ffc4f4 100755 --- a/test/drain/run.sh +++ b/test/drain/run.sh @@ -61,7 +61,7 @@ flag=() TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" [[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" -python3 "$WORKSPACE/test/lib/install.py" +python3 "$WORKSPACE/sh/js/install.py" frozen # ── relays ────────────────────────────────────────────────────────────────── # Start one relay on a reserved port: `start [extra relay args...]`. diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh index 220053c3b7f..b44bf824225 100755 --- a/test/ietf-conformance/run.sh +++ b/test/ietf-conformance/run.sh @@ -30,7 +30,7 @@ relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" # Cargo accepts an absolute target directory too. [[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" -python3 "$WORKSPACE/test/lib/install.py" +python3 "$WORKSPACE/sh/js/install.py" frozen harness_begin ietf-conformance "$(harness_env HARNESS_RESULTS)just test ietf-conformance$(harness_argv "$@")" python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" # LibreSSL in the dev shell has no system-wide openssl.cnf. diff --git a/test/interop/compat.sh b/test/interop/compat.sh index 7884411c35b..d92ad4b28ee 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -31,7 +31,7 @@ case "$*" in ;; esac cd "$WORKSPACE" -python3 test/lib/install.py +python3 sh/js/install.py frozen bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 545967060c2..66a7da161a8 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -709,7 +709,7 @@ prepare_js() { return fi echo "installing js clients (workspace @moq/* via bun)..." - if ! python3 "$WORKSPACE/test/lib/install.py" >"$HARNESS_RUN/js-install.log" 2>&1; then + if ! python3 "$WORKSPACE/sh/js/install.py" frozen >"$HARNESS_RUN/js-install.log" 2>&1; then for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done sed 's/^/ /' "$HARNESS_RUN/js-install.log" >&2 || true return diff --git a/test/json-properties.sh b/test/json-properties.sh index 3716418d543..e09180bdcac 100644 --- a/test/json-properties.sh +++ b/test/json-properties.sh @@ -7,7 +7,7 @@ MOQ_JSON_SEED=${MOQ_JSON_SEED:-$(python3 -c 'import secrets; print(secrets.randb MOQ_JSON_CASES=${MOQ_JSON_CASES:-4096} export MOQ_JSON_SEED MOQ_JSON_CASES echo "JSON codec properties: MOQ_JSON_SEED=$MOQ_JSON_SEED MOQ_JSON_CASES=$MOQ_JSON_CASES" -python3 test/lib/install.py +python3 sh/js/install.py frozen just rs test -p moq-json --lib -E 'test(snapshot_codec_property) | test(window_codec_property)' bun test ./js/json/src/snapshot/properties.test.ts ./js/json/src/window/properties.test.ts bunx tsc --noEmit --project js/json/bench/tsconfig.json diff --git a/test/justfile b/test/justfile index 93674a912d3..8c737b3563d 100644 --- a/test/justfile +++ b/test/justfile @@ -52,7 +52,7 @@ harness: python3 interop/report.test.py python3 interop/tone.test.py cd .. && bun test {{ replace(harness_compat_tests, "\n", " ") }} - python3 lib/install.py + python3 ../sh/js/install.py frozen cd interop/clients/js && bash ../../../../sh/local.sh bun test {{ replace(harness_browser_tests, "\n", " ") }} # Cross-language media interop test, built from this checkout. Stands up a @@ -66,7 +66,7 @@ harness: # Both runs first check the Rust/JS wire and session interop suites, in both directions. [positional-arguments] interop *args: - python3 lib/install.py + python3 ../sh/js/install.py frozen just rs test -p moq-net -p moq-crypto --lib \ auth_unsupported_interop \ ietf_compression_fallback_interop \ @@ -250,7 +250,7 @@ drain *args: # The @moq/video renderer in headless Chromium: WebGPU, a lost device, and the Canvas2D fallback. video: - python3 lib/install.py + python3 ../sh/js/install.py frozen # A desktop display makes software Vulkan request unsupported window surfaces. cd video && bash ../../sh/local.sh env -u DISPLAY -u WAYLAND_DISPLAY bun test ./renderer.browser.ts @@ -290,10 +290,10 @@ resume-model: # TypeScript-to-Rust qmux interop. qmux: cargo build --locked --quiet --manifest-path ../rs/qmux/Cargo.toml --example interop-server --example interop-client - python3 lib/install.py + python3 ../sh/js/install.py frozen cd .. && QMUX_INTEROP_BIN={{ quote(qmux_examples) }} bun test ./test/qmux/interop.test.ts # Complete MoQ SETUP and transfer a frame over a real browser data-channel pair. p2p: - python3 lib/install.py + python3 ../sh/js/install.py frozen cd p2p && bash ../../sh/local.sh bun test ./transport.browser.ts diff --git a/test/portrait/run.sh b/test/portrait/run.sh index ed04c80346e..d4d50054266 100755 --- a/test/portrait/run.sh +++ b/test/portrait/run.sh @@ -12,7 +12,7 @@ source "$DIR/../lib/harness.sh" harness_begin portrait 'just test portrait' cd "$WORKSPACE" cargo build --locked -p moq-relay -p moq-cli --features moq-cli/transcode,moq-cli/openh264 -python3 test/lib/install.py +python3 sh/js/install.py frozen (cd test/interop/clients/js && bunx vite build --outDir "$HARNESS_RUN/js-dist") harness_port relay URL="http://127.0.0.1:$HARNESS_PORT" diff --git a/test/resume-model.sh b/test/resume-model.sh index ee518199dab..015d9c80a23 100755 --- a/test/resume-model.sh +++ b/test/resume-model.sh @@ -8,5 +8,5 @@ MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} export MOQ_RESUME_SEED MOQ_RESUME_CASES echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" just rs test -p moq-net --features async --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' -python3 test/lib/install.py +python3 sh/js/install.py frozen bun test ./js/net/src/resume.model.test.ts From c7a4c4dec93eb66f7aac39c6592967942812d343 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:02:18 -0800 Subject: [PATCH 16/25] fix: expose coordinated installs through a JS recipe Co-Authored-By: GPT-6.1 Sol --- .github/workflows/interop.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release-js.yml | 2 +- js/justfile | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index a467867c50d..a09496895ee 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -55,7 +55,7 @@ jobs: - name: Workspace dependencies id: setup - run: nix develop --command python3 sh/js/install.py frozen + run: nix develop --command just js install frozen shell: bash -leo pipefail {0} # Each suite reports independently after setup, while a failed step still fails the job. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 989aa4d638a..5ae318cdb34 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -425,7 +425,7 @@ jobs: - name: Install publisher and player engines # Firefox loads libpulse at runtime; Playwright's engine dependencies omit it. run: | - nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" bash -c "python3 sh/js/install.py frozen && bunx playwright install --with-deps chromium firefox" + nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" bash -c "just js install frozen && bunx playwright install --with-deps chromium firefox" sudo apt-get install -y libpulse0 - name: Media and negative controls run: nix develop --command env PLAYWRIGHT_BROWSERS_PATH="${{ runner.temp }}/playwright" just test media --media-engines firefox diff --git a/.github/workflows/release-js.yml b/.github/workflows/release-js.yml index bbae5f20d7c..76cf4762d05 100644 --- a/.github/workflows/release-js.yml +++ b/.github/workflows/release-js.yml @@ -48,7 +48,7 @@ jobs: package-manager-cache: false - name: Install dependencies - run: nix develop --command python3 sh/js/install.py frozen + run: nix develop --command just js install frozen - name: Release packages env: diff --git a/js/justfile b/js/justfile index 7520733c60f..8119d0704f2 100644 --- a/js/justfile +++ b/js/justfile @@ -6,6 +6,10 @@ set working-directory := '.' +# Install root workspace dependencies: frozen for validation, update for lockfile changes. +install mode: + python3 ../sh/js/install.py {{ mode }} + # Run all checks by default. default: just check From 4eebe30b93f73f061bc7d99c51d718235b872c7c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:12:27 -0800 Subject: [PATCH 17/25] test: include coordinated installer in coverage fixtures Co-Authored-By: GPT-6.1 Sol --- sh/coverage.test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sh/coverage.test.py b/sh/coverage.test.py index 5aae214c404..f07df6c0ae9 100644 --- a/sh/coverage.test.py +++ b/sh/coverage.test.py @@ -17,16 +17,22 @@ def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) self.root = Path(self.directory.name) - for script in ("sh/js/coverage.py", "sh/rs/coverage.sh", "sh/rs/nextest.sh", "sh/local.sh"): + for script in ("sh/js/install.py", "sh/js/coverage.py", "sh/rs/coverage.sh", "sh/rs/nextest.sh", "sh/local.sh"): target = self.root / script target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ROOT / script, target) + subprocess.run(["git", "init", "-q", str(self.root)], check=True) self.bin = self.root / "bin" self.bin.mkdir() stub = """#!/usr/bin/env python3 import json, os from pathlib import Path import sys +if sys.argv[1:] == ["--version"]: + print("fixture") + sys.exit(0) +if "install" in sys.argv: + Path("node_modules").mkdir(exist_ok=True) with open(os.environ["COVERAGE_CALLS"], "a") as log: log.write(json.dumps([Path(sys.argv[0]).name, str(Path.cwd()), *sys.argv[1:]]) + "\\n") if os.environ.get("COVERAGE_FAIL") and ("nextest" in sys.argv or "test" in sys.argv): From 7c4cde42660ed5f9e5bfae5bf1f8a783d24bc1a0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:20:20 -0800 Subject: [PATCH 18/25] test: fail promptly when overlap fixture setup fails Co-Authored-By: GPT-6.1 Sol --- sh/js/install.test.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sh/js/install.test.py b/sh/js/install.test.py index c14eedbae08..c62a15feb21 100644 --- a/sh/js/install.test.py +++ b/sh/js/install.test.py @@ -32,7 +32,7 @@ def test_overlap(self): bin_dir.mkdir() bun = bin_dir / "bun" bun.write_text("""#!/usr/bin/env python3 -import os +from pathlib import Path import sys r = Path.cwd() if sys.argv[1] == '--version': @@ -41,8 +41,7 @@ def test_overlap(self): with (r / 'calls').open('a') as f: f.write('install\\n') # Signal the parent while holding the install lock, then block on a pipe. - with open(r / 'started', 'w') as f: - f.write('ready') + print('ready', flush=True) with open(r / 'continue') as f: f.read() (r / 'node_modules').mkdir(exist_ok=True) @@ -51,12 +50,11 @@ def test_overlap(self): (r / 'bun.lock').write_text('after') """) bun.chmod(0o755) - for name in ("started", "continue"): - os.mkfifo(root / name) + os.mkfifo(root / "continue") env = {**os.environ, "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"]} - recipe = subprocess.Popen(["just", "biome"], cwd=root, env=env) - with (root / "started").open() as started: - self.assertEqual(started.read(), "ready") + recipe = subprocess.Popen(["just", "biome"], cwd=root, env=env, stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: recipe.kill() if recipe.poll() is None else None) + self.assertEqual(recipe.stdout.readline(), "ready\n") test = subprocess.Popen(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env) with (root / "continue").open("w"): pass From be1ab151c363ba826e874b30415e7c1afa76daa9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:21:55 -0800 Subject: [PATCH 19/25] test: close overlap fixture output after completion Co-Authored-By: GPT-6.1 Sol --- sh/js/install.test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sh/js/install.test.py b/sh/js/install.test.py index c62a15feb21..36e8a908dba 100644 --- a/sh/js/install.test.py +++ b/sh/js/install.test.py @@ -53,6 +53,7 @@ def test_overlap(self): os.mkfifo(root / "continue") env = {**os.environ, "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"]} recipe = subprocess.Popen(["just", "biome"], cwd=root, env=env, stdout=subprocess.PIPE, text=True) + self.addCleanup(recipe.stdout.close) self.addCleanup(lambda: recipe.kill() if recipe.poll() is None else None) self.assertEqual(recipe.stdout.readline(), "ready\n") test = subprocess.Popen(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env) From 706a53278131b71f08e7119d62a1ca2b7708e53b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:25:46 -0800 Subject: [PATCH 20/25] test: stub installer in dispatcher selection fixture Co-Authored-By: GPT-6.1 Sol --- sh/js/workspace.test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sh/js/workspace.test.py b/sh/js/workspace.test.py index 9f9ad2bac50..dce937f7524 100644 --- a/sh/js/workspace.test.py +++ b/sh/js/workspace.test.py @@ -54,6 +54,8 @@ def test_dispatcher_passes_only_js_impact_paths(self): sh = self.root / "sh" shutil.copy(selector.ROOT / "sh/dispatch.sh", sh / "dispatch.sh") shutil.copy(selector.ROOT / "sh/quest.sh", sh / "quest.sh") + (sh / "js").mkdir() + (sh / "js/install.py").write_text("# Dependency setup is outside this selector fixture.\n") changed = sh / "changed.sh" changed.write_text("#!/bin/sh\nprintf '%s\\n' js/auth/src/token.ts demo/pub/worker.ts quest/unrelated.md\n") changed.chmod(0o755) From aa17b089a88b3a84bed46b4968762c6b7218fa76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:42:46 -0800 Subject: [PATCH 21/25] fix: support native Windows setup and validate install lock contention Co-Authored-By: GPT-6.1 Sol --- demo/boy/justfile | 8 ++--- demo/carla/justfile | 2 +- demo/justfile | 2 +- demo/relay/justfile | 2 +- demo/web/justfile | 2 +- js/justfile | 58 +++++++++++++++++----------------- justfile | 6 ++-- setup.bat | 4 +-- sh/api-diff/run.sh | 2 +- sh/dispatch.sh | 4 +-- sh/drafts/check.sh | 2 +- sh/js/README.md | 8 +++-- sh/js/coverage.py | 2 +- sh/js/install.py | 60 +++++++++++++++++++++++++++++++----- sh/js/install.test.py | 37 +++++++++++++++++++--- sh/js/workspace.py | 2 +- sh/size/collect.sh | 2 +- test/audio-quality/ptt.sh | 2 +- test/audio-quality/record.sh | 2 +- test/audio-quality/run.sh | 2 +- test/drain/run.sh | 2 +- test/ietf-conformance/run.sh | 2 +- test/interop/compat.sh | 2 +- test/interop/interop.sh | 2 +- test/json-properties.sh | 2 +- test/justfile | 10 +++--- test/portrait/run.sh | 2 +- test/resume-model.sh | 2 +- 28 files changed, 154 insertions(+), 79 deletions(-) diff --git a/demo/boy/justfile b/demo/boy/justfile index c7a0404774d..55e16cf7dbe 100644 --- a/demo/boy/justfile +++ b/demo/boy/justfile @@ -2,7 +2,7 @@ set fallback # Run the GB demo: relay + emulator publisher + web viewer. default: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update bun run concurrently --kill-others --prefix-colors auto \ "just relay" \ @@ -57,12 +57,12 @@ web url='http://localhost:4443': # Deploy the rom.moq.dev worker deploy: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update bun wrangler deploy # Upload a single ROM to R2 upload file: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update bun wrangler r2 object put "rom-moq-dev/{{ file }}" --file "rom/{{ file }}" --remote # Sync all ROMs to R2 @@ -72,5 +72,5 @@ sync dir="rom": # List files in the R2 bucket list: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update bun wrangler r2 object list "rom-moq-dev" diff --git a/demo/carla/justfile b/demo/carla/justfile index dd44da25e87..1d48b946c73 100644 --- a/demo/carla/justfile +++ b/demo/carla/justfile @@ -10,7 +10,7 @@ set fallback # # Run a local relay, both trucks, and the web player. default: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update bun run concurrently --kill-others --names relay,trucks,web --prefix-colors auto \ "just relay" \ "just wait && just wait-port 127.0.0.1 2000 && just worker" \ diff --git a/demo/justfile b/demo/justfile index b568b511bc6..9c431de0936 100644 --- a/demo/justfile +++ b/demo/justfile @@ -5,7 +5,7 @@ set fallback # Picks the first free port at/after 4443 so multiple worktrees can run # `just dev` without colliding; see sh/demo/run.sh. default: - python3 ../sh/js/install.py update + uv run --no-project python ../sh/js/install.py update ../sh/demo/run.sh # Wait for the localhost relay to be ready by repeatedly requesting the cert diff --git a/demo/relay/justfile b/demo/relay/justfile index c861ea818ef..793d7504ecc 100644 --- a/demo/relay/justfile +++ b/demo/relay/justfile @@ -10,7 +10,7 @@ default: # Cluster peers still authenticate to each other via mTLS (the cert/ca recipes). cluster: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update bun run concurrently --kill-others --names root,leaf0,leaf1,bbb,tos,web --prefix-colors auto \ "just root" \ diff --git a/demo/web/justfile b/demo/web/justfile index b9bf0df3d2d..8c4dd42402a 100644 --- a/demo/web/justfile +++ b/demo/web/justfile @@ -13,4 +13,4 @@ serve url='http://localhost:4443': install # Install the workspace deps that vite and the demo's imports need. install: - python3 ../../sh/js/install.py update + uv run --no-project python ../../sh/js/install.py update diff --git a/js/justfile b/js/justfile index 8119d0704f2..32a229b4320 100644 --- a/js/justfile +++ b/js/justfile @@ -6,14 +6,14 @@ set working-directory := '.' -# Install root workspace dependencies: frozen for validation, update for lockfile changes. -install mode: - python3 ../sh/js/install.py {{ mode }} - # Run all checks by default. default: just check +# Install root workspace dependencies: frozen for validation, update for lockfile changes. +install mode: + uv run --no-project python ../sh/js/install.py {{ mode }} + # `build` runs here rather than as its own gate: the per-package `check` script # is `tsc --noEmit`, which never runs declaration emit, so the errors that only # surface when emitting a `.d.ts` (TS4023 and friends) pass `check` and then @@ -50,7 +50,7 @@ fix: # Rebuild changed workspace packages (or `--all`) to update their API reports. fix-changed list: - python3 ../sh/js/install.py update + uv run --no-project python ../sh/js/install.py update MOQ_API_REPORT=update python3 ../sh/js/workspace.py build '{{ list }}' # Run all unit tests. @@ -67,44 +67,44 @@ coverage package="": # Run one net benchmark: broadcasts, forward, requests, fills, fetch, reader, frames, track, resume, observed, drift, ledger, bandwidth, replay, varint, lite-varint, priority, subscribe-drop, refresh, compression, or datagrams. bench name: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun net/bench/{{ name }}.ts # Sweep browser publisher fairness over backlog x viewers using the devShell's Chromium. serve-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/serve-browser.ts # Sweep catalog estimate publishes and subscribe updates over tracks x viewers. catalog-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun publish/bench/catalog.ts # Sweep QMux record decoding over record size x frames per record. qmux-record-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun qmux/bench/record.ts # Sweep QMux datagram receive-queue draining over backlog size (median of 7 runs). qmux-datagrams-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun qmux/bench/datagrams.ts # Sweep CMAF fragment decode over mdat bytes x declared sample count, including a hostile count. cmaf-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun hang/bench/cmaf.ts # Time Effect create and close, which pays for the creation stack its diagnostics print. signals-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun signals/bench/effect.ts # Bundle the worklets with Vite, webpack, esbuild, and Bun, and load them in Chromium, with and # without a strict CSP. Nightly, against the bundlers pinned in test/worklet. worklet-matrix: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun ../test/worklet/matrix.ts # Build all packages. @@ -117,87 +117,87 @@ dist-imports: # Compare eager and cold lazy WebSocket fallback setup in Chromium. qmux-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/qmux.ts # Sweep open arrival handles against their unread backlog. arrivals-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/arrivals.ts # Reduce and compare the worst known peer grant shapes, swept over AUTH_OK size. pattern-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project pattern/bench/tsconfig.json bun pattern/bench/algebra.ts # Sweep manual-clock armed timers x due timers, with independent and shared abort scopes. clock-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun clock/bench/timers.ts # Sweep buffered JSON stream reads over record count, payload size, and compression. json-stream-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/stream.ts # Sweep snapshot codec cost over document size and changed keys. json-codec-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/codec.ts # Compare the duplicate-refusing JSON frame parse against bare JSON.parse. json-parse-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project json/bench/tsconfig.json bun json/bench/parse.ts # Sweep a slow hang consumer's skipped groups against its warning traffic and catch-up time. slow-group-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun hang/bench/slow-group.ts # Sweep a hang consumer's buffered groups, frames per group, publishers and players across arrival orders. buffered-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project hang/bench/tsconfig.json bun hang/bench/buffered.ts # Sweep the audio jitter estimator's held history against in-order and reordered frames. jitter-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun hang/bench/jitter.ts # Sweep players against the main-thread CPU and allocation of feeding the audio ring, in Chromium. ring-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/ring.ts # Sweep audio rendition count against repeated mute/unmute catalog updates. audio-support-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/support.ts # Sweep decoder count against PCM callback size during audio warmup. audio-warmup-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun watch/bench/warmup.ts # Sweep data-channel qmux stream count and payload size. p2p-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun p2p/bench/transport.ts # Sweep E2EE frame size against writes in flight through the WebCrypto pump. e2ee-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bun crypto/bench/pump.ts # Sweep audio renditions x raw retention through native WebCodecs in Chromium. raw-audio-bench: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen bunx tsc --noEmit --project publish/bench/tsconfig.json bun publish/bench/audio.ts diff --git a/justfile b/justfile index 08a1270928b..59e7d4528ec 100644 --- a/justfile +++ b/justfile @@ -94,7 +94,7 @@ bench-browser *args: # Install repo-wide tooling. Per-language deps install on first check. install: - python3 sh/js/install.py update + uv run --no-project python sh/js/install.py update cargo install --locked cargo-shear cargo-sort cargo-semver-checks release-plz # Lints, compiles, and tests only the packages the branch changed plus @@ -139,12 +139,12 @@ ci-select-platform $BASE="": # Check every supported source file; exclusions live in biome.jsonc. biome *args: - python3 sh/js/install.py frozen + uv run --no-project python sh/js/install.py frozen bun biome check --error-on-warnings {{ args }} . # Apply Biome's fixes; installs without a frozen lockfile, like the other fixers. biome-fix: - python3 sh/js/install.py update + uv run --no-project python sh/js/install.py update bun biome check --error-on-warnings --write . # Auto-fix lint and formatting for what the branch changed since BASE. diff --git a/setup.bat b/setup.bat index 0d8b07c94d1..e0c23d1b4f8 100644 --- a/setup.bat +++ b/setup.bat @@ -41,7 +41,7 @@ echo --- FFmpeg ^(CLI, for the `just dev` demo media^) --- %WG% --id Gyan.FFmpeg echo --- GitHub CLI --- %WG% --id GitHub.cli -echo --- uv ^(Python, for the py/ workspace^) --- +echo --- uv ^(Python for dependency setup and the py/ workspace^) --- %WG% --id astral-sh.uv REM MSVC linker + C++ headers, required for any Rust MSVC build. winget skips @@ -56,7 +56,7 @@ where rustup >nul 2>nul && ( rustup default stable & rustup update stable ) || e echo. echo === Installing JS dependencies ^(bun install^) === -where bun >nul 2>nul && bun install || echo NOTE: bun not on PATH yet. Reopen your terminal, then run: bun install +where bun >nul 2>nul && where uv >nul 2>nul && uv run --no-project python sh/js/install.py update || echo NOTE: bun or uv not on PATH yet. Reopen your terminal, then run: just js install update echo. echo === Done === diff --git a/sh/api-diff/run.sh b/sh/api-diff/run.sh index 3330f12b3da..93ce238356d 100755 --- a/sh/api-diff/run.sh +++ b/sh/api-diff/run.sh @@ -214,7 +214,7 @@ js() { local manifest dir name version file entry surface # The renderer imports TypeDoc from this checkout's install, which a fresh runner lacks. The # current checkout uses the coordinated full install shared with other readers. - python3 sh/js/install.py frozen >&2 + uv run --no-project python sh/js/install.py frozen >&2 # Not local: the trap runs after a failure has unwound this function's locals. checkout=$out/js/checkout git worktree add --detach "$checkout" "$base" >&2 diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 6cfc2f0bc9b..a4301fae24a 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -372,7 +372,7 @@ step() { check:ts_paths) python3 sh/gh/ts-paths.py ;; check:biome) just biome ;; check:markdown) - python3 sh/js/install.py frozen + uv run --no-project python sh/js/install.py frozen sh/markdown.sh check ;; check:large_files) sh/large-files.sh "$large_base" ;; @@ -392,7 +392,7 @@ step() { fix:js) just js fix-changed "$js_list" ;; fix:biome) just biome-fix ;; fix:markdown) - python3 sh/js/install.py update + uv run --no-project python sh/js/install.py update sh/markdown.sh fix ;; fix:python) diff --git a/sh/drafts/check.sh b/sh/drafts/check.sh index f227a0738f2..ada82b1aa66 100755 --- a/sh/drafts/check.sh +++ b/sh/drafts/check.sh @@ -12,7 +12,7 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen cd drafts if grep -n "$(printf '\t')" draft-*.md; then diff --git a/sh/js/README.md b/sh/js/README.md index eb94f765fac..16833a8c5f3 100644 --- a/sh/js/README.md +++ b/sh/js/README.md @@ -34,14 +34,18 @@ The root devShell supplies Chromium and exports its Playwright version. Checks require every npm `playwright` dependency outside `test/smoke` to pin that exact version. A root `flake.nix` or `flake.lock` change selects JS checks too: when nixpkgs moves Playwright, update the reported manifests and regenerate `bun.lock` -with `python3 sh/js/install.py update` in the new shell. Smoke has its own flake and freshness check. +with `uv run --no-project python sh/js/install.py update` in the new shell. Smoke has its own flake and freshness check. Run `python3 sh/js/workspace.test.py` for graph and execution regressions. The same suite runs in `just gh test` on every check. To inspect work without executing it, pass a file containing relevant paths, one per line, to `python3 sh/js/workspace.py plan LISTFILE`, or pass `--all`. -All root workspace installers use `python3 sh/js/install.py frozen` for checks +All root workspace installers use `uv run --no-project python sh/js/install.py frozen` for checks and tests, or `update` when lockfile changes are allowed. Both reuse an unchanged installation under one checkout lock, including libav output. Separate Worker, smoke-client, and disposable baseline workspaces keep their own installs. + +The concurrency guarantee applies to unchanged dependency inputs. Stop active +consumers before changing dependencies or postinstall inputs. The installer uses +OS locks on Unix and Windows; uv supplies Python on both setup paths. diff --git a/sh/js/coverage.py b/sh/js/coverage.py index f2d877541c8..ff038d85867 100644 --- a/sh/js/coverage.py +++ b/sh/js/coverage.py @@ -66,7 +66,7 @@ def main(): (ROOT / "target/coverage/js/summary.json").unlink(missing_ok=True) for directory in selected: (ROOT / "target/coverage/js" / directory.name / "lcov.info").unlink(missing_ok=True) - subprocess.run(["python3", str(ROOT / "sh/js/install.py"), "frozen"], cwd=ROOT, check=True) + subprocess.run(["uv", "run", "--no-project", "python", str(ROOT / "sh/js/install.py"), "frozen"], cwd=ROOT, check=True) for directory in selected: out = ROOT / "target/coverage/js" / directory.name out.mkdir(parents=True, exist_ok=True) diff --git a/sh/js/install.py b/sh/js/install.py index ad39166ab88..37d8b4b4fa6 100644 --- a/sh/js/install.py +++ b/sh/js/install.py @@ -2,8 +2,9 @@ """Install the workspace once per dependency configuration, under a checkout lock.""" import argparse -import fcntl import hashlib +import json +import os import shutil import subprocess from pathlib import Path @@ -13,24 +14,34 @@ def fingerprint(): digest = hashlib.sha256(subprocess.check_output(["bun", "--version"])) - digest.update(str(shutil.which("nix")).encode()) + digest.update(str(shutil.which("nix") is not None).encode()) # Include the postinstall build inputs as well as dependency manifests. + manifest = json.loads((ROOT / "package.json").read_text()) + manifests = {b"package.json"} + for workspace in manifest["workspaces"]: + matches = list(ROOT.glob(workspace)) + # Exact workspace paths still contribute when their manifest was deleted. + for directory in matches or [ROOT / workspace]: + manifests.add(str((directory / "package.json").relative_to(ROOT)).encode()) paths = subprocess.check_output( [ "git", "ls-files", + "--cached", + "--others", + "--exclude-standard", "-z", "--", - "*package.json", "bun.lock", "flake.nix", "flake.lock", - "nix", + "nix/overlay.nix", "js/libav", "patches", ], cwd=ROOT, ).split(b"\0") + paths = set(paths) | manifests for name in sorted(filter(None, paths)): digest.update(name) path = ROOT / name.decode() @@ -46,12 +57,45 @@ def fingerprint(): args = parser.parse_args() (ROOT / ".tmp").mkdir(exist_ok=True) -with (ROOT / ".tmp/bun-install.lock").open("w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) +with (ROOT / ".tmp/bun-install.lock").open("ab") as lock: + if os.name == "nt": + import ctypes + import msvcrt + from ctypes import wintypes + + class Overlapped(ctypes.Structure): + _fields_ = [ + ("Internal", ctypes.c_size_t), + ("InternalHigh", ctypes.c_size_t), + ("Offset", wintypes.DWORD), + ("OffsetHigh", wintypes.DWORD), + ("hEvent", wintypes.HANDLE), + ] + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + acquire = kernel.LockFileEx + acquire.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + wintypes.DWORD, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(Overlapped), + ] + acquire.restype = wintypes.BOOL + overlapped = Overlapped() + # A synchronous, exclusive byte-range lock waits until the owning + # process closes its handle, including when that process crashes. + if not acquire(msvcrt.get_osfhandle(lock.fileno()), 2, 0, 1, 0, ctypes.byref(overlapped)): + raise ctypes.WinError(ctypes.get_last_error()) + else: + import fcntl + + fcntl.flock(lock, fcntl.LOCK_EX) current = fingerprint() stamp = ROOT / "node_modules/.moq-install" - # Another invocation must not relink dependencies or rebuild libav while - # the first invocation is importing them or playing its bundled page. + # Identical inputs reuse the tree while other invocations consume it. + # Dependency edits invalidate it; stop consumers before changing inputs. if ( not stamp.exists() or stamp.read_text() != current diff --git a/sh/js/install.test.py b/sh/js/install.test.py index 36e8a908dba..77d1705bd58 100644 --- a/sh/js/install.test.py +++ b/sh/js/install.test.py @@ -2,6 +2,7 @@ import os import shutil +import signal import subprocess import tempfile import unittest @@ -12,6 +13,7 @@ class Install(unittest.TestCase): def test_overlap(self): + (ROOT / ".tmp").mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: root = Path(directory) (root / "sh/js").mkdir(parents=True) @@ -24,7 +26,7 @@ def test_overlap(self): test_setup = next(line for line in test_setup if "install.py" in line) (root / "justfile").write_text("biome:\n" + recipe_setup + "\n") (root / "test/justfile").write_text("media:\n" + test_setup + "\n") - (root / "package.json").write_text("{}") + (root / "package.json").write_text('{"workspaces": []}') (root / "bun.lock").write_text("before") subprocess.run(["git", "init", "-q", str(root)], check=True) subprocess.run(["git", "-C", str(root), "add", "."], check=True) @@ -51,12 +53,37 @@ def test_overlap(self): """) bun.chmod(0o755) os.mkfifo(root / "continue") - env = {**os.environ, "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"]} - recipe = subprocess.Popen(["just", "biome"], cwd=root, env=env, stdout=subprocess.PIPE, text=True) + (root / "sitecustomize.py").write_text("""import fcntl, os +flock = fcntl.flock +def observed(fd, operation): + if os.environ.get('TRACE_LOCK'): + try: + flock(fd, operation | fcntl.LOCK_NB) + except BlockingIOError: + print('waiting', flush=True) + flock(fd, operation) + else: + flock(fd, operation) +fcntl.flock = observed +""") + env = {**os.environ, "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"], "PYTHONPATH": str(root)} + recipe = subprocess.Popen( + ["just", "biome"], cwd=root, env=env, stdout=subprocess.PIPE, text=True, start_new_session=True + ) self.addCleanup(recipe.stdout.close) - self.addCleanup(lambda: recipe.kill() if recipe.poll() is None else None) + self.addCleanup(lambda: os.killpg(recipe.pid, signal.SIGKILL) if recipe.poll() is None else None) self.assertEqual(recipe.stdout.readline(), "ready\n") - test = subprocess.Popen(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env) + test = subprocess.Popen( + ["just", "--justfile", "test/justfile", "media"], + cwd=root, + env={**env, "TRACE_LOCK": "1"}, + stdout=subprocess.PIPE, + text=True, + start_new_session=True, + ) + self.addCleanup(test.stdout.close) + self.addCleanup(lambda: os.killpg(test.pid, signal.SIGKILL) if test.poll() is None else None) + self.assertEqual(test.stdout.readline(), "waiting\n") with (root / "continue").open("w"): pass self.assertEqual(recipe.wait(), 0) diff --git a/sh/js/workspace.py b/sh/js/workspace.py index 124049de059..71cb530ed90 100644 --- a/sh/js/workspace.py +++ b/sh/js/workspace.py @@ -148,7 +148,7 @@ def reports(): if directories: command("bun", "js/common/api.ts", "check", *directories) - command("python3", str(ROOT / "sh/js/install.py"), "frozen") + command("uv", "run", "--no-project", "python", str(ROOT / "sh/js/install.py"), "frozen") if action == "test": command("python3", "sh/js/playwright.test.py") command( diff --git a/sh/size/collect.sh b/sh/size/collect.sh index 4534946fefe..96955f13365 100755 --- a/sh/size/collect.sh +++ b/sh/size/collect.sh @@ -15,7 +15,7 @@ esac cargo build --locked --release -p moq-ffi --no-default-features --target-dir "$target/minimal" cp "$target/minimal/release/libmoq_ffi.$dylib" "$out/native/libmoq_ffi-minimal.$dylib" cp "$target/minimal/release/libmoq_ffi.a" "$out/native/libmoq_ffi-minimal.a" -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen just js build bun sh/size/bundles.ts "$out/js" # Symbols are necessary only for attribution; they never replace the measured shipping sizes. diff --git a/test/audio-quality/ptt.sh b/test/audio-quality/ptt.sh index f04179b2c19..f00df4b7f54 100755 --- a/test/audio-quality/ptt.sh +++ b/test/audio-quality/ptt.sh @@ -19,7 +19,7 @@ export HARNESS_KEEP="${HARNESS_KEEP:-1}" exit 2 } -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen cargo build --locked -p moq-relay TARGET_BASE="${CARGO_TARGET_DIR:-$ROOT/target}" harness_port relay diff --git a/test/audio-quality/record.sh b/test/audio-quality/record.sh index 4a4a046204d..cb3c682668f 100755 --- a/test/audio-quality/record.sh +++ b/test/audio-quality/record.sh @@ -63,7 +63,7 @@ harness_begin audio-quality-record "$RERUN" echo "building the pages..." ( cd "$CLIENT" - python3 "$WORKSPACE/sh/js/install.py" frozen + uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir ) diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index 497bd85db8e..f1cfd36deed 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -241,7 +241,7 @@ done harness_begin audio-quality "$RERUN" echo "installing the workspace..." -(cd "$CLIENT" && python3 "$WORKSPACE/sh/js/install.py" frozen) +(cd "$CLIENT" && uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen) failed=0 diff --git a/test/drain/run.sh b/test/drain/run.sh index 796a9ffc4f4..1faa685bab2 100755 --- a/test/drain/run.sh +++ b/test/drain/run.sh @@ -61,7 +61,7 @@ flag=() TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" [[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" -python3 "$WORKSPACE/sh/js/install.py" frozen +uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen # ── relays ────────────────────────────────────────────────────────────────── # Start one relay on a reserved port: `start [extra relay args...]`. diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh index b44bf824225..387071eb488 100755 --- a/test/ietf-conformance/run.sh +++ b/test/ietf-conformance/run.sh @@ -30,7 +30,7 @@ relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" # Cargo accepts an absolute target directory too. [[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" -python3 "$WORKSPACE/sh/js/install.py" frozen +uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen harness_begin ietf-conformance "$(harness_env HARNESS_RESULTS)just test ietf-conformance$(harness_argv "$@")" python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" # LibreSSL in the dev shell has no system-wide openssl.cnf. diff --git a/test/interop/compat.sh b/test/interop/compat.sh index d92ad4b28ee..25c715557e8 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -31,7 +31,7 @@ case "$*" in ;; esac cd "$WORKSPACE" -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 66a7da161a8..014e654497a 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -709,7 +709,7 @@ prepare_js() { return fi echo "installing js clients (workspace @moq/* via bun)..." - if ! python3 "$WORKSPACE/sh/js/install.py" frozen >"$HARNESS_RUN/js-install.log" 2>&1; then + if ! uv run --no-project python "$WORKSPACE/sh/js/install.py" frozen >"$HARNESS_RUN/js-install.log" 2>&1; then for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done sed 's/^/ /' "$HARNESS_RUN/js-install.log" >&2 || true return diff --git a/test/json-properties.sh b/test/json-properties.sh index e09180bdcac..810694e1ba1 100644 --- a/test/json-properties.sh +++ b/test/json-properties.sh @@ -7,7 +7,7 @@ MOQ_JSON_SEED=${MOQ_JSON_SEED:-$(python3 -c 'import secrets; print(secrets.randb MOQ_JSON_CASES=${MOQ_JSON_CASES:-4096} export MOQ_JSON_SEED MOQ_JSON_CASES echo "JSON codec properties: MOQ_JSON_SEED=$MOQ_JSON_SEED MOQ_JSON_CASES=$MOQ_JSON_CASES" -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen just rs test -p moq-json --lib -E 'test(snapshot_codec_property) | test(window_codec_property)' bun test ./js/json/src/snapshot/properties.test.ts ./js/json/src/window/properties.test.ts bunx tsc --noEmit --project js/json/bench/tsconfig.json diff --git a/test/justfile b/test/justfile index 8c737b3563d..62ad49b80f1 100644 --- a/test/justfile +++ b/test/justfile @@ -52,7 +52,7 @@ harness: python3 interop/report.test.py python3 interop/tone.test.py cd .. && bun test {{ replace(harness_compat_tests, "\n", " ") }} - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen cd interop/clients/js && bash ../../../../sh/local.sh bun test {{ replace(harness_browser_tests, "\n", " ") }} # Cross-language media interop test, built from this checkout. Stands up a @@ -66,7 +66,7 @@ harness: # Both runs first check the Rust/JS wire and session interop suites, in both directions. [positional-arguments] interop *args: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen just rs test -p moq-net -p moq-crypto --lib \ auth_unsupported_interop \ ietf_compression_fallback_interop \ @@ -250,7 +250,7 @@ drain *args: # The @moq/video renderer in headless Chromium: WebGPU, a lost device, and the Canvas2D fallback. video: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen # A desktop display makes software Vulkan request unsupported window surfaces. cd video && bash ../../sh/local.sh env -u DISPLAY -u WAYLAND_DISPLAY bun test ./renderer.browser.ts @@ -290,10 +290,10 @@ resume-model: # TypeScript-to-Rust qmux interop. qmux: cargo build --locked --quiet --manifest-path ../rs/qmux/Cargo.toml --example interop-server --example interop-client - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen cd .. && QMUX_INTEROP_BIN={{ quote(qmux_examples) }} bun test ./test/qmux/interop.test.ts # Complete MoQ SETUP and transfer a frame over a real browser data-channel pair. p2p: - python3 ../sh/js/install.py frozen + uv run --no-project python ../sh/js/install.py frozen cd p2p && bash ../../sh/local.sh bun test ./transport.browser.ts diff --git a/test/portrait/run.sh b/test/portrait/run.sh index d4d50054266..c14fe333fca 100755 --- a/test/portrait/run.sh +++ b/test/portrait/run.sh @@ -12,7 +12,7 @@ source "$DIR/../lib/harness.sh" harness_begin portrait 'just test portrait' cd "$WORKSPACE" cargo build --locked -p moq-relay -p moq-cli --features moq-cli/transcode,moq-cli/openh264 -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen (cd test/interop/clients/js && bunx vite build --outDir "$HARNESS_RUN/js-dist") harness_port relay URL="http://127.0.0.1:$HARNESS_PORT" diff --git a/test/resume-model.sh b/test/resume-model.sh index 015d9c80a23..b91a8ed5821 100755 --- a/test/resume-model.sh +++ b/test/resume-model.sh @@ -8,5 +8,5 @@ MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} export MOQ_RESUME_SEED MOQ_RESUME_CASES echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" just rs test -p moq-net --features async --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen bun test ./js/net/src/resume.model.test.ts From 4054ccf1e7ebae9f973a11fee6da48cd9f07aedc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:48:48 -0800 Subject: [PATCH 22/25] style: format coordinated coverage setup command Co-Authored-By: GPT-6.1 Sol --- sh/js/coverage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sh/js/coverage.py b/sh/js/coverage.py index ff038d85867..6b59fb1494f 100644 --- a/sh/js/coverage.py +++ b/sh/js/coverage.py @@ -66,7 +66,9 @@ def main(): (ROOT / "target/coverage/js/summary.json").unlink(missing_ok=True) for directory in selected: (ROOT / "target/coverage/js" / directory.name / "lcov.info").unlink(missing_ok=True) - subprocess.run(["uv", "run", "--no-project", "python", str(ROOT / "sh/js/install.py"), "frozen"], cwd=ROOT, check=True) + subprocess.run( + ["uv", "run", "--no-project", "python", str(ROOT / "sh/js/install.py"), "frozen"], cwd=ROOT, check=True + ) for directory in selected: out = ROOT / "target/coverage/js" / directory.name out.mkdir(parents=True, exist_ok=True) From 9a2b7208157125923d8e37f02d64a1c7d5aecfb9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:52:23 -0800 Subject: [PATCH 23/25] fix: declare installer tools and preserve sequential Windows support Co-Authored-By: GPT-6.1 Sol --- bench/browser/run.sh | 2 +- sh/dispatch.sh | 10 +++++----- sh/js/README.md | 4 ++-- sh/js/install.py | 33 ++------------------------------- 4 files changed, 10 insertions(+), 39 deletions(-) diff --git a/bench/browser/run.sh b/bench/browser/run.sh index 106c9ae6404..e36a6bfc965 100755 --- a/bench/browser/run.sh +++ b/bench/browser/run.sh @@ -88,7 +88,7 @@ if [[ -z $RELAY ]]; then fi cd "$WORKSPACE" -python3 sh/js/install.py frozen +uv run --no-project python sh/js/install.py frozen # BROWSER_BIN overrides the Chromium provided by the root devShell. cd "$HERE" diff --git a/sh/dispatch.sh b/sh/dispatch.sh index a4301fae24a..7844a7290d7 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -135,12 +135,12 @@ module obs_compile '^(\.cargo/config\.toml$|cpp/obs/|cpp/moq/|sh/obs/|rs/moq-ffi # that fix runs it first, since Biome can fix sources that the package build # rejects, such as a type-only export. module biome '' \ - tools='bun' lane=bun fix + tools='bun uv' lane=bun fix # The JS tests also read Rust's shared JSON vectors; sh/js/workspace.py # fails a check that reads a cross-tree input this pattern misses. # Root Nix inputs own the browser version every npm Playwright pin matches. module js '^(js/|bench/browser/|doc/|drafts/|demo/(boy|web|carla/web)/|test/|rs/.*\.json$|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$|flake\.(nix|lock)$)' \ - tools='bun python3' lane=bun fix + tools='bun python3 uv' lane=bun fix module kt '^(\.cargo/config\.toml$|kt/|sh/kt/|rs/moq-ffi/|rs/uniffi-bindgen/|doc/lib/kt/|doc/lib/samples\.sh$)' \ tools='cargo gradle java ktfmt' compiles=check fix # Rust source is compiled below. Nix reads Cargo metadata, packaging inputs, @@ -161,11 +161,11 @@ module swift '^(\.cargo/config\.toml$|swift/|sh/swift/|rs/moq-ffi/|rs/uniffi-bin module go '^(\.cargo/config\.toml$|go/|sh/go/|rs/moq-ffi/|doc/lib/go/|doc/lib/samples\.sh$)' \ tools='cargo go uniffi-bindgen-go' compiles=check fix module drafts '^(drafts/|sh/drafts/|doc/\.vitepress/drafts\.ts$)' \ - tools='bun kramdown-rfc xml2rfc' lane=bun + tools='bun uv kramdown-rfc xml2rfc' lane=bun module markdown '' \ - tools='bun' lane=bun fix + tools='bun uv' lane=bun fix module gh '' \ - tools='actionlint zizmor bun python3 cargo release-plz clippy-driver dpkg-deb nfpm rpm' lane=bun + tools='actionlint zizmor bun python3 uv cargo release-plz clippy-driver dpkg-deb nfpm rpm' lane=bun # The drill mutations patch Rust source, so a Rust change can move the code # they target. Only nightly runs the drills; this catches a stale patch. module drill '^(rs/|test/drill/)' diff --git a/sh/js/README.md b/sh/js/README.md index 16833a8c5f3..8aaa93f744b 100644 --- a/sh/js/README.md +++ b/sh/js/README.md @@ -47,5 +47,5 @@ installation under one checkout lock, including libav output. Separate Worker, smoke-client, and disposable baseline workspaces keep their own installs. The concurrency guarantee applies to unchanged dependency inputs. Stop active -consumers before changing dependencies or postinstall inputs. The installer uses -OS locks on Unix and Windows; uv supplies Python on both setup paths. +consumers before changing dependencies or postinstall inputs. Unix installers share an OS lock. Native Windows runs one recipe at a time, +matching its existing development support; uv supplies Python on both setup paths. diff --git a/sh/js/install.py b/sh/js/install.py index 37d8b4b4fa6..a29d4d8853c 100644 --- a/sh/js/install.py +++ b/sh/js/install.py @@ -58,37 +58,8 @@ def fingerprint(): (ROOT / ".tmp").mkdir(exist_ok=True) with (ROOT / ".tmp/bun-install.lock").open("ab") as lock: - if os.name == "nt": - import ctypes - import msvcrt - from ctypes import wintypes - - class Overlapped(ctypes.Structure): - _fields_ = [ - ("Internal", ctypes.c_size_t), - ("InternalHigh", ctypes.c_size_t), - ("Offset", wintypes.DWORD), - ("OffsetHigh", wintypes.DWORD), - ("hEvent", wintypes.HANDLE), - ] - - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - acquire = kernel.LockFileEx - acquire.argtypes = [ - wintypes.HANDLE, - wintypes.DWORD, - wintypes.DWORD, - wintypes.DWORD, - wintypes.DWORD, - ctypes.POINTER(Overlapped), - ] - acquire.restype = wintypes.BOOL - overlapped = Overlapped() - # A synchronous, exclusive byte-range lock waits until the owning - # process closes its handle, including when that process crashes. - if not acquire(msvcrt.get_osfhandle(lock.fileno()), 2, 0, 1, 0, ctypes.byref(overlapped)): - raise ctypes.WinError(ctypes.get_last_error()) - else: + # Native Windows supports sequential recipes; Unix coordinates parallel runs. + if os.name != "nt": import fcntl fcntl.flock(lock, fcntl.LOCK_EX) From ccde7512afccd0d12abb625b2523296c57bc9221 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:20:20 -0800 Subject: [PATCH 24/25] test: capture installer contention through the recipe Co-Authored-By: GPT-6.1 Sol --- sh/js/install.test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sh/js/install.test.py b/sh/js/install.test.py index 805a421a1a5..8c3e79cd925 100644 --- a/sh/js/install.test.py +++ b/sh/js/install.test.py @@ -61,7 +61,7 @@ def test_overlap(self): self.addCleanup(lambda: os.killpg(recipe.pid, signal.SIGKILL) if recipe.poll() is None else None) self.assertEqual(recipe.stdout.readline(), "ready\n") test = subprocess.Popen( - ["just", "--quiet", "--justfile", "test/justfile", "media"], + ["just", "--justfile", "test/justfile", "media"], cwd=root, env=env, stderr=subprocess.PIPE, @@ -72,6 +72,7 @@ def test_overlap(self): self.addCleanup(test.stderr.close) self.addCleanup(test.stdout.close) self.addCleanup(lambda: os.killpg(test.pid, signal.SIGKILL) if test.poll() is None else None) + self.assertIn("install.py frozen", test.stderr.readline()) self.assertEqual(test.stderr.readline(), "waiting for another workspace dependency install\n") with (root / "continue").open("w"): pass @@ -79,7 +80,7 @@ def test_overlap(self): self.assertEqual(test.wait(), 0) self.assertEqual((root / "calls").read_text(), "install\n") # Frozen readers reuse the post-update fingerprint as well. - subprocess.run(["just", "--quiet", "--justfile", "test/justfile", "media"], cwd=root, env=env, check=True) + subprocess.run(["just", "--justfile", "test/justfile", "media"], cwd=root, env=env, check=True) self.assertEqual((root / "calls").read_text(), "install\n") From f47fcd364b0367ca7050c1cc2d7a2bdf02dbd83c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:37:53 -0800 Subject: [PATCH 25/25] test: fail overlap fixture when installer locking is absent Co-Authored-By: GPT-6.1 Sol --- sh/js/install.test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sh/js/install.test.py b/sh/js/install.test.py index 8c3e79cd925..7e76e09070c 100644 --- a/sh/js/install.test.py +++ b/sh/js/install.test.py @@ -64,16 +64,15 @@ def test_overlap(self): ["just", "--justfile", "test/justfile", "media"], cwd=root, env=env, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, text=True, start_new_session=True, ) - self.addCleanup(test.stderr.close) self.addCleanup(test.stdout.close) self.addCleanup(lambda: os.killpg(test.pid, signal.SIGKILL) if test.poll() is None else None) - self.assertIn("install.py frozen", test.stderr.readline()) - self.assertEqual(test.stderr.readline(), "waiting for another workspace dependency install\n") + self.assertIn("install.py frozen", test.stdout.readline()) + self.assertEqual(test.stdout.readline(), "waiting for another workspace dependency install\n") with (root / "continue").open("w"): pass self.assertEqual(recipe.wait(), 0)