From 6a12591bd829a8e46438ba672572b00a2f4bdf41 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 15 Aug 2026 16:45:41 -0700 Subject: [PATCH 01/29] chore: remove GitHub CLI from Nix dev shell (#2878) Co-authored-by: GPT-5 --- flake.nix | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index ea3a8c3c0cc..284f6388e3d 100644 --- a/flake.nix +++ b/flake.nix @@ -219,11 +219,9 @@ gzip ]; - # Developer workflow tooling not needed for builds: the GitHub CLI - # for opening/reviewing PRs, plus jq to read `cargo metadata` in - # `just rs check-changed`. + # Developer workflow tooling not needed for builds: jq reads + # `cargo metadata` in `just rs check-changed`. devTools = with pkgs; [ - gh jq ]; From 1bf2b94de927e1dce23095c916b8d88c5ad99d50 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 15 Aug 2026 17:02:10 -0700 Subject: [PATCH 02/29] feat(bench): chat workloads, JSONL stats, and a production-safe host sampler (M0) (#2877) Co-authored-by: Claude --- Cargo.lock | 29 ++- rs/justfile | 10 + rs/moq-bench/Cargo.toml | 12 + rs/moq-bench/README.md | 77 +++++- rs/moq-bench/config/chat-pub.toml | 21 ++ rs/moq-bench/config/chat-sub.toml | 14 ++ rs/moq-bench/config/chat.toml | 23 ++ rs/moq-bench/src/config.rs | 30 +++ rs/moq-bench/src/connection.rs | 261 ++++++++++++++++++-- rs/moq-bench/src/host.rs | 389 ++++++++++++++++++++++++++++++ rs/moq-bench/src/main.rs | 13 +- rs/moq-bench/src/stats.rs | 73 +++++- 12 files changed, 917 insertions(+), 35 deletions(-) create mode 100644 rs/moq-bench/config/chat-pub.toml create mode 100644 rs/moq-bench/config/chat-sub.toml create mode 100644 rs/moq-bench/config/chat.toml create mode 100644 rs/moq-bench/src/host.rs diff --git a/Cargo.lock b/Cargo.lock index 1789bd9119a..be446606850 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4653,6 +4653,7 @@ dependencies = [ "humantime-serde", "moq-native", "moq-net", + "procfs 0.18.0", "rand 0.10.2", "rustls", "serde", @@ -7024,10 +7025,23 @@ checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ "bitflags 2.13.1", "hex", - "procfs-core", + "procfs-core 0.17.0", "rustix 0.38.44", ] +[[package]] +name = "procfs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" +dependencies = [ + "bitflags 2.13.1", + "chrono", + "flate2", + "procfs-core 0.18.0", + "rustix 1.1.4", +] + [[package]] name = "procfs-core" version = "0.17.0" @@ -7038,6 +7052,17 @@ dependencies = [ "hex", ] +[[package]] +name = "procfs-core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" +dependencies = [ + "bitflags 2.13.1", + "chrono", + "hex", +] + [[package]] name = "profiling" version = "1.0.18" @@ -7056,7 +7081,7 @@ dependencies = [ "libc", "memchr", "parking_lot", - "procfs", + "procfs 0.17.0", "thiserror 2.0.20", ] diff --git a/rs/justfile b/rs/justfile index 3ed8ae75ff2..7920730210e 100644 --- a/rs/justfile +++ b/rs/justfile @@ -326,6 +326,16 @@ loom *args: build: cargo build +# Run a moq-bench preset against a relay, e.g. `just rs bench chat https://relay.example.com`. +# Extra args pass through, so `--connections 500 --output stats.jsonl` work as-is. +bench preset url *args: + cargo run --release -p moq-bench -- --file 'rs/moq-bench/config/{{ preset }}.toml' --client-connect '{{ url }}' {{ args }} + +# Sample a process's CPU/memory/context switches on this host (run it where the +# relay runs; it only reads /proc). E.g. `just rs bench-host --name moq-relay`. +bench-host *args: + cargo run --release -p moq-bench --bin moq-bench-host -- {{ args }} + # Remove the Rust target directory. clean: cargo clean diff --git a/rs/moq-bench/Cargo.toml b/rs/moq-bench/Cargo.toml index a2802af73b7..c22691ea483 100644 --- a/rs/moq-bench/Cargo.toml +++ b/rs/moq-bench/Cargo.toml @@ -13,12 +13,20 @@ keywords = ["quic", "http3", "webtransport", "media", "live"] categories = ["multimedia", "network-programming", "web-programming"] publish = false +# `cargo run -p moq-bench` means the load generator; the sampler needs --bin. +default-run = "moq-bench" [[bin]] name = "moq-bench" path = "src/main.rs" doc = false +# Host-side resource sampler, run next to the process under test (see src/host.rs). +[[bin]] +name = "moq-bench-host" +path = "src/host.rs" +doc = false + [features] default = ["quinn", "websocket"] iroh = ["moq-native/iroh"] @@ -42,6 +50,10 @@ tokio = { workspace = true, features = ["full"] } toml = "1.1" tracing = "0.1" +# The host sampler reads /proc, so it is Linux-only (production relays are Linux). +[target.'cfg(target_os = "linux")'.dependencies] +procfs = "0.18" + [dev-dependencies] # test-util enables tokio::time::pause/advance for the deterministic produce tests. tokio = { workspace = true, features = ["test-util"] } diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index db730adf159..3888dd84724 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -18,10 +18,14 @@ For a run, `moq-bench` establishes **A** connections. Each connection: The first frame of every group is a JSON keyframe describing the rolled parameters (connection id, broadcast path, group sequence, fps, frame size, -group size, and a wall-clock timestamp). The remaining **F** frames in the group -are zeroed. **F may be 0**, in which case each group is a lone JSON keyframe, -which is useful for stressing the announce/subscribe control plane rather than -the data path. +group size, and a wall-clock timestamp), padded up to **E** bytes so the +configured frame size holds even when it is the only frame. The keyframe's own +header is the floor: an **E** below roughly 170 bytes sends the header at its +natural size, so configure **E** at 200 or above when exact sizes matter. The +remaining **F** frames in the group are zeroed. **F may be 0**, in which case +each group is a lone JSON keyframe: that is the chat shape (every message pays +a full group), and it also stresses the announce/subscribe control plane rather +than the data path. To avoid a thundering herd at startup, connections and subscriptions are staggered over a `--startup` ramp window instead of all firing at once. @@ -73,6 +77,7 @@ table (`fps = { min = 24, max = 60 }`). | `--startup` | | Ramp window for staggering connections/subscriptions | | `--duration` | | Stop after this long (runs until interrupted otherwise) | | `--report` | | How often to log throughput stats | +| `--output` | | Also append the stats as JSON lines to this file | Client TLS/QUIC flags (`--client-tls-disable-verify`, `--client-bind`, ...) come from `moq-native` and behave the same as in `moq-cli` and `moq-relay`. @@ -85,3 +90,67 @@ The `config/` directory has a few starting points: - `sd.toml`: standard-definition video with more viewers per publisher. - `audio.toml`: small, frequent frames with short groups (Opus-like). - `announce.toml`: many broadcasts, near-zero media, to stress announcements. +- `chat.toml`: a chat message bus, 1:1. Many connections, tiny frames, one + group per message, so the cost is per-message overhead rather than bandwidth. +- `chat-pub.toml` / `chat-sub.toml`: chat rooms, 1:N. A pair of configs run as + two instances against the same relay: a few busy rooms and a large + subscriber-only audience that discovers them via announcements. Scale the + audience's `--connections` (or run more instances) to push the fanout. + +## Machine-readable output + +`--output stats.jsonl` mirrors every report interval to a file as one JSON +line, each stamped with `timestamp_ms`. The frame, byte, and group counters are +cumulative and monotonic, so a consumer diffs successive lines to compute +rates, the same convention as `moq-stats` frames. `connections`, `broadcasts`, +and `subscriptions` are live gauges that fall as work ends; diff only the +counters. + +## Host-side sampling: moq-bench-host + +`moq-bench` measures the load it generates; `moq-bench-host` measures what that +load costs. It runs on the host of the process under test, production included: +it only reads `/proc` (Linux-only), needs no privileges beyond visibility of the +target process, and never touches the process itself (no ptrace, no perf, no +signals). + +```bash +# On the relay host: sample the relay once per second, forever, to stdout. +moq-bench-host --name moq-relay + +# Bounded run to a file, with the per-thread breakdown. +moq-bench-host --name moq-relay --interval 1s --duration 5m \ + --threads --output host.jsonl +``` + +Each line carries cumulative CPU seconds (`cpu_user`, `cpu_system`), RSS, and +context switches summed across threads (`ctx_voluntary`, `ctx_involuntary`; +an exited thread's contribution is retained, so the counters stay monotonic), +plus the host's total busy CPU seconds (`host_cpu_busy`) and core count so the +process's share of the machine is computable. `--threads` adds a per-thread +breakdown including the core each thread last ran on, which is how to verify +pinning once the relay grows a thread-per-core mode. + +## Measuring CPU cost + +Run the load from one machine and the sampler on the relay's host, then join +the two JSONL files on `timestamp_ms` over the same steady-state window. Skip +the first two `--startup` windows: connections ramp across the first, and each +connection then gathers announcements for one more window before subscribing. `timestamp_ms` is wall clock from two different hosts, +so keep both NTP-synced; the join only has to agree on the window boundaries, +since every rate comes from deltas within a single file: + +- CPU per connection: `(delta cpu_user + delta cpu_system) / delta seconds / connections`. +- CPU per published message: `(delta cpu_user + delta cpu_system) / delta frames_sent`. + This charges each message once, including its whole fanout, so it is the + number that answers "what does one chat message cost the relay". +- CPU per delivered copy: same numerator over `delta frames_recv`, when the + per-subscriber cost is the question. Do not sum the two frame counters: in + the 1:1 preset every message is counted by both, so the sum double-counts + and halves the apparent cost. +- Context switches per message: either shape, with `delta ctx_voluntary + delta ctx_involuntary` + as the numerator. + +Comparisons are only meaningful between runs on the same hardware with the same +preset. Pin the relay build (`--version` appears in the logs) and record it next +to the results. diff --git a/rs/moq-bench/config/chat-pub.toml b/rs/moq-bench/config/chat-pub.toml new file mode 100644 index 00000000000..5747d606563 --- /dev/null +++ b/rs/moq-bench/config/chat-pub.toml @@ -0,0 +1,21 @@ +# Chat rooms, publisher half of the 1:N fanout pair. Run this against the relay +# alongside chat-sub.toml (another process or host): this instance is the small +# set of busy rooms, the other is the audience. Both use the default broadcast +# namespace, so the subscribers discover these rooms via announcements. + +startup = "5s" +report = "5s" + +# A: a handful of rooms +connections = 10 +# B: one broadcast per room +broadcasts = 1 +# C: rooms publish only +subscribe = 0 +# D: messages per second per room +fps = "1:10" +# E: bytes per message. The floor sits above the JSON keyframe header (roughly +# 170 bytes), so every rolled size is delivered exactly. +frame_size = "200:400" +# F: each group is a lone JSON keyframe +group_size = 0 diff --git a/rs/moq-bench/config/chat-sub.toml b/rs/moq-bench/config/chat-sub.toml new file mode 100644 index 00000000000..2f3eee9ea20 --- /dev/null +++ b/rs/moq-bench/config/chat-sub.toml @@ -0,0 +1,14 @@ +# Chat rooms, audience half of the 1:N fanout pair (see chat-pub.toml). +# Subscriber-only: publishes nothing and follows a few of the announced rooms. +# Scale --connections, or run several instances, to push the fanout: 10 rooms +# with 2000 subscribers at 1 to 3 rooms each is roughly 1:400 per room. + +startup = "60s" +report = "5s" + +# A: audience size +connections = 2000 +# B: the audience publishes nothing +broadcasts = 0 +# C: rooms followed per audience member +subscribe = "1:3" diff --git a/rs/moq-bench/config/chat.toml b/rs/moq-bench/config/chat.toml new file mode 100644 index 00000000000..46afc60a62f --- /dev/null +++ b/rs/moq-bench/config/chat.toml @@ -0,0 +1,23 @@ +# Chat message bus, 1:1. Every connection is one "user": it publishes a single +# broadcast and follows one peer, so egress is tiny and the cost is per-message +# overhead, not bandwidth. Each group is a lone JSON keyframe (group_size = 0), +# so every message pays a full group/stream open, the worst case per message. + +startup = "30s" +report = "5s" + +# A: number of connections (users) +connections = 1000 +# B: one broadcast per user +broadcasts = 1 +# C: each user follows one peer +subscribe = 1 +# D: one message per second per user. Hotter than real chat on purpose: it makes +# the CPU-per-message signal easier to read, and the math normalizes by message +# count rather than wall clock anyway. +fps = 1 +# E: bytes per message. The floor sits above the JSON keyframe header (roughly +# 170 bytes), so every rolled size is delivered exactly. +frame_size = "200:400" +# F: each group is a lone JSON keyframe +group_size = 0 diff --git a/rs/moq-bench/src/config.rs b/rs/moq-bench/src/config.rs index 7c3ee51bb50..dcbc8a15202 100644 --- a/rs/moq-bench/src/config.rs +++ b/rs/moq-bench/src/config.rs @@ -67,6 +67,12 @@ pub struct Config { #[serde(default, skip_serializing_if = "Option::is_none")] pub group_size: Option, + /// Write machine-readable stats to this file: one JSON line of cumulative + /// counters per report interval. Truncates on start. + #[arg(long, env = "MOQ_BENCH_OUTPUT")] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output: Option, + /// The MoQ client (QUIC/TLS) configuration. #[command(flatten)] #[serde(default)] @@ -197,6 +203,30 @@ tls.disable_verify = true assert_eq!(config.fps(), Range::new(24, 60)); } + #[test] + fn output_survives_toml_merge() { + // The TOML->CLI merge re-applies clap defaults, which clobbers bare fields; + // `output` must stay Option so an absent flag leaves the TOML value alone. + let toml = r#" +output = "stats.jsonl" + +[client] +connect = "https://example.com" +"#; + let dir = std::env::temp_dir().join("moq-bench-output-test"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("bench.toml"); + std::fs::write(&path, toml).unwrap(); + + let args = vec![ + std::ffi::OsString::from("moq-bench"), + std::ffi::OsString::from("--file"), + path.into(), + ]; + let config = Config::parse_and_merge(args).unwrap(); + assert_eq!(config.output.as_deref(), Some(std::path::Path::new("stats.jsonl"))); + } + #[test] fn zero_report_is_rejected() { // A zero report interval would panic `tokio::time::interval`; reject it early. diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index c7d875ad221..5c5ce6e9765 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -39,6 +39,10 @@ struct GroupHeader<'a> { subscribe: u64, /// Wall-clock milliseconds, handy for rough one-way latency when clocks agree. timestamp_ms: u128, + /// Zero padding sizing the keyframe up to the rolled frame size, so a + /// lone-keyframe group (`group_size = 0`, the chat shape) still costs + /// `frame_size` bytes on the wire. + pad: String, } /// The subset of [`GroupHeader`] a subscriber reads back to learn the shape of a @@ -82,15 +86,19 @@ pub async fn run(ctx: Connection) { let publish = Origin::random().produce(); // Consume side: the session fills this with peer announcements. let consume = Origin::random().produce(); - let announced = consume.consume().announced(); let name = config.name(); + let announced = discover(&consume, name); + let mut broadcasts = Vec::new(); let mut own = HashSet::new(); let mut tasks = JoinSet::new(); for index in 0..rolled.broadcasts { - let path = format!("{name}/{run_id:08x}/{connection}/{index}"); + // The announce consumer is rooted at `name`, so `own` (compared against + // announced paths) stays relative while the full path goes on the wire. + let relative = format!("{run_id:08x}/{connection}/{index}"); + let path = format!("{name}/{relative}"); let mut broadcast = match publish.create_broadcast(&path, broadcast::Route::new().with_announce(true)) { Ok(broadcast) => broadcast, @@ -106,7 +114,7 @@ pub async fn run(ctx: Connection) { continue; } }; - own.insert(path.clone()); + own.insert(relative); // Hold the broadcast producer for the connection's lifetime so it stays announced. broadcasts.push(broadcast); @@ -195,7 +203,7 @@ async fn produce( // Keyframe: the JSON header describing this connection's rolled parameters. ticker.tick().await; - let header = GroupHeader { + let mut header = GroupHeader { connection, broadcast: &path, group: sequence, @@ -208,8 +216,19 @@ async fn produce( .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_millis(), + pad: String::new(), }; - let header = Bytes::from(serde_json::to_vec(&header)?); + let mut payload = serde_json::to_vec(&header)?; + // Pad the keyframe up to the rolled frame size, so `frame_size` holds even + // when the keyframe is the only frame (`group_size = 0`, the chat shape). + // A header already at or past the target is sent as-is. + if let Some(n) = (rolled.frame_size as usize).checked_sub(payload.len()) + && n > 0 + { + header.pad = "0".repeat(n); + payload = serde_json::to_vec(&header)?; + } + let header = Bytes::from(payload); group.write_frame(moq_net::Timestamp::now(), header.clone())?; stats.frame_sent(header.len()); @@ -225,8 +244,31 @@ async fn produce( } } -/// Watch announcements and drain up to `want` peer broadcasts (excluding our own), -/// spreading each subscription's start over `startup` to avoid a thundering herd. +/// Announce consumer scoped to the bench namespace, emitting paths relative to it. +/// +/// The relay announces its own broadcasts too (`.stats/...` when stats publishing +/// is on, which production relays enable), and a subscription slot burned on one +/// of those is never retried, so an unscoped consumer starves the subscribe side. +fn discover(consume: &moq_net::origin::Producer, name: &str) -> moq_net::announce::Consumer { + consume + .consume() + .with_root(name) + .expect("origin must permit the bench namespace") + .announced() +} + +/// Watch announcements and drain up to `want` peer broadcasts (excluding our own). +/// +/// Candidates are gathered over the `startup` window and picked at random. The +/// relay replays existing announcements in deterministic path order, so a +/// first-come pick would put every subscriber on the same first few broadcasts +/// and collapse the 1:N presets into hotspots. Announcements arriving after the +/// window fill any remaining slots in arrival order. +/// +/// The gather window is the only delay added here: connections are already +/// staggered across `startup` by main, so subscriptions land spread over the +/// second startup window of the run, and the whole swarm is subscribed within +/// two of them. async fn subscribe( mut announced: moq_net::announce::Consumer, own: HashSet, @@ -236,33 +278,51 @@ async fn subscribe( ) -> anyhow::Result<()> { let mut tasks = JoinSet::new(); let mut seen: HashSet = HashSet::new(); + let mut pool = Vec::new(); + let mut eligible = 0; - while (seen.len() as u64) < want { + // Gather candidates until the startup window closes (or the announce stream ends). + let deadline = tokio::time::sleep(startup); + tokio::pin!(deadline); + loop { + tokio::select! { + // Deadline first: with random polling, a stream that always has another + // announcement ready could keep gathering past the startup window. + biased; + _ = &mut deadline => break, + update = announced.next() => { + let Some(moq_net::announce::Update { path, broadcast }) = update else { break }; + let Some(broadcast) = broadcast else { continue }; + let path = path.as_str().to_string(); + if own.contains(&path) || !seen.insert(path.clone()) { + continue; + } + eligible += 1; + reservoir_push(&mut pool, want as usize, eligible, (path, broadcast)); + } + } + } + + let mut selected = pool.len() as u64; + for (path, broadcast) in pool { + spawn_drain(&mut tasks, path, broadcast, stats.clone()); + } + + // Top up from late announcements, first-come: the pool was too small, so + // there is nothing to spread over. + while selected < want { let Some(moq_net::announce::Update { path, broadcast }) = announced.next().await else { break; }; let Some(broadcast) = broadcast else { continue; }; - let path = path.as_str().to_string(); if own.contains(&path) || !seen.insert(path.clone()) { continue; } - - // Stagger the subscription start somewhere within the startup window. - let delay = { - let mut rng = rand::rng(); - startup.mul_f64(rng.random_range(0.0..1.0)) - }; - - let stats = stats.clone(); - tasks.spawn(async move { - tokio::time::sleep(delay).await; - if let Err(err) = drain(broadcast, &stats).await { - tracing::debug!(%path, %err, "subscription ended"); - } - }); + selected += 1; + spawn_drain(&mut tasks, path, broadcast, stats.clone()); } // Keep the drain tasks alive; they run until their broadcasts close. @@ -270,6 +330,31 @@ async fn subscribe( Ok(()) } +/// Reservoir-sample (Algorithm R): offer the `eligible`-th stream item (1-based) +/// to a pool holding at most `want` uniform picks. Bounded memory, so a big +/// namespace never piles a copy of itself into every subscriber; dropped +/// candidates release their broadcast handles immediately. +fn reservoir_push(pool: &mut Vec, want: usize, eligible: usize, item: T) { + if pool.len() < want { + pool.push(item); + return; + } + let slot = rand::rng().random_range(0..eligible); + if slot < want { + pool[slot] = item; + } +} + +/// Queue one broadcast for draining. No extra delay: the caller's gather window +/// and main's connection stagger already spread subscription starts. +fn spawn_drain(tasks: &mut JoinSet<()>, path: String, broadcast: broadcast::Consumer, stats: Arc) { + tasks.spawn(async move { + if let Err(err) = drain(broadcast, &stats).await { + tracing::debug!(%path, %err, "subscription ended"); + } + }); +} + /// Subscribe to the broadcast's track, counting every frame received and tracking /// group-sequence gaps to report skipped groups. async fn drain(broadcast: broadcast::Consumer, stats: &Stats) -> anyhow::Result<()> { @@ -474,6 +559,136 @@ mod tests { task.abort(); } + /// A lone-keyframe group (`group_size = 0`) must still cost `frame_size` + /// bytes: the keyframe is padded via its `pad` field, and stays valid JSON. + /// Without this, the chat presets' frame_size setting had no effect at all. + #[tokio::test] + async fn keyframe_padded_to_frame_size() { + tokio::time::pause(); + + let stats = Arc::new(Stats::default()); + let mut broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track(TRACK, None).unwrap(); + let consumer = broadcast.consume(); + + // 10fps, 300-byte messages, lone-keyframe groups (the chat shape). + let task = tokio::spawn(produce( + 3, + "bench/test".into(), + rolled(10, 300, 0), + track, + stats.clone(), + )); + tokio::time::advance(Duration::from_millis(250)).await; + + let mut sub = consumer.track(TRACK).unwrap().subscribe(None).await.unwrap(); + let mut group = sub.next_group().await.unwrap().expect("a group"); + let keyframe = group.read_frame().await.unwrap().expect("keyframe").payload; + + assert_eq!(keyframe.len(), 300, "keyframe padded to the rolled frame size"); + let header: serde_json::Value = serde_json::from_slice(&keyframe).expect("padded keyframe is valid JSON"); + assert_eq!(header["frame_size"], 300); + + task.abort(); + } + + /// A frame size below the JSON header's own length is a floor, not an error: + /// the keyframe goes out at its natural size and still parses. The chat + /// presets stay above the floor so their configured sizes hold exactly. + #[tokio::test] + async fn keyframe_below_header_floor_is_unpadded() { + tokio::time::pause(); + + let stats = Arc::new(Stats::default()); + let mut broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track(TRACK, None).unwrap(); + let consumer = broadcast.consume(); + + // 50 bytes is well under the serialized header (roughly 170 bytes). + let task = tokio::spawn(produce(3, "bench/test".into(), rolled(10, 50, 0), track, stats.clone())); + tokio::time::advance(Duration::from_millis(250)).await; + + let mut sub = consumer.track(TRACK).unwrap().subscribe(None).await.unwrap(); + let mut group = sub.next_group().await.unwrap().expect("a group"); + let keyframe = group.read_frame().await.unwrap().expect("keyframe").payload; + + assert!(keyframe.len() > 50, "header is the floor; no truncation to fit"); + let header: serde_json::Value = serde_json::from_slice(&keyframe).expect("unpadded keyframe is valid JSON"); + assert_eq!(header["pad"], ""); + + task.abort(); + } + + /// Discovery must skip broadcasts outside the bench namespace: a relay with + /// stats publishing enabled announces `.stats/...` too, and a subscription + /// slot burned on it is never retried, so a chat-shaped run (`subscribe = 1`) + /// used to end up with zero working subscriptions. + #[tokio::test] + async fn subscribe_ignores_relay_internal_broadcasts() { + tokio::time::pause(); + + let stats = Arc::new(Stats::default()); + let origin = Origin::random().produce(); + + // The relay-internal broadcast: announced, but with no bench data track. + let _internal = origin + .create_broadcast(".stats/node/host", broadcast::Route::new().with_announce(true)) + .unwrap(); + + // Our own broadcast: in the namespace, but excluded via the `own` set + // (paths relative to the namespace, matching the scoped announce consumer). + let _own = origin + .create_broadcast("bench/00000000/9/9", broadcast::Route::new().with_announce(true)) + .unwrap(); + let own = HashSet::from(["00000000/9/9".to_string()]); + + // One legitimate peer under the bench namespace with a single finished group. + let mut peer = origin + .create_broadcast("bench/00000000/0/0", broadcast::Route::new().with_announce(true)) + .unwrap(); + let mut track = peer.create_track(TRACK, None).unwrap(); + let mut group = track.append_group().unwrap(); + group + .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) + .unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + + let announced = discover(&origin, "bench"); + subscribe(announced, own, 1, Duration::ZERO, stats.clone()) + .await + .unwrap(); + + // The one wanted slot went to the peer, not `.stats` and not our own. + assert_eq!(stats.frames_recv.load(Ordering::Relaxed), 1); + assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 1); + } + + /// Subscription targets must be picked at random from the announced stream. + /// Announcements replay in deterministic path order, so a first-come pick + /// put every subscriber on the same first rooms and turned the 1:N presets + /// into hotspots on one or two broadcasts. + #[test] + fn reservoir_spreads_selections() { + let mut distinct = HashSet::new(); + for _ in 0..64 { + let mut pool = Vec::new(); + for item in 0..8 { + reservoir_push(&mut pool, 1, item + 1, item); + } + distinct.insert(pool[0]); + } + // First-come always yields element 0. Randomness missing this over 64 + // draws from 8 elements has probability (1/8)^63, i.e. never. + assert!(distinct.len() > 1, "selection must not be deterministic"); + + // Fewer eligible items than want: everything survives. + let mut pool = Vec::new(); + reservoir_push(&mut pool, 5, 1, 1); + reservoir_push(&mut pool, 5, 2, 2); + assert_eq!(pool, vec![1, 2]); + } + fn lost(stats: &Stats) -> u64 { stats .groups_expected diff --git a/rs/moq-bench/src/host.rs b/rs/moq-bench/src/host.rs new file mode 100644 index 00000000000..8f2acbd2e07 --- /dev/null +++ b/rs/moq-bench/src/host.rs @@ -0,0 +1,389 @@ +//! Host-side resource sampler, the other half of `moq-bench`. +//! +//! `moq-bench` drives load from one machine; this binary runs next to the process +//! under test (a relay, usually) and records what that load costs. It is safe on +//! production hosts: it only reads `/proc`, needs no privileges beyond visibility +//! of the target process, and never touches the process itself (no ptrace, no +//! perf, no signals). +//! +//! Every interval it writes one JSON line per sampled process. Counters are +//! cumulative and monotonic, mirroring the moq-stats convention: a consumer diffs +//! successive lines to compute rates, and a counter going backwards means the +//! process restarted. The context-switch counters keep that promise across thread +//! churn: an exited thread's contribution is retained rather than vanishing with +//! its `/proc` entry. Combine with the load generator's `--output` to compute CPU +//! per connection and CPU per message (see the README). + +#[cfg(target_os = "linux")] +mod linux { + use std::collections::HashMap; + use std::io::Write; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use anyhow::Context; + use clap::Parser; + use procfs::CurrentSI; + use procfs::process::Process; + use serde::Serialize; + + /// Parse a duration and reject zero: a zero interval would busy-loop over + /// /proc and flood the output, which a sampler meant to sit on production + /// hosts must never do. + fn positive_duration(arg: &str) -> anyhow::Result { + let duration = humantime::parse_duration(arg)?; + anyhow::ensure!(!duration.is_zero(), "must be greater than 0s"); + Ok(duration) + } + + /// Sample CPU, memory, and context-switch counters for a running process. + #[derive(Parser, Debug)] + #[command(version = env!("VERSION"))] + pub struct Args { + /// Sample these PIDs. When set, --name is ignored. + #[arg(long)] + pub pid: Vec, + + /// Find target processes by name (/proc//comm, 15 bytes max). + #[arg(long, default_value = "moq-relay")] + pub name: String, + + /// How often to sample. + #[arg(long, value_parser = positive_duration, default_value = "1s")] + pub interval: Duration, + + /// Stop after this duration. Runs until interrupted (or the targets exit) otherwise. + #[arg(long, value_parser = humantime::parse_duration)] + pub duration: Option, + + /// Write JSON lines to this file instead of stdout. Truncates on start. + #[arg(long)] + pub output: Option, + + /// Include a per-thread breakdown in every sample. Costs one /proc read per + /// thread per interval, so leave it off for casual monitoring. + #[arg(long)] + pub threads: bool, + } + + /// One process snapshot. All counters are cumulative since process (or system) start. + #[derive(Serialize)] + struct Sample { + /// Wall-clock milliseconds since the Unix epoch. + timestamp_ms: u128, + pid: i32, + comm: String, + /// Seconds this process spent in user mode. + cpu_user: f64, + /// Seconds this process spent in kernel mode. + cpu_system: f64, + rss_bytes: u64, + threads: u64, + /// Voluntary context switches (blocked waiting for I/O or a lock), summed + /// across all threads the sampler has seen, including exited ones. + ctx_voluntary: u64, + /// Involuntary context switches (preempted by the scheduler), summed the + /// same way. + ctx_involuntary: u64, + /// Busy (non-idle, non-iowait) seconds summed across every core on the host, + /// so the process's share of the whole machine is computable. + host_cpu_busy: f64, + host_cores: u64, + #[serde(skip_serializing_if = "Option::is_none")] + per_thread: Option>, + } + + /// One thread's slice of a [`Sample`], for spotting scheduler imbalance and + /// verifying pinning (the `processor` field is the core it last ran on). + #[derive(Serialize)] + struct ThreadSample { + tid: i32, + name: String, + cpu_user: f64, + cpu_system: f64, + processor: i32, + ctx_voluntary: u64, + ctx_involuntary: u64, + } + + /// Monotonic context-switch totals for one process. + /// + /// `/proc//task/*` counters vanish when a thread exits, so a plain + /// per-sample sum can go backwards, which would read as a process restart + /// downstream. Instead, only nonnegative per-thread deltas accumulate, and an + /// exited thread's contribution stays in the total. + #[derive(Default)] + struct CtxTotals { + /// Last observed cumulative counters per live thread. + seen: HashMap, + voluntary: u64, + involuntary: u64, + } + + impl CtxTotals { + /// Fold one thread's current cumulative counters into the running totals. + fn observe(&mut self, tid: i32, voluntary: u64, involuntary: u64) { + let (prev_voluntary, prev_involuntary) = self.seen.get(&tid).copied().unwrap_or((0, 0)); + self.voluntary += voluntary.saturating_sub(prev_voluntary); + self.involuntary += involuntary.saturating_sub(prev_involuntary); + self.seen.insert(tid, (voluntary, involuntary)); + } + + /// Forget threads that no longer exist, so a reused tid starts a fresh + /// baseline instead of diffing against a dead thread's counters. + fn retain(&mut self, live: impl Fn(i32) -> bool) { + self.seen.retain(|tid, _| live(*tid)); + } + } + + /// One process under observation: the /proc handle plus the sampler-side + /// state that outlives individual threads. + struct Target { + proc: Process, + ctx: CtxTotals, + } + + /// Resolve the target processes from --pid or --name. + fn find(args: &Args) -> anyhow::Result> { + if !args.pid.is_empty() { + return args + .pid + .iter() + .map(|&pid| { + let proc = Process::new(pid).with_context(|| format!("pid {pid}"))?; + Ok(Target { + proc, + ctx: CtxTotals::default(), + }) + }) + .collect(); + } + + // A process can exit between the directory listing and the stat read, so + // per-process errors here mean "gone", not failure. + let mut found = Vec::new(); + for proc in procfs::process::all_processes()? { + let Ok(proc) = proc else { continue }; + let Ok(stat) = proc.stat() else { continue }; + if stat.comm == args.name { + found.push(Target { + proc, + ctx: CtxTotals::default(), + }); + } + } + Ok(found) + } + + /// Cumulative busy seconds across all cores, and the core count. + /// + /// `guest`/`guest_nice` are excluded: the kernel already accounts guest time + /// inside `user`/`nice`, so adding them again would double-count it. + fn host_cpu(tps: f64) -> anyhow::Result<(f64, u64)> { + let kernel = procfs::KernelStats::current()?; + let t = &kernel.total; + let busy = t.user + t.nice + t.system + t.irq.unwrap_or(0) + t.softirq.unwrap_or(0) + t.steal.unwrap_or(0); + Ok((busy as f64 / tps, kernel.cpu_time.len() as u64)) + } + + /// Snapshot one process; `Err` means it exited (or /proc denied us) and should + /// be dropped. Everything read here is per-target, so an error only ever + /// condemns this target; host-wide reads live in the caller and fail the run. + fn sample(target: &mut Target, tps: f64, page: u64, threads: bool, host: (f64, u64)) -> anyhow::Result { + let stat = target.proc.stat()?; + let (host_cpu_busy, host_cores) = host; + + // /proc//status reports the context-switch counters of the thread group + // leader alone (utime/stime in stat are the aggregated ones), so the + // process-wide numbers have to be folded over /proc//task/*. + let mut live = Vec::new(); + let mut per_thread = threads.then(Vec::new); + for task in target.proc.tasks()? { + // Threads come and go mid-iteration; skip the ones that vanished. + let Ok(task) = task else { continue }; + let Ok(tstatus) = task.status() else { continue }; + let voluntary = tstatus.voluntary_ctxt_switches.unwrap_or(0); + let involuntary = tstatus.nonvoluntary_ctxt_switches.unwrap_or(0); + target.ctx.observe(task.tid, voluntary, involuntary); + live.push(task.tid); + + if let Some(out) = &mut per_thread { + let Ok(tstat) = task.stat() else { continue }; + out.push(ThreadSample { + tid: task.tid, + name: tstat.comm, + cpu_user: tstat.utime as f64 / tps, + cpu_system: tstat.stime as f64 / tps, + processor: tstat.processor.unwrap_or(-1), + ctx_voluntary: voluntary, + ctx_involuntary: involuntary, + }); + } + } + target.ctx.retain(|tid| live.contains(&tid)); + + Ok(Sample { + timestamp_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + pid: target.proc.pid, + comm: stat.comm.clone(), + cpu_user: stat.utime as f64 / tps, + cpu_system: stat.stime as f64 / tps, + rss_bytes: stat.rss * page, + threads: stat.num_threads.max(0) as u64, + ctx_voluntary: target.ctx.voluntary, + ctx_involuntary: target.ctx.involuntary, + host_cpu_busy, + host_cores, + per_thread, + }) + } + + /// Resolve the targets, then sample them every interval until the duration + /// elapses, the process is interrupted, or every target exits (an error). + pub fn run() -> anyhow::Result<()> { + let args = Args::parse(); + let tps = procfs::ticks_per_second() as f64; + let page = procfs::page_size(); + + let mut out: Box = match &args.output { + Some(path) => Box::new(std::fs::File::create(path)?), + None => Box::new(std::io::stdout().lock()), + }; + + let mut targets = find(&args)?; + anyhow::ensure!( + !targets.is_empty(), + "no process matched (name = {:?}); pass --pid or --name", + args.name + ); + eprintln!( + "sampling {} process(es) every {}: {}", + targets.len(), + humantime::format_duration(args.interval), + targets + .iter() + .map(|t| t.proc.pid.to_string()) + .collect::>() + .join(", ") + ); + + let start = std::time::Instant::now(); + loop { + // Host-wide once per round: unlike the per-target reads inside `sample`, + // a failure here is not a target exiting, so it fails the run. + let host = host_cpu(tps).context("failed to read /proc/stat")?; + + // Drop targets that exited; sampling the survivors is still useful. + targets.retain_mut(|target| match sample(target, tps, page, args.threads, host) { + Ok(record) => { + // A serialization failure is a bug, not a runtime condition. + let line = serde_json::to_string(&record).expect("sample must serialize"); + if let Err(err) = writeln!(out, "{line}") { + eprintln!("write failed: {err}"); + std::process::exit(1); + } + true + } + Err(_) => { + eprintln!("pid {} exited, dropping", target.proc.pid); + false + } + }); + // Every line lands on disk before the sleep, so a Ctrl-C loses nothing. + out.flush()?; + + anyhow::ensure!(!targets.is_empty(), "all target processes exited"); + let Some(delay) = next_delay(args.interval, args.duration, start.elapsed()) else { + return Ok(()); + }; + std::thread::sleep(delay); + } + } + + /// How long to sleep before the next sample: the interval, capped at the time + /// left in `duration`. `None` means the duration has elapsed and the run is + /// done, so a `--duration` shorter than `--interval` cannot overshoot. + fn next_delay(interval: Duration, duration: Option, elapsed: Duration) -> Option { + let Some(duration) = duration else { + return Some(interval); + }; + let remaining = duration.checked_sub(elapsed)?; + if remaining.is_zero() { + return None; + } + Some(interval.min(remaining)) + } + + #[cfg(test)] + mod tests { + use super::*; + + /// `--interval 0s` must be rejected at parse time: with no pacing the + /// sampler busy-loops over /proc on the very hosts it is meant to be + /// harmless on. + #[test] + fn zero_interval_rejected() { + let err = Args::try_parse_from(["moq-bench-host", "--interval", "0s"]).unwrap_err(); + assert!(err.to_string().contains("greater than 0s"), "unexpected error: {err}"); + + // Nonzero still parses, and the default holds. + let args = Args::try_parse_from(["moq-bench-host"]).unwrap(); + assert_eq!(args.interval, Duration::from_secs(1)); + } + + /// A duration shorter than the interval must cap the sleep, not stretch the + /// run to the full interval (10ms of sampling used to take a whole second). + #[test] + fn short_duration_caps_the_sleep() { + let second = Duration::from_secs(1); + let ms = Duration::from_millis(10); + + // No duration: always the full interval. + assert_eq!(next_delay(second, None, second * 100), Some(second)); + // Time remains: sleep the smaller of interval and what's left. + assert_eq!(next_delay(second, Some(ms), Duration::ZERO), Some(ms)); + assert_eq!(next_delay(ms, Some(second), Duration::ZERO), Some(ms)); + // Elapsed at or past the duration: done. + assert_eq!(next_delay(second, Some(ms), ms), None); + assert_eq!(next_delay(second, Some(ms), second), None); + } + + /// The process-wide context-switch totals must stay monotonic when a + /// thread exits between samples. A naive per-sample sum would drop the + /// dead thread's counters, go backwards, and read as a process restart + /// downstream. + #[test] + fn ctx_totals_survive_thread_exit() { + let mut ctx = CtxTotals::default(); + + // Sample 1: two live threads. + ctx.observe(1, 10, 1); + ctx.observe(2, 5, 2); + ctx.retain(|tid| [1, 2].contains(&tid)); + assert_eq!((ctx.voluntary, ctx.involuntary), (15, 3)); + + // Sample 2: thread 2 exited; thread 1 advanced. Thread 2's counts stay. + ctx.observe(1, 12, 1); + ctx.retain(|tid| tid == 1); + assert_eq!((ctx.voluntary, ctx.involuntary), (17, 3)); + + // Sample 3: tid 2 is reused by a fresh thread; its counters start over + // and must add in full rather than diff against the dead thread's. + ctx.observe(1, 12, 1); + ctx.observe(2, 3, 1); + assert_eq!((ctx.voluntary, ctx.involuntary), (20, 4)); + } + } +} + +#[cfg(target_os = "linux")] +fn main() -> anyhow::Result<()> { + linux::run() +} + +#[cfg(not(target_os = "linux"))] +fn main() -> anyhow::Result<()> { + anyhow::bail!("moq-bench-host reads /proc, so it only runs on Linux") +} diff --git a/rs/moq-bench/src/main.rs b/rs/moq-bench/src/main.rs index 77c9e71d561..80a468efd40 100644 --- a/rs/moq-bench/src/main.rs +++ b/rs/moq-bench/src/main.rs @@ -31,12 +31,14 @@ async fn main() -> anyhow::Result<()> { let client = config.client.clone().init()?; let stats = Arc::new(Stats::default()); - // Periodic throughput reporter. - { + // Periodic throughput reporter, optionally mirrored to a JSONL file. Keep the + // handle: a failed stats write must fail the whole run (see Stats::report). + let mut reporter = { let stats = stats.clone(); let interval = config.report(); - tokio::spawn(async move { stats.report(interval).await }); - } + let output = config.output.as_ref().map(std::fs::File::create).transpose()?; + tokio::spawn(async move { stats.report(interval, output).await }) + }; // Roll the per-connection parameters up front: `ThreadRng` is not `Send`, so it // can't cross the spawn boundary. @@ -92,6 +94,9 @@ async fn main() -> anyhow::Result<()> { _ = stop => tracing::info!("duration elapsed, stopping"), _ = tokio::signal::ctrl_c() => tracing::info!("interrupted, stopping"), _ = drained => tracing::warn!("all connections ended"), + // The reporter only returns on a stats-output failure; die loudly rather + // than exit green with a partial JSONL file. + res = &mut reporter => res??, } Ok(()) diff --git a/rs/moq-bench/src/stats.rs b/rs/moq-bench/src/stats.rs index 91977891294..5a05caaeb18 100644 --- a/rs/moq-bench/src/stats.rs +++ b/rs/moq-bench/src/stats.rs @@ -1,5 +1,9 @@ +use std::io::Write; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::Context; +use serde::Serialize; /// Shared counters bumped by the connection tasks and drained by the reporter. #[derive(Default)] @@ -32,7 +36,13 @@ impl Stats { } /// Periodically log totals plus the throughput since the previous report. - pub async fn report(&self, interval: Duration) { + /// + /// With an `output` file, each report also appends one JSON line of the + /// cumulative counters, timestamped so it can be joined against the host + /// sampler's records (see `moq-bench-host`). Returns only on a failed write: + /// a benchmark whose recorded stats are partial is invalid, so the caller + /// must fail the run rather than exit green. + pub async fn report(&self, interval: Duration, mut output: Option) -> anyhow::Result<()> { let mut ticker = tokio::time::interval(interval); // Skip the immediate first tick so the first report covers a full interval. ticker.tick().await; @@ -41,6 +51,21 @@ impl Stats { loop { ticker.tick().await; let now = Snapshot::take(self); + + if let Some(file) = &mut output { + let record = Record { + timestamp_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + snapshot: &now, + }; + // A serialization failure is a bug, not a runtime condition. + let line = serde_json::to_string(&record).expect("stats must serialize"); + writeln!(file, "{line}") + .and_then(|_| file.flush()) + .context("failed to write stats output")?; + } let secs = interval.as_secs_f64().max(f64::MIN_POSITIVE); let send_mbps = (now.bytes_sent.saturating_sub(prev.bytes_sent) as f64 * 8.0) / secs / 1e6; @@ -75,6 +100,18 @@ impl Stats { } } +/// One machine-readable stats line: a timestamp plus the cumulative counters. +/// Cumulative and monotonic like moq-stats frames: consumers diff successive +/// lines to compute rates. +#[derive(Serialize)] +struct Record<'a> { + /// Wall-clock milliseconds since the Unix epoch. + timestamp_ms: u128, + #[serde(flatten)] + snapshot: &'a Snapshot, +} + +#[derive(Serialize)] struct Snapshot { connections: u64, broadcasts: u64, @@ -104,3 +141,35 @@ impl Snapshot { } } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + /// A failed stats write must surface as an error so the run dies loudly. + /// Silently dropping output leaves a partial JSONL file behind a green exit, + /// which reads as a valid benchmark that quietly lost data. + #[tokio::test] + async fn report_fails_on_output_error() { + tokio::time::pause(); + + let dir = std::env::temp_dir().join("moq-bench-stats-test"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("out.jsonl"); + std::fs::write(&path, b"").unwrap(); + // A read-only handle: the first write fails. + let file = std::fs::File::open(&path).unwrap(); + + let stats = Arc::new(Stats::default()); + let task = tokio::spawn({ + let stats = stats.clone(); + async move { stats.report(Duration::from_secs(1), Some(file)).await } + }); + + tokio::time::advance(Duration::from_secs(3)).await; + let result = task.await.unwrap(); + assert!(result.is_err(), "report must surface the output failure"); + } +} From c40afc40cf69d9f29d4bdeba964d7675c4719a4c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 15 Aug 2026 17:09:19 -0700 Subject: [PATCH 03/29] feat(obs): compile the plugin in CI, and type-check it without obs-deps (#2867) Co-authored-by: Claude Opus 5 --- .github/workflows/nightly.yml | 16 ++- .github/workflows/obs.yml | 100 +++++++++++++++++ CLAUDE.md | 6 +- cpp/obs/justfile | 201 ++++++++++++++++++++++++++++------ doc/bin/obs.md | 25 ++++- flake.nix | 67 ++++++++++-- justfile | 25 +++-- 7 files changed, 385 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/obs.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a7ddd5569be..174b8ca9cf5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,7 +7,10 @@ name: Nightly # stays silent on the days nobody pushes. `features` is diff-independent for a # different reason: it compiles the workspace at feature sets that share almost # no artifacts with the default one, which is several minutes of duplicate work -# that a PR gate can't justify for how rarely it catches something. +# that a PR gate can't justify for how rarely it catches something. `obs` is +# diff-independent because its PR-time counterpart can't be: obs.yml has to +# name the paths that can break an external link to libmoq.a, and no list of +# paths covers every one of them (see the comment there). # # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. @@ -60,3 +63,14 @@ jobs: - name: Audit if: ${{ !cancelled() }} run: nix develop --command just rs audit + + # The OBS plugin is the only thing that links libmoq.a from outside cargo, + # so a stale `rs/libmoq/native-libs/*.txt` breaks it and nothing else. + # obs.yml lists the inputs that can do that, but a list of paths can't be + # complete: a build script emitting `cargo::rustc-link-lib` needs no + # manifest change, and widening that filter to `rs/**` would run a link on + # nearly every PR. Being diff-independent, this run covers whatever the + # filter misses, at the cost of surfacing a day late rather than in review. + - name: OBS + if: ${{ !cancelled() }} + run: nix develop --command just obs ci diff --git a/.github/workflows/obs.yml b/.github/workflows/obs.yml new file mode 100644 index 00000000000..06a8c23aa05 --- /dev/null +++ b/.github/workflows/obs.yml @@ -0,0 +1,100 @@ +name: OBS + +# Compiles and links the OBS plugin, which nothing else does: `just check` +# lints cpp/obs but never builds it, and a local build needs the +# multi-hundred-MB obs-deps bundle on macOS and Windows. +# +# Linux is the platform that needs no bundle at all -- nixpkgs has obs-studio, +# Qt6 and ffmpeg, so the dev shell is the whole dependency set. The plugin is +# platform-independent C++ over libmoq's C ABI, so a Linux compile catches the +# breakage a macOS developer would otherwise ship uncompiled. +# +# Narrow trigger, like swift.yml: the plugin's own sources, the libmoq header +# they call through, and the flake that supplies the toolchain. +# +# The manifests and Cargo.lock are in there for the one way a crate the plugin +# never names can break it: linking libmoq.a from outside cargo needs the native +# libraries in `rs/libmoq/native-libs/*.txt`, which are maintained by hand. A +# dependency that starts pulling in a new one leaves that list stale, and every +# Rust gate stays green, because cargo passes the flag itself and only this job +# links without it. +# +# Manifests and build scripts rather than Cargo.lock alone, because neither a +# feature toggle nor a build script has to touch the lockfile. libmoq takes +# moq-native with default features, so flipping one of those defaults on +# (`jemalloc` gates `dep:tikv-jemallocator`) changes what libmoq.a needs at link +# time while Cargo.lock stays byte-identical, since moq-relay already put that +# package in the graph; a build.rs emitting `cargo::rustc-link-lib` needs no +# manifest change at all. +# +# A list of paths still can't be complete here -- a vendored C source or a +# .cargo/config.toml could do it too -- and `rs/**` would run this on nearly +# every PR, which is the cost that removed the Windows and macOS gates (#2600). +# So this filter covers the likely inputs at PR time and nightly.yml runs the +# same recipe diff-independently to cover the rest. + +permissions: + contents: read + +on: + pull_request: + # `closed` is here only so merging/closing a PR cancels its in-flight run + # via the concurrency group below; the job itself is skipped on close. + types: [opened, synchronize, reopened, closed] + paths: + - "cpp/obs/**" + - "rs/libmoq/**" + - "rs/**/Cargo.toml" + - "rs/**/build.rs" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "flake.nix" + - "flake.lock" + - ".github/workflows/obs.yml" + +concurrency: + group: obs-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: OBS + if: github.event.action != 'closed' + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main + with: + tool-cache: false + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, so the dev shell substitutes + # instead of building from source. See check.yml for the full story. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + # Restore only. cache.yml is the single writer, on `main`; see the comment + # there for why. `shared-key` has to match the one it saves under. It + # holds dev-profile artifacts, which is why `just obs ci` builds libmoq in + # debug rather than the preset's release. + - name: Rust Cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + shared-key: rust + save-if: false + + # The same recipe a developer runs on Linux. Warnings are errors here, via + # the platform's CI preset. + - name: Build + run: nix develop --command just obs ci diff --git a/CLAUDE.md b/CLAUDE.md index 1082226e7b4..2a9b2778461 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,9 @@ The Rust build cache is written by `main` and read by pull requests, never the o One tool stays unrequired: `swift` exists only on macOS, so swift.yml is its gate rather than the PR path. -Two gates live outside the PR path, in `.github/workflows/nightly.yml`: `just rs audit` (cargo-deny) because an advisory lands without this repo changing, and `just rs features` (the `--all-features` and `--no-default-features` compiles) because each is a full extra workspace compile that shares almost nothing with the default one. A break there lands on `main` rather than being caught in review, which is the accepted trade; anything that must block a merge belongs in `check`. +Two path-filtered workflows run recipes of their own alongside `check.yml`, each for something the Linux `just check` can't reach: swift.yml (`swift/scripts/check.sh`, on a Mac) and obs.yml (`just obs ci`, which links the OBS plugin against nixpkgs' libobs/Qt6 -- Linux is the only platform where that needs no obs-deps download). + +Three gates live outside the PR path, in `.github/workflows/nightly.yml`: `just rs audit` (cargo-deny) because an advisory lands without this repo changing, `just rs features` (the `--all-features` and `--no-default-features` compiles) because each is a full extra workspace compile that shares almost nothing with the default one, and `just obs ci` because obs.yml's path filter can't be complete (a build script can change what linking `libmoq.a` needs without touching a manifest). A break there lands on `main` rather than being caught in review, which is the accepted trade; anything that must block a merge belongs in `check`. ## Architecture @@ -60,7 +62,7 @@ Top-level layout only. Per-crate and per-package detail lives in the nested guid - `/rs/` - Rust crates: core networking (`moq-net`), native helpers, the relay, CLIs, media muxing/codecs, and the FFI/C bindings. See `rs/CLAUDE.md`. - `/js/` - TypeScript/JavaScript packages for the browser, published as `@moq/*`. See `js/CLAUDE.md`. - `/py/`, `/swift/`, `/kt/`, `/go/` - language wrappers over `rs/moq-ffi` (see [Language Bindings](#language-bindings)). `/py/` has `py/CLAUDE.md`; the others defer to their `README.md`. -- `/cpp/` - C/C++ consumers of `libmoq`. `cpp/obs/` is the OBS Studio plugin (CMake; links `libmoq` via `MOQ_LOCAL`), licensed GPL-2.0-or-later because it links `libobs`. PR CI never compiles it, so `just obs build` and `just obs test` (the latter runs `cpp/obs/test/` against stubbed libobs/libmoq under ThreadSanitizer) are manual gates, like `just rs macos`. See `doc/bin/obs.md`. +- `/cpp/` - C/C++ consumers of `libmoq`. `cpp/obs/` is the OBS Studio plugin (CMake; links `libmoq` via `MOQ_LOCAL`), licensed GPL-2.0-or-later because it links `libobs`. `just check` type-checks it via `just obs compile`, which needs headers rather than an obs-deps download, and obs.yml links it on Linux for `cpp/obs/` and `rs/libmoq/` PRs. Still manual, like `just rs macos`: `just obs build` (a loadable plugin, and the only path on macOS/Windows) and `just obs test` (`cpp/obs/test/` against stubbed libobs/libmoq under ThreadSanitizer). See `doc/bin/obs.md`. - `/demo/` - demos and test media: relay configs, the web demo, MoQ Boy, media hosting, and a network throttle script. - `/test/` - cross-language interop smoke tests (`test/smoke/`), run via `just test smoke[-full]`. - `/doc/` - documentation site (VitePress, deployed via Cloudflare). The `/draft/` section is generated from `drafts/` by `doc/.vitepress/drafts.ts`. diff --git a/cpp/obs/justfile b/cpp/obs/justfile index 7e36ca200f2..a6b7957ece1 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -1,6 +1,9 @@ #!/usr/bin/env just --justfile # OBS Studio plugin for MoQ. See doc/bin/obs.md for the full build guide. # +# `just obs compile` type-checks the sources on any platform with nothing to +# download. The recipes below build a loadable plugin, which is a bigger ask: +# # Linux: `nix develop` provides obs-studio/qt6/ffmpeg/cmake, then `just obs build`. # macOS: native, NOT nix. Needs full Xcode; run outside `nix develop` (its # toolchain breaks the Xcode build). `just obs setup` downloads @@ -58,39 +61,97 @@ run: cp -a build_macos/RelWithDebInfo/obs-moq.plugin "$dest/" RUST_LOG=debug RUST_BACKTRACE=1 OBS_LOG_LEVEL=debug /Applications/OBS.app/Contents/MacOS/OBS -# Unit-test the plugin sources against stubbed libobs/libmoq, under ThreadSanitizer. +# Type-check every plugin source, without linking or an obs-deps download. # -# Manual, like `just rs macos` and `just rs windows`: PR CI never compiles this -# plugin, and the libobs headers come from the multi-hundred-MB obs-deps download -# that `just obs setup` performs. Run it whenever you touch src/, especially the -# session callback plumbing, whose orderings the build can't check. -test: +# This is the gate to run in a worktree. `just obs build` needs the +# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per +# worktree; this needs neither, because the dev shell carries all three header +# sets: libobs pinned to the same OBS release buildspec.json downloads (see +# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux +# (.github/workflows/obs.yml). +# +# It compiles the Qt sources too, which the CMake build only does when +# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are +# set here to match. +compile: #!/usr/bin/env bash set -euo pipefail - # Unlike the lint recipes, this one never skips: a test gate that reports - # success without running anything is worse than no gate, and nothing else - # covers this code. + # Never skips, for the reason `test` doesn't: a compile gate that reports + # success without compiling is worse than no gate. Everything it needs is in + # the dev shell, so the fix is always to enter it. cxx="${CXX:-c++}" - unsupported="'$cxx' cannot build with -fsanitize=thread. These tests need a Clang or GCC - compiler that supports ThreadSanitizer: set CXX, or on Windows run them from WSL, since - neither MSVC nor Clang on Windows implements ThreadSanitizer." if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'." >&2 - echo "$unsupported" >&2 - exit 1 - fi - if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o /dev/null - >/dev/null 2>&1; then - echo "$unsupported" >&2 + echo "no C++ compiler at '$cxx'" >&2 exit 1 fi + for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do + # shellcheck disable=SC2086 + if ! pkg-config --exists $pkgs; then + echo "missing headers for: $pkgs" >&2 + echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 + exit 1 + fi + done - # libobs headers: the obs-deps framework (macOS/Windows), the OBS sources - # CMake unpacks beside it, or a system install (Linux/nix). + # Assign first so a failure in `_includes` still aborts under `set -e`, + # then split on newlines only: an include path may contain spaces. + includes_raw=$(just _includes) + includes=() + while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" + + qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) + ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) + + # MOQ_VERSION_STRING only reaches a label in the dock, so any value + # type-checks the same; CMake stamps the real libmoq version. + status=0 + for source in src/*.cpp; do + # shellcheck disable=SC2086 + # -Wno-unused-command-line-argument: the nix compiler wrapper injects + # link flags, which a syntax-only run reports as unused, once per file. + # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ + "${includes[@]}" $qt $ffmpeg \ + -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then + status=1 + fi + done + exit $status + +# Compile and link the plugin the way CI does, against the dev shell's own +# libobs/Qt6/ffmpeg. Linux-only in practice: it's the one platform where nixpkgs +# has obs-studio, so nothing has to be downloaded. Manual elsewhere, like +# `just rs macos`. +# +# CMAKE_BUILD_TYPE overrides the preset's RelWithDebInfo because the Rust cache +# CI restores holds dev-profile artifacts. A release build of libmoq would +# rebuild its whole dependency tree on every run, and the gate is about whether +# the plugin compiles, which the profile doesn't change. +ci preset="": + #!/usr/bin/env bash + set -euo pipefail + PRESET=$(just preset "{{ preset }}" ci) + cmake --preset "$PRESET" -DCMAKE_BUILD_TYPE=Debug + cmake --build --preset "$PRESET" + +# Print the -I flags for libobs and libmoq, shared by `compile` and `test`. +# +# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than +# just locating it: a stale header is how a plugin call to a since-changed +# libmoq function type-checks anyway. +[private] +_includes: + #!/usr/bin/env bash + set -euo pipefail + # The dev shell's pinned headers first, so every platform checks against the + # same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources + # CMake unpacks beside it, and a system install (Linux without nix). obs_include="" for candidate in \ + "${OBS_INCLUDE_DIR:-}" \ .deps/Frameworks/libobs.framework/Versions/A/Headers \ - .deps/obs-studio-*/libobs \ - "${OBS_INCLUDE_DIR:-}"; do + .deps/obs-studio-*/libobs; do if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then obs_include="$candidate" break @@ -100,22 +161,78 @@ test: obs_include=$(pkg-config --variable=includedir libobs)/obs fi if [ -z "$obs_include" ]; then - echo "libobs headers not found; run 'just obs setup' (or set OBS_INCLUDE_DIR)" >&2 + echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 exit 1 fi - # moq.h is generated by cbindgen when libmoq builds. - moq_include="../../target/include" + (cd ../.. && cargo check --quiet -p libmoq) + + # Ask cargo where its build script ran rather than reconstructing the path. + # CARGO_TARGET_DIR, a `--target` triple and a custom profile each move the + # generated header, and guessing wrong doesn't fail: it type-checks against + # whatever stale copy the default directory still holds. Replays the check + # above from cache, so it costs a process rather than a compile. + out_dir=$(cd ../.. && cargo check -p libmoq --message-format=json | + jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | + tail -1) + if [ -z "$out_dir" ]; then + echo "cargo reported no build script output for libmoq" >&2 + exit 1 + fi + # //build/libmoq-/out, and the header sits beside the + # profile directory. rs/libmoq/build.rs derives it from OUT_DIR the same way. + moq_include="$(dirname "$(dirname "$(dirname "$(dirname "$out_dir")")")")/include" if [ ! -f "$moq_include/moq.h" ]; then - echo "$moq_include/moq.h missing; run 'just obs build' or 'cargo build -p libmoq' first" >&2 + echo "$moq_include/moq.h missing after 'cargo check -p libmoq'" >&2 + exit 1 + fi + + # One flag per line, path glued to the -I, so a caller can rebuild the + # arguments without splitting on the spaces in a path. + printf -- '-I%s\n' src "$obs_include" "$moq_include" + +# Unit-test the plugin sources against stubbed libobs/libmoq, under ThreadSanitizer. +# +# Manual, like `just rs macos` and `just rs windows`: PR CI links the plugin but +# doesn't run this, since ThreadSanitizer needs its own build. Run it whenever +# you touch src/, especially the session callback plumbing, whose orderings the +# build can't check. +test: + #!/usr/bin/env bash + set -euo pipefail + # Unlike the lint recipes, this one never skips: a test gate that reports + # success without running anything is worse than no gate, and nothing else + # covers this code. + cxx="${CXX:-c++}" + unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a + Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run + them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer." + if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'." >&2 + echo "$unsupported" >&2 exit 1 fi out=$(mktemp -d) trap 'rm -rf "$out"' EXIT - echo "Building tests against $obs_include" - "$cxx" -std=c++17 -g -O0 -fsanitize=thread \ - -I "$obs_include" -I src -I "$moq_include" \ + + # Run the probe, don't just link it: a mismatch between the compiler's TSan + # runtime and the host kernel (macOS is where this shows up) links fine and + # then segfaults before main, which is an unreadable way for the real test + # to fail. + # The subshell keeps bash's own "Segmentation fault" job message out of the + # output, so the explanation below is all the reader gets. + if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$out/probe" - >/dev/null 2>&1 \ + || ! ("$out/probe" >/dev/null 2>&1) 2>/dev/null; then + echo "$unsupported" >&2 + exit 1 + fi + + includes_raw=$(just _includes) + includes=() + while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" + + "$cxx" -std=c++17 -g -O0 -fsanitize=thread "${includes[@]}" \ -o "$out/moq-output-test" test/moq-output-test.cpp src/moq-output.cpp TSAN_OPTIONS="halt_on_error=1" "$out/moq-output-test" @@ -127,6 +244,22 @@ check: echo "CMakePresets.json must not pin MOQ_VERSION" >&2 exit 1 fi + # The headers `just obs compile` type-checks against are a second copy of + # the OBS version this spec downloads, so a bump here has to reach flake.nix + # or the two gates check different libobs. + spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) + flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') + # Both are pattern matches against files this recipe doesn't own the shape + # of. An empty parse compares equal to an empty parse, so the guard would + # pass by reporting nothing rather than by finding the versions in sync. + if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then + echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 + exit 1 + fi + if [ "$spec_obs" != "$flake_obs" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 + exit 1 + fi if command -v cmake >/dev/null; then build_dir=$(mktemp -d) trap 'rm -rf "$build_dir"' EXIT @@ -158,17 +291,19 @@ fix: fi # Detect the CMake preset for the current platform (or use the override). -preset override="": +# Pass `ci` as the second argument for the warnings-as-errors variant, whose +# name upstream spells differently on each platform. +preset override="" variant="": #!/usr/bin/env bash set -euo pipefail if [[ -n "{{ override }}" ]]; then echo "{{ override }}" elif [[ "$OSTYPE" == "darwin"* ]]; then - echo "macos" + [[ "{{ variant }}" == "ci" ]] && echo "macos-ci" || echo "macos" elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - echo "ubuntu-x86_64" + [[ "{{ variant }}" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then - echo "windows-x64" + [[ "{{ variant }}" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" else echo "Unknown platform: $OSTYPE" >&2 exit 1 diff --git a/doc/bin/obs.md b/doc/bin/obs.md index 7d5b26ada5e..83f32988402 100644 --- a/doc/bin/obs.md +++ b/doc/bin/obs.md @@ -24,6 +24,8 @@ It loads into a stock OBS Studio install. You no longer need to build OBS from s The plugin lives in-tree under `cpp/obs/`. It links `libmoq`, which is built from the in-tree `rs/libmoq` crate via cargo (CMake's `MOQ_LOCAL` points at the repo root by default), so there is no prebuilt release to download. +Build it when you want to *run* it. To check that a change compiles, reach for [`just obs compile`](#type-checking) instead: on macOS and Windows a real build first downloads the multi-hundred-MB obs-deps bundle into the tree it's building in, which is per-worktree. + ### Linux (Nix) `libobs`, `Qt6`, and `ffmpeg` come from the dev shell; no system packages required. @@ -57,13 +59,32 @@ just obs setup just obs build ``` +### Type-checking + +`just obs compile` type-checks every plugin source without linking, and without downloading anything: + +```bash +nix develop +just obs compile +``` + +This is the gate to run while working, because it needs headers rather than libraries, and the dev shell carries all of them on every platform. `libobs` comes from the `libobs-headers` package in `flake.nix`, which unpacks the headers from the same OBS release `buildspec.json` pins (`just obs check` fails if the two versions drift); `Qt6` and `ffmpeg` come from nixpkgs. It regenerates `target/include/moq.h` first, so a call to a `libmoq` function whose signature has since changed is a compile error rather than something you find out about later. + +It compiles the Qt sources too, which the CMake build only does when `ENABLE_QT` and `ENABLE_FRONTEND_API` are on. `just check` runs it for you when a branch touches `cpp/obs/` or `rs/libmoq/`. + +### Compiling in CI + +[`obs.yml`](https://github.com/moq-dev/moq/blob/main/.github/workflows/obs.yml) compiles **and links** the plugin on every PR that touches the plugin, `rs/libmoq/`, a workspace manifest or build script, or the flake. It runs on Linux, the one platform where the whole dependency set (`libobs`, `Qt6`, `ffmpeg`) comes from nixpkgs with no obs-deps bundle to download. The plugin is platform-independent C++ over libmoq's C ABI, so this catches what a macOS developer would otherwise ship uncompiled. `just obs ci` is the same recipe locally. + +The filter reaches past `cpp/obs/` because this is the only place `libmoq.a` is linked from outside cargo, which needs the hand-maintained native-library lists in `rs/libmoq/native-libs/`. A dependency that starts pulling in a new native library leaves those stale, and every Rust gate stays green because cargo passes the flag itself. No list of paths catches all of those, so [`nightly.yml`](https://github.com/moq-dev/moq/blob/main/.github/workflows/nightly.yml) runs the same recipe diff-independently as the backstop. + ### Tests `just obs test` compiles the plugin sources against stubbed `libobs`/`libmoq` under ThreadSanitizer and drives the session status callback's orderings directly: a connection that fails permanently, a terminal arriving mid-`Start()`, a restart, and one arriving while the output is being destroyed. Run it after touching `cpp/obs/src/`. -It needs the `libobs` headers plus a built `libmoq` (`target/include/moq.h`). On macOS and Windows the headers come from the obs-deps bundle `just obs setup` downloads; on Linux they come from the Nix dev shell via `pkg-config`. Set `OBS_INCLUDE_DIR` to point it somewhere else. Like `just rs macos` and `just rs windows`, it is a manual gate: PR CI never compiles this plugin. +It finds the `libobs` headers the same way `just obs compile` does, and regenerates `moq.h` the same way; set `OBS_INCLUDE_DIR` to point it somewhere else. That shared step asks cargo where the header landed and reads the answer with `jq`, so outside the dev shell (running from WSL, say) `jq` has to be installed alongside cargo and the compiler. It stays a manual gate, like `just rs macos`: CI links the plugin but doesn't run these, since ThreadSanitizer needs its own build. -It also needs a Clang or GCC compiler that supports ThreadSanitizer, so it fails rather than skipping when one isn't available. Linux and macOS are covered by the toolchains the plugin already builds with; on Windows run it from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. +It also needs a Clang or GCC whose ThreadSanitizer runtime *runs* on the host, so it fails rather than skipping when one isn't available. On Windows run it from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. ```bash just obs test diff --git a/flake.nix b/flake.nix index 284f6388e3d..29aef0d5a70 100644 --- a/flake.nix +++ b/flake.nix @@ -290,19 +290,62 @@ uniffi-bindgen-go ]; - # Dependencies for building the OBS plugin (`just obs build`). - # Linux-only: nixpkgs marks obs-studio broken on Darwin, so macOS - # and Windows fetch libobs/Qt6 via the OBS buildspec instead (see - # cpp/obs/buildspec.json and doc/bin/obs.md). ffmpeg + cmake come from - # rustDeps. clang-tools/gersemi back `just obs check`. + # The libobs headers, unpacked from the same OBS release + # cpp/obs/buildspec.json downloads. Headers only: nothing here links, so + # it works on Darwin even though obs-studio itself does not build there, + # and it costs ~2 MB of store instead of the multi-hundred-MB obs-deps + # bundle. `just obs compile` type-checks the plugin against it. + # + # Keep `version` equal to buildspec.json's obs-studio version; `just obs + # check` fails when the two drift. + obs-headers = pkgs.stdenvNoCC.mkDerivation rec { + pname = "libobs-headers"; + version = "31.1.1"; + + src = pkgs.fetchzip { + url = "https://github.com/obsproject/obs-studio/archive/refs/tags/${version}.tar.gz"; + hash = "sha256-ycfROxgm3wUVyC2d1r3vIr7yWb6ErYIoDZX8xZrc+Vk="; + }; + + dontBuild = true; + + # The layout matches a real OBS install (everything flat under + # include/obs, frontend API included), so a plugin's include paths are + # the same here as against the system package on Linux. + installPhase = '' + runHook preInstall + (cd libobs && find . \( -name '*.h' -o -name '*.hpp' \) -exec install -Dm444 {} $out/include/obs/{} \;) + install -Dm444 frontend/api/obs-frontend-api.h $out/include/obs/obs-frontend-api.h + # obs.h includes obsconfig.h, which the OBS build generates. Only the + # two version macros have to hold a value; everything else it + # configures is internal to libobs, so leave the #cmakedefines unset. + sed -e 's/^#cmakedefine.*$//' -e 's/@OBS_RELEASE_CANDIDATE@/0/' -e 's/@OBS_BETA@/0/' \ + libobs/obsconfig.h.in > $out/include/obs/obsconfig.h + runHook postInstall + ''; + }; + + # Dependencies for the OBS plugin (`just obs build`, `just obs compile`, + # `just obs check`). ffmpeg + cmake come from rustDeps. + # + # Only the *build* is Linux-only: nixpkgs marks obs-studio broken on + # Darwin, so macOS and Windows link libobs/Qt6 from the OBS buildspec + # bundle instead (see cpp/obs/buildspec.json and doc/bin/obs.md). + # Type-checking needs headers rather than libraries, and those are + # cross-platform -- obs-headers above, plus qt6.qtbase, which does build + # on Darwin. So `just obs compile` and the lints run everywhere while + # `just obs build` stays native. obsDeps = with pkgs; - lib.optionals (!stdenv.isDarwin) [ - obs-studio + [ + obs-headers qt6.qtbase - ninja clang-tools gersemi + ] + ++ lib.optionals (!stdenv.isDarwin) [ + obs-studio + ninja ]; # Apply our overlay to get the package definitions @@ -370,7 +413,13 @@ # Nix's _FORTIFY_SOURCE hardening (requires -O). hardeningDisable = [ "fortify" ]; - env = pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) { + env = { + # Where `just obs compile` and `just obs test` look for libobs. Set + # on every platform so the plugin type-checks against the pinned OBS + # release everywhere, rather than whatever the host happens to have. + OBS_INCLUDE_DIR = "${obs-headers}/include/obs"; + } + // pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) { ALSA_PLUGIN_DIR = "${alsaPlugins}/lib/alsa-lib"; }; }; diff --git a/justfile b/justfile index 3801338dcbc..1d43cafda33 100644 --- a/justfile +++ b/justfile @@ -110,11 +110,10 @@ _tools $FILES="": # cargo because `go check` builds moq-ffi for the host, and skips on a # missing cargo the same way it skips on a missing go. scoped '^(go/|rs/moq-ffi/)' && tools+=(go uniffi-bindgen-go cargo) - # The OBS lints ship only in the Linux dev shell; nixpkgs marks obs-studio - # broken on Darwin. - if [[ "$(uname -s)" == "Linux" ]] && scoped '^cpp/obs/'; then - tools+=(clang-format gersemi) - fi + # cargo regenerates moq.h for the type-check; pkg-config locates Qt6 and + # ffmpeg. Every platform: the plugin type-checks against headers, and the + # dev shell ships those even on Darwin, where obs-studio can't build. + scoped '^(cpp/obs/|rs/libmoq/)' && tools+=(clang-format gersemi pkg-config cargo) # Scopes overlap (rs/moq-ffi/ is in four of them), so the same tool can land # in the list twice and be reported missing twice. Splitting on whitespace is @@ -162,9 +161,18 @@ check $BASE="": just kt check "$files" just swift check "$files" just go check "$files" - # The OBS plugin has no compile job in PR CI, so its lint + CMake - # guards are the only automated coverage it gets. - if echo "$files" | grep -q '^cpp/obs/'; then + # Type-checking the plugin needs only headers, so it runs here rather + # than waiting for obs.yml to link it on Linux. libmoq is in scope + # because the plugin calls through its generated C header, and flake.nix + # because it owns the libobs headers this compiles against -- obs.yml + # links against nixpkgs' obs-studio instead, so nothing else would notice + # that package going bad. + if echo "$files" | grep -qE '^(cpp/obs/|rs/libmoq/|flake\.nix$)'; then + just obs compile + fi + # flake.nix is in scope because `just obs check` is what compares the OBS + # version pinned there against buildspec.json, and either side can move. + if echo "$files" | grep -qE '^(cpp/obs/|flake\.nix$)'; then just obs check fi # Validates flake eval + dev shell build; it no longer compiles the @@ -191,6 +199,7 @@ check-all *args: just swift check just go check just obs check + just obs compile just _flake just _check-common From 7a21b35d4832a77e938b3ad31b2ff0e89faa2c6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:54:47 -0700 Subject: [PATCH 04/29] build(deps): bump Swatinem/rust-cache from e18b497796c12c097a38f9edb9d0641fb99eee32 to 258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 in the github-actions group (#2887) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cache.yml | 2 +- .github/workflows/check.yml | 4 ++-- .github/workflows/nightly.yml | 2 +- .github/workflows/obs.yml | 2 +- .github/workflows/release-go-ffi.yml | 4 ++-- .github/workflows/release-kt-ffi.yml | 4 ++-- .github/workflows/release-kt-lib.yml | 2 +- .github/workflows/release-swift-ffi.yml | 4 ++-- .github/workflows/smoke.yml | 2 +- .github/workflows/swift.yml | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 36477c5f018..7d0f88e23a2 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -89,7 +89,7 @@ jobs: # `shared-key` replaces the automatic per-job key, which is what lets # check.yml's differently-named jobs restore what this one saves. - name: Rust Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: shared-key: rust cache-on-failure: true diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 18ae5d3ec23..81ea06d697f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -61,7 +61,7 @@ jobs: # Restore only. cache.yml is the single writer, on `main`; see the comment # there for why. `shared-key` has to match the one it saves under. - name: Rust Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: shared-key: rust save-if: false @@ -112,7 +112,7 @@ jobs: extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - name: Rust Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: shared-key: rust save-if: false diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 174b8ca9cf5..081bbc82c28 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -49,7 +49,7 @@ jobs: determinate: false - name: Rust Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: cache-on-failure: true diff --git a/.github/workflows/obs.yml b/.github/workflows/obs.yml index 06a8c23aa05..1775e7c7a9c 100644 --- a/.github/workflows/obs.yml +++ b/.github/workflows/obs.yml @@ -89,7 +89,7 @@ jobs: # holds dev-profile artifacts, which is why `just obs ci` builds libmoq in # debug rather than the preset's release. - name: Rust Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: shared-key: rust save-if: false diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index 102c606ecb9..5a0295913c5 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -82,7 +82,7 @@ jobs: targets: ${{ matrix.target }} - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: # Don't let untrusted PR builds populate caches reused by trusted runs. save-if: ${{ github.event_name != 'pull_request' }} @@ -127,7 +127,7 @@ jobs: uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: # Don't let untrusted PR builds populate caches reused by trusted runs. save-if: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 86fc92527a7..8396208fd48 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -69,7 +69,7 @@ jobs: targets: ${{ matrix.target }} - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 - name: Install cross-compilation tools (Linux ARM64) if: matrix.target == 'aarch64-unknown-linux-gnu' @@ -115,7 +115,7 @@ jobs: uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 - name: Generate bindings env: diff --git a/.github/workflows/release-kt-lib.yml b/.github/workflows/release-kt-lib.yml index 5a4d1d7f6c6..2471b1577d7 100644 --- a/.github/workflows/release-kt-lib.yml +++ b/.github/workflows/release-kt-lib.yml @@ -126,7 +126,7 @@ jobs: uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 # Generate the UniFFI Kotlin so the sibling :moq-ffi project compiles. - name: Generate bindings diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index c05e7fef142..660ed451580 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -58,7 +58,7 @@ jobs: targets: ${{ matrix.target }} - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 - name: Build shell: bash @@ -93,7 +93,7 @@ jobs: uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 - name: Generate bindings env: diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index e9f0f81774f..c68efa8c0ee 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -57,7 +57,7 @@ jobs: # Reuse ~/.cargo + ./target across nightly runs so only changed crates # recompile (the relay, cli, moq-ffi, and libmoq all build from source). - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: cache-on-failure: true diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index eeed2381d89..4d7d9645609 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -38,7 +38,7 @@ jobs: targets: aarch64-apple-ios-sim,x86_64-apple-ios - name: Rust Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 with: cache-on-failure: true From b712bfe8b701d86b2675f4a63b6f399aced45616 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:55:10 -0700 Subject: [PATCH 05/29] build(deps): bump the bun group with 9 updates (#2885) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bun.lock | 200 ++++++++++++++-------- demo/boy/package.json | 4 +- demo/web/package.json | 4 +- doc/package.json | 4 +- js/clock/package.json | 2 +- js/moq-boy/package.json | 4 +- js/net/package.json | 4 +- js/publish/package.json | 4 +- js/token/package.json | 4 +- js/watch/package.json | 4 +- package.json | 4 +- test/smoke/clients/js-native/package.json | 2 +- test/smoke/clients/js/package.json | 4 +- 13 files changed, 149 insertions(+), 95 deletions(-) diff --git a/bun.lock b/bun.lock index 2e2fea44608..f6e4434f271 100644 --- a/bun.lock +++ b/bun.lock @@ -5,9 +5,9 @@ "": { "name": "moq", "devDependencies": { - "@biomejs/biome": "^2.5.6", + "@biomejs/biome": "^2.5.7", "concurrently": "^10.0.4", - "publint": "^0.3.22", + "publint": "^0.3.23", "remark-cli": "^12.0.1", "remark-frontmatter": "^5.0.0", "remark-lint-list-item-indent": "^4.0.1", @@ -26,9 +26,9 @@ "@moq/boy": "workspace:^", }, "devDependencies": { - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14", }, }, @@ -44,14 +44,14 @@ "devDependencies": { "@tailwindcss/typography": "^0.5.20", "@tailwindcss/vite": "^4.3.2", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "highlight.js": "^11.11.1", "open": "^11.0.0", "solid-element": "^1.9.2", "solid-js": "^1.9.14", "tailwindcss": "^4.1.13", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14", }, }, @@ -60,10 +60,10 @@ "version": "0.1.0", "devDependencies": { "@types/bun": "^1.3.14", - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "typescript": "7.0.2", "vitepress": "^1.6.4", - "wrangler": "^4.118.0", + "wrangler": "^4.120.0", "yaml": "^2.9.0", }, }, @@ -79,7 +79,7 @@ "@moq/net": "workspace:*", }, "devDependencies": { - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "typescript": "7.0.2", }, }, @@ -160,11 +160,11 @@ }, "devDependencies": { "@types/bun": "^1.3.14", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "rimraf": "^6.1.3", "solid-js": "^1.9.14", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14", }, }, @@ -184,13 +184,13 @@ "name": "@moq/net", "version": "0.3.0", "dependencies": { - "@moq/qmux": "^0.3.2", + "@moq/qmux": "^0.3.3", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0", }, "devDependencies": { "@types/bun": "^1.3.14", - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "@typescript/lib-dom": "npm:@types/web@^0.0.350", "rimraf": "^6.1.3", "typescript": "7.0.2", @@ -212,10 +212,10 @@ "@types/audioworklet": "^0.0.100", "@types/bun": "^1.3.14", "@typescript/lib-dom": "npm:@types/web@^0.0.350", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "rimraf": "^6.1.3", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", }, }, "js/signals": { @@ -249,12 +249,12 @@ "dependencies": { "@hexagon/base64": "^2.0.4", "commander": "^15.0.0", - "jose": "^6.2.7", + "jose": "^6.2.8", "zod": "^4.4.3", }, "devDependencies": { "@types/bun": "^1.3.14", - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "rimraf": "^6.1.3", "typescript": "7.0.2", }, @@ -276,10 +276,10 @@ "@types/audioworklet": "^0.0.100", "@types/bun": "^1.3.14", "@typescript/lib-dom": "npm:@types/web@^0.0.350", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "rimraf": "^6.1.3", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", }, }, "test/smoke/clients/js": { @@ -289,9 +289,9 @@ "@moq/watch": "workspace:*", }, "devDependencies": { - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "playwright": "^1.62.1", - "vite": "^8.2.0", + "vite": "^8.2.1", }, }, "test/smoke/clients/js-native": { @@ -304,7 +304,7 @@ "zod": "^4.0.0", }, "devDependencies": { - "tsx": "^4.23.4", + "tsx": "^4.23.11", }, }, }, @@ -384,37 +384,37 @@ "@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], - "@biomejs/biome": ["@biomejs/biome@2.5.6", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.5.6", "@biomejs/cli-darwin-x64": "2.5.6", "@biomejs/cli-linux-arm64": "2.5.6", "@biomejs/cli-linux-arm64-musl": "2.5.6", "@biomejs/cli-linux-x64": "2.5.6", "@biomejs/cli-linux-x64-musl": "2.5.6", "@biomejs/cli-win32-arm64": "2.5.6", "@biomejs/cli-win32-x64": "2.5.6" }, "bin": { "biome": "bin/biome" } }, "sha512-lxVNjv7UF6KfhMJfL9gaUHbWdJdHbsAj6OSmwSYNdhRuG67NxNQ4Xdvh3TUxsSK9sBzJBQhEJj3AopmmNJ5pSA=="], + "@biomejs/biome": ["@biomejs/biome@2.5.7", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.5.7", "@biomejs/cli-darwin-x64": "2.5.7", "@biomejs/cli-linux-arm64": "2.5.7", "@biomejs/cli-linux-arm64-musl": "2.5.7", "@biomejs/cli-linux-x64": "2.5.7", "@biomejs/cli-linux-x64-musl": "2.5.7", "@biomejs/cli-win32-arm64": "2.5.7", "@biomejs/cli-win32-x64": "2.5.7" }, "bin": { "biome": "bin/biome" } }, "sha512-zr8K/DcY5tYsQOQwqMJ0AWElo6QgmgNI7idXgXLhevVszlt8RGVpesEJPqx3ThazLaOwjJ5Y8fz3BtH5fGZNsw=="], - "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.5.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zMOLZP4oMrjh6m1zcSj1ud2awUPgTuMVbmQhYYWL7J8HwCnbHHBvTm7VBTRuY7epT5bez76IpKYQ11ZAqHFlnw=="], + "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.5.7", "", { "os": "darwin", "cpu": "arm64" }, "sha512-vxo/Ls3/PYdQWyLhYYcgMOCzQypAjcY+iihS8M0wW03l16TCLW4zqZzGo75gm1VdCMj38hTVZ31KBWrZ4G9dJw=="], - "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.5.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-JAC1VqzvO7Th5ZplU0G2uGfkZbxEe9uDDektPAhF0JLusoz1w+T4okp2bkykI0bbaO2vslKiRfj4gU43JaGreA=="], + "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.5.7", "", { "os": "darwin", "cpu": "x64" }, "sha512-Cd3Ga61amT/Yl/0x8elP5hhGYaFy4bw6WuysTgf7oo8TA5tJ5A1k+DkVoJ2BHbTVil51gTX9VPzArnrlLJ3Kyg=="], - "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.5.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-6XsYwCFkp5sMxl85ffhgeGpGgs6A7dRYFnkceZ7WVxvycuTnGdD5xa534Z3xfrBQ0JCMK/mujT6ZNPJoghedwg=="], + "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.5.7", "", { "os": "linux", "cpu": "arm64" }, "sha512-rR2QE0yF2GYSuYuKIa7pKvODGJqnOH+2eDREAM8wV+mWKSkMQKdAp4zXEZfTaxY8PMoNONnpgSWcBCyLDPDOKg=="], - "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.5.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-eUa3jeeYvfMt19LBeh6E5PUZpxnTC4JqNWo+EDjTtQjAr2xLGnWaxACtVU1DQqmHYbvThlJzLX+ZsYgrqh2qVw=="], + "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.5.7", "", { "os": "linux", "cpu": "arm64" }, "sha512-xPI5yB6XlpDbNkS+bm1t42olw5c4l3UrlOmLg7KtLJvjvkNF/1V4tnUgfkylGIeb3u/T+BzMGYqgQhzjAoJzuQ=="], - "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.5.6", "", { "os": "linux", "cpu": "x64" }, "sha512-Pop9VXCFUhFTMfFefZ39S+u2rOPyNp5iHlxbZRwXGACHLy2r0jjiRgJHmaEKJzL3SyxlVeGShXhvvElvWowonA=="], + "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.5.7", "", { "os": "linux", "cpu": "x64" }, "sha512-FQgqJhscrqJUFptGaRSUJWlXAExwWcDwLuK49dvKfkQ1bB5SEEyFssnsxQY83Xm6jR0EbbX3+8+D5bfvYqUG2Q=="], - "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.5.6", "", { "os": "linux", "cpu": "x64" }, "sha512-2Vp13QdKysH3HIWLaYLhUUwbK+jbZonJD1K+Lr0d0RO4wH7mkYd43vJixEDm8cUWrowoRz4UUHF1nm9Ae7ym8A=="], + "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.5.7", "", { "os": "linux", "cpu": "x64" }, "sha512-rE5VZi+qtmPgQH+l7jVxYoZ18b/TiHEhulhMpjmCZH1PltSbjRcxNWywC3HZ9tYottG7ORkeTtoscBilKSBm0g=="], - "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.5.6", "", { "os": "win32", "cpu": "arm64" }, "sha512-tDGshcm6BdkZOCGnTDX0Y8/U4IfBSlnUU7T56nNDuPEfed+aHg+u8G36NB43fJVl0Os6+QURXIE1yuD7AaEofA=="], + "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.5.7", "", { "os": "win32", "cpu": "arm64" }, "sha512-Oq4x0CCwP4jirrcTywXs5kOGZ4v5vuEP+gWrbtjApOA2CL9F3F9GlIdQIci8AKSCa/zURanMRpX/4wQ7Am6hHg=="], - "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.5.6", "", { "os": "win32", "cpu": "x64" }, "sha512-WN05KwXnTO/2J45RQPvzZMXf7tZUIofHoR35xIPfCo7pQ2RFidxI8sfb5mGsaTxdMmEOzHzOPRCdA5/fCpc7xQ=="], + "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.5.7", "", { "os": "win32", "cpu": "x64" }, "sha512-V+0wu/nrj2S+MhP4EQ0uHNolP0IALEsz45pg0WoKkHfDeh0+ItHwP/p7bX5RPoMOl9NkpHYWdYPhIcy2mACHvQ=="], "@cloudflare/kv-asset-handler": ["@cloudflare/kv-asset-handler@0.5.0", "", {}, "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg=="], "@cloudflare/unenv-preset": ["@cloudflare/unenv-preset@2.16.1", "", { "peerDependencies": { "unenv": "2.0.0-rc.24", "workerd": ">1.20260305.0 <2.0.0-0" }, "optionalPeers": ["workerd"] }, "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw=="], - "@cloudflare/workerd-darwin-64": ["@cloudflare/workerd-darwin-64@1.20260730.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA=="], + "@cloudflare/workerd-darwin-64": ["@cloudflare/workerd-darwin-64@1.20260801.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-wuJWbXpKvncJi1P0GKS+iYpN5tHdb7JPJJ/+6ZQe8zzovHHVMkLJPNBsgWpqeUhpM3g9qTwEKd2rglNKejuh5A=="], - "@cloudflare/workerd-darwin-arm64": ["@cloudflare/workerd-darwin-arm64@1.20260730.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ=="], + "@cloudflare/workerd-darwin-arm64": ["@cloudflare/workerd-darwin-arm64@1.20260801.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-kwoZiTpnhNrF3+APx84Q/oAqvJ3sU9yefGagwm/ASaH/2W19x0vghkW/r4qCoHCK0WW7EPugZ+aXjgPMRtlq1Q=="], - "@cloudflare/workerd-linux-64": ["@cloudflare/workerd-linux-64@1.20260730.1", "", { "os": "linux", "cpu": "x64" }, "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA=="], + "@cloudflare/workerd-linux-64": ["@cloudflare/workerd-linux-64@1.20260801.1", "", { "os": "linux", "cpu": "x64" }, "sha512-r0vAxCZH+Jih9Unm1yoyiByPNWNgawcKciOHDm5Q37ZVGOkKLsT9AtLe3yLSaul76WrKqtf+JP2n0WW32VBLJg=="], - "@cloudflare/workerd-linux-arm64": ["@cloudflare/workerd-linux-arm64@1.20260730.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ=="], + "@cloudflare/workerd-linux-arm64": ["@cloudflare/workerd-linux-arm64@1.20260801.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-zWgpdZtSozvIgzQNmQiDSF8yEOQJUkRAWNsDXXzAAoy+fCn8YUoSibj3mpFSbZRvbUldeBEaW6SCdC2VEMkhNQ=="], - "@cloudflare/workerd-windows-64": ["@cloudflare/workerd-windows-64@1.20260730.1", "", { "os": "win32", "cpu": "x64" }, "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ=="], + "@cloudflare/workerd-windows-64": ["@cloudflare/workerd-windows-64@1.20260801.1", "", { "os": "win32", "cpu": "x64" }, "sha512-2oQz+Ksu4ji6e/+ZoYX+tWQEcxAii2p7l+iR8kx48W1llMalaufAsmVxTlk+3/vrM7D3/2c0iK448e0UQTcIMg=="], "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], @@ -426,57 +426,57 @@ "@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ=="], - "@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + "@esbuild/android-arm": ["@esbuild/android-arm@0.28.2", "", { "os": "android", "cpu": "arm" }, "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg=="], - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.2", "", { "os": "android", "cpu": "arm64" }, "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A=="], - "@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + "@esbuild/android-x64": ["@esbuild/android-x64@0.28.2", "", { "os": "android", "cpu": "x64" }, "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q=="], - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw=="], - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw=="], - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw=="], - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg=="], - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.2", "", { "os": "linux", "cpu": "arm" }, "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w=="], - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug=="], - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.2", "", { "os": "linux", "cpu": "ia32" }, "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ=="], - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.2", "", { "os": "linux", "cpu": "none" }, "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ=="], - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.2", "", { "os": "linux", "cpu": "none" }, "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA=="], - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ=="], - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.2", "", { "os": "linux", "cpu": "none" }, "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA=="], - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg=="], - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.2", "", { "os": "linux", "cpu": "x64" }, "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ=="], - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.2", "", { "os": "none", "cpu": "arm64" }, "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw=="], - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.2", "", { "os": "none", "cpu": "x64" }, "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw=="], - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ=="], - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw=="], - "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.2", "", { "os": "none", "cpu": "arm64" }, "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q=="], - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g=="], - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ=="], - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA=="], - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.2", "", { "os": "win32", "cpu": "x64" }, "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g=="], "@fails-components/webtransport": ["@fails-components/webtransport@1.6.7", "", { "dependencies": { "@types/debug": "^4.1.7", "bindings": "^1.5.0", "debug": "^4.3.4" } }, "sha512-jBXB9TC7ItzhzXdSRSR8MbzvfdnQ7vuHiL4LFn2LGDc2nSsVdA9NWE1Zrf+XekbPlwK/K2trwXnQp6jOcw0OuA=="], @@ -586,7 +586,7 @@ "@moq/publish": ["@moq/publish@workspace:js/publish"], - "@moq/qmux": ["@moq/qmux@0.3.2", "", { "dependencies": { "@moq/web-socket-stream": "^0.1.1" } }, "sha512-8ymBriVxvD5ISfnTJ3k07Zzt4pBOoh/JnulGCjenY2NrUdCr5PUOy42y/9drSXA3SfCfOdaw4A+BSrWN92qVvg=="], + "@moq/qmux": ["@moq/qmux@0.3.3", "", { "dependencies": { "@moq/web-socket-stream": "^0.1.1" } }, "sha512-Ez1GCVwpyNGB0eWvhKIXOjXrfRlMdmO0lpJ7gqInZFzsOsYTyML0WRgh24Itczp9Kev0CXsUA/K+XrGs1dRKUA=="], "@moq/signals": ["@moq/signals@workspace:js/signals"], @@ -816,7 +816,7 @@ "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], - "@types/node": ["@types/node@26.1.2", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg=="], + "@types/node": ["@types/node@26.2.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg=="], "@types/react": ["@types/react@19.2.18", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w=="], @@ -1084,7 +1084,7 @@ "error-stack-parser-es": ["error-stack-parser-es@1.0.5", "", {}, "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA=="], - "esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + "esbuild": ["esbuild@0.28.2", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.2", "@esbuild/android-arm": "0.28.2", "@esbuild/android-arm64": "0.28.2", "@esbuild/android-x64": "0.28.2", "@esbuild/darwin-arm64": "0.28.2", "@esbuild/darwin-x64": "0.28.2", "@esbuild/freebsd-arm64": "0.28.2", "@esbuild/freebsd-x64": "0.28.2", "@esbuild/linux-arm": "0.28.2", "@esbuild/linux-arm64": "0.28.2", "@esbuild/linux-ia32": "0.28.2", "@esbuild/linux-loong64": "0.28.2", "@esbuild/linux-mips64el": "0.28.2", "@esbuild/linux-ppc64": "0.28.2", "@esbuild/linux-riscv64": "0.28.2", "@esbuild/linux-s390x": "0.28.2", "@esbuild/linux-x64": "0.28.2", "@esbuild/netbsd-arm64": "0.28.2", "@esbuild/netbsd-x64": "0.28.2", "@esbuild/openbsd-arm64": "0.28.2", "@esbuild/openbsd-x64": "0.28.2", "@esbuild/openharmony-arm64": "0.28.2", "@esbuild/sunos-x64": "0.28.2", "@esbuild/win32-arm64": "0.28.2", "@esbuild/win32-ia32": "0.28.2", "@esbuild/win32-x64": "0.28.2" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA=="], "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], @@ -1208,7 +1208,7 @@ "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], - "jose": ["jose@6.2.7", "", {}, "sha512-hq1OB1bALKfydZNoViyg6hPVGV4i93ny9Op+n4zP5RSf7SCZEXa/TsG2O3IEr7+WlHRTPnpqDmHfMH6qXAD60w=="], + "jose": ["jose@6.2.8", "", {}, "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ=="], "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], @@ -1340,7 +1340,7 @@ "mimic-response": ["mimic-response@3.1.0", "", {}, "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="], - "miniflare": ["miniflare@5.20260730.0-alpha", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.2", "undici": "7.28.0", "workerd": "1.20260730.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" } }, "sha512-8/dspSXDshP6nSkCpjKO7BYc2qZoYSXm7iM+QxY7qJyJpAB3onnQSaiu0cvKJlfuMGwULl55hG69FJCcCMXU1Q=="], + "miniflare": ["miniflare@5.20260801.1-alpha", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.2", "undici": "7.29.0", "workerd": "1.20260801.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" } }, "sha512-BHPVzIDA6mbx7LefxpvkXW7DHx9FKB9GorZatbnrrFTt3CVMU8zuUpbgyCuebwDKcTTOZos43ta8GQ0eMVEpxA=="], "minimatch": ["minimatch@10.2.5", "", { "dependencies": { "brace-expansion": "^5.0.5" } }, "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg=="], @@ -1454,7 +1454,7 @@ "property-information": ["property-information@7.2.0", "", {}, "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg=="], - "publint": ["publint@0.3.22", "", { "dependencies": { "@publint/pack": "^0.1.6", "package-manager-detector": "^1.7.0", "picocolors": "^1.1.1", "sade": "^1.8.1" }, "bin": { "publint": "./src/cli.js" } }, "sha512-6Z/scsr5CA7APdwyF35EY88CqgDj1textWuY788DVTJYPCWVv/Wn9G6KmLnrVRnStgYcahqN4wCDLZGSbQJ69w=="], + "publint": ["publint@0.3.23", "", { "dependencies": { "@publint/pack": "^0.1.6", "package-manager-detector": "^1.7.0", "picocolors": "^1.1.1", "sade": "^1.8.1" }, "bin": { "publint": "./src/cli.js" } }, "sha512-5MQipUPcB7MWw84zLUkHrg/H/UBtk3LL+A0GngTTBSsiNJLQurMUaSIRG3edlOrRz4UFe0AOKK9TZdIWviV+jQ=="], "pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="], @@ -1670,7 +1670,7 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "tsx": ["tsx@4.23.4", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-ZiUQ8oT/KzN51mJUWPqARYqwFLFJZtGZipRkw1ynHMr9vy3eU77m5yfF3Gzm6meEg/beW+lUu3fHYgskTN2oVQ=="], + "tsx": ["tsx@4.23.11", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-Ry2oTEUnhBdeEdWIztY8kf3/nBGnPnjMLVGL0YfdRXMORuPER5NlKmayqxtxRxwB1xBN+RivRaJfe7PM1rtiyw=="], "tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="], @@ -1680,7 +1680,7 @@ "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], - "undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], + "undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], @@ -1732,7 +1732,7 @@ "vfile-statistics": ["vfile-statistics@3.0.0", "", { "dependencies": { "vfile": "^6.0.0", "vfile-message": "^4.0.0" } }, "sha512-/qlwqwWBWFOmpXujL/20P+Iuydil0rZZNglR+VNm6J0gpLHwuVM5s7g2TfVoswbXjZ4HuIhLMySEyIw5i7/D8w=="], - "vite": ["vite@8.2.0", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.5", "postcss": "^8.5.23", "rolldown": "~1.2.0", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.4.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ=="], + "vite": ["vite@8.2.1", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.5", "postcss": "^8.5.25", "rolldown": "~1.2.1", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.4.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw=="], "vite-plugin-html": ["vite-plugin-html@3.2.2", "", { "dependencies": { "@rollup/pluginutils": "^4.2.0", "colorette": "^2.0.16", "connect-history-api-fallback": "^1.6.0", "consola": "^2.15.3", "dotenv": "^16.0.0", "dotenv-expand": "^8.0.2", "ejs": "^3.1.6", "fast-glob": "^3.2.11", "fs-extra": "^10.0.1", "html-minifier-terser": "^6.1.0", "node-html-parser": "^5.3.3", "pathe": "^0.2.0" }, "peerDependencies": { "vite": ">=2.0.0" } }, "sha512-vb9C9kcdzcIo/Oc3CLZVS03dL5pDlOFuhGlZYDCJ840BhWl/0nGeZWf3Qy7NlOayscY4Cm/QRgULCQkEZige5Q=="], @@ -1748,9 +1748,9 @@ "which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="], - "workerd": ["workerd@1.20260730.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260730.1", "@cloudflare/workerd-darwin-arm64": "1.20260730.1", "@cloudflare/workerd-linux-64": "1.20260730.1", "@cloudflare/workerd-linux-arm64": "1.20260730.1", "@cloudflare/workerd-windows-64": "1.20260730.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA=="], + "workerd": ["workerd@1.20260801.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260801.1", "@cloudflare/workerd-darwin-arm64": "1.20260801.1", "@cloudflare/workerd-linux-64": "1.20260801.1", "@cloudflare/workerd-linux-arm64": "1.20260801.1", "@cloudflare/workerd-windows-64": "1.20260801.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-/g9JGTyqnHtoIscpBHqKD8swE2V4StBs2i69PmLiOhH45OP95jCFICl4F1hKlgN57rqfni5LiCitttoX5OOkVA=="], - "wrangler": ["wrangler@4.118.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "5.20260730.0-alpha", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260730.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260730.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-9pkBw/b8zWqGx2S+oLhgHMR1M/4VOE8SynUFABnGWiSFGlcOQ4xiI/B71Xf66RYP2xzngU37IQFPtUruij3lYw=="], + "wrangler": ["wrangler@4.120.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "5.20260801.1-alpha", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260801.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260801.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-cBmu/MeaB/fPacC0JpATs4duTOCagBxrZo+vBzuTX06tLzwSyAHE1drlHUZ8rP0VqVz1fy3ReGYTiHdKkoHltg=="], "wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="], @@ -1900,6 +1900,8 @@ "vitepress/vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], + "wrangler/esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + "wrap-ansi-cjs/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "wrap-ansi-cjs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], @@ -1968,6 +1970,58 @@ "vitepress/vite/postcss": ["postcss@8.5.16", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg=="], + "wrangler/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + + "wrangler/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + + "wrangler/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + + "wrangler/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + + "wrangler/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + + "wrangler/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + + "wrangler/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + + "wrangler/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + + "wrangler/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + + "wrangler/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + + "wrangler/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + + "wrangler/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + + "wrangler/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + + "wrangler/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + + "wrangler/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + + "wrangler/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + + "wrangler/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + + "wrangler/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + + "wrangler/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + + "wrangler/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + + "wrangler/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + + "wrangler/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + + "wrangler/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + + "wrangler/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + + "wrangler/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + + "wrangler/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + "wrap-ansi-cjs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "wrap-ansi-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], diff --git a/demo/boy/package.json b/demo/boy/package.json index 42c345c8022..d1298f3de73 100644 --- a/demo/boy/package.json +++ b/demo/boy/package.json @@ -12,9 +12,9 @@ "@moq/boy": "workspace:^" }, "devDependencies": { - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14" } } diff --git a/demo/web/package.json b/demo/web/package.json index 878dafbe3b6..497f24d7d9e 100644 --- a/demo/web/package.json +++ b/demo/web/package.json @@ -21,14 +21,14 @@ "devDependencies": { "@tailwindcss/typography": "^0.5.20", "@tailwindcss/vite": "^4.3.2", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "highlight.js": "^11.11.1", "open": "^11.0.0", "solid-element": "^1.9.2", "solid-js": "^1.9.14", "tailwindcss": "^4.1.13", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14" } } diff --git a/doc/package.json b/doc/package.json index 5d1e6790499..cadb307f728 100644 --- a/doc/package.json +++ b/doc/package.json @@ -13,10 +13,10 @@ }, "devDependencies": { "@types/bun": "^1.3.14", - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "typescript": "7.0.2", "vitepress": "^1.6.4", - "wrangler": "^4.118.0", + "wrangler": "^4.120.0", "yaml": "^2.9.0" } } diff --git a/js/clock/package.json b/js/clock/package.json index f6b72778105..699e9e40596 100644 --- a/js/clock/package.json +++ b/js/clock/package.json @@ -22,7 +22,7 @@ "@fails-components/webtransport-transport-http3-quiche": "^1.6.7" }, "devDependencies": { - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "typescript": "7.0.2" } } diff --git a/js/moq-boy/package.json b/js/moq-boy/package.json index 265e6bb67c7..8fce68856a4 100644 --- a/js/moq-boy/package.json +++ b/js/moq-boy/package.json @@ -32,11 +32,11 @@ "//devDependencies": "These are only needed for local development with workspace packages. They are NOT needed when using the published npm packages.", "devDependencies": { "@types/bun": "^1.3.14", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "rimraf": "^6.1.3", "solid-js": "^1.9.14", "typescript": "7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14" } } diff --git a/js/net/package.json b/js/net/package.json index b86ee0ff764..861e47fd9c6 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -17,7 +17,7 @@ "release": "bun ../common/release.ts" }, "dependencies": { - "@moq/qmux": "^0.3.2", + "@moq/qmux": "^0.3.3", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0" }, @@ -26,7 +26,7 @@ }, "devDependencies": { "@types/bun": "^1.3.14", - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "@typescript/lib-dom": "npm:@types/web@^0.0.350", "rimraf": "^6.1.3", "typescript": "7.0.2", diff --git a/js/publish/package.json b/js/publish/package.json index d918cf99369..9181b9d27e9 100644 --- a/js/publish/package.json +++ b/js/publish/package.json @@ -33,9 +33,9 @@ "@types/audioworklet": "^0.0.100", "@types/bun": "^1.3.14", "@typescript/lib-dom": "npm:@types/web@^0.0.350", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "rimraf": "^6.1.3", "typescript": "7.0.2", - "vite": "^8.2.0" + "vite": "^8.2.1" } } diff --git a/js/token/package.json b/js/token/package.json index 0c82312b5e4..2d03b8e1638 100644 --- a/js/token/package.json +++ b/js/token/package.json @@ -21,12 +21,12 @@ "dependencies": { "@hexagon/base64": "^2.0.4", "commander": "^15.0.0", - "jose": "^6.2.7", + "jose": "^6.2.8", "zod": "^4.4.3" }, "devDependencies": { "@types/bun": "^1.3.14", - "@types/node": "^26.1.2", + "@types/node": "^26.2.0", "rimraf": "^6.1.3", "typescript": "7.0.2" } diff --git a/js/watch/package.json b/js/watch/package.json index 67382519981..bd158ddcd39 100644 --- a/js/watch/package.json +++ b/js/watch/package.json @@ -34,9 +34,9 @@ "@types/audioworklet": "^0.0.100", "@types/bun": "^1.3.14", "@typescript/lib-dom": "npm:@types/web@^0.0.350", - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "rimraf": "^6.1.3", "typescript": "7.0.2", - "vite": "^8.2.0" + "vite": "^8.2.1" } } diff --git a/package.json b/package.json index a5913edd382..2be40c9aa2f 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,9 @@ "name": "moq", "version": "0.0.0", "devDependencies": { - "@biomejs/biome": "^2.5.6", + "@biomejs/biome": "^2.5.7", "concurrently": "^10.0.4", - "publint": "^0.3.22", + "publint": "^0.3.23", "remark-cli": "^12.0.1", "remark-frontmatter": "^5.0.0", "remark-lint-list-item-indent": "^4.0.1", diff --git a/test/smoke/clients/js-native/package.json b/test/smoke/clients/js-native/package.json index bce53f7c7c0..cfeddb8324a 100644 --- a/test/smoke/clients/js-native/package.json +++ b/test/smoke/clients/js-native/package.json @@ -10,6 +10,6 @@ "zod": "^4.0.0" }, "devDependencies": { - "tsx": "^4.23.4" + "tsx": "^4.23.11" } } diff --git a/test/smoke/clients/js/package.json b/test/smoke/clients/js/package.json index d1e467dd893..819702853e7 100644 --- a/test/smoke/clients/js/package.json +++ b/test/smoke/clients/js/package.json @@ -10,8 +10,8 @@ "@moq/watch": "workspace:*" }, "devDependencies": { - "esbuild": "^0.28.1", + "esbuild": "^0.28.2", "playwright": "^1.62.1", - "vite": "^8.2.0" + "vite": "^8.2.1" } } From e67632b9ceb37b1645dcb3e58505d47d375424f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:55:27 -0700 Subject: [PATCH 06/29] build(deps-dev): bump ruff from 0.16.1 to 0.16.2 in the uv group (#2886) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/uv.lock b/uv.lock index 7391277d87d..576d256b541 100644 --- a/uv.lock +++ b/uv.lock @@ -195,27 +195,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.1" +version = "0.16.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, - { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, - { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, - { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, - { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, - { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, - { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, - { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, - { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, - { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, - { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, - { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, - { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, - { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, - { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, + { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, + { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, + { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, + { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, + { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, + { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, ] [[package]] From 619a8ffd7049cb39f26b3d944e3cc5f3a32ef4ea Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 11:29:14 -0700 Subject: [PATCH 07/29] fix(net): encode draft 17 message parameters by type (#2884) ## Summary - Encode draft 17+ JS message parameters using each parameter's defined value type instead of the legacy key-parity framing. - Preserve draft 14 through draft 16 behavior and add matching JS/Rust wire vectors for raw uint8 and Location values. - Preserve exact Location identifiers without JavaScript `Number` rounding, including full `uint64` values in drafts 17+. - Root cause: the JS codec continued inferring value framing from parameter ID parity after draft 17 introduced definition-specific value encodings. Its legacy Location cache also used number-only QUIC varint helpers despite exposing bigint identifiers. Closes #2864. ## Public API changes - Add `Varint.decodeBigInt` for exact QUIC varint decoding. - Allow `Varint.encode` to accept `bigint` as well as `number`. - `Parameters.largest` now preserves full `uint64` identifiers when using drafts 17+. - Bump `@moq/net` from 0.3.0 to 0.3.1. ## Test plan - [x] Original head: `just check` - [x] Original head: `bun test js/net/src/ietf/ietf.test.ts` (88 passed) - [x] Original head: `just rs test -p moq-net test_param_` (13 passed) - [x] Intermediate head `859a765`: GitHub Actions `Check` and `Test` - [x] Latest head `4890cbd`: GitHub Actions `Check` and `Test` - [ ] Draft 17 JS-to-Rust browser smoke test. Playwright Chromium crashed with SIGTRAP before loading the publisher, so no protocol session formed. ## Cross-package sync - Updated both `js/net` and `rs/moq-net` golden vectors. - No draft update is needed because this corrects the implementations to the existing draft 17 framing. - No other language package exposes the JS `Varint` API. Co-authored-by: GPT-5 --- bun.lock | 2 +- js/net/package.json | 2 +- js/net/src/ietf/ietf.test.ts | 91 ++++++++++++- js/net/src/ietf/parameters.ts | 211 ++++++++++++++++++++++++------ js/net/src/varint.test.ts | 9 ++ js/net/src/varint.ts | 30 +++-- rs/moq-net/src/ietf/parameters.rs | 54 +++++++- 7 files changed, 343 insertions(+), 56 deletions(-) diff --git a/bun.lock b/bun.lock index f6e4434f271..d214ae8f592 100644 --- a/bun.lock +++ b/bun.lock @@ -182,7 +182,7 @@ }, "js/net": { "name": "@moq/net", - "version": "0.3.0", + "version": "0.3.1", "dependencies": { "@moq/qmux": "^0.3.3", "@moq/signals": "workspace:*", diff --git a/js/net/package.json b/js/net/package.json index 861e47fd9c6..4192c369755 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -1,7 +1,7 @@ { "name": "@moq/net", "type": "module", - "version": "0.3.0", + "version": "0.3.1", "description": "The networking layer for Media over QUIC: real-time pub/sub with built-in caching, fan-out, and prioritization.", "license": "(MIT OR Apache-2.0)", "repository": "github:moq-dev/moq", diff --git a/js/net/src/ietf/ietf.test.ts b/js/net/src/ietf/ietf.test.ts index f7a4175d812..1e7282da2f9 100644 --- a/js/net/src/ietf/ietf.test.ts +++ b/js/net/src/ietf/ietf.test.ts @@ -6,7 +6,7 @@ import * as Varint from "../varint.ts"; import * as GoAway from "./goaway.ts"; import * as Namespace from "./namespace.ts"; import { Frame, Group, type GroupFlags } from "./object.ts"; -import { SetupOptions } from "./parameters.ts"; +import { Parameters, SetupOptions } from "./parameters.ts"; import { Publish, PublishDone } from "./publish.ts"; import * as Announce from "./publish_namespace.ts"; import { RequestError, RequestOk } from "./request.ts"; @@ -76,6 +76,95 @@ async function encodeFrameVersioned( return concatChunks(written); } +test("Message Parameters: uint8 wire encoding changes in draft 17", async () => { + const params = new Parameters(); + params.subscriberPriority = 255; + + expect(Array.from(await encodeVersioned(params, Version.DRAFT_16))).toEqual([ + 0x01, // parameter count + 0x20, // SUBSCRIBER_PRIORITY + 0x40, + 0xff, // QUIC varint 255 + ]); + + for (const version of [Version.DRAFT_17, Version.DRAFT_18, Version.DRAFT_19]) { + const expected = new Uint8Array([ + 0x01, // parameter count + 0x20, // SUBSCRIBER_PRIORITY + 0xff, // raw uint8 255 + ]); + expect(Array.from(await encodeVersioned(params, version))).toEqual(Array.from(expected)); + + const decoded = await decodeVersioned(expected, Parameters.decode, version); + expect(decoded.subscriberPriority).toBe(255); + } +}); + +test("Message Parameters: Location loses its length prefix in draft 17", async () => { + const params = new Parameters(); + params.largest = { groupId: 255n, objectId: 128n }; + + expect(Array.from(await encodeVersioned(params, Version.DRAFT_16))).toEqual([ + 0x01, // parameter count + 0x09, // LARGEST_OBJECT + 0x04, // byte-string length + 0x40, + 0xff, // QUIC varint 255 + 0x40, + 0x80, // QUIC varint 128 + ]); + + for (const version of [Version.DRAFT_17, Version.DRAFT_18, Version.DRAFT_19]) { + const expected = new Uint8Array([ + 0x01, // parameter count + 0x09, // LARGEST_OBJECT + 0x80, + 0xff, // leading-ones varint 255 + 0x80, + 0x80, // leading-ones varint 128 + ]); + expect(Array.from(await encodeVersioned(params, version))).toEqual(Array.from(expected)); + + const decoded = await decodeVersioned(expected, Parameters.decode, version); + expect(decoded.largest).toEqual({ groupId: 255n, objectId: 128n }); + } +}); + +test("Message Parameters: Location preserves bigint precision", async () => { + const largest = { groupId: 9007199254740993n, objectId: 9007199254740995n }; + const params = new Parameters(); + params.largest = largest; + + expect(params.largest).toEqual(largest); + + for (const version of [ + Version.DRAFT_14, + Version.DRAFT_15, + Version.DRAFT_16, + Version.DRAFT_17, + Version.DRAFT_18, + Version.DRAFT_19, + ]) { + const encoded = await encodeVersioned(params, version); + const decoded = await decodeVersioned(encoded, Parameters.decode, version); + expect(decoded.largest).toEqual(largest); + } +}); + +test("Message Parameters: Location preserves full uint64 values in draft 17", async () => { + const largest = { groupId: 2n ** 64n - 1n, objectId: 2n ** 62n }; + const params = new Parameters(); + params.largest = largest; + + expect(params.largest).toEqual(largest); + + for (const version of [Version.DRAFT_17, Version.DRAFT_18, Version.DRAFT_19]) { + const encoded = await encodeVersioned(params, version); + const decoded = await decodeVersioned(encoded, Parameters.decode, version); + expect(decoded.largest).toEqual(largest); + } +}); + // Subscribe tests (v14) test("Subscribe v14: round trip", async () => { const msg = new Subscribe.Subscribe({ diff --git a/js/net/src/ietf/parameters.ts b/js/net/src/ietf/parameters.ts index 1b265e3a251..4c16854726e 100644 --- a/js/net/src/ietf/parameters.ts +++ b/js/net/src/ietf/parameters.ts @@ -183,7 +183,8 @@ export class SetupOptions { } // ---- Message Parameters (used in Subscribe, Publish, Fetch, etc.) ---- -// Count-prefixed KVPs with delta-encoded keys (d16+). +// Count-prefixed KVPs through d16, then definition-specific Type-Value pairs. +// Parameter types are delta-encoded from d16 onward. // Varint parameter IDs (even) const MSG_PARAM_DELIVERY_TIMEOUT = 0x02n; @@ -198,17 +199,61 @@ const MSG_PARAM_GROUP_ORDER = 0x22n; const MSG_PARAM_LARGEST_OBJECT = 0x09n; const MSG_PARAM_SUBSCRIPTION_FILTER = 0x21n; -/// Message Parameters — count-prefixed KVPs used in control messages. +type MessageParamKind = "varint" | "uint8" | "bool" | "location" | "bytes"; +type MessageLocation = { groupId: bigint; objectId: bigint }; + +function getMessageParamKind(id: bigint): MessageParamKind { + switch (id) { + case MSG_PARAM_DELIVERY_TIMEOUT: + case MSG_PARAM_MAX_CACHE_DURATION: + case MSG_PARAM_EXPIRES: + return "varint"; + case MSG_PARAM_PUBLISHER_PRIORITY: + case MSG_PARAM_SUBSCRIBER_PRIORITY: + case MSG_PARAM_GROUP_ORDER: + return "uint8"; + case MSG_PARAM_FORWARD: + return "bool"; + case MSG_PARAM_LARGEST_OBJECT: + return "location"; + case MSG_PARAM_SUBSCRIPTION_FILTER: + return "bytes"; + default: + throw new Error(`unknown message parameter id: ${id.toString()}`); + } +} + +function decodeLocation(data: Uint8Array): MessageLocation { + const [groupId, objectData] = Varint.decodeBigInt(data); + const [objectId, trailing] = Varint.decodeBigInt(objectData); + if (trailing.length !== 0) { + throw new Error("trailing bytes in message parameter Location"); + } + return { groupId, objectId }; +} + +function encodeLocation({ groupId, objectId }: MessageLocation): Uint8Array { + const group = Varint.encode(groupId); + const object = Varint.encode(objectId); + const combined = new Uint8Array(group.length + object.length); + combined.set(group, 0); + combined.set(object, group.length); + return combined; +} + +/** Message parameters used in control messages. */ export class Parameters { vars: Map; bytes: Map; + #locations: Map; constructor() { this.vars = new Map(); this.bytes = new Map(); + this.#locations = new Map(); } - // --- Varint accessors --- + // --- Numeric accessors --- get subscriberPriority(): number | undefined { const v = this.vars.get(MSG_PARAM_SUBSCRIBER_PRIORITY); @@ -272,21 +317,13 @@ export class Parameters { // --- Bytes accessors --- - get largest(): { groupId: bigint; objectId: bigint } | undefined { - const data = this.bytes.get(MSG_PARAM_LARGEST_OBJECT); - if (!data || data.length === 0) return undefined; - const [groupId, rest] = Varint.decode(data); - const [objectId] = Varint.decode(rest); - return { groupId: BigInt(groupId), objectId: BigInt(objectId) }; + get largest(): MessageLocation | undefined { + const location = this.#locations.get(MSG_PARAM_LARGEST_OBJECT); + return location && { ...location }; } - set largest(v: { groupId: bigint; objectId: bigint }) { - const buf1 = Varint.encode(Number(v.groupId)); - const buf2 = Varint.encode(Number(v.objectId)); - const combined = new Uint8Array(buf1.length + buf2.length); - combined.set(buf1, 0); - combined.set(buf2, buf1.length); - this.bytes.set(MSG_PARAM_LARGEST_OBJECT, combined); + set largest(v: MessageLocation) { + this.#locations.set(MSG_PARAM_LARGEST_OBJECT, { ...v }); } get subscriptionFilter(): number | undefined { @@ -301,7 +338,7 @@ export class Parameters { } async encode(w: Writer, version: IetfVersion) { - await w.u53(this.vars.size + this.bytes.size); + await w.u53(this.vars.size + this.bytes.size + this.#locations.size); if (version === Version.DRAFT_14 || version === Version.DRAFT_15) { for (const [id, value] of this.vars) { @@ -314,28 +351,83 @@ export class Parameters { await w.u53(value.length); await w.write(value); } + + for (const [id, value] of this.#locations) { + const encoded = encodeLocation(value); + await w.u62(id); + await w.u53(encoded.length); + await w.write(encoded); + } } else { - // d16+: Delta encoding, merge vars and bytes, sort by key - const all: { key: bigint; isVar: boolean }[] = []; - for (const id of this.vars.keys()) all.push({ key: id, isVar: true }); - for (const id of this.bytes.keys()) all.push({ key: id, isVar: false }); + // d16+: Delta encoding, merge all parameter storage, sort by key + const all: { key: bigint; storage: "var" | "bytes" | "location" }[] = []; + for (const id of this.vars.keys()) all.push({ key: id, storage: "var" }); + for (const id of this.bytes.keys()) all.push({ key: id, storage: "bytes" }); + for (const id of this.#locations.keys()) all.push({ key: id, storage: "location" }); all.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); let prevId = 0n; for (let i = 0; i < all.length; i++) { - const { key, isVar } = all[i]; + const { key, storage } = all[i]; const delta = i === 0 ? key : key - prevId; prevId = key; await w.u62(delta); - if (isVar) { - // biome-ignore lint/style/noNonNullAssertion: key is guaranteed to exist in vars map - await w.u62(this.vars.get(key)!); - } else { - // biome-ignore lint/style/noNonNullAssertion: key is guaranteed to exist in bytes map - const value = this.bytes.get(key)!; - await w.u53(value.length); - await w.write(value); + if (version === Version.DRAFT_16) { + if (storage === "var") { + // biome-ignore lint/style/noNonNullAssertion: key is guaranteed to exist in vars map + await w.u62(this.vars.get(key)!); + } else { + const value = + storage === "bytes" + ? // biome-ignore lint/style/noNonNullAssertion: key is guaranteed to exist in bytes map + this.bytes.get(key)! + : // biome-ignore lint/style/noNonNullAssertion: key is guaranteed to exist in locations map + encodeLocation(this.#locations.get(key)!); + await w.u53(value.length); + await w.write(value); + } + continue; + } + + switch (getMessageParamKind(key)) { + case "varint": { + const value = this.vars.get(key); + if (value === undefined) throw new Error(`invalid varint message parameter: ${key.toString()}`); + await w.u62(value); + break; + } + case "uint8": { + const value = this.vars.get(key); + if (value === undefined || value < 0n || value > 0xffn) { + throw new Error(`invalid uint8 message parameter: ${key.toString()}`); + } + await w.u8(Number(value)); + break; + } + case "bool": { + const value = this.vars.get(key); + if (value !== 0n && value !== 1n) { + throw new Error(`invalid bool message parameter: ${key.toString()}`); + } + await w.bool(value === 1n); + break; + } + case "location": { + const location = this.#locations.get(key); + if (location === undefined) + throw new Error(`invalid Location message parameter: ${key.toString()}`); + await w.u62(location.groupId); + await w.u62(location.objectId); + break; + } + case "bytes": { + const value = this.bytes.get(key); + if (value === undefined) throw new Error(`invalid bytes message parameter: ${key.toString()}`); + await w.u53(value.length); + await w.write(value); + break; + } } } } @@ -358,19 +450,56 @@ export class Parameters { prevType = id; } - if (id % 2n === 0n) { - if (params.vars.has(id)) { - throw new Error(`duplicate message parameter id: ${id.toString()}`); + if (version === Version.DRAFT_14 || version === Version.DRAFT_15 || version === Version.DRAFT_16) { + if (id % 2n === 0n) { + if (params.vars.has(id)) { + throw new Error(`duplicate message parameter id: ${id.toString()}`); + } + const varint = await r.u62(); + params.vars.set(id, varint); + } else { + const size = await r.u53(); + const bytes = await r.read(size); + if (id === MSG_PARAM_LARGEST_OBJECT) { + if (params.#locations.has(id)) { + throw new Error(`duplicate message parameter id: ${id.toString()}`); + } + params.#locations.set(id, decodeLocation(bytes)); + } else { + if (params.bytes.has(id)) { + throw new Error(`duplicate message parameter id: ${id.toString()}`); + } + params.bytes.set(id, bytes); + } + } + continue; + } + + if (params.vars.has(id) || params.bytes.has(id) || params.#locations.has(id)) { + throw new Error(`duplicate message parameter id: ${id.toString()}`); + } + + switch (getMessageParamKind(id)) { + case "varint": + params.vars.set(id, await r.u62()); + break; + case "uint8": + params.vars.set(id, BigInt(await r.u8())); + break; + case "bool": + params.vars.set(id, (await r.bool()) ? 1n : 0n); + break; + case "location": { + const groupId = await r.u62(); + const objectId = await r.u62(); + params.#locations.set(id, { groupId, objectId }); + break; } - const varint = await r.u62(); - params.vars.set(id, varint); - } else { - if (params.bytes.has(id)) { - throw new Error(`duplicate message parameter id: ${id.toString()}`); + case "bytes": { + const size = await r.u53(); + params.bytes.set(id, await r.read(size)); + break; } - const size = await r.u53(); - const bytes = await r.read(size); - params.bytes.set(id, bytes); } } diff --git a/js/net/src/varint.test.ts b/js/net/src/varint.test.ts index fcf5a9f25de..a0b68944428 100644 --- a/js/net/src/varint.test.ts +++ b/js/net/src/varint.test.ts @@ -53,6 +53,15 @@ test("Varint encode/decode roundtrip - 8 byte values (1073741824+)", () => { } }); +test("Varint bigint roundtrip preserves 62-bit precision", () => { + for (const value of [9007199254740993n, 2n ** 62n - 1n]) { + const encoded = Varint.encode(value); + const [decoded, remaining] = Varint.decodeBigInt(encoded); + expect(decoded).toBe(value); + expect(remaining.byteLength).toBe(0); + } +}); + test("Varint size calculation", () => { expect(Varint.size(0)).toBe(1); expect(Varint.size(63)).toBe(1); diff --git a/js/net/src/varint.ts b/js/net/src/varint.ts index 3f2e375a1ca..afeb66ec722 100644 --- a/js/net/src/varint.ts +++ b/js/net/src/varint.ts @@ -251,18 +251,18 @@ export function encodeTo(dst: ArrayBuffer, v: number | bigint): Uint8Array { } /** - * Encodes a number as a QUIC variable-length integer. + * Encodes a number or bigint as a QUIC variable-length integer. * Returns a new Uint8Array containing the encoded bytes. */ -export function encode(v: number): Uint8Array { +export function encode(v: number | bigint): Uint8Array { return encodeTo(new ArrayBuffer(8), v); } /** - * Decodes a QUIC variable-length integer from a buffer. + * Decodes a QUIC variable-length integer exactly from a buffer. * Returns a tuple of [value, remaining buffer]. */ -export function decode(buf: Uint8Array): [number, Uint8Array] { +export function decodeBigInt(buf: Uint8Array): [bigint, Uint8Array] { if (buf.length === 0) { throw new Error("buffer is empty"); } @@ -276,20 +276,28 @@ export function decode(buf: Uint8Array): [number, Uint8Array] { const view = new DataView(buf.buffer, buf.byteOffset, size); const remain = buf.subarray(size); - let v: number; + let value: bigint; if (size === 1) { - v = buf[0] & 0x3f; + value = BigInt(buf[0] & 0x3f); } else if (size === 2) { - v = view.getUint16(0) & 0x3fff; + value = BigInt(view.getUint16(0) & 0x3fff); } else if (size === 4) { - v = view.getUint32(0) & 0x3fffffff; + value = BigInt(view.getUint32(0) & 0x3fffffff); } else if (size === 8) { - // NOTE: Precision loss above 2^53, but we're using number type - v = Number(view.getBigUint64(0) & 0x3fffffffffffffffn); + value = view.getBigUint64(0) & 0x3fffffffffffffffn; } else { throw new Error("impossible"); } - return [v, remain]; + return [value, remain]; +} + +/** + * Decodes a QUIC variable-length integer from a buffer. + * Values above 53 bits lose precision; use {@link decodeBigInt} for exact decoding. + */ +export function decode(buf: Uint8Array): [number, Uint8Array] { + const [value, remain] = decodeBigInt(buf); + return [Number(value), remain]; } diff --git a/rs/moq-net/src/ietf/parameters.rs b/rs/moq-net/src/ietf/parameters.rs index 7b6a8040c7b..0e5166d10f0 100644 --- a/rs/moq-net/src/ietf/parameters.rs +++ b/rs/moq-net/src/ietf/parameters.rs @@ -527,7 +527,7 @@ macro_rules! decode_params { #[cfg(test)] mod tests { use super::*; - use bytes::{Buf, BytesMut}; + use bytes::{Buf, Bytes, BytesMut}; // ---- Setup Parameters tests (unchanged) ---- @@ -642,6 +642,30 @@ mod tests { } } + #[test] + fn test_param_u8_wire_vectors() -> Result<(), EncodeError> { + for (version, expected) in [ + (Version::Draft16, &[0x01, 0x20, 0x40, 0xff][..]), + (Version::Draft17, &[0x01, 0x20, 0xff][..]), + (Version::Draft18, &[0x01, 0x20, 0xff][..]), + (Version::Draft19, &[0x01, 0x20, 0xff][..]), + ] { + let mut buf = BytesMut::new(); + encode_params!(&mut buf, version, 0x20 => u8::MAX); + assert_eq!(&buf[..], expected, "{version}"); + + let mut encoded = Bytes::copy_from_slice(expected); + let decoded = (|| -> Result, DecodeError> { + decode_params!(&mut encoded, version, 0x20 => value: Option); + Ok(value) + })() + .expect("fixed uint8 vector should decode"); + assert_eq!(decoded, Some(u8::MAX), "{version}"); + assert!(!encoded.has_remaining(), "{version}"); + } + Ok(()) + } + #[test] fn test_param_bool_all_versions() { for version in [ @@ -691,6 +715,34 @@ mod tests { } } + #[test] + fn test_param_location_wire_vectors() -> Result<(), EncodeError> { + let location = Location { + group: 255, + object: 128, + }; + for (version, expected) in [ + (Version::Draft16, &[0x01, 0x09, 0x04, 0x40, 0xff, 0x40, 0x80][..]), + (Version::Draft17, &[0x01, 0x09, 0x80, 0xff, 0x80, 0x80][..]), + (Version::Draft18, &[0x01, 0x09, 0x80, 0xff, 0x80, 0x80][..]), + (Version::Draft19, &[0x01, 0x09, 0x80, 0xff, 0x80, 0x80][..]), + ] { + let mut buf = BytesMut::new(); + encode_params!(&mut buf, version, 0x09 => location.clone()); + assert_eq!(&buf[..], expected, "{version}"); + + let mut encoded = Bytes::copy_from_slice(expected); + let decoded = (|| -> Result, DecodeError> { + decode_params!(&mut encoded, version, 0x09 => value: Option); + Ok(value) + })() + .expect("fixed Location vector should decode"); + assert_eq!(decoded, Some(location.clone()), "{version}"); + assert!(!encoded.has_remaining(), "{version}"); + } + Ok(()) + } + #[test] fn test_param_filter_type_all_versions() { for version in [ From c391176277ba8b3da727b09cabd3905333201f77 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 11:40:20 -0700 Subject: [PATCH 08/29] fix(video): validate PipeWire chunk ranges and accept NV12 (#2871) Co-authored-by: Claude Opus 5 Co-authored-by: OpenAI Codex --- rs/moq-video/src/capture/mod.rs | 5 + rs/moq-video/src/capture/pipewire.rs | 627 +++++++++++++++++++++++++-- rs/moq-video/src/encode/producer.rs | 2 + rs/moq-video/src/frame.rs | 8 +- 4 files changed, 605 insertions(+), 37 deletions(-) diff --git a/rs/moq-video/src/capture/mod.rs b/rs/moq-video/src/capture/mod.rs index a594d2d5b44..6b0dc816939 100644 --- a/rs/moq-video/src/capture/mod.rs +++ b/rs/moq-video/src/capture/mod.rs @@ -286,6 +286,11 @@ impl FrameStream { self.framerate } + /// The first frame's declared color space, when its capture backend knows it. + pub(crate) fn color(&self) -> Option { + self.pending.as_ref().and_then(Surface::color) + } + pub(crate) fn device(&self) -> &str { &self.device } diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index f10caa71cb1..72eeaf192ed 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -3,8 +3,9 @@ //! The ScreenCast portal owns source selection: [`open`] pops the compositor's //! picker dialog, the user chooses a monitor, and the portal hands us a PipeWire //! fd + node id. A dedicated thread then runs the PipeWire main loop, converting -//! each RGB frame to CPU [`I420`] and pushing it into the shared [`FrameChannel`] -//! (callback-driven like the macOS delegate, not a pull-style pump). +//! each packed RGB or NV12 frame to CPU [`I420`] and pushing it into the shared +//! [`FrameChannel`] (callback-driven like the macOS delegate, not a pull-style +//! pump). //! //! Two quirks worth knowing: //! - `publish_capture` releases the capture while unwatched and reopens it on @@ -18,6 +19,7 @@ //! the encoder. A loop timer re-emits the last frame whenever a frame interval //! passes without a fresh one, mirroring the Windows Desktop Duplication pacing. +use std::borrow::Cow; use std::cell::RefCell; use std::os::fd::OwnedFd; use std::rc::Rc; @@ -34,8 +36,8 @@ use spa::param::video::{VideoFormat, VideoInfoRaw}; use super::channel::FrameChannel; use super::pump::Geometry; use super::{Config, FrameStream}; -use crate::Error; use crate::frame::{I420, Surface}; +use crate::{Color, Error, Size}; const DEFAULT_FRAMERATE: u32 = 30; /// The compositor sends the negotiated format right after the stream connects; @@ -75,6 +77,7 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result, + /// The NV12 color space used to configure the encoder. + color: Option, geo_tx: Option>>, /// Most recent converted frame, re-emitted while the screen is static. last: Option, @@ -322,7 +326,7 @@ fn run_loop( } let mut state = state.borrow_mut(); - if let Err(e) = state.format.parse(param) { + if let Err(e) = replace_video_format(&mut state.format, |format| format.parse(param)) { tracing::warn!(error = %e, "failed to parse pipewire video format"); return; } @@ -335,9 +339,25 @@ fn run_loop( tracing::warn!(width = size.width, height = size.height, "unusable capture size"); return; } + let color = match pipewire_color(state.format, width, height) { + Ok(color) => color, + Err(e) => { + match state.geo_tx.take() { + Some(tx) => drop(tx.send(Err(e))), + None => { + tracing::warn!(error = %e, "unsupported pipewire video color space"); + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + } + return; + } + }; if let Some(tx) = state.geo_tx.take() { state.geometry = Some((width, height)); + state.color = color; // The compositor reports 0/1 for a variable rate; only a real // rate is worth forwarding to the encoder. let fr = state.format.framerate(); @@ -348,11 +368,10 @@ fn run_loop( framerate, device: format!("pipewire:{node_id}"), })); - } else if state.geometry != Some((width, height)) { - // Renegotiated to a new size (e.g. the monitor changed mode). - // End the stream; the encode loop reopens at the new geometry - // and the restore token skips the picker. - tracing::info!(width, height, "capture size changed; restarting the stream"); + } else if format_requires_restart(state.geometry, state.color, width, height, color) { + // The encoder's geometry and VUI are fixed when it opens. End the + // stream so the encode loop reopens it with the new format. + tracing::info!(width, height, ?color, "capture format changed; restarting the stream"); if let Some(mainloop) = mainloop.upgrade() { mainloop.quit(); } @@ -366,20 +385,66 @@ fn run_loop( move |stream, _| { let mut state = state.borrow_mut(); let Some((width, height)) = state.geometry else { return }; + let source_height = state.format.size().height; + let color = state.color; let Some(mut buffer) = stream.dequeue_buffer() else { return; }; let datas = buffer.datas_mut(); let Some(data) = datas.first_mut() else { return }; - let offset = data.chunk().offset() as usize; - let size = data.chunk().size() as usize; - let stride = data.chunk().stride(); - // Compositors mark skipped frames as empty or corrupted; drop those - // rather than treating them as a fatal conversion failure below. - if size == 0 || data.chunk().flags().contains(spa::buffer::ChunkFlags::CORRUPTED) { + let maxsize = data.as_raw().maxsize; + let size = clamp_chunk_size(data.chunk().size(), maxsize); + match chunk_kind(size, data.chunk().flags()) { + ChunkKind::Data => {} + ChunkKind::Empty => { + let Ok(frame) = neutral_frame(width, height, color) else { + return; + }; + chan.push(Surface::I420(frame.clone())); + state.last = Some(frame); + state.fresh = true; + return; + } + ChunkKind::Invalid => return, + } + let Some(offset) = normalize_chunk_offset(data.chunk().offset(), maxsize) else { + tracing::warn!("pipewire buffer has zero maximum size"); + return; + }; + // Fall back to the unclamped source width: for an odd-width source + // the real row is one pixel wider than the clamped `width`. The + // packing differs per format, so NV12 cannot assume 4 bytes/pixel. + let stride = match u32::try_from(data.chunk().stride()) { + Ok(stride) if stride > 0 => stride, + _ => match state.format.format() { + VideoFormat::NV12 => state.format.size().width, + _ => state.format.size().width.saturating_mul(4), + }, + }; + let layout = FrameLayout { + stride, + width, + height, + source_height, + }; + + // The chunk only says how many bytes the producer wrote. Check it + // actually spans every row `convert` will sample, so a short or + // mislabeled buffer is dropped here rather than read past. + let Some(required) = frame_data_size(state.format.format(), layout) else { + tracing::warn!("pipewire frame layout overflows its buffer"); + return; + }; + if required > size || required > maxsize as usize { + tracing::warn!( + required, + available = size, + "pipewire chunk does not contain a complete frame" + ); return; } + // Without dmabuf modifiers in our format offer the compositor uses // shared memory, which MAP_BUFFERS mmaps for us; `None` here means // it forced something we can't read, so give up cleanly. @@ -390,18 +455,14 @@ fn run_loop( } return; }; - let Some(bytes) = bytes.get(offset..offset + size) else { + let Some(allocation) = bytes.get(..maxsize as usize) else { return; }; - // Fall back to the unclamped source width: for an odd-width source - // the real row is one pixel wider than the clamped `width`. - let stride = if stride > 0 { - stride as u32 - } else { - state.format.size().width * 4 + let Some(bytes) = chunk_bytes(allocation, offset, required) else { + return; }; - match convert(state.format.format(), bytes, stride, width, height) { + match convert(state.format.format(), bytes.as_ref(), layout, color) { Ok(i420) => { chan.push(Surface::I420(i420.clone())); state.last = Some(i420); @@ -470,19 +531,262 @@ fn run_loop( Ok(()) } -/// Convert one strided RGB screen frame to tightly-packed I420. -fn convert(format: VideoFormat, bytes: &[u8], stride: u32, width: u32, height: u32) -> Result { +/// The geometry `convert` needs: the producer's row stride and source height, +/// plus the even-clamped output size. +#[derive(Clone, Copy)] +struct FrameLayout { + stride: u32, + width: u32, + height: u32, + /// Unclamped source height. NV12's chroma plane starts after this many luma + /// rows, which is one more than `height` for an odd-height source. + source_height: u32, +} + +const CHUNK_FLAG_EMPTY: i32 = 1 << 1; + +#[derive(Debug, PartialEq, Eq)] +enum ChunkKind { + Data, + Empty, + Invalid, +} + +fn chunk_kind(size: usize, flags: spa::buffer::ChunkFlags) -> ChunkKind { + if flags.contains(spa::buffer::ChunkFlags::CORRUPTED) { + ChunkKind::Invalid + } else if flags.bits() & CHUNK_FLAG_EMPTY != 0 { + ChunkKind::Empty + } else if size == 0 { + ChunkKind::Invalid + } else { + ChunkKind::Data + } +} + +fn neutral_frame(width: u32, height: u32, color: Option) -> Result { + let color = color.unwrap_or_else(|| Color::infer(Size::new(width, height))); + let luma = width as usize * height as usize; + let mut data = vec![128; I420::len(width, height)]; + data[..luma].fill(if color.limited() { 16 } else { 0 }); + Ok(I420::new(width, height, data)?.with_color(color)) +} + +/// Parse into a zeroed value before replacing the current format. libspa leaves +/// omitted optional properties untouched, so parsing into the reused value +/// would retain stale color metadata across renegotiation. +fn replace_video_format( + current: &mut VideoInfoRaw, + parse: impl FnOnce(&mut VideoInfoRaw) -> Result, +) -> Result { + let mut next = VideoInfoRaw::default(); + let result = parse(&mut next)?; + *current = next; + Ok(result) +} + +/// Preserve the color description that names NV12 samples. Unknown fields use +/// the same size-based, limited-range fallback as the encoder. Reject matrices +/// the crate cannot represent rather than writing a false 601/709 VUI. +fn pipewire_color(format: VideoInfoRaw, width: u32, height: u32) -> Result, Error> { + if format.format() != VideoFormat::NV12 { + return Ok(None); + } + let size = Size::new(width, height); + let color = color_from_pipewire(format.color_range(), format.color_matrix(), size)?; + validate_pipewire_description( + color.unwrap_or_else(|| Color::infer(size)), + format.color_primaries(), + format.transfer_function(), + )?; + Ok(color) +} + +fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result, Error> { + if range == spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN && matrix == spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN { + return Ok(None); + } + + let limited = match range { + spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN | spa::sys::SPA_VIDEO_COLOR_RANGE_16_235 => true, + spa::sys::SPA_VIDEO_COLOR_RANGE_0_255 => false, + _ => { + return Err(Error::Codec(anyhow::anyhow!( + "unsupported PipeWire NV12 color range {range}" + ))); + } + }; + let bt709 = match matrix { + spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN => { + matches!(Color::infer(size), Color::Bt709Limited | Color::Bt709Full) + } + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709 => true, + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT601 => false, + _ => { + return Err(Error::Codec(anyhow::anyhow!( + "unsupported PipeWire NV12 color matrix {matrix}" + ))); + } + }; + + Ok(Some(match (bt709, limited) { + (false, true) => Color::Bt601Limited, + (false, false) => Color::Bt601Full, + (true, true) => Color::Bt709Limited, + (true, false) => Color::Bt709Full, + })) +} + +fn validate_pipewire_description(color: Color, primaries: u32, transfer: u32) -> Result<(), Error> { + let expected_primaries = match color { + Color::Bt601Limited | Color::Bt601Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_SMPTE170M, + Color::Bt709Limited | Color::Bt709Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709, + }; + if primaries != spa::sys::SPA_VIDEO_COLOR_PRIMARIES_UNKNOWN && primaries != expected_primaries { + return Err(Error::Codec(anyhow::anyhow!( + "PipeWire NV12 primaries {primaries} do not match the negotiated matrix" + ))); + } + if !matches!( + transfer, + spa::sys::SPA_VIDEO_TRANSFER_UNKNOWN + | spa::sys::SPA_VIDEO_TRANSFER_BT709 + | spa::sys::SPA_VIDEO_TRANSFER_BT601 + | spa::sys::SPA_VIDEO_TRANSFER_BT2020_10 + ) { + return Err(Error::Codec(anyhow::anyhow!( + "unsupported PipeWire NV12 transfer function {transfer}" + ))); + } + Ok(()) +} + +fn format_requires_restart( + geometry: Option<(u32, u32)>, + color: Option, + width: u32, + height: u32, + next_color: Option, +) -> bool { + geometry.is_some_and(|geometry| geometry != (width, height) || color != next_color) +} + +/// A chunk offset is a ring position within the allocation, so wrap it rather +/// than trusting it to be in range. `None` means the allocation is unusable. +fn normalize_chunk_offset(offset: u32, maxsize: u32) -> Option { + (maxsize != 0).then(|| (offset % maxsize) as usize) +} + +fn clamp_chunk_size(size: u32, maxsize: u32) -> usize { + size.min(maxsize) as usize +} + +fn chunk_bytes(data: &[u8], offset: usize, size: usize) -> Option> { + if offset >= data.len() || size > data.len() { + return None; + } + let end = offset.checked_add(size)?; + if end <= data.len() { + return Some(Cow::Borrowed(&data[offset..end])); + } + + let mut wrapped = Vec::with_capacity(size); + wrapped.extend_from_slice(&data[offset..]); + let head = size - wrapped.len(); + wrapped.extend_from_slice(&data[..head]); + Some(Cow::Owned(wrapped)) +} + +/// Bytes from the chunk start through the span required by `convert`. NV12 stops +/// at the visible width of its final row; packed RGB requires every full stride. +fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { + let stride = layout.stride as usize; + let width = layout.width as usize; + let height = layout.height as usize; + let row_size = match format { + VideoFormat::NV12 => width, + VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => width.checked_mul(4)?, + _ => return None, + }; + if stride < row_size { + return None; + } + + match format { + // Chroma follows every source luma row, then runs at half height. + VideoFormat::NV12 => (layout.source_height as usize) + .checked_add(height / 2)? + .checked_sub(1)? + .checked_mul(stride)? + .checked_add(row_size), + _ => stride.checked_mul(height), + } +} + +/// Deinterleave strided NV12 into the crate's tightly packed I420 layout. +fn nv12_to_i420(data: &[u8], layout: FrameLayout) -> Result { + let (stride, width, height, source_height) = ( + layout.stride as usize, + layout.width as usize, + layout.height as usize, + layout.source_height as usize, + ); + if source_height < height { + return Err(Error::Codec(anyhow::anyhow!( + "NV12 source is shorter than the cropped output" + ))); + } + let y_len = stride + .checked_mul(source_height) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 luma size overflow")))?; + let uv_rows = height / 2; + let uv_len = uv_rows + .checked_sub(1) + .and_then(|rows| rows.checked_mul(stride)) + .and_then(|offset| offset.checked_add(width)) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 chroma size overflow")))?; + let frame_len = y_len + .checked_add(uv_len) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 frame size overflow")))?; + if stride < width || data.len() < frame_len { + return Err(Error::Codec(anyhow::anyhow!( + "NV12 frame is shorter than its declared rows" + ))); + } + + let mut packed = vec![0; I420::len(width as u32, height as u32)]; + for row in 0..height { + packed[row * width..(row + 1) * width].copy_from_slice(&data[row * stride..row * stride + width]); + } + let uv = &data[y_len..frame_len]; + let packed_uv = width * height; + for row in 0..uv_rows { + packed[packed_uv + row * width..packed_uv + (row + 1) * width] + .copy_from_slice(&uv[row * stride..row * stride + width]); + } + I420::from_nv12(&packed, width as u32, height as u32) +} + +/// Convert one strided screen frame to tightly-packed I420. +fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option) -> Result { match format { - VideoFormat::BGRx | VideoFormat::BGRA => I420::from_bgra(bytes, stride, width, height), - VideoFormat::RGBx | VideoFormat::RGBA => I420::from_rgba(bytes, stride, width, height), + VideoFormat::NV12 => { + let frame = nv12_to_i420(bytes, layout)?; + Ok(match color { + Some(color) => frame.with_color(color), + None => frame, + }) + } + VideoFormat::BGRx | VideoFormat::BGRA => I420::from_bgra(bytes, layout.stride, layout.width, layout.height), + VideoFormat::RGBx | VideoFormat::RGBA => I420::from_rgba(bytes, layout.stride, layout.width, layout.height), other => Err(Error::Codec(anyhow::anyhow!( "pipewire negotiated an unsupported video format {other:?}" ))), } } -/// Serialize the `EnumFormat` pod offering the RGB layouts we can convert, -/// any size, and a framerate range preferring `framerate`. +/// Serialize the `EnumFormat` pod offering the layouts we can convert (packed +/// RGB first, then NV12), any size, and a framerate range preferring `framerate`. fn format_offer(framerate: u32) -> Vec { let obj = spa::pod::object!( spa::utils::SpaTypes::ObjectParamFormat, @@ -507,6 +811,7 @@ fn format_offer(framerate: u32) -> Vec { VideoFormat::BGRA, VideoFormat::RGBx, VideoFormat::RGBA, + VideoFormat::NV12, ), spa::pod::property!( spa::param::format::FormatProperties::VideoSize, @@ -547,11 +852,267 @@ mod tests { use super::*; use crate::capture::Config; - /// The serialized format offer must parse back as a valid pod. + /// The serialized format offer must parse back as a valid pod, carrying every + /// layout `convert` knows how to handle. #[test] fn format_offer_is_valid_pod() { let bytes = format_offer(30); - assert!(spa::pod::Pod::from_bytes(&bytes).is_some(), "offer did not round-trip"); + let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(&bytes) + .expect("format offer did not round-trip"); + assert!(remaining.is_empty()); + let spa::pod::Value::Object(object) = value else { + panic!("format offer is not an object"); + }; + let property = object + .properties + .iter() + .find(|property| property.key == spa::param::format::FormatProperties::VideoFormat.as_raw()) + .expect("missing video format property"); + assert_eq!( + property.value, + spa::pod::Value::Choice(spa::pod::ChoiceValue::Id(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Enum { + default: spa::utils::Id(VideoFormat::BGRx.as_raw()), + alternatives: vec![ + spa::utils::Id(VideoFormat::BGRx.as_raw()), + spa::utils::Id(VideoFormat::BGRA.as_raw()), + spa::utils::Id(VideoFormat::RGBx.as_raw()), + spa::utils::Id(VideoFormat::RGBA.as_raw()), + spa::utils::Id(VideoFormat::NV12.as_raw()), + ], + }, + ))) + ); + } + + #[test] + fn chunk_offset_wraps_to_the_allocation() { + assert_eq!(normalize_chunk_offset(18, 16), Some(2)); + assert_eq!(normalize_chunk_offset(0, 0), None); + } + + #[test] + fn chunk_size_is_clamped_to_the_allocation() { + assert_eq!(clamp_chunk_size(18, 16), 16); + assert_eq!(clamp_chunk_size(8, 16), 8); + } + + #[test] + fn chunk_flags_distinguish_neutral_and_invalid_frames() { + let empty = spa::buffer::ChunkFlags::from_bits_retain(CHUNK_FLAG_EMPTY); + assert_eq!(chunk_kind(0, spa::buffer::ChunkFlags::empty()), ChunkKind::Invalid); + assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::CORRUPTED), ChunkKind::Invalid); + assert_eq!(chunk_kind(1, empty), ChunkKind::Empty); + assert_eq!(chunk_kind(0, empty), ChunkKind::Empty); + assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::empty()), ChunkKind::Data); + } + + #[test] + fn empty_chunk_is_limited_range_black() { + let frame = neutral_frame(4, 2, None).unwrap(); + assert_eq!(frame.y(), &[16; 8]); + assert_eq!(frame.u(), &[128; 2]); + assert_eq!(frame.v(), &[128; 2]); + } + + #[test] + fn negotiated_nv12_color_overrides_size_inference() { + let size = Size::new(1920, 1080); + assert_eq!( + color_from_pipewire( + spa::sys::SPA_VIDEO_COLOR_RANGE_0_255, + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT601, + size, + ) + .unwrap(), + Some(Color::Bt601Full) + ); + assert_eq!( + color_from_pipewire( + spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN, + spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN, + size, + ) + .unwrap(), + None + ); + assert!( + color_from_pipewire( + spa::sys::SPA_VIDEO_COLOR_RANGE_16_235, + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT2020, + size, + ) + .is_err() + ); + let mut format = VideoInfoRaw::default(); + format.set_format(VideoFormat::NV12); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_16_235); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709); + format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT2020); + format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_BT709); + assert!(pipewire_color(format, 1920, 1080).is_err()); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN); + format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_SMPTE2084); + assert!(pipewire_color(format, 1920, 1080).is_err()); + + let layout = FrameLayout { + stride: 4, + width: 4, + height: 2, + source_height: 2, + }; + let frame = convert( + VideoFormat::NV12, + &[16, 16, 16, 16, 16, 16, 16, 16, 128, 128, 128, 128], + layout, + Some(Color::Bt601Full), + ) + .unwrap(); + assert_eq!(frame.color(), Some(Color::Bt601Full)); + } + + #[test] + fn color_renegotiation_restarts_the_capture() { + let geometry = Some((1920, 1080)); + assert!(!format_requires_restart( + geometry, + Some(Color::Bt709Limited), + 1920, + 1080, + Some(Color::Bt709Limited), + )); + assert!(format_requires_restart( + geometry, + Some(Color::Bt709Limited), + 1920, + 1080, + Some(Color::Bt709Full), + )); + } + + #[test] + fn omitted_color_fields_reset_on_renegotiation() { + let mut format = VideoInfoRaw::default(); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_0_255); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709); + replace_video_format(&mut format, |next| { + next.set_format(VideoFormat::NV12); + Ok::<_, std::convert::Infallible>(()) + }) + .unwrap(); + assert_eq!(format.color_range(), spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN); + assert_eq!(format.color_matrix(), spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN); + } + + #[test] + fn empty_chunk_uses_full_range_black() { + let frame = neutral_frame(4, 2, Some(Color::Bt709Full)).unwrap(); + assert_eq!(frame.y(), &[0; 8]); + assert_eq!(frame.u(), &[128; 2]); + assert_eq!(frame.v(), &[128; 2]); + assert_eq!(frame.color(), Some(Color::Bt709Full)); + } + + /// The required size must reach the last sampled row, but not the padding + /// past it, or a tightly-sized final row would be rejected. + #[test] + fn frame_data_size_covers_every_sampled_row() { + let packed = FrameLayout { + stride: 20, + width: 4, + height: 2, + source_height: 2, + }; + assert_eq!(frame_data_size(VideoFormat::BGRx, packed), Some(40)); + + let nv12 = FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 3, + }; + assert_eq!(frame_data_size(VideoFormat::NV12, nv12), Some(22)); + + // A stride narrower than one row means the layout is nonsense. + assert_eq!( + frame_data_size(VideoFormat::BGRx, FrameLayout { stride: 15, ..packed }), + None + ); + } + + #[test] + fn wrapped_chunk_is_reassembled() { + let data = [0, 1, 2, 3, 4, 5]; + assert_eq!(chunk_bytes(&data, 1, 3).as_deref(), Some([1, 2, 3].as_slice())); + assert_eq!(chunk_bytes(&data, 4, 4).as_deref(), Some([4, 5, 0, 1].as_slice())); + } + + #[test] + fn nv12_stride_is_removed_and_chroma_is_deinterleaved() { + let data = [ + 1, 2, 3, 4, 99, 99, // Y row 0 plus padding + 5, 6, 7, 8, 99, 99, // Y row 1 plus padding + 9, 10, 11, 12, 99, 99, // UV row plus padding + ]; + let frame = nv12_to_i420( + &data, + FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 2, + }, + ) + .unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.u(), &[9, 11]); + assert_eq!(frame.v(), &[10, 12]); + } + + #[test] + fn nv12_accepts_a_width_precise_final_row() { + let layout = FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 2, + }; + let mut data = vec![99; frame_data_size(VideoFormat::NV12, layout).unwrap()]; + data[..4].copy_from_slice(&[1, 2, 3, 4]); + data[6..10].copy_from_slice(&[5, 6, 7, 8]); + data[12..16].copy_from_slice(&[9, 10, 11, 12]); + + let frame = nv12_to_i420(&data, layout).unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.u(), &[9, 11]); + assert_eq!(frame.v(), &[10, 12]); + } + + /// An odd-height source has one more luma row than the clamped output, and + /// chroma starts after all of them. + #[test] + fn nv12_crop_uses_the_source_height_for_chroma() { + let data = [ + 1, 2, 3, 4, // Y row 0 + 5, 6, 7, 8, // Y row 1 + 99, 99, 99, 99, // cropped Y row 2 + 9, 10, 11, 12, // UV row 0 + ]; + let frame = nv12_to_i420( + &data, + FrameLayout { + stride: 4, + width: 4, + height: 2, + source_height: 3, + }, + ) + .unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.u(), &[9, 11]); + assert_eq!(frame.v(), &[10, 12]); } /// Open the portal, grab a few frames, and check geometry. Ignored because it diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 493d530d130..251662c7e61 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -285,6 +285,7 @@ pub async fn publish_capture( probe_config.bitrate = encode.bitrate; probe_config.codec = encode.codec; probe_config.kind = encode.kind.clone(); + probe_config.color = camera.color(); probe_config.probe().await? }; @@ -431,6 +432,7 @@ async fn capture_loop( encoder_config.bitrate = encode.bitrate; encoder_config.codec = encode.codec; encoder_config.kind = encode.kind.clone(); + encoder_config.color = camera.color(); // Off macOS this opens the encoder on a dedicated thread; see `sink`. let mut encoder = Sink::open(&encoder_config).await?; // Force an IDR on the first frame of each (re)open so a viewer subscribing diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 06f3a5e154e..6627a187357 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -489,9 +489,9 @@ impl I420 { /// Split tightly-packed NV12 (Y plane `width * height`, then interleaved UV /// `width/2 * height/2` pairs) into planar I420. A chroma deinterleave, no - /// color-space conversion. Used for the Windows Media Foundation capture path, - /// whose source reader hands us NV12. - #[cfg(target_os = "windows")] + /// color-space conversion. Used by the Windows Media Foundation and Linux + /// PipeWire capture paths. + #[cfg(any(target_os = "windows", all(target_os = "linux", feature = "pipewire")))] pub(crate) fn from_nv12(nv12: &[u8], width: u32, height: u32) -> Result { let (w, h) = (width as usize, height as usize); let luma = w * h; @@ -631,7 +631,7 @@ pub(crate) fn interleave_uv(u: &[u8], v: &[u8], uv: &mut [u8]) { /// Split a packed NV12 chroma plane into separate U and V planes, the inverse of /// [`interleave_uv`]. -#[cfg(target_os = "windows")] +#[cfg(any(target_os = "windows", all(target_os = "linux", feature = "pipewire")))] pub(crate) fn deinterleave_uv(uv: &[u8], u: &mut [u8], v: &mut [u8]) { for (pair, (u, v)) in uv.chunks_exact(2).zip(u.iter_mut().zip(v)) { *u = pair[0]; From 96219416fa376559cfdeb4974bba34d6bb4a625c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 11:41:47 -0700 Subject: [PATCH 09/29] docs: reserve package bumps for releases (#2889) Co-authored-by: GPT-5 --- CLAUDE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 2a9b2778461..b8aed77f462 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,6 +89,10 @@ This root file holds only cross-cutting rules that apply everywhere (writing sty - When adding new dependencies, always use the **newest stable version** available. - **Prefer a maintained third-party crate over hand-rolling non-core functionality** (standard container/codec parsers, compression, serialization, etc.). Reserve bespoke code for the wire/protocol layers where we need full control or no suitable crate exists. +## Package Versions + +**Do not bump package versions unless the user explicitly asks for a version bump or release.** Feature and fix work must leave version fields and matching lockfile version metadata unchanged. Periodic release work owns those bumps. + ## Writing Style - **No em dashes (—)** in code, comments, doc comments, commit messages, or any prose. Use a period and start a new sentence, or use a comma/parenthesis if the clauses are tightly bound. From 9fe2f9a4c55b5a6b4859232ccd6545c8bf303f8d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 14:18:08 -0700 Subject: [PATCH 10/29] fix(moq-mux): accept refreshed clocks on TS duplicates (#2891) Co-authored-by: GPT-5 --- rs/moq-mux/src/container/ts/import.rs | 76 +++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index ca5fcc2d2f2..01e4726dabc 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -1005,14 +1005,13 @@ impl VerbatimStream { struct Continuity { /// Last continuity_counter seen on a packet with payload, to spot gaps. last_cc: Option, - /// Last payload packet, to skip ISO 13818-1 duplicates (same cc, identical bytes). + /// Last payload packet, to identify ISO 13818-1 retransmissions. last_pkt: Option<[u8; 188]>, } /// What one packet says about the bytes already accumulated for its PID. enum Continuation { - /// An exact retransmission, which ISO 13818-1 permits once. Ignore the packet entirely: - /// processing it would duplicate its bytes. + /// A retransmission, which may refresh PCR/OPCR. Ignore it entirely to avoid duplicate bytes. Duplicate, /// Contiguous with the previous payload packet. Contiguous, @@ -1024,6 +1023,32 @@ enum Continuation { Corrupt, } +/// Whether two packets differ only in the clock fields a retransmission may refresh. +fn is_duplicate(last: &[u8; 188], pkt: &[u8; 188]) -> bool { + if last == pkt { + return true; + } + + // A refreshed clock requires the same adaptation-field shape in both packets. Comparing + // through the flags byte also pins the PID, payload-unit start, counter, length, and every + // other header bit before any bytes are ignored. + let afc = (last[3] >> 4) & 0x3; + if afc & 0x2 == 0 || last[..6] != pkt[..6] { + return false; + } + + let af_len = last[4] as usize; + let flags = last[5]; + let clock_len = usize::from(flags & 0x10 != 0) * 6 + usize::from(flags & 0x08 != 0) * 6; + let clock_end = 6 + clock_len; + let adaptation_end = 5 + af_len; + if clock_len == 0 || clock_end > adaptation_end || adaptation_end > last.len() { + return false; + } + + last[clock_end..] == pkt[clock_end..] +} + impl Continuity { /// Classify one 188-byte packet, recording what the next call needs. fn observe(&mut self, pkt: &[u8; 188]) -> Continuation { @@ -1060,7 +1085,7 @@ impl Continuity { // Only the bytes tell them apart, which is why the counter alone can't decide: taking // every repeat for a duplicate would carry a partial straight across that loss and // join it to unrelated bytes. - if self.last_pkt.as_ref().is_some_and(|last| last == pkt) { + if self.last_pkt.as_ref().is_some_and(|last| is_duplicate(last, pkt)) { return Continuation::Duplicate; } self.last_pkt = Some(*pkt); @@ -2317,7 +2342,7 @@ impl Read for Feed { mod test { use mpeg2ts::es::StreamType; - use super::SectionReassembler; + use super::{Continuation, Continuity, SectionReassembler}; use moq_net::Timestamp; // libklvanc public-sample cue: table_id 0xFC, section_length 0x1b (27), 30 bytes total. @@ -2353,6 +2378,23 @@ mod test { p } + /// A continuation packet carrying PCR and/or OPCR in its adaptation field. + fn clock_packet(cc: u8, pcr: Option<[u8; 6]>, opcr: Option<[u8; 6]>, body: &[u8]) -> Vec { + let flags = if pcr.is_some() { 0x10 } else { 0 } | if opcr.is_some() { 0x08 } else { 0 }; + let length = 1 + usize::from(pcr.is_some()) * 6 + usize::from(opcr.is_some()) * 6; + let mut p = vec![0x47, 0x00, 0x21, 0x30 | (cc & 0x0f), length as u8, flags]; + if let Some(pcr) = pcr { + p.extend_from_slice(&pcr); + } + if let Some(opcr) = opcr { + p.extend_from_slice(&opcr); + } + p.extend_from_slice(body); + assert!(p.len() <= 188, "test packet body overflows 188 bytes"); + p.resize(188, 0xff); + p + } + /// A synthetic section: `table_id`, a 12-bit length, then `body_len` zero bytes. fn fake_section(table_id: u8, body_len: usize) -> Vec { let mut s = vec![table_id, ((body_len >> 8) & 0x0f) as u8, (body_len & 0xff) as u8]; @@ -4028,6 +4070,30 @@ mod test { assert_eq!(run(&[p1, p2.clone(), p2, p3]), vec![section]); } + #[test] + fn duplicate_with_refreshed_clock_is_skipped() { + let old = [0x00, 0x00, 0x00, 0x00, 0x7e, 0x00]; + let refreshed = [0x00, 0x00, 0x00, 0x01, 0x7e, 0x00]; + for (name, pcr, opcr) in [("PCR", Some(old), None), ("OPCR", None, Some(old))] { + let section = fake_section(0xfc, 390); + let p1 = packet(true, 0, 0, §ion[..183]); + let p2 = clock_packet(1, pcr, opcr, §ion[183..359]); + let duplicate = clock_packet(1, pcr.map(|_| refreshed), opcr.map(|_| refreshed), §ion[183..359]); + let p3 = packet(false, 2, 0, §ion[359..]); + assert_eq!(run(&[p1, p2, duplicate, p3]), vec![section], "refreshed {name}"); + } + } + + #[test] + fn repeated_counter_after_fifteen_lost_packets_is_broken() { + let mut continuity = Continuity::default(); + let previous: [u8; 188] = packet(false, 5, 0, &[0x11; 184]).try_into().unwrap(); + let after_loss: [u8; 188] = packet(false, 5, 0, &[0x22; 184]).try_into().unwrap(); + + assert!(matches!(continuity.observe(&previous), Continuation::Contiguous)); + assert!(matches!(continuity.observe(&after_loss), Continuation::Broken)); + } + #[test] fn duplicate_pusi_packet_emits_once() { // A complete cue in one PUSI packet sent twice (legal duplicate) emits once. From e32ca87d7525e0118fd1b417c255ea906e6a2430 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 16 Aug 2026 14:20:29 -0700 Subject: [PATCH 11/29] feat(video): render PipeWire DMA-BUF frames (#2839) Co-authored-by: OpenAI Codex Co-authored-by: Claude Opus 5 --- Cargo.lock | 2 + Cargo.toml | 2 + doc/lib/rs/crate/moq-video.md | 25 +- rs/moq-native/Cargo.toml | 2 +- rs/moq-video/Cargo.toml | 17 +- rs/moq-video/src/capture/pipewire.rs | 1364 ++++++++++++++++++++------ rs/moq-video/src/frame.rs | 272 +++++ rs/moq-video/src/lib.rs | 2 + rs/moq-video/src/render/dmabuf.rs | 102 ++ rs/moq-video/src/render/metal.rs | 3 +- rs/moq-video/src/render/mod.rs | 13 +- rs/moq-video/src/render/renderer.rs | 121 ++- rs/moq-video/src/render/shader.wgsl | 9 +- rs/moq-video/src/render/source.rs | 15 +- 14 files changed, 1637 insertions(+), 312 deletions(-) create mode 100644 rs/moq-video/src/render/dmabuf.rs diff --git a/Cargo.lock b/Cargo.lock index be446606850..552fbc5bfa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5130,7 +5130,9 @@ dependencies = [ "futures", "h264-reader", "hang", + "libc", "libloading 0.9.0", + "linux-raw-sys 0.12.1", "moq-mux", "moq-net", "moq-nvenc", diff --git a/Cargo.toml b/Cargo.toml index 593f397e9f3..8715f5c9d9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,8 @@ rust-version = "1.91" flate2 = "1.1" hang = { version = "0.20", path = "rs/hang" } kio = { version = "0.5", path = "rs/kio" } +libc = "0.2" +linux-raw-sys = { version = "0.12.1", features = ["ioctl"] } # Permutation testing for the kio primitives (and everything built on them). # Only compiled under `--cfg loom`; see `just rs loom`. loom = { version = "0.7.2", features = ["futures"] } diff --git a/doc/lib/rs/crate/moq-video.md b/doc/lib/rs/crate/moq-video.md index f830f4911db..1f5f553c402 100644 --- a/doc/lib/rs/crate/moq-video.md +++ b/doc/lib/rs/crate/moq-video.md @@ -28,7 +28,7 @@ Four role modules, symmetric on both ends of the wire: | `capture` | Camera, display, window, or application frames | AVFoundation + ScreenCaptureKit (macOS), V4L2 + PipeWire (Linux), Media Foundation + DXGI (Windows) | | `encode` | Raw frames to H.264/H.265, published through `moq-mux` | VideoToolbox, Media Foundation, NVENC, VAAPI, openh264 | | `decode` | A subscribed track back to raw frames | VideoToolbox, Media Foundation/DXVA, NVDEC, openh264 | -| `render` | A frame drawn on the GPU, handed back as a `wgpu` texture | wgpu, with a zero-copy Metal import on macOS | +| `render` | A frame drawn on the GPU, handed back as a `wgpu` texture | wgpu, with zero-copy Metal and Vulkan imports | A picture is a `Frame` wherever it crosses the API: a `moq_net::Timestamp` and a `Surface` holding the pixels. Capture and decode produce them, encode and render @@ -66,8 +66,8 @@ cargo add moq-video --features render,pipewire | `capture` | yes | Native device capture (`v4l` and `zune-jpeg` on Linux) | | `nvenc` / `nvdec` | yes | NVIDIA encode/decode on Linux (`cudarc`, `moq-nvenc`) | | `vaapi` | no | Intel/AMD encode on Linux (`moq-vaapi`), unvalidated on hardware | -| `render` | no | `wgpu` and the GPU renderer | -| `pipewire` | no | Wayland/X11 screen capture via xdg-desktop-portal | +| `render` | no | `wgpu`, the GPU renderer, and Linux DMA-BUF support | +| `pipewire` | no | Wayland/X11 screen capture via xdg-desktop-portal and DMA-BUF | `--no-default-features` gives a codec-only build that still encodes and decodes H.264 with openh264 but omits native capture and the Linux GPU dependencies. A @@ -133,9 +133,9 @@ while let Some(frame) = video.read().await? { ## Zero-copy `Surface` is a `#[non_exhaustive]` enum naming what actually holds a frame's -pixels: a `CVPixelBuffer` on macOS, a Direct3D 11 texture on Windows, CUDA memory -on Linux, or plain I420 anywhere. Keeping a decoded frame in the first three -avoids a round trip through system memory on every frame. +pixels: a `CVPixelBuffer` on macOS, a Direct3D 11 texture on Windows, CUDA or a +DMA-BUF on Linux, or plain I420 anywhere. Keeping a frame in one of the native +representations avoids a round trip through system memory on every frame. How far that gets today depends on the platform, so here is the honest matrix rather than a blanket promise: @@ -143,15 +143,16 @@ rather than a blanket promise: | Platform | Decode output | Zero-copy transcode | Zero-copy render | | --- | --- | --- | --- | | macOS | `PixelBuffer` (VideoToolbox) | yes | yes, via `CVMetalTextureCache` | -| Linux | `Cuda` (NVDEC) | yes, straight into NVENC | no, downloaded to I420 first | +| Linux | `Cuda` (NVDEC) | yes, straight into NVENC | decoded CUDA frames: no; packed PipeWire DMA-BUF capture: yes, via Vulkan | | Windows | `Texture` (Media Foundation / DXVA) | yes, through the Direct3D11 video processor | no, downloaded to I420 first | `Frame::resize` stays on the GPU through a `VTPixelTransferSession`, CUDA kernel, or Direct3D11 video processor. Call `Frame::resize_with` with `resize::Acceleration::Cpu` to force a download and CPU resize. A driver that -rejects GPU resizing returns to CPU scaling and warns once. Rendering is -zero-copy on macOS only; the Vulkan and EGL importers that would extend it are -tracked in [#2481](https://github.com/moq-dev/moq/issues/2481). +rejects GPU resizing returns to CPU scaling and warns once. Linux Vulkan can +import packed RGB DMA-BUF screen frames. Multi-plane NV12 import and retiling a +modifier that Vulkan rejects remain tracked in +[#2819](https://github.com/moq-dev/moq/issues/2819). Matching on `Surface` stays portable because every variant has a universal fallback in `Surface::into_i420()`: take the fast path you recognize and let the @@ -182,6 +183,10 @@ The `wgpu` version this was built against is re-exported as `moq_video::render::wgpu`, so you name the exact version rather than guessing at a compatible one. +On Linux, request `wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF` when creating +the device to activate the PipeWire DMA-BUF fast path. The renderer still works +without it and falls back to a CPU upload for linear allocations. + `Color` names the matrix and range (BT.601 or BT.709, limited or full), and the shader converts per frame rather than assuming one space. A capture labels what it produced, so a locally captured frame renders correctly with no help from you. diff --git a/rs/moq-native/Cargo.toml b/rs/moq-native/Cargo.toml index 34e39df7a5a..6b763b2adb3 100644 --- a/rs/moq-native/Cargo.toml +++ b/rs/moq-native/Cargo.toml @@ -96,7 +96,7 @@ webpki-roots = "1" [target.'cfg(unix)'.dependencies] # Errno constants for classifying an `accept(2)` failure (`accept::Failure`). # Their numeric values differ across unix platforms, so they can't be spelled out. -libc = "0.2" +libc = { workspace = true } [dev-dependencies] anyhow = "1" diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index 0b675740285..48f3e63ee83 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -37,13 +37,18 @@ nvdec = ["dep:cudarc", "dep:moq-nvenc", "dep:libloading"] # build: moq-vaapi dlopen's libva, so an enabled-vaapi build needs no libva at # build time and still starts on a libva-less host, where automatic backend # selection falls back to the next encoder like the NVENC backend does. -vaapi = ["dep:moq-vaapi"] +vaapi = ["dmabuf", "dep:moq-vaapi"] +# Linux DMA-BUF surface vocabulary and CPU fallback support. Enabled by every +# backend that can produce or consume one. It pulls no graphics API by itself. +dmabuf = ["dep:libc", "dep:linux-raw-sys"] # GPU rendering of decoded frames (the `render` module): a wgpu pipeline that # converts a frame to RGBA and hands back a texture the caller presents. Off by # default because wgpu pulls a graphics stack (and its backend drivers) that a # relay or a headless publisher has no use for; only an application that actually -# draws video turns it on. +# draws video turns it on. `dmabuf` adds the Linux surface vocabulary; Vulkan +# itself still comes from wgpu. render = [ + "dmabuf", "dep:bytemuck", "dep:wgpu", "dep:objc2-metal", @@ -55,7 +60,7 @@ render = [ # Off by default because the `pipewire` crate links libpipewire-0.3 via pkg-config # at build time (and its bindgen needs libclang), so it only belongs in builds # that actually capture displays. Camera capture (V4L2) needs nothing extra. -pipewire = ["capture", "dep:pipewire", "dep:ashpd"] +pipewire = ["capture", "dmabuf", "dep:pipewire", "dep:ashpd"] [dependencies] anyhow = "1" @@ -106,10 +111,16 @@ ashpd = { version = "0.13", optional = true, default-features = false, features # extra dependencies); libnvrtc itself is never loaded since we ship pre-built PTX # (see frame/nv12_resize.ptx) that the driver JIT-compiles. cudarc = { version = "0.19", optional = true, default-features = false, features = ["driver", "fallback-dynamic-loading", "cuda-12020", "nvrtc"] } +# Mapping a linear DMA-BUF for the universal CPU I420 fallback. Non-linear +# modifiers stay GPU-only and return an honest error instead of treating tiled +# memory as rows. +libc = { workspace = true, optional = true } # Probe for the NVIDIA driver libraries before calling cudarc / the NVENC SDK, # which panic (process-abort under release `panic = "abort"`) if their library is # absent. Lets a GPU-less host fall back to the next encoder instead of crashing. libloading = { version = "0.9", optional = true } +# Architecture-correct DMA_BUF_IOCTL_SYNC request values for CPU access. +linux-raw-sys = { workspace = true, optional = true } moq-nvenc = { workspace = true, optional = true } # Intel/AMD VAAPI hardware encoder, behind the opt-in `vaapi` feature. As of # moq-vaapi 0.0.3 libva is dlopen'd at runtime, so the binary carries no NEEDED diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index 72eeaf192ed..96d1bfe3084 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -2,8 +2,9 @@ //! //! The ScreenCast portal owns source selection: [`open`] pops the compositor's //! picker dialog, the user chooses a monitor, and the portal hands us a PipeWire -//! fd + node id. A dedicated thread then runs the PipeWire main loop, converting -//! each packed RGB or NV12 frame to CPU [`I420`] and pushing it into the shared +//! fd + node id. A dedicated thread then runs the PipeWire main loop, forwarding +//! DMA-BUF frames without copying when the compositor offers them and converting +//! shared-memory frames to CPU [`I420`] otherwise. It pushes both into the shared //! [`FrameChannel`] (callback-driven like the macOS delegate, not a pull-style //! pump). //! @@ -21,7 +22,7 @@ use std::borrow::Cow; use std::cell::RefCell; -use std::os::fd::OwnedFd; +use std::os::fd::{AsFd, BorrowedFd, OwnedFd}; use std::rc::Rc; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -31,15 +32,18 @@ use ashpd::desktop::PersistMode; use ashpd::desktop::screencast::{CursorMode, Screencast, SelectSourcesOptions, SourceType}; use pipewire as pw; use pw::spa; +use spa::buffer::DataType; use spa::param::video::{VideoFormat, VideoInfoRaw}; use super::channel::FrameChannel; use super::pump::Geometry; use super::{Config, FrameStream}; -use crate::frame::{I420, Surface}; +use crate::frame::{DmaBuf, DmaBufFrame, DmaBufPlane, DrmFormat, I420, Surface, wait_dma_buf_readable}; use crate::{Color, Error, Size}; const DEFAULT_FRAMERATE: u32 = 30; +// libspa 0.10 omits this flag from its safe wrapper, but exposes the raw bits. +const CHUNK_FLAG_EMPTY: i32 = 1 << 1; /// The compositor sends the negotiated format right after the stream connects; /// if nothing arrives the session is broken (or the grant was revoked mid-setup). const FORMAT_TIMEOUT: Duration = Duration::from_secs(10); @@ -70,6 +74,7 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result(); + let (return_tx, return_rx) = pw::channel::channel::(); let handle = std::thread::spawn({ let chan = chan.clone(); @@ -81,8 +86,19 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result, geo_tx: Option>>, /// Most recent converted frame, re-emitted while the screen is static. - last: Option, + last: Option, /// Whether a fresh frame arrived since the last pacing tick. fresh: bool, + /// Buffer-pool generation. Returns from superseded pools are discarded. + generation: u64, + /// Explicit DRM modifier from the negotiated DMA-BUF format. + dmabuf_modifier: Option, +} + +/// A retained frame for the static-screen pacing tick. +enum Last { + I420(I420), + DmaBuf(DmaBuf), +} + +impl Last { + fn surface(&self) -> Surface { + match self { + Self::I420(frame) => Surface::I420(frame.clone()), + Self::DmaBuf(frame) => Surface::DmaBuf(frame.clone()), + } + } +} + +#[derive(Clone, Copy)] +struct FrameLayout { + stride: u32, + width: u32, + height: u32, + source_height: u32, +} + +/// A dequeued PipeWire DMA-BUF kept out of the producer's pool until every +/// frame clone drops. Duplicating the fd alone is not enough: it preserves the +/// allocation, but the compositor may overwrite its pixels as soon as the +/// original buffer is queued again. +struct PipeWireDmaBuf { + fd: OwnedFd, + return_tx: pw::channel::Sender, + lease: Lease, + map_offset: u32, + allocation_size: Option, + data_offset: usize, + layout: FrameLayout, + format: DrmFormat, + modifier: u64, + color: Option, +} + +impl Drop for PipeWireDmaBuf { + fn drop(&mut self) { + // A failed send means the stream and its pool have already been destroyed. + // The duplicated fd still closes normally; the stale pointer is never used. + let _ = self.return_tx.send(self.lease); + } +} + +impl DmaBufFrame for PipeWireDmaBuf { + fn export(&self) -> std::io::Result { + self.fd.as_fd().try_clone_to_owned() + } + + fn download_i420(&self) -> Result { + // DRM_FORMAT_MOD_LINEAR is zero. A tiled allocation cannot be interpreted + // as strided rows; its fallback needs a GPU/VPP download instead. + if self.modifier != 0 { + return Err(Error::Codec(anyhow::anyhow!( + "cannot download DMA-BUF modifier {:#x} as linear rows", + self.modifier + ))); + } + + wait_dma_buf_readable(self.fd.as_fd()) + .map_err(|e| Error::Codec(anyhow::anyhow!("waiting for DMA-BUF producer: {e}")))?; + with_dma_buf_read(&self.fd, || { + let allocation_size = self + .allocation_size + .ok_or_else(|| Error::Codec(anyhow::anyhow!("DMA-BUF descriptor does not report a mappable size")))?; + let mapping = Mapping::new(&self.fd, self.map_offset, allocation_size)?; + let data = mapping + .as_slice() + .get(self.data_offset..) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("DMA-BUF chunk starts outside its allocation")))?; + + match self.format { + DrmFormat::NV12 => { + let frame = nv12_to_i420(data, self.layout)?; + Ok(match self.color { + Some(color) => frame.with_color(color), + None => frame, + }) + } + DrmFormat::XRGB8888 | DrmFormat::ARGB8888 => { + I420::from_bgra(data, self.layout.stride, self.layout.width, self.layout.height) + } + DrmFormat::XBGR8888 | DrmFormat::ABGR8888 => { + I420::from_rgba(data, self.layout.stride, self.layout.width, self.layout.height) + } + other => Err(Error::Codec(anyhow::anyhow!( + "cannot download DMA-BUF format {:#x}", + other.as_raw() + ))), + } + }) + } +} + +const DMA_BUF_SYNC_READ: u64 = 1 << 0; +const DMA_BUF_SYNC_END: u64 = 1 << 2; + +#[repr(C)] +struct DmaBufSync { + flags: u64, +} + +/// Brackets CPU reads with the cache-coherency protocol required by DMA-BUF. +fn with_dma_buf_read(fd: &OwnedFd, read: impl FnOnce() -> Result) -> Result { + let sync = DmaBufRead::new(fd)?; + let result = read(); + let end = sync.finish(); + match (result, end) { + (Ok(value), Ok(())) => Ok(value), + (Err(err), _) => Err(err), + (Ok(_), Err(err)) => Err(err), + } +} + +struct DmaBufRead<'a> { + fd: &'a OwnedFd, + finished: bool, +} + +impl<'a> DmaBufRead<'a> { + fn new(fd: &'a OwnedFd) -> Result { + dma_buf_sync(fd, DMA_BUF_SYNC_READ)?; + Ok(Self { fd, finished: false }) + } + + fn finish(mut self) -> Result<(), Error> { + self.finished = true; + dma_buf_sync(self.fd, DMA_BUF_SYNC_READ | DMA_BUF_SYNC_END) + } +} + +impl Drop for DmaBufRead<'_> { + fn drop(&mut self) { + if !self.finished + && let Err(err) = dma_buf_sync(self.fd, DMA_BUF_SYNC_READ | DMA_BUF_SYNC_END) + { + tracing::warn!(%err, "ending DMA-BUF CPU access failed"); + } + } +} + +fn dma_buf_sync(fd: &OwnedFd, flags: u64) -> Result<(), Error> { + let mut sync = DmaBufSync { flags }; + loop { + // SAFETY: this is the DMA-BUF sync ioctl with its matching UAPI payload, + // and `fd` stays open for the duration of the call. + let result = unsafe { + libc::ioctl( + std::os::fd::AsRawFd::as_raw_fd(fd), + linux_raw_sys::ioctl::DMA_BUF_IOCTL_SYNC as libc::Ioctl, + &mut sync, + ) + }; + if result == 0 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(Error::Codec(anyhow::anyhow!("DMA-BUF sync: {err}"))); + } + } +} + +fn dma_buf_allocation_size(fd: BorrowedFd<'_>, map_offset: u32) -> std::io::Result> { + let raw = std::os::fd::AsRawFd::as_raw_fd(&fd); + dma_buf_allocation_size_with_seek(map_offset, |offset, whence| seek_fd(raw, offset, whence)) +} + +fn dma_buf_allocation_size_with_seek( + map_offset: u32, + mut seek: impl FnMut(libc::off_t, libc::c_int) -> std::io::Result, +) -> std::io::Result> { + // DMA-BUF supports SEEK_END specifically so userspace can discover the + // allocation size even though fstat commonly reports zero. + let end = seek(0, libc::SEEK_END)?; + // DMA-BUF also supports SEEK_SET. Reset the shared file description to the + // canonical position before handing the descriptor to another subsystem. + seek(0, libc::SEEK_SET)?; + let Ok(end) = usize::try_from(end) else { + return Ok(None); + }; + Ok(end.checked_sub(map_offset as usize).filter(|size| *size > 0)) +} + +fn seek_fd(fd: std::os::fd::RawFd, offset: libc::off_t, whence: libc::c_int) -> std::io::Result { + loop { + // SAFETY: the caller keeps `fd` open for the duration of this syscall. + let result = unsafe { libc::lseek(fd, offset, whence) }; + if result >= 0 { + return Ok(result); + } + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(error); + } + } +} + +/// Read-only mmap of one linear DMA-BUF allocation. +struct Mapping { + ptr: *mut libc::c_void, + len: usize, +} + +impl Mapping { + fn new(fd: &OwnedFd, offset: u32, len: usize) -> Result { + if len == 0 { + return Err(Error::Codec(anyhow::anyhow!("cannot map an empty DMA-BUF"))); + } + // SAFETY: `fd` stays open for the mapping's lifetime, `len` is non-zero, + // and the kernel validates the PipeWire-provided offset and allocation. + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ, + libc::MAP_SHARED, + std::os::fd::AsRawFd::as_raw_fd(fd), + offset as libc::off_t, + ) + }; + if ptr == libc::MAP_FAILED { + return Err(Error::Codec(anyhow::anyhow!( + "DMA-BUF mmap: {}", + std::io::Error::last_os_error() + ))); + } + Ok(Self { ptr, len }) + } + + fn as_slice(&self) -> &[u8] { + // SAFETY: `ptr` and `len` came from a successful `mmap`, and `self` + // keeps that mapping alive for the returned slice's lifetime. + unsafe { std::slice::from_raw_parts(self.ptr.cast(), self.len) } + } +} + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: this exact pointer and length came from the successful `mmap` + // in `Mapping::new`, and the mapping is released exactly once here. + unsafe { + libc::munmap(self.ptr, self.len); + } + } +} + +/// Deinterleave strided NV12 into the crate's tightly packed I420 layout. +fn nv12_to_i420(data: &[u8], layout: FrameLayout) -> Result { + let (stride, width, height, source_height) = ( + layout.stride as usize, + layout.width as usize, + layout.height as usize, + layout.source_height as usize, + ); + if source_height < height { + return Err(Error::Codec(anyhow::anyhow!( + "NV12 source is shorter than the cropped output" + ))); + } + let y_len = stride + .checked_mul(source_height) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 luma size overflow")))?; + let uv_rows = height / 2; + let uv_len = uv_rows + .checked_sub(1) + .and_then(|rows| rows.checked_mul(stride)) + .and_then(|offset| offset.checked_add(width)) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 chroma size overflow")))?; + let frame_len = y_len + .checked_add(uv_len) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 frame size overflow")))?; + if stride < width || data.len() < frame_len { + return Err(Error::Codec(anyhow::anyhow!( + "NV12 frame is shorter than its declared rows" + ))); + } + + let mut packed = vec![0; width * height * 3 / 2]; + for row in 0..height { + packed[row * width..(row + 1) * width].copy_from_slice(&data[row * stride..row * stride + width]); + } + let uv = &data[y_len..frame_len]; + let packed_uv = width * height; + for row in 0..uv_rows { + packed[packed_uv + row * width..packed_uv + (row + 1) * width] + .copy_from_slice(&uv[row * stride..row * stride + width]); + } + I420::from_nv12(&packed, width as u32, height as u32) +} + +/// Queues a raw PipeWire buffer unless ownership is transferred to a DMA-BUF +/// surface. This mirrors `pipewire::buffer::Buffer` while allowing the surface +/// to return the buffer asynchronously on the PipeWire loop thread. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Lease { + buffer: usize, + generation: u64, +} + +impl Lease { + fn current(self, generation: u64) -> Option<*mut pw::sys::pw_buffer> { + (self.generation == generation).then_some(self.buffer as *mut pw::sys::pw_buffer) + } +} + +struct Dequeued<'a> { + stream: &'a pw::stream::Stream, + raw: *mut pw::sys::pw_buffer, + queue: bool, +} + +impl<'a> Dequeued<'a> { + unsafe fn new(stream: &'a pw::stream::Stream) -> Option { + // SAFETY: PipeWire permits one dequeue from this process callback. The + // wrapper below returns the pointer to this same stream unless leased. + let raw = unsafe { stream.dequeue_raw_buffer() }; + (!raw.is_null()).then_some(Self { + stream, + raw, + queue: true, + }) + } + + fn datas_mut(&mut self) -> &mut [spa::buffer::Data] { + // SAFETY: `raw` is a non-null buffer dequeued from `stream` and remains + // out of PipeWire's pool while this wrapper owns it. + let buffer = unsafe { (*self.raw).buffer }; + if buffer.is_null() || unsafe { (*buffer).n_datas == 0 || (*buffer).datas.is_null() } { + return &mut []; + } + // SAFETY: PipeWire owns this SPA buffer and guarantees its `datas` array + // has `n_datas` entries until the buffer is queued back. + unsafe { + std::slice::from_raw_parts_mut((*buffer).datas.cast::(), (*buffer).n_datas as usize) + } + } + + fn lease(mut self, generation: u64) -> Lease { + self.queue = false; + Lease { + buffer: self.raw as usize, + generation, + } + } } -/// Connect to the portal's PipeWire node and run the main loop until the stream -/// ends, `quit_rx` fires (the `FrameStream` dropped), or the format changes. -fn run_loop( +impl Drop for Dequeued<'_> { + fn drop(&mut self) { + if self.queue { + // SAFETY: the pointer came from `self.stream` and has not previously + // been returned or transferred to a leased surface. + unsafe { self.stream.queue_raw_buffer(self.raw) }; + } + } +} + +struct CaptureLoop { fd: OwnedFd, node_id: u32, framerate: u32, chan: Arc, state: Rc>, quit_rx: pw::channel::Receiver<()>, -) -> Result<(), Error> { + return_rx: pw::channel::Receiver, + return_tx: pw::channel::Sender, +} + +#[derive(Debug, PartialEq, Eq)] +enum NegotiatedMemory { + Fixating, + SharedMemory, + DmaBuf(u64), +} + +fn negotiated_memory(param: &spa::pod::Pod, format: VideoInfoRaw) -> Option { + let object = param.as_object().ok()?; + let modifier = object.find_prop(spa::utils::Id( + spa::param::format::FormatProperties::VideoModifier.as_raw(), + )); + Some(match modifier { + Some(modifier) if modifier.flags().contains(spa::pod::PodPropFlags::DONT_FIXATE) => NegotiatedMemory::Fixating, + Some(_) => NegotiatedMemory::DmaBuf(format.modifier()), + None => NegotiatedMemory::SharedMemory, + }) +} + +fn fixate_modifier(param: &spa::pod::Pod, modifier: u64) -> Option> { + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(param.as_bytes()).ok()?; + let spa::pod::Value::Object(mut object) = value else { + return None; + }; + let property = object + .properties + .iter_mut() + .find(|property| property.key == spa::param::format::FormatProperties::VideoModifier.as_raw())?; + property.flags.remove(spa::pod::PropertyFlags::from_bits_retain( + spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE, + )); + property.value = spa::pod::Value::Long(modifier as i64); + spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(object)) + .ok() + .map(|serialized| serialized.0.into_inner()) +} + +/// Connect to the portal's PipeWire node and run until the stream ends, the +/// consumer drops, or the format changes. +fn run_loop(args: CaptureLoop) -> Result<(), Error> { + let CaptureLoop { + fd, + node_id, + framerate, + chan, + state, + quit_rx, + return_rx, + return_tx, + } = args; pw::init(); let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|e| err("pipewire main loop", e))?; @@ -282,6 +714,23 @@ fn run_loop( ) .map_err(|e| err("pipewire stream", e))?; + // DMA-BUF surfaces return their dequeued PipeWire buffer here on the loop + // thread. A surface may drop on an encoder or renderer task, where calling + // `pw_stream_queue_buffer` directly would violate PipeWire's threading model. + let _returns = return_rx.attach(mainloop.loop_(), { + let stream = stream.downgrade(); + let state = state.clone(); + move |lease| { + if let Some(stream) = stream.upgrade() + && let Some(raw) = lease.current(state.borrow().generation) + { + // SAFETY: leased surfaces send each pointer exactly once, and every + // current-generation pointer was dequeued from this stream's pool. + unsafe { stream.queue_raw_buffer(raw) }; + } + } + }); + let _listener = stream .add_local_listener::<()>() .state_changed({ @@ -311,7 +760,7 @@ fn run_loop( .param_changed({ let state = state.clone(); let mainloop = mainloop.downgrade(); - move |_, _, id, param| { + move |stream, _, id, param| { let Some(param) = param else { return }; if id != spa::param::ParamType::Format.as_raw() { return; @@ -330,6 +779,38 @@ fn run_loop( tracing::warn!(error = %e, "failed to parse pipewire video format"); return; } + let Some(memory) = negotiated_memory(param, state.format) else { + return; + }; + let dmabuf_modifier = match memory { + NegotiatedMemory::DmaBuf(modifier) => Some(modifier), + NegotiatedMemory::SharedMemory => None, + NegotiatedMemory::Fixating => { + let Some(fixed) = fixate_modifier(param, state.format.modifier()) else { + tracing::warn!("failed to fixate the PipeWire DMA-BUF modifier"); + return; + }; + let Some(param) = spa::pod::Pod::from_bytes(&fixed) else { + return; + }; + let mut params = [param]; + if let Err(e) = stream.update_params(&mut params) { + tracing::warn!(error = %e, "failed to fixate the PipeWire DMA-BUF modifier"); + } + return; + } + }; + state.last = None; + state.dmabuf_modifier = dmabuf_modifier; + let buffers = buffer_offer(state.dmabuf_modifier.is_some()); + if let Some(param) = spa::pod::Pod::from_bytes(&buffers) { + let mut params = [param]; + if let Err(e) = stream.update_params(&mut params) { + tracing::warn!(error = %e, "DMA-BUF buffer negotiation failed; capture may use shared memory"); + } else { + state.generation = state.generation.wrapping_add(1); + } + } let size = state.format.size(); // I420 chroma is 2x2 subsampled; clamp down (drop the last odd @@ -382,44 +863,46 @@ fn run_loop( let state = state.clone(); let chan = chan.clone(); let mainloop = mainloop.downgrade(); + let return_tx = return_tx.clone(); move |stream, _| { let mut state = state.borrow_mut(); let Some((width, height)) = state.geometry else { return }; let source_height = state.format.size().height; let color = state.color; - let Some(mut buffer) = stream.dequeue_buffer() else { + // SAFETY: this is PipeWire's process callback for `stream`; `Dequeued` + // returns the buffer on drop unless a DMA-BUF surface leases it. + let Some(mut buffer) = (unsafe { Dequeued::new(stream) }) else { return; }; let datas = buffer.datas_mut(); - let Some(data) = datas.first_mut() else { return }; - - let maxsize = data.as_raw().maxsize; - let size = clamp_chunk_size(data.chunk().size(), maxsize); - match chunk_kind(size, data.chunk().flags()) { - ChunkKind::Data => {} - ChunkKind::Empty => { - let Ok(frame) = neutral_frame(width, height, color) else { - return; - }; - chan.push(Surface::I420(frame.clone())); - state.last = Some(frame); - state.fresh = true; - return; - } - ChunkKind::Invalid => return, - } - let Some(offset) = normalize_chunk_offset(data.chunk().offset(), maxsize) else { - tracing::warn!("pipewire buffer has zero maximum size"); + let [data] = datas else { + tracing::warn!( + blocks = datas.len(), + "pipewire ignored the negotiated single-block layout" + ); return; }; - // Fall back to the unclamped source width: for an odd-width source - // the real row is one pixel wider than the clamped `width`. The - // packing differs per format, so NV12 cannot assume 4 bytes/pixel. + + let chunk_offset = data.chunk().offset(); + let (map_offset, maxsize) = { + let raw = data.as_raw(); + (raw.mapoffset, raw.maxsize) + }; + let dmabuf = data.type_() == DataType::DmaBuf; + let chunk_size = data.chunk().size(); + let size = if dmabuf { + dma_buf_chunk_size(chunk_size, maxsize) + } else { + clamp_chunk_size(chunk_size, maxsize) + }; let stride = match u32::try_from(data.chunk().stride()) { Ok(stride) if stride > 0 => stride, _ => match state.format.format() { VideoFormat::NV12 => state.format.size().width, - _ => state.format.size().width.saturating_mul(4), + VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => { + state.format.size().width.saturating_mul(4) + } + _ => 0, }, }; let layout = FrameLayout { @@ -428,15 +911,110 @@ fn run_loop( height, source_height, }; - - // The chunk only says how many bytes the producer wrote. Check it - // actually spans every row `convert` will sample, so a short or - // mislabeled buffer is dropped here rather than read past. + match chunk_kind(size, data.chunk().flags(), dmabuf) { + ChunkKind::Data => {} + // An empty screencast chunk means there is no new damage. Keep + // the previous frame for the pacing timer instead of blanking it. + ChunkKind::Empty => return, + ChunkKind::Invalid => return, + } let Some(required) = frame_data_size(state.format.format(), layout) else { tracing::warn!("pipewire frame layout overflows its buffer"); return; }; - if required > size || required > maxsize as usize { + if dmabuf { + let Some(format) = drm_format(state.format.format()) else { + tracing::warn!(format = ?state.format.format(), "unsupported DMA-BUF pixel format"); + return; + }; + if data.fd() < 0 || stride == 0 { + tracing::warn!("DMA-BUF has no valid fd or row stride"); + return; + } + // SAFETY: PipeWire owns this fd while the buffer is dequeued. It is + // duplicated immediately, before the buffer can be queued again. + let fd = unsafe { BorrowedFd::borrow_raw(data.fd()) }; + let Ok(fd) = fd.try_clone_to_owned() else { + return; + }; + let offset = normalize_dma_buf_offset(chunk_offset, maxsize); + if !dma_buf_chunk_contains(required, chunk_size, maxsize) { + tracing::warn!( + required, + available = size, + "DMA-BUF chunk does not contain a complete frame" + ); + return; + } + let allocation_size = match dma_buf_allocation_size(fd.as_fd(), map_offset) { + Ok(Some(size)) if offset.checked_add(required).is_some_and(|end| end <= size) => Some(size), + Ok(Some(size)) => { + tracing::warn!( + required, + available = size, + "DMA-BUF allocation is shorter than its frame layout" + ); + return; + } + Ok(None) => None, + Err(error) => { + tracing::debug!(%error, "could not query DMA-BUF allocation size; CPU fallback is unavailable"); + None + } + }; + let Some(base) = map_offset.checked_add(offset as u32) else { + tracing::warn!("DMA-BUF plane offset overflow"); + return; + }; + let planes = if format == DrmFormat::NV12 { + let Some(uv) = layout + .stride + .checked_mul(layout.source_height) + .and_then(|size| base.checked_add(size)) + else { + tracing::warn!("DMA-BUF chroma plane offset overflow"); + return; + }; + vec![ + DmaBufPlane::new(base, layout.stride), + DmaBufPlane::new(uv, layout.stride), + ] + } else { + vec![DmaBufPlane::new(base, layout.stride)] + }; + let Some(modifier) = state.dmabuf_modifier else { + tracing::warn!("received a DMA-BUF without a negotiated modifier"); + return; + }; + let lease = buffer.lease(state.generation); + let inner = Arc::new(PipeWireDmaBuf { + fd, + return_tx: return_tx.clone(), + lease, + map_offset, + allocation_size, + data_offset: offset, + layout, + format, + modifier, + color, + }); + match DmaBuf::new(format, modifier, layout.width, layout.height, planes, color, inner) { + Ok(frame) => { + chan.push(Surface::DmaBuf(frame.clone())); + state.last = Some(Last::DmaBuf(frame)); + state.fresh = true; + } + Err(e) => tracing::warn!(error = %e, "invalid PipeWire DMA-BUF"), + } + return; + } + let allocation_size = maxsize as usize; + let Some(offset) = normalize_chunk_offset(chunk_offset, maxsize) else { + tracing::warn!("pipewire buffer has zero maximum size"); + return; + }; + if required > size || required > allocation_size { tracing::warn!( required, available = size, @@ -444,10 +1022,8 @@ fn run_loop( ); return; } - - // Without dmabuf modifiers in our format offer the compositor uses - // shared memory, which MAP_BUFFERS mmaps for us; `None` here means - // it forced something we can't read, so give up cleanly. + // MAP_BUFFERS maps MemPtr/MemFd for the CPU fallback. `None` here + // means the producer forced an unsupported buffer representation. let Some(bytes) = data.data() else { tracing::warn!("pipewire buffer is not CPU-mapped; stopping capture"); if let Some(mainloop) = mainloop.upgrade() { @@ -455,17 +1031,16 @@ fn run_loop( } return; }; - let Some(allocation) = bytes.get(..maxsize as usize) else { + let Some(allocation) = bytes.get(..allocation_size) else { return; }; let Some(bytes) = chunk_bytes(allocation, offset, required) else { return; }; - match convert(state.format.format(), bytes.as_ref(), layout, color) { Ok(i420) => { chan.push(Surface::I420(i420.clone())); - state.last = Some(i420); + state.last = Some(Last::I420(i420)); state.fresh = true; } Err(e) => { @@ -481,12 +1056,16 @@ fn run_loop( .register() .map_err(|e| err("pipewire listener", e))?; - // Offer the CPU-convertible RGB layouts; the compositor picks one and - // replies through `param_changed`. No dmabuf modifiers, so buffers stay in - // shared memory (the CPU path, like the other non-macOS backends). - let pod = format_offer(framerate); - let mut params = [spa::pod::Pod::from_bytes(&pod) - .ok_or_else(|| Error::Codec(anyhow::anyhow!("failed to build pipewire format offer")))?]; + // DMA-BUF formats come first so PipeWire prefers them. Each has a matching + // shared-memory offer without a modifier as the required fallback. + let offers = format_offers(framerate); + let mut params = offers + .iter() + .map(|offer| { + spa::pod::Pod::from_bytes(offer) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("failed to build pipewire format offer"))) + }) + .collect::, _>>()?; stream .connect( spa::utils::Direction::Input, @@ -507,7 +1086,7 @@ fn run_loop( return; } if let Some(last) = &state.last { - chan.push(Surface::I420(last.clone())); + chan.push(last.surface()); } } }); @@ -531,50 +1110,91 @@ fn run_loop( Ok(()) } -/// The geometry `convert` needs: the producer's row stride and source height, -/// plus the even-clamped output size. -#[derive(Clone, Copy)] -struct FrameLayout { - stride: u32, - width: u32, - height: u32, - /// Unclamped source height. NV12's chroma plane starts after this many luma - /// rows, which is one more than `height` for an odd-height source. - source_height: u32, +fn normalize_chunk_offset(offset: u32, maxsize: u32) -> Option { + (maxsize != 0).then(|| (offset % maxsize) as usize) } -const CHUNK_FLAG_EMPTY: i32 = 1 << 1; +fn normalize_dma_buf_offset(offset: u32, maxsize: u32) -> usize { + normalize_chunk_offset(offset, maxsize).unwrap_or(offset as usize) +} -#[derive(Debug, PartialEq, Eq)] -enum ChunkKind { - Data, - Empty, - Invalid, +fn clamp_chunk_size(size: u32, maxsize: u32) -> usize { + size.min(maxsize) as usize } -fn chunk_kind(size: usize, flags: spa::buffer::ChunkFlags) -> ChunkKind { - if flags.contains(spa::buffer::ChunkFlags::CORRUPTED) { - ChunkKind::Invalid - } else if flags.bits() & CHUNK_FLAG_EMPTY != 0 { - ChunkKind::Empty - } else if size == 0 { - ChunkKind::Invalid +fn dma_buf_chunk_size(size: u32, maxsize: u32) -> usize { + if maxsize == 0 { + size as usize } else { - ChunkKind::Data + clamp_chunk_size(size, maxsize) } } -fn neutral_frame(width: u32, height: u32, color: Option) -> Result { - let color = color.unwrap_or_else(|| Color::infer(Size::new(width, height))); - let luma = width as usize * height as usize; - let mut data = vec![128; I420::len(width, height)]; - data[..luma].fill(if color.limited() { 16 } else { 0 }); - Ok(I420::new(width, height, data)?.with_color(color)) +fn dma_buf_chunk_contains(required: usize, size: u32, maxsize: u32) -> bool { + size == 0 || required <= dma_buf_chunk_size(size, maxsize) } -/// Parse into a zeroed value before replacing the current format. libspa leaves -/// omitted optional properties untouched, so parsing into the reused value -/// would retain stale color metadata across renegotiation. +fn chunk_bytes(data: &[u8], offset: usize, size: usize) -> Option> { + if offset >= data.len() || size > data.len() { + return None; + } + let end = offset.checked_add(size)?; + if end <= data.len() { + return Some(Cow::Borrowed(&data[offset..end])); + } + + let mut wrapped = Vec::with_capacity(size); + wrapped.extend_from_slice(&data[offset..]); + let head = size - wrapped.len(); + wrapped.extend_from_slice(&data[..head]); + Some(Cow::Owned(wrapped)) +} + +fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { + let stride = layout.stride as usize; + let width = layout.width as usize; + let height = layout.height as usize; + let row_size = match format { + VideoFormat::NV12 => width, + VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => width.checked_mul(4)?, + _ => return None, + }; + if stride < row_size { + return None; + } + + match format { + VideoFormat::NV12 => (layout.source_height as usize) + .checked_add(height / 2)? + .checked_sub(1)? + .checked_mul(stride)? + .checked_add(row_size), + _ => height.checked_sub(1)?.checked_mul(stride)?.checked_add(row_size), + } +} + +#[derive(Debug, PartialEq, Eq)] +enum ChunkKind { + Data, + Empty, + Invalid, +} + +fn chunk_kind(size: usize, flags: spa::buffer::ChunkFlags, dmabuf: bool) -> ChunkKind { + if flags.contains(spa::buffer::ChunkFlags::CORRUPTED) { + ChunkKind::Invalid + } else if flags.bits() & CHUNK_FLAG_EMPTY != 0 { + ChunkKind::Empty + } else if size == 0 && !dmabuf { + ChunkKind::Invalid + } else { + ChunkKind::Data + } +} + +/// Parse into a zeroed value before replacing the current format. libspa leaves +/// omitted optional properties untouched, so parsing into the reused value +/// would retain stale color metadata across renegotiation. fn replace_video_format( current: &mut VideoInfoRaw, parse: impl FnOnce(&mut VideoInfoRaw) -> Result, @@ -671,102 +1291,6 @@ fn format_requires_restart( geometry.is_some_and(|geometry| geometry != (width, height) || color != next_color) } -/// A chunk offset is a ring position within the allocation, so wrap it rather -/// than trusting it to be in range. `None` means the allocation is unusable. -fn normalize_chunk_offset(offset: u32, maxsize: u32) -> Option { - (maxsize != 0).then(|| (offset % maxsize) as usize) -} - -fn clamp_chunk_size(size: u32, maxsize: u32) -> usize { - size.min(maxsize) as usize -} - -fn chunk_bytes(data: &[u8], offset: usize, size: usize) -> Option> { - if offset >= data.len() || size > data.len() { - return None; - } - let end = offset.checked_add(size)?; - if end <= data.len() { - return Some(Cow::Borrowed(&data[offset..end])); - } - - let mut wrapped = Vec::with_capacity(size); - wrapped.extend_from_slice(&data[offset..]); - let head = size - wrapped.len(); - wrapped.extend_from_slice(&data[..head]); - Some(Cow::Owned(wrapped)) -} - -/// Bytes from the chunk start through the span required by `convert`. NV12 stops -/// at the visible width of its final row; packed RGB requires every full stride. -fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { - let stride = layout.stride as usize; - let width = layout.width as usize; - let height = layout.height as usize; - let row_size = match format { - VideoFormat::NV12 => width, - VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => width.checked_mul(4)?, - _ => return None, - }; - if stride < row_size { - return None; - } - - match format { - // Chroma follows every source luma row, then runs at half height. - VideoFormat::NV12 => (layout.source_height as usize) - .checked_add(height / 2)? - .checked_sub(1)? - .checked_mul(stride)? - .checked_add(row_size), - _ => stride.checked_mul(height), - } -} - -/// Deinterleave strided NV12 into the crate's tightly packed I420 layout. -fn nv12_to_i420(data: &[u8], layout: FrameLayout) -> Result { - let (stride, width, height, source_height) = ( - layout.stride as usize, - layout.width as usize, - layout.height as usize, - layout.source_height as usize, - ); - if source_height < height { - return Err(Error::Codec(anyhow::anyhow!( - "NV12 source is shorter than the cropped output" - ))); - } - let y_len = stride - .checked_mul(source_height) - .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 luma size overflow")))?; - let uv_rows = height / 2; - let uv_len = uv_rows - .checked_sub(1) - .and_then(|rows| rows.checked_mul(stride)) - .and_then(|offset| offset.checked_add(width)) - .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 chroma size overflow")))?; - let frame_len = y_len - .checked_add(uv_len) - .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 frame size overflow")))?; - if stride < width || data.len() < frame_len { - return Err(Error::Codec(anyhow::anyhow!( - "NV12 frame is shorter than its declared rows" - ))); - } - - let mut packed = vec![0; I420::len(width as u32, height as u32)]; - for row in 0..height { - packed[row * width..(row + 1) * width].copy_from_slice(&data[row * stride..row * stride + width]); - } - let uv = &data[y_len..frame_len]; - let packed_uv = width * height; - for row in 0..uv_rows { - packed[packed_uv + row * width..packed_uv + (row + 1) * width] - .copy_from_slice(&uv[row * stride..row * stride + width]); - } - I420::from_nv12(&packed, width as u32, height as u32) -} - /// Convert one strided screen frame to tightly-packed I420. fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option) -> Result { match format { @@ -785,10 +1309,29 @@ fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option } } -/// Serialize the `EnumFormat` pod offering the layouts we can convert (packed -/// RGB first, then NV12), any size, and a framerate range preferring `framerate`. -fn format_offer(framerate: u32) -> Vec { - let obj = spa::pod::object!( +/// Map the negotiated SPA layout to its DRM fourcc. +fn drm_format(format: VideoFormat) -> Option { + match format { + VideoFormat::NV12 => Some(DrmFormat::NV12), + VideoFormat::BGRx => Some(DrmFormat::XRGB8888), + VideoFormat::BGRA => Some(DrmFormat::ARGB8888), + VideoFormat::RGBx => Some(DrmFormat::XBGR8888), + VideoFormat::RGBA => Some(DrmFormat::ABGR8888), + _ => None, + } +} + +const PIPEWIRE_FORMATS: [VideoFormat; 5] = [ + VideoFormat::BGRx, + VideoFormat::BGRA, + VideoFormat::RGBx, + VideoFormat::RGBA, + VideoFormat::NV12, +]; + +/// Serialize one `EnumFormat` pod for a concrete pixel format. +fn format_offer(framerate: u32, format: VideoFormat, dmabuf: bool) -> Vec { + let mut obj = spa::pod::object!( spa::utils::SpaTypes::ObjectParamFormat, spa::param::ParamType::EnumFormat, spa::pod::property!( @@ -801,18 +1344,7 @@ fn format_offer(framerate: u32) -> Vec { Id, spa::param::format::MediaSubtype::Raw ), - spa::pod::property!( - spa::param::format::FormatProperties::VideoFormat, - Choice, - Enum, - Id, - VideoFormat::BGRx, - VideoFormat::BGRx, - VideoFormat::BGRA, - VideoFormat::RGBx, - VideoFormat::RGBA, - VideoFormat::NV12, - ), + spa::pod::property!(spa::param::format::FormatProperties::VideoFormat, Id, format), spa::pod::property!( spa::param::format::FormatProperties::VideoSize, Choice, @@ -841,79 +1373,284 @@ fn format_offer(framerate: u32) -> Vec { spa::utils::Fraction { num: 1000, denom: 1 } ), ); + if dmabuf { + obj.properties.push(spa::pod::Property { + key: spa::param::format::FormatProperties::VideoModifier.as_raw(), + flags: spa::pod::PropertyFlags::from_bits_retain( + spa::sys::SPA_POD_PROP_FLAG_MANDATORY | spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE, + ), + value: spa::pod::Value::Choice(spa::pod::ChoiceValue::Long(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Enum { + default: 0, + alternatives: vec![0], + }, + ))), + }); + } spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(obj)) .expect("serializing a static format pod cannot fail") .0 .into_inner() } +/// Serialize DMA-BUF formats first and shared-memory fallbacks second. +fn format_offers(framerate: u32) -> Vec> { + PIPEWIRE_FORMATS + .into_iter() + .map(|format| format_offer(framerate, format, true)) + .chain( + PIPEWIRE_FORMATS + .into_iter() + .map(|format| format_offer(framerate, format, false)), + ) + .collect() +} + +/// Serialize `SPA_PARAM_Buffers` for the negotiated memory representation. +fn buffer_offer(dmabuf: bool) -> Vec { + let mem_ptr = 1 << DataType::MemPtr.as_raw(); + let mem_fd = 1 << DataType::MemFd.as_raw(); + let dma_buf = 1 << DataType::DmaBuf.as_raw(); + let data_types = if dmabuf { dma_buf } else { mem_fd | mem_ptr }; + + let obj = spa::pod::Object { + type_: spa::utils::SpaTypes::ObjectParamBuffers.as_raw(), + id: spa::param::ParamType::Buffers.as_raw(), + properties: vec![ + spa::pod::Property::new( + spa::sys::SPA_PARAM_BUFFERS_buffers, + spa::pod::Value::Choice(spa::pod::ChoiceValue::Int(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Range { + default: 8, + min: 2, + max: 64, + }, + ))), + ), + spa::pod::Property::new(spa::sys::SPA_PARAM_BUFFERS_blocks, spa::pod::Value::Int(1)), + spa::pod::Property::new( + spa::sys::SPA_PARAM_BUFFERS_dataType, + spa::pod::Value::Choice(spa::pod::ChoiceValue::Int(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Flags { + default: data_types, + flags: Vec::new(), + }, + ))), + ), + ], + }; + spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &spa::pod::Value::Object(obj)) + .expect("serializing a static buffer pod cannot fail") + .0 + .into_inner() +} + #[cfg(test)] mod tests { use super::*; use crate::capture::Config; - /// The serialized format offer must parse back as a valid pod, carrying every - /// layout `convert` knows how to handle. + /// The serialized format offer must parse back as a valid pod. #[test] fn format_offer_is_valid_pod() { - let bytes = format_offer(30); - let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(&bytes) - .expect("format offer did not round-trip"); - assert!(remaining.is_empty()); - let spa::pod::Value::Object(object) = value else { - panic!("format offer is not an object"); - }; - let property = object - .properties - .iter() - .find(|property| property.key == spa::param::format::FormatProperties::VideoFormat.as_raw()) - .expect("missing video format property"); - assert_eq!( - property.value, - spa::pod::Value::Choice(spa::pod::ChoiceValue::Id(spa::utils::Choice( - spa::utils::ChoiceFlags::empty(), - spa::utils::ChoiceEnum::Enum { - default: spa::utils::Id(VideoFormat::BGRx.as_raw()), - alternatives: vec![ - spa::utils::Id(VideoFormat::BGRx.as_raw()), - spa::utils::Id(VideoFormat::BGRA.as_raw()), - spa::utils::Id(VideoFormat::RGBx.as_raw()), - spa::utils::Id(VideoFormat::RGBA.as_raw()), - spa::utils::Id(VideoFormat::NV12.as_raw()), - ], - }, - ))) - ); + let offers = format_offers(30); + assert_eq!(offers.len(), PIPEWIRE_FORMATS.len() * 2); + for (index, bytes) in offers.iter().enumerate() { + let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(bytes) + .expect("format offer did not round-trip"); + assert!(remaining.is_empty()); + let spa::pod::Value::Object(object) = value else { + panic!("format offer is not an object"); + }; + let property = |key| object.properties.iter().find(|property| property.key == key); + let format = PIPEWIRE_FORMATS[index % PIPEWIRE_FORMATS.len()]; + assert_eq!( + property(spa::param::format::FormatProperties::VideoFormat.as_raw()).map(|p| &p.value), + Some(&spa::pod::Value::Id(spa::utils::Id(format.as_raw()))) + ); + let modifier = property(spa::param::format::FormatProperties::VideoModifier.as_raw()); + if index < PIPEWIRE_FORMATS.len() { + let modifier = modifier.expect("DMA-BUF offer has no modifier"); + assert_eq!( + modifier.flags.bits(), + spa::sys::SPA_POD_PROP_FLAG_MANDATORY | spa::sys::SPA_POD_PROP_FLAG_DONT_FIXATE + ); + assert_eq!( + modifier.value, + spa::pod::Value::Choice(spa::pod::ChoiceValue::Long(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Enum { + default: 0, + alternatives: vec![0], + }, + ))) + ); + } else { + assert!(modifier.is_none(), "shared-memory offer contains a modifier"); + } + } + } + + #[test] + fn negotiated_modifier_must_be_present_and_fixed() { + let shared = format_offer(30, VideoFormat::BGRx, false); + let shared = spa::pod::Pod::from_bytes(&shared).unwrap(); + let mut format = VideoInfoRaw::default(); + format.parse(shared).unwrap(); + assert_eq!(negotiated_memory(shared, format), Some(NegotiatedMemory::SharedMemory)); + + let offered = format_offer(30, VideoFormat::BGRx, true); + let offered = spa::pod::Pod::from_bytes(&offered).unwrap(); + format.parse(offered).unwrap(); + assert_eq!(negotiated_memory(offered, format), Some(NegotiatedMemory::Fixating)); + + let fixed = fixate_modifier(offered, format.modifier()).unwrap(); + let fixed = spa::pod::Pod::from_bytes(&fixed).unwrap(); + replace_video_format(&mut format, |format| format.parse(fixed)).unwrap(); + assert_eq!(negotiated_memory(fixed, format), Some(NegotiatedMemory::DmaBuf(0))); + } + + #[test] + fn buffer_offer_is_valid_pod() { + let dma_buf = 1 << DataType::DmaBuf.as_raw(); + let mem_fd = 1 << DataType::MemFd.as_raw(); + let mem_ptr = 1 << DataType::MemPtr.as_raw(); + for (dmabuf, data_types) in [(true, dma_buf), (false, mem_fd | mem_ptr)] { + let bytes = buffer_offer(dmabuf); + let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(&bytes) + .expect("buffer offer did not round-trip"); + assert!(remaining.is_empty()); + let spa::pod::Value::Object(object) = value else { + panic!("buffer offer is not an object"); + }; + let property = |key| { + &object + .properties + .iter() + .find(|property| property.key == key) + .unwrap_or_else(|| panic!("missing buffer property {key}")) + .value + }; + assert_eq!( + property(spa::sys::SPA_PARAM_BUFFERS_buffers), + &spa::pod::Value::Choice(spa::pod::ChoiceValue::Int(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Range { + default: 8, + min: 2, + max: 64, + }, + ))) + ); + assert_eq!(property(spa::sys::SPA_PARAM_BUFFERS_blocks), &spa::pod::Value::Int(1)); + assert_eq!( + property(spa::sys::SPA_PARAM_BUFFERS_dataType), + &spa::pod::Value::Choice(spa::pod::ChoiceValue::Int(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Flags { + default: data_types, + flags: Vec::new(), + }, + ))) + ); + } } #[test] fn chunk_offset_wraps_to_the_allocation() { assert_eq!(normalize_chunk_offset(18, 16), Some(2)); assert_eq!(normalize_chunk_offset(0, 0), None); + assert_eq!(normalize_dma_buf_offset(18, 16), 2); + assert_eq!(normalize_dma_buf_offset(18, 0), 18); } #[test] fn chunk_size_is_clamped_to_the_allocation() { assert_eq!(clamp_chunk_size(18, 16), 16); assert_eq!(clamp_chunk_size(8, 16), 8); + assert_eq!(dma_buf_chunk_size(8, 16), 8); + assert_eq!(dma_buf_chunk_size(8, 0), 8); + assert!(dma_buf_chunk_contains(16, 0, 0)); + assert!(!dma_buf_chunk_contains(16, 8, 0)); + assert!(dma_buf_chunk_contains(16, 16, 16)); } #[test] - fn chunk_flags_distinguish_neutral_and_invalid_frames() { + fn frame_data_size_covers_every_sampled_row() { + let packed = FrameLayout { + stride: 20, + width: 4, + height: 2, + source_height: 2, + }; + assert_eq!(frame_data_size(VideoFormat::BGRx, packed), Some(36)); + + let nv12 = FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 3, + }; + assert_eq!(frame_data_size(VideoFormat::NV12, nv12), Some(22)); + assert_eq!( + frame_data_size(VideoFormat::BGRx, FrameLayout { stride: 15, ..packed }), + None + ); + } + + #[test] + fn wrapped_chunk_is_reassembled() { + let data = [0, 1, 2, 3, 4, 5]; + assert_eq!(chunk_bytes(&data, 1, 3).as_deref(), Some([1, 2, 3].as_slice())); + assert_eq!(chunk_bytes(&data, 4, 4).as_deref(), Some([4, 5, 0, 1].as_slice())); + } + + #[test] + fn chunk_flags_distinguish_empty_and_invalid_frames() { let empty = spa::buffer::ChunkFlags::from_bits_retain(CHUNK_FLAG_EMPTY); - assert_eq!(chunk_kind(0, spa::buffer::ChunkFlags::empty()), ChunkKind::Invalid); - assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::CORRUPTED), ChunkKind::Invalid); - assert_eq!(chunk_kind(1, empty), ChunkKind::Empty); - assert_eq!(chunk_kind(0, empty), ChunkKind::Empty); - assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::empty()), ChunkKind::Data); + assert_eq!( + chunk_kind(0, spa::buffer::ChunkFlags::empty(), false), + ChunkKind::Invalid + ); + assert_eq!(chunk_kind(0, spa::buffer::ChunkFlags::empty(), true), ChunkKind::Data); + assert_eq!( + chunk_kind(1, spa::buffer::ChunkFlags::CORRUPTED, false), + ChunkKind::Invalid + ); + assert_eq!(chunk_kind(1, empty, false), ChunkKind::Empty); + assert_eq!(chunk_kind(0, empty, true), ChunkKind::Empty); + assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::empty(), false), ChunkKind::Data); } #[test] - fn empty_chunk_is_limited_range_black() { - let frame = neutral_frame(4, 2, None).unwrap(); - assert_eq!(frame.y(), &[16; 8]); - assert_eq!(frame.u(), &[128; 2]); - assert_eq!(frame.v(), &[128; 2]); + fn dmabuf_size_comes_from_seek_end_not_stat() { + let mut calls = Vec::new(); + let size = dma_buf_allocation_size_with_seek(1024, |offset, whence| { + calls.push((offset, whence)); + match whence { + libc::SEEK_END => Ok(4096), + libc::SEEK_SET => Ok(0), + _ => panic!("unexpected seek mode {whence}"), + } + }) + .unwrap(); + + // This models a DMA-BUF anon inode: no stat size is available, and the + // allocation length comes exclusively from its SEEK_END operation. + assert_eq!(size, Some(3072)); + assert_eq!(calls, [(0, libc::SEEK_END), (0, libc::SEEK_SET)]); + assert_eq!( + dma_buf_allocation_size_with_seek(4096, |_, whence| match whence { + libc::SEEK_END => Ok(4096), + libc::SEEK_SET => Ok(0), + _ => unreachable!(), + }) + .unwrap(), + None + ); } #[test] @@ -1007,46 +1744,14 @@ mod tests { } #[test] - fn empty_chunk_uses_full_range_black() { - let frame = neutral_frame(4, 2, Some(Color::Bt709Full)).unwrap(); - assert_eq!(frame.y(), &[0; 8]); - assert_eq!(frame.u(), &[128; 2]); - assert_eq!(frame.v(), &[128; 2]); - assert_eq!(frame.color(), Some(Color::Bt709Full)); - } - - /// The required size must reach the last sampled row, but not the padding - /// past it, or a tightly-sized final row would be rejected. - #[test] - fn frame_data_size_covers_every_sampled_row() { - let packed = FrameLayout { - stride: 20, - width: 4, - height: 2, - source_height: 2, - }; - assert_eq!(frame_data_size(VideoFormat::BGRx, packed), Some(40)); - - let nv12 = FrameLayout { - stride: 6, - width: 4, - height: 2, - source_height: 3, + fn stale_pool_lease_is_not_current() { + let buffer = std::ptr::dangling_mut::(); + let lease = Lease { + buffer: buffer as usize, + generation: 2, }; - assert_eq!(frame_data_size(VideoFormat::NV12, nv12), Some(22)); - - // A stride narrower than one row means the layout is nonsense. - assert_eq!( - frame_data_size(VideoFormat::BGRx, FrameLayout { stride: 15, ..packed }), - None - ); - } - - #[test] - fn wrapped_chunk_is_reassembled() { - let data = [0, 1, 2, 3, 4, 5]; - assert_eq!(chunk_bytes(&data, 1, 3).as_deref(), Some([1, 2, 3].as_slice())); - assert_eq!(chunk_bytes(&data, 4, 4).as_deref(), Some([4, 5, 0, 1].as_slice())); + assert_eq!(lease.current(1), None); + assert_eq!(lease.current(2), Some(buffer)); } #[test] @@ -1090,8 +1795,6 @@ mod tests { assert_eq!(frame.v(), &[10, 12]); } - /// An odd-height source has one more luma row than the clamped output, and - /// chroma starts after all of them. #[test] fn nv12_crop_uses_the_source_height_for_chroma() { let data = [ @@ -1115,6 +1818,105 @@ mod tests { assert_eq!(frame.v(), &[10, 12]); } + /// A duplicated fd does not protect the pixels from producer reuse. The + /// dequeued PipeWire buffer must be returned only after the last surface + /// clone drops, and its return crosses back onto the PipeWire loop. + #[test] + fn dmabuf_returns_on_last_drop() { + use std::cell::Cell; + use std::io::Write; + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + use std::rc::Rc; + + pw::init(); + let mainloop = pw::main_loop::MainLoopRc::new(None).expect("main loop"); + let (return_tx, return_rx) = pw::channel::channel::(); + let returned = Rc::new(Cell::new(None)); + let _returns = return_rx.attach(mainloop.loop_(), { + let mainloop = mainloop.downgrade(); + let returned = returned.clone(); + move |raw| { + returned.set(Some(raw)); + mainloop.upgrade().expect("live loop").quit(); + } + }); + let timer = mainloop.loop_().add_timer({ + let mainloop = mainloop.downgrade(); + move |_| mainloop.upgrade().expect("live loop").quit() + }); + + let (socket, mut peer) = UnixStream::pair().expect("fd pair"); + peer.write_all(&[0]).expect("signal readable"); + let inner = Arc::new(PipeWireDmaBuf { + fd: OwnedFd::from(socket), + return_tx, + lease: Lease { + buffer: 7, + generation: 1, + }, + map_offset: 0, + allocation_size: Some(6), + data_offset: 0, + layout: FrameLayout { + stride: 2, + width: 2, + height: 2, + source_height: 2, + }, + format: DrmFormat::NV12, + modifier: 0, + color: Some(Color::Bt709Full), + }); + let frame = DmaBuf::new( + DrmFormat::NV12, + 0, + 2, + 2, + vec![DmaBufPlane::new(0, 2), DmaBufPlane::new(4, 2)], + Some(Color::Bt709Full), + inner, + ) + .expect("DMA-BUF"); + assert_eq!(frame.format(), DrmFormat::NV12); + assert_eq!(frame.modifier(), 0); + assert_eq!((frame.width(), frame.height()), (2, 2)); + assert_eq!(frame.planes()[1], DmaBufPlane::new(4, 2)); + assert_eq!(Surface::DmaBuf(frame.clone()).color(), Some(Color::Bt709Full)); + let export = frame.export().expect("exported fd"); + let clone = frame.clone(); + + drop(frame); + timer + .update_timer(Some(Duration::from_millis(10)), None) + .into_result() + .expect("timer"); + mainloop.run(); + assert_eq!(returned.get(), None, "one live clone still owns the lease"); + + drop(clone); + timer + .update_timer(Some(Duration::from_millis(10)), None) + .into_result() + .expect("timer"); + mainloop.run(); + assert_eq!(returned.get(), None, "one live export still owns the lease"); + + drop(export); + timer + .update_timer(Some(Duration::from_secs(1)), None) + .into_result() + .expect("timeout timer"); + mainloop.run(); + assert_eq!( + returned.get(), + Some(Lease { + buffer: 7, + generation: 1 + }) + ); + } + /// Open the portal, grab a few frames, and check geometry. Ignored because it /// needs a desktop session, PipeWire, and a human clicking the picker dialog: /// `cargo test -p moq-video --features pipewire portal_capture -- --ignored`. diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 6627a187357..6165eebc188 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -13,6 +13,9 @@ //! `PixelBuffer` but has no Direct3D11 path yet. // `render` is deliberately not a doc link: the module sits behind a non-default // feature, so linking it fails the `-D warnings` rustdoc build of a plain build. +//! - `Surface::DmaBuf` is a Linux DRM allocation, produced by PipeWire capture. +//! The Vulkan renderer imports supported packed formats directly, while CPU +//! consumers map linear allocations only. //! - `Surface::I420` is CPU-resident planar I420, for the CPU encode path and //! platforms without a zero-copy capture. //! @@ -22,6 +25,11 @@ use std::borrow::Cow; +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +use std::sync::Arc; + use bytes::Bytes; use moq_net::Timestamp; @@ -74,6 +82,259 @@ impl Frame { } } +/// A DRM pixel format code carried by a Linux DMA-BUF. +/// +/// The four bytes are the kernel DRM fourcc, kept as a newtype so a stride, +/// PipeWire format id, or another bare integer cannot be passed accidentally. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DrmFormat(u32); + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl DrmFormat { + /// Semi-planar 8-bit 4:2:0 YUV. + pub const NV12: Self = Self::from_bytes(*b"NV12"); + /// Packed BGRx8888 as named by DRM (`XR24`). + pub const XRGB8888: Self = Self::from_bytes(*b"XR24"); + /// Packed BGRA8888 as named by DRM (`AR24`). + pub const ARGB8888: Self = Self::from_bytes(*b"AR24"); + /// Packed RGBx8888 as named by DRM (`XB24`). + pub const XBGR8888: Self = Self::from_bytes(*b"XB24"); + /// Packed RGBA8888 as named by DRM (`AB24`). + pub const ABGR8888: Self = Self::from_bytes(*b"AB24"); + + /// Build a DRM fourcc from its four ASCII bytes. + pub const fn from_bytes(bytes: [u8; 4]) -> Self { + Self(u32::from_le_bytes(bytes)) + } + + /// The integer value used by DRM, Vulkan, EGL, and VAAPI descriptors. + pub const fn as_raw(self) -> u32 { + self.0 + } +} + +/// One plane within a Linux DMA-BUF allocation. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DmaBufPlane { + offset: u32, + stride: u32, +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl DmaBufPlane { + #[cfg(feature = "pipewire")] + pub(crate) const fn new(offset: u32, stride: u32) -> Self { + Self { offset, stride } + } + + /// Byte offset of this plane from the start of the exported allocation. + pub const fn offset(&self) -> u32 { + self.offset + } + + /// Bytes between adjacent rows in this plane. + pub const fn stride(&self) -> u32 { + self.stride + } +} + +/// An exported Linux DMA-BUF descriptor and its producer lease. +/// +/// Keep this value alive for as long as an external device may read from the +/// descriptor returned by [`as_fd`](Self::as_fd). Dropping it releases the +/// producer's buffer when no other frame or export still owns that lease. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +pub struct DmaBufExport { + fd: OwnedFd, + inner: Arc, +} + +/// How long to wait on a producer's write fence before giving up. +/// +/// Vulkan does not adopt a DMA-BUF's implicit fence, so a reader has to wait for +/// it here. A screen frame's fence signals within a frame time; anything past +/// this is a wedged compositor, and the caller's CPU fallback beats blocking a +/// render thread forever. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +const DMA_BUF_FENCE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +pub(crate) fn wait_dma_buf_readable(fd: BorrowedFd<'_>) -> std::io::Result<()> { + let mut event = libc::pollfd { + fd: fd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let deadline = std::time::Instant::now() + DMA_BUF_FENCE_TIMEOUT; + loop { + // A signal restarts the wait against the same deadline rather than + // granting a fresh budget, so the total stall stays bounded. + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + return Err(std::io::Error::from(std::io::ErrorKind::TimedOut)); + } + // SAFETY: `event` is valid for this call and `fd` remains borrowed until + // the producer's current write fence has completed. + let result = unsafe { + libc::poll( + &mut event, + 1, + remaining.as_millis().min(i32::MAX as u128) as libc::c_int, + ) + }; + if result > 0 && event.revents & libc::POLLIN != 0 { + return Ok(()); + } + if result == 0 { + return Err(std::io::Error::from(std::io::ErrorKind::TimedOut)); + } + if result < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + return Err(std::io::Error::other(format!( + "DMA-BUF poll returned events {:#x}", + event.revents + ))); + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl DmaBufExport { + /// Borrow the exported descriptor without separating it from its producer lease. + pub fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { + std::os::fd::AsFd::as_fd(&self.fd) + } + + pub(crate) fn into_parts(self) -> (OwnedFd, Arc) { + (self.fd, self.inner) + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl std::os::fd::AsFd for DmaBufExport { + fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { + std::os::fd::AsFd::as_fd(&self.fd) + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl std::fmt::Debug for DmaBufExport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DmaBufExport").finish_non_exhaustive() + } +} + +/// A Linux DMA-BUF surface with an on-demand exported descriptor. +/// +/// Cloning this value retains the producer's surface but opens no file +/// descriptor. [`export`](Self::export) duplicates the descriptor only when a +/// consumer is ready to import it, avoiding one open fd for every buffered +/// frame. Dropping the last clone or [`DmaBufExport`] releases the producer's +/// buffer. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +#[derive(Clone)] +pub struct DmaBuf { + format: DrmFormat, + modifier: u64, + width: u32, + height: u32, + planes: Vec, + color: Option, + inner: Arc, +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl std::fmt::Debug for DmaBuf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DmaBuf") + .field("format", &self.format) + .field("modifier", &format_args!("{:#x}", self.modifier)) + .field("width", &self.width) + .field("height", &self.height) + .field("planes", &self.planes) + .finish_non_exhaustive() + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl DmaBuf { + #[cfg(feature = "pipewire")] + pub(crate) fn new( + format: DrmFormat, + modifier: u64, + width: u32, + height: u32, + planes: Vec, + color: Option, + inner: Arc, + ) -> Result { + Size::new(width, height).validate("DMA-BUF")?; + if planes.is_empty() { + return Err(Error::Codec(anyhow::anyhow!("DMA-BUF has no planes"))); + } + Ok(Self { + format, + modifier, + width, + height, + planes, + color, + inner, + }) + } + + /// Wait for producer writes, then export the descriptor with its producer lease. + pub fn export(&self) -> std::io::Result { + let fd = self.inner.export()?; + wait_dma_buf_readable(fd.as_fd())?; + Ok(DmaBufExport { + fd, + inner: self.inner.clone(), + }) + } + + /// DRM fourcc describing the plane layout. + pub const fn format(&self) -> DrmFormat { + self.format + } + + /// DRM format modifier describing the allocation's tiling. + pub const fn modifier(&self) -> u64 { + self.modifier + } + + /// Width of the coded allocation in pixels. + pub const fn width(&self) -> u32 { + self.width + } + + /// Height of the coded allocation in pixels. + pub const fn height(&self) -> u32 { + self.height + } + + /// Plane offsets and row strides, in format order. + pub fn planes(&self) -> &[DmaBufPlane] { + &self.planes + } +} + +/// The producer-owned half of a DMA-BUF surface. +/// +/// Kept private to the crate so backend lifetimes and download mechanisms do +/// not become public implementable API. [`DmaBuf`] is the stable consumer seam. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +pub(crate) trait DmaBufFrame: Send + Sync { + fn export(&self) -> std::io::Result; + fn download_i420(&self) -> Result; +} + /// Where a frame's pixels currently live. /// /// Decoders and capture sources hand these out; encoders and renderers consume @@ -105,6 +366,9 @@ pub enum Surface { /// decoder, consumed in place by the NVENC encoder. #[cfg(all(target_os = "linux", feature = "nvdec"))] Cuda(cuda::Frame), + /// Linux DMA-BUF, exported on access and retained until the last clone drops. + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaBuf(DmaBuf), /// CPU-resident planar I420. I420(I420), } @@ -119,6 +383,8 @@ impl Surface { Surface::Texture(t) => t.width, #[cfg(all(target_os = "linux", feature = "nvdec"))] Surface::Cuda(c) => c.width, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Surface::DmaBuf(d) => d.width, Surface::I420(i) => i.width, } } @@ -132,6 +398,8 @@ impl Surface { Surface::Texture(t) => t.height, #[cfg(all(target_os = "linux", feature = "nvdec"))] Surface::Cuda(c) => c.height, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Surface::DmaBuf(d) => d.height, Surface::I420(i) => i.height, } } @@ -288,6 +556,8 @@ impl Surface { Surface::Texture(_) => None, #[cfg(all(target_os = "linux", feature = "nvdec"))] Surface::Cuda(_) => None, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Surface::DmaBuf(d) => d.color, Surface::I420(i) => i.color(), } } @@ -301,6 +571,8 @@ impl Surface { Surface::Texture(t) => Ok(Cow::Owned(t.download_i420()?)), #[cfg(all(target_os = "linux", feature = "nvdec"))] Surface::Cuda(c) => Ok(Cow::Owned(c.download_i420()?)), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Surface::DmaBuf(d) => Ok(Cow::Owned(d.inner.download_i420()?)), Surface::I420(i) => Ok(Cow::Borrowed(i)), } } diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 86927be584d..4165a0b1059 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -84,6 +84,8 @@ mod mf; pub use color::Color; pub use error::Error; +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +pub use frame::{DmaBuf, DmaBufExport, DmaBufPlane, DrmFormat}; pub use frame::{Frame, I420, Surface}; pub use size::Size; diff --git a/rs/moq-video/src/render/dmabuf.rs b/rs/moq-video/src/render/dmabuf.rs new file mode 100644 index 00000000000..015447b7b39 --- /dev/null +++ b/rs/moq-video/src/render/dmabuf.rs @@ -0,0 +1,102 @@ +//! Zero-copy import of packed Linux DMA-BUFs into wgpu's Vulkan backend. +//! +//! wgpu owns the Vulkan image and imported fd once wrapping succeeds. The +//! [`DmaBuf`] itself rides with the submitted render work separately, keeping +//! the dequeued producer buffer out of PipeWire's pool until the GPU is done. + +use wgpu::hal::MemoryFlags; + +use super::source::{Layout, Source}; +use crate::{DmaBuf, DrmFormat, Error, Size}; + +fn err(message: impl std::fmt::Display) -> Error { + Error::Render(anyhow::anyhow!("{message}")) +} + +/// Alias one packed DMA-BUF allocation as a sampled Vulkan texture. +pub(super) fn import(device: &wgpu::Device, buffer: &DmaBuf) -> Result, Error> { + if !device + .features() + .contains(wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF) + { + return Ok(None); + } + + let format = match buffer.format() { + DrmFormat::XRGB8888 | DrmFormat::ARGB8888 => wgpu::TextureFormat::Bgra8Unorm, + DrmFormat::XBGR8888 | DrmFormat::ABGR8888 => wgpu::TextureFormat::Rgba8Unorm, + format => return Err(err(format!("cannot import DMA-BUF format {:#x}", format.as_raw()))), + }; + let [plane] = buffer.planes() else { + return Err(err("packed DMA-BUF must have exactly one plane")); + }; + let size = Size::new(buffer.width(), buffer.height()); + let extent = wgpu::Extent3d { + width: size.width, + height: size.height, + depth_or_array_layers: 1, + }; + let descriptor = wgpu::TextureDescriptor { + label: Some("moq-video imported DMA-BUF"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }; + let hal_descriptor = wgpu::hal::TextureDescriptor { + label: descriptor.label, + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUses::RESOURCE, + memory_flags: MemoryFlags::empty(), + view_formats: Vec::new(), + }; + + // SAFETY: the guard is only used to import a descriptor into the same + // Vulkan device. It drops before the resulting HAL texture is wrapped. + let Some(hal) = (unsafe { device.as_hal::() }) else { + return Ok(None); + }; + let export = buffer + .export() + .map_err(|e| Error::Render(anyhow::Error::new(e).context("export DMA-BUF")))?; + let (fd, keepalive) = export.into_parts(); + // SAFETY: `fd` is a fresh duplicate of this live DMA-BUF. Export waited for + // producer writes, and the format, modifier, extent, stride, and offset come + // from PipeWire's buffer metadata. Vulkan consumes the duplicate on success + // and wgpu-hal closes it on error. + let texture = unsafe { + hal.texture_from_dmabuf_fd( + fd, + &hal_descriptor, + buffer.modifier(), + plane.stride() as u64, + plane.offset() as u64, + ) + } + .map_err(|e| err(format!("Vulkan DMA-BUF import: {e:?}")))?; + drop(hal); + + // SAFETY: wgpu-hal created `texture` on this device from `hal_descriptor`, + // which exactly matches the public descriptor. Imported pixels are already + // initialized and will first be used as a sampled resource. + let texture = unsafe { + device.create_texture_from_hal::(texture, &descriptor, wgpu::TextureUses::RESOURCE) + }; + let view = texture.create_view(&Default::default()); + + Ok(Some(Source { + layout: Layout::Rgba, + color: None, + plane0: view.clone(), + plane1: view.clone(), + plane2: view, + keepalive: Some(Box::new(keepalive)), + })) +} diff --git a/rs/moq-video/src/render/metal.rs b/rs/moq-video/src/render/metal.rs index 27a6c8b5827..73aea95f89d 100644 --- a/rs/moq-video/src/render/metal.rs +++ b/rs/moq-video/src/render/metal.rs @@ -157,10 +157,11 @@ impl Import { Ok(Source { layout, - color, + color: Some(color), plane0, plane1, plane2, + keepalive: None, }) } diff --git a/rs/moq-video/src/render/mod.rs b/rs/moq-video/src/render/mod.rs index b9f66bcc285..e80530235e9 100644 --- a/rs/moq-video/src/render/mod.rs +++ b/rs/moq-video/src/render/mod.rs @@ -26,13 +26,15 @@ //! ## Zero-copy //! //! A hardware-decoded frame is imported by aliasing the decoder's surface as a -//! texture rather than copying it: `CVMetalTextureCache` on macOS, for the -//! `PixelBuffer` variant of [`Surface`](crate::Surface) that capture and a -//! VideoToolbox decode produce. +//! texture rather than copying it: `CVMetalTextureCache` on macOS for the +//! `PixelBuffer` variant of [`Surface`](crate::Surface), and Vulkan external +//! memory on Linux for packed PipeWire DMA-BUFs. // `PixelBuffer` is deliberately not a doc link: the variant is macOS-only, so a // link to it fails the `-D warnings` rustdoc build on every other platform. //! -//! Every other frame, and any import that fails, goes through +//! Linux callers must request +//! [`wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF`] when creating the device +//! to enable that import. Every other frame, and any import that fails, goes through //! [`Surface::into_i420`](crate::Surface::into_i420) and a plane upload. That //! path is always available, so which route a frame takes is a question of cost. //! An import path that keeps failing (a driver that cannot do it at all) retires @@ -44,6 +46,9 @@ mod color; mod renderer; mod source; +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +mod dmabuf; + #[cfg(target_os = "macos")] mod metal; diff --git a/rs/moq-video/src/render/renderer.rs b/rs/moq-video/src/render/renderer.rs index 8e521ce39b4..b82710d5c5f 100644 --- a/rs/moq-video/src/render/renderer.rs +++ b/rs/moq-video/src/render/renderer.rs @@ -12,6 +12,18 @@ use crate::{Color, Error, Frame, Size}; /// frame rate, so the path is retired and the CPU fallback takes over. const ZERO_COPY_STRIKES: u32 = 3; +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +fn dma_buf_import_timed_out(error: &Error) -> bool { + let Error::Render(error) = error else { + return false; + }; + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::TimedOut) + }) +} + /// Renderer configuration. /// /// `#[non_exhaustive]`: build via [`Config::new`] (or `default()`) and set the @@ -100,6 +112,8 @@ impl Config { pub struct Renderer { device: wgpu::Device, queue: wgpu::Queue, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + completion: Completion, config: Config, shader: Pipelines, @@ -125,15 +139,78 @@ struct Pipelines { /// everywhere, so it stays validated on every platform either way. #[cfg(target_os = "macos")] nv12: wgpu::RenderPipeline, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + /// Packed RGB or BGR imported from a Linux DMA-BUF. + rgba: wgpu::RenderPipeline, i420: wgpu::RenderPipeline, } +/// Waits for submitted GPU work before releasing its producer-owned surfaces. +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +struct Completion { + device: wgpu::Device, + tx: Option)>>, + thread: Option>, +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl Completion { + fn new(device: &wgpu::Device) -> Result { + let (tx, rx) = std::sync::mpsc::channel(); + let worker_device = device.clone(); + let thread = std::thread::Builder::new() + .name("moq-video-gpu-completion".into()) + .spawn(move || { + while let Ok((submission, keepalive)) = rx.recv() { + if let Err(err) = worker_device.poll(wgpu::PollType::Wait { + submission_index: Some(submission), + timeout: None, + }) { + tracing::warn!(%err, "waiting for imported GPU surface failed"); + } + drop(keepalive); + } + }) + .map_err(|err| Error::Render(anyhow::anyhow!("start GPU completion worker: {err}")))?; + + Ok(Self { + device: device.clone(), + tx: Some(tx), + thread: Some(thread), + }) + } + + fn submit(&self, submission: wgpu::SubmissionIndex, keepalive: Box) { + let tx = self.tx.as_ref().expect("completion sender lives until drop"); + if let Err(err) = tx.send((submission, keepalive)) { + let (submission, keepalive) = err.0; + let _ = self.device.poll(wgpu::PollType::Wait { + submission_index: Some(submission), + timeout: None, + }); + drop(keepalive); + } + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl Drop for Completion { + fn drop(&mut self) { + drop(self.tx.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + impl Renderer { /// Build a renderer on an existing `wgpu` device. /// /// The device and queue are the application's: the renderer draws into /// textures that application already owns, so it never creates a device of - /// its own. Both handles are cheap to clone and are kept. + /// its own. Both handles are cheap to clone and are kept. On Linux, request + /// [`wgpu::Features::VULKAN_EXTERNAL_MEMORY_DMA_BUF`] on the device to import + /// PipeWire DMA-BUFs instead of downloading them. pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, config: Config) -> Result { if let Some(size) = config.size { size.validate_nonzero("render output")?; @@ -150,6 +227,8 @@ impl Renderer { Ok(Self { device: device.clone(), queue: queue.clone(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + completion: Completion::new(device)?, config, shader, uniform, @@ -167,12 +246,14 @@ impl Renderer { /// call overwrites what you are holding. Present or copy it before rendering /// again. pub fn render(&mut self, frame: &Frame) -> Result { - let source = self.source(frame)?; - let color = self.config.color.unwrap_or(source.color); - if self.color != Some(color) { - self.queue - .write_buffer(&self.uniform, 0, bytemuck::cast_slice(&uniform(color))); - self.color = Some(color); + let mut source = self.source(frame)?; + if let Some(source_color) = source.color { + let color = self.config.color.unwrap_or(source_color); + if self.color != Some(color) { + self.queue + .write_buffer(&self.uniform, 0, bytemuck::cast_slice(&uniform(color))); + self.color = Some(color); + } } let output = self.output(frame.size())?; @@ -206,6 +287,8 @@ impl Renderer { }); let pipeline = match source.layout { + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Layout::Rgba => &self.shader.rgba, #[cfg(target_os = "macos")] Layout::Nv12 => &self.shader.nv12, Layout::I420 => &self.shader.i420, @@ -236,7 +319,13 @@ impl Renderer { pass.set_bind_group(0, &bind, &[]); pass.draw(0..3, 0..1); } - self.queue.submit([encoder.finish()]); + let submission = self.queue.submit([encoder.finish()]); + if let Some(keepalive) = source.keepalive.take() { + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + self.completion.submit(submission, keepalive); + #[cfg(not(all(target_os = "linux", feature = "dmabuf")))] + drop((submission, keepalive)); + } Ok(output) } @@ -254,6 +343,8 @@ impl Renderer { // failure, so it costs no strike: the CPU path is the answer // for this surface and always will be. Ok(None) => {} + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Err(err) if dma_buf_import_timed_out(&err) => return Err(err), Err(err) => { self.strikes += 1; self.retired = self.strikes >= ZERO_COPY_STRIKES; @@ -406,6 +497,8 @@ impl Pipelines { }); Ok(Self { + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + rgba: pipeline("rgba"), #[cfg(target_os = "macos")] nv12: pipeline("nv12"), i420: pipeline("i420"), @@ -422,6 +515,18 @@ mod tests { use super::*; use crate::Surface; + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + #[test] + fn dma_buf_fence_timeout_is_terminal_for_the_frame() { + let timed_out = Error::Render(anyhow::Error::new(std::io::Error::from(std::io::ErrorKind::TimedOut))); + let other = Error::Render(anyhow::Error::new(std::io::Error::from( + std::io::ErrorKind::PermissionDenied, + ))); + + assert!(dma_buf_import_timed_out(&timed_out)); + assert!(!dma_buf_import_timed_out(&other)); + } + /// Every test here draws on a real GPU, which a headless CI runner does not /// have (wgpu finds no adapter and `Renderer::new` never gets built). The /// color math itself is covered by [`super::super::color`]'s tests, which diff --git a/rs/moq-video/src/render/shader.wgsl b/rs/moq-video/src/render/shader.wgsl index db8918f6b80..a919b38f77f 100644 --- a/rs/moq-video/src/render/shader.wgsl +++ b/rs/moq-video/src/render/shader.wgsl @@ -1,5 +1,5 @@ -// YUV 4:2:0 -> RGB, for the two plane layouts the renderer feeds it: NV12 (luma -// plane + interleaved chroma plane) and I420 (three separate planes). +// The renderer feeds this shader packed RGBA, NV12 (luma plus interleaved +// chroma), or I420 (three separate planes). // // Chroma is sampled with a linear filter, so the half-resolution planes are // upsampled by the texture unit rather than in here. Output is gamma-encoded @@ -43,6 +43,11 @@ fn convert(yuv: vec3) -> vec4 { return vec4(clamp(rgb, vec3(0.0), vec3(1.0)), 1.0); } +@fragment +fn rgba(in: Vertex) -> @location(0) vec4 { + return vec4(textureSample(plane0, samp, in.uv).rgb, 1.0); +} + @fragment fn nv12(in: Vertex) -> @location(0) vec4 { let y = textureSample(plane0, samp, in.uv).r; diff --git a/rs/moq-video/src/render/source.rs b/rs/moq-video/src/render/source.rs index 49d045cd175..7d9dc1473a6 100644 --- a/rs/moq-video/src/render/source.rs +++ b/rs/moq-video/src/render/source.rs @@ -6,6 +6,10 @@ use crate::{Color, Error, Frame, Size, Surface}; /// How the planes of a [`Source`] are arranged, which picks the fragment shader. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Layout { + /// One packed RGBA or BGRA texture. The fragment shader ignores alpha so + /// both XRGB and ARGB DMA-BUFs produce an opaque frame. + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Rgba, /// Luma plane plus one interleaved chroma plane. What every hardware /// decoder hands back, so only a zero-copy import produces it: the CPU /// upload path is always I420. Gated to the platforms that have such an @@ -22,10 +26,14 @@ pub(super) enum Layout { /// that one bind group layout serves both shaders. pub(super) struct Source { pub layout: Layout, - pub color: Color, + /// The color conversion for YUV layouts. Packed RGB needs no conversion. + pub color: Option, pub plane0: wgpu::TextureView, pub plane1: wgpu::TextureView, pub plane2: wgpu::TextureView, + /// Producer-owned storage that must remain leased until the submitted draw + /// stops reading it. The renderer moves this to its completion worker. + pub keepalive: Option>, } /// The renderer's per-frame GPU state: the CPU path's plane textures, plus @@ -56,6 +64,8 @@ impl Cache { /// not, which is what the caller counts strikes against. pub fn import(&mut self, device: &wgpu::Device, surface: &Surface) -> Result, Error> { match surface { + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + Surface::DmaBuf(buffer) => super::dmabuf::import(device, buffer), #[cfg(target_os = "macos")] Surface::PixelBuffer(buffer) => { let metal = match &mut self.metal { @@ -97,10 +107,11 @@ impl Cache { // The conversion that produced these samples says which space they // are in where it knows. Only a passthrough (a decode, a camera) // leaves it open, and then the resolution is all there is to go on. - color: i420.color().unwrap_or_else(|| Color::infer(size)), + color: Some(i420.color().unwrap_or_else(|| Color::infer(size))), plane0, plane1, plane2, + keepalive: None, }) } } From 7b73c43381a7f9c309e3045a8f0f858aa32ef48c Mon Sep 17 00:00:00 2001 From: t0ms Date: Sun, 16 Aug 2026 23:27:00 +0100 Subject: [PATCH 12/29] test(ts): synthesise an EIT so the import path's SI routing is testable (#2828) Co-authored-by: Claude Co-authored-by: Cursor Co-authored-by: Luke Curley Co-authored-by: GPT-5 --- test/ts/README.md | 43 +++++ test/ts/make-eit-fixture.sh | 323 ++++++++++++++++++++++++++++++++++++ test/ts/run.sh | 36 ++++ 3 files changed, 402 insertions(+) create mode 100755 test/ts/make-eit-fixture.sh diff --git a/test/ts/README.md b/test/ts/README.md index d7c3b297d0a..a22896b4830 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -18,6 +18,7 @@ just test ts # generate a clip, round-trip, analyze just test ts --source cap.ts # round-trip a real capture instead just test ts --analyze-only x.ts # skip the round-trip, analyze a file just test ts --strict # also fail on broadcast-shape warnings +just test ts --with-eit # add a synthetic EPG first, report which SI survived ``` `--analyze-only` needs only TSDuck + Python, so you can point it at any captured @@ -72,6 +73,48 @@ Thresholds are CLI flags forwarded through `run.sh` (e.g. `--tb-size-bytes`, `--video-leak-bps`, `--audio-leak-bps`). `--report-json ` writes the full machine-readable report. +## EIT fixture + +`make-eit-fixture.sh` adds a synthetic DVB EPG to any transport stream, so the +import path's EIT handling can be exercised without a broadcast capture that +carries one. No capture in this repository does. + +```bash +./make-eit-fixture.sh in.ts out.ts # EIT p/f + schedule on PID 0x0012 +./make-eit-fixture.sh --pf-only in.ts out.ts # p/f only +``` + +Everything is derived from the input, so the EIT describes the service the +stream actually carries: the triplet (`original_network_id`, +`transport_stream_id`, `service_id`) comes from its PAT and SDT, and the EPG is +anchored to the stream's own TDT where it has one. Without a TDT (the +ffmpeg-generated clip has none), it falls back to a fixed date, so the output is +byte-reproducible for a given input. The SDT's `EIT_present_following_flag` and +`EIT_schedule_flag` are set to match, since a stream carrying an EIT while +advertising none is internally inconsistent. + +`tsp` replaces packets rather than creating them, so the EIT has to come out of +existing stuffing. A broadcast capture has plenty and keeps its exact mux rate; a +clip with none is padded to a constant bitrate first, and the script says so. + +`--with-eit` wires this into the round-trip and prints which SI PIDs came back: + +```text +### SI round-trip (source -> capture) + TABLE PID SOURCE CAPTURE + NIT 0x0010 7 5 + SDT 0x0011 31 21 + EIT 0x0012 1,007 0 + TDT/TOT 0x0014 40 0 +``` + +This is a report, not a gate. `SI_PIDS` in +[`catalog.rs`](../../rs/moq-mux/src/container/ts/catalog.rs) is the allowlist of +PIDs the import path routes, and a table outside it is dropped by design rather +than by malfunction; the census makes that visible instead of leaving it to be +inferred from the code. Counts differ for a table that *did* survive because the +exporter re-emits SI on its own repetition cadence rather than the source's. + ## CI `.github/workflows/smoke.yml` runs `just test ts` after the interop diff --git a/test/ts/make-eit-fixture.sh b/test/ts/make-eit-fixture.sh new file mode 100755 index 00000000000..18e660fcb89 --- /dev/null +++ b/test/ts/make-eit-fixture.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +# Add a synthetic DVB EPG (EIT) to any transport stream. +# +# The MPEG-TS import path routes SI by PID, but no capture in this repository carries +# EIT (PID 0x0012), so nothing exercises that route. This synthesises one from a clip +# that has none, including the ffmpeg-generated clip run.sh makes. This makes the EIT path +# testable without needing a broadcast capture. +# +# Everything is derived from the input: the service triplet comes from its PAT and SDT, so +# the generated EIT actually describes the service the stream carries rather than a +# plausible-looking one a receiver would ignore. The EPG is anchored to a fixed UTC +# reference by default, which makes the output byte-reproducible for a given input. +# +# Requires TSDuck (tsp, tstables) and python3. +# +# Usage: make-eit-fixture.sh [options] +# +# --pf-only EIT p/f actual only. The default also generates EIT schedule. +# --events N events in the EPG (default 12) +# --time T UTC reference, "YYYY/MM/DD:hh:mm:ss". Defaults to the input's own +# TDT if it has one, else a fixed date. +# --service-id N service to describe (default: the first service in the PAT) +# --quiet suppress the post-generation summary + +set -euo pipefail + +PF_ONLY="" +EVENTS=12 +TIME_REF="" +SERVICE_ID="" +QUIET="" +ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --pf-only) + PF_ONLY=1 + shift + ;; + --events) + EVENTS="$2" + shift 2 + ;; + --time) + TIME_REF="$2" + shift 2 + ;; + --service-id) + SERVICE_ID="$2" + shift 2 + ;; + --quiet) + QUIET=1 + shift + ;; + -h | --help) + sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + ARGS+=("$1") + shift + ;; + esac +done + +[[ ${#ARGS[@]} -eq 2 ]] || { + echo "usage: $(basename "$0") [options] " >&2 + exit 2 +} +IN="${ARGS[0]}" +OUT="${ARGS[1]}" +[[ -r "$IN" ]] || { + echo "error: no such input: $IN" >&2 + exit 1 +} + +for t in tsp tstables python3; do + command -v "$t" >/dev/null 2>&1 || { + echo "error: missing required tool: $t" >&2 + exit 1 + } +done + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# Read the service triplet from the stream rather than assuming it. An EIT whose +# (original_network_id, transport_stream_id, service_id) does not match the SDT describes +# nothing, and a receiver is right to ignore it. +tstables "$IN" --pid 0x0000 --pid 0x0011 --pid 0x0014 --max-tables 8 \ + --json-output "$TMP/si.json" --no-pager >/dev/null + +METADATA=$( + python3 - "$TMP/si.json" "$SERVICE_ID" <<'PY' +import json, sys + +path, requested_sid = sys.argv[1:3] + + +def fail(message): + print(f"error: {message}", file=sys.stderr) + sys.exit(1) + + +def integer(value, name): + if isinstance(value, bool): + fail(f"invalid {name} in SI metadata: {value!r}") + try: + return int(value) + except (TypeError, ValueError): + fail(f"invalid {name} in SI metadata: {value!r}") + + +try: + with open(path) as stream: + doc = json.load(stream) +except (OSError, json.JSONDecodeError) as error: + fail(f"cannot read complete PAT/SDT metadata: {error}") + +tables = doc.get("#nodes", []) if isinstance(doc, dict) else doc +if not isinstance(tables, list): + fail("PAT/SDT metadata is not a table list") + +pats = [table for table in tables if table.get("#name") == "PAT"] +sdts = [ + table + for table in tables + if table.get("#name") == "SDT" and table.get("actual", True) +] +if not pats: + fail("input has no complete PAT") +if not sdts: + fail("input has no complete actual SDT") + +pat_tsids = {integer(table.get("transport_stream_id"), "PAT transport_stream_id") for table in pats} +sdt_tsids = {integer(table.get("transport_stream_id"), "SDT transport_stream_id") for table in sdts} +if len(pat_tsids) != 1 or len(sdt_tsids) != 1 or pat_tsids != sdt_tsids: + fail(f"PAT/SDT transport_stream_id mismatch: PAT {sorted(pat_tsids)}, SDT {sorted(sdt_tsids)}") +tsid = pat_tsids.pop() + +onids = {integer(table.get("original_network_id"), "original_network_id") for table in sdts} +if len(onids) != 1: + fail(f"actual SDT tables disagree on original_network_id: {sorted(onids)}") +onid = onids.pop() + +pat_services = [ + integer(node.get("service_id"), "PAT service_id") + for table in pats + for node in table.get("#nodes", []) + if node.get("#name") == "service" +] +sdt_services = { + integer(node.get("service_id"), "SDT service_id") + for table in sdts + for node in table.get("#nodes", []) + if node.get("#name") == "service" +} +if requested_sid: + try: + sid = int(requested_sid, 0) + except ValueError: + fail(f"invalid --service-id: {requested_sid}") + if sid not in sdt_services: + fail(f"service {sid} is not present in the actual SDT") + if sid not in pat_services: + fail(f"service {sid} is not present in the PAT") +else: + try: + sid = next(service for service in pat_services if service in sdt_services) + except StopIteration: + fail("PAT and actual SDT have no service in common") + +tdt = "-" +for t in tables: + if t.get("#name") == "TDT" and tdt == "-": + raw = t.get("utc_time") or t.get("UTC_time") or "" + # "YYYY-MM-DD hh:mm:ss" -> the "YYYY/MM/DD:hh:mm:ss" eitinject --time wants. + if len(raw) == 19 and raw[4] == "-" and raw[10] == " ": + tdt = raw.replace("-", "/").replace(" ", ":") +print(sid, tsid, onid, tdt) +PY +) + +read -r SERVICE_ID TSID ONID TDT <<<"$METADATA" + +# Anchor to the stream's own clock where it has one, so the "present" event is genuinely +# present for a receiver decoding from the start. Otherwise a fixed date, which keeps the +# output reproducible; eitinject needs a reference either way and would otherwise take the +# wall clock, making every run differ. +if [[ -z "$TIME_REF" ]]; then + if [[ "$TDT" != "-" && -n "$TDT" ]]; then + TIME_REF="$TDT" + else + TIME_REF="2026/01/01:12:00:00" + fi +fi + +python3 - "$TMP/epg.xml" "$SERVICE_ID" "$TSID" "$ONID" "$TIME_REF" "$EVENTS" <<'PY' +import sys, datetime + +path, sid, tsid, onid, ref, count = sys.argv[1:7] +sid, tsid, onid, count = int(sid), int(tsid), int(onid), int(count) +t0 = datetime.datetime.strptime(ref, "%Y/%m/%d:%H:%M:%S") + +# The first event has to straddle the reference, not merely precede it: an event ending +# exactly at the reference is already obsolete and eitinject drops it, leaving p/f empty +# for the whole clip. +start = t0 - datetime.timedelta(minutes=15) +titles = [ + ("News", "International news, business and weather."), + ("Sport", "The day in sport."), + ("Documentary", "Long-form reporting."), + ("Talk", "Interviews with newsmakers."), + ("Markets", "Companies and the people who run them."), + ("Weather", "The outlook, region by region."), +] + +rows = [] +for i in range(count): + s = start + datetime.timedelta(minutes=30 * i) + name, text = titles[i % len(titles)] + rows.append( + f' \n' + f' \n' + f' {name} {i + 1}\n' + f' {text}\n' + f" \n" + f" " + ) + +# eitinject ignores this structure and re-derives p/f and schedule from the event list, so +# one EIT carrying every event is the intended input shape. +open(path, "w").write( + '\n\n' + f' \n' + + "\n".join(rows) + + "\n \n\n" +) +PY + +EIT_ARGS=(--actual) +[[ -n "$PF_ONLY" ]] && EIT_ARGS=(--actual-pf) + +# tsp packet processors replace packets, they do not create them, so EIT can only go where +# there is stuffing. A broadcast capture has plenty and keeps its mux rate; a clip muxed by +# ffmpeg has none, so pad it to a constant rate first and say so, since that changes the +# stream in a way the caller should know about. +SRC="$IN" +NULLS=$(tsp -I file "$IN" -P count --pid 0x1FFF --total -O drop 2>&1 | + sed -n 's/.*counted \([0-9,]*\) packets out of \([0-9,]*\).*/\1 \2/p' | head -1) +HAVE=$(echo "$NULLS" | cut -d' ' -f1 | tr -d ,) +TOTAL=$(echo "$NULLS" | cut -d' ' -f2 | tr -d ,) +if [[ -z "$HAVE" || -z "$TOTAL" || "$TOTAL" -eq 0 || $((HAVE * 200)) -lt "$TOTAL" ]]; then + RATE=$(tsp -I file "$IN" -P analyze --normalized -O drop 2>/dev/null | + sed -n 's/^ts:.*:bitrate=\([0-9]*\):.*/\1/p' | head -1) + [[ -n "$RATE" && "$RATE" -gt 0 ]] || { + echo "error: cannot determine input bitrate to pad to" >&2 + exit 1 + } + PADDED=$((RATE + 200000)) + [[ -n "$QUIET" ]] || echo "### input carries no stuffing; padding to ${PADDED} b/s CBR to make room" + tsstuff --bitrate "$PADDED" "$IN" >"$TMP/padded.ts" 2>"$TMP/stuff.log" || { + sed 's/^/ tsstuff: /' "$TMP/stuff.log" >&2 || true + exit 1 + } + SRC="$TMP/padded.ts" +fi + +# --wait-first-batch: without it injection races the EPG load and the head of the output +# carries no EIT, which a short capture would read as "the table was dropped". +# The SDT flags: a stream that carries EIT while advertising none is internally +# inconsistent, and a conformance analyser will say so. +tsp -I file "$SRC" \ + -P sdt --service-id "$SERVICE_ID" --eit-pf 1 --eit-schedule "$([[ -n "$PF_ONLY" ]] && echo 0 || echo 1)" \ + -P eitinject --files "$TMP/epg.xml" --wait-first-batch --time "$TIME_REF" "${EIT_ARGS[@]}" \ + -O file "$OUT" 2>"$TMP/tsp.log" || { + sed 's/^/ tsp: /' "$TMP/tsp.log" >&2 || true + exit 1 +} + +tstables "$OUT" --pid 0x0012 --json-output "$TMP/eit.json" --no-pager >/dev/null +python3 - "$TMP/eit.json" "$SERVICE_ID" "$TSID" "$ONID" <<'PY' +import json, sys + +path, sid, tsid, onid = sys.argv[1:5] +expected = tuple(map(int, (sid, tsid, onid))) +try: + with open(path) as stream: + doc = json.load(stream) +except (OSError, json.JSONDecodeError) as error: + print(f"error: cannot read generated EIT metadata: {error}", file=sys.stderr) + sys.exit(1) + +tables = doc.get("#nodes", []) if isinstance(doc, dict) else doc +observed = { + ( + table.get("service_id"), + table.get("transport_stream_id"), + table.get("original_network_id"), + ) + for table in tables + if table.get("#name") == "EIT" +} +if expected not in observed: + print( + f"error: no generated EIT for service triplet {expected}; observed {sorted(observed)}", + file=sys.stderr, + ) + sys.exit(1) +PY + +EIT_PKTS=$(tsp -I file "$OUT" -P count --pid 0x0012 --total -O drop 2>&1 | + sed -n 's/.*counted \([0-9,]*\) packets.*/\1/p' | head -1) + +[[ -n "$QUIET" ]] || { + echo "### EIT fixture: service $SERVICE_ID (ts $TSID, onid $ONID), reference $TIME_REF" + echo "### $EVENTS events, $([[ -n "$PF_ONLY" ]] && echo "p/f only" || echo "p/f + schedule")" + echo "### $EIT_PKTS packets on PID 0x0012 -> $OUT" +} diff --git a/test/ts/run.sh b/test/ts/run.sh index 04d5ed22e9a..e9d31d8c899 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -14,6 +14,7 @@ # ./run.sh --source cap.ts # round-trip a real capture instead # ./run.sh --analyze-only cap.ts # skip the round-trip, just analyze a file # ./run.sh --strict # fail on broadcast-shape warnings too +# ./run.sh --with-eit # add a synthetic EPG first, report which SI survived set -euo pipefail DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -26,6 +27,7 @@ BITRATE="${TSC_BITRATE:-10000000}" PORT="${TSC_PORT:-4443}" PROFILE="${TSC_PROFILE:-debug}" STRICT="" +WITH_EIT="" # add a synthetic EPG to the source and report which SI survived PASSTHRU=() # forwarded to compliance.py (thresholds, --report-json, ...) while [[ $# -gt 0 ]]; do @@ -54,6 +56,10 @@ while [[ $# -gt 0 ]]; do STRICT="--strict" shift ;; + --with-eit) + WITH_EIT=1 + shift + ;; *) PASSTHRU+=("$1") shift @@ -75,6 +81,11 @@ require_tools() { if [[ -z "$ANALYZE_ONLY" ]]; then for t in cargo ffmpeg curl timeout pgrep; do have "$t" || missing+=("$t"); done fi + # The EIT fixture reads the service triplet out of the stream and may need to pad a + # stuffing-free clip to make room for the table. + if [[ -n "$WITH_EIT" ]]; then + for t in tstables tsstuff; do have "$t" || missing+=("$t"); done + fi if [[ ${#missing[@]} -gt 0 ]]; then echo "error: missing required tools: ${missing[*]}" >&2 echo " TSDuck (tsp, tsanalyze) is required; install from https://tsduck.io" >&2 @@ -166,6 +177,13 @@ else -f mpegts -pes_payload_size 0 "$SRC_TS" fi +# No capture in this repository carries EIT, so the import path's EIT handling is +# otherwise untestable. Synthesise one, and report below which SI PIDs came back. +if [[ -n "$WITH_EIT" ]]; then + "$DIR/make-eit-fixture.sh" "$SRC_TS" "$TMP/source-eit.ts" + mv "$TMP/source-eit.ts" "$SRC_TS" +fi + echo "### starting relay on 127.0.0.1:${PORT}" sed "s/4443/${PORT}/g" "$DIR/../smoke/smoke.toml" >"$TMP/relay.toml" "$RELAY" "$TMP/relay.toml" >"$TMP/relay.log" 2>&1 & @@ -225,6 +243,24 @@ if [[ ! -s "$SUB_TS" ]]; then fi echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') bytes -> analyzing" + +# Which SI survived the round-trip. Informational: the exporter rebuilds SI from the +# catalog, so a PID the import path does not route simply is not there, and that is a +# statement about SI_PIDS rather than a malformed stream. compliance.py grades the stream +# an IRD receives; this says what it was carrying on the way in. +if [[ -n "$WITH_EIT" ]]; then + echo + echo "### SI round-trip (source -> capture)" + count_pid() { + tsp -I file "$1" -P count --pid "$2" --total -O drop 2>&1 | + sed -n 's/.*counted \([0-9,]*\) packets.*/\1/p' | head -1 + } + printf ' %-10s %-8s %12s %12s\n' TABLE PID SOURCE CAPTURE + for spec in "NIT:0x0010" "SDT:0x0011" "EIT:0x0012" "TDT/TOT:0x0014"; do + printf ' %-10s %-8s %12s %12s\n' "${spec%%:*}" "${spec##*:}" \ + "$(count_pid "$SRC_TS" "${spec##*:}")" "$(count_pid "$SUB_TS" "${spec##*:}")" + done +fi echo # Pass the source so duration-fidelity can pin the exported stream's rate. A tiny # capture still parses, so the round-trip can fail here with a non-empty file; From eab9601924c65f0351d59ff2fc290ca908aa9e00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:47:03 -0700 Subject: [PATCH 13/29] build(deps): bump the cargo group with 7 updates (#2888) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Luke Curley Co-authored-by: OpenAI Codex --- Cargo.lock | 62 +++++++++++++++++++++------------- Cargo.toml | 14 ++++---- rs/moq-native/src/noq.rs | 44 ++++++------------------ rs/moq-native/src/quinn.rs | 44 ++++++------------------ rs/moq-native/src/websocket.rs | 12 +++---- rs/moq-relay/src/websocket.rs | 12 +++---- rs/moq-rtc/Cargo.toml | 2 +- 7 files changed, 78 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 552fbc5bfa2..6a64f93bd1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4000,9 +4000,9 @@ dependencies = [ [[package]] name = "is" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cff4e962dd367e3b1c478f67c24a35e7120ec767090cb61305b855db9a529c4" +checksum = "d08f9118d003d441f79c1070e84d0a2a89f35721ca8ae0cd5e1f9026dc4a2517" dependencies = [ "crc", "serde", @@ -4277,6 +4277,16 @@ dependencies = [ "web-async", ] +[[package]] +name = "kio" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdaadab1a06c04819b8f7cabe4ead8c9f1e86016c4c2a9384829bbc555f477a7" +dependencies = [ + "loom", + "smallvec", +] + [[package]] name = "kqueue" version = "1.2.1" @@ -4771,7 +4781,7 @@ dependencies = [ "bytes", "hang", "humantime", - "kio", + "kio 0.5.4", "m3u8-rs", "moq-mux", "moq-net", @@ -4792,7 +4802,7 @@ dependencies = [ "bytes", "criterion", "json-patch", - "kio", + "kio 0.5.4", "moq-flate", "moq-net", "serde", @@ -4830,7 +4840,7 @@ dependencies = [ "futures", "h264-parser", "hang", - "kio", + "kio 0.5.4", "memchr", "moq-json", "moq-loc", @@ -4912,7 +4922,7 @@ dependencies = [ "criterion", "futures", "getrandom 0.4.3", - "kio", + "kio 0.5.4", "loom", "num_enum", "rand 0.10.2", @@ -7190,9 +7200,9 @@ dependencies = [ [[package]] name = "qmux" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a824572a7e027bd1d78ddd0df76447a810b881868dfee2cabdfe0e6301e24e5" +checksum = "98b28b59b131430bbfcc6687e98f33e0edd7d908e325c76583d62a9c4c876e53" dependencies = [ "bytes", "futures", @@ -8797,9 +8807,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "str0m" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed3d9290b349b6da18fd640d81a950e4b256dba80e36982be61ae6ef98fb025" +checksum = "4656b60e74d1a0cf7c407b0d992c5bc141c3e75193f4b1399817d54a180505f3" dependencies = [ "arrayvec", "base64ct", @@ -8818,9 +8828,9 @@ dependencies = [ [[package]] name = "str0m-aws-lc-rs" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e3e6efb440ad95c9370171da8fb033684f8237a4d4d4987887774149547b0d" +checksum = "73269314334fc6bfa8024f09dd652b9f5068360bf623c871e3d8d18e26d8035a" dependencies = [ "aws-lc-rs", "dimpl", @@ -8830,9 +8840,9 @@ dependencies = [ [[package]] name = "str0m-proto" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c99c3d805532aebd04585ba54e97e958c382759fb8f52256e289a49233f7e05b" +checksum = "e8f3d99f2cf6c76a502e45feb896ea71e54787ede8744bcdb215acfbfcb905dd" dependencies = [ "base64ct", "dimpl", @@ -10379,13 +10389,14 @@ dependencies = [ [[package]] name = "web-transport-iroh" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34a91cf241951b8fe02ff216fdf16fc0a52b910568053652d546ded52399f77" +checksum = "14c292c05ff2f4df8e582244a6a5f2ba89f4c13b1170ec5f59a727f9a4624baf" dependencies = [ "bytes", "http", "iroh", + "kio 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "n0-error", "n0-future", "tokio", @@ -10397,13 +10408,14 @@ dependencies = [ [[package]] name = "web-transport-noq" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0344a32000bbf11bd30e3c1244d223baac316c2a25e06eb638b5ca9322d589" +checksum = "56dfe08820de549204c3ab02e901c4a316616ba7daae26df4478a86f4a2d48cd" dependencies = [ "bytes", "futures", "http", + "kio 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "noq", "rustls", "rustls-native-certs", @@ -10431,15 +10443,16 @@ dependencies = [ [[package]] name = "web-transport-quiche" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3599414f964f4f1132dc8bd8f6c497b7a7b97043e4d5de53abbc155527d48fdf" +checksum = "153ece42b1d6e86015fcac192b6d4aae0ac8595045e844ac075169b22f1d37c6" dependencies = [ "boring", "bytes", "flume", "futures", "http", + "kio 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustls-pki-types", "thiserror 2.0.20", "tokio", @@ -10452,13 +10465,14 @@ dependencies = [ [[package]] name = "web-transport-quinn" -version = "0.11.12" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ada8c56f3dc126fa3c9e6b8f31f3bff5dfd24458f6664743d936ecad741d15" +checksum = "0b494e7a90c7c576bac11d7506128cea266d63ed9a8b0be39269c7b9ea402a5e" dependencies = [ "bytes", "futures", "http", + "kio 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "quinn", "rustls", "rustls-native-certs", @@ -10472,9 +10486,9 @@ dependencies = [ [[package]] name = "web-transport-trait" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6394e09415ddc1e7c88d605443a3d2f333bf278f74c17020a2b6c44e97507883" +checksum = "2a916df726e858e8e6cc21e852129f59e6277d3a01bb2254c0506dcf829fdd5d" dependencies = [ "bytes", ] diff --git a/Cargo.toml b/Cargo.toml index 8715f5c9d9b..a80c25b3fd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,19 +116,19 @@ moq-vaapi = "0.0.3" # whole device graph into every one of them. moq-video = { version = "0.0.17", path = "rs/moq-video", default-features = false } percent-encoding = "2" -qmux = { version = "0.4.0", default-features = false } +qmux = { version = "0.5.0", default-features = false } serde = { version = "1", features = ["derive"] } tokio = "1.48" web-async = { version = "0.1.5", features = ["tracing"] } -web-transport-iroh = "0.6" +web-transport-iroh = "0.7" # default-features off so the QUIC crypto provider is chosen by moq-native's # aws-lc-rs / ring features (mirroring how the quinn dependency is wired). -# 0.2.1 for the `qlog` passthrough feature moq-native's `qlog` forwards to. -web-transport-noq = { version = "0.2.1", default-features = false } +# Keep the `qlog` passthrough feature available for moq-native to forward. +web-transport-noq = { version = "0.3.0", default-features = false } web-transport-proto = "0.6" -web-transport-quiche = "0.5" -web-transport-quinn = "0.11" -web-transport-trait = "0.3.4" +web-transport-quiche = "0.6" +web-transport-quinn = "0.12" +web-transport-trait = "0.4.0" [profile.dev] panic = "abort" diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index 874f15fab6c..621a0bca1c6 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -366,28 +366,17 @@ impl NoqClient { .await?; tracing::Span::current().record("id", connection.stable_id()); - let mut request = web_transport_noq::proto::ConnectRequest::new(url.clone()); - for alpn in versions.alpns() { - request = request.with_protocol(alpn.to_string()); - } - let session = match url.scheme() { - "https" => web_transport_noq::Session::connect(connection, request) - .await - .map_err(map_client_error)?, - "moqt" | "moql" => { - let handshake = connection - .handshake_data() - .ok_or(Error::MissingHandshake)? - .downcast::() - .unwrap(); - - let alpn = handshake.protocol.ok_or(Error::MissingAlpn)?; - let alpn = String::from_utf8(alpn)?; - - let response = web_transport_noq::proto::ConnectResponse::OK.with_protocol(alpn); - web_transport_noq::Session::raw(connection, request, response) + "https" => { + let mut request = web_transport_noq::proto::ConnectRequest::new(url.clone()); + for alpn in versions.alpns() { + request = request.with_protocol(alpn.to_string()); + } + web_transport_noq::Session::connect(connection, request) + .await + .map_err(map_client_error)? } + "moqt" | "moql" => web_transport_noq::Session::raw(connection), _ => return Err(Error::UnsupportedScheme(url.scheme().to_string())), }; @@ -654,22 +643,9 @@ pub(crate) async fn accept( // this covers opt-in / work-in-progress versions (e.g. moq-lite-06-wip) that are // deliberately absent from `moq_net::ALPNS`. alpn if alpns.contains(&alpn) => { - // Raw QUIC carries no in-band request URL like WebTransport's CONNECT, so the TLS - // SNI is the only authority the client can offer, and it's optional. A client dialing - // a bare IP sends no SNI (RFC 6066 forbids IP literals), leaving `host` empty; the - // resulting hostless `moqt://` routes to the root path, exactly like a URL-less stream - // transport. `url()` returns `None` for the raw variant either way. - let host_str = if host.contains(':') { - format!("[{}]", host) - } else { - host.clone() - }; - let url = format!("moqt://{}", host_str).parse::().map_err(Error::BuildUrl)?; - let request = web_transport_noq::proto::ConnectRequest::new(url); - let response = web_transport_noq::proto::ConnectResponse::OK.with_protocol(alpn); let identity = crate::tls::PeerIdentity::from_any(conn.peer_identity()); // Raw QUIC carries no request URL; the path rides the SETUP. - let session = web_transport_noq::Session::raw(conn, request, response); + let session = web_transport_noq::Session::raw(conn); Ok((session, None, identity)) } _ => Err(Error::UnsupportedAlpn(alpn)), diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 163ed048ad9..dd4d818ef8c 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -379,28 +379,17 @@ impl QuinnClient { .await?; tracing::Span::current().record("id", connection.stable_id()); - let mut request = web_transport_quinn::proto::ConnectRequest::new(url.clone()); - for alpn in versions.alpns() { - request = request.with_protocol(alpn.to_string()); - } - let session = match url.scheme() { - "https" => web_transport_quinn::Session::connect(connection, request) - .await - .map_err(map_client_error)?, - "moqt" | "moql" => { - let handshake = connection - .handshake_data() - .ok_or(Error::MissingHandshake)? - .downcast::() - .unwrap(); - - let alpn = handshake.protocol.ok_or(Error::MissingAlpn)?; - let alpn = String::from_utf8(alpn)?; - - let response = web_transport_quinn::proto::ConnectResponse::OK.with_protocol(alpn); - web_transport_quinn::Session::raw(connection, request, response) + "https" => { + let mut request = web_transport_quinn::proto::ConnectRequest::new(url.clone()); + for alpn in versions.alpns() { + request = request.with_protocol(alpn.to_string()); + } + web_transport_quinn::Session::connect(connection, request) + .await + .map_err(map_client_error)? } + "moqt" | "moql" => web_transport_quinn::Session::raw(connection), _ => return Err(Error::UnsupportedScheme(url.scheme().to_string())), }; @@ -666,22 +655,9 @@ pub(crate) async fn accept( // this covers opt-in / work-in-progress versions (e.g. moq-lite-06-wip) that are // deliberately absent from `moq_net::ALPNS`. alpn if alpns.contains(&alpn) => { - // Raw QUIC carries no in-band request URL like WebTransport's CONNECT, so the TLS - // SNI is the only authority the client can offer, and it's optional. A client dialing - // a bare IP sends no SNI (RFC 6066 forbids IP literals), leaving `host` empty; the - // resulting hostless `moqt://` routes to the root path, exactly like a URL-less stream - // transport. `url()` returns `None` for the raw variant either way. - let host_str = if host.contains(':') { - format!("[{}]", host) - } else { - host.clone() - }; - let url = format!("moqt://{}", host_str).parse::().map_err(Error::BuildUrl)?; - let request = web_transport_quinn::proto::ConnectRequest::new(url); - let response = web_transport_quinn::proto::ConnectResponse::OK.with_protocol(alpn); let identity = crate::tls::PeerIdentity::from_any(conn.peer_identity()); // Raw QUIC carries no request URL; the path rides the SETUP. - let session = web_transport_quinn::Session::raw(conn, request, response); + let session = web_transport_quinn::Session::raw(conn); Ok((session, None, identity)) } _ => Err(Error::UnsupportedAlpn(alpn)), diff --git a/rs/moq-native/src/websocket.rs b/rs/moq-native/src/websocket.rs index a091b3e7484..aecffc9e25d 100644 --- a/rs/moq-native/src/websocket.rs +++ b/rs/moq-native/src/websocket.rs @@ -5,8 +5,8 @@ //! a small head start ([`Client::delay`]), so WebSocket only wins when QUIC can't get //! through. Servers accept it on a separate TCP port via [`Listener`]. -use qmux::tokio_tungstenite; -use qmux::tokio_tungstenite::tungstenite::{self, http}; +use qmux::ws::tokio_tungstenite; +use qmux::ws::tokio_tungstenite::tungstenite::{self, http}; use std::collections::HashSet; use std::sync::{Arc, LazyLock, Mutex}; use std::{net, time}; @@ -188,10 +188,10 @@ pub(crate) async fn connect( // restricts. qmux also offers the bare ALPNs (`qmux-01`, `qmux-00`, // `webtransport`) by default so we still interop with relays that only know a // wire-format version. - let session = qmux::Client::new() + let session = qmux::ws::Client::new() .with_protocols(alpns.iter().map(|&a| (a, qmux_versions_for(a)))) .with_connector(connector) - .with_keep_alive(qmux::KeepAlive::default()) // 5s ping / 30s deadline, parity with QUIC + .with_keep_alive(qmux::ws::KeepAlive::default()) // 5s ping / 30s deadline, parity with QUIC .connect(url.as_str()) .await .map_err(Error::Connect)?; @@ -202,7 +202,7 @@ pub(crate) async fn connect( Ok(session) } -/// The QMux drafts a moq ALPN is allowed to ride on, for `qmux::*::with_protocols`. +/// The QMux drafts a moq ALPN is allowed to ride on, for `qmux::ws::Client::with_protocols`. /// /// moq-transport-18 and -19 require qmux-01, so we never pair them with qmux-00. /// This mirrors the policy in `js/net`'s `connect.ts`. Every other ALPN returns @@ -349,7 +349,7 @@ impl Listener { .unwrap() .take() .expect("successful upgrade selected a protocol"); - let upgraded = qmux::ws::Upgraded::new(websocket).with_keep_alive(qmux::KeepAlive::default()); + let upgraded = qmux::ws::Upgraded::new(websocket).with_keep_alive(qmux::ws::KeepAlive::default()); let session = match protocol { Some(protocol) => upgraded.with_alpn(&protocol).accept(), None => upgraded.accept(), diff --git a/rs/moq-relay/src/websocket.rs b/rs/moq-relay/src/websocket.rs index 877da347378..bf671763e2c 100644 --- a/rs/moq-relay/src/websocket.rs +++ b/rs/moq-relay/src/websocket.rs @@ -1,5 +1,5 @@ use futures::{Sink, Stream}; -use qmux::tungstenite; +use qmux::ws::tungstenite; use std::{ pin::Pin, sync::{Arc, atomic::Ordering}, @@ -121,7 +121,7 @@ where // broadcast it published stays announced for that entire window and the // announce propagates to the rest of the cluster. QUIC gets this from its // idle timeout; WebSocket has no equivalent of its own. - let upgraded = qmux::ws::Upgraded::new(socket).with_keep_alive(qmux::KeepAlive::default()); + let upgraded = qmux::ws::Upgraded::new(socket).with_keep_alive(qmux::ws::KeepAlive::default()); let upgraded = match alpn.as_deref() { Some(alpn) => upgraded.with_alpn(alpn), None => upgraded, @@ -382,7 +382,7 @@ mod tests { } /// The newest moq ALPN both sides agree on. Derived from the same source - /// of truth that `supported_subprotocols` and `qmux::Client::with_protocols` + /// of truth that `supported_subprotocols` and `qmux::ws::Client::with_protocols` /// consume, so adding a new ALPN doesn't break these tests independently /// of the production logic. fn newest_moq_alpn() -> &'static str { @@ -644,7 +644,7 @@ mod tests { async fn axum_ws_negotiates_newest_moq_alpn() { let (addr, mut rx) = spawn_test_server().await; - let session = qmux::Client::new() + let session = qmux::ws::Client::new() .with_protocols(moq_net::ALPNS.iter().map(|&a| (a, &[] as &[qmux::Version]))) .connect(&format!("ws://{addr}/")) .await @@ -683,7 +683,7 @@ mod tests { continue; }; - let session = qmux::Client::new() + let session = qmux::ws::Client::new() .with_protocol(app, &[version]) .connect(&url) .await @@ -705,7 +705,7 @@ mod tests { // rather than failing: we always accept a bare web-transport connection, and // there's no other qmux version worth negotiating for moqt-18. It just never // lands on `moqt-18`. - let session = qmux::Client::new() + let session = qmux::ws::Client::new() .with_protocol("moqt-18", &[qmux::Version::QMux00]) .connect(&url) .await diff --git a/rs/moq-rtc/Cargo.toml b/rs/moq-rtc/Cargo.toml index f370aab7e1d..948dc9af4db 100644 --- a/rs/moq-rtc/Cargo.toml +++ b/rs/moq-rtc/Cargo.toml @@ -23,7 +23,7 @@ hang = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } reqwest = { version = "0.13", default-features = false, features = ["rustls"] } -str0m = "0.21" +str0m = "0.22" thiserror = "2" tokio = { workspace = true, features = ["full"] } tracing = "0.1" From 743c9d8b646fbd25bc659f97f2687cea85f3c1ba Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 18:58:29 -0700 Subject: [PATCH 14/29] fix(moq-mux): compile on wasm32, and gate it so it stays that way Two things kept moq-mux off wasm32, and the first is a latent bug on every target. The av1/h264/h265 splitters hold a `tokio::time::Instant`, but moq-mux declares tokio with only the `macros` feature. That compiles today purely because another workspace member enables tokio's `time`, and feature unification hands it over. Build moq-mux in a resolver context that doesn't, and it fails to find `tokio::time` at all. Swap it for `web_async::time::Instant`, which natively re-exports the very same tokio type (so nothing changes on native) and maps to wasmtimer on wasm. This is the migration moq-net already did. The catalog `Stream` trait then required `Send + 'static`. moq-net's handles are `Rc`-backed on wasm, so nothing there can satisfy it, and the bound exists to allow moving exporters between native threads rather than to express anything about the type. Add `kio::MaybeSend`, which is `Send` natively and vanishes on wasm, and bound `Stream` on that instead. Natively `MaybeSend` is implied by `Send` and implies it back, so every existing caller and impl is unaffected. kio is the right home rather than a private copy: moq-net already keeps its own `MaybeSend`-shaped helpers in `util.rs`, moq-mux is the second crate to need the bound, and moq-ffi is the third once it grows a wasm target. Both crates already depend on kio, and it carries no runtime. Extend `just rs wasm` to cover moq-mux so this cannot regress silently, since a host-target build will never notice. moq-mux has to share moq-wasm's cargo invocation to inherit `getrandom`'s `wasm_js` backend, and the recipe explains why splitting them fails. Verified: `just check` clean (workspace clippy plus the extended wasm gate), and 582 tests pass across kio and moq-mux. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 1 + rs/justfile | 12 +++++++++--- rs/kio/src/lib.rs | 2 ++ rs/kio/src/send.rs | 23 +++++++++++++++++++++++ rs/moq-mux/Cargo.toml | 1 + rs/moq-mux/src/catalog/producer.rs | 4 ++++ rs/moq-mux/src/catalog/stream.rs | 10 ++++++---- rs/moq-mux/src/codec/av1/split.rs | 4 ++-- rs/moq-mux/src/codec/h264/split.rs | 4 ++-- rs/moq-mux/src/codec/h265/split.rs | 4 ++-- 10 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 rs/kio/src/send.rs diff --git a/Cargo.lock b/Cargo.lock index 6a64f93bd1d..425cf60eacb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4858,6 +4858,7 @@ dependencies = [ "tokio", "tracing", "url", + "web-async", "webm-iterable", ] diff --git a/rs/justfile b/rs/justfile index 7920730210e..3e000aaf26c 100644 --- a/rs/justfile +++ b/rs/justfile @@ -209,17 +209,23 @@ macos *args: # which builds the shippable `@moq/wasm` package (wasm-bindgen, release profile); # this is the compile gate. +# `moq-mux` rides along in the same invocation on purpose: it needs `getrandom`'s +# `wasm_js` backend, which only `moq-wasm` declares, and feature unification only +# happens within one cargo invocation. Splitting these into two fails on getrandom. +# `--lib` rather than `--all-targets` because moq-mux's tests use tokio's native +# timers; moq-wasm has no test targets, so it loses no coverage. + # Compile and lint the browser/WASM bindings, which the host-target check skips. wasm *args: - cargo clippy -p moq-wasm --target wasm32-unknown-unknown --all-targets {{ args }} -- -D warnings + cargo clippy -p moq-wasm -p moq-mux --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings # The `fix` counterpart to `wasm`. `cargo fmt` is scoped rather than `--all` # because this runs alongside `fix`, which already formats everything else. # Auto-fix the browser/WASM bindings. wasm-fix *args: - cargo clippy --fix --allow-staged --allow-dirty -p moq-wasm --target wasm32-unknown-unknown --all-targets {{ args }} - cargo fmt -p moq-wasm + cargo clippy --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux --target wasm32-unknown-unknown --lib {{ args }} + cargo fmt -p moq-wasm -p moq-mux # Dependency advisories plus the license/ban policy. Time-based rather than # diff-based: an advisory lands without this repo changing, so a per-PR run both diff --git a/rs/kio/src/lib.rs b/rs/kio/src/lib.rs index 2e9be612d49..5743fe444f4 100644 --- a/rs/kio/src/lib.rs +++ b/rs/kio/src/lib.rs @@ -32,6 +32,7 @@ mod consumer; mod pollable; mod producer; mod queue; +mod send; mod shared; mod weak; @@ -52,6 +53,7 @@ pub use lock::{Lock, LockGuard, WeakLock}; pub use pollable::{Pending, Pollable}; pub use producer::{Mut, Producer, Ref}; pub use queue::{PushError, Queue}; +pub use send::MaybeSend; pub use shared::Shared; pub use waiter::{Fan, Hold, Park, Waiter, WaiterList, wait}; pub use weak::{ConsumerWeak, ProducerWeak, Weak}; diff --git a/rs/kio/src/send.rs b/rs/kio/src/send.rs new file mode 100644 index 00000000000..136f9e34cbf --- /dev/null +++ b/rs/kio/src/send.rs @@ -0,0 +1,23 @@ +//! A `Send` bound that disappears on wasm. + +/// `Send` on native targets, no bound at all on wasm. +/// +/// Browser wasm is single-threaded, and the futures and JS values that flow through it +/// are `!Send`. A `Send` bound in a public API therefore excludes types that are +/// perfectly safe on the one thread that exists. Use this wherever the bound is there +/// to allow moving work between native threads, rather than to express a real +/// requirement of the type itself. +/// +/// Natively this is implied by `Send`, so an existing `T: Send` caller keeps compiling +/// and a `T: MaybeSend` bound still yields `Send`. +#[cfg(not(target_family = "wasm"))] +pub trait MaybeSend: Send {} + +#[cfg(not(target_family = "wasm"))] +impl MaybeSend for T {} + +#[cfg(target_family = "wasm")] +pub trait MaybeSend {} + +#[cfg(target_family = "wasm")] +impl MaybeSend for T {} diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 1cfa104cd3e..8d7d12ef1f3 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -36,6 +36,7 @@ thiserror = "2" tokio = { workspace = true, features = ["macros"] } tracing = "0.1" url = "2" +web-async = { workspace = true } webm-iterable = "0.6" [dev-dependencies] diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index a237431b2fe..4b8cd4c4af3 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -176,6 +176,10 @@ impl Producer { json_config.compression = true; let hangz = moq_json::snapshot::Producer::new(hangz_track, json_config); + // The contents are `Send + Sync` natively; on wasm moq-net's handles are + // `Rc`-backed, so clippy sees a pointlessly atomic `Arc`. Keeping one type for + // both targets is worth the unused atomics on the single-threaded one. + #[allow(clippy::arc_with_non_send_sync)] Ok(Self { hang, hangz, diff --git a/rs/moq-mux/src/catalog/stream.rs b/rs/moq-mux/src/catalog/stream.rs index 8c4a351c0b5..444a2936980 100644 --- a/rs/moq-mux/src/catalog/stream.rs +++ b/rs/moq-mux/src/catalog/stream.rs @@ -20,16 +20,18 @@ use super::hang::{Catalog, CatalogExt}; /// `None` once the underlying track has ended. Late snapshots supersede /// earlier ones, so an implementation may drop intermediate snapshots. /// -/// Stream types are required to be `Send + 'static` so they can be moved -/// across threads and held inside exporters without per-call bounds. -pub trait Stream: Send + 'static { +/// Stream types are required to be `MaybeSend + 'static` so they can be moved +/// across threads and held inside exporters without per-call bounds. The bound is +/// `Send` natively and vanishes on wasm, where there is only one thread and +/// moq-net's handles are `!Send`. +pub trait Stream: kio::MaybeSend + 'static { /// The application extension carried by the yielded catalog (`()` for media-only). type Ext: CatalogExt; fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>>; /// Wait for the next snapshot. - fn next(&mut self) -> impl std::future::Future>>> + Send + fn next(&mut self) -> impl std::future::Future>>> + kio::MaybeSend where Self: Sized, { diff --git a/rs/moq-mux/src/codec/av1/split.rs b/rs/moq-mux/src/codec/av1/split.rs index b7ffc58feaf..7f1bf647c45 100644 --- a/rs/moq-mux/src/codec/av1/split.rs +++ b/rs/moq-mux/src/codec/av1/split.rs @@ -20,7 +20,7 @@ use crate::Result; /// stdin); call [`flush`](Self::flush) to emit the final in-flight temporal unit. pub struct Split { current: Au, - zero: Option, + zero: Option, pending: Vec, // Bytes carried across calls: a partial OBU at the tail of one `decode` waits // here for the rest to arrive on the next call. @@ -169,7 +169,7 @@ impl Split { if let Some(pts) = hint { return Ok(pts); } - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + let zero = self.zero.get_or_insert_with(web_async::time::Instant::now); Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) } } diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 29bf121f821..cbbcac9a8e3 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -38,7 +38,7 @@ pub struct Split { /// Retained PPS NALs. A keyframe may carry several (slices reference them by /// id); all are kept and re-injected, but a new GOP's set supersedes them. pps: Vec, - zero: Option, + zero: Option, pending: Vec, } @@ -238,7 +238,7 @@ impl Split { if let Some(pts) = hint { return Ok(pts); } - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + let zero = self.zero.get_or_insert_with(web_async::time::Instant::now); Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) } } diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs index fccb0643857..a0305118755 100644 --- a/rs/moq-mux/src/codec/h265/split.rs +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -36,7 +36,7 @@ pub struct Split { /// Retained PPS NALs. A keyframe may carry several (slices reference them by /// id); all are kept and re-injected, but a new GOP's set supersedes them. pps: Vec, - zero: Option, + zero: Option, pending: Vec, } @@ -235,7 +235,7 @@ impl Split { if let Some(pts) = hint { return Ok(pts); } - let zero = self.zero.get_or_insert_with(tokio::time::Instant::now); + let zero = self.zero.get_or_insert_with(web_async::time::Instant::now); Ok(moq_net::Timestamp::from_micros(zero.elapsed().as_micros() as u64)?) } } From 0a2fb13d8de82767a8142b1f44c01a301ee47b3b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 19:13:50 -0700 Subject: [PATCH 15/29] test(wasm): run the browser bindings in headless Chromium (#2904) Co-authored-by: Claude Opus 5 --- .github/workflows/wasm.yml | 106 ++++++++++ CLAUDE.md | 2 +- bun.lock | 12 ++ js/justfile | 2 +- js/wasm/README.md | 5 +- package.json | 3 +- rs/CLAUDE.md | 2 +- rs/moq-wasm/README.md | 6 + test/justfile | 8 + test/wasm/README.md | 92 +++++++++ test/wasm/driver.ts | 131 +++++++++++++ test/wasm/index.html | 5 + test/wasm/package.json | 12 ++ test/wasm/relay.toml | 24 +++ test/wasm/run.sh | 179 +++++++++++++++++ test/wasm/src/main.ts | 384 +++++++++++++++++++++++++++++++++++++ test/wasm/tsconfig.json | 8 + 17 files changed, 975 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/wasm.yml create mode 100644 test/wasm/README.md create mode 100644 test/wasm/driver.ts create mode 100644 test/wasm/index.html create mode 100644 test/wasm/package.json create mode 100644 test/wasm/relay.toml create mode 100755 test/wasm/run.sh create mode 100644 test/wasm/src/main.ts create mode 100644 test/wasm/tsconfig.json diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 00000000000..6be198e0ea8 --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,106 @@ +name: WASM + +# Runs the @moq/wasm bindings in a real browser, which nothing in `just check` +# can do. rs/moq-wasm is `#![cfg(target_arch = "wasm32")]`, so a host-target +# `cargo check --workspace` compiles it to nothing, and `just rs wasm` is a +# compile gate: it catches a moq-net change that stops the bindings building, +# never one that stops them working. See test/wasm/README.md. +# +# Narrow trigger, like obs.yml and swift.yml. The filter covers what the bindings +# are built out of: moq-net, because that is the break this exists to catch in +# review (they are a thin wrapper over it, and every other gate compiles them to +# nothing), and kio underneath it, because a lost wakeup or a timer regression +# reaches the browser through moq-net without touching a file named here +# otherwise. Those two are the whole in-repo dependency graph of the crate. +# +# The rest of the filter is the machinery that turns the crate into the package +# under test, which is just as able to break it: the root `justfile` owns the +# `wasm` recipe, and `.cargo/config.toml` carries the wasm32 rustflags +# (`getrandom_backend`, `web_sys_unstable_apis`) without which it does not +# compile at all. +# +# Two things the harness needs are deliberately out, because they are fixtures +# rather than the code under test, and both are already gated elsewhere: +# `js/net/**` (the publisher; its own unit tests plus the smoke matrix) and +# `rs/moq-relay/**` (the server; the Rust test suite plus smoke). Naming either +# would launch a browser on a large share of pull requests. The cost is that a +# break arriving through one of them lands on `main` and shows up the next time +# this runs. + +permissions: + contents: read + +on: + pull_request: + # `closed` is here only so merging/closing a PR cancels its in-flight run + # via the concurrency group below; the job itself is skipped on close. + types: [opened, synchronize, reopened, closed] + paths: + - "rs/moq-wasm/**" + - "rs/moq-net/**" + - "rs/kio/**" + - "js/wasm/**" + - "test/wasm/**" + - "justfile" + - "test/justfile" + - ".cargo/config.toml" + - "package.json" + - "bun.lock" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "flake.nix" + - "flake.lock" + - ".github/workflows/wasm.yml" + +concurrency: + group: wasm-${{ github.ref }} + cancel-in-progress: true + +jobs: + wasm: + name: WASM + if: github.event.action != 'closed' + # x64, not the arm runner the other jobs use: this is the configuration + # smoke.yml already launches Playwright's Chromium on. + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main + with: + tool-cache: false + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, so the dev shell substitutes + # instead of building from source. See check.yml for the full story. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + # Restore only. cache.yml is the single writer, on `main`; see the comment + # there for why. `shared-key` has to match the one it saves under. + - name: Rust Cache + uses: Swatinem/rust-cache@258712b0b7b1ddf8bddc9fc3b0faca682b2736c3 # v2 + with: + shared-key: rust + save-if: false + + # Playwright's Chromium isn't in the nix devShell. Install it plus its apt + # runtime libs (`--with-deps` needs the host package manager); run.sh's own + # `playwright install` then no-ops. + - name: Playwright Chromium + run: nix develop --command bash -c "bun install && bunx playwright install --with-deps chromium" + shell: bash -leo pipefail {0} + + - name: WASM + run: nix develop --command just test wasm + shell: bash -leo pipefail {0} diff --git a/CLAUDE.md b/CLAUDE.md index b8aed77f462..e0ddc88081d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,7 +64,7 @@ Top-level layout only. Per-crate and per-package detail lives in the nested guid - `/py/`, `/swift/`, `/kt/`, `/go/` - language wrappers over `rs/moq-ffi` (see [Language Bindings](#language-bindings)). `/py/` has `py/CLAUDE.md`; the others defer to their `README.md`. - `/cpp/` - C/C++ consumers of `libmoq`. `cpp/obs/` is the OBS Studio plugin (CMake; links `libmoq` via `MOQ_LOCAL`), licensed GPL-2.0-or-later because it links `libobs`. `just check` type-checks it via `just obs compile`, which needs headers rather than an obs-deps download, and obs.yml links it on Linux for `cpp/obs/` and `rs/libmoq/` PRs. Still manual, like `just rs macos`: `just obs build` (a loadable plugin, and the only path on macOS/Windows) and `just obs test` (`cpp/obs/test/` against stubbed libobs/libmoq under ThreadSanitizer). See `doc/bin/obs.md`. - `/demo/` - demos and test media: relay configs, the web demo, MoQ Boy, media hosting, and a network throttle script. -- `/test/` - cross-language interop smoke tests (`test/smoke/`), run via `just test smoke[-full]`. +- `/test/` - test harnesses that span more than one language or need a server. `test/smoke/` is the cross-language interop matrix (`just test smoke[-full]`); `test/wasm/` runs the `@moq/wasm` bindings in headless Chromium against a real relay (`just test wasm`), which is the only behavioral coverage `rs/moq-wasm` has. - `/doc/` - documentation site (VitePress, deployed via Cloudflare). The `/draft/` section is generated from `drafts/` by `doc/.vitepress/drafts.ts`. - `/drafts/` - IETF Internet-Drafts (kramdown-rfc) for the MoQ protocols implemented here. Built and published to the datatracker via `just drafts`. See `drafts/CLAUDE.md`. diff --git a/bun.lock b/bun.lock index d214ae8f592..93a986c5e8c 100644 --- a/bun.lock +++ b/bun.lock @@ -307,6 +307,16 @@ "tsx": "^4.23.11", }, }, + "test/wasm": { + "name": "@moq/wasm-test", + "dependencies": { + "@moq/net": "workspace:*", + "@moq/wasm": "workspace:*", + }, + "devDependencies": { + "playwright": "^1.62.1", + }, + }, }, "trustedDependencies": [ "@fails-components/webtransport-transport-http3-quiche", @@ -598,6 +608,8 @@ "@moq/wasm": ["@moq/wasm@workspace:js/wasm"], + "@moq/wasm-test": ["@moq/wasm-test@workspace:test/wasm"], + "@moq/watch": ["@moq/watch@workspace:js/watch"], "@moq/web-socket-stream": ["@moq/web-socket-stream@0.1.1", "", {}, "sha512-VKi/GE/CLqmbEZlgNbBLbAFk2pQTxT+KUD2DjCSoxqZxbS3KBe5BBZ0m3fila4fEGyll2JT5MmgEyVr4zkZEVA=="], diff --git a/js/justfile b/js/justfile index b4d05f58853..545de10fa3d 100644 --- a/js/justfile +++ b/js/justfile @@ -9,7 +9,7 @@ set working-directory := '.' # Paths a JS check depends on: every bun workspace (they are not all under js/, # see the root package.json) plus the config shared by all of them. `check`, # `fix`, and `ci` all skip themselves when a diff misses this. -scope := '^(js/|doc/|demo/(boy|web)/|test/smoke/clients/js|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' +scope := '^(js/|doc/|demo/(boy|web)/|test/smoke/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' # Run all checks by default. default: diff --git a/js/wasm/README.md b/js/wasm/README.md index af640a6e336..b836032e6c0 100644 --- a/js/wasm/README.md +++ b/js/wasm/README.md @@ -40,6 +40,7 @@ That compiles `rs/moq-wasm` for `wasm32-unknown-unknown`, runs `wasm-bindgen` Compiles and produces a typed, importable package. The consume path is runtime-portable: `moq-net`'s timers and `Instant` go through `web_async::time` -(wasmtimer on wasm), so they no longer panic. Not yet exercised end-to-end in a -browser against a relay, and media muxing (`moq-mux`) is still out. See +(wasmtimer on wasm), so they no longer panic. `just test wasm` exercises the +built package end-to-end in headless Chromium against a real relay +([`test/wasm/`](../../test/wasm)). Media muxing (`moq-mux`) is still out. See [`rs/moq-wasm/README.md`](../../rs/moq-wasm/README.md) for details. diff --git a/package.json b/package.json index 2be40c9aa2f..80473f4fb35 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "demo/web", "js/moq-boy", "test/smoke/clients/js", - "test/smoke/clients/js-native" + "test/smoke/clients/js-native", + "test/wasm" ] } diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 55f320ff43a..7941d520e32 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -178,7 +178,7 @@ Then `Config::load()?` (initializes tracing), build clients/servers via `.init() - Windows (moq-video's Media Foundation and D3D11 backends): `just rs windows`, which must run ON Windows. You can't reproduce it elsewhere, since cross-compiling dies in openh264-sys2's vendored C++. It names `moq-cli/play` explicitly, since that feature is what pulls in moq-video's wgpu renderer and moq-audio's cpal output; a default-feature build compiles neither. - macOS (moq-video's VideoToolbox and ScreenCaptureKit, moq-audio's system audio): `just rs macos`, which must run ON macOS. Scoped to moq-video + moq-audio, and needs `--all-features` because moq-audio's capture backend is off by default. - Linux: covered nightly, not per-PR. `just rs features` runs `--all-features` in a dev shell carrying PipeWire and ALSA. VAAPI loads libva dynamically, so nvenc/nvdec/vaapi/pipewire all compile without libva installed. - - wasm32 (moq-wasm): `just rs wasm`. The crate root is `#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` compiles it down to nothing and sees no errors at all. This one needs no special host (the Nix shell carries the target), so `just check-all` always runs it and `just check` runs it whenever the diff touches any crate directory under `rs/`, not just `rs/moq-wasm/`: moq-wasm builds on moq-net, so a break in a dependency is invisible to every host-target pass. It's a compile gate, distinct from the root `just wasm`, which builds the shippable `@moq/wasm` package. + - wasm32 (moq-wasm): `just rs wasm`. The crate root is `#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` compiles it down to nothing and sees no errors at all. This one needs no special host (the Nix shell carries the target), so `just check-all` always runs it and `just check` runs it whenever the diff touches any crate directory under `rs/`, not just `rs/moq-wasm/`: moq-wasm builds on moq-net, so a break in a dependency is invisible to every host-target pass. It's a compile gate, distinct from the root `just wasm`, which builds the shippable `@moq/wasm` package, and from `just test wasm`, which is the behavioral one: that runs the built bindings in headless Chromium against a real relay (`test/wasm/`, gated per-PR by `.github/workflows/wasm.yml`). Compiling says nothing about whether the bindings still open a session, which is how #2811 shipped on `main`. What still compiles these automatically, and when: diff --git a/rs/moq-wasm/README.md b/rs/moq-wasm/README.md index a63acbc5f0b..88d0fe5747f 100644 --- a/rs/moq-wasm/README.md +++ b/rs/moq-wasm/README.md @@ -30,6 +30,12 @@ What works today: methods, `.d.ts`). - Scope is the consume path (connect -> broadcast -> track -> group -> frame), the `@moq/watch` use case. The publish path follows the same shape. +- **It is tested in a browser**: `just test wasm` runs the built package in + headless Chromium against a real relay, one per protocol flavour, publishing + with `@moq/net` and subscribing with these bindings + ([`test/wasm/`](../../test/wasm)). `just rs wasm` only compiles the crate, so + that harness is the only thing that can tell a working binding from one that + merely builds. ### Three moq-net changes this requires diff --git a/test/justfile b/test/justfile index 7c9358e4c56..5af6a60a712 100644 --- a/test/justfile +++ b/test/justfile @@ -88,3 +88,11 @@ smoke-negative *args: # Check the subscriber's MPEG-TS output for IRD compliance. ts *args: ./ts/run.sh {{ args }} + +# Runs the @moq/wasm bindings in headless Chromium against a real relay, one per +# protocol flavour. The crate is `#![cfg(target_arch = "wasm32")]`, so nothing in +# `just check` or `just rs wasm` gets past compiling it. See wasm/README.md. + +# Browser coverage for the @moq/wasm bindings. +wasm *args: + ./wasm/run.sh {{ args }} diff --git a/test/wasm/README.md b/test/wasm/README.md new file mode 100644 index 00000000000..ee76bafd0b5 --- /dev/null +++ b/test/wasm/README.md @@ -0,0 +1,92 @@ +# @moq/wasm browser harness + +Runs the browser bindings ([`rs/moq-wasm`](../../rs/moq-wasm) -> `@moq/wasm`) in +headless Chromium against a real `moq-relay`, over real WebTransport. + +It exists because nothing else can see this code. The crate root is +`#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` +compiles it to nothing; `just rs wasm` compiles it for wasm32 but says nothing +about behavior. That gap shipped [#2811](https://github.com/moq-dev/moq/issues/2811) +on `main`: `ClientBuilder` never called `with_protocols`, the browser negotiated +no WebTransport subprotocol, and moq-net rejected the empty string as an unknown +ALPN. The bindings could not open a session at all and every gate stayed green. + +## Running + +```bash +just test wasm +``` + +```bash +just test wasm --timeout 60 +``` + +Everything is built from this checkout: `just wasm` for the bindings, `cargo +build -p moq-relay` for the relay. `WASM_PORT` moves the relay ports (three +consecutive, from 4460), `WASM_PROFILE` picks the relay's cargo profile, and +`RELAY_BIN` points at a prebuilt relay instead. + +## Shape + +`run.sh` builds, starts the relays, and hands their URLs to `driver.ts`, which +opens one Chromium tab on the bundled page and evaluates the suite in +`src/main.ts`. Each case reports back as data; the driver prints it and sets the +exit code. + +The publisher is `@moq/net` (the hand-written TypeScript implementation) and the +subscriber is `@moq/wasm` (the Rust one compiled to WebAssembly), so every case +is also a cross-implementation interop check. + +The page loads `js/wasm/dist/moq.js` from a URL rather than an import the +bundler can follow, so what runs is the generated output exactly as it ships: +the glue, the `.wasm` it fetches, and the JS names `#[wasm_bindgen]` chose. +`run.sh` also type-checks the harness against the generated `moq.d.ts`, which is +the only thing in the repo that reads it. That is worth doing: wasm-bindgen +resolves a type in a signature by its Rust identifier alone, so a binding can +compile, run, and still publish typings that name the wrong class. + +### Relays + +One per protocol flavour, since negotiation is the part that broke: + +| name | relay | negotiates | +| ------- | -------------------------------- | -------------------------------------- | +| `lite` | defaults | `moq-lite-05`, over its own ALPN | +| `ietf` | `--server-version moq-transport-19` | `moq-transport-19`, over its own ALPN | +| `setup` | `--server-version moq-lite-02` | the `moql` ALPN, version chosen by SETUP | + +The `setup` relay is as close as a real relay gets to the SETUP fallback path. +The branch that maps the browser's empty `protocol()` to `None` needs a server +that selects no subprotocol at all, which moq-relay never does, so it stays +uncovered here. + +### Cases + +- **negotiates the relay's version** -- the #2811 regression. Deleting the + `with_protocols` call fails this on `lite` (it falls back to `moq-lite-02` + over SETUP) and `ietf` (nothing left to negotiate). +- **reads a published broadcast** -- announce, subscribe, and read two + consecutive groups, checking every frame byte for byte. One frame is 128 KiB, + which is what exercises the chunked read path in the transport adapter. +- **refuses a track the publisher does not serve** -- the refusal has to + surface, as a rejected `subscribe` (lite-05, which looks a track's info up + first) or as a track that yields no group (IETF, lite-02). A hang fails on the + case timeout. + +### Known failures + +A case can name relay flavours where it is expected to fail, with the issue that +tracks why and a substring the failure has to contain. A matching failure prints +as `known` and does not fail the run. Anything else does: a failure that doesn't +match the signature is reported as the new regression it is, and so is a case +that starts passing, so the marker has to be removed with the fix. + +The signature is the point. Excusing a case by name alone would retire its +coverage, since an unrelated break in `consume`, `subscribe`, or frame copying +would land green under the same marker. + +## Not covered + +The publish direction: `moq-wasm` binds the consume path only. When +[#2814](https://github.com/moq-dev/moq/pull/2814) lands, the fixture publisher +here becomes a second wasm session and the interop runs both ways. diff --git a/test/wasm/driver.ts b/test/wasm/driver.ts new file mode 100644 index 00000000000..e23071fe51d --- /dev/null +++ b/test/wasm/driver.ts @@ -0,0 +1,131 @@ +/** + * Drives the `@moq/wasm` harness page in headless Chromium. + * + * Serves the bundled page plus the generated `js/wasm/dist` next to it, opens + * one tab, evaluates `moqWasmTest(config)`, and prints what each case reported. + * Relays are started by `run.sh`, which writes their URLs and expected versions + * into the `--relays` file. + * + * bun driver.ts --relays relays.json [--timeout 30] + * + * @module + */ +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { chromium } from "playwright"; +import type { CaseResult, Config, RelayFixture } from "./src/main.ts"; + +const { values } = parseArgs({ + options: { + relays: { type: "string" }, + timeout: { type: "string", default: "30" }, + }, +}); + +const timeoutMs = Number.parseFloat(values.timeout ?? "30") * 1000; +if (!values.relays || !Number.isFinite(timeoutMs) || timeoutMs <= 0) { + console.error("usage: driver.ts --relays relays.json [--timeout S>0]"); + process.exit(2); +} + +const relays = (await Bun.file(values.relays).json()) as RelayFixture[]; +if (relays.length === 0) { + console.error("error: no relays to test against"); + process.exit(2); +} + +const here = new URL(".", import.meta.url).pathname; +const dist = join(here, "dist"); +const wasm = join(here, "../../js/wasm/dist"); + +// localhost is a secure context, so WebTransport is available without TLS here. +// `/wasm/` serves the generated bindings unmodified; see src/main.ts. +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const path = new URL(req.url).pathname; + const file = path.startsWith("/wasm/") + ? Bun.file(join(wasm, path.slice("/wasm/".length))) + : Bun.file(join(dist, path === "/" ? "index.html" : path)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file); + }, +}); + +const config: Config = { + module: `http://127.0.0.1:${server.port}/wasm/moq.js`, + relays, + timeout: timeoutMs, +}; + +const browser = await chromium.launch({ + channel: "chromium", // full Chromium (new headless); the headless shell lacks WebTransport + headless: true, +}); + +let code = 1; +try { + const page = await browser.newPage(); + + // A Rust panic reaches the console through `console_error_panic_hook` rather + // than rejecting anything, so it can leave every case green. Fail on it, and + // on an uncaught exception, independently of the case results. + const fatal: string[] = []; + page.on("console", (message) => { + const text = message.text(); + console.error(`[page] ${text}`); + if (text.includes("panicked at")) fatal.push(`panic: ${text}`); + }); + page.on("pageerror", (error) => { + console.error(`[page error] ${error.message}`); + fatal.push(`uncaught: ${error.message}`); + }); + + await page.goto(`http://127.0.0.1:${server.port}/`, { waitUntil: "load" }); + + // The page bounds each case itself, so this only covers a wedge outside that + // loop: the wasm module never loading, or evaluate never returning. Generous + // on purpose, since the per-case budget is the tight one and a whole suite is + // seconds. Without it the job would sit until the workflow's own timeout. + const suiteTimeoutMs = timeoutMs * 10; + const results: CaseResult[] = await Promise.race([ + page.evaluate((cfg) => window.moqWasmTest(cfg), config), + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error(`suite did not finish within ${suiteTimeoutMs}ms`)), suiteTimeoutMs), + ), + ]); + + let failed = 0; + let known = 0; + for (const result of results) { + if (result.ok && !result.known) { + console.log(` ok ${result.name}`); + } else if (result.ok) { + // The bug it was waiting on is fixed, so the marker is now the lie. + failed++; + console.log(` FAIL ${result.name}: passes now; drop the ${result.known} marker`); + } else if (result.known) { + known++; + console.log(` known ${result.name} (${result.known}): ${result.error}`); + } else { + failed++; + console.log(` FAIL ${result.name}: ${result.error}`); + } + } + for (const message of fatal) console.log(` FAIL ${message}`); + + const total = results.length; + const summary = [`${total - failed - known}/${total} cases passed`]; + if (known > 0) summary.push(`${known} known`); + if (fatal.length > 0) summary.push(`${fatal.length} fatal`); + console.log(summary.join(", ")); + code = failed === 0 && fatal.length === 0 ? 0 : 1; +} catch (err) { + console.log(` FAIL ${err instanceof Error ? err.message : String(err)}`); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} + +process.exit(code); diff --git a/test/wasm/index.html b/test/wasm/index.html new file mode 100644 index 00000000000..f1ec8639b3c --- /dev/null +++ b/test/wasm/index.html @@ -0,0 +1,5 @@ + + +@moq/wasm browser harness +

Loaded by driver.ts; call moqWasmTest(config) from the console to run by hand.

+ diff --git a/test/wasm/package.json b/test/wasm/package.json new file mode 100644 index 00000000000..37d7bfeaae5 --- /dev/null +++ b/test/wasm/package.json @@ -0,0 +1,12 @@ +{ + "name": "@moq/wasm-test", + "private": true, + "type": "module", + "dependencies": { + "@moq/net": "workspace:*", + "@moq/wasm": "workspace:*" + }, + "devDependencies": { + "playwright": "^1.62.1" + } +} diff --git a/test/wasm/relay.toml b/test/wasm/relay.toml new file mode 100644 index 00000000000..93e133acf2e --- /dev/null +++ b/test/wasm/relay.toml @@ -0,0 +1,24 @@ +# Relay config for the @moq/wasm browser harness. +# Anonymous access, self-signed localhost cert, QUIC + HTTP on 127.0.0.1. +# +# run.sh starts one relay per protocol flavour, overriding the ports on the +# command line (`--server-bind` / `--web-http-listen`) and, for the non-default +# flavours, the version (`--server-version`). Only the settings every flavour +# shares live here. + +[log] +level = "info" + +[server] +# QUIC on UDP. 127.0.0.1 avoids IPv6 flakiness on CI runners. +bind = "127.0.0.1:4460" +tls.generate = ["localhost", "127.0.0.1"] + +[web.http] +# HTTP on TCP, serving /certificate.sha256 so the browser can pin the +# self-signed cert via WebTransport's serverCertificateHashes. +listen = "127.0.0.1:4460" + +[auth] +# Allow anonymous access to everything. +public = "" diff --git a/test/wasm/run.sh b/test/wasm/run.sh new file mode 100755 index 00000000000..d038e986941 --- /dev/null +++ b/test/wasm/run.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Headless-browser coverage for the @moq/wasm bindings. +# +# rs/moq-wasm is `#![cfg(target_arch = "wasm32")]`, so a host-target +# `cargo check --workspace` compiles it to nothing, and `just rs wasm` only +# compiles it. Neither says whether the bindings still work, which is how a +# missing `with_protocols` shipped on main unable to open a session at all +# (#2811). This runs them: a real moq-relay, real WebTransport, and the +# generated `js/wasm/dist` loaded by a real browser. +# +# One relay per protocol flavour, because negotiation is the part that broke: +# lite default versions -> moq-lite-05 over its own ALPN +# ietf --server-version 19 -> moq-transport-19 over its own ALPN +# setup --server-version lite-02 -> the "moql" ALPN, version chosen by SETUP +# +# The publisher is @moq/net (TypeScript) and the subscriber is @moq/wasm (Rust), +# so each case is also an interop check. See README.md. +set -euo pipefail + +WASM_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$WASM_DIR/../.." && pwd) + +TIMEOUT="${WASM_TIMEOUT:-30}" +PORT="${WASM_PORT:-4460}" + +# Cargo profile for the relay. Debug compiles faster, which is what a test +# fixture wants; the workload is three connections and a few hundred KiB. +PROFILE="${WASM_PROFILE:-debug}" +RELAY="${RELAY_BIN:-}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --timeout) + if [[ $# -lt 2 || -z "${2:-}" || "$2" == -* ]]; then + echo "error: --timeout requires a value" >&2 + exit 2 + fi + TIMEOUT="$2" + shift 2 + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +# Zero is rejected along with the non-numeric: it parses, but every case then +# times out on the first tick, which reads as nine broken bindings. +if [[ ! "$TIMEOUT" =~ ^[0-9]+(\.[0-9]+)?$ ]] || [[ "$TIMEOUT" =~ ^0+(\.0+)?$ ]]; then + echo "error: timeout must be a positive number (got '$TIMEOUT')" >&2 + exit 2 +fi + +# name:version-flag:expected-version. An empty flag leaves the relay at its +# defaults, which is the ALPN both sides prefer. +FLAVOURS=( + "lite::moq-lite-05" + "ietf:moq-transport-19:moq-transport-19" + "setup:moq-lite-02:moq-lite-02" +) + +# The relays take PORT and the next few, so the whole span has to be bindable. +# Port 0 is the trap worth naming: the relay would bind an arbitrary port while +# this script polls 0 forever. +if [[ ! "$PORT" =~ ^[0-9]+$ ]] || ((PORT < 1024 || PORT + ${#FLAVOURS[@]} - 1 > 65535)); then + echo "error: port must be 1024..$((65535 - ${#FLAVOURS[@]} + 1)) (got '$PORT')" >&2 + exit 2 +fi + +TMP=$(mktemp -d) +RELAY_PIDS=() + +# shellcheck disable=SC2329 # invoked indirectly via 'trap cleanup EXIT' +cleanup() { + local pid + for pid in ${RELAY_PIDS[@]+"${RELAY_PIDS[@]}"}; do + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + done + rm -rf "$TMP" +} +trap cleanup EXIT + +for tool in cargo bun; do + command -v "$tool" >/dev/null 2>&1 || { + echo "error: $tool not found; run inside 'nix develop'" >&2 + exit 1 + } +done +command -v wasm-bindgen >/dev/null 2>&1 || { + echo "error: wasm-bindgen not found; run inside 'nix develop'" >&2 + exit 1 +} + +# ── build ─────────────────────────────────────────────────────────────────── +echo "building @moq/wasm..." +(cd "$WORKSPACE" && just wasm) + +echo "building moq-relay ($PROFILE)..." +flag=() +[[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") +(cd "$WORKSPACE" && cargo build ${flag[@]+"${flag[@]}"} -p moq-relay) +TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" +[[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" + +cd "$WASM_DIR" +bun install +# Playwright ships its own Chromium; PLAYWRIGHT_BROWSERS_PATH means CI installed +# it already, in which case this is a no-op anyway. +bunx playwright install chromium + +# Type-check against the .d.ts wasm-bindgen just generated. This is the only +# thing that reads it, and it is worth reading: wasm-bindgen resolves a type in +# a signature by its Rust identifier alone, so a binding can compile, run, and +# still publish typings that name the wrong class. Not part of `just check`, +# which never builds js/wasm/dist. +echo "type-checking against js/wasm/dist/moq.d.ts..." +bunx tsc --noEmit + +echo "bundling the harness page..." +rm -rf dist +bun build src/main.ts --outdir dist --target browser >/dev/null +cp index.html dist/index.html + +# ── relays ────────────────────────────────────────────────────────────────── +entries=() +offset=0 +for flavour in "${FLAVOURS[@]}"; do + IFS=':' read -r name version_flag expected <<<"$flavour" + port=$((PORT + offset)) + offset=$((offset + 1)) + url="http://127.0.0.1:${port}" + + if curl -sf "$url/certificate.sha256" >/dev/null 2>&1; then + echo "error: something is already listening on 127.0.0.1:${port} (stale relay?)" >&2 + exit 1 + fi + + args=("$WASM_DIR/relay.toml" --server-bind "127.0.0.1:${port}" --web-http-listen "127.0.0.1:${port}") + [[ -z "$version_flag" ]] || args+=(--server-version "$version_flag") + + echo "starting $name relay on 127.0.0.1:${port}..." + "$RELAY" "${args[@]}" >"$TMP/relay-$name.log" 2>&1 & + RELAY_PIDS+=($!) + + # Polled tight rather than on a half-second tick: a relay binds in about + # 130ms, so a coarse interval spends most of the wait asleep, three times over. + for _ in $(seq 1 600); do + curl -sf "$url/certificate.sha256" >/dev/null 2>&1 && break + sleep 0.05 + done + if ! curl -sf "$url/certificate.sha256" >/dev/null 2>&1; then + echo "$name relay never became ready" >&2 + sed 's/^/ relay: /' "$TMP/relay-$name.log" >&2 || true + exit 1 + fi + + entries+=("{\"name\":\"$name\",\"url\":\"$url\",\"version\":\"$expected\"}") +done + +printf '[%s]\n' "$( + IFS=, + echo "${entries[*]}" +)" >"$TMP/relays.json" + +# ── run ───────────────────────────────────────────────────────────────────── +status=0 +bun driver.ts --relays "$TMP/relays.json" --timeout "$TIMEOUT" || status=$? + +if [[ $status -ne 0 ]]; then + for flavour in "${FLAVOURS[@]}"; do + name="${flavour%%:*}" + echo "── $name relay log ──" >&2 + sed 's/^/ /' "$TMP/relay-$name.log" >&2 || true + done +fi + +exit $status diff --git a/test/wasm/src/main.ts b/test/wasm/src/main.ts new file mode 100644 index 00000000000..616f0facd73 --- /dev/null +++ b/test/wasm/src/main.ts @@ -0,0 +1,384 @@ +/** + * The in-page half of the `@moq/wasm` browser harness. + * + * Registers `moqWasmTest(config)` on the window; `driver.ts` calls it from + * Playwright and reports what comes back. The publisher is `@moq/net` (the + * hand-written TypeScript implementation) and the subscriber is `@moq/wasm` + * (the Rust one compiled to WebAssembly), so every case is also a cross- + * implementation interop check against a real relay. + * + * The wasm module is loaded through a URL handed in at runtime rather than an + * `import` the bundler can see, so what runs here is `js/wasm/dist` exactly as + * it ships: the generated glue, the `.wasm` it fetches, and the JS names + * `#[wasm_bindgen]` chose. A binding that compiles but exports the wrong name + * fails here and nowhere else. + * + * @module + */ +import * as Moq from "@moq/net"; + +/** The generated `@moq/wasm` module, typed from its published `.d.ts`. */ +type Wasm = typeof import("@moq/wasm"); +type Session = import("@moq/wasm").Session; +type Broadcast = import("@moq/wasm").Broadcast; +type Track = import("@moq/wasm").Track; + +/** One relay the suite runs against, and the version it must negotiate. */ +export interface RelayFixture { + /** Label used in case names, e.g. "lite". */ + name: string; + /** The relay's `http://` origin. Rewritten to `https://` for WebTransport. */ + url: string; + /** What `Session.version()` has to report, e.g. "moq-lite-05". */ + version: string; +} + +/** Everything the driver hands the page. */ +export interface Config { + /** URL of the generated `moq.js`, served next to this page. */ + module: string; + /** The relays to run the suite against. */ + relays: RelayFixture[]; + /** Per-case budget in milliseconds. */ + timeout: number; +} + +/** The outcome of a single case, as reported back to the driver. */ +export interface CaseResult { + /** `: `, unique across the run. */ + name: string; + /** True when the case passed. */ + ok: boolean; + /** The failure message, when it didn't. */ + error?: string; + /** Set when this failure is the known one for this relay, naming its issue. */ + known?: string; +} + +/** The one track the fixture publisher serves. */ +const TRACK = "data"; + +/** + * Big enough to span several WebTransport read chunks, which is the only thing + * that exercises the copy loop in the transport adapter's `read`. + */ +const LARGE_FRAME_BYTES = 128 * 1024; + +/** The frames in every published group, in order. */ +const FIXTURE: Uint8Array[] = [encode("wasm-frame-0"), encode("wasm-frame-1"), pattern(LARGE_FRAME_BYTES)]; + +/** How many consecutive groups the round trip reads before it is satisfied. */ +const GROUPS_TO_READ = 2; + +/** How often the fixture publisher appends a group. */ +const GROUP_INTERVAL_MS = 50; + +function encode(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +/** A deterministic byte pattern; 251 is prime, so it never aligns with a chunk boundary. */ +function pattern(length: number): Uint8Array { + const bytes = new Uint8Array(length); + for (let i = 0; i < length; i++) bytes[i] = i % 251; + return bytes; +} + +function hexToBytes(hex: string): Uint8Array { + if (hex.length % 2 !== 0 || !/^[0-9a-fA-F]*$/.test(hex)) throw new Error(`not hex: ${hex}`); + const bytes = new Uint8Array(hex.length / 2); + for (let i = 0; i < bytes.length; i++) bytes[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + return bytes; +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Load and initialize the generated module once per page. */ +let loading: Promise | undefined; +function load(url: string): Promise { + loading ??= (async () => { + // The specifier is a variable so the bundler leaves it alone; see the module doc. + const wasm = (await import(url)) as Wasm; + await wasm.default(); + wasm.setup(); // panic hook + tracing, so a Rust panic reaches the console + return wasm; + })(); + return loading; +} + +/** + * Connect the wasm bindings to a relay, pinning its self-signed certificate. + * + * `http://` can't be a WebTransport origin, so the relay's HTTP listener serves + * the hash and the URL is rewritten, the same dance `@moq/net` does internally. + */ +async function connect(wasm: Wasm, relay: RelayFixture): Promise { + const url = new URL(relay.url); + const fingerprint = new URL(url); + fingerprint.pathname = "/certificate.sha256"; + + const response = await fetch(fingerprint); + if (!response.ok) throw new Error(`${fingerprint}: HTTP ${response.status}`); + const hash = hexToBytes((await response.text()).trim()); + + url.protocol = "https:"; + return await wasm.Session.connectWithHashes(url.toString(), [hash]); +} + +/** + * Publish {@link FIXTURE} on `path` for the duration of `run`, one group every + * {@link GROUP_INTERVAL_MS}. + * + * It keeps publishing rather than writing a fixed number of groups and closing: + * a track that ends before the subscriber's stream is wired up is only reachable + * through the relay's cache, so the test would be measuring retention, and would + * race. A live edge that keeps moving has neither problem. Every track name + * other than {@link TRACK} is rejected, which is what the refusal case reads. + */ +async function withPublisher(relay: RelayFixture, path: string, run: () => Promise): Promise { + const connection = await Moq.Connection.connect(new URL(relay.url)); + const broadcast = new Moq.Broadcast.Producer(); + connection.publish(Moq.Path.from(path), broadcast); + + let stopped = false; + const writers: Promise[] = []; + + // lite-05 looks a track's info up before subscribing, so the same name can be + // requested more than once. Each request gets its own producer writing the same + // groups; the subscriber only ever reads the one it asked for. + const serving = (async () => { + for (;;) { + const request = await broadcast.requested(); + if (!request) break; + if (request.name !== TRACK) { + request.reject(new Error(`no such track: ${request.name}`)); + continue; + } + const track = request.accept(); + writers.push( + (async () => { + while (!stopped && track.closed.peek() === undefined) { + const group = track.appendGroup(); + for (const frame of FIXTURE) { + group.writeFrame({ payload: frame, timestamp: Moq.Time.Timestamp.now() }); + } + group.close(); + await sleep(GROUP_INTERVAL_MS); + } + track.close(); + })(), + ); + } + })(); + + try { + return await run(); + } finally { + stopped = true; + broadcast.close(); + connection.close(); + await serving; + await Promise.all(writers); + } +} + +/** Run `body` against a connected wasm session, freeing it afterwards. */ +async function withSession(wasm: Wasm, relay: RelayFixture, body: (session: Session) => Promise): Promise { + const session = await connect(wasm, relay); + try { + return await body(session); + } finally { + // Dropping the Rust value closes the transport, which ends the driver task. + session.free(); + } +} + +/** Wait for `path` to be announced and return the broadcast, or fail. */ +async function consume(session: Session, path: string): Promise { + const broadcast = await session.consume(path); + if (!broadcast) throw new Error(`consume(${path}) resolved without a broadcast`); + return broadcast; +} + +/** + * Read `count` groups off a track and check each one against {@link FIXTURE}. + * + * The first sequence is whatever the live edge was at subscribe time, so only + * the step between groups is asserted, not the starting number. + */ +async function expectGroups(track: Track, count: number): Promise { + let previous: bigint | undefined; + + for (let i = 0; i < count; i++) { + const group = await track.recvGroup(); + if (!group) throw new Error(`track ended after ${i} groups, want ${count}`); + try { + if (previous !== undefined && group.sequence !== previous + 1n) { + throw new Error(`group ${i}: sequence is ${group.sequence}, want ${previous + 1n}`); + } + previous = group.sequence; + + for (const [j, want] of FIXTURE.entries()) { + const frame = await group.readFrame(); + if (!frame) throw new Error(`group ${i}: ended after ${j} frames, want ${FIXTURE.length}`); + expectBytes(frame, want, `group ${i} frame ${j}`); + } + if (await group.readFrame()) throw new Error(`group ${i}: got more than ${FIXTURE.length} frames`); + } finally { + group.free(); + } + } +} + +function expectBytes(got: Uint8Array, want: Uint8Array, where: string): void { + if (got.length !== want.length) throw new Error(`${where}: got ${got.length} bytes, want ${want.length}`); + for (let i = 0; i < want.length; i++) { + if (got[i] !== want[i]) throw new Error(`${where}: byte ${i} is ${got[i]}, want ${want[i]}`); + } +} + +/** An expected failure: the issue that tracks it, and how to recognize it. */ +interface Known { + /** The issue tracking the bug, e.g. "#2903". */ + issue: string; + /** + * Substring the failure has to contain to count as this bug. + * + * Excusing a whole case by name would excuse every future failure of it too, + * which would quietly retire the coverage: an unrelated regression in + * `consume`, `subscribe`, or frame copying would land green under the same + * marker. Anything that doesn't match is reported as the new failure it is. + */ + error: string; +} + +interface Case { + name: string; + /** + * Relay flavours where this case is expected to fail. The driver reports a + * matching failure as `known`, and reports a pass as a failure, so the marker + * has to be removed once the bug is fixed. + */ + known?: Record; + run: (wasm: Wasm, relay: RelayFixture) => Promise; +} + +const CASES: Case[] = [ + { + // The #2811 regression: without `with_protocols` the browser negotiates no + // subprotocol at all and moq-net rejects the empty string as an unknown ALPN, + // so this fails to connect rather than reporting the wrong version. + name: "negotiates the relay's version", + run: async (wasm, relay) => { + await withSession(wasm, relay, async (session) => { + const version = session.version(); + if (version !== relay.version) throw new Error(`version is "${version}", want "${relay.version}"`); + }); + }, + }, + { + name: "reads a published broadcast", + // The IETF subscriber takes the publisher's identity straight off the hop + // chain, so an advert from a publisher that has none (@moq/net speaks no + // cluster extension) is attributed to Origin::UNKNOWN. Every repeat NAMESPACE + // for that path then reads as a new publisher and detaches the live source, + // killing the subscription mid-read. moq-lite has the same rule but stamps the + // upstream session's id when the chain has none, so it never reaches it. + // See https://github.com/moq-dev/moq/issues/2903. The detached source surfaces + // as the recv stream being dropped mid-read, which is the signature below. + known: { ietf: { issue: "#2903", error: "dropped" } }, + run: async (wasm, relay) => { + const path = `wasm-test/${relay.name}`; + await withPublisher(relay, path, async () => { + await withSession(wasm, relay, async (session) => { + const broadcast = await consume(session, path); + const track = await broadcast.subscribe(TRACK); + try { + await expectGroups(track, GROUPS_TO_READ); + } finally { + track.free(); + broadcast.free(); + } + }); + }); + }, + }, + { + // A refusal surfaces at different points depending on the version: lite-05 + // looks a track's info up before subscribing, so `subscribe` itself rejects, + // while IETF and lite-02 have no such lookup and only find out when no group + // arrives. Either is fine. Delivering a group is not, and neither is never + // settling, which the case timeout catches. + name: "refuses a track the publisher does not serve", + run: async (wasm, relay) => { + const path = `wasm-test/${relay.name}-refuse`; + await withPublisher(relay, path, async () => { + await withSession(wasm, relay, async (session) => { + const broadcast = await consume(session, path); + let delivered = false; + try { + const track = await broadcast.subscribe("missing"); + try { + const group = await track.recvGroup(); + if (group) { + delivered = true; + group.free(); + } + } finally { + track.free(); + } + } catch { + // The refusal surfaced as a rejection, the other legal shape. + } finally { + broadcast.free(); + } + if (delivered) throw new Error("an unserved track delivered a group"); + }); + }); + }, + }, +]; + +/** Run every case against every relay, sequentially, and report each outcome. */ +async function run(config: Config): Promise { + const wasm = await load(config.module); + const results: CaseResult[] = []; + + for (const relay of config.relays) { + for (const example of CASES) { + const name = `${relay.name}: ${example.name}`; + const known = example.known?.[relay.name]; + let timer: ReturnType | undefined; + try { + await Promise.race([ + example.run(wasm, relay), + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`timed out after ${config.timeout}ms`)), + config.timeout, + ); + }), + ]); + // A pass still carries the marker, so the driver can fail on it. + results.push({ name, ok: true, ...(known ? { known: known.issue } : {}) }); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + const expected = known !== undefined && error.includes(known.error); + results.push({ name, ok: false, error, ...(expected ? { known: known.issue } : {}) }); + } finally { + clearTimeout(timer); + } + } + } + + return results; +} + +declare global { + interface Window { + /** Entry point the Playwright driver evaluates. */ + moqWasmTest: (config: Config) => Promise; + } +} + +window.moqWasmTest = run; diff --git a/test/wasm/tsconfig.json b/test/wasm/tsconfig.json new file mode 100644 index 00000000000..9dce13e427d --- /dev/null +++ b/test/wasm/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../js/tsconfig.json", + "compilerOptions": { + "types": ["bun"], + "noEmit": true + }, + "include": ["driver.ts", "src"] +} From f0d489813bdbd368f87ed7cfe839d0954e6aa488 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 17 Aug 2026 21:35:08 -0700 Subject: [PATCH 16/29] perf(kio): dedup waiter registration so Park can reuse a parked waiter (#2905) Co-authored-by: Claude Fable 5 --- Cargo.lock | 1 + Cargo.toml | 1 + rs/kio/Cargo.toml | 5 + rs/kio/benches/waiter.rs | 197 ++++++++++++++++++++ rs/kio/src/loom.rs | 58 ++++++ rs/kio/src/waiter.rs | 379 ++++++++++++++++++++++++++++++++++----- rs/moq-json/Cargo.toml | 2 +- rs/moq-net/Cargo.toml | 2 +- 8 files changed, 600 insertions(+), 45 deletions(-) create mode 100644 rs/kio/benches/waiter.rs diff --git a/Cargo.lock b/Cargo.lock index 425cf60eacb..af48953c7a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4271,6 +4271,7 @@ checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" name = "kio" version = "0.5.4" dependencies = [ + "criterion", "loom", "smallvec", "tokio", diff --git a/Cargo.toml b/Cargo.toml index a80c25b3fd6..6a93a0439e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ resolver = "2" rust-version = "1.91" [workspace.dependencies] +criterion = "0.8" flate2 = "1.1" hang = { version = "0.20", path = "rs/hang" } kio = { version = "0.5", path = "rs/kio" } diff --git a/rs/kio/Cargo.toml b/rs/kio/Cargo.toml index 079f3841ef2..0279994c0ac 100644 --- a/rs/kio/Cargo.toml +++ b/rs/kio/Cargo.toml @@ -30,7 +30,12 @@ web-async = { workspace = true, optional = true } loom = { workspace = true } [dev-dependencies] +criterion = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util", "time"] } [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] } + +[[bench]] +name = "waiter" +harness = false diff --git a/rs/kio/benches/waiter.rs b/rs/kio/benches/waiter.rs new file mode 100644 index 00000000000..e0e7f7483c1 --- /dev/null +++ b/rs/kio/benches/waiter.rs @@ -0,0 +1,197 @@ +//! Microbenchmarks for waiter registration. +//! +//! `WaiterList::register` runs on every pending poll of every kio channel, so it is +//! the per-wakeup overhead of the whole stack. These benchmarks isolate the cases +//! that matter: a fresh registration against lists of various sizes, the +//! steady-state re-registration of an already-parked waiter, reclaiming slots from +//! abandoned waiters, and the full park/wake/re-register cycle a poll function +//! actually drives. +//! +//! Contention is deliberately absent: a `WaiterList` is always behind its caller's +//! lock (`register` takes `&mut`), so cross-thread cost belongs to that lock, not to +//! the list. +//! +//! Run with `cargo bench -p kio`. + +use std::task::{Context, Waker}; + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use kio::{Park, Waiter, WaiterList}; + +/// Live-waiter populations to sweep. The top end is the inline capacity, past which +/// the list spills to the heap. +const SIZES: [usize; 4] = [1, 2, 8, 32]; + +/// A list holding `n` live registrations, plus the waiters keeping them live. +fn populated(n: usize) -> (WaiterList, Vec) { + let mut list = WaiterList::new(); + let waiters: Vec<_> = (0..n) + .map(|_| { + let waiter = Waiter::new(Waker::noop().clone()); + waiter.register(&mut list); + waiter + }) + .collect(); + (list, waiters) +} + +/// Register a fresh waiter into a list of N live entries: the dedup scan misses, +/// the dead-slot probe finds nothing, and the entry is appended. +fn bench_register(c: &mut Criterion) { + let mut g = c.benchmark_group("waiter_register_live"); + for &n in &SIZES { + g.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| { + b.iter_batched( + || { + let (list, waiters) = populated(n); + (list, waiters, Waiter::new(Waker::noop().clone())) + }, + |(mut list, waiters, fresh)| { + fresh.register(&mut list); + (list, waiters, fresh) + }, + BatchSize::SmallInput, + ) + }); + } + g.finish(); +} + +/// Re-register a waiter already in a list of N live entries: the record fast path, +/// which is the steady-state cost of a poll that re-runs while still parked. Flat +/// in N, since membership is settled by the waiter's own record, not a scan. +fn bench_reregister(c: &mut Criterion) { + let mut g = c.benchmark_group("waiter_reregister"); + for &n in &SIZES { + g.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| { + let (mut list, waiters) = populated(n); + let last = waiters.last().unwrap(); + b.iter(|| last.register(&mut list)); + }); + } + g.finish(); +} + +/// Register a fresh waiter into a list whose N entries are all dead: the dedup scan +/// misses and the probe reclaims a slot in place. +fn bench_register_dead(c: &mut Criterion) { + let mut g = c.benchmark_group("waiter_register_dead"); + for &n in &SIZES { + g.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| { + b.iter_batched( + || { + let (list, waiters) = populated(n); + drop(waiters); + (list, Waiter::new(Waker::noop().clone())) + }, + |(mut list, fresh)| { + fresh.register(&mut list); + (list, fresh) + }, + BatchSize::SmallInput, + ) + }); + } + g.finish(); +} + +/// Register-and-abandon without a wake: every iteration allocates a waiter, parks +/// it, and drops it, so the list must keep reclaiming the dead slot it leaves. +fn bench_cancel(c: &mut Criterion) { + c.bench_function("waiter_cancel", |b| { + let mut list = WaiterList::new(); + b.iter(|| Waiter::new(Waker::noop().clone()).register(&mut list)); + }); +} + +/// The cycle a poll function actually drives: hold the park, register with L lists, +/// then one list wakes. The re-poll arrives with live registrations still parked on +/// the other lists, which is exactly the case `Park` must not retire on. +fn bench_park_cycle(c: &mut Criterion) { + let mut g = c.benchmark_group("waiter_park_cycle"); + // 16 lists overflow the waiter's record table, exercising eviction and the + // scan fallback rather than the pure record fast path. + for &lists in &[1usize, 2, 4, 8, 16] { + g.bench_with_input(BenchmarkId::from_parameter(lists), &lists, |b, &n| { + let cx = Context::from_waker(Waker::noop()); + let mut park = Park::default(); + let mut lists: Vec<_> = (0..n).map(|_| WaiterList::new()).collect(); + b.iter(|| { + let waiter = park.hold(&cx); + for list in &mut lists { + waiter.register(list); + } + lists[0].wake(); + }); + }); + } + g.finish(); +} + +/// Fan-out sizes to sweep, crossing the inline capacity into heap spill. +const FANOUT: [usize; 4] = [8, 32, 128, 512]; + +/// The fan-out cycle: N waiters all parked on one list, a wake drains it, and every +/// waiter re-registers. Throughput is per registration, so a super-linear per-cycle +/// cost shows up as falling throughput at higher N. +fn bench_fanout_cycle(c: &mut Criterion) { + let mut g = c.benchmark_group("waiter_fanout_cycle"); + for &n in &FANOUT { + g.throughput(Throughput::Elements(n as u64)); + g.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| { + let mut list = WaiterList::new(); + let waiters: Vec<_> = (0..n).map(|_| Waiter::new(Waker::noop().clone())).collect(); + b.iter(|| { + for waiter in &waiters { + waiter.register(&mut list); + } + // Drain the production way: snapshot under the lock, wake outside. + list.take().wake(); + }); + }); + } + g.finish(); +} + +/// The fan-out cycle when every waiter is also parked on a second, never-woken list +/// (a value list and a closed list under one lock, say). The live second +/// registration is what forecloses the registered-nowhere fast path, so this is the +/// worst case for rebuilding the drained list. +fn bench_fanout_cycle_parked(c: &mut Criterion) { + let mut g = c.benchmark_group("waiter_fanout_cycle_parked"); + for &n in &FANOUT { + g.throughput(Throughput::Elements(n as u64)); + g.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| { + let mut list = WaiterList::new(); + let mut parked = WaiterList::new(); + let waiters: Vec<_> = (0..n) + .map(|_| { + let waiter = Waiter::new(Waker::noop().clone()); + waiter.register(&mut parked); + waiter + }) + .collect(); + b.iter(|| { + for waiter in &waiters { + waiter.register(&mut list); + waiter.register(&mut parked); + } + // Drain the production way: snapshot under the lock, wake outside. + list.take().wake(); + }); + }); + } + g.finish(); +} + +criterion_group!( + benches, + bench_register, + bench_reregister, + bench_register_dead, + bench_cancel, + bench_park_cycle, + bench_fanout_cycle, + bench_fanout_cycle_parked +); +criterion_main!(benches); diff --git a/rs/kio/src/loom.rs b/rs/kio/src/loom.rs index a63ada0ce3e..426bae6402b 100644 --- a/rs/kio/src/loom.rs +++ b/rs/kio/src/loom.rs @@ -253,6 +253,64 @@ fn queue_close_wakes_a_parked_pop() { }); } +/// A poll parked on two lists is re-woken by one while still registered with the +/// other. The re-poll reuses the parked waiter, so re-registering into the drained +/// list must actually land (not be skipped as a false duplicate), and re-registering +/// into the still-parked list must dedup rather than stack. +/// +/// The terminal wake is on the *drained* list, so an execution whose re-poll fails +/// to re-register there parks forever, which loom reports as a deadlock. `parked` is +/// never woken at all: it exists to keep the waiter's registrations live across +/// re-polls, which is exactly the state the old code retired on. +#[test] +fn a_reused_waiter_hears_the_drained_list_again() { + loom::model(|| { + let woken = Fan::new(); + let parked = Fan::new(); + let done = Arc::new(AtomicBool::new(false)); + + let waker = { + let woken = woken.clone(); + let done = done.clone(); + + thread::spawn(move || { + // Drains `woken`; a re-poll now re-registers while still parked on + // `parked`. + woken.wake(); + done.store(true, Ordering::SeqCst); + // Must reach the re-registered waiter, or the model deadlocks. + woken.wake(); + }) + }; + + // Register before reading `done`, as in `a_held_wake_is_never_lost`: the + // other order has a window where the store and both wakes land between the + // read and the registration. + // + // The closure holds its own waiter rather than using the parked one: loom's + // `block_on` waker fails `will_wake` between polls (its vtable is a + // const-promoted static, duplicated across codegen units in release builds), + // so `Park` retires every poll under loom and would never model the reuse + // this test exists for. Waking the first poll's waker still unparks the + // task, so a stable identity is safe here. + block_on({ + let mut stable = None; + wait(move |parked_waiter| { + let waiter = stable.get_or_insert_with(|| Waiter::new(parked_waiter.waker().clone())); + woken.register(waiter); + parked.register(waiter); + + match done.load(Ordering::SeqCst) { + true => Poll::Ready(()), + false => Poll::Pending, + } + }) + }); + + waker.join().unwrap(); + }); +} + /// `Fan` holds wakes back while a guard is out and delivers them when the last /// one drops. A wake racing that window must still reach the parked waiter, whichever /// side of the hold it lands on. diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index 7551db44287..a113a61a933 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -5,7 +5,7 @@ use std::{ pin::Pin, // std, not `crate::sync`: loom's Arc has no `downgrade`, and `Waker::from` takes // std's. See `sync.rs`. - sync::{Arc, OnceLock, Weak}, + sync::{Arc, OnceLock, Weak, atomic::AtomicU64}, task::{Context, Poll, Wake, Waker}, }; @@ -19,13 +19,93 @@ use crate::{ /// Number of slots stored inline before spilling to the heap. const INLINE_WAITERS: usize = 32; +/// Registrations remembered per waiter identity for O(1) dedup. A poll typically +/// parks on a small handful of lists; a poll cycling through more than this many +/// evicts its own records and falls back to the scan, so the cap bounds memory, +/// never correctness. +const RECORDED_LISTS: usize = 8; + +/// The shared identity behind a [`Waiter`], referenced weakly by every list entry. +struct Identity { + /// The task waker the lists deliver to. + waker: Waker, + + /// Registrations this identity has made. A record matching a list's current + /// epoch proves the registration is still in place, and a stale one proves it + /// is gone: live entries only leave through a drain, every drain bumps the + /// epoch, and every registration refreshes the record. Either way membership is + /// settled in O(1); only a missing record (never registered, or evicted) needs + /// the scan. + recorded: Mutex, +} + +/// The record table behind an [`Identity`]. +#[derive(Default)] +struct Records { + /// `(list id, list epoch)` pairs, at most [`RECORDED_LISTS`] of them. + entries: SmallVec<[(u64, u64); RECORDED_LISTS]>, + + /// State for choosing an eviction victim pseudo-randomly. Deterministic + /// policies (FIFO, LRU) evict exactly the next-needed record when a poll + /// cycles through more lists than the cap, degrading every registration to the + /// scan; a random victim keeps an expected `cap / lists` fraction of hits + /// instead. + seed: u64, +} + +/// What an identity's records prove about its membership in one list. +enum Presence { + /// Recorded at the list's current epoch: still registered. + Present, + /// Recorded at an older epoch: the drain that bumped it removed every entry, + /// and registering again would have refreshed the record, so: not registered. + Absent, + /// No record (never registered, or evicted): membership must be checked. + Unknown, +} + +impl Identity { + /// What the records prove about membership in list `id` at `epoch`. + /// + /// Also refreshes the record to `(id, epoch)` before returning. That is sound + /// even though the caller has not registered yet: every path out of + /// [`WaiterList::register`] leaves the waiter registered, and folding the write + /// into the lookup keeps registration at one lock round-trip. + fn presume(&self, id: u64, epoch: u64) -> Presence { + let mut recorded = self.recorded.lock().expect("mutex poisoned"); + + if let Some(entry) = recorded.entries.iter_mut().find(|entry| entry.0 == id) { + let presence = match entry.1 == epoch { + true => Presence::Present, + false => Presence::Absent, + }; + entry.1 = epoch; + return presence; + } + + if recorded.entries.len() < RECORDED_LISTS { + recorded.entries.push((id, epoch)); + return Presence::Unknown; + } + + // Evict a pseudo-random record (a Weyl-style step, no external RNG): + // eviction only ever costs a scan later, and a random victim avoids the + // deterministic worst case where a cyclic visit pattern one list wider + // than the cap evicts exactly the record it needs next, every time. + recorded.seed = recorded.seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + let victim = (recorded.seed >> 33) as usize % RECORDED_LISTS; + recorded.entries[victim] = (id, epoch); + Presence::Unknown + } +} + /// Handle passed to poll functions for registering with [`WaiterList`]s. /// -/// Holds the task's [`Waker`] by value and, lazily, a shared `Arc` that list -/// entries reference weakly. The `Arc` is allocated on the first [`Self::register`], -/// so a poll that resolves without ever parking never touches the heap. Its `Weak`s -/// go dead the moment the owning [`Waiter`] drops, which is how a [`WaiterList`] -/// reclaims slots with no explicit deregister. +/// Holds the task's [`Waker`] by value and, lazily, a shared identity that list +/// entries reference weakly. The identity is allocated on the first +/// [`Self::register`], so a poll that resolves without ever parking never touches +/// the heap. Its `Weak`s go dead the moment the owning [`Waiter`] drops, which is +/// how a [`WaiterList`] reclaims slots with no explicit deregister. /// /// A clone shares this identity: it wakes the same task, its registrations count as /// the original's, and they all stay live until every clone drops. @@ -36,7 +116,7 @@ pub struct Waiter { // The shared handle downgraded into every list this waiter registers with. Created on the // first `register` (a poll that never parks never allocates it), then reused so multiple // lists in one poll share a single allocation whose `Weak`s die together when the waiter drops. - shared: OnceLock>, + shared: OnceLock>, } impl Waiter { @@ -64,10 +144,15 @@ impl Waiter { &self.waker } - /// The shared waker handle downgraded into lists, allocated on first use and cached so + /// The shared identity downgraded into lists, allocated on first use and cached so /// repeat registrations (across polls, or across lists in one poll) share one allocation. - fn shared(&self) -> &Arc { - self.shared.get_or_init(|| Arc::new(self.waker.clone())) + fn shared(&self) -> &Arc { + self.shared.get_or_init(|| { + Arc::new(Identity { + waker: self.waker.clone(), + recorded: Mutex::new(Records::default()), + }) + }) } /// Poll a foreign [`Future`] against this waiter, so it re-wakes the enclosing @@ -90,6 +175,18 @@ impl Clone for Waiter { } } +/// Source of unique [`WaiterList`] ids, so a waiter's recorded registrations can +/// never confuse two lists (ids are handed out once and never reused). Exhausting +/// it would take 2^64 list creations, centuries at one per nanosecond, so wraparound +/// is unreachable in any process lifetime. +static NEXT_LIST_ID: AtomicU64 = AtomicU64::new(1); + +/// The id of a list that opts out of record-keeping: [`WaiterList::take`] snapshots, +/// which exist to be woken, not registered with. Never handed out by +/// [`NEXT_LIST_ID`], so no record can ever match one, and registering into such a +/// list just uses the scan fallback. +const UNTRACKED: u64 = 0; + /// A list of weak wakers waiting for notification. /// /// Slots live inline (up to `INLINE_WAITERS`) and only spill to the heap @@ -97,35 +194,94 @@ impl Clone for Waiter { /// collection across many `register` calls so the list doesn't grow /// unboundedly while keeping per-call cost O(1). pub struct WaiterList { - entries: SmallVec<[Weak; INLINE_WAITERS]>, + entries: SmallVec<[Weak; INLINE_WAITERS]>, /// Rotating cursor for opportunistic GC on `register`. cursor: usize, + /// Never-reused identity for this list, paired with `epoch` in waiter records. + id: u64, + /// Bumped on every drain. A waiter whose record carries the current epoch is + /// still registered: live entries only ever leave through a drain. + epoch: u64, } impl WaiterList { /// Create an empty list, allocating nothing until the first [`register`](Self::register). pub fn new() -> Self { + // A CAS loop rather than fetch_add, so exhaustion fails closed instead of + // wrapping into reissued ids (a reissued id could fake presence). List + // creation is cold, and the loop is contention-free in practice. + let mut id = NEXT_LIST_ID.load(std::sync::atomic::Ordering::Relaxed); + loop { + assert_ne!(id, u64::MAX, "waiter list id space exhausted"); + match NEXT_LIST_ID.compare_exchange_weak( + id, + id + 1, + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + ) { + Ok(_) => break, + Err(current) => id = current, + } + } + Self { entries: SmallVec::new(), cursor: 0, + id, + epoch: 0, } } - /// Register a waiter. + /// Register a waiter. Idempotent: a waiter already in the list stays as one entry. + /// + /// The cost is O(1) whenever the waiter's own records settle membership, which is + /// every steady state: still registered since the last drain (a re-poll while + /// parked) returns immediately, and registered before the last drain (rebuilding + /// after a wake) appends immediately. Only a waiter with no record of this list + /// falls back to a membership scan: pointer equality over contiguous entries, + /// exact because a `Weak` pins its allocation's address, so an equal pointer + /// always means the same waiter. The scan too is skipped when the waiter is + /// registered nowhere at all (one atomic load on its own allocation). /// - /// Performs a small, bounded amount of garbage collection: probes the - /// slot at the rotating cursor, replacing it in place if dead. The - /// cursor advances on each append so the probe window covers the - /// whole list over time. + /// An append also performs a small, bounded amount of garbage collection: probes + /// the slot at the rotating cursor, replacing it in place if dead. The cursor + /// advances on each append so the probe window covers the whole list over time. pub fn register(&mut self, waiter: &Waiter) { - let new_weak = Arc::downgrade(waiter.shared()); + let shared = waiter.shared(); + + let presence = match self.id { + UNTRACKED => Presence::Unknown, + id => shared.presume(id, self.epoch), + }; + + match presence { + // Still registered since the last drain: nothing to do. This is what + // lets `Park` keep reusing a still-registered waiter instead of + // retiring it every poll. + Presence::Present => return, + + // Provably not in the list: straight to the append. + Presence::Absent => {} + + // No record either way, so membership needs the scan, except when the + // waiter is registered nowhere at all. + Presence::Unknown => { + if Arc::weak_count(shared) > 0 { + let ptr = Arc::as_ptr(shared); + if self.entries.iter().any(|entry| std::ptr::eq(entry.as_ptr(), ptr)) { + return; + } + } + } + } + + let new_weak = Arc::downgrade(shared); for _ in 0..self.entries.len().min(2) { if self.entries[self.cursor].strong_count() == 0 { - // Reuse the dead slot in place. Each Waiter owns a - // unique Arc, so strong_count == 0 uniquely + // Reuse the dead slot in place. Each Waiter owns a unique + // identity allocation, so strong_count == 0 uniquely // identifies a slot whose owner has been dropped. - // No will_wake / pointer comparison needed. self.entries[self.cursor] = new_weak; return; } @@ -138,17 +294,27 @@ impl WaiterList { /// Drain all entries into a new [`WaiterList`], leaving this one empty. pub fn take(&mut self) -> Self { self.cursor = 0; + self.epoch = self.epoch.checked_add(1).expect("waiter list epoch overflow"); Self { entries: std::mem::take(&mut self.entries), cursor: 0, + // This runs on every notification, so the snapshot must not touch the + // global id counter (a shared cache line across all channels). It is + // untracked instead: it exists to be woken, and registering into it + // falls back to the scan rather than impersonating this list. + id: UNTRACKED, + epoch: 0, } } /// Wake all live waiters, draining the list. pub fn wake(&mut self) { self.cursor = 0; - for waker in self.entries.drain(..).filter_map(|w| w.upgrade()) { - waker.wake_by_ref(); + // Fail closed rather than wrap: a wrapped epoch would let a stale record + // claim presence. Unreachable in practice (2^64 drains of one list). + self.epoch = self.epoch.checked_add(1).expect("waiter list epoch overflow"); + for identity in self.entries.drain(..).filter_map(|w| w.upgrade()) { + identity.waker.wake_by_ref(); } } } @@ -215,19 +381,18 @@ impl Park { /// (the usual `ready!` on a nested poll) still leaves its registrations live. /// There is no second call to forget. /// - /// The held waiter is reused when it would wake the same task *and* has no live - /// list registrations (the usual case after a wakeup, which drains every entry), - /// so a steady-state park allocates nothing. Otherwise it is retired for a fresh - /// one: a still-registered waiter must not be registered again, because - /// [`WaiterList`] reclaims a slot only once its `Arc` dies, so reusing one with - /// live entries would stack duplicates the list could never collect. + /// The held waiter is reused whenever it would wake the same task, so a + /// steady-state park allocates nothing. Live registrations are no obstacle: a poll + /// that waits on several lists is re-woken by one while still parked on the rest, + /// and re-registering there is a no-op because [`WaiterList::register`] is + /// idempotent. Only a waker for a *different* task retires the waiter, dropping + /// its registrations so the lists can reclaim those slots. pub fn hold(&mut self, cx: &Context<'_>) -> &Waiter { - let reuse = self.0.as_ref().is_some_and(|waiter| { - cx.waker().will_wake(&waiter.waker) && waiter.shared.get().is_none_or(|shared| Arc::weak_count(shared) == 0) - }); + let reuse = self + .0 + .as_ref() + .is_some_and(|waiter| cx.waker().will_wake(&waiter.waker)); if !reuse { - // The outgoing waiter drops here, killing its registrations so the lists - // can reclaim those slots. self.0 = Some(Waiter::new(cx.waker().clone())); } self.0.as_ref().unwrap() @@ -729,30 +894,158 @@ mod tests { } #[test] - fn park_reuses_a_drained_waiter_and_retires_a_registered_one() { + fn park_reuses_a_still_registered_waiter() { let waker = Waker::noop().clone(); let cx = Context::from_waker(&waker); let mut park = Park::default(); let mut list = WaiterList::new(); - // Poll 1 parks with a live registration. Hold the Arc so a pointer comparison - // can't alias a recycled allocation. + // Poll 1 parks with a live registration. let waiter = park.hold(&cx); waiter.register(&mut list); let first = waiter.shared().clone(); - // Poll 2 with that registration still live must retire the waiter: reusing it - // would stack a duplicate entry the list could never reclaim. + // Poll 2 with that registration still live (a wake from some other list, say) + // reuses the waiter without allocating, and re-registering it dedups rather + // than stacking a duplicate. let waiter = park.hold(&cx); - assert!(!Arc::ptr_eq(&first, waiter.shared()), "a registered waiter was reused"); + assert!( + Arc::ptr_eq(&first, waiter.shared()), + "a still-registered waiter was retired" + ); + waiter.register(&mut list); + assert_eq!(list.entries.len(), 1, "re-registration stacked a duplicate"); + + // The wake drains the list; poll 3 reuses the same waiter and registers anew. + list.wake(); + let waiter = park.hold(&cx); + assert!(Arc::ptr_eq(&first, waiter.shared()), "a drained waiter was not reused"); + waiter.register(&mut list); + assert_eq!(list.entries.len(), 1); + assert!(list.entries[0].strong_count() > 0, "the registration must stay live"); + } + + #[test] + fn register_is_idempotent_per_identity() { + let mut list = WaiterList::new(); + + // Repeat registrations of one identity, directly or via a clone, are one entry. + let waiter = Waiter::new(Waker::noop().clone()); waiter.register(&mut list); - let second = waiter.shared().clone(); + waiter.register(&mut list); + waiter.clone().register(&mut list); + assert_eq!(list.entries.len(), 1, "one identity must occupy one slot"); + + // A distinct waiter is a distinct entry, even for the same task. + let other = Waiter::new(Waker::noop().clone()); + other.register(&mut list); + assert_eq!(list.entries.len(), 2); + } + + #[test] + fn dedup_survives_record_eviction() { + // More lists than the identity keeps records for: the evicted ones fall back + // to the scan, which must still find the registration. + let waiter = Waiter::new(Waker::noop().clone()); + let mut lists: Vec<_> = (0..2 * RECORDED_LISTS).map(|_| WaiterList::new()).collect(); + for list in &mut lists { + waiter.register(list); + } + + // Re-registering must not stack anywhere, recorded or evicted alike. + for list in lists.iter_mut().rev() { + waiter.register(list); + assert_eq!(list.entries.len(), 1, "re-registration stacked a duplicate"); + } + } - // The wake drains the list, so poll 3 reuses: same task, nothing left to - // duplicate, and no allocation. + #[test] + fn a_drain_invalidates_the_record() { + let waiter = Waiter::new(Waker::noop().clone()); + let mut list = WaiterList::new(); + + // A wake drains the registration; the stale record must not convince the + // next register that it is still present, or its wakeup is lost. + waiter.register(&mut list); list.wake(); + assert_eq!(list.entries.len(), 0); + waiter.register(&mut list); + assert_eq!(list.entries.len(), 1); + assert!(list.entries[0].strong_count() > 0, "the registration must be live"); + + // Same for a drain via take. + let taken = list.take(); + assert_eq!(list.entries.len(), 0); + drop(taken); + waiter.register(&mut list); + assert_eq!(list.entries.len(), 1); + assert!(list.entries[0].strong_count() > 0, "the registration must be live"); + } + + #[test] + fn a_taken_snapshot_tracks_nothing() { + let waiter = Waiter::new(Waker::noop().clone()); + let mut list = WaiterList::new(); + waiter.register(&mut list); + + // The snapshot inherits the entries but no identity of its own, so a + // register against it must settle membership by scan: the entry moved with + // the snapshot, so this dedups rather than stacking. + let mut taken = list.take(); + waiter.register(&mut taken); + assert_eq!(taken.entries.len(), 1, "the snapshot register stacked a duplicate"); + + // And a second waiter still gets in; untracked means unproven, not closed. + let other = Waiter::new(Waker::noop().clone()); + other.register(&mut taken); + assert_eq!(taken.entries.len(), 2); + } + + #[test] + fn abandoned_waiters_do_not_grow_the_list() { + let mut list = WaiterList::new(); + + // Register-and-drop, over and over, without a single wake. The rotating probe + // must keep reclaiming the dead slots, or the list grows by one per waiter. + for _ in 0..1_000 { + Waiter::new(Waker::noop().clone()).register(&mut list); + } + assert!(list.entries.len() <= 2, "the list grew to {}", list.entries.len()); + } + + /// The steady state this shape exists for: a poll waiting on several lists is + /// re-woken by one while still parked on the others, and the re-poll must not + /// allocate a new waiter or stack duplicate registrations. + #[test] + fn a_multi_list_poll_reuses_across_a_partial_wake() { + let waker = Waker::noop().clone(); + let cx = Context::from_waker(&waker); + let mut park = Park::default(); + let mut first = WaiterList::new(); + let mut second = WaiterList::new(); + + // Poll 1 waits on both lists. + let waiter = park.hold(&cx); + waiter.register(&mut first); + waiter.register(&mut second); + let shared = waiter.shared().clone(); + + // One list wakes; the other still holds a live registration. + first.wake(); + + // Poll 2 reuses the waiter and re-registers with both: an append into the + // drained list, a no-op in the still-parked one. let waiter = park.hold(&cx); - assert!(Arc::ptr_eq(&second, waiter.shared()), "a drained waiter was not reused"); + assert!( + Arc::ptr_eq(&shared, waiter.shared()), + "the waiter was retired mid-operation" + ); + waiter.register(&mut first); + waiter.register(&mut second); + + assert_eq!(first.entries.len(), 1); + assert_eq!(second.entries.len(), 1, "re-registration stacked a duplicate"); + assert!(second.entries[0].strong_count() > 0, "the registration must stay live"); } #[test] diff --git a/rs/moq-json/Cargo.toml b/rs/moq-json/Cargo.toml index 0a7842cb3dc..4acc4299c7b 100644 --- a/rs/moq-json/Cargo.toml +++ b/rs/moq-json/Cargo.toml @@ -25,7 +25,7 @@ thiserror = "2" tracing = "0.1" [dev-dependencies] -criterion = "0.8" +criterion = { workspace = true } [[bench]] name = "codec" diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index 939cdab2a62..0452a47bdeb 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -28,7 +28,7 @@ web-async = { workspace = true } web-transport-trait = { workspace = true } [dev-dependencies] -criterion = "0.8" +criterion = { workspace = true } serde_json = "1" # test-util (tokio::time::pause/advance) is test-only and is NOT supported on # wasm, so it must not leak into the normal dependency feature set. From 2ce48394b9a90df30f0e861d441e5ba48ddbc4a2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 09:55:27 -0700 Subject: [PATCH 17/29] docs(relay): spell out read-only and write-only tokens (#2912) Co-authored-by: Claude --- doc/bin/relay/auth.md | 40 ++++++++++++++++++++++++++++++++++------ doc/lib/js/@moq/token.md | 4 ++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index 9d4831fc249..4feb2b77baa 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -152,8 +152,8 @@ The JWT payload contains these claims: | Claim | Description | |-------|-------------| | `root` | Base path for publish/subscribe permissions. Optional, defaulting to the top-level path | -| `put` | Suffix, or list of suffixes, appended to root for publish permission | -| `get` | Suffix, or list of suffixes, appended to root for subscribe permission | +| `put` | Suffix, or list of suffixes, appended to root for publish permission. Omit for a token that cannot publish | +| `get` | Suffix, or list of suffixes, appended to root for subscribe permission. Omit for a token that cannot subscribe | | `exp` | Expiration time (Unix timestamp) | | `iat` | Issued-at time (Unix timestamp) | @@ -170,16 +170,44 @@ Full publish path = root + "/" + put Full subscribe path = root + "/" + get ``` -An empty suffix (`""`) allows access to anything under the root. Suffixes match on -path boundaries, so `foo` grants `foo` and `foo/bar` but never `foobar`. +Each claim is one suffix or a list of them, and the two kinds of empty mean +opposite things: -**Examples:** +- **The empty suffix** (`""`, or `[""]`) grants everything under the root. +- **An empty list** (`[]`), or the claim omitted entirely, grants nothing in that + role. That is how you mint a read-only token (no `put`) or a write-only one + (no `get`). + +Suffixes match on path boundaries, so `foo` grants `foo` and `foo/bar` but never +`foobar`. A token that grants neither role is useless, and is rejected both when +signing and when verifying. + +**Examples**, written in the list form. A bare string is the same claim, so +`"put": "alice"` and `"put": ["alice"]` produce the same token, and `[]` is the +same as leaving the claim out: | root | put | get | Can publish | Can subscribe | |------|-----|-----|-------------|---------------| | `demo` | `["my-stream"]` | `[""]` | `demo/my-stream` | `demo/*` | | `rooms/123` | `["alice"]` | `[""]` | `rooms/123/alice` | `rooms/123/*` | | `""` | `[""]` | `[""]` | Everything | Everything | +| `""` | `[]` | `[""]` | Nothing | Everything | +| `""` | `[""]` | `[]` | Everything | Nothing | +| `demo` | `[]` | `[""]` | Nothing | `demo/*` | + +On the CLI the difference is simply whether the flag appears at all. `--publish ""` +grants everything under the root; dropping `--publish` grants no publish access: + +```bash +# Read-only: watch anything under demo/, publish nothing. +moq-token sign --key my-key.jwk --root demo --subscribe "" + +# Write-only: publish demo/my-stream, subscribe to nothing. +moq-token sign --key my-key.jwk --root demo --publish my-stream +``` + +The same split applies to a [key scope](#scope-a-key): a key generated with only +`--subscribe` can never sign a token that publishes. ### Connection Path @@ -189,7 +217,7 @@ The client's connection URL path does **not** need to match the token's `root` e - If the connection path is a **parent** of the root (e.g., token root=`demo`, connect to `/`), permissions still apply but are scoped to the token's root. You can only access paths under `demo/`. - If the connection path is **unrelated** to the root (e.g., token root=`demo`, connect to `/other`), the connection is rejected. -The connection is also rejected if the resulting permissions are empty (no publish or subscribe paths remain after scoping). +The connection is also rejected when scoping leaves *both* roles empty, i.e. no publish and no subscribe path survives. Losing just one role is fine: that is what a read-only or write-only token looks like. ### Unified Auth API (`--auth-api`) diff --git a/doc/lib/js/@moq/token.md b/doc/lib/js/@moq/token.md index e0cbe853ba4..8c1080010bc 100644 --- a/doc/lib/js/@moq/token.md +++ b/doc/lib/js/@moq/token.md @@ -35,8 +35,8 @@ For a complete working example covering key loading, signing, and verification, | Claim | Type | Description | |-------|------|-------------| | `root` | string? | Root path for operations, defaulting to the top-level path | -| `put` | `string \| string[]?` | Publishing permission paths, relative to `root` | -| `get` | `string \| string[]?` | Subscription permission paths, relative to `root` | +| `put` | `string \| string[]?` | Publishing permission paths, relative to `root`. `""` or `[""]` is everything under the root; `[]` or omitted grants no publish access, i.e. a read-only token | +| `get` | `string \| string[]?` | Subscription permission paths, relative to `root`. `""` or `[""]` is everything under the root; `[]` or omitted grants no subscribe access, i.e. a write-only token | | `exp` | number? | Expiration timestamp | | `iat` | number? | Issued at timestamp | From 9f9f9fdb0ed7c80972c06c6c01fcb2ad2dd55a10 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 09:57:55 -0700 Subject: [PATCH 18/29] feat(js/net): implement the MoQ Cluster extension for moq-transport (#2910) Co-authored-by: Claude --- js/net/src/connection/accept.ts | 3 +- js/net/src/connection/connect.ts | 3 +- js/net/src/connection/handshake.ts | 28 ++- js/net/src/error.ts | 16 ++ js/net/src/ietf/cluster.test.ts | 229 +++++++++++++++++++++++ js/net/src/ietf/cluster.ts | 249 +++++++++++++++++++++++++ js/net/src/ietf/connection.ts | 32 +++- js/net/src/ietf/index.ts | 1 + js/net/src/ietf/parameters.ts | 30 ++- js/net/src/ietf/publish_namespace.ts | 35 +++- js/net/src/ietf/publisher.test.ts | 47 ++++- js/net/src/ietf/publisher.ts | 30 ++- js/net/src/ietf/subscribe_namespace.ts | 40 +++- js/net/src/ietf/subscriber.test.ts | 158 ++++++++++++++-- js/net/src/ietf/subscriber.ts | 133 ++++++++++++- js/net/src/lite/announce.test.ts | 2 +- js/net/src/lite/announce.ts | 14 +- js/net/src/lite/connection.ts | 2 +- js/net/src/lite/publisher.test.ts | 2 +- js/net/src/lite/publisher.ts | 2 +- js/net/src/lite/setup.test.ts | 2 +- js/net/src/lite/setup.ts | 2 +- js/net/src/lite/subscriber.test.ts | 2 +- js/net/src/lite/subscriber.ts | 2 +- js/net/src/{lite => }/origin.ts | 26 +++ 25 files changed, 1012 insertions(+), 78 deletions(-) create mode 100644 js/net/src/ietf/cluster.test.ts create mode 100644 js/net/src/ietf/cluster.ts rename js/net/src/{lite => }/origin.ts (62%) diff --git a/js/net/src/connection/accept.ts b/js/net/src/connection/accept.ts index 4c9b5ba5e8d..19c4d1ad1c5 100644 --- a/js/net/src/connection/accept.ts +++ b/js/net/src/connection/accept.ts @@ -65,7 +65,7 @@ async function acceptAlpn( version: Ietf.IetfVersion, discovery: boolean, ): Promise { - const { control, solicit } = await exchangeSetup(transport, version, "moq-lite-js"); + const { control, solicit, cluster } = await exchangeSetup(transport, version, "moq-lite-js"); return new Ietf.Connection({ discovery, @@ -74,6 +74,7 @@ async function acceptAlpn( quic: transport, control, solicit, + cluster, // v17+ uses NativeSession which manages its own request IDs; maxRequestId is unused. maxRequestId: 0n, version, diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index 2b792686365..d55022e924c 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -286,7 +286,7 @@ async function handshakeAlpn( version: Ietf.IetfVersion, discovery: boolean, ): Promise { - const { control, solicit } = await exchangeSetup(session, version, "moq-lite-js"); + const { control, solicit, cluster } = await exchangeSetup(session, version, "moq-lite-js"); return new Ietf.Connection({ discovery, @@ -295,6 +295,7 @@ async function handshakeAlpn( quic: session, control, solicit, + cluster, // v17+ uses NativeSession which manages its own request IDs; maxRequestId is unused. maxRequestId: 0n, version, diff --git a/js/net/src/connection/handshake.ts b/js/net/src/connection/handshake.ts index a967c42f54a..7ac6470790f 100644 --- a/js/net/src/connection/handshake.ts +++ b/js/net/src/connection/handshake.ts @@ -1,4 +1,5 @@ import * as Ietf from "../ietf/index.ts"; +import { type Origin, randomOrigin } from "../origin.ts"; import { Reader, Stream, Writer } from "../stream.ts"; /** @@ -7,20 +8,28 @@ import { Reader, Stream, Writer } from "../stream.ts"; * halves run in parallel and the protocol is symmetric, so both `connect` * (client) and `accept` (server) use this same function. * - * Returns the control stream plus whether the peer requires solicitation, which decides - * whether we announce namespaces unprompted (see the MoQ Solicit extension). We declare it - * ourselves on every session: we send SUBSCRIBE_NAMESPACE for each prefix we want, so an - * unsolicited advertisement can tell us nothing we won't have asked for. + * Returns the control stream plus what the peer's SETUP declared: whether it requires + * solicitation, which decides whether we announce namespaces unprompted (see the MoQ Solicit + * extension), and its Hop ID (see the MoQ Cluster extension). We declare both ourselves on + * every session: we send SUBSCRIBE_NAMESPACE for each prefix we want, so an unsolicited + * advertisement can tell us nothing we won't have asked for, and a peer that knows our Hop + * ID can withhold the advertisements that already flowed through us. */ export async function exchangeSetup( transport: WebTransport, version: Ietf.IetfVersion, implementation: string, -): Promise<{ control: Stream; solicit: boolean | undefined }> { +): Promise<{ control: Stream; solicit: boolean | undefined; cluster: Ietf.Cluster.Hops }> { const encoder = new TextEncoder(); const params = new Ietf.SetupOptions(); params.setBytes(Ietf.SetupOption.Implementation, encoder.encode(implementation)); Ietf.solicitIntoSetup(params); + + // One id per session, like the moq-lite connection: nothing in this process forwards + // between sessions, so there is nothing for a shared id to detect. + const self = randomOrigin(); + Ietf.Cluster.intoSetup(params, self, version); + const setupMsg = new Ietf.Setup({ parameters: params }); const [writer, received] = await Promise.all([ @@ -31,6 +40,7 @@ export async function exchangeSetup( return { control: new Stream({ writer, reader: received.reader }), solicit: received.solicit, + cluster: { self, peer: received.cluster }, }; } @@ -46,7 +56,7 @@ async function sendSetup(transport: WebTransport, version: Ietf.IetfVersion, set async function receiveSetup( transport: WebTransport, version: Ietf.IetfVersion, -): Promise<{ reader: Reader; solicit: boolean | undefined }> { +): Promise<{ reader: Reader; solicit: boolean | undefined; cluster: Origin | undefined }> { const uniReader = transport.incomingUnidirectionalStreams.getReader() as ReadableStreamDefaultReader< ReadableStream >; @@ -62,5 +72,9 @@ async function receiveSetup( } const setup = await Ietf.Setup.decode(reader, version); - return { reader, solicit: Ietf.solicitFromSetup(setup.parameters) }; + return { + reader, + solicit: Ietf.solicitFromSetup(setup.parameters), + cluster: Ietf.Cluster.fromSetup(setup.parameters, version), + }; } diff --git a/js/net/src/error.ts b/js/net/src/error.ts index 184b1e0ae01..26d39009734 100644 --- a/js/net/src/error.ts +++ b/js/net/src/error.ts @@ -38,6 +38,22 @@ export class RemoteError extends Error { } } +/** + * A peer broke the protocol in a way the spec says must end the session. + * + * Thrown where the violation is detected, rather than handled there: a decoder has no session + * to close. The dispatch that owns the session watches for it and closes, so a nonconforming + * peer cannot repeat the violation on the next stream. + * + * @internal + */ +export class ProtocolViolation extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "ProtocolViolation"; + } +} + /** The WebTransport-shaped fields a stream reset code arrives in. */ type StreamErrorLike = { source?: unknown; streamErrorCode?: unknown }; diff --git a/js/net/src/ietf/cluster.test.ts b/js/net/src/ietf/cluster.test.ts new file mode 100644 index 00000000000..9c429e5a1e7 --- /dev/null +++ b/js/net/src/ietf/cluster.test.ts @@ -0,0 +1,229 @@ +import { expect, test } from "bun:test"; +import { ProtocolViolation } from "../error.ts"; +import { type Origin, OriginSchema, UNKNOWN_ORIGIN } from "../origin.ts"; +import * as Path from "../path.ts"; +import { Reader, Writer } from "../stream.ts"; +import * as Cluster from "./cluster.ts"; +import { Parameters, SetupOption, SetupOptions } from "./parameters.ts"; +import { PublishNamespace } from "./publish_namespace.ts"; +import { SubscribeNamespaceEntry } from "./subscribe_namespace.ts"; +import { type IetfVersion, Version } from "./version.ts"; + +const VERSION = Version.DRAFT_19; + +function origin(id: bigint): Origin { + return OriginSchema.parse(id); +} + +function hops(...ids: bigint[]): Origin[] { + return ids.map(origin); +} + +async function encode }>( + message: T, + version: IetfVersion = VERSION, +): Promise { + const chunks: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ + write(chunk) { + chunks.push(new Uint8Array(chunk)); + }, + }), + version, + ); + await message.encode(writer, version); + writer.close(); + await writer.closed; + + const out = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.byteLength, 0)); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; +} + +function reader(bytes: Uint8Array, version: IetfVersion = VERSION): Reader { + return new Reader(undefined, bytes, version); +} + +test("Cluster: the registry code points and their parity", () => { + // Other implementations key on these, so a typo is invisible against ourselves and + // fatal against anyone else. The parity is load-bearing too: odd values are + // length-prefixed byte strings, even ones bare varints. + expect(SetupOption.RelayHops).toBe(0x40b55n); + expect(SetupOption.RelayCost).toBe(0x40b56n); + expect(SetupOption.RelayHops % 2n).toBe(1n); + expect(SetupOption.RelayCost % 2n).toBe(0n); +}); + +test("Cluster: the hop path carries no count of its own", async () => { + // The entries fill the parameter length; a count would make us unreadable to every + // other implementation. + const params = Cluster.intoParams({ hops: hops(1n, 2n, 3n), cost: 0n }); + expect(Array.from(await encode(params))).toEqual([ + 0x01, // parameter count + 0xc4, + 0x0b, + 0x57, // HOP_PATH (0x40b57), delta encoded from 0 + 0x03, // value length + 0x01, + 0x02, + 0x03, // the hop ids, filling the value + ]); +}); + +test("Cluster: an advertisement round trips through the parameters", async () => { + const advert = { hops: hops(7n, 9n), cost: 5n }; + const encoded = await encode(Cluster.intoParams(advert)); + const decoded = await Parameters.decode(reader(encoded), VERSION); + expect(Cluster.fromParams(decoded)).toEqual(advert); +}); + +test("Cluster: a zero route cost is absent from the wire", async () => { + const free = await encode(Cluster.intoParams({ hops: hops(7n), cost: 0n })); + const priced = await encode(Cluster.intoParams({ hops: hops(7n), cost: 1n })); + expect(free.length).toBeLessThan(priced.length); + + // Absent means 0, so it comes back as one. + const decoded = await Parameters.decode(reader(free), VERSION); + expect(Cluster.fromParams(decoded).cost).toBe(0n); +}); + +test("Cluster: a repeated zero is not a loop", async () => { + const advert = { hops: hops(0n, 5n, 0n), cost: 0n }; + const encoded = await encode(Cluster.intoParams(advert)); + const decoded = await Parameters.decode(reader(encoded), VERSION); + expect(Cluster.fromParams(decoded)).toEqual(advert); +}); + +test("Cluster: a malformed advertisement is a session-fatal violation", () => { + // The draft says a receiver MUST close the session over each of these, so they arrive + // as one error the session dispatch can act on, not a stream-local decode failure. + const missing = new Parameters(); + const empty = new Parameters(); + empty.hopPath = new Uint8Array(); + + for (const params of [missing, empty]) { + expect(() => Cluster.fromParams(params)).toThrow(ProtocolViolation); + } +}); + +test("Cluster: a path that cannot have come from a conforming sender is rejected", () => { + // A non-zero id appearing twice is a loop. + expect(() => Cluster.intoParams({ hops: hops(4n, 8n, 4n), cost: 0n })).toThrow(); + + // The list always has at least one entry: the original publisher. + const empty = new Parameters(); + empty.hopPath = new Uint8Array(); + expect(() => Cluster.fromParams(empty)).toThrow(); + + // A negotiated session that omits HOP_PATH entirely is a protocol violation. + expect(() => Cluster.fromParams(new Parameters())).toThrow(); + + // Entries must exactly fill the length: 300 takes two bytes, so one byte ends + // mid-varint. + const partial = new Parameters(); + partial.hopPath = new Uint8Array([0x81]); + expect(() => Cluster.fromParams(partial)).toThrow(); + + // Longer than we accept, which a real path never is and a loop is. + const long = Array.from({ length: 33 }, (_, i) => origin(BigInt(i + 1))); + expect(() => Cluster.intoParams({ hops: long, cost: 0n })).toThrow(); +}); + +test("Cluster: the setup options round trip", async () => { + for (const declared of [origin(42n), UNKNOWN_ORIGIN]) { + const params = new SetupOptions(); + Cluster.intoSetup(params, declared, VERSION); + + const encoded = await encode(params); + const decoded = await SetupOptions.decode(reader(encoded), VERSION); + expect(Cluster.fromSetup(decoded, VERSION)).toBe(declared); + } + + // A peer that declared nothing did not negotiate. + expect(Cluster.fromSetup(new SetupOptions(), VERSION)).toBeUndefined(); +}); + +test("Cluster: the setup options are skipped before draft-17", () => { + // Draft-14..16 exchange SETUP over the legacy control stream, which this extension + // does not cover; we neither send nor read the options there. + for (const version of [Version.DRAFT_14, Version.DRAFT_15, Version.DRAFT_16]) { + expect(Cluster.supported(version)).toBe(false); + + const params = new SetupOptions(); + Cluster.intoSetup(params, origin(42n), version); + expect(params.getBytes(SetupOption.RelayHops)).toBeUndefined(); + + // Even one that arrived anyway reads as "not negotiated" there. + Cluster.intoSetup(params, origin(42n), Version.DRAFT_19); + expect(Cluster.fromSetup(params, version)).toBeUndefined(); + } +}); + +test("Cluster: we advertise our own id, and only once negotiated", () => { + const self = origin(3n); + + // A peer that declared nothing gets no parameters at all. + expect(Cluster.advertise(undefined)).toBeUndefined(); + expect(Cluster.advertise({ self })).toBeUndefined(); + expect(Cluster.negotiated({ self })).toBe(false); + + // A peer that declared the reserved 0 negotiated, and excludes nothing. + expect(Cluster.negotiated({ self, peer: UNKNOWN_ORIGIN })).toBe(true); + expect(Cluster.advertise({ self, peer: origin(9n) })).toEqual({ hops: [self], cost: 0n }); +}); + +test("Cluster: a loop is detected by any non-zero id", () => { + const advert = { hops: hops(0n, 5n), cost: 0n }; + + // A receiver whose own id is 0 cannot detect loops through itself. + expect(Cluster.loops(advert, UNKNOWN_ORIGIN)).toBe(false); + expect(Cluster.loops(advert, origin(5n))).toBe(true); + expect(Cluster.loops(advert, origin(6n))).toBe(false); +}); + +test("Cluster: PUBLISH_NAMESPACE carries the parameters only once negotiated", async () => { + const advert = { hops: hops(7n, 9n), cost: 5n }; + const msg = new PublishNamespace({ requestId: 1n, trackNamespace: Path.from("alice.hang"), cluster: advert }); + const encoded = await encode(msg); + + const decoded = await PublishNamespace.decode(reader(encoded), VERSION, true); + expect(decoded.trackNamespace).toBe(Path.from("alice.hang")); + expect(decoded.cluster).toEqual(advert); + + // The message always has a Parameters field, so a session that negotiated nothing + // reads the same bytes and ignores what it did not ask for. + expect((await PublishNamespace.decode(reader(encoded), VERSION)).cluster).toBeUndefined(); + + // A negotiated session that omits HOP_PATH is a protocol violation, which the session + // dispatch closes over rather than losing only the stream. + const bare = new PublishNamespace({ requestId: 1n, trackNamespace: Path.from("alice.hang") }); + await expect(PublishNamespace.decode(reader(await encode(bare)), VERSION, true)).rejects.toThrow(ProtocolViolation); +}); + +test("Cluster: NAMESPACE grows a parameters field only once negotiated", async () => { + const advert = { hops: hops(7n), cost: 0n }; + const extended = await encode(new SubscribeNamespaceEntry({ suffix: Path.from("alice.hang"), cluster: advert })); + const base = await encode(new SubscribeNamespaceEntry({ suffix: Path.from("alice.hang") })); + + // The base form has no Parameters field at all, so it is strictly shorter. + expect(base.length).toBeLessThan(extended.length); + + const decoded = await SubscribeNamespaceEntry.decode(reader(extended), VERSION, true); + expect(decoded.suffix).toBe(Path.from("alice.hang")); + expect(decoded.cluster).toEqual(advert); + + expect((await SubscribeNamespaceEntry.decode(reader(base), VERSION)).cluster).toBeUndefined(); + + // Reading the extended form as the base one leaves the parameters behind rather than + // silently absorbing them. + await expect(SubscribeNamespaceEntry.decode(reader(extended), VERSION)).rejects.toThrow(); + + // The base form on a negotiated session ends before the parameters the peer owed us, + // which is the same violation as sending them without a HOP_PATH. + await expect(SubscribeNamespaceEntry.decode(reader(base), VERSION, true)).rejects.toThrow(ProtocolViolation); +}); diff --git a/js/net/src/ietf/cluster.ts b/js/net/src/ietf/cluster.ts new file mode 100644 index 00000000000..57deec09bb1 --- /dev/null +++ b/js/net/src/ietf/cluster.ts @@ -0,0 +1,249 @@ +/** + * The MoQ Cluster extension (draft-lcurley-moq-cluster-00). + * + * moq-transport carries no routing information, so a peer cannot tell whether an + * advertisement already passed through us. This extension adds it: + * + * - each endpoint declares its own Hop ID via the RELAY_HOPS Setup Option, which is also + * what negotiates the extension; + * - every advertisement carries the HOP_PATH it traversed and the accumulated ROUTE_COST + * of that path, as Key-Value-Pair message parameters on PUBLISH_NAMESPACE and NAMESPACE. + * + * These are the semantics moq-lite carries natively on every announcement (see + * `lite/announce.ts`); this module is the moq-transport binding, negotiated on draft-17+ + * only, where SETUP is a Key-Value-Pair block. + * + * We are a leaf, never a relay: the only path we can advertise is our own, so we stamp a + * single-entry HOP_PATH and seed the cost at 0. What we get out of declaring is the other + * direction. A relay that knows our Hop ID withholds the advertisements that already flowed + * through us, so publishing a broadcast no longer announces it back to us, which is what + * moq-lite has always done with its own hop chain. + * + * @module + * @internal + */ + +import { ProtocolViolation, reason } from "../error.ts"; +import { MAX_HOPS, type Origin, OriginSchema, UNKNOWN_ORIGIN } from "../origin.ts"; +import type { Reader } from "../stream.ts"; +import * as Varint from "../varint.ts"; +import { Parameters, SetupOption, type SetupOptions } from "./parameters.ts"; +import { type IetfVersion, Version } from "./version.ts"; + +/** + * Whether a version can negotiate the extension. + * + * Draft-17 is the first with the unified SETUP: a bare Key-Value-Pair block that a new + * option slots into without touching the message shape. Earlier drafts exchange SETUP over + * the legacy control stream, which this extension does not cover. + * + * @internal + */ +export function supported(version: IetfVersion): boolean { + return version !== Version.DRAFT_14 && version !== Version.DRAFT_15 && version !== Version.DRAFT_16; +} + +/** + * The Hop IDs this session declared, one per direction. + * + * @internal + */ +export interface Hops { + /** Our own Hop ID, declared in SETUP and stamped onto every advertisement we send. */ + self: Origin; + + /** + * The peer's Hop ID, or `undefined` when it declared no RELAY_HOPS. + * + * This is what decides whether advertisements carry the parameters at all, in both + * directions: a peer that declared nothing has not read ours either, so sending it a + * HOP_PATH would be a protocol violation and expecting one back would reject a legal + * message. A peer that declared the reserved 0 negotiated the extension but withheld its + * identity, so the parameters flow and there is simply nothing to exclude. + */ + peer?: Origin; +} + +/** + * The cluster parameters carried on one advertisement. + * + * Present on every PUBLISH_NAMESPACE and NAMESPACE of a session that negotiated the + * extension, and on none of a session that did not. + * + * @internal + */ +export interface Advert { + /** HOP_PATH: the path this advertisement traversed, publisher first, sender last. */ + hops: Origin[]; + + /** ROUTE_COST: the marginal cost of subscribing via this advertisement. Absent on the + * wire means 0, so an endpoint that prices nothing sends nothing. */ + cost: bigint; +} + +/** + * Read the peer's Hop ID out of its SETUP, or `undefined` when it did not negotiate. + * + * @internal + */ +export function fromSetup(params: SetupOptions, version: IetfVersion): Origin | undefined { + if (!supported(version)) return undefined; + + // RELAY_HOPS is odd, so its value is a length-prefixed byte string holding the sender's + // Hop ID as a single varint. 0 is legal: the peer speaks the extension but withholds + // its identity. + const value = params.getBytes(SetupOption.RelayHops); + if (value === undefined) return undefined; + + const [origin, rest] = Varint.decodeLeadingOnes(value); + if (rest.length !== 0) throw new Error("trailing bytes in RELAY_HOPS"); + return OriginSchema.parse(origin); +} + +/** + * Declare our own Hop ID, which is also what negotiates the extension. + * + * @internal + */ +export function intoSetup(params: SetupOptions, self: Origin, version: IetfVersion) { + if (!supported(version)) return; + params.setBytes(SetupOption.RelayHops, Varint.encodeLeadingOnes(self)); +} + +/** + * Whether the session negotiated the extension, which is what decides whether an + * advertisement carries the parameters in either direction. + * + * @internal + */ +export function negotiated(ids: Hops | undefined): boolean { + return ids?.peer !== undefined; +} + +/** + * The advertisement to send for one of our own broadcasts, or `undefined` on a session + * that negotiated nothing. + * + * The sender's own Hop ID is always the last entry, so a receiver reconstructs the full + * path without the sender restating it. We originate everything we advertise, so the path + * is just us and the cost is 0: a subscription costs the peer nothing but the transfer, + * since we are already producing the content. + * + * @internal + */ +export function advertise(ids: Hops | undefined): Advert | undefined { + if (ids === undefined || ids.peer === undefined) return undefined; + return { hops: [ids.self], cost: 0n }; +} + +/** + * Whether this advertisement looped back through `self`. + * + * A receiver discards such an advertisement: forwarding it would extend the loop, and + * subscribing through it would route the receiver back to itself. Hop ID 0 identifies + * nothing, so a receiver that withheld its own identity detects nothing. + * + * @internal + */ +export function loops(advert: Advert, self: Origin): boolean { + return self !== UNKNOWN_ORIGIN && advert.hops.includes(self); +} + +/** + * Write an advertisement's parameters, ready to encode onto the wire. + * + * @internal + */ +export function intoParams(advert: Advert): Parameters { + validate(advert.hops); + + const params = new Parameters(); + + // The entries fill the value, so they are written with no count of their own and framed + // by the parameter's own length prefix. + const hops = advert.hops.map((hop) => Varint.encodeLeadingOnes(hop)); + const value = new Uint8Array(hops.reduce((total, hop) => total + hop.length, 0)); + let offset = 0; + for (const hop of hops) { + value.set(hop, offset); + offset += hop.length; + } + params.hopPath = value; + + // Absent means 0, so a free path sends nothing. + if (advert.cost !== 0n) params.routeCost = advert.cost; + + return params; +} + +/** + * Read the parameters a negotiated session puts on every advertisement. + * + * The parameter block is mandatory there, so a message that ends before it (the base form) + * or one whose block does not parse is the peer's violation, same as a block that parses but + * carries no HOP_PATH. + * + * @internal + */ +export async function decodeParams(r: Reader, version: IetfVersion): Promise { + let params: Parameters; + try { + params = await Parameters.decode(r, version); + } catch (err) { + throw new ProtocolViolation(reason(err), { cause: err }); + } + + return fromParams(params); +} + +/** + * Read an advertisement out of a decoded parameter block. + * + * Every way this fails is one the draft says to close the session over: a negotiated session + * that omits HOP_PATH, entries that do not exactly fill the parameter, an empty list, or a + * non-zero Hop ID appearing twice. So they all surface as one {@link ProtocolViolation}, + * which the session dispatch acts on. + * + * @internal + */ +export function fromParams(params: Parameters): Advert { + try { + const value = params.hopPath; + if (value === undefined) throw new Error("advertisement is missing HOP_PATH"); + + const hops: Origin[] = []; + let rest = value; + while (rest.length > 0) { + // A short read here means the entries did not exactly fill the length. + const [hop, remain] = Varint.decodeLeadingOnes(rest); + hops.push(OriginSchema.parse(hop)); + rest = remain; + + // Bail before the buffer does: a hostile length would otherwise cost us one + // allocation per entry all the way to 64KB of parameter value. + if (hops.length > MAX_HOPS) throw new Error(`hop count exceeds maximum ${MAX_HOPS}`); + } + + validate(hops); + return { hops, cost: params.routeCost ?? 0n }; + } catch (err) { + throw new ProtocolViolation(reason(err), { cause: err }); + } +} + +/** + * Reject a path that cannot have come from a conforming sender: an empty list, one longer + * than we accept, or a non-zero Hop ID appearing twice (a loop). Duplicate zeros are legal, + * since 0 identifies nothing. + */ +function validate(hops: Origin[]) { + if (hops.length === 0) throw new Error("hop path is empty"); + if (hops.length > MAX_HOPS) throw new Error(`hop count ${hops.length} exceeds maximum ${MAX_HOPS}`); + + // MAX_HOPS is 32, so the quadratic scan is cheaper than allocating a set. + for (let i = 0; i < hops.length; i++) { + const hop = hops[i]; + if (hop === UNKNOWN_ORIGIN) continue; + if (hops.indexOf(hop, i + 1) !== -1) throw new Error(`hop ${hop} appears twice in the hop path`); + } +} diff --git a/js/net/src/ietf/connection.ts b/js/net/src/ietf/connection.ts index 138408d9d5e..e4b2a0d9ed7 100644 --- a/js/net/src/ietf/connection.ts +++ b/js/net/src/ietf/connection.ts @@ -4,9 +4,11 @@ import type * as broadcast from "../broadcast.ts"; import type { Established } from "../connection/established.ts"; import { type Probe, type Stats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; +import { ProtocolViolation } from "../error.ts"; import * as Path from "../path.ts"; import { type Reader, Readers, type Stream } from "../stream.ts"; import { ControlStreamAdapter, NativeSession, type Session } from "./adapter.ts"; +import * as Cluster from "./cluster.ts"; import { GoAway } from "./goaway.ts"; import { Group } from "./object.ts"; import { Publish } from "./publish.ts"; @@ -54,6 +56,9 @@ export class Connection implements Established { // What the peer declared about being solicited; see {@link Ietf.solicitFromSetup}. #solicit: boolean | undefined; + // The Hop IDs this session declared; see {@link Cluster}. + #cluster?: Cluster.Hops; + // Just to avoid logging when `close()` is called. #closed = false; @@ -65,6 +70,7 @@ export class Connection implements Established { * @param maxRequestId - The initial max request ID * @param version - The negotiated protocol version * @param solicit - What the peer's SETUP declared (undefined when it declared nothing) + * @param cluster - The Hop IDs the SETUP exchange settled, on the versions that negotiate them * * @internal */ @@ -77,6 +83,7 @@ export class Connection implements Established { client, discovery = true, solicit, + cluster, }: { url: URL; quic: WebTransport; @@ -91,6 +98,11 @@ export class Connection implements Established { * nothing, which is the one case where announcing at us unasked is not a bug. */ solicit?: boolean; + /** + * The Hop IDs this session declared (MoQ Cluster). `undefined` on a version that + * cannot negotiate the extension, as is a `peer` the peer never declared. + */ + cluster?: Cluster.Hops; }) { this.url = url; this.discovery = discovery; @@ -113,9 +125,15 @@ export class Connection implements Established { }); } - this.#publisher = new Publisher(this.#quic, this.#session, solicit ?? false); + this.#publisher = new Publisher({ + quic: this.#quic, + session: this.#session, + requiresSolicitation: solicit ?? false, + cluster, + }); this.#solicit = solicit; - this.#subscriber = new Subscriber(this.#session); + this.#cluster = cluster; + this.#subscriber = new Subscriber({ session: this.#session, cluster }); void this.#run(); } @@ -207,6 +225,10 @@ export class Connection implements Established { void this.#runBidi(stream).catch((err: unknown) => { console.error("error processing bidi stream", err); stream.abort(new Error("bidi stream error")); + + // The peer broke the protocol, so losing the stream is not enough: nothing + // stops it repeating the violation on the next one. + if (err instanceof ProtocolViolation) this.close(); }); } } @@ -251,7 +273,11 @@ export class Connection implements Established { // Subscriber handles incoming notifications case PublishNamespace.id: { - const msg = await PublishNamespace.decode(stream.reader, this.#session.version); + const msg = await PublishNamespace.decode( + stream.reader, + this.#session.version, + Cluster.negotiated(this.#cluster), + ); // We always declare that advertisements to us must be solicited (MoQ // Solicit), and writing the option at all proves the peer implements the diff --git a/js/net/src/ietf/index.ts b/js/net/src/ietf/index.ts index 91d45c9e1e1..9fd3ec84745 100644 --- a/js/net/src/ietf/index.ts +++ b/js/net/src/ietf/index.ts @@ -1,4 +1,5 @@ export * from "./adapter.ts"; +export * as Cluster from "./cluster.ts"; export * from "./connection.ts"; export * from "./control.ts"; export * from "./fetch.ts"; diff --git a/js/net/src/ietf/parameters.ts b/js/net/src/ietf/parameters.ts index 4c16854726e..8a9b7cb25f1 100644 --- a/js/net/src/ietf/parameters.ts +++ b/js/net/src/ietf/parameters.ts @@ -10,7 +10,11 @@ export const SetupOption = { MaxAuthTokenCacheSize: 4n, Authority: 5n, Implementation: 7n, - /// SOLICIT, from the MoQ Solicit extension. See `solicit.ts`. + /** RELAY_HOPS, from the MoQ Cluster extension. See `cluster.ts`. */ + RelayHops: 0x40b55n, + /** RELAY_COST, from the MoQ Cluster extension. See `cluster.ts`. */ + RelayCost: 0x40b56n, + /** SOLICIT, from the MoQ Solicit extension. See `solicit.ts`. */ Solicit: 0x40b5an, } as const; @@ -194,10 +198,14 @@ const MSG_PARAM_PUBLISHER_PRIORITY = 0x0en; const MSG_PARAM_FORWARD = 0x10n; const MSG_PARAM_SUBSCRIBER_PRIORITY = 0x20n; const MSG_PARAM_GROUP_ORDER = 0x22n; +/// ROUTE_COST, from the MoQ Cluster extension. See `cluster.ts`. +const MSG_PARAM_ROUTE_COST = 0x40b58n; // Bytes parameter IDs (odd) const MSG_PARAM_LARGEST_OBJECT = 0x09n; const MSG_PARAM_SUBSCRIPTION_FILTER = 0x21n; +/// HOP_PATH, from the MoQ Cluster extension. See `cluster.ts`. +const MSG_PARAM_HOP_PATH = 0x40b57n; type MessageParamKind = "varint" | "uint8" | "bool" | "location" | "bytes"; type MessageLocation = { groupId: bigint; objectId: bigint }; @@ -207,6 +215,7 @@ function getMessageParamKind(id: bigint): MessageParamKind { case MSG_PARAM_DELIVERY_TIMEOUT: case MSG_PARAM_MAX_CACHE_DURATION: case MSG_PARAM_EXPIRES: + case MSG_PARAM_ROUTE_COST: return "varint"; case MSG_PARAM_PUBLISHER_PRIORITY: case MSG_PARAM_SUBSCRIBER_PRIORITY: @@ -217,6 +226,7 @@ function getMessageParamKind(id: bigint): MessageParamKind { case MSG_PARAM_LARGEST_OBJECT: return "location"; case MSG_PARAM_SUBSCRIPTION_FILTER: + case MSG_PARAM_HOP_PATH: return "bytes"; default: throw new Error(`unknown message parameter id: ${id.toString()}`); @@ -337,6 +347,24 @@ export class Parameters { this.bytes.set(MSG_PARAM_SUBSCRIPTION_FILTER, new Uint8Array([v])); } + /** HOP_PATH: the hop chain an advertisement traversed, as its raw parameter value. */ + get hopPath(): Uint8Array | undefined { + return this.bytes.get(MSG_PARAM_HOP_PATH); + } + + set hopPath(v: Uint8Array) { + this.bytes.set(MSG_PARAM_HOP_PATH, v); + } + + /** ROUTE_COST: the accumulated cost of that path. Absent means 0. */ + get routeCost(): bigint | undefined { + return this.vars.get(MSG_PARAM_ROUTE_COST); + } + + set routeCost(v: bigint) { + this.vars.set(MSG_PARAM_ROUTE_COST, v); + } + async encode(w: Writer, version: IetfVersion) { await w.u53(this.vars.size + this.bytes.size + this.#locations.size); diff --git a/js/net/src/ietf/publish_namespace.ts b/js/net/src/ietf/publish_namespace.ts index 7ac8ad27989..52f9e0119b3 100644 --- a/js/net/src/ietf/publish_namespace.ts +++ b/js/net/src/ietf/publish_namespace.ts @@ -1,5 +1,6 @@ import type * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; +import * as Cluster from "./cluster.ts"; import * as Message from "./message.ts"; import * as Namespace from "./namespace.ts"; import { Parameters } from "./parameters.ts"; @@ -12,9 +13,19 @@ export class PublishNamespace { requestId: bigint; trackNamespace: Path.Valid; - constructor({ requestId, trackNamespace }: { requestId: bigint; trackNamespace: Path.Valid }) { + /** The MoQ Cluster parameters (see {@link Cluster}). Set on a session that negotiated + * the extension and `undefined` on one that did not, which is what decides whether they + * appear on the wire at all. */ + cluster?: Cluster.Advert; + + constructor({ + requestId, + trackNamespace, + cluster, + }: { requestId: bigint; trackNamespace: Path.Valid; cluster?: Cluster.Advert }) { this.requestId = requestId; this.trackNamespace = trackNamespace; + this.cluster = cluster; } async #encode(w: Writer, version: IetfVersion): Promise { @@ -23,23 +34,37 @@ export class PublishNamespace { await w.u62(0n); // required_request_id_delta: only 0 supported until stream-per-request } await Namespace.encode(w, this.trackNamespace); - await new Parameters().encode(w, version); + const params = this.cluster ? Cluster.intoParams(this.cluster) : new Parameters(); + await params.encode(w, version); } async encode(w: Writer, version: IetfVersion): Promise { return Message.encode(w, (wr) => this.#encode(wr, version)); } - static async decode(r: Reader, version: IetfVersion): Promise { - return Message.decode(r, (rd) => PublishNamespace.#decode(rd, version)); + /** + * Decode the message, expecting the cluster parameters when the session negotiated the + * extension. + * + * The negotiation is session state rather than anything in the message, so the caller + * supplies it. A negotiated session that omits HOP_PATH is a protocol violation, which + * surfaces here as a throw. + */ + static async decode(r: Reader, version: IetfVersion, negotiated = false): Promise { + return Message.decode(r, (rd) => PublishNamespace.#decode(rd, version, negotiated)); } - static async #decode(r: Reader, version: IetfVersion): Promise { + static async #decode(r: Reader, version: IetfVersion, negotiated: boolean): Promise { const requestId = await r.u62(); if (version === Version.DRAFT_17) { await r.u62(); // required_request_id_delta } const trackNamespace = await Namespace.decode(r); + if (negotiated) { + const cluster = await Cluster.decodeParams(r, version); + return new PublishNamespace({ requestId, trackNamespace, cluster }); + } + await Parameters.decode(r, version); // ignore parameters return new PublishNamespace({ requestId, trackNamespace }); } diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index 06d2dad6b81..d20894b7bc9 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -1,9 +1,11 @@ import { expect, test } from "bun:test"; import { Producer as BroadcastProducer } from "../broadcast.ts"; import { createMockTransportPair } from "../mock.ts"; +import { type Origin, OriginSchema } from "../origin.ts"; import * as Path from "../path.ts"; import { Stream } from "../stream.ts"; import { NativeSession, type Session } from "./adapter.ts"; +import type * as Cluster from "./cluster.ts"; import { PublishNamespace } from "./publish_namespace.ts"; import { Publisher } from "./publisher.ts"; import { RequestError, RequestOk } from "./request.ts"; @@ -82,9 +84,9 @@ async function declinePublishNamespace(stream: Stream, retryInterval = 1n): Prom } } -function publisher(transport: WebTransport): Publisher { +function publisher(transport: WebTransport, cluster?: Cluster.Hops): Publisher { const session = new NativeSession(transport, VERSION, true); - return new Publisher(transport, session, false); + return new Publisher({ quic: transport, session, requiresSolicitation: false, cluster }); } /** @@ -181,7 +183,7 @@ test("a failed stream open does not kill the announce loop", async () => { }, }; - const pub = new Publisher(pair.server, session, false); + const pub = new Publisher({ quic: pair.server, session, requiresSolicitation: false }); pub.publish(Path.from("first"), new BroadcastProducer()); void pub.runPublishNamespaces(); @@ -226,7 +228,7 @@ test("a namespace refused once is retried without anything else changing", async }, }; - const pub = new Publisher(pair.server, session, false); + const pub = new Publisher({ quic: pair.server, session, requiresSolicitation: false }); pub.publish(Path.from("lonely"), new BroadcastProducer()); void pub.runPublishNamespaces(); @@ -264,7 +266,7 @@ test("a solicited legacy advertisement refused once is retried", async () => { // The peer declared that advertisements to it must be solicited, so this is the loop // that answers its SUBSCRIBE_NAMESPACE. - const pub = new Publisher(pair.server, session, true); + const pub = new Publisher({ quic: pair.server, session, requiresSolicitation: true }); pub.publish(Path.from("lonely"), new BroadcastProducer()); const subscription = await Stream.open(pair.client, { version: Version.DRAFT_15 }); @@ -337,3 +339,38 @@ test("re-announcing a path clears a refusal that forbade retrying", async () => pub.close(); }); + +/** + * A peer that declared a Hop ID gets one on every advertisement: it is what lets the peer + * tell that an advertisement it hears back came from us. A peer that declared nothing has + * not read ours either, so sending it the parameters would be a protocol violation. + */ +test("an advertisement carries our hop id once the peer declared one", async () => { + const self: Origin = OriginSchema.parse(7n); + + for (const peer of [OriginSchema.parse(9n), undefined]) { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const pub = publisher(pair.server, { self, peer }); + pub.publish(Path.from("mine"), new BroadcastProducer()); + void pub.runPublishNamespaces(); + + const stream = await nextStream(pair.client); + if (!stream) throw new Error("no PUBLISH_NAMESPACE for the broadcast"); + expect(await stream.reader.u53()).toBe(PublishNamespace.id); + + if (peer === undefined) { + // Nothing negotiated, so the parameters are absent: reading the message as a + // negotiated one finds no HOP_PATH and rejects. + await expect(PublishNamespace.decode(stream.reader, VERSION, true)).rejects.toThrow(); + } else { + // Our own Hop ID is the last entry, and we originate everything we advertise, + // so it is the only one. The cost is 0: we are already producing the content. + const msg = await PublishNamespace.decode(stream.reader, VERSION, true); + expect(msg.trackNamespace).toBe(Path.from("mine")); + expect(msg.cluster).toEqual({ hops: [self], cost: 0n }); + await acceptPublishNamespace(stream); + } + + pub.close(); + } +}); diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index b226477abf7..5f428697ebb 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -7,6 +7,7 @@ import { type Stream, Writer } from "../stream.ts"; import type { Timescale } from "../time.ts"; import { withTimeout } from "../util/timeout.ts"; import type { Session } from "./adapter.ts"; +import * as Cluster from "./cluster.ts"; import { Frame, Group as GroupMessage } from "./object.ts"; import { fromWire } from "./priority.ts"; import { PublishDone } from "./publish.ts"; @@ -76,22 +77,39 @@ export class Publisher { #session: Session; #requiresSolicitation: boolean; + // What every advertisement carries on a session that negotiated the MoQ Cluster + // extension: a hop path holding our own id, so the peer can tell that what it hears + // back came from us. `undefined` when nothing negotiated it. + #advert?: Cluster.Advert; + // Our published broadcasts. // It's a signal so we can live update any subscribe_namespace streams. #broadcasts = new Signal | undefined>(new Map()); /** * Creates a new Publisher instance. - * @param quic - The WebTransport session (for uni streams) - * @param session - The session abstraction for bidi streams and request IDs - * @param requiresSolicitation - Whether the peer's SETUP asked to be told on request * * @internal */ - constructor(quic: WebTransport, session: Session, requiresSolicitation: boolean) { + constructor({ + quic, + session, + requiresSolicitation, + cluster, + }: { + /** The WebTransport session, for uni streams. */ + quic: WebTransport; + /** The session abstraction for bidi streams and request IDs. */ + session: Session; + /** Whether the peer's SETUP asked to be told on request (MoQ Solicit). */ + requiresSolicitation: boolean; + /** The Hop IDs the SETUP exchange settled (MoQ Cluster). */ + cluster?: Cluster.Hops; + }) { this.#quic = quic; this.#session = session; this.#requiresSolicitation = requiresSolicitation; + this.#advert = Cluster.advertise(cluster); } /** @@ -327,7 +345,7 @@ export class Publisher { } await stream.writer.u53(SubscribeNamespaceEntry.id); - await new SubscribeNamespaceEntry({ suffix }).encode(stream.writer, version); + await new SubscribeNamespaceEntry({ suffix, cluster: this.#advert }).encode(stream.writer, version); return true; }; const withdraw = async (suffix: Path.Valid) => { @@ -566,7 +584,7 @@ export class Publisher { await withTimeout( (async () => { await stream.writer.u53(PublishNamespace.id); - const msg = new PublishNamespace({ requestId, trackNamespace: path }); + const msg = new PublishNamespace({ requestId, trackNamespace: path, cluster: this.#advert }); await msg.encode(stream.writer, this.#session.version); // Read response (RequestOk and PublishNamespaceOk share 0x07) diff --git a/js/net/src/ietf/subscribe_namespace.ts b/js/net/src/ietf/subscribe_namespace.ts index a5d294ee72e..6d7944726f6 100644 --- a/js/net/src/ietf/subscribe_namespace.ts +++ b/js/net/src/ietf/subscribe_namespace.ts @@ -1,5 +1,6 @@ import type * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; +import * as Cluster from "./cluster.ts"; import * as Message from "./message.ts"; import * as Namespace from "./namespace.ts"; import { Parameters } from "./parameters.ts"; @@ -235,31 +236,52 @@ export class UnsubscribeNamespace { } } -/// NAMESPACE message (0x08) — v16 only, sent on SUBSCRIBE_NAMESPACE bidi stream +/** + * NAMESPACE message (0x08), v16+, sent on the SUBSCRIBE_NAMESPACE bidi stream. + * + * The base message carries only the suffix. A session that negotiated the MoQ Cluster + * extension uses the extended form instead, which appends a Parameters field so the + * advertisement can carry its HOP_PATH and ROUTE_COST (see `cluster.ts`). + */ export class SubscribeNamespaceEntry { static id = 0x08; suffix: Path.Valid; - constructor({ suffix }: { suffix: Path.Valid }) { + /** + * The MoQ Cluster parameters. Set selects the extended form, `undefined` the base one. + * An endpoint must not append them on a session that did not negotiate. + */ + cluster?: Cluster.Advert; + + constructor({ suffix, cluster }: { suffix: Path.Valid; cluster?: Cluster.Advert }) { this.suffix = suffix; + this.cluster = cluster; } - async #encode(w: Writer): Promise { + async #encode(w: Writer, version: IetfVersion): Promise { await Namespace.encode(w, this.suffix); + // The base form has no Parameters field at all, so there is nothing to write. + if (this.cluster) await Cluster.intoParams(this.cluster).encode(w, version); } - async encode(w: Writer, _version: IetfVersion): Promise { - return Message.encode(w, this.#encode.bind(this)); + async encode(w: Writer, version: IetfVersion): Promise { + return Message.encode(w, (wr) => this.#encode(wr, version)); } - static async decode(r: Reader, _version: IetfVersion): Promise { - return Message.decode(r, SubscribeNamespaceEntry.#decode); + /** + * Decode the message, expecting the extended form when the session negotiated the MoQ + * Cluster extension. See {@link PublishNamespace.decode}. + */ + static async decode(r: Reader, version: IetfVersion, negotiated = false): Promise { + return Message.decode(r, (rd) => SubscribeNamespaceEntry.#decode(rd, version, negotiated)); } - static async #decode(r: Reader): Promise { + static async #decode(r: Reader, version: IetfVersion, negotiated: boolean): Promise { const suffix = await Namespace.decode(r); - return new SubscribeNamespaceEntry({ suffix }); + if (!negotiated) return new SubscribeNamespaceEntry({ suffix }); + + return new SubscribeNamespaceEntry({ suffix, cluster: await Cluster.decodeParams(r, version) }); } } diff --git a/js/net/src/ietf/subscriber.test.ts b/js/net/src/ietf/subscriber.test.ts index 768713672ea..162e5e3d89e 100644 --- a/js/net/src/ietf/subscriber.test.ts +++ b/js/net/src/ietf/subscriber.test.ts @@ -1,11 +1,13 @@ import { expect, test } from "bun:test"; import type * as announce from "../announced.ts"; import { createMockTransportPair } from "../mock.ts"; +import { type Origin, OriginSchema } from "../origin.ts"; import * as Path from "../path.ts"; import { Stream } from "../stream.ts"; import { NativeSession } from "./adapter.ts"; +import type * as Cluster from "./cluster.ts"; import { PublishNamespace } from "./publish_namespace.ts"; -import { RequestOk } from "./request.ts"; +import { RequestError, RequestOk } from "./request.ts"; import { SubscribeNamespace, SubscribeNamespaceEntry, SubscribeNamespaceEntryDone } from "./subscribe_namespace.ts"; import { Subscriber } from "./subscriber.ts"; import { ALPN, Version } from "./version.ts"; @@ -51,7 +53,7 @@ test("every peer is asked", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); subscriber.announced(Path.empty()); expect(await nextStream(pair.client)).toBeDefined(); @@ -64,7 +66,7 @@ test("every peer is asked", async () => { test("an unsolicited announcement lands", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); const announced = subscriber.announced(Path.empty()); @@ -109,9 +111,9 @@ async function acceptSubscribeNamespace(transport: WebTransport): Promise { +async function inlineNamespace(stream: Stream, path: Path.Valid, cluster?: Cluster.Advert): Promise { await stream.writer.u53(SubscribeNamespaceEntry.id); - await new SubscribeNamespaceEntry({ suffix: path }).encode(stream.writer, VERSION); + await new SubscribeNamespaceEntry({ suffix: path, cluster }).encode(stream.writer, VERSION); } /** @@ -119,8 +121,8 @@ async function inlineNamespace(stream: Stream, path: Path.Valid): Promise * before it on the same stream have been read. Announcing an already-announced path is * silent by design, so it cannot be waited on directly. */ -async function syncInline(stream: Stream, announced: announce.Consumer): Promise { - await inlineNamespace(stream, Path.from("sentinel")); +async function syncInline(stream: Stream, announced: announce.Consumer, cluster?: Cluster.Advert): Promise { + await inlineNamespace(stream, Path.from("sentinel"), cluster); expect(await announced.next()).toMatchObject({ path: Path.from("sentinel"), active: true }); } @@ -135,7 +137,7 @@ async function syncInline(stream: Stream, announced: announce.Consumer): Promise test("an announcement survives the first of its two sources ending", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); const announced = subscriber.announced(Path.empty()); const subscription = await acceptSubscribeNamespace(pair.client); @@ -172,7 +174,7 @@ test("an announcement survives the first of its two sources ending", async () => test("an announcement ends once its last source does", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); const announced = subscriber.announced(Path.empty()); const subscription = await acceptSubscribeNamespace(pair.client); @@ -208,7 +210,7 @@ test("an announcement ends once its last source does", async () => { test("a subscription that dies releases what it advertised", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); // Two feeds, so one outlives the stream that carries the advertisement. const doomed = subscriber.announced(Path.empty()); @@ -236,7 +238,7 @@ test("a subscription that dies releases what it advertised", async () => { test("a duplicate legacy publish_namespace is still refused", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, Version.DRAFT_15, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); const announced = subscriber.announced(Path.empty()); @@ -272,7 +274,7 @@ test("a duplicate legacy publish_namespace is still refused", async () => { test("an entry buffered past a consumer close does not pin the path", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); const announced = subscriber.announced(Path.empty()); const subscription = await acceptSubscribeNamespace(pair.client); @@ -303,7 +305,7 @@ test("an entry buffered past a consumer close does not pin the path", async () = test("concurrent legacy publish_namespace requests take one reference", async () => { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, Version.DRAFT_15, true); - const subscriber = new Subscriber(session); + const subscriber = new Subscriber({ session }); const announced = subscriber.announced(Path.empty()); @@ -330,3 +332,133 @@ test("concurrent legacy publish_namespace requests take one reference", async () expect(await announced.next()).toMatchObject({ path: Path.from("raced"), active: false }); }); + +/** The Hop IDs a cluster-negotiated session declared, ours first. */ +const SELF: Origin = OriginSchema.parse(7n); +const PEER: Origin = OriginSchema.parse(9n); + +/** + * A peer that knows our Hop ID never advertises a path that already ran through us, so + * this is the backstop for one that does not conform: subscribing via such a path would + * route us back to ourselves, so the advertisement is dropped rather than announced. + */ +test("an inline NAMESPACE that looped back through us is dropped", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session, cluster: { self: SELF, peer: PEER } }); + + const announced = subscriber.announced(Path.empty()); + const subscription = await acceptSubscribeNamespace(pair.client); + + // Ours coming back, then someone else's. Only the second is news. + await inlineNamespace(subscription, Path.from("mine"), { hops: [SELF, PEER], cost: 0n }); + await syncInline(subscription, announced, { hops: [PEER], cost: 0n }); +}); + +/** + * An advertisement is updated in place, by re-sending it on the stream that carries it. One + * that now loops back has re-parented onto a route we cannot subscribe over, so the path is + * gone even though the message says active. + */ +test("an inline NAMESPACE that starts looping back is retracted", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session, cluster: { self: SELF, peer: PEER } }); + + const announced = subscriber.announced(Path.empty()); + const subscription = await acceptSubscribeNamespace(pair.client); + + await inlineNamespace(subscription, Path.from("theirs"), { hops: [PEER], cost: 0n }); + expect(await announced.next()).toMatchObject({ path: Path.from("theirs"), active: true }); + + await inlineNamespace(subscription, Path.from("theirs"), { hops: [SELF, PEER], cost: 0n }); + expect(await announced.next()).toMatchObject({ path: Path.from("theirs"), active: false }); +}); + +/** The same rule on the other kind of advertisement, which is a request we can refuse. */ +test("a PUBLISH_NAMESPACE that looped back through us is refused", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session, cluster: { self: SELF, peer: PEER } }); + + const announced = subscriber.announced(Path.empty()); + const subscription = await acceptSubscribeNamespace(pair.client); + + const request = await Stream.open(pair.server, { version: VERSION }); + await subscriber.runPublishNamespace( + new PublishNamespace({ + requestId: 0n, + trackNamespace: Path.from("mine"), + cluster: { hops: [SELF, PEER], cost: 0n }, + }), + request, + ); + + const peer = await nextStream(pair.client); + if (!peer) throw new Error("no PUBLISH_NAMESPACE stream"); + expect(await peer.reader.u53()).toBe(RequestError.id); + const err = await RequestError.decode(peer.reader, VERSION); + expect(err.errorCode).toBe(400); + + // Refused, so nothing was announced: the sentinel is the first thing a consumer hears. + await syncInline(subscription, announced, { hops: [PEER], cost: 0n }); +}); + +/** + * The cluster draft requires closing the session over an advertisement missing its HOP_PATH, + * not just the stream that carried it: a peer that broke the protocol once would otherwise + * repeat it on the next SUBSCRIBE_NAMESPACE. + */ +test("a NAMESPACE missing its hop path closes the session", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session, cluster: { self: SELF, peer: PEER } }); + + subscriber.announced(Path.empty()); + const subscription = await acceptSubscribeNamespace(pair.client); + + // The base form, which a negotiated session must never send. + await subscription.writer.u53(SubscribeNamespaceEntry.id); + await new SubscribeNamespaceEntry({ suffix: Path.from("nohops") }).encode(subscription.writer, VERSION); + + await Promise.race([ + pair.server.closed, + new Promise((_resolve, reject) => setTimeout(() => reject(new Error("session stayed up")), STREAM_WAIT)), + ]); +}); + +/** + * An advertisement is updated in place, by repeating PUBLISH_NAMESPACE on the stream that + * carries it. One re-parented onto a route through us is unusable, so the announcement has + * to go even though the stream stays open. + */ +test("a PUBLISH_NAMESPACE update that starts looping back is detached", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session, cluster: { self: SELF, peer: PEER } }); + + const announced = subscriber.announced(Path.empty()); + await acceptSubscribeNamespace(pair.client); + + const advert = (hops: Origin[]) => + new PublishNamespace({ + requestId: 0n, + trackNamespace: Path.from("theirs"), + cluster: { hops, cost: 0n }, + }); + + const request = await Stream.open(pair.server, { version: VERSION }); + const handler = subscriber.runPublishNamespace(advert([PEER]), request); + expect(await announced.next()).toMatchObject({ path: Path.from("theirs"), active: true }); + + // The peer re-parents the namespace onto a route that runs back through us. + const peer = await nextStream(pair.client); + if (!peer) throw new Error("no PUBLISH_NAMESPACE stream"); + await peer.writer.u53(PublishNamespace.id); + await advert([SELF, PEER]).encode(peer.writer, VERSION); + + expect(await announced.next()).toMatchObject({ path: Path.from("theirs"), active: false }); + + peer.close(); + await handler; +}); diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index ad0e630b362..66e6b9cc65e 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -1,7 +1,7 @@ import * as announce from "../announced.ts"; import * as broadcast from "../broadcast.ts"; import { BroadcastCache } from "../consume.ts"; -import { error, reason } from "../error.ts"; +import { error, ProtocolViolation, reason } from "../error.ts"; import * as netGroup from "../group.ts"; import * as Path from "../path.ts"; import type { Reader, Stream } from "../stream.ts"; @@ -10,10 +10,16 @@ import type * as track from "../track.ts"; import { withTimeout } from "../util/timeout.ts"; import type { Session } from "./adapter.ts"; import { TrackAliases } from "./aliases.ts"; +import * as Cluster from "./cluster.ts"; import { Frame, type Group as GroupMessage } from "./object.ts"; import { toWire } from "./priority.ts"; import { type Publish, PublishError } from "./publish.ts"; -import { type PublishNamespace, PublishNamespaceError, PublishNamespaceOk } from "./publish_namespace.ts"; +import { + PublishNamespace, + PublishNamespaceDone, + PublishNamespaceError, + PublishNamespaceOk, +} from "./publish_namespace.ts"; import { RequestError, RequestOk } from "./request.ts"; import { Subscribe, SubscribeError, SubscribeOk, Unsubscribe } from "./subscribe.ts"; import { @@ -49,6 +55,11 @@ type SubscribeSetupState = { export class Subscriber { #session: Session; + // The Hop IDs this session declared; see {@link Cluster}. What the peer declared is what + // says whether an advertisement carries a hop path, and ours is what a path looping back + // to us contains. + #cluster?: Cluster.Hops; + // Publisher-chosen aliases used by incoming group streams. #aliases = new TrackAliases(); @@ -79,12 +90,32 @@ export class Subscriber { /** * Creates a new Subscriber instance. - * @param session - The session abstraction for bidi streams and request IDs * * @internal */ - constructor(session: Session) { + constructor({ + session, + cluster, + }: { + /** The session abstraction for bidi streams and request IDs. */ + session: Session; + /** The Hop IDs the SETUP exchange settled (MoQ Cluster). */ + cluster?: Cluster.Hops; + }) { this.#session = session; + this.#cluster = cluster; + } + + /** + * Whether an advertisement is ours coming back: its hop path already ran through us, so + * subscribing via it would route us back to ourselves. + * + * A conforming peer withholds these (it knows our Hop ID), so this is the backstop that + * keeps a mesh working when one member does not. A session that negotiated nothing + * carries no path, and there is nothing to check. + */ + #reflected(advert: Cluster.Advert | undefined): boolean { + return advert !== undefined && this.#cluster !== undefined && Cluster.loops(advert, this.#cluster.self); } /** @@ -221,10 +252,23 @@ export class Subscriber { const msgType = await stream.reader.u53(); if (msgType === SubscribeNamespaceEntry.id) { - const entry = await SubscribeNamespaceEntry.decode(stream.reader, version); + const entry = await SubscribeNamespaceEntry.decode( + stream.reader, + version, + Cluster.negotiated(this.#cluster), + ); if (released) break; const path = Path.join(prefix, entry.suffix); + // A repeat updates the advertisement in place, so one that now + // loops back through us has taken a route we can't subscribe + // over: the path is gone even though the message says active. + if (this.#reflected(entry.cluster)) { + console.debug(`dropping reflected namespace: broadcast=${path}`); + if (live.delete(path)) this.#detachAnnounce(path); + continue; + } + // A repeat updates the advertisement; only the first is news. if (!live.has(path)) { live.add(path); @@ -249,6 +293,15 @@ export class Subscriber { } })(); + // The race below can end through the consumer's close while a message that + // already arrived is still decoding, which leaves the loop with nothing + // awaiting it: a violation decoded after that would land nowhere, and its + // rejection would go unhandled. Idempotent with the catch below, which is + // what runs when the loop is the one that ends the race. + readLoop.catch((err: unknown) => { + if (err instanceof ProtocolViolation) this.#session.close(); + }); + // Wait for either the read loop or the announced to close await Promise.race([readLoop, announced.closed]); @@ -272,6 +325,12 @@ export class Subscriber { const e = error(err); console.warn(`subscribe_namespace error: ${reason(e)}`); + // An advertisement is decoded here rather than in the session dispatch, so this + // is the only place a malformed one surfaces. The cluster draft requires closing + // the session over those, and the stream alone is not enough: the peer would + // just repeat it on the next SUBSCRIBE_NAMESPACE. + if (e instanceof ProtocolViolation) this.#session.close(); + // Abort the stream rather than letting the caller's `finally` close it cleanly. // A rejected namespace subscription is a failure, and a consumer that can't tell // it from "nothing is published under this prefix" waits forever on a broadcast @@ -477,6 +536,21 @@ export class Subscriber { const version = this.#session.version; const path = msg.trackNamespace; + // A path that already ran through us looped back. Refuse it rather than holding an + // advertisement we could never subscribe through. The peer knows our Hop ID, so a + // conforming one never offers it; 0 as the retry interval says not to come back. + if (this.#reflected(msg.cluster)) { + console.debug(`dropping reflected publish_namespace: broadcast=${path}`); + await stream.writer.u53(RequestError.id); + await new RequestError({ + requestId: msg.requestId, + errorCode: 400, + reasonPhrase: "route loops back", + }).encode(stream.writer, version); + stream.close(); + return; + } + // Draft-14/15 key their namespace-scoped messages by name, not request ID, so the // adapter can hold only one request per namespace: a second would overwrite the // first, and the withdrawals would then close the wrong stream and fail to find @@ -533,10 +607,51 @@ export class Subscriber { attached = true; this.#attachAnnounce(path); - // Wait for stream close (= PublishNamespaceDone) - console.debug(`runPublishNamespace: awaiting stream.reader.closed for ${path}`); - await stream.reader.closed; - console.debug(`runPublishNamespace: stream.reader.closed resolved for ${path}`); + // An advertisement is updated in place, by repeating PUBLISH_NAMESPACE on the + // stream that already carries it, so read until the stream ends rather than + // waiting on the close. Nothing else would deliver a re-parented route. + const done = version === Version.DRAFT_16 || legacy; + for (;;) { + if (await stream.reader.done()) break; + + const typeId = await stream.reader.u53(); + if (done && typeId === PublishNamespaceDone.id) { + await PublishNamespaceDone.decode(stream.reader, version); + break; + } + if (typeId !== PublishNamespace.id) { + throw new ProtocolViolation( + `unexpected message on publish_namespace stream: 0x${typeId.toString(16)}`, + ); + } + + const update = await PublishNamespace.decode(stream.reader, version, Cluster.negotiated(this.#cluster)); + + // The stream is the advertisement, so an update on it must name the same + // one. Applying a mismatched update would retarget this path with metadata + // meant for a different request. + if (update.requestId !== msg.requestId || update.trackNamespace !== path) { + throw new ProtocolViolation("publish_namespace update does not match its stream"); + } + + // A path that now runs through us is unusable, so give it back. Keep + // reading: this stream is the advertisement's only channel, so a later + // clean path arrives here or nowhere. + if (this.#reflected(update.cluster)) { + if (attached) { + attached = false; + console.debug(`publish_namespace now loops back, detaching: broadcast=${path}`); + this.#detachAnnounce(path); + } + continue; + } + + // Re-attach: a clean path replaced the reflected one we detached from. + if (!attached) { + attached = true; + this.#attachAnnounce(path); + } + } } finally { if (legacy) this.#legacyRequests.delete(path); diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 528ba520ea0..672606e5c64 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { OriginSchema } from "../origin.ts"; import * as Path from "../path.ts"; import { Reader, Writer } from "../stream.ts"; import { @@ -7,7 +8,6 @@ import { decodeAnnounceBroadcast, encodeAnnounceBroadcast, } from "./announce.ts"; -import { OriginSchema } from "./origin.ts"; import { Version } from "./version.ts"; function concat(chunks: Uint8Array[]): Uint8Array { diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index 767df13ee22..965742b5a1d 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -1,14 +1,9 @@ +import { MAX_HOPS, type Origin, OriginSchema, UNKNOWN_ORIGIN } from "../origin.ts"; import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import * as Message from "./message.ts"; -import { type Origin, OriginSchema } from "./origin.ts"; import { hasAnnounceId, hasAnnounceOk, hasExcludeHop, hasRouteCost, Version } from "./version.ts"; -// Must match the MAX_HOPS in Rust's model/origin.rs. Broadcasts with longer -// hop chains are rejected; this keeps loop-detection bounded and rejects -// pathological announcements across clusters with unbounded forwarding. -export const MAX_HOPS = 32; - // Pre-lite-06 inner status values, carried inside the single ANNOUNCE_BROADCAST body. const STATUS_ENDED = 0; const STATUS_ACTIVE = 1; @@ -77,10 +72,9 @@ async function decodeHops(r: Reader, version: Version): Promise { case Version.DRAFT_03: { const count = await r.u53(); if (count > MAX_HOPS) throw new Error(`hop count ${count} exceeds maximum ${MAX_HOPS}`); - // Lite03 carries only a hop count, not individual ids. Fill with - // the zero placeholder (OriginSchema accepts 0 as valid on-wire). - const placeholder = OriginSchema.parse(0n); - return new Array(count).fill(placeholder); + // Lite03 carries only a hop count, not individual ids, so every entry is + // the reserved "no identity" id. + return new Array(count).fill(UNKNOWN_ORIGIN); } default: { // Lite04+: hop count + individual Origin varints. diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 4611e561759..8e77a928e1d 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -4,13 +4,13 @@ import type * as broadcast from "../broadcast.ts"; import type { Established } from "../connection/established.ts"; import { type Probe, type Stats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; +import { type Origin, randomOrigin } from "../origin.ts"; import * as Path from "../path.ts"; import { type Reader, Readers, Stream, Writer } from "../stream.ts"; import { AnnounceRequest } from "./announce.ts"; import { Fetch } from "./fetch.ts"; import { Goaway } from "./goaway.ts"; import { Group } from "./group.ts"; -import { type Origin, randomOrigin } from "./origin.ts"; import { Publisher } from "./publisher.ts"; import { SessionInfo } from "./session.ts"; import { ProbeLevel, type Role, Setup } from "./setup.ts"; diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 2cc8198911a..d38bd82d690 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -2,11 +2,11 @@ import { expect, test } from "bun:test"; import { Producer as BroadcastProducer } from "../broadcast.ts"; import { Producer as GroupProducer } from "../group.ts"; import { createMockTransportPair } from "../mock.ts"; +import { randomOrigin } from "../origin.ts"; import * as Path from "../path.ts"; import { Reader, Stream } from "../stream.ts"; import { Fetch } from "./fetch.ts"; import { Group as GroupMessage } from "./group.ts"; -import { randomOrigin } from "./origin.ts"; import { sendOrder } from "./priority.ts"; import { Publisher } from "./publisher.ts"; import { decodeSubscribeResponse, Subscribe, SubscribeUpdate } from "./subscribe.ts"; diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index b23e062c151..34f3431f877 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -2,6 +2,7 @@ import { type Dispose, Signal } from "@moq/signals"; import type * as broadcast from "../broadcast.ts"; import { error, reason } from "../error.ts"; import type * as group from "../group.ts"; +import type { Origin } from "../origin.ts"; import * as Path from "../path.ts"; import { type Stream, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; @@ -11,7 +12,6 @@ import { Datagram as DatagramMessage } from "./datagram.ts"; import * as DatagramStream from "./datagram_stream.ts"; import type { Fetch } from "./fetch.ts"; import { Group as GroupMessage } from "./group.ts"; -import type { Origin } from "./origin.ts"; import { Priority, sendOrder } from "./priority.ts"; import { Probe } from "./probe.ts"; import { diff --git a/js/net/src/lite/setup.test.ts b/js/net/src/lite/setup.test.ts index 05ceb0825bf..160e0dac01e 100644 --- a/js/net/src/lite/setup.test.ts +++ b/js/net/src/lite/setup.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "bun:test"; +import { OriginSchema } from "../origin.ts"; import { Reader, Writer } from "../stream.ts"; import * as Varint from "../varint.ts"; -import { OriginSchema } from "./origin.ts"; import { ProbeLevel, Role, Setup } from "./setup.ts"; import { Version } from "./version.ts"; diff --git a/js/net/src/lite/setup.ts b/js/net/src/lite/setup.ts index c7d18574d3f..b231933f569 100644 --- a/js/net/src/lite/setup.ts +++ b/js/net/src/lite/setup.ts @@ -5,10 +5,10 @@ * @module */ +import { type Origin, OriginSchema } from "../origin.ts"; import type { Reader, Writer } from "../stream.ts"; import * as Varint from "../varint.ts"; import * as Message from "./message.ts"; -import { type Origin, OriginSchema } from "./origin.ts"; import { hasSetupStream, type Version } from "./version.ts"; /** Setup Parameter id for the Probe capability level. */ diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 8d6cd99094f..294c8b78b1e 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1,10 +1,10 @@ import { expect, spyOn, test } from "bun:test"; import { Signal } from "@moq/signals"; import type { Probe } from "../connection/stats.ts"; +import { OriginSchema } from "../origin.ts"; import * as Path from "../path.ts"; import { Writer } from "../stream.ts"; import { AnnounceOk, encodeAnnounceBroadcast } from "./announce.ts"; -import { OriginSchema } from "./origin.ts"; import { Subscriber } from "./subscriber.ts"; import { Version } from "./version.ts"; diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 790699a5147..6525f45102c 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -5,6 +5,7 @@ import type { Probe as ProbeStats } from "../connection/stats.ts"; import { BroadcastCache } from "../consume.ts"; import { error, reason } from "../error.ts"; import * as netGroup from "../group.ts"; +import type { Origin } from "../origin.ts"; import * as Path from "../path.ts"; import { type Reader, Stream } from "../stream.ts"; import * as Time from "../time.ts"; @@ -15,7 +16,6 @@ import { Datagram as DatagramMessage } from "./datagram.ts"; import * as DatagramStream from "./datagram_stream.ts"; import { Fetch as FetchMessage } from "./fetch.ts"; import type { Group as GroupMessage } from "./group.ts"; -import type { Origin } from "./origin.ts"; import { sendOrder } from "./priority.ts"; import { Probe } from "./probe.ts"; import { ProbeLevel, type Setup } from "./setup.ts"; diff --git a/js/net/src/lite/origin.ts b/js/net/src/origin.ts similarity index 62% rename from js/net/src/lite/origin.ts rename to js/net/src/origin.ts index ed069b934c1..076ef9fa99b 100644 --- a/js/net/src/lite/origin.ts +++ b/js/net/src/origin.ts @@ -1,3 +1,12 @@ +/** + * Endpoint identity within a hop chain, shared by both wire protocols. + * + * moq-lite carries these natively on every announcement; moq-transport carries them + * via the MoQ Cluster extension (see `ietf/cluster.ts`). Mirrors `Origin` in + * `rs/moq-net`. + * + * @module + */ import * as z from "zod/mini"; /** @@ -15,6 +24,23 @@ export const OriginSchema = z export type Origin = z.infer; +/** + * The reserved id 0, meaning "no identity". + * + * It stands in for an endpoint that never declared one, and any number of endpoints can + * be 0, so it identifies nothing: it is never a loop, never a publisher two chains have + * in common, and never excluded from an advertisement. + */ +export const UNKNOWN_ORIGIN: Origin = OriginSchema.parse(0n); + +/** + * Maximum length of a hop chain. Must match `MAX_HOPS` in Rust's `model/origin.rs`. + * + * Broadcasts with longer chains are rejected, which bounds loop detection and rejects + * pathological announcements across clusters with unbounded forwarding. + */ +export const MAX_HOPS = 32; + /** * Generate a fresh origin with a random non-zero id. * From 7b58d99bdbf3b1a69161833a8c255074a54c2fd5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 09:58:27 -0700 Subject: [PATCH 19/29] fix(moq-net): keep an anonymous publisher's source across a NAMESPACE update (#2908) Co-authored-by: Claude --- drafts/draft-lcurley-moq-cluster.md | 9 +- rs/moq-net/src/ietf/subscriber.rs | 152 ++++++++++++++++++++++++++-- test/wasm/src/main.ts | 9 -- 3 files changed, 147 insertions(+), 23 deletions(-) diff --git a/drafts/draft-lcurley-moq-cluster.md b/drafts/draft-lcurley-moq-cluster.md index 20984e16647..9ce287c6938 100644 --- a/drafts/draft-lcurley-moq-cluster.md +++ b/drafts/draft-lcurley-moq-cluster.md @@ -104,11 +104,11 @@ It is used for an endpoint that did not negotiate this extension, and an endpoin Because any number of endpoints can be 0, it identifies nothing, which constrains all three uses: - **Loop detection**: 0 in a HOP_PATH is never a loop. A receiver whose own Hop ID is 0 cannot detect loops through itself, and MUST NOT discard an advertisement merely because the path contains 0. -- **Origin identity**: an advertisement whose first entry is 0 has an unknown origin. A receiver MUST NOT treat two such advertisements as interchangeable ({{selection}}). +- **Origin identity**: an advertisement whose first entry is 0 has an unknown origin. A receiver MUST NOT treat two such advertisements as interchangeable ({{selection}}). Updating one advertisement is not two ({{updating}}). - **Filtering**: a peer that declared 0 excludes nothing, so the sender applies no filter to that session. Duplicate *non-zero* Hop IDs in one HOP_PATH are a loop; duplicate zeros are not. -Declaring 0 therefore trades loop detection, failover, and update continuity ({{updating}}) for anonymity. +Declaring 0 therefore trades loop detection and failover for anonymity. # Namespace Advertisements {#namespace} @@ -192,7 +192,10 @@ In {{moqt}} an advertisement lives for the lifetime of its stream, so an update An endpoint MUST NOT open a second stream for a namespace it already advertises on this session. Replacement is atomic, so a receiver MUST NOT tear down subscriptions or drop cached state merely because an update arrived. -What it means for existing subscriptions follows the first HOP_PATH entry ({{selection}}): unchanged and non-zero, the content is continuous and subscriptions MAY resume on the new route at a group boundary; changed or 0 ({{zero}}), a different publisher may have taken over and they do not carry over. +What it means for existing subscriptions follows the first HOP_PATH entry ({{selection}}): unchanged, the content is continuous and subscriptions MAY resume on the new route at a group boundary; changed, a different publisher has taken over and they do not carry over. + +This is the one comparison 0 ({{zero}}) does not decide: it identifies nothing, but there is one advertisement here and the stream carrying it is the continuity. +An endpoint whose publisher did change MUST withdraw the advertisement (NAMESPACE_DONE or PUBLISH_NAMESPACE_DONE) and advertise again rather than update in place. The expected case is a ROUTE_COST-only change, which is how a relay signals that it started or stopped carrying the namespace. diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 84a5bd2b32b..5866419c6b3 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -124,7 +124,8 @@ struct Advertised { /// The original publisher: the first entry of HOP_PATH. Two advertisements that /// share a non-zero one carry interchangeable content, so a new path splices in. /// A different one, or `Some(Origin::UNKNOWN)` (which identifies nothing), is a - /// distinct publisher reusing the namespace and must not splice. + /// distinct publisher reusing the namespace and must not splice. That comparison + /// only applies between separate advertisements; see [`Arrival`]. /// /// `None` when the advertisement carried no path, so there is no identity to /// compare and nothing ever replaces the source on identity grounds. That covers @@ -133,6 +134,22 @@ struct Advertised { publisher: Option, } +/// How an advertisement relates to whatever already holds the path. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Arrival { + /// A separate advertisement for a path another one already holds: a + /// PUBLISH_NAMESPACE alongside a NAMESPACE. Nothing links the two but the publisher + /// identity, so only a matching, real one proves they carry the same content. + Separate, + + /// A repeat of the advertisement already holding the path, on the stream that + /// carries it. That stream is the continuity: the peer never retracted the + /// advertisement, so only a *changed* publisher is a new publisher. The expected + /// repeat is a repricing, and a peer that withholds its identity reprices as often + /// as any other. + Repeat, +} + #[derive(Clone)] pub(super) struct Subscriber { session: S, @@ -876,7 +893,7 @@ impl Subscriber { fn start_announce(&mut self, path: PathOwned, advert: Advertised) -> Result { let mut state = self.state.lock(); let existing = state.broadcasts.contains_key(&path); - let producer = self.attach(&mut state, path.clone(), advert)?; + let producer = self.attach(&mut state, path.clone(), advert, Arrival::Separate)?; if existing && let Some(entry) = state.broadcasts.get_mut(&path) { // The path was already attached, so this is one more advertisement for it. // A replacement carries the previous count forward, so the increment still @@ -896,7 +913,7 @@ impl Subscriber { if !state.broadcasts.contains_key(&path) { return Err(Error::NotFound); } - self.attach(&mut state, path, advert)?; + self.attach(&mut state, path, advert, Arrival::Repeat)?; Ok(()) } @@ -904,14 +921,20 @@ impl Subscriber { /// /// The ingress announce stats (announced / announced_bytes) are driven in the model /// by `create_broadcast`'s route transitions below. - fn attach(&self, state: &mut State, path: PathOwned, advert: Advertised) -> Result { + fn attach( + &self, + state: &mut State, + path: PathOwned, + advert: Advertised, + arrival: Arrival, + ) -> Result { let publisher = advert.publisher; - // A different original publisher, or one that identifies nothing, means a - // distinct broadcast may have taken over this namespace. Its content is not - // interchangeable, so detach the old source and attach fresh instead of splicing - // the route in place; cached tracks and subscriptions must not carry over. The - // refcount rides along: the same advertisements still reference this path. + // A different original publisher means a distinct broadcast has taken over this + // namespace. Its content is not interchangeable, so detach the old source and + // attach fresh instead of splicing the route in place; cached tracks and + // subscriptions must not carry over. The refcount rides along: the same + // advertisements still reference this path. // // Both sides must be known for the comparison to mean anything. A session // without the MoQ Cluster extension has no identity on either, so nothing ever @@ -921,8 +944,17 @@ impl Subscriber { let mut carried = None; if let Entry::Occupied(entry) = state.broadcasts.entry(path.clone()) && let (Some(old), Some(new)) = (entry.get().publisher, publisher) - && (old != new || new == crate::Origin::UNKNOWN) - { + && match arrival { + // Two advertisements, so identity is the only link between them and + // UNKNOWN is no link at all: unrelated publishers all present the same + // one. Take the later as replacing the earlier. + Arrival::Separate => old != new || new == crate::Origin::UNKNOWN, + // One advertisement, repeated on the stream that carries it. Continuity + // is the stream, not the identity, so an unchanged UNKNOWN stays put; a + // peer that never named itself would otherwise lose every subscription + // each time it repriced. + Arrival::Repeat => old != new, + } { tracing::debug!(broadcast = %self.origin.absolute(&path), "publisher changed; replacing the source"); carried = Some(entry.get().count); entry.remove().producer.finish(); @@ -2226,6 +2258,104 @@ mod tests { assert!(consumer.get_broadcast("room/host").is_none()); } + /// Regression: a publisher that declares no identity of its own contributes + /// `Origin::UNKNOWN` as the first hop, which identifies nothing. A repeat NAMESPACE + /// is still the same advertisement being repriced (the expected update, and how a + /// relay signals that it started carrying the namespace), so the source and every + /// live subscription on it must survive. Reading the repeat as a new publisher + /// detached the source milliseconds after SUBSCRIBE went out. + #[tokio::test(start_paused = true)] + async fn anonymous_publisher_survives_a_repricing_update() { + let (mut subscriber, origin) = cluster_subscriber(crate::Origin::new(1).unwrap()); + let consumer = origin.consume(); + let path = crate::Path::new("room/host").to_owned(); + // A free link, so the route cost is exactly what the peer advertised. + let peer = cluster::Peer { + origin: Some(crate::Origin::new(9).unwrap()), + cost: Some(0), + }; + let hops = cluster::HopPath::new( + crate::OriginList::try_from(vec![crate::Origin::UNKNOWN, crate::Origin::new(9).unwrap()]).unwrap(), + ); + + let advertised = subscriber + .route( + Some(&cluster::Advert { + hops: hops.clone(), + cost: 2, + }), + &peer, + ) + .expect("route"); + assert_eq!( + advertised.publisher, + Some(crate::Origin::UNKNOWN), + "an anonymous publisher has no identity to carry", + ); + subscriber.start_announce(path.clone(), advertised).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + let original = consumer.get_broadcast("room/host").unwrap(); + + // The peer re-advertises the same path cheaper: it started carrying it. + let repriced = subscriber + .route(Some(&cluster::Advert { hops, cost: 1 }), &peer) + .expect("route"); + subscriber.update_announce(path.clone(), repriced).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + + assert!( + !original.is_closed(), + "a repricing update must not tear down the source" + ); + let broadcast = consumer.get_broadcast("room/host").unwrap(); + let routes = broadcast.routes(); + assert_eq!(routes.len(), 1, "the update replaced the route, it did not add one"); + assert_eq!(routes[0].cost, 1); + + // One advertisement, so one unannounce detaches it. + subscriber.stop_announce(path, Detach::Graceful).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + assert!(consumer.get_broadcast("room/host").is_none()); + } + + /// Two *separate* advertisements have nothing linking them but the publisher, and + /// `Origin::UNKNOWN` links nothing: any number of unrelated publishers present it. + /// So the later one replaces the earlier, unlike the repeat above. + #[tokio::test(start_paused = true)] + async fn separate_anonymous_adverts_replace_the_source() { + let (mut subscriber, origin) = cluster_subscriber(crate::Origin::new(1).unwrap()); + let consumer = origin.consume(); + let path = crate::Path::new("room/host").to_owned(); + let peer = cluster::Peer { + origin: Some(crate::Origin::new(9).unwrap()), + cost: None, + }; + let hops = cluster::HopPath::new( + crate::OriginList::try_from(vec![crate::Origin::UNKNOWN, crate::Origin::new(9).unwrap()]).unwrap(), + ); + let advert = cluster::Advert { hops, cost: 0 }; + + let first = subscriber.route(Some(&advert), &peer).expect("route"); + subscriber.start_announce(path.clone(), first).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + let original = consumer.get_broadcast("room/host").unwrap(); + + let second = subscriber.route(Some(&advert), &peer).expect("route"); + subscriber.start_announce(path.clone(), second).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + + assert!(original.is_closed(), "the old source was detached"); + assert!(consumer.get_broadcast("room/host").is_some()); + + // Two advertisements, so it takes two unannounces to detach. + subscriber.stop_announce(path.clone(), Detach::Graceful).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + assert!(consumer.get_broadcast("room/host").is_some()); + subscriber.stop_announce(path, Detach::Graceful).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + assert!(consumer.get_broadcast("room/host").is_none()); + } + /// Regression: without the MoQ Cluster extension an advertisement carries no path, /// so there is no publisher identity to compare. PUBLISH_NAMESPACE and NAMESPACE for /// one namespace are then two messages about a single source, and treating the diff --git a/test/wasm/src/main.ts b/test/wasm/src/main.ts index 616f0facd73..44620e993a7 100644 --- a/test/wasm/src/main.ts +++ b/test/wasm/src/main.ts @@ -278,15 +278,6 @@ const CASES: Case[] = [ }, { name: "reads a published broadcast", - // The IETF subscriber takes the publisher's identity straight off the hop - // chain, so an advert from a publisher that has none (@moq/net speaks no - // cluster extension) is attributed to Origin::UNKNOWN. Every repeat NAMESPACE - // for that path then reads as a new publisher and detaches the live source, - // killing the subscription mid-read. moq-lite has the same rule but stamps the - // upstream session's id when the chain has none, so it never reaches it. - // See https://github.com/moq-dev/moq/issues/2903. The detached source surfaces - // as the recv stream being dropped mid-read, which is the signature below. - known: { ietf: { issue: "#2903", error: "dropped" } }, run: async (wasm, relay) => { const path = `wasm-test/${relay.name}`; await withPublisher(relay, path, async () => { From c4b6c5d03f30fa94c323552a95c89c4c1e2b8e55 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 13:36:21 -0700 Subject: [PATCH 20/29] feat(moq-net): add Path::relative, replacing moq_transcode::source_reference (#2906) Co-authored-by: Claude Opus 5 --- doc/concept/layer/hang.md | 3 + js/net/src/path.test.ts | 55 ++++++++++ js/net/src/path.ts | 56 ++++++++++ rs/moq-cli/src/transcode.rs | 7 +- rs/moq-net/src/path.rs | 135 +++++++++++++++++++++++++ rs/moq-transcode/examples/transcode.rs | 9 +- rs/moq-transcode/src/config.rs | 18 +--- rs/moq-transcode/src/lib.rs | 5 +- 8 files changed, 268 insertions(+), 20 deletions(-) diff --git a/doc/concept/layer/hang.md b/doc/concept/layer/hang.md index b2193303cc8..1bdc33874d0 100644 --- a/doc/concept/layer/hang.md +++ b/doc/concept/layer/hang.md @@ -88,6 +88,9 @@ This lets a transcoder publish a sidecar catalog that adds new renditions while For example, a transcoder consuming `room/source` can publish `room/transcode` whose catalog contains a downscaled `480p` rendition plus the original `1080p` rendition marked `"broadcast": "./source"`. A viewer of `room/transcode` then pulls `480p` from the transcoder and `1080p` directly from the source, and the relay dedupes the source subscription with the transcoder's own. +Publishers author the reference with the inverse operation, so the two stay in step: `Path::relative` in Rust (`moq-net`) and `Path.relative` in TypeScript (`@moq/net`), each taking the target broadcast and the catalog broadcast it is named from. +Both refuse a target no reference can name, since a path segment may itself be called `.` or `..`. + `@moq/watch` resolves the reference automatically. In Rust, the `moq-mux` exporters do the same: they take a `Source::new(origin, path)`, and both the catalog broadcast and any referenced broadcast resolve through the origin over the same connection. ### Extensions diff --git a/js/net/src/path.test.ts b/js/net/src/path.test.ts index 834db511a09..92bfe8b02df 100644 --- a/js/net/src/path.test.ts +++ b/js/net/src/path.test.ts @@ -217,6 +217,61 @@ test("resolve excess dotdot clamps at empty", () => { expect(Path.resolve(Path.from("a"), "..")).toBe(Path.from("")); }); +test("relative inverts resolve", () => { + // Nested under the base: the base's own last segment is replaced, so it repeats. + expect(Path.relative(Path.from("foo/bar/baz"), Path.from("foo/bar"))).toBe("bar/baz"); + // Sibling. + expect(Path.relative(Path.from("foo/baz"), Path.from("foo/bar"))).toBe("baz"); + // Different subtree. + expect(Path.relative(Path.from("foo/baz/bar"), Path.from("foo/bar/baz"))).toBe("../baz/bar"); + // The base's parent, which only `.` can name. + expect(Path.relative(Path.from("a/b"), Path.from("a/b/transcode.hang"))).toBe("."); + expect(Path.relative(Path.from("a/b"), Path.from("a/b/one/two/transcode.hang"))).toBe("../.."); + // Roots. + expect(Path.relative(Path.from("foo/bar"), Path.empty())).toBe("foo/bar"); + expect(Path.relative(Path.empty(), Path.from("foo"))).toBe("."); + // The base itself, which only the empty reference names. + expect(Path.relative(Path.from("a/b"), Path.from("a/b"))).toBe(""); + expect(Path.relative(Path.empty(), Path.empty())).toBe(""); +}); + +test("relative rejects unnameable targets", () => { + // A segment literally named `.` or `..` is a legal path component, but resolution + // would walk on it instead of naming it. + expect(Path.relative(Path.from("a/../b"), Path.empty())).toBeUndefined(); + expect(Path.relative(Path.from("x/./y"), Path.from("x/z"))).toBeUndefined(); + expect(Path.relative(Path.from("a/.."), Path.from("a/b"))).toBeUndefined(); + + // A base is always nameable by itself, however its last segment is spelled. + expect(Path.relative(Path.from("a/.."), Path.from("a/.."))).toBe(""); + + // Dot segments inside the shared prefix are never emitted, so they are fine. + const rel = Path.relative(Path.from("a/../b/x"), Path.from("a/../b/c")); + expect(rel).toBe("x"); + expect(Path.resolve(Path.from("a/../b/c"), rel as string)).toBe(Path.from("a/../b/x")); +}); + +test("relative round trips through resolve", () => { + const paths = ["", "a", "b", "a/b", "a/c", "a/b/c", "a/b/c/d", "x/y/z", "a/../b", "a/./b", "a/..", "a/."].map((p) => + Path.from(p), + ); + + for (const base of paths) { + for (const target of paths) { + const rel = Path.relative(target, base); + if (rel === undefined) { + // Only an unnameable target may be refused, and never the base itself. + expect(target !== base && target.split("/").some((part) => part === "." || part === "..")).toBe(true); + continue; + } + + expect(Path.resolve(base, rel)).toBe(target); + // The reference is derived from a real target, so it never escapes the root. + expect(Path.tryResolve(base, rel)).toBe(target); + } + } +}); + test("resolve with empty base", () => { expect(Path.resolve(Path.empty(), "foo")).toBe(Path.from("foo")); expect(Path.resolve(Path.empty(), "..")).toBe(Path.from("")); diff --git a/js/net/src/path.ts b/js/net/src/path.ts index e98b34021ed..54862983227 100644 --- a/js/net/src/path.ts +++ b/js/net/src/path.ts @@ -255,3 +255,59 @@ export function tryResolve(base: Valid, rel: string): Valid | undefined { return segments.join("/") as Valid; } + +/** + * Express `target` relative to `base`: the inverse of {@link resolve}. + * + * The result round-trips (`resolve(base, relative(target, base)) === target`) and never + * walks above the root, so {@link tryResolve} accepts it too. + * + * A relative reference replaces the last segment of the base, matching relative URL + * resolution, so a target nested under the base repeats the base's own last segment. + * + * The empty reference names the base itself, so that is what a self-reference returns. + * + * Returns `undefined` for a target no reference can name: a path segment may literally be + * `.` or `..`, which resolution reads as navigation instead of as a name. Only the segments + * past the shared prefix matter, since the rest are never emitted. + * + * Mirrors the Rust `Path::relative`, used to author the cross-broadcast track + * references a hang catalog carries. Note the argument order: the target comes first, + * the opposite of Node's `path.relative(from, to)`. + * + * @example + * ```typescript + * Path.relative(Path.from("a/b/c"), Path.from("a/b")); // "b/c" + * Path.relative(Path.from("a/c"), Path.from("a/b")); // "c" + * Path.relative(Path.from("a"), Path.from("a/b")); // "." + * Path.relative(Path.from("a/b"), Path.from("a/b")); // "" + * Path.relative(Path.from("a/.."), Path.from("a/b")); // undefined + * ``` + */ +export function relative(target: Valid, base: Valid): string | undefined { + // Only the empty reference can name a base whose last segment is itself `.` or `..`, + // since resolution replaces that segment rather than emitting it. + if (target === base) return ""; + + // Resolution replaces the base's last segment, so walk from its parent. + const dir = base === "" ? [] : base.split("/"); + dir.pop(); + + const parts = target === "" ? [] : target.split("/"); + + let common = 0; + while (common < dir.length && common < parts.length && dir[common] === parts[common]) { + common += 1; + } + + const down = parts.slice(common); + // Resolution would walk on these instead of naming them. + if (down.some((part) => part === "." || part === "..")) return undefined; + + const rel = Array(dir.length - common) + .fill("..") + .concat(down); + + // An empty reference resolves to the base itself, so name the parent explicitly. + return rel.length === 0 ? "." : rel.join("/"); +} diff --git a/rs/moq-cli/src/transcode.rs b/rs/moq-cli/src/transcode.rs index dfe3cba3cdc..84bda01cac3 100644 --- a/rs/moq-cli/src/transcode.rs +++ b/rs/moq-cli/src/transcode.rs @@ -137,9 +137,10 @@ pub async fn run(moq: MoqSide, args: Args, net: Net) -> anyhow::Result<()> { name => moq_video::decode::Kind::Named(name.to_string()), }; config.resize.acceleration = args.resize_acceleration; - // Reference the source renditions relatively when the output nests under it; - // otherwise the derivative catalog advertises only the rungs. - config.source = moq_transcode::source_reference(&source_path, &output_path); + // Point the derivative catalog at the source renditions so players fetch them from the + // source directly. An empty reference would name the derivative broadcast itself, which + // publishes the rungs and nothing else. + config.source = source_path.relative(&output_path).filter(|rel| !rel.is_empty()); let output = publish .create_broadcast(&output_path, moq_net::broadcast::Route::new().with_announce(true)) diff --git a/rs/moq-net/src/path.rs b/rs/moq-net/src/path.rs index a4e32be5e55..963fc46f470 100644 --- a/rs/moq-net/src/path.rs +++ b/rs/moq-net/src/path.rs @@ -403,6 +403,72 @@ impl<'a> Path<'a> { })) } } + + /// Express this path relative to `base`: the inverse of [`Path::resolve`]. + /// + /// The result round-trips (`base.resolve(&rel) == self`) and never walks above the + /// root, so [`Path::try_resolve`] accepts it too. + /// + /// A relative reference replaces the last segment of the base, matching relative URL + /// resolution, so a target nested under the base repeats the base's own last segment. + /// + /// The empty reference names the base itself, so that is what a self-reference returns. + /// + /// Returns `None` for a target no reference can name: a path segment may literally be + /// `.` or `..`, which resolution reads as navigation instead of as a name. Only the + /// segments past the shared prefix matter, since the rest are never emitted. + /// + /// # Examples + /// ``` + /// use moq_net::Path; + /// + /// // The base names a broadcast, so its last segment is replaced, not descended into. + /// let base = Path::new("a/b"); + /// assert_eq!(Path::new("a/b/c").relative(&base).unwrap().as_str(), "b/c"); + /// assert_eq!(Path::new("a/c").relative(&base).unwrap().as_str(), "c"); + /// assert_eq!(Path::new("c").relative(&base).unwrap().as_str(), "../c"); + /// + /// // The lone `.` names the base's parent, which the empty reference cannot. + /// assert_eq!(Path::new("a").relative(&base).unwrap().as_str(), "."); + /// + /// // The base itself. + /// assert_eq!(Path::new("a/b").relative(&base).unwrap().as_str(), ""); + /// + /// // A segment named `..` is a legal path component but an unnameable target. + /// assert!(Path::new("a/..").relative(&base).is_none()); + /// ``` + pub fn relative(&self, base: impl AsPath) -> Option { + let base = base.as_path(); + + // Only the empty reference can name a base whose last segment is itself `.` or `..`, + // since resolution replaces that segment rather than emitting it. + if *self == base { + return Some(PathRelative::empty()); + } + + // Resolution replaces the base's last segment, so walk from its parent. + let mut dir: Vec<&str> = base.parts().collect(); + dir.pop(); + + let target: Vec<&str> = self.parts().collect(); + let common = dir.iter().zip(&target).take_while(|(a, b)| a == b).count(); + + let down = &target[common..]; + if down.iter().any(|part| *part == "." || *part == "..") { + // Resolution would walk on these instead of naming them. + return None; + } + + let mut rel: Vec<&str> = vec![".."; dir.len() - common]; + rel.extend(down); + + if rel.is_empty() { + // An empty reference resolves to the base itself, so name the parent explicitly. + return Some(PathRelative::new(".")); + } + + Some(PathRelativeOwned::from(rel.join("/"))) + } } // Comparisons, ordering, and hashing all go through `as_str()` so a borrowed and a @@ -1492,4 +1558,73 @@ mod tests { assert_eq!(nested.try_resolve(&PathRelative::new("..")).unwrap().as_str(), ""); assert!(nested.try_resolve(&PathRelative::new("../..")).is_none()); } + + #[test] + fn test_relative() { + let rel = |target: &str, base: &str| Path::new(target).relative(base).unwrap(); + + // Nested under the base: the base's own last segment is replaced, so it repeats. + assert_eq!(rel("foo/bar/baz", "foo/bar").as_str(), "bar/baz"); + // Sibling. + assert_eq!(rel("foo/baz", "foo/bar").as_str(), "baz"); + // Different subtree. + assert_eq!(rel("foo/baz/bar", "foo/bar/baz").as_str(), "../baz/bar"); + // The base's parent, which only `.` can name. + assert_eq!(rel("a/b", "a/b/transcode.hang").as_str(), "."); + assert_eq!(rel("a/b", "a/b/one/two/transcode.hang").as_str(), "../.."); + // Roots. + assert_eq!(rel("foo/bar", "").as_str(), "foo/bar"); + assert_eq!(rel("", "foo").as_str(), "."); + // The base itself, which only the empty reference names. + assert_eq!(rel("a/b", "a/b").as_str(), ""); + assert_eq!(rel("", "").as_str(), ""); + // Slashes are normalized first. + assert_eq!(rel("/a//b/", "//a/b/dir//").as_str(), "."); + } + + #[test] + fn test_relative_rejects_unnameable_targets() { + // A segment literally named `.` or `..` is a legal path component, but resolution + // would walk on it instead of naming it. + assert!(Path::new("a/../b").relative("").is_none()); + assert!(Path::new("x/./y").relative("x/z").is_none()); + assert!(Path::new("a/..").relative("a/b").is_none()); + + // A base is always nameable by itself, however its last segment is spelled. + assert_eq!(Path::new("a/..").relative("a/..").unwrap().as_str(), ""); + + // Dot segments inside the shared prefix are never emitted, so they are fine. + let rel = Path::new("a/../b/x").relative("a/../b/c").unwrap(); + assert_eq!(rel.as_str(), "x"); + assert_eq!(Path::new("a/../b/c").resolve(&rel).as_str(), "a/../b/x"); + } + + #[test] + fn test_relative_round_trips() { + let paths = [ + "", "a", "b", "a/b", "a/c", "a/b/c", "a/b/c/d", "x/y/z", "a/../b", "a/./b", "a/..", "a/.", + ]; + + for base in paths { + for target in paths { + let base = Path::new(base); + let target = Path::new(target); + let Some(rel) = target.relative(&base) else { + // Only an unnameable target may be refused, and never the base itself. + assert!( + target != base && target.parts().any(|part| part == "." || part == ".."), + "{base} -> {target} refused a nameable target" + ); + continue; + }; + + assert_eq!(base.resolve(&rel), target, "{base} -> {target} via {rel}"); + // The reference is derived from a real target, so it never escapes the root. + assert!( + base.try_resolve(&rel).is_some(), + "{base} -> {target} via {rel} escaped the root" + ); + } + } + } } diff --git a/rs/moq-transcode/examples/transcode.rs b/rs/moq-transcode/examples/transcode.rs index 51386517d8e..47c3ddd2ca7 100644 --- a/rs/moq-transcode/examples/transcode.rs +++ b/rs/moq-transcode/examples/transcode.rs @@ -74,10 +74,13 @@ async fn main() -> anyhow::Result<()> { .await .context("source broadcast unavailable")?; + // The default ladder and encoder (hardware first: NVENC on Linux) apply. let mut config = moq_transcode::Config::default(); - // Reference the source when the normalized output is nested beneath it. The - // default ladder and encoder (hardware first: NVENC on Linux) apply. - config.source = moq_transcode::source_reference(&source_path, &output_path); + + // Point the derivative catalog at the source renditions so players fetch them from the + // source directly. An empty reference would name the derivative broadcast itself, which + // publishes the rungs and nothing else. + config.source = source_path.relative(&output_path).filter(|rel| !rel.is_empty()); let output = publish .create_broadcast(&output_path, moq_net::broadcast::Route::new().with_announce(true)) diff --git a/rs/moq-transcode/src/config.rs b/rs/moq-transcode/src/config.rs index f7eae38c1d8..3f2f64ef81c 100644 --- a/rs/moq-transcode/src/config.rs +++ b/rs/moq-transcode/src/config.rs @@ -2,25 +2,16 @@ use moq_net::{AsPath, PathRelativeOwned}; -/// Compute the source broadcast reference for an output nested beneath it. -/// -/// Both paths are normalized before comparison. Returns `None` when `output` is -/// not a descendant of `source`, so callers can omit passthrough renditions. +#[doc(hidden)] +#[deprecated(note = "use moq_net::Path::relative")] pub fn source_reference(source: impl AsPath, output: impl AsPath) -> Option { let source = source.as_path(); let output = output.as_path(); - let rest = output.strip_prefix(&source)?; - if rest.is_empty() { + if output.strip_prefix(&source)?.is_empty() { return None; } - let parents = rest.parts().count().saturating_sub(1); - let rel = if parents == 0 { - ".".to_string() - } else { - vec![".."; parents].join("/") - }; - Some(PathRelativeOwned::from(rel)) + source.relative(&output) } /// One candidate output rendition: a target resolution (by height) and bitrate. @@ -102,6 +93,7 @@ impl Default for Config { } #[cfg(test)] +#[allow(deprecated)] mod tests { use super::*; diff --git a/rs/moq-transcode/src/lib.rs b/rs/moq-transcode/src/lib.rs index c5ec2a5df06..01c3e5f30cf 100644 --- a/rs/moq-transcode/src/lib.rs +++ b/rs/moq-transcode/src/lib.rs @@ -27,7 +27,10 @@ mod error; mod feed; mod rung; -pub use config::{Config, Rung, source_reference}; +pub use config::{Config, Rung}; + +#[allow(deprecated)] +pub use config::source_reference; pub use error::Error; /// Transcode `source` into `output` until the source broadcast ends. From 31942192863e7996c619299d5129f521159366ec Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 14:34:21 -0700 Subject: [PATCH 21/29] feat(moq-ffi): compile for wasm32 (#2911) Co-authored-by: Claude Opus 5 --- Cargo.lock | 5 + Cargo.toml | 2 + rs/justfile | 34 +++--- rs/moq-ffi/Cargo.toml | 51 +++++++-- rs/moq-ffi/src/error.rs | 5 + rs/moq-ffi/src/ffi.rs | 115 ++++++++++++++++---- rs/moq-ffi/src/json.rs | 14 +-- rs/moq-ffi/src/lib.rs | 18 +++- rs/moq-ffi/src/origin.rs | 16 +-- rs/moq-ffi/src/producer.rs | 98 ++++++++--------- rs/moq-ffi/src/session.rs | 203 +++++++++++++++++++++++++++++++++++- rs/moq-ffi/src/test.rs | 3 + rs/moq-ffi/src/transport.rs | 163 +++++++++++++++++++++++++++++ rs/moq-wasm/Cargo.toml | 4 +- 14 files changed, 606 insertions(+), 125 deletions(-) create mode 100644 rs/moq-ffi/src/transport.rs diff --git a/Cargo.lock b/Cargo.lock index af48953c7a9..ba95fb96bae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4730,7 +4730,9 @@ name = "moq-ffi" version = "0.3.11" dependencies = [ "bytes", + "getrandom 0.4.3", "hang", + "kio 0.5.4", "moq-audio", "moq-json", "moq-mux", @@ -4744,6 +4746,9 @@ dependencies = [ "tracing", "uniffi", "url", + "web-async", + "web-transport-trait", + "web-transport-wasm", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6a93a0439e8..3e6ce52ae13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ rust-version = "1.91" [workspace.dependencies] criterion = "0.8" flate2 = "1.1" +getrandom = { version = "0.4", features = ["wasm_js"] } hang = { version = "0.20", path = "rs/hang" } kio = { version = "0.5", path = "rs/kio" } libc = "0.2" @@ -130,6 +131,7 @@ web-transport-proto = "0.6" web-transport-quiche = "0.6" web-transport-quinn = "0.12" web-transport-trait = "0.4.0" +web-transport-wasm = "0.5" [profile.dev] panic = "abort" diff --git a/rs/justfile b/rs/justfile index 3e000aaf26c..cbf7becb5e3 100644 --- a/rs/justfile +++ b/rs/justfile @@ -129,10 +129,13 @@ check-changed $FILES: # moq-wasm's crate root is entirely `#![cfg(target_arch = "wasm32")]`, so # however it got selected the host-target pass above compiled it to nothing - # and saw no errors in it. This wasm32 pass is what actually checks it, and - # the selection already says whether moq-wasm or anything it depends on - # changed. - if [[ "$packages" == "ALL" ]] || grep -q -- '--package moq-wasm' <<< "$packages"; then + # and saw no errors in it. This wasm32 pass is what actually checks it. + # + # The alternation must list every package `wasm` compiles, not just moq-wasm: + # nothing in the workspace depends on moq-ffi, so an moq-ffi-only diff selects + # only itself and would never reach the gate that exists to cover it. moq-mux + # is selected by its dependents, none of which are moq-wasm. + if [[ "$packages" == "ALL" ]] || grep -qE -- '--package (moq-wasm|moq-mux|moq-ffi)' <<< "$packages"; then just rs wasm fi @@ -147,9 +150,9 @@ fix-changed $FILES: *) echo "rs: fixing ${packages//--package /}"; just rs fix $packages ;; esac - # Mirrors `check-changed`'s gate: without this a moq-wasm lint is never - # auto-fixed, and `check`'s wasm32 clippy pass fails on it later. - if [[ "$packages" == "ALL" ]] || grep -q -- '--package moq-wasm' <<< "$packages"; then + # Mirrors `check-changed`'s gate, alternation included: without this a wasm32 + # lint is never auto-fixed, and `check`'s wasm32 clippy pass fails on it later. + if [[ "$packages" == "ALL" ]] || grep -qE -- '--package (moq-wasm|moq-mux|moq-ffi)' <<< "$packages"; then just rs wasm-fix fi @@ -209,23 +212,24 @@ macos *args: # which builds the shippable `@moq/wasm` package (wasm-bindgen, release profile); # this is the compile gate. -# `moq-mux` rides along in the same invocation on purpose: it needs `getrandom`'s -# `wasm_js` backend, which only `moq-wasm` declares, and feature unification only -# happens within one cargo invocation. Splitting these into two fails on getrandom. -# `--lib` rather than `--all-targets` because moq-mux's tests use tokio's native -# timers; moq-wasm has no test targets, so it loses no coverage. +# `moq-mux` and `moq-ffi` ride along in the same invocation on purpose: `moq-mux` +# needs `getrandom`'s `wasm_js` backend that only its siblings declare, and feature +# unification only happens within one cargo invocation. Splitting them fails on +# getrandom. `--lib` rather than `--all-targets` because moq-mux's tests use tokio's +# native timers and moq-ffi's drive a native relay; moq-wasm has no test targets, so +# it loses no coverage. # Compile and lint the browser/WASM bindings, which the host-target check skips. wasm *args: - cargo clippy -p moq-wasm -p moq-mux --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings + cargo clippy -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings # The `fix` counterpart to `wasm`. `cargo fmt` is scoped rather than `--all` # because this runs alongside `fix`, which already formats everything else. # Auto-fix the browser/WASM bindings. wasm-fix *args: - cargo clippy --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux --target wasm32-unknown-unknown --lib {{ args }} - cargo fmt -p moq-wasm -p moq-mux + cargo clippy --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} + cargo fmt -p moq-wasm -p moq-mux -p moq-ffi # Dependency advisories plus the license/ban policy. Time-based rather than # diff-based: an advisory lands without this repo changing, so a per-PR run both diff --git a/rs/moq-ffi/Cargo.toml b/rs/moq-ffi/Cargo.toml index c21e0e167f3..e5682202376 100644 --- a/rs/moq-ffi/Cargo.toml +++ b/rs/moq-ffi/Cargo.toml @@ -11,6 +11,10 @@ edition = "2024" keywords = ["quic", "http3", "webtransport", "media", "live"] categories = ["multimedia", "network-programming", "web-programming"] +[package.metadata.cargo-shear] +# Required indirectly (a feature enabler with no `use`), so cargo-shear can't see it. +ignored = ["getrandom"] + [lib] name = "moq_ffi" crate-type = ["staticlib", "cdylib"] @@ -20,30 +24,59 @@ name = "uniffi-bindgen" path = "./uniffi-bindgen.rs" [features] +# On by default so every existing consumer (the language wrappers, `build.sh`) +# keeps the surface it has. Opt out with `default-features = false` to skip the +# ~40 crates that only the codecs pull in, including openh264's vendored C++. +# They are unavailable on wasm32 regardless: the browser encodes via WebCodecs. +default = ["audio", "video"] + android-logcat = ["moq-native/android-logcat"] +audio = ["dep:moq-audio"] +video = ["dep:moq-video", "dep:pollster"] [dependencies] bytes = "1" hang = { workspace = true } -moq-audio = { workspace = true } +# `kio::MaybeSend` is the `Send` bound that vanishes on wasm; see `ffi::Task`. +kio = { workspace = true } moq-json = { workspace = true } moq-mux = { workspace = true } -moq-native = { workspace = true, default-features = true } moq-net = { workspace = true } +serde_json = "1" +thiserror = "2" +# `sync` (Mutex, watch) and `macros` (select!) are the only tokio pieces that build +# on wasm32; the runtime and its drivers are native-only. +tokio = { workspace = true, features = ["macros", "sync"] } +tracing = "0.1" +url = "2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +moq-audio = { workspace = true, optional = true } +moq-native = { workspace = true, default-features = true } # Default features only: the Linux hardware codecs stay off so the wrappers' # cross-compiled targets (Android, iOS) don't pull a CUDA graph. -moq-video = { workspace = true } +moq-video = { workspace = true, optional = true } # Blocking on `encode::Sink` from the synchronous producer exports; see # `video::block_on` for why not a tokio helper. -pollster = "1.0" -serde_json = "1" -thiserror = "2" -tokio = { workspace = true, features = ["macros"] } -tracing = "0.1" +pollster = { version = "1.0", optional = true } +tokio = { workspace = true, features = ["rt", "macros", "sync"] } uniffi = { version = "0.31", features = ["cli"] } -url = "2" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +# Enables the `wasm_js` backend for rand-via-moq-net; the cfg flag lives in +# `.cargo/config.toml`. Referenced only as a feature enabler, hence the shear ignore. +getrandom = { workspace = true } +# `wasm-unstable-single-threaded` drops the `Send`/`Sync` bounds on +# `UniffiCompatibleFuture` and `FfiConverterArc`, which moq-net's wasm futures +# (LocalBoxFuture) and Rc-backed handles cannot satisfy. +uniffi = { version = "0.31", features = ["wasm-unstable-single-threaded"] } +web-async = { workspace = true } +web-transport-trait = { workspace = true } +web-transport-wasm = { workspace = true } [build-dependencies] uniffi = { version = "0.31", features = ["build"] } [dev-dependencies] +# One non-codec test blocks on a future; the main dep is `video`-only. +pollster = "1.0" diff --git a/rs/moq-ffi/src/error.rs b/rs/moq-ffi/src/error.rs index f95c6b2a500..ef326e8f110 100644 --- a/rs/moq-ffi/src/error.rs +++ b/rs/moq-ffi/src/error.rs @@ -15,9 +15,12 @@ pub enum MoqError { #[error(transparent)] JsonTrack(#[from] moq_json::Error), + // Native codec errors, behind the optional `audio`/`video` features. + #[cfg(all(feature = "audio", not(target_arch = "wasm32")))] #[error(transparent)] Audio(#[from] moq_audio::Error), + #[cfg(all(feature = "video", not(target_arch = "wasm32")))] #[error(transparent)] Video(#[from] moq_video::Error), @@ -30,6 +33,8 @@ pub enum MoqError { #[error(transparent)] LogLevel(#[from] tracing::metadata::ParseLevelError), + // Only the native path spawns onto a runtime, so only it can fail to join. + #[cfg(not(target_arch = "wasm32"))] #[error(transparent)] Task(#[from] tokio::task::JoinError), diff --git a/rs/moq-ffi/src/ffi.rs b/rs/moq-ffi/src/ffi.rs index b2e1a367917..6c52c5bdec7 100644 --- a/rs/moq-ffi/src/ffi.rs +++ b/rs/moq-ffi/src/ffi.rs @@ -1,9 +1,14 @@ use std::future::Future; -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; use crate::error::MoqError; -pub(crate) static RUNTIME: LazyLock = LazyLock::new(|| { +/// A dedicated runtime thread, so a foreign caller's thread never has to drive our futures. +/// +/// wasm32 has neither threads nor a tokio driver. uniffi's `RustFuture` is polled by the +/// JS event loop instead, so [`Task::run`] awaits in place there. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) static RUNTIME: std::sync::LazyLock = std::sync::LazyLock::new(|| { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -20,12 +25,55 @@ pub(crate) static RUNTIME: LazyLock = LazyLock::new(|| { handle }); -pub(crate) struct Task { +/// Enter the runtime context, so a handle built outside [`Task::run`] can still spawn. +/// +/// A no-op on wasm32, where the JS event loop is the only runtime. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn enter() -> tokio::runtime::EnterGuard<'static> { + RUNTIME.enter() +} + +/// Stands in for tokio's `EnterGuard` on wasm32, so callers bind a guard either way. +#[cfg(target_arch = "wasm32")] +pub(crate) struct EnterGuard; + +#[cfg(target_arch = "wasm32")] +pub(crate) fn enter() -> EnterGuard { + EnterGuard +} + +/// Run a future to completion off the caller's thread, mapping its error into [`MoqError`]. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) async fn detached(future: F) -> Result +where + F: Future> + Send + 'static, + T: Send + 'static, + E: Into + Send + 'static, +{ + match RUNTIME.spawn(future).await { + Ok(result) => result.map_err(Into::into), + Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), + Err(e) => Err(MoqError::Task(e)), + } +} + +/// Await a future in place: wasm32 has no other thread to hand it to. +#[cfg(target_arch = "wasm32")] +pub(crate) async fn detached(future: F) -> Result +where + F: Future> + 'static, + T: 'static, + E: Into + 'static, +{ + future.await.map_err(Into::into) +} + +pub(crate) struct Task { state: Arc>, cancel: tokio::sync::watch::Sender, } -impl Task { +impl Task { pub fn new(inner: T) -> Self { Self { state: Arc::new(tokio::sync::Mutex::new(inner)), @@ -40,31 +88,19 @@ impl Task { /// Spawn an async closure on the runtime. /// - /// The closure receives an [OwnedMutexGuard] which derefs to `T`. + /// The closure receives an [tokio::sync::OwnedMutexGuard] which derefs to `T`. /// If two calls are made concurrently, the second waits for the first to finish. + #[cfg(not(target_arch = "wasm32"))] pub async fn run(&self, f: F) -> Result where R: Send + 'static, F: FnOnce(tokio::sync::OwnedMutexGuard) -> Fut + Send + 'static, Fut: Future> + Send + 'static, { - let mut cancel = self.cancel.subscribe(); + let cancel = self.cancel.subscribe(); let state = self.state.clone(); - let handle = RUNTIME.spawn(async move { - let state = tokio::select! { - biased; - Ok(_) = cancel.wait_for(|&c| c) => return Err(MoqError::Cancelled), - state = state.lock_owned() => state, - }; - - let mut cancel = cancel; - tokio::select! { - biased; - Ok(_) = cancel.wait_for(|&c| c) => Err(MoqError::Cancelled), - result = f(state) => result, - } - }); + let handle = RUNTIME.spawn(async move { Self::drive(cancel, state, f).await }); match handle.await { Ok(result) => result, @@ -73,13 +109,50 @@ impl Task { } } + /// Run an async closure, awaiting it in place. + /// + /// Unlike the native path this does not detach, so dropping the returned future + /// cancels the work rather than leaving it running on a runtime thread. + #[cfg(target_arch = "wasm32")] + pub async fn run(&self, f: F) -> Result + where + R: 'static, + F: FnOnce(tokio::sync::OwnedMutexGuard) -> Fut + 'static, + Fut: Future> + 'static, + { + Self::drive(self.cancel.subscribe(), self.state.clone(), f).await + } + + /// Wait for the lock, then run `f`, with [Self::cancel] able to interrupt either. + async fn drive( + mut cancel: tokio::sync::watch::Receiver, + state: Arc>, + f: F, + ) -> Result + where + F: FnOnce(tokio::sync::OwnedMutexGuard) -> Fut, + Fut: Future>, + { + let state = tokio::select! { + biased; + Ok(_) = cancel.wait_for(|&c| c) => return Err(MoqError::Cancelled), + state = state.lock_owned() => state, + }; + + tokio::select! { + biased; + Ok(_) = cancel.wait_for(|&c| c) => Err(MoqError::Cancelled), + result = f(state) => result, + } + } + /// Cancel all current and future [Self::run] calls, causing them to return [MoqError::Cancelled]. pub fn cancel(&self) { let _ = self.cancel.send(true); } } -impl Drop for Task { +impl Drop for Task { fn drop(&mut self) { self.cancel(); } diff --git a/rs/moq-ffi/src/json.rs b/rs/moq-ffi/src/json.rs index df658319b21..cf3f244abe0 100644 --- a/rs/moq-ffi/src/json.rs +++ b/rs/moq-ffi/src/json.rs @@ -11,7 +11,7 @@ use serde_json::Value; use crate::consumer::MoqBroadcastConsumer; use crate::error::MoqError; -use crate::ffi::{RUNTIME, Task}; +use crate::ffi::Task; use crate::producer::MoqBroadcastProducer; /// Options for a JSON snapshot track (lossy latest-value mode). @@ -100,7 +100,7 @@ impl MoqBroadcastProducer { name: String, config: MoqJsonSnapshotConfig, ) -> Result, MoqError> { - let _guard = RUNTIME.enter(); + let _guard = crate::ffi::enter(); self.with_state(|state| { let mut broadcast = state.broadcast.clone(); let track = broadcast.create_track(name, None)?; @@ -117,7 +117,7 @@ impl MoqBroadcastProducer { name: String, config: MoqJsonStreamConfig, ) -> Result, MoqError> { - let _guard = RUNTIME.enter(); + let _guard = crate::ffi::enter(); self.with_state(|state| { let mut broadcast = state.broadcast.clone(); let track = broadcast.create_track(name, None)?; @@ -173,7 +173,7 @@ impl MoqJsonSnapshotProducer { /// Publish a new value, encoded as a snapshot or delta automatically. `value` is a JSON /// document. A no-op if unchanged from the previous update. pub fn update(&self, value: String) -> Result<(), MoqError> { - let _guard = RUNTIME.enter(); + let _guard = crate::ffi::enter(); let value: Value = serde_json::from_str(&value)?; let mut guard = self.inner.lock().unwrap(); let producer = guard.as_mut().ok_or(MoqError::Closed)?; @@ -183,7 +183,7 @@ impl MoqJsonSnapshotProducer { /// Finish the track, closing any open group. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut producer = self.inner.lock().unwrap().take().ok_or(MoqError::Closed)?; producer.finish()?; Ok(()) @@ -236,7 +236,7 @@ pub struct MoqJsonStreamProducer { impl MoqJsonStreamProducer { /// Append one record to the log. `value` is a JSON document. pub fn append(&self, value: String) -> Result<(), MoqError> { - let _guard = RUNTIME.enter(); + let _guard = crate::ffi::enter(); let value: Value = serde_json::from_str(&value)?; let mut guard = self.inner.lock().unwrap(); let producer = guard.as_mut().ok_or(MoqError::Closed)?; @@ -246,7 +246,7 @@ impl MoqJsonStreamProducer { /// Finish the track, closing the group. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut producer = self.inner.lock().unwrap().take().ok_or(MoqError::Closed)?; producer.finish()?; Ok(()) diff --git a/rs/moq-ffi/src/lib.rs b/rs/moq-ffi/src/lib.rs index 8fbe2d9b8cf..e626d471934 100644 --- a/rs/moq-ffi/src/lib.rs +++ b/rs/moq-ffi/src/lib.rs @@ -3,22 +3,38 @@ //! Provides a Kotlin/Swift-compatible API for real-time pub/sub over QUIC. //! Uses async UniFFI objects instead of callbacks for a native async experience. +// uniffi lowers every exported object as an `Arc`, and on wasm moq-net's handles are +// `Rc`-backed, so clippy sees a pointlessly atomic `Arc` at every construction site. The +// `Arc` is uniffi's ABI rather than our choice, and the target is single-threaded, so the +// wasted atomics are what one type serving both targets costs. +#![cfg_attr(target_arch = "wasm32", allow(clippy::arc_with_non_send_sync))] + #[cfg(target_os = "android")] mod android; +// The QUIC server and moq-native's logger have no browser equivalent, and the +// native codecs are both browser-inapplicable (WebCodecs does this) and optional, +// since they are the heaviest thing in the dependency graph. +#[cfg(all(feature = "audio", not(target_arch = "wasm32")))] pub mod audio; pub mod consumer; pub mod error; mod ffi; pub mod json; +#[cfg(not(target_arch = "wasm32"))] mod log; pub mod media; pub mod origin; pub mod producer; +#[cfg(not(target_arch = "wasm32"))] pub mod server; pub mod session; +#[cfg(target_arch = "wasm32")] +mod transport; +#[cfg(all(feature = "video", not(target_arch = "wasm32")))] pub mod video; uniffi::setup_scaffolding!("moq"); -#[cfg(test)] +// The test suite drives a native relay over a tokio runtime. +#[cfg(all(test, not(target_arch = "wasm32")))] mod test; diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index 685bf73f293..5c36c194e94 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -202,13 +202,13 @@ impl MoqOriginProducer { /// Create a new origin for publishing and/or consuming broadcasts. #[uniffi::constructor] pub fn new(options: MoqOriginOptions) -> Arc { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); Arc::new(Self::from_options(options)) } /// Create a consumer for this origin. pub fn consume(&self) -> Arc { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); Arc::new(MoqOriginConsumer { inner: self.inner.consume(), }) @@ -219,7 +219,7 @@ impl MoqOriginProducer { /// Hold the returned object while missing broadcast requests should be accepted. /// Dropping it makes future requests to unknown broadcasts fail. pub fn dynamic(&self) -> Arc { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); Arc::new(MoqOriginDynamic { task: Task::new(OriginDynamic { inner: self.inner.dynamic(), @@ -238,7 +238,7 @@ impl MoqOriginProducer { /// without finishing is treated as a failure: the path lingers briefly so a /// replacement publisher can take over without subscribers noticing. pub fn create_broadcast(&self, path: String) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); // Surfaces Error::Unauthorized (out of scope) via the MoqError::Protocol conversion. let broadcast = self .inner @@ -251,7 +251,7 @@ impl MoqOriginProducer { impl MoqOriginConsumer { /// Subscribe to all broadcast announcements under a prefix. pub fn announced(&self, prefix: String) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let origin = self.inner.with_root(prefix).ok_or(MoqError::Unauthorized)?; Ok(Arc::new(MoqAnnounced { task: Task::new(Announced { @@ -265,7 +265,7 @@ impl MoqOriginConsumer { /// This is how you resolve a path right after connecting: announcements arrive over the /// session after it opens, so `request_broadcast` on its own races them. pub fn announced_broadcast(&self, path: String) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let origin = self.inner.with_root(path).ok_or(MoqError::Unauthorized)?; Ok(Arc::new(MoqAnnouncedBroadcast { task: Task::new(Announced { @@ -335,7 +335,7 @@ impl MoqBroadcastRequest { /// Accept the request with an unannounced broadcast. pub fn accept(&self, broadcast: &MoqBroadcastProducer) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let consumer = broadcast.consume_inner()?; let request = self.take()?; request.accept(&consumer); @@ -344,7 +344,7 @@ impl MoqBroadcastRequest { /// Abort the request with an application error code. pub fn abort(&self, error_code: u16) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let request = self.take()?; request.reject(moq_net::Error::App(error_code)); Ok(()) diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 40161d59f56..62a9598a22b 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -237,7 +237,7 @@ pub struct MoqMediaStreamProducer { impl MoqBroadcastProducer { /// Create a consumer that reads from this broadcast's tracks. pub fn consume(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); Ok(Arc::new(MoqBroadcastConsumer::new(self.consume_inner()?))) } @@ -246,7 +246,7 @@ impl MoqBroadcastProducer { /// Hold the returned object for as long as missing track requests should be /// accepted. Dropping it makes future subscriptions to unknown tracks fail. pub fn dynamic(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.state.lock().unwrap(); let state = guard.as_ref().ok_or_else(|| MoqError::Closed)?; Ok(Arc::new(MoqBroadcastDynamic { @@ -263,7 +263,7 @@ impl MoqBroadcastProducer { /// [`MoqOriginProducer::create_broadcast`](crate::origin::MoqOriginProducer::create_broadcast) instead. #[uniffi::constructor] pub fn new() -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); Ok(Arc::new(Self::from_inner(moq_net::broadcast::Info::new().produce())?)) } @@ -273,7 +273,7 @@ impl MoqBroadcastProducer { /// once it is warm); consumers observe the change via /// `MoqBroadcastConsumer::route_updates` and sessions forward it downstream. pub fn set_route(&self, route: MoqRoute) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let route = route.try_into()?; self.with_state(|state| Ok(state.broadcast.set_route(route)?)) } @@ -284,7 +284,7 @@ impl MoqBroadcastProducer { /// broadcast stays reachable by exact path for subscribes and fetches. This is /// how a publisher goes on and off the air without tearing down the broadcast. pub fn set_announce(&self, announce: bool) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); self.with_state(|state| { let route = state.broadcast.consume().route(); Ok(state.broadcast.set_route(route.with_announce(announce))?) @@ -295,7 +295,7 @@ impl MoqBroadcastProducer { /// /// Rotation is clockwise and normalized to the nearest quarter turn. An absent field is removed from the next catalog update. pub fn set_video_properties(&self, properties: MoqVideoProperties) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut value = hang::catalog::VideoProperties::default(); value.display = properties.display.map(|display| hang::catalog::Display { width: display.width, @@ -319,7 +319,7 @@ impl MoqBroadcastProducer { /// error if `name` is `video`/`audio` (owned by the media pipeline). The section is /// republished on the catalog track immediately. pub fn set_catalog_section(&self, name: String, json: String) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let value: serde_json::Value = serde_json::from_str(&json)?; self.with_state(|state| Ok(state.catalog.lock().set_section(name, value)?)) } @@ -328,7 +328,7 @@ impl MoqBroadcastProducer { /// /// Republishes the catalog if the section existed; a no-op otherwise. pub fn remove_catalog_section(&self, name: String) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); self.with_state(|state| { state.catalog.lock().remove_section(&name); Ok(()) @@ -341,7 +341,7 @@ impl MoqBroadcastProducer { /// its hints seed the catalog. Hints apply to single-codec formats; container formats auto-detect /// every track. pub fn publish_media(&self, init: MoqInit) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.state.lock().unwrap(); let state = guard.as_ref().ok_or_else(|| MoqError::Closed)?; @@ -391,7 +391,7 @@ impl MoqBroadcastProducer { request: &MoqTrackRequest, init: MoqInit, ) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.state.lock().unwrap(); let state = guard.as_ref().ok_or_else(|| MoqError::Closed)?; @@ -419,7 +419,7 @@ impl MoqBroadcastProducer { /// (avc3, hev1, av01, fmp4, mkv). [`MoqInit`] carries the format, any /// seed bytes, and catalog hints. pub fn publish_media_stream(&self, init: MoqInit) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.state.lock().unwrap(); let state = guard.as_ref().ok_or_else(|| MoqError::Closed)?; @@ -458,7 +458,7 @@ impl MoqBroadcastProducer { /// bytes written directly to moq-lite groups with no media framing. `info` sets /// track properties (priority, latency_max, timescale); omit for defaults. pub fn publish_track(&self, name: String, info: Option) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.state.lock().unwrap(); let state = guard.as_ref().ok_or_else(|| MoqError::Closed)?; let info = raw_track_info(info)?; @@ -473,7 +473,7 @@ impl MoqBroadcastProducer { /// Finish this publisher, finalizing the catalog stream and cleanly closing the /// broadcast so subscribers see a normal end rather than `Error::Dropped`. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.state.lock().unwrap(); let mut state = guard.take().ok_or(MoqError::Closed)?; // Finish the broadcast first so the clean end reaches subscribers even if @@ -564,7 +564,7 @@ impl MoqGroupRequest { /// Accept the request and return a producer for filling the fetched group. pub fn accept(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let group = self.take()?.accept(None)?; Ok(Arc::new(MoqGroupProducer { sequence: group.sequence, @@ -574,7 +574,7 @@ impl MoqGroupRequest { /// Reject the fetch with an application error code. pub fn abort(&self, error_code: u16) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); self.take()?.reject(moq_net::Error::App(error_code)); Ok(()) } @@ -621,7 +621,7 @@ impl MoqTrackRequest { /// requested by a fetch. This keeps the pending group request serviceable across /// the transition from request to producer. pub fn dynamic(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.inner.lock().unwrap(); let request = guard.as_ref().ok_or(MoqError::Closed)?; Ok(Arc::new(MoqTrackDynamic::new(request.dynamic()))) @@ -632,7 +632,7 @@ impl MoqTrackRequest { /// For media use [`MoqBroadcastProducer::publish_media_on_track`] instead, which lets /// the importer pick the timescale. pub fn accept(&self, info: Option) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let info = raw_track_info(info)?; let request = self.take()?; Ok(Arc::new(MoqTrackProducer { @@ -642,7 +642,7 @@ impl MoqTrackRequest { /// Reject the request with an application error code, failing the waiting subscriber. pub fn abort(&self, error_code: u16) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let request = self.take()?; request.reject(moq_net::Error::App(error_code)); Ok(()) @@ -660,7 +660,7 @@ pub struct MoqTrackProducer { impl MoqTrackProducer { /// Return the name of this track. pub fn name(&self) -> Result { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.inner.lock().unwrap(); let track = guard.as_ref().ok_or(MoqError::Closed)?; Ok(track.name().to_string()) @@ -671,7 +671,7 @@ impl MoqTrackProducer { /// Hold the returned object for as long as cache misses should wait to be /// served. Without a live dynamic handler, a missing group fails with `NotFound`. pub fn dynamic(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.inner.lock().unwrap(); let track = guard.as_ref().ok_or(MoqError::Closed)?; Ok(Arc::new(MoqTrackDynamic::new(track.dynamic()))) @@ -680,21 +680,13 @@ impl MoqTrackProducer { /// Wait until this track has at least one active consumer. pub async fn used(&self) -> Result<(), MoqError> { let track = self.inner.lock().unwrap().as_ref().ok_or(MoqError::Closed)?.clone(); - match crate::ffi::RUNTIME.spawn(async move { track.used().await }).await { - Ok(result) => result.map_err(Into::into), - Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), - Err(e) => Err(MoqError::Task(e)), - } + crate::ffi::detached(async move { track.used().await }).await } /// Wait until this track has no active consumers. pub async fn unused(&self) -> Result<(), MoqError> { let track = self.inner.lock().unwrap().as_ref().ok_or(MoqError::Closed)?.clone(); - match crate::ffi::RUNTIME.spawn(async move { track.unused().await }).await { - Ok(result) => result.map_err(Into::into), - Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), - Err(e) => Err(MoqError::Task(e)), - } + crate::ffi::detached(async move { track.unused().await }).await } /// Create a consumer that reads from this producer's track. @@ -702,7 +694,7 @@ impl MoqTrackProducer { /// Useful for local pub/sub without going through an origin/broadcast. `subscription` /// tunes delivery priority, group ordering priority, and group range; omit for defaults. pub fn consume(&self, subscription: Option) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.inner.lock().unwrap(); let track = guard.as_ref().ok_or(MoqError::Closed)?; let subscription = subscription.map(moq_net::track::Subscription::from); @@ -711,7 +703,7 @@ impl MoqTrackProducer { /// Append a new group to the track, returning a producer for writing frames into it. pub fn append_group(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let track = guard.as_mut().ok_or(MoqError::Closed)?; let group = track.append_group()?; @@ -726,7 +718,7 @@ impl MoqTrackProducer { /// Use this for sparse or replayed tracks. [`append_group`](Self::append_group) /// remains the convenient live-stream path. pub fn create_group(&self, sequence: u64) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let track = guard.as_mut().ok_or(MoqError::Closed)?; let group = track.create_group(moq_net::group::Info { sequence })?; @@ -741,7 +733,7 @@ impl MoqTrackProducer { /// Raw tracks default to a microsecond timescale. Custom timescales may round /// the timestamp during conversion. pub fn write_frame(&self, frame: MoqFrame) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let timestamp = moq_net::Timestamp::from_micros(frame.timestamp_us)?; let mut guard = self.inner.lock().unwrap(); let track = guard.as_mut().ok_or(MoqError::Closed)?; @@ -754,7 +746,7 @@ impl MoqTrackProducer { /// The payload must be at most 1200 bytes. Datagrams are only delivered on transports and /// wire versions with a datagram channel; there is no stream fallback. pub fn append_datagram(&self, frame: MoqFrame) -> Result { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let timestamp = moq_net::Timestamp::from_micros(frame.timestamp_us)?; let mut guard = self.inner.lock().unwrap(); let track = guard.as_mut().ok_or(MoqError::Closed)?; @@ -763,7 +755,7 @@ impl MoqTrackProducer { /// Abort this track with an application error code. pub fn abort(&self, error_code: u16) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let track = guard.take().ok_or(MoqError::Closed)?; track.abort(moq_net::Error::App(error_code))?; @@ -775,7 +767,7 @@ impl MoqTrackProducer { /// [`finish_at`](Self::finish_at) declares the boundary ahead of time, so this keeps /// that boundary and only releases the producer. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let mut track = guard.take().ok_or(MoqError::Closed)?; if track.final_sequence().is_none() { @@ -790,7 +782,7 @@ impl MoqTrackProducer { /// above it are rejected. The producer remains open for groups below the boundary; /// call [`finish`](Self::finish) after producing the remaining groups. pub fn finish_at(&self, final_sequence: u64) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let track = guard.as_mut().ok_or(MoqError::Closed)?; track.finish_at(final_sequence)?; @@ -813,7 +805,7 @@ impl MoqGroupProducer { /// Create a consumer that reads frames from this group. pub fn consume(&self) -> Result, MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.inner.lock().unwrap(); let group = guard.as_ref().ok_or_else(|| MoqError::Closed)?; Ok(Arc::new(MoqGroupConsumer::new(group.consume()))) @@ -824,7 +816,7 @@ impl MoqGroupProducer { /// Raw tracks default to a microsecond timescale. Custom timescales may round /// the timestamp during conversion. pub fn write_frame(&self, frame: MoqFrame) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let timestamp = moq_net::Timestamp::from_micros(frame.timestamp_us)?; let mut guard = self.inner.lock().unwrap(); let group = guard.as_mut().ok_or_else(|| MoqError::Closed)?; @@ -834,7 +826,7 @@ impl MoqGroupProducer { /// Mark the group as complete. No more frames can be written. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let mut group = guard.take().ok_or_else(|| MoqError::Closed)?; group.finish()?; @@ -843,7 +835,7 @@ impl MoqGroupProducer { /// Abort this group with an application error code. pub fn abort(&self, error_code: u16) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let group = guard.take().ok_or(MoqError::Closed)?; group.abort(moq_net::Error::App(error_code))?; @@ -859,7 +851,7 @@ impl MoqMediaProducer { /// /// Errors for a multi-track container source, which has no single track name. pub fn name(&self) -> Result { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let guard = self.inner.lock().unwrap(); let media = guard.as_ref().ok_or_else(|| MoqError::Closed)?; let demand = media @@ -882,11 +874,7 @@ impl MoqMediaProducer { .demand .clone() .ok_or_else(|| MoqError::Codec("demand unavailable for a multi-track container".into()))?; - match crate::ffi::RUNTIME.spawn(async move { demand.used().await }).await { - Ok(result) => result.map_err(Into::into), - Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), - Err(e) => Err(MoqError::Task(e)), - } + crate::ffi::detached(async move { demand.used().await }).await } /// Wait until this media track has no active consumers. @@ -902,11 +890,7 @@ impl MoqMediaProducer { .demand .clone() .ok_or_else(|| MoqError::Codec("demand unavailable for a multi-track container".into()))?; - match crate::ffi::RUNTIME.spawn(async move { demand.unused().await }).await { - Ok(result) => result.map_err(Into::into), - Err(e) if e.is_cancelled() => Err(MoqError::Cancelled), - Err(e) => Err(MoqError::Task(e)), - } + crate::ffi::detached(async move { demand.unused().await }).await } /// Write `frame` to this media track. @@ -914,7 +898,7 @@ impl MoqMediaProducer { /// The importer derives keyframe status from the bitstream, so a [`MoqFrame`] carries only /// the payload and its timestamp. pub fn write_frame(&self, frame: MoqFrame) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let media = guard.as_mut().ok_or_else(|| MoqError::Closed)?; @@ -929,7 +913,7 @@ impl MoqMediaProducer { /// Finish this media track and finalize encoding. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let mut media = guard.take().ok_or_else(|| MoqError::Closed)?; media @@ -946,7 +930,7 @@ impl MoqMediaStreamProducer { /// frames whole access units and keeps any partial trailing frame for the /// next call, so callers can write arbitrary chunks. pub fn write(&self, payload: Vec) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let media = guard.as_mut().ok_or_else(|| MoqError::Closed)?; @@ -963,7 +947,7 @@ impl MoqMediaStreamProducer { /// arrives, so a trailing access unit with no following delimiter (e.g. the /// last frame at EOF) is not emitted. This matches moq-cli's stdin path. pub fn finish(&self) -> Result<(), MoqError> { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let mut media = guard.take().ok_or_else(|| MoqError::Closed)?; media diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 70a1be18a4e..8ba57a98a0e 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -6,12 +6,15 @@ use crate::error::MoqError; use crate::ffi::Task; use crate::origin::{MoqOriginConsumer, MoqOriginProducer}; +/// Native QUIC/WebTransport client configuration. +#[cfg(not(target_arch = "wasm32"))] struct Client { config: moq_native::ClientConfig, publish: Option>, consume: Option>, } +#[cfg(not(target_arch = "wasm32"))] impl Client { async fn connect(&self, url: Url) -> Result, MoqError> { let client = self.config.clone().init().map_err(map_connect_error)?; @@ -31,6 +34,7 @@ impl Client { } } +#[cfg(not(target_arch = "wasm32"))] fn map_connect_error(err: moq_native::Error) -> MoqError { match err.connect_error() { Some(moq_native::ConnectError::Unauthorized) => MoqError::Unauthorized, @@ -39,10 +43,59 @@ fn map_connect_error(err: moq_native::Error) -> MoqError { } } -#[cfg(test)] +#[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::*; + const VALID: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + #[test] + fn decodes_a_sha256_fingerprint() { + let bytes = decode_hex(VALID).unwrap(); + assert_eq!(bytes.len(), 32); + assert_eq!(bytes[0], 0x01); + + // Colons are the other shape `MoqServer::cert_fingerprints` and openssl print. + let colons = VALID + .as_bytes() + .chunks(2) + .map(|c| std::str::from_utf8(c).unwrap()) + .collect::>() + .join(":"); + assert_eq!(decode_hex(&colons).unwrap(), bytes); + } + + /// Byte-index slicing used to land inside a multi-byte character and panic, which an + /// FFI caller could reach with any non-ASCII string of even byte length. + #[test] + fn rejects_non_ascii_instead_of_panicking() { + for input in ["aéa", "é", "ééééééééééééééééééééééééééééééé"] { + assert!(matches!(decode_hex(input), Err(MoqError::Connect(_))), "{input}"); + } + } + + #[test] + fn rejects_a_wrong_length_fingerprint() { + assert!(decode_hex("").is_err()); + assert!(decode_hex("abcd").is_err()); + assert!(decode_hex(&VALID[..62]).is_err()); + assert!(decode_hex(&format!("{VALID}ab")).is_err()); + } + + #[test] + fn rejects_non_hex_digits() { + for bad in [ + format!("zz{}", &VALID[2..]), + // `u8::from_str_radix` accepts a leading sign, so an unchecked chunk would + // decode "+a" to 10 and silently yield a fingerprint matching no certificate. + format!("+0{}", &VALID[2..]), + format!("{}+0", &VALID[..62]), + format!(" 0{}", &VALID[2..]), + ] { + assert!(matches!(decode_hex(&bad), Err(MoqError::Connect(_))), "{bad}"); + } + } + #[test] fn maps_native_auth_connect_errors() { assert!(matches!( @@ -90,17 +143,157 @@ mod tests { } } +/// Browser WebTransport client configuration. +/// +/// The browser owns the socket and the trust store, so none of the native TLS knobs +/// (roots, mTLS, bind address) have an equivalent. Certificate hashes are the one +/// thing WebTransport does expose. +#[cfg(target_arch = "wasm32")] +struct Client { + fingerprints: Vec>, + publish: Option>, + consume: Option>, +} + +#[cfg(target_arch = "wasm32")] +impl Client { + async fn connect(&self, url: Url) -> Result, MoqError> { + let (publish, subscribe) = crate::origin::resolve_pair(self.publish.as_ref(), self.consume.as_ref()); + + let transport = match self.fingerprints.is_empty() { + true => crate::transport::connect(url).await, + false => crate::transport::connect_with_hashes(url, self.fingerprints.clone()).await, + } + .map_err(|err| MoqError::Connect(format!("{err}")))?; + + let (session, driver) = moq_net::Client::new() + .with_publisher(&publish) + .with_subscriber(subscribe.clone()) + .connect(transport) + .await?; + + // The session only progresses while the driver runs. The driver holds no session + // clone, so dropping the last handle still closes the transport and ends this task. + web_async::spawn(async move { + let _ = driver.await; + }); + + Ok(Arc::new(MoqSession::new(session, publish, subscribe))) + } +} + +/// Decode a hex-encoded SHA-256 certificate fingerprint into raw bytes. +/// +/// Not gated on wasm alone so the native test suite covers it: this parses a string an +/// FFI caller controls, and nothing in this repo runs a wasm test. +#[cfg(any(target_arch = "wasm32", test))] +fn decode_hex(hex: &str) -> Result, MoqError> { + /// A sha-256 digest is 32 bytes, so 64 hex characters. + const LEN: usize = 64; + + let hex = hex.replace(':', ""); + + // This string comes straight from the caller, and one check covers two hazards: + // non-ASCII would let a 2-byte chunk split a character, and `from_str_radix` accepts + // a leading sign, so an unchecked "+a" would decode to 10 rather than erroring. + if !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(MoqError::Connect(format!("fingerprint is not hex: {hex}"))); + } + + // WebTransport's `serverCertificateHashes` only accepts a 32-byte sha-256, so a + // different length can never match a certificate. Rejecting here beats failing + // opaquely inside the browser. + if hex.len() != LEN { + return Err(MoqError::Connect(format!( + "expected a {LEN}-character sha-256 fingerprint, got {}", + hex.len() + ))); + } + + hex.as_bytes() + .chunks(2) + .map(|pair| { + let pair = std::str::from_utf8(pair).expect("checked ascii above"); + u8::from_str_radix(pair, 16).map_err(|err| MoqError::Connect(format!("{err}"))) + }) + .collect() +} + +/// Builds a [`MoqSession`]: configure it, then [`connect`](Self::connect). +/// +/// The configuration differs by target, because the transport does. Native builds expose +/// the QUIC socket and TLS trust store; the browser owns both, so a wasm build exposes +/// only the certificate hashes WebTransport accepts. #[derive(uniffi::Object)] pub struct MoqClient { task: Task, } +#[cfg(target_arch = "wasm32")] +#[uniffi::export] +impl MoqClient { + /// Create a new MoQ client with default configuration. + #[uniffi::constructor] + pub fn new() -> Arc { + Arc::new(Self { + task: Task::new(Client { + fingerprints: Vec::new(), + publish: None, + consume: None, + }), + }) + } + + /// Pin the peer to a certificate with one of these SHA-256 fingerprints, encoded as hex. + /// + /// Passed through to WebTransport's `serverCertificateHashes`. An empty list restores + /// the browser's normal certificate verification. + pub fn set_tls_fingerprints(&self, fingerprints: Vec) -> Result<(), MoqError> { + let parsed = fingerprints + .iter() + .map(|hex| decode_hex(hex)) + .collect::, _>>()?; + if let Some(mut state) = self.task.lock() { + state.fingerprints = parsed; + } + Ok(()) + } + + /// Set the origin to publish local broadcasts to the remote. + pub fn set_publish(&self, origin: Option>) { + if let Some(mut state) = self.task.lock() { + state.publish = origin; + } + } + + /// Set the origin to consume remote broadcasts from the remote. + pub fn set_consume(&self, origin: Option>) { + if let Some(mut state) = self.task.lock() { + state.consume = origin; + } + } + + /// Connect to a MoQ server and wait for the session to be established. + /// + /// Can be cancelled by calling `cancel()`. + pub async fn connect(&self, url: String) -> Result, MoqError> { + let url = Url::parse(&url)?; + self.task.run(|state| async move { state.connect(url).await }).await + } + + /// Cancel all current and future `connect()` calls. + pub fn cancel(&self) { + self.task.cancel(); + } +} + +#[cfg(not(target_arch = "wasm32"))] #[uniffi::export] impl MoqClient { /// Create a new MoQ client with default configuration. #[uniffi::constructor] pub fn new() -> Arc { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); Arc::new(Self { task: Task::new(Client { config: moq_native::ClientConfig::default(), @@ -295,7 +488,7 @@ impl MoqSession { impl Drop for MoqSession { fn drop(&mut self) { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); // Close the transport while the runtime is entered. The backend spawns a // lingering CLOSE task, which panics (aborting under panic=abort) if no reactor // is in context. We can't leave this to the last `Session` clone's drop: that @@ -319,7 +512,7 @@ impl MoqSession { /// Close the session with the given error code. pub fn cancel(&self, code: u32) { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); if let Some(inner) = &self.inner { inner.abort(moq_net::Error::Remote(code)); } @@ -359,7 +552,7 @@ impl MoqSession { /// Individual fields are `None` when the transport backend doesn't report /// them; see [`MoqConnectionStats`]. pub fn stats(&self) -> MoqConnectionStats { - let _guard = crate::ffi::RUNTIME.enter(); + let _guard = crate::ffi::enter(); self.inner .as_ref() .map(moq_net::Session::stats) diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 9a6510f5605..2a719f0976d 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -1044,6 +1044,7 @@ async fn video_publish_consume() { /// The raw-video publish path: hand mid-gray RGBA to `publish_video` and check /// that the encoder's own output reaches a subscriber, described by a catalog /// rendition the importer built from the encoded keyframe. +#[cfg(feature = "video")] #[tokio::test] async fn video_raw_publish_consume() { use crate::video::*; @@ -1146,6 +1147,7 @@ async fn video_raw_publish_consume() { /// whichever thread dropped the object. The confinement itself is asserted in /// moq-video (`encode::sink`); this pins that the binding supports the usage /// end to end, including the drain on `finish`. +#[cfg(feature = "video")] #[tokio::test] async fn video_raw_publish_from_many_threads() { use crate::video::*; @@ -1221,6 +1223,7 @@ async fn video_raw_publish_from_many_threads() { /// A raw video producer rejects a buffer that isn't one picture at the /// configured resolution, rather than reinterpreting it, and rejects any write /// after `finish`. +#[cfg(feature = "video")] #[tokio::test] async fn video_raw_publish_rejects_bad_frames() { use crate::video::*; diff --git a/rs/moq-ffi/src/transport.rs b/rs/moq-ffi/src/transport.rs new file mode 100644 index 00000000000..6bf4b7f8606 --- /dev/null +++ b/rs/moq-ffi/src/transport.rs @@ -0,0 +1,163 @@ +//! Adapt `web-transport-wasm` (the browser WebTransport API) to the +//! `web-transport-trait` abstraction that `moq-net` is generic over. +//! +//! Both the trait and the wasm types are foreign, so the orphan rule forces +//! newtype wrappers. The shapes line up closely; the only real impedance +//! mismatches are documented inline (sync-vs-async datagrams, byte-count vs +//! write-all, string-vs-code stream resets). + +use bytes::Bytes; +use url::Url; +use web_transport_trait as wtt; + +/// A connected browser WebTransport session, usable by `moq-net`. +#[derive(Clone)] +pub struct Session(web_transport_wasm::Session); + +/// Advertise every version `moq-net` supports, so the peer picks one via ALPN. +/// Without this the browser negotiates no subprotocol and the session has no +/// version to start from. +fn builder() -> web_transport_wasm::ClientBuilder { + web_transport_wasm::ClientBuilder::new().with_protocols(moq_net::ALPNS.iter().copied()) +} + +/// Open a browser WebTransport connection to `url`. +pub async fn connect(url: Url) -> Result { + let client = builder().with_system_roots(); + let session = client.connect(url).await.map_err(Error)?; + Ok(Session(session)) +} + +/// Connect, trusting only the given sha-256 certificate hashes (serverless dev, +/// matching the browser's `serverCertificateHashes` option). +pub async fn connect_with_hashes(url: Url, hashes: Vec>) -> Result { + let client = builder().with_server_certificate_hashes(hashes); + let session = client.connect(url).await.map_err(Error)?; + Ok(Session(session)) +} + +/// The writable half of a browser WebTransport stream. +pub struct SendStream(web_transport_wasm::SendStream); +/// The readable half of a browser WebTransport stream. +pub struct RecvStream(web_transport_wasm::RecvStream); + +/// Wraps `web_transport_wasm::Error` so we can implement the foreign error trait. +#[derive(Debug, Clone, thiserror::Error)] +#[error(transparent)] +pub struct Error(web_transport_wasm::Error); + +impl wtt::Error for Error { + fn session_error(&self) -> Option<(u32, String)> { + self.0.code().map(|code| (code as u32, self.0.to_string())) + } + + fn stream_error(&self) -> Option { + self.0.code().map(|c| c as u32) + } +} + +impl wtt::Session for Session { + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = Error; + + async fn accept_uni(&self) -> Result { + Ok(RecvStream(self.0.accept_uni().await.map_err(Error)?)) + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + let (s, r) = self.0.accept_bi().await.map_err(Error)?; + Ok((SendStream(s), RecvStream(r))) + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + let (s, r) = self.0.open_bi().await.map_err(Error)?; + Ok((SendStream(s), RecvStream(r))) + } + + async fn open_uni(&self) -> Result { + Ok(SendStream(self.0.open_uni().await.map_err(Error)?)) + } + + fn send_datagram(&self, payload: Bytes) -> Result<(), Self::Error> { + // The browser datagram API is async; the trait is sync. moq drives all + // control/media over streams, so fire-and-forget is acceptable here. + let session = self.0.clone(); + web_async::spawn(async move { + let _ = session.send_datagram(payload).await; + }); + Ok(()) + } + + async fn recv_datagram(&self) -> Result { + self.0.recv_datagram().await.map_err(Error) + } + + fn max_datagram_size(&self) -> usize { + // The browser doesn't expose this; use the conservative QUIC default. + 1200 + } + + fn protocol(&self) -> Option<&str> { + // The browser reports "" when no subprotocol was negotiated, which means the + // same thing as `None`: fall back to negotiating the version over SETUP. + self.0.protocol().filter(|p| !p.is_empty()) + } + + fn close(&self, code: u32, reason: &str) { + self.0.close(code, reason); + } + + async fn closed(&self) -> Self::Error { + Error(self.0.closed().await) + } +} + +impl wtt::SendStream for SendStream { + type Error = Error; + + async fn write(&mut self, buf: &[u8]) -> Result { + // The wasm writer writes the whole slice or errors. + self.0.write(buf).await.map_err(Error)?; + Ok(buf.len()) + } + + fn set_priority(&mut self, order: u8) { + self.0.set_priority(order as i32); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + self.0.finish().map_err(Error) + } + + fn reset(&mut self, code: u32) { + self.0.reset(&code.to_string()); + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + self.0.closed().await.map(|_| ()).map_err(Error) + } +} + +impl wtt::RecvStream for RecvStream { + type Error = Error; + + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + match self.0.read(dst.len()).await.map_err(Error)? { + Some(chunk) => { + let n = chunk.len(); + dst[..n].copy_from_slice(&chunk); + Ok(Some(n)) + } + None => Ok(None), + } + } + + fn stop(&mut self, code: u32) { + self.0.stop(&code.to_string()); + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + self.0.closed().await.map(|_| ()).map_err(Error) + } +} diff --git a/rs/moq-wasm/Cargo.toml b/rs/moq-wasm/Cargo.toml index 294b2ecad6f..04a24cc472c 100644 --- a/rs/moq-wasm/Cargo.toml +++ b/rs/moq-wasm/Cargo.toml @@ -27,7 +27,7 @@ crate-type = ["cdylib", "rlib"] [target.'cfg(target_arch = "wasm32")'.dependencies] bytes = "1" console_error_panic_hook = "0.1" -getrandom = { version = "0.4", features = ["wasm_js"] } # wasm_js backend for rand-via-moq-net; cfg flag in .cargo/config.toml +getrandom = { workspace = true } # wasm_js backend for rand-via-moq-net; cfg flag in .cargo/config.toml js-sys = "0.3" moq-net = { workspace = true } thiserror = "2" @@ -39,4 +39,4 @@ wasm-bindgen = "=0.2.121" wasm-bindgen-futures = "0.4" web-async = { workspace = true } web-transport-trait = { workspace = true } -web-transport-wasm = "0.5" +web-transport-wasm = { workspace = true } From 2ba435e1a1806f0c12aec6a32d77299b7f50a0e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 18 Aug 2026 14:55:11 -0700 Subject: [PATCH 22/29] fix(js/watch): hide renditions whose broadcast isn't announced (#2918) Co-authored-by: Claude Opus 5 --- js/watch/src/broadcast.test.ts | 108 +++++++++++++++++++++++++++++++++ js/watch/src/broadcast.ts | 76 +++++++++++++++++------ 2 files changed, 164 insertions(+), 20 deletions(-) create mode 100644 js/watch/src/broadcast.test.ts diff --git a/js/watch/src/broadcast.test.ts b/js/watch/src/broadcast.test.ts new file mode 100644 index 00000000000..5015e00c884 --- /dev/null +++ b/js/watch/src/broadcast.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "bun:test"; +import * as Catalog from "@moq/hang/catalog"; +import type * as Moq from "@moq/net"; +import { Path } from "@moq/net"; +import { Signal } from "@moq/signals"; +import { Broadcast } from "./broadcast"; + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +async function settle(): Promise { + for (let i = 0; i < 5; i++) await flush(); +} + +type Announcement = { path: Moq.Path.Valid; active: boolean }; + +// A connection whose announcement stream the test drives by hand. +class FakeAnnounced { + #pending: Announcement[] = []; + #waiting: Array<(entry: Announcement | undefined) => void> = []; + + push(entry: Announcement): void { + const waiting = this.#waiting.shift(); + if (waiting) waiting(entry); + else this.#pending.push(entry); + } + + next(): Promise { + const pending = this.#pending.shift(); + if (pending) return Promise.resolve(pending); + return new Promise((resolve) => this.#waiting.push(resolve)); + } + + close(): void {} +} + +function mockConnection(announced: FakeAnnounced): Moq.Connection.Established { + return { + discovery: true, + announced: () => announced, + announcedBroadcast: () => ({ active: new Signal(undefined), close: () => {} }), + } as unknown as Moq.Connection.Established; +} + +function video(codec: string, fields: Record = {}): Catalog.VideoConfig { + return Catalog.VideoConfigSchema.parse({ codec, container: { kind: "legacy" }, ...fields }); +} + +function renditions(broadcast: Broadcast): string[] { + return Object.keys(broadcast.out.catalog.peek()?.video?.renditions ?? {}).sort(); +} + +describe("Broadcast cross-broadcast renditions", () => { + it("hides a rendition until its broadcast is announced", async () => { + // A transcoder under `public/` referencing a source under `private/`: a viewer scoped to + // `public/` is never told the source exists, and selecting it would render nothing. + const announced = new FakeAnnounced(); + const broadcast = new Broadcast({ + enabled: true, + connection: new Signal(mockConnection(announced)), + name: Path.from("public/transcode.hang"), + catalogFormat: "manual", + catalog: { + video: { + renditions: { + local: video("avc1.64001e", { bitrate: 1_000_000 }), + remote: video("avc1.640028", { bitrate: 9_000_000, broadcast: "../private/source" }), + }, + }, + }, + }); + + await settle(); + expect(renditions(broadcast)).toEqual(["local"]); + + // The source is announced, so it becomes usable without a new catalog. + announced.push({ path: Path.from("private/source"), active: true }); + await settle(); + expect(renditions(broadcast)).toEqual(["local", "remote"]); + + // ...and withdrawn again when it goes away. + announced.push({ path: Path.from("private/source"), active: false }); + await settle(); + expect(renditions(broadcast)).toEqual(["local"]); + + broadcast.close(); + }); +}); + +describe("Broadcast manual catalog", () => { + it("republishes a manual catalog mutated in place", async () => { + const catalog = new Signal({ + video: { renditions: { one: video("avc1.64001e") } }, + }); + const broadcast = new Broadcast({ enabled: true, catalogFormat: "manual", catalog }); + + await settle(); + expect(renditions(broadcast)).toEqual(["one"]); + + // The input is a signal the caller owns, so it can be updated in place. + catalog.mutate((c) => { + if (c.video) c.video.renditions.two = video("avc1.640028"); + }); + await settle(); + expect(renditions(broadcast)).toEqual(["one", "two"]); + + broadcast.close(); + }); +}); diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index 284d5cac1ba..2ff6393efdd 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -28,25 +28,24 @@ type ReferencedRendition = { broadcast?: string; }; +// Either the catalog's own broadcast, or a sibling to consume by path. +type RelativeTarget = { local: true } | { local: false; path: Moq.Path.Valid }; + function filterRenditions( - base: Moq.Path.Valid, renditions: Record, + usable: (rel: string | undefined) => boolean, ): Record { - return Object.fromEntries( - Object.entries(renditions).filter(([, config]) => - config.broadcast === undefined ? true : Path.tryResolve(base, config.broadcast) !== undefined, - ), - ); + return Object.fromEntries(Object.entries(renditions).filter(([, config]) => usable(config.broadcast))); } -function filterCatalog(base: Moq.Path.Valid, catalog: Catalog.Root): Catalog.Root { +function filterCatalog(catalog: Catalog.Root, usable: (rel: string | undefined) => boolean): Catalog.Root { return { ...catalog, video: catalog.video - ? { ...catalog.video, renditions: filterRenditions(base, catalog.video.renditions) } + ? { ...catalog.video, renditions: filterRenditions(catalog.video.renditions, usable) } : undefined, audio: catalog.audio - ? { ...catalog.audio, renditions: filterRenditions(base, catalog.audio.renditions) } + ? { ...catalog.audio, renditions: filterRenditions(catalog.audio.renditions, usable) } : undefined, }; } @@ -97,7 +96,9 @@ type BroadcastOutput = { status: Signal; active: Signal; - // The effective catalog: the fetched one, or a copy of input.catalog in manual mode. + // The effective catalog: the fetched one, or a copy of input.catalog in manual mode, minus + // any rendition this consumer can't use (see `#runFiltered`). A rendition referencing another + // broadcast appears once that broadcast is announced, so this can change without a new catalog. catalog: Signal; }; @@ -122,6 +123,14 @@ export class Broadcast { // no cross-broadcast renditions never opens the (broad) connection-scoped announcement stream. readonly #wantAnnounced = new Signal(false); + // The catalog as published, before renditions this consumer cannot use are dropped. Kept + // separate so the effective catalog can be re-derived when a reference becomes reachable. + // + // Writes notify unconditionally. A manual catalog can be mutated in place, and the rerun that + // delivers it lands inside the same flush, where a value-compare coalesces the write away and + // strands the filtered copy on the previous contents. + readonly #raw = new Signal(undefined); + #signals = new Effect(); constructor(props?: Inputs) { @@ -137,6 +146,7 @@ export class Broadcast { this.#signals.run(this.#runAnnounced.bind(this)); this.#signals.run(this.#runBroadcast.bind(this)); this.#signals.run(this.#runCatalog.bind(this)); + this.#signals.run(this.#runFiltered.bind(this)); } // Maintain the set of announced paths used by `relativeBroadcast`, by draining a connection-scoped @@ -168,6 +178,16 @@ export class Broadcast { }); } + // Publish the catalog minus every rendition this consumer cannot use: one whose reference + // escapes above the root, or names a broadcast that isn't announced to us. Selecting one of + // those would render nothing, since `relativeBroadcast` resolves it to no broadcast at all. + // Reruns as announcements arrive, so a rendition appears once its broadcast does. + #runFiltered(effect: Effect): void { + const raw = effect.get(this.#raw); + const usable = (rel: string | undefined) => this.#relativeTarget(effect, rel) !== undefined; + effect.set(this.#out.catalog, raw ? filterCatalog(raw, usable) : undefined); + } + // Whether `path` is currently announced, for `relativeBroadcast`'s cross-broadcast refs. Returns // true (subscribe immediately) when the gate can't apply: reload is off, or the relay doesn't // support discovery. Opens the announcement stream on first use. @@ -175,7 +195,10 @@ export class Broadcast { if (!effect.get(this.in.reload)) return true; const conn = effect.get(this.in.connection); - if (conn && skipDiscovery(conn)) return true; + // Nothing to ask yet. `relativeBroadcast` still bails on the missing connection, and this + // keeps a reconnect from briefly hiding every cross-broadcast rendition from selection. + if (!conn) return true; + if (skipDiscovery(conn)) return true; this.#wantAnnounced.set(true); @@ -226,7 +249,8 @@ export class Broadcast { if (format === "manual") { // Mirror the caller-supplied catalog into the effective output. const catalog = effect.get(this.in.catalog); - effect.set(this.#out.catalog, catalog ? filterCatalog(name, catalog) : undefined, undefined); + this.#raw.set(catalog, true); + effect.cleanup(() => this.#raw.set(undefined, true)); this.#out.status.set(catalog ? "live" : "loading"); return; } @@ -264,13 +288,13 @@ export class Broadcast { console.debug("received catalog", format, this.in.name.peek(), update); - this.#out.catalog.set(filterCatalog(name, update)); + this.#raw.set(update, true); this.#out.status.set("live"); } } catch (err) { console.warn("error fetching catalog", this.in.name.peek(), err); } finally { - this.#out.catalog.set(undefined); + this.#raw.set(undefined); this.#out.status.set("offline"); } }); @@ -287,8 +311,11 @@ export class Broadcast { * reference resolves lazily and reacts to `enabled` / connection / announcement * changes exactly like the catalog broadcast. */ - relativeBroadcast(effect: Effect, rel: string | undefined): Moq.Broadcast.Consumer | undefined { - if (!rel) return effect.get(this.out.active); + // Where a rendition's `broadcast` reference points once resolved and gated on the announcement + // stream, or `undefined` when it names nothing consumable right now. Playback and rendition + // selection both go through this so they cannot disagree about what is reachable. + #relativeTarget(effect: Effect, rel: string | undefined): RelativeTarget | undefined { + if (!rel) return { local: true }; const base = effect.get(this.in.name); const resolved = Path.tryResolve(base, rel); @@ -296,16 +323,25 @@ export class Broadcast { // Ignore a rendition whose reference escapes above the root. A valid empty result // names the root broadcast, while a reference back to the catalog uses its active handle. if (resolved === undefined) return undefined; - if (resolved === base) return effect.get(this.out.active); + if (resolved === base) return { local: true }; + + if (!this.#isPathAnnounced(effect, resolved)) return undefined; + + return { local: false, path: resolved }; + } + + /** Resolve a rendition's `broadcast` reference to a consumer, or `undefined` if unreachable. */ + relativeBroadcast(effect: Effect, rel: string | undefined): Moq.Broadcast.Consumer | undefined { + const target = this.#relativeTarget(effect, rel); + if (!target) return undefined; + if (target.local) return effect.get(this.out.active); if (!effect.get(this.in.enabled)) return undefined; const conn = effect.get(this.in.connection); if (!conn) return undefined; - if (!this.#isPathAnnounced(effect, resolved)) return undefined; - - const broadcast = conn.consume(resolved); + const broadcast = conn.consume(target.path); effect.cleanup(() => broadcast.close()); return broadcast; } From 70acf4c50a5345e41bd432983c92bf5ef4e04c14 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 19 Aug 2026 12:06:33 -0700 Subject: [PATCH 23/29] docs: describe the iroh P2P transport (#2938) Co-authored-by: Claude Opus 5 --- doc/.vitepress/config.ts | 1 + doc/bin/relay/config.md | 17 +++- doc/concept/layer/index.md | 9 ++ doc/concept/layer/iroh.md | 194 +++++++++++++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 doc/concept/layer/iroh.md diff --git a/doc/.vitepress/config.ts b/doc/.vitepress/config.ts index 0d67370f623..1100685a51a 100644 --- a/doc/.vitepress/config.ts +++ b/doc/.vitepress/config.ts @@ -98,6 +98,7 @@ export default defineConfig({ { text: "quic", link: "/concept/layer/quic" }, { text: "web-transport", link: "/concept/layer/web-transport" }, { text: "web-socket", link: "/concept/layer/web-socket" }, + { text: "iroh", link: "/concept/layer/iroh" }, { text: "moq-lite", link: "/concept/layer/moq-lite" }, { text: "hang", link: "/concept/layer/hang" }, ], diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 721d08808ed..25d9f3d23e5 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -522,17 +522,30 @@ All three flags also accept CLI arguments (`--cache-capacity`, ### \[iroh] -Experimental P2P support via iroh. +Experimental P2P support via [iroh](/concept/layer/iroh). Clients dial the relay's +endpoint id (`iroh:///`) instead of a hostname, with hole punching +and a relay fallback for peers that can't be reached directly. ```toml [iroh] # Enable iroh for P2P connections enabled = false -# Path to persist the iroh secret key +# Path to persist the iroh secret key, so the endpoint id survives a restart. +# Generated on first run if the file does not exist. secret = "./relay-iroh-secret.key" + +# UDP bind addresses. Default to an ephemeral port on both families. +# bind_v4 = "0.0.0.0:4444" +# bind_v6 = "[::]:4444" + +# Require a direct path, skipping the iroh relay fallback. +# disable_relay = false ``` +The endpoint id is logged at startup (`iroh listening endpoint_id=...`). Without `secret` +a fresh key is generated on every start, so the id changes each restart. + ## Example Configurations See the [`demo/relay/`](https://github.com/moq-dev/moq/tree/main/demo/relay) directory for working configuration files: diff --git a/doc/concept/layer/index.md b/doc/concept/layer/index.md index 2e823cc46b0..72d23458d7b 100644 --- a/doc/concept/layer/index.md +++ b/doc/concept/layer/index.md @@ -38,6 +38,15 @@ Everybody else can use QUIC directly instead. MoQ clients will automatically race the QUIC and WebSocket connections in parallel, using whatever wins the race. It sucks but sometimes *media over QUIC* isn't actually an option. +## iroh (optional) + +[iroh](/concept/layer/iroh) is a peer-to-peer alternative to dialing a server. + +You dial a public key instead of a hostname, and iroh handles discovery, hole punching, and +a relay fallback when a direct path can't be established. +It's still QUIC underneath, so nothing above this layer changes. +Native only, since a browser can't hole punch. + ## MoQ Transport [moq-lite](/concept/layer/moq-lite) is a forwards-compatible subset of the [MoqTransport](/concept/standard/moq-transport) specification. diff --git a/doc/concept/layer/iroh.md b/doc/concept/layer/iroh.md new file mode 100644 index 00000000000..9bf536249c2 --- /dev/null +++ b/doc/concept/layer/iroh.md @@ -0,0 +1,194 @@ +--- +title: iroh +description: Peer-to-peer QUIC, dialed by public key instead of a hostname. +--- + +# iroh + +[iroh](https://www.iroh.computer/) is an optional transport that swaps the "client dials a +server" model for "peer dials a peer". +It's still QUIC underneath, so every MoQ layer above it is unchanged. +What changes is *addressing* and *connectivity*: you dial a public key instead of a hostname, +and iroh does the NAT traversal to reach it. + +::: warning +This is experimental and **native only**. +Browsers can't do this. WebTransport gives a page a QUIC connection, not a QUIC endpoint, +so there's no hole punching and no way to accept an inbound connection. +Everything on this page applies to `moq-relay`, `moq`, and anything else built on +[`moq-native`](/lib/rs/crate/moq-native). +::: + +## Why + +The usual MoQ deployment is a [relay](/bin/relay/) with a public IP, a DNS name, and a TLS +certificate. That's the right answer at scale, but it's a lot of ceremony when: + +- Two machines on the same LAN want to exchange media and neither is a server. +- A laptop behind a NAT wants to publish, and you'd rather not run a relay in the middle. +- You want a stable identity for a node whose IP address keeps changing. +- You don't want to own a domain or provision a certificate just to move some frames. + +iroh solves all four with the same mechanism: the endpoint's key **is** its address. + +## Addressing + +An iroh endpoint is an ed25519 keypair. +The public half is the **endpoint id**, and that's what you dial: + +```text +iroh://k5lnrlndqpqcgh4d5nhbnbnhcyrgvw6ttxwrsvsu4nlt6foorxaa/room?jwt=... +``` + +The host is the endpoint id, not a hostname. The path and query mean exactly what they +mean on an `https://` URL: the [auth](/bin/relay/auth) path and an optional token. + +TLS still runs, but there's no certificate authority in the picture. +The peer proves it holds the private key matching the id you dialed, which is the +whole point: you already named the key, so nothing else needs to vouch for it. +Dial the wrong key and the handshake fails. There's no such thing as a misissued +certificate for an endpoint id. + +Note that the key is also the *only* identity. +iroh exposes no client certificate, so mTLS auth isn't available over this transport. +Authentication works like any other tokenized connection: a JWT in the URL, or anonymous +access to a public path. + +## Connectivity + +A single iroh endpoint plays both roles. The same bound socket dials out and accepts in, +which is what makes a true peer-to-peer topology possible. + +Getting a packet to a peer behind a NAT happens in stages: + +1. **Discovery** finds the peer's candidate addresses from its endpoint id. +2. **Hole punching** tries to establish a direct path between the two hosts. +3. **iroh relay** carries the traffic when a direct path can't be established. + +The connection starts over the iroh relay and upgrades to a direct path once hole punching +succeeds, so there's no connect-time stall while the two sides negotiate. +By default this uses the public discovery and relay servers operated by +[n0](https://n0.computer/). Pass `--iroh-disable-relay` to require a direct path and skip +the fallback entirely. + +::: tip Two different things called "relay" +An **iroh relay** forwards opaque UDP packets between two peers that can't reach each +other. It doesn't speak MoQ and doesn't know a track from a group. +A [**MoQ relay**](/bin/relay/) is a CDN node that understands broadcasts, fans them out, +and caches groups. +They're unrelated, and you can use either, both, or neither. +::: + +## Binding + +iroh connections negotiate their MoQ binding by ALPN, not by URL scheme. +The endpoint offers the moq-lite ALPNs first and `h3` last, so: + +- **Two MoQ endpoints** land on **raw QUIC**, skipping HTTP/3 entirely. There's no CONNECT + request to carry the request URI, so the path and token ride the moq-lite SETUP instead. +- **An H3 peer** falls back to **WebTransport**, where the path travels in the CONNECT URL + like it does over `https://`. + +You don't pick between them, and in practice you don't need to care. +It's worth knowing because it's why an `iroh://` URL's path can end up in either place, +and why the [`transport`](/bin/relay/auth) reported to the auth API is `iroh` rather than +`quic`. + +## Usage + +Enable the endpoint on both sides. It's off by default everywhere. + +A relay opts in through its [config file](/bin/relay/config#iroh): + +```toml +[iroh] +enabled = true + +# Persist the key so the endpoint id survives a restart. +# Generated on first run if the file doesn't exist. +secret = "./iroh-secret.key" +``` + +Without `secret` the relay generates a fresh key on every start, which means a new +endpoint id and a stale URL for everyone who wrote the old one down. +`MOQ_IROH_SECRET` sets the key directly if a file doesn't suit your deployment. + +Both the relay and the [CLI](/bin/cli) log the endpoint id at startup: + +```text +INFO iroh listening endpoint_id=k5lnrlndqpqcgh4d5nhbnbnhcyrgvw6ttxwrsvsu4nlt6foorxaa +``` + +The CLI takes the same settings as flags. `--iroh-enabled` binds the endpoint, and then an +`iroh://` URL works anywhere an `https://` one would: + +```bash +# Publish to a peer. +ffmpeg -i video.mp4 -c copy -f mpegts - | \ + moq --iroh-enabled \ + --client-connect "iroh://k5lnrlndqpqcgh4d5nhbnbnhcyrgvw6ttxwrsvsu4nlt6foorxaa/anon" \ + --broadcast my-stream.hang import ts + +# Play it back from another machine. +moq --iroh-enabled \ + --client-connect "iroh://k5lnrlndqpqcgh4d5nhbnbnhcyrgvw6ttxwrsvsu4nlt6foorxaa/anon" \ + --broadcast my-stream.hang play +``` + +Dialing an `iroh://` URL without `--iroh-enabled` is an error, not a silent fallback. + +| Flag | Environment | Purpose | +|---|---|---| +| `--iroh-enabled` | `MOQ_IROH_ENABLED` | Bind the endpoint. Required for `iroh://`. | +| `--iroh-secret` | `MOQ_IROH_SECRET` | Hex key or a path to persist one. | +| `--iroh-bind-v4` | `MOQ_IROH_BIND_V4` | UDP bind address, default `0.0.0.0:0`. | +| `--iroh-bind-v6` | `MOQ_IROH_BIND_V6` | UDP bind address, default `[::]:0`. | +| `--iroh-disable-relay` | `MOQ_IROH_DISABLE_RELAY` | Direct paths only, no relay fallback. | + +The endpoint is shared by the client and server halves of a process, so one set of flags +covers both dialing out and accepting in. + +## Building + +iroh is behind the `iroh` cargo feature. +It's on by default for `moq-cli` and `moq-relay`, and off by default for +[`moq-native`](/lib/rs/crate/moq-native): + +```bash +cargo install moq-cli --no-default-features --features "iroh,quinn,websocket" +``` + +Library users bind the endpoint once and hand it to both halves: + +```rust +let endpoint = iroh_config.bind(&client_config.quic).await?.unwrap(); +let client = client_config.init()?.with_iroh(endpoint.clone()); +let server = server_config.init()?.with_iroh(endpoint); +``` + +`Client::with_iroh_addrs` seeds known socket addresses for a peer, letting a LAN dial skip +discovery. + +## Limitations + +iroh runs on [noq](https://crates.io/crates/noq) rather than quinn, and it owns its own +transport config, so a few of the shared QUIC knobs behave differently: + +- **The client QUIC settings apply to both roles.** One endpoint serves both, and the + per-connection knobs are symmetric, so `[client.quic]` (`--client-quic-*`) is what it + reads. The `[server.quic]` section doesn't reach it. +- **Congestion control defaults to CUBIC**, unlike the quinn and quiche backends, which + default to BBR. iroh shares noq's BBRv3, which can underflow and panic on a packet loss, + so delay-based control stays reachable only when an operator asks for it by name. The + noq backend defaults to CUBIC for the same reason. See the + [relay config](/bin/relay/config). +- **GSO can't be disabled.** `--client-quic-gso=false` is refused rather than ignored. +- **No keep-alive knob.** Stream limits, idle timeout, MTU discovery, and congestion + control carry over; nothing else does. +- **No browser support**, as above. + +## More Info + +- [QUIC](/concept/layer/quic) - what iroh is carrying underneath. +- [Relay configuration](/bin/relay/config#iroh) - the `[iroh]` section. +- [CLI](/bin/cli) - publishing and playing over `iroh://`. From e8e4309fbabb47c4a2320f5d9f336b1aba05cf74 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 19 Aug 2026 15:01:16 -0700 Subject: [PATCH 24/29] feat(moq-native): size the UDP socket buffers, warning when the kernel clamps (#2940) Co-authored-by: Claude Opus 5 --- doc/setup/prod.md | 20 ++++ rs/moq-native/src/bind.rs | 215 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) diff --git a/doc/setup/prod.md b/doc/setup/prod.md index bba47e631e2..4784fe53e29 100644 --- a/doc/setup/prod.md +++ b/doc/setup/prod.md @@ -47,6 +47,26 @@ TLS is where most people get stuck. And of course, make sure UDP is allowed on your firewall. The default WebTransport port is UDP/443 but anything will work if you put it in the URL. +## Tuning + +A relay multiplexes every connection over a single UDP socket, so a burst that doesn't fit in the kernel's socket buffer is dropped before the relay ever sees it, and QUIC congestion control reads those drops as congestion. + +`moq-relay` and `moq-cli` request an 8 MiB socket buffer in each direction, but Linux silently clamps the request to `net.core.rmem_max` and `net.core.wmem_max`, which default to 208 KiB. +You'll get a warning on startup when that happens: + +```text +WARN moq_native::bind: UDP receive buffer is smaller than requested; raise `net.core.rmem_max` or expect packet loss under load wanted=8388608 granted=212992 +``` + +Raise both limits, persisting them across reboots: + +```bash +printf 'net.core.rmem_max = 8388608\nnet.core.wmem_max = 8388608\n' | sudo tee /etc/sysctl.d/60-moq.conf +sudo sysctl --system +``` + +macOS caps both directions with `kern.ipc.maxsockbuf` instead, and Windows sizes each socket on its own so there's nothing to raise. + ## Next Steps - Set up [Authentication](/bin/relay/auth) diff --git a/rs/moq-native/src/bind.rs b/rs/moq-native/src/bind.rs index 6f992826f4b..0f16fd4853e 100644 --- a/rs/moq-native/src/bind.rs +++ b/rs/moq-native/src/bind.rs @@ -11,6 +11,7 @@ use socket2::{Domain, Protocol, Socket, TcpKeepalive, Type}; use std::net::{SocketAddr, TcpListener, UdpSocket}; +use std::sync::Once; use std::time::Duration; /// TCP keepalive idle period before the kernel starts probing a silent peer, and @@ -23,15 +24,188 @@ use std::time::Duration; const KEEPALIVE_IDLE: Duration = Duration::from_secs(30); const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); +/// UDP socket buffer size requested in each direction, in bytes. +/// +/// A QUIC stack absorbs bursts in the kernel socket buffer: whatever doesn't fit +/// while the process is off the CPU is dropped before quinn ever sees it, and +/// congestion control reads those drops as congestion. The OS defaults are sized +/// for a chatty TCP-era socket (208 KiB on Linux), which a single relay socket +/// carrying every connection blows through in milliseconds. 8 MiB is roughly 64ms +/// of a saturated 1Gbps link, generous next to a scheduler delay and cheap next to +/// what a relay already spends per connection. quic-go asks for 7 MiB. +/// +/// Compiled in rather than derived from the NIC: the buffer is a ceiling on queued +/// bytes rather than an allocation, so an oversized request costs an idle socket +/// nothing, while a link's nominal speed says little about the path it feeds (a +/// VM's virtio NIC reports 10Gbps through a 100Mbps uplink). +const UDP_BUFFER: usize = 8 * 1024 * 1024; + +/// The sysctl capping `SO_RCVBUF`, named in the warning so an operator knows what +/// to raise. `None` on platforms that size socket buffers per socket only. +#[cfg(any(target_os = "linux", target_os = "android"))] +const RECV_SYSCTL: Option<&str> = Some("net.core.rmem_max"); +#[cfg(any( + target_vendor = "apple", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" +))] +const RECV_SYSCTL: Option<&str> = Some("kern.ipc.maxsockbuf"); +#[cfg(not(any( + target_os = "linux", + target_os = "android", + target_vendor = "apple", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" +)))] +const RECV_SYSCTL: Option<&str> = None; + +/// The sysctl capping `SO_SNDBUF`. See [`RECV_SYSCTL`]; the BSDs cap both +/// directions with the same knob. +#[cfg(any(target_os = "linux", target_os = "android"))] +const SEND_SYSCTL: Option<&str> = Some("net.core.wmem_max"); +#[cfg(not(any(target_os = "linux", target_os = "android")))] +const SEND_SYSCTL: Option<&str> = RECV_SYSCTL; + /// Bind a UDP socket, making an IPv6 socket dual-stack so it also serves IPv4. +/// +/// The socket buffers are grown to 8 MiB where the OS allows it, and a warning +/// names the sysctl to raise where it doesn't. pub fn udp(addr: SocketAddr) -> std::io::Result { let domain = if addr.is_ipv4() { Domain::IPV4 } else { Domain::IPV6 }; let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?; make_dual_stack(&socket, addr); + grow_buffers(&socket); socket.bind(&addr.into())?; Ok(socket.into()) } +/// Request [`UDP_BUFFER`] in both directions, best-effort. +fn grow_buffers(socket: &Socket) { + for direction in [Direction::Recv, Direction::Send] { + direction.grow(socket); + } +} + +/// One direction of a socket's buffering, owning everything that differs between +/// the two: the socket options, the sysctl to name, and the warning. +#[derive(Clone, Copy)] +enum Direction { + Recv, + Send, +} + +impl Direction { + /// Raise this direction's buffer to [`UDP_BUFFER`], then read back what the + /// kernel actually granted and warn once when it fell short. + /// + /// Reading back is the whole point: Linux silently clamps the request to its + /// sysctl, so `setsockopt` returning `Ok` says nothing about the size we ended + /// up with, and `SO_RCVBUFFORCE` (the clamp-free version) needs `CAP_NET_ADMIN` + /// that a relay shouldn't be asking for. + fn grow(self, socket: &Socket) { + // Never shrink a system that's already tuned above our default. + if self.size(socket).is_ok_and(sufficient) { + return; + } + + match self.set_size(socket, UDP_BUFFER).and_then(|()| self.size(socket)) { + Ok(reported) if sufficient(reported) => {} + Ok(reported) => self.warn_short(granted(reported)), + Err(err) => self.warn_failed(&err), + } + } + + fn size(self, socket: &Socket) -> std::io::Result { + match self { + Self::Recv => socket.recv_buffer_size(), + Self::Send => socket.send_buffer_size(), + } + } + + fn set_size(self, socket: &Socket, size: usize) -> std::io::Result<()> { + match self { + Self::Recv => socket.set_recv_buffer_size(size), + Self::Send => socket.set_send_buffer_size(size), + } + } + + fn name(self) -> &'static str { + match self { + Self::Recv => "receive", + Self::Send => "send", + } + } + + fn sysctl(self) -> Option<&'static str> { + match self { + Self::Recv => RECV_SYSCTL, + Self::Send => SEND_SYSCTL, + } + } + + /// One warning per direction per process: a client that reconnects rebinds, and + /// the operator only needs telling once. + fn warned(self) -> &'static Once { + static RECV: Once = Once::new(); + static SEND: Once = Once::new(); + + match self { + Self::Recv => &RECV, + Self::Send => &SEND, + } + } + + /// The kernel accepted the request and quietly handed back `granted` instead. + fn warn_short(self, granted: usize) { + self.warned().call_once(|| self.emit_short(granted)); + } + + /// The warning itself, minus the once-guard, so a test can read it back + /// whatever a previous bind on this host already consumed. + fn emit_short(self, granted: usize) { + let name = self.name(); + match self.sysctl() { + Some(sysctl) => tracing::warn!( + wanted = UDP_BUFFER, + granted, + "UDP {name} buffer is smaller than requested; raise `{sysctl}` or expect packet loss under load" + ), + None => tracing::warn!( + wanted = UDP_BUFFER, + granted, + "UDP {name} buffer is smaller than requested; expect packet loss under load" + ), + } + } + + /// The option itself was rejected, so we don't even know what we're running with. + fn warn_failed(self, err: &std::io::Error) { + let name = self.name(); + self.warned() + .call_once(|| tracing::warn!(%err, "failed to set the UDP {name} buffer size")); + } +} + +/// Whether a buffer size the kernel reported already covers [`UDP_BUFFER`]. +fn sufficient(reported: usize) -> bool { + granted(reported) >= UDP_BUFFER +} + +/// The usable size behind a buffer size the kernel reported. +/// +/// Linux reports back double what it granted, reserving the other half for +/// per-packet bookkeeping, so halving keeps the numbers we compare and log in the +/// same units an operator writes into the sysctl. +fn granted(reported: usize) -> usize { + if cfg!(any(target_os = "linux", target_os = "android")) { + reported / 2 + } else { + reported + } +} + /// Whether `socket` also reaches IPv4, through IPv4-mapped addresses. /// /// [`udp`] clears `IPV6_V6ONLY` best-effort, so this reads back what the @@ -124,6 +298,47 @@ mod tests { assert!(!socket.only_v6().unwrap(), "IPv6 socket should be dual-stack"); } + #[test] + fn udp_buffers_grow() { + fn check(direction: Direction) { + let plain = Socket::from(std::net::UdpSocket::bind("127.0.0.1:0").unwrap()); + let before = direction.size(&plain).unwrap(); + + let tuned = Socket::from(udp("127.0.0.1:0".parse().unwrap()).unwrap()); + let after = direction.size(&tuned).unwrap(); + + // A host whose default already covers UDP_BUFFER is left alone. Anywhere + // else the bind has to have actually raised it, whatever the sysctls + // clamped it to. + if sufficient(before) { + assert_eq!(after, before, "{} buffer should be left alone", direction.name()); + } else { + assert!(after > before, "{} buffer should grow past {before}", direction.name()); + } + } + + check(Direction::Recv); + check(Direction::Send); + } + + #[test] + fn sufficient_accounts_for_the_doubled_report() { + // Doubled, since that's how Linux reports back a buffer it granted. + assert!(sufficient(UDP_BUFFER * 2)); + assert!(!sufficient(512 * 1024)); + } + + #[tracing_test::traced_test] + #[test] + fn a_clamped_buffer_warns_and_names_the_sysctl() { + Direction::Recv.emit_short(512 * 1024); + + assert!(logs_contain("UDP receive buffer is smaller than requested")); + if let Some(sysctl) = Direction::Recv.sysctl() { + assert!(logs_contain(sysctl)); + } + } + #[test] fn udp_ipv4_still_binds() { let socket = udp("127.0.0.1:0".parse().unwrap()).unwrap(); From 67f12fead0950aa02c14e69eca8d56ecb9d0fdbe Mon Sep 17 00:00:00 2001 From: Alexandre-Fromantin <130671822+Alexandre-Fromantin@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:12:34 +0100 Subject: [PATCH 25/29] Reactive firefox 153+ via bowser (#2943) --- bun.lock | 3 +++ js/net/package.json | 3 ++- js/net/src/connection/browser.ts | 28 ++++++++++++++++------------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/bun.lock b/bun.lock index 93a986c5e8c..578dbdc38a6 100644 --- a/bun.lock +++ b/bun.lock @@ -187,6 +187,7 @@ "@moq/qmux": "^0.3.3", "@moq/signals": "workspace:*", "async-mutex": "^0.5.0", + "bowser": "^2.14.1", }, "devDependencies": { "@types/bun": "^1.3.14", @@ -960,6 +961,8 @@ "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], + "brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], diff --git a/js/net/package.json b/js/net/package.json index 4192c369755..35ca1829980 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -19,7 +19,8 @@ "dependencies": { "@moq/qmux": "^0.3.3", "@moq/signals": "workspace:*", - "async-mutex": "^0.5.0" + "async-mutex": "^0.5.0", + "bowser": "^2.14.1" }, "peerDependencies": { "zod": "^4.0.0" diff --git a/js/net/src/connection/browser.ts b/js/net/src/connection/browser.ts index e98848c18a0..21d3ffd7f24 100644 --- a/js/net/src/connection/browser.ts +++ b/js/net/src/connection/browser.ts @@ -1,19 +1,23 @@ +import Bowser from "bowser"; + /** Returns whether a browser user agent has a usable WebTransport implementation. */ export function isWebTransportUserAgentSupported(userAgent: string): boolean { - const normalized = userAgent.toLowerCase(); + const browser = Bowser.getParser(userAgent); + const supported = browser.satisfies({ + // Fixed with 153.0.0, Firefox only allows two concurrent remote-initiated streams: + // https://bugzilla.mozilla.org/show_bug.cgi?id=2046262 + firefox: ">=153.0", - // Firefox only allows two concurrent remote-initiated streams: - // https://bugzilla.mozilla.org/show_bug.cgi?id=2046262 - if (normalized.includes("firefox")) return false; + // Safari's flow-control window never refills, which permanently stalls sessions: + // https://bugs.webkit.org/show_bug.cgi?id=319818 + safari: "<0", + }); - // Safari's flow-control window never refills, which permanently stalls sessions: - // https://bugs.webkit.org/show_bug.cgi?id=319818 - const isSafari = - normalized.includes("safari") && - !normalized.includes("chrome") && - !normalized.includes("chromium") && - !normalized.includes("android"); - return !isSafari; + if (supported === undefined) { + //By default, other browsers are considered to support WebTransport. + return true; + } + return supported; } /** Returns whether this runtime can connect with WebTransport. */ From f37044bbd3a44453eec3662e59140e7e47fe1dd9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 19 Aug 2026 15:25:01 -0700 Subject: [PATCH 26/29] docs: scope P2P to the local network, relay across the internet (#2941) Write down a policy the iroh page was silent on, and correct two things it got wrong. P2P is a local-network feature; across the internet, run a MoQ relay. The page presented hole punching and the iroh relay as one feature, but they are not the same trade. A direct path between two peers costs nothing and is the point. The relay forwards opaque UDP through an n0 server, which is WebRTC's TURN with none of what a MoQ relay buys for that same hop: it cannot cache a group or serve a second viewer from the first one's fetch, and it adds whatever latency its location costs. Discovery is not local either. Both n0 presets install a PkarrPublisher, a PkarrResolver, and (outside browsers) a DnsAddressLookup against n0's iroh.link, and N0DisableRelay is N0 plus relay_mode(Disabled) and nothing else. So enabling iroh publishes this endpoint's addressing record to a third party whatever the relay setting; disabling the relay only changes what the record holds. Resolution is the half a caller can avoid, via Client::with_iroh_addrs; publishing has no equivalent escape hatch. Disabling the relay costs more than the fallback. A relay is the rendezvous both sides reach first, so it is also how they coordinate the punch, and its probes are how an endpoint learns the public address it advertises. Turning it off leaves direct addresses only. That is still right for two peers on one link, who need none of the three, but it is unsafe as a blanket recommendation: on a cloud VM behind 1:1 NAT it leaves a relay advertising a private address nobody can route to. The relay config sample therefore stays at the default with the trade written out. Also: iroh:// and https:// are separate MoQ connections to a relay, with no automatic fallback between them. Co-Authored-By: Claude Opus 5 --- doc/bin/relay/config.md | 29 ++++++++++++-- doc/concept/layer/index.md | 11 +++++- doc/concept/layer/iroh.md | 81 ++++++++++++++++++++++++++++++-------- 3 files changed, 99 insertions(+), 22 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 25d9f3d23e5..bfda1240e6c 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -523,8 +523,26 @@ All three flags also accept CLI arguments (`--cache-capacity`, ### \[iroh] Experimental P2P support via [iroh](/concept/layer/iroh). Clients dial the relay's -endpoint id (`iroh:///`) instead of a hostname, with hole punching -and a relay fallback for peers that can't be reached directly. +endpoint id (`iroh:///`) instead of a hostname. An n0 relay carries the +connection from the start and hole punching moves it onto a direct path, or fails and leaves +it there. + +Prefer `https://` for anything off this relay's own network. A client reaching this relay +over the internet gains nothing from iroh (this process is already the public address, and it +caches and fans out, which an n0 relay forwarding opaque packets does not), and there's no +automatic relationship between the two: `iroh://` and `https://` are separate MoQ connections +to this relay, and dialing one never falls back to the other. That choice belongs to whoever +writes the URL. + +`disable_relay = true` keeps an n0 relay out of the media path entirely. Read +[Connectivity](/concept/layer/iroh#connectivity) first: it also removes hole punching and the +probes an endpoint uses to learn its own public address, so it suits a relay meeting peers on +its own network and breaks one that clients dial over `iroh://` from elsewhere. On a cloud VM +behind 1:1 NAT it is the difference between advertising your public address and advertising a +private one nobody can route to. + +Either way, discovery still publishes this endpoint's addresses to n0's `iroh.link` DNS +server, so enabling iroh at all is visible off the network. ```toml [iroh] @@ -539,8 +557,11 @@ secret = "./relay-iroh-secret.key" # bind_v4 = "0.0.0.0:4444" # bind_v6 = "[::]:4444" -# Require a direct path, skipping the iroh relay fallback. -# disable_relay = false +# Uncomment for direct addresses only: no n0 relay, and no hole +# punching either. Right for peers on this relay's own network; see +# the note above before enabling it on a relay that clients dial +# over iroh:// from the internet. +# disable_relay = true ``` The endpoint id is logged at startup (`iroh listening endpoint_id=...`). Without `secret` diff --git a/doc/concept/layer/index.md b/doc/concept/layer/index.md index 72d23458d7b..68befa37f05 100644 --- a/doc/concept/layer/index.md +++ b/doc/concept/layer/index.md @@ -42,11 +42,18 @@ It sucks but sometimes *media over QUIC* isn't actually an option. [iroh](/concept/layer/iroh) is a peer-to-peer alternative to dialing a server. -You dial a public key instead of a hostname, and iroh handles discovery, hole punching, and -a relay fallback when a direct path can't be established. +You dial a public key instead of a hostname, and iroh handles discovery and hole punching. It's still QUIC underneath, so nothing above this layer changes. Native only, since a browser can't hole punch. +Reach for it on a local network, where two peers can talk without anything in the middle. +Across the internet, run a [MoQ relay](/bin/relay/) instead: iroh's own relay forwards opaque +packets like WebRTC's TURN, so it costs you a hop without the caching and fanout that hop +should be buying. + +Note that only the media path can be kept local. An endpoint publishes its address to n0's +discovery servers whichever way it's configured. + ## MoQ Transport [moq-lite](/concept/layer/moq-lite) is a forwards-compatible subset of the [MoqTransport](/concept/standard/moq-transport) specification. diff --git a/doc/concept/layer/iroh.md b/doc/concept/layer/iroh.md index 9bf536249c2..c692f0863f8 100644 --- a/doc/concept/layer/iroh.md +++ b/doc/concept/layer/iroh.md @@ -25,11 +25,34 @@ The usual MoQ deployment is a [relay](/bin/relay/) with a public IP, a DNS name, certificate. That's the right answer at scale, but it's a lot of ceremony when: - Two machines on the same LAN want to exchange media and neither is a server. -- A laptop behind a NAT wants to publish, and you'd rather not run a relay in the middle. - You want a stable identity for a node whose IP address keeps changing. - You don't want to own a domain or provision a certificate just to move some frames. -iroh solves all four with the same mechanism: the endpoint's key **is** its address. +iroh solves all three with the same mechanism: the endpoint's key **is** its address. + +## Scope + +Peer-to-peer is a **local network** feature. Keep it there. + +- **On one network**, peers dial each other directly, and with `--iroh-disable-relay` the + media never leaves the link. That's the case iroh is for here. (The addressing leaves + either way; see [Connectivity](#connectivity).) +- **Across the internet**, run a [MoQ relay](/bin/relay/). It has an address both sides can + already reach, it caches and fans out, and it's the only shape that scales past two peers. + +The iroh relay is the part to be careful with. By default your packets go through an n0 +server from the first byte, and stay there for good if the hole punch never lands. That's the +same shape as WebRTC's TURN: a third party in the media path that has no idea what it's +carrying, can't cache a group, can't serve the second viewer from the first one's fetch, and +adds however many milliseconds its location costs. If media has to cross a server anyway, +that server should be a MoQ relay. + +`--iroh-disable-relay` takes that off the table, but read [Connectivity](#connectivity) +before reaching for it: an iroh relay is not only the fallback path, it is also how two peers +coordinate a hole punch and how an endpoint learns its own public address. Turning it off +leaves direct addresses and nothing else, which is exactly right on one link and not much use +across the internet. That is the trade this page is recommending, not a way to keep NAT +traversal while dropping the forwarding. ## Addressing @@ -62,21 +85,37 @@ which is what makes a true peer-to-peer topology possible. Getting a packet to a peer behind a NAT happens in stages: 1. **Discovery** finds the peer's candidate addresses from its endpoint id. -2. **Hole punching** tries to establish a direct path between the two hosts. -3. **iroh relay** carries the traffic when a direct path can't be established. - -The connection starts over the iroh relay and upgrades to a direct path once hole punching -succeeds, so there's no connect-time stall while the two sides negotiate. -By default this uses the public discovery and relay servers operated by -[n0](https://n0.computer/). Pass `--iroh-disable-relay` to require a direct path and skip -the fallback entirely. +2. **iroh relay** carries the connection immediately, so there's no connect-time stall. +3. **Hole punching** then tries to open a direct path, and the connection migrates onto it. + +The relay does more than carry the traffic. It's the rendezvous both sides reach first, so +it's also how they coordinate the punch, and its probes are how an endpoint learns the public +address to advertise in the first place. Only once the direct path is up does it drop out, +and it stays in the path as a fallback if the punch never succeeds. + +So `--iroh-disable-relay` is not "punch, but don't fall back". It removes stages 2 and 3 +together and leaves stage 1: a peer is reachable if its advertised addresses reach you, and +unreachable otherwise. On one link that's all you need, and it's the [recommended](#scope) +setting there. Across a NAT it mostly means no connection, and an endpoint that can't see its +own public address advertises only its local ones. + +**Discovery runs either way.** An endpoint publishes its own addressing record to n0's +`iroh.link` DNS server whenever its addresses change, so turning iroh on tells a third party +which endpoint ids are live and where they are, whatever the relay setting. Disabling the +relay doesn't opt out; it only changes what the record holds, since with no relay URL to +publish it carries direct IP addresses instead. Resolution is the half you can avoid: +`Client::with_iroh_addrs` seeds a peer's addresses so a LAN dial needs no lookup. Publishing +has no such escape hatch. + +Weigh that before enabling iroh for something that has to stay local. The media can be pinned +to a direct path, the addressing can't. ::: tip Two different things called "relay" An **iroh relay** forwards opaque UDP packets between two peers that can't reach each other. It doesn't speak MoQ and doesn't know a track from a group. A [**MoQ relay**](/bin/relay/) is a CDN node that understands broadcasts, fans them out, and caches groups. -They're unrelated, and you can use either, both, or neither. +They're unrelated, and when media has to cross a server, it should be the second one. ::: ## Binding @@ -109,6 +148,11 @@ enabled = true secret = "./iroh-secret.key" ``` +Whether to add `disable_relay` depends on where this relay's iroh clients are, and the +[relay config reference](/bin/relay/config#iroh) has the trade. It keeps n0 out of the media +path, and it breaks a relay that clients dial over `iroh://` from anywhere but its own +network. + Without `secret` the relay generates a fresh key on every start, which means a new endpoint id and a stale URL for everyone who wrote the old one down. `MOQ_IROH_SECRET` sets the key directly if a file doesn't suit your deployment. @@ -123,18 +167,21 @@ The CLI takes the same settings as flags. `--iroh-enabled` binds the endpoint, a `iroh://` URL works anywhere an `https://` one would: ```bash -# Publish to a peer. +# Publish to a peer on this network. ffmpeg -i video.mp4 -c copy -f mpegts - | \ - moq --iroh-enabled \ + moq --iroh-enabled --iroh-disable-relay \ --client-connect "iroh://k5lnrlndqpqcgh4d5nhbnbnhcyrgvw6ttxwrsvsu4nlt6foorxaa/anon" \ --broadcast my-stream.hang import ts -# Play it back from another machine. -moq --iroh-enabled \ +# Play it back from another machine on the same network. +moq --iroh-enabled --iroh-disable-relay \ --client-connect "iroh://k5lnrlndqpqcgh4d5nhbnbnhcyrgvw6ttxwrsvsu4nlt6foorxaa/anon" \ --broadcast my-stream.hang play ``` +Drop `--iroh-disable-relay` and the two machines can be anywhere, at the cost of an n0 relay +in the media path until a punch succeeds. Prefer a [MoQ relay](/bin/relay/) for that case. + Dialing an `iroh://` URL without `--iroh-enabled` is an error, not a silent fallback. | Flag | Environment | Purpose | @@ -143,7 +190,7 @@ Dialing an `iroh://` URL without `--iroh-enabled` is an error, not a silent fall | `--iroh-secret` | `MOQ_IROH_SECRET` | Hex key or a path to persist one. | | `--iroh-bind-v4` | `MOQ_IROH_BIND_V4` | UDP bind address, default `0.0.0.0:0`. | | `--iroh-bind-v6` | `MOQ_IROH_BIND_V6` | UDP bind address, default `[::]:0`. | -| `--iroh-disable-relay` | `MOQ_IROH_DISABLE_RELAY` | Direct paths only, no relay fallback. | +| `--iroh-disable-relay` | `MOQ_IROH_DISABLE_RELAY` | Direct addresses only: no fallback, and no hole punching. Recommended on a LAN. | The endpoint is shared by the client and server halves of a process, so one set of flags covers both dialing out and accepting in. @@ -185,6 +232,8 @@ transport config, so a few of the shared QUIC knobs behave differently: - **GSO can't be disabled.** `--client-quic-gso=false` is refused rather than ignored. - **No keep-alive knob.** Stream limits, idle timeout, MTU discovery, and congestion control carry over; nothing else does. +- **Publishing is not local.** An endpoint's addressing record goes to n0's servers whatever + the relay setting, as above. Resolution can be skipped, publishing can't. - **No browser support**, as above. ## More Info From d8f94b37c0744172d184adb8515a0cbfd3f111e3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 19 Aug 2026 20:02:22 -0700 Subject: [PATCH 27/29] fix(libmoq): declare the video and container format enums in moq.h They cross the ABI as `uint32_t` so no signature reaches them, which is what the ENUMS allowlist in build.rs exists to compensate for. `moq_video_format` and `moq_container_format` were never added to it, so moq.h declared neither and a C caller of `moq_publish_video` or `moq_publish_container` had to hardcode the discriminants. cpp/obs did not, hence the build failure. Nothing caught it because PR CI never compiled a C consumer until obs.yml arrived from main. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017wFQ5wqKbvWwET3G5MJXY9 --- rs/libmoq/build.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/libmoq/build.rs b/rs/libmoq/build.rs index 0eede505cbe..5b254d01fb9 100644 --- a/rs/libmoq/build.rs +++ b/rs/libmoq/build.rs @@ -8,6 +8,8 @@ const LIB_NAME: &str = "moq"; const ENUMS: &[&str] = &[ "moq_container_kind", "moq_audio_format", + "moq_video_format", + "moq_container_format", "moq_video_pixel_format", "moq_video_codec", "moq_video_encoder_kind", From b3c9c51ee9db4aeeb918f12cd9d90b949ba79afc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 19 Aug 2026 20:29:47 -0700 Subject: [PATCH 28/29] fix(smoke): follow the media-publishing split in the python client #2927 split `publish_media_stream` into `publish_video_stream` and `publish_container_stream`, and updated `py/moq-rs`, but not the smoke client that calls it. Every `python -> *` case failed on the publisher with `'BroadcastProducer' object has no attribute 'publish_media_stream'`. smoke.yml only runs on a PR touching test/smoke, test/ts, test/justfile, package.json, or itself, so #2927 never triggered it and the break has been sitting on dev waiting for the nightly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017wFQ5wqKbvWwET3G5MJXY9 --- py/moq-rs/moq/publish.py | 2 +- test/smoke/clients/python/smoke.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py/moq-rs/moq/publish.py b/py/moq-rs/moq/publish.py index 8c8114fbd98..f2766cc57f1 100644 --- a/py/moq-rs/moq/publish.py +++ b/py/moq-rs/moq/publish.py @@ -166,7 +166,7 @@ class MediaStreamProducer: """Wraps MoqMediaStreamProducer: feed a raw byte stream (e.g. Annex-B H.264) and let the importer infer frame boundaries. - Built via :meth:`BroadcastProducer.publish_media_stream`. Unlike + Built via :meth:`BroadcastProducer.publish_video_stream`. Unlike :class:`MediaProducer`, no per-frame timestamps are needed; just push encoder bytes as they arrive. """ diff --git a/test/smoke/clients/python/smoke.py b/test/smoke/clients/python/smoke.py index ec82cc10dd2..0cf8304f319 100644 --- a/test/smoke/clients/python/smoke.py +++ b/test/smoke/clients/python/smoke.py @@ -23,7 +23,7 @@ async def publish(url: str, broadcast: str) -> None: async with moq.Client(url, tls_verify=False) as client: # Hold the producer for the lifetime of the publish loop; finish() unpublishes. producer = client.create_broadcast(broadcast) - media = producer.publish_media_stream("avc3") + media = producer.publish_video_stream(moq.VideoFormat.AVC3) print(f"publishing {broadcast!r} (Annex-B H.264 from stdin) to {url}") loop = asyncio.get_running_loop() From d50f2fb2b4cf68f35210c8b52aa649f41b515cf5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 19 Aug 2026 20:29:55 -0700 Subject: [PATCH 29/29] test(wasm): let the live edge pass a group without failing the case The harness subscribes with the default `Latency::REAL_TIME` budget, which drops a group the edge has moved past rather than delivering it late. #2890 made that enforcement real, so the publisher writing a group every 50ms can now retire one the reader is still on and the read fails with `old`. Requiring `previous + 1` and treating `old` as a case failure asserts a guarantee real-time delivery does not make. Skips are accepted and bounded by an attempt cap instead, so a subscription that never delivers a whole group still fails; what a delivered group has to be, in order and byte-exact, is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017wFQ5wqKbvWwET3G5MJXY9 --- test/wasm/src/main.ts | 44 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/test/wasm/src/main.ts b/test/wasm/src/main.ts index a52308afe7f..2b0df7b75fe 100644 --- a/test/wasm/src/main.ts +++ b/test/wasm/src/main.ts @@ -22,6 +22,7 @@ type Wasm = typeof import("@moq/wasm"); type Session = import("@moq/wasm").Session; type Broadcast = import("@moq/wasm").Broadcast; type Track = import("@moq/wasm").Track; +type Group = import("@moq/wasm").Group; /** One relay the suite runs against, and the version it must negotiate. */ export interface RelayFixture { @@ -211,28 +212,53 @@ async function consume(session: Session, path: string): Promise { */ async function expectGroups(track: Track, count: number): Promise { let previous: bigint | undefined; + let read = 0; - for (let i = 0; i < count; i++) { - const group = await track.recvGroup(); - if (!group) throw new Error(`track ended after ${i} groups, want ${count}`); + // A skip is legal (see `isOld`), so bound the attempts rather than the skips: a + // subscription that never delivers a whole group fails here instead of looping. + for (let attempt = 0; read < count; attempt++) { + if (attempt >= count * 4) throw new Error(`read ${read} whole groups in ${attempt} attempts, want ${count}`); + + let group: Group | undefined; try { - if (previous !== undefined && group.sequence !== previous + 1n) { - throw new Error(`group ${i}: sequence is ${group.sequence}, want ${previous + 1n}`); + group = await track.recvGroup(); + if (!group) throw new Error(`track ended after ${read} groups, want ${count}`); + + // Forward, not consecutive: a skipped group leaves a gap in the sequence. + if (previous !== undefined && group.sequence <= previous) { + throw new Error(`group ${read}: sequence is ${group.sequence}, want greater than ${previous}`); } previous = group.sequence; for (const [j, want] of FIXTURE.entries()) { const frame = await group.readFrame(); - if (!frame) throw new Error(`group ${i}: ended after ${j} frames, want ${FIXTURE.length}`); - expectBytes(frame, want, `group ${i} frame ${j}`); + if (!frame) throw new Error(`group ${read}: ended after ${j} frames, want ${FIXTURE.length}`); + expectBytes(frame, want, `group ${read} frame ${j}`); } - if (await group.readFrame()) throw new Error(`group ${i}: got more than ${FIXTURE.length} frames`); + if (await group.readFrame()) throw new Error(`group ${read}: got more than ${FIXTURE.length} frames`); + read++; + } catch (err) { + if (!isOld(err)) throw err; } finally { - group.free(); + group?.free(); } } } +/** + * Whether the live edge passed this group before it was delivered whole. + * + * The subscription's budget is the default `Latency::REAL_TIME`, which drops a group + * the edge has moved past rather than delivering it late, so the publisher writing one + * every {@link GROUP_INTERVAL_MS} can legitimately retire a group the reader is still + * on. Treating that as a case failure asserts a guarantee real-time delivery does not + * make, and loses the race on a loaded runner. What has to hold is that a group which + * *is* delivered arrives in order and whole, which the checks above still cover. + */ +function isOld(err: unknown): boolean { + return err instanceof Error && err.message === "old"; +} + function expectBytes(got: Uint8Array, want: Uint8Array, where: string): void { if (got.length !== want.length) throw new Error(`${where}: got ${got.length} bytes, want ${want.length}`); for (let i = 0; i < want.length; i++) {