From 9b9dc8e219c28a53ce694addcfb1f4deea3cc0b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:29:16 -0700 Subject: [PATCH 1/2] chore: claim sibling broadcasts quest Co-Authored-By: GPT-6 From 9bf93eec1fb20435b972dbb989744dba8c4a2597 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:55:20 -0700 Subject: [PATCH 2/2] docs(hang): publish sibling sources on one shared clock Co-Authored-By: GPT-6 --- doc/concept/hang.md | 2 +- doc/concept/use-case/conferencing.md | 2 +- js/publish/src/broadcast.test.ts | 23 +++++++++++ quest/m2/README.md | 1 - quest/m2/sibling-broadcasts.md | 62 ---------------------------- rs/moq-mux/src/catalog/producer.rs | 50 ++++++++++++++++++++++ 6 files changed, 75 insertions(+), 65 deletions(-) delete mode 100644 quest/m2/sibling-broadcasts.md diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 09c38ea9590..e924764cf93 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -69,7 +69,7 @@ A few things the catalog can express beyond decoder config: - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. - **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and, if recorded, the replay MoQ path, object-store URL, and format version. Every track is cut by one rule, at group boundaries between 2s and 10s (a zero minimum for sparse data such as the catalog), and tracks commit and expire independently. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. A reader that cannot decode the `archive` entry, such as one from a different version, ignores it and loses the timelines and replay, but live media still plays. -- **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a timeline. +- **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a timeline. Publish independent sources as sibling broadcasts under one prefix, such as `robot/front.hang` and `robot/rear.hang`; each broadcast's audio and video entries are renditions of that source. In one Rust publisher, pass the same `moq_mux::Clock` to each `catalog::Config::with_clock`; JS publishers on one page already share the page clock. Consumers discover the siblings by prefix and align their timestamps through the advertised wall mapping; `@moq/watch` plays one broadcast at a time. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. ## Text diff --git a/doc/concept/use-case/conferencing.md b/doc/concept/use-case/conferencing.md index 5ba66966623..98af2b0e5e0 100644 --- a/doc/concept/use-case/conferencing.md +++ b/doc/concept/use-case/conferencing.md @@ -13,7 +13,7 @@ features first. ## What MoQ offers -- **Publish and subscribe on one session.** Each participant is a broadcast; peers discover each other through announcements on a path prefix. No SDP offer/answer, no renegotiation when someone joins. +- **Publish and subscribe on one session.** Each source is a broadcast, such as `participant/camera.hang` and `participant/screen.hang`, under the participant's shared announcement prefix. Give siblings a shared [clock](/concept/hang#catalog) so consumers can align them by wall time. No SDP offer/answer, no renegotiation when someone joins. - **Latency you choose.** WebRTC decides when to drop. A MoQ subscriber sets its own budget per track, so audio can be near-lossless at 500 ms while video stays at the live edge. - **Scale from the same relay.** A relay that serves a two-person call also serves a ten-thousand-viewer webinar. There is no SFU/CDN split. - **Full pipeline control in the browser.** WebCodecs frames and `AudioData` samples are yours: render to a texture, run a model, apply effects. diff --git a/js/publish/src/broadcast.test.ts b/js/publish/src/broadcast.test.ts index b0d0c5f0be7..813141bced9 100644 --- a/js/publish/src/broadcast.test.ts +++ b/js/publish/src/broadcast.test.ts @@ -30,6 +30,29 @@ const audioConfig: Catalog.AudioConfig = { container: { kind: "legacy" }, }; +test("sibling broadcasts on one page advertise the same capture clock", async () => { + const origin = new Origin.Producer(); + const camera = new Broadcast({ enabled: true, origin, name: Path.from("participant/camera.hang") }); + const screen = new Broadcast({ enabled: true, origin, name: Path.from("participant/screen.hang") }); + try { + camera.video("video").config.set(videoConfig); + screen.video("video").config.set(videoConfig); + await settle(); + const cameraClock = (await readCatalog(camera))?.clock; + const screenClock = (await readCatalog(screen))?.clock; + if (!cameraClock || !screenClock) throw new Error("siblings must advertise their page clock"); + expect(cameraClock).toEqual(screenClock); + const captured = 1_000_000; + expect(Catalog.wallClockTime(cameraClock, captured, 1_000_000)).toEqual( + Catalog.wallClockTime(screenClock, captured, 1_000_000), + ); + } finally { + camera.close(); + screen.close(); + origin.close(); + } +}); + test("folds video and audio renditions into the catalog by full track name", async () => { const broadcast = new Broadcast({ enabled: true, display: { width: 1920, height: 1080 }, flip: true }); diff --git a/quest/m2/README.md b/quest/m2/README.md index d74752286f7..b3ea81243bd 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -167,7 +167,6 @@ ranked just below deterministic TS legs, which it requires. - [Runtime trait](/quest/m2/runtime-trait.md) - one runtime trait for spawn and time replaces web-async, with tokio, browser, io_uring, and sim implementations - [QUIC on the sim](/quest/m2/quic-sim.md) - the sim drives `moq-quic` over in-memory datagrams with latency and loss - [Seeded sim](/quest/m2/sim-seed.md) - sim tests draw every jitter from one seeded generator and replay bit-exact -- [Sibling broadcasts on one clock](/quest/m2/sibling-broadcasts.md) - one broadcast per source (webcam, screen, each camera, each language) under a shared prefix, sharing one clock, documented and tested in Rust and JS - [DVR rewind](/quest/m2/archive-dvr.md) - follow-on capability after the release contract is stable diff --git a/quest/m2/sibling-broadcasts.md b/quest/m2/sibling-broadcasts.md deleted file mode 100644 index 3d7e4e11934..00000000000 --- a/quest/m2/sibling-broadcasts.md +++ /dev/null @@ -1,62 +0,0 @@ -# [S] Sibling broadcasts on one clock - -## Goal - -Several distinct sources (a webcam and a screen share, each of a robot's -cameras, each audio language) publish as sibling broadcasts, one per source, -under a shared path prefix such as `robot/front.hang` and `robot/rear.hang`. -They advertise the same catalog `clock`, so a consumer that discovers them -through one announce prefix aligns them by wall time. The pattern is -documented and tested in Rust and JS. Requested downstream; the robot case -motivates it. - -Non-goals: - -- A source-grouped catalog, where one broadcast carries several sources each - with its own renditions. Decided 2026-10-12 by the maintainer: one broadcast - per camera angle, sharing a clock, instead of another level of catalog - complexity. Every `video` and `audio` entry stays a rendition of one picture - or sound. -- A player that renders several sibling broadcasts in lockstep. js/watch - plays one broadcast; aligning siblings is the consumer's job, through the - shared clock. -- Sharing a clock across broadcasts from the bindings, which expose no mux - `Clock`; extend [the shared timebase](/quest/m1/ffi-timebase.md) when a - binding app needs it. -- Tying a `json` or `binary` track to one source. Those sections are - application-defined; an application that needs the association defines it, - presumably in a catalog extension section or by publishing the track in - that source's broadcast. - -## Plan - -Most of the pattern ships already, so this is mostly docs and tests: - -- Rust: one `moq_mux::Clock` (it is `Copy`) passed to each sibling's - `catalog::Config::with_clock`. A separate process adopting another - broadcast's mapping needs the derivative-clock work planned in #5345, which - is related, not required: one publisher process covers the motivating - cases. -- JS: js/publish derives every broadcast's clock from - `performance.timeOrigin`, so siblings on one page already advertise the - same clock. js/room already publishes `{identity}/camera.hang` and - `{identity}/screen.hang` and discovers them by announce prefix. - -Tests: in Rust, two sibling catalogs on one clock advertise equal `clock` -sections, and frames captured at the same instant map to the same wall time -through `wall_clock`; in JS, two broadcasts on one page advertise the same -clock. Discovery by prefix is already covered by moq-net and js/room. - -Docs: edit the existing paragraphs rather than adding a page, likely the -`Clock` entry in `doc/concept/hang.md` and the conferencing use case, to say -that one source is one broadcast and that siblings share a clock. - -Decided 2026-10-12: m2, ranked last. Nothing blocks it, and nothing in m1 -waits on it. - -Public API: none. Wire: none. - -## Related - -- [Teleoperation](/quest/m2/teleop/README.md) - the robot line whose multi-camera case motivates this -- [Shared timebase in the bindings](/quest/m1/ffi-timebase.md) - shares one offset within a binding broadcast; sharing across broadcasts would extend it diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index ccb7cebdec5..125567f1b06 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -1727,6 +1727,56 @@ mod test { assert_eq!(snapshot.clock, catalog.clock().wall()); } + #[test] + fn sibling_broadcasts_share_a_clock_and_capture_time() { + use std::time::{Duration, Instant, SystemTime}; + + let captured = Instant::now(); + let wall = SystemTime::UNIX_EPOCH + Duration::from_millis(hang::catalog::MOQ_EPOCH_UNIX_MILLIS + 60_000); + let clock = crate::Clock::at(captured, wall).unwrap(); + let (origin, _driver) = moq_net::origin::Producer::new(Default::default()); + let mut front = origin.create_broadcast("robot/front.hang").unwrap(); + let mut rear = origin.create_broadcast("robot/rear.hang").unwrap(); + let catalogs = [ + Producer::new(&mut front, Config::default().with_clock(clock)).unwrap(), + Producer::new(&mut rear, Config::default().with_clock(clock)).unwrap(), + ]; + let mut broadcasts = [front, rear]; + for (catalog, broadcast) in catalogs.iter().zip(&mut broadcasts) { + assert_eq!(catalog.snapshot().clock, clock.wall()); + let timestamp = catalog.clock().capture(captured).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut video = catalog + .video( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Video), + VideoConfig::new(hang::catalog::VideoCodec::VP8), + ) + .unwrap(); + let mut consumer = video.consume(); + video + .write(crate::container::Frame { + timestamp, + duration: None, + payload: Bytes::from_static(b"keyframe"), + keyframe: true, + }) + .unwrap(); + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(Some(mut group))) = consumer.poll_recv_group(&waiter) else { + panic!("captured group is published"); + }; + let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { + panic!("captured frame is published"); + }; + let timestamp = frame.timestamp.expect("captured frame is timed"); + assert_eq!(catalog.clock().wall_clock(timestamp).unwrap(), wall); + assert_eq!(catalog.snapshot().clock.unwrap().wall_clock(timestamp).unwrap(), wall); + } + } + /// How far `pts` lands from `clock`'s reading now, which is zero for a frame live on arrival. fn lateness(clock: crate::Clock, pts: moq_net::Timestamp) -> std::time::Duration { let now = clock.now().as_micros();