From 9e8a3d29bbcfafb7a1ad0926284a4d648b6f01ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 20:50:26 -0700 Subject: [PATCH 01/13] quest: claim reference-announcements Co-Authored-By: Claude Opus 5.5 From 380b0386ecca2ee889eeb220409cec3e04455e6d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:18:57 -0700 Subject: [PATCH 02/13] feat(mux): follow referenced broadcasts' announcements Add moq_mux::Follow (Source::follow, Follow::new), built on announced_broadcast: each instance serving a sibling is handed out as it starts or takes over, and its end is reported. A reference naming an epoch follows only its own instance (best effort over epochless routes). moqsrc pumps follow their sibling: the first start binds the rendition, an end idles its pad, and another instance restarts the stream on the same pad. Exporters bind a sibling once it is first announced instead of failing, and end the rendition when that instance goes. @moq/watch drops a rendition whose broadcast proves another epoch from its effective catalog. Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 2 +- doc/concept/hang.md | 2 +- js/watch/src/broadcast.test.ts | 31 ++ js/watch/src/broadcast.ts | 35 ++- rs/moq-gst/src/source/imp.rs | 481 +++++++++++++++++++++-------- rs/moq-mux/src/container/source.rs | 115 ++++++- rs/moq-mux/src/lib.rs | 2 +- rs/moq-mux/src/source.rs | 348 +++++++++++++++++++-- 8 files changed, 844 insertions(+), 172 deletions(-) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 5b8c727fd14..2230e4a8256 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -124,7 +124,7 @@ rewound timestamps forward. Pads are named by kind and appear as the catalog announces renditions: `video_0`, `video_1`, `audio_0`. A rendition referencing another broadcast reads -that broadcast on the same connection. Link the pad you want by name; the terse +that broadcast on the same connection and follows its announcements the same way. Link the pad you want by name; the terse `moqsrc ! decodebin3` form links only the first pad offered, which may be audio. `moqsrc` follows the broadcast's announcements: when the publisher restarts, it switches to the new broadcast on the same pads, kept by rendition diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 35a5cab213d..4373b75c6dc 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. A consumer reads each referenced path from the first broadcast it resolves there. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance fails the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; an exporter starts on the first one announced, keeps it, and ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/js/watch/src/broadcast.test.ts b/js/watch/src/broadcast.test.ts index 317d0b279dc..b02fe01bd71 100644 --- a/js/watch/src/broadcast.test.ts +++ b/js/watch/src/broadcast.test.ts @@ -355,6 +355,37 @@ describe("relativeBroadcast", () => { } }); + it("drops a rendition from the catalog while its broadcast proves another epoch", async () => { + const owner = new Origin.Producer(); + const main = publish(owner, Path.from("a/b")); + const [oldEpoch, newEpoch] = [Moq.Epoch.mint(), Moq.Epoch.mint()]; + const old = owner.createBroadcast(Path.from("a/source")); + old.announce({ epoch: oldEpoch }); + const named = { ...video("avc1.64001f", "./source"), epoch: oldEpoch } as Catalog.VideoConfig; + const source = new Broadcast({ + origin: owner, + name: Path.from("a/b"), + enabled: true, + catalogFormat: "manual", + catalog: { video: { renditions: { plain: video("avc1.64001f", "./source"), named } } } as Catalog.Root, + }); + try { + await settle(); + expect(videoRenditions(source)).toEqual(["named", "plain"]); + + const replacement = owner.createBroadcast(Path.from("a/source")); + replacement.announce({ epoch: newEpoch }); + await withoutWarnings(settle); + expect(videoRenditions(source)).toEqual(["plain"]); + replacement.close(); + } finally { + source.close(); + old.close(); + main.close(); + owner.close(); + } + }); + it("uses the catalog's own broadcast when the reference is absent, empty, or self", async () => { const { source, owner } = broadcast("a/b"); const effect = new Effect(); diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index 7e76e00741f..62066c83821 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -9,16 +9,25 @@ import { toHang } from "./msf"; type ReferencedRendition = { broadcast?: Path.Relative; + epoch?: Catalog.ReferenceEpoch; }; +// Whether a rendition's reference names something this consumer can play. +type Usable = (rel: Path.Relative | undefined, epoch: Moq.Epoch.Valid | undefined) => boolean; + // Either the catalog's own broadcast, or a sibling to consume by path. type RelativeTarget = { local: true } | { local: false; path: Moq.Path.Valid }; -function filterRenditions( - renditions: Record, - usable: (rel: Path.Relative | undefined) => boolean, -): Record { - return Object.fromEntries(Object.entries(renditions).filter(([, config]) => usable(config.broadcast))); +function filterRenditions(renditions: Record, usable: Usable): Record { + return Object.fromEntries( + Object.entries(renditions).filter(([, config]) => usable(config.broadcast, config.epoch)), + ); +} + +// Whether `active` is proven to be another instance than `epoch` names. An epochless route cannot +// prove its instance, so only a known mismatch counts. +function provesOther(active: Moq.Broadcast.Consumer | undefined, epoch: Moq.Epoch.Valid | undefined): boolean { + return epoch !== undefined && active?.epoch !== undefined && active.epoch !== epoch; } // Whether an announced route covers `path`, until closed. Hidden routes count: a service claim @@ -47,7 +56,7 @@ function watchCovered(origin: Moq.Origin.Table, path: Moq.Path.Valid): { covered // Every section carrying a `broadcast` reference must be listed here, same as `Catalog.checkResolvable`; // one left out silently exempts its tracks from the reachability filter. -function filterCatalog(catalog: Catalog.Root, usable: (rel: Path.Relative | undefined) => boolean): Catalog.Root { +function filterCatalog(catalog: Catalog.Root, usable: Usable): Catalog.Root { return { ...catalog, video: catalog.video @@ -182,12 +191,16 @@ export class Broadcast { } // Publish the catalog minus every rendition this consumer cannot use: one naming a - // broadcast that isn't announced to us. Selecting one of those would render nothing, - // since `relativeBroadcast` resolves it to no broadcast at all. Reruns as announcements - // arrive, so a rendition appears once its broadcast does. + // broadcast that isn't announced to us, or one whose broadcast proves another epoch than it + // names. Selecting one of those would render nothing, since `relativeBroadcast` resolves it to + // no broadcast at all. Reruns as announcements arrive, so a rendition appears once its + // broadcast does. Only a rendition naming an epoch resolves its broadcast here, sharing the + // request playback uses. #runFiltered(effect: Effect): void { const raw = effect.get(this.#raw); - const usable = (rel: Path.Relative | undefined) => this.#relativeTarget(effect, rel) !== undefined; + const usable: Usable = (rel, epoch) => + this.#relativeTarget(effect, rel) !== undefined && + (epoch === undefined || !provesOther(this.#relativeActive(effect, rel), epoch)); effect.set(this.#out.catalog, raw ? filterCatalog(raw, usable) : undefined); } @@ -434,7 +447,7 @@ export class Broadcast { epoch?: Moq.Epoch.Valid, ): Moq.Broadcast.Consumer | undefined { const active = this.#relativeActive(effect, rel); - if (!active || epoch === undefined || active.epoch === undefined || active.epoch === epoch) return active; + if (!active || !provesOther(active, epoch)) return active; console.warn("ignoring rendition: its broadcast serves another epoch", active.path, active.epoch, epoch); return undefined; } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7dae3e7c2a1..546c24e52ec 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -988,15 +988,16 @@ fn reconcile( } }; - // Bind without waiting: an unavailable sibling or track must not park the catalog loop + // Follow without waiting: an unavailable sibling or track must not park the catalog loop // and hold up every other rendition. The pump resolves both under its cancellation scope. - let binding = match source.bind(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { - Ok(binding) => binding, + let follow = match source.follow(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { + Ok(follow) => follow, Err(err) => { gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); continue; } }; + let sibling = track.broadcast != source.resolve_reference(None); let (cancel_tx, cancel_rx) = watch::channel(false); let state = Arc::new(PumpState::new()); @@ -1008,9 +1009,11 @@ fn reconcile( kind: d.kind, name: name.clone(), caps: d.shape.caps.clone(), - binding, + follow, + sibling, track, container, + descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), finished: playback.finished.clone(), @@ -1102,9 +1105,16 @@ struct Pump { /// per-kind counter instead, and isn't known until the subscription resolves. name: String, caps: gst::Caps, - binding: moq_mux::Binding, + /// Each instance of the broadcast the rendition references. + follow: moq_mux::Follow, + /// The rendition references another broadcast than its catalog's, whose instances come and + /// go on their own announcements. + sibling: bool, track: Track, + /// The wire format of the first instance. container: moq_mux::catalog::hang::Container, + /// What builds it again for a later instance. + descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, /// The run's [`Playback::finished`]. @@ -1116,155 +1126,266 @@ struct Pump { shutdown: watch::Receiver, } +/// Why a pump stopped reading one instance without ending. +enum Stopped { + /// Another instance took over: restart on it. + Restart(moq_net::broadcast::Consumer), + /// The instance stopped serving: idle until the next one. + Idle, +} + impl Pump { /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the /// track ends, errors, or `cancel` fires. The pad goes back to the session without an EOS /// either way. Returns the rendition if its track ended rather than being cancelled, so the /// run can end the pad of one the catalog retired. + /// + /// A sibling's announcements drive the pump: its first start binds the rendition, an end + /// idles the pad, and another instance taking over restarts the stream on the same pad. async fn run(self) -> Option { let Pump { element, kind, name, caps, - binding, + mut follow, + sibling, track: media, container, + descriptor, pads, - timeline, + mut timeline, finished, state, mut cancel, shutdown, } = self; - // Resolves once the publisher answers, with the track info or with an error (which is - // what ending a broadcast produces for a name nobody served). A publisher that answers - // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and - // holding the wait here rather than in `reconcile` keeps it off every other - // rendition. - let subscriber = tokio::select! { - _ = cancel.changed() => return None, - subscriber = async { - let broadcast = binding.broadcast().await?; - let track = broadcast.track(&media.name)?; - Ok::<_, moq_mux::Error>(track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))).await?) - } => match subscriber { - Ok(subscriber) => subscriber, - Err(err) => { - gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); - return Some((kind, name)); - } - } - }; - let mut track = moq_mux::container::Consumer::new(subscriber, container); - - // Winning this is what earns a pad. Losing means a teardown got here while the - // subscription was still resolving (this rendition was removed, reshaped, or outlived by - // a closing catalog), and it must not publish a pad at all: the watch alone can't say - // that, since a cancel landing just after we read it would leave a pad exposed and then - // yanked without an EOS. - if !state.go_live() { - return None; - } + let mut container = Some(container); + // The pad once this pump owns it, from the first instance on. + let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; + let mut next: Option = None; - // A pad appears only once its track is live, so a pad downstream can link to is a promise - // that the rendition is actually flowing, and only a rendition that gets this far claims a - // pad id. A kept pad starts a new stream on it, with caps pushed even when they differ, so - // a format downstream can't take fails loudly as not-negotiated rather than going quiet. - let (pad, _owned) = match pads.claim(&element, (kind, name.clone()), &cancel)? { - Claim::New(pad, owned) => { - let obj = element.upgrade()?; - pad.set_active(true).ok()?; - begin(&pad, &name, &caps); - obj.add_pad(&pad).ok()?; - (pad, owned) - } - Claim::Kept(pad, owner) => { - let owned = tokio::select! { - biased; + loop { + let broadcast = match next.take() { + Some(broadcast) => broadcast, + None => tokio::select! { _ = cancel.changed() => return None, - owned = owner.lock_owned() => owned, - }; - // The pump this one replaces records its track as finished before letting go, so a - // replacement started while it was reading the end (a reshape) can't replay it. - if finished - .lock() - .unwrap() - .contains(&((kind, name.clone()), media.clone())) - { - return Some((kind, name)); - } - tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, owned) - } - }; - // Stop flushes only the pads it finds, and this one may have been added just after, while - // this pump's cancel is still on its way. Flushing it here keeps a push from blocking stop. - if *shutdown.borrow() { - let _ = pad.set_active(false); - } + followed = follow.next() => match followed { + Ok(moq_mux::Followed::Start(broadcast)) => broadcast, + Ok(moq_mux::Followed::End) => continue, + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + }, + }; - let mut sent_segment = false; - loop { - tokio::select! { - // This rendition is being torn down (its run ended, or a catalog update replaced it). An - // end that arrived alongside still finishes the track, or the replacement waiting for - // this pad would replay it. - _ = cancel.changed() => { - if ended(&mut track) { - finished.lock().unwrap().insert(((kind, name.clone()), media)); + // Resolves once the publisher answers, with the track info or with an error (which is + // what ending a broadcast produces for a name nobody served). A publisher that answers + // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and + // holding the wait here rather than in `reconcile` keeps it off every other + // rendition. + let subscribed = tokio::select! { + _ = cancel.changed() => return None, + followed = follow.next() => match followed { + Ok(moq_mux::Followed::Start(broadcast)) => Err(Stopped::Restart(broadcast)), + Ok(moq_mux::Followed::End) => Err(Stopped::Idle), + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + subscriber = async { + broadcast + .track(&media.name)? + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) + .await + } => match subscriber { + Ok(subscriber) => Ok(subscriber), + // The instance went while subscribing: that is its end, not this rendition's. + Err(_) if sibling && broadcast.is_closed() => Err(Stopped::Idle), + Err(err) => { + gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); + return Some((kind, name)); } - return None; } - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return Some((kind, name)); - } - }; - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other pumps. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); - } + }; + let subscriber = match subscribed { + Ok(subscriber) => subscriber, + Err(stopped) => { + next = stopped.restart(); + continue; + } + }; + let container = match container.take() { + Some(container) => container, + None => match moq_mux::catalog::hang::Container::new( + &descriptor, + match kind { + TrackKind::Video => moq_mux::container::Kind::Video, + TrackKind::Audio => moq_mux::container::Kind::Audio, + }, + ) { + Ok(container) => container, + Err(err) => { + gst::warning!(CAT, "rendition {name} has an unsupported container: {err:?}"); + return Some((kind, name)); + } + }, + }; + let mut track = moq_mux::container::Consumer::new(subscriber, container); + + let pad = match &owned { + // Another instance on the pad this pump owns: a new stream, on a timeline of its own, + // since the instance's timestamps share no origin with the run's. + Some((pad, _)) => { + timeline = Arc::default(); + tokio::task::block_in_place(|| begin(pad, &name, &caps)); + pad.clone() + } + None => { + // Winning this is what earns a pad. Losing means a teardown got here while the + // subscription was still resolving (this rendition was removed, reshaped, or + // outlived by a closing catalog), and it must not publish a pad at all: the + // watch alone can't say that, since a cancel landing just after we read it + // would leave a pad exposed and then yanked without an EOS. + if !state.go_live() { + return None; + } + + // A pad appears only once its track is live, so a pad downstream can link to is a + // promise that the rendition is actually flowing, and only a rendition that gets + // this far claims a pad id. A kept pad starts a new stream on it, with caps pushed + // even when they differ, so a format downstream can't take fails loudly as + // not-negotiated rather than going quiet. + let (pad, guard) = match pads.claim(&element, (kind, name.clone()), &cancel)? { + Claim::New(pad, owned) => { + let obj = element.upgrade()?; + pad.set_active(true).ok()?; + begin(&pad, &name, &caps); + obj.add_pad(&pad).ok()?; + (pad, owned) + } + Claim::Kept(pad, owner) => { + let owned = tokio::select! { + biased; + _ = cancel.changed() => return None, + owned = owner.lock_owned() => owned, + }; + // The pump this one replaces records its track as finished before letting + // go, so a replacement started while it was reading the end (a reshape) + // can't replay it. + if finished + .lock() + .unwrap() + .contains(&((kind, name.clone()), media.clone())) + { return Some((kind, name)); } - Err(_) => return Some((kind, name)), + tokio::task::block_in_place(|| begin(&pad, &name, &caps)); + (pad, owned) } + }; + // Stop flushes only the pads it finds, and this one may have been added just + // after, while this pump's cancel is still on its way. Flushing it here keeps a + // push from blocking stop. + if *shutdown.borrow() { + let _ = pad.set_active(false); } - Ok(None) => { - // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.lock().unwrap().insert(((kind, name.clone()), media)); - return Some((kind, name)); + owned = Some((pad.clone(), guard)); + pad + } + }; + + let mut sent_segment = false; + let stopped = loop { + tokio::select! { + // This rendition is being torn down (its run ended, or a catalog update replaced + // it). An end that arrived alongside still finishes the track, or the replacement + // waiting for this pad would replay it. + _ = cancel.changed() => { + if ended(&mut track) { + finished.lock().unwrap().insert(((kind, name.clone()), media)); + } + return None; } - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - return Some((kind, name)); + followed = follow.next() => match followed { + Ok(moq_mux::Followed::Start(broadcast)) => break Stopped::Restart(broadcast), + Ok(moq_mux::Followed::End) => break Stopped::Idle, + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + frame = track.read() => match frame { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + let buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); + } + return Some((kind, name)); + } + }; + let segment = (!sent_segment).then(|| { + let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other pumps. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); + } + return Some((kind, name)); + } + Err(_) => return Some((kind, name)), + } + } + // A sibling's track stopping with its instance is that instance's end, which + // idles the pad for the next one rather than ending the rendition. + Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, + Ok(None) => { + // Recorded while this pump still owns the pad, before any replacement can claim it. + finished.lock().unwrap().insert(((kind, name.clone()), media)); + return Some((kind, name)); + } + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + return Some((kind, name)); + } } } + }; + match &stopped { + Stopped::Restart(_) => gst::info!(CAT, "rendition {name} restarts on a new instance"), + Stopped::Idle => gst::info!(CAT, "rendition {name} is offline, holding its pad"), } + next = stopped.restart(); + } + } +} + +impl Stopped { + /// The instance to restart on, if any. + fn restart(self) -> Option { + match self { + Stopped::Restart(broadcast) => Some(broadcast), + Stopped::Idle => None, } } } @@ -2584,6 +2705,13 @@ mod session_tests { publisher } + /// The `cam` broadcast a rendition of `room` references as `./cam`, under `route`. + fn sibling(origin: &moq_net::origin::Producer, route: moq_net::origin::Route) -> Self { + let publisher = Self::build(origin.create_broadcast("cam").unwrap(), video_rendition()); + publisher.broadcast.announce(route).unwrap(); + publisher + } + /// A broadcast at no path, for a covering route to serve. fn standalone(config: VideoConfig) -> Self { Self::build(moq_net::broadcast::Info::new().produce(), config) @@ -2733,12 +2861,16 @@ mod session_tests { kind: super::TrackKind::Video, name: "video".into(), caps: gst::Caps::new_empty_simple("video/x-h264"), - binding: moq_mux::Binding::new(publisher.broadcast.consume()), + follow: moq_mux::Source::new(origin().consume(), publisher.broadcast.consume()) + .follow(None, None) + .unwrap(), + sibling: false, track: super::Track { broadcast: None, name: "video".into(), }, container: moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), finished: std::sync::Arc::default(), @@ -2993,6 +3125,109 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A `room` whose video rendition references the `cam` sibling, naming `epoch` when given. + fn referencing(origin: &moq_net::origin::Producer, epoch: Option) -> Publisher { + let mut config = video_rendition(); + config.broadcast = Some("./cam".into()); + config.epoch = epoch; + Publisher::new(origin, Default::default(), config) + } + + /// A sibling announced after the catalog starts its rendition, its end holds the pad, and its + /// return starts a new stream on that pad. + #[test] + fn a_late_sibling_starts_its_rendition_and_returns_after_an_end() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); + stop(session); + + assert!(recorder.seen().contains(&Seen::Start(pad)), "no new stream on the kept pad"); + recorder.assert_one_pad_held(); + } + + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on + /// the same pad, and nothing more from the old instance gets through. + fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let mut old = Publisher::sibling(&origin, route()); + let session = follow(&element, &origin); + let pad = recorder.written(&mut old, 1); + + let mut new = Publisher::sibling(&origin, route()); + assert_eq!(recorder.written(&mut new, 2), pad, "the restart moved to another pad"); + old.write(1); + recorder.written(&mut new, 3); + stop(session); + + let seen = recorder.seen(); + let switched = seen.iter().position(|seen| *seen == Seen::Buffer(pad.clone(), 2)).unwrap(); + assert!( + !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), + "the replaced sibling kept playing: {seen:?}" + ); + recorder.assert_one_pad_held(); + } + + #[test] + fn a_sibling_restart_switches_on_the_same_pad() { + sibling_restart(epoch); + } + + #[test] + fn an_epochless_sibling_restart_switches_on_the_same_pad() { + sibling_restart(moq_net::origin::Route::default); + } + + /// A reference naming its sibling's epoch never reads the instance replacing it: the pad + /// idles, and plays again once the named instance serves the path again. + #[test] + fn a_sibling_restart_idles_a_reference_naming_the_old_epoch() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let named = moq_net::Epoch::mint(); + let _room = referencing(&origin, Some(named.clone())); + let mut old = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(named)); + let session = follow(&element, &origin); + let pad = recorder.written(&mut old, 1); + + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + let mut new = Publisher::sibling(&origin, epoch()); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + new.write(2); + drop(new); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + + assert_eq!(recorder.written(&mut old, 3), pad, "the named instance moved to another pad"); + stop(session); + assert!( + !recorder.seen().contains(&Seen::Buffer(pad, 2)), + "another epoch played: {:?}", + recorder.seen() + ); + recorder.assert_one_pad_held(); + } + /// Re-pricing the same instance is an update, which the run rides out without a new stream. #[test] fn an_update_keeps_the_run() { diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 0335dfd1d5e..5910aa3fd8a 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -62,6 +62,8 @@ enum SourceState { /// The resolved consumer, reading frames. Boxed because it's much larger than /// the `Subscribing` variant (clippy `large_enum_variant`). Active(Box>), + /// The sibling instance it read went, and its rendition with it. + Ended, } /// A per-rendition source that normalizes frame shape (Annex-B → @@ -97,6 +99,8 @@ pub(crate) struct ExportSource { ends: Option<(u64, u64)>, /// The counters the export source reports for this rendition, as received, before any transform. received: std::sync::Arc, + /// The sibling broadcast the rendition reads, once resolved: its end ends the rendition. + instance: Option, } impl ExportSource { @@ -184,6 +188,7 @@ impl ExportSource { cuts: 0, ends: None, received: source.received(name), + instance: None, }; source.resolve_video_dimensions(&[])?; Ok(Some(source)) @@ -222,6 +227,7 @@ impl ExportSource { cuts: 0, ends: None, received: source.received(name), + instance: None, })) } @@ -253,6 +259,7 @@ impl ExportSource { cuts: 0, ends: None, received: source.received(name), + instance: None, }) } @@ -364,19 +371,31 @@ impl ExportSource { /// Read normalized media or a clean group boundary. pub fn poll_event(&mut self, waiter: &kio::Waiter) -> Poll>> { + if matches!(self.state, SourceState::Ended) { + return Poll::Ready(Ok(None)); + } + // Resolve a cross-broadcast reference into a broadcast before subscribing. if matches!(self.state, SourceState::Requesting(..)) { - let (broadcast, name) = { + let (broadcast, name, sibling) = { let SourceState::Requesting(pending, name) = &self.state else { unreachable!("just matched Requesting"); }; - (ready!(pending.poll_broadcast(waiter))?, name.clone()) + (ready!(pending.poll_broadcast(waiter))?, name.clone(), pending.sibling()) }; + if sibling { + self.instance = Some(broadcast.clone()); + } let mut subscription = moq_net::track::Subscription::default().with_max_delay(self.max_delay); if let Some(start) = self.start { subscription = subscription.with_groups(start..); } - self.state = SourceState::Subscribing(broadcast.track(&name)?.subscribe(subscription)); + let track = match broadcast.track(&name) { + Ok(track) => track, + Err(_) if self.instance_ended(waiter) => return self.end(), + Err(err) => return Poll::Ready(Err(err.into())), + }; + self.state = SourceState::Subscribing(track.subscribe(subscription)); } // Resolve the subscription before reading any frames. @@ -386,7 +405,12 @@ impl ExportSource { let SourceState::Subscribing(pending) = &self.state else { unreachable!("just matched Subscribing"); }; - ready!(pending.poll_ok(waiter))? + match pending.poll_ok(waiter) { + Poll::Ready(Ok(track)) => track, + Poll::Ready(Err(_)) | Poll::Pending if self.instance_ended(waiter) => return self.end(), + Poll::Ready(Err(err)) => return Poll::Ready(Err(err.into())), + Poll::Pending => return Poll::Pending, + } }; let media = self .media @@ -407,7 +431,21 @@ impl ExportSource { let SourceState::Active(consumer) = &mut self.state else { unreachable!("subscription resolved into an Active consumer"); }; - match ready!(consumer.poll_item(waiter))? { + let item = match consumer.poll_item(waiter) { + Poll::Ready(Ok(item)) => item, + // The field, not `instance_ended`, so the borrow stays off the consumer. + Poll::Ready(Err(_)) | Poll::Pending + if self + .instance + .as_ref() + .is_some_and(|instance| instance.poll_closed(waiter).is_ready()) => + { + return self.end(); + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + }; + match item { Some(Item::Read(Read::Frame { group, frame })) => { if self.ends.is_some_and(|(end_group, _)| end_group != group) { self.finish_ends(); @@ -485,6 +523,23 @@ impl ExportSource { } } + /// Whether the sibling broadcast this rendition reads has stopped serving, registering + /// `waiter` for when it does. Its track stalling or failing then is that instance's end, + /// which ends the rendition rather than the export once what it delivered is read; following + /// the instance that replaces it is the export's own call. + fn instance_ended(&self, waiter: &kio::Waiter) -> bool { + self.instance + .as_ref() + .is_some_and(|instance| instance.poll_closed(waiter).is_ready()) + } + + /// End the rendition with its instance. + fn end(&mut self) -> Poll>> { + self.finish_ends(); + self.state = SourceState::Ended; + Poll::Ready(Ok(None)) + } + fn finish_ends(&mut self) { if let Some((_, count)) = self.ends.take() { self.skips += count; @@ -665,6 +720,56 @@ mod tests { } } + /// A sibling rendition's first announcement starts it, a replacement is never read, and its + /// instance going ends the rendition instead of failing the export, both mid-stream and + /// before the subscription resolves. + #[tokio::test(start_paused = true)] + async fn a_sibling_rendition_starts_late_and_ends_with_its_instance() { + use std::time::Duration; + + use crate::container::test_util::raw_frame; + + let origin = crate::source::produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + tokio::task::yield_now().await; + let source = crate::source::resolved(&origin, "a/pub").await; + let config = audio(Some("./source")); + let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + let mut read = async || { + tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))) + .await + .ok() + .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros())) + }; + let wire = || HangContainer::Legacy(crate::container::Kind::Audio); + let info = || hang::container::track_info(hang::catalog::PRIORITY.audio); + + assert_eq!(read().await, None, "nothing announces the sibling yet"); + let old = origin.publish("a/source", Default::default()).unwrap(); + let mut track = crate::container::Producer::new(old.create_track("audio", info()).unwrap(), wire()); + track.write(raw_frame(0, b"old", true)).unwrap(); + assert_eq!(read().await, Some(Some(0)), "the late announcement starts it"); + + let new = origin.publish("a/source", Default::default()).unwrap(); + let mut replacement = crate::container::Producer::new(new.create_track("audio", info()).unwrap(), wire()); + replacement.write(raw_frame(1_000_000, b"new", true)).unwrap(); + track.write(raw_frame(24_000, b"old", true)).unwrap(); + assert_eq!(read().await, Some(Some(24_000)), "the replacement is never read"); + + drop(old); + assert_eq!(read().await, Some(None), "its end ends the rendition"); + + // A rendition still subscribing when its instance goes ends too. + let mut late = ExportSource::for_audio(&source, "late", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + let ended = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| late.poll_read(waiter))).await; + assert!(matches!(ended, Ok(Ok(None))), "{:?}", ended.map(|read| read.map(|_| ()))); + drop((track, replacement, new)); + } + /// The requested budget must be present on the first SUBSCRIBE, a live start's too. Updating /// it after SUBSCRIBE_OK cannot recover backlog the publisher already skipped, nor the groups a /// zero budget drops as superseded before the update lands. diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 879e436fbec..4b46cb87dde 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -48,7 +48,7 @@ pub mod timeline; pub use clock::Clock; pub use error::*; pub use pace::Pacer; -pub use source::{Binding, Source}; +pub use source::{Binding, Follow, Followed, Source}; /// Translate a catalog entry's declared compression into the flag the codecs take. /// diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 565911b29bf..1594c95ff40 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -23,12 +23,16 @@ type Pins = Arc> /// reference. Build one with [`Source::new`]. /// /// A source serves its own path from the broadcast it was built from, and keeps the first -/// broadcast it resolves for each sibling. Every later request for a path is served from that -/// handle ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), -/// [`resolve`](Self::resolve), [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), -/// and an export's track requests). A route replaced after that is never resolved, epoch or -/// not, so two publishers never splice into one export; a request on a held broadcast that has -/// ended fails instead. Clones share the held broadcasts. +/// broadcast it resolves for each sibling; a [binding](Self::bind) waits for the sibling's first +/// announcement to resolve it. Every later request for a path is served from that handle +/// ([`broadcast`](Self::broadcast), +/// [`catalog`](Self::catalog), [`resolve`](Self::resolve), [`bind`](Self::bind), +/// [`subscribe_track`](Self::subscribe_track), and an export's track requests). A route replaced +/// after that is never resolved, epoch or not, so two publishers never splice into one export; a +/// request on a held broadcast that has ended fails instead. Clones share the held broadcasts. +/// +/// A reader that follows a sibling's announcements instead, restarting on each new instance, +/// uses [`follow`](Self::follow), which holds nothing. /// /// A reference may name the instance it expects with an epoch (the catalog's optional `epoch` /// beside `broadcast`). A held or resolved broadcast serving another epoch fails that reference @@ -134,14 +138,23 @@ impl Source { } /// The broadcast `path` is pinned to, or a lookup that pins its answer, either checked - /// against `epoch` on every read. - fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>) -> Binding { + /// against `epoch` on every read. The lookup waits for the path's first announcement when + /// `announced`, and otherwise answers for the routes as they stand. + fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>, announced: bool) -> Binding { let bound = match self.pins.lock().expect("pins poisoned").get(path) { Some(pinned) => Bound::Ready(pinned.clone()), // Unpinned even with an epoch: an epochless route still answers, best effort, and // the read refuses a route that proves another instance. None => Bound::Requested { - request: self.origin.request_broadcast(path, None).into_inner(), + lookup: match announced { + true => { + let mut routing = self.origin.routed_broadcast(path).into_inner(); + // Issue the request now: a handler answers it whether or not anyone reads yet. + let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); + Lookup::Announced(Mutex::new(routing)) + } + false => Lookup::Now(self.origin.request_broadcast(path, None).into_inner()), + }, pins: self.pins.clone(), path: path.clone(), }, @@ -149,6 +162,7 @@ impl Source { Binding { bound, epoch: epoch.cloned(), + sibling: *path != self.path, } } @@ -184,7 +198,7 @@ impl Source { /// the same policy rather than a different one. pub(crate) fn try_request(&self, reference: &impl hang::catalog::Reference) -> Option { let path = self.resolve_reference(reference.broadcast())?; - Some(self.request_path(&path, reference.epoch())) + Some(self.request_path(&path, reference.epoch(), true)) } /// Remove the renditions an export must not select: a disabled one (catalog @@ -234,6 +248,10 @@ impl Source { /// broadcast must be (see [`Source`]). Use it when you need the broadcast handle itself /// (e.g. to FETCH individual groups) rather than a subscription. /// + /// Answers for the routes as they stand: a sibling nothing announces yet fails, rather than + /// waiting for it as [`bind`](Self::bind) does, so a reader choosing among renditions can + /// move on to the next. + /// /// A reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast): the rendition names /// no broadcast, so there is nothing to resolve. @@ -242,14 +260,15 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - self.bind(rel, epoch)?.broadcast().await + self.request_path(&self.target(rel)?, epoch, false).broadcast().await } /// Start one broadcast request now and retain its result for repeated reads. /// /// Unlike [`resolve`](Self::resolve), this issues the request without awaiting or polling. - /// A remote or dynamic handler may answer later. The binding never retries a failed request - /// or resolves the path again after a publisher is replaced. + /// A path nothing announces yet resolves once something does, and a remote or dynamic + /// handler may answer later. Once answered, the binding never resolves the path again, even + /// after its publisher is replaced or goes; [`follow`](Self::follow) does. /// /// A path this source already resolved binds to that broadcast; otherwise the origin is /// asked, deduplicating repeat requests for one path so the catalog and every rendition share @@ -263,15 +282,15 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - Ok(self.request_path(&self.target(rel)?, epoch)) + Ok(self.request_path(&self.target(rel)?, epoch, true)) } /// Resolve an optional cross-broadcast reference and subscribe to track `name`, /// awaiting SUBSCRIBE_OK. /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) - /// subscribes on the catalog broadcast; anything else fetches the referenced broadcast - /// from the origin first, checked against `epoch` as in [`resolve`](Self::resolve). A + /// subscribes on the catalog broadcast; anything else is bound as in [`bind`](Self::bind), + /// waiting for the referenced broadcast's first announcement, and checked against `epoch`. A /// reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// @@ -287,20 +306,160 @@ impl Source { let broadcast = self.bind(rel, epoch)?.broadcast().await?; Ok(broadcast.track(name)?.subscribe(None).await?) } + + /// Follow the broadcast a rendition's reference names across its announcements. + /// + /// Holds nothing: each instance serving a sibling is handed out as it comes online or takes + /// over, and its end is reported, so a reader restarts the rendition on each one. A + /// reference to the catalog broadcast itself hands out only the instance this source was + /// built from and never reports its end, since a reader follows its catalog's own end and + /// restarts by building a new source. + /// + /// With an `epoch`, only that instance (or an epochless route, which cannot prove another + /// one) is handed out: another taking over ends the rendition rather than substituting for + /// it. Rejects an escaping reference exactly as [`resolve`](Self::resolve) does. + pub fn follow( + &self, + rel: Option<&moq_net::path::Relative<'_>>, + epoch: Option<&moq_net::Epoch>, + ) -> crate::Result { + let path = self.target(rel)?; + let instances = match path == self.path { + true => Instances::Fixed(Some(self.broadcast())), + false => Instances::Announced(self.origin.announced_broadcast(&path, None)?), + }; + Ok(Follow::with(instances, epoch)) + } +} + +/// The publisher instances serving a broadcast, as each comes online, takes over, and goes. +/// +/// Built on [`announced_broadcast`](moq_net::origin::Consumer::announced_broadcast), since an +/// announcement is the online and offline signal. Create one with [`Source::follow`] for a +/// catalog reference, or [`Follow::new`] for a path. +pub struct Follow { + instances: Instances, + /// The instance a reference named, if any. + epoch: Option, + /// The instance handed out last, until it stops serving. + current: Option, +} + +enum Instances { + /// Every instance announced at a path. + Announced(moq_net::origin::AnnouncedBroadcast), + /// One instance, handed out once. + Fixed(Option), +} + +/// A change to the instance a [`Follow`] serves. +pub enum Followed { + /// Start on this instance, dropping whatever was read from the one before. + Start(moq_net::broadcast::Consumer), + /// The instance handed out last stopped serving: idle until the next start. + End, +} + +impl Follow { + /// Follow the broadcast at `path` through `origin`, handing out only `epoch` when given. + /// + /// Unlike [`Source::follow`], this needs no catalog broadcast, so a reader can follow the + /// broadcast whose catalog it reads and build a [`Source`] for each instance. + pub fn new( + origin: &moq_net::origin::Consumer, + path: impl moq_net::AsPath, + epoch: Option<&moq_net::Epoch>, + ) -> crate::Result { + let announced = origin.announced_broadcast(path, None)?; + Ok(Self::with(Instances::Announced(announced), epoch)) + } + + fn with(instances: Instances, epoch: Option<&moq_net::Epoch>) -> Self { + Self { + instances, + epoch: epoch.cloned(), + current: None, + } + } + + /// Poll for the next change, registering `waiter` when there is none yet. + /// + /// Each announcement that brings an instance hands it out, so the same one returns after an + /// end. Fails once the origin closes, and with any resolution failure, such as a handler + /// refusing the path. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + let next = match &mut self.instances { + // Followed unpinned and checked here: a pinned follow waits out an epochless route, + // which an epoch reads best effort. + Instances::Announced(announced) => announced.poll_available(waiter).map_err(crate::Error::from), + Instances::Fixed(broadcast) => match broadcast.take() { + Some(broadcast) => Poll::Ready(Ok(broadcast)), + None => Poll::Pending, + }, + }; + if let Poll::Ready(next) = next { + let broadcast = next?; + if proves_other(self.epoch.as_ref(), &broadcast) { + tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); + match self.current.take() { + Some(_) => return Poll::Ready(Ok(Followed::End)), + None => continue, + } + } + // The catalog's own instance ends with the catalog, which its reader follows. + if let Instances::Announced(_) = self.instances { + self.current = Some(broadcast.clone()); + } + return Poll::Ready(Ok(Followed::Start(broadcast))); + } + + if let Some(current) = &self.current + && current.poll_closed(waiter).is_ready() + { + self.current = None; + return Poll::Ready(Ok(Followed::End)); + } + return Poll::Pending; + } + } + + /// The next change; see [`Self::poll_next`]. + pub async fn next(&mut self) -> crate::Result { + kio::wait(|waiter| self.poll_next(waiter)).await + } +} + +/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route +/// cannot prove its instance, so only a known mismatch counts. +fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { + matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) } /// A held broadcast or the retained result of one eagerly issued broadcast request. /// -/// Reading the binding never looks up its path again, even after a failure or publisher -/// replacement. Pending requests follow the origin's routing semantics; issuing a request -/// does not establish an epoch relationship with a separate catalog broadcast. A binding made -/// with an epoch refuses a broadcast proven to serve another one. +/// A pending request waits for a path nothing announces yet and rides out route churn until it +/// resolves (see [`routed_broadcast`](moq_net::origin::Consumer::routed_broadcast)). Once +/// answered, reading the binding never looks up its path again, even after a failure or +/// publisher replacement. Issuing a request does not establish an epoch relationship with a +/// separate catalog broadcast. A binding made with an epoch refuses a broadcast proven to serve +/// another one. /// /// Build one with [`Source::bind`], or with [`Binding::new`] for a broadcast already in hand. pub struct Binding { bound: Bound, /// The instance the reference named, if any. epoch: Option, + /// The reference names another broadcast than the catalog's. + sibling: bool, +} + +/// How a [`Bound::Requested`] path is looked up. +enum Lookup { + /// For the routes as they stand. + Now(moq_net::origin::Requesting), + /// Once something announces the path, riding out route churn. + Announced(Mutex), } enum Bound { @@ -308,7 +467,7 @@ enum Bound { Ready(moq_net::broadcast::Consumer), /// A request issued when the binding was made. Requested { - request: moq_net::origin::Requesting, + lookup: Lookup, /// Where the answer is pinned, the first for its path winning. pins: Pins, path: moq_net::PathOwned, @@ -321,17 +480,30 @@ impl Binding { Self { bound: Bound::Ready(broadcast), epoch: None, + sibling: false, } } + /// Whether the reference names another broadcast than the catalog's. + pub(crate) fn sibling(&self) -> bool { + self.sibling + } + /// Poll for the bound broadcast without blocking. /// /// Concurrent polls and cancelled waits retain the same request and result. pub fn poll_broadcast(&self, waiter: &kio::Waiter) -> Poll> { let broadcast = match &self.bound { Bound::Ready(broadcast) => broadcast.clone(), - Bound::Requested { request, pins, path } => { - let broadcast = ready!(request.poll_ok(waiter))?; + Bound::Requested { lookup, pins, path } => { + let broadcast = match lookup { + Lookup::Now(request) => ready!(request.poll_ok(waiter))?, + // A resolved routing answers the same result on every poll. + Lookup::Announced(routing) => { + let mut routing = routing.lock().expect("routing poisoned"); + ready!(kio::Task::poll(&mut *routing, waiter))? + } + }; // A request that raced another for the path takes the one pinned first. The path's // first answer is pinned even when it fails this binding's epoch, so no later // reference resolves the path independently. @@ -452,20 +624,27 @@ mod tests { assert!(first.is_clone(&binding.broadcast().await.unwrap())); } - /// A failed lookup pins nothing: the guarantee starts at a path's first resolution. - #[tokio::test] - async fn failed_binding_does_not_retry_a_later_publisher() { + /// A binding made before anything announces its path resolves on the first announcement, + /// and keeps that instance once it is replaced. + #[tokio::test(start_paused = true)] + async fn a_binding_waits_for_the_first_announcement() { let origin = produce_origin(); let _catalog = origin.publish("pub", Default::default()).unwrap(); settle().await; let source = resolved(&origin, "pub").await; let media = Relative::new("media"); let binding = source.bind(Some(&media), None).unwrap(); - assert!(binding.broadcast().await.is_err()); - let _publisher = origin.publish("media", Default::default()).unwrap(); settle().await; - assert!(!source.resolve(Some(&media), None).await.unwrap().is_closed()); - assert!(binding.broadcast().await.is_err()); + assert!(binding.poll_broadcast(&kio::Waiter::noop()).is_pending()); + + let _first = origin.publish("media", Default::default()).unwrap(); + let first = binding.broadcast().await.unwrap(); + assert!(!first.is_closed()); + + let _second = origin.publish("media", Default::default()).unwrap(); + settle().await; + assert!(binding.broadcast().await.unwrap().is_clone(&first)); + assert!(source.resolve(Some(&media), None).await.unwrap().is_clone(&first)); } /// A source serves its path from the broadcast it was built from, even when a replacement @@ -603,6 +782,115 @@ mod tests { assert!(source.resolve(Some(&rel), None).await.unwrap().is_clone(&first)); } + /// The next change a follower reports, which must arrive within the origin's update hold. + async fn followed(follow: &mut Follow) -> Option { + let next = tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .expect("no change") + .expect("the follow failed"); + match next { + Followed::Start(broadcast) => Some(broadcast), + Followed::End => None, + } + } + + /// The follower stays quiet past the origin's update hold. + async fn quiet(follow: &mut Follow) { + if tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .is_ok() + { + panic!("expected no change"); + } + } + + /// A sibling announced after the follow starts it, its end idles it, and its return starts + /// it again. + #[tokio::test(start_paused = true)] + async fn a_follow_starts_late_and_survives_an_end() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./source")), None).unwrap(); + quiet(&mut follow).await; + + let first = origin.publish("a/source", Default::default()).unwrap(); + let started = followed(&mut follow).await.expect("a start"); + assert_eq!(started.info().path.as_str(), "a/source"); + + drop(first); + assert!(followed(&mut follow).await.is_none(), "an end"); + quiet(&mut follow).await; + + let _second = origin.publish("a/source", Default::default()).unwrap(); + let restarted = followed(&mut follow).await.expect("a start"); + assert!(!restarted.is_clone(&started)); + } + + /// A replacement restarts a reference that names no epoch, and ends one that names the old + /// instance until that instance serves again. + #[tokio::test(start_paused = true)] + async fn a_follow_restarts_only_without_an_epoch() { + let origin = produce_origin(); + let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + let _old = origin.publish("a/source", route(&old_epoch)).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let rel = Relative::new("./source"); + let mut any = source.follow(Some(&rel), None).unwrap(); + let mut named = source.follow(Some(&rel), Some(&old_epoch)).unwrap(); + let old = followed(&mut any).await.unwrap(); + assert!(followed(&mut named).await.unwrap().is_clone(&old)); + + let new = origin.publish("a/source", route(&new_epoch)).unwrap(); + let replaced = followed(&mut any).await.unwrap(); + assert_eq!(replaced.info().epoch.as_ref(), Some(&new_epoch)); + assert!(followed(&mut named).await.is_none(), "another epoch ends it"); + quiet(&mut named).await; + // The old instance still serves the subscriptions already on it. + assert!(!old.is_closed()); + + // The replacement goes and the old instance wins again. + drop(new); + let back = followed(&mut named).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + let back = followed(&mut any).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + } + + /// An epoch is best effort over an epochless route, which cannot prove another instance. + #[tokio::test(start_paused = true)] + async fn a_follow_reads_an_epochless_route_with_an_epoch() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut named = source + .follow(Some(&Relative::new("./source")), Some(&moq_net::Epoch::mint())) + .unwrap(); + let _sibling = origin.publish("a/source", Default::default()).unwrap(); + assert!(followed(&mut named).await.is_some()); + } + + /// A self-reference reads only the catalog instance, whatever replaces it at the path. + #[tokio::test(start_paused = true)] + async fn a_follow_of_the_catalog_keeps_its_instance() { + let origin = produce_origin(); + let old = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./pub")), None).unwrap(); + assert!(followed(&mut follow).await.unwrap().is_clone(&source.broadcast())); + + let _new = origin.publish("a/pub", Default::default()).unwrap(); + quiet(&mut follow).await; + // Its end is the catalog's, which the catalog's reader sees. + drop(old); + quiet(&mut follow).await; + } + /// A self-reference's epoch must not contradict the catalog broadcast's own. #[tokio::test(start_paused = true)] async fn a_self_reference_epoch_checks_the_catalog_instance() { From 3bba0a26421ec9770d439275b8462b6115906346 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:21:09 -0700 Subject: [PATCH 03/13] quest: finish reference announcements Delete the quest, point sink-restart-follow at moq_mux::Follow, and size the now-childless broadcast epoch epic. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/broadcast.ts | 5 +- quest/m0/broadcast-epoch/README.md | 11 ++--- .../reference-announcements.md | 48 ------------------- quest/m1/announce-takeover.md | 1 - quest/m1/identity-guide.md | 1 - quest/m1/reference-epoch-bindings.md | 4 -- quest/m1/sink-restart-follow.md | 16 ++++--- 7 files changed, 16 insertions(+), 70 deletions(-) delete mode 100644 quest/m0/broadcast-epoch/reference-announcements.md diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index 62066c83821..718e263da8c 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -18,7 +18,10 @@ type Usable = (rel: Path.Relative | undefined, epoch: Moq.Epoch.Valid | undefine // Either the catalog's own broadcast, or a sibling to consume by path. type RelativeTarget = { local: true } | { local: false; path: Moq.Path.Valid }; -function filterRenditions(renditions: Record, usable: Usable): Record { +function filterRenditions( + renditions: Record, + usable: Usable, +): Record { return Object.fromEntries( Object.entries(renditions).filter(([, config]) => usable(config.broadcast, config.epoch)), ); diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index 396e06f4d22..3cccb3adbe0 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -1,4 +1,4 @@ -# Broadcast epochs +# [L] Broadcast epochs ## Goal @@ -72,9 +72,8 @@ Decided: for the catalog instance's lifetime. Optional reference epochs constrain resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The - schemas, shared resolution, and compatibility documentation landed; - [reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - owns the native readers that follow announcements. + schemas, shared resolution, compatibility documentation, and `moq_mux::Follow` + (which `moqsrc` follows siblings with) landed. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players (`moq play`, `@moq/watch`, demo/web) follow the announce `Restart`. moq-stats mints one per @@ -124,10 +123,6 @@ constraints; it never mixes instances through an old handle. Check exact and covering routes, matching/different/absent epochs, and legacy End+Start. The child quests own their focused regressions. -## Required - -- [Reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - native readers restart a replaced referenced sibling on its announcements, never past a named epoch - ## Related - [JS consume identity](https://github.com/moq-dev/moq/pull/5254) - requests carry the serving prefix's epoch and new winners cannot reuse an old cached broadcast diff --git a/quest/m0/broadcast-epoch/reference-announcements.md b/quest/m0/broadcast-epoch/reference-announcements.md deleted file mode 100644 index cfde1ceeef0..00000000000 --- a/quest/m0/broadcast-epoch/reference-announcements.md +++ /dev/null @@ -1,48 +0,0 @@ -# [L] Rust readers follow referenced broadcasts' announcements - -## Goal - -Native readers follow a catalog-referenced sibling's announcements the way -`@moq/watch` already does, since announcement is the online/offline signal. -A reference that names an `epoch` never follows onto another instance. - -## Plan - -Split from the catalog references work (#5500's Codex review, decided by the -maintainer 2026-10-14). That work landed the optional reference `epoch`, its -shared resolution in `moq_mux::Source` and `@moq/watch`, and the Hang draft -text; this quest owns the following behavior it left out. - -- Share it in `moq_mux` (a `Source::follow`-style API; the shape is open) - rather than per reader, built on `origin::Consumer::announced_broadcast` - rather than another hand-rolled loop. Follow unpinned and check each - handed-out instance's epoch, as `Source::bind` does: a pinned - `announced_broadcast` waits out an epochless route, which would break the - best effort the reference epoch promises there. `Source::bind` binds once - today, as `failed_binding_does_not_retry_a_later_publisher` asserts; that - changes. The "no retarget" rule applies only to readers that follow no - announcement. -- `moqsrc`: Start binds the rendition, End idles its pad, and Restart - restarts its pump. -- Exporters: a late first announcement starts the rendition and End ends - it. Restart behavior belongs to - [sink restart follow](/quest/m1/sink-restart-follow.md). -- A replacement is a Restart for an announcement follower, never a seamless - same-epoch resume. A proven Takeover preserves the same instance. -- A reference with an `epoch` follows only its own instance: following never - substitutes a different epoch, so a Restart onto another one ends that - rendition, matching `relativeBroadcast` in - `@moq/watch` (`follows a replaced sibling only for a reference that names - no epoch`). -- `@moq/watch` returns nothing for a rendition whose followed broadcast proves - another epoch, but still lists it in `out.catalog`, so selection can pick a - rendition that never plays. Drop it from the effective catalog too. - -Test late first announcements, End then Start, Restart with and without an -epoch, and concurrent old/new consumers, in the moq-mux unit tests and the -`moqsrc` and exporter suites. - -## Related - -- [Sink restart follow](/quest/m1/sink-restart-follow.md) - exporters and sinks that follow an epochless Restart of their own broadcast -- [Export rendition rebind](/quest/m1/export-rendition-rebind.md) - exporters rebuild a rendition whose reference changes diff --git a/quest/m1/announce-takeover.md b/quest/m1/announce-takeover.md index 593b11e5366..3ff73d8184c 100644 --- a/quest/m1/announce-takeover.md +++ b/quest/m1/announce-takeover.md @@ -94,6 +94,5 @@ ANNOUNCE_TAKEOVER; published versions retain their framing. - [Claim answer epochs](https://github.com/moq-dev/moq/pull/5678) - `Request::prepare` then `publish` answers a claim at the exact path under a fresh epoch, the operation TAKEOVER continues - [IETF takeover](/quest/m1/ietf-takeover.md) - negotiated IETF counterpart -- [Reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - a proven continuation keeps a followed reference on its instance - [Retired requests](https://github.com/moq-dev/moq/pull/5294) - replacements never substitute content into held handles - [Identity and recovery guide](/quest/m1/identity-guide.md) - explains takeover alongside restart and legacy behavior diff --git a/quest/m1/identity-guide.md b/quest/m1/identity-guide.md index 27d5b876af0..4e9c3978aed 100644 --- a/quest/m1/identity-guide.md +++ b/quest/m1/identity-guide.md @@ -40,5 +40,4 @@ Public API and wire: none; this is a user guide for existing feature owners. ## Required -- [Reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - settles how readers pin and follow catalog references - [Announcement takeover](/quest/m1/announce-takeover.md) - establishes the continuity operation the guide demonstrates diff --git a/quest/m1/reference-epoch-bindings.md b/quest/m1/reference-epoch-bindings.md index 3e426407772..d6a0cacf219 100644 --- a/quest/m1/reference-epoch-bindings.md +++ b/quest/m1/reference-epoch-bindings.md @@ -24,7 +24,3 @@ field: the first instance resolved, without its guarantee. Test a matching, mismatched and epochless reference through `moq-ffi` and libmoq, and run `just test interop --all`. - -## Related - -- [Rust readers follow references](/quest/m0/broadcast-epoch/reference-announcements.md) - the following behavior the bindings inherit from `moq_mux` diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index 23d4db98a40..4b32c515f49 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -16,8 +16,8 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references -to other broadcasts stays with -[reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md). +to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; +`moq play` resolves a reference once with `Source::resolve`. Readers with their own follow loop today move onto `announced_broadcast`: @@ -47,9 +47,11 @@ Audit and fix the rest the same way: - `transcode`: its source broadcast; following means restarting the ladder on the new instance. -Share one helper (a `moq_mux` `Source::follow`-style API per -[reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md)) -rather than one rule per reader, built on `announced_broadcast`. Each fix -lands with a test that replaces the publisher while the reader runs. +Share `moq_mux::Follow::new(origin, path, None)` rather than one rule per +reader: it yields `Followed::Start` for each instance serving the path and +`Followed::End` once the one handed out stops serving. Build a +`moq_mux::Source` from each started instance, so its siblings follow through +`Source::follow` and its exporters pin through `Source::bind`. Each fix lands +with a test that replaces the publisher while the reader runs. -Public API: possibly a shared follow helper in `moq-mux`. Wire: none. +Public API: none beyond `moq_mux::Follow`. Wire: none. From 185dbc5f5f0a2421de3f7b0223328e62bc440ce4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:53:45 -0700 Subject: [PATCH 04/13] fix(mux): only export requests wait for a sibling's first announcement Public Source::bind, resolve, and subscribe_track keep answering for the routes as they stand: HLS answers per request and moq play picks renditions inline, so neither can wait on a sibling that left. Co-Authored-By: Claude Opus 5.5 --- doc/concept/hang.md | 2 +- rs/moq-gst/src/source/imp.rs | 24 +++++++-- rs/moq-mux/src/container/source.rs | 6 ++- rs/moq-mux/src/source.rs | 80 +++++++++++++++++------------- 4 files changed, 71 insertions(+), 41 deletions(-) diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 4373b75c6dc..149930b0963 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; an exporter starts on the first one announced, keeps it, and ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; the fMP4, TS, MKV, FLV, and HLS exporters start on the first one announced, keep it, and end the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 546c24e52ec..62e6b2f2278 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -3155,7 +3155,10 @@ mod session_tests { assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); stop(session); - assert!(recorder.seen().contains(&Seen::Start(pad)), "no new stream on the kept pad"); + assert!( + recorder.seen().contains(&Seen::Start(pad)), + "no new stream on the kept pad" + ); recorder.assert_one_pad_held(); } @@ -3178,7 +3181,10 @@ mod session_tests { stop(session); let seen = recorder.seen(); - let switched = seen.iter().position(|seen| *seen == Seen::Buffer(pad.clone(), 2)).unwrap(); + let switched = seen + .iter() + .position(|seen| *seen == Seen::Buffer(pad.clone(), 2)) + .unwrap(); assert!( !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), "the replaced sibling kept playing: {seen:?}" @@ -3213,12 +3219,20 @@ mod session_tests { let mut follow = origin.consume().follow("cam").unwrap(); await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); let mut new = Publisher::sibling(&origin, epoch()); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + await_event(&mut follow, |event| { + matches!(event, moq_net::announce::Event::Restart(_)) + }); new.write(2); drop(new); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + await_event(&mut follow, |event| { + matches!(event, moq_net::announce::Event::Restart(_)) + }); - assert_eq!(recorder.written(&mut old, 3), pad, "the named instance moved to another pad"); + assert_eq!( + recorder.written(&mut old, 3), + pad, + "the named instance moved to another pad" + ); stop(session); assert!( !recorder.seen().contains(&Seen::Buffer(pad, 2)), diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 5910aa3fd8a..21cae5c26c3 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -766,7 +766,11 @@ mod tests { .unwrap() .unwrap(); let ended = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| late.poll_read(waiter))).await; - assert!(matches!(ended, Ok(Ok(None))), "{:?}", ended.map(|read| read.map(|_| ()))); + assert!( + matches!(ended, Ok(Ok(None))), + "{:?}", + ended.map(|read| read.map(|_| ())) + ); drop((track, replacement, new)); } diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 1594c95ff40..72de27a2b12 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -23,13 +23,13 @@ type Pins = Arc> /// reference. Build one with [`Source::new`]. /// /// A source serves its own path from the broadcast it was built from, and keeps the first -/// broadcast it resolves for each sibling; a [binding](Self::bind) waits for the sibling's first -/// announcement to resolve it. Every later request for a path is served from that handle -/// ([`broadcast`](Self::broadcast), -/// [`catalog`](Self::catalog), [`resolve`](Self::resolve), [`bind`](Self::bind), -/// [`subscribe_track`](Self::subscribe_track), and an export's track requests). A route replaced -/// after that is never resolved, epoch or not, so two publishers never splice into one export; a -/// request on a held broadcast that has ended fails instead. Clones share the held broadcasts. +/// broadcast it resolves for each sibling. Every later request for a path is served from that +/// handle ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), +/// [`resolve`](Self::resolve), [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), +/// and an export's track requests, which wait for a sibling's first announcement). A route +/// replaced after that is never resolved, epoch or not, so two publishers never splice into one +/// export; a request on a held broadcast that has ended fails instead. Clones share the held +/// broadcasts. /// /// A reader that follows a sibling's announcements instead, restarting on each new instance, /// uses [`follow`](Self::follow), which holds nothing. @@ -151,7 +151,7 @@ impl Source { let mut routing = self.origin.routed_broadcast(path).into_inner(); // Issue the request now: a handler answers it whether or not anyone reads yet. let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); - Lookup::Announced(Mutex::new(routing)) + Lookup::Announced(Box::new(Mutex::new(routing))) } false => Lookup::Now(self.origin.request_broadcast(path, None).into_inner()), }, @@ -191,7 +191,8 @@ impl Source { } /// The skipping counterpart to [`Self::bind`] for a catalog entry, returning `None` when its - /// broadcast walks above the origin root. + /// broadcast walks above the origin root. Unlike a binding, it waits for a sibling nothing + /// announces yet, so a late first announcement starts the rendition. /// /// The exporters use it to drop a single bad rendition rather than fail the whole export. /// [`Self::retain_selectable`] already removes those renditions, so this is the second half of @@ -248,10 +249,6 @@ impl Source { /// broadcast must be (see [`Source`]). Use it when you need the broadcast handle itself /// (e.g. to FETCH individual groups) rather than a subscription. /// - /// Answers for the routes as they stand: a sibling nothing announces yet fails, rather than - /// waiting for it as [`bind`](Self::bind) does, so a reader choosing among renditions can - /// move on to the next. - /// /// A reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast): the rendition names /// no broadcast, so there is nothing to resolve. @@ -260,15 +257,14 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - self.request_path(&self.target(rel)?, epoch, false).broadcast().await + self.bind(rel, epoch)?.broadcast().await } /// Start one broadcast request now and retain its result for repeated reads. /// /// Unlike [`resolve`](Self::resolve), this issues the request without awaiting or polling. - /// A path nothing announces yet resolves once something does, and a remote or dynamic - /// handler may answer later. Once answered, the binding never resolves the path again, even - /// after its publisher is replaced or goes; [`follow`](Self::follow) does. + /// A remote or dynamic handler may answer later. The binding never retries a failed request + /// or resolves the path again after a publisher is replaced; [`follow`](Self::follow) does. /// /// A path this source already resolved binds to that broadcast; otherwise the origin is /// asked, deduplicating repeat requests for one path so the catalog and every rendition share @@ -282,15 +278,15 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - Ok(self.request_path(&self.target(rel)?, epoch, true)) + Ok(self.request_path(&self.target(rel)?, epoch, false)) } /// Resolve an optional cross-broadcast reference and subscribe to track `name`, /// awaiting SUBSCRIBE_OK. /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) - /// subscribes on the catalog broadcast; anything else is bound as in [`bind`](Self::bind), - /// waiting for the referenced broadcast's first announcement, and checked against `epoch`. A + /// subscribes on the catalog broadcast; anything else fetches the referenced broadcast + /// from the origin first, checked against `epoch` as in [`resolve`](Self::resolve). A /// reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// @@ -326,7 +322,7 @@ impl Source { let path = self.target(rel)?; let instances = match path == self.path { true => Instances::Fixed(Some(self.broadcast())), - false => Instances::Announced(self.origin.announced_broadcast(&path, None)?), + false => Instances::Announced(Box::new(self.origin.announced_broadcast(&path, None)?)), }; Ok(Follow::with(instances, epoch)) } @@ -347,7 +343,7 @@ pub struct Follow { enum Instances { /// Every instance announced at a path. - Announced(moq_net::origin::AnnouncedBroadcast), + Announced(Box), /// One instance, handed out once. Fixed(Option), } @@ -371,7 +367,7 @@ impl Follow { epoch: Option<&moq_net::Epoch>, ) -> crate::Result { let announced = origin.announced_broadcast(path, None)?; - Ok(Self::with(Instances::Announced(announced), epoch)) + Ok(Self::with(Instances::Announced(Box::new(announced)), epoch)) } fn with(instances: Instances, epoch: Option<&moq_net::Epoch>) -> Self { @@ -438,12 +434,10 @@ fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast:: /// A held broadcast or the retained result of one eagerly issued broadcast request. /// -/// A pending request waits for a path nothing announces yet and rides out route churn until it -/// resolves (see [`routed_broadcast`](moq_net::origin::Consumer::routed_broadcast)). Once -/// answered, reading the binding never looks up its path again, even after a failure or -/// publisher replacement. Issuing a request does not establish an epoch relationship with a -/// separate catalog broadcast. A binding made with an epoch refuses a broadcast proven to serve -/// another one. +/// Reading the binding never looks up its path again, even after a failure or publisher +/// replacement. Pending requests follow the origin's routing semantics; issuing a request +/// does not establish an epoch relationship with a separate catalog broadcast. A binding made +/// with an epoch refuses a broadcast proven to serve another one. /// /// Build one with [`Source::bind`], or with [`Binding::new`] for a broadcast already in hand. pub struct Binding { @@ -459,7 +453,7 @@ enum Lookup { /// For the routes as they stand. Now(moq_net::origin::Requesting), /// Once something announces the path, riding out route churn. - Announced(Mutex), + Announced(Box>), } enum Bound { @@ -624,16 +618,34 @@ mod tests { assert!(first.is_clone(&binding.broadcast().await.unwrap())); } - /// A binding made before anything announces its path resolves on the first announcement, - /// and keeps that instance once it is replaced. - #[tokio::test(start_paused = true)] - async fn a_binding_waits_for_the_first_announcement() { + /// A failed lookup pins nothing: the guarantee starts at a path's first resolution. + #[tokio::test] + async fn failed_binding_does_not_retry_a_later_publisher() { let origin = produce_origin(); let _catalog = origin.publish("pub", Default::default()).unwrap(); settle().await; let source = resolved(&origin, "pub").await; let media = Relative::new("media"); let binding = source.bind(Some(&media), None).unwrap(); + assert!(binding.broadcast().await.is_err()); + let _publisher = origin.publish("media", Default::default()).unwrap(); + settle().await; + assert!(!source.resolve(Some(&media), None).await.unwrap().is_closed()); + assert!(binding.broadcast().await.is_err()); + } + + /// An export's request made before anything announces its sibling resolves on the first + /// announcement, and keeps that instance once it is replaced. + #[tokio::test(start_paused = true)] + async fn an_export_request_waits_for_the_first_announcement() { + let origin = produce_origin(); + let _catalog = origin.publish("pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "pub").await; + let media = Relative::new("media"); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + config.broadcast = Some(media.to_owned()); + let binding = source.try_request(&config).unwrap(); settle().await; assert!(binding.poll_broadcast(&kio::Waiter::noop()).is_pending()); From 24206edeee7f4ecd4df2ae3b6c836569fd4a7f1b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 00:32:03 -0700 Subject: [PATCH 05/13] feat(mux): bind waits for a sibling's first announcement; rename to follow::* Source::bind, resolve, and subscribe_track wait for a sibling nothing announces yet, through routed_broadcast, instead of failing Unroutable. A binding answers its pinned instance once the path is pinned, so a sibling that announced and then left answers its ended broadcast rather than waiting for the next. moq-hls drops its own re-issue loop, and its disconnected-sibling test now names the sibling it means to bind. Move Follow/Followed to moq_mux::follow::{Instances, Event}. Co-Authored-By: Claude Opus 5.5 --- cpp/moq/MIGRATE.md | 5 + dart/moq/MIGRATE.md | 5 + dart/moq_ffi/MIGRATE.md | 6 + dart/moq_ffi/lib/src/moq.dart | 2 +- doc/concept/hang.md | 2 +- go/ffi/MIGRATE.md | 6 + go/wrapper/MIGRATE.md | 5 + kt/moq-ffi/MIGRATE.md | 6 + kt/moq/MIGRATE.md | 5 + py/moq-ffi/MIGRATE.md | 6 + py/moq-rs/MIGRATE.md | 5 + quest/m0/broadcast-epoch/README.md | 2 +- quest/m1/sink-restart-follow.md | 12 +- rs/moq-c/MIGRATE.md | 5 + rs/moq-ffi/MIGRATE.md | 5 + rs/moq-ffi/src/consumer.rs | 5 +- rs/moq-gst/src/source/imp.rs | 14 +- rs/moq-hls/src/export/mod.rs | 4 +- rs/moq-hls/src/export/rendition.rs | 53 +--- rs/moq-hls/src/server/routes.rs | 2 +- rs/moq-mux/MIGRATE.md | 5 + rs/moq-mux/src/follow.rs | 243 ++++++++++++++++++ rs/moq-mux/src/lib.rs | 5 +- rs/moq-mux/src/source.rs | 382 ++++++----------------------- swift/MIGRATE.md | 5 + swift/ffi/MIGRATE.md | 6 + 26 files changed, 435 insertions(+), 366 deletions(-) create mode 100644 rs/moq-mux/src/follow.rs diff --git a/cpp/moq/MIGRATE.md b/cpp/moq/MIGRATE.md index ec0b7243cd4..6e087da914a 100644 --- a/cpp/moq/MIGRATE.md +++ b/cpp/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer::resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel it. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer::finish` lets go. diff --git a/dart/moq/MIGRATE.md b/dart/moq/MIGRATE.md index 7a1f1be6ca3..c145997e2ac 100644 --- a/dart/moq/MIGRATE.md +++ b/dart/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference in the rendition-based subscribes and fetches now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: add a `.timeout` to the future. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/dart/moq_ffi/MIGRATE.md b/dart/moq_ffi/MIGRATE.md index 3e8c381a845..61c02c56f61 100644 --- a/dart/moq_ffi/MIGRATE.md +++ b/dart/moq_ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: add a `.timeout` to the future. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 1d0d4121dcd..31703ab149c 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -19310,7 +19310,7 @@ void _checkApiChecksums() { 18633) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqbroadcastconsumer_resolve() != 62317) { + if (uniffi_moq_ffi_checksum_method_moqbroadcastconsumer_resolve() != 48256) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqbroadcastconsumer_subscribe_track() != diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 149930b0963..841f74d2b0b 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; the fMP4, TS, MKV, FLV, and HLS exporters start on the first one announced, keep it, and end the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; every other reader, such as `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/go/ffi/MIGRATE.md b/go/ffi/MIGRATE.md index d7abfc288f8..818a4eca19f 100644 --- a/go/ffi/MIGRATE.md +++ b/go/ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.Resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: pass a `ctx` with a deadline. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.Finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index b5e60cf4cc7..04a184e7131 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer.Resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: pass a `ctx` with a deadline. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.Finish()` lets go. diff --git a/kt/moq-ffi/MIGRATE.md b/kt/moq-ffi/MIGRATE.md index 6e9627fd6a6..dc1d04ae17e 100644 --- a/kt/moq-ffi/MIGRATE.md +++ b/kt/moq-ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the coroutine or wrap it in `withTimeout`. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/kt/moq/MIGRATE.md b/kt/moq/MIGRATE.md index 2ff652f9212..59c92a5d82f 100644 --- a/kt/moq/MIGRATE.md +++ b/kt/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference in the rendition-based subscribes and fetches now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the coroutine or wrap it in `withTimeout`. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast ends or the new `BroadcastProducer.finish()` lets go. diff --git a/py/moq-ffi/MIGRATE.md b/py/moq-ffi/MIGRATE.md index a009932892c..5fa95305f84 100644 --- a/py/moq-ffi/MIGRATE.md +++ b/py/moq-ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve()` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task or wrap it in `asyncio.wait_for`. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 4175ecd3ec6..24f4f3a2212 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer.resolve()` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task or wrap it in `asyncio.wait_for`. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index 3cccb3adbe0..addef2d7b7d 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -72,7 +72,7 @@ Decided: for the catalog instance's lifetime. Optional reference epochs constrain resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The - schemas, shared resolution, compatibility documentation, and `moq_mux::Follow` + schemas, shared resolution, compatibility documentation, and `moq_mux::follow` (which `moqsrc` follows siblings with) landed. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index ea5b6b3e45d..b76f09ecbfe 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -17,7 +17,8 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; -`moq play` resolves a reference once with `Source::resolve`. +`moq play` resolves a reference once with `Source::resolve`, which waits for +a sibling's first announcement and keeps it. Readers with their own follow loop today move onto `announced_broadcast`: @@ -47,12 +48,13 @@ Audit and fix the rest the same way: - `transcode`: its source broadcast; following means restarting the ladder on the new instance. -Share `moq_mux::Follow::new(origin, path, None)` (decided 2026-10-20, +Share `moq_mux::follow::Instances::new(origin, path, None)` (decided 2026-10-20, maintainer: no reader hand-rolls its own follow loop) rather than one rule per -reader: it yields `Followed::Start` for each instance serving the path and -`Followed::End` once the one handed out stops serving. Build a +reader: it yields `follow::Event::Start` for each instance serving the path and +`follow::Event::End` once the one handed out stops serving. Build a `moq_mux::Source` from each started instance, so its siblings follow through -`Source::follow` and its exporters pin through `Source::bind`. Each fix lands +`Source::follow` and its exporters pin through `Source::bind`, which waits for +a sibling's first announcement. Each fix lands with a test that replaces the publisher while the reader runs. Public API: none. Wire: none. diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 45a2f272283..1e52cf8b7e7 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +`moq_consume_video` and `moq_consume_audio` on a rendition whose catalog `broadcast` field names a sibling now wait for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, give up on it yourself with `moq_consume_video_cancel` or `moq_consume_audio_cancel`. + ### Finished tracks stay until the broadcast ends (#6157) `moq_publish_track_finish`, `moq_publish_media_finish`, `moq_publish_container_finish`, and the encoder finishes now release the handle while the broadcast keeps the finished track listed, with its catalog rendition and demand, until `moq_publish_close` or the new `moq_publish_finish_close`. diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index 59e13b01410..b8c81b79eb5 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,6 +5,11 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer::resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel it or race it against a timeout. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer::finish` lets go, instead of unlisting when its producer finishes or is released. diff --git a/rs/moq-ffi/src/consumer.rs b/rs/moq-ffi/src/consumer.rs index 17fab728081..1a6328e4c82 100644 --- a/rs/moq-ffi/src/consumer.rs +++ b/rs/moq-ffi/src/consumer.rs @@ -260,9 +260,8 @@ impl MoqBroadcastConsumer { /// `MoqVideoConsumer::subscribe`, `MoqAudioConsumer::subscribe`) resolve it themselves. /// /// Errors if this broadcast came from a local producer rather than an origin, since a - /// standalone broadcast has no sibling to name, and reports a sibling that exists but is not - /// announced yet as unroutable rather than waiting for it (see - /// [`MoqOriginConsumer::request_broadcast`](crate::origin::MoqOriginConsumer::request_broadcast)). + /// standalone broadcast has no sibling to name. A sibling nothing announces yet is waited for, + /// so a late one resolves once it is announced; cancel the call to give up on it. pub async fn resolve(&self, reference: Option) -> Result, MoqError> { let broadcast = self.resolve_inner(reference.as_deref()).await?; Ok(Arc::new(match self.origin.clone() { diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 62e6b2f2278..8564c8a94b3 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1106,7 +1106,7 @@ struct Pump { name: String, caps: gst::Caps, /// Each instance of the broadcast the rendition references. - follow: moq_mux::Follow, + follow: moq_mux::follow::Instances, /// The rendition references another broadcast than its catalog's, whose instances come and /// go on their own announcements. sibling: bool, @@ -1171,8 +1171,8 @@ impl Pump { None => tokio::select! { _ = cancel.changed() => return None, followed = follow.next() => match followed { - Ok(moq_mux::Followed::Start(broadcast)) => broadcast, - Ok(moq_mux::Followed::End) => continue, + Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, + Ok(moq_mux::follow::Event::End) => continue, Err(err) => { gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); return Some((kind, name)); @@ -1189,8 +1189,8 @@ impl Pump { let subscribed = tokio::select! { _ = cancel.changed() => return None, followed = follow.next() => match followed { - Ok(moq_mux::Followed::Start(broadcast)) => Err(Stopped::Restart(broadcast)), - Ok(moq_mux::Followed::End) => Err(Stopped::Idle), + Ok(moq_mux::follow::Event::Start(broadcast)) => Err(Stopped::Restart(broadcast)), + Ok(moq_mux::follow::Event::End) => Err(Stopped::Idle), Err(err) => { gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); return Some((kind, name)); @@ -1311,8 +1311,8 @@ impl Pump { return None; } followed = follow.next() => match followed { - Ok(moq_mux::Followed::Start(broadcast)) => break Stopped::Restart(broadcast), - Ok(moq_mux::Followed::End) => break Stopped::Idle, + Ok(moq_mux::follow::Event::Start(broadcast)) => break Stopped::Restart(broadcast), + Ok(moq_mux::follow::Event::End) => break Stopped::Idle, Err(err) => { gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); return Some((kind, name)); diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 8f846c51f08..c2283e13e37 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -3865,8 +3865,8 @@ mod tests { drop((track, catalog, live)); } - // A sibling not yet announced when the rendition binds it fails that first lookup. The - // catalog's rows are still listed, and the bind is issued again until the sibling resolves. + // A sibling not yet announced when the rendition binds it is waited for. The catalog's rows + // are still listed, and the binding resolves on the sibling's first announcement. #[tokio::test] async fn a_sibling_announced_after_export_start_is_bound() { tokio::time::pause(); diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 72b4146188f..baed6031b62 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -115,9 +115,8 @@ enum Config { struct Media { handle: Mutex, track: String, - /// The source, reference, and epoch a sibling is bound through, to re-issue a bind that - /// failed before it ever resolved. - sibling: Option<(moq_mux::Source, moq_net::path::RelativeOwned, Option)>, + /// The rendition reads another broadcast than the catalog's. + sibling: bool, } struct Handle { @@ -154,7 +153,8 @@ impl Media { Ok(Self { handle: Mutex::new(handle), track: track.to_string(), - sibling: sibling(source, rel).map(|rel| (source.clone(), rel, reference.epoch().cloned())), + // `bind` already rejected an escaping reference, so both sides name a path. + sibling: source.resolve_reference(rel) != source.resolve_reference(None), }) } @@ -172,33 +172,16 @@ impl Media { /// Register the bound broadcast's close and retain its track before new lookups are refused. fn poll_sync(&self, waiter: &kio::Waiter, window: &segments::Producer) -> Arc { let mut handle = self.handle.lock().expect("media lock poisoned"); - let mut reissued = false; - loop { - match handle.binding.poll_broadcast(waiter) { - Poll::Ready(Ok(broadcast)) => { - handle.capture(&broadcast, &self.track); - if self.sibling.is_some() && !handle.ended && broadcast.poll_closed(waiter).is_ready() { - // End and Restart both drain only the pinned instance. Looking its path - // up again could serve a replacement under the old timeline's rows. - handle.ended = true; - window.end(); - } - } - // Only a binding that never resolved may be reissued. The source pins every - // successful result, so a replacement can never answer these old rows. - Poll::Ready(Err(_)) if !reissued => { - reissued = true; - if let Some((source, rel, epoch)) = &self.sibling - && let Ok(next) = source.bind(Some(rel), epoch.as_ref()) - { - handle.binding = Arc::new(next); - continue; - } - } - _ => {} + if let Poll::Ready(Ok(broadcast)) = handle.binding.poll_broadcast(waiter) { + handle.capture(&broadcast, &self.track); + if self.sibling && !handle.ended && broadcast.poll_closed(waiter).is_ready() { + // End and Restart both drain only the pinned instance. Looking its path up again + // could serve a replacement under the old timeline's rows. + handle.ended = true; + window.end(); } - return handle.binding.clone(); } + handle.binding.clone() } fn poll_track( @@ -215,18 +198,6 @@ impl Media { } } -/// A catalog `broadcast` reference that names a different path than the catalog itself. -fn sibling( - source: &moq_mux::Source, - rel: Option<&moq_net::path::RelativeOwned>, -) -> Option { - let target = source.resolve_reference(rel)?; - if source.resolve_reference(None).as_ref() == Some(&target) { - return None; - } - Some(rel.cloned().unwrap_or_else(moq_net::path::Relative::empty)) -} - /// Clear the video fields that don't change how the rendition decodes or is muxed, so a config /// the publisher only re-measured or re-labelled compares equal to the one already being served. /// diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index dda6d035e60..9b0b95e810b 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -1061,7 +1061,7 @@ mod tests { moq_mux::catalog::Producer::new(&mut catalog_broadcast, moq_mux::catalog::Config::default()).unwrap(); let reserved = catalog.reserve(); let mut config = video_config(); - config.broadcast = Some(moq_net::path::Relative::new("../source").to_owned()); + config.broadcast = Some(moq_net::path::Relative::new("./source").to_owned()); let track = media_broadcast.create_track("video0", timed()).unwrap(); let mut media = reserved .video( diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index 0dcc889ce36..085b10a56c2 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Source` waits for a sibling's first announcement (#6254) + +`Source::bind`, `resolve`, and `subscribe_track` now wait for a sibling nothing announces yet instead of failing with `Unroutable`, and keep the first instance they resolve: once it goes, they answer that ended broadcast rather than waiting for the next. +Where you skipped a rendition on that error, bound the call yourself (`tokio::time::timeout`, or drop it), or follow the sibling's instances with `Source::follow`. + ### `Source` resolution takes the reference's epoch (#5699) `Source::bind(rel)`, `resolve(rel)`, and `subscribe_track(rel, name)` take the catalog entry's optional epoch after `rel`. diff --git a/rs/moq-mux/src/follow.rs b/rs/moq-mux/src/follow.rs new file mode 100644 index 00000000000..eb149f7a593 --- /dev/null +++ b/rs/moq-mux/src/follow.rs @@ -0,0 +1,243 @@ +//! Following a broadcast across the publisher instances that serve it. +//! +//! An announcement is the online and offline signal, so a reader that restarts on each instance +//! follows [`Instances`] rather than pinning one broadcast as a [`Binding`](crate::Binding) does. + +use std::task::Poll; + +/// The publisher instances serving a broadcast, as each comes online, takes over, and goes. +/// +/// Built on [`announced_broadcast`](moq_net::origin::Consumer::announced_broadcast). Create one +/// with [`Source::follow`](crate::Source::follow) for a catalog reference, or [`Instances::new`] +/// for a path. +pub struct Instances { + feed: Feed, + /// The instance a reference named, if any. + epoch: Option, + /// The instance handed out last, until it stops serving. + current: Option, +} + +enum Feed { + /// Every instance announced at a path. + Announced(Box), + /// One instance, handed out once. + Fixed(Option), +} + +/// A change to the instance [`Instances`] serves. +pub enum Event { + /// Start on this instance, dropping whatever was read from the one before. + Start(moq_net::broadcast::Consumer), + /// The instance handed out last stopped serving: idle until the next start. + End, +} + +impl Instances { + /// Follow the broadcast at `path` through `origin`, handing out only `epoch` when given. + /// + /// Unlike [`Source::follow`](crate::Source::follow), this needs no catalog broadcast, so a + /// reader can follow the broadcast whose catalog it reads and build a + /// [`Source`](crate::Source) for each instance. + pub fn new( + origin: &moq_net::origin::Consumer, + path: impl moq_net::AsPath, + epoch: Option<&moq_net::Epoch>, + ) -> crate::Result { + let announced = origin.announced_broadcast(path, None)?; + Ok(Self::with(Feed::Announced(Box::new(announced)), epoch)) + } + + /// Hand out `broadcast` once and never report its end, which its owner follows. + pub(crate) fn fixed(broadcast: moq_net::broadcast::Consumer, epoch: Option<&moq_net::Epoch>) -> Self { + Self::with(Feed::Fixed(Some(broadcast)), epoch) + } + + fn with(feed: Feed, epoch: Option<&moq_net::Epoch>) -> Self { + Self { + feed, + epoch: epoch.cloned(), + current: None, + } + } + + /// Poll for the next change, registering `waiter` when there is none yet. + /// + /// Each announcement that brings an instance hands it out, so the same one returns after an + /// end. Fails once the origin closes, and with any resolution failure, such as a handler + /// refusing the path. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + let next = match &mut self.feed { + // Followed unpinned and checked here: a pinned follow waits out an epochless route, + // which an epoch reads best effort. + Feed::Announced(announced) => announced.poll_available(waiter).map_err(crate::Error::from), + Feed::Fixed(broadcast) => match broadcast.take() { + Some(broadcast) => Poll::Ready(Ok(broadcast)), + None => Poll::Pending, + }, + }; + if let Poll::Ready(next) = next { + let broadcast = next?; + if proves_other(self.epoch.as_ref(), &broadcast) { + tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); + match self.current.take() { + Some(_) => return Poll::Ready(Ok(Event::End)), + None => continue, + } + } + // A fixed instance ends with its owner, which follows that itself. + if let Feed::Announced(_) = self.feed { + self.current = Some(broadcast.clone()); + } + return Poll::Ready(Ok(Event::Start(broadcast))); + } + + if let Some(current) = &self.current + && current.poll_closed(waiter).is_ready() + { + self.current = None; + return Poll::Ready(Ok(Event::End)); + } + return Poll::Pending; + } + } + + /// The next change; see [`Self::poll_next`]. + pub async fn next(&mut self) -> crate::Result { + kio::wait(|waiter| self.poll_next(waiter)).await + } +} + +/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route +/// cannot prove its instance, so only a known mismatch counts. +fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { + matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::source::{produce_origin, resolved}; + use moq_net::path::Relative; + + /// Let the origin's driver run the fronts that announcements started. + async fn settle() { + for _ in 0..10 { + tokio::task::yield_now().await; + } + } + + fn route(epoch: &moq_net::Epoch) -> moq_net::origin::Route { + moq_net::origin::Route::default().with_epoch(epoch.clone()) + } + + /// The next change a follower reports, which must arrive within the origin's update hold. + async fn followed(follow: &mut Instances) -> Option { + let next = tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .expect("no change") + .expect("the follow failed"); + match next { + Event::Start(broadcast) => Some(broadcast), + Event::End => None, + } + } + + /// The follower stays quiet past the origin's update hold. + async fn quiet(follow: &mut Instances) { + if tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .is_ok() + { + panic!("expected no change"); + } + } + + /// A sibling announced after the follow starts it, its end idles it, and its return starts + /// it again. + #[tokio::test(start_paused = true)] + async fn a_follow_starts_late_and_survives_an_end() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./source")), None).unwrap(); + quiet(&mut follow).await; + + let first = origin.publish("a/source", Default::default()).unwrap(); + let started = followed(&mut follow).await.expect("a start"); + assert_eq!(started.info().path.as_str(), "a/source"); + + drop(first); + assert!(followed(&mut follow).await.is_none(), "an end"); + quiet(&mut follow).await; + + let _second = origin.publish("a/source", Default::default()).unwrap(); + let restarted = followed(&mut follow).await.expect("a start"); + assert!(!restarted.is_clone(&started)); + } + + /// A replacement restarts a reference that names no epoch, and ends one that names the old + /// instance until that instance serves again. + #[tokio::test(start_paused = true)] + async fn a_follow_restarts_only_without_an_epoch() { + let origin = produce_origin(); + let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + let _old = origin.publish("a/source", route(&old_epoch)).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let rel = Relative::new("./source"); + let mut any = source.follow(Some(&rel), None).unwrap(); + let mut named = source.follow(Some(&rel), Some(&old_epoch)).unwrap(); + let old = followed(&mut any).await.unwrap(); + assert!(followed(&mut named).await.unwrap().is_clone(&old)); + + let new = origin.publish("a/source", route(&new_epoch)).unwrap(); + let replaced = followed(&mut any).await.unwrap(); + assert_eq!(replaced.info().epoch.as_ref(), Some(&new_epoch)); + assert!(followed(&mut named).await.is_none(), "another epoch ends it"); + quiet(&mut named).await; + // The old instance still serves the subscriptions already on it. + assert!(!old.is_closed()); + + // The replacement goes and the old instance wins again. + drop(new); + let back = followed(&mut named).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + let back = followed(&mut any).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + } + + /// An epoch is best effort over an epochless route, which cannot prove another instance. + #[tokio::test(start_paused = true)] + async fn a_follow_reads_an_epochless_route_with_an_epoch() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut named = source + .follow(Some(&Relative::new("./source")), Some(&moq_net::Epoch::mint())) + .unwrap(); + let _sibling = origin.publish("a/source", Default::default()).unwrap(); + assert!(followed(&mut named).await.is_some()); + } + + /// A self-reference reads only the catalog instance, whatever replaces it at the path. + #[tokio::test(start_paused = true)] + async fn a_follow_of_the_catalog_keeps_its_instance() { + let origin = produce_origin(); + let old = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./pub")), None).unwrap(); + assert!(followed(&mut follow).await.unwrap().is_clone(&source.broadcast())); + + let _new = origin.publish("a/pub", Default::default()).unwrap(); + quiet(&mut follow).await; + // Its end is the catalog's, which the catalog's reader sees. + drop(old); + quiet(&mut follow).await; + } +} diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 4b46cb87dde..e76649e1859 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -28,6 +28,8 @@ //! record per aligned segment, mapping a span of content time to the group //! ranges that carry it on each track, so consumers can seek or build //! HLS/DASH playlists without downloading media. +//! - [`follow`] follows a broadcast across the publisher instances that +//! serve it, for a reader that restarts on each one. pub mod binary; pub mod catalog; @@ -35,6 +37,7 @@ mod clock; pub mod codec; pub mod container; mod error; +pub mod follow; pub mod import; mod jitter; pub mod json; @@ -48,7 +51,7 @@ pub mod timeline; pub use clock::Clock; pub use error::*; pub use pace::Pacer; -pub use source::{Binding, Follow, Followed, Source}; +pub use source::{Binding, Source}; /// Translate a catalog entry's declared compression into the flag the codecs take. /// diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 72de27a2b12..499580260a4 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -23,13 +23,13 @@ type Pins = Arc> /// reference. Build one with [`Source::new`]. /// /// A source serves its own path from the broadcast it was built from, and keeps the first -/// broadcast it resolves for each sibling. Every later request for a path is served from that -/// handle ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), -/// [`resolve`](Self::resolve), [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), -/// and an export's track requests, which wait for a sibling's first announcement). A route -/// replaced after that is never resolved, epoch or not, so two publishers never splice into one -/// export; a request on a held broadcast that has ended fails instead. Clones share the held -/// broadcasts. +/// broadcast it resolves for each sibling, waiting for the sibling's first announcement if +/// nothing serves it yet. Every later request for a path is served from that handle +/// ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), [`resolve`](Self::resolve), +/// [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), and an export's track +/// requests). A route replaced after that is never resolved, epoch or not, so two publishers +/// never splice into one export; a request on a held broadcast that has ended fails instead of +/// waiting for the next one. Clones share the held broadcasts. /// /// A reader that follows a sibling's announcements instead, restarting on each new instance, /// uses [`follow`](Self::follow), which holds nothing. @@ -137,27 +137,23 @@ impl Source { Binding::new(self.broadcast()) } - /// The broadcast `path` is pinned to, or a lookup that pins its answer, either checked - /// against `epoch` on every read. The lookup waits for the path's first announcement when - /// `announced`, and otherwise answers for the routes as they stand. - fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>, announced: bool) -> Binding { + /// The broadcast `path` is pinned to, or a lookup that waits for the path's first + /// announcement and pins its answer, either checked against `epoch` on every read. + fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>) -> Binding { let bound = match self.pins.lock().expect("pins poisoned").get(path) { Some(pinned) => Bound::Ready(pinned.clone()), // Unpinned even with an epoch: an epochless route still answers, best effort, and // the read refuses a route that proves another instance. - None => Bound::Requested { - lookup: match announced { - true => { - let mut routing = self.origin.routed_broadcast(path).into_inner(); - // Issue the request now: a handler answers it whether or not anyone reads yet. - let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); - Lookup::Announced(Box::new(Mutex::new(routing))) - } - false => Lookup::Now(self.origin.request_broadcast(path, None).into_inner()), - }, - pins: self.pins.clone(), - path: path.clone(), - }, + None => { + let mut routing = self.origin.routed_broadcast(path).into_inner(); + // Issue the request now: a handler answers it whether or not anyone reads yet. + let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); + Bound::Requested { + routing: Box::new(Mutex::new(routing)), + pins: self.pins.clone(), + path: path.clone(), + } + } }; Binding { bound, @@ -191,15 +187,14 @@ impl Source { } /// The skipping counterpart to [`Self::bind`] for a catalog entry, returning `None` when its - /// broadcast walks above the origin root. Unlike a binding, it waits for a sibling nothing - /// announces yet, so a late first announcement starts the rendition. + /// broadcast walks above the origin root. /// /// The exporters use it to drop a single bad rendition rather than fail the whole export. /// [`Self::retain_selectable`] already removes those renditions, so this is the second half of /// the same policy rather than a different one. pub(crate) fn try_request(&self, reference: &impl hang::catalog::Reference) -> Option { let path = self.resolve_reference(reference.broadcast())?; - Some(self.request_path(&path, reference.epoch(), true)) + Some(self.request_path(&path, reference.epoch())) } /// Remove the renditions an export must not select: a disabled one (catalog @@ -245,9 +240,10 @@ impl Source { /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) /// resolves the catalog broadcast itself; anything else fetches the referenced - /// broadcast from the origin. `epoch` is the rendition's `epoch` field, the instance the - /// broadcast must be (see [`Source`]). Use it when you need the broadcast handle itself - /// (e.g. to FETCH individual groups) rather than a subscription. + /// broadcast from the origin, waiting for its first announcement if nothing serves it yet. + /// `epoch` is the rendition's `epoch` field, the instance the broadcast must be (see + /// [`Source`]). Use it when you need the broadcast handle itself (e.g. to FETCH individual + /// groups) rather than a subscription. /// /// A reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast): the rendition names @@ -263,8 +259,9 @@ impl Source { /// Start one broadcast request now and retain its result for repeated reads. /// /// Unlike [`resolve`](Self::resolve), this issues the request without awaiting or polling. - /// A remote or dynamic handler may answer later. The binding never retries a failed request - /// or resolves the path again after a publisher is replaced; [`follow`](Self::follow) does. + /// A path nothing announces yet resolves once something does, and a remote or dynamic + /// handler may answer later. Once answered, the binding never resolves the path again, even + /// after its publisher is replaced or goes; [`follow`](Self::follow) does. /// /// A path this source already resolved binds to that broadcast; otherwise the origin is /// asked, deduplicating repeat requests for one path so the catalog and every rendition share @@ -278,15 +275,16 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - Ok(self.request_path(&self.target(rel)?, epoch, false)) + Ok(self.request_path(&self.target(rel)?, epoch)) } /// Resolve an optional cross-broadcast reference and subscribe to track `name`, /// awaiting SUBSCRIBE_OK. /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) - /// subscribes on the catalog broadcast; anything else fetches the referenced broadcast - /// from the origin first, checked against `epoch` as in [`resolve`](Self::resolve). A + /// subscribes on the catalog broadcast; anything else is resolved as in + /// [`resolve`](Self::resolve), waiting for its first announcement and checked against + /// `epoch`. A /// reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// @@ -318,126 +316,23 @@ impl Source { &self, rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, - ) -> crate::Result { + ) -> crate::Result { let path = self.target(rel)?; - let instances = match path == self.path { - true => Instances::Fixed(Some(self.broadcast())), - false => Instances::Announced(Box::new(self.origin.announced_broadcast(&path, None)?)), - }; - Ok(Follow::with(instances, epoch)) - } -} - -/// The publisher instances serving a broadcast, as each comes online, takes over, and goes. -/// -/// Built on [`announced_broadcast`](moq_net::origin::Consumer::announced_broadcast), since an -/// announcement is the online and offline signal. Create one with [`Source::follow`] for a -/// catalog reference, or [`Follow::new`] for a path. -pub struct Follow { - instances: Instances, - /// The instance a reference named, if any. - epoch: Option, - /// The instance handed out last, until it stops serving. - current: Option, -} - -enum Instances { - /// Every instance announced at a path. - Announced(Box), - /// One instance, handed out once. - Fixed(Option), -} - -/// A change to the instance a [`Follow`] serves. -pub enum Followed { - /// Start on this instance, dropping whatever was read from the one before. - Start(moq_net::broadcast::Consumer), - /// The instance handed out last stopped serving: idle until the next start. - End, -} - -impl Follow { - /// Follow the broadcast at `path` through `origin`, handing out only `epoch` when given. - /// - /// Unlike [`Source::follow`], this needs no catalog broadcast, so a reader can follow the - /// broadcast whose catalog it reads and build a [`Source`] for each instance. - pub fn new( - origin: &moq_net::origin::Consumer, - path: impl moq_net::AsPath, - epoch: Option<&moq_net::Epoch>, - ) -> crate::Result { - let announced = origin.announced_broadcast(path, None)?; - Ok(Self::with(Instances::Announced(Box::new(announced)), epoch)) - } - - fn with(instances: Instances, epoch: Option<&moq_net::Epoch>) -> Self { - Self { - instances, - epoch: epoch.cloned(), - current: None, - } - } - - /// Poll for the next change, registering `waiter` when there is none yet. - /// - /// Each announcement that brings an instance hands it out, so the same one returns after an - /// end. Fails once the origin closes, and with any resolution failure, such as a handler - /// refusing the path. - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { - loop { - let next = match &mut self.instances { - // Followed unpinned and checked here: a pinned follow waits out an epochless route, - // which an epoch reads best effort. - Instances::Announced(announced) => announced.poll_available(waiter).map_err(crate::Error::from), - Instances::Fixed(broadcast) => match broadcast.take() { - Some(broadcast) => Poll::Ready(Ok(broadcast)), - None => Poll::Pending, - }, - }; - if let Poll::Ready(next) = next { - let broadcast = next?; - if proves_other(self.epoch.as_ref(), &broadcast) { - tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); - match self.current.take() { - Some(_) => return Poll::Ready(Ok(Followed::End)), - None => continue, - } - } - // The catalog's own instance ends with the catalog, which its reader follows. - if let Instances::Announced(_) = self.instances { - self.current = Some(broadcast.clone()); - } - return Poll::Ready(Ok(Followed::Start(broadcast))); - } - - if let Some(current) = &self.current - && current.poll_closed(waiter).is_ready() - { - self.current = None; - return Poll::Ready(Ok(Followed::End)); - } - return Poll::Pending; - } - } - - /// The next change; see [`Self::poll_next`]. - pub async fn next(&mut self) -> crate::Result { - kio::wait(|waiter| self.poll_next(waiter)).await + Ok(match path == self.path { + true => crate::follow::Instances::fixed(self.broadcast(), epoch), + false => crate::follow::Instances::new(&self.origin, &path, epoch)?, + }) } } -/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route -/// cannot prove its instance, so only a known mismatch counts. -fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { - matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) -} - /// A held broadcast or the retained result of one eagerly issued broadcast request. /// -/// Reading the binding never looks up its path again, even after a failure or publisher -/// replacement. Pending requests follow the origin's routing semantics; issuing a request -/// does not establish an epoch relationship with a separate catalog broadcast. A binding made -/// with an epoch refuses a broadcast proven to serve another one. +/// A pending request waits for a path nothing announces yet and rides out route churn until it +/// resolves (see [`routed_broadcast`](moq_net::origin::Consumer::routed_broadcast)). Once the +/// path is pinned, reading the binding answers that pin and never looks the path up again, even +/// after its publisher is replaced or goes. Issuing a request does not establish an epoch +/// relationship with a separate catalog broadcast. A binding made with an epoch refuses a +/// broadcast proven to serve another one. /// /// Build one with [`Source::bind`], or with [`Binding::new`] for a broadcast already in hand. pub struct Binding { @@ -448,20 +343,12 @@ pub struct Binding { sibling: bool, } -/// How a [`Bound::Requested`] path is looked up. -enum Lookup { - /// For the routes as they stand. - Now(moq_net::origin::Requesting), - /// Once something announces the path, riding out route churn. - Announced(Box>), -} - enum Bound { /// A broadcast the caller already holds. Ready(moq_net::broadcast::Consumer), - /// A request issued when the binding was made. + /// A request issued when the binding was made, resolved once something announces the path. Requested { - lookup: Lookup, + routing: Box>, /// Where the answer is pinned, the first for its path winning. pins: Pins, path: moq_net::PathOwned, @@ -489,20 +376,22 @@ impl Binding { pub fn poll_broadcast(&self, waiter: &kio::Waiter) -> Poll> { let broadcast = match &self.bound { Bound::Ready(broadcast) => broadcast.clone(), - Bound::Requested { lookup, pins, path } => { - let broadcast = match lookup { - Lookup::Now(request) => ready!(request.poll_ok(waiter))?, - // A resolved routing answers the same result on every poll. - Lookup::Announced(routing) => { + Bound::Requested { routing, pins, path } => { + // Once a path is pinned, its pin is the answer: a request that raced another for + // the path takes the one pinned first, and the routing is never polled again, since + // it would wait out the gap after the pinned instance goes. + if let Some(pinned) = pins.lock().expect("pins poisoned").get(path) { + pinned.clone() + } else { + let broadcast = { let mut routing = routing.lock().expect("routing poisoned"); ready!(kio::Task::poll(&mut *routing, waiter))? - } - }; - // A request that raced another for the path takes the one pinned first. The path's - // first answer is pinned even when it fails this binding's epoch, so no later - // reference resolves the path independently. - let mut pins = pins.lock().expect("pins poisoned"); - pins.entry(path.clone()).or_insert(broadcast).clone() + }; + // The path's first answer is pinned even when it fails this binding's epoch, so + // no later reference resolves the path independently. + let mut pins = pins.lock().expect("pins poisoned"); + pins.entry(path.clone()).or_insert(broadcast).clone() + } } }; // An epochless route cannot prove its instance, so only a known mismatch fails. @@ -618,38 +507,21 @@ mod tests { assert!(first.is_clone(&binding.broadcast().await.unwrap())); } - /// A failed lookup pins nothing: the guarantee starts at a path's first resolution. - #[tokio::test] - async fn failed_binding_does_not_retry_a_later_publisher() { - let origin = produce_origin(); - let _catalog = origin.publish("pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "pub").await; - let media = Relative::new("media"); - let binding = source.bind(Some(&media), None).unwrap(); - assert!(binding.broadcast().await.is_err()); - let _publisher = origin.publish("media", Default::default()).unwrap(); - settle().await; - assert!(!source.resolve(Some(&media), None).await.unwrap().is_closed()); - assert!(binding.broadcast().await.is_err()); - } - - /// An export's request made before anything announces its sibling resolves on the first - /// announcement, and keeps that instance once it is replaced. + /// A binding made before anything announces its sibling resolves on the first announcement + /// and keeps that instance: past its replacement, and once it goes, answering the ended + /// instance at once rather than waiting for the next one. #[tokio::test(start_paused = true)] - async fn an_export_request_waits_for_the_first_announcement() { + async fn a_binding_waits_for_the_first_announcement_and_keeps_it() { let origin = produce_origin(); let _catalog = origin.publish("pub", Default::default()).unwrap(); settle().await; let source = resolved(&origin, "pub").await; let media = Relative::new("media"); - let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); - config.broadcast = Some(media.to_owned()); - let binding = source.try_request(&config).unwrap(); + let binding = source.bind(Some(&media), None).unwrap(); settle().await; assert!(binding.poll_broadcast(&kio::Waiter::noop()).is_pending()); - let _first = origin.publish("media", Default::default()).unwrap(); + let first_publisher = origin.publish("media", Default::default()).unwrap(); let first = binding.broadcast().await.unwrap(); assert!(!first.is_closed()); @@ -657,6 +529,19 @@ mod tests { settle().await; assert!(binding.broadcast().await.unwrap().is_clone(&first)); assert!(source.resolve(Some(&media), None).await.unwrap().is_clone(&first)); + + drop(first_publisher); + first.closed().await; + assert!(binding.broadcast().await.unwrap().is_clone(&first)); + let late = source.bind(Some(&media), None).unwrap(); + assert!( + late.broadcast().await.unwrap().is_clone(&first), + "a later binding reads the pin" + ); + + // A new catalog instance starts fresh pins, which resolve the replacement. + let fresh = source.switched(&source.broadcast()); + assert!(!fresh.resolve(Some(&media), None).await.unwrap().is_closed()); } /// A source serves its path from the broadcast it was built from, even when a replacement @@ -794,115 +679,6 @@ mod tests { assert!(source.resolve(Some(&rel), None).await.unwrap().is_clone(&first)); } - /// The next change a follower reports, which must arrive within the origin's update hold. - async fn followed(follow: &mut Follow) -> Option { - let next = tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) - .await - .expect("no change") - .expect("the follow failed"); - match next { - Followed::Start(broadcast) => Some(broadcast), - Followed::End => None, - } - } - - /// The follower stays quiet past the origin's update hold. - async fn quiet(follow: &mut Follow) { - if tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) - .await - .is_ok() - { - panic!("expected no change"); - } - } - - /// A sibling announced after the follow starts it, its end idles it, and its return starts - /// it again. - #[tokio::test(start_paused = true)] - async fn a_follow_starts_late_and_survives_an_end() { - let origin = produce_origin(); - let _catalog = origin.publish("a/pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let mut follow = source.follow(Some(&Relative::new("./source")), None).unwrap(); - quiet(&mut follow).await; - - let first = origin.publish("a/source", Default::default()).unwrap(); - let started = followed(&mut follow).await.expect("a start"); - assert_eq!(started.info().path.as_str(), "a/source"); - - drop(first); - assert!(followed(&mut follow).await.is_none(), "an end"); - quiet(&mut follow).await; - - let _second = origin.publish("a/source", Default::default()).unwrap(); - let restarted = followed(&mut follow).await.expect("a start"); - assert!(!restarted.is_clone(&started)); - } - - /// A replacement restarts a reference that names no epoch, and ends one that names the old - /// instance until that instance serves again. - #[tokio::test(start_paused = true)] - async fn a_follow_restarts_only_without_an_epoch() { - let origin = produce_origin(); - let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); - let _catalog = origin.publish("a/pub", Default::default()).unwrap(); - let _old = origin.publish("a/source", route(&old_epoch)).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let rel = Relative::new("./source"); - let mut any = source.follow(Some(&rel), None).unwrap(); - let mut named = source.follow(Some(&rel), Some(&old_epoch)).unwrap(); - let old = followed(&mut any).await.unwrap(); - assert!(followed(&mut named).await.unwrap().is_clone(&old)); - - let new = origin.publish("a/source", route(&new_epoch)).unwrap(); - let replaced = followed(&mut any).await.unwrap(); - assert_eq!(replaced.info().epoch.as_ref(), Some(&new_epoch)); - assert!(followed(&mut named).await.is_none(), "another epoch ends it"); - quiet(&mut named).await; - // The old instance still serves the subscriptions already on it. - assert!(!old.is_closed()); - - // The replacement goes and the old instance wins again. - drop(new); - let back = followed(&mut named).await.unwrap(); - assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); - let back = followed(&mut any).await.unwrap(); - assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); - } - - /// An epoch is best effort over an epochless route, which cannot prove another instance. - #[tokio::test(start_paused = true)] - async fn a_follow_reads_an_epochless_route_with_an_epoch() { - let origin = produce_origin(); - let _catalog = origin.publish("a/pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let mut named = source - .follow(Some(&Relative::new("./source")), Some(&moq_net::Epoch::mint())) - .unwrap(); - let _sibling = origin.publish("a/source", Default::default()).unwrap(); - assert!(followed(&mut named).await.is_some()); - } - - /// A self-reference reads only the catalog instance, whatever replaces it at the path. - #[tokio::test(start_paused = true)] - async fn a_follow_of_the_catalog_keeps_its_instance() { - let origin = produce_origin(); - let old = origin.publish("a/pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let mut follow = source.follow(Some(&Relative::new("./pub")), None).unwrap(); - assert!(followed(&mut follow).await.unwrap().is_clone(&source.broadcast())); - - let _new = origin.publish("a/pub", Default::default()).unwrap(); - quiet(&mut follow).await; - // Its end is the catalog's, which the catalog's reader sees. - drop(old); - quiet(&mut follow).await; - } - /// A self-reference's epoch must not contradict the catalog broadcast's own. #[tokio::test(start_paused = true)] async fn a_self_reference_epoch_checks_the_catalog_instance() { diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index 044aa7fa273..d35929168fc 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer.resolve(_:)` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/swift/ffi/MIGRATE.md b/swift/ffi/MIGRATE.md index 3552e06292f..9ac7e18f93d 100644 --- a/swift/ffi/MIGRATE.md +++ b/swift/ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve(reference:)` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. From cd9fb914fb0084a41bbe68e88839ce952fcd5bc3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 00:43:07 -0700 Subject: [PATCH 06/13] fix(gst,mux): share a restarted sibling's timeline; fail a contradicted self-follow moqsrc renditions restarting on the same sibling instance now share that instance's timeline, so audio and video fix one segment mapping instead of each starting from its own first frame. Source::follow fails a self-reference whose epoch the catalog instance contradicts with EpochMismatch, rather than parking forever on a fixed feed no other instance can join. Binding and follow share one epoch check. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 68 +++++++++++++++++++++++++++++++++--- rs/moq-mux/src/follow.rs | 12 ++----- rs/moq-mux/src/source.rs | 46 ++++++++++++++++-------- 3 files changed, 99 insertions(+), 27 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 8564c8a94b3..186ee91c4b5 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -767,6 +767,7 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), + restarts: Restarts::default(), finished: Arc::default(), }; let mut active: HashMap = HashMap::new(); @@ -1016,6 +1017,7 @@ fn reconcile( descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), + restarts: playback.restarts.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1071,9 +1073,37 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, + restarts: Restarts, finished: Finished, } +/// The timeline of each sibling instance a rendition restarted on, shared by every rendition +/// reading that instance so their first frames fix one mapping for all of them. +#[derive(Clone, Default)] +struct Restarts(Arc>>); + +/// An instance some rendition restarted on, and the timeline its renditions share while any reads it. +type Restart = (moq_net::broadcast::Consumer, std::sync::Weak); + +impl Restarts { + /// The timeline of `instance`, new unless another rendition already restarted on it. + fn timeline(&self, instance: &moq_net::broadcast::Consumer) -> Arc { + let mut restarts = self.0.lock().unwrap(); + // Forget an instance no rendition reads any more, releasing its handle. + restarts.retain(|(_, timeline)| timeline.strong_count() > 0); + if let Some(timeline) = restarts + .iter() + .find(|(known, _)| known.is_clone(instance)) + .and_then(|(_, timeline)| timeline.upgrade()) + { + return timeline; + } + let timeline = Arc::default(); + restarts.push((instance.clone(), Arc::downgrade(&timeline))); + timeline + } +} + /// Each pad's rendition with a track it played to the end. A new subscription to a finished /// track replays its last group, so a catalog update that still lists the rendition (a /// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A @@ -1117,6 +1147,8 @@ struct Pump { descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, + /// The run's [`Playback::restarts`]. + restarts: Restarts, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1155,6 +1187,7 @@ impl Pump { descriptor, pads, mut timeline, + restarts, finished, state, mut cancel, @@ -1237,10 +1270,10 @@ impl Pump { let mut track = moq_mux::container::Consumer::new(subscriber, container); let pad = match &owned { - // Another instance on the pad this pump owns: a new stream, on a timeline of its own, - // since the instance's timestamps share no origin with the run's. + // Another instance on the pad this pump owns: a new stream, on that instance's own + // timeline, since its timestamps share no origin with the run's. Some((pad, _)) => { - timeline = Arc::default(); + timeline = restarts.timeline(&broadcast); tokio::task::block_in_place(|| begin(pad, &name, &caps)); pad.clone() } @@ -1524,9 +1557,10 @@ fn audio_caps(config: &hang::catalog::AudioConfig) -> Result { #[cfg(test)] mod tests { - use super::{PumpState, Timeline, ended, plan_reconcile}; + use super::{PumpState, Restarts, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; + use std::sync::Arc; /// A pump cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -1624,6 +1658,31 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } + /// Renditions restarting on the same sibling instance share its timeline, and the next + /// instance starts another. + #[tokio::test(start_paused = true)] + async fn renditions_restarting_on_one_instance_share_its_timeline() { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + tokio::spawn(moq_net::time::run(driver)); + let consumer = origin.consume(); + let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { + moq_mux::follow::Event::Start(broadcast) => broadcast, + moq_mux::follow::Event::End => panic!("expected a start"), + }; + let restarts = Restarts::default(); + + let _first = origin.publish("source", Default::default()).unwrap(); + let shared = restarts.timeline(&started(&mut audio).await); + assert!(Arc::ptr_eq(&shared, &restarts.timeline(&started(&mut video).await))); + + let _second = origin.publish("source", Default::default()).unwrap(); + let next = restarts.timeline(&started(&mut audio).await); + assert!(!Arc::ptr_eq(&shared, &next), "another instance has its own timeline"); + assert!(Arc::ptr_eq(&next, &restarts.timeline(&started(&mut video).await))); + } + #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -2873,6 +2932,7 @@ mod session_tests { descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), + restarts: super::Restarts::default(), finished: std::sync::Arc::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, diff --git a/rs/moq-mux/src/follow.rs b/rs/moq-mux/src/follow.rs index eb149f7a593..9823777dd5d 100644 --- a/rs/moq-mux/src/follow.rs +++ b/rs/moq-mux/src/follow.rs @@ -49,8 +49,8 @@ impl Instances { } /// Hand out `broadcast` once and never report its end, which its owner follows. - pub(crate) fn fixed(broadcast: moq_net::broadcast::Consumer, epoch: Option<&moq_net::Epoch>) -> Self { - Self::with(Feed::Fixed(Some(broadcast)), epoch) + pub(crate) fn fixed(broadcast: moq_net::broadcast::Consumer) -> Self { + Self::with(Feed::Fixed(Some(broadcast)), None) } fn with(feed: Feed, epoch: Option<&moq_net::Epoch>) -> Self { @@ -79,7 +79,7 @@ impl Instances { }; if let Poll::Ready(next) = next { let broadcast = next?; - if proves_other(self.epoch.as_ref(), &broadcast) { + if crate::source::check_epoch(self.epoch.as_ref(), &broadcast).is_err() { tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); match self.current.take() { Some(_) => return Poll::Ready(Ok(Event::End)), @@ -109,12 +109,6 @@ impl Instances { } } -/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route -/// cannot prove its instance, so only a known mismatch counts. -fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { - matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) -} - #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 499580260a4..a1a5cf58aeb 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -311,7 +311,9 @@ impl Source { /// /// With an `epoch`, only that instance (or an epochless route, which cannot prove another /// one) is handed out: another taking over ends the rendition rather than substituting for - /// it. Rejects an escaping reference exactly as [`resolve`](Self::resolve) does. + /// it. A self-reference whose epoch the catalog instance contradicts fails with + /// [`Error::EpochMismatch`](crate::Error::EpochMismatch), since no other instance can come. + /// Rejects an escaping reference exactly as [`resolve`](Self::resolve) does. pub fn follow( &self, rel: Option<&moq_net::path::Relative<'_>>, @@ -319,12 +321,34 @@ impl Source { ) -> crate::Result { let path = self.target(rel)?; Ok(match path == self.path { - true => crate::follow::Instances::fixed(self.broadcast(), epoch), + true => { + let broadcast = self.broadcast(); + check_epoch(epoch, &broadcast)?; + crate::follow::Instances::fixed(broadcast) + } false => crate::follow::Instances::new(&self.origin, &path, epoch)?, }) } } +/// Fail with [`Error::EpochMismatch`](crate::Error::EpochMismatch) when `broadcast` is proven to +/// be another instance than `epoch` names. An epochless route cannot prove its instance, so only +/// a known mismatch fails. +pub(crate) fn check_epoch( + epoch: Option<&moq_net::Epoch>, + broadcast: &moq_net::broadcast::Consumer, +) -> crate::Result<()> { + let info = broadcast.info(); + match (epoch, &info.epoch) { + (Some(expected), Some(actual)) if expected != actual => Err(crate::Error::EpochMismatch { + path: info.path.to_string(), + expected: expected.clone(), + actual: actual.clone(), + }), + _ => Ok(()), + } +} + /// A held broadcast or the retained result of one eagerly issued broadcast request. /// /// A pending request waits for a path nothing announces yet and rides out route churn until it @@ -394,18 +418,7 @@ impl Binding { } } }; - // An epochless route cannot prove its instance, so only a known mismatch fails. - let info = broadcast.info(); - if let (Some(expected), Some(actual)) = (&self.epoch, &info.epoch) - && expected != actual - { - return Poll::Ready(Err(crate::Error::EpochMismatch { - path: info.path.to_string(), - expected: expected.clone(), - actual: actual.clone(), - })); - } - Poll::Ready(Ok(broadcast)) + Poll::Ready(check_epoch(self.epoch.as_ref(), &broadcast).map(|()| broadcast)) } /// The bound broadcast, awaiting the origin's answer when it hasn't arrived yet. @@ -693,6 +706,11 @@ mod tests { let own = source.resolve(rel.as_ref(), Some(&catalog)).await.unwrap(); assert!(own.is_clone(&source.broadcast()), "{reference:?}"); assert!(is_mismatch(source.resolve(rel.as_ref(), Some(&other)).await, &other)); + assert!(source.follow(rel.as_ref(), Some(&catalog)).is_ok()); + assert!(matches!( + source.follow(rel.as_ref(), Some(&other)), + Err(crate::Error::EpochMismatch { .. }) + )); } } From a36fb1cf9cb4a7c3f16ff1d8012d233c1ed9726a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 00:52:38 -0700 Subject: [PATCH 07/13] fix(gst): key sibling timelines by instance; a finished sibling track idles Every moqsrc rendition picks its sibling instance's timeline when it starts reading it, not only once it owns a pad, so one still subscribing when a replacement takes over shares the replacement's mapping. The first instance at a path keeps the run's timeline. A sibling track finishing while its instance stays up idles the pad and keeps following, so the instance that replaces it starts a new stream there; the finish record clears once a new instance starts on the pad. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 129 ++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 38 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 186ee91c4b5..b8d43d916e8 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -767,7 +767,7 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), - restarts: Restarts::default(), + siblings: Siblings::default(), finished: Arc::default(), }; let mut active: HashMap = HashMap::new(); @@ -1017,7 +1017,7 @@ fn reconcile( descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), - restarts: playback.restarts.clone(), + siblings: playback.siblings.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1073,33 +1073,46 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, - restarts: Restarts, + siblings: Siblings, finished: Finished, } -/// The timeline of each sibling instance a rendition restarted on, shared by every rendition -/// reading that instance so their first frames fix one mapping for all of them. +/// The timeline of each sibling instance, shared by every rendition reading that instance so +/// their first frames fix one mapping for all of them. #[derive(Clone, Default)] -struct Restarts(Arc>>); - -/// An instance some rendition restarted on, and the timeline its renditions share while any reads it. -type Restart = (moq_net::broadcast::Consumer, std::sync::Weak); - -impl Restarts { - /// The timeline of `instance`, new unless another rendition already restarted on it. - fn timeline(&self, instance: &moq_net::broadcast::Consumer) -> Arc { - let mut restarts = self.0.lock().unwrap(); - // Forget an instance no rendition reads any more, releasing its handle. - restarts.retain(|(_, timeline)| timeline.strong_count() > 0); - if let Some(timeline) = restarts +struct Siblings(Arc>); + +#[derive(Default)] +struct SiblingTimelines { + /// Each sibling path some instance served this run. + seen: HashSet, + /// Each instance still serving, and the timeline its renditions share while any reads it. + live: Vec<(moq_net::broadcast::Consumer, std::sync::Weak)>, +} + +impl Siblings { + /// The timeline of `instance`: the run's for the first instance at its path, which the + /// catalog's own renditions share, and one of its own for any later instance, whose + /// timestamps share no origin with the run's. + fn timeline(&self, instance: &moq_net::broadcast::Consumer, run: &Arc) -> Arc { + let mut siblings = self.0.lock().unwrap(); + // Forget an instance that ended or that no rendition reads any more, releasing its handle. + siblings + .live + .retain(|(known, timeline)| !known.is_closed() && timeline.strong_count() > 0); + if let Some(timeline) = siblings + .live .iter() .find(|(known, _)| known.is_clone(instance)) .and_then(|(_, timeline)| timeline.upgrade()) { return timeline; } - let timeline = Arc::default(); - restarts.push((instance.clone(), Arc::downgrade(&timeline))); + let timeline = match siblings.seen.insert(instance.info().path.clone()) { + true => run.clone(), + false => Arc::default(), + }; + siblings.live.push((instance.clone(), Arc::downgrade(&timeline))); timeline } } @@ -1147,8 +1160,8 @@ struct Pump { descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, - /// The run's [`Playback::restarts`]. - restarts: Restarts, + /// The run's [`Playback::siblings`]. + siblings: Siblings, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1186,8 +1199,8 @@ impl Pump { container, descriptor, pads, - mut timeline, - restarts, + timeline: run, + siblings, finished, state, mut cancel, @@ -1268,12 +1281,16 @@ impl Pump { }, }; let mut track = moq_mux::container::Consumer::new(subscriber, container); + let timeline = match sibling { + true => siblings.timeline(&broadcast, &run), + false => run.clone(), + }; let pad = match &owned { - // Another instance on the pad this pump owns: a new stream, on that instance's own - // timeline, since its timestamps share no origin with the run's. + // Another instance on the pad this pump owns: a new stream, and a track that + // finished on the old instance is not finished on this one. Some((pad, _)) => { - timeline = restarts.timeline(&broadcast); + finished.lock().unwrap().remove(&((kind, name.clone()), media.clone())); tokio::task::block_in_place(|| begin(pad, &name, &caps)); pad.clone() } @@ -1392,6 +1409,13 @@ impl Pump { // A sibling's track stopping with its instance is that instance's end, which // idles the pad for the next one rather than ending the rendition. Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, + // A sibling's track finishing ends it for this instance only: the pad idles + // for the next instance, and the record keeps a reshape on this one from + // replaying it. + Ok(None) if sibling => { + finished.lock().unwrap().insert(((kind, name.clone()), media.clone())); + break Stopped::Idle; + } Ok(None) => { // Recorded while this pump still owns the pad, before any replacement can claim it. finished.lock().unwrap().insert(((kind, name.clone()), media)); @@ -1557,7 +1581,7 @@ fn audio_caps(config: &hang::catalog::AudioConfig) -> Result { #[cfg(test)] mod tests { - use super::{PumpState, Restarts, Timeline, ended, plan_reconcile}; + use super::{PumpState, Siblings, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; use std::sync::Arc; @@ -1658,29 +1682,31 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } - /// Renditions restarting on the same sibling instance share its timeline, and the next - /// instance starts another. + /// Renditions on the same sibling instance share its timeline, whenever each starts on it: + /// the first instance shares the run's, and the next one starts another. #[tokio::test(start_paused = true)] - async fn renditions_restarting_on_one_instance_share_its_timeline() { + async fn renditions_on_one_sibling_instance_share_its_timeline() { let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); tokio::spawn(moq_net::time::run(driver)); let consumer = origin.consume(); let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { moq_mux::follow::Event::Start(broadcast) => broadcast, moq_mux::follow::Event::End => panic!("expected a start"), }; - let restarts = Restarts::default(); + let (siblings, run) = (Siblings::default(), Arc::::default()); let _first = origin.publish("source", Default::default()).unwrap(); - let shared = restarts.timeline(&started(&mut audio).await); - assert!(Arc::ptr_eq(&shared, &restarts.timeline(&started(&mut video).await))); + let first = started(&mut audio).await; + assert!(Arc::ptr_eq(&run, &siblings.timeline(&first, &run))); + // A video rendition that first starts on the replacement, as one still subscribing when it + // took over does, shares its timeline with the audio rendition restarting onto it. let _second = origin.publish("source", Default::default()).unwrap(); - let next = restarts.timeline(&started(&mut audio).await); - assert!(!Arc::ptr_eq(&shared, &next), "another instance has its own timeline"); - assert!(Arc::ptr_eq(&next, &restarts.timeline(&started(&mut video).await))); + let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let next = siblings.timeline(&started(&mut video).await, &run); + assert!(!Arc::ptr_eq(&run, &next), "another instance has its own timeline"); + assert!(Arc::ptr_eq(&next, &siblings.timeline(&started(&mut audio).await, &run))); } #[test] @@ -2932,7 +2958,7 @@ mod session_tests { descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), - restarts: super::Restarts::default(), + siblings: super::Siblings::default(), finished: std::sync::Arc::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, @@ -3222,6 +3248,33 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A sibling track finishing while its instance stays up idles the pad, and the instance that + /// replaces it starts a new stream there. + #[test] + fn a_finished_sibling_track_returns_on_the_next_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + assert!(!recorder.seen().contains(&Seen::Eos(pad)), "the finish ended the pad"); + recorder.assert_one_pad_held(); + drop(first); + } + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on /// the same pad, and nothing more from the old instance gets through. fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { From 92f6749ba4944c0acda68d91258e5d1a13f49a22 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:03:34 -0700 Subject: [PATCH 08/13] fix(gst): a delisted sibling rendition follows no later instance A rendition the catalog delists drains the instance it reads, then ends its pad instead of idling for, or restarting on, the sibling's next instance. One already idle when the delist lands ends right away. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 90 ++++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index b8d43d916e8..ec11523829d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -418,7 +418,12 @@ struct Shape { /// stopping it means it must never create one; after, it owns a pad and has to drop it. One /// compare-exchange settles which of the two happened, so a pad can't slip out between a /// teardown's check and the pump's creation. -struct PumpState(AtomicU8); +struct PumpState { + phase: AtomicU8, + /// The latest catalog still lists the rendition. A delisted pump drains the instance it reads + /// and follows no other. + listed: watch::Sender, +} impl PumpState { const SUBSCRIBING: u8 = 0; @@ -426,12 +431,15 @@ impl PumpState { const CANCELLED: u8 = 2; fn new() -> Self { - Self(AtomicU8::new(Self::SUBSCRIBING)) + Self { + phase: AtomicU8::new(Self::SUBSCRIBING), + listed: watch::Sender::new(true), + } } /// Claim the right to create a pad, false once a teardown got here first. fn go_live(&self) -> bool { - self.0 + self.phase .compare_exchange(Self::SUBSCRIBING, Self::LIVE, Ordering::AcqRel, Ordering::Acquire) .is_ok() } @@ -439,10 +447,19 @@ impl PumpState { /// Stop a pump that hasn't taken a pad, false if it already has one and must be torn down /// through its cancel watch instead. fn cancel_before_live(&self) -> bool { - self.0 + self.phase .compare_exchange(Self::SUBSCRIBING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + + fn set_listed(&self, listed: bool) { + self.listed + .send_if_modified(|current| std::mem::replace(current, listed) != listed); + } + + fn is_listed(&self) -> bool { + *self.listed.borrow() + } } /// A rendition by kind and moq track name: what a pad is kept by. @@ -931,6 +948,12 @@ fn reconcile( .collect(), ); + // A delisted pump drains the instance it reads and follows no other; a relisted one follows + // again. + for (name, track) in active.iter() { + track.state.set_listed(desired.contains_key(name)); + } + // Cancel anything that changed shape; each cancelled pump hands its pad back. Changed // renditions also land in `plan.add`, so they respawn below on the same pad. // @@ -1186,7 +1209,8 @@ impl Pump { /// run can end the pad of one the catalog retired. /// /// A sibling's announcements drive the pump: its first start binds the rendition, an end - /// idles the pad, and another instance taking over restarts the stream on the same pad. + /// idles the pad, and another instance taking over restarts the stream on the same pad, until + /// the catalog delists the rendition. async fn run(self) -> Option { let Pump { element, @@ -1210,12 +1234,18 @@ impl Pump { // The pad once this pump owns it, from the first instance on. let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; let mut next: Option = None; + let mut listed = state.listed.subscribe(); loop { + // A delisted rendition drained the instance it read, and follows no other. + if owned.is_some() && !state.is_listed() { + return Some((kind, name)); + } let broadcast = match next.take() { Some(broadcast) => broadcast, None => tokio::select! { _ = cancel.changed() => return None, + _ = listed.wait_for(|listed| !listed), if owned.is_some() => return Some((kind, name)), followed = follow.next() => match followed { Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, Ok(moq_mux::follow::Event::End) => continue, @@ -1409,10 +1439,10 @@ impl Pump { // A sibling's track stopping with its instance is that instance's end, which // idles the pad for the next one rather than ending the rendition. Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, - // A sibling's track finishing ends it for this instance only: the pad idles - // for the next instance, and the record keeps a reshape on this one from - // replaying it. - Ok(None) if sibling => { + // A listed sibling's track finishing ends it for this instance only: the pad + // idles for the next instance, and the record keeps a reshape on this one + // from replaying it. + Ok(None) if sibling && state.is_listed() => { finished.lock().unwrap().insert(((kind, name.clone()), media.clone())); break Stopped::Idle; } @@ -3275,6 +3305,48 @@ mod session_tests { drop(first); } + /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next + /// instance plays nothing on it. + #[test] + fn a_delisted_sibling_rendition_follows_no_later_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + room.catalog.modify().unwrap().video.renditions.clear(); + for _ in 0..200 { + if recorder.seen().contains(&Seen::Eos(pad.clone())) { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + recorder.seen().contains(&Seen::Eos(pad.clone())), + "the idle pad never ended" + ); + + let mut second = Publisher::sibling(&origin, Default::default()); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + for _ in 0..10 { + second.write(2); + std::thread::sleep(Duration::from_millis(50)); + } + stop(session); + + let seen = recorder.seen(); + assert!(!seen.iter().any(|seen| matches!(seen, Seen::Buffer(_, 2))), "{seen:?}"); + } + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on /// the same pad, and nothing more from the old instance gets through. fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { From 94292e0f4a726bf917a7fd9e5d69a692695b0862 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:15:28 -0700 Subject: [PATCH 09/13] fix(gst): scope a finished track to the instance that served it moqsrc recorded a finished track by rendition and path, so a catalog update that replaced an idle sibling pump (a disable and enable, or an epoch handoff) skipped the rendition for every later instance, leaving its pad silent. The record now keeps the instance that finished, and the pump checks it once it knows which instance it reads, instead of reconcile skipping the rendition. A sibling pump idles past a finished instance, holding its pad if it has one; the catalog's own rendition still stops. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 187 +++++++++++++++++++++++++++++------ 1 file changed, 159 insertions(+), 28 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index ec11523829d..89ff06207d7 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -785,7 +785,7 @@ async fn follow_catalog( pads: pads.clone(), timeline: Arc::default(), siblings: Siblings::default(), - finished: Arc::default(), + finished: Finished::default(), }; let mut active: HashMap = HashMap::new(); let mut pumps: tokio::task::JoinSet> = tokio::task::JoinSet::new(); @@ -994,10 +994,6 @@ fn reconcile( broadcast: source.resolve_reference(d.shape.broadcast.as_ref()), name: d.shape.track.clone(), }; - let played = ((d.kind, name.clone()), track.clone()); - if playback.finished.lock().unwrap().contains(&played) { - continue; - } let container = match moq_mux::catalog::hang::Container::new( &d.shape.container, match d.kind { @@ -1140,12 +1136,30 @@ impl Siblings { } } -/// Each pad's rendition with a track it played to the end. A new subscription to a finished -/// track replays its last group, so a catalog update that still lists the rendition (a -/// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A -/// pad that never played the track joins it like any late subscriber. Shared with the pumps, -/// since a pump that replaces another on its pad learns this only once the other lets go. -type Finished = Arc>>; +/// Each pad's rendition with a track it played to the end, and the instance that served it. A new +/// subscription to a finished track replays its last group, so a catalog update that still lists +/// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on +/// that pad again. A later instance carries a new track, and a pad that never played the track +/// joins it like any late subscriber. Shared with the pumps, since a pump that replaces another on +/// its pad learns this only once the other lets go. +#[derive(Clone, Default)] +struct Finished(Arc>>); + +impl Finished { + /// Record that `instance` served `played` to its end, replacing any earlier instance's record. + fn insert(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) { + self.0.lock().unwrap().insert(played.clone(), instance.clone()); + } + + /// Whether `instance` already served `played` to its end on this pad. + fn contains(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) -> bool { + self.0 + .lock() + .unwrap() + .get(played) + .is_some_and(|finished| finished.is_clone(instance)) + } +} /// The first decoded frame fixes one mapping for every rendition of this broadcast run. #[derive(Default)] @@ -1230,6 +1244,7 @@ impl Pump { mut cancel, shutdown, } = self; + let played = ((kind, name.clone()), media.clone()); let mut container = Some(container); // The pad once this pump owns it, from the first instance on. let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; @@ -1257,6 +1272,15 @@ impl Pump { }, }; + // An instance that already played this track to its end on the pad would replay its + // last group. A sibling waits for the next instance; the catalog's own has no other. + if finished.contains(&played, &broadcast) { + match sibling { + true => continue, + false => return None, + } + } + // Resolves once the publisher answers, with the track info or with an error (which is // what ending a broadcast produces for a name nobody served). A publisher that answers // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and @@ -1317,10 +1341,8 @@ impl Pump { }; let pad = match &owned { - // Another instance on the pad this pump owns: a new stream, and a track that - // finished on the old instance is not finished on this one. + // Another instance on the pad this pump owns: a new stream. Some((pad, _)) => { - finished.lock().unwrap().remove(&((kind, name.clone()), media.clone())); tokio::task::block_in_place(|| begin(pad, &name, &caps)); pad.clone() } @@ -1348,23 +1370,23 @@ impl Pump { (pad, owned) } Claim::Kept(pad, owner) => { - let owned = tokio::select! { + let guard = tokio::select! { biased; _ = cancel.changed() => return None, - owned = owner.lock_owned() => owned, + guard = owner.lock_owned() => guard, }; // The pump this one replaces records its track as finished before letting // go, so a replacement started while it was reading the end (a reshape) - // can't replay it. - if finished - .lock() - .unwrap() - .contains(&((kind, name.clone()), media.clone())) - { - return Some((kind, name)); + // can't replay it. A sibling holds the pad for its next instance. + if finished.contains(&played, &broadcast) { + if !sibling { + return Some((kind, name)); + } + owned = Some((pad, guard)); + continue; } tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, owned) + (pad, guard) } }; // Stop flushes only the pads it finds, and this one may have been added just @@ -1386,7 +1408,7 @@ impl Pump { // waiting for this pad would replay it. _ = cancel.changed() => { if ended(&mut track) { - finished.lock().unwrap().insert(((kind, name.clone()), media)); + finished.insert(&played, &broadcast); } return None; } @@ -1443,12 +1465,12 @@ impl Pump { // idles for the next instance, and the record keeps a reshape on this one // from replaying it. Ok(None) if sibling && state.is_listed() => { - finished.lock().unwrap().insert(((kind, name.clone()), media.clone())); + finished.insert(&played, &broadcast); break Stopped::Idle; } Ok(None) => { // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.lock().unwrap().insert(((kind, name.clone()), media)); + finished.insert(&played, &broadcast); return Some((kind, name)); } Err(err) => { @@ -2989,7 +3011,7 @@ mod session_tests { pads: Pads::default(), timeline: std::sync::Arc::default(), siblings: super::Siblings::default(), - finished: std::sync::Arc::default(), + finished: super::Finished::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, shutdown, @@ -3305,6 +3327,115 @@ mod session_tests { drop(first); } + /// Block until `publisher`'s video track reads to its end, as a pump reading it does. + fn await_finished(publisher: &Publisher) { + let broadcast = publisher.broadcast.consume(); + super::RUNTIME.block_on(async { + let finished = async { + let subscriber = broadcast + .track("video") + .unwrap() + .subscribe(moq_net::track::Subscription::default()) + .await + .unwrap(); + let mut track = moq_mux::container::Consumer::new( + subscriber, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + while track.read().await.unwrap().is_some() {} + }; + tokio::time::timeout(Duration::from_secs(10), finished) + .await + .expect("the track never finished"); + }); + } + + /// A sibling track that finished stays finished for its instance only: a catalog update that + /// replaces the idle pump, here a disable and an enable, still plays the next instance. + #[test] + fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let _audio = room + .broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + + // The catalog consumer only yields the newest snapshot, so the same update lists an audio + // rendition: its pad appearing proves the run acted on the disable. + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().enabled = false; + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + await_pad(&element, "audio_"); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .enabled = true; + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + recorder.assert_one_pad_held(); + drop(first); + } + + /// A publisher handoff after a sibling track finished, the catalog naming the new instance's + /// epoch, plays that instance on the same pad. + #[test] + fn an_epoch_handoff_after_a_sibling_finish_plays_the_new_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let (old, new) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let mut room = referencing(&origin, Some(old.clone())); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(old)); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + + let mut second = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(new.clone())); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .epoch = Some(new); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the new instance moved to another pad" + ); + stop(session); + + recorder.assert_one_pad_held(); + drop(first); + } + /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next /// instance plays nothing on it. #[test] From fe98261afb17e1f90b0c29e9cbe40ca0f7e2eb8a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 14:15:31 -0700 Subject: [PATCH 10/13] refactor(gst): split moqsrc sibling following into its own PR moqsrc keeps today's sibling handling: a rendition binds its sibling through `Source::bind`, which now waits for the first announcement and keeps that instance. Following a sibling's later instances moves to the moqsrc sibling follow quest. Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 2 +- doc/concept/hang.md | 2 +- quest/m0/broadcast-epoch/README.md | 8 +- .../broadcast-epoch/moqsrc-sibling-follow.md | 41 + quest/m1/sink-restart-follow.md | 3 +- rs/moq-gst/src/source/imp.rs | 839 +++--------------- 6 files changed, 188 insertions(+), 707 deletions(-) create mode 100644 quest/m0/broadcast-epoch/moqsrc-sibling-follow.md diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index b77e58889fb..3b053b30a2d 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -124,7 +124,7 @@ rewound timestamps forward. Pads are named by kind and appear as the catalog announces renditions: `video_0`, `video_1`, `audio_0`. A rendition referencing another broadcast reads -that broadcast on the same connection and follows its announcements the same way. Link the pad you want by name; the terse +that broadcast on the same connection, waiting for it to be announced. Link the pad you want by name; the terse `moqsrc ! decodebin3` form links only the first pad offered, which may be audio. `moqsrc` follows the broadcast's announcements: when the publisher restarts, it switches to the new broadcast on the same pads, kept by rendition diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 841f74d2b0b..714ce485f61 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; every other reader, such as `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` follows a referenced path's announcements like its own, restarting the rendition on each instance that serves it; every other reader, such as `moqsrc`, `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index ff1b59f49dc..e3e052afc07 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -1,4 +1,4 @@ -# [L] Broadcast epochs +# Broadcast epochs ## Goal @@ -73,7 +73,7 @@ Decided: resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The schemas, shared resolution, compatibility documentation, and `moq_mux::follow` - (which `moqsrc` follows siblings with) landed. + landed; `moqsrc` following siblings with it remains. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players (`moq play`, `@moq/watch`, demo/web) follow the announce `Restart`. moq-stats mints one per @@ -122,3 +122,7 @@ restarts that sibling while preserving the catalog's explicit epoch constraints; it never mixes instances through an old handle. Check exact and covering routes, matching/different/absent epochs, and legacy End+Start. The child quests own their focused regressions. + +## Required + +- [moqsrc sibling follow](/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md) - moqsrc restarts a referenced sibling's rendition on each instance its announcements hand out, never past a named epoch diff --git a/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md b/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md new file mode 100644 index 00000000000..f14cd2dd940 --- /dev/null +++ b/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md @@ -0,0 +1,41 @@ +# [M] moqsrc follows referenced broadcasts' announcements + +## Goal + +`moqsrc` follows a catalog-referenced sibling's announcements the way +`@moq/watch` does: each rendition reads the instance `moq_mux::Source::follow` +hands out, idles its pad (no EOS) while the sibling is offline, and restarts on +the same pad when another instance serves it. A reference that names an +`epoch` never plays another instance. Until this lands, `moqsrc` waits for a +sibling's first announcement and keeps that instance, like every other +non-following reader. + +## Plan + +Split from the reference announcements work (maintainer, 2026-10-22), which +landed `moq_mux::follow` and the waiting `Source::bind`. The moqsrc half drew a +new state-machine finding every review round, so it gets its own design pass. + +- Start binds the rendition; End idles its pad; another instance restarts the + stream on the same pad with a fresh stream-start, caps, and a segment. +- Renditions reading one sibling instance share that instance's timeline, + whenever each starts on it, including one still subscribing when a + replacement takes over. +- A sibling track that finishes while its instance stays up idles the pad and + keeps following; a finished record names the instance that served it, so a + reshape (disable and enable, an epoch handoff) still plays the next one. +- A rendition the catalog delists drains the instance it reads, then ends its + pad and follows no later instance. +- A pump that has returned but is still in `active` must not swallow a relist: + a relist arriving before the pump is joined matches its shape and starts + nothing, for an idle delisted sibling and for a delisted catalog rendition + whose track ends just before the relist. Cover both cases. + +Weigh a simpler state model (one task per sibling instance owning its +renditions, or reconciling after each pump exits) before patching each case. + +Public API: none (moqsrc behavior). Wire: none. + +## Related + +- [Sink restart follow](/quest/m1/sink-restart-follow.md) - moves moqsrc's own broadcast follow onto `follow::Instances` diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index b76f09ecbfe..5ff849bc074 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -16,7 +16,8 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references -to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; +to other broadcasts is [moqsrc sibling follow](/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md) +for `moqsrc`, through `moq_mux::Source::follow`; `moq play` resolves a reference once with `Source::resolve`, which waits for a sibling's first announcement and keeps it. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index e54be6bfbbf..9316007c684 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -437,12 +437,7 @@ struct Shape { /// stopping it means it must never create one; after, it owns a pad and has to drop it. One /// compare-exchange settles which of the two happened, so a pad can't slip out between a /// teardown's check and the pump's creation. -struct PumpState { - phase: AtomicU8, - /// The latest catalog still lists the rendition. A delisted pump drains the instance it reads - /// and follows no other. - listed: watch::Sender, -} +struct PumpState(AtomicU8); impl PumpState { const SUBSCRIBING: u8 = 0; @@ -450,15 +445,12 @@ impl PumpState { const CANCELLED: u8 = 2; fn new() -> Self { - Self { - phase: AtomicU8::new(Self::SUBSCRIBING), - listed: watch::Sender::new(true), - } + Self(AtomicU8::new(Self::SUBSCRIBING)) } /// Claim the right to create a pad, false once a teardown got here first. fn go_live(&self) -> bool { - self.phase + self.0 .compare_exchange(Self::SUBSCRIBING, Self::LIVE, Ordering::AcqRel, Ordering::Acquire) .is_ok() } @@ -466,19 +458,10 @@ impl PumpState { /// Stop a pump that hasn't taken a pad, false if it already has one and must be torn down /// through its cancel watch instead. fn cancel_before_live(&self) -> bool { - self.phase + self.0 .compare_exchange(Self::SUBSCRIBING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire) .is_ok() } - - fn set_listed(&self, listed: bool) { - self.listed - .send_if_modified(|current| std::mem::replace(current, listed) != listed); - } - - fn is_listed(&self) -> bool { - *self.listed.borrow() - } } /// A rendition by kind and moq track name: what a pad is kept by. @@ -803,8 +786,7 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), - siblings: Siblings::default(), - finished: Finished::default(), + finished: Arc::default(), }; let mut active: HashMap = HashMap::new(); let mut pumps: tokio::task::JoinSet> = tokio::task::JoinSet::new(); @@ -967,12 +949,6 @@ fn reconcile( .collect(), ); - // A delisted pump drains the instance it reads and follows no other; a relisted one follows - // again. - for (name, track) in active.iter() { - track.state.set_listed(desired.contains_key(name)); - } - // Cancel anything that changed shape; each cancelled pump hands its pad back. Changed // renditions also land in `plan.add`, so they respawn below on the same pad. // @@ -1013,6 +989,10 @@ fn reconcile( broadcast: source.resolve_reference(d.shape.broadcast.as_ref()), name: d.shape.track.clone(), }; + let played = ((d.kind, name.clone()), track.clone()); + if playback.finished.lock().unwrap().contains(&played) { + continue; + } let container = match moq_mux::catalog::hang::Container::new( &d.shape.container, match d.kind { @@ -1027,16 +1007,15 @@ fn reconcile( } }; - // Follow without waiting: an unavailable sibling or track must not park the catalog loop + // Bind without waiting: an unavailable sibling or track must not park the catalog loop // and hold up every other rendition. The pump resolves both under its cancellation scope. - let follow = match source.follow(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { - Ok(follow) => follow, + let binding = match source.bind(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { + Ok(binding) => binding, Err(err) => { gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); continue; } }; - let sibling = track.broadcast != source.resolve_reference(None); let (cancel_tx, cancel_rx) = watch::channel(false); let state = Arc::new(PumpState::new()); @@ -1048,14 +1027,11 @@ fn reconcile( kind: d.kind, name: name.clone(), caps: d.shape.caps.clone(), - follow, - sibling, + binding, track, container, - descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), - siblings: playback.siblings.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1111,74 +1087,15 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, - siblings: Siblings, finished: Finished, } -/// The timeline of each sibling instance, shared by every rendition reading that instance so -/// their first frames fix one mapping for all of them. -#[derive(Clone, Default)] -struct Siblings(Arc>); - -#[derive(Default)] -struct SiblingTimelines { - /// Each sibling path some instance served this run. - seen: HashSet, - /// Each instance still serving, and the timeline its renditions share while any reads it. - live: Vec<(moq_net::broadcast::Consumer, std::sync::Weak)>, -} - -impl Siblings { - /// The timeline of `instance`: the run's for the first instance at its path, which the - /// catalog's own renditions share, and one of its own for any later instance, whose - /// timestamps share no origin with the run's. - fn timeline(&self, instance: &moq_net::broadcast::Consumer, run: &Arc) -> Arc { - let mut siblings = self.0.lock().unwrap(); - // Forget an instance that ended or that no rendition reads any more, releasing its handle. - siblings - .live - .retain(|(known, timeline)| !known.is_closed() && timeline.strong_count() > 0); - if let Some(timeline) = siblings - .live - .iter() - .find(|(known, _)| known.is_clone(instance)) - .and_then(|(_, timeline)| timeline.upgrade()) - { - return timeline; - } - let timeline = match siblings.seen.insert(instance.info().path.clone()) { - true => run.clone(), - false => Arc::default(), - }; - siblings.live.push((instance.clone(), Arc::downgrade(&timeline))); - timeline - } -} - -/// Each pad's rendition with a track it played to the end, and the instance that served it. A new -/// subscription to a finished track replays its last group, so a catalog update that still lists -/// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on -/// that pad again. A later instance carries a new track, and a pad that never played the track -/// joins it like any late subscriber. Shared with the pumps, since a pump that replaces another on -/// its pad learns this only once the other lets go. -#[derive(Clone, Default)] -struct Finished(Arc>>); - -impl Finished { - /// Record that `instance` served `played` to its end, replacing any earlier instance's record. - fn insert(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) { - self.0.lock().unwrap().insert(played.clone(), instance.clone()); - } - - /// Whether `instance` already served `played` to its end on this pad. - fn contains(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) -> bool { - self.0 - .lock() - .unwrap() - .get(played) - .is_some_and(|finished| finished.is_clone(instance)) - } -} +/// Each pad's rendition with a track it played to the end. A new subscription to a finished +/// track replays its last group, so a catalog update that still lists the rendition (a +/// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A +/// pad that never played the track joins it like any late subscriber. Shared with the pumps, +/// since a pump that replaces another on its pad learns this only once the other lets go. +type Finished = Arc>>; /// The first decoded frame fixes one mapping for every rendition of this broadcast run. #[derive(Default)] @@ -1204,20 +1121,11 @@ struct Pump { /// per-kind counter instead, and isn't known until the subscription resolves. name: String, caps: gst::Caps, - /// Each instance of the broadcast the rendition references. - follow: moq_mux::follow::Instances, - /// The rendition references another broadcast than its catalog's, whose instances come and - /// go on their own announcements. - sibling: bool, + binding: moq_mux::Binding, track: Track, - /// The wire format of the first instance. container: moq_mux::catalog::hang::Container, - /// What builds it again for a later instance. - descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, - /// The run's [`Playback::siblings`]. - siblings: Siblings, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1227,293 +1135,155 @@ struct Pump { shutdown: watch::Receiver, } -/// Why a pump stopped reading one instance without ending. -enum Stopped { - /// Another instance took over: restart on it. - Restart(moq_net::broadcast::Consumer), - /// The instance stopped serving: idle until the next one. - Idle, -} - impl Pump { /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the /// track ends, errors, or `cancel` fires. The pad goes back to the session without an EOS /// either way. Returns the rendition if its track ended rather than being cancelled, so the /// run can end the pad of one the catalog retired. - /// - /// A sibling's announcements drive the pump: its first start binds the rendition, an end - /// idles the pad, and another instance taking over restarts the stream on the same pad, until - /// the catalog delists the rendition. async fn run(self) -> Option { let Pump { element, kind, name, caps, - mut follow, - sibling, + binding, track: media, container, - descriptor, pads, - timeline: run, - siblings, + timeline, finished, state, mut cancel, shutdown, } = self; - let played = ((kind, name.clone()), media.clone()); - let mut container = Some(container); - // The pad once this pump owns it, from the first instance on. - let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; - let mut next: Option = None; - let mut listed = state.listed.subscribe(); + // Resolves once the publisher answers, with the track info or with an error (which is + // what ending a broadcast produces for a name nobody served). A publisher that answers + // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and + // holding the wait here rather than in `reconcile` keeps it off every other + // rendition. + let subscriber = tokio::select! { + _ = cancel.changed() => return None, + subscriber = async { + let broadcast = binding.broadcast().await?; + let track = broadcast.track(&media.name)?; + Ok::<_, moq_mux::Error>(track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))).await?) + } => match subscriber { + Ok(subscriber) => subscriber, + Err(err) => { + gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); + return Some((kind, name)); + } + } + }; + let mut track = moq_mux::container::Consumer::new(subscriber, container); + + // Winning this is what earns a pad. Losing means a teardown got here while the + // subscription was still resolving (this rendition was removed, reshaped, or outlived by + // a closing catalog), and it must not publish a pad at all: the watch alone can't say + // that, since a cancel landing just after we read it would leave a pad exposed and then + // yanked without an EOS. + if !state.go_live() { + return None; + } - loop { - // A delisted rendition drained the instance it read, and follows no other. - if owned.is_some() && !state.is_listed() { - return Some((kind, name)); + // A pad appears only once its track is live, so a pad downstream can link to is a promise + // that the rendition is actually flowing, and only a rendition that gets this far claims a + // pad id. A kept pad starts a new stream on it, with caps pushed even when they differ, so + // a format downstream can't take fails loudly as not-negotiated rather than going quiet. + let (pad, _owned) = match pads.claim(&element, (kind, name.clone()), &cancel)? { + Claim::New(pad, owned) => { + let obj = element.upgrade()?; + pad.set_active(true).ok()?; + begin(&pad, &name, &caps); + obj.add_pad(&pad).ok()?; + (pad, owned) } - let broadcast = match next.take() { - Some(broadcast) => broadcast, - None => tokio::select! { + Claim::Kept(pad, owner) => { + let owned = tokio::select! { + biased; _ = cancel.changed() => return None, - _ = listed.wait_for(|listed| !listed), if owned.is_some() => return Some((kind, name)), - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, - Ok(moq_mux::follow::Event::End) => continue, - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - }, - }; - - // An instance that already played this track to its end on the pad would replay its - // last group. A sibling waits for the next instance; the catalog's own has no other. - if finished.contains(&played, &broadcast) { - match sibling { - true => continue, - false => return None, + owned = owner.lock_owned() => owned, + }; + // The pump this one replaces records its track as finished before letting go, so a + // replacement started while it was reading the end (a reshape) can't replay it. + if finished + .lock() + .unwrap() + .contains(&((kind, name.clone()), media.clone())) + { + return Some((kind, name)); } + tokio::task::block_in_place(|| begin(&pad, &name, &caps)); + (pad, owned) } + }; + // Stop flushes only the pads it finds, and this one may have been added just after, while + // this pump's cancel is still on its way. Flushing it here keeps a push from blocking stop. + if *shutdown.borrow() { + let _ = pad.set_active(false); + } - // Resolves once the publisher answers, with the track info or with an error (which is - // what ending a broadcast produces for a name nobody served). A publisher that answers - // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and - // holding the wait here rather than in `reconcile` keeps it off every other - // rendition. - let subscribed = tokio::select! { - _ = cancel.changed() => return None, - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => Err(Stopped::Restart(broadcast)), - Ok(moq_mux::follow::Event::End) => Err(Stopped::Idle), - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - subscriber = async { - broadcast - .track(&media.name)? - .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) - .await - } => match subscriber { - Ok(subscriber) => Ok(subscriber), - // The instance went while subscribing: that is its end, not this rendition's. - Err(_) if sibling && broadcast.is_closed() => Err(Stopped::Idle), - Err(err) => { - gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); - return Some((kind, name)); - } - } - }; - let subscriber = match subscribed { - Ok(subscriber) => subscriber, - Err(stopped) => { - next = stopped.restart(); - continue; - } - }; - let container = match container.take() { - Some(container) => container, - None => match moq_mux::catalog::hang::Container::new( - &descriptor, - match kind { - TrackKind::Video => moq_mux::container::Kind::Video, - TrackKind::Audio => moq_mux::container::Kind::Audio, - }, - ) { - Ok(container) => container, - Err(err) => { - gst::warning!(CAT, "rendition {name} has an unsupported container: {err:?}"); - return Some((kind, name)); + let mut sent_segment = false; + loop { + tokio::select! { + // This rendition is being torn down (its run ended, or a catalog update replaced it). An + // end that arrived alongside still finishes the track, or the replacement waiting for + // this pad would replay it. + _ = cancel.changed() => { + if ended(&mut track) { + finished.lock().unwrap().insert(((kind, name.clone()), media)); } - }, - }; - let mut track = moq_mux::container::Consumer::new(subscriber, container); - let timeline = match sibling { - true => siblings.timeline(&broadcast, &run), - false => run.clone(), - }; - - let pad = match &owned { - // Another instance on the pad this pump owns: a new stream. - Some((pad, _)) => { - tokio::task::block_in_place(|| begin(pad, &name, &caps)); - pad.clone() + return None; } - None => { - // Winning this is what earns a pad. Losing means a teardown got here while the - // subscription was still resolving (this rendition was removed, reshaped, or - // outlived by a closing catalog), and it must not publish a pad at all: the - // watch alone can't say that, since a cancel landing just after we read it - // would leave a pad exposed and then yanked without an EOS. - if !state.go_live() { - return None; - } - - // A pad appears only once its track is live, so a pad downstream can link to is a - // promise that the rendition is actually flowing, and only a rendition that gets - // this far claims a pad id. A kept pad starts a new stream on it, with caps pushed - // even when they differ, so a format downstream can't take fails loudly as - // not-negotiated rather than going quiet. - let (pad, guard) = match pads.claim(&element, (kind, name.clone()), &cancel)? { - Claim::New(pad, owned) => { - let obj = element.upgrade()?; - pad.set_active(true).ok()?; - begin(&pad, &name, &caps); - obj.add_pad(&pad).ok()?; - (pad, owned) - } - Claim::Kept(pad, owner) => { - let guard = tokio::select! { - biased; - _ = cancel.changed() => return None, - guard = owner.lock_owned() => guard, - }; - // The pump this one replaces records its track as finished before letting - // go, so a replacement started while it was reading the end (a reshape) - // can't replay it. A sibling holds the pad for its next instance. - if finished.contains(&played, &broadcast) { - if !sibling { - return Some((kind, name)); + frame = track.read() => match frame { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + let buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); } - owned = Some((pad, guard)); - continue; + return Some((kind, name)); } - tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, guard) - } - }; - // Stop flushes only the pads it finds, and this one may have been added just - // after, while this pump's cancel is still on its way. Flushing it here keeps a - // push from blocking stop. - if *shutdown.borrow() { - let _ = pad.set_active(false); - } - owned = Some((pad.clone(), guard)); - pad - } - }; - - let mut sent_segment = false; - let stopped = loop { - tokio::select! { - // This rendition is being torn down (its run ended, or a catalog update replaced - // it). An end that arrived alongside still finishes the track, or the replacement - // waiting for this pad would replay it. - _ = cancel.changed() => { - if ended(&mut track) { - finished.insert(&played, &broadcast); - } - return None; - } - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => break Stopped::Restart(broadcast), - Ok(moq_mux::follow::Event::End) => break Stopped::Idle, - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return Some((kind, name)); - } - }; - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other pumps. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); - } - return Some((kind, name)); + }; + let segment = (!sent_segment).then(|| { + let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other pumps. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); } - Err(_) => return Some((kind, name)), + return Some((kind, name)); } + Err(_) => return Some((kind, name)), } - // A sibling's track stopping with its instance is that instance's end, which - // idles the pad for the next one rather than ending the rendition. - Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, - // A listed sibling's track finishing ends it for this instance only: the pad - // idles for the next instance, and the record keeps a reshape on this one - // from replaying it. - Ok(None) if sibling && state.is_listed() => { - finished.insert(&played, &broadcast); - break Stopped::Idle; - } - Ok(None) => { - // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.insert(&played, &broadcast); - return Some((kind, name)); - } - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - return Some((kind, name)); - } + } + Ok(None) => { + // Recorded while this pump still owns the pad, before any replacement can claim it. + finished.lock().unwrap().insert(((kind, name.clone()), media)); + return Some((kind, name)); + } + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + return Some((kind, name)); } } - }; - match &stopped { - Stopped::Restart(_) => gst::info!(CAT, "rendition {name} restarts on a new instance"), - Stopped::Idle => gst::info!(CAT, "rendition {name} is offline, holding its pad"), } - next = stopped.restart(); - } - } -} - -impl Stopped { - /// The instance to restart on, if any. - fn restart(self) -> Option { - match self { - Stopped::Restart(broadcast) => Some(broadcast), - Stopped::Idle => None, } } } @@ -1700,10 +1470,9 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{PumpState, Siblings, Timeline, ended, plan_reconcile}; + use super::{PumpState, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; - use std::sync::Arc; /// A pump cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -1835,33 +1604,6 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } - /// Renditions on the same sibling instance share its timeline, whenever each starts on it: - /// the first instance shares the run's, and the next one starts another. - #[tokio::test(start_paused = true)] - async fn renditions_on_one_sibling_instance_share_its_timeline() { - let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); - tokio::spawn(moq_net::time::run(driver)); - let consumer = origin.consume(); - let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { - moq_mux::follow::Event::Start(broadcast) => broadcast, - moq_mux::follow::Event::End => panic!("expected a start"), - }; - let (siblings, run) = (Siblings::default(), Arc::::default()); - - let _first = origin.publish("source", Default::default()).unwrap(); - let first = started(&mut audio).await; - assert!(Arc::ptr_eq(&run, &siblings.timeline(&first, &run))); - - // A video rendition that first starts on the replacement, as one still subscribing when it - // took over does, shares its timeline with the audio rendition restarting onto it. - let _second = origin.publish("source", Default::default()).unwrap(); - let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let next = siblings.timeline(&started(&mut video).await, &run); - assert!(!Arc::ptr_eq(&run, &next), "another instance has its own timeline"); - assert!(Arc::ptr_eq(&next, &siblings.timeline(&started(&mut audio).await, &run))); - } - #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -2966,13 +2708,6 @@ mod session_tests { publisher } - /// The `cam` broadcast a rendition of `room` references as `./cam`, under `route`. - fn sibling(origin: &moq_net::origin::Producer, route: moq_net::origin::Route) -> Self { - let publisher = Self::build(origin.create_broadcast("cam").unwrap(), video_rendition()); - publisher.broadcast.announce(route).unwrap(); - publisher - } - /// A broadcast at no path, for a covering route to serve. fn standalone(config: VideoConfig) -> Self { Self::build(moq_net::broadcast::Info::new().produce(), config) @@ -3122,20 +2857,15 @@ mod session_tests { kind: super::TrackKind::Video, name: "video".into(), caps: gst::Caps::new_empty_simple("video/x-h264"), - follow: moq_mux::Source::new(origin().consume(), publisher.broadcast.consume()) - .follow(None, None) - .unwrap(), - sibling: false, + binding: moq_mux::Binding::new(publisher.broadcast.consume()), track: super::Track { broadcast: None, name: "video".into(), }, container: moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), - siblings: super::Siblings::default(), - finished: super::Finished::default(), + finished: std::sync::Arc::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, shutdown, @@ -3387,301 +3117,6 @@ mod session_tests { recorder.assert_one_pad_held(); } - /// A `room` whose video rendition references the `cam` sibling, naming `epoch` when given. - fn referencing(origin: &moq_net::origin::Producer, epoch: Option) -> Publisher { - let mut config = video_rendition(); - config.broadcast = Some("./cam".into()); - config.epoch = epoch; - Publisher::new(origin, Default::default(), config) - } - - /// A sibling announced after the catalog starts its rendition, its end holds the pad, and its - /// return starts a new stream on that pad. - #[test] - fn a_late_sibling_starts_its_rendition_and_returns_after_an_end() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let _room = referencing(&origin, None); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - let mut follow = origin.consume().follow("cam").unwrap(); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - drop(first); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); - - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); - stop(session); - - assert!( - recorder.seen().contains(&Seen::Start(pad)), - "no new stream on the kept pad" - ); - recorder.assert_one_pad_held(); - } - - /// A sibling track finishing while its instance stays up idles the pad, and the instance that - /// replaces it starts a new stream there. - #[test] - fn a_finished_sibling_track_returns_on_the_next_instance() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let _room = referencing(&origin, None); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!( - recorder.written(&mut second, 2), - pad, - "the next instance moved to another pad" - ); - stop(session); - - assert!(!recorder.seen().contains(&Seen::Eos(pad)), "the finish ended the pad"); - recorder.assert_one_pad_held(); - drop(first); - } - - /// Block until `publisher`'s video track reads to its end, as a pump reading it does. - fn await_finished(publisher: &Publisher) { - let broadcast = publisher.broadcast.consume(); - super::RUNTIME.block_on(async { - let finished = async { - let subscriber = broadcast - .track("video") - .unwrap() - .subscribe(moq_net::track::Subscription::default()) - .await - .unwrap(); - let mut track = moq_mux::container::Consumer::new( - subscriber, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - ); - while track.read().await.unwrap().is_some() {} - }; - tokio::time::timeout(Duration::from_secs(10), finished) - .await - .expect("the track never finished"); - }); - } - - /// A sibling track that finished stays finished for its instance only: a catalog update that - /// replaces the idle pump, here a disable and an enable, still plays the next instance. - #[test] - fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let mut room = referencing(&origin, None); - let _audio = room - .broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - await_finished(&first); - - // The catalog consumer only yields the newest snapshot, so the same update lists an audio - // rendition: its pad appearing proves the run acted on the disable. - { - let mut guard = room.catalog.modify().unwrap(); - guard.video.renditions.get_mut("video").unwrap().enabled = false; - guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); - } - await_pad(&element, "audio_"); - room.catalog - .modify() - .unwrap() - .video - .renditions - .get_mut("video") - .unwrap() - .enabled = true; - - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!( - recorder.written(&mut second, 2), - pad, - "the next instance moved to another pad" - ); - stop(session); - - recorder.assert_one_pad_held(); - drop(first); - } - - /// A publisher handoff after a sibling track finished, the catalog naming the new instance's - /// epoch, plays that instance on the same pad. - #[test] - fn an_epoch_handoff_after_a_sibling_finish_plays_the_new_instance() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let (old, new) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); - let mut room = referencing(&origin, Some(old.clone())); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(old)); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - await_finished(&first); - - let mut second = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(new.clone())); - room.catalog - .modify() - .unwrap() - .video - .renditions - .get_mut("video") - .unwrap() - .epoch = Some(new); - assert_eq!( - recorder.written(&mut second, 2), - pad, - "the new instance moved to another pad" - ); - stop(session); - - recorder.assert_one_pad_held(); - drop(first); - } - - /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next - /// instance plays nothing on it. - #[test] - fn a_delisted_sibling_rendition_follows_no_later_instance() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let mut room = referencing(&origin, None); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - let mut follow = origin.consume().follow("cam").unwrap(); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - drop(first); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); - - room.catalog.modify().unwrap().video.renditions.clear(); - for _ in 0..200 { - if recorder.seen().contains(&Seen::Eos(pad.clone())) { - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - assert!( - recorder.seen().contains(&Seen::Eos(pad.clone())), - "the idle pad never ended" - ); - - let mut second = Publisher::sibling(&origin, Default::default()); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - for _ in 0..10 { - second.write(2); - std::thread::sleep(Duration::from_millis(50)); - } - stop(session); - - let seen = recorder.seen(); - assert!(!seen.iter().any(|seen| matches!(seen, Seen::Buffer(_, 2))), "{seen:?}"); - } - - /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on - /// the same pad, and nothing more from the old instance gets through. - fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let _room = referencing(&origin, None); - let mut old = Publisher::sibling(&origin, route()); - let session = follow(&element, &origin); - let pad = recorder.written(&mut old, 1); - - let mut new = Publisher::sibling(&origin, route()); - assert_eq!(recorder.written(&mut new, 2), pad, "the restart moved to another pad"); - old.write(1); - recorder.written(&mut new, 3); - stop(session); - - let seen = recorder.seen(); - let switched = seen - .iter() - .position(|seen| *seen == Seen::Buffer(pad.clone(), 2)) - .unwrap(); - assert!( - !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), - "the replaced sibling kept playing: {seen:?}" - ); - recorder.assert_one_pad_held(); - } - - #[test] - fn a_sibling_restart_switches_on_the_same_pad() { - sibling_restart(epoch); - } - - #[test] - fn an_epochless_sibling_restart_switches_on_the_same_pad() { - sibling_restart(moq_net::origin::Route::default); - } - - /// A reference naming its sibling's epoch never reads the instance replacing it: the pad - /// idles, and plays again once the named instance serves the path again. - #[test] - fn a_sibling_restart_idles_a_reference_naming_the_old_epoch() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let named = moq_net::Epoch::mint(); - let _room = referencing(&origin, Some(named.clone())); - let mut old = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(named)); - let session = follow(&element, &origin); - let pad = recorder.written(&mut old, 1); - - let mut follow = origin.consume().follow("cam").unwrap(); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - let mut new = Publisher::sibling(&origin, epoch()); - await_event(&mut follow, |event| { - matches!(event, moq_net::announce::Event::Restart(_)) - }); - new.write(2); - drop(new); - await_event(&mut follow, |event| { - matches!(event, moq_net::announce::Event::Restart(_)) - }); - - assert_eq!( - recorder.written(&mut old, 3), - pad, - "the named instance moved to another pad" - ); - stop(session); - assert!( - !recorder.seen().contains(&Seen::Buffer(pad, 2)), - "another epoch played: {:?}", - recorder.seen() - ); - recorder.assert_one_pad_held(); - } - /// Re-pricing the same instance is an update, which the run rides out without a new stream. #[test] fn an_update_keeps_the_run() { From 286f5493bea0c9aff6ba5565c256055b80ef70e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 14:33:15 -0700 Subject: [PATCH 11/13] fix(mux): a sibling's decode error fails the export even once it closes ExportSource treated any read error as the instance's end once the sibling closed, so malformed media followed by a close exported as a clean EOF. Only a transport error ends the rendition with its instance. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/source.rs | 49 ++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 27795ce2797..606e7c9727c 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -433,8 +433,10 @@ impl ExportSource { }; let item = match consumer.poll_item(waiter) { Poll::Ready(Ok(item)) => item, - // The field, not `instance_ended`, so the borrow stays off the consumer. - Poll::Ready(Err(_)) | Poll::Pending + // Only the transport ending with the instance is its end: media that fails to + // decode still fails the export. The field, not `instance_ended`, so the borrow + // stays off the consumer. + Poll::Ready(Err(crate::Error::Moq(_))) | Poll::Pending if self .instance .as_ref() @@ -775,6 +777,49 @@ mod tests { drop((track, replacement, new)); } + /// Media that fails to decode still fails the export when its sibling instance goes: only + /// the transport ending with the instance is the rendition's end. + #[tokio::test(start_paused = true)] + async fn a_sibling_decode_error_outlives_its_instance() { + use std::time::Duration; + + let origin = crate::source::produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + tokio::task::yield_now().await; + let source = crate::source::resolved(&origin, "a/pub").await; + let config = audio(Some("./source")); + let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + + let sibling = origin.publish("a/source", Default::default()).unwrap(); + let track = sibling + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + // Subscribe before anything is written, so the malformed frame and the instance's end + // arrive together. + let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); + assert!(settle.await.is_err(), "nothing is written yet"); + // An empty Legacy audio frame is malformed. + let mut group = track.append_group().unwrap(); + group + .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) + .unwrap(); + drop(sibling); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); + + let read = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) + .await + .expect("the read settles"); + assert!( + read.is_err(), + "{:?}", + read.map(|frame| frame.map(|frame| frame.timestamp)) + ); + drop((group, track)); + } + /// The requested budget must be present on the first SUBSCRIBE, a live start's too. Updating /// it after SUBSCRIBE_OK cannot recover backlog the publisher already skipped, nor the groups a /// zero budget drops as superseded before the update lands. From a1dd7fce25bd24bd5a0d79777e4bd0879f5aa021 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 15:14:56 -0700 Subject: [PATCH 12/13] fix(mux): only an orderly sibling shutdown ends its rendition A transport failure such as a protocol violation read after the sibling closes failed nothing: any moq-net error counted as the instance's end. Only an orderly shutdown (dropped, cancelled, closed, going away, or a cancelled session) ends the rendition; every other error fails the export. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/source.rs | 122 +++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 34 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 606e7c9727c..e8aa71b1c75 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -433,10 +433,18 @@ impl ExportSource { }; let item = match consumer.poll_item(waiter) { Poll::Ready(Ok(item)) => item, - // Only the transport ending with the instance is its end: media that fails to - // decode still fails the export. The field, not `instance_ended`, so the borrow - // stays off the consumer. - Poll::Ready(Err(crate::Error::Moq(_))) | Poll::Pending + // Only an orderly shutdown of the instance is its end: malformed media or a + // failed transport still fails the export. The field, not `instance_ended`, so + // the borrow stays off the consumer. + Poll::Ready(Err(crate::Error::Moq( + moq_net::Error::Dropped + | moq_net::Error::Cancel + | moq_net::Error::Closed + | moq_net::Error::GoingAway + | moq_net::Error::SessionClosed + | moq_net::Error::Session(moq_net::SessionError::Cancel), + ))) + | Poll::Pending if self .instance .as_ref() @@ -777,47 +785,93 @@ mod tests { drop((track, replacement, new)); } - /// Media that fails to decode still fails the export when its sibling instance goes: only - /// the transport ending with the instance is the rendition's end. + /// A sibling's failure still fails the export when its instance goes, whether its media is + /// malformed or its track is aborted with a protocol error: only an orderly shutdown of the + /// instance is the rendition's end. #[tokio::test(start_paused = true)] - async fn a_sibling_decode_error_outlives_its_instance() { + async fn a_sibling_failure_outlives_its_instance() { use std::time::Duration; + for malformed in [true, false] { + let origin = crate::source::produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + tokio::task::yield_now().await; + let source = crate::source::resolved(&origin, "a/pub").await; + let config = audio(Some("./source")); + let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + + let sibling = origin.publish("a/source", Default::default()).unwrap(); + let track = sibling + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + // Subscribe before anything is written, so the failure and the instance's end arrive + // together. + let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); + assert!(settle.await.is_err(), "nothing is written yet"); + let mut group = track.append_group().unwrap(); + if malformed { + // An empty Legacy audio frame is malformed. + group + .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) + .unwrap(); + } else { + track.clone().abort(moq_net::Error::ProtocolViolation).unwrap(); + } + drop(sibling); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); + + let read = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) + .await + .expect("the read settles"); + assert!( + read.is_err(), + "malformed: {malformed}: {:?}", + read.map(|frame| frame.map(|frame| frame.timestamp)) + ); + drop((group, track)); + } + } + + /// A sibling served over a session ends the rendition cleanly when its publisher goes, which + /// the session reports as its own cancellation. + #[tokio::test(start_paused = true)] + async fn a_remote_sibling_ends_with_its_instance() { + use std::time::Duration; + + use crate::container::test_util::{Link, raw_frame}; + + let remote = crate::source::produce_origin(); let origin = crate::source::produce_origin(); let _catalog = origin.publish("a/pub", Default::default()).unwrap(); tokio::task::yield_now().await; let source = crate::source::resolved(&origin, "a/pub").await; - let config = audio(Some("./source")); - let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + let _link = Link::new(&remote, &origin).await; + let sibling = remote.publish("a/source", Default::default()).unwrap(); + let mut track = crate::container::Producer::new( + sibling + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(), + HangContainer::Legacy(crate::container::Kind::Audio), + ); + track.write(raw_frame(0, b"x", true)).unwrap(); + let mut export = ExportSource::for_audio(&source, "audio", &audio(Some("./source")), Duration::from_secs(10)) .unwrap() .unwrap(); + let mut read = async || { + tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) + .await + .expect("the read settles") + .unwrap() + .map(|frame| frame.timestamp.as_micros()) + }; + assert_eq!(read().await, Some(0)); - let sibling = origin.publish("a/source", Default::default()).unwrap(); - let track = sibling - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - // Subscribe before anything is written, so the malformed frame and the instance's end - // arrive together. - let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); - assert!(settle.await.is_err(), "nothing is written yet"); - // An empty Legacy audio frame is malformed. - let mut group = track.append_group().unwrap(); - group - .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) - .unwrap(); drop(sibling); - tokio::time::sleep(Duration::from_millis(10)).await; - assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); - - let read = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) - .await - .expect("the read settles"); - assert!( - read.is_err(), - "{:?}", - read.map(|frame| frame.map(|frame| frame.timestamp)) - ); - drop((group, track)); + assert_eq!(read().await, None, "its end ends the rendition"); + drop(track); } /// The requested budget must be present on the first SUBSCRIBE, a live start's too. Updating From f6ab8e6f0987b9635e2268ea26eae6f6a8003c3f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 15:56:12 -0700 Subject: [PATCH 13/13] fix(mux): a sibling's unfinished track fails the export `Dropped` means a producer went away without finishing, so content is incomplete rather than ended. It no longer counts as an orderly end of the sibling instance. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/source.rs | 38 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index e8aa71b1c75..eb6d0f3d274 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -437,8 +437,7 @@ impl ExportSource { // failed transport still fails the export. The field, not `instance_ended`, so // the borrow stays off the consumer. Poll::Ready(Err(crate::Error::Moq( - moq_net::Error::Dropped - | moq_net::Error::Cancel + moq_net::Error::Cancel | moq_net::Error::Closed | moq_net::Error::GoingAway | moq_net::Error::SessionClosed @@ -786,13 +785,13 @@ mod tests { } /// A sibling's failure still fails the export when its instance goes, whether its media is - /// malformed or its track is aborted with a protocol error: only an orderly shutdown of the - /// instance is the rendition's end. + /// malformed, its track is aborted with a protocol error, or its producers went away + /// unfinished: only an orderly shutdown of the instance is the rendition's end. #[tokio::test(start_paused = true)] async fn a_sibling_failure_outlives_its_instance() { use std::time::Duration; - for malformed in [true, false] { + for failure in ["malformed", "violation", "dropped"] { let origin = crate::source::produce_origin(); let _catalog = origin.publish("a/pub", Default::default()).unwrap(); tokio::task::yield_now().await; @@ -811,14 +810,23 @@ mod tests { let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); assert!(settle.await.is_err(), "nothing is written yet"); let mut group = track.append_group().unwrap(); - if malformed { - // An empty Legacy audio frame is malformed. - group - .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) - .unwrap(); - } else { - track.clone().abort(moq_net::Error::ProtocolViolation).unwrap(); - } + let held = match failure { + "malformed" => { + // An empty Legacy audio frame is malformed. + group + .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) + .unwrap(); + Some((group, track)) + } + "violation" => { + track.clone().abort(moq_net::Error::ProtocolViolation).unwrap(); + Some((group, track)) + } + _ => { + drop((group, track)); + None + } + }; drop(sibling); tokio::time::sleep(Duration::from_millis(10)).await; assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); @@ -828,10 +836,10 @@ mod tests { .expect("the read settles"); assert!( read.is_err(), - "malformed: {malformed}: {:?}", + "{failure}: {:?}", read.map(|frame| frame.map(|frame| frame.timestamp)) ); - drop((group, track)); + drop(held); } }