From 9e8a3d29bbcfafb7a1ad0926284a4d648b6f01ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 20:50:26 -0700 Subject: [PATCH 01/27] quest: claim reference-announcements Co-Authored-By: Claude Opus 5.5 From 380b0386ecca2ee889eeb220409cec3e04455e6d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:18:57 -0700 Subject: [PATCH 02/27] feat(mux): follow referenced broadcasts' announcements Add moq_mux::Follow (Source::follow, Follow::new), built on announced_broadcast: each instance serving a sibling is handed out as it starts or takes over, and its end is reported. A reference naming an epoch follows only its own instance (best effort over epochless routes). moqsrc pumps follow their sibling: the first start binds the rendition, an end idles its pad, and another instance restarts the stream on the same pad. Exporters bind a sibling once it is first announced instead of failing, and end the rendition when that instance goes. @moq/watch drops a rendition whose broadcast proves another epoch from its effective catalog. Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 2 +- doc/concept/hang.md | 2 +- js/watch/src/broadcast.test.ts | 31 ++ js/watch/src/broadcast.ts | 35 ++- rs/moq-gst/src/source/imp.rs | 481 +++++++++++++++++++++-------- rs/moq-mux/src/container/source.rs | 115 ++++++- rs/moq-mux/src/lib.rs | 2 +- rs/moq-mux/src/source.rs | 348 +++++++++++++++++++-- 8 files changed, 844 insertions(+), 172 deletions(-) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 5b8c727fd14..2230e4a8256 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -124,7 +124,7 @@ rewound timestamps forward. Pads are named by kind and appear as the catalog announces renditions: `video_0`, `video_1`, `audio_0`. A rendition referencing another broadcast reads -that broadcast on the same connection. Link the pad you want by name; the terse +that broadcast on the same connection and follows its announcements the same way. Link the pad you want by name; the terse `moqsrc ! decodebin3` form links only the first pad offered, which may be audio. `moqsrc` follows the broadcast's announcements: when the publisher restarts, it switches to the new broadcast on the same pads, kept by rendition diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 35a5cab213d..4373b75c6dc 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. A consumer reads each referenced path from the first broadcast it resolves there. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance fails the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; an exporter starts on the first one announced, keeps it, and ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/js/watch/src/broadcast.test.ts b/js/watch/src/broadcast.test.ts index 317d0b279dc..b02fe01bd71 100644 --- a/js/watch/src/broadcast.test.ts +++ b/js/watch/src/broadcast.test.ts @@ -355,6 +355,37 @@ describe("relativeBroadcast", () => { } }); + it("drops a rendition from the catalog while its broadcast proves another epoch", async () => { + const owner = new Origin.Producer(); + const main = publish(owner, Path.from("a/b")); + const [oldEpoch, newEpoch] = [Moq.Epoch.mint(), Moq.Epoch.mint()]; + const old = owner.createBroadcast(Path.from("a/source")); + old.announce({ epoch: oldEpoch }); + const named = { ...video("avc1.64001f", "./source"), epoch: oldEpoch } as Catalog.VideoConfig; + const source = new Broadcast({ + origin: owner, + name: Path.from("a/b"), + enabled: true, + catalogFormat: "manual", + catalog: { video: { renditions: { plain: video("avc1.64001f", "./source"), named } } } as Catalog.Root, + }); + try { + await settle(); + expect(videoRenditions(source)).toEqual(["named", "plain"]); + + const replacement = owner.createBroadcast(Path.from("a/source")); + replacement.announce({ epoch: newEpoch }); + await withoutWarnings(settle); + expect(videoRenditions(source)).toEqual(["plain"]); + replacement.close(); + } finally { + source.close(); + old.close(); + main.close(); + owner.close(); + } + }); + it("uses the catalog's own broadcast when the reference is absent, empty, or self", async () => { const { source, owner } = broadcast("a/b"); const effect = new Effect(); diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index 7e76e00741f..62066c83821 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -9,16 +9,25 @@ import { toHang } from "./msf"; type ReferencedRendition = { broadcast?: Path.Relative; + epoch?: Catalog.ReferenceEpoch; }; +// Whether a rendition's reference names something this consumer can play. +type Usable = (rel: Path.Relative | undefined, epoch: Moq.Epoch.Valid | undefined) => boolean; + // Either the catalog's own broadcast, or a sibling to consume by path. type RelativeTarget = { local: true } | { local: false; path: Moq.Path.Valid }; -function filterRenditions( - renditions: Record, - usable: (rel: Path.Relative | undefined) => boolean, -): Record { - return Object.fromEntries(Object.entries(renditions).filter(([, config]) => usable(config.broadcast))); +function filterRenditions(renditions: Record, usable: Usable): Record { + return Object.fromEntries( + Object.entries(renditions).filter(([, config]) => usable(config.broadcast, config.epoch)), + ); +} + +// Whether `active` is proven to be another instance than `epoch` names. An epochless route cannot +// prove its instance, so only a known mismatch counts. +function provesOther(active: Moq.Broadcast.Consumer | undefined, epoch: Moq.Epoch.Valid | undefined): boolean { + return epoch !== undefined && active?.epoch !== undefined && active.epoch !== epoch; } // Whether an announced route covers `path`, until closed. Hidden routes count: a service claim @@ -47,7 +56,7 @@ function watchCovered(origin: Moq.Origin.Table, path: Moq.Path.Valid): { covered // Every section carrying a `broadcast` reference must be listed here, same as `Catalog.checkResolvable`; // one left out silently exempts its tracks from the reachability filter. -function filterCatalog(catalog: Catalog.Root, usable: (rel: Path.Relative | undefined) => boolean): Catalog.Root { +function filterCatalog(catalog: Catalog.Root, usable: Usable): Catalog.Root { return { ...catalog, video: catalog.video @@ -182,12 +191,16 @@ export class Broadcast { } // Publish the catalog minus every rendition this consumer cannot use: one naming a - // broadcast that isn't announced to us. Selecting one of those would render nothing, - // since `relativeBroadcast` resolves it to no broadcast at all. Reruns as announcements - // arrive, so a rendition appears once its broadcast does. + // broadcast that isn't announced to us, or one whose broadcast proves another epoch than it + // names. Selecting one of those would render nothing, since `relativeBroadcast` resolves it to + // no broadcast at all. Reruns as announcements arrive, so a rendition appears once its + // broadcast does. Only a rendition naming an epoch resolves its broadcast here, sharing the + // request playback uses. #runFiltered(effect: Effect): void { const raw = effect.get(this.#raw); - const usable = (rel: Path.Relative | undefined) => this.#relativeTarget(effect, rel) !== undefined; + const usable: Usable = (rel, epoch) => + this.#relativeTarget(effect, rel) !== undefined && + (epoch === undefined || !provesOther(this.#relativeActive(effect, rel), epoch)); effect.set(this.#out.catalog, raw ? filterCatalog(raw, usable) : undefined); } @@ -434,7 +447,7 @@ export class Broadcast { epoch?: Moq.Epoch.Valid, ): Moq.Broadcast.Consumer | undefined { const active = this.#relativeActive(effect, rel); - if (!active || epoch === undefined || active.epoch === undefined || active.epoch === epoch) return active; + if (!active || !provesOther(active, epoch)) return active; console.warn("ignoring rendition: its broadcast serves another epoch", active.path, active.epoch, epoch); return undefined; } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7dae3e7c2a1..546c24e52ec 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -988,15 +988,16 @@ fn reconcile( } }; - // Bind without waiting: an unavailable sibling or track must not park the catalog loop + // Follow without waiting: an unavailable sibling or track must not park the catalog loop // and hold up every other rendition. The pump resolves both under its cancellation scope. - let binding = match source.bind(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { - Ok(binding) => binding, + let follow = match source.follow(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { + Ok(follow) => follow, Err(err) => { gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); continue; } }; + let sibling = track.broadcast != source.resolve_reference(None); let (cancel_tx, cancel_rx) = watch::channel(false); let state = Arc::new(PumpState::new()); @@ -1008,9 +1009,11 @@ fn reconcile( kind: d.kind, name: name.clone(), caps: d.shape.caps.clone(), - binding, + follow, + sibling, track, container, + descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), finished: playback.finished.clone(), @@ -1102,9 +1105,16 @@ struct Pump { /// per-kind counter instead, and isn't known until the subscription resolves. name: String, caps: gst::Caps, - binding: moq_mux::Binding, + /// Each instance of the broadcast the rendition references. + follow: moq_mux::Follow, + /// The rendition references another broadcast than its catalog's, whose instances come and + /// go on their own announcements. + sibling: bool, track: Track, + /// The wire format of the first instance. container: moq_mux::catalog::hang::Container, + /// What builds it again for a later instance. + descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, /// The run's [`Playback::finished`]. @@ -1116,155 +1126,266 @@ struct Pump { shutdown: watch::Receiver, } +/// Why a pump stopped reading one instance without ending. +enum Stopped { + /// Another instance took over: restart on it. + Restart(moq_net::broadcast::Consumer), + /// The instance stopped serving: idle until the next one. + Idle, +} + impl Pump { /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the /// track ends, errors, or `cancel` fires. The pad goes back to the session without an EOS /// either way. Returns the rendition if its track ended rather than being cancelled, so the /// run can end the pad of one the catalog retired. + /// + /// A sibling's announcements drive the pump: its first start binds the rendition, an end + /// idles the pad, and another instance taking over restarts the stream on the same pad. async fn run(self) -> Option { let Pump { element, kind, name, caps, - binding, + mut follow, + sibling, track: media, container, + descriptor, pads, - timeline, + mut timeline, finished, state, mut cancel, shutdown, } = self; - // Resolves once the publisher answers, with the track info or with an error (which is - // what ending a broadcast produces for a name nobody served). A publisher that answers - // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and - // holding the wait here rather than in `reconcile` keeps it off every other - // rendition. - let subscriber = tokio::select! { - _ = cancel.changed() => return None, - subscriber = async { - let broadcast = binding.broadcast().await?; - let track = broadcast.track(&media.name)?; - Ok::<_, moq_mux::Error>(track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))).await?) - } => match subscriber { - Ok(subscriber) => subscriber, - Err(err) => { - gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); - return Some((kind, name)); - } - } - }; - let mut track = moq_mux::container::Consumer::new(subscriber, container); - - // Winning this is what earns a pad. Losing means a teardown got here while the - // subscription was still resolving (this rendition was removed, reshaped, or outlived by - // a closing catalog), and it must not publish a pad at all: the watch alone can't say - // that, since a cancel landing just after we read it would leave a pad exposed and then - // yanked without an EOS. - if !state.go_live() { - return None; - } + let mut container = Some(container); + // The pad once this pump owns it, from the first instance on. + let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; + let mut next: Option = None; - // A pad appears only once its track is live, so a pad downstream can link to is a promise - // that the rendition is actually flowing, and only a rendition that gets this far claims a - // pad id. A kept pad starts a new stream on it, with caps pushed even when they differ, so - // a format downstream can't take fails loudly as not-negotiated rather than going quiet. - let (pad, _owned) = match pads.claim(&element, (kind, name.clone()), &cancel)? { - Claim::New(pad, owned) => { - let obj = element.upgrade()?; - pad.set_active(true).ok()?; - begin(&pad, &name, &caps); - obj.add_pad(&pad).ok()?; - (pad, owned) - } - Claim::Kept(pad, owner) => { - let owned = tokio::select! { - biased; + loop { + let broadcast = match next.take() { + Some(broadcast) => broadcast, + None => tokio::select! { _ = cancel.changed() => return None, - owned = owner.lock_owned() => owned, - }; - // The pump this one replaces records its track as finished before letting go, so a - // replacement started while it was reading the end (a reshape) can't replay it. - if finished - .lock() - .unwrap() - .contains(&((kind, name.clone()), media.clone())) - { - return Some((kind, name)); - } - tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, owned) - } - }; - // Stop flushes only the pads it finds, and this one may have been added just after, while - // this pump's cancel is still on its way. Flushing it here keeps a push from blocking stop. - if *shutdown.borrow() { - let _ = pad.set_active(false); - } + followed = follow.next() => match followed { + Ok(moq_mux::Followed::Start(broadcast)) => broadcast, + Ok(moq_mux::Followed::End) => continue, + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + }, + }; - let mut sent_segment = false; - loop { - tokio::select! { - // This rendition is being torn down (its run ended, or a catalog update replaced it). An - // end that arrived alongside still finishes the track, or the replacement waiting for - // this pad would replay it. - _ = cancel.changed() => { - if ended(&mut track) { - finished.lock().unwrap().insert(((kind, name.clone()), media)); + // Resolves once the publisher answers, with the track info or with an error (which is + // what ending a broadcast produces for a name nobody served). A publisher that answers + // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and + // holding the wait here rather than in `reconcile` keeps it off every other + // rendition. + let subscribed = tokio::select! { + _ = cancel.changed() => return None, + followed = follow.next() => match followed { + Ok(moq_mux::Followed::Start(broadcast)) => Err(Stopped::Restart(broadcast)), + Ok(moq_mux::Followed::End) => Err(Stopped::Idle), + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + subscriber = async { + broadcast + .track(&media.name)? + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) + .await + } => match subscriber { + Ok(subscriber) => Ok(subscriber), + // The instance went while subscribing: that is its end, not this rendition's. + Err(_) if sibling && broadcast.is_closed() => Err(Stopped::Idle), + Err(err) => { + gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); + return Some((kind, name)); } - return None; } - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return Some((kind, name)); - } - }; - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other pumps. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); - } + }; + let subscriber = match subscribed { + Ok(subscriber) => subscriber, + Err(stopped) => { + next = stopped.restart(); + continue; + } + }; + let container = match container.take() { + Some(container) => container, + None => match moq_mux::catalog::hang::Container::new( + &descriptor, + match kind { + TrackKind::Video => moq_mux::container::Kind::Video, + TrackKind::Audio => moq_mux::container::Kind::Audio, + }, + ) { + Ok(container) => container, + Err(err) => { + gst::warning!(CAT, "rendition {name} has an unsupported container: {err:?}"); + return Some((kind, name)); + } + }, + }; + let mut track = moq_mux::container::Consumer::new(subscriber, container); + + let pad = match &owned { + // Another instance on the pad this pump owns: a new stream, on a timeline of its own, + // since the instance's timestamps share no origin with the run's. + Some((pad, _)) => { + timeline = Arc::default(); + tokio::task::block_in_place(|| begin(pad, &name, &caps)); + pad.clone() + } + None => { + // Winning this is what earns a pad. Losing means a teardown got here while the + // subscription was still resolving (this rendition was removed, reshaped, or + // outlived by a closing catalog), and it must not publish a pad at all: the + // watch alone can't say that, since a cancel landing just after we read it + // would leave a pad exposed and then yanked without an EOS. + if !state.go_live() { + return None; + } + + // A pad appears only once its track is live, so a pad downstream can link to is a + // promise that the rendition is actually flowing, and only a rendition that gets + // this far claims a pad id. A kept pad starts a new stream on it, with caps pushed + // even when they differ, so a format downstream can't take fails loudly as + // not-negotiated rather than going quiet. + let (pad, guard) = match pads.claim(&element, (kind, name.clone()), &cancel)? { + Claim::New(pad, owned) => { + let obj = element.upgrade()?; + pad.set_active(true).ok()?; + begin(&pad, &name, &caps); + obj.add_pad(&pad).ok()?; + (pad, owned) + } + Claim::Kept(pad, owner) => { + let owned = tokio::select! { + biased; + _ = cancel.changed() => return None, + owned = owner.lock_owned() => owned, + }; + // The pump this one replaces records its track as finished before letting + // go, so a replacement started while it was reading the end (a reshape) + // can't replay it. + if finished + .lock() + .unwrap() + .contains(&((kind, name.clone()), media.clone())) + { return Some((kind, name)); } - Err(_) => return Some((kind, name)), + tokio::task::block_in_place(|| begin(&pad, &name, &caps)); + (pad, owned) } + }; + // Stop flushes only the pads it finds, and this one may have been added just + // after, while this pump's cancel is still on its way. Flushing it here keeps a + // push from blocking stop. + if *shutdown.borrow() { + let _ = pad.set_active(false); } - Ok(None) => { - // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.lock().unwrap().insert(((kind, name.clone()), media)); - return Some((kind, name)); + owned = Some((pad.clone(), guard)); + pad + } + }; + + let mut sent_segment = false; + let stopped = loop { + tokio::select! { + // This rendition is being torn down (its run ended, or a catalog update replaced + // it). An end that arrived alongside still finishes the track, or the replacement + // waiting for this pad would replay it. + _ = cancel.changed() => { + if ended(&mut track) { + finished.lock().unwrap().insert(((kind, name.clone()), media)); + } + return None; } - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - return Some((kind, name)); + followed = follow.next() => match followed { + Ok(moq_mux::Followed::Start(broadcast)) => break Stopped::Restart(broadcast), + Ok(moq_mux::Followed::End) => break Stopped::Idle, + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + frame = track.read() => match frame { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + let buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); + } + return Some((kind, name)); + } + }; + let segment = (!sent_segment).then(|| { + let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other pumps. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); + } + return Some((kind, name)); + } + Err(_) => return Some((kind, name)), + } + } + // A sibling's track stopping with its instance is that instance's end, which + // idles the pad for the next one rather than ending the rendition. + Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, + Ok(None) => { + // Recorded while this pump still owns the pad, before any replacement can claim it. + finished.lock().unwrap().insert(((kind, name.clone()), media)); + return Some((kind, name)); + } + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + return Some((kind, name)); + } } } + }; + match &stopped { + Stopped::Restart(_) => gst::info!(CAT, "rendition {name} restarts on a new instance"), + Stopped::Idle => gst::info!(CAT, "rendition {name} is offline, holding its pad"), } + next = stopped.restart(); + } + } +} + +impl Stopped { + /// The instance to restart on, if any. + fn restart(self) -> Option { + match self { + Stopped::Restart(broadcast) => Some(broadcast), + Stopped::Idle => None, } } } @@ -2584,6 +2705,13 @@ mod session_tests { publisher } + /// The `cam` broadcast a rendition of `room` references as `./cam`, under `route`. + fn sibling(origin: &moq_net::origin::Producer, route: moq_net::origin::Route) -> Self { + let publisher = Self::build(origin.create_broadcast("cam").unwrap(), video_rendition()); + publisher.broadcast.announce(route).unwrap(); + publisher + } + /// A broadcast at no path, for a covering route to serve. fn standalone(config: VideoConfig) -> Self { Self::build(moq_net::broadcast::Info::new().produce(), config) @@ -2733,12 +2861,16 @@ mod session_tests { kind: super::TrackKind::Video, name: "video".into(), caps: gst::Caps::new_empty_simple("video/x-h264"), - binding: moq_mux::Binding::new(publisher.broadcast.consume()), + follow: moq_mux::Source::new(origin().consume(), publisher.broadcast.consume()) + .follow(None, None) + .unwrap(), + sibling: false, track: super::Track { broadcast: None, name: "video".into(), }, container: moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), finished: std::sync::Arc::default(), @@ -2993,6 +3125,109 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A `room` whose video rendition references the `cam` sibling, naming `epoch` when given. + fn referencing(origin: &moq_net::origin::Producer, epoch: Option) -> Publisher { + let mut config = video_rendition(); + config.broadcast = Some("./cam".into()); + config.epoch = epoch; + Publisher::new(origin, Default::default(), config) + } + + /// A sibling announced after the catalog starts its rendition, its end holds the pad, and its + /// return starts a new stream on that pad. + #[test] + fn a_late_sibling_starts_its_rendition_and_returns_after_an_end() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); + stop(session); + + assert!(recorder.seen().contains(&Seen::Start(pad)), "no new stream on the kept pad"); + recorder.assert_one_pad_held(); + } + + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on + /// the same pad, and nothing more from the old instance gets through. + fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let mut old = Publisher::sibling(&origin, route()); + let session = follow(&element, &origin); + let pad = recorder.written(&mut old, 1); + + let mut new = Publisher::sibling(&origin, route()); + assert_eq!(recorder.written(&mut new, 2), pad, "the restart moved to another pad"); + old.write(1); + recorder.written(&mut new, 3); + stop(session); + + let seen = recorder.seen(); + let switched = seen.iter().position(|seen| *seen == Seen::Buffer(pad.clone(), 2)).unwrap(); + assert!( + !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), + "the replaced sibling kept playing: {seen:?}" + ); + recorder.assert_one_pad_held(); + } + + #[test] + fn a_sibling_restart_switches_on_the_same_pad() { + sibling_restart(epoch); + } + + #[test] + fn an_epochless_sibling_restart_switches_on_the_same_pad() { + sibling_restart(moq_net::origin::Route::default); + } + + /// A reference naming its sibling's epoch never reads the instance replacing it: the pad + /// idles, and plays again once the named instance serves the path again. + #[test] + fn a_sibling_restart_idles_a_reference_naming_the_old_epoch() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let named = moq_net::Epoch::mint(); + let _room = referencing(&origin, Some(named.clone())); + let mut old = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(named)); + let session = follow(&element, &origin); + let pad = recorder.written(&mut old, 1); + + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + let mut new = Publisher::sibling(&origin, epoch()); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + new.write(2); + drop(new); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + + assert_eq!(recorder.written(&mut old, 3), pad, "the named instance moved to another pad"); + stop(session); + assert!( + !recorder.seen().contains(&Seen::Buffer(pad, 2)), + "another epoch played: {:?}", + recorder.seen() + ); + recorder.assert_one_pad_held(); + } + /// Re-pricing the same instance is an update, which the run rides out without a new stream. #[test] fn an_update_keeps_the_run() { diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 0335dfd1d5e..5910aa3fd8a 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -62,6 +62,8 @@ enum SourceState { /// The resolved consumer, reading frames. Boxed because it's much larger than /// the `Subscribing` variant (clippy `large_enum_variant`). Active(Box>), + /// The sibling instance it read went, and its rendition with it. + Ended, } /// A per-rendition source that normalizes frame shape (Annex-B → @@ -97,6 +99,8 @@ pub(crate) struct ExportSource { ends: Option<(u64, u64)>, /// The counters the export source reports for this rendition, as received, before any transform. received: std::sync::Arc, + /// The sibling broadcast the rendition reads, once resolved: its end ends the rendition. + instance: Option, } impl ExportSource { @@ -184,6 +188,7 @@ impl ExportSource { cuts: 0, ends: None, received: source.received(name), + instance: None, }; source.resolve_video_dimensions(&[])?; Ok(Some(source)) @@ -222,6 +227,7 @@ impl ExportSource { cuts: 0, ends: None, received: source.received(name), + instance: None, })) } @@ -253,6 +259,7 @@ impl ExportSource { cuts: 0, ends: None, received: source.received(name), + instance: None, }) } @@ -364,19 +371,31 @@ impl ExportSource { /// Read normalized media or a clean group boundary. pub fn poll_event(&mut self, waiter: &kio::Waiter) -> Poll>> { + if matches!(self.state, SourceState::Ended) { + return Poll::Ready(Ok(None)); + } + // Resolve a cross-broadcast reference into a broadcast before subscribing. if matches!(self.state, SourceState::Requesting(..)) { - let (broadcast, name) = { + let (broadcast, name, sibling) = { let SourceState::Requesting(pending, name) = &self.state else { unreachable!("just matched Requesting"); }; - (ready!(pending.poll_broadcast(waiter))?, name.clone()) + (ready!(pending.poll_broadcast(waiter))?, name.clone(), pending.sibling()) }; + if sibling { + self.instance = Some(broadcast.clone()); + } let mut subscription = moq_net::track::Subscription::default().with_max_delay(self.max_delay); if let Some(start) = self.start { subscription = subscription.with_groups(start..); } - self.state = SourceState::Subscribing(broadcast.track(&name)?.subscribe(subscription)); + let track = match broadcast.track(&name) { + Ok(track) => track, + Err(_) if self.instance_ended(waiter) => return self.end(), + Err(err) => return Poll::Ready(Err(err.into())), + }; + self.state = SourceState::Subscribing(track.subscribe(subscription)); } // Resolve the subscription before reading any frames. @@ -386,7 +405,12 @@ impl ExportSource { let SourceState::Subscribing(pending) = &self.state else { unreachable!("just matched Subscribing"); }; - ready!(pending.poll_ok(waiter))? + match pending.poll_ok(waiter) { + Poll::Ready(Ok(track)) => track, + Poll::Ready(Err(_)) | Poll::Pending if self.instance_ended(waiter) => return self.end(), + Poll::Ready(Err(err)) => return Poll::Ready(Err(err.into())), + Poll::Pending => return Poll::Pending, + } }; let media = self .media @@ -407,7 +431,21 @@ impl ExportSource { let SourceState::Active(consumer) = &mut self.state else { unreachable!("subscription resolved into an Active consumer"); }; - match ready!(consumer.poll_item(waiter))? { + let item = match consumer.poll_item(waiter) { + Poll::Ready(Ok(item)) => item, + // The field, not `instance_ended`, so the borrow stays off the consumer. + Poll::Ready(Err(_)) | Poll::Pending + if self + .instance + .as_ref() + .is_some_and(|instance| instance.poll_closed(waiter).is_ready()) => + { + return self.end(); + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + }; + match item { Some(Item::Read(Read::Frame { group, frame })) => { if self.ends.is_some_and(|(end_group, _)| end_group != group) { self.finish_ends(); @@ -485,6 +523,23 @@ impl ExportSource { } } + /// Whether the sibling broadcast this rendition reads has stopped serving, registering + /// `waiter` for when it does. Its track stalling or failing then is that instance's end, + /// which ends the rendition rather than the export once what it delivered is read; following + /// the instance that replaces it is the export's own call. + fn instance_ended(&self, waiter: &kio::Waiter) -> bool { + self.instance + .as_ref() + .is_some_and(|instance| instance.poll_closed(waiter).is_ready()) + } + + /// End the rendition with its instance. + fn end(&mut self) -> Poll>> { + self.finish_ends(); + self.state = SourceState::Ended; + Poll::Ready(Ok(None)) + } + fn finish_ends(&mut self) { if let Some((_, count)) = self.ends.take() { self.skips += count; @@ -665,6 +720,56 @@ mod tests { } } + /// A sibling rendition's first announcement starts it, a replacement is never read, and its + /// instance going ends the rendition instead of failing the export, both mid-stream and + /// before the subscription resolves. + #[tokio::test(start_paused = true)] + async fn a_sibling_rendition_starts_late_and_ends_with_its_instance() { + use std::time::Duration; + + use crate::container::test_util::raw_frame; + + let origin = crate::source::produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + tokio::task::yield_now().await; + let source = crate::source::resolved(&origin, "a/pub").await; + let config = audio(Some("./source")); + let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + let mut read = async || { + tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))) + .await + .ok() + .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros())) + }; + let wire = || HangContainer::Legacy(crate::container::Kind::Audio); + let info = || hang::container::track_info(hang::catalog::PRIORITY.audio); + + assert_eq!(read().await, None, "nothing announces the sibling yet"); + let old = origin.publish("a/source", Default::default()).unwrap(); + let mut track = crate::container::Producer::new(old.create_track("audio", info()).unwrap(), wire()); + track.write(raw_frame(0, b"old", true)).unwrap(); + assert_eq!(read().await, Some(Some(0)), "the late announcement starts it"); + + let new = origin.publish("a/source", Default::default()).unwrap(); + let mut replacement = crate::container::Producer::new(new.create_track("audio", info()).unwrap(), wire()); + replacement.write(raw_frame(1_000_000, b"new", true)).unwrap(); + track.write(raw_frame(24_000, b"old", true)).unwrap(); + assert_eq!(read().await, Some(Some(24_000)), "the replacement is never read"); + + drop(old); + assert_eq!(read().await, Some(None), "its end ends the rendition"); + + // A rendition still subscribing when its instance goes ends too. + let mut late = ExportSource::for_audio(&source, "late", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + let ended = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| late.poll_read(waiter))).await; + assert!(matches!(ended, Ok(Ok(None))), "{:?}", ended.map(|read| read.map(|_| ()))); + drop((track, replacement, new)); + } + /// The requested budget must be present on the first SUBSCRIBE, a live start's too. Updating /// it after SUBSCRIBE_OK cannot recover backlog the publisher already skipped, nor the groups a /// zero budget drops as superseded before the update lands. diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 879e436fbec..4b46cb87dde 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -48,7 +48,7 @@ pub mod timeline; pub use clock::Clock; pub use error::*; pub use pace::Pacer; -pub use source::{Binding, Source}; +pub use source::{Binding, Follow, Followed, Source}; /// Translate a catalog entry's declared compression into the flag the codecs take. /// diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 565911b29bf..1594c95ff40 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -23,12 +23,16 @@ type Pins = Arc> /// reference. Build one with [`Source::new`]. /// /// A source serves its own path from the broadcast it was built from, and keeps the first -/// broadcast it resolves for each sibling. Every later request for a path is served from that -/// handle ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), -/// [`resolve`](Self::resolve), [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), -/// and an export's track requests). A route replaced after that is never resolved, epoch or -/// not, so two publishers never splice into one export; a request on a held broadcast that has -/// ended fails instead. Clones share the held broadcasts. +/// broadcast it resolves for each sibling; a [binding](Self::bind) waits for the sibling's first +/// announcement to resolve it. Every later request for a path is served from that handle +/// ([`broadcast`](Self::broadcast), +/// [`catalog`](Self::catalog), [`resolve`](Self::resolve), [`bind`](Self::bind), +/// [`subscribe_track`](Self::subscribe_track), and an export's track requests). A route replaced +/// after that is never resolved, epoch or not, so two publishers never splice into one export; a +/// request on a held broadcast that has ended fails instead. Clones share the held broadcasts. +/// +/// A reader that follows a sibling's announcements instead, restarting on each new instance, +/// uses [`follow`](Self::follow), which holds nothing. /// /// A reference may name the instance it expects with an epoch (the catalog's optional `epoch` /// beside `broadcast`). A held or resolved broadcast serving another epoch fails that reference @@ -134,14 +138,23 @@ impl Source { } /// The broadcast `path` is pinned to, or a lookup that pins its answer, either checked - /// against `epoch` on every read. - fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>) -> Binding { + /// against `epoch` on every read. The lookup waits for the path's first announcement when + /// `announced`, and otherwise answers for the routes as they stand. + fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>, announced: bool) -> Binding { let bound = match self.pins.lock().expect("pins poisoned").get(path) { Some(pinned) => Bound::Ready(pinned.clone()), // Unpinned even with an epoch: an epochless route still answers, best effort, and // the read refuses a route that proves another instance. None => Bound::Requested { - request: self.origin.request_broadcast(path, None).into_inner(), + lookup: match announced { + true => { + let mut routing = self.origin.routed_broadcast(path).into_inner(); + // Issue the request now: a handler answers it whether or not anyone reads yet. + let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); + Lookup::Announced(Mutex::new(routing)) + } + false => Lookup::Now(self.origin.request_broadcast(path, None).into_inner()), + }, pins: self.pins.clone(), path: path.clone(), }, @@ -149,6 +162,7 @@ impl Source { Binding { bound, epoch: epoch.cloned(), + sibling: *path != self.path, } } @@ -184,7 +198,7 @@ impl Source { /// the same policy rather than a different one. pub(crate) fn try_request(&self, reference: &impl hang::catalog::Reference) -> Option { let path = self.resolve_reference(reference.broadcast())?; - Some(self.request_path(&path, reference.epoch())) + Some(self.request_path(&path, reference.epoch(), true)) } /// Remove the renditions an export must not select: a disabled one (catalog @@ -234,6 +248,10 @@ impl Source { /// broadcast must be (see [`Source`]). Use it when you need the broadcast handle itself /// (e.g. to FETCH individual groups) rather than a subscription. /// + /// Answers for the routes as they stand: a sibling nothing announces yet fails, rather than + /// waiting for it as [`bind`](Self::bind) does, so a reader choosing among renditions can + /// move on to the next. + /// /// A reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast): the rendition names /// no broadcast, so there is nothing to resolve. @@ -242,14 +260,15 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - self.bind(rel, epoch)?.broadcast().await + self.request_path(&self.target(rel)?, epoch, false).broadcast().await } /// Start one broadcast request now and retain its result for repeated reads. /// /// Unlike [`resolve`](Self::resolve), this issues the request without awaiting or polling. - /// A remote or dynamic handler may answer later. The binding never retries a failed request - /// or resolves the path again after a publisher is replaced. + /// A path nothing announces yet resolves once something does, and a remote or dynamic + /// handler may answer later. Once answered, the binding never resolves the path again, even + /// after its publisher is replaced or goes; [`follow`](Self::follow) does. /// /// A path this source already resolved binds to that broadcast; otherwise the origin is /// asked, deduplicating repeat requests for one path so the catalog and every rendition share @@ -263,15 +282,15 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - Ok(self.request_path(&self.target(rel)?, epoch)) + Ok(self.request_path(&self.target(rel)?, epoch, true)) } /// Resolve an optional cross-broadcast reference and subscribe to track `name`, /// awaiting SUBSCRIBE_OK. /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) - /// subscribes on the catalog broadcast; anything else fetches the referenced broadcast - /// from the origin first, checked against `epoch` as in [`resolve`](Self::resolve). A + /// subscribes on the catalog broadcast; anything else is bound as in [`bind`](Self::bind), + /// waiting for the referenced broadcast's first announcement, and checked against `epoch`. A /// reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// @@ -287,20 +306,160 @@ impl Source { let broadcast = self.bind(rel, epoch)?.broadcast().await?; Ok(broadcast.track(name)?.subscribe(None).await?) } + + /// Follow the broadcast a rendition's reference names across its announcements. + /// + /// Holds nothing: each instance serving a sibling is handed out as it comes online or takes + /// over, and its end is reported, so a reader restarts the rendition on each one. A + /// reference to the catalog broadcast itself hands out only the instance this source was + /// built from and never reports its end, since a reader follows its catalog's own end and + /// restarts by building a new source. + /// + /// With an `epoch`, only that instance (or an epochless route, which cannot prove another + /// one) is handed out: another taking over ends the rendition rather than substituting for + /// it. Rejects an escaping reference exactly as [`resolve`](Self::resolve) does. + pub fn follow( + &self, + rel: Option<&moq_net::path::Relative<'_>>, + epoch: Option<&moq_net::Epoch>, + ) -> crate::Result { + let path = self.target(rel)?; + let instances = match path == self.path { + true => Instances::Fixed(Some(self.broadcast())), + false => Instances::Announced(self.origin.announced_broadcast(&path, None)?), + }; + Ok(Follow::with(instances, epoch)) + } +} + +/// The publisher instances serving a broadcast, as each comes online, takes over, and goes. +/// +/// Built on [`announced_broadcast`](moq_net::origin::Consumer::announced_broadcast), since an +/// announcement is the online and offline signal. Create one with [`Source::follow`] for a +/// catalog reference, or [`Follow::new`] for a path. +pub struct Follow { + instances: Instances, + /// The instance a reference named, if any. + epoch: Option, + /// The instance handed out last, until it stops serving. + current: Option, +} + +enum Instances { + /// Every instance announced at a path. + Announced(moq_net::origin::AnnouncedBroadcast), + /// One instance, handed out once. + Fixed(Option), +} + +/// A change to the instance a [`Follow`] serves. +pub enum Followed { + /// Start on this instance, dropping whatever was read from the one before. + Start(moq_net::broadcast::Consumer), + /// The instance handed out last stopped serving: idle until the next start. + End, +} + +impl Follow { + /// Follow the broadcast at `path` through `origin`, handing out only `epoch` when given. + /// + /// Unlike [`Source::follow`], this needs no catalog broadcast, so a reader can follow the + /// broadcast whose catalog it reads and build a [`Source`] for each instance. + pub fn new( + origin: &moq_net::origin::Consumer, + path: impl moq_net::AsPath, + epoch: Option<&moq_net::Epoch>, + ) -> crate::Result { + let announced = origin.announced_broadcast(path, None)?; + Ok(Self::with(Instances::Announced(announced), epoch)) + } + + fn with(instances: Instances, epoch: Option<&moq_net::Epoch>) -> Self { + Self { + instances, + epoch: epoch.cloned(), + current: None, + } + } + + /// Poll for the next change, registering `waiter` when there is none yet. + /// + /// Each announcement that brings an instance hands it out, so the same one returns after an + /// end. Fails once the origin closes, and with any resolution failure, such as a handler + /// refusing the path. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + let next = match &mut self.instances { + // Followed unpinned and checked here: a pinned follow waits out an epochless route, + // which an epoch reads best effort. + Instances::Announced(announced) => announced.poll_available(waiter).map_err(crate::Error::from), + Instances::Fixed(broadcast) => match broadcast.take() { + Some(broadcast) => Poll::Ready(Ok(broadcast)), + None => Poll::Pending, + }, + }; + if let Poll::Ready(next) = next { + let broadcast = next?; + if proves_other(self.epoch.as_ref(), &broadcast) { + tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); + match self.current.take() { + Some(_) => return Poll::Ready(Ok(Followed::End)), + None => continue, + } + } + // The catalog's own instance ends with the catalog, which its reader follows. + if let Instances::Announced(_) = self.instances { + self.current = Some(broadcast.clone()); + } + return Poll::Ready(Ok(Followed::Start(broadcast))); + } + + if let Some(current) = &self.current + && current.poll_closed(waiter).is_ready() + { + self.current = None; + return Poll::Ready(Ok(Followed::End)); + } + return Poll::Pending; + } + } + + /// The next change; see [`Self::poll_next`]. + pub async fn next(&mut self) -> crate::Result { + kio::wait(|waiter| self.poll_next(waiter)).await + } +} + +/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route +/// cannot prove its instance, so only a known mismatch counts. +fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { + matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) } /// A held broadcast or the retained result of one eagerly issued broadcast request. /// -/// Reading the binding never looks up its path again, even after a failure or publisher -/// replacement. Pending requests follow the origin's routing semantics; issuing a request -/// does not establish an epoch relationship with a separate catalog broadcast. A binding made -/// with an epoch refuses a broadcast proven to serve another one. +/// A pending request waits for a path nothing announces yet and rides out route churn until it +/// resolves (see [`routed_broadcast`](moq_net::origin::Consumer::routed_broadcast)). Once +/// answered, reading the binding never looks up its path again, even after a failure or +/// publisher replacement. Issuing a request does not establish an epoch relationship with a +/// separate catalog broadcast. A binding made with an epoch refuses a broadcast proven to serve +/// another one. /// /// Build one with [`Source::bind`], or with [`Binding::new`] for a broadcast already in hand. pub struct Binding { bound: Bound, /// The instance the reference named, if any. epoch: Option, + /// The reference names another broadcast than the catalog's. + sibling: bool, +} + +/// How a [`Bound::Requested`] path is looked up. +enum Lookup { + /// For the routes as they stand. + Now(moq_net::origin::Requesting), + /// Once something announces the path, riding out route churn. + Announced(Mutex), } enum Bound { @@ -308,7 +467,7 @@ enum Bound { Ready(moq_net::broadcast::Consumer), /// A request issued when the binding was made. Requested { - request: moq_net::origin::Requesting, + lookup: Lookup, /// Where the answer is pinned, the first for its path winning. pins: Pins, path: moq_net::PathOwned, @@ -321,17 +480,30 @@ impl Binding { Self { bound: Bound::Ready(broadcast), epoch: None, + sibling: false, } } + /// Whether the reference names another broadcast than the catalog's. + pub(crate) fn sibling(&self) -> bool { + self.sibling + } + /// Poll for the bound broadcast without blocking. /// /// Concurrent polls and cancelled waits retain the same request and result. pub fn poll_broadcast(&self, waiter: &kio::Waiter) -> Poll> { let broadcast = match &self.bound { Bound::Ready(broadcast) => broadcast.clone(), - Bound::Requested { request, pins, path } => { - let broadcast = ready!(request.poll_ok(waiter))?; + Bound::Requested { lookup, pins, path } => { + let broadcast = match lookup { + Lookup::Now(request) => ready!(request.poll_ok(waiter))?, + // A resolved routing answers the same result on every poll. + Lookup::Announced(routing) => { + let mut routing = routing.lock().expect("routing poisoned"); + ready!(kio::Task::poll(&mut *routing, waiter))? + } + }; // A request that raced another for the path takes the one pinned first. The path's // first answer is pinned even when it fails this binding's epoch, so no later // reference resolves the path independently. @@ -452,20 +624,27 @@ mod tests { assert!(first.is_clone(&binding.broadcast().await.unwrap())); } - /// A failed lookup pins nothing: the guarantee starts at a path's first resolution. - #[tokio::test] - async fn failed_binding_does_not_retry_a_later_publisher() { + /// A binding made before anything announces its path resolves on the first announcement, + /// and keeps that instance once it is replaced. + #[tokio::test(start_paused = true)] + async fn a_binding_waits_for_the_first_announcement() { let origin = produce_origin(); let _catalog = origin.publish("pub", Default::default()).unwrap(); settle().await; let source = resolved(&origin, "pub").await; let media = Relative::new("media"); let binding = source.bind(Some(&media), None).unwrap(); - assert!(binding.broadcast().await.is_err()); - let _publisher = origin.publish("media", Default::default()).unwrap(); settle().await; - assert!(!source.resolve(Some(&media), None).await.unwrap().is_closed()); - assert!(binding.broadcast().await.is_err()); + assert!(binding.poll_broadcast(&kio::Waiter::noop()).is_pending()); + + let _first = origin.publish("media", Default::default()).unwrap(); + let first = binding.broadcast().await.unwrap(); + assert!(!first.is_closed()); + + let _second = origin.publish("media", Default::default()).unwrap(); + settle().await; + assert!(binding.broadcast().await.unwrap().is_clone(&first)); + assert!(source.resolve(Some(&media), None).await.unwrap().is_clone(&first)); } /// A source serves its path from the broadcast it was built from, even when a replacement @@ -603,6 +782,115 @@ mod tests { assert!(source.resolve(Some(&rel), None).await.unwrap().is_clone(&first)); } + /// The next change a follower reports, which must arrive within the origin's update hold. + async fn followed(follow: &mut Follow) -> Option { + let next = tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .expect("no change") + .expect("the follow failed"); + match next { + Followed::Start(broadcast) => Some(broadcast), + Followed::End => None, + } + } + + /// The follower stays quiet past the origin's update hold. + async fn quiet(follow: &mut Follow) { + if tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .is_ok() + { + panic!("expected no change"); + } + } + + /// A sibling announced after the follow starts it, its end idles it, and its return starts + /// it again. + #[tokio::test(start_paused = true)] + async fn a_follow_starts_late_and_survives_an_end() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./source")), None).unwrap(); + quiet(&mut follow).await; + + let first = origin.publish("a/source", Default::default()).unwrap(); + let started = followed(&mut follow).await.expect("a start"); + assert_eq!(started.info().path.as_str(), "a/source"); + + drop(first); + assert!(followed(&mut follow).await.is_none(), "an end"); + quiet(&mut follow).await; + + let _second = origin.publish("a/source", Default::default()).unwrap(); + let restarted = followed(&mut follow).await.expect("a start"); + assert!(!restarted.is_clone(&started)); + } + + /// A replacement restarts a reference that names no epoch, and ends one that names the old + /// instance until that instance serves again. + #[tokio::test(start_paused = true)] + async fn a_follow_restarts_only_without_an_epoch() { + let origin = produce_origin(); + let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + let _old = origin.publish("a/source", route(&old_epoch)).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let rel = Relative::new("./source"); + let mut any = source.follow(Some(&rel), None).unwrap(); + let mut named = source.follow(Some(&rel), Some(&old_epoch)).unwrap(); + let old = followed(&mut any).await.unwrap(); + assert!(followed(&mut named).await.unwrap().is_clone(&old)); + + let new = origin.publish("a/source", route(&new_epoch)).unwrap(); + let replaced = followed(&mut any).await.unwrap(); + assert_eq!(replaced.info().epoch.as_ref(), Some(&new_epoch)); + assert!(followed(&mut named).await.is_none(), "another epoch ends it"); + quiet(&mut named).await; + // The old instance still serves the subscriptions already on it. + assert!(!old.is_closed()); + + // The replacement goes and the old instance wins again. + drop(new); + let back = followed(&mut named).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + let back = followed(&mut any).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + } + + /// An epoch is best effort over an epochless route, which cannot prove another instance. + #[tokio::test(start_paused = true)] + async fn a_follow_reads_an_epochless_route_with_an_epoch() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut named = source + .follow(Some(&Relative::new("./source")), Some(&moq_net::Epoch::mint())) + .unwrap(); + let _sibling = origin.publish("a/source", Default::default()).unwrap(); + assert!(followed(&mut named).await.is_some()); + } + + /// A self-reference reads only the catalog instance, whatever replaces it at the path. + #[tokio::test(start_paused = true)] + async fn a_follow_of_the_catalog_keeps_its_instance() { + let origin = produce_origin(); + let old = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./pub")), None).unwrap(); + assert!(followed(&mut follow).await.unwrap().is_clone(&source.broadcast())); + + let _new = origin.publish("a/pub", Default::default()).unwrap(); + quiet(&mut follow).await; + // Its end is the catalog's, which the catalog's reader sees. + drop(old); + quiet(&mut follow).await; + } + /// A self-reference's epoch must not contradict the catalog broadcast's own. #[tokio::test(start_paused = true)] async fn a_self_reference_epoch_checks_the_catalog_instance() { From 3bba0a26421ec9770d439275b8462b6115906346 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:21:09 -0700 Subject: [PATCH 03/27] quest: finish reference announcements Delete the quest, point sink-restart-follow at moq_mux::Follow, and size the now-childless broadcast epoch epic. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/broadcast.ts | 5 +- quest/m0/broadcast-epoch/README.md | 11 ++--- .../reference-announcements.md | 48 ------------------- quest/m1/announce-takeover.md | 1 - quest/m1/identity-guide.md | 1 - quest/m1/reference-epoch-bindings.md | 4 -- quest/m1/sink-restart-follow.md | 16 ++++--- 7 files changed, 16 insertions(+), 70 deletions(-) delete mode 100644 quest/m0/broadcast-epoch/reference-announcements.md diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index 62066c83821..718e263da8c 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -18,7 +18,10 @@ type Usable = (rel: Path.Relative | undefined, epoch: Moq.Epoch.Valid | undefine // Either the catalog's own broadcast, or a sibling to consume by path. type RelativeTarget = { local: true } | { local: false; path: Moq.Path.Valid }; -function filterRenditions(renditions: Record, usable: Usable): Record { +function filterRenditions( + renditions: Record, + usable: Usable, +): Record { return Object.fromEntries( Object.entries(renditions).filter(([, config]) => usable(config.broadcast, config.epoch)), ); diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index 396e06f4d22..3cccb3adbe0 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -1,4 +1,4 @@ -# Broadcast epochs +# [L] Broadcast epochs ## Goal @@ -72,9 +72,8 @@ Decided: for the catalog instance's lifetime. Optional reference epochs constrain resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The - schemas, shared resolution, and compatibility documentation landed; - [reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - owns the native readers that follow announcements. + schemas, shared resolution, compatibility documentation, and `moq_mux::Follow` + (which `moqsrc` follows siblings with) landed. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players (`moq play`, `@moq/watch`, demo/web) follow the announce `Restart`. moq-stats mints one per @@ -124,10 +123,6 @@ constraints; it never mixes instances through an old handle. Check exact and covering routes, matching/different/absent epochs, and legacy End+Start. The child quests own their focused regressions. -## Required - -- [Reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - native readers restart a replaced referenced sibling on its announcements, never past a named epoch - ## Related - [JS consume identity](https://github.com/moq-dev/moq/pull/5254) - requests carry the serving prefix's epoch and new winners cannot reuse an old cached broadcast diff --git a/quest/m0/broadcast-epoch/reference-announcements.md b/quest/m0/broadcast-epoch/reference-announcements.md deleted file mode 100644 index cfde1ceeef0..00000000000 --- a/quest/m0/broadcast-epoch/reference-announcements.md +++ /dev/null @@ -1,48 +0,0 @@ -# [L] Rust readers follow referenced broadcasts' announcements - -## Goal - -Native readers follow a catalog-referenced sibling's announcements the way -`@moq/watch` already does, since announcement is the online/offline signal. -A reference that names an `epoch` never follows onto another instance. - -## Plan - -Split from the catalog references work (#5500's Codex review, decided by the -maintainer 2026-10-14). That work landed the optional reference `epoch`, its -shared resolution in `moq_mux::Source` and `@moq/watch`, and the Hang draft -text; this quest owns the following behavior it left out. - -- Share it in `moq_mux` (a `Source::follow`-style API; the shape is open) - rather than per reader, built on `origin::Consumer::announced_broadcast` - rather than another hand-rolled loop. Follow unpinned and check each - handed-out instance's epoch, as `Source::bind` does: a pinned - `announced_broadcast` waits out an epochless route, which would break the - best effort the reference epoch promises there. `Source::bind` binds once - today, as `failed_binding_does_not_retry_a_later_publisher` asserts; that - changes. The "no retarget" rule applies only to readers that follow no - announcement. -- `moqsrc`: Start binds the rendition, End idles its pad, and Restart - restarts its pump. -- Exporters: a late first announcement starts the rendition and End ends - it. Restart behavior belongs to - [sink restart follow](/quest/m1/sink-restart-follow.md). -- A replacement is a Restart for an announcement follower, never a seamless - same-epoch resume. A proven Takeover preserves the same instance. -- A reference with an `epoch` follows only its own instance: following never - substitutes a different epoch, so a Restart onto another one ends that - rendition, matching `relativeBroadcast` in - `@moq/watch` (`follows a replaced sibling only for a reference that names - no epoch`). -- `@moq/watch` returns nothing for a rendition whose followed broadcast proves - another epoch, but still lists it in `out.catalog`, so selection can pick a - rendition that never plays. Drop it from the effective catalog too. - -Test late first announcements, End then Start, Restart with and without an -epoch, and concurrent old/new consumers, in the moq-mux unit tests and the -`moqsrc` and exporter suites. - -## Related - -- [Sink restart follow](/quest/m1/sink-restart-follow.md) - exporters and sinks that follow an epochless Restart of their own broadcast -- [Export rendition rebind](/quest/m1/export-rendition-rebind.md) - exporters rebuild a rendition whose reference changes diff --git a/quest/m1/announce-takeover.md b/quest/m1/announce-takeover.md index 593b11e5366..3ff73d8184c 100644 --- a/quest/m1/announce-takeover.md +++ b/quest/m1/announce-takeover.md @@ -94,6 +94,5 @@ ANNOUNCE_TAKEOVER; published versions retain their framing. - [Claim answer epochs](https://github.com/moq-dev/moq/pull/5678) - `Request::prepare` then `publish` answers a claim at the exact path under a fresh epoch, the operation TAKEOVER continues - [IETF takeover](/quest/m1/ietf-takeover.md) - negotiated IETF counterpart -- [Reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - a proven continuation keeps a followed reference on its instance - [Retired requests](https://github.com/moq-dev/moq/pull/5294) - replacements never substitute content into held handles - [Identity and recovery guide](/quest/m1/identity-guide.md) - explains takeover alongside restart and legacy behavior diff --git a/quest/m1/identity-guide.md b/quest/m1/identity-guide.md index 27d5b876af0..4e9c3978aed 100644 --- a/quest/m1/identity-guide.md +++ b/quest/m1/identity-guide.md @@ -40,5 +40,4 @@ Public API and wire: none; this is a user guide for existing feature owners. ## Required -- [Reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md) - settles how readers pin and follow catalog references - [Announcement takeover](/quest/m1/announce-takeover.md) - establishes the continuity operation the guide demonstrates diff --git a/quest/m1/reference-epoch-bindings.md b/quest/m1/reference-epoch-bindings.md index 3e426407772..d6a0cacf219 100644 --- a/quest/m1/reference-epoch-bindings.md +++ b/quest/m1/reference-epoch-bindings.md @@ -24,7 +24,3 @@ field: the first instance resolved, without its guarantee. Test a matching, mismatched and epochless reference through `moq-ffi` and libmoq, and run `just test interop --all`. - -## Related - -- [Rust readers follow references](/quest/m0/broadcast-epoch/reference-announcements.md) - the following behavior the bindings inherit from `moq_mux` diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index 23d4db98a40..4b32c515f49 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -16,8 +16,8 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references -to other broadcasts stays with -[reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md). +to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; +`moq play` resolves a reference once with `Source::resolve`. Readers with their own follow loop today move onto `announced_broadcast`: @@ -47,9 +47,11 @@ Audit and fix the rest the same way: - `transcode`: its source broadcast; following means restarting the ladder on the new instance. -Share one helper (a `moq_mux` `Source::follow`-style API per -[reference announcements](/quest/m0/broadcast-epoch/reference-announcements.md)) -rather than one rule per reader, built on `announced_broadcast`. Each fix -lands with a test that replaces the publisher while the reader runs. +Share `moq_mux::Follow::new(origin, path, None)` rather than one rule per +reader: it yields `Followed::Start` for each instance serving the path and +`Followed::End` once the one handed out stops serving. Build a +`moq_mux::Source` from each started instance, so its siblings follow through +`Source::follow` and its exporters pin through `Source::bind`. Each fix lands +with a test that replaces the publisher while the reader runs. -Public API: possibly a shared follow helper in `moq-mux`. Wire: none. +Public API: none beyond `moq_mux::Follow`. Wire: none. From 185dbc5f5f0a2421de3f7b0223328e62bc440ce4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:53:45 -0700 Subject: [PATCH 04/27] fix(mux): only export requests wait for a sibling's first announcement Public Source::bind, resolve, and subscribe_track keep answering for the routes as they stand: HLS answers per request and moq play picks renditions inline, so neither can wait on a sibling that left. Co-Authored-By: Claude Opus 5.5 --- doc/concept/hang.md | 2 +- rs/moq-gst/src/source/imp.rs | 24 +++++++-- rs/moq-mux/src/container/source.rs | 6 ++- rs/moq-mux/src/source.rs | 80 +++++++++++++++++------------- 4 files changed, 71 insertions(+), 41 deletions(-) diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 4373b75c6dc..149930b0963 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; an exporter starts on the first one announced, keeps it, and ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; the fMP4, TS, MKV, FLV, and HLS exporters start on the first one announced, keep it, and end the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 546c24e52ec..62e6b2f2278 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -3155,7 +3155,10 @@ mod session_tests { assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); stop(session); - assert!(recorder.seen().contains(&Seen::Start(pad)), "no new stream on the kept pad"); + assert!( + recorder.seen().contains(&Seen::Start(pad)), + "no new stream on the kept pad" + ); recorder.assert_one_pad_held(); } @@ -3178,7 +3181,10 @@ mod session_tests { stop(session); let seen = recorder.seen(); - let switched = seen.iter().position(|seen| *seen == Seen::Buffer(pad.clone(), 2)).unwrap(); + let switched = seen + .iter() + .position(|seen| *seen == Seen::Buffer(pad.clone(), 2)) + .unwrap(); assert!( !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), "the replaced sibling kept playing: {seen:?}" @@ -3213,12 +3219,20 @@ mod session_tests { let mut follow = origin.consume().follow("cam").unwrap(); await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); let mut new = Publisher::sibling(&origin, epoch()); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + await_event(&mut follow, |event| { + matches!(event, moq_net::announce::Event::Restart(_)) + }); new.write(2); drop(new); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Restart(_))); + await_event(&mut follow, |event| { + matches!(event, moq_net::announce::Event::Restart(_)) + }); - assert_eq!(recorder.written(&mut old, 3), pad, "the named instance moved to another pad"); + assert_eq!( + recorder.written(&mut old, 3), + pad, + "the named instance moved to another pad" + ); stop(session); assert!( !recorder.seen().contains(&Seen::Buffer(pad, 2)), diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 5910aa3fd8a..21cae5c26c3 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -766,7 +766,11 @@ mod tests { .unwrap() .unwrap(); let ended = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| late.poll_read(waiter))).await; - assert!(matches!(ended, Ok(Ok(None))), "{:?}", ended.map(|read| read.map(|_| ()))); + assert!( + matches!(ended, Ok(Ok(None))), + "{:?}", + ended.map(|read| read.map(|_| ())) + ); drop((track, replacement, new)); } diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 1594c95ff40..72de27a2b12 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -23,13 +23,13 @@ type Pins = Arc> /// reference. Build one with [`Source::new`]. /// /// A source serves its own path from the broadcast it was built from, and keeps the first -/// broadcast it resolves for each sibling; a [binding](Self::bind) waits for the sibling's first -/// announcement to resolve it. Every later request for a path is served from that handle -/// ([`broadcast`](Self::broadcast), -/// [`catalog`](Self::catalog), [`resolve`](Self::resolve), [`bind`](Self::bind), -/// [`subscribe_track`](Self::subscribe_track), and an export's track requests). A route replaced -/// after that is never resolved, epoch or not, so two publishers never splice into one export; a -/// request on a held broadcast that has ended fails instead. Clones share the held broadcasts. +/// broadcast it resolves for each sibling. Every later request for a path is served from that +/// handle ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), +/// [`resolve`](Self::resolve), [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), +/// and an export's track requests, which wait for a sibling's first announcement). A route +/// replaced after that is never resolved, epoch or not, so two publishers never splice into one +/// export; a request on a held broadcast that has ended fails instead. Clones share the held +/// broadcasts. /// /// A reader that follows a sibling's announcements instead, restarting on each new instance, /// uses [`follow`](Self::follow), which holds nothing. @@ -151,7 +151,7 @@ impl Source { let mut routing = self.origin.routed_broadcast(path).into_inner(); // Issue the request now: a handler answers it whether or not anyone reads yet. let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); - Lookup::Announced(Mutex::new(routing)) + Lookup::Announced(Box::new(Mutex::new(routing))) } false => Lookup::Now(self.origin.request_broadcast(path, None).into_inner()), }, @@ -191,7 +191,8 @@ impl Source { } /// The skipping counterpart to [`Self::bind`] for a catalog entry, returning `None` when its - /// broadcast walks above the origin root. + /// broadcast walks above the origin root. Unlike a binding, it waits for a sibling nothing + /// announces yet, so a late first announcement starts the rendition. /// /// The exporters use it to drop a single bad rendition rather than fail the whole export. /// [`Self::retain_selectable`] already removes those renditions, so this is the second half of @@ -248,10 +249,6 @@ impl Source { /// broadcast must be (see [`Source`]). Use it when you need the broadcast handle itself /// (e.g. to FETCH individual groups) rather than a subscription. /// - /// Answers for the routes as they stand: a sibling nothing announces yet fails, rather than - /// waiting for it as [`bind`](Self::bind) does, so a reader choosing among renditions can - /// move on to the next. - /// /// A reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast): the rendition names /// no broadcast, so there is nothing to resolve. @@ -260,15 +257,14 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - self.request_path(&self.target(rel)?, epoch, false).broadcast().await + self.bind(rel, epoch)?.broadcast().await } /// Start one broadcast request now and retain its result for repeated reads. /// /// Unlike [`resolve`](Self::resolve), this issues the request without awaiting or polling. - /// A path nothing announces yet resolves once something does, and a remote or dynamic - /// handler may answer later. Once answered, the binding never resolves the path again, even - /// after its publisher is replaced or goes; [`follow`](Self::follow) does. + /// A remote or dynamic handler may answer later. The binding never retries a failed request + /// or resolves the path again after a publisher is replaced; [`follow`](Self::follow) does. /// /// A path this source already resolved binds to that broadcast; otherwise the origin is /// asked, deduplicating repeat requests for one path so the catalog and every rendition share @@ -282,15 +278,15 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - Ok(self.request_path(&self.target(rel)?, epoch, true)) + Ok(self.request_path(&self.target(rel)?, epoch, false)) } /// Resolve an optional cross-broadcast reference and subscribe to track `name`, /// awaiting SUBSCRIBE_OK. /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) - /// subscribes on the catalog broadcast; anything else is bound as in [`bind`](Self::bind), - /// waiting for the referenced broadcast's first announcement, and checked against `epoch`. A + /// subscribes on the catalog broadcast; anything else fetches the referenced broadcast + /// from the origin first, checked against `epoch` as in [`resolve`](Self::resolve). A /// reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// @@ -326,7 +322,7 @@ impl Source { let path = self.target(rel)?; let instances = match path == self.path { true => Instances::Fixed(Some(self.broadcast())), - false => Instances::Announced(self.origin.announced_broadcast(&path, None)?), + false => Instances::Announced(Box::new(self.origin.announced_broadcast(&path, None)?)), }; Ok(Follow::with(instances, epoch)) } @@ -347,7 +343,7 @@ pub struct Follow { enum Instances { /// Every instance announced at a path. - Announced(moq_net::origin::AnnouncedBroadcast), + Announced(Box), /// One instance, handed out once. Fixed(Option), } @@ -371,7 +367,7 @@ impl Follow { epoch: Option<&moq_net::Epoch>, ) -> crate::Result { let announced = origin.announced_broadcast(path, None)?; - Ok(Self::with(Instances::Announced(announced), epoch)) + Ok(Self::with(Instances::Announced(Box::new(announced)), epoch)) } fn with(instances: Instances, epoch: Option<&moq_net::Epoch>) -> Self { @@ -438,12 +434,10 @@ fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast:: /// A held broadcast or the retained result of one eagerly issued broadcast request. /// -/// A pending request waits for a path nothing announces yet and rides out route churn until it -/// resolves (see [`routed_broadcast`](moq_net::origin::Consumer::routed_broadcast)). Once -/// answered, reading the binding never looks up its path again, even after a failure or -/// publisher replacement. Issuing a request does not establish an epoch relationship with a -/// separate catalog broadcast. A binding made with an epoch refuses a broadcast proven to serve -/// another one. +/// Reading the binding never looks up its path again, even after a failure or publisher +/// replacement. Pending requests follow the origin's routing semantics; issuing a request +/// does not establish an epoch relationship with a separate catalog broadcast. A binding made +/// with an epoch refuses a broadcast proven to serve another one. /// /// Build one with [`Source::bind`], or with [`Binding::new`] for a broadcast already in hand. pub struct Binding { @@ -459,7 +453,7 @@ enum Lookup { /// For the routes as they stand. Now(moq_net::origin::Requesting), /// Once something announces the path, riding out route churn. - Announced(Mutex), + Announced(Box>), } enum Bound { @@ -624,16 +618,34 @@ mod tests { assert!(first.is_clone(&binding.broadcast().await.unwrap())); } - /// A binding made before anything announces its path resolves on the first announcement, - /// and keeps that instance once it is replaced. - #[tokio::test(start_paused = true)] - async fn a_binding_waits_for_the_first_announcement() { + /// A failed lookup pins nothing: the guarantee starts at a path's first resolution. + #[tokio::test] + async fn failed_binding_does_not_retry_a_later_publisher() { let origin = produce_origin(); let _catalog = origin.publish("pub", Default::default()).unwrap(); settle().await; let source = resolved(&origin, "pub").await; let media = Relative::new("media"); let binding = source.bind(Some(&media), None).unwrap(); + assert!(binding.broadcast().await.is_err()); + let _publisher = origin.publish("media", Default::default()).unwrap(); + settle().await; + assert!(!source.resolve(Some(&media), None).await.unwrap().is_closed()); + assert!(binding.broadcast().await.is_err()); + } + + /// An export's request made before anything announces its sibling resolves on the first + /// announcement, and keeps that instance once it is replaced. + #[tokio::test(start_paused = true)] + async fn an_export_request_waits_for_the_first_announcement() { + let origin = produce_origin(); + let _catalog = origin.publish("pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "pub").await; + let media = Relative::new("media"); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + config.broadcast = Some(media.to_owned()); + let binding = source.try_request(&config).unwrap(); settle().await; assert!(binding.poll_broadcast(&kio::Waiter::noop()).is_pending()); From 24206edeee7f4ecd4df2ae3b6c836569fd4a7f1b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 00:32:03 -0700 Subject: [PATCH 05/27] feat(mux): bind waits for a sibling's first announcement; rename to follow::* Source::bind, resolve, and subscribe_track wait for a sibling nothing announces yet, through routed_broadcast, instead of failing Unroutable. A binding answers its pinned instance once the path is pinned, so a sibling that announced and then left answers its ended broadcast rather than waiting for the next. moq-hls drops its own re-issue loop, and its disconnected-sibling test now names the sibling it means to bind. Move Follow/Followed to moq_mux::follow::{Instances, Event}. Co-Authored-By: Claude Opus 5.5 --- cpp/moq/MIGRATE.md | 5 + dart/moq/MIGRATE.md | 5 + dart/moq_ffi/MIGRATE.md | 6 + dart/moq_ffi/lib/src/moq.dart | 2 +- doc/concept/hang.md | 2 +- go/ffi/MIGRATE.md | 6 + go/wrapper/MIGRATE.md | 5 + kt/moq-ffi/MIGRATE.md | 6 + kt/moq/MIGRATE.md | 5 + py/moq-ffi/MIGRATE.md | 6 + py/moq-rs/MIGRATE.md | 5 + quest/m0/broadcast-epoch/README.md | 2 +- quest/m1/sink-restart-follow.md | 12 +- rs/moq-c/MIGRATE.md | 5 + rs/moq-ffi/MIGRATE.md | 5 + rs/moq-ffi/src/consumer.rs | 5 +- rs/moq-gst/src/source/imp.rs | 14 +- rs/moq-hls/src/export/mod.rs | 4 +- rs/moq-hls/src/export/rendition.rs | 53 +--- rs/moq-hls/src/server/routes.rs | 2 +- rs/moq-mux/MIGRATE.md | 5 + rs/moq-mux/src/follow.rs | 243 ++++++++++++++++++ rs/moq-mux/src/lib.rs | 5 +- rs/moq-mux/src/source.rs | 382 ++++++----------------------- swift/MIGRATE.md | 5 + swift/ffi/MIGRATE.md | 6 + 26 files changed, 435 insertions(+), 366 deletions(-) create mode 100644 rs/moq-mux/src/follow.rs diff --git a/cpp/moq/MIGRATE.md b/cpp/moq/MIGRATE.md index ec0b7243cd4..6e087da914a 100644 --- a/cpp/moq/MIGRATE.md +++ b/cpp/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer::resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel it. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer::finish` lets go. diff --git a/dart/moq/MIGRATE.md b/dart/moq/MIGRATE.md index 7a1f1be6ca3..c145997e2ac 100644 --- a/dart/moq/MIGRATE.md +++ b/dart/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference in the rendition-based subscribes and fetches now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: add a `.timeout` to the future. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/dart/moq_ffi/MIGRATE.md b/dart/moq_ffi/MIGRATE.md index 3e8c381a845..61c02c56f61 100644 --- a/dart/moq_ffi/MIGRATE.md +++ b/dart/moq_ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: add a `.timeout` to the future. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 1d0d4121dcd..31703ab149c 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -19310,7 +19310,7 @@ void _checkApiChecksums() { 18633) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqbroadcastconsumer_resolve() != 62317) { + if (uniffi_moq_ffi_checksum_method_moqbroadcastconsumer_resolve() != 48256) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqbroadcastconsumer_subscribe_track() != diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 149930b0963..841f74d2b0b 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; the fMP4, TS, MKV, FLV, and HLS exporters start on the first one announced, keep it, and end the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; every other reader, such as `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/go/ffi/MIGRATE.md b/go/ffi/MIGRATE.md index d7abfc288f8..818a4eca19f 100644 --- a/go/ffi/MIGRATE.md +++ b/go/ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.Resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: pass a `ctx` with a deadline. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.Finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index b5e60cf4cc7..04a184e7131 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer.Resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: pass a `ctx` with a deadline. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.Finish()` lets go. diff --git a/kt/moq-ffi/MIGRATE.md b/kt/moq-ffi/MIGRATE.md index 6e9627fd6a6..dc1d04ae17e 100644 --- a/kt/moq-ffi/MIGRATE.md +++ b/kt/moq-ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the coroutine or wrap it in `withTimeout`. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/kt/moq/MIGRATE.md b/kt/moq/MIGRATE.md index 2ff652f9212..59c92a5d82f 100644 --- a/kt/moq/MIGRATE.md +++ b/kt/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference in the rendition-based subscribes and fetches now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the coroutine or wrap it in `withTimeout`. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast ends or the new `BroadcastProducer.finish()` lets go. diff --git a/py/moq-ffi/MIGRATE.md b/py/moq-ffi/MIGRATE.md index a009932892c..5fa95305f84 100644 --- a/py/moq-ffi/MIGRATE.md +++ b/py/moq-ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve()` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task or wrap it in `asyncio.wait_for`. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 4175ecd3ec6..24f4f3a2212 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer.resolve()` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task or wrap it in `asyncio.wait_for`. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index 3cccb3adbe0..addef2d7b7d 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -72,7 +72,7 @@ Decided: for the catalog instance's lifetime. Optional reference epochs constrain resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The - schemas, shared resolution, compatibility documentation, and `moq_mux::Follow` + schemas, shared resolution, compatibility documentation, and `moq_mux::follow` (which `moqsrc` follows siblings with) landed. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index ea5b6b3e45d..b76f09ecbfe 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -17,7 +17,8 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; -`moq play` resolves a reference once with `Source::resolve`. +`moq play` resolves a reference once with `Source::resolve`, which waits for +a sibling's first announcement and keeps it. Readers with their own follow loop today move onto `announced_broadcast`: @@ -47,12 +48,13 @@ Audit and fix the rest the same way: - `transcode`: its source broadcast; following means restarting the ladder on the new instance. -Share `moq_mux::Follow::new(origin, path, None)` (decided 2026-10-20, +Share `moq_mux::follow::Instances::new(origin, path, None)` (decided 2026-10-20, maintainer: no reader hand-rolls its own follow loop) rather than one rule per -reader: it yields `Followed::Start` for each instance serving the path and -`Followed::End` once the one handed out stops serving. Build a +reader: it yields `follow::Event::Start` for each instance serving the path and +`follow::Event::End` once the one handed out stops serving. Build a `moq_mux::Source` from each started instance, so its siblings follow through -`Source::follow` and its exporters pin through `Source::bind`. Each fix lands +`Source::follow` and its exporters pin through `Source::bind`, which waits for +a sibling's first announcement. Each fix lands with a test that replaces the publisher while the reader runs. Public API: none. Wire: none. diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 45a2f272283..1e52cf8b7e7 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +`moq_consume_video` and `moq_consume_audio` on a rendition whose catalog `broadcast` field names a sibling now wait for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, give up on it yourself with `moq_consume_video_cancel` or `moq_consume_audio_cancel`. + ### Finished tracks stay until the broadcast ends (#6157) `moq_publish_track_finish`, `moq_publish_media_finish`, `moq_publish_container_finish`, and the encoder finishes now release the handle while the broadcast keeps the finished track listed, with its catalog rendition and demand, until `moq_publish_close` or the new `moq_publish_finish_close`. diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index 59e13b01410..b8c81b79eb5 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,6 +5,11 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer::resolve` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel it or race it against a timeout. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer::finish` lets go, instead of unlisting when its producer finishes or is released. diff --git a/rs/moq-ffi/src/consumer.rs b/rs/moq-ffi/src/consumer.rs index 17fab728081..1a6328e4c82 100644 --- a/rs/moq-ffi/src/consumer.rs +++ b/rs/moq-ffi/src/consumer.rs @@ -260,9 +260,8 @@ impl MoqBroadcastConsumer { /// `MoqVideoConsumer::subscribe`, `MoqAudioConsumer::subscribe`) resolve it themselves. /// /// Errors if this broadcast came from a local producer rather than an origin, since a - /// standalone broadcast has no sibling to name, and reports a sibling that exists but is not - /// announced yet as unroutable rather than waiting for it (see - /// [`MoqOriginConsumer::request_broadcast`](crate::origin::MoqOriginConsumer::request_broadcast)). + /// standalone broadcast has no sibling to name. A sibling nothing announces yet is waited for, + /// so a late one resolves once it is announced; cancel the call to give up on it. pub async fn resolve(&self, reference: Option) -> Result, MoqError> { let broadcast = self.resolve_inner(reference.as_deref()).await?; Ok(Arc::new(match self.origin.clone() { diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 62e6b2f2278..8564c8a94b3 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1106,7 +1106,7 @@ struct Pump { name: String, caps: gst::Caps, /// Each instance of the broadcast the rendition references. - follow: moq_mux::Follow, + follow: moq_mux::follow::Instances, /// The rendition references another broadcast than its catalog's, whose instances come and /// go on their own announcements. sibling: bool, @@ -1171,8 +1171,8 @@ impl Pump { None => tokio::select! { _ = cancel.changed() => return None, followed = follow.next() => match followed { - Ok(moq_mux::Followed::Start(broadcast)) => broadcast, - Ok(moq_mux::Followed::End) => continue, + Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, + Ok(moq_mux::follow::Event::End) => continue, Err(err) => { gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); return Some((kind, name)); @@ -1189,8 +1189,8 @@ impl Pump { let subscribed = tokio::select! { _ = cancel.changed() => return None, followed = follow.next() => match followed { - Ok(moq_mux::Followed::Start(broadcast)) => Err(Stopped::Restart(broadcast)), - Ok(moq_mux::Followed::End) => Err(Stopped::Idle), + Ok(moq_mux::follow::Event::Start(broadcast)) => Err(Stopped::Restart(broadcast)), + Ok(moq_mux::follow::Event::End) => Err(Stopped::Idle), Err(err) => { gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); return Some((kind, name)); @@ -1311,8 +1311,8 @@ impl Pump { return None; } followed = follow.next() => match followed { - Ok(moq_mux::Followed::Start(broadcast)) => break Stopped::Restart(broadcast), - Ok(moq_mux::Followed::End) => break Stopped::Idle, + Ok(moq_mux::follow::Event::Start(broadcast)) => break Stopped::Restart(broadcast), + Ok(moq_mux::follow::Event::End) => break Stopped::Idle, Err(err) => { gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); return Some((kind, name)); diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 8f846c51f08..c2283e13e37 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -3865,8 +3865,8 @@ mod tests { drop((track, catalog, live)); } - // A sibling not yet announced when the rendition binds it fails that first lookup. The - // catalog's rows are still listed, and the bind is issued again until the sibling resolves. + // A sibling not yet announced when the rendition binds it is waited for. The catalog's rows + // are still listed, and the binding resolves on the sibling's first announcement. #[tokio::test] async fn a_sibling_announced_after_export_start_is_bound() { tokio::time::pause(); diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 72b4146188f..baed6031b62 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -115,9 +115,8 @@ enum Config { struct Media { handle: Mutex, track: String, - /// The source, reference, and epoch a sibling is bound through, to re-issue a bind that - /// failed before it ever resolved. - sibling: Option<(moq_mux::Source, moq_net::path::RelativeOwned, Option)>, + /// The rendition reads another broadcast than the catalog's. + sibling: bool, } struct Handle { @@ -154,7 +153,8 @@ impl Media { Ok(Self { handle: Mutex::new(handle), track: track.to_string(), - sibling: sibling(source, rel).map(|rel| (source.clone(), rel, reference.epoch().cloned())), + // `bind` already rejected an escaping reference, so both sides name a path. + sibling: source.resolve_reference(rel) != source.resolve_reference(None), }) } @@ -172,33 +172,16 @@ impl Media { /// Register the bound broadcast's close and retain its track before new lookups are refused. fn poll_sync(&self, waiter: &kio::Waiter, window: &segments::Producer) -> Arc { let mut handle = self.handle.lock().expect("media lock poisoned"); - let mut reissued = false; - loop { - match handle.binding.poll_broadcast(waiter) { - Poll::Ready(Ok(broadcast)) => { - handle.capture(&broadcast, &self.track); - if self.sibling.is_some() && !handle.ended && broadcast.poll_closed(waiter).is_ready() { - // End and Restart both drain only the pinned instance. Looking its path - // up again could serve a replacement under the old timeline's rows. - handle.ended = true; - window.end(); - } - } - // Only a binding that never resolved may be reissued. The source pins every - // successful result, so a replacement can never answer these old rows. - Poll::Ready(Err(_)) if !reissued => { - reissued = true; - if let Some((source, rel, epoch)) = &self.sibling - && let Ok(next) = source.bind(Some(rel), epoch.as_ref()) - { - handle.binding = Arc::new(next); - continue; - } - } - _ => {} + if let Poll::Ready(Ok(broadcast)) = handle.binding.poll_broadcast(waiter) { + handle.capture(&broadcast, &self.track); + if self.sibling && !handle.ended && broadcast.poll_closed(waiter).is_ready() { + // End and Restart both drain only the pinned instance. Looking its path up again + // could serve a replacement under the old timeline's rows. + handle.ended = true; + window.end(); } - return handle.binding.clone(); } + handle.binding.clone() } fn poll_track( @@ -215,18 +198,6 @@ impl Media { } } -/// A catalog `broadcast` reference that names a different path than the catalog itself. -fn sibling( - source: &moq_mux::Source, - rel: Option<&moq_net::path::RelativeOwned>, -) -> Option { - let target = source.resolve_reference(rel)?; - if source.resolve_reference(None).as_ref() == Some(&target) { - return None; - } - Some(rel.cloned().unwrap_or_else(moq_net::path::Relative::empty)) -} - /// Clear the video fields that don't change how the rendition decodes or is muxed, so a config /// the publisher only re-measured or re-labelled compares equal to the one already being served. /// diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index dda6d035e60..9b0b95e810b 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -1061,7 +1061,7 @@ mod tests { moq_mux::catalog::Producer::new(&mut catalog_broadcast, moq_mux::catalog::Config::default()).unwrap(); let reserved = catalog.reserve(); let mut config = video_config(); - config.broadcast = Some(moq_net::path::Relative::new("../source").to_owned()); + config.broadcast = Some(moq_net::path::Relative::new("./source").to_owned()); let track = media_broadcast.create_track("video0", timed()).unwrap(); let mut media = reserved .video( diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index 0dcc889ce36..085b10a56c2 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Source` waits for a sibling's first announcement (#6254) + +`Source::bind`, `resolve`, and `subscribe_track` now wait for a sibling nothing announces yet instead of failing with `Unroutable`, and keep the first instance they resolve: once it goes, they answer that ended broadcast rather than waiting for the next. +Where you skipped a rendition on that error, bound the call yourself (`tokio::time::timeout`, or drop it), or follow the sibling's instances with `Source::follow`. + ### `Source` resolution takes the reference's epoch (#5699) `Source::bind(rel)`, `resolve(rel)`, and `subscribe_track(rel, name)` take the catalog entry's optional epoch after `rel`. diff --git a/rs/moq-mux/src/follow.rs b/rs/moq-mux/src/follow.rs new file mode 100644 index 00000000000..eb149f7a593 --- /dev/null +++ b/rs/moq-mux/src/follow.rs @@ -0,0 +1,243 @@ +//! Following a broadcast across the publisher instances that serve it. +//! +//! An announcement is the online and offline signal, so a reader that restarts on each instance +//! follows [`Instances`] rather than pinning one broadcast as a [`Binding`](crate::Binding) does. + +use std::task::Poll; + +/// The publisher instances serving a broadcast, as each comes online, takes over, and goes. +/// +/// Built on [`announced_broadcast`](moq_net::origin::Consumer::announced_broadcast). Create one +/// with [`Source::follow`](crate::Source::follow) for a catalog reference, or [`Instances::new`] +/// for a path. +pub struct Instances { + feed: Feed, + /// The instance a reference named, if any. + epoch: Option, + /// The instance handed out last, until it stops serving. + current: Option, +} + +enum Feed { + /// Every instance announced at a path. + Announced(Box), + /// One instance, handed out once. + Fixed(Option), +} + +/// A change to the instance [`Instances`] serves. +pub enum Event { + /// Start on this instance, dropping whatever was read from the one before. + Start(moq_net::broadcast::Consumer), + /// The instance handed out last stopped serving: idle until the next start. + End, +} + +impl Instances { + /// Follow the broadcast at `path` through `origin`, handing out only `epoch` when given. + /// + /// Unlike [`Source::follow`](crate::Source::follow), this needs no catalog broadcast, so a + /// reader can follow the broadcast whose catalog it reads and build a + /// [`Source`](crate::Source) for each instance. + pub fn new( + origin: &moq_net::origin::Consumer, + path: impl moq_net::AsPath, + epoch: Option<&moq_net::Epoch>, + ) -> crate::Result { + let announced = origin.announced_broadcast(path, None)?; + Ok(Self::with(Feed::Announced(Box::new(announced)), epoch)) + } + + /// Hand out `broadcast` once and never report its end, which its owner follows. + pub(crate) fn fixed(broadcast: moq_net::broadcast::Consumer, epoch: Option<&moq_net::Epoch>) -> Self { + Self::with(Feed::Fixed(Some(broadcast)), epoch) + } + + fn with(feed: Feed, epoch: Option<&moq_net::Epoch>) -> Self { + Self { + feed, + epoch: epoch.cloned(), + current: None, + } + } + + /// Poll for the next change, registering `waiter` when there is none yet. + /// + /// Each announcement that brings an instance hands it out, so the same one returns after an + /// end. Fails once the origin closes, and with any resolution failure, such as a handler + /// refusing the path. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + let next = match &mut self.feed { + // Followed unpinned and checked here: a pinned follow waits out an epochless route, + // which an epoch reads best effort. + Feed::Announced(announced) => announced.poll_available(waiter).map_err(crate::Error::from), + Feed::Fixed(broadcast) => match broadcast.take() { + Some(broadcast) => Poll::Ready(Ok(broadcast)), + None => Poll::Pending, + }, + }; + if let Poll::Ready(next) = next { + let broadcast = next?; + if proves_other(self.epoch.as_ref(), &broadcast) { + tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); + match self.current.take() { + Some(_) => return Poll::Ready(Ok(Event::End)), + None => continue, + } + } + // A fixed instance ends with its owner, which follows that itself. + if let Feed::Announced(_) = self.feed { + self.current = Some(broadcast.clone()); + } + return Poll::Ready(Ok(Event::Start(broadcast))); + } + + if let Some(current) = &self.current + && current.poll_closed(waiter).is_ready() + { + self.current = None; + return Poll::Ready(Ok(Event::End)); + } + return Poll::Pending; + } + } + + /// The next change; see [`Self::poll_next`]. + pub async fn next(&mut self) -> crate::Result { + kio::wait(|waiter| self.poll_next(waiter)).await + } +} + +/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route +/// cannot prove its instance, so only a known mismatch counts. +fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { + matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::source::{produce_origin, resolved}; + use moq_net::path::Relative; + + /// Let the origin's driver run the fronts that announcements started. + async fn settle() { + for _ in 0..10 { + tokio::task::yield_now().await; + } + } + + fn route(epoch: &moq_net::Epoch) -> moq_net::origin::Route { + moq_net::origin::Route::default().with_epoch(epoch.clone()) + } + + /// The next change a follower reports, which must arrive within the origin's update hold. + async fn followed(follow: &mut Instances) -> Option { + let next = tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .expect("no change") + .expect("the follow failed"); + match next { + Event::Start(broadcast) => Some(broadcast), + Event::End => None, + } + } + + /// The follower stays quiet past the origin's update hold. + async fn quiet(follow: &mut Instances) { + if tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) + .await + .is_ok() + { + panic!("expected no change"); + } + } + + /// A sibling announced after the follow starts it, its end idles it, and its return starts + /// it again. + #[tokio::test(start_paused = true)] + async fn a_follow_starts_late_and_survives_an_end() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./source")), None).unwrap(); + quiet(&mut follow).await; + + let first = origin.publish("a/source", Default::default()).unwrap(); + let started = followed(&mut follow).await.expect("a start"); + assert_eq!(started.info().path.as_str(), "a/source"); + + drop(first); + assert!(followed(&mut follow).await.is_none(), "an end"); + quiet(&mut follow).await; + + let _second = origin.publish("a/source", Default::default()).unwrap(); + let restarted = followed(&mut follow).await.expect("a start"); + assert!(!restarted.is_clone(&started)); + } + + /// A replacement restarts a reference that names no epoch, and ends one that names the old + /// instance until that instance serves again. + #[tokio::test(start_paused = true)] + async fn a_follow_restarts_only_without_an_epoch() { + let origin = produce_origin(); + let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + let _old = origin.publish("a/source", route(&old_epoch)).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let rel = Relative::new("./source"); + let mut any = source.follow(Some(&rel), None).unwrap(); + let mut named = source.follow(Some(&rel), Some(&old_epoch)).unwrap(); + let old = followed(&mut any).await.unwrap(); + assert!(followed(&mut named).await.unwrap().is_clone(&old)); + + let new = origin.publish("a/source", route(&new_epoch)).unwrap(); + let replaced = followed(&mut any).await.unwrap(); + assert_eq!(replaced.info().epoch.as_ref(), Some(&new_epoch)); + assert!(followed(&mut named).await.is_none(), "another epoch ends it"); + quiet(&mut named).await; + // The old instance still serves the subscriptions already on it. + assert!(!old.is_closed()); + + // The replacement goes and the old instance wins again. + drop(new); + let back = followed(&mut named).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + let back = followed(&mut any).await.unwrap(); + assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); + } + + /// An epoch is best effort over an epochless route, which cannot prove another instance. + #[tokio::test(start_paused = true)] + async fn a_follow_reads_an_epochless_route_with_an_epoch() { + let origin = produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut named = source + .follow(Some(&Relative::new("./source")), Some(&moq_net::Epoch::mint())) + .unwrap(); + let _sibling = origin.publish("a/source", Default::default()).unwrap(); + assert!(followed(&mut named).await.is_some()); + } + + /// A self-reference reads only the catalog instance, whatever replaces it at the path. + #[tokio::test(start_paused = true)] + async fn a_follow_of_the_catalog_keeps_its_instance() { + let origin = produce_origin(); + let old = origin.publish("a/pub", Default::default()).unwrap(); + settle().await; + let source = resolved(&origin, "a/pub").await; + let mut follow = source.follow(Some(&Relative::new("./pub")), None).unwrap(); + assert!(followed(&mut follow).await.unwrap().is_clone(&source.broadcast())); + + let _new = origin.publish("a/pub", Default::default()).unwrap(); + quiet(&mut follow).await; + // Its end is the catalog's, which the catalog's reader sees. + drop(old); + quiet(&mut follow).await; + } +} diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 4b46cb87dde..e76649e1859 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -28,6 +28,8 @@ //! record per aligned segment, mapping a span of content time to the group //! ranges that carry it on each track, so consumers can seek or build //! HLS/DASH playlists without downloading media. +//! - [`follow`] follows a broadcast across the publisher instances that +//! serve it, for a reader that restarts on each one. pub mod binary; pub mod catalog; @@ -35,6 +37,7 @@ mod clock; pub mod codec; pub mod container; mod error; +pub mod follow; pub mod import; mod jitter; pub mod json; @@ -48,7 +51,7 @@ pub mod timeline; pub use clock::Clock; pub use error::*; pub use pace::Pacer; -pub use source::{Binding, Follow, Followed, Source}; +pub use source::{Binding, Source}; /// Translate a catalog entry's declared compression into the flag the codecs take. /// diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 72de27a2b12..499580260a4 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -23,13 +23,13 @@ type Pins = Arc> /// reference. Build one with [`Source::new`]. /// /// A source serves its own path from the broadcast it was built from, and keeps the first -/// broadcast it resolves for each sibling. Every later request for a path is served from that -/// handle ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), -/// [`resolve`](Self::resolve), [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), -/// and an export's track requests, which wait for a sibling's first announcement). A route -/// replaced after that is never resolved, epoch or not, so two publishers never splice into one -/// export; a request on a held broadcast that has ended fails instead. Clones share the held -/// broadcasts. +/// broadcast it resolves for each sibling, waiting for the sibling's first announcement if +/// nothing serves it yet. Every later request for a path is served from that handle +/// ([`broadcast`](Self::broadcast), [`catalog`](Self::catalog), [`resolve`](Self::resolve), +/// [`bind`](Self::bind), [`subscribe_track`](Self::subscribe_track), and an export's track +/// requests). A route replaced after that is never resolved, epoch or not, so two publishers +/// never splice into one export; a request on a held broadcast that has ended fails instead of +/// waiting for the next one. Clones share the held broadcasts. /// /// A reader that follows a sibling's announcements instead, restarting on each new instance, /// uses [`follow`](Self::follow), which holds nothing. @@ -137,27 +137,23 @@ impl Source { Binding::new(self.broadcast()) } - /// The broadcast `path` is pinned to, or a lookup that pins its answer, either checked - /// against `epoch` on every read. The lookup waits for the path's first announcement when - /// `announced`, and otherwise answers for the routes as they stand. - fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>, announced: bool) -> Binding { + /// The broadcast `path` is pinned to, or a lookup that waits for the path's first + /// announcement and pins its answer, either checked against `epoch` on every read. + fn request_path(&self, path: &moq_net::PathOwned, epoch: Option<&moq_net::Epoch>) -> Binding { let bound = match self.pins.lock().expect("pins poisoned").get(path) { Some(pinned) => Bound::Ready(pinned.clone()), // Unpinned even with an epoch: an epochless route still answers, best effort, and // the read refuses a route that proves another instance. - None => Bound::Requested { - lookup: match announced { - true => { - let mut routing = self.origin.routed_broadcast(path).into_inner(); - // Issue the request now: a handler answers it whether or not anyone reads yet. - let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); - Lookup::Announced(Box::new(Mutex::new(routing))) - } - false => Lookup::Now(self.origin.request_broadcast(path, None).into_inner()), - }, - pins: self.pins.clone(), - path: path.clone(), - }, + None => { + let mut routing = self.origin.routed_broadcast(path).into_inner(); + // Issue the request now: a handler answers it whether or not anyone reads yet. + let _ = kio::Task::poll(&mut routing, &kio::Waiter::noop()); + Bound::Requested { + routing: Box::new(Mutex::new(routing)), + pins: self.pins.clone(), + path: path.clone(), + } + } }; Binding { bound, @@ -191,15 +187,14 @@ impl Source { } /// The skipping counterpart to [`Self::bind`] for a catalog entry, returning `None` when its - /// broadcast walks above the origin root. Unlike a binding, it waits for a sibling nothing - /// announces yet, so a late first announcement starts the rendition. + /// broadcast walks above the origin root. /// /// The exporters use it to drop a single bad rendition rather than fail the whole export. /// [`Self::retain_selectable`] already removes those renditions, so this is the second half of /// the same policy rather than a different one. pub(crate) fn try_request(&self, reference: &impl hang::catalog::Reference) -> Option { let path = self.resolve_reference(reference.broadcast())?; - Some(self.request_path(&path, reference.epoch(), true)) + Some(self.request_path(&path, reference.epoch())) } /// Remove the renditions an export must not select: a disabled one (catalog @@ -245,9 +240,10 @@ impl Source { /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) /// resolves the catalog broadcast itself; anything else fetches the referenced - /// broadcast from the origin. `epoch` is the rendition's `epoch` field, the instance the - /// broadcast must be (see [`Source`]). Use it when you need the broadcast handle itself - /// (e.g. to FETCH individual groups) rather than a subscription. + /// broadcast from the origin, waiting for its first announcement if nothing serves it yet. + /// `epoch` is the rendition's `epoch` field, the instance the broadcast must be (see + /// [`Source`]). Use it when you need the broadcast handle itself (e.g. to FETCH individual + /// groups) rather than a subscription. /// /// A reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast): the rendition names @@ -263,8 +259,9 @@ impl Source { /// Start one broadcast request now and retain its result for repeated reads. /// /// Unlike [`resolve`](Self::resolve), this issues the request without awaiting or polling. - /// A remote or dynamic handler may answer later. The binding never retries a failed request - /// or resolves the path again after a publisher is replaced; [`follow`](Self::follow) does. + /// A path nothing announces yet resolves once something does, and a remote or dynamic + /// handler may answer later. Once answered, the binding never resolves the path again, even + /// after its publisher is replaced or goes; [`follow`](Self::follow) does. /// /// A path this source already resolved binds to that broadcast; otherwise the origin is /// asked, deduplicating repeat requests for one path so the catalog and every rendition share @@ -278,15 +275,16 @@ impl Source { rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, ) -> crate::Result { - Ok(self.request_path(&self.target(rel)?, epoch, false)) + Ok(self.request_path(&self.target(rel)?, epoch)) } /// Resolve an optional cross-broadcast reference and subscribe to track `name`, /// awaiting SUBSCRIBE_OK. /// /// `rel` is a rendition's catalog `broadcast` field: `None` (or an empty reference) - /// subscribes on the catalog broadcast; anything else fetches the referenced broadcast - /// from the origin first, checked against `epoch` as in [`resolve`](Self::resolve). A + /// subscribes on the catalog broadcast; anything else is resolved as in + /// [`resolve`](Self::resolve), waiting for its first announcement and checked against + /// `epoch`. A /// reference that escapes above the origin root is /// [`Error::EscapingBroadcast`](crate::Error::EscapingBroadcast). /// @@ -318,126 +316,23 @@ impl Source { &self, rel: Option<&moq_net::path::Relative<'_>>, epoch: Option<&moq_net::Epoch>, - ) -> crate::Result { + ) -> crate::Result { let path = self.target(rel)?; - let instances = match path == self.path { - true => Instances::Fixed(Some(self.broadcast())), - false => Instances::Announced(Box::new(self.origin.announced_broadcast(&path, None)?)), - }; - Ok(Follow::with(instances, epoch)) - } -} - -/// The publisher instances serving a broadcast, as each comes online, takes over, and goes. -/// -/// Built on [`announced_broadcast`](moq_net::origin::Consumer::announced_broadcast), since an -/// announcement is the online and offline signal. Create one with [`Source::follow`] for a -/// catalog reference, or [`Follow::new`] for a path. -pub struct Follow { - instances: Instances, - /// The instance a reference named, if any. - epoch: Option, - /// The instance handed out last, until it stops serving. - current: Option, -} - -enum Instances { - /// Every instance announced at a path. - Announced(Box), - /// One instance, handed out once. - Fixed(Option), -} - -/// A change to the instance a [`Follow`] serves. -pub enum Followed { - /// Start on this instance, dropping whatever was read from the one before. - Start(moq_net::broadcast::Consumer), - /// The instance handed out last stopped serving: idle until the next start. - End, -} - -impl Follow { - /// Follow the broadcast at `path` through `origin`, handing out only `epoch` when given. - /// - /// Unlike [`Source::follow`], this needs no catalog broadcast, so a reader can follow the - /// broadcast whose catalog it reads and build a [`Source`] for each instance. - pub fn new( - origin: &moq_net::origin::Consumer, - path: impl moq_net::AsPath, - epoch: Option<&moq_net::Epoch>, - ) -> crate::Result { - let announced = origin.announced_broadcast(path, None)?; - Ok(Self::with(Instances::Announced(Box::new(announced)), epoch)) - } - - fn with(instances: Instances, epoch: Option<&moq_net::Epoch>) -> Self { - Self { - instances, - epoch: epoch.cloned(), - current: None, - } - } - - /// Poll for the next change, registering `waiter` when there is none yet. - /// - /// Each announcement that brings an instance hands it out, so the same one returns after an - /// end. Fails once the origin closes, and with any resolution failure, such as a handler - /// refusing the path. - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { - loop { - let next = match &mut self.instances { - // Followed unpinned and checked here: a pinned follow waits out an epochless route, - // which an epoch reads best effort. - Instances::Announced(announced) => announced.poll_available(waiter).map_err(crate::Error::from), - Instances::Fixed(broadcast) => match broadcast.take() { - Some(broadcast) => Poll::Ready(Ok(broadcast)), - None => Poll::Pending, - }, - }; - if let Poll::Ready(next) = next { - let broadcast = next?; - if proves_other(self.epoch.as_ref(), &broadcast) { - tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); - match self.current.take() { - Some(_) => return Poll::Ready(Ok(Followed::End)), - None => continue, - } - } - // The catalog's own instance ends with the catalog, which its reader follows. - if let Instances::Announced(_) = self.instances { - self.current = Some(broadcast.clone()); - } - return Poll::Ready(Ok(Followed::Start(broadcast))); - } - - if let Some(current) = &self.current - && current.poll_closed(waiter).is_ready() - { - self.current = None; - return Poll::Ready(Ok(Followed::End)); - } - return Poll::Pending; - } - } - - /// The next change; see [`Self::poll_next`]. - pub async fn next(&mut self) -> crate::Result { - kio::wait(|waiter| self.poll_next(waiter)).await + Ok(match path == self.path { + true => crate::follow::Instances::fixed(self.broadcast(), epoch), + false => crate::follow::Instances::new(&self.origin, &path, epoch)?, + }) } } -/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route -/// cannot prove its instance, so only a known mismatch counts. -fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { - matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) -} - /// A held broadcast or the retained result of one eagerly issued broadcast request. /// -/// Reading the binding never looks up its path again, even after a failure or publisher -/// replacement. Pending requests follow the origin's routing semantics; issuing a request -/// does not establish an epoch relationship with a separate catalog broadcast. A binding made -/// with an epoch refuses a broadcast proven to serve another one. +/// A pending request waits for a path nothing announces yet and rides out route churn until it +/// resolves (see [`routed_broadcast`](moq_net::origin::Consumer::routed_broadcast)). Once the +/// path is pinned, reading the binding answers that pin and never looks the path up again, even +/// after its publisher is replaced or goes. Issuing a request does not establish an epoch +/// relationship with a separate catalog broadcast. A binding made with an epoch refuses a +/// broadcast proven to serve another one. /// /// Build one with [`Source::bind`], or with [`Binding::new`] for a broadcast already in hand. pub struct Binding { @@ -448,20 +343,12 @@ pub struct Binding { sibling: bool, } -/// How a [`Bound::Requested`] path is looked up. -enum Lookup { - /// For the routes as they stand. - Now(moq_net::origin::Requesting), - /// Once something announces the path, riding out route churn. - Announced(Box>), -} - enum Bound { /// A broadcast the caller already holds. Ready(moq_net::broadcast::Consumer), - /// A request issued when the binding was made. + /// A request issued when the binding was made, resolved once something announces the path. Requested { - lookup: Lookup, + routing: Box>, /// Where the answer is pinned, the first for its path winning. pins: Pins, path: moq_net::PathOwned, @@ -489,20 +376,22 @@ impl Binding { pub fn poll_broadcast(&self, waiter: &kio::Waiter) -> Poll> { let broadcast = match &self.bound { Bound::Ready(broadcast) => broadcast.clone(), - Bound::Requested { lookup, pins, path } => { - let broadcast = match lookup { - Lookup::Now(request) => ready!(request.poll_ok(waiter))?, - // A resolved routing answers the same result on every poll. - Lookup::Announced(routing) => { + Bound::Requested { routing, pins, path } => { + // Once a path is pinned, its pin is the answer: a request that raced another for + // the path takes the one pinned first, and the routing is never polled again, since + // it would wait out the gap after the pinned instance goes. + if let Some(pinned) = pins.lock().expect("pins poisoned").get(path) { + pinned.clone() + } else { + let broadcast = { let mut routing = routing.lock().expect("routing poisoned"); ready!(kio::Task::poll(&mut *routing, waiter))? - } - }; - // A request that raced another for the path takes the one pinned first. The path's - // first answer is pinned even when it fails this binding's epoch, so no later - // reference resolves the path independently. - let mut pins = pins.lock().expect("pins poisoned"); - pins.entry(path.clone()).or_insert(broadcast).clone() + }; + // The path's first answer is pinned even when it fails this binding's epoch, so + // no later reference resolves the path independently. + let mut pins = pins.lock().expect("pins poisoned"); + pins.entry(path.clone()).or_insert(broadcast).clone() + } } }; // An epochless route cannot prove its instance, so only a known mismatch fails. @@ -618,38 +507,21 @@ mod tests { assert!(first.is_clone(&binding.broadcast().await.unwrap())); } - /// A failed lookup pins nothing: the guarantee starts at a path's first resolution. - #[tokio::test] - async fn failed_binding_does_not_retry_a_later_publisher() { - let origin = produce_origin(); - let _catalog = origin.publish("pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "pub").await; - let media = Relative::new("media"); - let binding = source.bind(Some(&media), None).unwrap(); - assert!(binding.broadcast().await.is_err()); - let _publisher = origin.publish("media", Default::default()).unwrap(); - settle().await; - assert!(!source.resolve(Some(&media), None).await.unwrap().is_closed()); - assert!(binding.broadcast().await.is_err()); - } - - /// An export's request made before anything announces its sibling resolves on the first - /// announcement, and keeps that instance once it is replaced. + /// A binding made before anything announces its sibling resolves on the first announcement + /// and keeps that instance: past its replacement, and once it goes, answering the ended + /// instance at once rather than waiting for the next one. #[tokio::test(start_paused = true)] - async fn an_export_request_waits_for_the_first_announcement() { + async fn a_binding_waits_for_the_first_announcement_and_keeps_it() { let origin = produce_origin(); let _catalog = origin.publish("pub", Default::default()).unwrap(); settle().await; let source = resolved(&origin, "pub").await; let media = Relative::new("media"); - let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); - config.broadcast = Some(media.to_owned()); - let binding = source.try_request(&config).unwrap(); + let binding = source.bind(Some(&media), None).unwrap(); settle().await; assert!(binding.poll_broadcast(&kio::Waiter::noop()).is_pending()); - let _first = origin.publish("media", Default::default()).unwrap(); + let first_publisher = origin.publish("media", Default::default()).unwrap(); let first = binding.broadcast().await.unwrap(); assert!(!first.is_closed()); @@ -657,6 +529,19 @@ mod tests { settle().await; assert!(binding.broadcast().await.unwrap().is_clone(&first)); assert!(source.resolve(Some(&media), None).await.unwrap().is_clone(&first)); + + drop(first_publisher); + first.closed().await; + assert!(binding.broadcast().await.unwrap().is_clone(&first)); + let late = source.bind(Some(&media), None).unwrap(); + assert!( + late.broadcast().await.unwrap().is_clone(&first), + "a later binding reads the pin" + ); + + // A new catalog instance starts fresh pins, which resolve the replacement. + let fresh = source.switched(&source.broadcast()); + assert!(!fresh.resolve(Some(&media), None).await.unwrap().is_closed()); } /// A source serves its path from the broadcast it was built from, even when a replacement @@ -794,115 +679,6 @@ mod tests { assert!(source.resolve(Some(&rel), None).await.unwrap().is_clone(&first)); } - /// The next change a follower reports, which must arrive within the origin's update hold. - async fn followed(follow: &mut Follow) -> Option { - let next = tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) - .await - .expect("no change") - .expect("the follow failed"); - match next { - Followed::Start(broadcast) => Some(broadcast), - Followed::End => None, - } - } - - /// The follower stays quiet past the origin's update hold. - async fn quiet(follow: &mut Follow) { - if tokio::time::timeout(std::time::Duration::from_secs(1), follow.next()) - .await - .is_ok() - { - panic!("expected no change"); - } - } - - /// A sibling announced after the follow starts it, its end idles it, and its return starts - /// it again. - #[tokio::test(start_paused = true)] - async fn a_follow_starts_late_and_survives_an_end() { - let origin = produce_origin(); - let _catalog = origin.publish("a/pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let mut follow = source.follow(Some(&Relative::new("./source")), None).unwrap(); - quiet(&mut follow).await; - - let first = origin.publish("a/source", Default::default()).unwrap(); - let started = followed(&mut follow).await.expect("a start"); - assert_eq!(started.info().path.as_str(), "a/source"); - - drop(first); - assert!(followed(&mut follow).await.is_none(), "an end"); - quiet(&mut follow).await; - - let _second = origin.publish("a/source", Default::default()).unwrap(); - let restarted = followed(&mut follow).await.expect("a start"); - assert!(!restarted.is_clone(&started)); - } - - /// A replacement restarts a reference that names no epoch, and ends one that names the old - /// instance until that instance serves again. - #[tokio::test(start_paused = true)] - async fn a_follow_restarts_only_without_an_epoch() { - let origin = produce_origin(); - let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); - let _catalog = origin.publish("a/pub", Default::default()).unwrap(); - let _old = origin.publish("a/source", route(&old_epoch)).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let rel = Relative::new("./source"); - let mut any = source.follow(Some(&rel), None).unwrap(); - let mut named = source.follow(Some(&rel), Some(&old_epoch)).unwrap(); - let old = followed(&mut any).await.unwrap(); - assert!(followed(&mut named).await.unwrap().is_clone(&old)); - - let new = origin.publish("a/source", route(&new_epoch)).unwrap(); - let replaced = followed(&mut any).await.unwrap(); - assert_eq!(replaced.info().epoch.as_ref(), Some(&new_epoch)); - assert!(followed(&mut named).await.is_none(), "another epoch ends it"); - quiet(&mut named).await; - // The old instance still serves the subscriptions already on it. - assert!(!old.is_closed()); - - // The replacement goes and the old instance wins again. - drop(new); - let back = followed(&mut named).await.unwrap(); - assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); - let back = followed(&mut any).await.unwrap(); - assert_eq!(back.info().epoch.as_ref(), Some(&old_epoch)); - } - - /// An epoch is best effort over an epochless route, which cannot prove another instance. - #[tokio::test(start_paused = true)] - async fn a_follow_reads_an_epochless_route_with_an_epoch() { - let origin = produce_origin(); - let _catalog = origin.publish("a/pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let mut named = source - .follow(Some(&Relative::new("./source")), Some(&moq_net::Epoch::mint())) - .unwrap(); - let _sibling = origin.publish("a/source", Default::default()).unwrap(); - assert!(followed(&mut named).await.is_some()); - } - - /// A self-reference reads only the catalog instance, whatever replaces it at the path. - #[tokio::test(start_paused = true)] - async fn a_follow_of_the_catalog_keeps_its_instance() { - let origin = produce_origin(); - let old = origin.publish("a/pub", Default::default()).unwrap(); - settle().await; - let source = resolved(&origin, "a/pub").await; - let mut follow = source.follow(Some(&Relative::new("./pub")), None).unwrap(); - assert!(followed(&mut follow).await.unwrap().is_clone(&source.broadcast())); - - let _new = origin.publish("a/pub", Default::default()).unwrap(); - quiet(&mut follow).await; - // Its end is the catalog's, which the catalog's reader sees. - drop(old); - quiet(&mut follow).await; - } - /// A self-reference's epoch must not contradict the catalog broadcast's own. #[tokio::test(start_paused = true)] async fn a_self_reference_epoch_checks_the_catalog_instance() { diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index 044aa7fa273..d35929168fc 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `BroadcastConsumer.resolve(_:)` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/swift/ffi/MIGRATE.md b/swift/ffi/MIGRATE.md index 3552e06292f..9ac7e18f93d 100644 --- a/swift/ffi/MIGRATE.md +++ b/swift/ffi/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A broadcast reference waits for its sibling's announcement (#6254) + +Resolving a rendition's `broadcast` reference, through `MoqBroadcastConsumer.resolve(reference:)` and the rendition-based subscribes and fetches, now waits for a sibling nothing announces yet instead of failing as unroutable, so a late sibling plays once it is announced. +Where you skipped such a rendition on that error, bound the call yourself: cancel the task. +Regenerate the bindings for the changed `resolve` checksum. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer.finish()` lets go, instead of unlisting when its producer finishes or is released. From cd9fb914fb0084a41bbe68e88839ce952fcd5bc3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 00:43:07 -0700 Subject: [PATCH 06/27] fix(gst,mux): share a restarted sibling's timeline; fail a contradicted self-follow moqsrc renditions restarting on the same sibling instance now share that instance's timeline, so audio and video fix one segment mapping instead of each starting from its own first frame. Source::follow fails a self-reference whose epoch the catalog instance contradicts with EpochMismatch, rather than parking forever on a fixed feed no other instance can join. Binding and follow share one epoch check. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 68 +++++++++++++++++++++++++++++++++--- rs/moq-mux/src/follow.rs | 12 ++----- rs/moq-mux/src/source.rs | 46 ++++++++++++++++-------- 3 files changed, 99 insertions(+), 27 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 8564c8a94b3..186ee91c4b5 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -767,6 +767,7 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), + restarts: Restarts::default(), finished: Arc::default(), }; let mut active: HashMap = HashMap::new(); @@ -1016,6 +1017,7 @@ fn reconcile( descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), + restarts: playback.restarts.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1071,9 +1073,37 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, + restarts: Restarts, finished: Finished, } +/// The timeline of each sibling instance a rendition restarted on, shared by every rendition +/// reading that instance so their first frames fix one mapping for all of them. +#[derive(Clone, Default)] +struct Restarts(Arc>>); + +/// An instance some rendition restarted on, and the timeline its renditions share while any reads it. +type Restart = (moq_net::broadcast::Consumer, std::sync::Weak); + +impl Restarts { + /// The timeline of `instance`, new unless another rendition already restarted on it. + fn timeline(&self, instance: &moq_net::broadcast::Consumer) -> Arc { + let mut restarts = self.0.lock().unwrap(); + // Forget an instance no rendition reads any more, releasing its handle. + restarts.retain(|(_, timeline)| timeline.strong_count() > 0); + if let Some(timeline) = restarts + .iter() + .find(|(known, _)| known.is_clone(instance)) + .and_then(|(_, timeline)| timeline.upgrade()) + { + return timeline; + } + let timeline = Arc::default(); + restarts.push((instance.clone(), Arc::downgrade(&timeline))); + timeline + } +} + /// Each pad's rendition with a track it played to the end. A new subscription to a finished /// track replays its last group, so a catalog update that still lists the rendition (a /// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A @@ -1117,6 +1147,8 @@ struct Pump { descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, + /// The run's [`Playback::restarts`]. + restarts: Restarts, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1155,6 +1187,7 @@ impl Pump { descriptor, pads, mut timeline, + restarts, finished, state, mut cancel, @@ -1237,10 +1270,10 @@ impl Pump { let mut track = moq_mux::container::Consumer::new(subscriber, container); let pad = match &owned { - // Another instance on the pad this pump owns: a new stream, on a timeline of its own, - // since the instance's timestamps share no origin with the run's. + // Another instance on the pad this pump owns: a new stream, on that instance's own + // timeline, since its timestamps share no origin with the run's. Some((pad, _)) => { - timeline = Arc::default(); + timeline = restarts.timeline(&broadcast); tokio::task::block_in_place(|| begin(pad, &name, &caps)); pad.clone() } @@ -1524,9 +1557,10 @@ fn audio_caps(config: &hang::catalog::AudioConfig) -> Result { #[cfg(test)] mod tests { - use super::{PumpState, Timeline, ended, plan_reconcile}; + use super::{PumpState, Restarts, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; + use std::sync::Arc; /// A pump cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -1624,6 +1658,31 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } + /// Renditions restarting on the same sibling instance share its timeline, and the next + /// instance starts another. + #[tokio::test(start_paused = true)] + async fn renditions_restarting_on_one_instance_share_its_timeline() { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + tokio::spawn(moq_net::time::run(driver)); + let consumer = origin.consume(); + let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { + moq_mux::follow::Event::Start(broadcast) => broadcast, + moq_mux::follow::Event::End => panic!("expected a start"), + }; + let restarts = Restarts::default(); + + let _first = origin.publish("source", Default::default()).unwrap(); + let shared = restarts.timeline(&started(&mut audio).await); + assert!(Arc::ptr_eq(&shared, &restarts.timeline(&started(&mut video).await))); + + let _second = origin.publish("source", Default::default()).unwrap(); + let next = restarts.timeline(&started(&mut audio).await); + assert!(!Arc::ptr_eq(&shared, &next), "another instance has its own timeline"); + assert!(Arc::ptr_eq(&next, &restarts.timeline(&started(&mut video).await))); + } + #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -2873,6 +2932,7 @@ mod session_tests { descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), + restarts: super::Restarts::default(), finished: std::sync::Arc::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, diff --git a/rs/moq-mux/src/follow.rs b/rs/moq-mux/src/follow.rs index eb149f7a593..9823777dd5d 100644 --- a/rs/moq-mux/src/follow.rs +++ b/rs/moq-mux/src/follow.rs @@ -49,8 +49,8 @@ impl Instances { } /// Hand out `broadcast` once and never report its end, which its owner follows. - pub(crate) fn fixed(broadcast: moq_net::broadcast::Consumer, epoch: Option<&moq_net::Epoch>) -> Self { - Self::with(Feed::Fixed(Some(broadcast)), epoch) + pub(crate) fn fixed(broadcast: moq_net::broadcast::Consumer) -> Self { + Self::with(Feed::Fixed(Some(broadcast)), None) } fn with(feed: Feed, epoch: Option<&moq_net::Epoch>) -> Self { @@ -79,7 +79,7 @@ impl Instances { }; if let Poll::Ready(next) = next { let broadcast = next?; - if proves_other(self.epoch.as_ref(), &broadcast) { + if crate::source::check_epoch(self.epoch.as_ref(), &broadcast).is_err() { tracing::debug!(path = %broadcast.info().path, "skipping an instance of another epoch"); match self.current.take() { Some(_) => return Poll::Ready(Ok(Event::End)), @@ -109,12 +109,6 @@ impl Instances { } } -/// Whether `broadcast` is proven to be another instance than `epoch` names. An epochless route -/// cannot prove its instance, so only a known mismatch counts. -fn proves_other(epoch: Option<&moq_net::Epoch>, broadcast: &moq_net::broadcast::Consumer) -> bool { - matches!((epoch, &broadcast.info().epoch), (Some(expected), Some(actual)) if expected != actual) -} - #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 499580260a4..a1a5cf58aeb 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -311,7 +311,9 @@ impl Source { /// /// With an `epoch`, only that instance (or an epochless route, which cannot prove another /// one) is handed out: another taking over ends the rendition rather than substituting for - /// it. Rejects an escaping reference exactly as [`resolve`](Self::resolve) does. + /// it. A self-reference whose epoch the catalog instance contradicts fails with + /// [`Error::EpochMismatch`](crate::Error::EpochMismatch), since no other instance can come. + /// Rejects an escaping reference exactly as [`resolve`](Self::resolve) does. pub fn follow( &self, rel: Option<&moq_net::path::Relative<'_>>, @@ -319,12 +321,34 @@ impl Source { ) -> crate::Result { let path = self.target(rel)?; Ok(match path == self.path { - true => crate::follow::Instances::fixed(self.broadcast(), epoch), + true => { + let broadcast = self.broadcast(); + check_epoch(epoch, &broadcast)?; + crate::follow::Instances::fixed(broadcast) + } false => crate::follow::Instances::new(&self.origin, &path, epoch)?, }) } } +/// Fail with [`Error::EpochMismatch`](crate::Error::EpochMismatch) when `broadcast` is proven to +/// be another instance than `epoch` names. An epochless route cannot prove its instance, so only +/// a known mismatch fails. +pub(crate) fn check_epoch( + epoch: Option<&moq_net::Epoch>, + broadcast: &moq_net::broadcast::Consumer, +) -> crate::Result<()> { + let info = broadcast.info(); + match (epoch, &info.epoch) { + (Some(expected), Some(actual)) if expected != actual => Err(crate::Error::EpochMismatch { + path: info.path.to_string(), + expected: expected.clone(), + actual: actual.clone(), + }), + _ => Ok(()), + } +} + /// A held broadcast or the retained result of one eagerly issued broadcast request. /// /// A pending request waits for a path nothing announces yet and rides out route churn until it @@ -394,18 +418,7 @@ impl Binding { } } }; - // An epochless route cannot prove its instance, so only a known mismatch fails. - let info = broadcast.info(); - if let (Some(expected), Some(actual)) = (&self.epoch, &info.epoch) - && expected != actual - { - return Poll::Ready(Err(crate::Error::EpochMismatch { - path: info.path.to_string(), - expected: expected.clone(), - actual: actual.clone(), - })); - } - Poll::Ready(Ok(broadcast)) + Poll::Ready(check_epoch(self.epoch.as_ref(), &broadcast).map(|()| broadcast)) } /// The bound broadcast, awaiting the origin's answer when it hasn't arrived yet. @@ -693,6 +706,11 @@ mod tests { let own = source.resolve(rel.as_ref(), Some(&catalog)).await.unwrap(); assert!(own.is_clone(&source.broadcast()), "{reference:?}"); assert!(is_mismatch(source.resolve(rel.as_ref(), Some(&other)).await, &other)); + assert!(source.follow(rel.as_ref(), Some(&catalog)).is_ok()); + assert!(matches!( + source.follow(rel.as_ref(), Some(&other)), + Err(crate::Error::EpochMismatch { .. }) + )); } } From a36fb1cf9cb4a7c3f16ff1d8012d233c1ed9726a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 00:52:38 -0700 Subject: [PATCH 07/27] fix(gst): key sibling timelines by instance; a finished sibling track idles Every moqsrc rendition picks its sibling instance's timeline when it starts reading it, not only once it owns a pad, so one still subscribing when a replacement takes over shares the replacement's mapping. The first instance at a path keeps the run's timeline. A sibling track finishing while its instance stays up idles the pad and keeps following, so the instance that replaces it starts a new stream there; the finish record clears once a new instance starts on the pad. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 129 ++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 38 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 186ee91c4b5..b8d43d916e8 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -767,7 +767,7 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), - restarts: Restarts::default(), + siblings: Siblings::default(), finished: Arc::default(), }; let mut active: HashMap = HashMap::new(); @@ -1017,7 +1017,7 @@ fn reconcile( descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), - restarts: playback.restarts.clone(), + siblings: playback.siblings.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1073,33 +1073,46 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, - restarts: Restarts, + siblings: Siblings, finished: Finished, } -/// The timeline of each sibling instance a rendition restarted on, shared by every rendition -/// reading that instance so their first frames fix one mapping for all of them. +/// The timeline of each sibling instance, shared by every rendition reading that instance so +/// their first frames fix one mapping for all of them. #[derive(Clone, Default)] -struct Restarts(Arc>>); - -/// An instance some rendition restarted on, and the timeline its renditions share while any reads it. -type Restart = (moq_net::broadcast::Consumer, std::sync::Weak); - -impl Restarts { - /// The timeline of `instance`, new unless another rendition already restarted on it. - fn timeline(&self, instance: &moq_net::broadcast::Consumer) -> Arc { - let mut restarts = self.0.lock().unwrap(); - // Forget an instance no rendition reads any more, releasing its handle. - restarts.retain(|(_, timeline)| timeline.strong_count() > 0); - if let Some(timeline) = restarts +struct Siblings(Arc>); + +#[derive(Default)] +struct SiblingTimelines { + /// Each sibling path some instance served this run. + seen: HashSet, + /// Each instance still serving, and the timeline its renditions share while any reads it. + live: Vec<(moq_net::broadcast::Consumer, std::sync::Weak)>, +} + +impl Siblings { + /// The timeline of `instance`: the run's for the first instance at its path, which the + /// catalog's own renditions share, and one of its own for any later instance, whose + /// timestamps share no origin with the run's. + fn timeline(&self, instance: &moq_net::broadcast::Consumer, run: &Arc) -> Arc { + let mut siblings = self.0.lock().unwrap(); + // Forget an instance that ended or that no rendition reads any more, releasing its handle. + siblings + .live + .retain(|(known, timeline)| !known.is_closed() && timeline.strong_count() > 0); + if let Some(timeline) = siblings + .live .iter() .find(|(known, _)| known.is_clone(instance)) .and_then(|(_, timeline)| timeline.upgrade()) { return timeline; } - let timeline = Arc::default(); - restarts.push((instance.clone(), Arc::downgrade(&timeline))); + let timeline = match siblings.seen.insert(instance.info().path.clone()) { + true => run.clone(), + false => Arc::default(), + }; + siblings.live.push((instance.clone(), Arc::downgrade(&timeline))); timeline } } @@ -1147,8 +1160,8 @@ struct Pump { descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, - /// The run's [`Playback::restarts`]. - restarts: Restarts, + /// The run's [`Playback::siblings`]. + siblings: Siblings, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1186,8 +1199,8 @@ impl Pump { container, descriptor, pads, - mut timeline, - restarts, + timeline: run, + siblings, finished, state, mut cancel, @@ -1268,12 +1281,16 @@ impl Pump { }, }; let mut track = moq_mux::container::Consumer::new(subscriber, container); + let timeline = match sibling { + true => siblings.timeline(&broadcast, &run), + false => run.clone(), + }; let pad = match &owned { - // Another instance on the pad this pump owns: a new stream, on that instance's own - // timeline, since its timestamps share no origin with the run's. + // Another instance on the pad this pump owns: a new stream, and a track that + // finished on the old instance is not finished on this one. Some((pad, _)) => { - timeline = restarts.timeline(&broadcast); + finished.lock().unwrap().remove(&((kind, name.clone()), media.clone())); tokio::task::block_in_place(|| begin(pad, &name, &caps)); pad.clone() } @@ -1392,6 +1409,13 @@ impl Pump { // A sibling's track stopping with its instance is that instance's end, which // idles the pad for the next one rather than ending the rendition. Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, + // A sibling's track finishing ends it for this instance only: the pad idles + // for the next instance, and the record keeps a reshape on this one from + // replaying it. + Ok(None) if sibling => { + finished.lock().unwrap().insert(((kind, name.clone()), media.clone())); + break Stopped::Idle; + } Ok(None) => { // Recorded while this pump still owns the pad, before any replacement can claim it. finished.lock().unwrap().insert(((kind, name.clone()), media)); @@ -1557,7 +1581,7 @@ fn audio_caps(config: &hang::catalog::AudioConfig) -> Result { #[cfg(test)] mod tests { - use super::{PumpState, Restarts, Timeline, ended, plan_reconcile}; + use super::{PumpState, Siblings, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; use std::sync::Arc; @@ -1658,29 +1682,31 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } - /// Renditions restarting on the same sibling instance share its timeline, and the next - /// instance starts another. + /// Renditions on the same sibling instance share its timeline, whenever each starts on it: + /// the first instance shares the run's, and the next one starts another. #[tokio::test(start_paused = true)] - async fn renditions_restarting_on_one_instance_share_its_timeline() { + async fn renditions_on_one_sibling_instance_share_its_timeline() { let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); tokio::spawn(moq_net::time::run(driver)); let consumer = origin.consume(); let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { moq_mux::follow::Event::Start(broadcast) => broadcast, moq_mux::follow::Event::End => panic!("expected a start"), }; - let restarts = Restarts::default(); + let (siblings, run) = (Siblings::default(), Arc::::default()); let _first = origin.publish("source", Default::default()).unwrap(); - let shared = restarts.timeline(&started(&mut audio).await); - assert!(Arc::ptr_eq(&shared, &restarts.timeline(&started(&mut video).await))); + let first = started(&mut audio).await; + assert!(Arc::ptr_eq(&run, &siblings.timeline(&first, &run))); + // A video rendition that first starts on the replacement, as one still subscribing when it + // took over does, shares its timeline with the audio rendition restarting onto it. let _second = origin.publish("source", Default::default()).unwrap(); - let next = restarts.timeline(&started(&mut audio).await); - assert!(!Arc::ptr_eq(&shared, &next), "another instance has its own timeline"); - assert!(Arc::ptr_eq(&next, &restarts.timeline(&started(&mut video).await))); + let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let next = siblings.timeline(&started(&mut video).await, &run); + assert!(!Arc::ptr_eq(&run, &next), "another instance has its own timeline"); + assert!(Arc::ptr_eq(&next, &siblings.timeline(&started(&mut audio).await, &run))); } #[test] @@ -2932,7 +2958,7 @@ mod session_tests { descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), - restarts: super::Restarts::default(), + siblings: super::Siblings::default(), finished: std::sync::Arc::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, @@ -3222,6 +3248,33 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A sibling track finishing while its instance stays up idles the pad, and the instance that + /// replaces it starts a new stream there. + #[test] + fn a_finished_sibling_track_returns_on_the_next_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + assert!(!recorder.seen().contains(&Seen::Eos(pad)), "the finish ended the pad"); + recorder.assert_one_pad_held(); + drop(first); + } + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on /// the same pad, and nothing more from the old instance gets through. fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { From 92f6749ba4944c0acda68d91258e5d1a13f49a22 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:03:34 -0700 Subject: [PATCH 08/27] fix(gst): a delisted sibling rendition follows no later instance A rendition the catalog delists drains the instance it reads, then ends its pad instead of idling for, or restarting on, the sibling's next instance. One already idle when the delist lands ends right away. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 90 ++++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index b8d43d916e8..ec11523829d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -418,7 +418,12 @@ struct Shape { /// stopping it means it must never create one; after, it owns a pad and has to drop it. One /// compare-exchange settles which of the two happened, so a pad can't slip out between a /// teardown's check and the pump's creation. -struct PumpState(AtomicU8); +struct PumpState { + phase: AtomicU8, + /// The latest catalog still lists the rendition. A delisted pump drains the instance it reads + /// and follows no other. + listed: watch::Sender, +} impl PumpState { const SUBSCRIBING: u8 = 0; @@ -426,12 +431,15 @@ impl PumpState { const CANCELLED: u8 = 2; fn new() -> Self { - Self(AtomicU8::new(Self::SUBSCRIBING)) + Self { + phase: AtomicU8::new(Self::SUBSCRIBING), + listed: watch::Sender::new(true), + } } /// Claim the right to create a pad, false once a teardown got here first. fn go_live(&self) -> bool { - self.0 + self.phase .compare_exchange(Self::SUBSCRIBING, Self::LIVE, Ordering::AcqRel, Ordering::Acquire) .is_ok() } @@ -439,10 +447,19 @@ impl PumpState { /// Stop a pump that hasn't taken a pad, false if it already has one and must be torn down /// through its cancel watch instead. fn cancel_before_live(&self) -> bool { - self.0 + self.phase .compare_exchange(Self::SUBSCRIBING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + + fn set_listed(&self, listed: bool) { + self.listed + .send_if_modified(|current| std::mem::replace(current, listed) != listed); + } + + fn is_listed(&self) -> bool { + *self.listed.borrow() + } } /// A rendition by kind and moq track name: what a pad is kept by. @@ -931,6 +948,12 @@ fn reconcile( .collect(), ); + // A delisted pump drains the instance it reads and follows no other; a relisted one follows + // again. + for (name, track) in active.iter() { + track.state.set_listed(desired.contains_key(name)); + } + // Cancel anything that changed shape; each cancelled pump hands its pad back. Changed // renditions also land in `plan.add`, so they respawn below on the same pad. // @@ -1186,7 +1209,8 @@ impl Pump { /// run can end the pad of one the catalog retired. /// /// A sibling's announcements drive the pump: its first start binds the rendition, an end - /// idles the pad, and another instance taking over restarts the stream on the same pad. + /// idles the pad, and another instance taking over restarts the stream on the same pad, until + /// the catalog delists the rendition. async fn run(self) -> Option { let Pump { element, @@ -1210,12 +1234,18 @@ impl Pump { // The pad once this pump owns it, from the first instance on. let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; let mut next: Option = None; + let mut listed = state.listed.subscribe(); loop { + // A delisted rendition drained the instance it read, and follows no other. + if owned.is_some() && !state.is_listed() { + return Some((kind, name)); + } let broadcast = match next.take() { Some(broadcast) => broadcast, None => tokio::select! { _ = cancel.changed() => return None, + _ = listed.wait_for(|listed| !listed), if owned.is_some() => return Some((kind, name)), followed = follow.next() => match followed { Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, Ok(moq_mux::follow::Event::End) => continue, @@ -1409,10 +1439,10 @@ impl Pump { // A sibling's track stopping with its instance is that instance's end, which // idles the pad for the next one rather than ending the rendition. Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, - // A sibling's track finishing ends it for this instance only: the pad idles - // for the next instance, and the record keeps a reshape on this one from - // replaying it. - Ok(None) if sibling => { + // A listed sibling's track finishing ends it for this instance only: the pad + // idles for the next instance, and the record keeps a reshape on this one + // from replaying it. + Ok(None) if sibling && state.is_listed() => { finished.lock().unwrap().insert(((kind, name.clone()), media.clone())); break Stopped::Idle; } @@ -3275,6 +3305,48 @@ mod session_tests { drop(first); } + /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next + /// instance plays nothing on it. + #[test] + fn a_delisted_sibling_rendition_follows_no_later_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + room.catalog.modify().unwrap().video.renditions.clear(); + for _ in 0..200 { + if recorder.seen().contains(&Seen::Eos(pad.clone())) { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + recorder.seen().contains(&Seen::Eos(pad.clone())), + "the idle pad never ended" + ); + + let mut second = Publisher::sibling(&origin, Default::default()); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + for _ in 0..10 { + second.write(2); + std::thread::sleep(Duration::from_millis(50)); + } + stop(session); + + let seen = recorder.seen(); + assert!(!seen.iter().any(|seen| matches!(seen, Seen::Buffer(_, 2))), "{seen:?}"); + } + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on /// the same pad, and nothing more from the old instance gets through. fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { From 94292e0f4a726bf917a7fd9e5d69a692695b0862 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 01:15:28 -0700 Subject: [PATCH 09/27] fix(gst): scope a finished track to the instance that served it moqsrc recorded a finished track by rendition and path, so a catalog update that replaced an idle sibling pump (a disable and enable, or an epoch handoff) skipped the rendition for every later instance, leaving its pad silent. The record now keeps the instance that finished, and the pump checks it once it knows which instance it reads, instead of reconcile skipping the rendition. A sibling pump idles past a finished instance, holding its pad if it has one; the catalog's own rendition still stops. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 187 +++++++++++++++++++++++++++++------ 1 file changed, 159 insertions(+), 28 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index ec11523829d..89ff06207d7 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -785,7 +785,7 @@ async fn follow_catalog( pads: pads.clone(), timeline: Arc::default(), siblings: Siblings::default(), - finished: Arc::default(), + finished: Finished::default(), }; let mut active: HashMap = HashMap::new(); let mut pumps: tokio::task::JoinSet> = tokio::task::JoinSet::new(); @@ -994,10 +994,6 @@ fn reconcile( broadcast: source.resolve_reference(d.shape.broadcast.as_ref()), name: d.shape.track.clone(), }; - let played = ((d.kind, name.clone()), track.clone()); - if playback.finished.lock().unwrap().contains(&played) { - continue; - } let container = match moq_mux::catalog::hang::Container::new( &d.shape.container, match d.kind { @@ -1140,12 +1136,30 @@ impl Siblings { } } -/// Each pad's rendition with a track it played to the end. A new subscription to a finished -/// track replays its last group, so a catalog update that still lists the rendition (a -/// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A -/// pad that never played the track joins it like any late subscriber. Shared with the pumps, -/// since a pump that replaces another on its pad learns this only once the other lets go. -type Finished = Arc>>; +/// Each pad's rendition with a track it played to the end, and the instance that served it. A new +/// subscription to a finished track replays its last group, so a catalog update that still lists +/// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on +/// that pad again. A later instance carries a new track, and a pad that never played the track +/// joins it like any late subscriber. Shared with the pumps, since a pump that replaces another on +/// its pad learns this only once the other lets go. +#[derive(Clone, Default)] +struct Finished(Arc>>); + +impl Finished { + /// Record that `instance` served `played` to its end, replacing any earlier instance's record. + fn insert(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) { + self.0.lock().unwrap().insert(played.clone(), instance.clone()); + } + + /// Whether `instance` already served `played` to its end on this pad. + fn contains(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) -> bool { + self.0 + .lock() + .unwrap() + .get(played) + .is_some_and(|finished| finished.is_clone(instance)) + } +} /// The first decoded frame fixes one mapping for every rendition of this broadcast run. #[derive(Default)] @@ -1230,6 +1244,7 @@ impl Pump { mut cancel, shutdown, } = self; + let played = ((kind, name.clone()), media.clone()); let mut container = Some(container); // The pad once this pump owns it, from the first instance on. let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; @@ -1257,6 +1272,15 @@ impl Pump { }, }; + // An instance that already played this track to its end on the pad would replay its + // last group. A sibling waits for the next instance; the catalog's own has no other. + if finished.contains(&played, &broadcast) { + match sibling { + true => continue, + false => return None, + } + } + // Resolves once the publisher answers, with the track info or with an error (which is // what ending a broadcast produces for a name nobody served). A publisher that answers // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and @@ -1317,10 +1341,8 @@ impl Pump { }; let pad = match &owned { - // Another instance on the pad this pump owns: a new stream, and a track that - // finished on the old instance is not finished on this one. + // Another instance on the pad this pump owns: a new stream. Some((pad, _)) => { - finished.lock().unwrap().remove(&((kind, name.clone()), media.clone())); tokio::task::block_in_place(|| begin(pad, &name, &caps)); pad.clone() } @@ -1348,23 +1370,23 @@ impl Pump { (pad, owned) } Claim::Kept(pad, owner) => { - let owned = tokio::select! { + let guard = tokio::select! { biased; _ = cancel.changed() => return None, - owned = owner.lock_owned() => owned, + guard = owner.lock_owned() => guard, }; // The pump this one replaces records its track as finished before letting // go, so a replacement started while it was reading the end (a reshape) - // can't replay it. - if finished - .lock() - .unwrap() - .contains(&((kind, name.clone()), media.clone())) - { - return Some((kind, name)); + // can't replay it. A sibling holds the pad for its next instance. + if finished.contains(&played, &broadcast) { + if !sibling { + return Some((kind, name)); + } + owned = Some((pad, guard)); + continue; } tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, owned) + (pad, guard) } }; // Stop flushes only the pads it finds, and this one may have been added just @@ -1386,7 +1408,7 @@ impl Pump { // waiting for this pad would replay it. _ = cancel.changed() => { if ended(&mut track) { - finished.lock().unwrap().insert(((kind, name.clone()), media)); + finished.insert(&played, &broadcast); } return None; } @@ -1443,12 +1465,12 @@ impl Pump { // idles for the next instance, and the record keeps a reshape on this one // from replaying it. Ok(None) if sibling && state.is_listed() => { - finished.lock().unwrap().insert(((kind, name.clone()), media.clone())); + finished.insert(&played, &broadcast); break Stopped::Idle; } Ok(None) => { // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.lock().unwrap().insert(((kind, name.clone()), media)); + finished.insert(&played, &broadcast); return Some((kind, name)); } Err(err) => { @@ -2989,7 +3011,7 @@ mod session_tests { pads: Pads::default(), timeline: std::sync::Arc::default(), siblings: super::Siblings::default(), - finished: std::sync::Arc::default(), + finished: super::Finished::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, shutdown, @@ -3305,6 +3327,115 @@ mod session_tests { drop(first); } + /// Block until `publisher`'s video track reads to its end, as a pump reading it does. + fn await_finished(publisher: &Publisher) { + let broadcast = publisher.broadcast.consume(); + super::RUNTIME.block_on(async { + let finished = async { + let subscriber = broadcast + .track("video") + .unwrap() + .subscribe(moq_net::track::Subscription::default()) + .await + .unwrap(); + let mut track = moq_mux::container::Consumer::new( + subscriber, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + while track.read().await.unwrap().is_some() {} + }; + tokio::time::timeout(Duration::from_secs(10), finished) + .await + .expect("the track never finished"); + }); + } + + /// A sibling track that finished stays finished for its instance only: a catalog update that + /// replaces the idle pump, here a disable and an enable, still plays the next instance. + #[test] + fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let _audio = room + .broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + + // The catalog consumer only yields the newest snapshot, so the same update lists an audio + // rendition: its pad appearing proves the run acted on the disable. + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().enabled = false; + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + await_pad(&element, "audio_"); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .enabled = true; + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + recorder.assert_one_pad_held(); + drop(first); + } + + /// A publisher handoff after a sibling track finished, the catalog naming the new instance's + /// epoch, plays that instance on the same pad. + #[test] + fn an_epoch_handoff_after_a_sibling_finish_plays_the_new_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let (old, new) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let mut room = referencing(&origin, Some(old.clone())); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(old)); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + + let mut second = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(new.clone())); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .epoch = Some(new); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the new instance moved to another pad" + ); + stop(session); + + recorder.assert_one_pad_held(); + drop(first); + } + /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next /// instance plays nothing on it. #[test] From fe98261afb17e1f90b0c29e9cbe40ca0f7e2eb8a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 14:15:31 -0700 Subject: [PATCH 10/27] refactor(gst): split moqsrc sibling following into its own PR moqsrc keeps today's sibling handling: a rendition binds its sibling through `Source::bind`, which now waits for the first announcement and keeps that instance. Following a sibling's later instances moves to the moqsrc sibling follow quest. Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 2 +- doc/concept/hang.md | 2 +- quest/m0/broadcast-epoch/README.md | 8 +- .../broadcast-epoch/moqsrc-sibling-follow.md | 41 + quest/m1/sink-restart-follow.md | 3 +- rs/moq-gst/src/source/imp.rs | 839 +++--------------- 6 files changed, 188 insertions(+), 707 deletions(-) create mode 100644 quest/m0/broadcast-epoch/moqsrc-sibling-follow.md diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index b77e58889fb..3b053b30a2d 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -124,7 +124,7 @@ rewound timestamps forward. Pads are named by kind and appear as the catalog announces renditions: `video_0`, `video_1`, `audio_0`. A rendition referencing another broadcast reads -that broadcast on the same connection and follows its announcements the same way. Link the pad you want by name; the terse +that broadcast on the same connection, waiting for it to be announced. Link the pad you want by name; the terse `moqsrc ! decodebin3` form links only the first pad offered, which may be audio. `moqsrc` follows the broadcast's announcements: when the publisher restarts, it switches to the new broadcast on the same pads, kept by rendition diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 841f74d2b0b..714ce485f61 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; every other reader, such as `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` follows a referenced path's announcements like its own, restarting the rendition on each instance that serves it; every other reader, such as `moqsrc`, `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index ff1b59f49dc..e3e052afc07 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -1,4 +1,4 @@ -# [L] Broadcast epochs +# Broadcast epochs ## Goal @@ -73,7 +73,7 @@ Decided: resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The schemas, shared resolution, compatibility documentation, and `moq_mux::follow` - (which `moqsrc` follows siblings with) landed. + landed; `moqsrc` following siblings with it remains. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players (`moq play`, `@moq/watch`, demo/web) follow the announce `Restart`. moq-stats mints one per @@ -122,3 +122,7 @@ restarts that sibling while preserving the catalog's explicit epoch constraints; it never mixes instances through an old handle. Check exact and covering routes, matching/different/absent epochs, and legacy End+Start. The child quests own their focused regressions. + +## Required + +- [moqsrc sibling follow](/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md) - moqsrc restarts a referenced sibling's rendition on each instance its announcements hand out, never past a named epoch diff --git a/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md b/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md new file mode 100644 index 00000000000..f14cd2dd940 --- /dev/null +++ b/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md @@ -0,0 +1,41 @@ +# [M] moqsrc follows referenced broadcasts' announcements + +## Goal + +`moqsrc` follows a catalog-referenced sibling's announcements the way +`@moq/watch` does: each rendition reads the instance `moq_mux::Source::follow` +hands out, idles its pad (no EOS) while the sibling is offline, and restarts on +the same pad when another instance serves it. A reference that names an +`epoch` never plays another instance. Until this lands, `moqsrc` waits for a +sibling's first announcement and keeps that instance, like every other +non-following reader. + +## Plan + +Split from the reference announcements work (maintainer, 2026-10-22), which +landed `moq_mux::follow` and the waiting `Source::bind`. The moqsrc half drew a +new state-machine finding every review round, so it gets its own design pass. + +- Start binds the rendition; End idles its pad; another instance restarts the + stream on the same pad with a fresh stream-start, caps, and a segment. +- Renditions reading one sibling instance share that instance's timeline, + whenever each starts on it, including one still subscribing when a + replacement takes over. +- A sibling track that finishes while its instance stays up idles the pad and + keeps following; a finished record names the instance that served it, so a + reshape (disable and enable, an epoch handoff) still plays the next one. +- A rendition the catalog delists drains the instance it reads, then ends its + pad and follows no later instance. +- A pump that has returned but is still in `active` must not swallow a relist: + a relist arriving before the pump is joined matches its shape and starts + nothing, for an idle delisted sibling and for a delisted catalog rendition + whose track ends just before the relist. Cover both cases. + +Weigh a simpler state model (one task per sibling instance owning its +renditions, or reconciling after each pump exits) before patching each case. + +Public API: none (moqsrc behavior). Wire: none. + +## Related + +- [Sink restart follow](/quest/m1/sink-restart-follow.md) - moves moqsrc's own broadcast follow onto `follow::Instances` diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index b76f09ecbfe..5ff849bc074 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -16,7 +16,8 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references -to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; +to other broadcasts is [moqsrc sibling follow](/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md) +for `moqsrc`, through `moq_mux::Source::follow`; `moq play` resolves a reference once with `Source::resolve`, which waits for a sibling's first announcement and keeps it. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index e54be6bfbbf..9316007c684 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -437,12 +437,7 @@ struct Shape { /// stopping it means it must never create one; after, it owns a pad and has to drop it. One /// compare-exchange settles which of the two happened, so a pad can't slip out between a /// teardown's check and the pump's creation. -struct PumpState { - phase: AtomicU8, - /// The latest catalog still lists the rendition. A delisted pump drains the instance it reads - /// and follows no other. - listed: watch::Sender, -} +struct PumpState(AtomicU8); impl PumpState { const SUBSCRIBING: u8 = 0; @@ -450,15 +445,12 @@ impl PumpState { const CANCELLED: u8 = 2; fn new() -> Self { - Self { - phase: AtomicU8::new(Self::SUBSCRIBING), - listed: watch::Sender::new(true), - } + Self(AtomicU8::new(Self::SUBSCRIBING)) } /// Claim the right to create a pad, false once a teardown got here first. fn go_live(&self) -> bool { - self.phase + self.0 .compare_exchange(Self::SUBSCRIBING, Self::LIVE, Ordering::AcqRel, Ordering::Acquire) .is_ok() } @@ -466,19 +458,10 @@ impl PumpState { /// Stop a pump that hasn't taken a pad, false if it already has one and must be torn down /// through its cancel watch instead. fn cancel_before_live(&self) -> bool { - self.phase + self.0 .compare_exchange(Self::SUBSCRIBING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire) .is_ok() } - - fn set_listed(&self, listed: bool) { - self.listed - .send_if_modified(|current| std::mem::replace(current, listed) != listed); - } - - fn is_listed(&self) -> bool { - *self.listed.borrow() - } } /// A rendition by kind and moq track name: what a pad is kept by. @@ -803,8 +786,7 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), - siblings: Siblings::default(), - finished: Finished::default(), + finished: Arc::default(), }; let mut active: HashMap = HashMap::new(); let mut pumps: tokio::task::JoinSet> = tokio::task::JoinSet::new(); @@ -967,12 +949,6 @@ fn reconcile( .collect(), ); - // A delisted pump drains the instance it reads and follows no other; a relisted one follows - // again. - for (name, track) in active.iter() { - track.state.set_listed(desired.contains_key(name)); - } - // Cancel anything that changed shape; each cancelled pump hands its pad back. Changed // renditions also land in `plan.add`, so they respawn below on the same pad. // @@ -1013,6 +989,10 @@ fn reconcile( broadcast: source.resolve_reference(d.shape.broadcast.as_ref()), name: d.shape.track.clone(), }; + let played = ((d.kind, name.clone()), track.clone()); + if playback.finished.lock().unwrap().contains(&played) { + continue; + } let container = match moq_mux::catalog::hang::Container::new( &d.shape.container, match d.kind { @@ -1027,16 +1007,15 @@ fn reconcile( } }; - // Follow without waiting: an unavailable sibling or track must not park the catalog loop + // Bind without waiting: an unavailable sibling or track must not park the catalog loop // and hold up every other rendition. The pump resolves both under its cancellation scope. - let follow = match source.follow(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { - Ok(follow) => follow, + let binding = match source.bind(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { + Ok(binding) => binding, Err(err) => { gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); continue; } }; - let sibling = track.broadcast != source.resolve_reference(None); let (cancel_tx, cancel_rx) = watch::channel(false); let state = Arc::new(PumpState::new()); @@ -1048,14 +1027,11 @@ fn reconcile( kind: d.kind, name: name.clone(), caps: d.shape.caps.clone(), - follow, - sibling, + binding, track, container, - descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), - siblings: playback.siblings.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1111,74 +1087,15 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, - siblings: Siblings, finished: Finished, } -/// The timeline of each sibling instance, shared by every rendition reading that instance so -/// their first frames fix one mapping for all of them. -#[derive(Clone, Default)] -struct Siblings(Arc>); - -#[derive(Default)] -struct SiblingTimelines { - /// Each sibling path some instance served this run. - seen: HashSet, - /// Each instance still serving, and the timeline its renditions share while any reads it. - live: Vec<(moq_net::broadcast::Consumer, std::sync::Weak)>, -} - -impl Siblings { - /// The timeline of `instance`: the run's for the first instance at its path, which the - /// catalog's own renditions share, and one of its own for any later instance, whose - /// timestamps share no origin with the run's. - fn timeline(&self, instance: &moq_net::broadcast::Consumer, run: &Arc) -> Arc { - let mut siblings = self.0.lock().unwrap(); - // Forget an instance that ended or that no rendition reads any more, releasing its handle. - siblings - .live - .retain(|(known, timeline)| !known.is_closed() && timeline.strong_count() > 0); - if let Some(timeline) = siblings - .live - .iter() - .find(|(known, _)| known.is_clone(instance)) - .and_then(|(_, timeline)| timeline.upgrade()) - { - return timeline; - } - let timeline = match siblings.seen.insert(instance.info().path.clone()) { - true => run.clone(), - false => Arc::default(), - }; - siblings.live.push((instance.clone(), Arc::downgrade(&timeline))); - timeline - } -} - -/// Each pad's rendition with a track it played to the end, and the instance that served it. A new -/// subscription to a finished track replays its last group, so a catalog update that still lists -/// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on -/// that pad again. A later instance carries a new track, and a pad that never played the track -/// joins it like any late subscriber. Shared with the pumps, since a pump that replaces another on -/// its pad learns this only once the other lets go. -#[derive(Clone, Default)] -struct Finished(Arc>>); - -impl Finished { - /// Record that `instance` served `played` to its end, replacing any earlier instance's record. - fn insert(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) { - self.0.lock().unwrap().insert(played.clone(), instance.clone()); - } - - /// Whether `instance` already served `played` to its end on this pad. - fn contains(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) -> bool { - self.0 - .lock() - .unwrap() - .get(played) - .is_some_and(|finished| finished.is_clone(instance)) - } -} +/// Each pad's rendition with a track it played to the end. A new subscription to a finished +/// track replays its last group, so a catalog update that still lists the rendition (a +/// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A +/// pad that never played the track joins it like any late subscriber. Shared with the pumps, +/// since a pump that replaces another on its pad learns this only once the other lets go. +type Finished = Arc>>; /// The first decoded frame fixes one mapping for every rendition of this broadcast run. #[derive(Default)] @@ -1204,20 +1121,11 @@ struct Pump { /// per-kind counter instead, and isn't known until the subscription resolves. name: String, caps: gst::Caps, - /// Each instance of the broadcast the rendition references. - follow: moq_mux::follow::Instances, - /// The rendition references another broadcast than its catalog's, whose instances come and - /// go on their own announcements. - sibling: bool, + binding: moq_mux::Binding, track: Track, - /// The wire format of the first instance. container: moq_mux::catalog::hang::Container, - /// What builds it again for a later instance. - descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, - /// The run's [`Playback::siblings`]. - siblings: Siblings, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1227,293 +1135,155 @@ struct Pump { shutdown: watch::Receiver, } -/// Why a pump stopped reading one instance without ending. -enum Stopped { - /// Another instance took over: restart on it. - Restart(moq_net::broadcast::Consumer), - /// The instance stopped serving: idle until the next one. - Idle, -} - impl Pump { /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the /// track ends, errors, or `cancel` fires. The pad goes back to the session without an EOS /// either way. Returns the rendition if its track ended rather than being cancelled, so the /// run can end the pad of one the catalog retired. - /// - /// A sibling's announcements drive the pump: its first start binds the rendition, an end - /// idles the pad, and another instance taking over restarts the stream on the same pad, until - /// the catalog delists the rendition. async fn run(self) -> Option { let Pump { element, kind, name, caps, - mut follow, - sibling, + binding, track: media, container, - descriptor, pads, - timeline: run, - siblings, + timeline, finished, state, mut cancel, shutdown, } = self; - let played = ((kind, name.clone()), media.clone()); - let mut container = Some(container); - // The pad once this pump owns it, from the first instance on. - let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; - let mut next: Option = None; - let mut listed = state.listed.subscribe(); + // Resolves once the publisher answers, with the track info or with an error (which is + // what ending a broadcast produces for a name nobody served). A publisher that answers + // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and + // holding the wait here rather than in `reconcile` keeps it off every other + // rendition. + let subscriber = tokio::select! { + _ = cancel.changed() => return None, + subscriber = async { + let broadcast = binding.broadcast().await?; + let track = broadcast.track(&media.name)?; + Ok::<_, moq_mux::Error>(track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))).await?) + } => match subscriber { + Ok(subscriber) => subscriber, + Err(err) => { + gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); + return Some((kind, name)); + } + } + }; + let mut track = moq_mux::container::Consumer::new(subscriber, container); + + // Winning this is what earns a pad. Losing means a teardown got here while the + // subscription was still resolving (this rendition was removed, reshaped, or outlived by + // a closing catalog), and it must not publish a pad at all: the watch alone can't say + // that, since a cancel landing just after we read it would leave a pad exposed and then + // yanked without an EOS. + if !state.go_live() { + return None; + } - loop { - // A delisted rendition drained the instance it read, and follows no other. - if owned.is_some() && !state.is_listed() { - return Some((kind, name)); + // A pad appears only once its track is live, so a pad downstream can link to is a promise + // that the rendition is actually flowing, and only a rendition that gets this far claims a + // pad id. A kept pad starts a new stream on it, with caps pushed even when they differ, so + // a format downstream can't take fails loudly as not-negotiated rather than going quiet. + let (pad, _owned) = match pads.claim(&element, (kind, name.clone()), &cancel)? { + Claim::New(pad, owned) => { + let obj = element.upgrade()?; + pad.set_active(true).ok()?; + begin(&pad, &name, &caps); + obj.add_pad(&pad).ok()?; + (pad, owned) } - let broadcast = match next.take() { - Some(broadcast) => broadcast, - None => tokio::select! { + Claim::Kept(pad, owner) => { + let owned = tokio::select! { + biased; _ = cancel.changed() => return None, - _ = listed.wait_for(|listed| !listed), if owned.is_some() => return Some((kind, name)), - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, - Ok(moq_mux::follow::Event::End) => continue, - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - }, - }; - - // An instance that already played this track to its end on the pad would replay its - // last group. A sibling waits for the next instance; the catalog's own has no other. - if finished.contains(&played, &broadcast) { - match sibling { - true => continue, - false => return None, + owned = owner.lock_owned() => owned, + }; + // The pump this one replaces records its track as finished before letting go, so a + // replacement started while it was reading the end (a reshape) can't replay it. + if finished + .lock() + .unwrap() + .contains(&((kind, name.clone()), media.clone())) + { + return Some((kind, name)); } + tokio::task::block_in_place(|| begin(&pad, &name, &caps)); + (pad, owned) } + }; + // Stop flushes only the pads it finds, and this one may have been added just after, while + // this pump's cancel is still on its way. Flushing it here keeps a push from blocking stop. + if *shutdown.borrow() { + let _ = pad.set_active(false); + } - // Resolves once the publisher answers, with the track info or with an error (which is - // what ending a broadcast produces for a name nobody served). A publisher that answers - // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and - // holding the wait here rather than in `reconcile` keeps it off every other - // rendition. - let subscribed = tokio::select! { - _ = cancel.changed() => return None, - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => Err(Stopped::Restart(broadcast)), - Ok(moq_mux::follow::Event::End) => Err(Stopped::Idle), - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - subscriber = async { - broadcast - .track(&media.name)? - .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) - .await - } => match subscriber { - Ok(subscriber) => Ok(subscriber), - // The instance went while subscribing: that is its end, not this rendition's. - Err(_) if sibling && broadcast.is_closed() => Err(Stopped::Idle), - Err(err) => { - gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); - return Some((kind, name)); - } - } - }; - let subscriber = match subscribed { - Ok(subscriber) => subscriber, - Err(stopped) => { - next = stopped.restart(); - continue; - } - }; - let container = match container.take() { - Some(container) => container, - None => match moq_mux::catalog::hang::Container::new( - &descriptor, - match kind { - TrackKind::Video => moq_mux::container::Kind::Video, - TrackKind::Audio => moq_mux::container::Kind::Audio, - }, - ) { - Ok(container) => container, - Err(err) => { - gst::warning!(CAT, "rendition {name} has an unsupported container: {err:?}"); - return Some((kind, name)); + let mut sent_segment = false; + loop { + tokio::select! { + // This rendition is being torn down (its run ended, or a catalog update replaced it). An + // end that arrived alongside still finishes the track, or the replacement waiting for + // this pad would replay it. + _ = cancel.changed() => { + if ended(&mut track) { + finished.lock().unwrap().insert(((kind, name.clone()), media)); } - }, - }; - let mut track = moq_mux::container::Consumer::new(subscriber, container); - let timeline = match sibling { - true => siblings.timeline(&broadcast, &run), - false => run.clone(), - }; - - let pad = match &owned { - // Another instance on the pad this pump owns: a new stream. - Some((pad, _)) => { - tokio::task::block_in_place(|| begin(pad, &name, &caps)); - pad.clone() + return None; } - None => { - // Winning this is what earns a pad. Losing means a teardown got here while the - // subscription was still resolving (this rendition was removed, reshaped, or - // outlived by a closing catalog), and it must not publish a pad at all: the - // watch alone can't say that, since a cancel landing just after we read it - // would leave a pad exposed and then yanked without an EOS. - if !state.go_live() { - return None; - } - - // A pad appears only once its track is live, so a pad downstream can link to is a - // promise that the rendition is actually flowing, and only a rendition that gets - // this far claims a pad id. A kept pad starts a new stream on it, with caps pushed - // even when they differ, so a format downstream can't take fails loudly as - // not-negotiated rather than going quiet. - let (pad, guard) = match pads.claim(&element, (kind, name.clone()), &cancel)? { - Claim::New(pad, owned) => { - let obj = element.upgrade()?; - pad.set_active(true).ok()?; - begin(&pad, &name, &caps); - obj.add_pad(&pad).ok()?; - (pad, owned) - } - Claim::Kept(pad, owner) => { - let guard = tokio::select! { - biased; - _ = cancel.changed() => return None, - guard = owner.lock_owned() => guard, - }; - // The pump this one replaces records its track as finished before letting - // go, so a replacement started while it was reading the end (a reshape) - // can't replay it. A sibling holds the pad for its next instance. - if finished.contains(&played, &broadcast) { - if !sibling { - return Some((kind, name)); + frame = track.read() => match frame { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + let buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); } - owned = Some((pad, guard)); - continue; + return Some((kind, name)); } - tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, guard) - } - }; - // Stop flushes only the pads it finds, and this one may have been added just - // after, while this pump's cancel is still on its way. Flushing it here keeps a - // push from blocking stop. - if *shutdown.borrow() { - let _ = pad.set_active(false); - } - owned = Some((pad.clone(), guard)); - pad - } - }; - - let mut sent_segment = false; - let stopped = loop { - tokio::select! { - // This rendition is being torn down (its run ended, or a catalog update replaced - // it). An end that arrived alongside still finishes the track, or the replacement - // waiting for this pad would replay it. - _ = cancel.changed() => { - if ended(&mut track) { - finished.insert(&played, &broadcast); - } - return None; - } - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => break Stopped::Restart(broadcast), - Ok(moq_mux::follow::Event::End) => break Stopped::Idle, - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return Some((kind, name)); - } - }; - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other pumps. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); - } - return Some((kind, name)); + }; + let segment = (!sent_segment).then(|| { + let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other pumps. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); } - Err(_) => return Some((kind, name)), + return Some((kind, name)); } + Err(_) => return Some((kind, name)), } - // A sibling's track stopping with its instance is that instance's end, which - // idles the pad for the next one rather than ending the rendition. - Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, - // A listed sibling's track finishing ends it for this instance only: the pad - // idles for the next instance, and the record keeps a reshape on this one - // from replaying it. - Ok(None) if sibling && state.is_listed() => { - finished.insert(&played, &broadcast); - break Stopped::Idle; - } - Ok(None) => { - // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.insert(&played, &broadcast); - return Some((kind, name)); - } - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - return Some((kind, name)); - } + } + Ok(None) => { + // Recorded while this pump still owns the pad, before any replacement can claim it. + finished.lock().unwrap().insert(((kind, name.clone()), media)); + return Some((kind, name)); + } + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + return Some((kind, name)); } } - }; - match &stopped { - Stopped::Restart(_) => gst::info!(CAT, "rendition {name} restarts on a new instance"), - Stopped::Idle => gst::info!(CAT, "rendition {name} is offline, holding its pad"), } - next = stopped.restart(); - } - } -} - -impl Stopped { - /// The instance to restart on, if any. - fn restart(self) -> Option { - match self { - Stopped::Restart(broadcast) => Some(broadcast), - Stopped::Idle => None, } } } @@ -1700,10 +1470,9 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{PumpState, Siblings, Timeline, ended, plan_reconcile}; + use super::{PumpState, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; - use std::sync::Arc; /// A pump cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -1835,33 +1604,6 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } - /// Renditions on the same sibling instance share its timeline, whenever each starts on it: - /// the first instance shares the run's, and the next one starts another. - #[tokio::test(start_paused = true)] - async fn renditions_on_one_sibling_instance_share_its_timeline() { - let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); - tokio::spawn(moq_net::time::run(driver)); - let consumer = origin.consume(); - let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { - moq_mux::follow::Event::Start(broadcast) => broadcast, - moq_mux::follow::Event::End => panic!("expected a start"), - }; - let (siblings, run) = (Siblings::default(), Arc::::default()); - - let _first = origin.publish("source", Default::default()).unwrap(); - let first = started(&mut audio).await; - assert!(Arc::ptr_eq(&run, &siblings.timeline(&first, &run))); - - // A video rendition that first starts on the replacement, as one still subscribing when it - // took over does, shares its timeline with the audio rendition restarting onto it. - let _second = origin.publish("source", Default::default()).unwrap(); - let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let next = siblings.timeline(&started(&mut video).await, &run); - assert!(!Arc::ptr_eq(&run, &next), "another instance has its own timeline"); - assert!(Arc::ptr_eq(&next, &siblings.timeline(&started(&mut audio).await, &run))); - } - #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -2966,13 +2708,6 @@ mod session_tests { publisher } - /// The `cam` broadcast a rendition of `room` references as `./cam`, under `route`. - fn sibling(origin: &moq_net::origin::Producer, route: moq_net::origin::Route) -> Self { - let publisher = Self::build(origin.create_broadcast("cam").unwrap(), video_rendition()); - publisher.broadcast.announce(route).unwrap(); - publisher - } - /// A broadcast at no path, for a covering route to serve. fn standalone(config: VideoConfig) -> Self { Self::build(moq_net::broadcast::Info::new().produce(), config) @@ -3122,20 +2857,15 @@ mod session_tests { kind: super::TrackKind::Video, name: "video".into(), caps: gst::Caps::new_empty_simple("video/x-h264"), - follow: moq_mux::Source::new(origin().consume(), publisher.broadcast.consume()) - .follow(None, None) - .unwrap(), - sibling: false, + binding: moq_mux::Binding::new(publisher.broadcast.consume()), track: super::Track { broadcast: None, name: "video".into(), }, container: moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), - siblings: super::Siblings::default(), - finished: super::Finished::default(), + finished: std::sync::Arc::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, shutdown, @@ -3387,301 +3117,6 @@ mod session_tests { recorder.assert_one_pad_held(); } - /// A `room` whose video rendition references the `cam` sibling, naming `epoch` when given. - fn referencing(origin: &moq_net::origin::Producer, epoch: Option) -> Publisher { - let mut config = video_rendition(); - config.broadcast = Some("./cam".into()); - config.epoch = epoch; - Publisher::new(origin, Default::default(), config) - } - - /// A sibling announced after the catalog starts its rendition, its end holds the pad, and its - /// return starts a new stream on that pad. - #[test] - fn a_late_sibling_starts_its_rendition_and_returns_after_an_end() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let _room = referencing(&origin, None); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - let mut follow = origin.consume().follow("cam").unwrap(); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - drop(first); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); - - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); - stop(session); - - assert!( - recorder.seen().contains(&Seen::Start(pad)), - "no new stream on the kept pad" - ); - recorder.assert_one_pad_held(); - } - - /// A sibling track finishing while its instance stays up idles the pad, and the instance that - /// replaces it starts a new stream there. - #[test] - fn a_finished_sibling_track_returns_on_the_next_instance() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let _room = referencing(&origin, None); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!( - recorder.written(&mut second, 2), - pad, - "the next instance moved to another pad" - ); - stop(session); - - assert!(!recorder.seen().contains(&Seen::Eos(pad)), "the finish ended the pad"); - recorder.assert_one_pad_held(); - drop(first); - } - - /// Block until `publisher`'s video track reads to its end, as a pump reading it does. - fn await_finished(publisher: &Publisher) { - let broadcast = publisher.broadcast.consume(); - super::RUNTIME.block_on(async { - let finished = async { - let subscriber = broadcast - .track("video") - .unwrap() - .subscribe(moq_net::track::Subscription::default()) - .await - .unwrap(); - let mut track = moq_mux::container::Consumer::new( - subscriber, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - ); - while track.read().await.unwrap().is_some() {} - }; - tokio::time::timeout(Duration::from_secs(10), finished) - .await - .expect("the track never finished"); - }); - } - - /// A sibling track that finished stays finished for its instance only: a catalog update that - /// replaces the idle pump, here a disable and an enable, still plays the next instance. - #[test] - fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let mut room = referencing(&origin, None); - let _audio = room - .broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - await_finished(&first); - - // The catalog consumer only yields the newest snapshot, so the same update lists an audio - // rendition: its pad appearing proves the run acted on the disable. - { - let mut guard = room.catalog.modify().unwrap(); - guard.video.renditions.get_mut("video").unwrap().enabled = false; - guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); - } - await_pad(&element, "audio_"); - room.catalog - .modify() - .unwrap() - .video - .renditions - .get_mut("video") - .unwrap() - .enabled = true; - - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!( - recorder.written(&mut second, 2), - pad, - "the next instance moved to another pad" - ); - stop(session); - - recorder.assert_one_pad_held(); - drop(first); - } - - /// A publisher handoff after a sibling track finished, the catalog naming the new instance's - /// epoch, plays that instance on the same pad. - #[test] - fn an_epoch_handoff_after_a_sibling_finish_plays_the_new_instance() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let (old, new) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); - let mut room = referencing(&origin, Some(old.clone())); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(old)); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - await_finished(&first); - - let mut second = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(new.clone())); - room.catalog - .modify() - .unwrap() - .video - .renditions - .get_mut("video") - .unwrap() - .epoch = Some(new); - assert_eq!( - recorder.written(&mut second, 2), - pad, - "the new instance moved to another pad" - ); - stop(session); - - recorder.assert_one_pad_held(); - drop(first); - } - - /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next - /// instance plays nothing on it. - #[test] - fn a_delisted_sibling_rendition_follows_no_later_instance() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let mut room = referencing(&origin, None); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - let mut follow = origin.consume().follow("cam").unwrap(); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - drop(first); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); - - room.catalog.modify().unwrap().video.renditions.clear(); - for _ in 0..200 { - if recorder.seen().contains(&Seen::Eos(pad.clone())) { - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - assert!( - recorder.seen().contains(&Seen::Eos(pad.clone())), - "the idle pad never ended" - ); - - let mut second = Publisher::sibling(&origin, Default::default()); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - for _ in 0..10 { - second.write(2); - std::thread::sleep(Duration::from_millis(50)); - } - stop(session); - - let seen = recorder.seen(); - assert!(!seen.iter().any(|seen| matches!(seen, Seen::Buffer(_, 2))), "{seen:?}"); - } - - /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on - /// the same pad, and nothing more from the old instance gets through. - fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let _room = referencing(&origin, None); - let mut old = Publisher::sibling(&origin, route()); - let session = follow(&element, &origin); - let pad = recorder.written(&mut old, 1); - - let mut new = Publisher::sibling(&origin, route()); - assert_eq!(recorder.written(&mut new, 2), pad, "the restart moved to another pad"); - old.write(1); - recorder.written(&mut new, 3); - stop(session); - - let seen = recorder.seen(); - let switched = seen - .iter() - .position(|seen| *seen == Seen::Buffer(pad.clone(), 2)) - .unwrap(); - assert!( - !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), - "the replaced sibling kept playing: {seen:?}" - ); - recorder.assert_one_pad_held(); - } - - #[test] - fn a_sibling_restart_switches_on_the_same_pad() { - sibling_restart(epoch); - } - - #[test] - fn an_epochless_sibling_restart_switches_on_the_same_pad() { - sibling_restart(moq_net::origin::Route::default); - } - - /// A reference naming its sibling's epoch never reads the instance replacing it: the pad - /// idles, and plays again once the named instance serves the path again. - #[test] - fn a_sibling_restart_idles_a_reference_naming_the_old_epoch() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let named = moq_net::Epoch::mint(); - let _room = referencing(&origin, Some(named.clone())); - let mut old = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(named)); - let session = follow(&element, &origin); - let pad = recorder.written(&mut old, 1); - - let mut follow = origin.consume().follow("cam").unwrap(); - await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); - let mut new = Publisher::sibling(&origin, epoch()); - await_event(&mut follow, |event| { - matches!(event, moq_net::announce::Event::Restart(_)) - }); - new.write(2); - drop(new); - await_event(&mut follow, |event| { - matches!(event, moq_net::announce::Event::Restart(_)) - }); - - assert_eq!( - recorder.written(&mut old, 3), - pad, - "the named instance moved to another pad" - ); - stop(session); - assert!( - !recorder.seen().contains(&Seen::Buffer(pad, 2)), - "another epoch played: {:?}", - recorder.seen() - ); - recorder.assert_one_pad_held(); - } - /// Re-pricing the same instance is an update, which the run rides out without a new stream. #[test] fn an_update_keeps_the_run() { From 667a334d4402052283be943debdb29296113c5ee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 14:21:11 -0700 Subject: [PATCH 11/27] feat(gst): moqsrc follows referenced broadcasts' announcements Each pump follows its rendition's `moq_mux::follow::Instances`: Start binds, End idles the pad, and another instance restarts the stream on the same pad with a timeline shared by every rendition reading that instance. A finished sibling track idles, recorded per instance; a delisted rendition drains its instance and ends its pad. Split from #6254 with the code its review rounds covered. Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 2 +- doc/concept/hang.md | 2 +- quest/m0/broadcast-epoch/README.md | 8 +- .../broadcast-epoch/moqsrc-sibling-follow.md | 41 - quest/m1/sink-restart-follow.md | 3 +- rs/moq-gst/src/source/imp.rs | 839 +++++++++++++++--- 6 files changed, 707 insertions(+), 188 deletions(-) delete mode 100644 quest/m0/broadcast-epoch/moqsrc-sibling-follow.md diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 3b053b30a2d..b77e58889fb 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -124,7 +124,7 @@ rewound timestamps forward. Pads are named by kind and appear as the catalog announces renditions: `video_0`, `video_1`, `audio_0`. A rendition referencing another broadcast reads -that broadcast on the same connection, waiting for it to be announced. Link the pad you want by name; the terse +that broadcast on the same connection and follows its announcements the same way. Link the pad you want by name; the terse `moqsrc ! decodebin3` form links only the first pad offered, which may be audio. `moqsrc` follows the broadcast's announcements: when the publisher restarts, it switches to the new broadcast on the same pads, kept by rendition diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 714ce485f61..841f74d2b0b 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -63,7 +63,7 @@ decoder once per track. A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the rendition's ID, so labels need not be unique and renaming one doesn't rename the track. -- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` follows a referenced path's announcements like its own, restarting the rendition on each instance that serves it; every other reader, such as `moqsrc`, `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. +- **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. `@moq/watch` and `moqsrc` follow a referenced path's announcements like their own, restarting the rendition on each instance that serves it; every other reader, such as `moq play`, the exporters, and the bindings, waits for the first one announced and keeps it, and an exporter ends the rendition with it. An optional `epoch` beside the path names the [publisher instance](/concept/moq-lite#publisher-epochs) it expects: a route proving another instance ends the rendition instead of falling back to it, while a route without an epoch (an older version) cannot prove one, so there it is best effort. - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. diff --git a/quest/m0/broadcast-epoch/README.md b/quest/m0/broadcast-epoch/README.md index e3e052afc07..ff1b59f49dc 100644 --- a/quest/m0/broadcast-epoch/README.md +++ b/quest/m0/broadcast-epoch/README.md @@ -1,4 +1,4 @@ -# Broadcast epochs +# [L] Broadcast epochs ## Goal @@ -73,7 +73,7 @@ Decided: resolution on epoch-aware transports and remain best effort on epochless transports. Following never turns a replacement into seamless resume. The schemas, shared resolution, compatibility documentation, and `moq_mux::follow` - landed; `moqsrc` following siblings with it remains. + (which `moqsrc` follows siblings with) landed. - Every first-party publisher that can restart mints its own: the apps, moq-boy, the ingest gateways, moqsink, OBS, and the bindings. Players (`moq play`, `@moq/watch`, demo/web) follow the announce `Restart`. moq-stats mints one per @@ -122,7 +122,3 @@ restarts that sibling while preserving the catalog's explicit epoch constraints; it never mixes instances through an old handle. Check exact and covering routes, matching/different/absent epochs, and legacy End+Start. The child quests own their focused regressions. - -## Required - -- [moqsrc sibling follow](/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md) - moqsrc restarts a referenced sibling's rendition on each instance its announcements hand out, never past a named epoch diff --git a/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md b/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md deleted file mode 100644 index f14cd2dd940..00000000000 --- a/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] moqsrc follows referenced broadcasts' announcements - -## Goal - -`moqsrc` follows a catalog-referenced sibling's announcements the way -`@moq/watch` does: each rendition reads the instance `moq_mux::Source::follow` -hands out, idles its pad (no EOS) while the sibling is offline, and restarts on -the same pad when another instance serves it. A reference that names an -`epoch` never plays another instance. Until this lands, `moqsrc` waits for a -sibling's first announcement and keeps that instance, like every other -non-following reader. - -## Plan - -Split from the reference announcements work (maintainer, 2026-10-22), which -landed `moq_mux::follow` and the waiting `Source::bind`. The moqsrc half drew a -new state-machine finding every review round, so it gets its own design pass. - -- Start binds the rendition; End idles its pad; another instance restarts the - stream on the same pad with a fresh stream-start, caps, and a segment. -- Renditions reading one sibling instance share that instance's timeline, - whenever each starts on it, including one still subscribing when a - replacement takes over. -- A sibling track that finishes while its instance stays up idles the pad and - keeps following; a finished record names the instance that served it, so a - reshape (disable and enable, an epoch handoff) still plays the next one. -- A rendition the catalog delists drains the instance it reads, then ends its - pad and follows no later instance. -- A pump that has returned but is still in `active` must not swallow a relist: - a relist arriving before the pump is joined matches its shape and starts - nothing, for an idle delisted sibling and for a delisted catalog rendition - whose track ends just before the relist. Cover both cases. - -Weigh a simpler state model (one task per sibling instance owning its -renditions, or reconciling after each pump exits) before patching each case. - -Public API: none (moqsrc behavior). Wire: none. - -## Related - -- [Sink restart follow](/quest/m1/sink-restart-follow.md) - moves moqsrc's own broadcast follow onto `follow::Instances` diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index 5ff849bc074..b76f09ecbfe 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -16,8 +16,7 @@ route dies, since its subscriptions then end with an error. Decided with the maintainer on 2026-10-20: widened from the server and push sinks to every moq-cli and moq-gst reader. Following a catalog's references -to other broadcasts is [moqsrc sibling follow](/quest/m0/broadcast-epoch/moqsrc-sibling-follow.md) -for `moqsrc`, through `moq_mux::Source::follow`; +to other broadcasts landed for `moqsrc` through `moq_mux::Source::follow`; `moq play` resolves a reference once with `Source::resolve`, which waits for a sibling's first announcement and keeps it. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 9316007c684..e54be6bfbbf 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -437,7 +437,12 @@ struct Shape { /// stopping it means it must never create one; after, it owns a pad and has to drop it. One /// compare-exchange settles which of the two happened, so a pad can't slip out between a /// teardown's check and the pump's creation. -struct PumpState(AtomicU8); +struct PumpState { + phase: AtomicU8, + /// The latest catalog still lists the rendition. A delisted pump drains the instance it reads + /// and follows no other. + listed: watch::Sender, +} impl PumpState { const SUBSCRIBING: u8 = 0; @@ -445,12 +450,15 @@ impl PumpState { const CANCELLED: u8 = 2; fn new() -> Self { - Self(AtomicU8::new(Self::SUBSCRIBING)) + Self { + phase: AtomicU8::new(Self::SUBSCRIBING), + listed: watch::Sender::new(true), + } } /// Claim the right to create a pad, false once a teardown got here first. fn go_live(&self) -> bool { - self.0 + self.phase .compare_exchange(Self::SUBSCRIBING, Self::LIVE, Ordering::AcqRel, Ordering::Acquire) .is_ok() } @@ -458,10 +466,19 @@ impl PumpState { /// Stop a pump that hasn't taken a pad, false if it already has one and must be torn down /// through its cancel watch instead. fn cancel_before_live(&self) -> bool { - self.0 + self.phase .compare_exchange(Self::SUBSCRIBING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + + fn set_listed(&self, listed: bool) { + self.listed + .send_if_modified(|current| std::mem::replace(current, listed) != listed); + } + + fn is_listed(&self) -> bool { + *self.listed.borrow() + } } /// A rendition by kind and moq track name: what a pad is kept by. @@ -786,7 +803,8 @@ async fn follow_catalog( let playback = Playback { pads: pads.clone(), timeline: Arc::default(), - finished: Arc::default(), + siblings: Siblings::default(), + finished: Finished::default(), }; let mut active: HashMap = HashMap::new(); let mut pumps: tokio::task::JoinSet> = tokio::task::JoinSet::new(); @@ -949,6 +967,12 @@ fn reconcile( .collect(), ); + // A delisted pump drains the instance it reads and follows no other; a relisted one follows + // again. + for (name, track) in active.iter() { + track.state.set_listed(desired.contains_key(name)); + } + // Cancel anything that changed shape; each cancelled pump hands its pad back. Changed // renditions also land in `plan.add`, so they respawn below on the same pad. // @@ -989,10 +1013,6 @@ fn reconcile( broadcast: source.resolve_reference(d.shape.broadcast.as_ref()), name: d.shape.track.clone(), }; - let played = ((d.kind, name.clone()), track.clone()); - if playback.finished.lock().unwrap().contains(&played) { - continue; - } let container = match moq_mux::catalog::hang::Container::new( &d.shape.container, match d.kind { @@ -1007,15 +1027,16 @@ fn reconcile( } }; - // Bind without waiting: an unavailable sibling or track must not park the catalog loop + // Follow without waiting: an unavailable sibling or track must not park the catalog loop // and hold up every other rendition. The pump resolves both under its cancellation scope. - let binding = match source.bind(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { - Ok(binding) => binding, + let follow = match source.follow(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { + Ok(follow) => follow, Err(err) => { gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); continue; } }; + let sibling = track.broadcast != source.resolve_reference(None); let (cancel_tx, cancel_rx) = watch::channel(false); let state = Arc::new(PumpState::new()); @@ -1027,11 +1048,14 @@ fn reconcile( kind: d.kind, name: name.clone(), caps: d.shape.caps.clone(), - binding, + follow, + sibling, track, container, + descriptor: d.shape.container.clone(), pads: pads.clone(), timeline: playback.timeline.clone(), + siblings: playback.siblings.clone(), finished: playback.finished.clone(), state: state.clone(), cancel: cancel_rx, @@ -1087,15 +1111,74 @@ fn plan_reconcile(desired: &HashMap, active: &HashMap, + siblings: Siblings, finished: Finished, } -/// Each pad's rendition with a track it played to the end. A new subscription to a finished -/// track replays its last group, so a catalog update that still lists the rendition (a -/// publisher's final snapshot lands after its tracks end) must not start it on that pad again. A -/// pad that never played the track joins it like any late subscriber. Shared with the pumps, -/// since a pump that replaces another on its pad learns this only once the other lets go. -type Finished = Arc>>; +/// The timeline of each sibling instance, shared by every rendition reading that instance so +/// their first frames fix one mapping for all of them. +#[derive(Clone, Default)] +struct Siblings(Arc>); + +#[derive(Default)] +struct SiblingTimelines { + /// Each sibling path some instance served this run. + seen: HashSet, + /// Each instance still serving, and the timeline its renditions share while any reads it. + live: Vec<(moq_net::broadcast::Consumer, std::sync::Weak)>, +} + +impl Siblings { + /// The timeline of `instance`: the run's for the first instance at its path, which the + /// catalog's own renditions share, and one of its own for any later instance, whose + /// timestamps share no origin with the run's. + fn timeline(&self, instance: &moq_net::broadcast::Consumer, run: &Arc) -> Arc { + let mut siblings = self.0.lock().unwrap(); + // Forget an instance that ended or that no rendition reads any more, releasing its handle. + siblings + .live + .retain(|(known, timeline)| !known.is_closed() && timeline.strong_count() > 0); + if let Some(timeline) = siblings + .live + .iter() + .find(|(known, _)| known.is_clone(instance)) + .and_then(|(_, timeline)| timeline.upgrade()) + { + return timeline; + } + let timeline = match siblings.seen.insert(instance.info().path.clone()) { + true => run.clone(), + false => Arc::default(), + }; + siblings.live.push((instance.clone(), Arc::downgrade(&timeline))); + timeline + } +} + +/// Each pad's rendition with a track it played to the end, and the instance that served it. A new +/// subscription to a finished track replays its last group, so a catalog update that still lists +/// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on +/// that pad again. A later instance carries a new track, and a pad that never played the track +/// joins it like any late subscriber. Shared with the pumps, since a pump that replaces another on +/// its pad learns this only once the other lets go. +#[derive(Clone, Default)] +struct Finished(Arc>>); + +impl Finished { + /// Record that `instance` served `played` to its end, replacing any earlier instance's record. + fn insert(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) { + self.0.lock().unwrap().insert(played.clone(), instance.clone()); + } + + /// Whether `instance` already served `played` to its end on this pad. + fn contains(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) -> bool { + self.0 + .lock() + .unwrap() + .get(played) + .is_some_and(|finished| finished.is_clone(instance)) + } +} /// The first decoded frame fixes one mapping for every rendition of this broadcast run. #[derive(Default)] @@ -1121,11 +1204,20 @@ struct Pump { /// per-kind counter instead, and isn't known until the subscription resolves. name: String, caps: gst::Caps, - binding: moq_mux::Binding, + /// Each instance of the broadcast the rendition references. + follow: moq_mux::follow::Instances, + /// The rendition references another broadcast than its catalog's, whose instances come and + /// go on their own announcements. + sibling: bool, track: Track, + /// The wire format of the first instance. container: moq_mux::catalog::hang::Container, + /// What builds it again for a later instance. + descriptor: hang::catalog::Container, pads: Pads, timeline: Arc, + /// The run's [`Playback::siblings`]. + siblings: Siblings, /// The run's [`Playback::finished`]. finished: Finished, /// Shared with this rendition's [`ActiveTrack::state`]. @@ -1135,155 +1227,293 @@ struct Pump { shutdown: watch::Receiver, } +/// Why a pump stopped reading one instance without ending. +enum Stopped { + /// Another instance took over: restart on it. + Restart(moq_net::broadcast::Consumer), + /// The instance stopped serving: idle until the next one. + Idle, +} + impl Pump { /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the /// track ends, errors, or `cancel` fires. The pad goes back to the session without an EOS /// either way. Returns the rendition if its track ended rather than being cancelled, so the /// run can end the pad of one the catalog retired. + /// + /// A sibling's announcements drive the pump: its first start binds the rendition, an end + /// idles the pad, and another instance taking over restarts the stream on the same pad, until + /// the catalog delists the rendition. async fn run(self) -> Option { let Pump { element, kind, name, caps, - binding, + mut follow, + sibling, track: media, container, + descriptor, pads, - timeline, + timeline: run, + siblings, finished, state, mut cancel, shutdown, } = self; - // Resolves once the publisher answers, with the track info or with an error (which is - // what ending a broadcast produces for a name nobody served). A publisher that answers - // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and - // holding the wait here rather than in `reconcile` keeps it off every other - // rendition. - let subscriber = tokio::select! { - _ = cancel.changed() => return None, - subscriber = async { - let broadcast = binding.broadcast().await?; - let track = broadcast.track(&media.name)?; - Ok::<_, moq_mux::Error>(track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))).await?) - } => match subscriber { - Ok(subscriber) => subscriber, - Err(err) => { - gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); - return Some((kind, name)); - } - } - }; - let mut track = moq_mux::container::Consumer::new(subscriber, container); - - // Winning this is what earns a pad. Losing means a teardown got here while the - // subscription was still resolving (this rendition was removed, reshaped, or outlived by - // a closing catalog), and it must not publish a pad at all: the watch alone can't say - // that, since a cancel landing just after we read it would leave a pad exposed and then - // yanked without an EOS. - if !state.go_live() { - return None; - } + let played = ((kind, name.clone()), media.clone()); + let mut container = Some(container); + // The pad once this pump owns it, from the first instance on. + let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; + let mut next: Option = None; + let mut listed = state.listed.subscribe(); - // A pad appears only once its track is live, so a pad downstream can link to is a promise - // that the rendition is actually flowing, and only a rendition that gets this far claims a - // pad id. A kept pad starts a new stream on it, with caps pushed even when they differ, so - // a format downstream can't take fails loudly as not-negotiated rather than going quiet. - let (pad, _owned) = match pads.claim(&element, (kind, name.clone()), &cancel)? { - Claim::New(pad, owned) => { - let obj = element.upgrade()?; - pad.set_active(true).ok()?; - begin(&pad, &name, &caps); - obj.add_pad(&pad).ok()?; - (pad, owned) + loop { + // A delisted rendition drained the instance it read, and follows no other. + if owned.is_some() && !state.is_listed() { + return Some((kind, name)); } - Claim::Kept(pad, owner) => { - let owned = tokio::select! { - biased; + let broadcast = match next.take() { + Some(broadcast) => broadcast, + None => tokio::select! { _ = cancel.changed() => return None, - owned = owner.lock_owned() => owned, - }; - // The pump this one replaces records its track as finished before letting go, so a - // replacement started while it was reading the end (a reshape) can't replay it. - if finished - .lock() - .unwrap() - .contains(&((kind, name.clone()), media.clone())) - { - return Some((kind, name)); + _ = listed.wait_for(|listed| !listed), if owned.is_some() => return Some((kind, name)), + followed = follow.next() => match followed { + Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, + Ok(moq_mux::follow::Event::End) => continue, + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + }, + }; + + // An instance that already played this track to its end on the pad would replay its + // last group. A sibling waits for the next instance; the catalog's own has no other. + if finished.contains(&played, &broadcast) { + match sibling { + true => continue, + false => return None, } - tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, owned) } - }; - // Stop flushes only the pads it finds, and this one may have been added just after, while - // this pump's cancel is still on its way. Flushing it here keeps a push from blocking stop. - if *shutdown.borrow() { - let _ = pad.set_active(false); - } - let mut sent_segment = false; - loop { - tokio::select! { - // This rendition is being torn down (its run ended, or a catalog update replaced it). An - // end that arrived alongside still finishes the track, or the replacement waiting for - // this pad would replay it. - _ = cancel.changed() => { - if ended(&mut track) { - finished.lock().unwrap().insert(((kind, name.clone()), media)); + // Resolves once the publisher answers, with the track info or with an error (which is + // what ending a broadcast produces for a name nobody served). A publisher that answers + // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and + // holding the wait here rather than in `reconcile` keeps it off every other + // rendition. + let subscribed = tokio::select! { + _ = cancel.changed() => return None, + followed = follow.next() => match followed { + Ok(moq_mux::follow::Event::Start(broadcast)) => Err(Stopped::Restart(broadcast)), + Ok(moq_mux::follow::Event::End) => Err(Stopped::Idle), + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + subscriber = async { + broadcast + .track(&media.name)? + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) + .await + } => match subscriber { + Ok(subscriber) => Ok(subscriber), + // The instance went while subscribing: that is its end, not this rendition's. + Err(_) if sibling && broadcast.is_closed() => Err(Stopped::Idle), + Err(err) => { + gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); + return Some((kind, name)); } - return None; } - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return Some((kind, name)); - } - }; - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other pumps. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); + }; + let subscriber = match subscribed { + Ok(subscriber) => subscriber, + Err(stopped) => { + next = stopped.restart(); + continue; + } + }; + let container = match container.take() { + Some(container) => container, + None => match moq_mux::catalog::hang::Container::new( + &descriptor, + match kind { + TrackKind::Video => moq_mux::container::Kind::Video, + TrackKind::Audio => moq_mux::container::Kind::Audio, + }, + ) { + Ok(container) => container, + Err(err) => { + gst::warning!(CAT, "rendition {name} has an unsupported container: {err:?}"); + return Some((kind, name)); + } + }, + }; + let mut track = moq_mux::container::Consumer::new(subscriber, container); + let timeline = match sibling { + true => siblings.timeline(&broadcast, &run), + false => run.clone(), + }; + + let pad = match &owned { + // Another instance on the pad this pump owns: a new stream. + Some((pad, _)) => { + tokio::task::block_in_place(|| begin(pad, &name, &caps)); + pad.clone() + } + None => { + // Winning this is what earns a pad. Losing means a teardown got here while the + // subscription was still resolving (this rendition was removed, reshaped, or + // outlived by a closing catalog), and it must not publish a pad at all: the + // watch alone can't say that, since a cancel landing just after we read it + // would leave a pad exposed and then yanked without an EOS. + if !state.go_live() { + return None; + } + + // A pad appears only once its track is live, so a pad downstream can link to is a + // promise that the rendition is actually flowing, and only a rendition that gets + // this far claims a pad id. A kept pad starts a new stream on it, with caps pushed + // even when they differ, so a format downstream can't take fails loudly as + // not-negotiated rather than going quiet. + let (pad, guard) = match pads.claim(&element, (kind, name.clone()), &cancel)? { + Claim::New(pad, owned) => { + let obj = element.upgrade()?; + pad.set_active(true).ok()?; + begin(&pad, &name, &caps); + obj.add_pad(&pad).ok()?; + (pad, owned) + } + Claim::Kept(pad, owner) => { + let guard = tokio::select! { + biased; + _ = cancel.changed() => return None, + guard = owner.lock_owned() => guard, + }; + // The pump this one replaces records its track as finished before letting + // go, so a replacement started while it was reading the end (a reshape) + // can't replay it. A sibling holds the pad for its next instance. + if finished.contains(&played, &broadcast) { + if !sibling { + return Some((kind, name)); } - return Some((kind, name)); + owned = Some((pad, guard)); + continue; } - Err(_) => return Some((kind, name)), + tokio::task::block_in_place(|| begin(&pad, &name, &caps)); + (pad, guard) } + }; + // Stop flushes only the pads it finds, and this one may have been added just + // after, while this pump's cancel is still on its way. Flushing it here keeps a + // push from blocking stop. + if *shutdown.borrow() { + let _ = pad.set_active(false); } - Ok(None) => { - // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.lock().unwrap().insert(((kind, name.clone()), media)); - return Some((kind, name)); + owned = Some((pad.clone(), guard)); + pad + } + }; + + let mut sent_segment = false; + let stopped = loop { + tokio::select! { + // This rendition is being torn down (its run ended, or a catalog update replaced + // it). An end that arrived alongside still finishes the track, or the replacement + // waiting for this pad would replay it. + _ = cancel.changed() => { + if ended(&mut track) { + finished.insert(&played, &broadcast); + } + return None; } - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - return Some((kind, name)); + followed = follow.next() => match followed { + Ok(moq_mux::follow::Event::Start(broadcast)) => break Stopped::Restart(broadcast), + Ok(moq_mux::follow::Event::End) => break Stopped::Idle, + Err(err) => { + gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); + return Some((kind, name)); + } + }, + frame = track.read() => match frame { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + let buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); + } + return Some((kind, name)); + } + }; + let segment = (!sent_segment).then(|| { + let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other pumps. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); + } + return Some((kind, name)); + } + Err(_) => return Some((kind, name)), + } + } + // A sibling's track stopping with its instance is that instance's end, which + // idles the pad for the next one rather than ending the rendition. + Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, + // A listed sibling's track finishing ends it for this instance only: the pad + // idles for the next instance, and the record keeps a reshape on this one + // from replaying it. + Ok(None) if sibling && state.is_listed() => { + finished.insert(&played, &broadcast); + break Stopped::Idle; + } + Ok(None) => { + // Recorded while this pump still owns the pad, before any replacement can claim it. + finished.insert(&played, &broadcast); + return Some((kind, name)); + } + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + return Some((kind, name)); + } } } + }; + match &stopped { + Stopped::Restart(_) => gst::info!(CAT, "rendition {name} restarts on a new instance"), + Stopped::Idle => gst::info!(CAT, "rendition {name} is offline, holding its pad"), } + next = stopped.restart(); + } + } +} + +impl Stopped { + /// The instance to restart on, if any. + fn restart(self) -> Option { + match self { + Stopped::Restart(broadcast) => Some(broadcast), + Stopped::Idle => None, } } } @@ -1470,9 +1700,10 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{PumpState, Timeline, ended, plan_reconcile}; + use super::{PumpState, Siblings, Timeline, ended, plan_reconcile}; use moq_net::Timestamp; use std::collections::HashMap; + use std::sync::Arc; /// A pump cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -1604,6 +1835,33 @@ mod tests { assert_eq!(sorted(plan.add), vec!["a", "b"]); } + /// Renditions on the same sibling instance share its timeline, whenever each starts on it: + /// the first instance shares the run's, and the next one starts another. + #[tokio::test(start_paused = true)] + async fn renditions_on_one_sibling_instance_share_its_timeline() { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + tokio::spawn(moq_net::time::run(driver)); + let consumer = origin.consume(); + let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { + moq_mux::follow::Event::Start(broadcast) => broadcast, + moq_mux::follow::Event::End => panic!("expected a start"), + }; + let (siblings, run) = (Siblings::default(), Arc::::default()); + + let _first = origin.publish("source", Default::default()).unwrap(); + let first = started(&mut audio).await; + assert!(Arc::ptr_eq(&run, &siblings.timeline(&first, &run))); + + // A video rendition that first starts on the replacement, as one still subscribing when it + // took over does, shares its timeline with the audio rendition restarting onto it. + let _second = origin.publish("source", Default::default()).unwrap(); + let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); + let next = siblings.timeline(&started(&mut video).await, &run); + assert!(!Arc::ptr_eq(&run, &next), "another instance has its own timeline"); + assert!(Arc::ptr_eq(&next, &siblings.timeline(&started(&mut audio).await, &run))); + } + #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -2708,6 +2966,13 @@ mod session_tests { publisher } + /// The `cam` broadcast a rendition of `room` references as `./cam`, under `route`. + fn sibling(origin: &moq_net::origin::Producer, route: moq_net::origin::Route) -> Self { + let publisher = Self::build(origin.create_broadcast("cam").unwrap(), video_rendition()); + publisher.broadcast.announce(route).unwrap(); + publisher + } + /// A broadcast at no path, for a covering route to serve. fn standalone(config: VideoConfig) -> Self { Self::build(moq_net::broadcast::Info::new().produce(), config) @@ -2857,15 +3122,20 @@ mod session_tests { kind: super::TrackKind::Video, name: "video".into(), caps: gst::Caps::new_empty_simple("video/x-h264"), - binding: moq_mux::Binding::new(publisher.broadcast.consume()), + follow: moq_mux::Source::new(origin().consume(), publisher.broadcast.consume()) + .follow(None, None) + .unwrap(), + sibling: false, track: super::Track { broadcast: None, name: "video".into(), }, container: moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + descriptor: Container::Legacy, pads: Pads::default(), timeline: std::sync::Arc::default(), - finished: std::sync::Arc::default(), + siblings: super::Siblings::default(), + finished: super::Finished::default(), state: std::sync::Arc::new(super::PumpState::new()), cancel, shutdown, @@ -3117,6 +3387,301 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A `room` whose video rendition references the `cam` sibling, naming `epoch` when given. + fn referencing(origin: &moq_net::origin::Producer, epoch: Option) -> Publisher { + let mut config = video_rendition(); + config.broadcast = Some("./cam".into()); + config.epoch = epoch; + Publisher::new(origin, Default::default(), config) + } + + /// A sibling announced after the catalog starts its rendition, its end holds the pad, and its + /// return starts a new stream on that pad. + #[test] + fn a_late_sibling_starts_its_rendition_and_returns_after_an_end() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); + stop(session); + + assert!( + recorder.seen().contains(&Seen::Start(pad)), + "no new stream on the kept pad" + ); + recorder.assert_one_pad_held(); + } + + /// A sibling track finishing while its instance stays up idles the pad, and the instance that + /// replaces it starts a new stream there. + #[test] + fn a_finished_sibling_track_returns_on_the_next_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + assert!(!recorder.seen().contains(&Seen::Eos(pad)), "the finish ended the pad"); + recorder.assert_one_pad_held(); + drop(first); + } + + /// Block until `publisher`'s video track reads to its end, as a pump reading it does. + fn await_finished(publisher: &Publisher) { + let broadcast = publisher.broadcast.consume(); + super::RUNTIME.block_on(async { + let finished = async { + let subscriber = broadcast + .track("video") + .unwrap() + .subscribe(moq_net::track::Subscription::default()) + .await + .unwrap(); + let mut track = moq_mux::container::Consumer::new( + subscriber, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + while track.read().await.unwrap().is_some() {} + }; + tokio::time::timeout(Duration::from_secs(10), finished) + .await + .expect("the track never finished"); + }); + } + + /// A sibling track that finished stays finished for its instance only: a catalog update that + /// replaces the idle pump, here a disable and an enable, still plays the next instance. + #[test] + fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let _audio = room + .broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + + // The catalog consumer only yields the newest snapshot, so the same update lists an audio + // rendition: its pad appearing proves the run acted on the disable. + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().enabled = false; + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + await_pad(&element, "audio_"); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .enabled = true; + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + recorder.assert_one_pad_held(); + drop(first); + } + + /// A publisher handoff after a sibling track finished, the catalog naming the new instance's + /// epoch, plays that instance on the same pad. + #[test] + fn an_epoch_handoff_after_a_sibling_finish_plays_the_new_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let (old, new) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let mut room = referencing(&origin, Some(old.clone())); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(old)); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + + let mut second = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(new.clone())); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .epoch = Some(new); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the new instance moved to another pad" + ); + stop(session); + + recorder.assert_one_pad_held(); + drop(first); + } + + /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next + /// instance plays nothing on it. + #[test] + fn a_delisted_sibling_rendition_follows_no_later_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + room.catalog.modify().unwrap().video.renditions.clear(); + for _ in 0..200 { + if recorder.seen().contains(&Seen::Eos(pad.clone())) { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + recorder.seen().contains(&Seen::Eos(pad.clone())), + "the idle pad never ended" + ); + + let mut second = Publisher::sibling(&origin, Default::default()); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + for _ in 0..10 { + second.write(2); + std::thread::sleep(Duration::from_millis(50)); + } + stop(session); + + let seen = recorder.seen(); + assert!(!seen.iter().any(|seen| matches!(seen, Seen::Buffer(_, 2))), "{seen:?}"); + } + + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on + /// the same pad, and nothing more from the old instance gets through. + fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let mut old = Publisher::sibling(&origin, route()); + let session = follow(&element, &origin); + let pad = recorder.written(&mut old, 1); + + let mut new = Publisher::sibling(&origin, route()); + assert_eq!(recorder.written(&mut new, 2), pad, "the restart moved to another pad"); + old.write(1); + recorder.written(&mut new, 3); + stop(session); + + let seen = recorder.seen(); + let switched = seen + .iter() + .position(|seen| *seen == Seen::Buffer(pad.clone(), 2)) + .unwrap(); + assert!( + !seen[switched..].contains(&Seen::Buffer(pad.clone(), 1)), + "the replaced sibling kept playing: {seen:?}" + ); + recorder.assert_one_pad_held(); + } + + #[test] + fn a_sibling_restart_switches_on_the_same_pad() { + sibling_restart(epoch); + } + + #[test] + fn an_epochless_sibling_restart_switches_on_the_same_pad() { + sibling_restart(moq_net::origin::Route::default); + } + + /// A reference naming its sibling's epoch never reads the instance replacing it: the pad + /// idles, and plays again once the named instance serves the path again. + #[test] + fn a_sibling_restart_idles_a_reference_naming_the_old_epoch() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let named = moq_net::Epoch::mint(); + let _room = referencing(&origin, Some(named.clone())); + let mut old = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(named)); + let session = follow(&element, &origin); + let pad = recorder.written(&mut old, 1); + + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + let mut new = Publisher::sibling(&origin, epoch()); + await_event(&mut follow, |event| { + matches!(event, moq_net::announce::Event::Restart(_)) + }); + new.write(2); + drop(new); + await_event(&mut follow, |event| { + matches!(event, moq_net::announce::Event::Restart(_)) + }); + + assert_eq!( + recorder.written(&mut old, 3), + pad, + "the named instance moved to another pad" + ); + stop(session); + assert!( + !recorder.seen().contains(&Seen::Buffer(pad, 2)), + "another epoch played: {:?}", + recorder.seen() + ); + recorder.assert_one_pad_held(); + } + /// Re-pricing the same instance is an update, which the run rides out without a new stream. #[test] fn an_update_keeps_the_run() { From 286f5493bea0c9aff6ba5565c256055b80ef70e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 14:33:15 -0700 Subject: [PATCH 12/27] fix(mux): a sibling's decode error fails the export even once it closes ExportSource treated any read error as the instance's end once the sibling closed, so malformed media followed by a close exported as a clean EOF. Only a transport error ends the rendition with its instance. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/source.rs | 49 ++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 27795ce2797..606e7c9727c 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -433,8 +433,10 @@ impl ExportSource { }; let item = match consumer.poll_item(waiter) { Poll::Ready(Ok(item)) => item, - // The field, not `instance_ended`, so the borrow stays off the consumer. - Poll::Ready(Err(_)) | Poll::Pending + // Only the transport ending with the instance is its end: media that fails to + // decode still fails the export. The field, not `instance_ended`, so the borrow + // stays off the consumer. + Poll::Ready(Err(crate::Error::Moq(_))) | Poll::Pending if self .instance .as_ref() @@ -775,6 +777,49 @@ mod tests { drop((track, replacement, new)); } + /// Media that fails to decode still fails the export when its sibling instance goes: only + /// the transport ending with the instance is the rendition's end. + #[tokio::test(start_paused = true)] + async fn a_sibling_decode_error_outlives_its_instance() { + use std::time::Duration; + + let origin = crate::source::produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + tokio::task::yield_now().await; + let source = crate::source::resolved(&origin, "a/pub").await; + let config = audio(Some("./source")); + let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + + let sibling = origin.publish("a/source", Default::default()).unwrap(); + let track = sibling + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + // Subscribe before anything is written, so the malformed frame and the instance's end + // arrive together. + let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); + assert!(settle.await.is_err(), "nothing is written yet"); + // An empty Legacy audio frame is malformed. + let mut group = track.append_group().unwrap(); + group + .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) + .unwrap(); + drop(sibling); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); + + let read = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) + .await + .expect("the read settles"); + assert!( + read.is_err(), + "{:?}", + read.map(|frame| frame.map(|frame| frame.timestamp)) + ); + drop((group, track)); + } + /// The requested budget must be present on the first SUBSCRIBE, a live start's too. Updating /// it after SUBSCRIBE_OK cannot recover backlog the publisher already skipped, nor the groups a /// zero budget drops as superseded before the update lands. From a1dd7fce25bd24bd5a0d79777e4bd0879f5aa021 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 15:14:56 -0700 Subject: [PATCH 13/27] fix(mux): only an orderly sibling shutdown ends its rendition A transport failure such as a protocol violation read after the sibling closes failed nothing: any moq-net error counted as the instance's end. Only an orderly shutdown (dropped, cancelled, closed, going away, or a cancelled session) ends the rendition; every other error fails the export. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/source.rs | 122 +++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 34 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 606e7c9727c..e8aa71b1c75 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -433,10 +433,18 @@ impl ExportSource { }; let item = match consumer.poll_item(waiter) { Poll::Ready(Ok(item)) => item, - // Only the transport ending with the instance is its end: media that fails to - // decode still fails the export. The field, not `instance_ended`, so the borrow - // stays off the consumer. - Poll::Ready(Err(crate::Error::Moq(_))) | Poll::Pending + // Only an orderly shutdown of the instance is its end: malformed media or a + // failed transport still fails the export. The field, not `instance_ended`, so + // the borrow stays off the consumer. + Poll::Ready(Err(crate::Error::Moq( + moq_net::Error::Dropped + | moq_net::Error::Cancel + | moq_net::Error::Closed + | moq_net::Error::GoingAway + | moq_net::Error::SessionClosed + | moq_net::Error::Session(moq_net::SessionError::Cancel), + ))) + | Poll::Pending if self .instance .as_ref() @@ -777,47 +785,93 @@ mod tests { drop((track, replacement, new)); } - /// Media that fails to decode still fails the export when its sibling instance goes: only - /// the transport ending with the instance is the rendition's end. + /// A sibling's failure still fails the export when its instance goes, whether its media is + /// malformed or its track is aborted with a protocol error: only an orderly shutdown of the + /// instance is the rendition's end. #[tokio::test(start_paused = true)] - async fn a_sibling_decode_error_outlives_its_instance() { + async fn a_sibling_failure_outlives_its_instance() { use std::time::Duration; + for malformed in [true, false] { + let origin = crate::source::produce_origin(); + let _catalog = origin.publish("a/pub", Default::default()).unwrap(); + tokio::task::yield_now().await; + let source = crate::source::resolved(&origin, "a/pub").await; + let config = audio(Some("./source")); + let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + .unwrap() + .unwrap(); + + let sibling = origin.publish("a/source", Default::default()).unwrap(); + let track = sibling + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + // Subscribe before anything is written, so the failure and the instance's end arrive + // together. + let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); + assert!(settle.await.is_err(), "nothing is written yet"); + let mut group = track.append_group().unwrap(); + if malformed { + // An empty Legacy audio frame is malformed. + group + .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) + .unwrap(); + } else { + track.clone().abort(moq_net::Error::ProtocolViolation).unwrap(); + } + drop(sibling); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); + + let read = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) + .await + .expect("the read settles"); + assert!( + read.is_err(), + "malformed: {malformed}: {:?}", + read.map(|frame| frame.map(|frame| frame.timestamp)) + ); + drop((group, track)); + } + } + + /// A sibling served over a session ends the rendition cleanly when its publisher goes, which + /// the session reports as its own cancellation. + #[tokio::test(start_paused = true)] + async fn a_remote_sibling_ends_with_its_instance() { + use std::time::Duration; + + use crate::container::test_util::{Link, raw_frame}; + + let remote = crate::source::produce_origin(); let origin = crate::source::produce_origin(); let _catalog = origin.publish("a/pub", Default::default()).unwrap(); tokio::task::yield_now().await; let source = crate::source::resolved(&origin, "a/pub").await; - let config = audio(Some("./source")); - let mut export = ExportSource::for_audio(&source, "audio", &config, Duration::from_secs(10)) + let _link = Link::new(&remote, &origin).await; + let sibling = remote.publish("a/source", Default::default()).unwrap(); + let mut track = crate::container::Producer::new( + sibling + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(), + HangContainer::Legacy(crate::container::Kind::Audio), + ); + track.write(raw_frame(0, b"x", true)).unwrap(); + let mut export = ExportSource::for_audio(&source, "audio", &audio(Some("./source")), Duration::from_secs(10)) .unwrap() .unwrap(); + let mut read = async || { + tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) + .await + .expect("the read settles") + .unwrap() + .map(|frame| frame.timestamp.as_micros()) + }; + assert_eq!(read().await, Some(0)); - let sibling = origin.publish("a/source", Default::default()).unwrap(); - let track = sibling - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - // Subscribe before anything is written, so the malformed frame and the instance's end - // arrive together. - let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); - assert!(settle.await.is_err(), "nothing is written yet"); - // An empty Legacy audio frame is malformed. - let mut group = track.append_group().unwrap(); - group - .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) - .unwrap(); drop(sibling); - tokio::time::sleep(Duration::from_millis(10)).await; - assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); - - let read = tokio::time::timeout(Duration::from_secs(1), kio::wait(|waiter| export.poll_read(waiter))) - .await - .expect("the read settles"); - assert!( - read.is_err(), - "{:?}", - read.map(|frame| frame.map(|frame| frame.timestamp)) - ); - drop((group, track)); + assert_eq!(read().await, None, "its end ends the rendition"); + drop(track); } /// The requested budget must be present on the first SUBSCRIBE, a live start's too. Updating From f6ab8e6f0987b9635e2268ea26eae6f6a8003c3f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 15:56:12 -0700 Subject: [PATCH 14/27] fix(mux): a sibling's unfinished track fails the export `Dropped` means a producer went away without finishing, so content is incomplete rather than ended. It no longer counts as an orderly end of the sibling instance. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/source.rs | 38 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index e8aa71b1c75..eb6d0f3d274 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -437,8 +437,7 @@ impl ExportSource { // failed transport still fails the export. The field, not `instance_ended`, so // the borrow stays off the consumer. Poll::Ready(Err(crate::Error::Moq( - moq_net::Error::Dropped - | moq_net::Error::Cancel + moq_net::Error::Cancel | moq_net::Error::Closed | moq_net::Error::GoingAway | moq_net::Error::SessionClosed @@ -786,13 +785,13 @@ mod tests { } /// A sibling's failure still fails the export when its instance goes, whether its media is - /// malformed or its track is aborted with a protocol error: only an orderly shutdown of the - /// instance is the rendition's end. + /// malformed, its track is aborted with a protocol error, or its producers went away + /// unfinished: only an orderly shutdown of the instance is the rendition's end. #[tokio::test(start_paused = true)] async fn a_sibling_failure_outlives_its_instance() { use std::time::Duration; - for malformed in [true, false] { + for failure in ["malformed", "violation", "dropped"] { let origin = crate::source::produce_origin(); let _catalog = origin.publish("a/pub", Default::default()).unwrap(); tokio::task::yield_now().await; @@ -811,14 +810,23 @@ mod tests { let settle = tokio::time::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); assert!(settle.await.is_err(), "nothing is written yet"); let mut group = track.append_group().unwrap(); - if malformed { - // An empty Legacy audio frame is malformed. - group - .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) - .unwrap(); - } else { - track.clone().abort(moq_net::Error::ProtocolViolation).unwrap(); - } + let held = match failure { + "malformed" => { + // An empty Legacy audio frame is malformed. + group + .write_frame(moq_net::Timestamp::from_micros(0).unwrap(), bytes::Bytes::new()) + .unwrap(); + Some((group, track)) + } + "violation" => { + track.clone().abort(moq_net::Error::ProtocolViolation).unwrap(); + Some((group, track)) + } + _ => { + drop((group, track)); + None + } + }; drop(sibling); tokio::time::sleep(Duration::from_millis(10)).await; assert!(export.instance_ended(&kio::Waiter::noop()), "the instance has gone"); @@ -828,10 +836,10 @@ mod tests { .expect("the read settles"); assert!( read.is_err(), - "malformed: {malformed}: {:?}", + "{failure}: {:?}", read.map(|frame| frame.map(|frame| frame.timestamp)) ); - drop((group, track)); + drop(held); } } From 9776abfef42c9f1b7e1fc1c23578e28ce3d80a40 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 17:05:58 -0700 Subject: [PATCH 15/27] refactor(moq-gst): one follower per referenced broadcast drives its renditions moqsrc kept one long-lived pump per rendition, each following its own broadcast's announcements, plus an `active` map the catalog loop diffed against. A pump that exited on its own stayed in `active` until pruned, so a relist landing in that window started nothing. Each broadcast the catalog references now gets one follower task that owns its instances (`Source::follow`) and its renditions' state: a start reads every rendition on that instance, an end idles them, a takeover restarts them on the same pads. A reader is a task per (rendition, instance), and its exit reaches the follower that owns it, so idle and draining are follower state rather than tasks. The pad table tracks which renditions the latest catalog lists, so a claim and a retirement decide under one lock, replacing the PumpState compare-exchange. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 1576 +++++++++++++++++----------------- 1 file changed, 772 insertions(+), 804 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index e54be6bfbbf..d91d0780372 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1,5 +1,5 @@ use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, LazyLock, Mutex, OnceLock}; use std::time::Duration; @@ -30,7 +30,7 @@ static RUNTIME: LazyLock = LazyLock::new(|| { /// silently breaking those pipelines. A session keeps each pad by rendition across restarts /// and format changes, so only a new or renamed rendition takes another id. /// -/// An id is claimed where the pad is created, not where its pump is spawned. A rendition whose +/// An id is claimed where the pad is created, not where its reader is spawned. A rendition whose /// subscription never resolves therefore reserves nothing, so it can't leave `video_0` pointing /// at a pad that will never exist while the rendition that does arrive lands on `video_1`. static NEXT_VIDEO_PAD_ID: AtomicU64 = AtomicU64::new(0); @@ -104,14 +104,14 @@ impl TrackKind { } tokio::task_local! { - /// The element whose session task or pump is running, so [`SessionController::stop`] can tell + /// The element whose session task or reader is running, so [`SessionController::stop`] can tell /// it was reached from that element's own streaming context. static STREAMING: glib::WeakRef; } /// The session task drives everything: it connects, follows the catalog, and -/// runs one [`Pump`] per active rendition. The element just starts and -/// stops it. No control-plane channel is needed because pumps push to their pads +/// runs one [`Reader`] per rendition and instance. The element just starts and +/// stops it. No control-plane channel is needed because readers push to their pads /// directly from their own task (a source pad's push *is* its streaming thread), /// so there's nothing to marshal back onto the element. struct SessionController { @@ -169,7 +169,7 @@ impl SessionController { } } - /// Stop the session, returning once its pumps have removed their pads and the connection closed. + /// Stop the session, returning once its readers have removed their pads and the connection closed. /// /// A dial still running once the element reached NULL can outlive `main` (`gst-launch` exits right /// after), and aws-lc aborts the process when a thread asks it for randomness after its exit @@ -177,14 +177,14 @@ impl SessionController { fn stop(self, element: &super::MoqSrc) { let _ = self.shutdown.send(true); - // A pump blocked in a push returns once its pad flushes, the same unlock a GStreamer source + // A reader blocked in a push returns once its pad flushes, the same unlock a GStreamer source // gives its streaming thread on stop. A push held up inside a downstream element is released // by that element leaving PAUSED, which a pipeline does before it reaches its source. for pad in element.src_pads() { let _ = pad.set_active(false); } - // Reached from this element's own session task or pump (a bus sync or pad handler), waiting + // Reached from this element's own session task or reader (a bus sync or pad handler), waiting // for the session would wait on the caller's own stack. GStreamer refuses the same for its // sources; the connection still closes before this returns. let own = STREAMING @@ -420,9 +420,9 @@ impl MoqSrc { } } -/// The identity we reconcile a rendition on: a change to any field tears the pad down and -/// recreates it. Caps cover codec/resolution; the container descriptor covers the wire framing -/// (e.g. legacy -> cmaf); the broadcast and track identify the media, apart from its catalog ID. +/// The identity a rendition plays under: a change to any field restarts its stream on the same +/// pad. Caps cover codec/resolution; the container descriptor covers the wire framing (e.g. legacy +/// -> cmaf); the broadcast and track identify the media, apart from its catalog ID. #[derive(Clone, PartialEq)] struct Shape { caps: gst::Caps, @@ -432,55 +432,6 @@ struct Shape { epoch: Option, } -/// A pump's progress, shared with its [`ActiveTrack`] so teardown and pad creation can't both -/// win. A pump is torn down two different ways depending on how far it got: before it owns a pad, -/// stopping it means it must never create one; after, it owns a pad and has to drop it. One -/// compare-exchange settles which of the two happened, so a pad can't slip out between a -/// teardown's check and the pump's creation. -struct PumpState { - phase: AtomicU8, - /// The latest catalog still lists the rendition. A delisted pump drains the instance it reads - /// and follows no other. - listed: watch::Sender, -} - -impl PumpState { - const SUBSCRIBING: u8 = 0; - const LIVE: u8 = 1; - const CANCELLED: u8 = 2; - - fn new() -> Self { - Self { - phase: AtomicU8::new(Self::SUBSCRIBING), - listed: watch::Sender::new(true), - } - } - - /// Claim the right to create a pad, false once a teardown got here first. - fn go_live(&self) -> bool { - self.phase - .compare_exchange(Self::SUBSCRIBING, Self::LIVE, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - - /// Stop a pump that hasn't taken a pad, false if it already has one and must be torn down - /// through its cancel watch instead. - fn cancel_before_live(&self) -> bool { - self.phase - .compare_exchange(Self::SUBSCRIBING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - - fn set_listed(&self, listed: bool) { - self.listed - .send_if_modified(|current| std::mem::replace(current, listed) != listed); - } - - fn is_listed(&self) -> bool { - *self.listed.borrow() - } -} - /// A rendition by kind and moq track name: what a pad is kept by. type Rendition = (TrackKind, String); @@ -488,55 +439,39 @@ type Rendition = (TrackKind, String); /// it or however it spells the reference. #[derive(Clone, PartialEq, Eq, Hash)] struct Track { - broadcast: Option, + broadcast: moq_net::PathOwned, name: String, } -/// A rendition a run is serving, keyed in the run by moq track name. -struct ActiveTrack { - kind: TrackKind, - /// Identity we diff against on each catalog update; a change hands the pad to a new pump. - shape: Shape, - /// Tells the pump to hand its pad back and exit (set when the run ends or reconcile - /// replaces the rendition). - cancel: watch::Sender, - /// Handle to the pump task in the run's `JoinSet`. We only read - /// `is_finished()` to prune this entry once the pump ends (the `JoinSet` owns - /// the task and reaps it); teardown goes through `cancel`, never `abort()`. - task: tokio::task::AbortHandle, - /// Shared with the pump, so teardown and pad creation agree on which of them happened. - state: Arc, -} +/// The session's pads by rendition, and the renditions the latest catalog lists. The pads outlive +/// the readers and runs that stream to them, so a pipeline linked by name (`s.video_0 ! ...`) keeps +/// flowing across a restart. +#[derive(Clone, Default)] +struct Pads(Arc>); -impl ActiveTrack { - /// Tear the pump down whatever stage it reached: one still subscribing never takes a pad, - /// one that has hands it back when it sees the watch. - /// - /// Terminal, so it consumes the handle: a cancelled rendition is removed from the run's - /// active set, and respawns as a fresh pump if the catalog names it again. - fn cancel(self) { - self.state.cancel_before_live(); - let _ = self.cancel.send(true); - } +#[derive(Default)] +struct PadTable { + slots: HashMap, + /// Only a listed rendition takes a pad, or keeps one once its reader lets go. + listed: HashSet, } -/// The session's pads by rendition. They outlive the pumps and runs that stream to them, so a -/// pipeline linked by name (`s.video_0 ! ...`) keeps flowing across a restart. -#[derive(Clone, Default)] -struct Pads(Arc>>); - struct Slot { pad: gst::Pad, - /// Held by the pump streaming to the pad, so its replacement waits for it to let go. + /// Held by the reader streaming to the pad, so the next one waits for it to let go. owner: Arc>, + /// A retirement waits for the reader to let go. + retiring: bool, } -/// A pad for a pump to stream to, from [`Pads::claim`]. +/// A pad for a reader to stream to, from [`Pads::claim`]. enum Claim { - /// A new pad, already owned, for the pump to add to the element. + /// A new pad, already owned, for the reader to add to the element. New(gst::Pad, tokio::sync::OwnedMutexGuard<()>), - /// A kept pad, owned once its previous pump lets go. + /// A kept pad, owned once its previous reader lets go. Kept(gst::Pad, Arc>), + /// No pad: the latest catalog does not list the rendition. + Unlisted, } impl Pads { @@ -545,17 +480,20 @@ impl Pads { fn claim( &self, element: &glib::WeakRef, - rendition: Rendition, + rendition: &Rendition, cancel: &watch::Receiver, ) -> Option { let obj = element.upgrade()?; - let mut slots = self.0.lock().unwrap(); - // Checked under the lock: a run cancels a pump before it retires the pump's pad, so a - // cancelled pump never claims one behind the retirement. + let mut table = self.0.lock().unwrap(); + // Checked under the lock [`Self::list`] takes, so a reader that an update cancels or delists + // never claims a pad behind the update's retirements. if *cancel.borrow() { return None; } - if let Some(slot) = slots.get(&rendition) { + if !table.listed.contains(rendition) { + return Some(Claim::Unlisted); + } + if let Some(slot) = table.slots.get(rendition) { return Some(Claim::Kept(slot.pad.clone(), slot.owner.clone())); } @@ -566,36 +504,80 @@ impl Pads { .build(); let owner = Arc::new(tokio::sync::Mutex::new(())); let owned = owner.clone().try_lock_owned().expect("a new lock is free"); - slots.insert( - rendition, + table.slots.insert( + rendition.clone(), Slot { pad: pad.clone(), owner, + retiring: false, }, ); Some(Claim::New(pad, owned)) } - /// Take the rendition's pad out of the session, returning the task that ends it with EOS once - /// its pump lets go. - fn retire(&self, rendition: &Rendition) -> Option> + Send + 'static> { - let slot = self.0.lock().unwrap().remove(rendition)?; - Some(async move { - let _owned = slot.owner.clone().lock_owned().await; - slot.end(true); - None - }) + /// Whether the latest catalog lists the rendition. + fn is_listed(&self, rendition: &Rendition) -> bool { + self.0.lock().unwrap().listed.contains(rendition) } - /// The renditions with a pad that `keep` rejects. - fn unwanted(&self, keep: impl Fn(&Rendition) -> bool) -> Vec { - let slots = self.0.lock().unwrap(); - slots.keys().filter(|rendition| !keep(rendition)).cloned().collect() + /// List what the latest catalog names, returning a retirement for each pad it no longer lists. + /// Each ends its pad with EOS once its reader lets go, unless the rendition is listed again by + /// then, so a delisted rendition still playing out its track keeps its tail. + /// Nothing once `cancel` fired, since a later run lists the pads then. + fn list( + &self, + listed: HashSet, + cancel: &watch::Receiver, + ) -> Vec + Send + 'static> { + let mut table = self.0.lock().unwrap(); + if *cancel.borrow() { + return Vec::new(); + } + table.listed = listed; + let PadTable { slots, listed } = &mut *table; + slots + .iter_mut() + .filter(|(rendition, slot)| !slot.retiring && !listed.contains(*rendition)) + .map(|(rendition, slot)| { + slot.retiring = true; + let (pads, rendition, owner) = (self.clone(), rendition.clone(), slot.owner.clone()); + async move { + let _owned = owner.clone().lock_owned().await; + let slot = { + let mut table = pads.0.lock().unwrap(); + let PadTable { slots, listed } = &mut *table; + match slots.get_mut(&rendition) { + Some(slot) if Arc::ptr_eq(&slot.owner, &owner) => match listed.contains(&rendition) { + true => { + slot.retiring = false; + None + } + false => slots.remove(&rendition), + }, + _ => None, + } + }; + if let Some(slot) = slot { + slot.end(true); + } + } + }) + .collect() + } + + /// Whether `owner` still guards the rendition's pad, rather than a retirement having ended it. + fn holds(&self, rendition: &Rendition, owner: &Arc>) -> bool { + self.0 + .lock() + .unwrap() + .slots + .get(rendition) + .is_some_and(|slot| Arc::ptr_eq(&slot.owner, owner)) } /// Remove every pad once nothing streams to them any more, after an EOS if `eos`. fn release(&self, eos: bool) { - let slots = std::mem::take(&mut *self.0.lock().unwrap()); + let slots = std::mem::take(&mut self.0.lock().unwrap().slots); for slot in slots.into_values() { slot.end(eos); } @@ -683,7 +665,7 @@ async fn follow_path( }; if start { - // Cut over at once: the old run's pumps hand their pads back without waiting for their + // Cut over at once: the old run's readers hand their pads back without waiting for their // subscriptions to end. if let Some(cancel) = latest.take() { let _ = cancel.send(true); @@ -769,9 +751,9 @@ fn catalog_lost(err: &moq_mux::Error) -> bool { } } -/// Follow one broadcast's catalog, keeping one [`Pump`] per announced rendition in sync with it. -/// Returns once the catalog closes and the last pump drains, or `cancel` fires. The pads stay with -/// the session, except those of renditions the catalog retired. +/// Follow one broadcast's catalog, with one [`Follower`] per broadcast its renditions reference. +/// Returns once the catalog closes and the last follower drains, or `cancel` fires. The pads stay +/// with the session, except those of renditions the catalog retired. async fn follow_catalog( source: moq_mux::Source, pads: &Pads, @@ -795,29 +777,30 @@ async fn follow_catalog( }; let mut catalog_consumer = moq_mux::catalog::hang::Consumer::new(catalog_track); - // Follow the catalog for the whole run and reconcile our pumps against every update, - // rather than building them once from the first frame. This covers reactive publishers - // (the browser via @moq/hang) that announce an empty catalog before their encoder - // configures, then add renditions a beat later, as well as renditions appearing, - // disappearing, or changing codec/resolution mid-stream. + // Follow the catalog for the whole run and reconcile against every update, rather than + // building the renditions once from the first frame. This covers reactive publishers (the + // browser via @moq/hang) that announce an empty catalog before their encoder configures, then + // add renditions a beat later, as well as renditions appearing, disappearing, or changing + // codec/resolution mid-stream. + let (stop, stopped) = watch::channel(false); let playback = Playback { + element, pads: pads.clone(), + path: broadcast.info().path.clone(), timeline: Arc::default(), siblings: Siblings::default(), finished: Finished::default(), + shutdown: shutdown.clone(), + stop: stopped, }; - let mut active: HashMap = HashMap::new(); - let mut pumps: tokio::task::JoinSet> = tokio::task::JoinSet::new(); - // The renditions the latest catalog lists. - let mut listed: HashSet = HashSet::new(); + let mut followers: HashMap> = HashMap::new(); + // The followers, and the retirements of pads the catalog no longer lists. + let mut tasks = tokio::task::JoinSet::new(); let mut catalog_closed = false; let mut result = Ok(()); loop { - // Prune metadata for pumps that have ended (the JoinSet has already reaped the - // tasks). Once the catalog is closed and the last pump drains, the run is done. - active.retain(|_, track| !track.task.is_finished()); - if catalog_closed && pumps.is_empty() { + if catalog_closed && tasks.is_empty() { break; } @@ -825,94 +808,72 @@ async fn follow_catalog( biased; // Ended by a restart, or the session ending. _ = cancel.changed() => break, - // A pump finished; loop back so the `retain` above prunes its entry and the - // break condition sees the drained set. - joined = pumps.join_next(), if !pumps.is_empty() => { - // A rendition the catalog retired ends with EOS once its track does. One still - // listed keeps its pad, since the track, the catalog, and the announcements arrive - // on different streams in either order. - if let Some(Ok(Some(rendition))) = joined - && !listed.contains(&rendition) - && let Some(retire) = pads.retire(&rendition) - { - pumps.spawn_on(STREAMING.scope(element.clone(), retire), RUNTIME.handle()); - } - } + _ = tasks.join_next(), if !tasks.is_empty() => {} // The guard stops us polling a closed catalog track (which would spin the loop - // returning None) while we wait for the remaining pumps to drain. - next = catalog_consumer.next(), if !catalog_closed => { - match next { - Ok(Some(catalog)) => { - listed = reconcile(&catalog, &mut active, &mut pumps, &source, &playback, &element, shutdown); - } - // Catalog track closed. Don't cancel the pumps: let each reach its natural end, - // and just stop reconciling while they drain. - // - // That includes a pump still resolving its subscription, which is - // indistinguishable here from one that has simply not been polled yet: - // a final snapshot naming a track the publisher does serve arrives this - // way, and cancelling on "not live yet" would drop it. Whether a - // rendition will ever be served is the publisher's to answer, and - // ending the broadcast is where it does: every name it never served - // resolves with an error, which fails that pump's subscribe and ends - // it here. - Ok(None) => catalog_closed = true, - Err(err) if catalog_lost(&err) => { - tracing::warn!(%err, "catalog lost, holding pads"); - catalog_closed = true; - } - Err(err) => { - result = Err(err.into()); - break; - } + // returning None) while the followers drain. + next = catalog_consumer.next(), if !catalog_closed => match next { + Ok(Some(catalog)) => reconcile(&catalog, &source, &playback, &mut followers, &mut tasks, cancel), + // Catalog track closed. Don't stop the followers: let each reader reach its natural + // end, and just stop reconciling while they drain. + // + // That includes a reader still resolving its subscription, which is + // indistinguishable here from one that has simply not been polled yet: a final + // snapshot naming a track the publisher does serve arrives this way, and stopping on + // "not live yet" would drop it. Whether a rendition will ever be served is the + // publisher's to answer, and ending the broadcast is where it does: every name it + // never served resolves with an error, which fails that reader's subscribe and ends + // it. + Ok(None) => catalog_closed = true, + Err(err) if catalog_lost(&err) => { + tracing::warn!(%err, "catalog lost, holding pads"); + catalog_closed = true; } + Err(err) => { + result = Err(err.into()); + break; + } + }, + } + + if catalog_closed { + for (_, plan) in followers.drain() { + plan.send_modify(|plan| plan.closed = true); } } } - // Cancel every pump, then wait for them all to hand their pads back. Cancel all up front - // (pumps only exit on their own `cancel`), or the not-yet-cancelled ones would keep streaming - // while we await the rest. On the clean catalog-closed exit `active`/`pumps` are already - // drained, so this is a no-op. - for (_, track) in active.drain() { - track.cancel(); - } - while pumps.join_next().await.is_some() {} + // Stop every follower, then wait for each to hand its pads back. On the clean catalog-closed + // exit the tasks already drained, so this is a no-op. + let _ = stop.send(true); + while tasks.join_next().await.is_some() {} result } -/// Bring the run's pumps in line with `catalog`: spawn pumps for newly announced renditions, -/// hand the pad of any whose [`Shape`] changed to a new pump, leave ones that vanished to -/// end with their track, stop disabled ones, and end the kept pads of renditions it no longer -/// lists. A rendition that already played its track to the end keeps its pad but gets no pump. -/// Returns the renditions it lists. +/// The broadcast a [`Follower`] follows: the path a reference resolves to, and the instance it +/// names, if any. +type Followed = (moq_net::PathOwned, Option); + +/// Bring the run in line with `catalog`: list its renditions on the pads, retiring the pads of +/// those it dropped, and hand each referenced broadcast's follower the renditions that read it. /// /// Infallible by design: every way a single rendition can be unusable (unsupported codec, -/// malformed init, a name the broadcast refuses) skips just that rendition, so one bad entry in -/// the catalog can never tear down the ones already streaming. +/// malformed init, a reference that escapes the root) skips just that rendition, so one bad entry +/// in the catalog can never tear down the ones already streaming. fn reconcile( catalog: &moq_mux::catalog::hang::Catalog, - active: &mut HashMap, - pumps: &mut tokio::task::JoinSet>, source: &moq_mux::Source, playback: &Playback, - element: &glib::WeakRef, - shutdown: &watch::Receiver, -) -> HashSet { - let pads = &playback.pads; - struct Desired { - kind: TrackKind, - shape: Shape, - } - + followers: &mut HashMap>, + tasks: &mut tokio::task::JoinSet<()>, + cancel: &watch::Receiver, +) { // Build the desired shape for each rendition. This is deliberately cheap: caps come from the - // catalog config and the container is just the hang descriptor. We defer parsing the wire - // container (which re-parses the CMAF init) to spawn time below, so an unchanged rendition - // costs nothing here. A rendition whose caps we can't build (unsupported codec) is logged and - // skipped rather than failing the whole run, so one bad rendition can't tear down the - // others we're already serving. - let mut desired: HashMap = HashMap::new(); + // catalog config and the container is just the hang descriptor. The reader parses the wire + // container (which re-parses the CMAF init) when it starts, so an unchanged rendition costs + // nothing here. A rendition whose caps we can't build (unsupported codec) is logged and + // skipped. + let mut desired: HashMap = HashMap::new(); let mut insert = |name: &String, kind, caps: Result, @@ -926,7 +887,7 @@ fn reconcile( broadcast: reference.broadcast().cloned(), epoch: reference.epoch().cloned(), }; - desired.insert(name.clone(), Desired { kind, shape }); + desired.insert((kind, name.clone()), shape); } Err(err) => gst::warning!(CAT, "ignoring {kind:?} rendition {name}: {err:?}"), }; @@ -937,9 +898,9 @@ fn reconcile( insert(name, TrackKind::Audio, audio_caps(config), &config.container, config); } - // A disabled rendition stays listed: its pump stops, but its pad waits idle for a re-enable, + // A disabled rendition stays listed: its reader stops, but its pad waits idle for a re-enable, // since a pipeline links pads by name (`s.video_0 ! ...`). - let disabled: HashSet = catalog + let disabled = catalog .video .renditions .iter() @@ -952,70 +913,393 @@ fn reconcile( .iter() .filter(|(_, config)| !config.enabled) .map(|(name, _)| (TrackKind::Audio, name.clone())), - ) - .collect(); + ); + // Listed before any follower hears of the update, so a reader it stops can't claim a pad. + let listed = desired.keys().cloned().chain(disabled).collect(); + for retire in playback.pads.list(listed, cancel) { + tasks.spawn_on(STREAMING.scope(playback.element.clone(), retire), RUNTIME.handle()); + } - // Pure set math: which pumps to tear down, which renditions to spawn. - let plan = plan_reconcile( - &desired - .iter() - .map(|(name, d)| (name.clone(), (d.kind, d.shape.clone()))) - .collect(), - &active - .iter() - .map(|(name, t)| (name.clone(), (t.kind, t.shape.clone()))) - .collect(), - ); + let mut plans: HashMap = HashMap::new(); + for (rendition, shape) in desired { + let Some(path) = source.resolve_reference(shape.broadcast.as_ref()) else { + gst::warning!( + CAT, + "ignoring rendition {}: its broadcast escapes the root", + rendition.1 + ); + continue; + }; + plans + .entry((path, shape.epoch.clone())) + .or_default() + .renditions + .insert(rendition, shape); + } + + // A broadcast no rendition reads any more is not followed: its readers stop, apart from a + // delisted one playing out its tail. + followers.retain(|followed, _| plans.contains_key(followed)); + for (followed, plan) in plans { + // A follower whose broadcast failed exited, and this update follows it again. + if let Some(follower) = followers.get(&followed) + && !follower.is_closed() + { + follower.send_replace(plan); + continue; + } + + // Follow without waiting: an unavailable sibling must not park the catalog loop and hold + // up every other rendition. The follower resolves its instances under the run. + let shape = plan.renditions.values().next().expect("a plan names a rendition"); + let instances = match source.follow(shape.broadcast.as_ref(), followed.1.as_ref()) { + Ok(instances) => instances, + Err(err) => { + gst::warning!(CAT, "ignoring the renditions of {}: {err:?}", followed.0); + continue; + } + }; + let (follower, plan) = watch::channel(plan); + let follower_task = Follower { + playback: playback.clone(), + path: followed.0.clone(), + renditions: HashMap::new(), + readers: tokio::task::JoinSet::new(), + current: None, + next_id: 0, + } + .run(instances, plan); + tasks.spawn_on( + STREAMING.scope(playback.element.clone(), follower_task), + RUNTIME.handle(), + ); + followers.insert(followed, follower); + } +} + +/// What one broadcast run shares with every follower and reader it starts. Pads survive a +/// broadcast switch; its timing and finished tracks do not. +#[derive(Clone)] +struct Playback { + element: glib::WeakRef, + pads: Pads, + /// The catalog's own path, whose renditions read the run's broadcast rather than following + /// announcements. + path: moq_net::PathOwned, + timeline: Arc, + siblings: Siblings, + finished: Finished, + /// The session's shutdown, which [`SessionController::stop`] sets before it flushes the pads. + shutdown: watch::Receiver, + /// Set once the run ends, stopping every follower. + stop: watch::Receiver, +} + +/// The renditions a run asks one [`Follower`] to play. +#[derive(Clone, Default)] +struct Plan { + renditions: HashMap, + /// The catalog closed: no rendition starts again, and the follower ends once its readers + /// drain. + closed: bool, +} + +/// Follows one broadcast's instances and plays the renditions that read it on each in turn: a +/// start reads them on that instance, an end idles their pads, and another instance taking over +/// restarts each stream on the same pad. The catalog's own broadcast is a single instance that +/// never ends. +/// +/// It owns its renditions' readers, so whether a rendition is reading, idle, or playing out its +/// tail is its own state rather than a task's. +struct Follower { + playback: Playback, + path: moq_net::PathOwned, + renditions: HashMap, + readers: tokio::task::JoinSet<(Rendition, u64, Exit)>, + /// The instance being read, and the timeline its renditions share. + current: Option<(moq_net::broadcast::Consumer, Arc)>, + next_id: u64, +} + +/// A rendition a [`Follower`] plays. +struct Entry { + shape: Shape, + /// Reading the current instance, if it is. + reader: Option, + /// Delisted while reading: plays out the instance it reads, then leaves. + draining: bool, +} + +/// A running [`Reader`], cancelled when dropped. +struct ReaderHandle { + id: u64, + /// Set once the reader owns the rendition's pad. + live: Arc, + cancel: watch::Sender, +} + +impl Drop for ReaderHandle { + fn drop(&mut self) { + let _ = self.cancel.send(true); + } +} - // A delisted pump drains the instance it reads and follows no other; a relisted one follows - // again. - for (name, track) in active.iter() { - track.state.set_listed(desired.contains_key(name)); - } - - // Cancel anything that changed shape; each cancelled pump hands its pad back. Changed - // renditions also land in `plan.add`, so they respawn below on the same pad. - // - // A rendition that vanished is only no longer selectable: a publisher retires one by - // delisting it and finishing its track, and the two arrive in either order. A pump that owns - // a pad stays in `active` and ends with its track, where `follow_catalog` ends its pad. One - // that never took a pad is stopped. - for name in plan.remove { - let Some(track) = active.get(&name) else { continue }; - // An explicit disable stops a live pump too; a delist still drains its tail. - let drains = !desired.contains_key(&name) - && !disabled.contains(&(track.kind, name.clone())) - && !track.state.cancel_before_live(); - if !drains && let Some(track) = active.remove(&name) { - track.cancel(); +impl Follower { + /// Play the renditions `plan` names on each instance until the plan closes and the last reader + /// drains, the run stops, or the broadcast fails. + async fn run(mut self, mut instances: moq_mux::follow::Instances, mut plan: watch::Receiver) { + let mut stop = self.playback.stop.clone(); + // The catalog's own broadcast is ready at once, so its renditions play even when the + // catalog closed before this follower ran. + match instances.poll_next(&moq_net::kio::Waiter::noop()) { + std::task::Poll::Ready(Ok(moq_mux::follow::Event::Start(instance))) => self.start(instance), + std::task::Poll::Ready(Err(err)) => { + gst::warning!(CAT, "lost {}: {err:?}", self.path); + return; + } + _ => {} } + let first = plan.borrow_and_update().clone(); + let mut following = !first.closed; + self.apply(first.renditions); + self.play(); + + loop { + if !following && self.readers.is_empty() { + break; + } + + tokio::select! { + biased; + _ = stop.changed() => break, + changed = plan.changed(), if following => { + // A dropped plan reads no rendition any more. + let next = match changed { + Ok(()) => plan.borrow_and_update().clone(), + Err(_) => Plan { closed: true, ..Default::default() }, + }; + following = !next.closed; + self.apply(next.renditions); + if following { + self.play(); + } + } + Some(joined) = self.readers.join_next() => { + if let Ok((rendition, id, exit)) = joined { + self.exited(rendition, id, exit); + } + } + event = instances.next(), if following => match event { + Ok(moq_mux::follow::Event::Start(instance)) => { + gst::info!(CAT, "{} is online, starting its renditions", self.path); + self.start(instance); + } + Ok(moq_mux::follow::Event::End) => { + gst::info!(CAT, "{} is offline, holding its pads", self.path); + self.current = None; + self.idle(); + } + Err(err) => { + gst::warning!(CAT, "lost {}: {err:?}", self.path); + break; + } + }, + } + } + + self.renditions.clear(); + while self.readers.join_next().await.is_some() {} + } + + /// Bring the renditions in line with `wanted`, stopping each one reshaped or gone. [`Self::play`] + /// starts the rest. + fn apply(&mut self, mut wanted: HashMap) { + let pads = &self.playback.pads; + self.renditions + .retain(|rendition, entry| match wanted.remove(rendition) { + Some(shape) => { + // A new shape hands the pad to a new reader. + if shape != entry.shape { + entry.shape = shape; + entry.reader = None; + } + entry.draining = false; + true + } + // A delisted rendition plays out the instance it reads, then leaves. One that never + // took its pad, or that is disabled or now reads another broadcast, stops now. + None => { + entry.draining = !pads.is_listed(rendition) + && entry + .reader + .as_ref() + .is_some_and(|reader| reader.live.load(Ordering::Acquire)); + entry.draining + } + }); + for (rendition, shape) in wanted { + self.renditions.insert( + rendition, + Entry { + shape, + reader: None, + draining: false, + }, + ); + } + } + + /// Read `instance`, which took over from any before it. + fn start(&mut self, instance: moq_net::broadcast::Consumer) { + let timeline = match self.path == self.playback.path { + true => self.playback.timeline.clone(), + false => self.playback.siblings.timeline(&instance, &self.playback.timeline), + }; + self.current = Some((instance, timeline)); + self.idle(); + self.play(); } - // A kept pad the catalog no longer lists ends with EOS, unless a pump still drains its track. - let unwanted = pads.unwanted(|rendition| { - let (kind, name) = rendition; - desired.get(name).is_some_and(|d| d.kind == *kind) - || disabled.contains(rendition) - || active.get(name).is_some_and(|t| t.kind == *kind) - }); - for rendition in unwanted { - if let Some(retire) = pads.retire(&rendition) { - pumps.spawn_on(STREAMING.scope(element.clone(), retire), RUNTIME.handle()); + /// Stop reading the instance. A draining rendition leaves with it. + fn idle(&mut self) { + self.renditions.retain(|_, entry| !entry.draining); + for entry in self.renditions.values_mut() { + entry.reader = None; } } - // Spawn pumps for new or changed renditions. The wire container is parsed here, lazily and - // only for renditions we're actually starting, since parsing a CMAF init is wasted work for - // renditions that didn't change. A parse failure (malformed init) skips just this rendition. - for name in plan.add { - let d = &desired[&name]; - let track = Track { - broadcast: source.resolve_reference(d.shape.broadcast.as_ref()), - name: d.shape.track.clone(), + /// Start a reader on the current instance for each rendition without one. + fn play(&mut self) { + let Some((instance, timeline)) = &self.current else { + return; + }; + for (rendition, entry) in &mut self.renditions { + if entry.reader.is_none() { + self.next_id += 1; + let (cancel, cancelled) = watch::channel(false); + let live = Arc::new(AtomicBool::new(false)); + let reader = Reader { + playback: self.playback.clone(), + rendition: rendition.clone(), + shape: entry.shape.clone(), + track: Track { + broadcast: self.path.clone(), + name: entry.shape.track.clone(), + }, + instance: instance.clone(), + timeline: timeline.clone(), + live: live.clone(), + cancel: cancelled, + }; + let (rendition, id) = (rendition.clone(), self.next_id); + self.readers.spawn_on( + STREAMING.scope(self.playback.element.clone(), async move { + let exit = reader.run().await; + (rendition, id, exit) + }), + RUNTIME.handle(), + ); + entry.reader = Some(ReaderHandle { id, live, cancel }); + } + } + } + + /// The rendition's reader `id` exited on its own. + fn exited(&mut self, rendition: Rendition, id: u64, exit: Exit) { + let Some(entry) = self.renditions.get_mut(&rendition) else { + return; + }; + if entry.reader.as_ref().is_none_or(|reader| reader.id != id) { + return; + } + entry.reader = None; + if entry.draining { + self.renditions.remove(&rendition); + } else if let Exit::Unlisted = exit + && self.playback.pads.is_listed(&rendition) + { + // Delisted and listed again while it went to claim its pad, which the plan coalesced. + self.play(); + } + } +} + +/// Why a [`Reader`] exited. +enum Exit { + /// It is done with the instance. + Done, + /// The catalog did not list the rendition when it went to claim its pad. + Unlisted, +} + +/// One rendition reading one instance: everything [`Follower::play`] hands a task it spawns. +struct Reader { + playback: Playback, + rendition: Rendition, + shape: Shape, + track: Track, + instance: moq_net::broadcast::Consumer, + timeline: Arc, + /// Shared with this reader's [`ReaderHandle::live`]. + live: Arc, + cancel: watch::Receiver, +} + +impl Reader { + /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the + /// track ends, errors, its instance stops serving, or `cancel` fires. The pad stays with the + /// session either way. + async fn run(self) -> Exit { + let Reader { + playback, + rendition, + shape, + track: media, + instance, + timeline, + live, + mut cancel, + } = self; + let Playback { + element, + pads, + finished, + shutdown, + .. + } = playback; + let (kind, name) = (rendition.0, &rendition.1); + let played = (rendition.clone(), media.clone()); + + // An instance that already played this track to its end on the pad would replay its last + // group. + if finished.contains(&played, &instance) { + return Exit::Done; + } + + // Resolves once the publisher answers, with the track info or with an error (which is what + // ending a broadcast produces for a name nobody served). A publisher that answers neither + // leaves this waiting, so racing `cancel` keeps such a reader reapable, and holding the + // wait here keeps it off every other rendition. + let subscriber = tokio::select! { + _ = cancel.changed() => return Exit::Done, + subscriber = async { + instance + .track(&media.name)? + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) + .await + } => match subscriber { + Ok(subscriber) => subscriber, + // The instance went while subscribing: that is its end, not this rendition's. + Err(_) if instance.is_closed() => return Exit::Done, + Err(err) => { + gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); + return Exit::Done; + } + } }; let container = match moq_mux::catalog::hang::Container::new( - &d.shape.container, - match d.kind { + &shape.container, + match kind { TrackKind::Video => moq_mux::container::Kind::Video, TrackKind::Audio => moq_mux::container::Kind::Audio, }, @@ -1023,96 +1307,122 @@ fn reconcile( Ok(container) => container, Err(err) => { gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); - continue; + return Exit::Done; } }; - - // Follow without waiting: an unavailable sibling or track must not park the catalog loop - // and hold up every other rendition. The pump resolves both under its cancellation scope. - let follow = match source.follow(d.shape.broadcast.as_ref(), d.shape.epoch.as_ref()) { - Ok(follow) => follow, - Err(err) => { - gst::warning!(CAT, "ignoring rendition {name}: {err:?}"); - continue; + let mut track = moq_mux::container::Consumer::new(subscriber, container); + + // A pad appears only once its track is live, so a pad downstream can link to is a promise + // that the rendition is actually flowing, and only a rendition that gets this far claims a + // pad id. A kept pad starts a new stream on it, with caps pushed even when they differ, so + // a format downstream can't take fails loudly as not-negotiated rather than going quiet. + let (pad, _owned) = match pads.claim(&element, &rendition, &cancel) { + None => return Exit::Done, + Some(Claim::Unlisted) => return Exit::Unlisted, + Some(Claim::New(pad, owned)) => { + let Some(obj) = element.upgrade() else { + return Exit::Done; + }; + if pad.set_active(true).is_err() { + return Exit::Done; + } + begin(&pad, name, &shape.caps); + if obj.add_pad(&pad).is_err() { + return Exit::Done; + } + (pad, owned) } - }; - let sibling = track.broadcast != source.resolve_reference(None); - - let (cancel_tx, cancel_rx) = watch::channel(false); - let state = Arc::new(PumpState::new()); - let task = pumps.spawn_on( - STREAMING.scope( - element.clone(), - Pump { - element: element.clone(), - kind: d.kind, - name: name.clone(), - caps: d.shape.caps.clone(), - follow, - sibling, - track, - container, - descriptor: d.shape.container.clone(), - pads: pads.clone(), - timeline: playback.timeline.clone(), - siblings: playback.siblings.clone(), - finished: playback.finished.clone(), - state: state.clone(), - cancel: cancel_rx, - shutdown: shutdown.clone(), + Some(Claim::Kept(pad, owner)) => { + let owned = tokio::select! { + biased; + _ = cancel.changed() => return Exit::Done, + owned = owner.clone().lock_owned() => owned, + }; + // The reader this one replaces records its track as finished before letting go, so + // one started while it was reading the end (a reshape) can't replay it. A + // retirement that got the pad first ended it. + if !pads.holds(&rendition, &owner) || finished.contains(&played, &instance) { + return Exit::Done; } - .run(), - ), - RUNTIME.handle(), - ); + tokio::task::block_in_place(|| begin(&pad, name, &shape.caps)); + (pad, owned) + } + }; + live.store(true, Ordering::Release); + // Stop flushes only the pads it finds, and this one may have been added just after, while + // this reader's cancel is still on its way. Flushing it here keeps a push from blocking + // stop. + if *shutdown.borrow() { + let _ = pad.set_active(false); + } - active.insert( - name, - ActiveTrack { - kind: d.kind, - shape: d.shape.clone(), - cancel: cancel_tx, - task, - state, - }, - ); + let mut sent_segment = false; + loop { + tokio::select! { + // The follower stopped this reader (the instance changed, the rendition was + // reshaped or delisted, or the run ended). An end that arrived alongside still + // finishes the track, or a reader waiting for this pad would replay it. + _ = cancel.changed() => { + if ended(&mut track) { + finished.insert(&played, &instance); + } + return Exit::Done; + } + frame = track.read() => match frame { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { + let buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); + } + return Exit::Done; + } + }; + let segment = (!sent_segment).then(|| { + let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other readers. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {}", shape.caps]); + } + return Exit::Done; + } + Err(_) => return Exit::Done, + } + } + // The track stopping with its instance is that instance's end, which idles the + // pad for the next one rather than ending the rendition. + Ok(None) | Err(_) if instance.is_closed() => return Exit::Done, + // Recorded while this reader still owns the pad, before any other can claim it, + // so a reshape on this instance does not replay it. + Ok(None) => { + finished.insert(&played, &instance); + return Exit::Done; + } + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + return Exit::Done; + } + } + } + } } - - desired - .into_iter() - .map(|(name, d)| (d.kind, name)) - .chain(disabled) - .collect() -} - -/// Tear-down / spawn decisions for one catalog update, computed purely from the desired and -/// active rendition sets. A name present in both with an equal shape is left untouched; a name -/// whose shape changed lands in both lists (cancel the old pump, spawn a fresh one). -struct ReconcilePlan { - remove: Vec, - add: Vec, -} - -fn plan_reconcile(desired: &HashMap, active: &HashMap) -> ReconcilePlan { - let remove = active - .iter() - .filter(|(name, shape)| desired.get(*name) != Some(*shape)) - .map(|(name, _)| name.clone()) - .collect(); - let add = desired - .iter() - .filter(|(name, shape)| active.get(*name) != Some(*shape)) - .map(|(name, _)| name.clone()) - .collect(); - ReconcilePlan { remove, add } -} - -/// Pads survive a broadcast switch; its timing and finished tracks do not. -struct Playback { - pads: Pads, - timeline: Arc, - siblings: Siblings, - finished: Finished, } /// The timeline of each sibling instance, shared by every rendition reading that instance so @@ -1159,8 +1469,8 @@ impl Siblings { /// subscription to a finished track replays its last group, so a catalog update that still lists /// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on /// that pad again. A later instance carries a new track, and a pad that never played the track -/// joins it like any late subscriber. Shared with the pumps, since a pump that replaces another on -/// its pad learns this only once the other lets go. +/// joins it like any late subscriber. Shared with the readers, since a reader that replaces +/// another on its pad learns this only once the other lets go. #[derive(Clone, Default)] struct Finished(Arc>>); @@ -1180,7 +1490,7 @@ impl Finished { } } -/// The first decoded frame fixes one mapping for every rendition of this broadcast run. +/// The first decoded frame fixes one mapping for every rendition reading an instance. #[derive(Default)] struct Timeline(OnceLock>); @@ -1196,328 +1506,6 @@ impl Timeline { } } -/// One rendition's pump: everything [`reconcile`] hands a task it spawns. -struct Pump { - element: glib::WeakRef, - kind: TrackKind, - /// The rendition ID, which is also the pad's stream id. The pad's own name comes from a - /// per-kind counter instead, and isn't known until the subscription resolves. - name: String, - caps: gst::Caps, - /// Each instance of the broadcast the rendition references. - follow: moq_mux::follow::Instances, - /// The rendition references another broadcast than its catalog's, whose instances come and - /// go on their own announcements. - sibling: bool, - track: Track, - /// The wire format of the first instance. - container: moq_mux::catalog::hang::Container, - /// What builds it again for a later instance. - descriptor: hang::catalog::Container, - pads: Pads, - timeline: Arc, - /// The run's [`Playback::siblings`]. - siblings: Siblings, - /// The run's [`Playback::finished`]. - finished: Finished, - /// Shared with this rendition's [`ActiveTrack::state`]. - state: Arc, - cancel: watch::Receiver, - /// The session's shutdown, which [`SessionController::stop`] sets before it flushes the pads. - shutdown: watch::Receiver, -} - -/// Why a pump stopped reading one instance without ending. -enum Stopped { - /// Another instance took over: restart on it. - Restart(moq_net::broadcast::Consumer), - /// The instance stopped serving: idle until the next one. - Idle, -} - -impl Pump { - /// Subscribe to the track, then stream its frames to the rendition's pad, owning it until the - /// track ends, errors, or `cancel` fires. The pad goes back to the session without an EOS - /// either way. Returns the rendition if its track ended rather than being cancelled, so the - /// run can end the pad of one the catalog retired. - /// - /// A sibling's announcements drive the pump: its first start binds the rendition, an end - /// idles the pad, and another instance taking over restarts the stream on the same pad, until - /// the catalog delists the rendition. - async fn run(self) -> Option { - let Pump { - element, - kind, - name, - caps, - mut follow, - sibling, - track: media, - container, - descriptor, - pads, - timeline: run, - siblings, - finished, - state, - mut cancel, - shutdown, - } = self; - let played = ((kind, name.clone()), media.clone()); - let mut container = Some(container); - // The pad once this pump owns it, from the first instance on. - let mut owned: Option<(gst::Pad, tokio::sync::OwnedMutexGuard<()>)> = None; - let mut next: Option = None; - let mut listed = state.listed.subscribe(); - - loop { - // A delisted rendition drained the instance it read, and follows no other. - if owned.is_some() && !state.is_listed() { - return Some((kind, name)); - } - let broadcast = match next.take() { - Some(broadcast) => broadcast, - None => tokio::select! { - _ = cancel.changed() => return None, - _ = listed.wait_for(|listed| !listed), if owned.is_some() => return Some((kind, name)), - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => broadcast, - Ok(moq_mux::follow::Event::End) => continue, - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - }, - }; - - // An instance that already played this track to its end on the pad would replay its - // last group. A sibling waits for the next instance; the catalog's own has no other. - if finished.contains(&played, &broadcast) { - match sibling { - true => continue, - false => return None, - } - } - - // Resolves once the publisher answers, with the track info or with an error (which is - // what ending a broadcast produces for a name nobody served). A publisher that answers - // neither leaves this waiting, so racing `cancel` keeps such a pump reapable, and - // holding the wait here rather than in `reconcile` keeps it off every other - // rendition. - let subscribed = tokio::select! { - _ = cancel.changed() => return None, - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => Err(Stopped::Restart(broadcast)), - Ok(moq_mux::follow::Event::End) => Err(Stopped::Idle), - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - subscriber = async { - broadcast - .track(&media.name)? - .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) - .await - } => match subscriber { - Ok(subscriber) => Ok(subscriber), - // The instance went while subscribing: that is its end, not this rendition's. - Err(_) if sibling && broadcast.is_closed() => Err(Stopped::Idle), - Err(err) => { - gst::warning!(CAT, "track {name} failed to subscribe: {err:?}"); - return Some((kind, name)); - } - } - }; - let subscriber = match subscribed { - Ok(subscriber) => subscriber, - Err(stopped) => { - next = stopped.restart(); - continue; - } - }; - let container = match container.take() { - Some(container) => container, - None => match moq_mux::catalog::hang::Container::new( - &descriptor, - match kind { - TrackKind::Video => moq_mux::container::Kind::Video, - TrackKind::Audio => moq_mux::container::Kind::Audio, - }, - ) { - Ok(container) => container, - Err(err) => { - gst::warning!(CAT, "rendition {name} has an unsupported container: {err:?}"); - return Some((kind, name)); - } - }, - }; - let mut track = moq_mux::container::Consumer::new(subscriber, container); - let timeline = match sibling { - true => siblings.timeline(&broadcast, &run), - false => run.clone(), - }; - - let pad = match &owned { - // Another instance on the pad this pump owns: a new stream. - Some((pad, _)) => { - tokio::task::block_in_place(|| begin(pad, &name, &caps)); - pad.clone() - } - None => { - // Winning this is what earns a pad. Losing means a teardown got here while the - // subscription was still resolving (this rendition was removed, reshaped, or - // outlived by a closing catalog), and it must not publish a pad at all: the - // watch alone can't say that, since a cancel landing just after we read it - // would leave a pad exposed and then yanked without an EOS. - if !state.go_live() { - return None; - } - - // A pad appears only once its track is live, so a pad downstream can link to is a - // promise that the rendition is actually flowing, and only a rendition that gets - // this far claims a pad id. A kept pad starts a new stream on it, with caps pushed - // even when they differ, so a format downstream can't take fails loudly as - // not-negotiated rather than going quiet. - let (pad, guard) = match pads.claim(&element, (kind, name.clone()), &cancel)? { - Claim::New(pad, owned) => { - let obj = element.upgrade()?; - pad.set_active(true).ok()?; - begin(&pad, &name, &caps); - obj.add_pad(&pad).ok()?; - (pad, owned) - } - Claim::Kept(pad, owner) => { - let guard = tokio::select! { - biased; - _ = cancel.changed() => return None, - guard = owner.lock_owned() => guard, - }; - // The pump this one replaces records its track as finished before letting - // go, so a replacement started while it was reading the end (a reshape) - // can't replay it. A sibling holds the pad for its next instance. - if finished.contains(&played, &broadcast) { - if !sibling { - return Some((kind, name)); - } - owned = Some((pad, guard)); - continue; - } - tokio::task::block_in_place(|| begin(&pad, &name, &caps)); - (pad, guard) - } - }; - // Stop flushes only the pads it finds, and this one may have been added just - // after, while this pump's cancel is still on its way. Flushing it here keeps a - // push from blocking stop. - if *shutdown.borrow() { - let _ = pad.set_active(false); - } - owned = Some((pad.clone(), guard)); - pad - } - }; - - let mut sent_segment = false; - let stopped = loop { - tokio::select! { - // This rendition is being torn down (its run ended, or a catalog update replaced - // it). An end that arrived alongside still finishes the track, or the replacement - // waiting for this pad would replay it. - _ = cancel.changed() => { - if ended(&mut track) { - finished.insert(&played, &broadcast); - } - return None; - } - followed = follow.next() => match followed { - Ok(moq_mux::follow::Event::Start(broadcast)) => break Stopped::Restart(broadcast), - Ok(moq_mux::follow::Event::End) => break Stopped::Idle, - Err(err) => { - gst::warning!(CAT, "rendition {name} lost its broadcast: {err:?}"); - return Some((kind, name)); - } - }, - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return Some((kind, name)); - } - }; - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other pumps. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {caps}"]); - } - return Some((kind, name)); - } - Err(_) => return Some((kind, name)), - } - } - // A sibling's track stopping with its instance is that instance's end, which - // idles the pad for the next one rather than ending the rendition. - Ok(None) | Err(_) if sibling && broadcast.is_closed() => break Stopped::Idle, - // A listed sibling's track finishing ends it for this instance only: the pad - // idles for the next instance, and the record keeps a reshape on this one - // from replaying it. - Ok(None) if sibling && state.is_listed() => { - finished.insert(&played, &broadcast); - break Stopped::Idle; - } - Ok(None) => { - // Recorded while this pump still owns the pad, before any replacement can claim it. - finished.insert(&played, &broadcast); - return Some((kind, name)); - } - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - return Some((kind, name)); - } - } - } - }; - match &stopped { - Stopped::Restart(_) => gst::info!(CAT, "rendition {name} restarts on a new instance"), - Stopped::Idle => gst::info!(CAT, "rendition {name} is offline, holding its pad"), - } - next = stopped.restart(); - } - } -} - -impl Stopped { - /// The instance to restart on, if any. - fn restart(self) -> Option { - match self { - Stopped::Restart(broadcast) => Some(broadcast), - Stopped::Idle => None, - } - } -} - /// Whether the track's end is the next media item, without waiting for it. Markers ready before it /// are passed over; a frame or anything not ready yet means it has not ended. fn ended(track: &mut moq_mux::container::Consumer) -> bool { @@ -1700,12 +1688,11 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{PumpState, Siblings, Timeline, ended, plan_reconcile}; + use super::{Siblings, Timeline, ended}; use moq_net::Timestamp; - use std::collections::HashMap; use std::sync::Arc; - /// A pump cancelled as its track ends records the end only when no media is left to read, + /// A reader cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. #[test] fn ended_sees_only_an_end_with_no_media_before_it() { @@ -1788,53 +1775,6 @@ mod tests { } } - // The shape type is generic, so the set math can be exercised with a plain integer standing - // in for (caps, container): equal value == unchanged rendition, different value == reshape. - fn renditions(pairs: &[(&str, u32)]) -> HashMap { - pairs.iter().map(|(name, shape)| (name.to_string(), *shape)).collect() - } - - fn sorted(mut names: Vec) -> Vec { - names.sort(); - names - } - - #[test] - fn plan_reconcile_diffs_by_name_and_shape() { - // keep: same shape (untouched). gone: removed. added: new. changed: same name, new - // shape, so it must be both torn down and respawned. - let active = renditions(&[("keep", 1), ("gone", 1), ("changed", 1)]); - let desired = renditions(&[("keep", 1), ("changed", 2), ("added", 9)]); - - let plan = plan_reconcile(&desired, &active); - assert_eq!(sorted(plan.remove), vec!["changed", "gone"]); - assert_eq!(sorted(plan.add), vec!["added", "changed"]); - } - - #[test] - fn plan_reconcile_noops_on_identical_sets() { - let set = renditions(&[("a", 1), ("b", 2)]); - let plan = plan_reconcile(&set, &set); - assert!(plan.remove.is_empty()); - assert!(plan.add.is_empty()); - } - - #[test] - fn plan_reconcile_empty_desired_removes_all() { - let active = renditions(&[("a", 1), ("b", 2)]); - let plan = plan_reconcile(&HashMap::new(), &active); - assert_eq!(sorted(plan.remove), vec!["a", "b"]); - assert!(plan.add.is_empty()); - } - - #[test] - fn plan_reconcile_empty_active_adds_all() { - let desired = renditions(&[("a", 1), ("b", 2)]); - let plan = plan_reconcile(&desired, &HashMap::new()); - assert!(plan.remove.is_empty()); - assert_eq!(sorted(plan.add), vec!["a", "b"]); - } - /// Renditions on the same sibling instance share its timeline, whenever each starts on it: /// the first instance shares the run's, and the next one starts another. #[tokio::test(start_paused = true)] @@ -1898,37 +1838,14 @@ mod tests { }; assert!(super::build_buffer(frame, super::TrackKind::Video).is_err()); } - - /// The pad claim and the teardown race for every pump, and only one may win: a subscription - /// resolving after a teardown must not publish a rendition the session has finished with and - /// then yank it without an EOS. The cancel watch alone cannot say which happened, so this is - /// the state that does. - #[test] - fn a_pump_either_goes_live_or_is_cancelled() { - let cancelled = PumpState::new(); - assert!(cancelled.cancel_before_live()); - assert!(!cancelled.go_live(), "a cancelled pump still claimed a pad"); - assert!( - !cancelled.cancel_before_live(), - "a second teardown claimed the same transition" - ); - - let live = PumpState::new(); - assert!(live.go_live()); - assert!( - !live.cancel_before_live(), - "a live pump was dropped without its cancel watch" - ); - assert!(!live.go_live(), "a live pump claimed a second pad"); - } } #[cfg(test)] #[allow(clippy::disallowed_methods, clippy::disallowed_types)] mod session_tests { - use std::collections::BTreeMap; - use std::sync::Mutex; + use std::collections::{BTreeMap, HashSet}; use std::sync::atomic::Ordering; + use std::sync::{Arc, Mutex}; use std::time::Duration; use gst::glib; @@ -1950,7 +1867,7 @@ mod session_tests { PAD_IDS.lock().unwrap_or_else(|err| err.into_inner()) } - /// The pumps push from their own tasks, so the element only has to exist and own pads. + /// The readers push from their own tasks, so the element only has to exist and own pads. fn element() -> super::super::MoqSrc { gst::init().unwrap(); glib::Object::new() @@ -2007,7 +1924,7 @@ mod session_tests { } /// Real wire consumers and source pads, with each write acknowledged by a pad probe. - /// No arrival-time guesses: the first acknowledgement fixes which pump won the run. + /// No arrival-time guesses: the first acknowledgement fixes which reader won the run. #[test] fn audio_and_video_share_a_run_timeline_across_scales_and_restarts() { let _pad_ids = pad_ids(); @@ -2112,7 +2029,7 @@ mod session_tests { }) .unwrap(); rx.recv_timeout(Duration::from_secs(10)) - .expect("pump did not deliver frame") + .expect("reader did not deliver frame") }; let (name, pts, segment) = write(first, reference); assert!(name.starts_with(if audio_first { "audio" } else { "video" })); @@ -2156,7 +2073,7 @@ mod session_tests { .expect("broadcast closed") } - /// Poll for a pad rather than sleeping a fixed beat: the pumps run on another runtime, so + /// Poll for a pad rather than sleeping a fixed beat: the readers run on another runtime, so /// the only ordering we have is "eventually". Fails the test if it never shows up. fn await_pad(element: &super::super::MoqSrc, kind: &str) -> gst::Pad { for _ in 0..100 { @@ -2218,7 +2135,7 @@ mod session_tests { stopping.send(true).unwrap(); super::RUNTIME.block_on(following).unwrap().unwrap(); assert_eq!( - next.expect("the broadcast change did not replace its pump") + next.expect("the broadcast change did not replace its reader") .unwrap() .name(), "encoded" @@ -2304,12 +2221,11 @@ mod session_tests { super::RUNTIME.block_on(session).unwrap().unwrap(); } - /// A rendition delisted while its pump is still subscribing ends that pump, and answering - /// the subscription afterwards must not resurrect it into a pad. Which of the two the pump - /// sees first is the runtime's to decide, so the state machine that refuses the losing side - /// is covered by `a_pump_either_goes_live_or_is_cancelled` instead. + /// A rendition delisted while its reader is still subscribing ends that reader, and answering + /// the subscription afterwards must not resurrect it into a pad. Listing it again starts it + /// anew, since the reader that left is no longer the rendition's. #[test] - fn a_rendition_delisted_while_subscribing_takes_no_pad() { + fn a_rendition_delisted_while_subscribing_takes_no_pad_until_relisted() { let _pad_ids = pad_ids(); let element = element(); @@ -2327,7 +2243,7 @@ mod session_tests { let weak = element.downgrade(); let session = run(consumer, weak, shutdown_rx); - // Hold the subscription pending, then drop the rendition from the catalog so its pump + // Hold the subscription pending, then drop the rendition from the catalog so its reader // is cancelled while it is still waiting. let request = await_request(&mut dynamic); { @@ -2335,9 +2251,9 @@ mod session_tests { guard.video.renditions.clear(); } - // A cancelled pump returns out of its subscribe, dropping the only consumer this request + // A cancelled reader returns out of its subscribe, dropping the only consumer this request // has: that edge says the session reconciled the removal, which a fixed beat can only - // guess at. Answering before it lands is a pump that legitimately goes live, so the + // guess at. Answering before it lands is a reader that legitimately goes live, so the // wait is what the assertion below is about. let mut demand = request.demand(); let unwanted = async { @@ -2346,13 +2262,18 @@ mod session_tests { }; super::RUNTIME .block_on(async { tokio::time::timeout(Duration::from_secs(10), unwanted).await }) - .expect("the cancelled pump never dropped its subscription") + .expect("the cancelled reader never dropped its subscription") .expect("the pending request is still open"); - // Only now answer it. The pump is gone and its state is terminal, so no later scheduling - // can produce a pad. + // Only now answer it. The reader is gone, so no later scheduling can produce a pad. let _serving = request.accept(moq_net::track::Info::default()); - assert!(pads(&element, "video_").is_empty(), "a cancelled pump still took a pad"); + assert!( + pads(&element, "video_").is_empty(), + "a cancelled reader still took a pad" + ); + + catalog.modify().unwrap().video.renditions = BTreeMap::from([("stalled".to_string(), video_rendition())]); + await_pad(&element, "video_"); let _ = shutdown.send(true); super::RUNTIME.block_on(session).unwrap().unwrap(); @@ -2390,7 +2311,7 @@ mod session_tests { /// The flip side of the test above: a rendition the publisher reserved and never served is /// answerable once the publisher ends the broadcast, which resolves every name it never - /// filled. That pump must end on its own so the run ends on the media it was serving, and the + /// filled. That reader must end on its own so the run ends on the media it was serving, and the /// session can follow the path to its next start. The pad it served is held, not ended: the /// broadcast is offline, not retired. #[test] @@ -2448,7 +2369,7 @@ mod session_tests { /// A publisher that ends one rendition finishes its track and retires it from the catalog, /// which is what `moqsink` does on EOS. The two travel on different tracks, so the catalog - /// update can reach the subscriber before the tail of the media does. It must not cut the pump + /// update can reach the subscriber before the tail of the media does. It must not cut the reader /// short: the pad owes downstream every frame of the track and an EOS. /// /// The subscriber's view of that arrival order is reproduced exactly: the head of the track, @@ -2532,7 +2453,7 @@ mod session_tests { ); // The catalog update retiring the rendition arrives next. The catalog and the broadcast - // stay open, so nothing else can end the pump. The same update lists an audio rendition: + // stay open, so nothing else can end the reader. The same update lists an audio rendition: // its pad appearing proves the session acted on the update before the tail is written. { let mut guard = catalog.modify().unwrap(); @@ -2541,7 +2462,7 @@ mod session_tests { } await_pad(&element, "audio_"); - // A pump the update cancelled still has to notice. It takes its pad with it when it does, + // A reader the update cancelled still has to notice. It takes its pad with it when it does, // so the wait is only ever served in full by one that is still reading. for _ in 0..10 { if pads(&element, "video_").is_empty() { @@ -2556,14 +2477,14 @@ mod session_tests { } producer.finish().unwrap(); - // The pump removes its pad on the way out, whichever way it goes. + // The reader removes its pad on the way out, whichever way it goes. for _ in 0..100 { if pads(&element, "video_").is_empty() { break; } std::thread::sleep(Duration::from_millis(50)); } - assert!(pads(&element, "video_").is_empty(), "the pump never ended"); + assert!(pads(&element, "video_").is_empty(), "the reader never ended"); let _ = shutdown.send(true); super::RUNTIME.block_on(session).unwrap().unwrap(); @@ -2577,7 +2498,7 @@ mod session_tests { } /// A publisher that delists a rendition without finishing its track and lists it again later - /// (the browser toggling a source) is resuming the same track. The pump that is still reading + /// (the browser toggling a source) is resuming the same track. The reader that is still reading /// it carries on under the same pad, and a track that ends while listed keeps its pad without /// an EOS, since the announcements say whether the broadcast comes back. #[test] @@ -2729,7 +2650,7 @@ mod session_tests { .unwrap(); catalog.modify().unwrap().video.renditions = BTreeMap::from([("video".to_string(), video_rendition())]); - // Count what reaches the first video pad once it is added. A pump adds a new pad from the + // Count what reaches the first video pad once it is added. A reader adds a new pad from the // signal, before it pushes anything, so every buffer and every later stream is seen. let buffers = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); let restarts = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); @@ -2785,15 +2706,15 @@ mod session_tests { } producer.finish().unwrap(); - // The pump owns its pad until it has pushed the whole track and read its end. - let owner = pads.0.lock().unwrap()[&(TrackKind::Video, "video".to_string())] + // The reader owns its pad until it has pushed the whole track and read its end. + let owner = pads.0.lock().unwrap().slots[&(TrackKind::Video, "video".to_string())] .owner .clone(); drop(super::RUNTIME.block_on(owner.lock_owned())); assert_eq!(buffers.load(Ordering::Relaxed), FRAMES, "the track never played"); // The update reshapes the finished rendition over the same track, which hands its pad to a - // new pump, and lists a second rendition of that track. Its audio pad proves the session + // new reader, and lists a second rendition of that track. Its audio pad proves the session // acted on it. { let mut guard = catalog.modify().unwrap(); @@ -2811,7 +2732,7 @@ mod session_tests { "the update added {added:?}" ); - // The run ends only once every pump drains, a replaying one included. + // The run ends only once every reader drains, a replaying one included. audio.finish().unwrap(); catalog.finish().unwrap(); super::RUNTIME.block_on(session).unwrap().unwrap(); @@ -3075,7 +2996,7 @@ mod session_tests { assert!(is_closed(&connection)); } - /// Stop waits for every pump to remove its pad, so a pump held in a push has to be let go + /// Stop waits for every reader to remove its pad, so a reader held in a push has to be let go /// first, or stop never returns. #[test] fn stop_releases_a_blocked_push() { @@ -3101,8 +3022,8 @@ mod session_tests { assert!(is_closed(&connection)); } - /// A pump whose subscription resolves while stop flushes the pads adds its own just after, with - /// its cancel still on the way. A push on that pad must not block stop either. + /// A reader whose subscription resolves while stop flushes the pads adds its own just after, + /// with its cancel still on the way. A push on that pad must not block stop either. #[test] fn a_pad_added_after_stop_flushed_does_not_block() { let _pad_ids = pad_ids(); @@ -3115,41 +3036,51 @@ mod session_tests { }); publisher.write(0); + let rendition = (TrackKind::Video, "video".to_string()); let (_cancel, cancel) = watch::channel(false); let (_shutdown, shutdown) = watch::channel(true); - let pump = super::Pump { - element: element.downgrade(), - kind: super::TrackKind::Video, - name: "video".into(), - caps: gst::Caps::new_empty_simple("video/x-h264"), - follow: moq_mux::Source::new(origin().consume(), publisher.broadcast.consume()) - .follow(None, None) - .unwrap(), - sibling: false, - track: super::Track { + let (_stop, stop) = watch::channel(false); + let pads = Pads::default(); + assert!(pads.list(HashSet::from([rendition.clone()]), &cancel).is_empty()); + let instance = publisher.broadcast.consume(); + let reader = super::Reader { + playback: super::Playback { + element: element.downgrade(), + pads, + path: instance.info().path.clone(), + timeline: Arc::default(), + siblings: super::Siblings::default(), + finished: super::Finished::default(), + shutdown, + stop, + }, + rendition, + shape: super::Shape { + caps: gst::Caps::new_empty_simple("video/x-h264"), + container: Container::Legacy, + track: "video".into(), broadcast: None, + epoch: None, + }, + track: super::Track { + broadcast: instance.info().path.clone(), name: "video".into(), }, - container: moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - descriptor: Container::Legacy, - pads: Pads::default(), - timeline: std::sync::Arc::default(), - siblings: super::Siblings::default(), - finished: super::Finished::default(), - state: std::sync::Arc::new(super::PumpState::new()), + instance, + timeline: Arc::default(), + live: Arc::default(), cancel, - shutdown, }; super::RUNTIME - .block_on(async { tokio::time::timeout(Duration::from_secs(10), super::RUNTIME.spawn(pump.run())).await }) + .block_on(async { tokio::time::timeout(Duration::from_secs(10), super::RUNTIME.spawn(reader.run())).await }) .expect("a push on a pad added after the flush blocked") .unwrap(); } - /// A bus sync or pad handler can stop the element from inside a pump's push. Waiting for the - /// session there would wait on that pump, so stop settles for the connection closing. + /// A bus sync or pad handler can stop the element from inside a reader's push. Waiting for the + /// session there would wait on that reader, so stop settles for the connection closing. #[test] - fn stop_from_its_own_pump_does_not_deadlock() { + fn stop_from_its_own_reader_does_not_deadlock() { let _pad_ids = pad_ids(); let element = element(); let origin = origin(); @@ -3172,7 +3103,7 @@ mod session_tests { publisher.write(0); returned .recv_timeout(Duration::from_secs(10)) - .expect("stop from the pump never returned"); + .expect("stop from the reader never returned"); assert!(is_closed(&connection)); } @@ -3451,7 +3382,7 @@ mod session_tests { drop(first); } - /// Block until `publisher`'s video track reads to its end, as a pump reading it does. + /// Block until `publisher`'s video track reads to its end, as a reader reading it does. fn await_finished(publisher: &Publisher) { let broadcast = publisher.broadcast.consume(); super::RUNTIME.block_on(async { @@ -3475,7 +3406,7 @@ mod session_tests { } /// A sibling track that finished stays finished for its instance only: a catalog update that - /// replaces the idle pump, here a disable and an enable, still plays the next instance. + /// restarts the idle rendition, here a disable and an enable, still plays the next instance. #[test] fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { let _pad_ids = pad_ids(); @@ -3602,6 +3533,43 @@ mod session_tests { assert!(!seen.iter().any(|seen| matches!(seen, Seen::Buffer(_, 2))), "{seen:?}"); } + /// A rendition delisted while its sibling is offline and listed again follows the sibling's next + /// instance, even though the reader that left took its pad with it. + #[test] + fn a_relisted_sibling_rendition_follows_the_next_instance() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + let listed = room.catalog.modify().unwrap().video.renditions.clone(); + room.catalog.modify().unwrap().video.renditions.clear(); + for _ in 0..200 { + if recorder.seen().contains(&Seen::Eos(pad.clone())) { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + recorder.seen().contains(&Seen::Eos(pad.clone())), + "the idle pad never ended" + ); + room.catalog.modify().unwrap().video.renditions = listed; + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_ne!(recorder.written(&mut second, 2), pad, "the ended pad came back"); + stop(session); + } + /// A sibling replaced while the old one stays up restarts a reference that names no epoch, on /// the same pad, and nothing more from the old instance gets through. fn sibling_restart(route: impl Fn() -> moq_net::origin::Route) { @@ -3736,7 +3704,7 @@ mod session_tests { recorder.assert_one_pad_held(); } - /// Disabling a rendition stops its pump but holds its pad idle, without an EOS, so a pipeline + /// Disabling a rendition stops its reader but holds its pad idle, without an EOS, so a pipeline /// linked by name resumes on it once the rendition is enabled again. #[test] fn a_reenabled_rendition_resumes_on_the_same_pad() { @@ -3781,7 +3749,7 @@ mod session_tests { let seen = recorder.seen(); assert!( seen.contains(&Seen::Start(pad)), - "the disabled pump kept running: {seen:?}" + "the disabled reader kept running: {seen:?}" ); recorder.assert_one_pad_held(); } From e940e7ea3b7f27f32080d78c481c91297b6c0de1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 17:19:37 -0700 Subject: [PATCH 16/27] refactor(moq-gst): key followers by path and keep instance state in them A follower per (path, epoch) needed a run-wide registry to share an instance's timeline across followers, which kept every sibling path seen in the run and parked broadcast clones in it and in the finished records. One follower per path now follows every instance and checks each rendition's named epoch itself. It numbers its instances, gives the first the run's timeline and each later one its own, and records finished tracks by instance number, so it holds no broadcast it no longer reads and forgets a path once the catalog stops referencing it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 319 ++++++++++++++++++----------------- 1 file changed, 166 insertions(+), 153 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index d91d0780372..7aca8da79a4 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -435,14 +435,6 @@ struct Shape { /// A rendition by kind and moq track name: what a pad is kept by. type Rendition = (TrackKind, String); -/// The media a rendition plays: its resolved broadcast and track name, whatever the catalog calls -/// it or however it spells the reference. -#[derive(Clone, PartialEq, Eq, Hash)] -struct Track { - broadcast: moq_net::PathOwned, - name: String, -} - /// The session's pads by rendition, and the renditions the latest catalog lists. The pads outlive /// the readers and runs that stream to them, so a pipeline linked by name (`s.video_0 ! ...`) keeps /// flowing across a restart. @@ -786,14 +778,11 @@ async fn follow_catalog( let playback = Playback { element, pads: pads.clone(), - path: broadcast.info().path.clone(), timeline: Arc::default(), - siblings: Siblings::default(), - finished: Finished::default(), shutdown: shutdown.clone(), stop: stopped, }; - let mut followers: HashMap> = HashMap::new(); + let mut followers: HashMap> = HashMap::new(); // The followers, and the retirements of pads the catalog no longer lists. let mut tasks = tokio::task::JoinSet::new(); let mut catalog_closed = false; @@ -850,10 +839,6 @@ async fn follow_catalog( result } -/// The broadcast a [`Follower`] follows: the path a reference resolves to, and the instance it -/// names, if any. -type Followed = (moq_net::PathOwned, Option); - /// Bring the run in line with `catalog`: list its renditions on the pads, retiring the pads of /// those it dropped, and hand each referenced broadcast's follower the renditions that read it. /// @@ -864,7 +849,7 @@ fn reconcile( catalog: &moq_mux::catalog::hang::Catalog, source: &moq_mux::Source, playback: &Playback, - followers: &mut HashMap>, + followers: &mut HashMap>, tasks: &mut tokio::task::JoinSet<()>, cancel: &watch::Receiver, ) { @@ -920,7 +905,7 @@ fn reconcile( tasks.spawn_on(STREAMING.scope(playback.element.clone(), retire), RUNTIME.handle()); } - let mut plans: HashMap = HashMap::new(); + let mut plans: HashMap = HashMap::new(); for (rendition, shape) in desired { let Some(path) = source.resolve_reference(shape.broadcast.as_ref()) else { gst::warning!( @@ -930,19 +915,15 @@ fn reconcile( ); continue; }; - plans - .entry((path, shape.epoch.clone())) - .or_default() - .renditions - .insert(rendition, shape); + plans.entry(path).or_default().renditions.insert(rendition, shape); } // A broadcast no rendition reads any more is not followed: its readers stop, apart from a // delisted one playing out its tail. - followers.retain(|followed, _| plans.contains_key(followed)); - for (followed, plan) in plans { + followers.retain(|path, _| plans.contains_key(path)); + for (path, plan) in plans { // A follower whose broadcast failed exited, and this update follows it again. - if let Some(follower) = followers.get(&followed) + if let Some(follower) = followers.get(&path) && !follower.is_closed() { follower.send_replace(plan); @@ -950,22 +931,25 @@ fn reconcile( } // Follow without waiting: an unavailable sibling must not park the catalog loop and hold - // up every other rendition. The follower resolves its instances under the run. + // up every other rendition. The follower resolves its instances under the run. Every + // instance is followed, and each rendition checks the epoch it names. let shape = plan.renditions.values().next().expect("a plan names a rendition"); - let instances = match source.follow(shape.broadcast.as_ref(), followed.1.as_ref()) { + let instances = match source.follow(shape.broadcast.as_ref(), None) { Ok(instances) => instances, Err(err) => { - gst::warning!(CAT, "ignoring the renditions of {}: {err:?}", followed.0); + gst::warning!(CAT, "ignoring the renditions of {path}: {err:?}"); continue; } }; let (follower, plan) = watch::channel(plan); let follower_task = Follower { playback: playback.clone(), - path: followed.0.clone(), + path: path.clone(), renditions: HashMap::new(), readers: tokio::task::JoinSet::new(), current: None, + finished: Finished::default(), + started: 0, next_id: 0, } .run(instances, plan); @@ -973,22 +957,18 @@ fn reconcile( STREAMING.scope(playback.element.clone(), follower_task), RUNTIME.handle(), ); - followers.insert(followed, follower); + followers.insert(path, follower); } } /// What one broadcast run shares with every follower and reader it starts. Pads survive a -/// broadcast switch; its timing and finished tracks do not. +/// broadcast switch; its timing does not. #[derive(Clone)] struct Playback { element: glib::WeakRef, pads: Pads, - /// The catalog's own path, whose renditions read the run's broadcast rather than following - /// announcements. - path: moq_net::PathOwned, + /// Shared by the catalog's own renditions and the first instance of each sibling. timeline: Arc, - siblings: Siblings, - finished: Finished, /// The session's shutdown, which [`SessionController::stop`] sets before it flushes the pads. shutdown: watch::Receiver, /// Set once the run ends, stopping every follower. @@ -1004,23 +984,36 @@ struct Plan { closed: bool, } -/// Follows one broadcast's instances and plays the renditions that read it on each in turn: a +/// Follows the instances at one path and plays the renditions that read it on each in turn: a /// start reads them on that instance, an end idles their pads, and another instance taking over /// restarts each stream on the same pad. The catalog's own broadcast is a single instance that /// never ends. /// -/// It owns its renditions' readers, so whether a rendition is reading, idle, or playing out its -/// tail is its own state rather than a task's. +/// It owns its renditions' readers and the state of the instance they read, so whether a +/// rendition is reading, idle, or playing out its tail is its own state rather than a task's. struct Follower { playback: Playback, path: moq_net::PathOwned, renditions: HashMap, readers: tokio::task::JoinSet<(Rendition, u64, Exit)>, - /// The instance being read, and the timeline its renditions share. - current: Option<(moq_net::broadcast::Consumer, Arc)>, + current: Option, + /// Shared with the readers. + finished: Finished, + /// How many instances started, numbering each. + started: u64, next_id: u64, } +/// The instance a [`Follower`] reads. +struct Current { + instance: moq_net::broadcast::Consumer, + /// Shared by every rendition reading the instance, so their first frames fix one mapping for + /// all of them. + timeline: Arc, + /// Its number among the follower's instances. + serial: u64, +} + /// A rendition a [`Follower`] plays. struct Entry { shape: Shape, @@ -1147,15 +1140,25 @@ impl Follower { }, ); } + self.finished + .retain(|rendition, _| self.renditions.contains_key(rendition)); } /// Read `instance`, which took over from any before it. fn start(&mut self, instance: moq_net::broadcast::Consumer) { - let timeline = match self.path == self.playback.path { - true => self.playback.timeline.clone(), - false => self.playback.siblings.timeline(&instance, &self.playback.timeline), + self.started += 1; + // The first instance is the one the run started alongside. A later one's timestamps share + // no origin with the run's. + let timeline = match self.started { + 1 => self.playback.timeline.clone(), + _ => Arc::default(), }; - self.current = Some((instance, timeline)); + self.current = Some(Current { + instance, + timeline, + serial: self.started, + }); + self.finished.retain(|_, serial| serial == self.started); self.idle(); self.play(); } @@ -1168,13 +1171,13 @@ impl Follower { } } - /// Start a reader on the current instance for each rendition without one. + /// Start a reader on the current instance for each rendition without one that it may serve. fn play(&mut self) { - let Some((instance, timeline)) = &self.current else { + let Some(current) = &self.current else { return; }; for (rendition, entry) in &mut self.renditions { - if entry.reader.is_none() { + if entry.reader.is_none() && serves(¤t.instance, entry.shape.epoch.as_ref()) { self.next_id += 1; let (cancel, cancelled) = watch::channel(false); let live = Arc::new(AtomicBool::new(false)); @@ -1182,12 +1185,10 @@ impl Follower { playback: self.playback.clone(), rendition: rendition.clone(), shape: entry.shape.clone(), - track: Track { - broadcast: self.path.clone(), - name: entry.shape.track.clone(), - }, - instance: instance.clone(), - timeline: timeline.clone(), + instance: current.instance.clone(), + serial: current.serial, + timeline: current.timeline.clone(), + finished: self.finished.clone(), live: live.clone(), cancel: cancelled, }; @@ -1224,6 +1225,15 @@ impl Follower { } } +/// Whether `instance` may serve a rendition naming `epoch`. An epochless route cannot prove its +/// instance, so only a known mismatch refuses it. +fn serves(instance: &moq_net::broadcast::Consumer, epoch: Option<&moq_net::Epoch>) -> bool { + match (epoch, &instance.info().epoch) { + (Some(expected), Some(actual)) => expected == actual, + _ => true, + } +} + /// Why a [`Reader`] exited. enum Exit { /// It is done with the instance. @@ -1237,9 +1247,12 @@ struct Reader { playback: Playback, rendition: Rendition, shape: Shape, - track: Track, instance: moq_net::broadcast::Consumer, + /// The instance's [`Current::serial`]. + serial: u64, timeline: Arc, + /// The follower's [`Follower::finished`]. + finished: Finished, /// Shared with this reader's [`ReaderHandle::live`]. live: Arc, cancel: watch::Receiver, @@ -1254,25 +1267,25 @@ impl Reader { playback, rendition, shape, - track: media, instance, + serial, timeline, + finished, live, mut cancel, } = self; let Playback { element, pads, - finished, shutdown, .. } = playback; let (kind, name) = (rendition.0, &rendition.1); - let played = (rendition.clone(), media.clone()); + let played = (rendition.clone(), shape.track.clone()); // An instance that already played this track to its end on the pad would replay its last // group. - if finished.contains(&played, &instance) { + if finished.contains(&played, serial) { return Exit::Done; } @@ -1284,7 +1297,7 @@ impl Reader { _ = cancel.changed() => return Exit::Done, subscriber = async { instance - .track(&media.name)? + .track(&shape.track)? .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(1))) .await } => match subscriber { @@ -1341,7 +1354,7 @@ impl Reader { // The reader this one replaces records its track as finished before letting go, so // one started while it was reading the end (a reshape) can't replay it. A // retirement that got the pad first ended it. - if !pads.holds(&rendition, &owner) || finished.contains(&played, &instance) { + if !pads.holds(&rendition, &owner) || finished.contains(&played, serial) { return Exit::Done; } tokio::task::block_in_place(|| begin(&pad, name, &shape.caps)); @@ -1364,7 +1377,7 @@ impl Reader { // finishes the track, or a reader waiting for this pad would replay it. _ = cancel.changed() => { if ended(&mut track) { - finished.insert(&played, &instance); + finished.insert(&played, serial); } return Exit::Done; } @@ -1412,7 +1425,7 @@ impl Reader { // Recorded while this reader still owns the pad, before any other can claim it, // so a reshape on this instance does not replay it. Ok(None) => { - finished.insert(&played, &instance); + finished.insert(&played, serial); return Exit::Done; } Err(err) => { @@ -1425,68 +1438,35 @@ impl Reader { } } -/// The timeline of each sibling instance, shared by every rendition reading that instance so -/// their first frames fix one mapping for all of them. +/// Each of a follower's renditions with a track it played to the end, and the serial of the +/// instance that served it. A new subscription to a finished track replays its last group, so a +/// catalog update that still lists the rendition (a publisher's final snapshot lands after its +/// tracks end) must not start it on that pad again. A later instance carries a new track, and a +/// pad that never played the track joins it like any late subscriber. Shared with the readers, +/// since a reader that replaces another on its pad learns this only once the other lets go. #[derive(Clone, Default)] -struct Siblings(Arc>); - -#[derive(Default)] -struct SiblingTimelines { - /// Each sibling path some instance served this run. - seen: HashSet, - /// Each instance still serving, and the timeline its renditions share while any reads it. - live: Vec<(moq_net::broadcast::Consumer, std::sync::Weak)>, -} +struct Finished(Arc>>); -impl Siblings { - /// The timeline of `instance`: the run's for the first instance at its path, which the - /// catalog's own renditions share, and one of its own for any later instance, whose - /// timestamps share no origin with the run's. - fn timeline(&self, instance: &moq_net::broadcast::Consumer, run: &Arc) -> Arc { - let mut siblings = self.0.lock().unwrap(); - // Forget an instance that ended or that no rendition reads any more, releasing its handle. - siblings - .live - .retain(|(known, timeline)| !known.is_closed() && timeline.strong_count() > 0); - if let Some(timeline) = siblings - .live - .iter() - .find(|(known, _)| known.is_clone(instance)) - .and_then(|(_, timeline)| timeline.upgrade()) - { - return timeline; - } - let timeline = match siblings.seen.insert(instance.info().path.clone()) { - true => run.clone(), - false => Arc::default(), - }; - siblings.live.push((instance.clone(), Arc::downgrade(&timeline))); - timeline +impl Finished { + /// Record that instance `serial` served `played` to its end. A reader of an earlier instance + /// never overwrites a later one's record. + fn insert(&self, played: &(Rendition, String), serial: u64) { + let mut finished = self.0.lock().unwrap(); + let recorded = finished.entry(played.clone()).or_insert(serial); + *recorded = (*recorded).max(serial); } -} -/// Each pad's rendition with a track it played to the end, and the instance that served it. A new -/// subscription to a finished track replays its last group, so a catalog update that still lists -/// the rendition (a publisher's final snapshot lands after its tracks end) must not start it on -/// that pad again. A later instance carries a new track, and a pad that never played the track -/// joins it like any late subscriber. Shared with the readers, since a reader that replaces -/// another on its pad learns this only once the other lets go. -#[derive(Clone, Default)] -struct Finished(Arc>>); - -impl Finished { - /// Record that `instance` served `played` to its end, replacing any earlier instance's record. - fn insert(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) { - self.0.lock().unwrap().insert(played.clone(), instance.clone()); + /// Whether instance `serial` already served `played` to its end on this pad. + fn contains(&self, played: &(Rendition, String), serial: u64) -> bool { + self.0.lock().unwrap().get(played) == Some(&serial) } - /// Whether `instance` already served `played` to its end on this pad. - fn contains(&self, played: &(Rendition, Track), instance: &moq_net::broadcast::Consumer) -> bool { + /// Keep only the records `keep` accepts. + fn retain(&self, keep: impl Fn(&Rendition, u64) -> bool) { self.0 .lock() .unwrap() - .get(played) - .is_some_and(|finished| finished.is_clone(instance)) + .retain(|(rendition, _), serial| keep(rendition, *serial)); } } @@ -1688,9 +1668,8 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{Siblings, Timeline, ended}; + use super::{Timeline, ended}; use moq_net::Timestamp; - use std::sync::Arc; /// A reader cancelled as its track ends records the end only when no media is left to read, /// passing over the markers a publisher closes with. @@ -1775,33 +1754,6 @@ mod tests { } } - /// Renditions on the same sibling instance share its timeline, whenever each starts on it: - /// the first instance shares the run's, and the next one starts another. - #[tokio::test(start_paused = true)] - async fn renditions_on_one_sibling_instance_share_its_timeline() { - let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); - tokio::spawn(moq_net::time::run(driver)); - let consumer = origin.consume(); - let mut audio = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let started = async |follow: &mut moq_mux::follow::Instances| match follow.next().await.unwrap() { - moq_mux::follow::Event::Start(broadcast) => broadcast, - moq_mux::follow::Event::End => panic!("expected a start"), - }; - let (siblings, run) = (Siblings::default(), Arc::::default()); - - let _first = origin.publish("source", Default::default()).unwrap(); - let first = started(&mut audio).await; - assert!(Arc::ptr_eq(&run, &siblings.timeline(&first, &run))); - - // A video rendition that first starts on the replacement, as one still subscribing when it - // took over does, shares its timeline with the audio rendition restarting onto it. - let _second = origin.publish("source", Default::default()).unwrap(); - let mut video = moq_mux::follow::Instances::new(&consumer, "source", None).unwrap(); - let next = siblings.timeline(&started(&mut video).await, &run); - assert!(!Arc::ptr_eq(&run, &next), "another instance has its own timeline"); - assert!(Arc::ptr_eq(&next, &siblings.timeline(&started(&mut audio).await, &run))); - } - #[test] fn the_first_frame_fixes_the_segment_start_and_base() { gst::init().unwrap(); @@ -3047,10 +2999,7 @@ mod session_tests { playback: super::Playback { element: element.downgrade(), pads, - path: instance.info().path.clone(), timeline: Arc::default(), - siblings: super::Siblings::default(), - finished: super::Finished::default(), shutdown, stop, }, @@ -3062,12 +3011,10 @@ mod session_tests { broadcast: None, epoch: None, }, - track: super::Track { - broadcast: instance.info().path.clone(), - name: "video".into(), - }, instance, + serial: 1, timeline: Arc::default(), + finished: super::Finished::default(), live: Arc::default(), cancel, }; @@ -3118,6 +3065,8 @@ mod session_tests { /// A buffer, by its tag. Buffer(String, u8), Eos(String), + /// A segment, by its start. + Segment(String, i64), } /// Records what reaches the element's video pads. A new pad's first stream-start and caps @@ -3151,6 +3100,9 @@ mod session_tests { seen.push(Seen::Caps(name.clone(), media)); } gst::EventView::Eos(_) => seen.push(Seen::Eos(name.clone())), + gst::EventView::Segment(segment) => { + seen.push(Seen::Segment(name.clone(), segment.segment().start().value())) + } _ => {} }, _ => {} @@ -3355,6 +3307,67 @@ mod session_tests { recorder.assert_one_pad_held(); } + /// A rendition that joins a sibling instance after another started on it shares that instance's + /// timeline, so both map the instance's timestamps to one running time. + #[test] + fn a_rendition_joining_a_sibling_instance_shares_its_timeline() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let session = follow(&element, &origin); + + // A replacement starts a timeline of its own, which its first frame on `first` fixes. + let mut old = Publisher::sibling(&origin, Default::default()); + let first = recorder.written(&mut old, 1); + let mut new = Publisher::sibling(&origin, Default::default()); + assert_eq!(recorder.written(&mut new, 2), first); + // Frames past the subscription's one second of delay, so the joining rendition starts on a + // later frame than `first` did. + for _ in 0..30 { + new.write(3); + std::thread::sleep(Duration::from_millis(50)); + } + + let mut again = room.catalog.modify().unwrap().video.renditions["video"].clone(); + again.track = Some("video".into()); + room.catalog + .modify() + .unwrap() + .video + .renditions + .insert("again".into(), again); + let mut second = None; + for _ in 0..200 { + new.write(4); + std::thread::sleep(Duration::from_millis(50)); + second = recorder.seen().into_iter().find_map(|seen| match seen { + Seen::Buffer(pad, 4) if pad != first => Some(pad), + _ => None, + }); + if second.is_some() { + break; + } + } + let second = second.expect("the joining rendition never played"); + stop(session); + + let segment = |pad: &String| { + recorder.seen().into_iter().rev().find_map(|seen| match seen { + Seen::Segment(segmented, start) if segmented == *pad => Some(start), + _ => None, + }) + }; + assert_eq!( + segment(&second), + segment(&first), + "the joining rendition mapped the instance on its own: {:?}", + recorder.seen() + ); + drop(old); + } + /// A sibling track finishing while its instance stays up idles the pad, and the instance that /// replaces it starts a new stream there. #[test] From 165a66c41b669f19a10e853caf5f45fa06fd1be2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 17:58:06 -0700 Subject: [PATCH 17/27] test(moq-gst): make the joining-rendition timeline test deterministic Jump the sibling's timestamps past the subscription's delay instead of sleeping past it, so the groups before the jump go stale on media time. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7aca8da79a4..7feb396b315 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -2886,6 +2886,11 @@ mod session_tests { }) .unwrap(); } + + /// Move the timestamps of later frames `by` ahead. + fn skip(&mut self, by: Duration) { + self.start -= by; + } } // A dial still running once the element reached NULL can outlive `main`, and aws-lc aborts the @@ -3323,12 +3328,11 @@ mod session_tests { let first = recorder.written(&mut old, 1); let mut new = Publisher::sibling(&origin, Default::default()); assert_eq!(recorder.written(&mut new, 2), first); - // Frames past the subscription's one second of delay, so the joining rendition starts on a - // later frame than `first` did. - for _ in 0..30 { - new.write(3); - std::thread::sleep(Duration::from_millis(50)); - } + // Media past the subscription's one second of delay makes the groups before it stale, so + // the joining rendition starts on a later frame than `first` did. + recorder.written(&mut new, 3); + new.skip(Duration::from_secs(2)); + recorder.written(&mut new, 4); let mut again = room.catalog.modify().unwrap().video.renditions["video"].clone(); again.track = Some("video".into()); @@ -3340,10 +3344,10 @@ mod session_tests { .insert("again".into(), again); let mut second = None; for _ in 0..200 { - new.write(4); + new.write(5); std::thread::sleep(Duration::from_millis(50)); second = recorder.seen().into_iter().find_map(|seen| match seen { - Seen::Buffer(pad, 4) if pad != first => Some(pad), + Seen::Buffer(pad, 5) if pad != first => Some(pad), _ => None, }); if second.is_some() { From 0f79bfebb65c5443c1da013b52ec5678133d5fa4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 18:12:26 -0700 Subject: [PATCH 18/27] fix(moq-gst): keep a finished sibling track finished across a re-enable Disabling a sibling's only rendition dropped its follower, and with it the record that the rendition's track finished on the current instance, so a re-enable replayed the track's last group. Pruning records of renditions a follower no longer plays did the same while another rendition kept it. A disabled rendition now keeps its path's follower, and a follower keeps one record per listed rendition until the next instance starts. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 138 +++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 46 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7feb396b315..0042d95291d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -885,27 +885,49 @@ fn reconcile( // A disabled rendition stays listed: its reader stops, but its pad waits idle for a re-enable, // since a pipeline links pads by name (`s.video_0 ! ...`). - let disabled = catalog + let disabled: Vec<(Rendition, &dyn hang::catalog::Reference)> = catalog .video .renditions .iter() .filter(|(_, config)| !config.enabled) - .map(|(name, _)| (TrackKind::Video, name.clone())) + .map(|(name, config)| { + ( + (TrackKind::Video, name.clone()), + config as &dyn hang::catalog::Reference, + ) + }) .chain( catalog .audio .renditions .iter() .filter(|(_, config)| !config.enabled) - .map(|(name, _)| (TrackKind::Audio, name.clone())), - ); + .map(|(name, config)| { + ( + (TrackKind::Audio, name.clone()), + config as &dyn hang::catalog::Reference, + ) + }), + ) + .collect(); // Listed before any follower hears of the update, so a reader it stops can't claim a pad. - let listed = desired.keys().cloned().chain(disabled).collect(); + let listed = desired + .keys() + .cloned() + .chain(disabled.iter().map(|(rendition, _)| rendition.clone())) + .collect(); for retire in playback.pads.list(listed, cancel) { tasks.spawn_on(STREAMING.scope(playback.element.clone(), retire), RUNTIME.handle()); } - let mut plans: HashMap = HashMap::new(); + // A disabled rendition keeps its broadcast's follower, and with it the record of a track it + // finished, so a re-enable on the same instance does not replay it. + let mut plans: HashMap = disabled + .iter() + .filter_map(|(_, reference)| source.resolve_reference(reference.broadcast())) + .filter(|path| followers.contains_key(path)) + .map(|path| (path, Plan::default())) + .collect(); for (rendition, shape) in desired { let Some(path) = source.resolve_reference(shape.broadcast.as_ref()) else { gst::warning!( @@ -933,7 +955,10 @@ fn reconcile( // Follow without waiting: an unavailable sibling must not park the catalog loop and hold // up every other rendition. The follower resolves its instances under the run. Every // instance is followed, and each rendition checks the epoch it names. - let shape = plan.renditions.values().next().expect("a plan names a rendition"); + let Some(shape) = plan.renditions.values().next() else { + // Only disabled renditions read it, and their follower is gone. + continue; + }; let instances = match source.follow(shape.broadcast.as_ref(), None) { Ok(instances) => instances, Err(err) => { @@ -1140,8 +1165,8 @@ impl Follower { }, ); } - self.finished - .retain(|rendition, _| self.renditions.contains_key(rendition)); + // A disabled rendition keeps its pad, and the record with it. A retired pad needs none. + self.finished.retain(|rendition, _| pads.is_listed(rendition)); } /// Read `instance`, which took over from any before it. @@ -1281,11 +1306,10 @@ impl Reader { .. } = playback; let (kind, name) = (rendition.0, &rendition.1); - let played = (rendition.clone(), shape.track.clone()); // An instance that already played this track to its end on the pad would replay its last // group. - if finished.contains(&played, serial) { + if finished.contains(&rendition, &shape.track, serial) { return Exit::Done; } @@ -1354,7 +1378,7 @@ impl Reader { // The reader this one replaces records its track as finished before letting go, so // one started while it was reading the end (a reshape) can't replay it. A // retirement that got the pad first ended it. - if !pads.holds(&rendition, &owner) || finished.contains(&played, serial) { + if !pads.holds(&rendition, &owner) || finished.contains(&rendition, &shape.track, serial) { return Exit::Done; } tokio::task::block_in_place(|| begin(&pad, name, &shape.caps)); @@ -1377,7 +1401,7 @@ impl Reader { // finishes the track, or a reader waiting for this pad would replay it. _ = cancel.changed() => { if ended(&mut track) { - finished.insert(&played, serial); + finished.insert(&rendition, &shape.track, serial); } return Exit::Done; } @@ -1425,7 +1449,7 @@ impl Reader { // Recorded while this reader still owns the pad, before any other can claim it, // so a reshape on this instance does not replay it. Ok(None) => { - finished.insert(&played, serial); + finished.insert(&rendition, &shape.track, serial); return Exit::Done; } Err(err) => { @@ -1438,27 +1462,33 @@ impl Reader { } } -/// Each of a follower's renditions with a track it played to the end, and the serial of the -/// instance that served it. A new subscription to a finished track replays its last group, so a -/// catalog update that still lists the rendition (a publisher's final snapshot lands after its -/// tracks end) must not start it on that pad again. A later instance carries a new track, and a -/// pad that never played the track joins it like any late subscriber. Shared with the readers, -/// since a reader that replaces another on its pad learns this only once the other lets go. +/// The track each of a follower's renditions last played to the end on its pad, and the serial of +/// the instance that served it. A new subscription to a finished track replays its last group, so +/// a catalog update that still lists the rendition (a publisher's final snapshot lands after its +/// tracks end, or a disable and an enable) must not start it on that pad again. A later instance +/// carries a new track, and a pad that never played the track joins it like any late subscriber. +/// Shared with the readers, since a reader that replaces another on its pad learns this only once +/// the other lets go. #[derive(Clone, Default)] -struct Finished(Arc>>); +struct Finished(Arc>>); impl Finished { - /// Record that instance `serial` served `played` to its end. A reader of an earlier instance - /// never overwrites a later one's record. - fn insert(&self, played: &(Rendition, String), serial: u64) { + /// Record that instance `serial` served the rendition's `track` to its end. A reader of an + /// earlier instance never overwrites a later one's record. + fn insert(&self, rendition: &Rendition, track: &str, serial: u64) { let mut finished = self.0.lock().unwrap(); - let recorded = finished.entry(played.clone()).or_insert(serial); - *recorded = (*recorded).max(serial); + if finished.get(rendition).is_none_or(|(_, recorded)| *recorded <= serial) { + finished.insert(rendition.clone(), (track.to_string(), serial)); + } } - /// Whether instance `serial` already served `played` to its end on this pad. - fn contains(&self, played: &(Rendition, String), serial: u64) -> bool { - self.0.lock().unwrap().get(played) == Some(&serial) + /// Whether instance `serial` already served the rendition's `track` to its end on its pad. + fn contains(&self, rendition: &Rendition, track: &str, serial: u64) -> bool { + self.0 + .lock() + .unwrap() + .get(rendition) + .is_some_and(|(finished, recorded)| finished == track && *recorded == serial) } /// Keep only the records `keep` accepts. @@ -1466,7 +1496,7 @@ impl Finished { self.0 .lock() .unwrap() - .retain(|(rendition, _), serial| keep(rendition, *serial)); + .retain(|rendition, (_, serial)| keep(rendition, *serial)); } } @@ -3422,8 +3452,8 @@ mod session_tests { }); } - /// A sibling track that finished stays finished for its instance only: a catalog update that - /// restarts the idle rendition, here a disable and an enable, still plays the next instance. + /// A sibling track that finished stays finished for its instance only: a disable and an enable + /// replays nothing on that instance, and the next instance still plays. #[test] fn a_reenabled_sibling_rendition_plays_the_next_instance_after_a_finish() { let _pad_ids = pad_ids(); @@ -3431,10 +3461,11 @@ mod session_tests { let recorder = Recorder::attach(&element); let origin = origin(); let mut room = referencing(&origin, None); - let _audio = room - .broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); + let _audio = ["audio", "again"].map(|name| { + room.broadcast + .create_track(name, hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap() + }); let session = follow(&element, &origin); let mut first = Publisher::sibling(&origin, Default::default()); @@ -3442,22 +3473,29 @@ mod session_tests { first.video.finish().unwrap(); await_finished(&first); - // The catalog consumer only yields the newest snapshot, so the same update lists an audio - // rendition: its pad appearing proves the run acted on the disable. + // The catalog consumer only yields the newest snapshot, so each update lists another audio + // rendition: its pad appearing proves the run acted on it. { let mut guard = room.catalog.modify().unwrap(); guard.video.renditions.get_mut("video").unwrap().enabled = false; guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); } await_pad(&element, "audio_"); - room.catalog - .modify() - .unwrap() - .video - .renditions - .get_mut("video") - .unwrap() - .enabled = true; + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().enabled = true; + let mut again = audio_rendition(); + again.track = Some("again".into()); + guard.audio.renditions.insert("again".to_string(), again); + } + for _ in 0..200 { + if pads(&element, "audio_").len() == 2 { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert_eq!(pads(&element, "audio_").len(), 2, "the re-enable never landed"); + let replayed = recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)); let mut second = Publisher::sibling(&origin, Default::default()); assert_eq!( @@ -3467,6 +3505,14 @@ mod session_tests { ); stop(session); + // Only the next instance starts a stream: the finished one stays finished on its pad. + let seen = recorder.seen(); + assert_eq!(recorder.count(|seen| *seen == Seen::Start(pad.clone())), 1, "{seen:?}"); + assert_eq!( + recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)), + replayed, + "{seen:?}" + ); recorder.assert_one_pad_held(); drop(first); } From 5c2444fc4a290ba18136473621892e2f2e8d4b32 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 18:57:48 -0700 Subject: [PATCH 19/27] fix(moq-gst): follow siblings past a closed catalog until its broadcast ends A publisher may finish its catalog and keep serving. A closed plan stopped every follower from polling its instances, so a sibling's next instance never played after the catalog's final snapshot. A sibling's follower now keeps following its instances once the catalog closes, until the catalog's broadcast ends. The catalog's own follower, whose single instance has no successor, still ends once its readers drain. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 70 ++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 0042d95291d..d48f087ef6b 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -777,6 +777,7 @@ async fn follow_catalog( let (stop, stopped) = watch::channel(false); let playback = Playback { element, + broadcast: broadcast.clone(), pads: pads.clone(), timeline: Arc::default(), shutdown: shutdown.clone(), @@ -991,6 +992,8 @@ fn reconcile( #[derive(Clone)] struct Playback { element: glib::WeakRef, + /// The catalog's broadcast. A sibling outlives its closed catalog until it ends. + broadcast: moq_net::broadcast::Consumer, pads: Pads, /// Shared by the catalog's own renditions and the first instance of each sibling. timeline: Arc, @@ -1004,8 +1007,9 @@ struct Playback { #[derive(Clone, Default)] struct Plan { renditions: HashMap, - /// The catalog closed: no rendition starts again, and the follower ends once its readers - /// drain. + /// The catalog closed: the renditions no longer change. The catalog's own follower ends once + /// its readers drain, and a sibling's keeps following its instances until the catalog's + /// broadcast ends. closed: bool, } @@ -1077,8 +1081,13 @@ impl Follower { } _ => {} } + let catalog = self.playback.broadcast.clone(); + let sibling = self.path != catalog.info().path; let first = plan.borrow_and_update().clone(); - let mut following = !first.closed; + // Taking plan updates, until the catalog closes or stops referencing this path. + let mut listening = !first.closed; + // Following instances, which a sibling does past its catalog's close. + let mut following = listening || sibling; self.apply(first.renditions); self.play(); @@ -1090,18 +1099,22 @@ impl Follower { tokio::select! { biased; _ = stop.changed() => break, - changed = plan.changed(), if following => { - // A dropped plan reads no rendition any more. - let next = match changed { - Ok(()) => plan.borrow_and_update().clone(), - Err(_) => Plan { closed: true, ..Default::default() }, - }; - following = !next.closed; - self.apply(next.renditions); - if following { - self.play(); + changed = plan.changed(), if listening => match changed { + Ok(()) => { + let next = plan.borrow_and_update().clone(); + listening = !next.closed; + following = listening || sibling; + self.apply(next.renditions); + if listening { + self.play(); + } } - } + // No rendition reads this path any more. + Err(_) => { + (listening, following) = (false, false); + self.apply(HashMap::new()); + } + }, Some(joined) = self.readers.join_next() => { if let Ok((rendition, id, exit)) = joined { self.exited(rendition, id, exit); @@ -1122,6 +1135,9 @@ impl Follower { break; } }, + // The catalog's broadcast ended after its catalog, and nothing reads this sibling + // any more. + _ = catalog.closed(), if following && !listening => following = false, } } @@ -3033,6 +3049,7 @@ mod session_tests { let reader = super::Reader { playback: super::Playback { element: element.downgrade(), + broadcast: instance.clone(), pads, timeline: Arc::default(), shutdown, @@ -3402,6 +3419,31 @@ mod session_tests { drop(old); } + /// A catalog finished while its broadcast stays up still follows its siblings: the next + /// instance plays on the same pad. + #[test] + fn a_sibling_is_followed_past_its_catalogs_close() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + room.catalog.finish().unwrap(); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); + stop(session); + recorder.assert_one_pad_held(); + } + /// A sibling track finishing while its instance stays up idles the pad, and the instance that /// replaces it starts a new stream there. #[test] From fbb897c1da94132c391c70d731f449144959423a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 19:05:57 -0700 Subject: [PATCH 20/27] fix(moq-gst): idle sibling renditions when the catalog's broadcast ends Once the catalog closed, the end of its broadcast only stopped a sibling's follower from taking later instances, and the reader on the current one kept streaming while the run was offline. That end now idles the sibling's renditions, holding their pads, as an end of the sibling itself does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index d48f087ef6b..bd155e9ac04 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1135,9 +1135,13 @@ impl Follower { break; } }, - // The catalog's broadcast ended after its catalog, and nothing reads this sibling - // any more. - _ = catalog.closed(), if following && !listening => following = false, + // The catalog's broadcast ended after its catalog: the run is offline, so its + // sibling renditions idle with it. + _ = catalog.closed(), if following && !listening => { + following = false; + self.current = None; + self.idle(); + } } } @@ -3420,7 +3424,7 @@ mod session_tests { } /// A catalog finished while its broadcast stays up still follows its siblings: the next - /// instance plays on the same pad. + /// instance plays on the same pad, until the catalog's broadcast ends. #[test] fn a_sibling_is_followed_past_its_catalogs_close() { let _pad_ids = pad_ids(); @@ -3440,6 +3444,18 @@ mod session_tests { let mut second = Publisher::sibling(&origin, Default::default()); assert_eq!(recorder.written(&mut second, 2), pad, "the return moved to another pad"); + + // The catalog's broadcast ending lets go of the sibling it read. + let mut demand = second.video.demand(); + drop(room); + let unwanted = async { + while demand.wanted_changed().await?.is_some() {} + moq_net::Result::Ok(()) + }; + super::RUNTIME + .block_on(async { tokio::time::timeout(Duration::from_secs(10), unwanted).await }) + .expect("the sibling outlived its catalog's broadcast") + .expect("the sibling track closed"); stop(session); recorder.assert_one_pad_held(); } From 2cb82306954fb835dd9c5045733fb0e42e11b925 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 19:15:59 -0700 Subject: [PATCH 21/27] fix(moq-gst): keep a draining follower reachable across a relist Delisting a path's last live rendition dropped its follower while that rendition drained, so a relist spawned a second follower for the path. Its reader took the pad once the first let go and, without the first's finished records, replayed the track's last group. The run now keeps a follower reachable while a disabled rendition names its path or a delisted one drains, and lets it go on the first update after it plays nothing. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 126 ++++++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 23 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index bd155e9ac04..46862b3fe4e 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -783,7 +783,7 @@ async fn follow_catalog( shutdown: shutdown.clone(), stop: stopped, }; - let mut followers: HashMap> = HashMap::new(); + let mut followers: HashMap = HashMap::new(); // The followers, and the retirements of pads the catalog no longer lists. let mut tasks = tokio::task::JoinSet::new(); let mut catalog_closed = false; @@ -826,8 +826,8 @@ async fn follow_catalog( } if catalog_closed { - for (_, plan) in followers.drain() { - plan.send_modify(|plan| plan.closed = true); + for (_, follower) in followers.drain() { + follower.plan.send_modify(|plan| plan.closed = true); } } } @@ -850,7 +850,7 @@ fn reconcile( catalog: &moq_mux::catalog::hang::Catalog, source: &moq_mux::Source, playback: &Playback, - followers: &mut HashMap>, + followers: &mut HashMap, tasks: &mut tokio::task::JoinSet<()>, cancel: &watch::Receiver, ) { @@ -921,14 +921,7 @@ fn reconcile( tasks.spawn_on(STREAMING.scope(playback.element.clone(), retire), RUNTIME.handle()); } - // A disabled rendition keeps its broadcast's follower, and with it the record of a track it - // finished, so a re-enable on the same instance does not replay it. - let mut plans: HashMap = disabled - .iter() - .filter_map(|(_, reference)| source.resolve_reference(reference.broadcast())) - .filter(|path| followers.contains_key(path)) - .map(|path| (path, Plan::default())) - .collect(); + let mut plans: HashMap = HashMap::new(); for (rendition, shape) in desired { let Some(path) = source.resolve_reference(shape.broadcast.as_ref()) else { gst::warning!( @@ -941,25 +934,38 @@ fn reconcile( plans.entry(path).or_default().renditions.insert(rendition, shape); } - // A broadcast no rendition reads any more is not followed: its readers stop, apart from a - // delisted one playing out its tail. - followers.retain(|path, _| plans.contains_key(path)); + // A broadcast no enabled rendition reads any more stops its readers, apart from a delisted + // one playing out its tail. Its follower stays reachable, with its finished records, while a + // disabled rendition still names it or a delisted one drains, so a re-enable or relist reaches + // it rather than a new follower. Once it plays nothing, the next update lets it go. + let held: HashSet = disabled + .iter() + .filter_map(|(_, reference)| source.resolve_reference(reference.broadcast())) + .collect(); + followers.retain(|path, follower| { + if plans.contains_key(path) { + return true; + } + let idle = *follower.empty.borrow() && follower.plan.borrow().renditions.is_empty(); + if follower.plan.is_closed() || (idle && !held.contains(path)) { + return false; + } + follower.plan.send_replace(Plan::default()); + true + }); for (path, plan) in plans { // A follower whose broadcast failed exited, and this update follows it again. if let Some(follower) = followers.get(&path) - && !follower.is_closed() + && !follower.plan.is_closed() { - follower.send_replace(plan); + follower.plan.send_replace(plan); continue; } // Follow without waiting: an unavailable sibling must not park the catalog loop and hold // up every other rendition. The follower resolves its instances under the run. Every // instance is followed, and each rendition checks the epoch it names. - let Some(shape) = plan.renditions.values().next() else { - // Only disabled renditions read it, and their follower is gone. - continue; - }; + let shape = plan.renditions.values().next().expect("a plan names a rendition"); let instances = match source.follow(shape.broadcast.as_ref(), None) { Ok(instances) => instances, Err(err) => { @@ -967,7 +973,8 @@ fn reconcile( continue; } }; - let (follower, plan) = watch::channel(plan); + let (plan_tx, plan) = watch::channel(plan); + let (empty_tx, empty) = watch::channel(false); let follower_task = Follower { playback: playback.clone(), path: path.clone(), @@ -977,16 +984,24 @@ fn reconcile( finished: Finished::default(), started: 0, next_id: 0, + empty: empty_tx, } .run(instances, plan); tasks.spawn_on( STREAMING.scope(playback.element.clone(), follower_task), RUNTIME.handle(), ); - followers.insert(path, follower); + followers.insert(path, Followed { plan: plan_tx, empty }); } } +/// A [`Follower`] as its run reaches it. +struct Followed { + plan: watch::Sender, + /// Set while it plays no rendition, so the run may let it go. + empty: watch::Receiver, +} + /// What one broadcast run shares with every follower and reader it starts. Pads survive a /// broadcast switch; its timing does not. #[derive(Clone)] @@ -1031,6 +1046,8 @@ struct Follower { /// How many instances started, numbering each. started: u64, next_id: u64, + /// The run's [`Followed::empty`]. + empty: watch::Sender, } /// The instance a [`Follower`] reads. @@ -1095,6 +1112,9 @@ impl Follower { if !following && self.readers.is_empty() { break; } + let empty = self.renditions.is_empty(); + self.empty + .send_if_modified(|was| std::mem::replace(was, empty) != empty); tokio::select! { biased; @@ -3575,6 +3595,66 @@ mod session_tests { drop(first); } + /// A sibling rendition delisted while it plays and listed again before its track finishes stays + /// with the reader that drained it: the finished track does not replay on its pad, and the + /// next instance plays there. + #[test] + fn a_sibling_rendition_relisted_while_draining_does_not_replay_its_finish() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let _audio = ["audio", "again"].map(|name| { + room.broadcast + .create_track(name, hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap() + }); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + + // The catalog consumer only yields the newest snapshot, so each update lists another audio + // rendition: its pad appearing proves the run acted on it. + let listed = room.catalog.modify().unwrap().video.renditions.clone(); + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.clear(); + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + await_pad(&element, "audio_"); + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions = listed; + let mut again = audio_rendition(); + again.track = Some("again".into()); + guard.audio.renditions.insert("again".to_string(), again); + } + for _ in 0..200 { + if pads(&element, "audio_").len() == 2 { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert_eq!(pads(&element, "audio_").len(), 2, "the relist never landed"); + + first.video.finish().unwrap(); + await_finished(&first); + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + let seen = recorder.seen(); + assert_eq!(recorder.count(|seen| *seen == Seen::Start(pad.clone())), 1, "{seen:?}"); + recorder.assert_one_pad_held(); + drop(first); + } + /// A publisher handoff after a sibling track finished, the catalog naming the new instance's /// epoch, plays that instance on the same pad. #[test] From 1f44597dc3fbfff7fac945cbf13287051b6a2556 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 19:43:34 -0700 Subject: [PATCH 22/27] fix(moq-gst): resume a sibling instance that is announced again follow::Instances hands out the same instance again after it returns, and the follower treated it as a new one: a fresh serial dropped the record of a track it finished, so its last group replayed on the pad, and a fresh timeline remapped it. The follower keeps its last instance across an end. The same channel announced again keeps its readers; the same epoch served again after an end resumes with its serial and timeline. An epochless return after an end cannot prove its identity and still counts as new. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 96 +++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 19 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 46862b3fe4e..a770f19682d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -981,6 +981,7 @@ fn reconcile( renditions: HashMap::new(), readers: tokio::task::JoinSet::new(), current: None, + online: false, finished: Finished::default(), started: 0, next_id: 0, @@ -1030,8 +1031,8 @@ struct Plan { /// Follows the instances at one path and plays the renditions that read it on each in turn: a /// start reads them on that instance, an end idles their pads, and another instance taking over -/// restarts each stream on the same pad. The catalog's own broadcast is a single instance that -/// never ends. +/// restarts each stream on the same pad. The same instance returning resumes its state. The +/// catalog's own broadcast is a single instance that never ends. /// /// It owns its renditions' readers and the state of the instance they read, so whether a /// rendition is reading, idle, or playing out its tail is its own state rather than a task's. @@ -1040,7 +1041,10 @@ struct Follower { path: moq_net::PathOwned, renditions: HashMap, readers: tokio::task::JoinSet<(Rendition, u64, Exit)>, + /// The instance last started, kept across its end so its return resumes it. current: Option, + /// Whether [`Self::current`] is announced. + online: bool, /// Shared with the readers. finished: Finished, /// How many instances started, numbering each. @@ -1147,7 +1151,7 @@ impl Follower { } Ok(moq_mux::follow::Event::End) => { gst::info!(CAT, "{} is offline, holding its pads", self.path); - self.current = None; + self.online = false; self.idle(); } Err(err) => { @@ -1159,7 +1163,7 @@ impl Follower { // sibling renditions idle with it. _ = catalog.closed(), if following && !listening => { following = false; - self.current = None; + self.online = false; self.idle(); } } @@ -1211,20 +1215,36 @@ impl Follower { /// Read `instance`, which took over from any before it. fn start(&mut self, instance: moq_net::broadcast::Consumer) { - self.started += 1; - // The first instance is the one the run started alongside. A later one's timestamps share - // no origin with the run's. - let timeline = match self.started { - 1 => self.playback.timeline.clone(), - _ => Arc::default(), - }; - self.current = Some(Current { - instance, - timeline, - serial: self.started, - }); - self.finished.retain(|_, serial| serial == self.started); - self.idle(); + self.online = true; + match &mut self.current { + // Announced again without an end, its readers carry on. + Some(current) if current.instance.is_clone(&instance) => {} + // The same publisher instance served again after an end keeps its timeline and the + // tracks it finished. An epochless one cannot prove it is the same. + Some(current) + if current.instance.info().epoch.is_some() + && current.instance.info().epoch == instance.info().epoch => + { + current.instance = instance; + self.idle(); + } + _ => { + self.started += 1; + // The first instance is the one the run started alongside. A later one's + // timestamps share no origin with the run's. + let timeline = match self.started { + 1 => self.playback.timeline.clone(), + _ => Arc::default(), + }; + self.current = Some(Current { + instance, + timeline, + serial: self.started, + }); + self.finished.retain(|_, serial| serial == self.started); + self.idle(); + } + } self.play(); } @@ -1238,7 +1258,7 @@ impl Follower { /// Start a reader on the current instance for each rendition without one that it may serve. fn play(&mut self) { - let Some(current) = &self.current else { + let Some(current) = self.current.as_ref().filter(|_| self.online) else { return; }; for (rendition, entry) in &mut self.renditions { @@ -3443,6 +3463,44 @@ mod session_tests { drop(old); } + /// The same sibling instance announced again resumes its state, whether or not its end reached + /// the follower first: a track it finished does not replay on its pad. + #[test] + fn a_reannounced_sibling_instance_does_not_replay_its_finish() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + let session = follow(&element, &origin); + let mut follow = origin.consume().follow("cam").unwrap(); + + let route = epoch(); + let mut first = Publisher::sibling(&origin, route.clone()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + first.broadcast.unannounce(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::End(_))); + first.broadcast.announce(route).unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + + let mut second = Publisher::sibling(&origin, epoch()); + assert_eq!( + recorder.written(&mut second, 2), + pad, + "the next instance moved to another pad" + ); + stop(session); + + // Only the next instance starts a stream on the pad. + let seen = recorder.seen(); + assert_eq!(recorder.count(|seen| *seen == Seen::Start(pad.clone())), 1, "{seen:?}"); + recorder.assert_one_pad_held(); + drop(first); + } + /// A catalog finished while its broadcast stays up still follows its siblings: the next /// instance plays on the same pad, until the catalog's broadcast ends. #[test] From 46ea01cbfbb7e41f52ee61e4ec18c949032b3fe3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 19:59:51 -0700 Subject: [PATCH 23/27] fix(moq-gst): let go of an emptied follower without another catalog update A sibling path the catalog stopped reading kept its follower, with its instance consumers, until the next catalog update or the end of the run. A follower now wakes the run when it runs out of renditions, and the run drops it once the plan it sent last is empty and no disabled rendition names the path, the same rule an update applies. A follower with no renditions left also ends once the catalog closes, since no plan can add any. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 108 ++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index a770f19682d..1f81bc6e5a8 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -784,6 +784,9 @@ async fn follow_catalog( stop: stopped, }; let mut followers: HashMap = HashMap::new(); + // Woken when a follower runs out of renditions, so the run can let it go without waiting for + // another catalog update. + let emptied = Arc::new(tokio::sync::Notify::new()); // The followers, and the retirements of pads the catalog no longer lists. let mut tasks = tokio::task::JoinSet::new(); let mut catalog_closed = false; @@ -799,10 +802,11 @@ async fn follow_catalog( // Ended by a restart, or the session ending. _ = cancel.changed() => break, _ = tasks.join_next(), if !tasks.is_empty() => {} + _ = emptied.notified() => followers.retain(|_, follower| !follower.plan.is_closed() && !follower.idle()), // The guard stops us polling a closed catalog track (which would spin the loop // returning None) while the followers drain. next = catalog_consumer.next(), if !catalog_closed => match next { - Ok(Some(catalog)) => reconcile(&catalog, &source, &playback, &mut followers, &mut tasks, cancel), + Ok(Some(catalog)) => reconcile(&catalog, &source, &playback, &mut followers, &mut tasks, &emptied, cancel), // Catalog track closed. Don't stop the followers: let each reader reach its natural // end, and just stop reconciling while they drain. // @@ -852,6 +856,7 @@ fn reconcile( playback: &Playback, followers: &mut HashMap, tasks: &mut tokio::task::JoinSet<()>, + emptied: &Arc, cancel: &watch::Receiver, ) { // Build the desired shape for each rendition. This is deliberately cheap: caps come from the @@ -937,31 +942,21 @@ fn reconcile( // A broadcast no enabled rendition reads any more stops its readers, apart from a delisted // one playing out its tail. Its follower stays reachable, with its finished records, while a // disabled rendition still names it or a delisted one drains, so a re-enable or relist reaches - // it rather than a new follower. Once it plays nothing, the next update lets it go. + // it rather than a new follower. Once it plays nothing, the run lets it go. let held: HashSet = disabled .iter() .filter_map(|(_, reference)| source.resolve_reference(reference.broadcast())) .collect(); followers.retain(|path, follower| { - if plans.contains_key(path) { - return true; - } - let idle = *follower.empty.borrow() && follower.plan.borrow().renditions.is_empty(); - if follower.plan.is_closed() || (idle && !held.contains(path)) { + // A follower whose broadcast failed exited, and this update follows it again. + if follower.plan.is_closed() { return false; } - follower.plan.send_replace(Plan::default()); - true + follower.held = held.contains(path); + follower.plan.send_replace(plans.remove(path).unwrap_or_default()); + !follower.idle() }); for (path, plan) in plans { - // A follower whose broadcast failed exited, and this update follows it again. - if let Some(follower) = followers.get(&path) - && !follower.plan.is_closed() - { - follower.plan.send_replace(plan); - continue; - } - // Follow without waiting: an unavailable sibling must not park the catalog loop and hold // up every other rendition. The follower resolves its instances under the run. Every // instance is followed, and each rendition checks the epoch it names. @@ -975,6 +970,7 @@ fn reconcile( }; let (plan_tx, plan) = watch::channel(plan); let (empty_tx, empty) = watch::channel(false); + let held = held.contains(&path); let follower_task = Follower { playback: playback.clone(), path: path.clone(), @@ -986,21 +982,40 @@ fn reconcile( started: 0, next_id: 0, empty: empty_tx, + emptied: emptied.clone(), } .run(instances, plan); tasks.spawn_on( STREAMING.scope(playback.element.clone(), follower_task), RUNTIME.handle(), ); - followers.insert(path, Followed { plan: plan_tx, empty }); + followers.insert( + path, + Followed { + plan: plan_tx, + empty, + held, + }, + ); } } /// A [`Follower`] as its run reaches it. struct Followed { plan: watch::Sender, - /// Set while it plays no rendition, so the run may let it go. + /// Set while it plays no rendition. empty: watch::Receiver, + /// A disabled rendition names its path, so a re-enable needs its finished records. + held: bool, +} + +impl Followed { + /// Whether nothing reaches it again: it plays nothing, was asked for nothing, and keeps no + /// record a re-enable needs. Judged against the plan sent last, so a plan it has yet to take + /// keeps it. + fn idle(&self) -> bool { + !self.held && *self.empty.borrow() && self.plan.borrow().renditions.is_empty() + } } /// What one broadcast run shares with every follower and reader it starts. Pads survive a @@ -1052,6 +1067,8 @@ struct Follower { next_id: u64, /// The run's [`Followed::empty`]. empty: watch::Sender, + /// Woken as [`Self::empty`] sets, so the run checks whether to let it go. + emptied: Arc, } /// The instance a [`Follower`] reads. @@ -1113,12 +1130,19 @@ impl Follower { self.play(); loop { - if !following && self.readers.is_empty() { + // Done once nothing it reads remains, or nothing is left to read and no plan can add + // any: a closed catalog stops renditions changing. + if (!following || !listening && self.renditions.is_empty()) && self.readers.is_empty() { break; } let empty = self.renditions.is_empty(); - self.empty - .send_if_modified(|was| std::mem::replace(was, empty) != empty); + if self + .empty + .send_if_modified(|was| std::mem::replace(was, empty) != empty) + && empty + { + self.emptied.notify_one(); + } tokio::select! { biased; @@ -3501,6 +3525,46 @@ mod session_tests { drop(first); } + /// A sibling the catalog stopped reading is let go: once the catalog closes, the run ends while + /// both broadcasts stay up. + #[test] + fn a_sibling_no_rendition_reads_does_not_outlive_its_catalog() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let (_shutdown, shutdown) = watch::channel(false); + let (_cancel, mut cancelled) = watch::channel(false); + let run = { + let weak = element.downgrade(); + let source = moq_mux::Source::new(origin.consume(), room.broadcast.consume()); + super::RUNTIME + .spawn(async move { follow_catalog(source, &Pads::default(), weak, &shutdown, &mut cancelled).await }) + }; + + let mut cam = Publisher::sibling(&origin, Default::default()); + recorder.written(&mut cam, 1); + // The delisted rendition plays out its track before its pad retires. + room.catalog.modify().unwrap().video.renditions.clear(); + cam.video.finish().unwrap(); + for _ in 0..200 { + if pads(&element, "video_").is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!(pads(&element, "video_").is_empty(), "the removal never landed"); + + room.catalog.finish().unwrap(); + super::RUNTIME + .block_on(async { tokio::time::timeout(Duration::from_secs(10), run).await }) + .expect("the sibling's follower outlived the catalog") + .unwrap() + .unwrap(); + drop((room, cam)); + } + /// A catalog finished while its broadcast stays up still follows its siblings: the next /// instance plays on the same pad, until the catalog's broadcast ends. #[test] From c2bfc01f0e6e5aa77c03438fe5c4d9bde93c5696 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 20:08:22 -0700 Subject: [PATCH 24/27] fix(moq-gst): release a sibling instance at its end The follower kept the ended instance's consumer to recognize its return, which held the broadcast's state while the sibling stayed offline. It now keeps only the epoch, serial, and timeline across an end, and holds the consumer only while the instance serves. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 47 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 1f81bc6e5a8..60f5bc72ed6 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -977,7 +977,6 @@ fn reconcile( renditions: HashMap::new(), readers: tokio::task::JoinSet::new(), current: None, - online: false, finished: Finished::default(), started: 0, next_id: 0, @@ -1058,8 +1057,6 @@ struct Follower { readers: tokio::task::JoinSet<(Rendition, u64, Exit)>, /// The instance last started, kept across its end so its return resumes it. current: Option, - /// Whether [`Self::current`] is announced. - online: bool, /// Shared with the readers. finished: Finished, /// How many instances started, numbering each. @@ -1073,7 +1070,10 @@ struct Follower { /// The instance a [`Follower`] reads. struct Current { - instance: moq_net::broadcast::Consumer, + /// Released at its end, so an offline sibling holds none of its state. + instance: Option, + /// Identifies it after its end, if its route named one. + epoch: Option, /// Shared by every rendition reading the instance, so their first frames fix one mapping for /// all of them. timeline: Arc, @@ -1175,8 +1175,7 @@ impl Follower { } Ok(moq_mux::follow::Event::End) => { gst::info!(CAT, "{} is offline, holding its pads", self.path); - self.online = false; - self.idle(); + self.offline(); } Err(err) => { gst::warning!(CAT, "lost {}: {err:?}", self.path); @@ -1187,8 +1186,7 @@ impl Follower { // sibling renditions idle with it. _ = catalog.closed(), if following && !listening => { following = false; - self.online = false; - self.idle(); + self.offline(); } } } @@ -1239,17 +1237,16 @@ impl Follower { /// Read `instance`, which took over from any before it. fn start(&mut self, instance: moq_net::broadcast::Consumer) { - self.online = true; match &mut self.current { // Announced again without an end, its readers carry on. - Some(current) if current.instance.is_clone(&instance) => {} + Some(Current { + instance: Some(current), + .. + }) if current.is_clone(&instance) => {} // The same publisher instance served again after an end keeps its timeline and the // tracks it finished. An epochless one cannot prove it is the same. - Some(current) - if current.instance.info().epoch.is_some() - && current.instance.info().epoch == instance.info().epoch => - { - current.instance = instance; + Some(current) if current.epoch.is_some() && current.epoch == instance.info().epoch => { + current.instance = Some(instance); self.idle(); } _ => { @@ -1261,7 +1258,8 @@ impl Follower { _ => Arc::default(), }; self.current = Some(Current { - instance, + epoch: instance.info().epoch.clone(), + instance: Some(instance), timeline, serial: self.started, }); @@ -1272,6 +1270,14 @@ impl Follower { self.play(); } + /// The instance ended: release it and idle until the next one. + fn offline(&mut self) { + if let Some(current) = &mut self.current { + current.instance = None; + } + self.idle(); + } + /// Stop reading the instance. A draining rendition leaves with it. fn idle(&mut self) { self.renditions.retain(|_, entry| !entry.draining); @@ -1282,11 +1288,14 @@ impl Follower { /// Start a reader on the current instance for each rendition without one that it may serve. fn play(&mut self) { - let Some(current) = self.current.as_ref().filter(|_| self.online) else { + let Some(current) = &self.current else { + return; + }; + let Some(instance) = ¤t.instance else { return; }; for (rendition, entry) in &mut self.renditions { - if entry.reader.is_none() && serves(¤t.instance, entry.shape.epoch.as_ref()) { + if entry.reader.is_none() && serves(instance, entry.shape.epoch.as_ref()) { self.next_id += 1; let (cancel, cancelled) = watch::channel(false); let live = Arc::new(AtomicBool::new(false)); @@ -1294,7 +1303,7 @@ impl Follower { playback: self.playback.clone(), rendition: rendition.clone(), shape: entry.shape.clone(), - instance: current.instance.clone(), + instance: instance.clone(), serial: current.serial, timeline: current.timeline.clone(), finished: self.finished.clone(), From 93fad07b517d10e7a6ff0386111c0547a17d6bf8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 21:40:11 -0700 Subject: [PATCH 25/27] fix(moq-gst): keep finished tracks across a pinned failover and a track switch Two ways a finished sibling track replayed its final group on its pad: - An instance a rendition pins finished its track, another instance won the path, and the pinned one won it back with a new serial. The follower now keeps the serial and timeline of each instance a rendition pins, by epoch, next to the current one, so its return resumes them. - A rendition finished one track, then another on the same instance, then switched back. The finished records now hold every track per instance rather than only the last. Records stay bounded by the instances the follower keeps and the tracks they served on listed pads. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 279 +++++++++++++++++++++++++++++------ 1 file changed, 236 insertions(+), 43 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 60f5bc72ed6..57177a7d592 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -977,6 +977,7 @@ fn reconcile( renditions: HashMap::new(), readers: tokio::task::JoinSet::new(), current: None, + resumable: HashMap::new(), finished: Finished::default(), started: 0, next_id: 0, @@ -1055,8 +1056,11 @@ struct Follower { path: moq_net::PathOwned, renditions: HashMap, readers: tokio::task::JoinSet<(Rendition, u64, Exit)>, - /// The instance last started, kept across its end so its return resumes it. + /// The instance last started. current: Option, + /// The number and timeline of each instance a return would resume: the current one and each + /// a rendition pins, by epoch. An epochless instance cannot prove it is the one returning. + resumable: HashMap)>, /// Shared with the readers. finished: Finished, /// How many instances started, numbering each. @@ -1072,7 +1076,7 @@ struct Follower { struct Current { /// Released at its end, so an offline sibling holds none of its state. instance: Option, - /// Identifies it after its end, if its route named one. + /// Its route's, which [`Follower::resumable`] keeps it by. epoch: Option, /// Shared by every rendition reading the instance, so their first frames fix one mapping for /// all of them. @@ -1231,25 +1235,27 @@ impl Follower { }, ); } - // A disabled rendition keeps its pad, and the record with it. A retired pad needs none. - self.finished.retain(|rendition, _| pads.is_listed(rendition)); + self.forget(); } /// Read `instance`, which took over from any before it. fn start(&mut self, instance: moq_net::broadcast::Consumer) { - match &mut self.current { - // Announced again without an end, its readers carry on. - Some(Current { - instance: Some(current), - .. - }) if current.is_clone(&instance) => {} - // The same publisher instance served again after an end keeps its timeline and the - // tracks it finished. An epochless one cannot prove it is the same. - Some(current) if current.epoch.is_some() && current.epoch == instance.info().epoch => { - current.instance = Some(instance); - self.idle(); - } - _ => { + // Announced again without an end, its readers carry on. + if let Some(Current { + instance: Some(current), + .. + }) = &self.current + && current.is_clone(&instance) + { + self.play(); + return; + } + + // An instance served again keeps its timeline and the tracks it finished. + let epoch = instance.info().epoch.clone(); + let (serial, timeline) = match epoch.as_ref().and_then(|epoch| self.resumable.get(epoch)) { + Some((serial, timeline)) => (*serial, timeline.clone()), + None => { self.started += 1; // The first instance is the one the run started alongside. A later one's // timestamps share no origin with the run's. @@ -1257,19 +1263,44 @@ impl Follower { 1 => self.playback.timeline.clone(), _ => Arc::default(), }; - self.current = Some(Current { - epoch: instance.info().epoch.clone(), - instance: Some(instance), - timeline, - serial: self.started, - }); - self.finished.retain(|_, serial| serial == self.started); - self.idle(); + (self.started, timeline) } + }; + if let Some(epoch) = &epoch { + self.resumable.insert(epoch.clone(), (serial, timeline.clone())); } + self.current = Some(Current { + instance: Some(instance), + epoch, + timeline, + serial, + }); + self.forget(); + self.idle(); self.play(); } + /// Drop the state no return can resume: an instance neither current nor pinned by a rendition, + /// and the records of a retired pad. A disabled rendition keeps its pad, and its records. + fn forget(&mut self) { + let pinned: HashSet<&moq_net::Epoch> = self + .renditions + .values() + .filter_map(|entry| entry.shape.epoch.as_ref()) + .chain(self.current.as_ref().and_then(|current| current.epoch.as_ref())) + .collect(); + self.resumable.retain(|epoch, _| pinned.contains(epoch)); + let serials: HashSet = self + .resumable + .values() + .map(|(serial, _)| *serial) + .chain(self.current.as_ref().map(|current| current.serial)) + .collect(); + let pads = &self.playback.pads; + self.finished + .retain(|rendition, serial| pads.is_listed(rendition) && serials.contains(&serial)); + } + /// The instance ended: release it and idle until the next one. fn offline(&mut self) { if let Some(current) = &mut self.current { @@ -1555,24 +1586,24 @@ impl Reader { } } -/// The track each of a follower's renditions last played to the end on its pad, and the serial of -/// the instance that served it. A new subscription to a finished track replays its last group, so -/// a catalog update that still lists the rendition (a publisher's final snapshot lands after its -/// tracks end, or a disable and an enable) must not start it on that pad again. A later instance -/// carries a new track, and a pad that never played the track joins it like any late subscriber. -/// Shared with the readers, since a reader that replaces another on its pad learns this only once -/// the other lets go. +/// Each track a follower's renditions played to the end on their pads, with the serial of the +/// instance that served it. A new subscription to a finished track replays its last group, so a +/// catalog update that still lists the rendition (a publisher's final snapshot lands after its +/// tracks end, a disable and an enable, or a switch back to a track) must not start it on that pad +/// again. Another instance carries a new track, and a pad that never played the track joins it +/// like any late subscriber. At most one record per track an instance the follower keeps served, +/// on a listed pad. Shared with the readers, since a reader that replaces another on its pad +/// learns this only once the other lets go. #[derive(Clone, Default)] -struct Finished(Arc>>); +struct Finished(Arc>>); impl Finished { - /// Record that instance `serial` served the rendition's `track` to its end. A reader of an - /// earlier instance never overwrites a later one's record. + /// Record that instance `serial` served the rendition's `track` to its end. fn insert(&self, rendition: &Rendition, track: &str, serial: u64) { - let mut finished = self.0.lock().unwrap(); - if finished.get(rendition).is_none_or(|(_, recorded)| *recorded <= serial) { - finished.insert(rendition.clone(), (track.to_string(), serial)); - } + self.0 + .lock() + .unwrap() + .insert((rendition.clone(), track.to_string(), serial)); } /// Whether instance `serial` already served the rendition's `track` to its end on its pad. @@ -1580,8 +1611,7 @@ impl Finished { self.0 .lock() .unwrap() - .get(rendition) - .is_some_and(|(finished, recorded)| finished == track && *recorded == serial) + .contains(&(rendition.clone(), track.to_string(), serial)) } /// Keep only the records `keep` accepts. @@ -1589,7 +1619,7 @@ impl Finished { self.0 .lock() .unwrap() - .retain(|rendition, (_, serial)| keep(rendition, *serial)); + .retain(|(rendition, _, serial)| keep(rendition, *serial)); } } @@ -3640,11 +3670,16 @@ mod session_tests { /// Block until `publisher`'s video track reads to its end, as a reader reading it does. fn await_finished(publisher: &Publisher) { + await_track_finished(publisher, "video"); + } + + /// [`await_finished`] for the track `name`. + fn await_track_finished(publisher: &Publisher, name: &str) { let broadcast = publisher.broadcast.consume(); super::RUNTIME.block_on(async { let finished = async { let subscriber = broadcast - .track("video") + .track(name) .unwrap() .subscribe(moq_net::track::Subscription::default()) .await @@ -3823,6 +3858,164 @@ mod session_tests { drop(first); } + /// An instance a rendition pins that wins its path back from another resumes its state: the + /// track it finished replays on no pad, pinned or not. + #[test] + fn a_pinned_instance_winning_its_path_back_does_not_replay_its_finish() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let (pinned, other, next) = (moq_net::Epoch::mint(), moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let mut room = referencing(&origin, Some(pinned.clone())); + { + let mut guard = room.catalog.modify().unwrap(); + let mut again = guard.video.renditions["video"].clone(); + again.track = Some("video".into()); + again.epoch = None; + guard.video.renditions.insert("again".into(), again); + } + let session = follow(&element, &origin); + + // Each rendition plays the first instance on a pad of its own. + let reached = |tag: u8| -> HashSet { + recorder + .seen() + .into_iter() + .filter_map(|seen| match seen { + Seen::Buffer(pad, seen) if seen == tag => Some(pad), + _ => None, + }) + .collect() + }; + let mut first = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(pinned)); + for _ in 0..200 { + if reached(1).len() == 2 { + break; + } + first.write(1); + std::thread::sleep(Duration::from_millis(50)); + } + assert_eq!(reached(1).len(), 2, "{:?}", recorder.seen()); + first.video.finish().unwrap(); + await_finished(&first); + + // Another instance wins the path, which only the unpinned rendition plays, then leaves it + // to the pinned one. + let mut second = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(other)); + let unpinned = recorder.written(&mut second, 2); + let pad = reached(1).into_iter().find(|pad| *pad != unpinned).unwrap(); + let mut follow = origin.consume().follow("cam").unwrap(); + await_event(&mut follow, |event| matches!(event, moq_net::announce::Event::Start(_))); + drop(second); + await_event(&mut follow, |event| { + matches!( + event, + moq_net::announce::Event::Restart(_) | moq_net::announce::Event::Start(_) + ) + }); + + let mut third = Publisher::sibling(&origin, moq_net::origin::Route::default().with_epoch(next.clone())); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .epoch = Some(next); + for _ in 0..200 { + if reached(3).contains(&pad) { + break; + } + third.write(3); + std::thread::sleep(Duration::from_millis(50)); + } + assert!(reached(3).contains(&pad), "the next instance moved to another pad"); + stop(session); + + // Neither pad replays the first instance's track on its return. + let seen = recorder.seen(); + for pad in [&pad, &unpinned] { + assert_eq!( + recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)), + 1, + "{seen:?}" + ); + } + drop(first); + } + + /// A rendition that finished one track, then another, and switches back to the first replays + /// nothing of it. + #[test] + fn a_rendition_switched_back_to_a_finished_track_does_not_replay_it() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let mut room = referencing(&origin, None); + let _audio = room + .broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let session = follow(&element, &origin); + + let mut first = Publisher::sibling(&origin, Default::default()); + let pad = recorder.written(&mut first, 1); + first.video.finish().unwrap(); + await_finished(&first); + let played = recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)); + + // Writing through the publisher now feeds the other track. + let other = first + .broadcast + .create_track("other", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut finished = moq_mux::container::Producer::new( + other, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + std::mem::swap(&mut first.video, &mut finished); + room.catalog + .modify() + .unwrap() + .video + .renditions + .get_mut("video") + .unwrap() + .track = Some("other".into()); + assert_eq!(recorder.written(&mut first, 2), pad, "the switch moved to another pad"); + first.video.finish().unwrap(); + await_track_finished(&first, "other"); + + // The update also lists an audio rendition: its pad appearing proves the run acted on it. + { + let mut guard = room.catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().track = None; + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + await_pad(&element, "audio_"); + let mut second = Publisher::sibling(&origin, Default::default()); + assert_eq!( + recorder.written(&mut second, 3), + pad, + "the next instance moved to another pad" + ); + stop(session); + + // The switch and the next instance each start a stream on the pad; the switch back none. + let seen = recorder.seen(); + assert_eq!(recorder.count(|seen| *seen == Seen::Start(pad.clone())), 2, "{seen:?}"); + assert_eq!( + recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)), + played, + "{seen:?}" + ); + recorder.assert_one_pad_held(); + drop((first, finished)); + } + /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next /// instance plays nothing on it. #[test] From 9ce3862a7e4604a561b112192a594f7d859a69dc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 21:54:35 -0700 Subject: [PATCH 26/27] fix(moq-gst): bound the finished records to one per rendition and instance Keeping every track a rendition finished let a publisher grow the records without bound by renaming a rendition's track and finishing each one. Keep only the last finished track per rendition and kept instance, which the catalog bounds. A rendition switched back to a track it finished before another replays that track's last group, which is the accepted cost. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 100 +++++------------------------------ 1 file changed, 13 insertions(+), 87 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 57177a7d592..efe4b301ace 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1586,16 +1586,16 @@ impl Reader { } } -/// Each track a follower's renditions played to the end on their pads, with the serial of the -/// instance that served it. A new subscription to a finished track replays its last group, so a +/// The track each of a follower's renditions last played to the end on its pad, per instance the +/// follower keeps, by serial. A new subscription to a finished track replays its last group, so a /// catalog update that still lists the rendition (a publisher's final snapshot lands after its -/// tracks end, a disable and an enable, or a switch back to a track) must not start it on that pad -/// again. Another instance carries a new track, and a pad that never played the track joins it -/// like any late subscriber. At most one record per track an instance the follower keeps served, -/// on a listed pad. Shared with the readers, since a reader that replaces another on its pad -/// learns this only once the other lets go. +/// tracks end, or a disable and an enable) must not start it on that pad again. Another instance +/// carries a new track, and a pad that never played the track joins it like any late subscriber. +/// Only the last track is kept, so the records stay bounded by the catalog: a rendition switched +/// back to a track it finished before another replays its last group. Shared with the readers, +/// since a reader that replaces another on its pad learns this only once the other lets go. #[derive(Clone, Default)] -struct Finished(Arc>>); +struct Finished(Arc>>); impl Finished { /// Record that instance `serial` served the rendition's `track` to its end. @@ -1603,7 +1603,7 @@ impl Finished { self.0 .lock() .unwrap() - .insert((rendition.clone(), track.to_string(), serial)); + .insert((rendition.clone(), serial), track.to_string()); } /// Whether instance `serial` already served the rendition's `track` to its end on its pad. @@ -1611,7 +1611,8 @@ impl Finished { self.0 .lock() .unwrap() - .contains(&(rendition.clone(), track.to_string(), serial)) + .get(&(rendition.clone(), serial)) + .is_some_and(|finished| finished == track) } /// Keep only the records `keep` accepts. @@ -1619,7 +1620,7 @@ impl Finished { self.0 .lock() .unwrap() - .retain(|(rendition, _, serial)| keep(rendition, *serial)); + .retain(|(rendition, serial), _| keep(rendition, *serial)); } } @@ -3670,16 +3671,11 @@ mod session_tests { /// Block until `publisher`'s video track reads to its end, as a reader reading it does. fn await_finished(publisher: &Publisher) { - await_track_finished(publisher, "video"); - } - - /// [`await_finished`] for the track `name`. - fn await_track_finished(publisher: &Publisher, name: &str) { let broadcast = publisher.broadcast.consume(); super::RUNTIME.block_on(async { let finished = async { let subscriber = broadcast - .track(name) + .track("video") .unwrap() .subscribe(moq_net::track::Subscription::default()) .await @@ -3946,76 +3942,6 @@ mod session_tests { drop(first); } - /// A rendition that finished one track, then another, and switches back to the first replays - /// nothing of it. - #[test] - fn a_rendition_switched_back_to_a_finished_track_does_not_replay_it() { - let _pad_ids = pad_ids(); - let element = element(); - let recorder = Recorder::attach(&element); - let origin = origin(); - let mut room = referencing(&origin, None); - let _audio = room - .broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let session = follow(&element, &origin); - - let mut first = Publisher::sibling(&origin, Default::default()); - let pad = recorder.written(&mut first, 1); - first.video.finish().unwrap(); - await_finished(&first); - let played = recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)); - - // Writing through the publisher now feeds the other track. - let other = first - .broadcast - .create_track("other", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let mut finished = moq_mux::container::Producer::new( - other, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - ); - std::mem::swap(&mut first.video, &mut finished); - room.catalog - .modify() - .unwrap() - .video - .renditions - .get_mut("video") - .unwrap() - .track = Some("other".into()); - assert_eq!(recorder.written(&mut first, 2), pad, "the switch moved to another pad"); - first.video.finish().unwrap(); - await_track_finished(&first, "other"); - - // The update also lists an audio rendition: its pad appearing proves the run acted on it. - { - let mut guard = room.catalog.modify().unwrap(); - guard.video.renditions.get_mut("video").unwrap().track = None; - guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); - } - await_pad(&element, "audio_"); - let mut second = Publisher::sibling(&origin, Default::default()); - assert_eq!( - recorder.written(&mut second, 3), - pad, - "the next instance moved to another pad" - ); - stop(session); - - // The switch and the next instance each start a stream on the pad; the switch back none. - let seen = recorder.seen(); - assert_eq!(recorder.count(|seen| *seen == Seen::Start(pad.clone())), 2, "{seen:?}"); - assert_eq!( - recorder.count(|seen| *seen == Seen::Buffer(pad.clone(), 1)), - played, - "{seen:?}" - ); - recorder.assert_one_pad_held(); - drop((first, finished)); - } - /// A rendition delisted while its sibling is offline ends its idle pad, and the sibling's next /// instance plays nothing on it. #[test] From 3787aa6f0d8027a18d218a116e4f78828339220a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 22:33:12 -0700 Subject: [PATCH 27/27] fix(moq-gst): play what the catalog's final snapshot adds A final snapshot and the catalog's close could reach a follower as one plan, and a closed plan started no reader, so a rendition the snapshot added or reshaped never played. The final plan now starts exactly the renditions it added or reshaped, and still restarts no reader that already ended. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 57 +++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index efe4b301ace..9184e1e13e3 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1156,9 +1156,12 @@ impl Follower { let next = plan.borrow_and_update().clone(); listening = !next.closed; following = listening || sibling; - self.apply(next.renditions); - if listening { - self.play(); + let fresh = self.apply(next.renditions); + // The final plan starts what it adds or reshapes, but restarts no reader + // that already ended. + match listening { + true => self.play(), + false => self.play_where(|rendition| fresh.contains(rendition)), } } // No rendition reads this path any more. @@ -1199,10 +1202,11 @@ impl Follower { while self.readers.join_next().await.is_some() {} } - /// Bring the renditions in line with `wanted`, stopping each one reshaped or gone. [`Self::play`] - /// starts the rest. - fn apply(&mut self, mut wanted: HashMap) { + /// Bring the renditions in line with `wanted`, stopping each one reshaped or gone, and return + /// those added or reshaped. [`Self::play`] starts the rest. + fn apply(&mut self, mut wanted: HashMap) -> HashSet { let pads = &self.playback.pads; + let mut fresh = HashSet::new(); self.renditions .retain(|rendition, entry| match wanted.remove(rendition) { Some(shape) => { @@ -1210,6 +1214,7 @@ impl Follower { if shape != entry.shape { entry.shape = shape; entry.reader = None; + fresh.insert(rendition.clone()); } entry.draining = false; true @@ -1226,6 +1231,7 @@ impl Follower { } }); for (rendition, shape) in wanted { + fresh.insert(rendition.clone()); self.renditions.insert( rendition, Entry { @@ -1236,6 +1242,7 @@ impl Follower { ); } self.forget(); + fresh } /// Read `instance`, which took over from any before it. @@ -1319,6 +1326,11 @@ impl Follower { /// Start a reader on the current instance for each rendition without one that it may serve. fn play(&mut self) { + self.play_where(|_| true); + } + + /// [`Self::play`] for the renditions `only` accepts. + fn play_where(&mut self, only: impl Fn(&Rendition) -> bool) { let Some(current) = &self.current else { return; }; @@ -1326,7 +1338,7 @@ impl Follower { return; }; for (rendition, entry) in &mut self.renditions { - if entry.reader.is_none() && serves(instance, entry.shape.epoch.as_ref()) { + if entry.reader.is_none() && only(rendition) && serves(instance, entry.shape.epoch.as_ref()) { self.next_id += 1; let (cancel, cancelled) = watch::channel(false); let live = Arc::new(AtomicBool::new(false)); @@ -2415,6 +2427,37 @@ mod session_tests { super::RUNTIME.block_on(session).unwrap().unwrap(); } + /// A rendition the catalog's final snapshot adds plays, even when the snapshot and the close + /// reach the follower together. + #[test] + fn a_closing_catalog_plays_the_rendition_its_final_snapshot_adds() { + let _pad_ids = pad_ids(); + let element = element(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let _video = broadcast.create_track("video", None).unwrap(); + let _audio = broadcast.create_track("audio", None).unwrap(); + { + let mut guard = catalog.modify().unwrap(); + guard.video.renditions = BTreeMap::from([("video".to_string(), video_rendition())]); + } + + let (shutdown, shutdown_rx) = watch::channel(false); + let session = run(broadcast.consume(), element.downgrade(), shutdown_rx); + await_pad(&element, "video_"); + + { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + catalog.finish().unwrap(); + await_pad(&element, "audio_"); + + let _ = shutdown.send(true); + super::RUNTIME.block_on(session).unwrap().unwrap(); + } + /// The flip side of the test above: a rendition the publisher reserved and never served is /// answerable once the publisher ends the broadcast, which resolves every name it never /// filled. That reader must end on its own so the run ends on the media it was serving, and the