From 97baa2c4211902fc307817d14c30eb307cc5613a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 08:14:44 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/test-flakes-2/gst-sink-restart-pads.md Quest-Claim: 99b585c5cd98dca9 From d1bb46cc21bb510c3efe42ddacae5c295e0b2155 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 09:24:08 -0800 Subject: [PATCH 2/3] fix(gst): a failed track is not replayed on its pad A track that failed on an instance still up, such as a relay's copy cut off by its publisher's session closing before the instance ends, left no finished record. A later catalog update, such as the publisher's final snapshot, then started the rendition again on the same instance, and the new subscription could replay a group the held pad already played, behind a second stream-start. That second stream-start is the likely cause of the load-only failure of `a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times`, whose audio pad played in a generation it was dropped from. A failure now records the track like an end, in the stream and in the cancel race beside it. Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-2/README.md | 7 +- .../m0/test-flakes-2/gst-sink-restart-pads.md | 17 -- rs/moq-gst/src/source/imp.rs | 152 +++++++++++++++--- 3 files changed, 140 insertions(+), 36 deletions(-) delete mode 100644 quest/m0/test-flakes-2/gst-sink-restart-pads.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index f448acb0bc2..a8ec32a5e34 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -148,6 +148,12 @@ skipped the timing report. A TERM landing after a lane forked but before hung. The exit trap now waits again, and cancellation reads the job table, fixed by [moq-dev/moq#8138](https://github.com/moq-dev/moq/pull/8138). +moq-gst's sink-restart pad test caught a real source bug: a track that failed on +an instance still up, such as a relay's copy cut off as its publisher's session +closed, left no finished record, so the final catalog snapshot started the +rendition again and replayed a group on its held pad. A failure now counts as +the track's end, fixed by [moq-dev/moq#8160](https://github.com/moq-dev/moq/pull/8160). + Public API: none. Wire: none. ## Required @@ -155,4 +161,3 @@ Public API: none. Wire: none. - [Fallback playback](/quest/m0/test-flakes-2/ietf22-fallback.md) - a browser viewer follows a fallback within the media lane's bound without dropping its session - [The datagram-end close follow-up](/quest/m0/test-flakes-2/datagram-end-close.md) - investigate whether duplicate SETUP can override an accepted lite-06 SUBSCRIBE_END during shutdown - [moq-transcode's idle-rung cancel test survives load](/quest/m0/test-flakes-2/transcode-idle-rung-cancel.md) - `cancelling_an_idle_rung_reports_cancel` stops seeing `serve` return before its abort lands -- [moq-gst's sink-restart pad test survives load](/quest/m0/test-flakes-2/gst-sink-restart-pads.md) - `a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times` keeps its pads under load diff --git a/quest/m0/test-flakes-2/gst-sink-restart-pads.md b/quest/m0/test-flakes-2/gst-sink-restart-pads.md deleted file mode 100644 index 4355ab0e32e..00000000000 --- a/quest/m0/test-flakes-2/gst-sink-restart-pads.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] moq-gst's sink-restart pad test survives load - -## Goal - -`source::imp::session_tests::a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times` -passes reliably under host load, fixed at the cause rather than retried. - -## Plan - -Seen 2026-11-07 on [moq-dev/moq#8122](https://github.com/moq-dev/moq/pull/8122)'s -Check ([job](https://github.com/moq-dev/moq/actions/runs/42341390803/job/128309636226)), -a PR that doesn't touch moq-gst: `assertion failed: playing.iter().all(|pad| -held.contains(pad) && pad.starts_with("video_"))` at -`rs/moq-gst/src/source/imp.rs:5830`, followed by a GStreamer-CRITICAL that -`moqsink0` was disposed in PAUSED instead of NULL. Reproduce under load first, -then find whether the test reads the pad set before the restart settles or the -source really drops a pad. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 0105f7ef25f..9b082fddb35 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1766,11 +1766,14 @@ impl Play { loop { tokio::select! { // The follower stopped this reader (the instance changed, the rendition was - // reshaped or delisted, or the run ended). An end that arrived alongside still - // finishes the track, or the play queued behind this one would replay it. + // reshaped or delisted, or the run ended). An end or failure that arrived alongside + // still finishes the track, as the arms below would, or the play queued behind this + // one would replay it. _ = cancel.changed() => { - if ended(&mut track) { - finished.insert(rendition, &shape.track, serial); + match ended(&mut track) { + Some(Ok(())) => finished.insert(rendition, &shape.track, serial), + Some(Err(_)) if !instance.is_closed() => finished.insert(rendition, &shape.track, serial), + _ => {} } return; } @@ -1838,8 +1841,11 @@ impl Play { finished.insert(rendition, &shape.track, serial); return; } + // The track is over on an instance still up, which may hand a new subscription + // the groups already played, so it is recorded like an end. Err(err) => { gst::warning!(CAT, "track {name} failed: {err:?}"); + finished.insert(rendition, &shape.track, serial); return; } } @@ -1848,11 +1854,12 @@ impl Play { } } -/// The track each of a follower's renditions last played to the end on its pad, per instance the -/// follower keeps, by serial. A new subscription to a finished track replays its last group, so a -/// catalog update that still lists the rendition (a publisher's final snapshot lands after its -/// tracks end, or a disable and an enable) must not start it on that pad again. Another instance -/// carries a new track, and a pad that never played the track joins it like any late subscriber. +/// The track each of a follower's renditions last played to its end or failure on its pad, per +/// instance the follower keeps, by serial. A new subscription to an ended track replays its last +/// group, so a catalog update that still lists the rendition (a publisher's final snapshot lands +/// after its tracks end, or a disable and an enable) must not start it on that pad again. Another +/// instance carries a new track, and a pad that never played the track joins it like any late +/// subscriber. /// Only the last track is kept, so the records stay bounded by the catalog: a rendition switched /// back to a track it finished before another replays its last group. Shared with the plays, /// since a play queued behind another on its pad learns this only once the other is done. @@ -1902,18 +1909,20 @@ impl Timeline { } } -/// Whether the track's end is the next media item, without waiting for it. Markers ready before it -/// are passed over; a frame or anything not ready yet means it has not ended. -fn ended(track: &mut moq_mux::container::Consumer) -> bool { +/// The track's end, or its failure, if that is the next media item, without waiting for it. +/// Markers ready before it are passed over; a frame or anything not ready yet means it has not +/// ended. +fn ended(track: &mut moq_mux::container::Consumer) -> Option> { let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); loop { let mut read = std::pin::pin!(track.read()); match read.as_mut().poll(&mut cx) { - std::task::Poll::Ready(Ok(None)) => return true, + std::task::Poll::Ready(Ok(None)) => return Some(Ok(())), std::task::Poll::Ready(Ok(Some( moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }, ))) => continue, - _ => return false, + std::task::Poll::Ready(Err(err)) => return Some(Err(err)), + std::task::Poll::Pending | std::task::Poll::Ready(Ok(Some(_))) => return None, } } } @@ -2169,7 +2178,7 @@ mod tests { use moq_net::Timestamp; use std::time::Duration; - /// A reader cancelled as its track ends records the end only when no media is left to read, + /// A reader cancelled as its track ends or fails records it only when no media is left to read, /// passing over the markers a publisher closes with. #[test] fn ended_sees_only_an_end_with_no_media_before_it() { @@ -2178,7 +2187,7 @@ mod tests { let legacy = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); let mut producer = moq_mux::container::Producer::new(track, legacy()); let mut consumer = moq_mux::container::Consumer::new(producer.consume(), legacy()); - assert!(!ended(&mut consumer), "an open track has not ended"); + assert!(ended(&mut consumer).is_none(), "an open track has not ended"); producer .write(moq_mux::container::Frame { @@ -2188,11 +2197,22 @@ mod tests { duration: None, }) .unwrap(); - assert!(!ended(&mut consumer), "a frame was still unread"); + assert!(ended(&mut consumer).is_none(), "a frame was still unread"); producer.discontinuity().unwrap(); producer.finish().unwrap(); - assert!(ended(&mut consumer), "the closing marker hid the end"); + assert!( + matches!(ended(&mut consumer), Some(Ok(()))), + "the closing marker hid the end" + ); + + let track = broadcast + .create_track("failed", hang::container::track_info(0)) + .unwrap(); + let producer = moq_mux::container::Producer::new(track, legacy()); + let mut consumer = moq_mux::container::Consumer::new(producer.consume(), legacy()); + producer.abort(moq_net::Error::Cancel); + assert!(matches!(ended(&mut consumer), Some(Err(_))), "a failure is an end too"); } #[test] @@ -3575,6 +3595,102 @@ mod session_tests { assert_eq!(buffers.load(Ordering::Relaxed), FRAMES, "the finished track replayed"); } + /// A track failing on an instance still up is over there, as a finished one is, yet the + /// instance may still hand a new subscription what the pad already played: a relay's copy cut + /// off by its publisher's session closing keeps its cache until the instance ends. A catalog + /// update, even one reshaping the rendition, must not start it again on that instance. + #[test] + fn a_failed_track_is_not_replayed_after_a_catalog_update() { + let _pad_ids = pad_ids(); + let element = element(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let info = hang::container::track_info(hang::catalog::PRIORITY.video); + let container = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let keyframe = moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_micros(0).unwrap(), + payload: bytes::Bytes::from(vec![0; 64]), + keyframe: true, + duration: None, + }; + let mut producer = + moq_mux::container::Producer::new(broadcast.create_track("video", info.clone()).unwrap(), container()); + producer.write(keyframe.clone()).unwrap(); + catalog.modify().unwrap().video.renditions = BTreeMap::from([("video".to_string(), video_rendition())]); + + // The probe goes on before the pad's first push, so it sees every buffer and every later + // stream start. + let restarts = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let restarted = restarts.clone(); + let (pushed, mut buffers) = tokio::sync::mpsc::unbounded_channel(); + let (added, mut adds) = tokio::sync::mpsc::unbounded_channel(); + element.connect_pad_added(move |_, pad| { + if pad.name().starts_with("video_") { + let (pushed, restarted) = (pushed.clone(), restarted.clone()); + pad.add_probe( + gst::PadProbeType::BUFFER | gst::PadProbeType::EVENT_DOWNSTREAM, + move |_, info| match &info.data { + Some(gst::PadProbeData::Buffer(_)) => { + let _ = pushed.send(()); + gst::PadProbeReturn::Drop + } + Some(gst::PadProbeData::Event(event)) if event.type_() == gst::EventType::StreamStart => { + restarted.fetch_add(1, Ordering::Relaxed); + gst::PadProbeReturn::Ok + } + _ => gst::PadProbeReturn::Ok, + }, + ); + } + added.send(pad.name().to_string()).unwrap(); + }); + + let pads = Pads::default(); + let (_shutdown, shutdown) = watch::channel(false); + let (_cancel, mut cancelled) = watch::channel(false); + let session = { + let source = moq_mux::Source::new(origin().consume(), broadcast.consume()); + let session = Session { + max_delay: DEFAULT_MAX_DELAY, + element: element.downgrade(), + pads: pads.clone(), + shutdown, + }; + super::RUNTIME.spawn(async move { session.follow_catalog(source, &mut cancelled).await }) + }; + super::RUNTIME.block_on(buffers.recv()).unwrap(); + + // An abort drops what its readers have yet to take, so it waits for the frame above. + producer.abort(moq_net::Error::Cancel); + super::RUNTIME.block_on(pads.settled(&(TrackKind::Video, "video".to_string()))); + + // The instance serves the track again, with the frame the pad already played. + let mut again = moq_mux::container::Producer::new(broadcast.create_track("video", info).unwrap(), container()); + again.write(keyframe).unwrap(); + + // The update reshapes the rendition, which hands its pad to a new reader, and lists an + // audio one, whose pad proves the session acted on it. + let audio = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + { + let mut guard = catalog.modify().unwrap(); + guard.video.renditions.get_mut("video").unwrap().description = Some(bytes::Bytes::from_static(&[1])); + guard.audio.renditions = BTreeMap::from([("audio".to_string(), audio_rendition())]); + } + super::RUNTIME.block_on(async { while !adds.recv().await.unwrap().starts_with("audio_") {} }); + + // The run ends only once every reader drains, a replaying one included. + again.finish().unwrap(); + audio.finish().unwrap(); + catalog.finish().unwrap(); + super::RUNTIME.block_on(session).unwrap().unwrap(); + + assert_eq!(restarts.load(Ordering::Relaxed), 0, "the failed track started again"); + assert!(buffers.try_recv().is_err(), "the failed track replayed"); + } + /// Pipelines link `moqsrc`'s pads by name, so the first video rendition that actually /// arrives has to be `video_0`. A rendition announced but never served must not claim that /// name and leave the real one on `video_1`, where `s.video_0 ! ...` never links. From 9d86e15c3965e655f65d4f7cc3937a86d147a4e5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 10:45:17 -0800 Subject: [PATCH 3/3] fix(gst): a failure holds only for the start of the instance it happened on A track can fail as its instance goes away, before the instance reports closed. Recording that by serial suppressed the rendition when the same instance returned. A failure record now names the start it happened on, so a return resumes the track while a clean end stays suppressed. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 156 +++++++++++++++++++++++++++-------- 1 file changed, 123 insertions(+), 33 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 9b082fddb35..46bd3c189ee 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -733,7 +733,11 @@ impl PadActor { /// Start a new stream on the pad and stream `play` to it. The finished record is checked only /// now, since the play before records its track as finished before it lets go. async fn play(&mut self, play: Play) -> Exit { - if *play.cancel.borrow() || play.finished.contains(&self.rendition, &play.shape.track, play.serial) { + if *play.cancel.borrow() + || play + .finished + .contains(&self.rendition, &play.shape.track, play.attached) + { return Exit::Done; } let name = &self.rendition.1; @@ -1201,6 +1205,7 @@ fn reconcile( resumable: HashMap::new(), finished: Finished::default(), started: 0, + starts: 0, next_id: 0, empty: empty_tx, emptied: emptied.clone(), @@ -1276,6 +1281,8 @@ struct Follower { finished: Finished, /// How many instances started, numbering each. started: u64, + /// How many times any instance started, a return included. + starts: u64, next_id: u64, /// The run's [`Followed::empty`]. empty: watch::Sender, @@ -1294,6 +1301,17 @@ struct Current { timeline: Arc, /// Its number among the follower's instances. serial: u64, + /// Its number among the follower's starts, which a return does not keep. + start: u64, +} + +/// The instance a reader reads, and the start of it that the reader belongs to. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Attached { + /// The instance's [`Current::serial`]. + serial: u64, + /// The instance's [`Current::start`]. + start: u64, } /// A rendition a [`Follower`] plays. @@ -1495,11 +1513,13 @@ impl Follower { if let Some(id) = &id { self.resumable.insert(id.clone(), (serial, timeline.clone())); } + self.starts += 1; self.current = Some(Current { instance: Some(instance), id, timeline, serial, + start: self.starts, }); self.forget(); self.idle(); @@ -1566,7 +1586,10 @@ impl Follower { rendition: rendition.clone(), shape: entry.shape.clone(), instance: instance.clone(), - serial: current.serial, + attached: Attached { + serial: current.serial, + start: current.start, + }, timeline: current.timeline.clone(), finished: self.finished.clone(), live: live.clone(), @@ -1631,8 +1654,7 @@ struct Reader { rendition: Rendition, shape: Shape, instance: moq_net::broadcast::Consumer, - /// The instance's [`Current::serial`]. - serial: u64, + attached: Attached, timeline: Arc, /// The follower's [`Follower::finished`]. finished: Finished, @@ -1651,7 +1673,7 @@ impl Reader { rendition, shape, instance, - serial, + attached, timeline, finished, live, @@ -1667,7 +1689,7 @@ impl Reader { // An instance that already played this track to its end on the pad would replay its last // group. - if finished.contains(&rendition, &shape.track, serial) { + if finished.contains(&rendition, &shape.track, attached) { return Exit::Done; } @@ -1716,7 +1738,7 @@ impl Reader { shape, track, instance, - serial, + attached, timeline, finished, live, @@ -1733,8 +1755,7 @@ struct Play { shape: Shape, track: moq_mux::container::Consumer, instance: moq_net::broadcast::Consumer, - /// The instance's [`Current::serial`]. - serial: u64, + attached: Attached, timeline: Arc, finished: Finished, /// Shared with the reader's [`ReaderHandle::live`], set once the play has the pad. @@ -1752,7 +1773,7 @@ impl Play { shape, mut track, instance, - serial, + attached, timeline, finished, mut cancel, @@ -1767,13 +1788,11 @@ impl Play { tokio::select! { // The follower stopped this reader (the instance changed, the rendition was // reshaped or delisted, or the run ended). An end or failure that arrived alongside - // still finishes the track, as the arms below would, or the play queued behind this - // one would replay it. + // is recorded as the arms below would, or the play queued behind this one would + // replay it. _ = cancel.changed() => { - match ended(&mut track) { - Some(Ok(())) => finished.insert(rendition, &shape.track, serial), - Some(Err(_)) if !instance.is_closed() => finished.insert(rendition, &shape.track, serial), - _ => {} + if let Some(end) = ended(&mut track) { + finished.insert(rendition, &shape.track, attached, end.is_err()); } return; } @@ -1838,14 +1857,14 @@ impl Play { // Recorded before the actor takes the next play, // so a reshape on this instance does not replay it. Ok(None) => { - finished.insert(rendition, &shape.track, serial); + finished.insert(rendition, &shape.track, attached, false); return; } - // The track is over on an instance still up, which may hand a new subscription - // the groups already played, so it is recorded like an end. + // The instance may still hand a new subscription the groups already played, + // such as a relay's copy cut off by its publisher's session closing. Err(err) => { gst::warning!(CAT, "track {name} failed: {err:?}"); - finished.insert(rendition, &shape.track, serial); + finished.insert(rendition, &shape.track, attached, true); return; } } @@ -1860,28 +1879,41 @@ impl Play { /// after its tracks end, or a disable and an enable) must not start it on that pad again. Another /// instance carries a new track, and a pad that never played the track joins it like any late /// subscriber. +/// An end holds for every return of its instance. A failure holds only for the start it happened +/// on: it may be the instance going away, seen before the instance reports closed, and the same +/// instance returning serves the track again. /// Only the last track is kept, so the records stay bounded by the catalog: a rendition switched /// back to a track it finished before another replays its last group. Shared with the plays, /// since a play queued behind another on its pad learns this only once the other is done. #[derive(Clone, Default)] -struct Finished(Arc>>); +struct Finished(Arc>>); + +/// A [`Finished`] record. +struct Ended { + track: String, + /// The [`Attached::start`] a failure happened on, or none for an end. + failed: Option, +} impl Finished { - /// Record that instance `serial` served the rendition's `track` to its end. - fn insert(&self, rendition: &Rendition, track: &str, serial: u64) { - self.0 - .lock() - .unwrap() - .insert((rendition.clone(), serial), track.to_string()); + /// Record that the rendition's `track` ended, or `failed`, while `attached`. + fn insert(&self, rendition: &Rendition, track: &str, attached: Attached, failed: bool) { + self.0.lock().unwrap().insert( + (rendition.clone(), attached.serial), + Ended { + track: track.to_string(), + failed: failed.then_some(attached.start), + }, + ); } - /// Whether instance `serial` already served the rendition's `track` to its end on its pad. - fn contains(&self, rendition: &Rendition, track: &str, serial: u64) -> bool { + /// Whether the rendition's `track` already played out on its pad for a reader `attached`. + fn contains(&self, rendition: &Rendition, track: &str, attached: Attached) -> bool { self.0 .lock() .unwrap() - .get(&(rendition.clone(), serial)) - .is_some_and(|finished| finished == track) + .get(&(rendition.clone(), attached.serial)) + .is_some_and(|ended| ended.track == track && ended.failed.is_none_or(|start| start == attached.start)) } /// Keep only the records `keep` accepts. @@ -4032,7 +4064,7 @@ mod session_tests { warmup: Duration::ZERO, }, instance, - serial: 1, + attached: super::Attached { serial: 1, start: 1 }, timeline: Arc::default(), finished: super::Finished::default(), live: Arc::default(), @@ -4083,7 +4115,7 @@ mod session_tests { warmup: Duration::ZERO, }, instance, - serial: 1, + attached: super::Attached { serial: 1, start: 1 }, timeline: Arc::default(), finished: super::Finished::default(), live: Arc::default(), @@ -4632,6 +4664,64 @@ mod session_tests { drop(first); } + /// A track failing may be its instance going away before the instance reports closed, as a + /// relay's copy does when its publisher's session closes, so the same instance announced again + /// plays the track's later groups on its pad. + #[test] + fn a_reannounced_sibling_instance_plays_a_track_that_failed() { + let _pad_ids = pad_ids(); + let element = element(); + let recorder = Recorder::attach(&element); + let origin = origin(); + let _room = referencing(&origin, None); + // [`follow`], keeping the pads to wait on. + let pads = Pads::default(); + let session = { + let (shutdown, stopping) = watch::channel(false); + let session = Session { + max_delay: DEFAULT_MAX_DELAY, + element: element.downgrade(), + pads: pads.clone(), + shutdown: stopping, + }; + let origin = origin.consume(); + let task = + super::RUNTIME.spawn(async move { session.follow_path(&origin, "room", std::future::pending()).await }); + (shutdown, task) + }; + let mut follow = watch(&origin, "cam"); + + let mut first = Publisher::sibling(&origin, None); + let pad = recorder.written(&mut first, 1); + await_event(&mut follow, |event| { + matches!(event, moq_net::origin::RouteEvent::Start(_)) + }); + // The pad plays out the failure while the instance is still announced. + first.video.abort(moq_net::Error::Cancel); + super::RUNTIME.block_on(pads.settled(&(TrackKind::Video, "video".to_string()))); + first.route.as_ref().unwrap().withdraw(); + await_event(&mut follow, |event| { + matches!(event, moq_net::origin::RouteEvent::End(_)) + }); + + // The returning instance serves the track again, from a later group. + first.video = moq_mux::container::Producer::new( + first + .broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(), + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + first.route.as_ref().unwrap().announce().unwrap(); + assert_eq!( + recorder.written(&mut first, 2), + pad, + "the returning instance moved to another pad" + ); + stop(session); + recorder.assert_one_pad_held(); + } + /// A sibling the catalog stopped reading is let go: once the catalog closes, the run ends while /// both broadcasts stay up. #[test]