From 9a084af322aa6e9043c973419e5128c08cbd1963 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:04:01 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m0/test-flakes-2/gst-sibling-reannounce.md Quest-Claim: 54d4cb257f29d19b From da4f9c616e720352eea7b8ae50c654bcd6345907 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:51:30 -0800 Subject: [PATCH 2/4] fix(gst): record a track's end even when its instance closed first A play that read its track's end after the instance closed returned without recording it, so the same instance announced again replayed the finished track's last group. Re-enables the test that caught it. Co-Authored-By: Claude Opus 5.5 --- .../test-flakes-2/gst-sibling-reannounce.md | 19 ------------------- rs/moq-gst/src/source/imp.rs | 12 ++++++------ sh/gh/test-inventory.allow | 1 - 3 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 quest/m0/test-flakes-2/gst-sibling-reannounce.md diff --git a/quest/m0/test-flakes-2/gst-sibling-reannounce.md b/quest/m0/test-flakes-2/gst-sibling-reannounce.md deleted file mode 100644 index 25d3f97b178..00000000000 --- a/quest/m0/test-flakes-2/gst-sibling-reannounce.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] A reannounced gst sibling never replays its finish under load - -## Goal - -`moq-gst`'s `source::imp::session_tests::a_reannounced_sibling_instance_does_not_replay_its_finish` -passes reliably under a loaded `just check --all`, fixed at its cause, and is -re-enabled: its `#[ignore]` and its `sh/gh/test-inventory.allow` entry go. - -## Plan - -[moq-dev/moq#8438](https://github.com/moq-dev/moq/pull/8438) disabled it after -one failure in a loaded `just check`: the finished track replayed on its pad -(2 starts instead of 1). It passed 3/3 alone. No single PR is known to have -caused it; find out whether the test races real time or `moqsrc` really can -replay a finished track when the sibling's re-announce wins the race against -its end, and fix whichever it is. If it is a product bug, the PR is a `fix`. - -Decided 2026-11-11 (maintainer, follow-up planning in /quest-complete): ✅ -plan it as a child of this m0 epic (rejected: leave the test ignored). diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index ac47d12a88d..3ffdc872eba 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1879,15 +1879,16 @@ impl Play { Err(_) => return, } } - // The track stopping with its instance is that instance's end, which idles the - // pad for the next one rather than ending the rendition. - Ok(None) | Err(_) if instance.is_closed() => return, - // Recorded before the actor takes the next play, - // so a reshape on this instance does not replay it. + // Recorded before the actor takes the next play, so a reshape on this instance + // does not replay it, even when the instance closed before the end was read: the + // same instance announced again would hand a new subscription its last group. Ok(None) => { finished.insert(rendition, &shape.track, attached, false); return; } + // The track failing with its instance is that instance's end, which idles the + // pad for the next one rather than ending the rendition. + Err(_) if instance.is_closed() => return, // The instance may still hand a new subscription the groups already played, // such as a relay's copy cut off by its publisher's session closing. Err(err) => { @@ -4703,7 +4704,6 @@ mod session_tests { /// The same sibling instance announced again resumes its state, whether or not its end reached /// the follower first: a track it finished does not replay on its pad. #[test] - #[ignore = "flaky: replayed a finished track once under host load; owned by /quest/m0/test-flakes-2/README.md"] fn a_reannounced_sibling_instance_does_not_replay_its_finish() { let _pad_ids = pad_ids(); let element = element(); diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index cb03f2626a8..436bbb7c0c2 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -43,4 +43,3 @@ ignore moq-tokio::datagram_tracks::datagram_tracks_javascript /quest/m2/cluster- recipe test::datagram-tracks /quest/m2/cluster-routing/routes/js.md # Flaked under a loaded `just check`; disabled until fixed at its cause. -ignore moq-gst::source::imp::session_tests::a_reannounced_sibling_instance_does_not_replay_its_finish /quest/m0/test-flakes-2/README.md From 0a3a0764e1619f66e2eedd84bc7e1c77a0a59f63 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 16:41:01 -0800 Subject: [PATCH 3/4] test(gst): hold the play's push so the reannounce test reaches a closed instance Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 26 +++++++++++++++++++++++++- sh/gh/test-inventory.allow | 1 - 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 3ffdc872eba..cc46e8d55bc 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -4707,6 +4707,25 @@ mod session_tests { fn a_reannounced_sibling_instance_does_not_replay_its_finish() { let _pad_ids = pad_ids(); let element = element(); + // Hold the play in its push of the first group until the instance has closed, so it may + // read the track's end only after the withdraw, the ordering a loaded host produces. The + // follower's cancel races that read, so a regression fails often rather than always. + let (entered, pushing) = std::sync::mpsc::channel(); + let (release, released) = std::sync::mpsc::channel::<()>(); + let gate = Mutex::new(Some((entered, released))); + element.connect_pad_added(move |_, pad| { + let Some((entered, released)) = gate.lock().unwrap().take() else { + return; + }; + let gate = Mutex::new(Some((entered, released))); + pad.add_probe(gst::PadProbeType::BUFFER, move |_, _| { + if let Some((entered, released)) = gate.lock().unwrap().take() { + entered.send(()).unwrap(); + let _ = released.recv(); + } + gst::PadProbeReturn::Ok + }); + }); let recorder = Recorder::attach(&element); let origin = origin(); let _room = referencing(&origin, None); @@ -4714,7 +4733,10 @@ mod session_tests { let mut follow = watch(&origin, "cam"); let mut first = Publisher::sibling(&origin, None); - let pad = recorder.written(&mut first, 1); + first.write(1); + pushing + .recv_timeout(Duration::from_secs(10)) + .expect("1 never reached a pad"); first.finish_video(); await_finished(&first); await_event(&mut follow, |event| { @@ -4736,6 +4758,8 @@ mod session_tests { } ) }); + release.send(()).unwrap(); + let pad = recorder.wait("1 never reached a pad", |seen| reached(seen, 1).into_iter().next()); first.route.as_ref().unwrap().announce().unwrap(); await_event(&mut follow, |event| { matches!( diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 436bbb7c0c2..cc1ed1710da 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -42,4 +42,3 @@ ignore moq-net::test_interop::lite_tracks_interop /quest/m2/cluster-routing/rout ignore moq-tokio::datagram_tracks::datagram_tracks_javascript /quest/m2/cluster-routing/routes/js.md recipe test::datagram-tracks /quest/m2/cluster-routing/routes/js.md -# Flaked under a loaded `just check`; disabled until fixed at its cause. From c03d92c201c90fdbe36f956465257ad9b7f4e0db Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 17:18:10 -0800 Subject: [PATCH 4/4] fix(gst): poll the play's track before its cancel, and test the end record directly The play loop's select! picked at random between the cancel and the read, so which arm saw a track's end was chance. Poll the track first, and let a frame yield to a cancel, so an end or failure is always handled by the read arm. This also drops the ended() drain the cancel arm needed. The reannounce session test cannot fail on every run with the fix reverted: the follower may fold the re-announce and the next instance into one hand-out. A direct Play::stream test with a closed instance and a finished track covers the end record, with and without a cancel alongside. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 308 +++++++++++++++++------------------ 1 file changed, 153 insertions(+), 155 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index cc46e8d55bc..630b5761424 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1813,90 +1813,113 @@ impl Play { let mut clip = OpusClip::new(&shape.caps, shape.warmup); let mut first = true; loop { - tokio::select! { - // The follower stopped this reader (the instance changed, the rendition was - // reshaped or delisted, or the run ended). An end or failure that arrived alongside - // is recorded as the arms below would, or the play queued behind this one would - // replay it. - _ = cancel.changed() => { - if let Some(end) = ended(&mut track) { - finished.insert(rendition, &shape.track, attached, end.is_err()); + // The track wins over the cancel, so an end that arrived alongside one is still recorded. + let read = { + let mut cancelled = std::pin::pin!(cancel.changed()); + kio::wait(|waiter| match track.poll_read(waiter) { + std::task::Poll::Ready(read) => std::task::Poll::Ready(Some(read)), + std::task::Poll::Pending => waiter.poll_future(cancelled.as_mut()).map(|_| None), + }) + .await + }; + // The follower stopped this reader (the instance changed, the rendition was reshaped or + // delisted, or the run ended). + let Some(read) = read else { + return; + }; + match read { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, group })) => { + // A frame yields to a cancel, so a backlog never keeps a stopped reader playing. + if cancel.has_changed().unwrap_or(true) { + return; } - return; - } - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, group })) => { - // Nothing precedes group 0, and a group delivers from its frame 0 or not at - // all, so a play that reads it first decodes from the encoder's first packet, - // which has no history to warm up from. - if std::mem::take(&mut first) - && group == 0 - && let Some(clip) = &mut clip - { - clip.track_start(); - } - let mut buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return; - } - }; - if let Some(clip) = &mut clip - && let Err(err) = clip.apply(buffer.get_mut().unwrap()) - { + // Nothing precedes group 0, and a group delivers from its frame 0 or not at + // all, so a play that reads it first decodes from the encoder's first packet, + // which has no history to warm up from. + if std::mem::take(&mut first) + && group == 0 && let Some(clip) = &mut clip + { + clip.track_start(); + } + let mut buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid Opus packet"), ["{err:#}"]); + gst::element_error!( + obj, + gst::StreamError::Format, + ("track {name} has an invalid timestamp"), + ["{err:#}"] + ); } return; } - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other readers. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {}", shape.caps]); - } - return; - } - Err(_) => return, + }; + if let Some(clip) = &mut clip + && let Err(err) = clip.apply(buffer.get_mut().unwrap()) + { + if let Some(obj) = element.upgrade() { + gst::element_error!( + obj, + gst::StreamError::Format, + ("track {name} has an invalid Opus packet"), + ["{err:#}"] + ); } - } - // Recorded before the actor takes the next play, so a reshape on this instance - // does not replay it, even when the instance closed before the end was read: the - // same instance announced again would hand a new subscription its last group. - Ok(None) => { - finished.insert(rendition, &shape.track, attached, false); return; } - // The track failing with its instance is that instance's end, which idles the - // pad for the next one rather than ending the rendition. - Err(_) if instance.is_closed() => return, - // The instance may still hand a new subscription the groups already played, - // such as a relay's copy cut off by its publisher's session closing. - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - finished.insert(rendition, &shape.track, attached, true); - return; + let segment = (!sent_segment).then(|| { + let base = element + .upgrade() + .and_then(|obj| obj.current_running_time()) + .unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other readers. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!( + obj, + gst::StreamError::Format, + ("track {name} was not negotiated"), + ["caps {}", shape.caps] + ); + } + return; + } + Err(_) => return, } } + // Recorded before the actor takes the next play, so a reshape on this instance + // does not replay it, even when the instance closed before the end was read: the + // same instance announced again would hand a new subscription its last group. + Ok(None) => { + finished.insert(rendition, &shape.track, attached, false); + return; + } + // The track failing with its instance is that instance's end, which idles the + // pad for the next one rather than ending the rendition. + Err(_) if instance.is_closed() => return, + // The instance may still hand a new subscription the groups already played, + // such as a relay's copy cut off by its publisher's session closing. + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + finished.insert(rendition, &shape.track, attached, true); + return; + } } } } @@ -1970,24 +1993,6 @@ impl Timeline { } } -/// The track's end, or its failure, if that is the next media item, without waiting for it. -/// Markers ready before it are passed over; a frame or anything not ready yet means it has not -/// ended. -fn ended(track: &mut moq_mux::container::Consumer) -> Option> { - let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); - loop { - let mut read = std::pin::pin!(track.read()); - match read.as_mut().poll(&mut cx) { - std::task::Poll::Ready(Ok(None)) => return Some(Ok(())), - std::task::Poll::Ready(Ok(Some( - moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }, - ))) => continue, - std::task::Poll::Ready(Err(err)) => return Some(Err(err)), - std::task::Poll::Pending | std::task::Poll::Ready(Ok(Some(_))) => return None, - } - } -} - /// Start a new stream on the pad: a fresh stream-start and the rendition's caps. The segment /// waits for this pad's first buffer, then uses the run's shared mapping. fn begin(pad: &gst::Pad, name: &str, caps: &gst::Caps) { @@ -2235,47 +2240,10 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{Timeline, ended}; + use super::Timeline; use moq_net::Timestamp; use std::time::Duration; - /// A reader cancelled as its track ends or fails records it only when no media is left to read, - /// passing over the markers a publisher closes with. - #[test] - fn ended_sees_only_an_end_with_no_media_before_it() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast.create_track("video", hang::container::track_info(0)).unwrap(); - let legacy = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - let mut producer = moq_mux::container::Producer::new(track, legacy()); - let mut consumer = moq_mux::container::Consumer::new(producer.consume(), legacy()); - assert!(ended(&mut consumer).is_none(), "an open track has not ended"); - - producer - .write(moq_mux::container::Frame { - timestamp: Timestamp::ZERO, - payload: bytes::Bytes::from_static(&[0]), - keyframe: true, - duration: None, - }) - .unwrap(); - assert!(ended(&mut consumer).is_none(), "a frame was still unread"); - - producer.discontinuity().unwrap(); - let _finished = producer.finish().unwrap(); - assert!( - matches!(ended(&mut consumer), Some(Ok(()))), - "the closing marker hid the end" - ); - - let track = broadcast - .create_track("failed", hang::container::track_info(0)) - .unwrap(); - let producer = moq_mux::container::Producer::new(track, legacy()); - let mut consumer = moq_mux::container::Consumer::new(producer.consume(), legacy()); - producer.abort(moq_net::Error::Cancel); - assert!(matches!(ended(&mut consumer), Some(Err(_))), "a failure is an end too"); - } - #[test] fn video_caps_preserve_nal_framing() { gst::init().unwrap(); @@ -4707,25 +4675,6 @@ mod session_tests { fn a_reannounced_sibling_instance_does_not_replay_its_finish() { let _pad_ids = pad_ids(); let element = element(); - // Hold the play in its push of the first group until the instance has closed, so it may - // read the track's end only after the withdraw, the ordering a loaded host produces. The - // follower's cancel races that read, so a regression fails often rather than always. - let (entered, pushing) = std::sync::mpsc::channel(); - let (release, released) = std::sync::mpsc::channel::<()>(); - let gate = Mutex::new(Some((entered, released))); - element.connect_pad_added(move |_, pad| { - let Some((entered, released)) = gate.lock().unwrap().take() else { - return; - }; - let gate = Mutex::new(Some((entered, released))); - pad.add_probe(gst::PadProbeType::BUFFER, move |_, _| { - if let Some((entered, released)) = gate.lock().unwrap().take() { - entered.send(()).unwrap(); - let _ = released.recv(); - } - gst::PadProbeReturn::Ok - }); - }); let recorder = Recorder::attach(&element); let origin = origin(); let _room = referencing(&origin, None); @@ -4733,10 +4682,7 @@ mod session_tests { let mut follow = watch(&origin, "cam"); let mut first = Publisher::sibling(&origin, None); - first.write(1); - pushing - .recv_timeout(Duration::from_secs(10)) - .expect("1 never reached a pad"); + let pad = recorder.written(&mut first, 1); first.finish_video(); await_finished(&first); await_event(&mut follow, |event| { @@ -4758,8 +4704,6 @@ mod session_tests { } ) }); - release.send(()).unwrap(); - let pad = recorder.wait("1 never reached a pad", |seen| reached(seen, 1).into_iter().next()); first.route.as_ref().unwrap().announce().unwrap(); await_event(&mut follow, |event| { matches!( @@ -4786,6 +4730,60 @@ mod session_tests { drop(first); } + /// A track's end read after its instance closed is still recorded, with or without a cancel + /// alongside it: the same instance announced again would replay its last group to a new + /// subscription. + #[test] + fn an_end_read_after_its_instance_closed_is_recorded() { + let element = element(); + for cancelled in [false, true] { + let broadcast = moq_net::broadcast::Info::new().produce(); + let legacy = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let producer = moq_mux::container::Producer::new( + broadcast.create_track("video", hang::container::track_info(0)).unwrap(), + legacy(), + ); + let track = moq_mux::container::Consumer::new(producer.consume(), legacy()); + let _finished = producer.finish().unwrap(); + let instance = broadcast.consume(); + broadcast.close(); + assert!(instance.is_closed()); + + let rendition = (TrackKind::Video, "video".to_string()); + let finished = super::Finished::default(); + let (cancelling, cancel) = watch::channel(false); + if cancelled { + cancelling.send_replace(true); + } + let (_shutdown, shutdown) = watch::channel(false); + let play = super::Play { + element: element.downgrade(), + shape: super::Shape { + caps: gst::Caps::new_empty_simple("video/x-h264"), + container: Container::Legacy, + track: "video".into(), + broadcast: None, + instance: None, + warmup: Duration::ZERO, + }, + track, + instance, + attached: super::Attached { serial: 1, start: 1 }, + timeline: Arc::default(), + finished: finished.clone(), + live: Arc::default(), + cancel, + shutdown, + }; + let pad = gst::Pad::builder(gst::PadDirection::Src).build(); + super::RUNTIME.block_on(play.stream(&pad, &rendition)); + assert!( + finished.contains(&rendition, "video", super::Attached { serial: 1, start: 2 }), + "the end was not recorded for the instance's next start (cancelled: {cancelled})" + ); + } + } + /// A track failing may be its instance going away before the instance reports closed, as a /// relay's copy does when its publisher's session closes, so the same instance announced again /// plays the track's later groups on its pad.