diff --git a/quest/m0/test-flakes-2/gst-sibling-reannounce.md b/quest/m0/test-flakes-2/gst-sibling-reannounce.md deleted file mode 100644 index 25d3f97b178..00000000000 --- a/quest/m0/test-flakes-2/gst-sibling-reannounce.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] A reannounced gst sibling never replays its finish under load - -## Goal - -`moq-gst`'s `source::imp::session_tests::a_reannounced_sibling_instance_does_not_replay_its_finish` -passes reliably under a loaded `just check --all`, fixed at its cause, and is -re-enabled: its `#[ignore]` and its `sh/gh/test-inventory.allow` entry go. - -## Plan - -[moq-dev/moq#8438](https://github.com/moq-dev/moq/pull/8438) disabled it after -one failure in a loaded `just check`: the finished track replayed on its pad -(2 starts instead of 1). It passed 3/3 alone. No single PR is known to have -caused it; find out whether the test races real time or `moqsrc` really can -replay a finished track when the sibling's re-announce wins the race against -its end, and fix whichever it is. If it is a product bug, the PR is a `fix`. - -Decided 2026-11-11 (maintainer, follow-up planning in /quest-complete): ✅ -plan it as a child of this m0 epic (rejected: leave the test ignored). diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index ac47d12a88d..630b5761424 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1813,89 +1813,113 @@ impl Play { let mut clip = OpusClip::new(&shape.caps, shape.warmup); let mut first = true; loop { - tokio::select! { - // The follower stopped this reader (the instance changed, the rendition was - // reshaped or delisted, or the run ended). An end or failure that arrived alongside - // is recorded as the arms below would, or the play queued behind this one would - // replay it. - _ = cancel.changed() => { - if let Some(end) = ended(&mut track) { - finished.insert(rendition, &shape.track, attached, end.is_err()); + // The track wins over the cancel, so an end that arrived alongside one is still recorded. + let read = { + let mut cancelled = std::pin::pin!(cancel.changed()); + kio::wait(|waiter| match track.poll_read(waiter) { + std::task::Poll::Ready(read) => std::task::Poll::Ready(Some(read)), + std::task::Poll::Pending => waiter.poll_future(cancelled.as_mut()).map(|_| None), + }) + .await + }; + // The follower stopped this reader (the instance changed, the rendition was reshaped or + // delisted, or the run ended). + let Some(read) = read else { + return; + }; + match read { + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, group })) => { + // A frame yields to a cancel, so a backlog never keeps a stopped reader playing. + if cancel.has_changed().unwrap_or(true) { + return; } - return; - } - frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} - Ok(Some(moq_mux::container::Read::Frame { frame, group })) => { - // Nothing precedes group 0, and a group delivers from its frame 0 or not at - // all, so a play that reads it first decodes from the encoder's first packet, - // which has no history to warm up from. - if std::mem::take(&mut first) - && group == 0 - && let Some(clip) = &mut clip - { - clip.track_start(); - } - let mut buffer = match build_buffer(frame, kind) { - Ok(buffer) => buffer, - Err(err) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid timestamp"), ["{err:#}"]); - } - return; - } - }; - if let Some(clip) = &mut clip - && let Err(err) = clip.apply(buffer.get_mut().unwrap()) - { + // Nothing precedes group 0, and a group delivers from its frame 0 or not at + // all, so a play that reads it first decodes from the encoder's first packet, + // which has no history to warm up from. + if std::mem::take(&mut first) + && group == 0 && let Some(clip) = &mut clip + { + clip.track_start(); + } + let mut buffer = match build_buffer(frame, kind) { + Ok(buffer) => buffer, + Err(err) => { if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid Opus packet"), ["{err:#}"]); + gst::element_error!( + obj, + gst::StreamError::Format, + ("track {name} has an invalid timestamp"), + ["{err:#}"] + ); } return; } - let segment = (!sent_segment).then(|| { - let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); - sent_segment = true; - gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) - }); - // pad.push() blocks until downstream accepts the buffer (full queues, a - // clock-synced sink). block_in_place hands our sibling tasks to another - // worker so a stalled downstream can't pin a runtime thread and starve - // the session loop or other readers. - let pushed = tokio::task::block_in_place(|| { - if let Some(segment) = segment { - pad.push_event(segment); - } - pad.push(buffer) - }); - match pushed { - Ok(_) => {} - Err(gst::FlowError::NotNegotiated) => { - if let Some(obj) = element.upgrade() { - gst::element_error!(obj, gst::StreamError::Format, ("track {name} was not negotiated"), ["caps {}", shape.caps]); - } - return; - } - Err(_) => return, + }; + if let Some(clip) = &mut clip + && let Err(err) = clip.apply(buffer.get_mut().unwrap()) + { + if let Some(obj) = element.upgrade() { + gst::element_error!( + obj, + gst::StreamError::Format, + ("track {name} has an invalid Opus packet"), + ["{err:#}"] + ); } - } - // The track stopping with its instance is that instance's end, which idles the - // pad for the next one rather than ending the rendition. - Ok(None) | Err(_) if instance.is_closed() => return, - // Recorded before the actor takes the next play, - // so a reshape on this instance does not replay it. - Ok(None) => { - finished.insert(rendition, &shape.track, attached, false); return; } - // The instance may still hand a new subscription the groups already played, - // such as a relay's copy cut off by its publisher's session closing. - Err(err) => { - gst::warning!(CAT, "track {name} failed: {err:?}"); - finished.insert(rendition, &shape.track, attached, true); - return; + let segment = (!sent_segment).then(|| { + let base = element + .upgrade() + .and_then(|obj| obj.current_running_time()) + .unwrap_or(gst::ClockTime::ZERO); + sent_segment = true; + gst::event::Segment::new(timeline.segment(buffer.pts().unwrap(), base)) + }); + // pad.push() blocks until downstream accepts the buffer (full queues, a + // clock-synced sink). block_in_place hands our sibling tasks to another + // worker so a stalled downstream can't pin a runtime thread and starve + // the session loop or other readers. + let pushed = tokio::task::block_in_place(|| { + if let Some(segment) = segment { + pad.push_event(segment); + } + pad.push(buffer) + }); + match pushed { + Ok(_) => {} + Err(gst::FlowError::NotNegotiated) => { + if let Some(obj) = element.upgrade() { + gst::element_error!( + obj, + gst::StreamError::Format, + ("track {name} was not negotiated"), + ["caps {}", shape.caps] + ); + } + return; + } + Err(_) => return, } } + // Recorded before the actor takes the next play, so a reshape on this instance + // does not replay it, even when the instance closed before the end was read: the + // same instance announced again would hand a new subscription its last group. + Ok(None) => { + finished.insert(rendition, &shape.track, attached, false); + return; + } + // The track failing with its instance is that instance's end, which idles the + // pad for the next one rather than ending the rendition. + Err(_) if instance.is_closed() => return, + // The instance may still hand a new subscription the groups already played, + // such as a relay's copy cut off by its publisher's session closing. + Err(err) => { + gst::warning!(CAT, "track {name} failed: {err:?}"); + finished.insert(rendition, &shape.track, attached, true); + return; + } } } } @@ -1969,24 +1993,6 @@ impl Timeline { } } -/// The track's end, or its failure, if that is the next media item, without waiting for it. -/// Markers ready before it are passed over; a frame or anything not ready yet means it has not -/// ended. -fn ended(track: &mut moq_mux::container::Consumer) -> Option> { - let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); - loop { - let mut read = std::pin::pin!(track.read()); - match read.as_mut().poll(&mut cx) { - std::task::Poll::Ready(Ok(None)) => return Some(Ok(())), - std::task::Poll::Ready(Ok(Some( - moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }, - ))) => continue, - std::task::Poll::Ready(Err(err)) => return Some(Err(err)), - std::task::Poll::Pending | std::task::Poll::Ready(Ok(Some(_))) => return None, - } - } -} - /// Start a new stream on the pad: a fresh stream-start and the rendition's caps. The segment /// waits for this pad's first buffer, then uses the run's shared mapping. fn begin(pad: &gst::Pad, name: &str, caps: &gst::Caps) { @@ -2234,47 +2240,10 @@ fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> R #[cfg(test)] mod tests { - use super::{Timeline, ended}; + use super::Timeline; use moq_net::Timestamp; use std::time::Duration; - /// A reader cancelled as its track ends or fails records it only when no media is left to read, - /// passing over the markers a publisher closes with. - #[test] - fn ended_sees_only_an_end_with_no_media_before_it() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast.create_track("video", hang::container::track_info(0)).unwrap(); - let legacy = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - let mut producer = moq_mux::container::Producer::new(track, legacy()); - let mut consumer = moq_mux::container::Consumer::new(producer.consume(), legacy()); - assert!(ended(&mut consumer).is_none(), "an open track has not ended"); - - producer - .write(moq_mux::container::Frame { - timestamp: Timestamp::ZERO, - payload: bytes::Bytes::from_static(&[0]), - keyframe: true, - duration: None, - }) - .unwrap(); - assert!(ended(&mut consumer).is_none(), "a frame was still unread"); - - producer.discontinuity().unwrap(); - let _finished = producer.finish().unwrap(); - assert!( - matches!(ended(&mut consumer), Some(Ok(()))), - "the closing marker hid the end" - ); - - let track = broadcast - .create_track("failed", hang::container::track_info(0)) - .unwrap(); - let producer = moq_mux::container::Producer::new(track, legacy()); - let mut consumer = moq_mux::container::Consumer::new(producer.consume(), legacy()); - producer.abort(moq_net::Error::Cancel); - assert!(matches!(ended(&mut consumer), Some(Err(_))), "a failure is an end too"); - } - #[test] fn video_caps_preserve_nal_framing() { gst::init().unwrap(); @@ -4703,7 +4672,6 @@ mod session_tests { /// The same sibling instance announced again resumes its state, whether or not its end reached /// the follower first: a track it finished does not replay on its pad. #[test] - #[ignore = "flaky: replayed a finished track once under host load; owned by /quest/m0/test-flakes-2/README.md"] fn a_reannounced_sibling_instance_does_not_replay_its_finish() { let _pad_ids = pad_ids(); let element = element(); @@ -4762,6 +4730,60 @@ mod session_tests { drop(first); } + /// A track's end read after its instance closed is still recorded, with or without a cancel + /// alongside it: the same instance announced again would replay its last group to a new + /// subscription. + #[test] + fn an_end_read_after_its_instance_closed_is_recorded() { + let element = element(); + for cancelled in [false, true] { + let broadcast = moq_net::broadcast::Info::new().produce(); + let legacy = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let producer = moq_mux::container::Producer::new( + broadcast.create_track("video", hang::container::track_info(0)).unwrap(), + legacy(), + ); + let track = moq_mux::container::Consumer::new(producer.consume(), legacy()); + let _finished = producer.finish().unwrap(); + let instance = broadcast.consume(); + broadcast.close(); + assert!(instance.is_closed()); + + let rendition = (TrackKind::Video, "video".to_string()); + let finished = super::Finished::default(); + let (cancelling, cancel) = watch::channel(false); + if cancelled { + cancelling.send_replace(true); + } + let (_shutdown, shutdown) = watch::channel(false); + let play = super::Play { + element: element.downgrade(), + shape: super::Shape { + caps: gst::Caps::new_empty_simple("video/x-h264"), + container: Container::Legacy, + track: "video".into(), + broadcast: None, + instance: None, + warmup: Duration::ZERO, + }, + track, + instance, + attached: super::Attached { serial: 1, start: 1 }, + timeline: Arc::default(), + finished: finished.clone(), + live: Arc::default(), + cancel, + shutdown, + }; + let pad = gst::Pad::builder(gst::PadDirection::Src).build(); + super::RUNTIME.block_on(play.stream(&pad, &rendition)); + assert!( + finished.contains(&rendition, "video", super::Attached { serial: 1, start: 2 }), + "the end was not recorded for the instance's next start (cancelled: {cancelled})" + ); + } + } + /// A track failing may be its instance going away before the instance reports closed, as a /// relay's copy does when its publisher's session closes, so the same instance announced again /// plays the track's later groups on its pad. diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index cb03f2626a8..cc1ed1710da 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -42,5 +42,3 @@ ignore moq-net::test_interop::lite_tracks_interop /quest/m2/cluster-routing/rout ignore moq-tokio::datagram_tracks::datagram_tracks_javascript /quest/m2/cluster-routing/routes/js.md recipe test::datagram-tracks /quest/m2/cluster-routing/routes/js.md -# Flaked under a loaded `just check`; disabled until fixed at its cause. -ignore moq-gst::source::imp::session_tests::a_reannounced_sibling_instance_does_not_replay_its_finish /quest/m0/test-flakes-2/README.md