From ef094265ae6f696999bdd75ada7ce83c337f3635 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 10:25:49 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m0/test-flakes-3/webrtc-reoffer-load.md Quest-Claim: 1a6d45b26bf073fa From 07ff0af0710bbe24346e049bd2d63f132c4cf5f0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 11:56:22 -0800 Subject: [PATCH 2/2] test(webrtc): a re-offer test keeps its source live until each viewer plays Co-Authored-By: Claude Sonnet 5.5 --- quest/m0/test-flakes-3/webrtc-live-source.md | 11 +++++ quest/m0/test-flakes-3/webrtc-reoffer-load.md | 12 ------ quest/m3/webrtc/reoffer-dtls-flight.md | 15 +++++++ rs/moq-webrtc/src/lib.rs | 42 +++++++++++-------- 4 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 quest/m0/test-flakes-3/webrtc-live-source.md delete mode 100644 quest/m0/test-flakes-3/webrtc-reoffer-load.md create mode 100644 quest/m3/webrtc/reoffer-dtls-flight.md diff --git a/quest/m0/test-flakes-3/webrtc-live-source.md b/quest/m0/test-flakes-3/webrtc-live-source.md new file mode 100644 index 00000000000..e0af30be7b3 --- /dev/null +++ b/quest/m0/test-flakes-3/webrtc-live-source.md @@ -0,0 +1,11 @@ +# [S] The WHIP and WHEP round trips play under load + +## Goal + +The `moq-webrtc` tests that wait on a packet the source pushed once before its viewer connected pass however loaded the host is: `a_whep_viewer_ends_with_its_broadcast`, `a_whep_viewer_joining_after_the_catalog_ends_plays_the_rest`, `a_whep_viewer_stays_on_its_instance_until_the_route_dies`, `a_whip_push_stays_on_its_instance_until_the_route_dies`, and `whip_and_whep_round_trip_opus`. + +## Plan + +Found while fixing [a_reoffer_from_the_same_address_plays](https://github.com/moq-dev/moq/pull/9241), which failed for the same cause. Run sixteen copies of one of these at once inside `systemd-run --user --scope -p CPUQuota=100%` and every copy times out at 10 s waiting for its first packet. None has flaked in a real `just check` yet, only a_reoffer did. + +The cause: `Bridge::push` cuts a group per packet, so a packet's group has an (empty) successor at once and goes stale 100 ms later (`egress::MAX_DELAY`). A session that connects past that budget under load reads nothing, and the test's source never pushes again. `a_reoffer_from_the_same_address_plays` now keeps a live source sending (`played_live`) until its viewer plays. Do the same here, sharing the helper rather than copying it: the sources that also need an exact payload (`OLD_PACKET` after a take-over) push it as their live packet instead of a one-shot. diff --git a/quest/m0/test-flakes-3/webrtc-reoffer-load.md b/quest/m0/test-flakes-3/webrtc-reoffer-load.md deleted file mode 100644 index 2b46389691e..00000000000 --- a/quest/m0/test-flakes-3/webrtc-reoffer-load.md +++ /dev/null @@ -1,12 +0,0 @@ -# [S] A WHEP re-offer plays under load - -## Goal - -`moq-webrtc`'s `tests::a_reoffer_from_the_same_address_plays` passes however loaded the host is. - -## Plan - -It failed once in a loaded `just check` on 2026-11-19 (while building [moq-dev/moq#9222](https://github.com/moq-dev/moq/pull/9222), which touches no WebRTC code): `received` hit its 10s `TIMEOUT` ("the viewer neither received nor closed", `rs/moq-webrtc/src/lib.rs:300`) after 10.18s, and it passed alone in 0.9s. -The panic does not say which `received` call timed out: the first viewer's packet, or the re-offered session's loop that pushes a packet every 20ms. -Reproduce first under CPU pressure like the failing run's: a one-core scope alone starves only the test's own threads, so also keep the other cores busy, such as `cargo nextest run -p moq-webrtc a_reoffer --stress-count 50` inside `systemd-run --user --scope -p CPUQuota=100%` while a workspace build runs beside it, then find which wait stalls and why: ICE over real sockets racing the 20ms push loop, or the new session joining past the delay budget so every pushed packet ages out before it reads. -Keep the epic's rules: no longer timeout, no retry. diff --git a/quest/m3/webrtc/reoffer-dtls-flight.md b/quest/m3/webrtc/reoffer-dtls-flight.md new file mode 100644 index 00000000000..c145128b5c3 --- /dev/null +++ b/quest/m3/webrtc/reoffer-dtls-flight.md @@ -0,0 +1,15 @@ +# [S] A re-offered session's first DTLS flight reaches it + +## Goal + +A viewer that re-offers from the address its abandoned session still holds completes DTLS without waiting out a DTLS retransmit (about 0.9 s), because its first flight reaches the new session. + +## Plan + +Found while fixing [a_reoffer_from_the_same_address_plays](https://github.com/moq-dev/moq/pull/9241), which takes ~1 s of its 1.1 s in this wait on an idle host. In `server/mux.rs`, a datagram that is not STUN goes to the address's holder alone (`Datagram::Other`). The re-offering peer starts DTLS the moment its ICE completes, and its ClientHello can reach the mux 40 us before the new session's ICE connects and `Route::confirm` moves the address, so the old session receives and drops it. Only dimpl's first retransmit recovers. + +Measured with a debug print in `Registry::route` and `Route::confirm`: ClientHello routed to the old holder with `claims.contains_key(src)`, then `confirm` 43 us later. + +Options: route `Datagram::Other` to the claimant as well as the holder, as `Datagram::Answer` already does (check that str0m drops DTLS from a pair ICE has not verified, so a forged source gains nothing), or hold the claimant's own first flight until it confirms. Prefer the first unless the second is simpler once the code is read. Test with a re-offer whose first flight is sent before the confirm, asserting the new session plays well under one retransmit interval on a paused or manual clock if the sockets allow it, otherwise by a lower bound on the flight count. + +Public API: none. Wire: none. diff --git a/rs/moq-webrtc/src/lib.rs b/rs/moq-webrtc/src/lib.rs index b280725fd1c..f4e6577aadd 100644 --- a/rs/moq-webrtc/src/lib.rs +++ b/rs/moq-webrtc/src/lib.rs @@ -408,41 +408,47 @@ mod tests { http.abort(); } + /// The next payload `viewer` receives while `bridge` keeps sending from `at`, as a live + /// source does. Each packet's group ends whole where the packet does, so a paused source's + /// last one ages out before a session that connects past its delay budget reads it, which + /// a loaded host does to the first packet as readily as the second. + async fn played_live( + bridge: &mut dyn Bridge, + at: &mut u64, + viewer: &mut tokio::sync::mpsc::UnboundedReceiver>, + ) -> Vec { + let read = received(viewer); + tokio::pin!(read); + loop { + push(bridge, *at, NEW_PACKET); + *at += 20_000; + tokio::select! { + played = &mut read => return played.expect("the viewer was closed"), + _ = kio::clock::sleep(Duration::from_millis(20)) => {} + } + } + } + /// A viewer re-offering from the address its abandoned session still holds plays on its /// new session: the new session's ICE confirms the path, so the address moves to it. #[tokio::test] async fn a_reoffer_from_the_same_address_plays() { let origin = moq_tokio::origin::spawn(); let (_route, mut bridge, _catalog) = publish_opus(&origin, moq_net::Instance::random()); - push(&mut *bridge, 20_000, OPUS_PACKET); + let mut at = 20_000; let (url, http, mut sessions) = serve_whep(&origin).await; let udp = std::sync::Arc::new(tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap()); let (old, mut viewer) = raw_view_on(url.clone(), udp.clone()).await; let _old_session = sessions.recv().await.unwrap(); - assert_eq!(received(&mut viewer).await.as_deref(), Some(OPUS_PACKET)); + assert_eq!(played_live(&mut *bridge, &mut at, &mut viewer).await, NEW_PACKET); // The peer drops its connection without a word, then offers again from the same address. old.abort(); let _ = old.await; let (_new, mut viewer) = raw_view_on(url, udp).await; let _new_session = sessions.recv().await.unwrap(); - // The source keeps sending, as a live one does: each packet's group ends whole where the - // packet does, so a paused source's last one ages out before a session arriving past its - // delay budget reads it. - let read = received(&mut viewer); - tokio::pin!(read); - let mut at = 40_000; - let played = loop { - push(&mut *bridge, at, NEW_PACKET); - at += 20_000; - tokio::select! { - played = &mut read => break played, - _ = kio::clock::sleep(Duration::from_millis(20)) => {} - } - }; - let played = played.expect("the new session never played"); - assert!([OPUS_PACKET, NEW_PACKET].contains(&played.as_slice()), "{played:?}"); + assert_eq!(played_live(&mut *bridge, &mut at, &mut viewer).await, NEW_PACKET); http.abort(); }