From eb7cca22f7643897d5e1276265dcfc5b1f4575ab Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 15:03:38 -0700 Subject: [PATCH 01/11] chore(quest): claim quest/m0/nightly/browser-baseline.md Quest-Claim: f38699da0b8c8798 From 658a14a297114cd4bbaad1c8a3f1b06e2db9d145 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 15:18:28 -0700 Subject: [PATCH 02/11] fix(js): stamp a timed track's objects under INCLUDE_PROPERTIES=0 A Rust relay fills a cache miss with a draft-22 FETCH that sends INCLUDE_PROPERTIES=0, since it already learned the track from SUBSCRIBE_OK. The JS publisher read that as opting the track out of timestamps and sent untimed objects on a timed track, which the relay rejects as a malformed track and aborts for every subscriber. That is the browser backend baseline's IETF failure: downstream it surfaces as PUBLISH_DONE INTERNAL_ERROR, and through moq-ffi as MoqError.Internal. INCLUDE_PROPERTIES=0 only empties the answer's Track Properties. A timed track stamps every object, as the Rust publisher already does and draft-lcurley-moq-timestamp requires. Co-Authored-By: Claude Opus 5.5 --- js/net/src/ietf/fetch.test.ts | 36 +++++++++++++++++++++++++++---- js/net/src/ietf/publisher.test.ts | 17 ++++++--------- js/net/src/ietf/publisher.ts | 21 +++++++++--------- 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/js/net/src/ietf/fetch.test.ts b/js/net/src/ietf/fetch.test.ts index 49483a43678..3c55e3ad5f5 100644 --- a/js/net/src/ietf/fetch.test.ts +++ b/js/net/src/ietf/fetch.test.ts @@ -6,12 +6,12 @@ import { error } from "../error.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; -import { Reader, Stream, Writer } from "../stream.ts"; +import { Cursor, Reader, Stream, Writer } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import type { Order, Producer as TrackProducer } from "../track.ts"; import { Fetch, FetchHeader, FetchOk, type FetchType, type Location } from "./fetch.ts"; import type * as Filter from "./filter.ts"; -import { Group as GroupMessage } from "./object.ts"; +import { decodeObjectTime, Group as GroupMessage } from "./object.ts"; import { Publisher } from "./publisher.ts"; import { RequestError } from "./request.ts"; import { NativeSession } from "./session.ts"; @@ -217,7 +217,12 @@ async function answer(fx: Served, wire: Reader, order: Order): Promise { * Decode a fetch stream's objects as `[group, object, payload]`, its groups in `order`, and * each End of Range as `[group, object, "end "]`. */ -async function readObjects(reader: Reader, order: Order): Promise<[number, number, string][]> { +async function readObjects( + reader: Reader, + order: Order, + /** Each object's Timestamp in milliseconds, or undefined when it carries none. */ + stamps?: (number | undefined)[], +): Promise<[number, number, string][]> { const objects: [number, number, string][] = []; let group = 0; let object = 0; @@ -236,7 +241,11 @@ async function readObjects(reader: Reader, order: Order): Promise<[number, numbe } object = flags & 0x04 ? await reader.u53() : object + 1; if (flags & 0x10) await reader.u8(); - if (flags & 0x20) await reader.read(await reader.u53()); + const properties = flags & 0x20 ? await reader.read(await reader.u53()) : undefined; + stamps?.push( + properties && + decodeObjectTime(new Cursor(properties, reader.version), Timescale.MILLI)?.as(Timescale.MILLI), + ); const payload = new TextDecoder().decode(await reader.read(await reader.u53())); objects.push([group, object, payload]); } @@ -320,6 +329,25 @@ test("a FETCH stops at its end object", async () => { } }); +// INCLUDE_PROPERTIES=0 leaves FETCH_OK's properties empty, since a relay filling its cache +// already knows the track. The objects still carry their Timestamps: a Rust relay reads a timed +// object without one as a malformed track and aborts it for every subscriber. +test("draft-22: an INCLUDE_PROPERTIES=0 FETCH still stamps a timed track", async () => { + using fx = serve(D22); + publishPairs(fx.track, 3); + const msg = new Fetch({ requestId: FETCH_ID, type: range(D22, at(1, 0), at(1, 0)), propertiesWanted: false }); + const answer = served(await fetch(fx, msg)); + expect(answer.ok.properties).toEqual({}); + expect(answer.objects).toEqual(pairs([1])); + + const reader = new Reader(undefined, answer.raw, D22); + expect(await reader.u53()).toBe(FetchHeader.type); + await FetchHeader.decode(reader, D22); + const stamps: (number | undefined)[] = []; + await readObjects(reader, "ascending", stamps); + expect(stamps).toEqual([0, 1]); +}); + // The groups that do not exist are left out, and the rest go out in the order asked for. test("a FETCH serves several groups in order", async () => { for (const version of VERSIONS) { diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index 690ed2a3c03..b7bc141799c 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -2951,8 +2951,9 @@ test("LARGEST_OBJECT is the live edge", async () => { }); /** - * INCLUDE_PROPERTIES=0 opts the response out of Track Properties, which also opts the track - * out of timestamps: with no declared Timescale there are no units to read one in. + * INCLUDE_PROPERTIES=0 opts the response out of Track Properties, since the peer already knows + * them. A timed track still stamps every object: a receiver reads a timed object without a + * Timestamp as malformed, which is how a Rust relay's FETCH lost a JS publisher's track. */ test("draft-22: an opt-out peer gets no track properties", async () => { for (const propertiesWanted of [true, false]) { @@ -2973,22 +2974,18 @@ test("draft-22: an opt-out peer gets no track properties", async () => { expect(ok.properties.timescale !== undefined).toBe(propertiesWanted); expect(ok.properties.groupOrder !== undefined).toBe(propertiesWanted); - // With no TIMESCALE declared there are no units to read a timestamp in, so the objects - // must not carry one either: a bare value invites a peer to read it as some default. writeGroup(track, 1); const served = await nextUni(fx.uni); if (!served) throw new Error("the group was never served"); const reader = new Reader(served, undefined, V22); const header = await GroupMessage.decode(reader, V22); - expect(header.flags.hasExtensions).toBe(propertiesWanted); + expect(header.flags.hasExtensions).toBe(true); await reader.u53(); // object id delta - if (propertiesWanted) { - const length = await reader.u53(); - expect(length).toBeGreaterThan(0); // the properties block, carrying the timestamp - await reader.read(length); - } + const length = await reader.u53(); + expect(length).toBeGreaterThan(0); // the properties block, carrying the timestamp + await reader.read(length); expect(await reader.read(await reader.u53())).toEqual(new TextEncoder().encode("0.0")); fx.close(); diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index 77cd37cccc0..92484499948 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -232,10 +232,10 @@ interface RunGroup { * The units each object's presentation timestamp is written in, or undefined when objects * carry none. * - * Undefined when SUBSCRIBE_OK carries no TIMESCALE: the track has no timeline, the - * subscriber sent INCLUDE_PROPERTIES=0. A - * timestamp whose units were never declared is worse than none. Our own reader discards - * it; another may read it as some default and time the media wrong. + * Undefined only for an untimed track, which has no timeline. A timed track stamps every + * object even for a subscriber that sent INCLUDE_PROPERTIES=0: that says it already knows + * the track's properties, not that the track lost its timeline, and a receiver treats a + * timed object without a Timestamp as malformed (draft-lcurley-moq-timestamp). */ timescale?: Timescale; @@ -743,8 +743,8 @@ export class Publisher { let cache: TrackSubscriber | undefined; let ranking: Priority | undefined; try { - // A Timestamp goes out only when this SUBSCRIBE_OK actually carries TIMESCALE. - const timescale = msg.propertiesWanted ? (info.timescale ?? undefined) : undefined; + // A timed track stamps every object, INCLUDE_PROPERTIES=0 or not; see RunGroup.timescale. + const timescale = info.timescale ?? undefined; // The model ranks higher-first, the IETF wire lower-first. Every group this // subscription serves carries the same publisher priority, which is what lets a // relay prefer catalog and audio over video when it has no subscriber preference @@ -3163,8 +3163,9 @@ export class Publisher { groups, last, ok: new FetchOk({ endOfTrack, endLocation, properties: trackProperties(info, msg.propertiesWanted) }), - // A Timestamp goes out only when FETCH_OK actually carries TIMESCALE. - timescale: msg.propertiesWanted ? (info.timescale ?? undefined) : undefined, + // Stamped even under INCLUDE_PROPERTIES=0, which a relay sends for a track it + // already knows; see RunGroup.timescale. + timescale: info.timescale ?? undefined, // Its own rank, so it competes with group streams by its priority alone. sendOrder: sendOrder({ priority }), order: msg.groupOrder ?? "ascending", @@ -3726,8 +3727,8 @@ function cachedSequences(cache: TrackSubscriber, groups: Groups, priority: numbe /** The Track Properties a SUBSCRIBE_OK, TRACK_STATUS, or FETCH_OK answers with. */ function trackProperties(info: Info, wanted: boolean): Properties.Properties { - // INCLUDE_PROPERTIES=0. The block stays present but empty, which also means the track opts - // out of timestamps for this requester. + // INCLUDE_PROPERTIES=0: the requester already knows them, so the block stays present but + // empty. A timed track's objects still carry their timestamps. if (!wanted) return {}; // TIMESCALE is what opts the track into timestamps, on drafts that can send it. return { From 9e63743d4c142c9c9e2fa6d70b4502020b183999 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 16:40:49 -0700 Subject: [PATCH 03/11] fix(net): hold a relay's cache-miss FETCH behind its own fill A relay joining a moq-transport-22 track mid-group subscribes with a fill for the head of that group. A downstream join that misses the same head also queued a cache-miss FETCH for it. When the FETCH's objects landed first, they cached the group as a fetched one, and the live tail stream that ends the group was refused as a duplicate. Live readers then waited on that group until they timed out, which the browser backend baseline hit intermittently on its IETF relay. The miss now waits until the subscription's fill has headed its group or settled without it. The cache then answers it, and a fill that failed still leaves the miss to a FETCH. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 25331e67790..2f1da3cb1b3 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -3686,7 +3686,16 @@ impl Subscriber { requested, timescale, ); - group_fetches.push(fetch); + // A join asks for the head of the group it lands in, which this + // subscription's own fill is already bringing. A FETCH racing it can + // cache that group first, and then the tail stream, which is what + // ends the group for live readers, is refused as a duplicate. So the + // miss waits for the head, and the cache answers it once that lands. + let heading = fill.clone(); + group_fetches.push(async move { + kio::wait(|waiter| poll_headed(&heading, waiter)).await; + fetch.await + }); } End::Idle => { tracing::debug!(broadcast = %crate::path::Rooted(self.origin.root(), broadcast_path), track = %track_name, "subscribe cancelled (idle)"); @@ -15146,6 +15155,35 @@ mod joining_fetch_tests { } } + /// A reader missing the head of the group the subscription joined waits for the + /// subscription's own fill, which is bringing that head. A cache-miss FETCH racing it + /// could cache the group first, refusing the tail that ends it for live readers. Only a + /// fill that settled without the head leaves the miss to a FETCH. + #[moq_clock::test] + async fn a_miss_waits_for_the_subscriptions_own_fill() { + let version = Version::Draft22; + let largest = Some(ietf::Location { group: 4, object: 1 }); + let run = JoinRun::start(version, None, subscribe_ok(version, largest), fetch_ok(version)).await; + let fetches = || { + decode_messages(&run.session.log, version) + .iter() + .filter(|(id, _)| *id == ietf::Fetch::ID) + .count() + }; + assert!(run.fill_outstanding(), "the subscription asked for a fill"); + + let mut fetch = Box::pin(run._hold.1.fetch_group(4, None)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + settle().await; + assert_eq!(fetches(), 0, "the miss waits for the subscription's fill"); + + // The fill fails, so the head is the miss's to fetch after all. + let fill = run.subscriber.state.lock().subscribes.values().next().unwrap().fill.clone(); + *fill.write().ok().unwrap() = Fill::Done; + settle().await; + assert_eq!(fetches(), 1, "a settled fill leaves the miss to a FETCH"); + } + /// A malformed answer to the joining FETCH is not a refusal: the peer broke the /// protocol, so the session closes. #[moq_clock::test] From b30bbd187cf3b8eb3b68ee274c2ef502e2c80696 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 17:20:59 -0700 Subject: [PATCH 04/11] style(net): format the fill-wait test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 2f1da3cb1b3..fcee15d34ae 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -15178,7 +15178,16 @@ mod joining_fetch_tests { assert_eq!(fetches(), 0, "the miss waits for the subscription's fill"); // The fill fails, so the head is the miss's to fetch after all. - let fill = run.subscriber.state.lock().subscribes.values().next().unwrap().fill.clone(); + let fill = run + .subscriber + .state + .lock() + .subscribes + .values() + .next() + .unwrap() + .fill + .clone(); *fill.write().ok().unwrap() = Fill::Done; settle().await; assert_eq!(fetches(), 1, "a settled fill leaves the miss to a FETCH"); From 9daaeae57f8c9525cd408a0aa1eeefd668a2dd46 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 17:25:48 -0700 Subject: [PATCH 05/11] chore(quest): finish quest/m0/nightly/browser-baseline.md Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/README.md | 1 - quest/m0/nightly/browser-baseline.md | 20 -------------------- 2 files changed, 21 deletions(-) delete mode 100644 quest/m0/nightly/browser-baseline.md diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 22bfd31dfb1..18ff3d12ce3 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -35,7 +35,6 @@ Public API: none. Wire: none. - [The software renderer tests both capability paths](/quest/m0/nightly/render-capabilities.md) - the software-renderer lane verifies output selection with and without DMA-BUF import support without requiring hardware the runner lacks - [Shipping size history admits the generated C package](/quest/m0/nightly/shipping-history.md) - nightly measures the generated C shipping archives while preserving comparable existing size history and refusing missing or renamed artifacts - [The browser publisher completes its fairness sweep](/quest/m0/nightly/publisher-fairness.md) - the publisher fairness benchmark completes its backlog on the hosted ARM runner, yielding browser tasks while serving every group -- [The browser backend baseline completes on IETF22](/quest/m0/nightly/browser-baseline.md) - the WASM and UniFFI backend baseline completes every supported transport case without an internal error - [A Firefox late join reaches live playback](/quest/m0/nightly/firefox-late-join.md) - a Firefox player joining a live broadcast catches up within the media harness's derived playback deadline - [A stitched restart signals every TS discontinuity](/quest/m0/nightly/stitch-discontinuity.md) - a restarted publisher exported with stitching signals the program break on every affected PID - [Resolve the paired network microbenchmark regressions](/quest/m0/nightly/paired-net.md) - the reported network microbenchmark regressions are attributed and genuine extra costs are fixed with paired measurements diff --git a/quest/m0/nightly/browser-baseline.md b/quest/m0/nightly/browser-baseline.md deleted file mode 100644 index ff173f85878..00000000000 --- a/quest/m0/nightly/browser-baseline.md +++ /dev/null @@ -1,20 +0,0 @@ -# [M] The browser backend baseline completes on IETF22 - -## Goal - -The WASM and UniFFI backend baseline completes every supported transport case without an internal error. - -## Plan - -[The browser baseline job](https://github.com/moq-dev/moq/actions/runs/40698329026/job/122851001597) fails under -`just test wasm-baseline --uniffi` with `evaluate: MoqError.Internal` during -the moq-transport-22 benchmark. Its relay log also contains a uni-stream -`not found` error; that is evidence to investigate, not an established cause. -The retained `wasm-backend-baseline` artifact contains the run's diagnostics. - -Reproduce the failing cell on current main and trace the original transport -or model error through the bindings. Preserve completion and frame-integrity -checks. Review the already-landed reset/error mapping fixes before attributing -this to UniFFI itself. - -Public API: none expected. Wire: none expected. From 1ced942a7e9f394b614990a69c9d43c3db5eb977 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 19:24:10 -0700 Subject: [PATCH 06/11] fix(net): wait only the joined group on the subscription's fill A cache miss waited for the subscription's fill before any FETCH, so a head that never arrived held back unrelated history, and a cancelled request kept its task until the head settled. Split the joined group off into its own run, wait only that run, and stop waiting once no reader wants it or the track aborts. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 169 +++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 26 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index fcee15d34ae..81b881bbd2b 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -3685,17 +3685,12 @@ impl Subscriber { track_name.to_owned(), requested, timescale, + Some(Heading { + request_id, + fill: fill.clone(), + }), ); - // A join asks for the head of the group it lands in, which this - // subscription's own fill is already bringing. A FETCH racing it can - // cache that group first, and then the tail stream, which is what - // ends the group for live readers, is refused as a duplicate. So the - // miss waits for the head, and the cache answers it once that lands. - let heading = fill.clone(); - group_fetches.push(async move { - kio::wait(|waiter| poll_headed(&heading, waiter)).await; - fetch.await - }); + group_fetches.push(fetch); } End::Idle => { tracing::debug!(broadcast = %crate::path::Rooted(self.origin.root(), broadcast_path), track = %track_name, "subscribe cancelled (idle)"); @@ -3914,9 +3909,13 @@ impl Subscriber { match step { Step::Fill(fill) => { - let fetch = - self.clone() - .serve_fill(broadcast_path.clone(), track_name.to_owned(), fill, idle.timescale); + let fetch = self.clone().serve_fill( + broadcast_path.clone(), + track_name.to_owned(), + fill, + idle.timescale, + None, + ); group_fetches.push(fetch); } Step::Subscribe => break true, @@ -5275,25 +5274,53 @@ impl Subscriber { /// group always goes alone: a run including the group its source is still writing would /// hold the earlier groups until it finishes, or be cut short at the Largest Object by a /// publisher following draft-22 section 9.11. - async fn serve_fill(self, broadcast: Path, name: String, fill: track::Fill, timescale: Option) { + /// + /// `heading` is the subscription's own fill, when it may be bringing a joined group's + /// head. That group waits for it instead: a FETCH racing it can cache the group first, + /// and then the tail stream, which is what ends the group for live readers, is refused + /// as a duplicate. The cache answers the group once the head lands, and a fill that + /// settles without one leaves it to a FETCH. Only that group waits, and only while a + /// reader still wants it. + async fn serve_fill( + self, + broadcast: Path, + name: String, + fill: track::Fill, + timescale: Option, + heading: Option, + ) { let fill = std::sync::Arc::new(fill); let mut last = None; while fill.is_used() && let Some(run) = fill.next_run() { - let run = match fill.live_group() { - Some(live) if run.contains(&live) => { - let mut pieces = [run.start..live, live..live + 1, live + 1..run.end] - .into_iter() - .filter(|piece| !piece.is_empty()); - let piece = match fill.order() { - track::Order::Ascending => pieces.next(), - track::Order::Descending => pieces.next_back(), - }; - piece.expect("the run holds the live group") - } - _ => run, + let mut run = match fill.live_group() { + Some(live) => first_piece(run, live, fill.order()), + None => run, }; + if let Some(heading) = &heading + && let Some(head) = heading.pending(&self.state) + { + if let Head::Group(group) = head { + run = first_piece(run, group, fill.order()); + } + if head.overlaps(&run) { + let waited = kio::wait(|waiter| { + if let Poll::Ready(err) = fill.poll_aborted(waiter) { + return Poll::Ready(Err(err)); + } + if fill.poll_unwanted_groups(run.clone(), waiter).is_ready() { + return Poll::Ready(Ok(())); + } + heading.poll_changed(&self.state, head, waiter).map(Ok) + }) + .await; + if let Err(err) = waited { + fill.reject_groups(run, err); + } + continue; + } + } // Every fetch answers its run, so the same one coming back is a bug; ending the // fill names it dropped rather than fetching it forever. if last.as_ref() == Some(&run) { @@ -5897,6 +5924,74 @@ fn poll_headed(fill: &kio::Producer, waiter: &kio::Waiter) -> Poll<()> { } } +/// A subscription's own fill, which may still be bringing the head of the group it joined. +struct Heading { + request_id: RequestId, + fill: kio::Producer, +} + +/// The group a subscription's fill may still bring a head for. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Head { + /// A widening's fill awaits the answer that names its group, so it could be any. + Unnamed, + /// The group the subscription joined, whose head is on its way. + Group(u64), +} + +impl Head { + fn overlaps(self, run: &std::ops::Range) -> bool { + match self { + Head::Unnamed => true, + Head::Group(group) => run.contains(&group), + } + } +} + +impl Heading { + /// The group whose head the fill may still bring, or `None` once the head landed or + /// none is coming. + fn pending(&self, state: &Lock) -> Option { + // The state lock first, as SUBSCRIBE_OK takes it to name the group and serve the fill. + let state = state.lock(); + let largest = state + .subscribes + .get(&self.request_id) + .and_then(|held| held.fill_largest); + match *self.fill.read() { + Fill::Requested => Some(Head::Unnamed), + Fill::Serving(_) | Fill::Active => largest.map(Head::Group), + Fill::Ready { .. } | Fill::Done | Fill::Ended => None, + } + } + + /// Ready once [`Self::pending`] no longer answers `head`. + fn poll_changed(&self, state: &Lock, head: Head, waiter: &kio::Waiter) -> Poll<()> { + // Registered before reading, so a change after the read still wakes us. The group + // only changes with the fill's state. + let _ = self.fill.poll_ref(waiter, |_| Poll::<()>::Pending); + match self.pending(state) == Some(head) { + true => Poll::Pending, + false => Poll::Ready(()), + } + } +} + +/// The first piece of `run` in `order` once `group` is split off into a run of its own. +fn first_piece(run: std::ops::Range, group: u64, order: track::Order) -> std::ops::Range { + if !run.contains(&group) { + return run; + } + let mut pieces = [run.start..group, group..group + 1, group + 1..run.end] + .into_iter() + .filter(|piece| !piece.is_empty()); + let piece = match order { + track::Order::Ascending => pieces.next(), + track::Order::Descending => pieces.next_back(), + }; + piece.expect("the run holds the group") +} + /// A Normal object's timestamp: none on an untimed track, and required on a timed one, /// where an object without a Timestamp is malformed. fn object_time( @@ -15193,6 +15288,28 @@ mod joining_fetch_tests { assert_eq!(fetches(), 1, "a settled fill leaves the miss to a FETCH"); } + /// Only the joined group waits for the subscription's fill. A head that never arrives + /// leaves its fill outstanding for as long as the subscription lasts, which must not hold + /// back older groups it brings nothing for. + #[moq_clock::test] + async fn a_miss_for_older_history_skips_the_fill_wait() { + let version = Version::Draft22; + let largest = Some(ietf::Location { group: 4, object: 1 }); + let run = JoinRun::start(version, None, subscribe_ok(version, largest), fetch_ok(version)).await; + let fetches = || { + decode_messages(&run.session.log, version) + .iter() + .filter(|(id, _)| *id == ietf::Fetch::ID) + .count() + }; + assert!(run.fill_outstanding(), "the subscription asked for a fill"); + + let mut fetch = Box::pin(run._hold.1.fetch_group(3, None)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + settle().await; + assert_eq!(fetches(), 1, "group 3 is fetched without waiting for group 4's head"); + } + /// A malformed answer to the joining FETCH is not a refusal: the peer broke the /// protocol, so the session closes. #[moq_clock::test] From 6f3b699ba295c0416681bd9e646dde39bebd6cf4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 29 Oct 2026 11:30:23 -0700 Subject: [PATCH 07/11] test(net): cover cache misses answered by the subscription fill Co-Authored-By: Claude Opus 5.5 Co-Authored-By: GPT-6.1 Sol --- rs/moq-net/src/ietf/subscriber.rs | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 7e3d7047c55..e7facadab8d 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -15294,6 +15294,47 @@ mod joining_fetch_tests { assert_eq!(fetches(), 1, "a settled fill leaves the miss to a FETCH"); } + /// The head landing ends the wait, and the cache answers the miss: no FETCH duplicates + /// the group the fill wrote. + #[moq_clock::test] + async fn a_miss_is_answered_by_the_head_its_fill_lands() { + let version = Version::Draft22; + let largest = Some(ietf::Location { group: 4, object: 1 }); + let run = JoinRun::start(version, None, subscribe_ok(version, largest), fetch_ok(version)).await; + let fetches = || { + decode_messages(&run.session.log, version) + .iter() + .filter(|(id, _)| *id == ietf::Fetch::ID) + .count() + }; + + let mut fetch = Box::pin(run._hold.1.fetch_group(4, None)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + settle().await; + assert_eq!(fetches(), 0, "the miss waits for the subscription's fill"); + + // The fill writes the head, as `run_fill` does. + let (mut track, fill) = { + let state = run.subscriber.state.lock(); + let held = state.subscribes.values().next().unwrap(); + (held.producer.clone().unwrap(), held.fill.clone()) + }; + let mut head = None; + open_fill_group(&mut track, &mut head, 4).unwrap(); + let (sequence, next, producer) = head.unwrap(); + *fill.write().ok().unwrap() = Fill::Ready { + sequence, + next, + producer: producer.into_inner(), + }; + settle().await; + assert_eq!(fetches(), 0, "the cache answers the miss"); + let Poll::Ready(Ok(group)) = futures::poll!(fetch.as_mut()) else { + panic!("the miss must resolve from the cache"); + }; + assert_eq!(group.sequence, 4); + } + /// Only the joined group waits for the subscription's fill. A head that never arrives /// leaves its fill outstanding for as long as the subscription lasts, which must not hold /// back older groups it brings nothing for. From ec2172e70083bbf5fe57d68133e006072171cf81 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:38:24 -0700 Subject: [PATCH 08/11] fix(net): follow a widened fill and serve groups around a pending head A fill wait read the subscription's original fill, so a widening's replacement went unseen and a cache-miss FETCH could race its head. Look the fill up on each read. A pending head also held back the groups split off after it; serve the groups around it first so it holds back only its own group. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 117 ++++++++++++++++++++++++------ 1 file changed, 96 insertions(+), 21 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 96f3707b8ac..cf9935303f3 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -3773,10 +3773,7 @@ impl Subscriber { track_name.to_owned(), requested, timescale, - Some(Heading { - request_id, - fill: fill.clone(), - }), + Some(Heading { request_id }), ); group_fetches.push(fetch); } @@ -5362,7 +5359,7 @@ impl Subscriber { && let Some(head) = heading.pending(&self.state) { if let Head::Group(group) = head { - run = first_piece(run, group, fill.order()); + run = first_piece_without(run, group, fill.order()); } if head.overlaps(&run) { let waited = kio::wait(|waiter| { @@ -5986,9 +5983,9 @@ fn poll_headed(fill: &kio::Producer, waiter: &kio::Waiter) -> Poll<()> { } /// A subscription's own fill, which may still be bringing the head of the group it joined. +/// It is looked up on each read, since a widening replaces the fill with its own. struct Heading { request_id: RequestId, - fill: kio::Producer, } /// The group a subscription's fill may still bring a head for. @@ -6013,29 +6010,34 @@ impl Heading { /// The group whose head the fill may still bring, or `None` once the head landed or /// none is coming. fn pending(&self, state: &Lock) -> Option { - // The state lock first, as SUBSCRIBE_OK takes it to name the group and serve the fill. - let state = state.lock(); - let largest = state - .subscribes - .get(&self.request_id) - .and_then(|held| held.fill_largest); - match *self.fill.read() { - Fill::Requested => Some(Head::Unnamed), - Fill::Serving(_) | Fill::Active => largest.map(Head::Group), - Fill::Ready { .. } | Fill::Done | Fill::Ended => None, - } + self.read(state, None) } /// Ready once [`Self::pending`] no longer answers `head`. fn poll_changed(&self, state: &Lock, head: Head, waiter: &kio::Waiter) -> Poll<()> { - // Registered before reading, so a change after the read still wakes us. The group - // only changes with the fill's state. - let _ = self.fill.poll_ref(waiter, |_| Poll::<()>::Pending); - match self.pending(state) == Some(head) { + match self.read(state, Some(waiter)) == Some(head) { true => Poll::Pending, false => Poll::Ready(()), } } + + /// [`Self::pending`], registering `waiter` on the fill read before reading it, so a change + /// after the read still wakes it. The group only changes with the fill's state, and a + /// widening cancels the fill it replaces, which wakes a waiter on that one. + fn read(&self, state: &Lock, waiter: Option<&kio::Waiter>) -> Option { + // The state lock first, as SUBSCRIBE_OK takes it to name the group and serve the fill. + // Holding it across the read keeps a widening from swapping the fill in between. + let state = state.lock(); + let held = state.subscribes.get(&self.request_id)?; + if let Some(waiter) = waiter { + let _ = held.fill.poll_ref(waiter, |_| Poll::<()>::Pending); + } + match *held.fill.read() { + Fill::Requested => Some(Head::Unnamed), + Fill::Serving(_) | Fill::Active => held.fill_largest.map(Head::Group), + Fill::Ready { .. } | Fill::Done | Fill::Ended => None, + } + } } /// The first piece of `run` in `order` once `group` is split off into a run of its own. @@ -6053,6 +6055,22 @@ fn first_piece(run: std::ops::Range, group: u64, order: track::Order) -> st piece.expect("the run holds the group") } +/// The first piece of `run` in `order` that leaves `group` out, so the groups around it are +/// served while it waits, or `group` alone once nothing else is left. +fn first_piece_without(run: std::ops::Range, group: u64, order: track::Order) -> std::ops::Range { + if !run.contains(&group) { + return run; + } + let mut pieces = [run.start..group, group + 1..run.end] + .into_iter() + .filter(|piece| !piece.is_empty()); + let piece = match order { + track::Order::Ascending => pieces.next(), + track::Order::Descending => pieces.next_back(), + }; + piece.unwrap_or(group..group + 1) +} + /// A Normal object's timestamp: none on an untimed track, and required on a timed one, /// where an object without a Timestamp is malformed. fn object_time( @@ -15571,6 +15589,63 @@ mod joining_fetch_tests { assert_eq!(fetches(), 1, "group 3 is fetched without waiting for group 4's head"); } + /// A widening replaces the subscription's fill with its own, so a miss follows the new + /// one: until its answer names the group it heads, every miss waits for it. + #[moq_clock::test] + async fn a_miss_follows_the_fill_a_widening_requested() { + let version = Version::Draft22; + let largest = Some(ietf::Location { group: 4, object: 1 }); + let run = JoinRun::start(version, None, subscribe_ok(version, largest), fetch_ok(version)).await; + let fetches = || { + decode_messages(&run.session.log, version) + .iter() + .filter(|(id, _)| *id == ietf::Fetch::ID) + .count() + }; + + // As the update arm does: cancel the old fill and request a new one in its place. + { + let mut state = run.subscriber.state.lock(); + let held = state.subscribes.values_mut().next().unwrap(); + held.fill.write().ok().unwrap().cancel(); + held.fill = kio::Producer::new(Fill::Requested); + } + + let mut fetch = Box::pin(run._hold.1.fetch_group(3, None)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + settle().await; + assert_eq!(fetches(), 0, "the miss waits for the widening's fill"); + + let fill = run + .subscriber + .state + .lock() + .subscribes + .values() + .next() + .unwrap() + .fill + .clone(); + *fill.write().ok().unwrap() = Fill::Done; + settle().await; + assert_eq!(fetches(), 1, "a settled fill leaves the miss to a FETCH"); + } + + /// The groups around a pending head are served before it, in the fill's order, so a + /// head that never arrives holds back only its own group. + #[test] + fn a_pending_head_is_split_off_last() { + use track::Order::{Ascending, Descending}; + assert_eq!(first_piece_without(0..5, 4, Descending), 0..4); + assert_eq!(first_piece_without(0..5, 4, Ascending), 0..4); + assert_eq!(first_piece_without(3..8, 4, Ascending), 3..4); + assert_eq!(first_piece_without(4..8, 4, Ascending), 5..8); + assert_eq!(first_piece_without(3..8, 4, Descending), 5..8); + assert_eq!(first_piece_without(3..5, 4, Descending), 3..4); + assert_eq!(first_piece_without(4..5, 4, Descending), 4..5); + assert_eq!(first_piece_without(0..3, 4, Ascending), 0..3); + } + /// A malformed answer to the joining FETCH is not a refusal: the peer broke the /// protocol, so the session closes. #[moq_clock::test] From abfe8310c97ff03953709101c47a5b15f04e111f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 09:05:22 -0700 Subject: [PATCH 09/11] fix(net): serve every other run before waiting on a pending head The cache can split a fill so the pending head's run comes up alone, and waiting on it then held back older runs too. Pick the next run passing over that group, and wait on it only once nothing else is owed. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 45 +++++++----------------------- rs/moq-net/src/model/track.rs | 35 +++++++++++++++++++++++ rs/moq-net/src/model/track/fill.rs | 42 ++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 38 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index cf9935303f3..f5b71984d48 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -5348,9 +5348,15 @@ impl Subscriber { ) { let fill = std::sync::Arc::new(fill); let mut last = None; - while fill.is_used() - && let Some(run) = fill.next_run() - { + while fill.is_used() { + // The groups around a pending head go first, so it holds back only its own group. + let pending = match heading.as_ref().and_then(|heading| heading.pending(&self.state)) { + Some(Head::Group(group)) => fill.next_run_except(group), + _ => None, + }; + let Some(run) = pending.or_else(|| fill.next_run()) else { + break; + }; let mut run = match fill.live_group() { Some(live) => first_piece(run, live, fill.order()), None => run, @@ -5359,7 +5365,7 @@ impl Subscriber { && let Some(head) = heading.pending(&self.state) { if let Head::Group(group) = head { - run = first_piece_without(run, group, fill.order()); + run = first_piece(run, group, fill.order()); } if head.overlaps(&run) { let waited = kio::wait(|waiter| { @@ -6055,22 +6061,6 @@ fn first_piece(run: std::ops::Range, group: u64, order: track::Order) -> st piece.expect("the run holds the group") } -/// The first piece of `run` in `order` that leaves `group` out, so the groups around it are -/// served while it waits, or `group` alone once nothing else is left. -fn first_piece_without(run: std::ops::Range, group: u64, order: track::Order) -> std::ops::Range { - if !run.contains(&group) { - return run; - } - let mut pieces = [run.start..group, group + 1..run.end] - .into_iter() - .filter(|piece| !piece.is_empty()); - let piece = match order { - track::Order::Ascending => pieces.next(), - track::Order::Descending => pieces.next_back(), - }; - piece.unwrap_or(group..group + 1) -} - /// A Normal object's timestamp: none on an untimed track, and required on a timed one, /// where an object without a Timestamp is malformed. fn object_time( @@ -15631,21 +15621,6 @@ mod joining_fetch_tests { assert_eq!(fetches(), 1, "a settled fill leaves the miss to a FETCH"); } - /// The groups around a pending head are served before it, in the fill's order, so a - /// head that never arrives holds back only its own group. - #[test] - fn a_pending_head_is_split_off_last() { - use track::Order::{Ascending, Descending}; - assert_eq!(first_piece_without(0..5, 4, Descending), 0..4); - assert_eq!(first_piece_without(0..5, 4, Ascending), 0..4); - assert_eq!(first_piece_without(3..8, 4, Ascending), 3..4); - assert_eq!(first_piece_without(4..8, 4, Ascending), 5..8); - assert_eq!(first_piece_without(3..8, 4, Descending), 5..8); - assert_eq!(first_piece_without(3..5, 4, Descending), 3..4); - assert_eq!(first_piece_without(4..5, 4, Descending), 4..5); - assert_eq!(first_piece_without(0..3, 4, Ascending), 0..3); - } - /// A malformed answer to the joining FETCH is not a refusal: the peer broke the /// protocol, so the session closes. #[moq_clock::test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index da4102ad639..e0a40904b77 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -18831,6 +18831,41 @@ mod test { ); } + /// A run that passes over a group another request is bringing serves the groups around + /// it, across the runs that answered groups split the fill into, and leaves that group + /// to [`Fill::next_run`] once nothing else is owed. + #[test] + #[cfg(feature = "async")] + fn a_run_passes_over_a_skipped_group() { + let mut producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let mut descending = producer.subscribe(replay().with_range(0..)); + producer.start_at(6).unwrap(); + write_group(&producer, 6); + let _ = descending.poll_recv_group(&kio::Waiter::noop()); + let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&kio::Waiter::noop()) else { + panic!("the miss below the start is asked"); + }; + assert_eq!(fill.order(), Order::Descending); + assert_eq!(fill.next_run(), Some(0..6)); + assert_eq!(fill.next_run_except(5), Some(0..5)); + assert_eq!(fill.next_run_except(2), Some(3..6)); + + fill.drop_groups(3..4); + assert_eq!(fill.next_run(), Some(4..6)); + assert_eq!(fill.next_run_except(5), Some(4..5)); + assert_eq!(fill.next_run_except(4), Some(5..6)); + fill.drop_groups(4..5); + assert_eq!( + fill.next_run_except(5), + Some(0..3), + "older runs are served while 5 waits" + ); + fill.drop_groups(0..3); + assert_eq!(fill.next_run_except(5), None); + assert_eq!(fill.next_run(), Some(5..6)); + } + /// A reader changing its priority, budget, or order moves the fill it waits on, queued /// or taken, so the handler serves the rest under the new values, even ones lower than /// it was asked with. diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index c529bd39465..b0273961cd8 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -587,6 +587,15 @@ impl FillState { } } +/// The pieces of `gap` once `skip` is taken out of it. +fn without(gap: ops::Range, skip: Option) -> impl Iterator> { + let pieces = match skip { + Some(skip) if gap.contains(&skip) => [gap.start..skip, skip + 1..gap.end], + _ => [gap, 0..0], + }; + pieces.into_iter().filter(|piece| !piece.is_empty()) +} + /// Stop routing readers wanting `groups` to fill `id`: they were answered. Reads first, so /// an answer that changes nothing wakes no handler parked on the queue. fn uncover(fills: &kio::Shared, id: u64, groups: ops::Range) { @@ -778,11 +787,19 @@ impl Fill { /// The next group still owed in the fill's order: a reader still reads it, and it was /// neither answered nor delivered live since it was asked. `None` once none is owed. pub fn next(&self) -> Option { + self.next_except(None) + } + + /// [`Self::next`], passing over `skip`. + fn next_except(&self, skip: Option) -> Option { loop { let wanted = self.inner.wanted(); let sequence = { let outcome = self.inner.result.read(); - let mut gaps = wanted.iter().flat_map(|run| outcome.resolved.gaps(run)); + let mut gaps = wanted + .iter() + .flat_map(|run| outcome.resolved.gaps(run)) + .flat_map(|gap| without(gap, skip)); match self.order() { Order::Ascending => gaps.next().map(|gap| gap.start), Order::Descending => gaps.last().map(|gap| gap.end - 1), @@ -821,10 +838,29 @@ impl Fill { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn next_run(&self) -> Option> { let sequence = self.next()?; + Some(self.run_from(sequence, self.inner.owed_around(sequence))) + } + + /// [`Self::next_run`], passing over `skip`: a group another request is already bringing, + /// so the groups around it are served while it waits. `None` once nothing else is owed. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn next_run_except(&self, skip: u64) -> Option> { + let sequence = self.next_except(Some(skip))?; let owed = self.inner.owed_around(sequence); + let owed = match skip < sequence { + true => owed.start.max(skip + 1)..owed.end, + false => owed.start..owed.end.min(skip), + }; + Some(self.run_from(sequence, owed)) + } + + /// The run from `sequence` over the `owed` groups beside it, in the fill's order, up to + /// the nearest one the cache holds. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + fn run_from(&self, sequence: u64, owed: ops::Range) -> ops::Range { let state = self.inner.state.read(); let held = |(_, slot): &(&u64, &super::Slot)| !slot.is_aborted(); - Some(match self.order() { + match self.order() { Order::Ascending => { let end = state .lookup @@ -842,7 +878,7 @@ impl Fill { .map_or(owed.start, |(held, _)| held + 1); start..sequence + 1 } - }) + } } /// The newest group the track's live feed produced, while a feed still writes the track: From d94527ec382a2578b919cddb82a6aa862029c019 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 23:54:07 -0700 Subject: [PATCH 10/11] fix(net): let a failed fill head fall back to a FETCH main's Fill::Failed (moq-dev/moq#7322) means the head is never coming, so the fill wait stops holding that group back and the cache miss FETCHes it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 97d48b38d4c..4f9d728a23e 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -6173,7 +6173,7 @@ impl Heading { match *held.fill.read() { Fill::Requested => Some(Head::Unnamed), Fill::Serving(_) | Fill::Active => held.fill_largest.map(Head::Group), - Fill::Ready { .. } | Fill::Done | Fill::Ended => None, + Fill::Ready { .. } | Fill::Done | Fill::Failed { .. } | Fill::Ended => None, } } } From 5ff6e7ee609bcaa53e9f989be378cd2dd8caafc0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 20:00:09 -0800 Subject: [PATCH 11/11] chore(quest): size the nightly epic now its last child is done Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 12d2934766b..720f0a44712 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -1,4 +1,4 @@ -# Nightly passes on main +# [S] Nightly passes on main ## Goal