From da303f1815da6477769731cf479e0db6cc4578c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 00:02:16 -0700 Subject: [PATCH 01/18] chore(quest): claim quest/m1/container-backpressure.md Quest-Claim: 515cc5c1958084f4 From dc5172183f020a1b2c1d7fc03e63908ad0c07ea6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 00:54:11 -0700 Subject: [PATCH 02/18] feat(container): bound decoded lookahead by a max buffer Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 125 +++++++++ js/hang/src/container/consumer.ts | 122 +++++++- rs/moq-mux/src/container/consumer.rs | 261 +++++++++++++++++- 3 files changed, 496 insertions(+), 12 deletions(-) create mode 100644 js/hang/src/container/consumer.backpressure.test.ts diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts new file mode 100644 index 00000000000..0ad0d5e3116 --- /dev/null +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -0,0 +1,125 @@ +import { expect, spyOn, test } from "bun:test"; +import { type Micro, Milli } from "@moq/clock"; +import { Time, Track } from "@moq/net"; +import { Consumer, type Read } from "./consumer.ts"; +import { encodeFrame, Format } from "./legacy.ts"; +import type { Frame } from "./types.ts"; + +// Counts decoded samples, so a test can tell how many the consumer holds ahead of its reads. +class Counting extends Format { + decoded = 0; + + override decode(frame: Uint8Array): Frame[] { + const frames = super.decode(frame); + this.decoded += frames.length; + return frames; + } +} + +const PAYLOAD = new Uint8Array([0xde, 0xad]); + +function write(group: { writeFrame(frame: { payload: Uint8Array; timestamp: Time.Timestamp }): void }, ts: number) { + group.writeFrame({ payload: encodeFrame(PAYLOAD, ts as Micro), timestamp: Time.Timestamp.now() }); +} + +// The most frames the consumer may hold: past the max buffer, one more wire frame. Each held frame +// costs at least its payload and its own entry. +function maxHeld(maxBuffer: number): number { + return Math.ceil(maxBuffer / (64 + PAYLOAD.byteLength)) + 1; +} + +// Read every item. With `bounded`, check after each that the consumer holds no more than the max +// buffer allows, which only holds while no skip drops decoded frames unread. +async function readAll(consumer: Consumer, format: Counting, bounded?: number): Promise { + const reads: Read[] = []; + let delivered = 0; + for (;;) { + const read = await Promise.race([ + consumer.next(), + new Promise((_, reject) => setTimeout(() => reject(new Error("consumer hung")), 5_000)), + ]); + if (read === undefined) return reads; + reads.push(read); + if (read.kind === "frame") delivered++; + if (bounded !== undefined) expect(format.decoded - delivered).toBeLessThanOrEqual(maxHeld(bounded)); + } +} + +function timestamps(reads: Read[]): number[] { + return reads.flatMap((read) => (read.kind === "frame" ? [read.frame.timestamp] : [])); +} + +test("Consumer holds frames sharing a timestamp within the max buffer", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const maxBuffer = 4096; + const format = new Counting("data"); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format, + maxDelay: Milli(5_000), + maxBuffer, + }); + try { + // Frames packed into one timestamp have no span, so the age budget never skips the stalled + // group they queue behind. They fill the max buffer instead, which skips it once its one frame + // is read, so nothing decoded is dropped. + const stalled = track.createGroup(0); + write(stalled, 0); + const packed = track.createGroup(1); + for (let i = 0; i < 2_000; i++) write(packed, 10_000); + packed.finish(); + const tail = track.createGroup(2); + write(tail, 20_000); + write(tail, 30_000); + tail.finish(); + track.finish(); + + const reads = await readAll(consumer, format, maxBuffer); + const ts = timestamps(reads); + expect(ts.length).toBe(1 + 2_000 + 2); + expect(ts.at(0)).toBe(0); + expect(ts.at(-1)).toBe(30_000); + // The stalled group's tail is lost. + expect(reads[1]?.kind).toBe("gap"); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + +test("Consumer holds many groups within the max buffer and reaches the live edge", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const maxBuffer = 8192; + const format = new Counting("data"); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format, + maxDelay: Milli(100), + maxBuffer, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + for (let g = 1; g <= 500; g++) { + const group = track.createGroup(g); + write(group, g * 1_000); + group.finish(); + } + track.finish(); + + // The groups' overhead and first frames fill the max buffer, and the rest wait unpulled. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(format.decoded).toBeLessThanOrEqual(maxHeld(maxBuffer)); + expect(format.decoded).toBeGreaterThan(1); + + const reads = await readAll(consumer, format); + const ts = timestamps(reads); + expect(ts.at(0)).toBe(0); + expect(ts.at(-1)).toBe(500_000); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index fd3d1723161..17f37d039df 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -20,6 +20,16 @@ export interface ConsumerProps { */ // Read-only: a Getter (e.g. another component's output) is accepted directly. maxDelay?: GetterInit; + /** + * Cap the bytes of decoded frames held ahead of the reader, each frame's and group's own overhead + * included (default: {@link DEFAULT_MAX_BUFFER}). Mirrors Rust's `Consumer::set_max_buffer`. + * + * Once it is full the consumer stops decoding ahead and stops pulling new groups, so the undecoded + * rest stays in the transport's cache, which bounds it. The group being read still decodes one wire + * frame at a time, so reading never waits on its own lookahead. If that group stalls while later + * groups fill the cap, it is skipped like a slow group: no more of what it waits behind can be held. + */ + maxBuffer?: number; /** * The consumer this one replaces on the same path, such as the previous rendition. * @@ -50,6 +60,14 @@ export type Read = */ | { kind: "end"; group: number; end: Micro }; +/** The default {@link ConsumerProps.maxBuffer}: one group at the transport's size cap. Mirrors Rust's. */ +export const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024; + +// Rough sizes of a buffered frame's and a group's own objects, charged to the max buffer alongside +// payload bytes. JS cannot measure them; these only need to be the right order of magnitude. +const FRAME_COST = 64; +const GROUP_COST = 256; + interface Group { consumer: Moq.Group.Consumer; // From then on, any sequence missing below this group is late: the earliest arrival among it @@ -66,6 +84,8 @@ interface Group { done?: boolean; // Set when #runGroup finishes reading all frames truncated?: boolean; // The missing tail becomes a gap after the buffered frames are delivered. fault?: Error; // The payload that failed to decode, an error after the buffered frames are delivered. + charged: number; // What this group holds against the max buffer + released?: boolean; // Dropped from #groups, with what it held refunded } // Two adjacent groups are treated as timeline-contiguous when the next group's first PTS is within @@ -106,6 +126,11 @@ export class Consumer { #track: Moq.Track.Subscriber; #format: Format; #maxDelay: Getter; + #maxBuffer: number; + // What the buffered groups hold, against the max buffer. + #used = 0; + // Group tasks and the track loop parked on a full max buffer. + #room: (() => void)[] = []; #groups: Group[] = []; #active?: number; // the active group sequence number // When the missing sequence at the cursor became late, while one is. @@ -156,6 +181,7 @@ export class Consumer { this.#track = track; this.#format = props.format; this.#maxDelay = getter(props.maxDelay ?? Milli.zero); + this.#maxBuffer = props.maxBuffer ?? DEFAULT_MAX_BUFFER; this.#jitter = props.predecessor ? props.predecessor.#jitter.fork() : new Jitter(); this.#spread = new Signal(this.#jitter.measured); this.spread = this.#spread; @@ -167,6 +193,8 @@ export class Consumer { group.consumer.close(); } this.#groups.length = 0; + // Parked tasks see the abort and return. + this.#wake(); }); } @@ -180,6 +208,14 @@ export class Consumer { // Start fetching groups in the background try { for (;;) { + // Past the max buffer, later groups wait in the transport's cache, unless the group being + // read has not arrived yet and might be the next one. + await this.#waitRoom(() => { + const head = this.#groups.at(0); + return head === undefined || this.#active === undefined || head.consumer.sequence > this.#active; + }); + if (this.#signals.abort.aborted) break; + const consumer = await this.#track.recvGroup(); if (!consumer) break; @@ -200,7 +236,9 @@ export class Consumer { frames: [], empty: true, media: false, + charged: 0, }; + this.#charge(group, GROUP_COST); // Insert into #groups based on the group sequence number (ascending). // This is used to cancel old groups. @@ -225,6 +263,11 @@ export class Consumer { let index = 0; for (;;) { + // Only the head may decode past a full max buffer, and only once it has nothing left to read, + // so reading never waits on its own lookahead. A dropped group goes on to find its close. + await this.#waitRoom(() => group.released === true || (group === this.#groups[0] && group.frames.length === 0)); + if (this.#signals.abort.aborted) return; + const next = await group.consumer.readFrame(); if (!next) break; group.empty = false; @@ -281,6 +324,7 @@ export class Consumer { group.start ??= frame.timestamp; group.frames.push(frame); + this.#charge(group, FRAME_COST + frame.payload.byteLength); if (group.latest === undefined || frame.timestamp > group.latest) { group.latest = frame.timestamp; @@ -319,7 +363,8 @@ export class Consumer { // buffered group is the delivery head regardless of id scheme; next()'s // promotion guard advances #active to it. Works for sequential and // non-sequential ids alike. - if (skipped || group.consumer.sequence === this.#active || group === this.#groups[0]) { + // Once the max buffer fills, next() may have to skip the stalled head it is waiting on. + if (skipped || group.consumer.sequence === this.#active || group === this.#groups[0] || this.#full()) { this.#notify?.(); this.#notify = undefined; } @@ -459,6 +504,71 @@ export class Consumer { return true; } + // Skip the stalled active group once later groups fill the max buffer, which can hold no more of + // what it waits behind. Only fires when the active group is fully consumed and still open, so we + // never drop frames the consumer hasn't seen. Returns true if a group was skipped. + #tryPressureSkip(): boolean { + if (this.#active === undefined || !this.#full()) return false; + + const active = this.#groups[0]; + if (!active || active.consumer.sequence !== this.#active) return false; + if (active.done || active.frames.length > 0) return false; + + const next = this.#groups[1]; + const nextStart = next?.frames.at(0)?.timestamp; + if (!next || nextStart === undefined) return false; + + this.#groups.shift(); + console.warn( + `skipping a stalled group past the max buffer: track=${this.#track.name} ${active.consumer.sequence} -> ${next.consumer.sequence}`, + ); + // Still open, so its tail is lost unless the next group picks up where it ended. + if (active.truncated || !ptsContiguous(active.end ?? this.#presentedEnd, nextStart)) this.#lost = true; + this.#recordPresented(active); + this.#active = next.consumer.sequence; + + this.#discard(active); + this.#updateBuffered(); + return true; + } + + #full(): boolean { + return this.#used >= this.#maxBuffer; + } + + #charge(group: Group, bytes: number): void { + group.charged += bytes; + this.#used += bytes; + } + + #refund(group: Group, bytes: number): void { + group.charged -= bytes; + this.#used -= bytes; + this.#wake(); + } + + // Refund everything a group dropped from #groups still holds. + #release(group: Group): void { + if (group.released) return; + group.released = true; + this.#refund(group, group.charged); + } + + // Wait until the max buffer has room, or `exempt` holds, re-checked whenever it may have changed. + async #waitRoom(exempt: () => boolean): Promise { + const abort = this.#signals.abort; + while (this.#full() && !exempt() && !abort.aborted) { + await new Promise((resolve) => this.#room.push(resolve)); + } + } + + // Let parked tasks re-check the max buffer: something was refunded, or the head or cursor moved. + #wake(): void { + const room = this.#room; + this.#room = []; + for (const resolve of room) resolve(); + } + // Drop a group without delivering it. A drop must not hide what was already found, so one holding // a fault, or with frames below the latest delivered group start, ends the reader instead. What // its task has not decoded yet stays unchecked: decoding data only to throw it away is work a @@ -466,6 +576,7 @@ export class Consumer { #discard(group: Group): boolean { group.consumer.close(); group.frames.length = 0; + this.#release(group); const start = this.#start; if ( start !== undefined && @@ -553,7 +664,9 @@ export class Consumer { head.latest !== undefined && head.latest - this.#presentedEnd > maxDelay)); const elapsed = Instant.durationSince(now(), this.#hole) >= Nano.fromMicro(maxDelay); - if (startup || empty || contiguous || skipHole || elapsed || ended !== undefined) { + // The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it. + const full = head.frames.length > 0 && this.#full(); + if (startup || empty || contiguous || skipHole || elapsed || full || ended !== undefined) { if (!contiguous && !empty) { this.#lost = true; if (!startup) { @@ -575,6 +688,7 @@ export class Consumer { ) { const frame = this.#groups[0].frames.shift(); if (frame) { + this.#refund(this.#groups[0], FRAME_COST + frame.payload.byteLength); const seq = this.#groups[0].consumer.sequence; const end = this.#format.end?.(frame); if (end !== undefined) { @@ -620,6 +734,7 @@ export class Consumer { const group = this.#groups.shift(); if (group) { + this.#release(group); if (group.fault) { this.#abort(group.fault); throw group.fault; @@ -639,11 +754,14 @@ export class Consumer { // has already been reached by this group's duration, skip ahead now // rather than waiting for the stalled group to resume. if (this.#tryDurationSkip()) continue; + if (this.#tryPressureSkip()) continue; } if (this.#notify) { throw new Error("multiple calls to next not supported"); } + // The head or cursor may have moved, which can exempt a parked task from the max buffer. + this.#wake(); const abort = this.#signals.abort; if (abort.aborted) return undefined; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 1c010612bec..a539e8641dd 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -1,4 +1,6 @@ use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::{Poll, ready}; use moq_clock::Instant; @@ -100,6 +102,15 @@ pub(crate) enum Item { /// failed track once the groups already here drain. A skip returns the error of a malformed group /// it already read rather than a gap; a group dropped unread is not validated, since decoding data /// that is thrown away is work a peer could force. After an error the reader reads as finished. +/// +/// ## Memory +/// +/// Decoded frames waiting to be read are charged to a byte cap, [`set_max_buffer`](Self::set_max_buffer), +/// along with each buffered frame's and group's own overhead. Once it is full the consumer stops +/// decoding ahead and stops pulling new groups, so the undecoded rest stays in the transport's cache, +/// which bounds it. The group the cursor is on still decodes one wire frame at a time, so reading never +/// waits on its own lookahead. If that group stalls while later groups fill the cap, it is skipped like +/// a slow group: the consumer can hold no more of what it would wait behind. pub struct Consumer { // Dropped once an error ends the reader, releasing the publisher's demand. track: Option, @@ -113,6 +124,9 @@ pub struct Consumer { // Groups that we are monitoring, sorted by sequence ascending. pending: VecDeque>, + // What the pending groups hold, against the max buffer. + memory: Memory, + // Latches the cursor onto the publisher's first served group when the // subscription names no start. Cleared once a first group is chosen. startup: bool, @@ -162,6 +176,48 @@ pub struct Consumer { /// about one group duration. const CONTIGUITY_TOLERANCE: std::time::Duration = std::time::Duration::from_millis(1); +/// The default [`Consumer::set_max_buffer`]: one group at the transport's size cap. +pub const DEFAULT_MAX_BUFFER: usize = moq_net::group::MAX_CACHE_BYTES as usize; + +/// The decoded bytes a consumer's groups hold, shared with each group so one dropped by any path +/// refunds what it held. +#[derive(Clone)] +struct Memory(Arc); + +struct MemoryState { + used: AtomicUsize, + max: AtomicUsize, +} + +impl Memory { + fn new(max: usize) -> Self { + Self(Arc::new(MemoryState { + used: AtomicUsize::new(0), + max: AtomicUsize::new(max), + })) + } + + fn used(&self) -> usize { + self.0.used.load(Ordering::Relaxed) + } + + fn full(&self) -> bool { + self.used() >= self.0.max.load(Ordering::Relaxed) + } + + fn set_max(&self, max: usize) { + self.0.max.store(max, Ordering::Relaxed); + } + + fn charge(&self, bytes: usize) { + self.0.used.fetch_add(bytes, Ordering::Relaxed); + } + + fn refund(&self, bytes: usize) { + self.0.used.fetch_sub(bytes, Ordering::Relaxed); + } +} + fn pts_contiguous(end: Option, next_start: std::time::Duration) -> bool { end.is_some_and(|end| next_start <= end.saturating_add(CONTIGUITY_TOLERANCE)) } @@ -208,6 +264,7 @@ impl Consumer { format, current: start.unwrap_or(0), pending: VecDeque::new(), + memory: Memory::new(DEFAULT_MAX_BUFFER), startup: start.is_none(), newest: false, max_delay, @@ -550,7 +607,10 @@ impl Consumer { .track .as_ref() .is_some_and(|track| track.next_in_range(self.current).is_none_or(|next| next >= sequence)); - if finished || elapsed || aged || excluded { + // The groups past the hole already fill the max buffer, so it can hold none of what + // would arrive into it. + let full = self.memory.full(); + if finished || elapsed || aged || excluded || full { let _ = self.pending.front_mut().unwrap().look_all(waiter, &self.format); let front = self.pending.front_mut().unwrap(); let empty = front.empty && matches!(front.group.poll_finished(waiter), Poll::Ready(Ok(_))); @@ -580,10 +640,13 @@ impl Consumer { // short whenever it blocks behind a newer one, as a join's backlog does // while the newer groups race ahead on their own streams. Skip early too // once it has presented up to where the next group begins (duration - // coverage), since nothing is left worth waiting for. + // coverage), since nothing is left worth waiting for, or once the later groups fill the max + // buffer, since no more of them can be held while it waits. + let full = self.memory.full(); let should_skip = next_group.is_some_and(|(_, next_start)| { max_timestamp.saturating_sub(next_start) >= self.max_delay || current_end.is_some_and(|end| end >= next_start) + || full }); if let Some((new_idx, next_start)) = next_group @@ -655,6 +718,11 @@ impl Consumer { return Poll::Ready(()); } loop { + // Past the max buffer, later groups wait in the transport's cache, unless the cursor's own + // group has not arrived yet and might be the next one. + if self.memory.full() && self.pending.front().is_some_and(|group| group.sequence <= self.current) { + return Poll::Pending; + } let next = match ready!(self.subscriber().poll_recv_group(waiter)) { Ok(next) => next, Err(err) => { @@ -668,7 +736,7 @@ impl Consumer { return Poll::Ready(()); }; - let mut reader = GroupBuffer::new(group); + let mut reader = GroupBuffer::new(group, self.memory.clone()); let sequence = reader.group.sequence; if sequence < self.current { @@ -734,6 +802,19 @@ impl Consumer { let subscription = track.subscription().with_max_delay(max_delay); let _ = track.update(subscription); } + + /// Cap the bytes of decoded frames held ahead of the reader, each frame's and group's own + /// overhead included (default [`DEFAULT_MAX_BUFFER`]). Past it the consumer stops decoding ahead + /// and pulling new groups; see the type docs. + pub fn set_max_buffer(&mut self, bytes: usize) { + self.memory.set_max(bytes); + } + + /// The bytes currently charged against [`set_max_buffer`](Self::set_max_buffer). + #[cfg(test)] + pub(crate) fn buffered_bytes(&self) -> usize { + self.memory.used() + } } /// Why a group stopped buffering. @@ -779,6 +860,10 @@ struct GroupBuffer { markers: VecDeque<(usize, Timestamp)>, delivered: usize, + // The consumer's max buffer, and how much of it this group holds, refunded when it drops. + memory: Memory, + charged: usize, + // The minimum timestamp in the group. min_timestamp: Option, @@ -792,8 +877,8 @@ struct GroupBuffer { } impl GroupBuffer { - fn new(group: moq_net::group::Consumer) -> Self { - Self { + fn new(group: moq_net::group::Consumer, memory: Memory) -> Self { + let mut this = Self { group, late: moq_clock::now(), index: 0, @@ -808,7 +893,11 @@ impl GroupBuffer { max_timestamp: None, min_timestamp: None, max_end: None, - } + memory, + charged: 0, + }; + this.charge(std::mem::size_of::()); + this } /// Poll for the next frame from this group. @@ -816,10 +905,12 @@ impl GroupBuffer { loop { if self.markers.front().is_some_and(|(index, _)| *index <= self.delivered) { let (index, end) = self.markers.pop_front().unwrap(); + self.refund(MARKER_COST); return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: index == 0 }))); } if let Some((frame, arrived)) = self.buffered.pop_front() { self.delivered += 1; + self.refund(frame_cost(&frame)); return Poll::Ready(Ok(Some(Event::Frame { frame, arrived }))); } if !ready!(self.buffer_once(waiter, format)?) { @@ -859,6 +950,7 @@ impl GroupBuffer { if let Some(bound) = format.end(&frame) { self.note_end(bound); self.markers.push_back((self.index, bound)); + self.charge(MARKER_COST); continue; } @@ -887,6 +979,7 @@ impl GroupBuffer { self.index += 1; self.media = true; + self.charge(frame_cost(&frame)); self.buffered.push_back((frame, arrived)); } @@ -897,11 +990,15 @@ impl GroupBuffer { // decode error is kept for the reader rather than dropped with the frame it consumed; a // transport abort is left for the cursor to find. // - // Returns false once this group can buffer nothing more. + // Returns false once this group can buffer nothing more, and pends without registering the waiter + // while the max buffer is full: only the reader taking frames frees it. fn look_once(&mut self, waiter: &kio::Waiter, format: &F) -> Poll { if self.fault.is_some() { return Poll::Ready(false); } + if self.memory.full() { + return Poll::Pending; + } match ready!(self.buffer_once(waiter, format)) { Ok(more) => Poll::Ready(more), Err(Failure::Decode(err)) => { @@ -920,22 +1017,36 @@ impl GroupBuffer { /// Poll for the maximum timestamp in this group, or `None` if it ended without one. fn poll_max_timestamp(&mut self, waiter: &kio::Waiter, format: &F) -> Poll> { // Keep reading more frames just to advance the max timestamp. - let _ = self.look_all(waiter, format); + let looked = self.look_all(waiter, format); if let Some(max) = self.max_timestamp { return Poll::Ready(Some(max)); } + // Frames left unread behind a full max buffer may still carry one. + ready!(looked); self.poll_ended(waiter) } /// Poll for the minimum timestamp in this group, or `None` if it ended without one. fn poll_min_timestamp(&mut self, waiter: &kio::Waiter, format: &F) -> Poll> { // A wire frame can decode to no media frames, so keep going until one is buffered. - while self.buffered.is_empty() && matches!(self.look_once(waiter, format), Poll::Ready(true)) {} + let mut looked = Poll::Ready(()); + while self.buffered.is_empty() { + match self.look_once(waiter, format) { + Poll::Ready(true) => continue, + Poll::Ready(false) => break, + Poll::Pending => { + looked = Poll::Pending; + break; + } + } + } if let Some(min) = self.min_timestamp { return Poll::Ready(Some(min)); } + // Frames left unread behind a full max buffer may still carry one. + ready!(looked); self.poll_ended(waiter) } @@ -961,6 +1072,30 @@ impl GroupBuffer { fn marker(&self) -> bool { !self.empty && !self.media } + + fn charge(&mut self, bytes: usize) { + self.charged += bytes; + self.memory.charge(bytes); + } + + fn refund(&mut self, bytes: usize) { + self.charged -= bytes; + self.memory.refund(bytes); + } +} + +impl Drop for GroupBuffer { + fn drop(&mut self) { + self.memory.refund(self.charged); + } +} + +/// What a buffered endpoint marker holds against the max buffer. +const MARKER_COST: usize = std::mem::size_of::<(usize, Timestamp)>(); + +/// What a buffered frame holds against the max buffer: its payload and its own entry. +fn frame_cost(frame: &Frame) -> usize { + std::mem::size_of::<(Frame, Instant)>() + frame.payload.len() } impl std::ops::Deref for GroupBuffer { @@ -3010,7 +3145,7 @@ mod tests { group.finish().unwrap(); // Finding the start buffers one frame; the read behind it already holds the rest. - let mut buffer = GroupBuffer::::new(group.consume()); + let mut buffer = GroupBuffer::::new(group.consume(), Memory::new(DEFAULT_MAX_BUFFER)); assert_eq!( buffer.poll_min_timestamp(&waiter, &FailingDecode), Poll::Ready(Some(ts(0))) @@ -3910,4 +4045,110 @@ mod tests { let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); assert!(matches!(event, Some(Event::FrameEnd { end, opens: false }) if end == ts(15_000))); } + + // ---- Max buffer ---- + + /// Write one unfinished Legacy data frame into a new group, leaving it open. + fn open_group( + track: &mut moq_net::track::Producer, + sequence: u64, + timestamp: Timestamp, + ) -> moq_net::group::Producer { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + Container::Legacy(crate::container::Kind::Data) + .write( + &mut group, + &[Frame { + timestamp, + payload: Bytes::from_static(&[0xDE, 0xAD]), + keyframe: false, + duration: None, + }], + ) + .unwrap(); + group + } + + /// Read every item, checking after each that the consumer holds about the max buffer at most: + /// past it, one more wire frame and the groups the cursor still needs. + async fn read_bounded(consumer: &mut Consumer, max_buffer: usize) -> Vec { + let slack = 4 * std::mem::size_of::>() + 1024; + let mut reads = Vec::new(); + loop { + let read = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + .await + .expect("consumer hung") + .unwrap(); + let used = consumer.buffered_bytes(); + assert!( + used <= max_buffer + slack, + "holding {used} bytes past a {max_buffer} byte max buffer" + ); + match read { + Some(read) => reads.push(read), + None => return reads, + } + } + } + + fn timestamps(reads: &[Read]) -> Vec { + reads + .iter() + .filter_map(|read| match read { + Read::Frame { frame, .. } => Some(frame.timestamp), + _ => None, + }) + .collect() + } + + /// Frames packed into one timestamp have no span, so the age budget never skips the stalled group + /// they queue behind. They fill the max buffer instead, which skips it, and the rest stays unread + /// in the transport's cache until the reader gets to it. + #[tokio::test] + async fn frames_sharing_a_timestamp_stay_within_the_max_buffer() { + tokio::time::pause(); + const MAX_BUFFER: usize = 4096; + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = container_max_delay_only(track.subscribe(None), Duration::from_secs(5)); + consumer.set_max_buffer(MAX_BUFFER); + + let _stalled = open_group(&mut track, 0, ts(0)); + write_group(&mut track, 1, &vec![ts(10_000); 2_000]); + write_group(&mut track, 2, &[ts(20_000), ts(30_000)]); + track.finish().unwrap(); + + let reads = read_bounded(&mut consumer, MAX_BUFFER).await; + let timestamps = timestamps(&reads); + assert_eq!(timestamps.len(), 1 + 2_000 + 2); + assert_eq!(timestamps.first(), Some(&ts(0))); + assert_eq!(timestamps.last(), Some(&ts(30_000))); + // The stalled group's tail is lost. + assert!(matches!(reads[1], Read::Gap { .. }), "{:?}", reads[1]); + } + + /// A publisher opening many groups fills the max buffer with their overhead and first frames, so + /// the rest wait in the transport's cache, where its own budget drops what falls behind: playback + /// still reaches the live edge. + #[tokio::test] + async fn many_groups_stay_within_the_max_buffer() { + tokio::time::pause(); + const MAX_BUFFER: usize = 8192; + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let subscriber = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_millis(100))); + let mut consumer = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Data)); + consumer.set_max_buffer(MAX_BUFFER); + + let _stalled = open_group(&mut track, 0, ts(0)); + for g in 1..=500u64 { + write_group(&mut track, g, &[ts(g * 1_000)]); + } + track.finish().unwrap(); + + let reads = read_bounded(&mut consumer, MAX_BUFFER).await; + let timestamps = timestamps(&reads); + assert_eq!(timestamps.first(), Some(&ts(0))); + assert_eq!(timestamps.last(), Some(&ts(500_000))); + assert!(timestamps.len() < 501, "nothing was skipped: {}", timestamps.len()); + } } From 4445cb7bff836ab5b71b050e1d82f6a529d7184e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 02:03:52 -0700 Subject: [PATCH 03/18] feat(container): finish the max buffer and retire its quest Export DEFAULT_MAX_BUFFER, let the oldest group and the first group past the cursor decode past a full cap, and gate group pulls on the cap alone. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 8 +-- js/hang/src/container/consumer.ts | 4 +- js/hang/src/container/index.ts | 2 +- quest/m1/README.md | 1 - quest/m1/container-backpressure.md | 46 --------------- rs/moq-mux/src/container/consumer.rs | 56 ++++++++++--------- rs/moq-mux/src/container/mod.rs | 2 +- 7 files changed, 37 insertions(+), 82 deletions(-) delete mode 100644 quest/m1/container-backpressure.md diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 0ad0d5e3116..ccd6e020618 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -68,17 +68,13 @@ test("Consumer holds frames sharing a timestamp within the max buffer", async () const packed = track.createGroup(1); for (let i = 0; i < 2_000; i++) write(packed, 10_000); packed.finish(); - const tail = track.createGroup(2); - write(tail, 20_000); - write(tail, 30_000); - tail.finish(); track.finish(); const reads = await readAll(consumer, format, maxBuffer); const ts = timestamps(reads); - expect(ts.length).toBe(1 + 2_000 + 2); + expect(ts.length).toBe(1 + 2_000); expect(ts.at(0)).toBe(0); - expect(ts.at(-1)).toBe(30_000); + expect(ts.at(-1)).toBe(10_000); // The stalled group's tail is lost. expect(reads[1]?.kind).toBe("gap"); } finally { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 17f37d039df..1289434db77 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -265,7 +265,9 @@ export class Consumer { for (;;) { // Only the head may decode past a full max buffer, and only once it has nothing left to read, // so reading never waits on its own lookahead. A dropped group goes on to find its close. - await this.#waitRoom(() => group.released === true || (group === this.#groups[0] && group.frames.length === 0)); + await this.#waitRoom( + () => group.released === true || (group === this.#groups[0] && group.frames.length === 0), + ); if (this.#signals.abort.aborted) return; const next = await group.consumer.readFrame(); diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts index c05edf1aaa4..14d2edf2cc7 100644 --- a/js/hang/src/container/index.ts +++ b/js/hang/src/container/index.ts @@ -7,7 +7,7 @@ /** Re-export of {@link https://jsr.io/@moq/loc | @moq/loc}, the LOC container. */ export * as Loc from "@moq/loc"; -export { Consumer, type ConsumerProps, type Read } from "./consumer"; +export { Consumer, type ConsumerProps, DEFAULT_MAX_BUFFER, type Read } from "./consumer"; export type { Format } from "./format"; /** The legacy varint container. */ export * as Legacy from "./legacy"; diff --git a/quest/m1/README.md b/quest/m1/README.md index c8fd34f95eb..755039ccce9 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -167,7 +167,6 @@ removed). - [Relay demand gaps](/quest/m1/relay-demand-gaps.md) - pre-07 backlog, IETF warm-relay newcomer, and double upstream pulls are fixed or ruled out - [Segment cuts hold bounded state](/quest/m1/segmenter-cuts.md) - the HLS segmenter keeps one pending cut, so `styp` input without sync samples cannot grow it - [Next Object on a cold Rust copy](/quest/m1/next-object-cold-copy.md) - a join after an upstream group report starts at the first frame received, never skipping -- [Container consumers buffer by backpressure](/quest/m1/container-backpressure.md) - Rust and JS container consumers decode ahead only a byte budget, so a hostile publisher cannot grow them - [Settled copies versus whole groups](/quest/m1/settled-copies.md) - every model check of whether a group is done means either a settled copy or the whole group, as it needs - [An HLS viewer starts capture](/quest/m1/hls-record-demand.md) - a record-track subscriber counts as demand for its rendition's media - [One moq-ffi build definition](/quest/m1/ffi-build-flags.md) - every binding takes moq-ffi's default features from one place diff --git a/quest/m1/container-backpressure.md b/quest/m1/container-backpressure.md deleted file mode 100644 index 4f971d84e6d..00000000000 --- a/quest/m1/container-backpressure.md +++ /dev/null @@ -1,46 +0,0 @@ -# [M] Container consumers buffer by backpressure - -## Goal - -A hostile publisher cannot make a container consumer hold unbounded decoded -frames. `@moq/hang`'s `container/consumer.ts` and moq-mux's -`container/consumer.rs` decode ahead at most a byte budget of frames; past -it, reads wait, and the undecoded data stays in the net cache, which the -pool already bounds. Many frames packed into one timestamp, or many groups -in flight, no longer grow the consumer. - -## Plan - -Decided 2026-10-27 (maintainer), from -[moq-dev/moq#6903](https://github.com/moq-dev/moq/pull/6903): bound by -backpressure in Rust and JS together, not a frame-count cap that refuses a -group. Release keeps the gap until the next cut; no backport. - -Today both consumers bound only by timestamp span: - -- JS pushes every decoded sample (`group.frames.push` in - `js/hang/src/container/consumer.ts`), and `#checkMaxDelay` measures the - span across at least two groups, so frames that share a timestamp have - zero span and are never skipped. -- Rust buffers a `VecDeque<(Frame, Instant)>` per group - (`rs/moq-mux/src/container/consumer.rs`) bounded by span and elapsed - time only. -- The net layers cap a group (32 MiB, 8192 frames) but not how many groups - a consumer holds, and one CMAF wire frame can decode to many samples. - -Steps: - -- Charge decoded frames to a per-consumer byte budget and stop pulling from - net groups while it is full, so pressure reaches the net cache and QUIC - flow control instead of a refusal. Keep the existing span skip for - latency. -- Charge each buffered sample's and group's own overhead to the budget, not - only payload bytes, and keep the oldest group moving even when later groups - fill the budget, so the consumer cannot deadlock on its own lookahead. -- Mirror names across Rust and JS. -- Test a publisher that packs many frames into one timestamp, and one that - opens many groups: memory stays near the budget and playback still skips - to the live edge. - -Public API: the consumer config may gain a budget field; mirror it in JS. -Wire: none. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index a539e8641dd..29ed9452729 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -407,9 +407,10 @@ impl Consumer { if self.startup { // NOTE: poll_min_timestamp buffers at least one frame per group and // registers the waiter on the ones still empty. Any wire frame counts, so a - // paused track whose only group is a marker still reads its end. - let any_frame = self.pending.iter_mut().any(|group| { - let _ = group.poll_min_timestamp(waiter, &self.format); + // paused track whose only group is a marker still reads its end. The oldest + // buffers past a full max buffer, so a start is always found. + let any_frame = self.pending.iter_mut().enumerate().any(|(i, group)| { + let _ = group.poll_min_timestamp(waiter, &self.format, i == 0); !group.empty }); if any_frame { @@ -547,7 +548,7 @@ impl Consumer { let current_end = if let Some(current) = self.pending.front_mut() && current.sequence <= self.current { - match current.poll_min_timestamp(waiter, &self.format) { + match current.poll_min_timestamp(waiter, &self.format, true) { Poll::Ready(Some(_)) => current.max_end, _ => None, } @@ -555,17 +556,21 @@ impl Consumer { None }; - // Find the first newer group with data (our skip target) and where it starts. + // Find the first newer group with data (our skip target) and where it starts. The first + // one past the cursor buffers past a full max buffer, so a stalled cursor always has a + // target to skip to. let mut next_group = None; + let mut lead = true; for (i, group) in self.pending.iter_mut().enumerate() { if group.sequence <= self.current { continue; } - if let Poll::Ready(Some(ts)) = group.poll_min_timestamp(waiter, &self.format) { + if let Poll::Ready(Some(ts)) = group.poll_min_timestamp(waiter, &self.format, lead) { next_group = Some((i, std::time::Duration::from(ts))); break; } + lead = false; } // Find the max timestamp across all newer groups. @@ -718,9 +723,9 @@ impl Consumer { return Poll::Ready(()); } loop { - // Past the max buffer, later groups wait in the transport's cache, unless the cursor's own - // group has not arrived yet and might be the next one. - if self.memory.full() && self.pending.front().is_some_and(|group| group.sequence <= self.current) { + // Past the max buffer, later groups wait in the transport's cache. A missing group at the + // cursor then expires like one past the budget: see the hole below. + if self.memory.full() { return Poll::Pending; } let next = match ready!(self.subscriber().poll_recv_group(waiter)) { @@ -990,13 +995,13 @@ impl GroupBuffer { // decode error is kept for the reader rather than dropped with the frame it consumed; a // transport abort is left for the cursor to find. // - // Returns false once this group can buffer nothing more, and pends without registering the waiter - // while the max buffer is full: only the reader taking frames frees it. - fn look_once(&mut self, waiter: &kio::Waiter, format: &F) -> Poll { + // Returns false once this group can buffer nothing more. When `capped`, pends without registering + // the waiter while the max buffer is full: only the reader taking frames frees it. + fn look_once(&mut self, waiter: &kio::Waiter, format: &F, capped: bool) -> Poll { if self.fault.is_some() { return Poll::Ready(false); } - if self.memory.full() { + if capped && self.memory.full() { return Poll::Pending; } match ready!(self.buffer_once(waiter, format)) { @@ -1010,7 +1015,7 @@ impl GroupBuffer { } fn look_all(&mut self, waiter: &kio::Waiter, format: &F) -> Poll<()> { - while ready!(self.look_once(waiter, format)) {} + while ready!(self.look_once(waiter, format, true)) {} Poll::Ready(()) } @@ -1027,12 +1032,13 @@ impl GroupBuffer { self.poll_ended(waiter) } - /// Poll for the minimum timestamp in this group, or `None` if it ended without one. - fn poll_min_timestamp(&mut self, waiter: &kio::Waiter, format: &F) -> Poll> { + /// Poll for the minimum timestamp in this group, or `None` if it ended without one. A `lead` group + /// buffers its first frame past a full max buffer. + fn poll_min_timestamp(&mut self, waiter: &kio::Waiter, format: &F, lead: bool) -> Poll> { // A wire frame can decode to no media frames, so keep going until one is buffered. let mut looked = Poll::Ready(()); while self.buffered.is_empty() { - match self.look_once(waiter, format) { + match self.look_once(waiter, format, !lead) { Poll::Ready(true) => continue, Poll::Ready(false) => break, Poll::Pending => { @@ -3147,7 +3153,7 @@ mod tests { // Finding the start buffers one frame; the read behind it already holds the rest. let mut buffer = GroupBuffer::::new(group.consume(), Memory::new(DEFAULT_MAX_BUFFER)); assert_eq!( - buffer.poll_min_timestamp(&waiter, &FailingDecode), + buffer.poll_min_timestamp(&waiter, &FailingDecode, false), Poll::Ready(Some(ts(0))) ); group.abort(moq_net::Error::Old).unwrap(); @@ -4102,8 +4108,8 @@ mod tests { } /// Frames packed into one timestamp have no span, so the age budget never skips the stalled group - /// they queue behind. They fill the max buffer instead, which skips it, and the rest stays unread - /// in the transport's cache until the reader gets to it. + /// they queue behind. Read ahead as the newest group, they fill the max buffer instead, which skips + /// it, and the rest stays unread in the transport's cache until the reader gets to it. #[tokio::test] async fn frames_sharing_a_timestamp_stay_within_the_max_buffer() { tokio::time::pause(); @@ -4114,21 +4120,20 @@ mod tests { let _stalled = open_group(&mut track, 0, ts(0)); write_group(&mut track, 1, &vec![ts(10_000); 2_000]); - write_group(&mut track, 2, &[ts(20_000), ts(30_000)]); track.finish().unwrap(); let reads = read_bounded(&mut consumer, MAX_BUFFER).await; let timestamps = timestamps(&reads); - assert_eq!(timestamps.len(), 1 + 2_000 + 2); + assert_eq!(timestamps.len(), 1 + 2_000); assert_eq!(timestamps.first(), Some(&ts(0))); - assert_eq!(timestamps.last(), Some(&ts(30_000))); + assert_eq!(timestamps.last(), Some(&ts(10_000))); // The stalled group's tail is lost. assert!(matches!(reads[1], Read::Gap { .. }), "{:?}", reads[1]); } /// A publisher opening many groups fills the max buffer with their overhead and first frames, so - /// the rest wait in the transport's cache, where its own budget drops what falls behind: playback - /// still reaches the live edge. + /// the rest wait in the transport's cache: playback still reaches the live edge, and what arrives + /// below the cursor once there is room is dropped as old. #[tokio::test] async fn many_groups_stay_within_the_max_buffer() { tokio::time::pause(); @@ -4147,7 +4152,6 @@ mod tests { let reads = read_bounded(&mut consumer, MAX_BUFFER).await; let timestamps = timestamps(&reads); - assert_eq!(timestamps.first(), Some(&ts(0))); assert_eq!(timestamps.last(), Some(&ts(500_000))); assert!(timestamps.len() < 501, "nothing was skipped: {}", timestamps.len()); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 562f5c15948..2c11f751514 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -30,7 +30,7 @@ pub mod loc; pub mod mkv; pub mod ts; -pub use consumer::Consumer; +pub use consumer::{Consumer, DEFAULT_MAX_BUFFER}; pub use group::GroupConsumer; pub use producer::Producer; pub(crate) use source::{ExportSource, Resume}; From 649e5c788e0dbf72746ddb85f1b97a9c08fc3cfa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 02:42:45 -0700 Subject: [PATCH 04/18] fix(container): bound every buffered item and in-flight read - JS: admit groups on room alone, let the head's successor lead so a pressure skip always has a target, re-admit each read before decoding it, and stop a dropped group's task instead of charging what no one refunds. - Rust: end the lead exemption once a group buffers an endpoint marker. - Tests: sync on task drains instead of wall-clock sleeps, cover a successor behind group overhead and an endpoint-only successor. Co-Authored-By: Claude Opus 5.5 --- js/hang/api/container.api.md | 3 ++ .../container/consumer.backpressure.test.ts | 42 +++++++++++++++--- js/hang/src/container/consumer.ts | 43 +++++++++++-------- rs/moq-mux/src/container/consumer.rs | 41 +++++++++++++++--- 4 files changed, 100 insertions(+), 29 deletions(-) diff --git a/js/hang/api/container.api.md b/js/hang/api/container.api.md index 44a7eda96c1..54efd4ea80a 100644 --- a/js/hang/api/container.api.md +++ b/js/hang/api/container.api.md @@ -26,10 +26,13 @@ export class Consumer { export interface ConsumerProps { format: Format; + maxBuffer?: number; maxDelay?: Signals.GetterInit; predecessor?: Consumer; } +export const DEFAULT_MAX_BUFFER: number; + export interface Format { decode(payload: Uint8Array, timestamp: Net.Time.Timestamp | undefined): Frame[]; end?(frame: Frame): Micro | undefined; diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index ccd6e020618..86be10b29fd 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -18,6 +18,9 @@ class Counting extends Format { const PAYLOAD = new Uint8Array([0xde, 0xad]); +// Let the consumer's group tasks run until they park: on a read, or on a full max buffer. +const flush = () => new Promise((resolve) => setImmediate(resolve)); + function write(group: { writeFrame(frame: { payload: Uint8Array; timestamp: Time.Timestamp }): void }, ts: number) { group.writeFrame({ payload: encodeFrame(PAYLOAD, ts as Micro), timestamp: Time.Timestamp.now() }); } @@ -28,16 +31,14 @@ function maxHeld(maxBuffer: number): number { return Math.ceil(maxBuffer / (64 + PAYLOAD.byteLength)) + 1; } -// Read every item. With `bounded`, check after each that the consumer holds no more than the max -// buffer allows, which only holds while no skip drops decoded frames unread. +// Read every item. With `bounded`, check after each, once decoding has caught up, that the consumer +// holds no more than the max buffer allows, which only holds while no skip drops decoded frames unread. async function readAll(consumer: Consumer, format: Counting, bounded?: number): Promise { const reads: Read[] = []; let delivered = 0; for (;;) { - const read = await Promise.race([ - consumer.next(), - new Promise((_, reject) => setTimeout(() => reject(new Error("consumer hung")), 5_000)), - ]); + const read = await consumer.next(); + await flush(); if (read === undefined) return reads; reads.push(read); if (read.kind === "frame") delivered++; @@ -105,7 +106,7 @@ test("Consumer holds many groups within the max buffer and reaches the live edge track.finish(); // The groups' overhead and first frames fill the max buffer, and the rest wait unpulled. - await new Promise((resolve) => setTimeout(resolve, 50)); + await flush(); expect(format.decoded).toBeLessThanOrEqual(maxHeld(maxBuffer)); expect(format.decoded).toBeGreaterThan(1); @@ -119,3 +120,30 @@ test("Consumer holds many groups within the max buffer and reaches the live edge warn.mockRestore(); } }); + +test("Consumer skips a stalled group once group overhead alone fills the max buffer", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const format = new Counting("data"); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format, + maxDelay: Milli(5_000), + // Two groups' overhead, so nothing past the stalled head would decode without its successor leading. + maxBuffer: 512, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + const next = track.createGroup(1); + write(next, 10_000); + next.finish(); + + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 0 } }); + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 10_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 1289434db77..ff29586c58b 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -25,9 +25,10 @@ export interface ConsumerProps { * included (default: {@link DEFAULT_MAX_BUFFER}). Mirrors Rust's `Consumer::set_max_buffer`. * * Once it is full the consumer stops decoding ahead and stops pulling new groups, so the undecoded - * rest stays in the transport's cache, which bounds it. The group being read still decodes one wire - * frame at a time, so reading never waits on its own lookahead. If that group stalls while later - * groups fill the cap, it is skipped like a slow group: no more of what it waits behind can be held. + * rest stays in the transport's cache, which bounds it. The group being read, and the one after it, + * still decode a wire frame whenever they hold nothing, so reading never waits on its own lookahead. + * If the group being read stalls while later groups fill the cap, it is skipped like a slow group: + * no more of what it waits behind can be held. */ maxBuffer?: number; /** @@ -208,12 +209,9 @@ export class Consumer { // Start fetching groups in the background try { for (;;) { - // Past the max buffer, later groups wait in the transport's cache, unless the group being - // read has not arrived yet and might be the next one. - await this.#waitRoom(() => { - const head = this.#groups.at(0); - return head === undefined || this.#active === undefined || head.consumer.sequence > this.#active; - }); + // Past the max buffer, later groups wait in the transport's cache. A missing group at the + // cursor then expires like one past the budget: see next(). + await this.#waitRoom(() => false); if (this.#signals.abort.aborted) break; const consumer = await this.#track.recvGroup(); @@ -263,15 +261,12 @@ export class Consumer { let index = 0; for (;;) { - // Only the head may decode past a full max buffer, and only once it has nothing left to read, - // so reading never waits on its own lookahead. A dropped group goes on to find its close. - await this.#waitRoom( - () => group.released === true || (group === this.#groups[0] && group.frames.length === 0), - ); - if (this.#signals.abort.aborted) return; - + if (!(await this.#admit(group))) return; const next = await group.consumer.readFrame(); if (!next) break; + // The read may have waited while a skip dropped this group or other groups filled the max + // buffer, so admit what it decodes again: the max buffer bounds reads in flight too. + if (!(await this.#admit(group))) return; group.empty = false; const arrival = Nano.toMilli(Nano(now())); @@ -382,7 +377,8 @@ export class Consumer { } finally { group.done = true; - if (group.consumer.sequence === this.#active) { + // A dropped group no longer holds the cursor. + if (!group.released && group.consumer.sequence === this.#active) { this.#recordPresented(group); // Advance to the next buffered group's actual sequence, but ONLY if it continues this @@ -556,6 +552,19 @@ export class Consumer { this.#refund(group, group.charged); } + // Wait until `group` may decode another wire frame: the max buffer has room, or the group leads with + // nothing buffered. The head leads so reading never waits on its own lookahead, and the group past it + // so a stalled head always has a successor to skip to. False once the group is dropped or the + // consumer closes, so its task stops rather than charge what no one will refund. + async #admit(group: Group): Promise { + await this.#waitRoom( + () => + group.released === true || + (group.frames.length === 0 && (group === this.#groups[0] || group === this.#groups[1])), + ); + return group.released !== true && !this.#signals.abort.aborted; + } + // Wait until the max buffer has room, or `exempt` holds, re-checked whenever it may have changed. async #waitRoom(exempt: () => boolean): Promise { const abort = this.#signals.abort; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 29ed9452729..9b0c1653444 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -108,9 +108,10 @@ pub(crate) enum Item { /// Decoded frames waiting to be read are charged to a byte cap, [`set_max_buffer`](Self::set_max_buffer), /// along with each buffered frame's and group's own overhead. Once it is full the consumer stops /// decoding ahead and stops pulling new groups, so the undecoded rest stays in the transport's cache, -/// which bounds it. The group the cursor is on still decodes one wire frame at a time, so reading never -/// waits on its own lookahead. If that group stalls while later groups fill the cap, it is skipped like -/// a slow group: the consumer can hold no more of what it would wait behind. +/// which bounds it. The group the cursor is on, and the first one past it, still decode a wire frame +/// whenever they hold nothing, so reading never waits on its own lookahead. If the cursor's group stalls +/// while later groups fill the cap, it is skipped like a slow group: the consumer can hold no more of +/// what it would wait behind. pub struct Consumer { // Dropped once an error ends the reader, releasing the publisher's demand. track: Option, @@ -1033,12 +1034,13 @@ impl GroupBuffer { } /// Poll for the minimum timestamp in this group, or `None` if it ended without one. A `lead` group - /// buffers its first frame past a full max buffer. + /// buffers past a full max buffer only while it holds nothing, endpoint markers included. fn poll_min_timestamp(&mut self, waiter: &kio::Waiter, format: &F, lead: bool) -> Poll> { // A wire frame can decode to no media frames, so keep going until one is buffered. let mut looked = Poll::Ready(()); while self.buffered.is_empty() { - match self.look_once(waiter, format, !lead) { + let capped = !lead || !self.markers.is_empty(); + match self.look_once(waiter, format, capped) { Poll::Ready(true) => continue, Poll::Ready(false) => break, Poll::Pending => { @@ -4155,4 +4157,33 @@ mod tests { assert_eq!(timestamps.last(), Some(&ts(500_000))); assert!(timestamps.len() < 501, "nothing was skipped: {}", timestamps.len()); } + + /// A successor holding only endpoint markers stops buffering at the max buffer like any other + /// lookahead, rather than reading past it in search of a media frame. + #[test] + fn an_endpoint_only_successor_stays_within_the_max_buffer() { + const MAX_BUFFER: usize = 1024; + let waiter = kio::Waiter::noop(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let subscriber = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(30))); + let mut consumer = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); + consumer.set_max_buffer(MAX_BUFFER); + + let _stalled = open_group(&mut track, 0, ts(0)); + let mut markers = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + for _ in 0..2_000 { + write_marker(&mut markers, ts(10_000)); + } + + let first = consumer.poll_read(&waiter); + assert!(matches!(first, Poll::Ready(Ok(Some(Read::Frame { .. })))), "{first:?}"); + assert!(consumer.poll_read(&waiter).is_pending()); + let used = consumer.buffered_bytes(); + let slack = 2 * std::mem::size_of::>() + MARKER_COST; + assert!( + used <= MAX_BUFFER + slack, + "holding {used} bytes past a {MAX_BUFFER} byte max buffer" + ); + } } From cc5563a37d842aa1f21f44c070b91beaf29bf31d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 03:01:04 -0700 Subject: [PATCH 05/18] fix(container): move past a stalled group under pressure, read or not Design change: once the max buffer is full and the cursor's group stalls, the cursor moves to the next pending group even with nothing of it read. It reads as the cursor's group, and each skip refunds what the group left behind held, so progress no longer depends on any later group decoding past the cap. This drops the successor exemption. - Rust: re-pull groups held back by a full cap once a skip refunds room. - JS: check admission synchronously before each decode and charge, so tasks woken together cannot all spend the same room. - Tests: an endpoint-only successor with media past its markers, a stalled successor in front of a readable group, and reads completing together. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 57 +++++++++ js/hang/src/container/consumer.ts | 72 ++++++----- rs/moq-mux/src/container/consumer.rs | 119 ++++++++++++++---- 3 files changed, 190 insertions(+), 58 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 86be10b29fd..b9e2e201c20 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -147,3 +147,60 @@ test("Consumer skips a stalled group once group overhead alone fills the max buf warn.mockRestore(); } }); + +test("Consumer reaches a readable group past a stalled successor once group overhead fills the max buffer", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const format = new Counting("data"); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format, + maxDelay: Milli.zero, + // Three groups' overhead: the readable group cannot decode until the stalled ones are moved past. + maxBuffer: 768, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + track.createGroup(1); + const readable = track.createGroup(2); + write(readable, 10_000); + write(readable, 20_000); + readable.finish(); + + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 0 } }); + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 10_000 } }); + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 20_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + +test("Consumer admits reads completing together one at a time against the max buffer", async () => { + const format = new Counting("data"); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format, + maxDelay: Milli(5_000), + maxBuffer: 4096, + }); + try { + // Ten open groups park on their first read with room to spare, then large frames land in all of + // them at once. + const groups = Array.from({ length: 10 }, (_, sequence) => track.createGroup(sequence)); + await flush(); + const large = new Uint8Array(2_000); + for (const group of groups) { + group.writeFrame({ payload: encodeFrame(large, 0 as Micro), timestamp: Time.Timestamp.now() }); + } + await flush(); + + // The first decode fills the max buffer, so only the head may decode past it. + expect(format.decoded).toBeLessThanOrEqual(2); + } finally { + consumer.close(); + track.close(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index ff29586c58b..ff4362ab869 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -25,10 +25,10 @@ export interface ConsumerProps { * included (default: {@link DEFAULT_MAX_BUFFER}). Mirrors Rust's `Consumer::set_max_buffer`. * * Once it is full the consumer stops decoding ahead and stops pulling new groups, so the undecoded - * rest stays in the transport's cache, which bounds it. The group being read, and the one after it, - * still decode a wire frame whenever they hold nothing, so reading never waits on its own lookahead. - * If the group being read stalls while later groups fill the cap, it is skipped like a slow group: - * no more of what it waits behind can be held. + * rest stays in the transport's cache, which bounds it. The group being read still decodes a wire + * frame whenever it holds nothing, so reading never waits on its own lookahead. If it stalls while the + * cap is full, the consumer moves on to the next group like a skip past a slow group, read or not: no + * more of what it waits behind can be held. */ maxBuffer?: number; /** @@ -211,7 +211,7 @@ export class Consumer { for (;;) { // Past the max buffer, later groups wait in the transport's cache. A missing group at the // cursor then expires like one past the budget: see next(). - await this.#waitRoom(() => false); + while (this.#full() && !this.#signals.abort.aborted) await this.#park(); if (this.#signals.abort.aborted) break; const consumer = await this.#track.recvGroup(); @@ -246,6 +246,11 @@ export class Consumer { const next = this.#groups.at(index); if (next && next.late < group.late) group.late = next.late; this.#groups.splice(index, 0, group); + // Group overhead alone can fill the max buffer, past which next() moves on from a stalled head. + if (this.#full()) { + this.#notify?.(); + this.#notify = undefined; + } // Start buffering frames from this group this.#signals.spawn(this.#runGroup.bind(this, group)); @@ -261,12 +266,16 @@ export class Consumer { let index = 0; for (;;) { - if (!(await this.#admit(group))) return; + while (!this.#stopped(group) && !this.#admits(group)) await this.#park(); + if (this.#stopped(group)) return; const next = await group.consumer.readFrame(); if (!next) break; // The read may have waited while a skip dropped this group or other groups filled the max - // buffer, so admit what it decodes again: the max buffer bounds reads in flight too. - if (!(await this.#admit(group))) return; + // buffer, so admit what it decodes again: the max buffer bounds reads in flight too. Nothing + // yields between this check and the charge below, or every task woken together would spend + // the same room. + while (!this.#stopped(group) && !this.#admits(group)) await this.#park(); + if (this.#stopped(group)) return; group.empty = false; const arrival = Nano.toMilli(Nano(now())); @@ -502,9 +511,11 @@ export class Consumer { return true; } - // Skip the stalled active group once later groups fill the max buffer, which can hold no more of - // what it waits behind. Only fires when the active group is fully consumed and still open, so we - // never drop frames the consumer hasn't seen. Returns true if a group was skipped. + // Move on from the stalled active group once the max buffer is full, since it can hold no more of + // what it waits behind, to the next group even with nothing of it read yet: that one reads as the + // head, and each such skip refunds what the group left behind held. Only fires when the active group + // is fully consumed and still open, so we never drop frames the consumer hasn't seen. Returns true if + // a group was skipped. #tryPressureSkip(): boolean { if (this.#active === undefined || !this.#full()) return false; @@ -513,14 +524,15 @@ export class Consumer { if (active.done || active.frames.length > 0) return false; const next = this.#groups[1]; - const nextStart = next?.frames.at(0)?.timestamp; - if (!next || nextStart === undefined) return false; + if (!next) return false; this.#groups.shift(); console.warn( `skipping a stalled group past the max buffer: track=${this.#track.name} ${active.consumer.sequence} -> ${next.consumer.sequence}`, ); - // Still open, so its tail is lost unless the next group picks up where it ended. + // Still open, so its tail is lost unless the next group picks up where it ended, which nothing + // proves before its start is read. + const nextStart = next.frames.at(0)?.timestamp; if (active.truncated || !ptsContiguous(active.end ?? this.#presentedEnd, nextStart)) this.#lost = true; this.#recordPresented(active); this.#active = next.consumer.sequence; @@ -552,25 +564,21 @@ export class Consumer { this.#refund(group, group.charged); } - // Wait until `group` may decode another wire frame: the max buffer has room, or the group leads with - // nothing buffered. The head leads so reading never waits on its own lookahead, and the group past it - // so a stalled head always has a successor to skip to. False once the group is dropped or the - // consumer closes, so its task stops rather than charge what no one will refund. - async #admit(group: Group): Promise { - await this.#waitRoom( - () => - group.released === true || - (group.frames.length === 0 && (group === this.#groups[0] || group === this.#groups[1])), - ); - return group.released !== true && !this.#signals.abort.aborted; + // Whether `group` may decode another wire frame: the max buffer has room, or it is the head with + // nothing buffered, so reading never waits on its own lookahead. + #admits(group: Group): boolean { + return !this.#full() || (group === this.#groups[0] && group.frames.length === 0); } - // Wait until the max buffer has room, or `exempt` holds, re-checked whenever it may have changed. - async #waitRoom(exempt: () => boolean): Promise { - const abort = this.#signals.abort; - while (this.#full() && !exempt() && !abort.aborted) { - await new Promise((resolve) => this.#room.push(resolve)); - } + // Whether `group`'s task should stop: the group was dropped, so what it decodes would never be + // refunded, or the consumer closed. + #stopped(group: Group): boolean { + return group.released === true || this.#signals.abort.aborted; + } + + // Park until something may have freed room or moved the head; see #wake. + #park(): Promise { + return new Promise((resolve) => this.#room.push(resolve)); } // Let parked tasks re-check the max buffer: something was refunded, or the head or cursor moved. @@ -676,7 +684,7 @@ export class Consumer { head.latest - this.#presentedEnd > maxDelay)); const elapsed = Instant.durationSince(now(), this.#hole) >= Nano.fromMicro(maxDelay); // The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it. - const full = head.frames.length > 0 && this.#full(); + const full = this.#full(); if (startup || empty || contiguous || skipHole || elapsed || full || ended !== undefined) { if (!contiguous && !empty) { this.#lost = true; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 9b0c1653444..32defd54b8a 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -108,10 +108,10 @@ pub(crate) enum Item { /// Decoded frames waiting to be read are charged to a byte cap, [`set_max_buffer`](Self::set_max_buffer), /// along with each buffered frame's and group's own overhead. Once it is full the consumer stops /// decoding ahead and stops pulling new groups, so the undecoded rest stays in the transport's cache, -/// which bounds it. The group the cursor is on, and the first one past it, still decode a wire frame -/// whenever they hold nothing, so reading never waits on its own lookahead. If the cursor's group stalls -/// while later groups fill the cap, it is skipped like a slow group: the consumer can hold no more of -/// what it would wait behind. +/// which bounds it. The group the cursor is on still decodes a wire frame whenever it holds nothing, so +/// reading never waits on its own lookahead. If that group stalls while the cap is full, the cursor +/// moves on to the next group like a skip past a slow group, read or not: the consumer can hold no +/// more of what it would wait behind. pub struct Consumer { // Dropped once an error ends the reader, releasing the publisher's demand. track: Option, @@ -128,6 +128,10 @@ pub struct Consumer { // What the pending groups hold, against the max buffer. memory: Memory, + // The last pull stopped at a full max buffer rather than at the track, so nothing wakes the reader + // once a skip refunds room: it pulls again itself. + gated: bool, + // Latches the cursor onto the publisher's first served group when the // subscription names no start. Cleared once a first group is chosen. startup: bool, @@ -266,6 +270,7 @@ impl Consumer { current: start.unwrap_or(0), pending: VecDeque::new(), memory: Memory::new(DEFAULT_MAX_BUFFER), + gated: false, startup: start.is_none(), newest: false, max_delay, @@ -388,6 +393,16 @@ impl Consumer { } fn poll_event_inner(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + loop { + match self.poll_event_once(waiter) { + // A skip refunded room for groups the max buffer held back, so pull them. + Poll::Pending if std::mem::take(&mut self.gated) && !self.memory.full() => continue, + res => return res, + } + } + } + + fn poll_event_once(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { // Grab any new groups from the track, recording whether the track is finished. let finished = self.poll_read_finish(waiter).is_ready(); @@ -557,21 +572,17 @@ impl Consumer { None }; - // Find the first newer group with data (our skip target) and where it starts. The first - // one past the cursor buffers past a full max buffer, so a stalled cursor always has a - // target to skip to. + // Find the first newer group with data (our skip target) and where it starts. let mut next_group = None; - let mut lead = true; for (i, group) in self.pending.iter_mut().enumerate() { if group.sequence <= self.current { continue; } - if let Poll::Ready(Some(ts)) = group.poll_min_timestamp(waiter, &self.format, lead) { + if let Poll::Ready(Some(ts)) = group.poll_min_timestamp(waiter, &self.format, false) { next_group = Some((i, std::time::Duration::from(ts))); break; } - lead = false; } // Find the max timestamp across all newer groups. @@ -646,19 +657,27 @@ impl Consumer { // short whenever it blocks behind a newer one, as a join's backlog does // while the newer groups race ahead on their own streams. Skip early too // once it has presented up to where the next group begins (duration - // coverage), since nothing is left worth waiting for, or once the later groups fill the max - // buffer, since no more of them can be held while it waits. - let full = self.memory.full(); + // coverage), since nothing is left worth waiting for. let should_skip = next_group.is_some_and(|(_, next_start)| { max_timestamp.saturating_sub(next_start) >= self.max_delay || current_end.is_some_and(|end| end >= next_start) - || full }); + let target = match should_skip { + true => next_group.map(|(i, start)| (i, Some(start))), + // Once the later groups fill the max buffer, no more of them can be held while it waits, + // so move on to the next group even with nothing of it read yet: it reads as the + // cursor's group, and each such skip refunds what the group left behind held. + false if self.memory.full() => self + .pending + .iter() + .position(|group| group.sequence > self.current) + .map(|i| (i, self.pending[i].min_timestamp.map(std::time::Duration::from))), + false => None, + }; - if let Some((new_idx, next_start)) = next_group - && should_skip - { - let hole = !pts_contiguous(current_end.or(self.presented_end), next_start); + if let Some((new_idx, next_start)) = target { + // Without a start read yet, nothing proves the timeline continues. + let hole = next_start.is_none_or(|start| !pts_contiguous(current_end.or(self.presented_end), start)); let had_marker = self.pending.iter().take(new_idx).any(GroupBuffer::marker); // Buffered frames are thrown away, and `current_end` counts them, so contiguity alone would hide them. let discarded = self @@ -720,6 +739,7 @@ impl Consumer { // // Returns Pending until all groups have been consumed. fn poll_read_finish(&mut self, waiter: &kio::Waiter) -> Poll<()> { + self.gated = false; if self.track_error.is_some() { return Poll::Ready(()); } @@ -727,6 +747,7 @@ impl Consumer { // Past the max buffer, later groups wait in the transport's cache. A missing group at the // cursor then expires like one past the budget: see the hole below. if self.memory.full() { + self.gated = true; return Poll::Pending; } let next = match ready!(self.subscriber().poll_recv_group(waiter)) { @@ -4175,15 +4196,61 @@ mod tests { for _ in 0..2_000 { write_marker(&mut markers, ts(10_000)); } + Container::Legacy(crate::container::Kind::Data) + .write( + &mut markers, + &[Frame { + timestamp: ts(20_000), + payload: Bytes::from_static(b"video"), + keyframe: false, + duration: None, + }], + ) + .unwrap(); + markers.finish().unwrap(); - let first = consumer.poll_read(&waiter); - assert!(matches!(first, Poll::Ready(Ok(Some(Read::Frame { .. })))), "{first:?}"); - assert!(consumer.poll_read(&waiter).is_pending()); - let used = consumer.buffered_bytes(); + // The markers fill the max buffer behind the stalled group, which moves the cursor on to them, + // and the media past them still plays. let slack = 2 * std::mem::size_of::>() + MARKER_COST; - assert!( - used <= MAX_BUFFER + slack, - "holding {used} bytes past a {MAX_BUFFER} byte max buffer" - ); + let mut frames = Vec::new(); + loop { + let read = consumer.poll_read(&waiter); + let used = consumer.buffered_bytes(); + assert!( + used <= MAX_BUFFER + slack, + "holding {used} bytes past a {MAX_BUFFER} byte max buffer" + ); + match read { + Poll::Ready(Ok(Some(Read::Frame { frame, .. }))) => frames.push(frame.timestamp), + Poll::Ready(Ok(Some(_))) => continue, + Poll::Pending => break, + other => panic!("unexpected read: {other:?}"), + } + } + assert_eq!(frames, vec![ts(0), ts(20_000)]); + } + + /// A stalled successor does not hide a readable group behind it once group overhead alone fills the + /// max buffer: the cursor moves on group by group until one has something to read. + #[test] + fn a_stalled_successor_does_not_hide_a_readable_group() { + let waiter = kio::Waiter::noop(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = container_max_delay_only(track.subscribe(None), Duration::from_secs(5)); + consumer.set_max_buffer(2 * std::mem::size_of::>()); + + let _stalled = open_group(&mut track, 0, ts(0)); + let _empty = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + write_group(&mut track, 2, &[ts(20_000)]); + + let mut reads = Vec::new(); + while let Poll::Ready(read) = consumer.poll_read(&waiter) { + match read.unwrap() { + Some(Read::Frame { frame, .. }) => reads.push(Some(frame.timestamp)), + Some(Read::Gap { .. }) => reads.push(None), + other => panic!("unexpected read: {other:?}"), + } + } + assert_eq!(reads, vec![Some(ts(0)), None, Some(ts(20_000))]); } } From bb0985e62618054a4d40d77c16a773e3799beb8c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 03:12:31 -0700 Subject: [PATCH 06/18] fix(container): hold no payload while parked, and read through any cap - JS: wait for a frame with readable() and take it only once admitted, so a group parked on the max buffer holds no wire payload. - Rust and JS: always pull a successor to the cursor's group, so a stalled group can be skipped however small the cap, zero included. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 28 ++++++++++++++ js/hang/src/container/consumer.ts | 23 ++++++------ rs/moq-mux/src/container/consumer.rs | 37 +++++++++++++++++-- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index b9e2e201c20..7110abdd58d 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -204,3 +204,31 @@ test("Consumer admits reads completing together one at a time against the max bu track.close(); } }); + +for (const maxBuffer of [0, 256]) { + test(`Consumer moves past a stalled group with a ${maxBuffer} byte max buffer`, async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + // No more than one group's overhead: the successor is pulled anyway, so the head can be skipped. + maxBuffer, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + const next = track.createGroup(1); + write(next, 10_000); + next.finish(); + + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 0 } }); + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 10_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } + }); +} diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index ff4362ab869..82615f1a523 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -24,8 +24,8 @@ export interface ConsumerProps { * Cap the bytes of decoded frames held ahead of the reader, each frame's and group's own overhead * included (default: {@link DEFAULT_MAX_BUFFER}). Mirrors Rust's `Consumer::set_max_buffer`. * - * Once it is full the consumer stops decoding ahead and stops pulling new groups, so the undecoded - * rest stays in the transport's cache, which bounds it. The group being read still decodes a wire + * Once it is full the consumer stops decoding ahead and stops pulling new groups past one successor + * to the group being read, so the undecoded rest stays in the transport's cache, which bounds it. The group being read still decodes a wire * frame whenever it holds nothing, so reading never waits on its own lookahead. If it stalls while the * cap is full, the consumer moves on to the next group like a skip past a slow group, read or not: no * more of what it waits behind can be held. @@ -209,9 +209,10 @@ export class Consumer { // Start fetching groups in the background try { for (;;) { - // Past the max buffer, later groups wait in the transport's cache. A missing group at the - // cursor then expires like one past the budget: see next(). - while (this.#full() && !this.#signals.abort.aborted) await this.#park(); + // Past the max buffer, later groups wait in the transport's cache, once the head has a successor + // to move on to: however small the cap, a stalled head can always be skipped. A missing group + // at the cursor then expires like one past the budget: see next(). + while (this.#full() && this.#groups.length >= 2 && !this.#signals.abort.aborted) await this.#park(); if (this.#signals.abort.aborted) break; const consumer = await this.#track.recvGroup(); @@ -266,15 +267,15 @@ export class Consumer { let index = 0; for (;;) { + // Wait for a frame without taking it, so a group parked on the max buffer holds nothing. + await group.consumer.readable(); while (!this.#stopped(group) && !this.#admits(group)) await this.#park(); if (this.#stopped(group)) return; - const next = await group.consumer.readFrame(); + // Nothing yields between the check above and the charge below, or every task woken together + // would spend the same room. A buffered frame is taken synchronously; only an end or a + // failure, which carries no frame, is left for readFrame to report. + const next = group.consumer.tryReadFrame() ?? (await group.consumer.readFrame()); if (!next) break; - // The read may have waited while a skip dropped this group or other groups filled the max - // buffer, so admit what it decodes again: the max buffer bounds reads in flight too. Nothing - // yields between this check and the charge below, or every task woken together would spend - // the same room. - while (!this.#stopped(group) && !this.#admits(group)) await this.#park(); if (this.#stopped(group)) return; group.empty = false; const arrival = Nano.toMilli(Nano(now())); diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 32defd54b8a..84edd7e230b 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -107,8 +107,8 @@ pub(crate) enum Item { /// /// Decoded frames waiting to be read are charged to a byte cap, [`set_max_buffer`](Self::set_max_buffer), /// along with each buffered frame's and group's own overhead. Once it is full the consumer stops -/// decoding ahead and stops pulling new groups, so the undecoded rest stays in the transport's cache, -/// which bounds it. The group the cursor is on still decodes a wire frame whenever it holds nothing, so +/// decoding ahead and stops pulling new groups past one successor to the cursor's group, so the +/// undecoded rest stays in the transport's cache, which bounds it. The group the cursor is on still decodes a wire frame whenever it holds nothing, so /// reading never waits on its own lookahead. If that group stalls while the cap is full, the cursor /// moves on to the next group like a skip past a slow group, read or not: the consumer can hold no /// more of what it would wait behind. @@ -396,7 +396,7 @@ impl Consumer { loop { match self.poll_event_once(waiter) { // A skip refunded room for groups the max buffer held back, so pull them. - Poll::Pending if std::mem::take(&mut self.gated) && !self.memory.full() => continue, + Poll::Pending if std::mem::take(&mut self.gated) && !self.pull_gated() => continue, res => return res, } } @@ -746,7 +746,7 @@ impl Consumer { loop { // Past the max buffer, later groups wait in the transport's cache. A missing group at the // cursor then expires like one past the budget: see the hole below. - if self.memory.full() { + if self.pull_gated() { self.gated = true; return Poll::Pending; } @@ -782,6 +782,12 @@ impl Consumer { } } + // Whether the max buffer holds back the next group: it is full, and the cursor's group already has a + // successor to move on to, so however small the cap, a stalled group can always be skipped. + fn pull_gated(&self) -> bool { + self.memory.full() && self.pending.len() >= 2 + } + // Drop the pending groups `pick` selects without delivering them. A drop must not hide what // was already found, so the lowest one holding a fault, or with frames below the start of the // latest delivered group, ends the reader instead. What was never read stays unread: decoding @@ -4253,4 +4259,27 @@ mod tests { } assert_eq!(reads, vec![Some(ts(0)), None, Some(ts(20_000))]); } + + /// No cap is too small to read through: the cursor's group and one successor are always pulled, so + /// a stalled group can be moved past even when one group's overhead fills the max buffer. + #[test] + fn a_zero_max_buffer_still_moves_past_a_stalled_group() { + let waiter = kio::Waiter::noop(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = container_max_delay_only(track.subscribe(None), Duration::from_secs(5)); + consumer.set_max_buffer(0); + + let _stalled = open_group(&mut track, 0, ts(0)); + write_group(&mut track, 1, &[ts(10_000)]); + + let mut reads = Vec::new(); + while let Poll::Ready(read) = consumer.poll_read(&waiter) { + match read.unwrap() { + Some(Read::Frame { frame, .. }) => reads.push(Some(frame.timestamp)), + Some(Read::Gap { .. }) => reads.push(None), + other => panic!("unexpected read: {other:?}"), + } + } + assert_eq!(reads, vec![Some(ts(0)), None, Some(ts(10_000))]); + } } From 0e9e8f9c8d874be6e7173c159c77d8115b2f68de Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 03:22:15 -0700 Subject: [PATCH 07/18] fix(container): move past a stalled backlog head under pressure JS admits groups below the cursor, so a stalled backlog head could hold the max buffer while the pressure skip only looked at the active group. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 28 ++++++++++++++ js/hang/src/container/consumer.ts | 37 +++++++++++-------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 7110abdd58d..129ff75510a 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -232,3 +232,31 @@ for (const maxBuffer of [0, 256]) { } }); } + +test("Consumer moves past a stalled backlog group below the cursor once the max buffer is full", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + // Two groups' overhead: the backlog group and the active one leave no room for the newer group. + maxBuffer: 512, + }); + try { + const active = track.createGroup(3); + write(active, 30_000); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 3 }); + + track.createGroup(1); + const newer = track.createGroup(4); + write(newer, 40_000); + newer.finish(); + + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 4, frame: { timestamp: 40_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 82615f1a523..54666939e4d 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -512,33 +512,38 @@ export class Consumer { return true; } - // Move on from the stalled active group once the max buffer is full, since it can hold no more of - // what it waits behind, to the next group even with nothing of it read yet: that one reads as the - // head, and each such skip refunds what the group left behind held. Only fires when the active group - // is fully consumed and still open, so we never drop frames the consumer hasn't seen. Returns true if - // a group was skipped. + // Move on from the stalled head once the max buffer is full, since it can hold no more of what it + // waits behind, to the next group even with nothing of it read yet: that one reads as the head, and + // each such skip refunds what the group left behind held. The head may be the active group or a + // backlog group admitted below it. Only fires when the head is fully consumed and still open, so we + // never drop frames the consumer hasn't seen. Returns true if a group was skipped. #tryPressureSkip(): boolean { if (this.#active === undefined || !this.#full()) return false; - const active = this.#groups[0]; - if (!active || active.consumer.sequence !== this.#active) return false; - if (active.done || active.frames.length > 0) return false; + const head = this.#groups[0]; + if (!head || head.consumer.sequence > this.#active) return false; + if (head.done || head.frames.length > 0) return false; const next = this.#groups[1]; if (!next) return false; this.#groups.shift(); console.warn( - `skipping a stalled group past the max buffer: track=${this.#track.name} ${active.consumer.sequence} -> ${next.consumer.sequence}`, + `skipping a stalled group past the max buffer: track=${this.#track.name} ${head.consumer.sequence} -> ${next.consumer.sequence}`, ); - // Still open, so its tail is lost unless the next group picks up where it ended, which nothing - // proves before its start is read. - const nextStart = next.frames.at(0)?.timestamp; - if (active.truncated || !ptsContiguous(active.end ?? this.#presentedEnd, nextStart)) this.#lost = true; - this.#recordPresented(active); - this.#active = next.consumer.sequence; + if (head.consumer.sequence === this.#active) { + // Still open, so its tail is lost unless the next group picks up where it ended, which nothing + // proves before its start is read. + const nextStart = next.frames.at(0)?.timestamp; + if (head.truncated || !ptsContiguous(head.end ?? this.#presentedEnd, nextStart)) this.#lost = true; + this.#recordPresented(head); + this.#active = next.consumer.sequence; + } else { + // A backlog group below the cursor, which stays put: what the group would still have delivered is lost. + this.#lost = true; + } - this.#discard(active); + this.#discard(head); this.#updateBuffered(); return true; } From d6cda6ada0b73cd76390a318f4c2eb6652ff326b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 03:32:14 -0700 Subject: [PATCH 08/18] fix(container): read a promoted head before skipping it under pressure Extract the per-frame decode into a synchronous #decodeBuffered, shared by each group's task and the pressure skip, so a head whose task has not run since its promotion reads what the transport already holds instead of being skipped as stalled. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 35 +++ js/hang/src/container/consumer.ts | 230 ++++++++++-------- 2 files changed, 158 insertions(+), 107 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 129ff75510a..b67db0cbe18 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -260,3 +260,38 @@ test("Consumer moves past a stalled backlog group below the cursor once the max warn.mockRestore(); } }); + +test("Consumer reads what a promoted head already holds before skipping it under pressure", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + maxBuffer: 1024, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0 }); + + // A later group's large frame fills the max buffer, then a frame lands in the group between, + // stamped on the wire before the later group's. + const between = track.createGroup(1); + const later = track.createGroup(2); + const early = Time.Timestamp.now(); + later.writeFrame({ + payload: encodeFrame(new Uint8Array(2_000), 20_000 as Micro), + timestamp: Time.Timestamp.now(), + }); + await flush(); + between.writeFrame({ payload: encodeFrame(PAYLOAD, 10_000 as Micro), timestamp: early }); + + // Group 0 is skipped, but group 1 is read once promoted rather than skipped before its task runs. + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 1, frame: { timestamp: 10_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 54666939e4d..05d85b7b344 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -85,6 +85,7 @@ interface Group { done?: boolean; // Set when #runGroup finishes reading all frames truncated?: boolean; // The missing tail becomes a gap after the buffered frames are delivered. fault?: Error; // The payload that failed to decode, an error after the buffered frames are delivered. + index: number; // Media frames decoded so far: the first is a keyframe by protocol invariant charged: number; // What this group holds against the max buffer released?: boolean; // Dropped from #groups, with what it held refunded } @@ -235,6 +236,7 @@ export class Consumer { frames: [], empty: true, media: false, + index: 0, charged: 0, }; this.#charge(group, GROUP_COST); @@ -264,117 +266,19 @@ export class Consumer { async #runGroup(group: Group) { try { - let index = 0; - for (;;) { // Wait for a frame without taking it, so a group parked on the max buffer holds nothing. await group.consumer.readable(); while (!this.#stopped(group) && !this.#admits(group)) await this.#park(); if (this.#stopped(group)) return; - // Nothing yields between the check above and the charge below, or every task woken together - // would spend the same room. A buffered frame is taken synchronously; only an end or a - // failure, which carries no frame, is left for readFrame to report. - const next = group.consumer.tryReadFrame() ?? (await group.consumer.readFrame()); - if (!next) break; - if (this.#stopped(group)) return; - group.empty = false; - const arrival = Nano.toMilli(Nano(now())); - - // A malformed payload is not loss: it ends the reader where it sits, rather than truncating the group. - let decoded: Frame[]; - try { - decoded = this.#format.decode(next.payload, next.timestamp); - } catch (err) { - group.fault = err instanceof Error ? err : new Error(String(err)); - return; - } - if (this.#format.kind === "data") { - // A data track's marker is one object with no bytes, alone in its group. - if (group.bare || (decoded.length === 0 && group.media)) { - group.fault = new Error("a data track's discontinuity marker shares its group"); - return; - } - group.bare = decoded.length === 0; - } - - for (const sample of decoded) { - const marker = this.#format.end?.(sample) !== undefined; - - // Observed here, before any gap handling or age budget: a target derived from - // what survives the budget would only ever confirm the budget it was cut to. - if (!marker) { - this.#jitter.observe(arrival, Milli.fromMicro(sample.timestamp)); - this.#spread.set(this.#jitter.measured); - } - const frame: ReceivedFrame = { - arrival, - payload: sample.payload, - timestamp: sample.timestamp, - // Protocol invariant: groups always start at a keyframe. - // For index 0, we enforce this regardless of what the format reports. - // For index > 0, we trust the format's keyframe detection. - keyframe: !marker && index === 0 ? true : sample.keyframe, - // Carry the container's per-sample duration through so group.end is the real - // presentation end (ts + duration), not just the last frame's ts. This is what - // makes the PTS-contiguity check (next.firstPTS <= group.end) work; without it a - // contiguous next group looks one frame past the end. Undefined for Legacy (no duration). - duration: sample.duration, - }; - - if (!marker) { - index++; - group.media = true; - if (group.minMedia === undefined || frame.timestamp < group.minMedia) { - group.minMedia = frame.timestamp; - } - } - - group.start ??= frame.timestamp; - group.frames.push(frame); - this.#charge(group, FRAME_COST + frame.payload.byteLength); - - if (group.latest === undefined || frame.timestamp > group.latest) { - group.latest = frame.timestamp; - } - - const end = (frame.timestamp + (frame.duration ?? 0)) as Micro; - if (group.end === undefined || end > group.end) { - group.end = end; - } - } - - if (decoded.length === 0) continue; - - // Skip and wake only once the whole batch is in its group: a skip that drops this group - // checks it for a rewind, which must see every sample one fragment decoded to. - this.#updateBuffered(); - - let skipped = false; - if (group.consumer.sequence !== this.#active) { - // A non-active group can also be too slow to wait for. This runs even when - // the group is the delivery head, because that is exactly the stalled case - // (#active sits below every buffered group) where the max delay budget is - // what eventually breaks the stall. - this.#checkMaxDelay(); - - // A newer group reaching back to where the stalled active group has - // already presented means we can advance now instead of waiting. - skipped = this.#tryDurationSkip(); - } - - // Wake next() for the current delivery head so its frames surface as they - // arrive. Gating only on `=== #active` assumed +1 group numbering: with - // non-sequential group ids (large jumps between groups) #active lags one group - // behind as a stale `+1` phantom, so the real head never matched and its whole - // group was held until completion, then flushed in a burst. The earliest - // buffered group is the delivery head regardless of id scheme; next()'s - // promotion guard advances #active to it. Works for sequential and - // non-sequential ids alike. - // Once the max buffer fills, next() may have to skip the stalled head it is waiting on. - if (skipped || group.consumer.sequence === this.#active || group === this.#groups[0] || this.#full()) { - this.#notify?.(); - this.#notify = undefined; + if (this.#decodeBuffered(group)) { + if (group.fault) return; + continue; } + // Nothing buffered: next() took the frame, or the group ended or failed, which readFrame + // reports without carrying a frame. + if (!group.consumer.done) continue; + if (!(await group.consumer.readFrame())) break; } } catch (err) { if (this.#error) return; @@ -416,6 +320,114 @@ export class Consumer { } } + // Take and decode `group`'s next wire frame if the transport already holds one, charging what it + // decodes to. Synchronous, so it shares one step with the admission check before it: nothing yields + // between them, or every task woken together would spend the same room. Returns false when nothing + // is buffered; a payload that fails to decode sets the group's fault. + #decodeBuffered(group: Group): boolean { + const next = group.consumer.tryReadFrame(); + if (!next) return false; + group.empty = false; + const arrival = Nano.toMilli(Nano(now())); + + // A malformed payload is not loss: it ends the reader where it sits, rather than truncating the group. + let decoded: Frame[]; + try { + decoded = this.#format.decode(next.payload, next.timestamp); + } catch (err) { + group.fault = err instanceof Error ? err : new Error(String(err)); + return true; + } + if (this.#format.kind === "data") { + // A data track's marker is one object with no bytes, alone in its group. + if (group.bare || (decoded.length === 0 && group.media)) { + group.fault = new Error("a data track's discontinuity marker shares its group"); + return true; + } + group.bare = decoded.length === 0; + } + + for (const sample of decoded) { + const marker = this.#format.end?.(sample) !== undefined; + + // Observed here, before any gap handling or age budget: a target derived from + // what survives the budget would only ever confirm the budget it was cut to. + if (!marker) { + this.#jitter.observe(arrival, Milli.fromMicro(sample.timestamp)); + this.#spread.set(this.#jitter.measured); + } + const frame: ReceivedFrame = { + arrival, + payload: sample.payload, + timestamp: sample.timestamp, + // Protocol invariant: groups always start at a keyframe. + // For index 0, we enforce this regardless of what the format reports. + // For index > 0, we trust the format's keyframe detection. + keyframe: !marker && group.index === 0 ? true : sample.keyframe, + // Carry the container's per-sample duration through so group.end is the real + // presentation end (ts + duration), not just the last frame's ts. This is what + // makes the PTS-contiguity check (next.firstPTS <= group.end) work; without it a + // contiguous next group looks one frame past the end. Undefined for Legacy (no duration). + duration: sample.duration, + }; + + if (!marker) { + group.index++; + group.media = true; + if (group.minMedia === undefined || frame.timestamp < group.minMedia) { + group.minMedia = frame.timestamp; + } + } + + group.start ??= frame.timestamp; + group.frames.push(frame); + this.#charge(group, FRAME_COST + frame.payload.byteLength); + + if (group.latest === undefined || frame.timestamp > group.latest) { + group.latest = frame.timestamp; + } + + const end = (frame.timestamp + (frame.duration ?? 0)) as Micro; + if (group.end === undefined || end > group.end) { + group.end = end; + } + } + + if (decoded.length === 0) return true; + + // Skip and wake only once the whole batch is in its group: a skip that drops this group + // checks it for a rewind, which must see every sample one fragment decoded to. + this.#updateBuffered(); + + let skipped = false; + if (group.consumer.sequence !== this.#active) { + // A non-active group can also be too slow to wait for. This runs even when + // the group is the delivery head, because that is exactly the stalled case + // (#active sits below every buffered group) where the max delay budget is + // what eventually breaks the stall. + this.#checkMaxDelay(); + + // A newer group reaching back to where the stalled active group has + // already presented means we can advance now instead of waiting. + skipped = this.#tryDurationSkip(); + } + + // Wake next() for the current delivery head so its frames surface as they + // arrive. Gating only on `=== #active` assumed +1 group numbering: with + // non-sequential group ids (large jumps between groups) #active lags one group + // behind as a stale `+1` phantom, so the real head never matched and its whole + // group was held until completion, then flushed in a burst. The earliest + // buffered group is the delivery head regardless of id scheme; next()'s + // promotion guard advances #active to it. Works for sequential and + // non-sequential ids alike. + // Once the max buffer fills, next() may have to skip the stalled head it is waiting on. + if (skipped || group.consumer.sequence === this.#active || group === this.#groups[0] || this.#full()) { + this.#notify?.(); + this.#notify = undefined; + } + return true; + } + // Record where a group's content ends as the cursor advances past it. next()'s promotion guard // compares the following group's first PTS against this to tell an unbroken timeline from a real // gap, so EVERY site that moves #active past a group must call this; a site that forgets leaves a @@ -516,17 +528,21 @@ export class Consumer { // waits behind, to the next group even with nothing of it read yet: that one reads as the head, and // each such skip refunds what the group left behind held. The head may be the active group or a // backlog group admitted below it. Only fires when the head is fully consumed and still open, so we - // never drop frames the consumer hasn't seen. Returns true if a group was skipped. + // never drop frames the consumer hasn't seen. Returns true if it skipped a group or read one. #tryPressureSkip(): boolean { if (this.#active === undefined || !this.#full()) return false; const head = this.#groups[0]; if (!head || head.consumer.sequence > this.#active) return false; - if (head.done || head.frames.length > 0) return false; + if (head.done || head.fault || head.frames.length > 0) return false; const next = this.#groups[1]; if (!next) return false; + // Its task may not have run since it became the head, so read what the transport already holds + // before calling it stalled, as its own task would: the head decodes past a full max buffer. + if (this.#decodeBuffered(head)) return true; + this.#groups.shift(); console.warn( `skipping a stalled group past the max buffer: track=${this.#track.name} ${head.consumer.sequence} -> ${next.consumer.sequence}`, From e31546b271e0013fcd09c2e4937ed3a806673bff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 08:20:22 -0700 Subject: [PATCH 09/18] refactor(hang): decode the container lookahead from one synchronous owner Group tasks become watchers: they await readiness and wake the reader, and never decode or settle a group themselves. One synchronous #look decodes what the transport holds within the max buffer, run by next() and by every watcher wake. next() owns completion, faults and the pressure skip, reading the transport's state as it stands rather than flags a task set later. This removes the race class behind the last two review findings: a malformed frame found during a pressure skip now ends the reader instead of hanging it, and a finished data-marker group reads its end before the skip. Both are covered in JS and mirrored in Rust. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 82 +++++ js/hang/src/container/consumer.ts | 286 ++++++++---------- rs/moq-mux/src/container/consumer.rs | 78 +++++ 3 files changed, 282 insertions(+), 164 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index b67db0cbe18..2a6a10fb733 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -295,3 +295,85 @@ test("Consumer reads what a promoted head already holds before skipping it under warn.mockRestore(); } }); + +// Settle `promise` within a second, so a reader that hangs fails the test instead of timing it out. +async function within(promise: Promise): Promise { + let timer: ReturnType | undefined; + const hung = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("the reader hung")), 1_000); + }); + try { + return await Promise.race([promise, hung]); + } finally { + clearTimeout(timer); + } +} + +test("Consumer raises a malformed frame in a head promoted under pressure", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + maxBuffer: 1024, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0 }); + + // A later group's large frame fills the max buffer, then a malformed frame lands in the open group between. + const between = track.createGroup(1); + const later = track.createGroup(2); + const early = Time.Timestamp.now(); + later.writeFrame({ + payload: encodeFrame(new Uint8Array(2_000), 20_000 as Micro), + timestamp: Time.Timestamp.now(), + }); + await flush(); + between.writeFrame({ payload: new Uint8Array([0x80]), timestamp: early }); + + // Group 0 is skipped, and group 1's decode error ends the reader rather than leaving it waiting. + await expect(within(consumer.next())).rejects.toThrow("buffer too short"); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + +test("Consumer reads a finished marker group's end before skipping under pressure", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + maxBuffer: 1024, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0 }); + + // A later group's large frame fills the max buffer, then the group between finishes as a data marker. + const between = track.createGroup(1); + const later = track.createGroup(2); + const early = Time.Timestamp.now(); + later.writeFrame({ + payload: encodeFrame(new Uint8Array(2_000), 20_000 as Micro), + timestamp: Time.Timestamp.now(), + }); + await flush(); + between.writeFrame({ payload: new Uint8Array(), timestamp: early }); + between.finish(); + + // Group 0 is skipped, and group 1 declares its end before group 2 is read. + expect(await within(consumer.next())).toMatchObject({ kind: "gap" }); + expect(await within(consumer.next())).toMatchObject({ kind: "end", group: 1 }); + expect(await within(consumer.next())).toMatchObject({ kind: "frame", group: 2, frame: { timestamp: 20_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 05d85b7b344..7548827d813 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -1,5 +1,5 @@ import { Instant, Micro, Milli, Nano, now } from "@moq/clock"; -import * as Moq from "@moq/net"; +import type * as Moq from "@moq/net"; import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq/signals"; import type { Format } from "./format"; import { Jitter } from "./jitter"; @@ -82,8 +82,6 @@ interface Group { minMedia?: Micro; // Lowest decodable timestamp latest?: Micro; // The timestamp of the latest known frame end?: Micro; // The furthest presentation point so far, i.e. max(timestamp + duration) - done?: boolean; // Set when #runGroup finishes reading all frames - truncated?: boolean; // The missing tail becomes a gap after the buffered frames are delivered. fault?: Error; // The payload that failed to decode, an error after the buffered frames are delivered. index: number; // Media frames decoded so far: the first is a keyframe by protocol invariant charged: number; // What this group holds against the max buffer @@ -131,7 +129,7 @@ export class Consumer { #maxBuffer: number; // What the buffered groups hold, against the max buffer. #used = 0; - // Group tasks and the track loop parked on a full max buffer. + // Group watchers and the track loop parked on a full max buffer. #room: (() => void)[] = []; #groups: Group[] = []; #active?: number; // the active group sequence number @@ -195,7 +193,7 @@ export class Consumer { group.consumer.close(); } this.#groups.length = 0; - // Parked tasks see the abort and return. + // Parked watchers see the abort and return. this.#wake(); }); } @@ -255,8 +253,7 @@ export class Consumer { this.#notify = undefined; } - // Start buffering frames from this group - this.#signals.spawn(this.#runGroup.bind(this, group)); + this.#signals.spawn(this.#watch.bind(this, group)); } this.#finish(null); } catch (err) { @@ -264,85 +261,74 @@ export class Consumer { } } - async #runGroup(group: Group) { - try { - for (;;) { - // Wait for a frame without taking it, so a group parked on the max buffer holds nothing. - await group.consumer.readable(); - while (!this.#stopped(group) && !this.#admits(group)) await this.#park(); - if (this.#stopped(group)) return; - if (this.#decodeBuffered(group)) { - if (group.fault) return; - continue; - } - // Nothing buffered: next() took the frame, or the group ended or failed, which readFrame - // reports without carrying a frame. - if (!group.consumer.done) continue; - if (!(await group.consumer.readFrame())) break; - } - } catch (err) { - if (this.#error) return; - // Stop reading the group but keep already-decoded frames. - // A stream RESET or eviction truncates the tail of the GoP; - // frames decoded before it are still valid and playable. - // The tail is gone though, so the next group does not continue this one. - group.truncated = true; - if (!(err instanceof Moq.Error.Stream)) throw err; - } finally { - group.done = true; - - // A dropped group no longer holds the cursor. - if (!group.released && group.consumer.sequence === this.#active) { - this.#recordPresented(group); - - // Advance to the next buffered group's actual sequence, but ONLY if it continues this - // group's timeline. Some encoders number groups non-sequentially with large gaps (not - // +1), so a bare `+= 1` would point #active at a nonexistent sequence and stall next() - // until #checkMaxDelay skipped it -- every group through the skip path, i.e. constant - // stutter. A real PTS gap is different: an intermediate group may still be in transit, - // so fall back to +1 there (next()'s promotion guard fixes it up once a continuous - // group arrives) and let #checkMaxDelay / #tryDurationSkip skip the gap only once - // age or duration coverage proves it too old. - const next = this.#groups[this.#groups.indexOf(group) + 1]; - this.#active = continues(group, next) ? next.consumer.sequence : group.consumer.sequence + 1; - } - - // Recompute buffered ranges now that this group is done, - // so consecutive done groups can merge into a single range. - this.#updateBuffered(); - - // Always notify - the consumer may need to advance past this group - // even if it wasn't active when this task finished. + // Watch `group` for wire frames and its end, waking the reader. Only #look decodes, and the + // watcher runs it at once rather than leave it to the next read, so buffered ranges, arrival times + // and the age budget keep up while the reader is busy. Completion, faults and the pressure skip are + // next()'s alone: it reads the transport's state as it stands, so nothing here settles a group + // behind its back. + async #watch(group: Group) { + while (!this.#stopped(group)) { + await group.consumer.readable(); + if (this.#stopped(group)) return; + this.#look(); this.#notify?.(); this.#notify = undefined; + if (group.fault || group.consumer.done) return; + // A frame left unread behind a full max buffer waits for room or for the group to lead. + if (!this.#admits(group)) await this.#park(); + } + } - group.consumer.close(); + // Take and decode every wire frame the transport already holds, oldest group first, each only while + // the max buffer admits it, then skip what the new frames make too stale to wait for. The one place + // frames come off the wire. Synchronous, so each admission check shares one step with the decode it + // admits, and nothing can spend the same room twice or see a group half decoded. + #look(): void { + for (;;) { + let decoded = false; + // A frame past the cursor, which can show the groups ahead of it are too slow to wait for. + let ahead = false; + for (const group of this.#groups) { + while (!group.fault && this.#admits(group)) { + const wire = group.consumer.tryReadFrame(); + if (!wire) break; + this.#decode(group, wire); + decoded = true; + if (group.consumer.sequence !== this.#active) ahead = true; + } + } + if (!decoded) return; + this.#updateBuffered(); + // This runs even when the decoded group is the delivery head, because that is exactly the + // stalled case (#active sits below every buffered group) where the max delay budget is what + // eventually breaks the stall. A newer group reaching back to where the stalled active group + // has already presented means it can advance now instead of waiting. + if (!ahead) return; + const aged = this.#checkMaxDelay(); + if (!this.#tryDurationSkip() && !aged) return; + // The skip refunded room and moved the head, which may admit frames left unread. } } - // Take and decode `group`'s next wire frame if the transport already holds one, charging what it - // decodes to. Synchronous, so it shares one step with the admission check before it: nothing yields - // between them, or every task woken together would spend the same room. Returns false when nothing - // is buffered; a payload that fails to decode sets the group's fault. - #decodeBuffered(group: Group): boolean { - const next = group.consumer.tryReadFrame(); - if (!next) return false; + // Decode one wire frame into `group`, charging what it buffers. A payload that fails to decode sets + // the group's fault, which next() raises once the frames ahead of it are read. + #decode(group: Group, wire: Moq.Group.Frame): void { group.empty = false; const arrival = Nano.toMilli(Nano(now())); // A malformed payload is not loss: it ends the reader where it sits, rather than truncating the group. let decoded: Frame[]; try { - decoded = this.#format.decode(next.payload, next.timestamp); + decoded = this.#format.decode(wire.payload, wire.timestamp); } catch (err) { group.fault = err instanceof Error ? err : new Error(String(err)); - return true; + return; } if (this.#format.kind === "data") { // A data track's marker is one object with no bytes, alone in its group. if (group.bare || (decoded.length === 0 && group.media)) { group.fault = new Error("a data track's discontinuity marker shares its group"); - return true; + return; } group.bare = decoded.length === 0; } @@ -392,40 +378,13 @@ export class Consumer { group.end = end; } } + } - if (decoded.length === 0) return true; - - // Skip and wake only once the whole batch is in its group: a skip that drops this group - // checks it for a rewind, which must see every sample one fragment decoded to. - this.#updateBuffered(); - - let skipped = false; - if (group.consumer.sequence !== this.#active) { - // A non-active group can also be too slow to wait for. This runs even when - // the group is the delivery head, because that is exactly the stalled case - // (#active sits below every buffered group) where the max delay budget is - // what eventually breaks the stall. - this.#checkMaxDelay(); - - // A newer group reaching back to where the stalled active group has - // already presented means we can advance now instead of waiting. - skipped = this.#tryDurationSkip(); - } - - // Wake next() for the current delivery head so its frames surface as they - // arrive. Gating only on `=== #active` assumed +1 group numbering: with - // non-sequential group ids (large jumps between groups) #active lags one group - // behind as a stale `+1` phantom, so the real head never matched and its whole - // group was held until completion, then flushed in a burst. The earliest - // buffered group is the delivery head regardless of id scheme; next()'s - // promotion guard advances #active to it. Works for sequential and - // non-sequential ids alike. - // Once the max buffer fills, next() may have to skip the stalled head it is waiting on. - if (skipped || group.consumer.sequence === this.#active || group === this.#groups[0] || this.#full()) { - this.#notify?.(); - this.#notify = undefined; - } - return true; + // The transport's verdict on a group with nothing left to read: null once it finished cleanly, the + // error that cut its tail off, or undefined while more may arrive. + #ended(group: Group): Error | null | undefined { + if (!group.consumer.done) return undefined; + return group.consumer.closed.peek() ?? null; } // Record where a group's content ends as the cursor advances past it. next()'s promotion guard @@ -438,8 +397,9 @@ export class Consumer { if (group.end !== undefined) this.#presentedEnd = group.end; } - #checkMaxDelay() { - if (this.#active === undefined) return; + // Skip the oldest groups while the buffered span exceeds the max delay. Returns true if any was skipped. + #checkMaxDelay(): boolean { + if (this.#active === undefined) return false; let skipped = 0; const start = this.#groups[0]?.consumer.sequence; @@ -479,10 +439,11 @@ export class Consumer { const discarded = first.frames.some((frame) => this.#format.end?.(frame) === undefined); // Adjacency proves nothing is missing only once the group finished cleanly: a truncated // or still-open group may have lost its tail. - const finished = first.done && !first.truncated; + const ended = this.#ended(first); + const truncated = ended instanceof Error; const adjacent = next?.consumer.sequence === first.consumer.sequence + 1; - const continued = (finished && adjacent) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); - if (marker || discarded || first.truncated || !continued) this.#lost = true; + const continued = (ended === null && adjacent) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); + if (marker || discarded || truncated || !continued) this.#lost = true; skipped++; if (!this.#discard(first)) break; } @@ -492,11 +453,8 @@ export class Consumer { `skipping slow groups: track=${this.#track.name} ${start} -> ${this.#active} count=${skipped}`, ); this.#updateBuffered(); - - // Wake up any consumers waiting for a new frame. - this.#notify?.(); - this.#notify = undefined; } + return skipped > 0; } // Skip the stalled active group once it has presented up to where the next group @@ -508,7 +466,7 @@ export class Consumer { const active = this.#groups[0]; if (!active || active.consumer.sequence !== this.#active) return false; - if (active.done || active.frames.length > 0 || active.end === undefined) return false; + if (this.#ended(active) !== undefined || active.frames.length > 0 || active.end === undefined) return false; const next = this.#groups[1]; const nextStart = next?.frames.at(0)?.timestamp; @@ -528,30 +486,28 @@ export class Consumer { // waits behind, to the next group even with nothing of it read yet: that one reads as the head, and // each such skip refunds what the group left behind held. The head may be the active group or a // backlog group admitted below it. Only fires when the head is fully consumed and still open, so we - // never drop frames the consumer hasn't seen. Returns true if it skipped a group or read one. + // never drop frames the consumer hasn't seen: #look already read what the transport holds for it. + // Returns true if it skipped a group. #tryPressureSkip(): boolean { if (this.#active === undefined || !this.#full()) return false; const head = this.#groups[0]; if (!head || head.consumer.sequence > this.#active) return false; - if (head.done || head.fault || head.frames.length > 0) return false; + if (head.fault || head.frames.length > 0 || this.#ended(head) !== undefined) return false; const next = this.#groups[1]; if (!next) return false; - // Its task may not have run since it became the head, so read what the transport already holds - // before calling it stalled, as its own task would: the head decodes past a full max buffer. - if (this.#decodeBuffered(head)) return true; - this.#groups.shift(); console.warn( `skipping a stalled group past the max buffer: track=${this.#track.name} ${head.consumer.sequence} -> ${next.consumer.sequence}`, ); if (head.consumer.sequence === this.#active) { // Still open, so its tail is lost unless the next group picks up where it ended, which nothing - // proves before its start is read. + // proves before its start is read. A marker group stops the timeline, so moving past one loses it. const nextStart = next.frames.at(0)?.timestamp; - if (head.truncated || !ptsContiguous(head.end ?? this.#presentedEnd, nextStart)) this.#lost = true; + const marker = !head.empty && !head.media; + if (marker || !ptsContiguous(head.end ?? this.#presentedEnd, nextStart)) this.#lost = true; this.#recordPresented(head); this.#active = next.consumer.sequence; } else { @@ -592,7 +548,7 @@ export class Consumer { return !this.#full() || (group === this.#groups[0] && group.frames.length === 0); } - // Whether `group`'s task should stop: the group was dropped, so what it decodes would never be + // Whether `group`'s watcher should stop: the group was dropped, so what it decodes would never be // refunded, or the consumer closed. #stopped(group: Group): boolean { return group.released === true || this.#signals.abort.aborted; @@ -603,7 +559,7 @@ export class Consumer { return new Promise((resolve) => this.#room.push(resolve)); } - // Let parked tasks re-check the max buffer: something was refunded, or the head or cursor moved. + // Let parked watchers re-check the max buffer: something was refunded, or the head or cursor moved. #wake(): void { const room = this.#room; this.#room = []; @@ -612,7 +568,7 @@ export class Consumer { // Drop a group without delivering it. A drop must not hide what was already found, so one holding // a fault, or with frames below the latest delivered group start, ends the reader instead. What - // its task has not decoded yet stays unchecked: decoding data only to throw it away is work a + // #look has not decoded yet stays unchecked: decoding data only to throw it away is work a // peer could force. Returns false if it ended the reader. #discard(group: Group): boolean { group.consumer.close(); @@ -640,7 +596,7 @@ export class Consumer { this.#track.close(error); this.#notify?.(); this.#notify = undefined; - // Nothing more is read, so stop the group tasks and release their groups rather than let a + // Nothing more is read, so stop the group watchers and release their groups rather than let a // peer keep them decoding and buffering. After the wake: next() reads a closed effect as closed. this.#signals.close(); } @@ -662,6 +618,8 @@ export class Consumer { } for (;;) { if (this.#error) throw this.#error; + // Decode what has arrived, so every decision below sees it. + this.#look(); const ended = this.#closed.peek(); if (this.#groups[0]?.consumer.sequence === this.#active || this.#groups.length === 0) { @@ -674,7 +632,7 @@ export class Consumer { } } - // If #active points below all buffered groups -- e.g. the finally block's `+ 1` + // If #active points below all buffered groups -- e.g. a finished group's `+ 1` // fallback fired because no later group was buffered yet, and the real (large-gap, // non-sequential) next group has since arrived -- promote #active to the first real // group so delivery resumes instead of stalling on a nonexistent sequence. @@ -722,15 +680,12 @@ export class Consumer { } } - if ( - this.#groups.length > 0 && - this.#active !== undefined && - this.#groups[0].consumer.sequence <= this.#active - ) { - const frame = this.#groups[0].frames.shift(); + const head = this.#groups.at(0); + if (head && this.#active !== undefined && head.consumer.sequence <= this.#active) { + const seq = head.consumer.sequence; + const frame = head.frames.shift(); if (frame) { - this.#refund(this.#groups[0], FRAME_COST + frame.payload.byteLength); - const seq = this.#groups[0].consumer.sequence; + this.#refund(head, FRAME_COST + frame.payload.byteLength); const end = this.#format.end?.(frame); if (end !== undefined) { this.#updateBuffered(); @@ -756,39 +711,42 @@ export class Consumer { return this.#emit({ kind: "frame", group: seq, frame }); } - // Check if the group is done and then remove it. A group is removable only - // once its #runGroup task has finished (done) and all frames are consumed: - // a below-#active group (a backlog group admitted behind the live edge) may - // still be downloading when its buffer momentarily drains, and removing it - // then silently truncates its tail. #runGroup notifies whenever the head - // group gains a frame, so waiting here is woken, and #checkMaxDelay bounds - // how long a stalled head can hold delivery up. - if (this.#groups[0].done) { - if (this.#groups[0].consumer.sequence === this.#active) { - // The cursor moves past this group here rather than in #runGroup's finally - // block whenever the group finished before it became active, so this is the - // site that has to record its presentation end. Advance by +1 and let the - // promotion guard above resolve the real successor on the next iteration. - this.#recordPresented(this.#groups[0]); - this.#active += 1; - } + // A payload that failed to decode ends the reader once the frames ahead of it are read. + if (head.fault) { + this.#abort(head.fault); + throw head.fault; + } - const group = this.#groups.shift(); - if (group) { - this.#release(group); - if (group.fault) { - this.#abort(group.fault); - throw group.fault; - } - if (group.truncated) this.#lost = true; - this.#updateBuffered(); - // A data track's marker carries no timestamp, so it ends where this reader got to. - // Before the first item there is nothing to end. - if (group.bare && !group.truncated && this.#edge !== undefined) { - return this.#emit({ kind: "end", group: group.consumer.sequence, end: this.#edge }); - } - continue; + // Remove the group once the transport has settled it and every frame is read. A + // below-#active group (a backlog group admitted behind the live edge) may still be + // downloading when its buffer momentarily drains, and removing it then silently + // truncates its tail. Its watcher wakes this read whenever the group gains a frame or + // ends, and #checkMaxDelay bounds how long a stalled head can hold delivery up. + const settled = this.#ended(head); + if (settled !== undefined) { + this.#groups.shift(); + if (seq === this.#active) { + // Advance to the next buffered group's actual sequence, but ONLY if it continues this + // group's timeline. Some encoders number groups non-sequentially with large gaps (not + // +1), so a bare `+= 1` would point #active at a nonexistent sequence and stall until + // the hole expired -- every group through the skip path, i.e. constant stutter. A real + // PTS gap is different: an intermediate group may still be in transit, so fall back to + // +1 there (the promotion guard above fixes it up once a continuous group arrives). + this.#recordPresented(head); + const next = this.#groups.at(0); + this.#active = continues(head, next) ? next.consumer.sequence : seq + 1; } + head.consumer.close(); + this.#release(head); + const truncated = settled instanceof Error; + if (truncated) this.#lost = true; + this.#updateBuffered(); + // A data track's marker carries no timestamp, so it ends where this reader got to. + // Before the first item there is nothing to end. + if (head.bare && !truncated && this.#edge !== undefined) { + return this.#emit({ kind: "end", group: seq, end: this.#edge }); + } + continue; } // The active group is stalled with nothing buffered. If a later group @@ -801,7 +759,7 @@ export class Consumer { if (this.#notify) { throw new Error("multiple calls to next not supported"); } - // The head or cursor may have moved, which can exempt a parked task from the max buffer. + // The head or cursor may have moved, which can exempt a parked watcher from the max buffer. this.#wake(); const abort = this.#signals.abort; @@ -864,7 +822,7 @@ export class Consumer { const end = Milli.fromMicro(group.latest); const last = ranges.at(-1); - const contiguous = prev?.done && prev.consumer.sequence + 1 === group.consumer.sequence; + const contiguous = prev?.consumer.done && prev.consumer.sequence + 1 === group.consumer.sequence; if (last && (last.end >= start || contiguous)) { last.end = Milli(Math.max(last.end, end)); } else { diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 84edd7e230b..2a665a4d479 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -4260,6 +4260,84 @@ mod tests { assert_eq!(reads, vec![Some(ts(0)), None, Some(ts(20_000))]); } + /// Groups 0 and 1 open behind a cursor that read group 0's frame, and group 2's large frame that + /// fills the max buffer once read ahead. Returns groups 0 and 1. + fn fill_behind_a_stalled_group( + track: &mut moq_net::track::Producer, + consumer: &mut Consumer, + waiter: &kio::Waiter, + ) -> (moq_net::group::Producer, moq_net::group::Producer) { + let stalled = open_group(track, 0, ts(0)); + assert!(matches!( + consumer.poll_read(waiter), + Poll::Ready(Ok(Some(Read::Frame { group: 0, .. }))) + )); + let between = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut later = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + Container::Legacy(crate::container::Kind::Data) + .write( + &mut later, + &[Frame { + timestamp: ts(20_000), + payload: Bytes::from(vec![0; 4096]), + keyframe: false, + duration: None, + }], + ) + .unwrap(); + later.finish().unwrap(); + (stalled, between) + } + + // A max buffer that holds three groups' overhead with room to spare, short of group 2's large frame. + fn pressured_data_consumer(track: &mut moq_net::track::Producer) -> Consumer { + let mut consumer = container_max_delay_only(track.subscribe(None), Duration::from_secs(5)); + consumer.set_max_buffer(1024 + 3 * std::mem::size_of::>()); + consumer + } + + /// A malformed frame landing in the group the cursor moves on to under pressure fails the reader + /// rather than leaving it waiting on the group. + #[test] + fn a_malformed_frame_in_a_group_moved_to_under_pressure_fails_the_reader() { + let waiter = kio::Waiter::noop(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = pressured_data_consumer(&mut track); + let (_stalled, mut between) = fill_behind_a_stalled_group(&mut track, &mut consumer, &waiter); + + between.write_frame(ts(10_000), Bytes::from_static(&[0x80])).unwrap(); + let read = loop { + match consumer.poll_read(&waiter) { + Poll::Ready(Ok(Some(Read::Gap { .. }))) => continue, + other => break other, + } + }; + assert!(matches!(read, Poll::Ready(Err(_))), "{read:?}"); + } + + /// A data marker group that finished before the cursor moved on to it under pressure still reads + /// its end before the group after it. + #[test] + fn a_finished_marker_group_moved_to_under_pressure_reads_its_end() { + let waiter = kio::Waiter::noop(); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = pressured_data_consumer(&mut track); + let (_stalled, mut between) = fill_behind_a_stalled_group(&mut track, &mut consumer, &waiter); + + between.write_frame(ts(10_000), Bytes::new()).unwrap(); + between.finish().unwrap(); + let mut reads = Vec::new(); + while let Poll::Ready(read) = consumer.poll_read(&waiter) { + reads.push(match read.unwrap() { + Some(Read::Frame { group, frame }) => format!("frame {group} {}", frame.timestamp.as_micros()), + Some(Read::Gap { .. }) => "gap".to_string(), + Some(Read::End { group, .. }) => format!("end {group}"), + None => break, + }); + } + assert_eq!(reads, ["gap", "end 1", "frame 2 20000"]); + } + /// No cap is too small to read through: the cursor's group and one successor are always pulled, so /// a stalled group can be moved past even when one group's overhead fills the max buffer. #[test] From 4d7400b7fc524da9f821cb00c605d2409aaa89cb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 08:31:29 -0700 Subject: [PATCH 10/18] fix(hang): wake the reader when a group's frame fails to read A compressed frame inflates lazily inside tryReadFrame, which throws and fails the group on a slice that will not inflate. #look let that escape, so a watcher died before waking a parked reader. It now stops at the failure, leaving the group ended with the error, which next() reads as a cut tail; the watcher also wakes the reader however it exits. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 39 +++++++++++++++++++ js/hang/src/container/consumer.ts | 29 ++++++++++---- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 2a6a10fb733..2a7bfece34a 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -377,3 +377,42 @@ test("Consumer reads a finished marker group's end before skipping under pressur warn.mockRestore(); } }); + +test("Consumer reads a group whose frame fails to read as cut short while the reader waits", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0 }); + const next = track.createGroup(1); + write(next, 10_000); + next.finish(); + + // The reader parks on group 0, then a frame lands whose payload fails once read, as a compressed + // slice that will not inflate does. The read that takes it fails the group. + const pending = within(consumer.next()); + await flush(); + const payload = encodeFrame(PAYLOAD, 5_000 as Micro); + let armed = false; + stalled.writeFrame({ + get payload() { + if (armed) throw new Error("the slice will not inflate"); + return payload; + }, + timestamp: Time.Timestamp.now(), + }); + armed = true; + + expect(await pending).toMatchObject({ kind: "gap" }); + expect(await within(consumer.next())).toMatchObject({ kind: "frame", group: 1, frame: { timestamp: 10_000 } }); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 7548827d813..d854cc9ff38 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -267,15 +267,21 @@ export class Consumer { // next()'s alone: it reads the transport's state as it stands, so nothing here settles a group // behind its back. async #watch(group: Group) { - while (!this.#stopped(group)) { - await group.consumer.readable(); - if (this.#stopped(group)) return; - this.#look(); + try { + while (!this.#stopped(group)) { + await group.consumer.readable(); + if (this.#stopped(group)) return; + this.#look(); + this.#notify?.(); + this.#notify = undefined; + if (group.fault || group.consumer.done) return; + // A frame left unread behind a full max buffer waits for room or for the group to lead. + if (!this.#admits(group)) await this.#park(); + } + } finally { + // However the watch ends, the reader looks at the group again: it may have ended or failed. this.#notify?.(); this.#notify = undefined; - if (group.fault || group.consumer.done) return; - // A frame left unread behind a full max buffer waits for room or for the group to lead. - if (!this.#admits(group)) await this.#park(); } } @@ -290,7 +296,14 @@ export class Consumer { let ahead = false; for (const group of this.#groups) { while (!group.fault && this.#admits(group)) { - const wire = group.consumer.tryReadFrame(); + let wire: Moq.Group.Frame | undefined; + try { + wire = group.consumer.tryReadFrame(); + } catch { + // The transport failed the group, such as on a slice that would not inflate. It now + // reads as ended with that error, which next() takes as a cut tail. + break; + } if (!wire) break; this.#decode(group, wire); decoded = true; From d6b662337111a79b0ce5ce333269166414c0353a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 09:59:41 -0700 Subject: [PATCH 11/18] fix(net,hang): frames carry when they landed, so held-back frames keep their arrival JS @moq/net frames now carry `arrived`, as Rust's `Frame::arrived()` does: the group stamps each frame as it is written and returns the stamp on every read, compressed and resumed copies included. The container consumer stamps `ReceivedFrame.arrival` from it instead of from decode time, so frames the max buffer leaves in the transport no longer count the consumer's own wait as jitter. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 31 +++++++++- js/hang/src/container/consumer.ts | 4 +- js/net/src/flate/group.ts | 9 ++- js/net/src/group.test.ts | 60 ++++++++++++++++++- js/net/src/group.ts | 22 +++++-- 5 files changed, 116 insertions(+), 10 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 2a7bfece34a..5f450a84407 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -1,5 +1,5 @@ import { expect, spyOn, test } from "bun:test"; -import { type Micro, Milli } from "@moq/clock"; +import { Manual, type Micro, Milli, Nano } from "@moq/clock"; import { Time, Track } from "@moq/net"; import { Consumer, type Read } from "./consumer.ts"; import { encodeFrame, Format } from "./legacy.ts"; @@ -205,6 +205,35 @@ test("Consumer admits reads completing together one at a time against the max bu } }); +test("Consumer stamps frames the max buffer held back with when they landed, not when they decoded", async () => { + const clock = new Manual(); + const installed = clock.install(); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(5_000), + // One group's overhead and one frame: the second frame waits in the transport until the first is read. + maxBuffer: 300, + }); + try { + const group = track.createGroup(0); + write(group, 0); + await clock.advance(Nano.fromMilli(Milli(250))); + write(group, 10_000); + group.finish(); + track.finish(); + // The reader is slow: nothing is read until well after both frames landed. + await clock.advance(Nano.fromMilli(Milli(750))); + + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 0, arrival: 0 } }); + expect(await consumer.next()).toMatchObject({ kind: "frame", frame: { timestamp: 10_000, arrival: 250 } }); + } finally { + consumer.close(); + track.close(); + installed.close(); + } +}); + for (const maxBuffer of [0, 256]) { test(`Consumer moves past a stalled group with a ${maxBuffer} byte max buffer`, async () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index d854cc9ff38..e2e4559b375 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -327,7 +327,9 @@ export class Consumer { // the group's fault, which next() raises once the frames ahead of it are read. #decode(group: Group, wire: Moq.Group.Frame): void { group.empty = false; - const arrival = Nano.toMilli(Nano(now())); + // When the frame landed, not when it was decoded: a frame the max buffer left in the transport + // would otherwise count the consumer's own wait as network jitter. + const arrival = Nano.toMilli(Nano(wire.arrived ?? now())); // A malformed payload is not loss: it ends the reader where it sits, rather than truncating the group. let decoded: Frame[]; diff --git a/js/net/src/flate/group.ts b/js/net/src/flate/group.ts index 5b62662a5b6..5be070f4191 100644 --- a/js/net/src/flate/group.ts +++ b/js/net/src/flate/group.ts @@ -1,3 +1,4 @@ +import type { Instant } from "@moq/clock"; import type { Frame } from "../group.ts"; import type { Timestamp } from "../time.ts"; import { Decoder } from "./decoder.ts"; @@ -14,9 +15,11 @@ export interface Stored { // Class identity keeps reactive equality from reading the lazy payload getter. class SliceFrame implements Frame { readonly timestamp: Timestamp | undefined; + readonly arrived: Instant | undefined; #read: () => Uint8Array; - constructor(timestamp: Timestamp | undefined, read: () => Uint8Array) { + constructor(timestamp: Timestamp | undefined, arrived: Instant | undefined, read: () => Uint8Array) { this.timestamp = timestamp; + this.arrived = arrived; this.#read = read; } get payload(): Uint8Array { @@ -44,7 +47,7 @@ export function datagramSlice(payload: Uint8Array, inflated?: number): Stored { /** One reader's lazy plain view of a slice, so no queue retains a copy another reader inflated. */ export function sliceView(timestamp: Timestamp | undefined, slice: Stored): Frame { let plain: Uint8Array | undefined; - const frame = new SliceFrame(timestamp, () => { + const frame = new SliceFrame(timestamp, undefined, () => { plain ??= new Decoder().frame(slice.payload, slice.inflated); return plain; }); @@ -79,7 +82,7 @@ export class Compression { this.bytes += slice.payload.length; const index = this.#frames.length; this.#frames.push(slice); - const result = new SliceFrame(frame.timestamp, () => this.#inflate(index)); + const result = new SliceFrame(frame.timestamp, frame.arrived, () => this.#inflate(index)); slices.set(result, slice); return result; } diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index 663fcb40d89..3065ee13e99 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { Manual } from "@moq/clock"; +import { Manual, Milli, Nano } from "@moq/clock"; import { Signal } from "@moq/signals"; import { FrameTooLarge, GroupTooLarge, StreamCode, StreamError } from "./error.ts"; import { MAX_GROUP_CACHE_BYTES, MAX_GROUP_FRAMES, Producer } from "./group.ts"; @@ -65,6 +65,64 @@ test("reordered frames keep the highest group PTS in live and later mirrors", () } }); +test("a frame read late keeps the instant it landed, plain, compressed, or resumed", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const plain = new Producer(0); + const compressed = new Producer(0); + // Frame 0 lands before compression binds, so the restored copy must carry its stamp too. + compressed.writeFrame(textFrame("a")); + hooks.bindGroupCompression(compressed, true); + const first = clock.now(); + plain.writeFrame(textFrame("a")); + clock.jump(Nano.fromMilli(Milli(10))); + const second = clock.now(); + plain.writeFrame(textFrame("b")); + compressed.writeFrame(textFrame("b")); + clock.jump(Nano.fromMilli(Milli(1000))); + + const reader = plain.consume(); + expect(reader.tryReadFrame()?.arrived).toBe(first); + expect((await reader.readFrameSequence())?.arrived).toBe(second); + const stored = compressed.consume(); + expect((await hooks.readGroupFrame(stored, true))?.frame.arrived).toBe(first); + expect((await stored.readFrame())?.arrived).toBe(second); + + // A frame written in with a stamp of its own still gets this node's. + const forwarded = new Producer(0); + forwarded.writeFrame({ payload: new Uint8Array([1]), arrived: first }); + expect(forwarded.consume().tryReadFrame()?.arrived).toBe(clock.now()); + + // A resumed read takes the stamp the replacement copy holds, not when it was adopted. + const failing = new Producer(0); + const resumed = failing.mirror(); + const replacement = new Producer(0); + replacement.writeFrame(textFrame("a")); + const landed = clock.now(); + replacement.writeFrame(textFrame("b")); + hooks.recoverGroup(resumed, { + changed: new Signal(0), + wants: (failed) => failed !== undefined, + replace: async () => ({ + group: replacement.mirror(), + expiry: { expired: () => false, changed: [] }, + adopt: () => {}, + discard: () => {}, + }), + }); + failing.writeFrame(textFrame("a")); + await resumed.readFrame(); + clock.jump(Nano.fromMilli(Milli(1000))); + failing.abort(new Error("copy failed")); + expect((await resumed.readFrame())?.arrived).toBe(landed); + + for (const producer of [plain, compressed, forwarded, replacement]) producer.finish(); + } finally { + installed.close(); + } +}); + test("zero group PTS keeps its scale and untimed groups stay unstamped", () => { const installed = new Manual().install(); try { diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 155034bb8f9..48050daeb33 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -39,6 +39,14 @@ export interface Frame { * refused with {@link TimestampMismatch}. */ timestamp?: Timestamp; + /** + * When the frame landed in this node's group, on the installed clock. + * + * Every frame a group returns carries the instant it was written, received, or fetched + * into it, so a reader judges lateness on that rather than on when it got around to + * reading. A write ignores it and stamps its own. + */ + arrived?: Instant; } /** Options for a sequence-aware frame read. */ @@ -382,13 +390,15 @@ export class Producer { this.#overflow(); } + const arrived = now(); + frame = { payload: frame.payload, timestamp: frame.timestamp, arrived }; if (this.#state.flate) { frame = this.#state.flate.push(frame, inflated); if (this.#state.flate.stored > MAX_GROUP_CACHE_BYTES) { this.#overflow(); } } - this.#activity = now(); + this.#activity = arrived; appendFrame(this.#state, frame); if (commits) { this.#admits = undefined; @@ -887,7 +897,11 @@ export class Consumer { const slice = compressed ? stored(buffered) : undefined; let frame: Frame; try { - frame = { payload: slice?.payload ?? buffered.payload, timestamp: buffered.timestamp }; + frame = { + payload: slice?.payload ?? buffered.payload, + timestamp: buffered.timestamp, + arrived: buffered.arrived, + }; } catch (error) { this.#terminal = error instanceof Error ? error : new Error(String(error)); if (this.#verdict.peek() === undefined) this.#verdict.set(this.#terminal); @@ -945,7 +959,7 @@ export class Consumer { if (this.#terminal) return undefined; const read = this.#readBufferedFrame(); if (!read) return undefined; - return { sequence: read.sequence, payload: read.frame.payload, timestamp: read.frame.timestamp }; + return { sequence: read.sequence, ...read.frame }; } /** Resolves once {@link readFrame} would not block. */ @@ -987,7 +1001,7 @@ export class Consumer { async readFrameSequence(options?: ReadOptions): Promise<({ sequence: number } & Frame) | undefined> { const read = await this.#readFrame(options?.from ?? 0); if (!read) return undefined; - return { sequence: read.sequence, payload: read.frame.payload, timestamp: read.frame.timestamp }; + return { sequence: read.sequence, ...read.frame }; } async #readFrame(from: number, compressed = false): Promise { From 9e393982044f9f87870c995ff309036571e12688 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 10:02:23 -0700 Subject: [PATCH 12/18] chore(net): regenerate the API report for Frame.arrived Co-Authored-By: Claude Opus 5.5 --- js/net/api/index.api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 550c115edfa..af9b9271bfd 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -2,6 +2,7 @@ ```ts import { $ZodBranded } from "zod/v4/core"; +import { Instant } from "@moq/clock"; import { Milli } from "@moq/clock"; import { ZodMiniBigInt } from "@zod/mini"; import { infer } from "@zod/mini"; @@ -329,6 +330,7 @@ export namespace Group { } & Group.Frame | undefined; } export interface Frame { + arrived?: Instant; payload: Uint8Array; timestamp?: Time.Timestamp; } From 26f1ad64fe1e70d5ef51ac93c040bf65349dc2fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 10:13:11 -0700 Subject: [PATCH 13/18] fix(net): keep the writer's frame behind its arrival stamp Copying the payload at write read a lazy payload before any reader took it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/group.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 48050daeb33..1ae4b134d75 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -105,6 +105,23 @@ class GroupState { } } +// A written frame and when it landed. It keeps the writer's frame rather than copying it, so a +// lazy payload is read only when a reader takes it. +class Landed implements Frame { + readonly arrived: Instant; + #frame: Frame; + constructor(frame: Frame, arrived: Instant) { + this.#frame = frame; + this.arrived = arrived; + } + get payload(): Uint8Array { + return this.#frame.payload; + } + get timestamp(): Timestamp | undefined { + return this.#frame.timestamp; + } +} + // Refuse a frame whose timedness disagrees with the track's. function checkTimed(timed: boolean | undefined, frame: Frame) { if (timed !== undefined && timed !== (frame.timestamp !== undefined)) throw new TimestampMismatch(); @@ -391,7 +408,7 @@ export class Producer { } const arrived = now(); - frame = { payload: frame.payload, timestamp: frame.timestamp, arrived }; + frame = new Landed(frame, arrived); if (this.#state.flate) { frame = this.#state.flate.push(frame, inflated); if (this.#state.flate.stored > MAX_GROUP_CACHE_BYTES) { From 11ebbd252bbdd22e09dcd5ad1be075e9195175e7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 10:36:44 -0700 Subject: [PATCH 14/18] fix(container): charge the max buffer for the wire frame a decoded view keeps alive Decoded payloads are views of their wire frame (LOC returns the bytes past its property block), so a frame with a large header and a tiny payload kept its whole wire frame allocated while charging only the view. The last frame decoded from a wire frame now also holds whatever of it the views leave out, in both Rust and JS, and each buffered frame keeps its charge for the refund. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 43 +++++++ js/hang/src/container/consumer.ts | 23 +++- rs/moq-mux/src/container/consumer.rs | 107 ++++++++++++++++-- 3 files changed, 160 insertions(+), 13 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 5f450a84407..089d132e897 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -205,6 +205,49 @@ test("Consumer admits reads completing together one at a time against the max bu } }); +// Keeps one byte of each decoded payload, a view of a wire frame mostly made of header, as LOC's is +// behind a large property block. +class Header extends Format { + override decode(frame: Uint8Array): Frame[] { + return super.decode(frame).map((decoded) => ({ ...decoded, payload: decoded.payload.subarray(0, 1) })); + } +} + +test("Consumer charges a small view of a large wire frame the wire frame", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Header("data"), + maxDelay: Milli(5_000), + maxBuffer: 16 * 1024, + }); + try { + const stalled = track.createGroup(0); + write(stalled, 0); + const padded = track.createGroup(1); + const padding = new Uint8Array(60_000); + for (let i = 0; i < 8; i++) { + padded.writeFrame({ + payload: encodeFrame(padding, (10_000 + i) as Micro), + timestamp: Time.Timestamp.now(), + }); + } + padded.finish(); + + // The padded frames fill the max buffer behind the stalled group, which moves the cursor on to them. + // Charging only the one byte views, the buffer never fills and the reader waits on the stall. + expect(await within(consumer.next())).toMatchObject({ kind: "frame", frame: { timestamp: 0 } }); + expect(await within(consumer.next())).toMatchObject({ kind: "gap" }); + for (let i = 0; i < 8; i++) { + expect(await within(consumer.next())).toMatchObject({ kind: "frame", frame: { timestamp: 10_000 + i } }); + } + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + test("Consumer stamps frames the max buffer held back with when they landed, not when they decoded", async () => { const clock = new Manual(); const installed = clock.install(); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index e2e4559b375..3554b59647d 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -21,8 +21,9 @@ export interface ConsumerProps { // Read-only: a Getter (e.g. another component's output) is accepted directly. maxDelay?: GetterInit; /** - * Cap the bytes of decoded frames held ahead of the reader, each frame's and group's own overhead - * included (default: {@link DEFAULT_MAX_BUFFER}). Mirrors Rust's `Consumer::set_max_buffer`. + * Cap the bytes of decoded frames held ahead of the reader, counting the whole wire payload they + * share and each frame's and group's own overhead (default: {@link DEFAULT_MAX_BUFFER}). Mirrors + * Rust's `Consumer::set_max_buffer`. * * Once it is full the consumer stops decoding ahead and stops pulling new groups past one successor * to the group being read, so the undecoded rest stays in the transport's cache, which bounds it. The group being read still decodes a wire @@ -75,6 +76,7 @@ interface Group { // and the groups past it, kept on insert so a hole reads it without scanning the backlog. late: Instant; frames: ReceivedFrame[]; // decode order + costs: number[]; // What each of frames holds against the max buffer empty: boolean; // no wire frame was published; empty groups mean nothing media: boolean; // a decodable (non-marker) frame was buffered bare?: boolean; // a data track's marker group: one object with no bytes @@ -232,6 +234,7 @@ export class Consumer { consumer, late: now(), frames: [], + costs: [], empty: true, media: false, index: 0, @@ -348,6 +351,9 @@ export class Consumer { group.bare = decoded.length === 0; } + // Decoded payloads are views of the wire payload, which stays allocated until the last of them is + // read, so that one also holds whatever of it the views leave out, such as a large header. + let views = 0; for (const sample of decoded) { const marker = this.#format.end?.(sample) !== undefined; @@ -382,7 +388,10 @@ export class Consumer { group.start ??= frame.timestamp; group.frames.push(frame); - this.#charge(group, FRAME_COST + frame.payload.byteLength); + const cost = FRAME_COST + frame.payload.byteLength; + group.costs.push(cost); + this.#charge(group, cost); + views += frame.payload.byteLength; if (group.latest === undefined || frame.timestamp > group.latest) { group.latest = frame.timestamp; @@ -393,6 +402,11 @@ export class Consumer { group.end = end; } } + const rest = wire.payload.byteLength - views; + if (decoded.length > 0 && rest > 0) { + group.costs[group.costs.length - 1] += rest; + this.#charge(group, rest); + } } // The transport's verdict on a group with nothing left to read: null once it finished cleanly, the @@ -588,6 +602,7 @@ export class Consumer { #discard(group: Group): boolean { group.consumer.close(); group.frames.length = 0; + group.costs.length = 0; this.#release(group); const start = this.#start; if ( @@ -700,7 +715,7 @@ export class Consumer { const seq = head.consumer.sequence; const frame = head.frames.shift(); if (frame) { - this.#refund(head, FRAME_COST + frame.payload.byteLength); + this.#refund(head, head.costs.shift() ?? 0); const end = this.#format.end?.(frame); if (end !== undefined) { this.#updateBuffered(); diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 2a665a4d479..261045e1582 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -106,7 +106,8 @@ pub(crate) enum Item { /// ## Memory /// /// Decoded frames waiting to be read are charged to a byte cap, [`set_max_buffer`](Self::set_max_buffer), -/// along with each buffered frame's and group's own overhead. Once it is full the consumer stops +/// counting the whole wire payload they share, along with each buffered frame's and group's own +/// overhead. Once it is full the consumer stops /// decoding ahead and stops pulling new groups past one successor to the cursor's group, so the /// undecoded rest stays in the transport's cache, which bounds it. The group the cursor is on still decodes a wire frame whenever it holds nothing, so /// reading never waits on its own lookahead. If that group stalls while the cap is full, the cursor @@ -836,8 +837,8 @@ impl Consumer { let _ = track.update(subscription); } - /// Cap the bytes of decoded frames held ahead of the reader, each frame's and group's own - /// overhead included (default [`DEFAULT_MAX_BUFFER`]). Past it the consumer stops decoding ahead + /// Cap the bytes of decoded frames held ahead of the reader, counting the wire payload they share + /// and each frame's and group's own overhead (default [`DEFAULT_MAX_BUFFER`]). Past it the consumer stops decoding ahead /// and pulling new groups; see the type docs. pub fn set_max_buffer(&mut self, bytes: usize) { self.memory.set_max(bytes); @@ -889,7 +890,8 @@ struct GroupBuffer { fault: Option, // Read frames that haven't been consumed yet, each with when its wire frame landed. - buffered: VecDeque<(Frame, Instant)>, + // Each with when it arrived and what it holds against the max buffer. + buffered: VecDeque<(Frame, Instant, usize)>, markers: VecDeque<(usize, Timestamp)>, delivered: usize, @@ -941,9 +943,9 @@ impl GroupBuffer { self.refund(MARKER_COST); return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: index == 0 }))); } - if let Some((frame, arrived)) = self.buffered.pop_front() { + if let Some((frame, arrived, cost)) = self.buffered.pop_front() { self.delivered += 1; - self.refund(frame_cost(&frame)); + self.refund(cost); return Poll::Ready(Ok(Some(Event::Frame { frame, arrived }))); } if !ready!(self.buffer_once(waiter, format)?) { @@ -969,6 +971,7 @@ impl GroupBuffer { }; self.empty = false; let arrived = wire.arrived(); + let size = wire.payload.len(); let frames = format.decode(wire).map_err(Failure::Decode)?; if format.kind() == super::Kind::Data { // A data track's marker is one object with no bytes, alone in its group. @@ -979,6 +982,10 @@ impl GroupBuffer { self.bare = frames.is_empty(); } + // Decoded payloads are views of the wire payload, which stays allocated until the last of them + // is read, so that one also holds whatever of it the views leave out, such as a large header. + let mut views = 0; + let first = self.buffered.len(); for mut frame in frames { if let Some(bound) = format.end(&frame) { self.note_end(bound); @@ -1012,8 +1019,18 @@ impl GroupBuffer { self.index += 1; self.media = true; - self.charge(frame_cost(&frame)); - self.buffered.push_back((frame, arrived)); + let cost = frame_cost(&frame); + views += frame.payload.len(); + self.charge(cost); + self.buffered.push_back((frame, arrived, cost)); + } + let rest = size.saturating_sub(views); + if self.buffered.len() > first + && rest > 0 + && let Some((_, _, cost)) = self.buffered.back_mut() + { + *cost += rest; + self.charge(rest); } Poll::Ready(Ok(true)) @@ -1130,7 +1147,7 @@ const MARKER_COST: usize = std::mem::size_of::<(usize, Timestamp)>(); /// What a buffered frame holds against the max buffer: its payload and its own entry. fn frame_cost(frame: &Frame) -> usize { - std::mem::size_of::<(Frame, Instant)>() + frame.payload.len() + std::mem::size_of::<(Frame, Instant, usize)>() + frame.payload.len() } impl std::ops::Deref for GroupBuffer { @@ -4236,6 +4253,78 @@ mod tests { assert_eq!(frames, vec![ts(0), ts(20_000)]); } + /// A decoded payload is a view of its wire frame, which stays allocated while the frame is held, so + /// the max buffer charges the whole wire frame rather than the view. Here each is a 1 byte payload + /// behind a large LOC property block. + #[test] + fn a_small_view_of_a_large_wire_frame_is_charged_the_wire_frame() { + const PADDING: usize = 60_000; + const MAX_BUFFER: usize = 16 * 1024; + + // A QUIC varint, which is all LOC's framing needs. + fn varint(buf: &mut Vec, value: usize) { + match value { + 0..64 => buf.push(value as u8), + 64..16_384 => buf.extend_from_slice(&(value as u16 | 0x4000).to_be_bytes()), + _ => buf.extend_from_slice(&(value as u32 | 0x8000_0000).to_be_bytes()), + } + } + + let waiter = kio::Waiter::noop(); + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let subscriber = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(30))); + let loc = Container::Loc(crate::container::Kind::Data); + let mut consumer = Consumer::new(subscriber, Container::Loc(crate::container::Kind::Data)); + consumer.set_max_buffer(MAX_BUFFER); + + let mut stalled = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let frame = Frame { + timestamp: ts(0), + payload: Bytes::from_static(&[0xDE]), + keyframe: false, + duration: None, + }; + loc.write(&mut stalled, &[frame]).unwrap(); + let mut padded = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + for i in 0..8 { + let timestamp = 10_000 + i * 1_000; + // An unknown video-config property (0x0d) padding the frame, then the timestamp (0x10). + let mut props = Vec::new(); + varint(&mut props, 0x0d); + varint(&mut props, PADDING); + props.resize(props.len() + PADDING, 0); + varint(&mut props, 0x10 - 0x0d); + varint(&mut props, timestamp); + let mut wire = Vec::new(); + varint(&mut wire, props.len()); + wire.extend_from_slice(&props); + wire.push(0xAD); + padded.write_frame(ts(timestamp as u64), wire).unwrap(); + } + padded.finish().unwrap(); + + // The padded frames fill the max buffer behind the stalled group, which moves the cursor on to them. + // Charging only the 1 byte views, the buffer never fills and the reader waits on the stall forever. + let slack = PADDING + 1024 + 2 * std::mem::size_of::>(); + let mut reads = Vec::new(); + while let Poll::Ready(read) = consumer.poll_read(&waiter) { + let used = consumer.buffered_bytes(); + assert!( + used <= MAX_BUFFER + slack, + "holding {used} bytes past a {MAX_BUFFER} byte max buffer" + ); + match read.unwrap() { + Some(Read::Frame { frame, .. }) => reads.push(Some(frame.timestamp)), + Some(Read::Gap { .. }) => reads.push(None), + other => panic!("unexpected read: {other:?}"), + } + } + let mut expected = vec![Some(ts(0)), None]; + expected.extend((0..8).map(|i| Some(ts(10_000 + i * 1_000)))); + assert_eq!(reads, expected); + } + /// A stalled successor does not hide a readable group behind it once group overhead alone fills the /// max buffer: the cursor moves on group by group until one has something to read. #[test] From 13c595e6f6e6fe774c094dc15d97eeca744f9c91 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 11:45:57 -0700 Subject: [PATCH 15/18] fix(container): expire a hole from when the group past it landed A full max buffer delays pulling a group, and its `late` was stamped at the pull, so an already-expired hole waited another full max delay. Expose when a group arrived at this hop as `group::Consumer::arrived()` (Rust) and `Group.Consumer.arrived` (JS), and seed `late` from it. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 33 +++++++++++++ js/hang/src/container/consumer.ts | 3 +- js/net/api/index.api.md | 1 + js/net/src/group.test.ts | 47 +++++++++++++++++++ js/net/src/group.ts | 24 ++++++++-- rs/moq-mux/src/container/consumer.rs | 45 +++++++++++++++++- rs/moq-net/src/model/group.rs | 42 +++++++++++++++++ 7 files changed, 190 insertions(+), 5 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index 089d132e897..f6063971e47 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -277,6 +277,39 @@ test("Consumer stamps frames the max buffer held back with when they landed, not } }); +test("Consumer expires a hole behind a full max buffer from when the group past it landed", async () => { + const clock = new Manual(); + const installed = clock.install(); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(500), + // Two groups' overhead: group 3 waits in the transport until groups 0 and 1 are read. + maxBuffer: 512, + }); + try { + for (const sequence of [0, 1]) { + const group = track.createGroup(sequence); + write(group, sequence * 10_000); + group.finish(); + } + const marker = track.createGroup(3); + marker.writeFrame({ payload: new Uint8Array(), timestamp: Time.Timestamp.now() }); + marker.finish(); + // The reader is slow: the hole before group 3 expired before anything is read. + await clock.advance(Nano.fromMilli(Milli(1_000))); + + expect(await within(consumer.next())).toMatchObject({ kind: "frame", group: 0 }); + expect(await within(consumer.next())).toMatchObject({ kind: "frame", group: 1 }); + expect(await within(consumer.next())).toMatchObject({ kind: "gap" }); + expect(await within(consumer.next())).toMatchObject({ kind: "end", group: 3 }); + } finally { + consumer.close(); + track.close(); + installed.close(); + } +}); + for (const maxBuffer of [0, 256]) { test(`Consumer moves past a stalled group with a ${maxBuffer} byte max buffer`, async () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 3554b59647d..3d68ee999b6 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -232,7 +232,8 @@ export class Consumer { // may be is the subscription's own max delay, applied before it ever reaches here. const group: Group = { consumer, - late: now(), + // When the group landed, not when it was pulled: a full max buffer delays pulls. + late: consumer.arrived, frames: [], costs: [], empty: true, diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index af9b9271bfd..b6b296c3992 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -310,6 +310,7 @@ export namespace Error { export namespace Group { export class Consumer { + get arrived(): Instant; close(): void; get closed(): Signals.GetPromise; get done(): boolean; diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index 3065ee13e99..5c73f3332a8 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -6,6 +6,7 @@ import { MAX_GROUP_CACHE_BYTES, MAX_GROUP_FRAMES, Producer } from "./group.ts"; import { hooks, type Replacement } from "./internal.ts"; import { textFrame } from "./mock.ts"; import { Timestamp } from "./time.ts"; +import { Producer as TrackProducer } from "./track.ts"; const dec = new TextDecoder(); @@ -123,6 +124,52 @@ test("a frame read late keeps the instant it landed, plain, compressed, or resum } }); +test("a group read late keeps the instant it joined, alone, on a track, or resumed", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const created = clock.now(); + const alone = new Producer(0); + const track = new TrackProducer("test"); + const subscriber = track.subscribe(); + const live = track.createGroup(0); + live.writeFrame(textFrame("a")); + clock.jump(Nano.fromMilli(Milli(1000))); + + expect(alone.consume().arrived).toBe(created); + expect((await subscriber.recvGroup())?.arrived).toBe(created); + + // A resumed read keeps the original group's arrival, not the replacement copy's. + const failing = new Producer(0); + const resumed = failing.mirror(); + clock.jump(Nano.fromMilli(Milli(1000))); + const replacement = new Producer(0); + replacement.writeFrame(textFrame("a")); + replacement.writeFrame(textFrame("b")); + hooks.recoverGroup(resumed, { + changed: new Signal(0), + wants: (failed) => failed !== undefined, + replace: async () => ({ + group: replacement.mirror(), + expiry: { expired: () => false, changed: [] }, + adopt: () => {}, + discard: () => {}, + }), + }); + failing.writeFrame(textFrame("a")); + await resumed.readFrame(); + failing.abort(new Error("copy failed")); + expect((await resumed.readFrame())?.payload).toEqual(textFrame("b").payload); + expect(resumed.arrived).toBe(created + Nano.fromMilli(Milli(1000))); + + for (const producer of [alone, live, replacement]) producer.finish(); + subscriber.close(); + track.close(); + } finally { + installed.close(); + } +}); + test("zero group PTS keeps its scale and untimed groups stay unstamped", () => { const installed = new Manual().install(); try { diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 1ae4b134d75..3d5f8fc6be6 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -3,7 +3,7 @@ * * @module */ -import { Instant, now } from "@moq/clock"; +import { type Instant, now } from "@moq/clock"; import { type Dispose, type GetPromise, type Getter, Once, race, Signal } from "@moq/signals"; import { FrameTooLarge, @@ -94,8 +94,9 @@ class GroupState { // so this mirror drives no media-time expiry. unverified = false; // When the group joined the track this mirror belongs to, at this hop: the wall clock a - // superseded predecessor ages from. Per track, since one group can join several. - arrived = Instant.zero; + // superseded predecessor ages from. Per track, since one group can join several. A group + // no track holds arrived when it was created. + arrived = now(); // Called with the frame count on every write, synchronously: a track's largest must not // trail a write that already returned, as a batched `total` notification would. progress = new Set<(total: number) => void>(); @@ -329,6 +330,8 @@ export class Producer { dst.latest = this.#state.latest; dst.flate = this.#state.flate; dst.compressed = this.#state.compressed; + // A track restamps its own mirror with when the group joined it. + dst.arrived = this.#state.arrived; for (const frame of this.#state.frames.peek()) appendFrame(dst, frame); dst.start = this.#state.start; dst.offset = this.#state.offset; @@ -677,12 +680,26 @@ export class Consumer { #expired?: Expired; // Closed by its reader, which ends it whatever is left unread. #released = false; + // The arrival of the copy first read, kept once a replacement copy is adopted. + #arrived?: Instant; private constructor(state: GroupState) { this.#state = state; this.sequence = state.sequence; } + /** + * When the group joined its track at this hop, before its first frame, on the installed + * clock: its header was received, or it was fetched or created here. + * + * A reader that gets to the group late still measures from when it landed, as + * {@link Frame.arrived} does for each frame. A consumer resumed onto a replacement copy + * keeps the original group's arrival. + */ + get arrived(): Instant { + return this.#arrived ?? this.#state.arrived; + } + /** * Settles when this consumer reaches a terminal state: `null` after a clean close has no * unread or in-flight frames, or the terminal {@link Error}. @@ -887,6 +904,7 @@ export class Consumer { replacement.adopt(); // The copy left behind stops teeing into the state this consumer read. if (this.#state.closed.peek() === undefined) this.#state.closed.set(null); + this.#arrived ??= this.#state.arrived; this.#state = next; this.#expiry = replacement.expiry; this.#skip = index; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 261045e1582..6c498b99644 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -914,8 +914,9 @@ struct GroupBuffer { impl GroupBuffer { fn new(group: moq_net::group::Consumer, memory: Memory) -> Self { let mut this = Self { + // When the group landed, not when it was pulled: a full max buffer delays pulls. + late: group.arrived(), group, - late: moq_clock::now(), index: 0, empty: true, media: false, @@ -4427,6 +4428,48 @@ mod tests { assert_eq!(reads, ["gap", "end 1", "frame 2 20000"]); } + /// A hole's budget runs from when the group past it landed, not from when the reader pulled it: + /// a full max buffer holds pulls back, so a hole that already expired is passed at once. + #[tokio::test(start_paused = true)] + async fn a_hole_behind_a_full_max_buffer_expires_from_its_successors_arrival() { + let max_delay = Duration::from_millis(500); + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = container_max_delay_only(track.subscribe(None), max_delay); + consumer.set_max_buffer(4096); + for sequence in 0..2u64 { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + Container::Legacy(crate::container::Kind::Data) + .write( + &mut group, + &[Frame { + timestamp: ts(sequence * 10_000), + payload: Bytes::from(vec![0; 4096]), + keyframe: false, + duration: None, + }], + ) + .unwrap(); + group.finish().unwrap(); + } + let mut marker = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + marker.write_frame(ts(20_000), Bytes::new()).unwrap(); + marker.finish().unwrap(); + + // The reader gets to the track only after the hole before group 3 has expired. + tokio::time::advance(2 * max_delay).await; + let waiter = kio::Waiter::noop(); + let mut reads = Vec::new(); + while let Poll::Ready(read) = consumer.poll_read(&waiter) { + reads.push(match read.unwrap() { + Some(Read::Frame { group, frame }) => format!("frame {group} {}", frame.timestamp.as_micros()), + Some(Read::Gap { .. }) => "gap".to_string(), + Some(Read::End { group, .. }) => format!("end {group}"), + None => break, + }); + } + assert_eq!(reads, ["frame 0 0", "frame 1 10000", "gap", "end 3"]); + } + /// No cap is too small to read through: the cursor's group and one successor are always pulled, so /// a stalled group can be moved past even when one group's overhead fills the max buffer. #[test] diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 96bae34053b..d13722215d8 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1164,6 +1164,8 @@ struct Alive { // A reader of the frame log, cloned into each cursor so it reads frames without the // group lock. log: kio::log::Reader, + // When the group was created here, on the ambient clock; see [`Consumer::arrived`]. + arrived: crate::time::Instant, } impl Drop for Producer { @@ -1293,6 +1295,7 @@ impl Producer { started: AtomicBool::new(false), access, log, + arrived: super::clock::now(), }); Self { handle: Handle { @@ -2517,6 +2520,7 @@ impl Handle { adopted: false, stale_counted: Arc::default(), recover: None, + arrived: self.alive.arrived, } } @@ -2919,6 +2923,9 @@ pub struct Consumer { // Handed out from a front's logical track: carries the read across route changes. // Boxed: it is the rare case. recover: Option>, + // When the group was created here. Kept across an adopted replacement copy, which the + // group arrived before. + arrived: crate::time::Instant, } /// Subscriber-specific policy for expiring a group after it was handed out. @@ -2999,6 +3006,7 @@ impl Clone for Consumer { adopted: self.adopted, stale_counted: self.stale_counted.clone(), recover: self.recover.clone(), + arrived: self.arrived, } } } @@ -3255,6 +3263,16 @@ impl Consumer { self.track.timescale } + /// When the group was created at this hop, before its first frame, on the ambient clock: + /// when its header was received, or when it was fetched or created here. + /// + /// Fixed for the consumer's life, so a reader that pulls the group late still measures + /// from when it landed, as [`frame::Frame::arrived`] does for each frame. A consumer + /// resumed onto a replacement copy keeps the original group's arrival. + pub fn arrived(&self) -> crate::time::Instant { + self.arrived + } + /// The timescale a moq-lite hop serves the parent track at (see /// [`track::Info::lite_timescale`]). #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -4986,6 +5004,30 @@ mod test { assert_eq!(arrived, [0, 20, 40, 40], "stamped on completion, a batch at its write"); } + /// A group is stamped when it is created here, and a reader pulling it late from its track + /// still sees that stamp. + #[moq_clock::test] + #[cfg(feature = "async")] + async fn groups_carry_when_they_landed() { + let start = moq_clock::now(); + let mut alone = Info { sequence: 0 }.produce(); + let mut track = crate::broadcast::Info::new() + .produce() + .create_track("test", None) + .unwrap(); + let mut subscriber = track.subscribe(None); + let mut live = track.create_group(Info { sequence: 0 }).unwrap(); + + moq_clock::sleep(std::time::Duration::from_secs(1)).await; + assert_eq!(alone.consume().arrived(), start); + let group = subscriber.recv_group().await.unwrap().expect("the live group"); + assert_eq!(group.arrived(), start); + assert_eq!(group.clone().arrived(), start, "a clone keeps it"); + + alone.finish().unwrap(); + live.finish().unwrap(); + } + /// A frame whose timestamp is at a different scale is converted to the group's /// scale by `create_frame`. #[test] From 8f587e710f2a8d58b1645dba453aa63db522b5d0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 11:52:47 -0700 Subject: [PATCH 16/18] test: drop unneeded muts and a branded-instant sum Co-Authored-By: Claude Opus 5.5 --- js/net/src/group.test.ts | 3 ++- rs/moq-mux/src/container/consumer.rs | 2 +- rs/moq-net/src/model/group.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index 5c73f3332a8..14a22fe0a91 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -140,6 +140,7 @@ test("a group read late keeps the instant it joined, alone, on a track, or resum expect((await subscriber.recvGroup())?.arrived).toBe(created); // A resumed read keeps the original group's arrival, not the replacement copy's. + const original = clock.now(); const failing = new Producer(0); const resumed = failing.mirror(); clock.jump(Nano.fromMilli(Milli(1000))); @@ -160,7 +161,7 @@ test("a group read late keeps the instant it joined, alone, on a track, or resum await resumed.readFrame(); failing.abort(new Error("copy failed")); expect((await resumed.readFrame())?.payload).toEqual(textFrame("b").payload); - expect(resumed.arrived).toBe(created + Nano.fromMilli(Milli(1000))); + expect(resumed.arrived).toBe(original); for (const producer of [alone, live, replacement]) producer.finish(); subscriber.close(); diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 6c498b99644..be9374df9c0 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -4433,7 +4433,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn a_hole_behind_a_full_max_buffer_expires_from_its_successors_arrival() { let max_delay = Duration::from_millis(500); - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = container_max_delay_only(track.subscribe(None), max_delay); consumer.set_max_buffer(4096); for sequence in 0..2u64 { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index d13722215d8..c6fa532db04 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -5011,7 +5011,7 @@ mod test { async fn groups_carry_when_they_landed() { let start = moq_clock::now(); let mut alone = Info { sequence: 0 }.produce(); - let mut track = crate::broadcast::Info::new() + let track = crate::broadcast::Info::new() .produce() .create_track("test", None) .unwrap(); From 21f109067c6e4d0aab59b6d6c4bb904bcb42e594 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 12:19:55 -0700 Subject: [PATCH 17/18] fix(container): keep a hole late from the earliest arrival past it, and raise a skip's fault A drained backlog comes in sequence order, not arrival order, so an arrival now lowers the `late` of the groups below it. In JS, a fault an age skip in `#look()` raises after the track finished was read as a clean end; `next()` now throws it. Co-Authored-By: Claude Opus 5.5 --- .../container/consumer.backpressure.test.ts | 30 ++++++++++++ js/hang/src/container/consumer.holes.test.ts | 49 +++++++++++++++++++ js/hang/src/container/consumer.ts | 6 ++- rs/moq-mux/src/container/consumer.rs | 40 ++++++++++++++- 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/js/hang/src/container/consumer.backpressure.test.ts b/js/hang/src/container/consumer.backpressure.test.ts index f6063971e47..381e5d92654 100644 --- a/js/hang/src/container/consumer.backpressure.test.ts +++ b/js/hang/src/container/consumer.backpressure.test.ts @@ -447,6 +447,36 @@ test("Consumer raises a malformed frame in a head promoted under pressure", asyn } }); +test("Consumer raises a malformed group an age skip passes after the track finished", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(500), + }); + try { + const first = track.createGroup(0); + write(first, 0); + first.finish(); + const malformed = track.createGroup(1); + const later = track.createGroup(2); + track.finish(); + expect(await within(consumer.next())).toMatchObject({ kind: "frame", group: 0 }); + await flush(); + + // Group 1 holds a valid frame then a malformed one, and group 2 starts past the max delay, so the + // next read's decode skips group 1 for its age. + write(malformed, 10_000); + malformed.writeFrame({ payload: new Uint8Array([0x80]), timestamp: Time.Timestamp.now() }); + write(later, 2_000_000); + await expect(within(consumer.next())).rejects.toThrow("buffer too short"); + } finally { + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + test("Consumer reads a finished marker group's end before skipping under pressure", async () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); const track = new Track.Producer("test"); diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index 9b0d50f540e..7b0d7fecc7f 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -186,3 +186,52 @@ test("a hole is late from the earliest arrival past it", async () => { installed.close(); } }); + +test("a hole in a drained backlog is late from the earliest arrival past it, not the first admitted", async () => { + const clock = new Manual(); + const installed = clock.install(); + const track = new Track.Producer("data"); + const marker = (sequence: number) => { + const group = track.createGroup(sequence); + group.writeFrame({ payload: new Uint8Array(), timestamp: Time.Timestamp.fromMicros(sequence * 10_000) }); + group.finish(); + }; + const first = track.createGroup(0); + first.writeFrame({ payload: encodeFrame(new Uint8Array([1]), Micro(0)), timestamp: Time.Timestamp.fromMicros(0) }); + first.finish(); + marker(5); + await clock.advance(Nano.fromMilli(Milli(900))); + marker(3); + await clock.advance(Nano.fromMilli(Milli(100))); + + // Joined now, the backlog drains in sequence order: group 3 before group 5, which landed first. + const consumer = new Consumer(track.subscribe({ maxDelay: Milli(30_000) }), { + format: new Format("data"), + maxDelay: Milli(500), + }); + const results: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + results.push(read); + } + })(); + try { + await flush(); + // Group 5 landed a second ago, so the holes before groups 3 and 5 both expired. + expect(results.map((read) => (read.kind === "end" ? `end ${read.group}` : read.kind))).toEqual([ + "frame", + "gap", + "end 3", + "gap", + "end 5", + ]); + } finally { + consumer.close(); + track.finish(); + track.close(); + await reading; + installed.close(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 3d68ee999b6..4d9a8f83239 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -247,9 +247,11 @@ export class Consumer { // This is used to cancel old groups. let index = this.#groups.findIndex((other) => other.consumer.sequence > consumer.sequence); if (index === -1) index = this.#groups.length; - // Arrivals only move forward, so the groups below this one keep their earliest arrival. + // Each group is late from the earliest arrival among it and the groups past it. A drained + // backlog comes in sequence order, not arrival order, so an arrival can lower those below it. const next = this.#groups.at(index); if (next && next.late < group.late) group.late = next.late; + for (let i = index - 1; i >= 0 && this.#groups[i].late > group.late; i--) this.#groups[i].late = group.late; this.#groups.splice(index, 0, group); // Group overhead alone can fill the max buffer, past which next() moves on from a stalled head. if (this.#full()) { @@ -651,6 +653,8 @@ export class Consumer { if (this.#error) throw this.#error; // Decode what has arrived, so every decision below sees it. this.#look(); + // Its skips can raise a fault, which a finished track's settled `#closed` cannot carry. + if (this.#error) throw this.#error; const ended = this.#closed.peek(); if (this.#groups[0]?.consumer.sequence === this.#active || this.#groups.length === 0) { diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index be9374df9c0..f93bec2bf59 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -775,10 +775,17 @@ impl Consumer { let idx = self .pending .partition_point(|g| g.group.sequence < reader.group.sequence); - // Arrivals only move forward, so the groups below this one keep their earliest arrival. + // Each group is late from the earliest arrival among it and the groups past it. A drained + // backlog comes in sequence order, not arrival order, so an arrival can lower those below it. if let Some(next) = self.pending.get(idx) { reader.late = reader.late.min(next.late); } + for below in self.pending.range_mut(..idx).rev() { + if below.late <= reader.late { + break; + } + below.late = reader.late; + } self.pending.insert(idx, reader); } } @@ -4470,6 +4477,37 @@ mod tests { assert_eq!(reads, ["frame 0 0", "frame 1 10000", "gap", "end 3"]); } + /// A joining reader drains the backlog in sequence order, not arrival order, so a group can land + /// before the one below it is pulled: the hole before that lower group is late from the earlier one. + #[tokio::test(start_paused = true)] + async fn a_hole_in_a_drained_backlog_is_late_from_the_earliest_arrival_past_it() { + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let marker = |track: &mut moq_net::track::Producer, sequence: u64| { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + group.write_frame(ts(sequence * 10_000), Bytes::new()).unwrap(); + group.finish().unwrap(); + }; + write_group(&mut track, 0, &[ts(0)]); + marker(&mut track, 5); + tokio::time::advance(Duration::from_millis(900)).await; + marker(&mut track, 3); + tokio::time::advance(Duration::from_millis(100)).await; + + let mut consumer = container_max_delay_only(track.subscribe(None), Duration::from_millis(500)); + let waiter = kio::Waiter::noop(); + let mut reads = Vec::new(); + while let Poll::Ready(read) = consumer.poll_read(&waiter) { + reads.push(match read.unwrap() { + Some(Read::Frame { group, .. }) => format!("frame {group}"), + Some(Read::Gap { .. }) => "gap".to_string(), + Some(Read::End { group, .. }) => format!("end {group}"), + None => break, + }); + } + // Group 5 landed a second ago, so the holes before groups 3 and 5 both expired. + assert_eq!(reads, ["frame 0", "gap", "end 3", "gap", "end 5"]); + } + /// No cap is too small to read through: the cursor's group and one successor are always pulled, so /// a stalled group can be moved past even when one group's overhead fills the max buffer. #[test] From 99a5924bb2bd138e3c0bfaa8ad309d9483460ccd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 12:25:55 -0700 Subject: [PATCH 18/18] style(hang): format the late propagation loop Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 4d9a8f83239..d38796a1f44 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -251,7 +251,8 @@ export class Consumer { // backlog comes in sequence order, not arrival order, so an arrival can lower those below it. const next = this.#groups.at(index); if (next && next.late < group.late) group.late = next.late; - for (let i = index - 1; i >= 0 && this.#groups[i].late > group.late; i--) this.#groups[i].late = group.late; + for (let i = index - 1; i >= 0 && this.#groups[i].late > group.late; i--) + this.#groups[i].late = group.late; this.#groups.splice(index, 0, group); // Group overhead alone can fill the max buffer, past which next() moves on from a stalled head. if (this.#full()) {