From acb5154c0ed5914414f2bef31e81c48cca8cb2b9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 10:24:29 -0700 Subject: [PATCH 1/2] fix(audio): bound the jitter reference history The reference search scanned every arrival still inside the window, and the window is media time, which the publisher chooses. A track stamped a microsecond apart holds millions of entries inside two seconds, so every frame's scan grew with the peer's timestamp rate and a viewer's main thread stalled. Keep a monotonic deque of the prefix minimum beside the history, so an in-order frame reads its reference in O(1) amortized, and fall back to the scan only for a reordered frame, where the prefix is shorter than the history. Cap the history at 8192 arrivals, far above any real track's two-second window, so memory and the reordered scan stay bounded too. Both the browser and native estimators implement the page, so both change together, with the cap named in doc/concept/audio-jitter.md. Co-Authored-By: DeepSeek V4.1 Flash --- doc/concept/audio-jitter.md | 24 ++++- js/hang/src/container/jitter.test.ts | 8 ++ js/hang/src/container/jitter.ts | 99 +++++++++++++++++--- rs/moq-audio/src/jitter.rs | 132 +++++++++++++++++++++++++-- 4 files changed, 242 insertions(+), 21 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index a857987e31d..29ecd16186f 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -91,9 +91,14 @@ observe(arrival, media): if media > newest: # a reordered frame never joins the history newest = media drop history entries with entry.media < media - WINDOW + drop history entries from the front while MAX_HISTORY are held push (arrival, media) onto history + keep the prefix minimum of (arrival - media) beside it - ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media) + if media >= newest: # in media order, every entry is the prefix + ref = the prefix minimum # stored as the entry itself, not a reassociated value + else: # reordered, a shorter prefix + ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media) if there is none: return # older than the whole window, not measured delay = max(0, (arrival - ref.arrival) - (media - ref.media)) @@ -139,6 +144,22 @@ special case, and neither does a group the relay dropped. fills over ten seconds is measured as the part of it that happened in the last two. +### Bounding the history + +The history holds at most `MAX_HISTORY` (8192) arrivals. The window is media +time and the publisher chooses the timestamp rate, so without a cap a stream +that advances a microsecond per frame holds millions of entries inside two +seconds, and every frame's reference search scans them. The cap stays far above +the two-second window of any real track, including a 1000 fps one (2000 +entries); a stream dense enough to reach it measures against a window shorter +than `WINDOW` instead of growing without bound. + +The reference is a prefix minimum, so an in-order frame reads it from a deque +kept beside the history in O(1) amortized. A reordered frame sees only a prefix +the deque cannot answer, and scans the capped history. Both produce exactly the +minimum the definition above names; the structure is an implementation detail, +and the cap is the only place where this page deliberately narrows the window. + ### Reordered arrivals are measured against what came before them A frame whose media timestamp is not newer than the newest already observed is @@ -473,6 +494,7 @@ The exceptions are marked. | `FRESH_MAX` | 0.1 | Ours, twice the quantile's tail. NetEq ramps the forget factor from a start weight of 2 instead; see [the histogram](#the-histogram) | | `PRIOR[i]` | `0.5^(i+1)` | NetEq's histogram seed | | `IDLE_MAX` | 1024 intervals | Ours. A work bound with no effect on output | +| `MAX_HISTORY` | 8192 arrivals | Ours. Bounds a rate the publisher chooses; see [Bounding the history](#bounding-the-history) | | Out-of-range delays | clamped, not dropped | Ours. See [the histogram](#the-histogram) | ## Implementing from this page diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index 62a8b49b144..f41dd2fd114 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -114,4 +114,12 @@ describe("Jitter", () => { expect(fork.measured).toBeLessThanOrEqual(learned); expect(warm.measured).toBe(learned); }); + + it("caps the retained history against a timestamp rate the peer chooses", () => { + const jitter = new Jitter(); + // One frame per microsecond of media, for well past any real window. + for (let i = 0; i < 20_000; i++) jitter.observe(i as Milli, (i / 1000) as Milli); + expect(jitter.retained).toBeLessThanOrEqual(8192); + expect(Number.isFinite(jitter.measured)).toBe(true); + }); }); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index 012131fb353..be0cacfeee8 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -28,6 +28,16 @@ const FRESH_MAX = 0.1; /** Idle intervals folded at once, purely to bound the work. */ const IDLE_MAX = 1024; +/** + * Arrivals kept in the reference window. + * + * The window is media time, and a publisher chooses the timestamp rate, so an uncapped + * history can hold millions of entries from a stream that advances a microsecond per + * frame. This bounds both memory and a reordered frame's scan, and stays far above the + * 2 s window of any real track (a 1000 fps track holds 2000). + */ +const MAX_HISTORY = 8192; + /** The cold-start distribution, which is also what an idle interval relaxes back to. */ const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1)); @@ -41,7 +51,15 @@ const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1)); */ export class Jitter { /** Arrivals still inside the reference window, in media order. */ - #history: { arrival: number; media: number }[] = []; + #history: { seq: number; arrival: number; media: number }[] = []; + /** The first live entry in `#history`. */ + #historyStart = 0; + /** The prefix minimum of `arrival - media` over `#history`, front first. */ + #minima: { seq: number; key: number; arrival: number; media: number }[] = []; + /** The first live entry in `#minima`. */ + #minimaStart = 0; + /** The next entry's position in the arrival sequence. */ + #nextSeq = 0; /** The largest media timestamp observed, so a reordered frame is recognized. */ #newest = Number.NEGATIVE_INFINITY; @@ -58,6 +76,8 @@ export class Jitter { /** Fold one frame in: `arrival` on the receiver's monotonic clock, `media` its timestamp. */ observe(arrival: Milli, media: Milli): void { + const key = arrival - media; + // A reordered frame is measured but never joins the window: it arrived after a newer frame, // so it can never be the fastest, and a later frame must not be measured against it. if (media > this.#newest) { @@ -65,10 +85,22 @@ export class Jitter { // Pruning by media time is what makes a timestamp jump self-correcting: everything before // it leaves the window at once and the new frame becomes its own reference. - let stale = 0; - while (stale < this.#history.length && this.#history[stale].media < media - WINDOW) stale++; - if (stale > 0) this.#history.splice(0, stale); - this.#history.push({ arrival, media }); + while ( + this.#historyStart < this.#history.length && + this.#history[this.#historyStart].media < media - WINDOW + ) { + this.#historyStart++; + } + while (this.#history.length - this.#historyStart >= MAX_HISTORY) this.#historyStart++; + + const seq = this.#nextSeq++; + this.#history.push({ seq, arrival, media }); + while (this.#minima.length > this.#minimaStart && this.#minima[this.#minima.length - 1].key >= key) { + this.#minima.pop(); + } + this.#minima.push({ seq, key, arrival, media }); + this.#trimMinima(); + this.#compact(); } // Delay against the fastest frame in the window at or before this one, not the previous one, @@ -76,12 +108,10 @@ export class Jitter { // held behind newer ones reads as the wait playing it in order costs. A frame older than the // whole window, such as the backlog a subscription delivers after its live edge, has no // reference and is not measured. - let reference: { arrival: number; media: number } | undefined; - for (const entry of this.#history) { - if (entry.media > media) break; - if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; - } - if (!reference) return; + const reference = this.#reference(media); + if (reference === undefined) return; + // Written in this order on purpose: the document requires bit-identical arithmetic, + // so the difference is not reassociated. const delay = Math.max(0, arrival - reference.arrival - (media - reference.media)); if (this.#origin === undefined) { @@ -106,6 +136,48 @@ export class Jitter { this.#peak = delay; } + /** The fastest arrival at or before `media`, as its `(arrival, media)`. */ + #reference(media: number): { arrival: number; media: number } | undefined { + if (media >= this.#newest) { + // Every entry is at or before a frame in media order, and the deque's front is their minimum. + return this.#minimaStart < this.#minima.length ? this.#minima[this.#minimaStart] : undefined; + } + + // A reordered frame sees only a prefix, which the minimum deque cannot answer. + // The history cap bounds the scan. + let reference: { arrival: number; media: number } | undefined; + for (let i = this.#historyStart; i < this.#history.length; i++) { + const entry = this.#history[i]; + if (entry.media > media) break; + if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; + } + return reference; + } + + /** Drop minimums whose arrivals left the history. */ + #trimMinima(): void { + const front = this.#historyStart < this.#history.length ? this.#history[this.#historyStart].seq : undefined; + if (front === undefined) { + this.#minima.length = 0; + this.#minimaStart = 0; + return; + } + while (this.#minimaStart < this.#minima.length && this.#minima[this.#minimaStart].seq < front) + this.#minimaStart++; + } + + /** Reclaim the dropped prefix once it dominates, so memory does not grow with the stream. */ + #compact(): void { + if (this.#historyStart > 1024 && this.#historyStart * 2 >= this.#history.length) { + this.#history = this.#history.slice(this.#historyStart); + this.#historyStart = 0; + } + if (this.#minimaStart > 1024 && this.#minimaStart * 2 >= this.#minima.length) { + this.#minima = this.#minima.slice(this.#minimaStart); + this.#minimaStart = 0; + } + } + /** * A new estimator that keeps what this one learned but measures arrivals afresh, for a track that * replaces this one on the same path. Its frames are measured against their own fastest, never @@ -118,6 +190,11 @@ export class Jitter { return fork; } + /** @internal The history entries currently retained, for the cap's regression test. */ + get retained(): number { + return this.#history.length - this.#historyStart; + } + /** The measured term: the upper edge of the bucket holding the 95th percentile. */ get measured(): Milli { let cumulative = 0; diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs index 4aa764aade8..4fa318fe660 100644 --- a/rs/moq-audio/src/jitter.rs +++ b/rs/moq-audio/src/jitter.rs @@ -36,6 +36,15 @@ const FRESH_MAX: f64 = 0.1; /// Idle intervals folded at once, purely to bound the work. const IDLE_MAX: f64 = 1024.0; +/// Arrivals kept in the reference window. +/// +/// The window is media time, and a publisher chooses the timestamp rate, so an +/// uncapped history can hold millions of entries from a stream that advances a +/// microsecond per frame. This bounds both memory and a reordered frame's scan, +/// and stays far above the 2 s window of any real track (a 1000 fps track holds +/// 2000). +const MAX_HISTORY: usize = 8192; + /// The playout target, fixed or estimated. pub(crate) enum Target { /// An explicit delay, taken literally: no floor, no frame term, no estimate. @@ -89,6 +98,16 @@ fn prior(i: usize) -> f64 { /// One arrival still inside the reference window. struct Entry { + /// Position in the arrival sequence, so the minimum deque can drop retired entries. + seq: u64, + arrival: f64, + media: f64, +} + +/// A candidate reference: the smallest delay seen at or before its arrival. +struct MinKey { + seq: u64, + key: f64, arrival: f64, media: f64, } @@ -102,6 +121,14 @@ pub(crate) struct Estimator { /// Arrivals still inside the reference window, in media order. history: VecDeque, + /// The prefix minimum of `arrival - media` over `history`, front first. + /// + /// Keys increase from front to back, so the front is the smallest, and a retired + /// entry is dropped by its sequence number. This makes an in-order frame's reference + /// O(1) instead of a scan over a count the publisher chose. + minima: VecDeque, + /// The next entry's position in the arrival sequence. + next_seq: u64, /// The largest media timestamp observed, so a reordered frame is recognized. newest: f64, @@ -123,6 +150,8 @@ impl Estimator { frame: 0.0, advertised, history: VecDeque::new(), + minima: VecDeque::new(), + next_seq: 0, newest: f64::NEG_INFINITY, origin: None, interval: 0.0, @@ -143,7 +172,24 @@ impl Estimator { while self.history.front().is_some_and(|entry| entry.media < media - WINDOW) { self.history.pop_front(); } - self.history.push_back(Entry { arrival, media }); + while self.history.len() >= MAX_HISTORY { + self.history.pop_front(); + } + + let seq = self.next_seq; + self.next_seq += 1; + let key = arrival - media; + self.history.push_back(Entry { seq, arrival, media }); + while self.minima.back().is_some_and(|min| min.key >= key) { + self.minima.pop_back(); + } + self.minima.push_back(MinKey { + seq, + key, + arrival, + media, + }); + self.trim_minima(); } // The fastest frame in the window at or before this one, so delay is measured @@ -151,17 +197,13 @@ impl Estimator { // frame held behind newer ones reads as the wait playing it in order costs. A // frame older than the whole window, such as a subscription's backlog behind its // live edge, has nothing to be measured against. - let mut reference: Option<&Entry> = None; - for entry in self.history.iter().take_while(|entry| entry.media <= media) { - if reference.is_none_or(|best| entry.arrival - entry.media < best.arrival - best.media) { - reference = Some(entry); - } - } - let Some(reference) = reference else { + let Some((reference, reference_media)) = self.reference(media) else { return; }; - let delay = f64::max(0.0, arrival - reference.arrival - (media - reference.media)); + // Written in this order on purpose: the document requires bit-identical + // arithmetic, so the difference is not reassociated. + let delay = f64::max(0.0, arrival - reference - (media - reference_media)); let Some(origin) = self.origin else { self.origin = Some(arrival); @@ -186,6 +228,34 @@ impl Estimator { self.peak = delay; } + /// The fastest arrival at or before `media`, as its `(arrival, media)`. + fn reference(&self, media: f64) -> Option<(f64, f64)> { + if media >= self.newest { + // Every entry is at or before a frame in media order, and the deque's front + // is their minimum. + return self.minima.front().map(|min| (min.arrival, min.media)); + } + + // A reordered frame sees only a prefix, which the minimum deque cannot answer. + // The history cap bounds the scan. + self.history + .iter() + .take_while(|entry| entry.media <= media) + .min_by(|a, b| (a.arrival - a.media).total_cmp(&(b.arrival - b.media))) + .map(|entry| (entry.arrival, entry.media)) + } + + /// Drop minimums whose arrivals left the history. + fn trim_minima(&mut self) { + let Some(front) = self.history.front().map(|entry| entry.seq) else { + self.minima.clear(); + return; + }; + while self.minima.front().is_some_and(|min| min.seq < front) { + self.minima.pop_front(); + } + } + fn target(&self) -> f64 { let mut cumulative = 0.0; let mut measured = BUCKETS as f64 * BUCKET; @@ -340,4 +410,48 @@ pub(crate) mod tests { } assert_eq!(target.millis(), BUCKETS as f64 * BUCKET + 20.0); } + + /// The minimum deque must answer exactly what the documented scan would. + #[test] + fn reference_matches_a_naive_prefix_scan() { + let mut estimator = Estimator::new(0.0); + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + for index in 0..400 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + // Mostly in order, with an occasional reordered frame and a timestamp jump. + let media = if index % 37 == 0 { + (index as f64) * 20.0 - 900.0 + } else if index % 89 == 0 { + (index as f64) * 20.0 + 3000.0 + } else { + (index as f64) * 20.0 + }; + let arrival = (index as f64) * 20.0 + (seed >> 40) as f64 / 1024.0; + estimator.observe(arrival, media); + + for query in [media - 10.0, media, media + 10.0, media + 1.0e6] { + let expected = estimator + .history + .iter() + .take_while(|entry| entry.media <= query) + .map(|entry| entry.arrival - entry.media) + .min_by(f64::total_cmp); + let actual = estimator.reference(query).map(|(arrival, media)| arrival - media); + assert_eq!(actual, expected, "frame {index}, query {query}"); + } + } + } + + /// A peer choosing a microsecond per frame cannot grow the history without bound. + #[test] + fn the_history_is_capped() { + let mut estimator = Estimator::new(0.0); + // One frame per microsecond of media, for well past the cap. + for index in 0..(MAX_HISTORY + 5000) { + estimator.observe(index as f64, index as f64 / 1000.0); + } + assert_eq!(estimator.history.len(), MAX_HISTORY); + assert!(estimator.minima.len() <= MAX_HISTORY); + assert!(estimator.target().is_finite()); + } } From c155a479457781c295812da414533ef7759f060e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 11:40:07 -0700 Subject: [PATCH 2/2] fix(audio): answer any prefix's reference in logarithmic time A reordered frame's reference is a minimum over a prefix of the history, and the scan let a peer spend MAX_HISTORY comparisons per frame by sending only reordered frames. Replace the minimum deque and the scan with a segment tree over a ring, which answers in-order and reordered frames the same way in O(log n), and keeps the oldest entry on a tie as the scan did. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 31 ++- js/hang/bench/jitter.ts | 31 +++ js/hang/src/container/jitter.test.ts | 55 ++++- js/hang/src/container/jitter.ts | 229 +++++++++++++-------- js/justfile | 5 + rs/moq-audio/src/jitter.rs | 287 ++++++++++++++++++--------- 6 files changed, 433 insertions(+), 205 deletions(-) create mode 100644 js/hang/bench/jitter.ts diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 29ecd16186f..66388cd34d6 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -91,14 +91,11 @@ observe(arrival, media): if media > newest: # a reordered frame never joins the history newest = media drop history entries with entry.media < media - WINDOW - drop history entries from the front while MAX_HISTORY are held + if MAX_HISTORY entries are held: drop the oldest push (arrival, media) onto history - keep the prefix minimum of (arrival - media) beside it - if media >= newest: # in media order, every entry is the prefix - ref = the prefix minimum # stored as the entry itself, not a reassociated value - else: # reordered, a shorter prefix - ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media) + ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media), + the oldest on a tie if there is none: return # older than the whole window, not measured delay = max(0, (arrival - ref.arrival) - (media - ref.media)) @@ -148,17 +145,17 @@ two. The history holds at most `MAX_HISTORY` (8192) arrivals. The window is media time and the publisher chooses the timestamp rate, so without a cap a stream -that advances a microsecond per frame holds millions of entries inside two -seconds, and every frame's reference search scans them. The cap stays far above -the two-second window of any real track, including a 1000 fps one (2000 -entries); a stream dense enough to reach it measures against a window shorter -than `WINDOW` instead of growing without bound. - -The reference is a prefix minimum, so an in-order frame reads it from a deque -kept beside the history in O(1) amortized. A reordered frame sees only a prefix -the deque cannot answer, and scans the capped history. Both produce exactly the -minimum the definition above names; the structure is an implementation detail, -and the cap is the only place where this page deliberately narrows the window. +that advances a nanosecond per frame keeps every frame it sends. The cap stays +far above the two-second window of any real track, including a 1000 fps one +(2000 entries); a stream dense enough to reach it measures against a window +shorter than `WINDOW` instead of growing without bound. + +The reference is a minimum over a prefix of the history, and the publisher +chooses the prefix by reordering, so scanning for it lets every frame cost +`MAX_HISTORY` comparisons. Keep the history in a structure that answers any +prefix's minimum in logarithmic time, such as a segment tree over a ring. +Keep the oldest entry on a tie: tied entries with a different `arrival` and +`media` round `delay` differently. ### Reordered arrivals are measured against what came before them diff --git a/js/hang/bench/jitter.ts b/js/hang/bench/jitter.ts new file mode 100644 index 00000000000..35da4373424 --- /dev/null +++ b/js/hang/bench/jitter.ts @@ -0,0 +1,31 @@ +// Sweep the jitter estimator's held history against in-order and reordered frames, the per-frame +// cost a publisher controls through its timestamp rate. + +import type { Milli } from "@moq/clock"; +import { Jitter } from "../src/container/jitter.ts"; + +const FRAMES = 100_000; + +console.log("held,order,ns_per_frame"); +for (const held of [64, 512, 4096, 8192]) { + for (const order of ["in", "reordered"] as const) { + const jitter = new Jitter(); + // Spaced so `held` frames fit inside the two-second window. + const step = 2000 / held; + let media = 0; + for (let i = 0; i < held; i++, media += step) jitter.observe(media as Milli, media as Milli); + + const started = performance.now(); + for (let i = 0; i < FRAMES; i++) { + if (order === "in") { + jitter.observe(media as Milli, media as Milli); + media += step; + } else { + // Just behind the newest, so the reference covers almost the whole history. + jitter.observe(media as Milli, (media - 2 * step) as Milli); + } + } + const ns = ((performance.now() - started) * 1e6) / FRAMES; + console.log(`${held},${order},${ns.toFixed(1)}`); + } +} diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index f41dd2fd114..4e2c8c32bcc 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -3,7 +3,7 @@ import { readdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import type { Milli } from "@moq/clock"; -import { Jitter } from "./jitter"; +import { type Entry, History, Jitter } from "./jitter"; // The conformance corpus beside doc/concept/audio-jitter.md, read directly so this is graded // against the document rather than against another implementation. @@ -114,12 +114,53 @@ describe("Jitter", () => { expect(fork.measured).toBeLessThanOrEqual(learned); expect(warm.measured).toBe(learned); }); +}); - it("caps the retained history against a timestamp rate the peer chooses", () => { - const jitter = new Jitter(); - // One frame per microsecond of media, for well past any real window. - for (let i = 0; i < 20_000; i++) jitter.observe(i as Milli, (i / 1000) as Milli); - expect(jitter.retained).toBeLessThanOrEqual(8192); - expect(Number.isFinite(jitter.measured)).toBe(true); +// The documented scan: the first entry with the smallest key at or before `media`. +function scan(history: History, media: number): Entry | undefined { + let reference: Entry | undefined; + for (let i = 0; i < history.length; i++) { + const entry = history.at(i); + if (entry.media > media) break; + if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; + } + return reference; +} + +describe("History", () => { + // A tied entry with a different (arrival, media) rounds the delay differently, so the tree must + // pick exactly the entry the scan would. + it("picks the entry the scan would, ties included", () => { + const history = new History(); + let seed = 1; + let newest = Number.NEGATIVE_INFINITY; + for (let i = 0; i < 400; i++) { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + // Mostly in order, with an occasional reordered frame and a timestamp jump. + const media = i % 37 === 0 ? i * 20 - 900 : i % 89 === 0 ? i * 20 + 3000 : i * 20; + // Whole milliseconds of jitter tie often; the occasional fraction does not. + const jitter = i % 5 === 0 ? seed / 2 ** 21 : seed % 8; + if (media > newest) { + newest = media; + history.push(i * 20 + jitter, media); + } + for (const query of [media - 30, media - 10, media, media + 10, media + 1e6]) { + expect(history.reference(query)).toEqual(scan(history, query)); + } + } + }); + + it("caps a timestamp rate the peer chooses, and the wrapped ring answers every prefix", () => { + const history = new History(); + for (let i = 0; i < 8192 + 5000; i++) { + const media = i / 1000; + history.push(i + (i % 7), media); + if (i % 997 === 0) { + for (const query of [media - 3, media - 0.5, media]) { + expect(history.reference(query)).toEqual(scan(history, query)); + } + } + } + expect(history.length).toBe(8192); }); }); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index be0cacfeee8..0759bf70b6d 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -29,12 +29,12 @@ const FRESH_MAX = 0.1; const IDLE_MAX = 1024; /** - * Arrivals kept in the reference window. + * Arrivals kept in the reference window, a power of two so it is also the ring's largest capacity. * - * The window is media time, and a publisher chooses the timestamp rate, so an uncapped - * history can hold millions of entries from a stream that advances a microsecond per - * frame. This bounds both memory and a reordered frame's scan, and stays far above the - * 2 s window of any real track (a 1000 fps track holds 2000). + * The window is media time, and a publisher chooses the timestamp rate, so an uncapped history + * holds every frame of a stream that advances a nanosecond per frame. Lookups are logarithmic + * either way; this bounds the memory, and stays far above the 2 s window of any real track (a + * 1000 fps track holds 2000). */ const MAX_HISTORY = 8192; @@ -51,15 +51,7 @@ const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1)); */ export class Jitter { /** Arrivals still inside the reference window, in media order. */ - #history: { seq: number; arrival: number; media: number }[] = []; - /** The first live entry in `#history`. */ - #historyStart = 0; - /** The prefix minimum of `arrival - media` over `#history`, front first. */ - #minima: { seq: number; key: number; arrival: number; media: number }[] = []; - /** The first live entry in `#minima`. */ - #minimaStart = 0; - /** The next entry's position in the arrival sequence. */ - #nextSeq = 0; + #history = new History(); /** The largest media timestamp observed, so a reordered frame is recognized. */ #newest = Number.NEGATIVE_INFINITY; @@ -76,31 +68,11 @@ export class Jitter { /** Fold one frame in: `arrival` on the receiver's monotonic clock, `media` its timestamp. */ observe(arrival: Milli, media: Milli): void { - const key = arrival - media; - // A reordered frame is measured but never joins the window: it arrived after a newer frame, // so it can never be the fastest, and a later frame must not be measured against it. if (media > this.#newest) { this.#newest = media; - - // Pruning by media time is what makes a timestamp jump self-correcting: everything before - // it leaves the window at once and the new frame becomes its own reference. - while ( - this.#historyStart < this.#history.length && - this.#history[this.#historyStart].media < media - WINDOW - ) { - this.#historyStart++; - } - while (this.#history.length - this.#historyStart >= MAX_HISTORY) this.#historyStart++; - - const seq = this.#nextSeq++; - this.#history.push({ seq, arrival, media }); - while (this.#minima.length > this.#minimaStart && this.#minima[this.#minima.length - 1].key >= key) { - this.#minima.pop(); - } - this.#minima.push({ seq, key, arrival, media }); - this.#trimMinima(); - this.#compact(); + this.#history.push(arrival, media); } // Delay against the fastest frame in the window at or before this one, not the previous one, @@ -108,7 +80,7 @@ export class Jitter { // held behind newer ones reads as the wait playing it in order costs. A frame older than the // whole window, such as the backlog a subscription delivers after its live edge, has no // reference and is not measured. - const reference = this.#reference(media); + const reference = this.#history.reference(media); if (reference === undefined) return; // Written in this order on purpose: the document requires bit-identical arithmetic, // so the difference is not reassociated. @@ -136,48 +108,6 @@ export class Jitter { this.#peak = delay; } - /** The fastest arrival at or before `media`, as its `(arrival, media)`. */ - #reference(media: number): { arrival: number; media: number } | undefined { - if (media >= this.#newest) { - // Every entry is at or before a frame in media order, and the deque's front is their minimum. - return this.#minimaStart < this.#minima.length ? this.#minima[this.#minimaStart] : undefined; - } - - // A reordered frame sees only a prefix, which the minimum deque cannot answer. - // The history cap bounds the scan. - let reference: { arrival: number; media: number } | undefined; - for (let i = this.#historyStart; i < this.#history.length; i++) { - const entry = this.#history[i]; - if (entry.media > media) break; - if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; - } - return reference; - } - - /** Drop minimums whose arrivals left the history. */ - #trimMinima(): void { - const front = this.#historyStart < this.#history.length ? this.#history[this.#historyStart].seq : undefined; - if (front === undefined) { - this.#minima.length = 0; - this.#minimaStart = 0; - return; - } - while (this.#minimaStart < this.#minima.length && this.#minima[this.#minimaStart].seq < front) - this.#minimaStart++; - } - - /** Reclaim the dropped prefix once it dominates, so memory does not grow with the stream. */ - #compact(): void { - if (this.#historyStart > 1024 && this.#historyStart * 2 >= this.#history.length) { - this.#history = this.#history.slice(this.#historyStart); - this.#historyStart = 0; - } - if (this.#minimaStart > 1024 && this.#minimaStart * 2 >= this.#minima.length) { - this.#minima = this.#minima.slice(this.#minimaStart); - this.#minimaStart = 0; - } - } - /** * A new estimator that keeps what this one learned but measures arrivals afresh, for a track that * replaces this one on the same path. Its frames are measured against their own fastest, never @@ -190,11 +120,6 @@ export class Jitter { return fork; } - /** @internal The history entries currently retained, for the cap's regression test. */ - get retained(): number { - return this.#history.length - this.#historyStart; - } - /** The measured term: the upper edge of the bucket holding the 95th percentile. */ get measured(): Milli { let cumulative = 0; @@ -223,3 +148,141 @@ export class Jitter { this.#prior = left; } } + +/** One arrival in the reference window. */ +export interface Entry { + arrival: number; + media: number; +} + +/** + * The arrivals inside the reference window, in media order, answering any prefix's fastest arrival + * in logarithmic time. + * + * A reordered frame is measured against the prefix at or before its timestamp, so a peer that sends + * only reordered frames would otherwise force a scan per frame. Entries live in a ring indexed from + * `#head`, and a segment tree over the ring's slots holds each node's fastest slot, the older on a + * tie, as the scan would pick. A query never spans the ring's wrap point within one node, so a node + * fully inside it is in arrival order. + * + * @internal Exported for the equivalence test. + */ +export class History { + /** The ring's arrivals, its length a power of two that grows up to `MAX_HISTORY`. */ + #arrival = new Float64Array(0); + /** The ring's media timestamps, slot for slot with `#arrival`. */ + #media = new Float64Array(0); + /** The slot of the oldest entry. */ + #head = 0; + /** The number of entries held. */ + #len = 0; + /** Leaf `capacity + slot` holds `slot`; each parent holds the faster of its children. */ + #tree = new Uint32Array(0); + + /** The entries held. */ + get length(): number { + return this.#len; + } + + /** The entry `index` places after the oldest. */ + at(index: number): Entry { + const slot = (this.#head + index) & (this.#arrival.length - 1); + return { arrival: this.#arrival[slot], media: this.#media[slot] }; + } + + /** Append an arrival newer than every entry, after dropping those it pushes out of the window. */ + push(arrival: number, media: number): void { + // Pruning by media time is what makes a timestamp jump self-correcting: everything before + // it leaves the window at once and the new frame becomes its own reference. + while (this.#len > 0 && this.#media[this.#head] < media - WINDOW) this.#pop(); + if (this.#len === MAX_HISTORY) this.#pop(); + if (this.#len === this.#arrival.length) this.#grow(); + + const capacity = this.#arrival.length; + const slot = (this.#head + this.#len) & (capacity - 1); + this.#arrival[slot] = arrival; + this.#media[slot] = media; + this.#len++; + + for (let node = (capacity + slot) >> 1; node > 0; node >>= 1) { + this.#tree[node] = this.#faster(this.#tree[2 * node], this.#tree[2 * node + 1]); + } + } + + /** The fastest entry with `entry.media <= media`, the oldest on a tie. */ + reference(media: number): Entry | undefined { + // Media timestamps increase along the ring, so the prefix ends at a binary search. + const mask = this.#arrival.length - 1; + let lo = 0; + let hi = this.#len; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (this.#media[(this.#head + mid) & mask] <= media) lo = mid + 1; + else hi = mid; + } + if (lo === 0) return undefined; + + const end = (this.#head + lo - 1) & mask; + const slot = + this.#head <= end + ? this.#range(this.#head, end) + : this.#faster(this.#range(this.#head, mask), this.#range(0, end)); + return { arrival: this.#arrival[slot], media: this.#media[slot] }; + } + + #pop(): void { + this.#head = (this.#head + 1) & (this.#arrival.length - 1); + this.#len--; + } + + /** Double the ring, unwrapping it to start at slot zero, and rebuild the tree. */ + #grow(): void { + const old = this.#arrival.length; + const capacity = Math.max(1, old * 2); + const arrival = new Float64Array(capacity); + const media = new Float64Array(capacity); + for (let i = 0; i < this.#len; i++) { + const slot = (this.#head + i) & (old - 1); + arrival[i] = this.#arrival[slot]; + media[i] = this.#media[slot]; + } + this.#arrival = arrival; + this.#media = media; + this.#head = 0; + + this.#tree = new Uint32Array(2 * capacity); + for (let slot = 0; slot < capacity; slot++) this.#tree[capacity + slot] = slot; + for (let node = capacity - 1; node > 0; node--) { + this.#tree[node] = this.#faster(this.#tree[2 * node], this.#tree[2 * node + 1]); + } + } + + /** The fastest slot in `first..=last`, which must not wrap. */ + #range(first: number, last: number): number { + const capacity = this.#arrival.length; + let lo = first + capacity; + let hi = last + capacity + 1; + let older = -1; + let newer = -1; + while (lo < hi) { + if (lo & 1) { + older = older < 0 ? this.#tree[lo] : this.#faster(older, this.#tree[lo]); + lo++; + } + if (hi & 1) { + hi--; + newer = newer < 0 ? this.#tree[hi] : this.#faster(this.#tree[hi], newer); + } + lo >>= 1; + hi >>= 1; + } + if (older < 0) return newer; + if (newer < 0) return older; + return this.#faster(older, newer); + } + + /** Of two slots, `older` first in arrival order, the faster one, keeping `older` on a tie. */ + #faster(older: number, newer: number): number { + return this.#arrival[newer] - this.#media[newer] < this.#arrival[older] - this.#media[older] ? newer : older; + } +} diff --git a/js/justfile b/js/justfile index 06b5ce872ff..6c15e6f05d9 100644 --- a/js/justfile +++ b/js/justfile @@ -133,6 +133,11 @@ slow-group-bench: bun install --frozen-lockfile bun hang/bench/slow-group.ts +# Sweep the audio jitter estimator's held history against in-order and reordered frames. +jitter-bench: + bun install --frozen-lockfile + bun hang/bench/jitter.ts + # Sweep decoder count against PCM callback size during audio warmup. audio-warmup-bench: bun install --frozen-lockfile diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs index 4fa318fe660..21330ae1986 100644 --- a/rs/moq-audio/src/jitter.rs +++ b/rs/moq-audio/src/jitter.rs @@ -9,7 +9,6 @@ //! Every value is `f64` milliseconds, evaluated in the order the page writes it, //! so both languages produce bit-identical targets from the same trace. -use std::collections::VecDeque; use std::time::Duration; /// Milliseconds of media time the reference search looks back over. @@ -36,13 +35,13 @@ const FRESH_MAX: f64 = 0.1; /// Idle intervals folded at once, purely to bound the work. const IDLE_MAX: f64 = 1024.0; -/// Arrivals kept in the reference window. +/// Arrivals kept in the reference window, a power of two so it is also the ring's +/// largest capacity. /// /// The window is media time, and a publisher chooses the timestamp rate, so an -/// uncapped history can hold millions of entries from a stream that advances a -/// microsecond per frame. This bounds both memory and a reordered frame's scan, -/// and stays far above the 2 s window of any real track (a 1000 fps track holds -/// 2000). +/// uncapped history holds every frame of a stream that advances a nanosecond per +/// frame. Lookups are logarithmic either way; this bounds the memory, and stays far +/// above the 2 s window of any real track (a 1000 fps track holds 2000). const MAX_HISTORY: usize = 8192; /// The playout target, fixed or estimated. @@ -97,19 +96,148 @@ fn prior(i: usize) -> f64 { } /// One arrival still inside the reference window. +#[derive(Clone, Copy, Default)] struct Entry { - /// Position in the arrival sequence, so the minimum deque can drop retired entries. - seq: u64, arrival: f64, media: f64, } -/// A candidate reference: the smallest delay seen at or before its arrival. -struct MinKey { - seq: u64, - key: f64, - arrival: f64, - media: f64, +impl Entry { + /// What the reference minimises: how late this frame arrived for its timestamp. + fn key(&self) -> f64 { + self.arrival - self.media + } +} + +/// The arrivals inside the reference window, in media order, answering any +/// prefix's fastest arrival in logarithmic time. +/// +/// A reordered frame is measured against the prefix at or before its timestamp, +/// so a peer that sends only reordered frames would otherwise force a scan per +/// frame. Entries live in a ring indexed from `head`, and a segment tree over the +/// ring's slots holds each node's fastest slot, the older on a tie, as the scan +/// would pick. A query never spans the ring's wrap point within one node, so a +/// node fully inside it is in arrival order. +#[derive(Default)] +struct History { + /// The ring, its length a power of two that grows up to `MAX_HISTORY`. + ring: Vec, + /// The slot of the oldest entry. + head: usize, + /// The number of entries held. + len: usize, + /// Leaf `ring.len() + slot` holds `slot`; each parent holds the faster of its children. + tree: Vec, +} + +impl History { + /// Append an arrival newer than every entry, after dropping those it pushes out of the window. + fn push(&mut self, arrival: f64, media: f64) { + // Pruning by media time rather than by arrival is what makes a timestamp jump + // self-correcting: everything before it leaves the window at once. + while self.len > 0 && self.ring[self.head].media < media - WINDOW { + self.pop(); + } + if self.len == MAX_HISTORY { + self.pop(); + } + if self.len == self.ring.len() { + self.grow(); + } + + let capacity = self.ring.len(); + let slot = (self.head + self.len) & (capacity - 1); + self.ring[slot] = Entry { arrival, media }; + self.len += 1; + + let mut node = (capacity + slot) / 2; + while node > 0 { + self.tree[node] = self.faster(self.tree[2 * node], self.tree[2 * node + 1]); + node /= 2; + } + } + + fn pop(&mut self) { + self.head = (self.head + 1) & (self.ring.len() - 1); + self.len -= 1; + } + + /// Double the ring, unwrapping it to start at slot zero, and rebuild the tree. + fn grow(&mut self) { + let old = self.ring.len(); + let capacity = (old * 2).max(1); + let mut ring = vec![Entry::default(); capacity]; + for (index, entry) in ring.iter_mut().enumerate().take(self.len) { + *entry = self.ring[(self.head + index) & (old - 1)]; + } + self.ring = ring; + self.head = 0; + + self.tree = vec![0; 2 * capacity]; + for slot in 0..capacity { + self.tree[capacity + slot] = slot; + } + for node in (1..capacity).rev() { + self.tree[node] = self.faster(self.tree[2 * node], self.tree[2 * node + 1]); + } + } + + /// The fastest entry with `entry.media <= media`, the oldest on a tie. + fn reference(&self, media: f64) -> Option { + // Media timestamps increase along the ring, so the prefix ends at a binary search. + let mask = self.ring.len().wrapping_sub(1); + let (mut lo, mut hi) = (0, self.len); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self.ring[(self.head + mid) & mask].media <= media { + lo = mid + 1; + } else { + hi = mid; + } + } + let last = lo.checked_sub(1)?; + + let end = (self.head + last) & mask; + let slot = if self.head <= end { + self.range(self.head, end) + } else { + let older = self.range(self.head, mask); + self.faster(older, self.range(0, end)) + }; + Some(self.ring[slot]) + } + + /// The fastest slot in `first..=last`, which must not wrap. + fn range(&self, first: usize, last: usize) -> usize { + let capacity = self.ring.len(); + let (mut lo, mut hi) = (first + capacity, last + capacity + 1); + let (mut older, mut newer): (Option, Option) = (None, None); + while lo < hi { + if lo & 1 == 1 { + older = Some(older.map_or(self.tree[lo], |slot| self.faster(slot, self.tree[lo]))); + lo += 1; + } + if hi & 1 == 1 { + hi -= 1; + newer = Some(newer.map_or(self.tree[hi], |slot| self.faster(self.tree[hi], slot))); + } + lo /= 2; + hi /= 2; + } + match (older, newer) { + (Some(older), Some(newer)) => self.faster(older, newer), + (older, newer) => older.or(newer).expect("a non-empty range"), + } + } + + /// Of two slots, `older` first in arrival order, the faster one, keeping `older` on a tie. + fn faster(&self, older: usize, newer: usize) -> usize { + if self.ring[newer].key() < self.ring[older].key() { + newer + } else { + older + } + } } /// Estimates the playout target from arrival timing alone. @@ -120,15 +248,7 @@ pub(crate) struct Estimator { advertised: f64, /// Arrivals still inside the reference window, in media order. - history: VecDeque, - /// The prefix minimum of `arrival - media` over `history`, front first. - /// - /// Keys increase from front to back, so the front is the smallest, and a retired - /// entry is dropped by its sequence number. This makes an in-order frame's reference - /// O(1) instead of a scan over a count the publisher chose. - minima: VecDeque, - /// The next entry's position in the arrival sequence. - next_seq: u64, + history: History, /// The largest media timestamp observed, so a reordered frame is recognized. newest: f64, @@ -149,9 +269,7 @@ impl Estimator { Self { frame: 0.0, advertised, - history: VecDeque::new(), - minima: VecDeque::new(), - next_seq: 0, + history: History::default(), newest: f64::NEG_INFINITY, origin: None, interval: 0.0, @@ -166,30 +284,7 @@ impl Estimator { // newer frame, so it is never the fastest, and nothing later is measured against it. if media > self.newest { self.newest = media; - - // Pruning by media time rather than by arrival is what makes a timestamp jump - // self-correcting: everything before it leaves the window at once. - while self.history.front().is_some_and(|entry| entry.media < media - WINDOW) { - self.history.pop_front(); - } - while self.history.len() >= MAX_HISTORY { - self.history.pop_front(); - } - - let seq = self.next_seq; - self.next_seq += 1; - let key = arrival - media; - self.history.push_back(Entry { seq, arrival, media }); - while self.minima.back().is_some_and(|min| min.key >= key) { - self.minima.pop_back(); - } - self.minima.push_back(MinKey { - seq, - key, - arrival, - media, - }); - self.trim_minima(); + self.history.push(arrival, media); } // The fastest frame in the window at or before this one, so delay is measured @@ -197,13 +292,13 @@ impl Estimator { // frame held behind newer ones reads as the wait playing it in order costs. A // frame older than the whole window, such as a subscription's backlog behind its // live edge, has nothing to be measured against. - let Some((reference, reference_media)) = self.reference(media) else { + let Some(reference) = self.history.reference(media) else { return; }; // Written in this order on purpose: the document requires bit-identical // arithmetic, so the difference is not reassociated. - let delay = f64::max(0.0, arrival - reference - (media - reference_media)); + let delay = f64::max(0.0, arrival - reference.arrival - (media - reference.media)); let Some(origin) = self.origin else { self.origin = Some(arrival); @@ -228,34 +323,6 @@ impl Estimator { self.peak = delay; } - /// The fastest arrival at or before `media`, as its `(arrival, media)`. - fn reference(&self, media: f64) -> Option<(f64, f64)> { - if media >= self.newest { - // Every entry is at or before a frame in media order, and the deque's front - // is their minimum. - return self.minima.front().map(|min| (min.arrival, min.media)); - } - - // A reordered frame sees only a prefix, which the minimum deque cannot answer. - // The history cap bounds the scan. - self.history - .iter() - .take_while(|entry| entry.media <= media) - .min_by(|a, b| (a.arrival - a.media).total_cmp(&(b.arrival - b.media))) - .map(|entry| (entry.arrival, entry.media)) - } - - /// Drop minimums whose arrivals left the history. - fn trim_minima(&mut self) { - let Some(front) = self.history.front().map(|entry| entry.seq) else { - self.minima.clear(); - return; - }; - while self.minima.front().is_some_and(|min| min.seq < front) { - self.minima.pop_front(); - } - } - fn target(&self) -> f64 { let mut cumulative = 0.0; let mut measured = BUCKETS as f64 * BUCKET; @@ -411,9 +478,25 @@ pub(crate) mod tests { assert_eq!(target.millis(), BUCKETS as f64 * BUCKET + 20.0); } - /// The minimum deque must answer exactly what the documented scan would. + /// The documented scan: the first entry with the smallest key at or before `media`. + fn scan(history: &History, media: f64) -> Option<(f64, f64)> { + let mut reference: Option = None; + for index in 0..history.len { + let entry = history.ring[(history.head + index) & (history.ring.len() - 1)]; + if entry.media > media { + break; + } + if reference.is_none_or(|best| entry.key() < best.key()) { + reference = Some(entry); + } + } + reference.map(|entry| (entry.arrival, entry.media)) + } + + /// The tree must pick exactly the entry the documented scan would, ties included, + /// since a tied entry with a different `(arrival, media)` rounds the delay differently. #[test] - fn reference_matches_a_naive_prefix_scan() { + fn reference_matches_the_scan() { let mut estimator = Estimator::new(0.0); let mut seed = 0x9e37_79b9_7f4a_7c15u64; for index in 0..400 { @@ -426,32 +509,40 @@ pub(crate) mod tests { } else { (index as f64) * 20.0 }; - let arrival = (index as f64) * 20.0 + (seed >> 40) as f64 / 1024.0; - estimator.observe(arrival, media); + // Whole milliseconds of jitter tie often; the occasional fraction does not. + let jitter = if index % 5 == 0 { + (seed >> 40) as f64 / 1024.0 + } else { + (seed >> 61) as f64 + }; + estimator.observe((index as f64) * 20.0 + jitter, media); - for query in [media - 10.0, media, media + 10.0, media + 1.0e6] { - let expected = estimator - .history - .iter() - .take_while(|entry| entry.media <= query) - .map(|entry| entry.arrival - entry.media) - .min_by(f64::total_cmp); - let actual = estimator.reference(query).map(|(arrival, media)| arrival - media); - assert_eq!(actual, expected, "frame {index}, query {query}"); + let history = &estimator.history; + for query in [media - 30.0, media - 10.0, media, media + 10.0, media + 1.0e6] { + let actual = history.reference(query).map(|entry| (entry.arrival, entry.media)); + assert_eq!(actual, scan(history, query), "frame {index}, query {query}"); } } } - /// A peer choosing a microsecond per frame cannot grow the history without bound. + /// A peer choosing a microsecond per frame cannot grow the history without bound, + /// and the full, wrapped ring still answers every prefix. #[test] fn the_history_is_capped() { let mut estimator = Estimator::new(0.0); - // One frame per microsecond of media, for well past the cap. for index in 0..(MAX_HISTORY + 5000) { - estimator.observe(index as f64, index as f64 / 1000.0); + let media = index as f64 / 1000.0; + estimator.observe(index as f64 + (index % 7) as f64, media); + if index % 997 == 0 { + let history = &estimator.history; + for query in [media - 3.0, media - 0.5, media] { + let actual = history.reference(query).map(|entry| (entry.arrival, entry.media)); + assert_eq!(actual, scan(history, query), "frame {index}, query {query}"); + } + } } - assert_eq!(estimator.history.len(), MAX_HISTORY); - assert!(estimator.minima.len() <= MAX_HISTORY); + assert_eq!(estimator.history.len, MAX_HISTORY); + assert_eq!(estimator.history.ring.len(), MAX_HISTORY); assert!(estimator.target().is_finite()); } }