diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index a857987e31d..66388cd34d6 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -91,9 +91,11 @@ observe(arrival, media): if media > newest: # a reordered frame never joins the history newest = media drop history entries with entry.media < media - WINDOW + if MAX_HISTORY entries are held: drop the oldest push (arrival, media) onto history - ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media) + ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media), + the oldest on a tie if there is none: return # older than the whole window, not measured delay = max(0, (arrival - ref.arrival) - (media - ref.media)) @@ -139,6 +141,22 @@ special case, and neither does a group the relay dropped. fills over ten seconds is measured as the part of it that happened in the last two. +### Bounding the history + +The history holds at most `MAX_HISTORY` (8192) arrivals. The window is media +time and the publisher chooses the timestamp rate, so without a cap a stream +that advances a nanosecond per frame keeps every frame it sends. The cap stays +far above the two-second window of any real track, including a 1000 fps one +(2000 entries); a stream dense enough to reach it measures against a window +shorter than `WINDOW` instead of growing without bound. + +The reference is a minimum over a prefix of the history, and the publisher +chooses the prefix by reordering, so scanning for it lets every frame cost +`MAX_HISTORY` comparisons. Keep the history in a structure that answers any +prefix's minimum in logarithmic time, such as a segment tree over a ring. +Keep the oldest entry on a tie: tied entries with a different `arrival` and +`media` round `delay` differently. + ### Reordered arrivals are measured against what came before them A frame whose media timestamp is not newer than the newest already observed is @@ -473,6 +491,7 @@ The exceptions are marked. | `FRESH_MAX` | 0.1 | Ours, twice the quantile's tail. NetEq ramps the forget factor from a start weight of 2 instead; see [the histogram](#the-histogram) | | `PRIOR[i]` | `0.5^(i+1)` | NetEq's histogram seed | | `IDLE_MAX` | 1024 intervals | Ours. A work bound with no effect on output | +| `MAX_HISTORY` | 8192 arrivals | Ours. Bounds a rate the publisher chooses; see [Bounding the history](#bounding-the-history) | | Out-of-range delays | clamped, not dropped | Ours. See [the histogram](#the-histogram) | ## Implementing from this page diff --git a/js/hang/bench/jitter.ts b/js/hang/bench/jitter.ts new file mode 100644 index 00000000000..35da4373424 --- /dev/null +++ b/js/hang/bench/jitter.ts @@ -0,0 +1,31 @@ +// Sweep the jitter estimator's held history against in-order and reordered frames, the per-frame +// cost a publisher controls through its timestamp rate. + +import type { Milli } from "@moq/clock"; +import { Jitter } from "../src/container/jitter.ts"; + +const FRAMES = 100_000; + +console.log("held,order,ns_per_frame"); +for (const held of [64, 512, 4096, 8192]) { + for (const order of ["in", "reordered"] as const) { + const jitter = new Jitter(); + // Spaced so `held` frames fit inside the two-second window. + const step = 2000 / held; + let media = 0; + for (let i = 0; i < held; i++, media += step) jitter.observe(media as Milli, media as Milli); + + const started = performance.now(); + for (let i = 0; i < FRAMES; i++) { + if (order === "in") { + jitter.observe(media as Milli, media as Milli); + media += step; + } else { + // Just behind the newest, so the reference covers almost the whole history. + jitter.observe(media as Milli, (media - 2 * step) as Milli); + } + } + const ns = ((performance.now() - started) * 1e6) / FRAMES; + console.log(`${held},${order},${ns.toFixed(1)}`); + } +} diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index 62a8b49b144..4e2c8c32bcc 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -3,7 +3,7 @@ import { readdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import type { Milli } from "@moq/clock"; -import { Jitter } from "./jitter"; +import { type Entry, History, Jitter } from "./jitter"; // The conformance corpus beside doc/concept/audio-jitter.md, read directly so this is graded // against the document rather than against another implementation. @@ -115,3 +115,52 @@ describe("Jitter", () => { expect(warm.measured).toBe(learned); }); }); + +// The documented scan: the first entry with the smallest key at or before `media`. +function scan(history: History, media: number): Entry | undefined { + let reference: Entry | undefined; + for (let i = 0; i < history.length; i++) { + const entry = history.at(i); + if (entry.media > media) break; + if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; + } + return reference; +} + +describe("History", () => { + // A tied entry with a different (arrival, media) rounds the delay differently, so the tree must + // pick exactly the entry the scan would. + it("picks the entry the scan would, ties included", () => { + const history = new History(); + let seed = 1; + let newest = Number.NEGATIVE_INFINITY; + for (let i = 0; i < 400; i++) { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + // Mostly in order, with an occasional reordered frame and a timestamp jump. + const media = i % 37 === 0 ? i * 20 - 900 : i % 89 === 0 ? i * 20 + 3000 : i * 20; + // Whole milliseconds of jitter tie often; the occasional fraction does not. + const jitter = i % 5 === 0 ? seed / 2 ** 21 : seed % 8; + if (media > newest) { + newest = media; + history.push(i * 20 + jitter, media); + } + for (const query of [media - 30, media - 10, media, media + 10, media + 1e6]) { + expect(history.reference(query)).toEqual(scan(history, query)); + } + } + }); + + it("caps a timestamp rate the peer chooses, and the wrapped ring answers every prefix", () => { + const history = new History(); + for (let i = 0; i < 8192 + 5000; i++) { + const media = i / 1000; + history.push(i + (i % 7), media); + if (i % 997 === 0) { + for (const query of [media - 3, media - 0.5, media]) { + expect(history.reference(query)).toEqual(scan(history, query)); + } + } + } + expect(history.length).toBe(8192); + }); +}); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index 012131fb353..0759bf70b6d 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -28,6 +28,16 @@ const FRESH_MAX = 0.1; /** Idle intervals folded at once, purely to bound the work. */ const IDLE_MAX = 1024; +/** + * Arrivals kept in the reference window, a power of two so it is also the ring's largest capacity. + * + * The window is media time, and a publisher chooses the timestamp rate, so an uncapped history + * holds every frame of a stream that advances a nanosecond per frame. Lookups are logarithmic + * either way; this bounds the memory, and stays far above the 2 s window of any real track (a + * 1000 fps track holds 2000). + */ +const MAX_HISTORY = 8192; + /** The cold-start distribution, which is also what an idle interval relaxes back to. */ const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1)); @@ -41,7 +51,7 @@ const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1)); */ export class Jitter { /** Arrivals still inside the reference window, in media order. */ - #history: { arrival: number; media: number }[] = []; + #history = new History(); /** The largest media timestamp observed, so a reordered frame is recognized. */ #newest = Number.NEGATIVE_INFINITY; @@ -62,13 +72,7 @@ export class Jitter { // so it can never be the fastest, and a later frame must not be measured against it. if (media > this.#newest) { this.#newest = media; - - // Pruning by media time is what makes a timestamp jump self-correcting: everything before - // it leaves the window at once and the new frame becomes its own reference. - let stale = 0; - while (stale < this.#history.length && this.#history[stale].media < media - WINDOW) stale++; - if (stale > 0) this.#history.splice(0, stale); - this.#history.push({ arrival, media }); + this.#history.push(arrival, media); } // Delay against the fastest frame in the window at or before this one, not the previous one, @@ -76,12 +80,10 @@ export class Jitter { // held behind newer ones reads as the wait playing it in order costs. A frame older than the // whole window, such as the backlog a subscription delivers after its live edge, has no // reference and is not measured. - let reference: { arrival: number; media: number } | undefined; - for (const entry of this.#history) { - if (entry.media > media) break; - if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; - } - if (!reference) return; + const reference = this.#history.reference(media); + if (reference === undefined) return; + // Written in this order on purpose: the document requires bit-identical arithmetic, + // so the difference is not reassociated. const delay = Math.max(0, arrival - reference.arrival - (media - reference.media)); if (this.#origin === undefined) { @@ -146,3 +148,141 @@ export class Jitter { this.#prior = left; } } + +/** One arrival in the reference window. */ +export interface Entry { + arrival: number; + media: number; +} + +/** + * The arrivals inside the reference window, in media order, answering any prefix's fastest arrival + * in logarithmic time. + * + * A reordered frame is measured against the prefix at or before its timestamp, so a peer that sends + * only reordered frames would otherwise force a scan per frame. Entries live in a ring indexed from + * `#head`, and a segment tree over the ring's slots holds each node's fastest slot, the older on a + * tie, as the scan would pick. A query never spans the ring's wrap point within one node, so a node + * fully inside it is in arrival order. + * + * @internal Exported for the equivalence test. + */ +export class History { + /** The ring's arrivals, its length a power of two that grows up to `MAX_HISTORY`. */ + #arrival = new Float64Array(0); + /** The ring's media timestamps, slot for slot with `#arrival`. */ + #media = new Float64Array(0); + /** The slot of the oldest entry. */ + #head = 0; + /** The number of entries held. */ + #len = 0; + /** Leaf `capacity + slot` holds `slot`; each parent holds the faster of its children. */ + #tree = new Uint32Array(0); + + /** The entries held. */ + get length(): number { + return this.#len; + } + + /** The entry `index` places after the oldest. */ + at(index: number): Entry { + const slot = (this.#head + index) & (this.#arrival.length - 1); + return { arrival: this.#arrival[slot], media: this.#media[slot] }; + } + + /** Append an arrival newer than every entry, after dropping those it pushes out of the window. */ + push(arrival: number, media: number): void { + // Pruning by media time is what makes a timestamp jump self-correcting: everything before + // it leaves the window at once and the new frame becomes its own reference. + while (this.#len > 0 && this.#media[this.#head] < media - WINDOW) this.#pop(); + if (this.#len === MAX_HISTORY) this.#pop(); + if (this.#len === this.#arrival.length) this.#grow(); + + const capacity = this.#arrival.length; + const slot = (this.#head + this.#len) & (capacity - 1); + this.#arrival[slot] = arrival; + this.#media[slot] = media; + this.#len++; + + for (let node = (capacity + slot) >> 1; node > 0; node >>= 1) { + this.#tree[node] = this.#faster(this.#tree[2 * node], this.#tree[2 * node + 1]); + } + } + + /** The fastest entry with `entry.media <= media`, the oldest on a tie. */ + reference(media: number): Entry | undefined { + // Media timestamps increase along the ring, so the prefix ends at a binary search. + const mask = this.#arrival.length - 1; + let lo = 0; + let hi = this.#len; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (this.#media[(this.#head + mid) & mask] <= media) lo = mid + 1; + else hi = mid; + } + if (lo === 0) return undefined; + + const end = (this.#head + lo - 1) & mask; + const slot = + this.#head <= end + ? this.#range(this.#head, end) + : this.#faster(this.#range(this.#head, mask), this.#range(0, end)); + return { arrival: this.#arrival[slot], media: this.#media[slot] }; + } + + #pop(): void { + this.#head = (this.#head + 1) & (this.#arrival.length - 1); + this.#len--; + } + + /** Double the ring, unwrapping it to start at slot zero, and rebuild the tree. */ + #grow(): void { + const old = this.#arrival.length; + const capacity = Math.max(1, old * 2); + const arrival = new Float64Array(capacity); + const media = new Float64Array(capacity); + for (let i = 0; i < this.#len; i++) { + const slot = (this.#head + i) & (old - 1); + arrival[i] = this.#arrival[slot]; + media[i] = this.#media[slot]; + } + this.#arrival = arrival; + this.#media = media; + this.#head = 0; + + this.#tree = new Uint32Array(2 * capacity); + for (let slot = 0; slot < capacity; slot++) this.#tree[capacity + slot] = slot; + for (let node = capacity - 1; node > 0; node--) { + this.#tree[node] = this.#faster(this.#tree[2 * node], this.#tree[2 * node + 1]); + } + } + + /** The fastest slot in `first..=last`, which must not wrap. */ + #range(first: number, last: number): number { + const capacity = this.#arrival.length; + let lo = first + capacity; + let hi = last + capacity + 1; + let older = -1; + let newer = -1; + while (lo < hi) { + if (lo & 1) { + older = older < 0 ? this.#tree[lo] : this.#faster(older, this.#tree[lo]); + lo++; + } + if (hi & 1) { + hi--; + newer = newer < 0 ? this.#tree[hi] : this.#faster(this.#tree[hi], newer); + } + lo >>= 1; + hi >>= 1; + } + if (older < 0) return newer; + if (newer < 0) return older; + return this.#faster(older, newer); + } + + /** Of two slots, `older` first in arrival order, the faster one, keeping `older` on a tie. */ + #faster(older: number, newer: number): number { + return this.#arrival[newer] - this.#media[newer] < this.#arrival[older] - this.#media[older] ? newer : older; + } +} diff --git a/js/justfile b/js/justfile index 06b5ce872ff..6c15e6f05d9 100644 --- a/js/justfile +++ b/js/justfile @@ -133,6 +133,11 @@ slow-group-bench: bun install --frozen-lockfile bun hang/bench/slow-group.ts +# Sweep the audio jitter estimator's held history against in-order and reordered frames. +jitter-bench: + bun install --frozen-lockfile + bun hang/bench/jitter.ts + # Sweep decoder count against PCM callback size during audio warmup. audio-warmup-bench: bun install --frozen-lockfile diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs index 4aa764aade8..21330ae1986 100644 --- a/rs/moq-audio/src/jitter.rs +++ b/rs/moq-audio/src/jitter.rs @@ -9,7 +9,6 @@ //! Every value is `f64` milliseconds, evaluated in the order the page writes it, //! so both languages produce bit-identical targets from the same trace. -use std::collections::VecDeque; use std::time::Duration; /// Milliseconds of media time the reference search looks back over. @@ -36,6 +35,15 @@ const FRESH_MAX: f64 = 0.1; /// Idle intervals folded at once, purely to bound the work. const IDLE_MAX: f64 = 1024.0; +/// Arrivals kept in the reference window, a power of two so it is also the ring's +/// largest capacity. +/// +/// The window is media time, and a publisher chooses the timestamp rate, so an +/// uncapped history holds every frame of a stream that advances a nanosecond per +/// frame. Lookups are logarithmic either way; this bounds the memory, and stays far +/// above the 2 s window of any real track (a 1000 fps track holds 2000). +const MAX_HISTORY: usize = 8192; + /// The playout target, fixed or estimated. pub(crate) enum Target { /// An explicit delay, taken literally: no floor, no frame term, no estimate. @@ -88,11 +96,150 @@ fn prior(i: usize) -> f64 { } /// One arrival still inside the reference window. +#[derive(Clone, Copy, Default)] struct Entry { arrival: f64, media: f64, } +impl Entry { + /// What the reference minimises: how late this frame arrived for its timestamp. + fn key(&self) -> f64 { + self.arrival - self.media + } +} + +/// The arrivals inside the reference window, in media order, answering any +/// prefix's fastest arrival in logarithmic time. +/// +/// A reordered frame is measured against the prefix at or before its timestamp, +/// so a peer that sends only reordered frames would otherwise force a scan per +/// frame. Entries live in a ring indexed from `head`, and a segment tree over the +/// ring's slots holds each node's fastest slot, the older on a tie, as the scan +/// would pick. A query never spans the ring's wrap point within one node, so a +/// node fully inside it is in arrival order. +#[derive(Default)] +struct History { + /// The ring, its length a power of two that grows up to `MAX_HISTORY`. + ring: Vec, + /// The slot of the oldest entry. + head: usize, + /// The number of entries held. + len: usize, + /// Leaf `ring.len() + slot` holds `slot`; each parent holds the faster of its children. + tree: Vec, +} + +impl History { + /// Append an arrival newer than every entry, after dropping those it pushes out of the window. + fn push(&mut self, arrival: f64, media: f64) { + // Pruning by media time rather than by arrival is what makes a timestamp jump + // self-correcting: everything before it leaves the window at once. + while self.len > 0 && self.ring[self.head].media < media - WINDOW { + self.pop(); + } + if self.len == MAX_HISTORY { + self.pop(); + } + if self.len == self.ring.len() { + self.grow(); + } + + let capacity = self.ring.len(); + let slot = (self.head + self.len) & (capacity - 1); + self.ring[slot] = Entry { arrival, media }; + self.len += 1; + + let mut node = (capacity + slot) / 2; + while node > 0 { + self.tree[node] = self.faster(self.tree[2 * node], self.tree[2 * node + 1]); + node /= 2; + } + } + + fn pop(&mut self) { + self.head = (self.head + 1) & (self.ring.len() - 1); + self.len -= 1; + } + + /// Double the ring, unwrapping it to start at slot zero, and rebuild the tree. + fn grow(&mut self) { + let old = self.ring.len(); + let capacity = (old * 2).max(1); + let mut ring = vec![Entry::default(); capacity]; + for (index, entry) in ring.iter_mut().enumerate().take(self.len) { + *entry = self.ring[(self.head + index) & (old - 1)]; + } + self.ring = ring; + self.head = 0; + + self.tree = vec![0; 2 * capacity]; + for slot in 0..capacity { + self.tree[capacity + slot] = slot; + } + for node in (1..capacity).rev() { + self.tree[node] = self.faster(self.tree[2 * node], self.tree[2 * node + 1]); + } + } + + /// The fastest entry with `entry.media <= media`, the oldest on a tie. + fn reference(&self, media: f64) -> Option { + // Media timestamps increase along the ring, so the prefix ends at a binary search. + let mask = self.ring.len().wrapping_sub(1); + let (mut lo, mut hi) = (0, self.len); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self.ring[(self.head + mid) & mask].media <= media { + lo = mid + 1; + } else { + hi = mid; + } + } + let last = lo.checked_sub(1)?; + + let end = (self.head + last) & mask; + let slot = if self.head <= end { + self.range(self.head, end) + } else { + let older = self.range(self.head, mask); + self.faster(older, self.range(0, end)) + }; + Some(self.ring[slot]) + } + + /// The fastest slot in `first..=last`, which must not wrap. + fn range(&self, first: usize, last: usize) -> usize { + let capacity = self.ring.len(); + let (mut lo, mut hi) = (first + capacity, last + capacity + 1); + let (mut older, mut newer): (Option, Option) = (None, None); + while lo < hi { + if lo & 1 == 1 { + older = Some(older.map_or(self.tree[lo], |slot| self.faster(slot, self.tree[lo]))); + lo += 1; + } + if hi & 1 == 1 { + hi -= 1; + newer = Some(newer.map_or(self.tree[hi], |slot| self.faster(self.tree[hi], slot))); + } + lo /= 2; + hi /= 2; + } + match (older, newer) { + (Some(older), Some(newer)) => self.faster(older, newer), + (older, newer) => older.or(newer).expect("a non-empty range"), + } + } + + /// Of two slots, `older` first in arrival order, the faster one, keeping `older` on a tie. + fn faster(&self, older: usize, newer: usize) -> usize { + if self.ring[newer].key() < self.ring[older].key() { + newer + } else { + older + } + } +} + /// Estimates the playout target from arrival timing alone. pub(crate) struct Estimator { /// The codec's frame duration, never learned from observed timestamps. @@ -101,7 +248,7 @@ pub(crate) struct Estimator { advertised: f64, /// Arrivals still inside the reference window, in media order. - history: VecDeque, + history: History, /// The largest media timestamp observed, so a reordered frame is recognized. newest: f64, @@ -122,7 +269,7 @@ impl Estimator { Self { frame: 0.0, advertised, - history: VecDeque::new(), + history: History::default(), newest: f64::NEG_INFINITY, origin: None, interval: 0.0, @@ -137,13 +284,7 @@ impl Estimator { // newer frame, so it is never the fastest, and nothing later is measured against it. if media > self.newest { self.newest = media; - - // Pruning by media time rather than by arrival is what makes a timestamp jump - // self-correcting: everything before it leaves the window at once. - while self.history.front().is_some_and(|entry| entry.media < media - WINDOW) { - self.history.pop_front(); - } - self.history.push_back(Entry { arrival, media }); + self.history.push(arrival, media); } // The fastest frame in the window at or before this one, so delay is measured @@ -151,16 +292,12 @@ impl Estimator { // frame held behind newer ones reads as the wait playing it in order costs. A // frame older than the whole window, such as a subscription's backlog behind its // live edge, has nothing to be measured against. - let mut reference: Option<&Entry> = None; - for entry in self.history.iter().take_while(|entry| entry.media <= media) { - if reference.is_none_or(|best| entry.arrival - entry.media < best.arrival - best.media) { - reference = Some(entry); - } - } - let Some(reference) = reference else { + let Some(reference) = self.history.reference(media) else { return; }; + // Written in this order on purpose: the document requires bit-identical + // arithmetic, so the difference is not reassociated. let delay = f64::max(0.0, arrival - reference.arrival - (media - reference.media)); let Some(origin) = self.origin else { @@ -340,4 +477,72 @@ pub(crate) mod tests { } assert_eq!(target.millis(), BUCKETS as f64 * BUCKET + 20.0); } + + /// The documented scan: the first entry with the smallest key at or before `media`. + fn scan(history: &History, media: f64) -> Option<(f64, f64)> { + let mut reference: Option = None; + for index in 0..history.len { + let entry = history.ring[(history.head + index) & (history.ring.len() - 1)]; + if entry.media > media { + break; + } + if reference.is_none_or(|best| entry.key() < best.key()) { + reference = Some(entry); + } + } + reference.map(|entry| (entry.arrival, entry.media)) + } + + /// The tree must pick exactly the entry the documented scan would, ties included, + /// since a tied entry with a different `(arrival, media)` rounds the delay differently. + #[test] + fn reference_matches_the_scan() { + let mut estimator = Estimator::new(0.0); + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + for index in 0..400 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + // Mostly in order, with an occasional reordered frame and a timestamp jump. + let media = if index % 37 == 0 { + (index as f64) * 20.0 - 900.0 + } else if index % 89 == 0 { + (index as f64) * 20.0 + 3000.0 + } else { + (index as f64) * 20.0 + }; + // Whole milliseconds of jitter tie often; the occasional fraction does not. + let jitter = if index % 5 == 0 { + (seed >> 40) as f64 / 1024.0 + } else { + (seed >> 61) as f64 + }; + estimator.observe((index as f64) * 20.0 + jitter, media); + + let history = &estimator.history; + for query in [media - 30.0, media - 10.0, media, media + 10.0, media + 1.0e6] { + let actual = history.reference(query).map(|entry| (entry.arrival, entry.media)); + assert_eq!(actual, scan(history, query), "frame {index}, query {query}"); + } + } + } + + /// A peer choosing a microsecond per frame cannot grow the history without bound, + /// and the full, wrapped ring still answers every prefix. + #[test] + fn the_history_is_capped() { + let mut estimator = Estimator::new(0.0); + for index in 0..(MAX_HISTORY + 5000) { + let media = index as f64 / 1000.0; + estimator.observe(index as f64 + (index % 7) as f64, media); + if index % 997 == 0 { + let history = &estimator.history; + for query in [media - 3.0, media - 0.5, media] { + let actual = history.reference(query).map(|entry| (entry.arrival, entry.media)); + assert_eq!(actual, scan(history, query), "frame {index}, query {query}"); + } + } + } + assert_eq!(estimator.history.len, MAX_HISTORY); + assert_eq!(estimator.history.ring.len(), MAX_HISTORY); + assert!(estimator.target().is_finite()); + } }