Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion doc/concept/audio-jitter.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,11 @@ observe(arrival, media):
if media > newest: # a reordered frame never joins the history
newest = media
drop history entries with entry.media < media - WINDOW
if MAX_HISTORY entries are held: drop the oldest
push (arrival, media) onto history
ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media)
ref = entry in history with entry.media <= media, minimising (entry.arrival - entry.media),
the oldest on a tie
if there is none: return # older than the whole window, not measured
delay = max(0, (arrival - ref.arrival) - (media - ref.media))
Expand Down Expand Up @@ -139,6 +141,22 @@ special case, and neither does a group the relay dropped.
fills over ten seconds is measured as the part of it that happened in the last
two.

### Bounding the history

The history holds at most `MAX_HISTORY` (8192) arrivals. The window is media
time and the publisher chooses the timestamp rate, so without a cap a stream
that advances a nanosecond per frame keeps every frame it sends. The cap stays
far above the two-second window of any real track, including a 1000 fps one
(2000 entries); a stream dense enough to reach it measures against a window
shorter than `WINDOW` instead of growing without bound.

The reference is a minimum over a prefix of the history, and the publisher
chooses the prefix by reordering, so scanning for it lets every frame cost
`MAX_HISTORY` comparisons. Keep the history in a structure that answers any
prefix's minimum in logarithmic time, such as a segment tree over a ring.
Keep the oldest entry on a tie: tied entries with a different `arrival` and
`media` round `delay` differently.

### Reordered arrivals are measured against what came before them

A frame whose media timestamp is not newer than the newest already observed is
Expand Down Expand Up @@ -473,6 +491,7 @@ The exceptions are marked.
| `FRESH_MAX` | 0.1 | Ours, twice the quantile's tail. NetEq ramps the forget factor from a start weight of 2 instead; see [the histogram](#the-histogram) |
| `PRIOR[i]` | `0.5^(i+1)` | NetEq's histogram seed |
| `IDLE_MAX` | 1024 intervals | Ours. A work bound with no effect on output |
| `MAX_HISTORY` | 8192 arrivals | Ours. Bounds a rate the publisher chooses; see [Bounding the history](#bounding-the-history) |
| Out-of-range delays | clamped, not dropped | Ours. See [the histogram](#the-histogram) |

## Implementing from this page
Expand Down
31 changes: 31 additions & 0 deletions js/hang/bench/jitter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Sweep the jitter estimator's held history against in-order and reordered frames, the per-frame
// cost a publisher controls through its timestamp rate.

import type { Milli } from "@moq/clock";
import { Jitter } from "../src/container/jitter.ts";

const FRAMES = 100_000;

console.log("held,order,ns_per_frame");
for (const held of [64, 512, 4096, 8192]) {
for (const order of ["in", "reordered"] as const) {
const jitter = new Jitter();
// Spaced so `held` frames fit inside the two-second window.
const step = 2000 / held;
let media = 0;
for (let i = 0; i < held; i++, media += step) jitter.observe(media as Milli, media as Milli);

const started = performance.now();
for (let i = 0; i < FRAMES; i++) {
if (order === "in") {
jitter.observe(media as Milli, media as Milli);
media += step;
} else {
// Just behind the newest, so the reference covers almost the whole history.
jitter.observe(media as Milli, (media - 2 * step) as Milli);
}
}
const ns = ((performance.now() - started) * 1e6) / FRAMES;
console.log(`${held},${order},${ns.toFixed(1)}`);
}
}
51 changes: 50 additions & 1 deletion js/hang/src/container/jitter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { readdirSync, readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import type { Milli } from "@moq/clock";
import { Jitter } from "./jitter";
import { type Entry, History, Jitter } from "./jitter";

// The conformance corpus beside doc/concept/audio-jitter.md, read directly so this is graded
// against the document rather than against another implementation.
Expand Down Expand Up @@ -115,3 +115,52 @@ describe("Jitter", () => {
expect(warm.measured).toBe(learned);
});
});

// The documented scan: the first entry with the smallest key at or before `media`.
function scan(history: History, media: number): Entry | undefined {
let reference: Entry | undefined;
for (let i = 0; i < history.length; i++) {
const entry = history.at(i);
if (entry.media > media) break;
if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry;
}
return reference;
}

describe("History", () => {
// A tied entry with a different (arrival, media) rounds the delay differently, so the tree must
// pick exactly the entry the scan would.
it("picks the entry the scan would, ties included", () => {
const history = new History();
let seed = 1;
let newest = Number.NEGATIVE_INFINITY;
for (let i = 0; i < 400; i++) {
seed = (seed * 1103515245 + 12345) % 2 ** 31;
// Mostly in order, with an occasional reordered frame and a timestamp jump.
const media = i % 37 === 0 ? i * 20 - 900 : i % 89 === 0 ? i * 20 + 3000 : i * 20;
// Whole milliseconds of jitter tie often; the occasional fraction does not.
const jitter = i % 5 === 0 ? seed / 2 ** 21 : seed % 8;
if (media > newest) {
newest = media;
history.push(i * 20 + jitter, media);
}
for (const query of [media - 30, media - 10, media, media + 10, media + 1e6]) {
expect(history.reference(query)).toEqual(scan(history, query));
}
}
});

it("caps a timestamp rate the peer chooses, and the wrapped ring answers every prefix", () => {
const history = new History();
for (let i = 0; i < 8192 + 5000; i++) {
const media = i / 1000;
history.push(i + (i % 7), media);
if (i % 997 === 0) {
for (const query of [media - 3, media - 0.5, media]) {
expect(history.reference(query)).toEqual(scan(history, query));
}
}
}
expect(history.length).toBe(8192);
});
});
168 changes: 154 additions & 14 deletions js/hang/src/container/jitter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ const FRESH_MAX = 0.1;
/** Idle intervals folded at once, purely to bound the work. */
const IDLE_MAX = 1024;

/**
* Arrivals kept in the reference window, a power of two so it is also the ring's largest capacity.
*
* The window is media time, and a publisher chooses the timestamp rate, so an uncapped history
* holds every frame of a stream that advances a nanosecond per frame. Lookups are logarithmic
* either way; this bounds the memory, and stays far above the 2 s window of any real track (a
* 1000 fps track holds 2000).
*/
const MAX_HISTORY = 8192;

/** The cold-start distribution, which is also what an idle interval relaxes back to. */
const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1));

Expand All @@ -41,7 +51,7 @@ const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1));
*/
export class Jitter {
/** Arrivals still inside the reference window, in media order. */
#history: { arrival: number; media: number }[] = [];
#history = new History();
/** The largest media timestamp observed, so a reordered frame is recognized. */
#newest = Number.NEGATIVE_INFINITY;

Expand All @@ -62,26 +72,18 @@ export class Jitter {
// so it can never be the fastest, and a later frame must not be measured against it.
if (media > this.#newest) {
this.#newest = media;

// Pruning by media time is what makes a timestamp jump self-correcting: everything before
// it leaves the window at once and the new frame becomes its own reference.
let stale = 0;
while (stale < this.#history.length && this.#history[stale].media < media - WINDOW) stale++;
if (stale > 0) this.#history.splice(0, stale);
this.#history.push({ arrival, media });
this.#history.push(arrival, media);
}

// Delay against the fastest frame in the window at or before this one, not the previous one,
// so a queue that fills a little per frame reads as the build-up it is, and an older frame
// held behind newer ones reads as the wait playing it in order costs. A frame older than the
// whole window, such as the backlog a subscription delivers after its live edge, has no
// reference and is not measured.
let reference: { arrival: number; media: number } | undefined;
for (const entry of this.#history) {
if (entry.media > media) break;
if (!reference || entry.arrival - entry.media < reference.arrival - reference.media) reference = entry;
}
if (!reference) return;
const reference = this.#history.reference(media);
if (reference === undefined) return;
// Written in this order on purpose: the document requires bit-identical arithmetic,
// so the difference is not reassociated.
const delay = Math.max(0, arrival - reference.arrival - (media - reference.media));

if (this.#origin === undefined) {
Expand Down Expand Up @@ -146,3 +148,141 @@ export class Jitter {
this.#prior = left;
}
}

/** One arrival in the reference window. */
export interface Entry {
arrival: number;
media: number;
}

/**
* The arrivals inside the reference window, in media order, answering any prefix's fastest arrival
* in logarithmic time.
*
* A reordered frame is measured against the prefix at or before its timestamp, so a peer that sends
* only reordered frames would otherwise force a scan per frame. Entries live in a ring indexed from
* `#head`, and a segment tree over the ring's slots holds each node's fastest slot, the older on a
* tie, as the scan would pick. A query never spans the ring's wrap point within one node, so a node
* fully inside it is in arrival order.
*
* @internal Exported for the equivalence test.
*/
export class History {
/** The ring's arrivals, its length a power of two that grows up to `MAX_HISTORY`. */
#arrival = new Float64Array(0);
/** The ring's media timestamps, slot for slot with `#arrival`. */
#media = new Float64Array(0);
/** The slot of the oldest entry. */
#head = 0;
/** The number of entries held. */
#len = 0;
/** Leaf `capacity + slot` holds `slot`; each parent holds the faster of its children. */
#tree = new Uint32Array(0);

/** The entries held. */
get length(): number {
return this.#len;
}

/** The entry `index` places after the oldest. */
at(index: number): Entry {
const slot = (this.#head + index) & (this.#arrival.length - 1);
return { arrival: this.#arrival[slot], media: this.#media[slot] };
}

/** Append an arrival newer than every entry, after dropping those it pushes out of the window. */
push(arrival: number, media: number): void {
// Pruning by media time is what makes a timestamp jump self-correcting: everything before
// it leaves the window at once and the new frame becomes its own reference.
while (this.#len > 0 && this.#media[this.#head] < media - WINDOW) this.#pop();
if (this.#len === MAX_HISTORY) this.#pop();
if (this.#len === this.#arrival.length) this.#grow();

const capacity = this.#arrival.length;
const slot = (this.#head + this.#len) & (capacity - 1);
this.#arrival[slot] = arrival;
this.#media[slot] = media;
this.#len++;

for (let node = (capacity + slot) >> 1; node > 0; node >>= 1) {
this.#tree[node] = this.#faster(this.#tree[2 * node], this.#tree[2 * node + 1]);
}
}

/** The fastest entry with `entry.media <= media`, the oldest on a tie. */
reference(media: number): Entry | undefined {
// Media timestamps increase along the ring, so the prefix ends at a binary search.
const mask = this.#arrival.length - 1;
let lo = 0;
let hi = this.#len;
while (lo < hi) {
const mid = (lo + hi) >> 1;
if (this.#media[(this.#head + mid) & mask] <= media) lo = mid + 1;
else hi = mid;
}
if (lo === 0) return undefined;

const end = (this.#head + lo - 1) & mask;
const slot =
this.#head <= end
? this.#range(this.#head, end)
: this.#faster(this.#range(this.#head, mask), this.#range(0, end));
return { arrival: this.#arrival[slot], media: this.#media[slot] };
}

#pop(): void {
this.#head = (this.#head + 1) & (this.#arrival.length - 1);
this.#len--;
}

/** Double the ring, unwrapping it to start at slot zero, and rebuild the tree. */
#grow(): void {
const old = this.#arrival.length;
const capacity = Math.max(1, old * 2);
const arrival = new Float64Array(capacity);
const media = new Float64Array(capacity);
for (let i = 0; i < this.#len; i++) {
const slot = (this.#head + i) & (old - 1);
arrival[i] = this.#arrival[slot];
media[i] = this.#media[slot];
}
this.#arrival = arrival;
this.#media = media;
this.#head = 0;

this.#tree = new Uint32Array(2 * capacity);
for (let slot = 0; slot < capacity; slot++) this.#tree[capacity + slot] = slot;
for (let node = capacity - 1; node > 0; node--) {
this.#tree[node] = this.#faster(this.#tree[2 * node], this.#tree[2 * node + 1]);
}
}

/** The fastest slot in `first..=last`, which must not wrap. */
#range(first: number, last: number): number {
const capacity = this.#arrival.length;
let lo = first + capacity;
let hi = last + capacity + 1;
let older = -1;
let newer = -1;
while (lo < hi) {
if (lo & 1) {
older = older < 0 ? this.#tree[lo] : this.#faster(older, this.#tree[lo]);
lo++;
}
if (hi & 1) {
hi--;
newer = newer < 0 ? this.#tree[hi] : this.#faster(this.#tree[hi], newer);
}
lo >>= 1;
hi >>= 1;
}
if (older < 0) return newer;
if (newer < 0) return older;
return this.#faster(older, newer);
}

/** Of two slots, `older` first in arrival order, the faster one, keeping `older` on a tie. */
#faster(older: number, newer: number): number {
return this.#arrival[newer] - this.#media[newer] < this.#arrival[older] - this.#media[older] ? newer : older;
}
}
5 changes: 5 additions & 0 deletions js/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,11 @@ slow-group-bench:
bun install --frozen-lockfile
bun hang/bench/slow-group.ts

# Sweep the audio jitter estimator's held history against in-order and reordered frames.
jitter-bench:
bun install --frozen-lockfile
bun hang/bench/jitter.ts

# Sweep decoder count against PCM callback size during audio warmup.
audio-warmup-bench:
bun install --frozen-lockfile
Expand Down
Loading
Loading