From d7bb096c02bf47a47f6ecb6e91d3bfbe9a8eb2d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 11:50:33 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m2/audio-playout/join-backlog.md Quest-Claim: 4be84cc4568e9f7f From 06223d9a0d90dac5ffdbd07120c2a9e041de38d3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 13:22:54 -0800 Subject: [PATCH 2/8] feat(hang,moq-mux,moq-audio): both consumers report the join's backlog they skip Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 33 ++++++++-- doc/concept/audio-jitter/corpus.ts | 48 ++++++++++++-- doc/concept/audio-jitter/reference.ts | 19 +++++- doc/concept/audio-jitter/skipped.json | 1 + js/hang/src/container/consumer.ts | 3 + js/hang/src/container/jitter.test.ts | 37 ++++++++++- js/hang/src/container/jitter.ts | 26 +++++++- quest/m2/audio-playout/README.md | 7 +- quest/m2/audio-playout/join-backlog.md | 37 ----------- rs/moq-audio/src/decode/consumer.rs | 91 ++++++++++++++++++++++++++ rs/moq-audio/src/jitter.rs | 77 +++++++++++++++++++++- rs/moq-mux/src/container/consumer.rs | 51 ++++++++++++++- test/audio-quality/budgets.json | 18 ++--- 13 files changed, 381 insertions(+), 67 deletions(-) create mode 100644 doc/concept/audio-jitter/skipped.json delete mode 100644 quest/m2/audio-playout/join-backlog.md diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 31b6a6f9643..4e1b0a64887 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -38,8 +38,9 @@ frame: Both sit below the decoder and above nothing: no reordering, no gap handling, no age budget. Except one skip: both consumers close a group below where delivery -stands unread, so a join's backlog they skip is never observed, and a join -delayed only by it is measured as lateness. +stands unread, so its frames are never observed. They report its arrival +instead, which ends a [join](#the-joins-backlog) as late as its observed backlog +would. Natively that holds only as far as the caller lets it. The container consumer reads groups in sequence order and applies its age budget inside `read()`, so a @@ -125,8 +126,8 @@ observe(arrival, media): if oldest + fastest < joined: # the join, not the path: folds nothing pending += 1 - else: - close(bucket of peak) + else: # open starts true + open = false; close(bucket of peak) if resumed: # a declared silence folds nothing settle(); origin = arrival; index = 0 else: # held until this interval closes @@ -153,6 +154,10 @@ settle(): # a stall is one observation if a stall is open: fold(stall.bucket); fold(nothing) min(stall.span, IDLE_MAX) times; close it +backlog(arrival): # a group skipped unread arrived + if not first observation and open and arrival < start + JOIN_MAX: + joined = max(joined, arrival) + end(): # a declared end precedes the next observation if not first observation: ended = true @@ -324,6 +329,18 @@ is measured from its first flush past the backlog. A first frame slower than the path after it is itself backlog, so most subscriptions skip their first interval, which leaves the prior 500 ms later. +A backlog the consumer skips unread carries no timestamp to test, only its +arrival, which `backlog` reports. It extends the join only while the join is +**open**, before any interval has measured the path: after that, a skipped group +is an outage's, whose frames queued behind it belong to the stall rule. On +`relay-mic-rtt40` the browser closes 267 groups 5 to 6 s old, arriving 1.0 to +1.3 s in, and the live frames queued behind them are up to 300 ms late; observed +or reported, they fold nothing, where unreported they held the target at 320 ms +for a 20 s half-life. A skip that lands before an interval closes keeps that +interval measuring the join, so a path dropping a late group every interval +would hold the join open for good: a skip extends it only `JOIN_MAX` past the +start, after which the estimator measures what it observes. + ### The histogram An empirical distribution of `BUCKETS` buckets `BUCKET` wide, holding total @@ -656,6 +673,7 @@ The exceptions are marked. | `FRESH_MAX` | 0.1 | Ours, twice the quantile's tail. NetEq ramps the forget factor from a start weight of 2 instead; see [the histogram](#the-histogram) | | `PRIOR[i]` | `0.5^(i+1)` | NetEq's histogram seed | | `IDLE_MAX` | 1024 intervals | Ours. A work bound with no effect on output | +| `JOIN_MAX` | 2000 ms | Ours. How long after the first arrival a skipped group can extend the join; see [the join's backlog](#the-joins-backlog) | | `MAX_HISTORY` | 8192 arrivals | Ours. Bounds a rate the publisher chooses; see [Bounding the history](#bounding-the-history) | | Out-of-range delays | clamped, not dropped | Ours. See [the histogram](#the-histogram) | @@ -696,8 +714,10 @@ directly, so neither is graded against the other. `arrival` and `media` are parallel arrays in **delivery order**, the order the receiver observes them, so `media` is deliberately out of order in the `reorder` and `backlog` cases. `advertised` and `delay` are present only where the case uses -them, as is `ends`: the indices, ascending, of the frames observed right after a -[declared end](#declared-silence), each of which calls `end()` before it is observed. `target` lists `[frame index, target]` at every frame where the target +them, as are `ends`: the indices, ascending, of the frames observed right after a +[declared end](#declared-silence), each of which calls `end()` before it is observed, +and `skipped`: the indices, ascending, of the frames a consumer closed unread, each +reported with `backlog(arrival)` instead of observed. `target` lists `[frame index, target]` at every frame where the target differs from the previous frame's, starting at frame 0, so an implementation folds the trace one frame at a time and compares its target at each listed index. Tolerance is 1e-6 ms, which the `f64` rules above make generous. @@ -711,6 +731,7 @@ index. Tolerance is 1e-6 ms, which the `f64` rules above make generous. | `reorder` | A run delivered after a newer one is measured against the frames before it, so the target covers waiting for it | | `backlog` | A backlog delivered newest first behind the live edge is not measured, so its media span never reaches the target | | `join` | A recorded join whose backlog arrives interleaved with the live edge, and delays the live frames queued behind it, folds nothing, so the target leaves the prior for the path's own | +| `skipped` | The `join` recording with the stale backlog a consumer closes below its cursor reported rather than observed still folds nothing for the join, so the target follows `join`'s | | `idle` | A 30 s gap relaxes the histogram toward the prior | | `silence` | A declared 60 s silence and a declared pause leave the target where it was, and a later undeclared gap still relaxes it | | `tunein` | A stale frame then the live edge 14.5 s later leaves the target near the frame duration. This is the #3517 regression | diff --git a/doc/concept/audio-jitter/corpus.ts b/doc/concept/audio-jitter/corpus.ts index fc56d430e46..5e828e62a54 100644 --- a/doc/concept/audio-jitter/corpus.ts +++ b/doc/concept/audio-jitter/corpus.ts @@ -26,6 +26,8 @@ export interface Case { media: number[]; /** Indices of the frames observed right after a declared end, in ascending order. */ ends?: number[]; + /** Indices of the frames skipped unread, reported by arrival instead of observed, in ascending order. */ + skipped?: number[]; /** `[frame index, target]` wherever the target differs from the previous frame's. */ target: [number, number][]; } @@ -107,7 +109,7 @@ function paced(frames: number, frame: number, base: number): Trace { /** Replay a trace through the reference and keep only the frames where the target moved. */ function run( - trace: Trace & { ends?: number[] }, + trace: Trace & { ends?: number[]; skipped?: number[] }, config: { frame: number; advertised?: number; delay?: number }, ): [number, number][] { if (config.delay !== undefined) return [[0, config.delay]]; @@ -117,9 +119,11 @@ function run( let previous: number | undefined; const ends = new Set(trace.ends); + const skipped = new Set(trace.skipped); for (let i = 0; i < trace.arrival.length; i++) { if (ends.has(i)) estimator.end(); - estimator.observe({ arrival: trace.arrival[i], media: trace.media[i] }); + if (skipped.has(i)) estimator.backlog(trace.arrival[i]); + else estimator.observe({ arrival: trace.arrival[i], media: trace.media[i] }); const value = estimator.target; if (value !== previous) target.push([i, value]); previous = value; @@ -130,15 +134,41 @@ function run( const FRAME = 20; -/** The first `until` ms of a recorded replay trace under `test/audio-quality/traces`. */ -function recorded(name: string, until: number): Trace { +/** A recorded replay trace under `test/audio-quality/traces`: each frame's arrival, media, and group. */ +function arrivals(name: string): [number, number, number][] { const here = dirname(fileURLToPath(import.meta.url)); const path = join(here, "../../../test/audio-quality/traces", `${name}.json`); - const { arrivals } = JSON.parse(readFileSync(path, "utf8")) as { arrivals: [number, number][] }; - const kept = arrivals.filter(([arrival]) => arrival < until); + return (JSON.parse(readFileSync(path, "utf8")) as { arrivals: [number, number, number][] }).arrivals; +} + +/** The first `until` ms of a recorded replay trace. */ +function recorded(name: string, until: number): Trace { + const kept = arrivals(name).filter(([arrival]) => arrival < until); return { arrival: kept.map(([arrival]) => arrival), media: kept.map(([, media]) => media) }; } +/** + * The first `until` ms of a recorded replay trace, with the frames of every group that arrives two or + * more below the newest one within `[from, to)` ms skipped, as a container consumer closes the groups + * that arrive below its cursor. + */ +function skipping(name: string, until: number, from: number, to: number): Trace & { skipped: number[] } { + const kept = arrivals(name).filter(([arrival]) => arrival < until); + const skipped: number[] = []; + const closed = new Set(); + const opened = new Set(); + let newest = Number.NEGATIVE_INFINITY; + kept.forEach(([arrival, , group], i) => { + if (!opened.has(group)) { + opened.add(group); + if (arrival >= from && arrival < to && group < newest - 1) closed.add(group); + } + if (closed.has(group)) skipped.push(i); + newest = Math.max(newest, group); + }); + return { arrival: kept.map(([arrival]) => arrival), media: kept.map(([, media]) => media), skipped }; +} + const traces: Omit[] = [ { name: "paced", @@ -204,6 +234,12 @@ const traces: Omit[] = [ frame: FRAME, ...recorded("relay-mic-rtt40", 30_000), }, + { + name: "skipped", + about: "The `join` recording as a container consumer reads it: from 0.9 s on, it closes every group arriving two or more below the newest unread, the join's stale backlog among them, and reports each one's arrival instead of observing it. The estimator never sees those frames are backlog, but the join still lasts until they arrived, so the target leaves the prior for the path's own as in `join`, instead of folding the live frames queued behind the backlog as a 300 ms stall. A group skipped once an interval has measured the path is an outage's, and leaves the join where it ended.", + frame: FRAME, + ...skipping("relay-mic-rtt40", 30_000, 900, 30_000), + }, { name: "idle", about: "A 30 s gap in both clocks, as a suspended tab produces. The idle intervals relax the histogram back toward the prior rather than freezing it.", diff --git a/doc/concept/audio-jitter/reference.ts b/doc/concept/audio-jitter/reference.ts index 224839e219b..833526abbee 100644 --- a/doc/concept/audio-jitter/reference.ts +++ b/doc/concept/audio-jitter/reference.ts @@ -30,6 +30,9 @@ export const FRESH_MAX = 0.1; /** Idle intervals folded at once, purely to bound the work. */ export const IDLE_MAX = 1024; +/** Milliseconds after the first arrival a skipped group can still extend the join. */ +export const JOIN_MAX = 2000; + /** The cold-start distribution, which is also what an idle receiver relaxes back to. */ export function prior(): Float64Array { const out = new Float64Array(BUCKETS); @@ -79,6 +82,8 @@ export class Estimator { #start?: number; /** The latest a frame produced before the subscription arrived, starting at `#start`. */ #joined = 0; + /** Every interval closed so far measured the join, so a skipped backlog can still extend it. */ + #open = true; /** A declared end precedes the next measured observation, so the silence before it is not idle. */ #ended = false; /** The earliest a frame stalled in this interval was due. */ @@ -167,7 +172,10 @@ export class Estimator { // join, not the path: those frames queued behind the backlog, and the player skipped onto // the clock past all of it. It folds nothing, like an interval with no arrival. if (this.#oldest + this.#fastest() < this.#joined) this.#pending += 1; - else this.#close(Math.min(BUCKETS - 1, Math.floor(this.#peak / BUCKET))); + else { + this.#open = false; + this.#close(Math.min(BUCKETS - 1, Math.floor(this.#peak / BUCKET))); + } if (resumed) { // The publisher declared the silence, so it is not evidence of anything: the @@ -244,6 +252,15 @@ export class Estimator { this.#stall = undefined; } + /** A frame produced before the subscription arrived at `arrival` but was skipped unread. */ + backlog(arrival: number): void { + // Before any observation the join has not started, and starts later than this. + // Once an interval measured the path the join is over, and a skip is an outage's. + if (this.#start !== undefined && this.#open && arrival < this.#start + JOIN_MAX) { + this.#joined = Math.max(this.#joined, arrival); + } + } + /** A declared end lies between the previous observation and the next. */ end(): void { // Before any observation there is no interval for the silence to stretch. diff --git a/doc/concept/audio-jitter/skipped.json b/doc/concept/audio-jitter/skipped.json new file mode 100644 index 00000000000..002f15004c8 --- /dev/null +++ b/doc/concept/audio-jitter/skipped.json @@ -0,0 +1 @@ +{"name":"skipped","about":"The `join` recording as a container consumer reads it: from 0.9 s on, it closes every group arriving two or more below the newest unread, the join's stale backlog among them, and reports each one's arrival instead of observing it. The estimator never sees those frames are backlog, but the join still lasts until they arrived, so the target leaves the prior for the path's own as in `join`, instead of folding the live frames queued behind the backlog as a 300 ms stall. A group skipped once an interval has measured the path is an outage's, and leaves the join where it ended.","frame":20,"arrival":[0,1.6,2.6,3,7.6,8,8.1,8.3,9.1,9.3,9.6,9.7,10,15.7,16.1,16.4,16.5,16.8,16.9,17.3,17.4,17.4,17.6,17.8,18,18,18.2,18.5,18.6,18.8,19.2,20.1,20.3,20.5,20.5,20.6,20.7,20.8,20.9,21.1,21.4,21.5,21.6,21.6,21.8,22.1,22.2,22.3,22.4,22.4,22.6,22.7,22.8,22.9,23,23.1,23.2,23.8,23.9,24,24,24.1,24.5,24.5,24.7,24.9,24.9,25.1,25.1,25.3,25.4,25.5,25.7,25.7,25.9,26,26.2,26.3,26.4,26.6,26.7,26.8,27,27,27.4,27.5,27.5,27.6,27.7,28.2,28.3,28.5,31.4,31.5,31.5,31.6,31.7,31.7,31.7,31.7,31.7,31.9,68.1,68.3,68.4,68.5,68.5,68.6,68.8,68.9,69,69,69.2,69.3,69.5,69.7,69.8,70.4,70.6,70.8,71.5,71.6,71.7,71.8,71.8,71.9,72,72.5,72.5,72.6,72.7,72.8,72.8,73,73.1,73.2,73.4,73.4,73.5,73.6,73.7,73.9,73.9,74,74.4,74.5,74.5,74.7,74.8,74.8,74.9,75,75,75.2,75.3,75.4,75.5,75.7,75.7,75.7,77.5,77.8,77.9,77.9,78,78,78.1,78.3,78.3,78.5,78.5,78.7,78.7,78.8,79,79.1,79.2,117.6,117.6,117.7,117.8,117.9,117.9,118.1,118.2,118.3,118.4,118.4,118.5,118.6,118.7,118.8,118.9,119.2,119.4,119.4,119.5,119.5,119.7,119.8,119.9,120,120.2,120.2,120.8,121,141,141.1,141.3,141.4,141.5,141.6,141.7,141.8,141.9,142,142.1,142.2,142.4,142.5,142.6,142.8,142.9,143.4,143.8,144,144.2,144.3,144.5,144.6,145.7,145.8,145.8,145.9,146,146.1,146.2,146.5,146.6,146.8,146.9,147,147.1,147.2,147.3,147.4,147.5,147.6,147.7,147.8,148,148.1,148.1,148.3,148.4,148.6,148.8,149,149.1,149.3,149.4,149.7,149.8,150,150.3,150.5,150.7,150.9,151,151.1,151.2,154.5,154.7,154.8,154.8,155,155,155,155.2,155.4,193.3,193.5,193.5,193.7,193.7,193.8,193.9,194,194.1,194.3,194.4,194.5,194.6,194.8,194.9,195,195.1,195.3,195.4,195.6,195.7,195.9,196,196.2,196.3,196.4,196.6,196.8,197.5,242.8,243,244.8,461.1,489.7,594.3,640.4,640.5,712.4,712.6,766.2,766.3,814.3,814.4,889.6,889.7,943.7,944.2,991.4,993.7,1018.4,1019.2,1019.6,1020,1020.4,1020.7,1020.8,1021.1,1021.3,1021.6,1021.8,1022.1,1022.4,1022.7,1022.9,1023.1,1023.4,1023.8,1024,1024.2,1024.6,1024.8,1025.1,1025.6,1025.8,1026.1,1026.3,1026.6,1026.9,1027,1027.2,1027.6,1027.9,1028.1,1028.4,1028.6,1028.8,1029,1029.1,1029.4,1029.6,1029.8,1030,1030.3,1030.4,1030.7,1030.9,1031.1,1031.4,1031.6,1031.7,1031.9,1032.2,1033.1,1033.2,1033.3,1033.5,1033.5,1033.6,1033.7,1033.8,1033.9,1034,1034.1,1034.2,1034.3,1034.4,1034.4,1034.5,1034.6,1034.8,1034.9,1035.2,1035.4,1035.7,1036.3,1036.3,1068.1,1068.3,1068.4,1068.6,1068.8,1068.9,1069.1,1069.2,1069.3,1069.6,1069.8,1069.9,1070,1070.1,1070.1,1070.2,1070.3,1070.3,1070.3,1125,1125.6,1125.9,1126.4,1127,1127.3,1127.7,1128,1128.2,1128.5,1128.8,1129.1,1129.5,1129.6,1129.9,1130.1,1130.5,1130.8,1131,1131.4,1131.6,1132.1,1132.6,1132.9,1133.4,1133.9,1134.1,1134.3,1134.6,1134.8,1135,1135.4,1135.8,1136.1,1136.3,1136.5,1136.8,1137.2,1137.3,1137.5,1137.5,1137.7,1138,1138.2,1138.2,1138.3,1138.4,1138.4,1138.5,1138.6,1138.6,1138.7,1138.8,1138.8,1138.9,1139.1,1139.2,1139.3,1139.4,1139.5,1139.7,1139.9,1140,1140.1,1140.2,1140.2,1140.4,1140.6,1140.6,1140.6,1168.7,1168.9,1169.1,1169.2,1169.4,1169.5,1169.6,1169.7,1169.9,1170,1170.1,1170.3,1170.3,1170.4,1170.4,1170.4,1170.5,1191,1191.5,1191.9,1192.8,1192.9,1193,1193.1,1196.7,1197,1197.2,1197.5,1197.8,1198,1198.5,1199.1,1199.4,1199.7,1200,1200.4,1200.8,1201.2,1201.6,1201.9,1202.2,1202.6,1203.1,1203.5,1204,1204.3,1204.8,1205.3,1205.8,1206.2,1206.6,1206.9,1209.9,1210.2,1210.3,1212.7,1215,1215.1,1215.3,1215.4,1215.5,1215.5,1215.7,1215.8,1215.9,1215.9,1216.1,1216.1,1216.3,1216.3,1216.5,1239.3,1239.3,1239.4,1239.5,1239.6,1239.7,1239.7,1239.7,1239.9,1240.1,1240.1,1240.2,1240.4,1240.4,1240.5,1240.6,1240.7,1240.9,1240.9,1243.2,1243.2,1243.4,1243.4,1243.6,1243.6,1243.8,1243.9,1289,1289.4,1289.8,1290,1290.3,1290.5,1290.7,1291.1,1291.4,1291.6,1291.9,1292.1,1292.4,1292.5,1292.6,1292.7,1292.9,1292.9,1303.9,1320.6,1343,1362.8,1382.8,1404.5,1423.9,1445,1462.3,1482.4,1506.5,1524.2,1545,1563,1583.4,1603.5,1622.1,1642.1,1662.5,1682.5,1703.8,1724.2,1744.3,1762,1783.2,1804.8,1825.3,1844.5,1862.7,1884.7,1905,1922.3,1943.7,1964.7,1984.3,2004.8,2022.2,2043,2068.1,2084.4,2105.2,2124.4,2142.7,2163.2,2183.9,2203.3,2224.2,2243.8,2263.9,2285.3,2306.3,2323.4,2345.6,2369.2,2383.3,2405.8,2425.1,2445.5,2464.7,2493.7,2505.6,2524.2,2545.5,2564.2,2584.4,2604.3,2624.1,2644.3,2665.3,2682.8,2702.2,2724.4,2745.8,2763.1,2781.6,2804.7,2826,2843.5,2865.6,2887.2,2904.7,2924.8,2945.4,2969.9,2999.8,3012.5,3059.2,3072.3,3072.7,3093,3130.3,3154.8,3159.9,3174.8,3207.3,3226.9,3235.7,3256,3283.8,3296,3317.5,3324.9,3344.7,3363.8,3383.7,3402.7,3424.6,3443.9,3464.6,3481.8,3502.7,3524.6,3545.5,3565.2,3583.7,3604,3625,3643.4,3665.1,3682.8,3703.3,3724.1,3745,3766.3,3785.2,3806,3824.2,3844.2,3865.1,3884.3,3904.4,3924.8,3945.2,3963.8,3985,4004,4024.2,4043.7,4063.3,4083.3,4103.1,4123.5,4145.2,4165.3,4184.8,4202.9,4225,4243.5,4262,4282,4302.7,4324.8,4345.4,4362.6,4384.9,4404.7,4424.4,4444.8,4465.7,4482.5,4503.3,4524.7,4544.6,4564.6,4584.3,4604.4,4624.5,4644.4,4664.4,4683.1,4704.1,4725.4,4742.9,4764.8,4784.2,4805,4823.8,4847.1,4863.4,4883.5,4904.4,4924.6,4944,4964.5,4985.1,5004.1,5023.8,5044.5,5066.1,5085.4,5107,5123.6,5143.6,5163.9,5184.1,5203.1,5223,5242.8,5263.6,5284,5304.3,5323.2,5343.1,5365.1,5385.7,5408.4,5422.1,5444.3,5465.7,5483.4,5504.3,5524.2,5544.5,5564.6,5584.5,5605.6,5625.9,5642.9,5665.3,5686.2,5703.6,5723.6,5744.7,5766.9,5785,5802.4,5824.5,5844.1,5863.3,5884.6,5903.7,5924.8,5947.7,5969.5,5983.8,6002.9,6023.2,6045.8,6070.1,6085.1,6103.3,6124.3,6144.4,6162.9,6186.9,6207.7,6225.8,6244.5,6264.7,6284.5,6304.7,6324.1,6343.2,6363,6384.3,6405,6424.2,6445.7,6488.5,6510.7,6520.5,6537.5,6581.5,6581.9,6601,6619,6644.3,6644.8,6663.8,6684.3,6704.2,6725,6744.1,6764.7,6785.6,6804.7,6824.1,6844.1,6865.3,6885.7,6905.1,6924.2,6944.3,6966.9,6986.6,7002.8,7024.6,7044.7,7063,7084.2,7104,7124.5,7143.4,7163.2,7186.5,7206.8,7224.9,7244,7263.7,7284.2,7304.7,7325,7343.5,7364.2,7384,7405.6,7421.3,7445.5,7462.6,7484.5,7507.2,7544.8,7545.5,7564.5,7584.2,7604.9,7624,7642,7664.6,7685.8,7702.5,7723.5,7748.8,7765.2,7784.4,7805,7824.5,7845.1,7863.6,7883.6,7904.3,7924.4,7944.3,7964.3,7985.1,8004.7,8024.9,8043.3,8064.2,8085.4,8101.5,8124.4,8144.2,8164.9,8185.9,8204.4,8223.5,8242.7,8264.7,8283.3,8303.9,8325.6,8343.2,8364.2,8383.1,8404.3,8424.1,8444.8,8464.7,8487.1,8502.8,8528.6,8543.6,8563.4,8584.6,8605.2,8624.5,8644,8664,8684.3,8703.8,8723.7,8744.5,8764.9,8785.3,8802,8830,8844.3,8864.1,8885,8903,8924.2,8944.3,8964,8984.2,9003,9024.5,9044.2,9065.1,9084.3,9104.6,9124.1,9145.3,9164.2,9184.4,9205.1,9224,9244,9264.1,9284.8,9304.5,9326.8,9342.9,9364.4,9385,9406,9422.5,9443,9463.7,9484.1,9503.4,9523.3,9544.1,9562.5,9584,9607.3,9623.7,9644.2,9664.3,9684.6,9704.8,9724.2,9744.1,9763.4,9785.1,9808.2,9828.1,9847.2,9864.4,9885,9905.3,9922.4,9944.3,9962.9,9984.5,10004.9,10024.4,10044.7,10065,10084,10102.1,10122.9,10143.3,10165,10182.8,10204.5,10225.2,10243.7,10264.1,10285.4,10306.2,10325.2,10342,10367.1,10383.1,10402.9,10424.8,10446,10513,10513.8,10514.7,10556.9,10578.6,10579.2,10591.5,10634,10638.7,10661.2,10668.1,10688.9,10726.5,10740.6,10743.8,10764,10785.6,10803.8,10825.8,10845.6,10862.4,10883.1,10903.6,10923,10942.9,10964,10983.1,11003.5,11023.2,11044.1,11065.5,11082.4,11105.2,11123,11144.7,11162.9,11184.6,11204.5,11224.5,11243.9,11264.9,11282.5,11304.7,11324.6,11344.2,11364.8,11383.5,11403.7,11423.5,11443.6,11464,11491.5,11502,11524.7,11545.7,11565.7,11584.7,11603.7,11622.6,11644.4,11665.1,11684.1,11704.4,11724.5,11744.2,11764.3,11782.9,11803.9,11825.5,11847,11863.9,11885,11903.2,11924.6,11944.1,11963.7,11982.8,12002.5,12022.7,12042.4,12064,12084.4,12104.1,12125.7,12151.8,12164.8,12185.1,12204,12224.1,12244.1,12264.8,12282.4,12304.9,12323,12343,12362.9,12384,12403.9,12423.4,12444,12464.4,12486.1,12507.8,12523.8,12544.1,12564.9,12583.8,12603.9,12623.6,12643.9,12663.8,12686.2,12704.5,12724.8,12744.1,12765.6,12783,12804.2,12824.4,12845.4,12864.1,12884.3,12904.5,12923.5,12944,12964.8,12985.2,13005.5,13024.5,13044.1,13065.1,13083.9,13104.9,13124.8,13143.8,13163.3,13184.9,13205.5,13226.1,13249.1,13263.5,13283.9,13302.7,13323,13343.2,13364.1,13385.1,13403.5,13423.3,13444.3,13463.4,13483.9,13504.3,13525.4,13542.7,13562.3,13583.7,13604.9,13623.8,13643.3,13661.9,13683.3,13703.6,13725.3,13746.3,13763.5,13782.8,13805.7,13821.8,13845.5,13862.6,13883.5,13904.3,13925.4,13942.5,13962.6,13983.5,14005.8,14022.5,14043.1,14062.9,14084.1,14103.1,14124.1,14144.9,14164.6,14185.8,14203.7,14223.7,14246.2,14263.9,14284.3,14305.3,14324.1,14345.7,14364.4,14384.5,14404.8,14423.1,14445.2,14468.9,14493.3,14509.9,14554.3,14568,14588.4,14591.6,14641.5,14642.2,14681.3,14681.9,14701.6,14722.7,14737.8,14768.9,14780.3,14815.3,14815.9,14824.6,14843.4,14863.5,14883.3,14903.4,14924.2,14944.8,14963.2,14982.2,15003.2,15023.3,15043,15064.1,15082.8,15106.4,15123.9,15144,15165,15184.3,15204.9,15222.3,15242.8,15264.1,15284.3,15304.7,15324.7,15344.4,15363.9,15384.1,15405.2,15423.7,15443.3,15463.6,15483.3,15502,15523.8,15543.6,15563.6,15584,15604.1,15622.9,15643.6,15664.1,15684.9,15703.4,15723.4,15742.5,15765,15784,15803.1,15824.4,15845.6,15862.7,15884.9,15903.5,15923.9,15943.4,15963.8,15984.9,16003.5,16025,16045.2,16079.4,16083.6,16103.2,16123.7,16143.8,16162.4,16182.7,16204,16223.1,16243.4,16264.5,16285,16303.7,16323.4,16344.2,16366.1,16381.8,16403.9,16425.4,16443.2,16463.3,16483.6,16503.8,16523.4,16544.1,16564.9,16583.1,16604,16623.4,16644.5,16664.4,16684.6,16703.8,16724.4,16745,16763.4,16783.1,16803.4,16824.5,16842,16863.2,16883.8,16903.7,16922.8,16943.9,16965.1,16983.2,17003.2,17021.8,17044.3,17064.1,17082.6,17103.9,17123.6,17143.9,17164,17185.9,17203,17224.1,17243.8,17264.2,17285.4,17302.6,17325.3,17343.5,17364.4,17384.3,17404.4,17425.2,17444,17463.6,17483.9,17504.4,17527.2,17544.4,17564.4,17584.3,17604.5,17624,17643.4,17664.3,17685.5,17702.9,17724.2,17744.9,17762.2,17783.9,17804.4,17824,17844.5,17863.7,17883.8,17905.3,17924.1,17944.7,17964.1,17984.2,18005.2,18023.3,18042.1,18064.2,18085.2,18103.5,18123.7,18143.9,18164.1,18185,18204.1,18223.4,18242.9,18264.4,18284.3,18304.4,18324.8,18344.4,18365.3,18383,18403.3,18423.8,18443.6,18465.6,18486.8,18524.6,18535.4,18565.1,18573.9,18602.7,18619.5,18655.2,18678,18687.1,18707,18718.1,18740.8,18787.1,18787.5,18788,18804,18823.8,18844.7,18865.2,18883.8,18905.3,18927.7,18942.9,18963.6,18982.6,19002.5,19023.9,19042.3,19063.1,19083.6,19102.7,19122.5,19144.9,19162,19186.5,19203.2,19223.9,19243.9,19263.6,19282.7,19302.7,19323.5,19344,19364.5,19383.3,19403.8,19424,19445,19463.3,19483.3,19504,19525.2,19543.5,19564.1,19585.8,19601.7,19622.1,19642,19662.6,19684.6,19705,19723.9,19753,19762,19783.6,19802.7,19824,19848.7,19864.6,19882.4,19902.7,19924.1,19943.8,19964,19985.6,20004.5,20023.6,20044.4,20065.2,20086.9,20104,20123.7,20143.4,20164,20187.5,20202.8,20222.9,20243.8,20266,20284.5,20303.2,20325.8,20346.1,20364.3,20384.4,20405.3,20433,20444.1,20472.3,20484.1,20507.6,20535.6,20548.3,20564.2,20583.9,20604,20623.2,20645.7,20666.3,20684.4,20703.2,20724.7,20745.3,20767.9,20785.1,20804,20821.6,20844,20864.5,20884.3,20904.9,20924.2,20944.2,20964.5,20984.2,21006.8,21025.5,21048.1,21065.4,21089.6,21105.2,21130,21144.1,21170.3,21183.8,21209.1,21223.9,21247.5,21265.1,21283.9,21304.3,21340.3,21357.4,21363.8,21397,21405.6,21434.5,21444,21470.1,21485.4,21509.1,21523.9,21547,21563.4,21583.7,21603.4,21623.5,21643.1,21663.3,21681.9,21702.9,21725.1,21744.1,21763.5,21783.2,21803.7,21823.8,21843.4,21863.4,21884.3,21903.4,21925.3,21943.9,21963.8,21986,22015.9,22042.6,22064.5,22078.7,22105.9,22108.3,22142.7,22148.2,22183.7,22193.8,22229.3,22236,22302.4,22303.2,22303.5,22322,22333.6,22356.4,22380.3,22417.8,22418.7,22438.2,22451.1,22464.7,22485,22502.7,22524.4,22543.1,22563.1,22584,22604.2,22625.1,22643.8,22665.2,22687.1,22702.7,22723.1,22744.3,22765.1,22787.4,22803.4,22828.5,22845.6,22867.6,22885.1,22911.5,22926.5,22950.5,22964.5,22996.1,23004.9,23039,23045.1,23076.4,23083.9,23116.1,23124.1,23154.1,23163.6,23194.1,23204.2,23223.9,23245.2,23264.3,23284.4,23305.7,23325.8,23344.7,23365.3,23385.8,23404.3,23427.9,23450.3,23466,23488.7,23517.8,23527.1,23551.5,23573,23585.7,23607,23626.2,23648.6,23689.5,23690.3,23732.4,23733.2,23747.4,23768.6,23789.4,23809.2,23829.5,23862.6,23866.3,23902.2,23908.2,23927,23970.4,23980.4,24008.3,24008.7,24024.5,24044.4,24066,24086.1,24105,24127.8,24150.4,24164.1,24186,24206.3,24226.4,24255.6,24266.8,24289,24309.8,24325.8,24370.4,24371.1,24389.7,24405,24426.7,24452.1,24465.9,24485.5,24507.2,24530,24547.2,24578.5,24584.4,24610.1,24625.9,24646.3,24673.3,24688.8,24706,24728.4,24755.4,24768.2,24790.1,24810.3,24824.9,24844.6,24869,24885.4,24913.3,24926.8,24946.1,24973,24990.6,25003.8,25024.6,25047,25075.9,25085.1,25110.3,25129,25151.9,25165.7,25184.9,25209.6,25225.4,25248.3,25269.8,25294.4,25308.1,25325.4,25350,25372.3,25393.1,25412.9,25431.9,25450.7,25472.5,25489.9,25503.6,25526.1,25546.7,25571.5,25592.9,25608.2,25631.1,25647.2,25665.4,25685.5,25713.7,25730,25751.6,25772.5,25790.4,25802,25823.9,25843.8,25864.1,25885.4,25903.3,25922.3,25964,25966.5,25999.9,26014.1,26032.3,26059.1,26094,26104.7,26112.4,26124.7,26142.9,26165,26199.7,26203.4,26222.2,26244.3,26264.8,26283.5,26304.1,26323.8,26344.1,26364.5,26384.7,26404.7,26423.1,26443.7,26463.9,26482.8,26503.9,26525.7,26544.2,26562.5,26583.1,26603.8,26623.7,26642.5,26663.5,26683.4,26704,26723,26744.2,26765.8,26781.5,26801.6,26824.6,26844.4,26864,26885.2,26906.7,26922,26942.6,26963.9,26986.3,27003.6,27023.2,27043.3,27063,27083.7,27103.4,27122.2,27143.8,27163.8,27184.6,27203.4,27223.3,27244.5,27264.4,27284.1,27307.1,27327.4,27344.8,27364.2,27384.4,27404.2,27425.1,27443,27464.3,27485.2,27503.3,27523.5,27542.4,27563.6,27582.7,27602.5,27623.6,27643.4,27663.6,27684.2,27702.7,27723.6,27743.9,27763.2,27784.6,27804.7,27823.8,27842.8,27864.4,27883.1,27903.8,27923.7,27944,27964.1,27984.3,28006.6,28024.7,28045.2,28062.2,28082.2,28102.9,28124.7,28144.1,28165.4,28186.4,28203.9,28224,28244.6,28264.1,28284.8,28302.6,28322.2,28344,28362.6,28383.2,28404.8,28423.8,28444.7,28463.8,28485.4,28502.7,28523.8,28543.2,28563.1,28584,28602.5,28624.8,28643.7,28662.1,28683.6,28704,28722.9,28744.1,28764.7,28784,28803,28822.5,28844.7,28866.8,28883.1,28903.2,28923.3,28943.8,28967.5,28982.3,29002.6,29022.7,29043.9,29064.8,29083.8,29102.1,29122.2,29144.4,29165.5,29182.3,29202.9,29223,29243,29263,29283.9,29302.9,29321.9,29343,29362.2,29381.9,29403.1,29423.7,29442.2,29461.7,29482.6,29503.2,29521.3,29543.3,29563.3,29581.5,29604.5,29624.1,29642,29662.7,29682.2,29702.7,29722.3,29743,29761.4,29781.3,29803.2,29822.7,29843.9,29863.9,29887,29904.3,29923.3,29942.6,29963.9,29983.7],"media":[222738.333,679218.333,679238.333,679198.333,679298.333,679338.333,679318.333,679278.333,679358.333,679378.333,679398.333,679418.333,679258.333,679438.333,679458.332,679698.333,679718.332,679738.333,679658.333,679758.333,679618.333,679638.333,679578.333,679558.333,679518.333,679538.333,679598.333,679818.333,679798.333,679678.333,679778.333,679958.333,679938.333,679918.333,679898.333,679878.333,679858.333,680118.333,680078.333,680058.333,680038.333,680158.333,680178.333,680198.333,679498.333,680358.333,680338.333,680318.333,680298.333,680278.333,680258.333,680378.333,680398.333,680018.333,679998.333,679978.333,680138.333,679838.333,680418.333,680458.333,680478.333,680498.333,680518.333,680538.333,680558.333,680598.333,680638.333,680618.333,680658.333,680678.333,680718.333,680698.333,680758.333,680778.333,680798.333,680818.333,680838.333,680858.333,680878.333,680918.333,680938.333,680958.333,680978.333,680998.333,681018.333,681058.333,681038.333,681098.333,681078.333,681118.333,681138.333,681178.333,681158.333,680898.333,680438.333,680578.333,680218.333,680238.332,680098.333,680738.333,679478.333,681198.333,681218.333,681238.333,681258.332,681278.332,681298.333,681338.333,681378.333,681318.333,681358.333,681398.333,681418.333,681478.333,681438.333,681458.333,681498.333,681518.333,681538.333,681558.333,681578.333,681598.333,681638.333,681618.333,681658.333,681738.333,681858.333,681678.333,681878.333,681758.333,681778.333,681698.333,681898.333,681938.333,681918.333,681798.333,681718.333,681818.333,681838.333,681958.333,681978.333,681998.333,682018.333,682038.333,682058.333,682078.333,682098.333,682118.333,682138.333,682158.333,682178.333,682198.333,682218.333,682258.333,682278.333,682298.333,682318.333,682338.333,682358.333,682378.333,682398.333,682418.333,682438.333,682458.333,682478.333,682518.333,682498.333,682538.333,682718.333,682698.333,682658.333,682598.333,682678.333,682638.333,682558.333,682578.333,682238.333,683278.333,683298.332,683258.333,683238.333,683218.333,683198.333,683898.333,683878.333,683858.333,683838.333,684558.333,683818.333,683798.333,683778.333,683758.333,683738.333,686458.333,686418.333,686358.333,686338.333,686318.333,686298.333,684798.333,686398.333,686378.333,686138.333,685998.333,685978.333,685958.333,685938.333,686118.333,684758.333,684778.333,686098.333,686078.333,684838.333,684818.333,685398.333,686038.333,686158.333,685918.333,685698.333,685678.333,685658.333,685358.333,685638.333,685898.333,686198.333,685018.333,684998.333,685538.333,684978.333,684958.333,684938.333,684918.333,684898.333,684858.333,684878.333,685438.333,685798.333,685198.333,685178.333,685158.333,685138.333,685118.333,685098.333,685418.333,685078.333,685058.333,685038.333,685878.333,685858.333,685838.333,685818.333,685378.333,686178.333,685778.333,685758.333,685738.333,686278.333,686258.333,685618.333,686238.333,686218.333,685498.333,685598.333,685458.333,685218.333,685558.333,685338.333,685318.333,686058.333,685718.333,686018.333,685298.333,685278.333,685258.333,685238.333,685478.333,685578.333,685518.333,691518.333,691498.333,696438.333,696418.333,696398.333,696378.333,696358.333,696338.333,696318.333,696298.333,696278.333,696258.333,696238.333,696218.333,696198.333,696178.333,696158.333,696138.333,696118.333,696098.333,696078.333,696058.333,696038.333,696018.333,695998.333,695978.333,695958.332,695938.332,695898.332,695918.332,697958.333,703478.333,699178.333,701338.333,709618.333,709658.333,709738.333,709818.333,709798.333,709858.333,709878.333,709918.333,709938.333,709998.333,709978.333,710058.333,710038.333,710118.333,710098.333,710158.333,708838.333,703498.333,704098.333,704118.333,704138.333,704158.333,704178.333,704198.333,704218.333,704238.333,704258.333,704278.333,704298.333,704318.333,704338.333,704358.333,704378.333,704398.333,704418.333,704438.333,704458.333,704478.333,704498.333,704538.333,704558.333,704578.333,704618.333,704638.333,704658.333,704678.333,704698.333,704718.333,704758.333,704778.333,704798.333,704818.333,704838.333,704858.333,704878.333,704898.333,704918.333,704938.333,704958.333,704978.333,704998.333,705018.333,705038.333,705058.333,705098.333,705118.333,705138.333,705158.333,705178.333,705198.333,705218.333,705238.333,705258.333,705278.333,705298.333,705318.333,705338.333,705358.333,705378.333,705398.333,705418.333,705438.333,705458.333,705478.333,705498.333,705538.333,705558.333,705578.333,705598.333,705618.333,705638.333,705658.333,705678.333,705698.333,705718.333,705738.333,705758.333,705778.333,705798.333,705818.333,705838.333,705858.333,705898.333,705918.333,705938.333,705958.333,705978.333,705998.333,706038.333,706058.333,706078.333,706098.333,706118.333,706138.333,706158.333,706178.333,706258.333,706278.333,706318.333,706338.333,706358.333,706378.333,706418.333,706438.333,706458.333,706498.333,706518.333,706538.333,706578.333,706598.333,706618.333,706638.333,706678.333,706698.333,706718.333,706758.333,706798.333,706818.333,706838.333,706858.333,706878.333,706898.333,706918.333,706938.333,706958.333,706978.333,706998.333,707018.333,707038.333,707058.333,707078.333,707098.333,707118.333,707138.333,707158.333,707198.333,707218.333,707238.333,707258.333,707278.333,707298.333,707318.333,707338.333,707358.333,707378.333,707398.333,707418.333,707458.333,707478.333,707498.333,707518.333,707538.333,707558.333,707578.333,707598.333,707618.333,707638.333,707658.333,707698.333,707718.333,707738.333,707758.333,707778.333,707818.333,707838.333,707858.333,707898.333,707918.333,707938.333,707958.333,707978.333,707998.333,708018.333,708038.333,708058.333,708078.333,708098.333,708118.333,708138.333,708158.333,708178.333,708218.333,708198.333,708238.333,708258.333,708278.333,708298.333,709498.333,709478.333,709438.333,709458.333,709418.333,709398.333,708338.333,708358.333,708378.333,708398.333,708418.333,708438.333,708458.333,708478.333,708518.333,708498.333,708538.333,708558.333,708578.333,708598.333,708618.333,708638.333,708658.333,708678.333,708698.333,708718.333,708738.333,708778.333,708758.333,708798.333,708818.333,708318.333,709378.333,709358.333,709338.333,709318.333,709298.333,709278.333,709258.333,709238.333,709218.333,709198.333,709178.333,709158.333,709138.333,709118.333,709078.333,709098.333,709058.333,709038.333,709018.333,708998.333,708978.333,708958.333,708938.333,708918.333,709558.333,709638.333,709718.333,708898.333,708878.333,708858.333,709518.333,709538.333,709598.333,709578.333,709698.333,709678.333,709758.333,709778.333,709838.333,709898.333,709958.333,710018.333,710078.333,710178.333,710138.333,710198.333,710238.333,710218.333,710258.333,710298.333,710278.333,710358.333,710318.333,710338.333,710418.333,710398.333,710378.333,710478.333,710438.333,710458.333,710498.333,710518.333,710538.333,710558.333,710578.333,710598.333,710618.333,710638.333,710658.333,710678.333,710698.333,710718.333,710738.333,710758.333,710778.333,710798.333,710818.333,710838.333,710858.333,710878.333,710898.333,710918.333,710938.333,710958.333,710978.333,710998.333,711018.333,711038.333,711058.333,711078.333,711098.333,711118.333,711138.333,711158.333,711178.333,711198.333,711218.333,711238.333,711258.333,711278.333,711298.333,711318.333,711338.333,711358.333,711378.333,711398.333,711418.333,711438.333,711458.333,711478.333,711498.333,711518.333,711538.333,711558.333,711578.333,711598.333,711618.333,711638.333,711658.333,711678.333,711698.333,711718.333,711738.333,711758.333,711778.333,711798.333,711818.333,711838.333,711858.333,711878.333,711898.333,711918.333,711938.333,711958.333,711978.332,711998.332,712018.332,712038.332,712058.332,712078.332,712098.332,712118.332,712138.332,712158.332,712178.332,712198.332,712218.332,712258.332,712238.332,712278.332,712298.332,712318.332,712338.332,712358.332,712378.332,712398.332,712418.332,712438.332,712458.332,712478.332,712498.332,712518.332,712538.332,712558.332,712578.332,712598.332,712618.332,712638.332,712658.332,712678.332,712698.332,712718.332,712738.332,712758.333,712778.333,712798.333,712818.333,712838.333,712858.333,712878.333,712898.333,712918.333,712938.333,712958.333,712978.333,712998.333,713018.333,713038.333,713058.333,713078.333,713098.333,713118.333,713138.333,713158.333,713178.333,713198.333,713218.333,713238.333,713258.333,713278.333,713298.333,713318.333,713338.333,713358.333,713378.333,713398.333,713418.333,713438.333,713458.333,713478.333,713498.333,713518.333,713538.333,713558.333,713578.333,713598.333,713618.333,713638.333,713658.333,713678.333,713698.333,713718.333,713738.333,713758.333,713778.333,713798.333,713818.333,713838.333,713858.333,713878.333,713898.333,713918.333,713938.333,713958.333,713978.333,713998.333,714018.333,714038.333,714058.333,714078.333,714098.333,714118.333,714138.333,714158.333,714178.333,714198.333,714218.333,714238.333,714258.333,714278.333,714298.333,714318.333,714338.333,714358.333,714378.333,714398.333,714418.333,714438.333,714458.333,714478.333,714498.333,714518.333,714538.333,714558.333,714578.333,714598.333,714618.333,714638.333,714658.333,714678.333,714698.333,714718.333,714738.333,714758.333,714778.333,714798.333,714818.333,714838.333,714858.333,714878.333,714898.333,714918.333,714938.333,714958.333,714978.333,714998.333,715018.333,715038.333,715058.333,715078.333,715098.333,715118.333,715138.333,715158.333,715178.333,715198.333,715218.333,715238.333,715258.333,715278.333,715298.333,715318.333,715338.333,715358.333,715378.333,715398.333,715418.333,715438.333,715458.333,715478.333,715498.333,715518.333,715538.333,715558.333,715578.333,715598.333,715618.333,715638.333,715658.333,715678.333,715698.333,715718.333,715738.333,715758.333,715778.333,715798.333,715818.333,715838.333,715858.333,715878.333,715898.333,715918.333,715938.333,715958.333,715978.333,715998.333,716018.333,716038.333,716058.333,716078.333,716098.333,716118.333,716138.333,716158.333,716178.333,716198.333,716218.333,716238.333,716258.333,716278.333,716298.333,716318.333,716338.333,716358.333,716378.333,716398.333,716418.333,716438.333,716458.333,716478.333,716498.333,716518.333,716538.333,716558.333,716578.333,716598.333,716618.333,716638.333,716658.333,716678.333,716698.333,716718.333,716738.333,716758.333,716778.333,716798.333,716818.333,716838.333,716858.333,716878.333,716898.333,716918.333,716938.333,716958.333,716978.333,716998.333,717018.333,717038.333,717058.333,717078.333,717098.333,717118.333,717138.333,717158.333,717178.333,717198.333,717218.333,717238.333,717258.333,717278.333,717298.333,717318.333,717338.333,717358.333,717378.333,717398.333,717418.333,717438.333,717458.333,717478.333,717498.333,717518.333,717538.333,717558.333,717578.333,717598.333,717618.333,717638.333,717658.333,717678.333,717698.333,717718.333,717738.333,717758.333,717778.333,717798.333,717818.333,717838.333,717858.333,717878.333,717898.333,717918.333,717938.333,717958.333,717978.333,717998.333,718018.333,718038.333,718058.333,718078.333,718098.333,718118.333,718138.333,718158.333,718178.333,718198.333,718218.333,718238.333,718258.333,718278.333,718298.333,718318.333,718338.333,718358.333,718378.333,718398.333,718418.333,718438.333,718458.333,718478.333,718498.333,718518.333,718538.333,718558.333,718578.333,718598.333,718618.333,718638.333,718658.333,718678.333,718698.333,718718.333,718738.333,718758.333,718778.333,718798.333,718818.333,718838.333,718858.333,718878.333,718898.333,718918.333,718938.333,718958.333,718978.333,718998.333,719018.333,719038.333,719058.333,719078.333,719098.333,719118.333,719138.333,719158.333,719178.333,719198.333,719218.333,719238.333,719258.333,719278.333,719298.333,719318.333,719338.333,719358.333,719378.333,719398.333,719418.333,719438.333,719458.333,719478.333,719498.333,719518.333,719538.333,719558.333,719578.333,719598.333,719618.333,719638.333,719658.333,719678.333,719698.333,719718.333,719738.333,719758.333,719778.333,719798.333,719818.333,719838.333,719858.333,719878.333,719898.333,719918.333,719938.333,719958.333,719978.333,719998.333,720018.333,720038.333,720058.333,720078.333,720098.333,720118.333,720138.333,720158.333,720178.333,720198.333,720218.333,720238.333,720258.333,720278.333,720298.333,720318.333,720338.333,720358.333,720378.333,720398.333,720418.333,720438.333,720458.333,720478.333,720498.333,720518.333,720538.333,720558.333,720578.333,720598.333,720618.333,720638.333,720658.333,720678.333,720698.333,720718.333,720738.333,720758.333,720778.333,720798.333,720818.333,720838.333,720858.333,720878.333,720898.333,720918.333,720938.333,720958.333,720978.333,720998.333,721018.333,721038.333,721058.333,721078.333,721098.333,721118.333,721138.333,721158.333,721178.333,721198.333,721218.333,721238.333,721258.333,721278.333,721298.333,721318.333,721338.333,721358.333,721378.333,721398.333,721418.333,721438.333,721458.333,721478.333,721498.333,721518.333,721538.333,721558.333,721578.333,721598.333,721618.333,721638.333,721658.333,721678.333,721698.333,721718.333,721738.333,721758.333,721778.333,721798.333,721818.333,721838.333,721858.333,721878.333,721898.333,721918.333,721938.333,721958.333,721978.333,721998.333,722018.333,722038.333,722058.333,722078.333,722098.333,722118.333,722138.333,722158.333,722178.333,722198.333,722218.333,722238.333,722258.333,722278.333,722298.333,722318.333,722338.333,722358.333,722378.333,722398.333,722418.333,722438.333,722458.333,722478.333,722498.333,722518.333,722538.333,722558.333,722578.333,722598.333,722618.333,722638.333,722658.333,722678.333,722698.333,722718.333,722738.333,722758.333,722778.333,722798.333,722818.333,722838.333,722858.333,722878.333,722898.333,722918.333,722938.333,722958.333,722978.333,722998.333,723018.333,723038.333,723058.333,723078.333,723098.333,723118.333,723138.333,723158.333,723178.333,723198.333,723218.333,723238.333,723258.333,723278.333,723298.333,723318.333,723338.333,723358.333,723378.333,723398.333,723418.333,723438.333,723458.333,723478.333,723498.333,723518.333,723538.333,723558.333,723578.333,723598.333,723618.333,723638.333,723658.333,723678.333,723698.333,723718.333,723738.333,723758.333,723778.333,723798.333,723818.333,723858.333,723838.333,723878.333,723898.333,723918.333,723938.333,723958.333,723978.333,723998.333,724018.333,724038.333,724058.333,724078.333,724098.333,724118.333,724138.333,724158.333,724178.333,724198.333,724218.333,724238.333,724258.333,724278.333,724298.333,724318.333,724338.333,724358.333,724378.333,724398.333,724418.333,724438.333,724458.333,724478.333,724498.333,724518.333,724538.333,724558.333,724578.333,724598.333,724618.333,724638.333,724658.333,724678.333,724698.333,724718.333,724738.333,724758.333,724778.333,724798.333,724818.333,724838.333,724858.333,724878.333,724898.333,724918.333,724938.333,724958.333,724978.333,724998.333,725018.333,725038.333,725058.333,725078.333,725098.333,725118.333,725138.333,725158.333,725178.333,725198.333,725218.333,725238.333,725258.333,725278.333,725298.333,725318.333,725338.333,725358.333,725378.333,725398.333,725418.333,725438.333,725458.333,725478.333,725498.333,725518.333,725538.333,725558.333,725578.333,725598.333,725618.333,725638.333,725658.333,725678.333,725698.333,725718.333,725738.333,725758.333,725778.333,725798.333,725818.333,725838.333,725858.333,725878.333,725898.333,725918.333,725938.333,725958.333,725978.333,725998.333,726018.333,726038.333,726058.333,726078.333,726098.333,726118.333,726138.333,726158.333,726178.333,726198.333,726218.333,726238.333,726258.333,726278.333,726298.333,726318.333,726338.333,726358.333,726378.333,726398.333,726418.333,726438.333,726458.333,726478.333,726498.333,726518.333,726538.333,726558.333,726578.333,726598.333,726618.333,726638.333,726658.333,726678.333,726698.333,726718.333,726738.333,726758.333,726778.333,726798.333,726818.333,726838.333,726858.333,726878.333,726898.333,726918.333,726938.333,726958.333,726978.333,726998.333,727018.333,727038.333,727058.333,727078.333,727098.333,727118.333,727138.333,727158.333,727178.333,727198.333,727218.333,727238.333,727258.333,727278.333,727298.333,727318.333,727338.333,727358.333,727378.333,727398.333,727418.333,727438.333,727458.333,727478.333,727498.333,727518.333,727538.333,727558.333,727578.333,727598.333,727618.333,727638.333,727658.333,727678.333,727698.333,727718.333,727738.333,727758.333,727778.333,727798.333,727818.333,727838.333,727858.333,727878.333,727898.333,727918.333,727938.333,727958.333,727978.333,727998.333,728018.333,728038.333,728058.333,728078.333,728098.333,728118.333,728138.333,728158.333,728178.333,728198.333,728218.333,728238.333,728258.333,728278.333,728298.333,728318.333,728338.333,728358.333,728378.333,728398.333,728418.333,728438.333,728458.333,728478.333,728498.333,728518.333,728538.333,728558.333,728578.333,728598.333,728618.333,728638.333,728658.333,728678.333,728698.333,728718.333,728738.333,728758.333,728778.333,728798.333,728818.333,728838.333,728858.333,728878.333,728898.333,728918.333,728938.333,728958.333,728978.333,728998.333,729018.333,729038.333,729058.333,729078.333,729098.333,729118.333,729138.333,729158.333,729178.333,729198.333,729218.333,729238.333,729258.333,729278.333,729298.333,729318.333,729338.333,729358.333,729378.333,729398.333,729418.333,729438.333,729458.333,729478.333,729498.333,729518.333,729538.333,729558.333,729578.333,729598.333,729618.333,729638.333,729658.333,729678.333,729698.333,729718.333,729738.333,729758.333,729778.333,729798.333,729818.333,729838.333,729858.333,729878.333,729898.333,729918.333,729938.333,729958.333,729978.333,729998.333,730018.333,730038.333,730058.333,730078.333,730098.333,730118.333,730138.333,730158.333,730178.333,730198.333,730218.333,730238.333,730258.333,730278.333,730298.333,730318.333,730338.333,730358.333,730378.333,730398.333,730418.333,730438.333,730458.333,730478.333,730498.333,730518.333,730538.333,730558.333,730578.333,730598.333,730618.333,730638.333,730658.333,730678.333,730698.333,730718.333,730738.333,730758.333,730778.333,730798.333,730818.333,730838.333,730858.333,730878.333,730898.333,730918.333,730938.333,730958.333,730978.333,730998.333,731018.333,731038.333,731058.333,731078.333,731098.333,731118.333,731138.333,731158.333,731178.333,731198.333,731218.333,731238.333,731258.333,731278.333,731298.333,731318.333,731338.333,731358.333,731378.333,731398.333,731418.333,731438.333,731478.333,731458.333,731498.333,731518.333,731538.333,731558.333,731578.333,731598.333,731618.333,731638.333,731658.333,731678.333,731698.333,731718.333,731738.333,731758.333,731778.333,731798.333,731818.333,731838.333,731858.333,731878.333,731898.333,731918.333,731938.333,731958.333,731978.333,731998.333,732018.333,732038.333,732058.333,732078.333,732098.333,732118.333,732138.333,732158.333,732178.333,732198.333,732218.333,732238.333,732258.333,732278.333,732298.333,732318.333,732338.333,732358.333,732378.333,732398.333,732418.333,732438.333,732458.333,732478.333,732498.333,732518.333,732538.333,732558.333,732578.333,732598.333,732618.333,732638.333,732658.333,732678.333,732698.333,732718.333,732738.333,732758.333,732778.333,732798.333,732818.333,732838.333,732858.333,732878.333,732898.333,732918.333,732938.333,732958.333,732978.333,732998.333,733018.333,733038.333,733058.333,733078.333,733098.333,733118.333,733138.333,733158.333,733178.333,733198.333,733218.333,733238.333,733258.333,733278.333,733298.333,733318.333,733338.333,733358.333,733378.333,733398.333,733418.333,733438.333,733458.333,733478.333,733498.333,733518.333,733538.333,733558.333,733578.333,733598.333,733618.333,733638.333,733658.333,733678.333,733698.333,733718.333,733738.333,733758.333,733778.333,733798.333,733818.333,733838.333,733858.333,733878.333,733898.333,733918.333,733938.333,733958.333,733978.333,733998.333,734018.333,734038.333,734058.333,734078.333,734098.333,734118.333,734138.333,734158.333,734178.333,734198.333,734218.333,734238.333,734258.333,734278.333,734298.333,734318.333,734338.333,734358.333,734378.333,734398.333,734418.333,734438.333,734458.333,734478.333,734498.333,734518.333,734538.333,734558.333,734578.333,734598.333,734618.333,734638.333,734658.333,734678.333,734698.333,734718.333,734738.333,734758.333,734778.333,734798.333,734818.333,734838.333,734858.333,734878.333,734898.333,734918.333,734938.333,734958.333,734978.333,734998.333,735018.333,735038.333,735058.333,735078.333,735098.333,735118.333,735138.333,735158.333,735178.333,735198.333,735218.333,735238.333,735258.333,735278.333,735298.333,735318.333,735338.333,735358.333,735378.333,735398.333,735418.333,735438.333,735458.333,735478.333,735498.333,735518.333,735538.333,735558.333,735578.333,735598.333,735618.333,735638.333,735658.333,735678.333,735698.333,735718.333,735738.333,735758.333,735778.333,735798.333,735818.333,735838.333,735858.333,735878.333,735898.333,735918.333,735938.333,735958.333,735978.333,735998.333,736018.333,736038.333,736058.333,736078.333,736098.333,736118.333,736138.333,736158.333,736178.333,736198.333,736218.333,736238.333,736258.333,736278.333,736298.333,736318.333,736338.333,736358.333,736378.333,736398.333,736418.333,736438.333,736458.333,736478.333,736498.333,736518.333,736538.333,736558.333,736578.333,736598.333,736618.333,736638.333,736658.333,736678.333,736698.333,736718.333,736738.333,736758.333,736778.333,736798.333,736818.333,736838.333,736858.333,736878.333,736898.333,736918.333,736938.333,736958.333,736978.333,736998.333,737018.333,737038.333,737058.333,737078.333,737098.333,737118.333,737138.333,737158.333,737178.333,737198.333,737218.333,737238.333,737258.333,737278.333,737298.333,737318.333,737338.333,737358.333,737378.333,737398.333,737418.333,737438.333,737458.333,737478.333,737498.333,737518.333,737538.333,737558.333,737578.333,737598.333,737618.333,737638.333,737658.333,737678.333,737698.333,737718.333,737738.333,737758.333,737778.333,737798.333,737818.333,737838.333,737858.333,737878.333,737898.333,737918.333,737938.333,737958.333,737978.333,737998.333,738018.333,738038.333,738058.333,738078.333,738098.333,738118.333,738138.333,738158.333,738178.333,738198.333,738218.333,738238.333,738258.333,738278.333,738298.333,738318.333,738338.333,738358.333,738378.333,738398.333,738418.333,738438.333,738458.333,738478.333,738498.333,738518.333,738538.333,738558.333,738578.333,738598.333,738618.333,738638.333,738658.333,738678.333,738698.333,738718.333,738738.333,738758.333,738778.333,738798.333,738818.333,738838.333,738858.333,738878.333,738898.333,738918.333,738938.333,738958.333,738978.333,738998.333,739018.333,739038.333,739058.333,739078.333,739098.333,739118.333,739138.333,739158.333,739178.333],"skipped":[328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,595,603,607,609],"target":[[0,120],[696,100],[796,80],[846,60],[1671,80],[1871,60]]} diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 0114eebf5b9..43aaea5d42c 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -317,6 +317,9 @@ export class Consumer { // gap that moved the cursor, or was never read. Mirrors Rust's consumer. While the start // still waits for a group covering its reach, it starts at the lowest that arrived. if (!this.#starting && consumer.sequence < this.#active) { + // Its frames go unobserved, so the estimator learns when they arrived: a join's backlog + // still has to end before the join's intervals count. + this.#jitter.backlog(consumer.arrived.since(Instant.ZERO).asMillis()); consumer.close(); continue; } diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index f5c1403d18e..fd6b5baa084 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -16,6 +16,7 @@ interface Case { arrival: number[]; media: number[]; ends?: number[]; + skipped?: number[]; target: [number, number][]; } @@ -45,6 +46,7 @@ describe("audio jitter conformance", () => { "paced", "reorder", "silence", + "skipped", "spike", "straggler", "tunein", @@ -56,10 +58,12 @@ describe("audio jitter conformance", () => { const jitter = new Jitter(); const expected = new Map(c.target); const ends = new Set(c.ends); + const skipped = new Set(c.skipped); let current: number | undefined; for (let i = 0; i < c.arrival.length; i++) { if (ends.has(i)) jitter.end(); - jitter.observe(c.arrival[i], c.media[i]); + if (skipped.has(i)) jitter.backlog(c.arrival[i]); + else jitter.observe(c.arrival[i], c.media[i]); const value = target(jitter, c); const listed = expected.get(i); if (listed !== undefined) { @@ -103,6 +107,37 @@ describe("Jitter", () => { expect(jitter.measured).toBeGreaterThan(settled); }); + it("leaves the join where it ended once an interval measured the path", () => { + // Every frame of `outage` from its second interval on also reported as a skipped group's + // arrival once observed, as a consumer closing groups below its cursor would. Taken as + // backlog, each would make its interval fold nothing: the prior would linger, and the + // outages that raise the target would go unmeasured. + const outage = cases.find((c) => c.name === "outage") as Case; + const plain = new Jitter(); + const reported = new Jitter(); + for (let i = 0; i < outage.arrival.length; i++) { + plain.observe(outage.arrival[i], outage.media[i]); + reported.observe(outage.arrival[i], outage.media[i]); + if (outage.arrival[i] >= outage.arrival[0] + 500) reported.backlog(outage.arrival[i]); + expect(reported.measured).toBe(plain.measured); + } + }); + + it("ends a join that skips hold open", () => { + // A skip reported ahead of every frame of `outage` keeps each interval measuring the join, but + // only for its first 2 s: the outages after it are still measured. + const outage = cases.find((c) => c.name === "outage") as Case; + const plain = new Jitter(); + const reported = new Jitter(); + for (let i = 0; i < outage.arrival.length; i++) { + plain.observe(outage.arrival[i], outage.media[i]); + reported.backlog(outage.arrival[i]); + reported.observe(outage.arrival[i], outage.media[i]); + } + expect(plain.measured).toBeGreaterThan(new Jitter().measured); + expect(reported.measured).toBe(plain.measured); + }); + it("does not follow the absolute delay", () => { // Only the spread of arrivals matters, not how far away the sender is. const near = new Jitter(); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index ba070ea1ed9..c21a9c6b2e4 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -23,6 +23,13 @@ const RESAMPLE = 500; /** The most weight one observation takes, so a lone one holds the quantile for one half-life at most. */ const FRESH_MAX = 0.1; +/** + * Milliseconds after the first arrival a skipped group can still extend the join. A skip carries no + * timestamp, so a path dropping a late group every interval from the start would otherwise hold the + * join open, and the estimate unmeasured, for as long as it lasts. + */ +const JOIN_MAX = 2000; + /** Idle intervals folded at once, purely to bound the work. */ const IDLE_MAX = 1024; @@ -70,6 +77,8 @@ export class Jitter { #start?: number; /** The latest a frame produced before the subscription arrived, starting at `#start`. */ #joined = 0; + /** Every interval closed so far measured the join, so a skipped backlog can still extend it. */ + #open = true; /** A declared end precedes the next measured observation, so the silence before it is not idle. */ #ended = false; /** The earliest a frame stalled in this interval was due. */ @@ -140,7 +149,10 @@ export class Jitter { } if (this.#joining()) this.#pending += 1; - else this.#close(bucket(this.#peak)); + else { + this.#open = false; + this.#close(bucket(this.#peak)); + } if (resumed) { // A silence the publisher declared says nothing about the path, so the estimate keeps @@ -216,6 +228,18 @@ export class Jitter { this.#stall = undefined; } + /** + * A frame the subscription delivered from before it began, which arrived at `arrival` in + * milliseconds and was skipped unread, so the join lasts until then as if it had been observed. + */ + backlog(arrival: number): void { + // Before any observation the join has not started, and starts later than this. + // Once an interval measured the path the join is over, and a skip is an outage's. + if (this.#start !== undefined && this.#open && arrival < this.#start + JOIN_MAX) { + this.#joined = Math.max(this.#joined, arrival); + } + } + /** * A declared end lies between the previous observation and the next, so the silence it declares * does not relax the estimate the way an unexplained gap does. Call it before observing the first diff --git a/quest/m2/audio-playout/README.md b/quest/m2/audio-playout/README.md index 13321dddbd4..1fa9c79fad7 100644 --- a/quest/m2/audio-playout/README.md +++ b/quest/m2/audio-playout/README.md @@ -97,9 +97,10 @@ Order: the 200 ms snap both ways and the that replaced the NetEq controller and splices under it, are done, and the [clock step](/quest/m2/audio-playout/clock-step.md) builds on the lag rule. The estimator stops folding a join's intervals -([moq-dev/moq#8730](https://github.com/moq-dev/moq/pull/8730)); -[output latency](/quest/m2/audio-playout/output-latency.md) and the -[join's skipped backlog](/quest/m2/audio-playout/join-backlog.md) are independent. +([moq-dev/moq#8730](https://github.com/moq-dev/moq/pull/8730)), including +those behind a backlog the consumers skip unread +([moq-dev/moq#8761](https://github.com/moq-dev/moq/pull/8761)); +[output latency](/quest/m2/audio-playout/output-latency.md) is independent. The [calibration](/quest/m3/audio-calibrate.md) re-measures every ceiling after this epic. diff --git a/quest/m2/audio-playout/join-backlog.md b/quest/m2/audio-playout/join-backlog.md deleted file mode 100644 index af3ef82338d..00000000000 --- a/quest/m2/audio-playout/join-backlog.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] Both consumers report the join's backlog they skip - -## Goal - -The playout estimator learns when a subscription's opening backlog finished -arriving even when the container consumer skips that backlog unread, so a -join whose backlog the consumer drops no longer holds the target above the -steady state. `relay-mic-rtt40`'s replay target p95 falls from 320 ms to the -path's own, in both players. - -## Plan - -Found by [moq-dev/moq#8730](https://github.com/moq-dev/moq/pull/8730), which -made the estimator fold nothing for an interval holding a frame due before the -join's backlog finished arriving (`doc/concept/audio-jitter.md`, "The join's -backlog"). The estimator knows a frame is backlog from its timestamp, so it -only sees the backlog it observes. The browser consumer -(`js/hang/src/container/consumer.ts` `#run`) closes a group below its cursor -as soon as it arrives, unread, and `moq-mux`'s `container::Consumer` discards -groups below its start the same way. In `relay-mic-rtt40`, 267 groups 5 to 6 s -old arrive 1.0 to 1.3 s into the join and delay the live frames behind them by -about 300 ms. The player never observes those groups, so the delay reads as -an ordinary stall, which the cold start folds at `FRESH_MAX`: 320 ms for a 20 s -half-life. The raw trace with every frame observed holds 60 to 100 ms (the -corpus `join` case). - -Recommended: when either consumer drops a group below where delivery started, -it reports that group's arrival to the estimator as backlog (a new -`Jitter.backlog(arrival)` and Rust counterpart that raises `joined`), and still -closes the group unread. A group dropped below a cursor that a later gap moved -is not reported: it is an outage's, which the stall rule owns. Add a corpus -field for frames reported this way instead of observed. - -Rejected: reading and observing every frame of a group below the cursor before -closing it, as the observation point's "before any group can be skipped" -reads. That keeps stale groups open, so it spends the bandwidth closing them -saves, which for video's keyframes is large. diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 5c17c3bcb2e..7206abb4e70 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -648,6 +648,12 @@ impl Consumer { Encoded::Gap { .. } if self.declared.take().is_some() => self.target.end(), _ => {} } + // A group the container dropped unread is never observed, so the estimator + // learns when it arrived instead: a join lasts until its skipped backlog has. + if let Some(skipped) = self.track.skipped() { + self.target + .backlog(jitter::millis(skipped.duration_since(self.origin))); + } // Every packet the container hands over is an arrival, including one the // decoder will refuse, so it is observed before anything can fail. match &read { @@ -2689,6 +2695,91 @@ mod tests { } } + /// Replay a join through the whole decode path, one PCM packet per group written at + /// its arrival, and return the delay once it has settled. Observed backlog opens the + /// join and the live edge queues behind it, then 15 stale groups land 1.0 to 1.3 s + /// in, after the cursor waited out their sequences, and the live packets produced + /// meanwhile queue behind them. Without `delivered`, those groups never reach this + /// node, though the live packets still queued behind them. + async fn join_delay(delivered: bool) -> f64 { + use moq_mux::container::Container as _; + + // `(arrival, group, media)` in ms. + let mut events: Vec<(u64, u64, u64)> = Vec::new(); + for i in 0..=40 { + events.push((10 * i, 100 + i, 90_000 + 20 * i)); + } + if delivered { + // Produced before the subscription, within the transport's budget. + for s in 0..15 { + events.push((1000 + 20 * s, 141 + s, 99_700 + 20 * s)); + } + } + let (mut opening, mut held, mut last) = (0, 0, 0); + for k in 0..300 { + let mut at = 20 * k + 40; + if at <= 400 { + opening += 1; + at = 400 + 2 * opening; + } else if (1000..1300).contains(&at) { + held += 1; + at = 1300 + 2 * held; + } + // A queue drains in order, so nothing overtakes the packets still in it. + at = at.max(last); + last = at; + events.push((at, 156 + k, 100_000 + 20 * k)); + } + events.sort_by_key(|&(at, sequence, _)| (at, sequence)); + + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 8_000, 1); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + // The estimate's 2 s ceiling past the delay, as `moq play` subscribes. + let options = Options { + delay: Delay { + max: Some(Duration::from_secs(2)), + ..Delay::default() + }, + estimate: true, + ..Options::new() + }; + let mut consumer = Consumer::new(&subscriber, &catalog, "audio", options).await.unwrap(); + + let container = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio); + let payload: Bytes = vec![0u8; 160 * size_of::()].into(); + let start = kio::clock::now(); + for (at, sequence, media) in events { + tokio::time::advance(Duration::from_millis(at).saturating_sub(start.elapsed())).await; + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let frame = moq_mux::container::Frame { + timestamp: Timestamp::from_micros(media * 1000).unwrap(), + duration: None, + payload: payload.clone(), + keyframe: true, + }; + container.write(&mut group, &[frame]).unwrap(); + group.finish().unwrap(); + if !(141..156).contains(&sequence) { + next_frame(&mut consumer).await.unwrap().expect("one frame per packet"); + } + } + crate::jitter::millis(consumer.delay()) + } + + /// A join's backlog that arrives below the cursor is dropped unread, yet the target + /// still learns when it arrived: the live packets queued behind it measured the join, + /// not the path, so they fold nothing rather than a 300 ms stall. + #[tokio::test(start_paused = true)] + async fn a_join_backlog_skipped_below_the_cursor_folds_nothing() { + // The path's own 40 ms plus the frame, where an unseen backlog folds the stall. + assert_eq!(join_delay(true).await, 60.0); + assert_eq!(join_delay(false).await, 340.0); + } + /// A group lost between a declared end and what resumes leaves the end declared, as the /// browser's consumer does: the estimator measures the same as when nothing was lost. #[tokio::test] diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs index 54c1e9e6a66..ae0b8bb0425 100644 --- a/rs/moq-audio/src/jitter.rs +++ b/rs/moq-audio/src/jitter.rs @@ -32,6 +32,12 @@ const RESAMPLE: f64 = 500.0; /// The most weight one observation takes, so a lone one holds the quantile for one half-life at most. const FRESH_MAX: f64 = 0.1; +/// Milliseconds after the first arrival a skipped group can still extend the join. A +/// skip carries no timestamp, so a path dropping a late group every interval from the +/// start would otherwise hold the join open, and the estimate unmeasured, for as long +/// as it lasts. +const JOIN_MAX: f64 = 2000.0; + /// Idle intervals folded at once, purely to bound the work. const IDLE_MAX: f64 = 1024.0; @@ -77,6 +83,14 @@ impl Target { } } + /// A frame produced before the subscription, which arrived at `arrival` but was + /// skipped unread. A fixed target ignores it. + pub(crate) fn backlog(&mut self, arrival: f64) { + if let Self::Auto(estimator) = self { + estimator.backlog(arrival); + } + } + /// Set the codec's frame duration, the term added on top of the measurement. pub(crate) fn set_frame(&mut self, frame: f64) { if let Self::Auto(estimator) = self { @@ -290,6 +304,8 @@ pub(crate) struct Estimator { start: Option, /// The latest a frame produced before the subscription arrived, starting at `start`. joined: f64, + /// Every interval closed so far measured the join, so a skipped backlog can still extend it. + open: bool, /// A declared end precedes the next measured observation, so the silence before it is not idle. ended: bool, /// The earliest a frame stalled in this interval was due. @@ -320,6 +336,7 @@ impl Estimator { oldest: 0.0, start: None, joined: 0.0, + open: true, ended: false, due: f64::INFINITY, late: f64::NEG_INFINITY, @@ -385,6 +402,7 @@ impl Estimator { if self.joining() { self.pending += 1.0; } else { + self.open = false; self.close(((self.peak / BUCKET).floor() as usize).min(BUCKETS - 1)); } @@ -470,6 +488,19 @@ impl Estimator { } } + /// A frame produced before the subscription, skipped unread, so the join lasts + /// until it arrived as if it had been observed. + fn backlog(&mut self, arrival: f64) { + // Before any observation the join has not started, and starts later than this. Once + // an interval measured the path the join is over, and a skip is an outage's. + if let Some(start) = self.start + && self.open + && arrival < start + JOIN_MAX + { + self.joined = self.joined.max(arrival); + } + } + /// A declared end lies between the previous observation and the next, so the /// silence it declares does not relax the estimate the way an unexplained gap /// does. Called before observing the first frame after the end, not at the @@ -535,6 +566,9 @@ pub(crate) mod tests { /// Indices of the frames observed right after a declared end. #[serde(default)] pub ends: Vec, + /// Indices of the frames skipped unread, reported by arrival instead of observed. + #[serde(default)] + pub skipped: Vec, pub target: Vec<(usize, f64)>, } @@ -573,7 +607,7 @@ pub(crate) mod tests { /// Every case the document lists. Named rather than globbed, so a case that /// goes missing fails here instead of silently shrinking the suite. - const CORPUS: [&str; 15] = [ + const CORPUS: [&str; 16] = [ "advertised", "backlog", "batches", @@ -586,6 +620,7 @@ pub(crate) mod tests { "paced", "reorder", "silence", + "skipped", "spike", "straggler", "tunein", @@ -607,7 +642,10 @@ pub(crate) mod tests { if case.ends.contains(&index) { target.end(); } - target.observe(arrival, media); + match case.skipped.contains(&index) { + true => target.backlog(arrival), + false => target.observe(arrival, media), + } let actual = target.millis(); assert!( (actual - series[index]).abs() <= 1e-6, @@ -618,6 +656,41 @@ pub(crate) mod tests { } } + /// Every frame of `outage` from its second interval on also reported as a skipped + /// group's arrival once observed, as a consumer closing groups below its cursor + /// would. Taken as backlog, each would make its interval fold nothing: the prior + /// would linger, and the outages that raise the target would go unmeasured. + #[test] + fn leaves_the_join_where_it_ended_once_an_interval_measured_the_path() { + let case = case("outage"); + let mut plain = Target::new(Duration::ZERO, true, None); + let mut reported = Target::new(Duration::ZERO, true, None); + for (&arrival, &media) in case.arrival.iter().zip(&case.media) { + plain.observe(arrival, media); + reported.observe(arrival, media); + if arrival >= case.arrival[0] + RESAMPLE { + reported.backlog(arrival); + } + assert_eq!(reported.millis(), plain.millis()); + } + } + + /// A skip reported ahead of every frame of `outage` keeps each interval measuring + /// the join, but only for its first 2 s: the outages after it are still measured. + #[test] + fn ends_a_join_that_skips_hold_open() { + let case = case("outage"); + let mut plain = Target::new(Duration::ZERO, true, None); + let mut reported = Target::new(Duration::ZERO, true, None); + for (&arrival, &media) in case.arrival.iter().zip(&case.media) { + plain.observe(arrival, media); + reported.backlog(arrival); + reported.observe(arrival, media); + } + assert!(plain.millis() > Target::new(Duration::ZERO, true, None).millis()); + assert_eq!(reported.millis(), plain.millis()); + } + /// The #3517 regression, without the corpus: a stale frame then paced audio far /// ahead of it must not carry the media-time gap into the target. #[test] diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 26b683d8137..5472b0ec320 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -205,6 +205,9 @@ pub struct Consumer { // When the wire frame behind the last frame read landed in this node's cache. arrived: Option, + + // When the latest group dropped without delivering all of it landed. + skipped: Option, } /// Whether a [`Consumer`] still reads its subscription. @@ -419,6 +422,7 @@ impl Consumer { next: None, group: 0, arrived: None, + skipped: None, } } @@ -441,6 +445,16 @@ impl Consumer { self.arrived } + /// When the latest group this reader dropped without delivering all of it landed in this + /// node's cache, or `None` before the first. + /// + /// Its frames were never read, so a caller timing [`arrival`](Self::arrival)s learns when a + /// join's backlog finished arriving even when the reader skipped it: one that arrived below + /// the cursor, or below the start, or that a skip passed over. + pub fn skipped(&self) -> Option { + self.skipped + } + /// The group sequence of the last event read. pub(crate) fn group(&self) -> u64 { self.group @@ -567,7 +581,14 @@ impl Consumer { (floor, served) => floor.or(served), }; if let Some(start) = start { - self.pending.retain(|group| group.sequence >= start); + let skipped = &mut self.skipped; + self.pending.retain(|group| { + let keep = group.sequence >= start; + if !keep { + *skipped = (*skipped).max(Some(group.group.arrived())); + } + keep + }); self.current = start; self.startup = Startup::Started; } @@ -1119,6 +1140,7 @@ impl Consumer { if sequence < self.current { tracing::debug!(old = ?sequence, current = ?self.current, "skipping old group"); + self.skipped = self.skipped.max(Some(reader.group.arrived())); continue; } @@ -1167,15 +1189,18 @@ impl Consumer { fn discard(&mut self, mut pick: impl FnMut(&mut GroupBuffer) -> bool) -> Result<(), F::Error> { let start = self.start; let mut fault = None; + let mut skipped = self.skipped; self.pending.retain_mut(|group| { if !pick(group) { return true; } + skipped = skipped.max(Some(group.group.arrived())); if fault.is_none() { fault = Self::discard_fault(start, group); } false }); + self.skipped = skipped; self.discarded(fault) } @@ -1185,6 +1210,7 @@ impl Consumer { let start = self.start; let mut fault = None; for mut group in self.pending.drain(..count) { + self.skipped = self.skipped.max(Some(group.group.arrived())); if fault.is_none() { fault = Self::discard_fault(start, &mut group); } @@ -1994,6 +2020,29 @@ mod tests { assert_eq!(reader.arrival(), Some(second), "an end carries no arrival of its own"); } + /// A group that arrives below the cursor is dropped unread, and the reader reports when it + /// landed, so a caller timing arrivals knows when the backlog it never saw arrived. + #[tokio::test(start_paused = true)] + async fn a_group_below_the_cursor_reports_when_it_landed() { + let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscriber = track.subscribe( + moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))), + ); + let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Audio)); + + write_group(&mut track, 5, &[ts(100_000)]); + assert!(matches!(reader.read().await.unwrap(), Some(Read::Frame { group: 5, .. }))); + assert_eq!(reader.skipped(), None, "nothing dropped yet"); + + tokio::time::advance(Duration::from_millis(30)).await; + let below = kio::clock::now(); + write_group(&mut track, 2, &[ts(40_000)]); + tokio::time::advance(Duration::from_millis(30)).await; + write_group(&mut track, 6, &[ts(120_000)]); + assert!(matches!(reader.read().await.unwrap(), Some(Read::Frame { group: 6, .. }))); + assert_eq!(reader.skipped(), Some(below)); + } + /// The holes a source left behind when it stopped expire together, a budget after the groups /// past them arrived, rather than one budget each as the cursor reaches them. #[tokio::test(start_paused = true)] diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 30a831652ee..e023f53ced7 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows. Also on 2026-11-13, the estimator began counting one outage, the intervals its stalled frames span and the empty ones before them, as one observation, and every replay row was re-measured on top of the snap: only relay-mic-stall moved. Its single stall no longer raises the target, so its target p95 fell from 1540 to 80 ms and the trail the outage leaves is skipped onto the clock at once rather than through the target's falls: 2 skips a minute where it took 6.1, 5122.3 trimmed ms a minute where it trimmed 4678.2, and its A/V lag p95 from 118.6 to 19.7 ms. With 80 ms of buffer rather than 1540, more of the outage plays as silence: its stalled share rose from 0.067 to 0.092 and silence from 0.085 to 0.092. On 2026-11-14 every player began following the clock by its lag alone: a continuous WSOLA time stretch for any content, at a rate proportional to the lag and capped at 10%, which leaves a lag within half a 10 ms hop alone, and slows down while the audio received ahead of the playhead is about to run out, replacing NetEq's buffer-level decision and pitch-period splices. Every replay row was re-measured. Audio now rests within half a hop of the clock rather than within a pitch period: the A/V lag p50 fell from 10 to 0.2 ms on bursty-aac, whose p95 fell from 49.2 to 0.2, from 1.7 to -1.3 on local-aac, 2.2 to 1.8 on relay-mic and 0.9 to 0.6 on relay-mic-rtt40, and moved from -2 to 2.7 on relay-bbb and 2.2 to 3.1 on relay-mic-stall, each within the half hop the rule leaves alone. The p95 rose from 3.6 to 6.2 ms on relay-bbb, 4.5 to 6.9 on relay-mic, 2.9 to 6.9 on relay-mic-rtt40 and 19.7 to 32.1 on relay-mic-stall, and fell from 15.1 to 14.3 on local-aac: a lag under a hop is corrected only once its pace adds up to one. player_underruns_per_min rose from 4.4 to 6.6 on bursty-aac and 4 to 8.1 on local-aac, whose recorded outages outlast what slowing down at 10% bridges, and fell from 3.1 to 1 on relay-mic-rtt40, whose tap's underruns fell from 0.7 to 0.3 a minute; bursty-aac's silence rose from 0.007 to 0.008. Nothing more is trimmed on any row. The browser and native rows were measured on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row) against main the same day and left to the nightly runner: every browser row's A/V lag p50 now sits between 27 and 39 ms, where the fixed-250 control does on that host, where wide and bursty sat up to 58 ms behind, and every native row's within 15 ms of the clock. Also on 2026-11-14 the estimator stopped folding the intervals a subscription's join spans, those holding a frame due before its backlog finished arriving, and every replay row was re-measured: only relay-mic and relay-mic-stall moved. Their first frame arrives slower than the path after it, so their first interval folds nothing and the target leaves the prior 500 ms later, which moves where audio rests within the half hop the rule leaves alone: relay-mic's A/V lag p50 from 1.8 to -0.3 ms and p95 from 6.9 to 9, and relay-mic-stall's p50 from 3.1 to 1.7. relay-mic-rtt40 did not move: the browser consumer closes its join's stale groups below the cursor unread, so the estimator never sees when that backlog ended, and its target p95 holds at 320 ms.", + "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows. Also on 2026-11-13, the estimator began counting one outage, the intervals its stalled frames span and the empty ones before them, as one observation, and every replay row was re-measured on top of the snap: only relay-mic-stall moved. Its single stall no longer raises the target, so its target p95 fell from 1540 to 80 ms and the trail the outage leaves is skipped onto the clock at once rather than through the target's falls: 2 skips a minute where it took 6.1, 5122.3 trimmed ms a minute where it trimmed 4678.2, and its A/V lag p95 from 118.6 to 19.7 ms. With 80 ms of buffer rather than 1540, more of the outage plays as silence: its stalled share rose from 0.067 to 0.092 and silence from 0.085 to 0.092. On 2026-11-14 every player began following the clock by its lag alone: a continuous WSOLA time stretch for any content, at a rate proportional to the lag and capped at 10%, which leaves a lag within half a 10 ms hop alone, and slows down while the audio received ahead of the playhead is about to run out, replacing NetEq's buffer-level decision and pitch-period splices. Every replay row was re-measured. Audio now rests within half a hop of the clock rather than within a pitch period: the A/V lag p50 fell from 10 to 0.2 ms on bursty-aac, whose p95 fell from 49.2 to 0.2, from 1.7 to -1.3 on local-aac, 2.2 to 1.8 on relay-mic and 0.9 to 0.6 on relay-mic-rtt40, and moved from -2 to 2.7 on relay-bbb and 2.2 to 3.1 on relay-mic-stall, each within the half hop the rule leaves alone. The p95 rose from 3.6 to 6.2 ms on relay-bbb, 4.5 to 6.9 on relay-mic, 2.9 to 6.9 on relay-mic-rtt40 and 19.7 to 32.1 on relay-mic-stall, and fell from 15.1 to 14.3 on local-aac: a lag under a hop is corrected only once its pace adds up to one. player_underruns_per_min rose from 4.4 to 6.6 on bursty-aac and 4 to 8.1 on local-aac, whose recorded outages outlast what slowing down at 10% bridges, and fell from 3.1 to 1 on relay-mic-rtt40, whose tap's underruns fell from 0.7 to 0.3 a minute; bursty-aac's silence rose from 0.007 to 0.008. Nothing more is trimmed on any row. The browser and native rows were measured on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row) against main the same day and left to the nightly runner: every browser row's A/V lag p50 now sits between 27 and 39 ms, where the fixed-250 control does on that host, where wide and bursty sat up to 58 ms behind, and every native row's within 15 ms of the clock. Also on 2026-11-14 the estimator stopped folding the intervals a subscription's join spans, those holding a frame due before its backlog finished arriving, and every replay row was re-measured: only relay-mic and relay-mic-stall moved. Their first frame arrives slower than the path after it, so their first interval folds nothing and the target leaves the prior 500 ms later, which moves where audio rests within the half hop the rule leaves alone: relay-mic's A/V lag p50 from 1.8 to -0.3 ms and p95 from 6.9 to 9, and relay-mic-stall's p50 from 3.1 to 1.7. relay-mic-rtt40 did not move: the browser consumer closes its join's stale groups below the cursor unread, so the estimator never sees when that backlog ended, and its target p95 holds at 320 ms. Later on 2026-11-14 both container consumers began reporting when each group they close unread arrived, which extends a join still open, and every replay row was re-measured: only relay-mic-rtt40 moved. Its 267 stale groups no longer leave the live frames queued behind them to fold as a 300 ms stall, so its target p95 fell from 320 to 120 ms, the 120 it settles at, and the 320 to 80 ms fall it skipped is gone: player_skips_per_min from 0.3 to 0 and trimmed_ms_per_min from 83 to 0. player_underruns_per_min fell from 1 to 0.7, while the tap's underruns rose from 0.3 to 0.7 a minute (0.2 to 0.8 ms), and its A/V lag p50 and p95 rose from 0.6 and 6.9 to 2.3 and 8.6 ms, within the half hop the rule leaves alone, since it holds 200 ms less audio from the join on.", "rows": [ { "runtime": "chromium", @@ -490,16 +490,16 @@ "codec": "opus", "rate": 48000, "profile": "relay-mic-rtt40", - "player_underruns_per_min": 1, - "player_skips_per_min": 0.3, - "underruns_per_min": 0.3, - "underrun_ms_per_min": 0.2, - "trimmed_ms_per_min": 83, + "player_underruns_per_min": 0.7, + "player_skips_per_min": 0, + "underruns_per_min": 0.7, + "underrun_ms_per_min": 0.8, + "trimmed_ms_per_min": 0, "stalled_share": 0, "silence_share": 0, - "target_ms_p95": 320, - "av_lag_ms_p50": 0.6, - "av_lag_ms_p95": 6.9 + "target_ms_p95": 120, + "av_lag_ms_p50": 2.3, + "av_lag_ms_p95": 8.6 }, { "runtime": "replay", From 7d69ec3b590b573df0e18ea4a1f824f802dfca1f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 13:35:28 -0800 Subject: [PATCH 3/8] style: format Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/decode/consumer.rs | 3 +-- rs/moq-mux/src/container/consumer.rs | 10 ++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 7206abb4e70..6c532ec9c8e 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -651,8 +651,7 @@ impl Consumer { // A group the container dropped unread is never observed, so the estimator // learns when it arrived instead: a join lasts until its skipped backlog has. if let Some(skipped) = self.track.skipped() { - self.target - .backlog(jitter::millis(skipped.duration_since(self.origin))); + self.target.backlog(jitter::millis(skipped.duration_since(self.origin))); } // Every packet the container hands over is an arrival, including one the // decoder will refuse, so it is observed before anything can fail. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 5472b0ec320..5bdcb6e71bd 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -2031,7 +2031,10 @@ mod tests { let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Audio)); write_group(&mut track, 5, &[ts(100_000)]); - assert!(matches!(reader.read().await.unwrap(), Some(Read::Frame { group: 5, .. }))); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 5, .. }) + )); assert_eq!(reader.skipped(), None, "nothing dropped yet"); tokio::time::advance(Duration::from_millis(30)).await; @@ -2039,7 +2042,10 @@ mod tests { write_group(&mut track, 2, &[ts(40_000)]); tokio::time::advance(Duration::from_millis(30)).await; write_group(&mut track, 6, &[ts(120_000)]); - assert!(matches!(reader.read().await.unwrap(), Some(Read::Frame { group: 6, .. }))); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 6, .. }) + )); assert_eq!(reader.skipped(), Some(below)); } From b0f206e51053b5725509be9bd58026d012327012 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:52:46 -0800 Subject: [PATCH 4/8] feat(hang,moq-mux,moq-audio): tell the join's backlog by the publisher's largest at subscribe A skipped group counts as the join's backlog only at or below the publisher's largest group as the subscription began, replacing the 2 s JOIN_MAX bound. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 27 ++-- doc/concept/audio-jitter/reference.ts | 7 +- .../src/container/consumer.backlog.test.ts | 123 +++++++++++++++ js/hang/src/container/consumer.ts | 14 +- js/hang/src/container/jitter.test.ts | 15 -- js/hang/src/container/jitter.ts | 11 +- rs/moq-audio/src/decode/consumer.rs | 142 ++++++++++++------ rs/moq-audio/src/jitter.rs | 27 +--- rs/moq-mux/src/container/consumer.rs | 142 ++++++++++++++---- test/audio-quality/budgets.json | 18 +-- 10 files changed, 367 insertions(+), 159 deletions(-) create mode 100644 js/hang/src/container/consumer.backlog.test.ts diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index ea3e0e63da2..0fb976195a1 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -38,9 +38,9 @@ frame: Both sit below the decoder and above nothing: no reordering, no gap handling, no age budget. Except one skip: both consumers close a group below where delivery -stands unread, so its frames are never observed. They report its arrival -instead, which ends a [join](#the-joins-backlog) as late as its observed backlog -would. +stands unread, so its frames are never observed. When it is +[backlog](#the-joins-backlog), they report its arrival instead, which ends the +join as late as its observed backlog would. Natively that holds only as far as the caller lets it. The container consumer reads groups in sequence order and applies its age budget inside `read()`, so a @@ -154,8 +154,8 @@ settle(): # a stall is one observation if a stall is open: fold(stall.bucket); fold(nothing) min(stall.span, IDLE_MAX) times; close it -backlog(arrival): # a group skipped unread arrived - if not first observation and open and arrival < start + JOIN_MAX: +backlog(arrival): # a backlog group skipped unread arrived + if not first observation and open: joined = max(joined, arrival) end(): # a declared end precedes the next observation @@ -329,17 +329,19 @@ is measured from its first flush past the backlog. A first frame slower than the path after it is itself backlog, so most subscriptions skip their first interval, which leaves the prior 500 ms later. -A backlog the consumer skips unread carries no timestamp to test, only its -arrival, which `backlog` reports. It extends the join only while the join is +A group the consumer skips unread carries no timestamp to test, so the consumer +tells backlog by its sequence instead: one at or below the publisher's largest +group when the subscription began, the first largest the subscriber learns (from +moq-lite's subscribe answer, or the first group to land when the publisher +reports none), was produced before it. Only such a group's arrival is reported, +with `backlog`, so a path that drops late groups from the start, all produced +since, holds no join open. A report extends the join only while the join is **open**, before any interval has measured the path: after that, a skipped group is an outage's, whose frames queued behind it belong to the stall rule. On `relay-mic-rtt40` the browser closes 267 groups 5 to 6 s old, arriving 1.0 to 1.3 s in, and the live frames queued behind them are up to 300 ms late; observed -or reported, they fold nothing, where unreported they held the target at 320 ms -for a 20 s half-life. A skip that lands before an interval closes keeps that -interval measuring the join, so a path dropping a late group every interval -would hold the join open for good: a skip extends it only `JOIN_MAX` past the -start, after which the estimator measures what it observes. +or reported, they fold nothing, where unreported they hold the target at 320 ms +for a 20 s half-life. ### The histogram @@ -741,7 +743,6 @@ The exceptions are marked. | `FRESH_MAX` | 0.1 | Ours, twice the quantile's tail. NetEq ramps the forget factor from a start weight of 2 instead; see [the histogram](#the-histogram) | | `PRIOR[i]` | `0.5^(i+1)` | NetEq's histogram seed | | `IDLE_MAX` | 1024 intervals | Ours. A work bound with no effect on output | -| `JOIN_MAX` | 2000 ms | Ours. How long after the first arrival a skipped group can extend the join; see [the join's backlog](#the-joins-backlog) | | `MAX_HISTORY` | 8192 arrivals | Ours. Bounds a rate the publisher chooses; see [Bounding the history](#bounding-the-history) | | Out-of-range delays | clamped, not dropped | Ours. See [the histogram](#the-histogram) | diff --git a/doc/concept/audio-jitter/reference.ts b/doc/concept/audio-jitter/reference.ts index 833526abbee..0829e64021b 100644 --- a/doc/concept/audio-jitter/reference.ts +++ b/doc/concept/audio-jitter/reference.ts @@ -30,9 +30,6 @@ export const FRESH_MAX = 0.1; /** Idle intervals folded at once, purely to bound the work. */ export const IDLE_MAX = 1024; -/** Milliseconds after the first arrival a skipped group can still extend the join. */ -export const JOIN_MAX = 2000; - /** The cold-start distribution, which is also what an idle receiver relaxes back to. */ export function prior(): Float64Array { const out = new Float64Array(BUCKETS); @@ -256,9 +253,7 @@ export class Estimator { backlog(arrival: number): void { // Before any observation the join has not started, and starts later than this. // Once an interval measured the path the join is over, and a skip is an outage's. - if (this.#start !== undefined && this.#open && arrival < this.#start + JOIN_MAX) { - this.#joined = Math.max(this.#joined, arrival); - } + if (this.#start !== undefined && this.#open) this.#joined = Math.max(this.#joined, arrival); } /** A declared end lies between the previous observation and the next. */ diff --git a/js/hang/src/container/consumer.backlog.test.ts b/js/hang/src/container/consumer.backlog.test.ts new file mode 100644 index 00000000000..c49c41506e6 --- /dev/null +++ b/js/hang/src/container/consumer.backlog.test.ts @@ -0,0 +1,123 @@ +// Pairs `rs/moq-audio`'s decode-path tests of the join's backlog: a group the consumer closes below +// its cursor unread still tells the estimator when it arrived, but only when the publisher's largest +// at the subscription's start shows it was produced before. + +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { Duration, type Installed, Instant, now } from "@moq/clock"; +import { Manual } from "@moq/clock/test"; +import { Time, Track, Varint } from "@moq/net"; +import { Consumer } from "./consumer.ts"; +import { Jitter } from "./jitter.ts"; +import { Format } from "./legacy.ts"; + +let clock: Manual; +let installed: Installed; +beforeEach(() => { + clock = new Manual(); + installed = clock.install(); +}); +afterEach(() => { + installed.close(); +}); + +/** `[arrival, group, media]` in ms, in arrival order. */ +type Event = [number, number, number]; + +/** + * Write `events`, one frame per group, reading everything readable as each lands, and return the + * consumer's measured term once they all have, beside the estimator fed the same arrivals with each + * group `backlog` names reported rather than observed. + */ +async function play(events: Event[], backlog: (group: number) => boolean): Promise<[number, number]> { + const track = new Track.Producer("audio"); + const consumer = new Consumer(track.subscribe({ delay: { max: Duration.millis(2000) } }), { + format: new Format("audio"), + delay: { max: Duration.millis(2000) }, + wait: Duration.millis(100), + }); + const oracle = new Jitter(); + const base = now().since(Instant.ZERO).asMillis(); + let read = consumer.next(); + try { + for (const [at, sequence, media] of events) { + await clock.advance(Duration.millis(base + at).saturatingSub(now().since(Instant.ZERO))); + const group = track.createGroup(BigInt(sequence)); + const payload = new Uint8Array([...Varint.encode(media * 1000), 1]); + group.writeFrame({ payload, timestamp: Time.Timestamp.fromMicros(media * 1000) }); + group.finish(); + if (backlog(sequence)) oracle.backlog(base + at); + else oracle.observe(base + at, media); + // Read whatever is ready, so the cursor moves as a player's does. + for (;;) { + const flushed = clock.advance(Duration.ZERO); + const done = await Promise.race([read.then(() => true), flushed.then(() => false)]); + await flushed; + if (!done) break; + read = consumer.next(); + } + } + return [consumer.spread.peek().asMillis(), oracle.measured]; + } finally { + consumer.close(); + track.finish(); + track.close(); + } +} + +/** The live edge, group 200 onward, a frame every 20 ms over a 40 ms path, arrivals moved by `delay` in order. */ +function live(delay: (produced: number) => number): Event[] { + let last = 0; + return Array.from({ length: 300 }, (_, k) => { + const at = Math.max(delay(20 * k), last); + last = at; + return [at, 200 + k, 100_000 + 20 * k]; + }); +} + +const order = (a: Event, b: Event) => a[0] - b[0] || a[1] - b[1]; + +// The publisher stood at group 200 as the subscription began. Its backlog lands behind it, 41 groups +// over the first 400 ms and 15 more 1.0 to 1.3 s in, with the live frames produced meanwhile queued +// behind them. Without `delivered`, that backlog never reaches this node, though the live frames still +// queue behind it. +async function join(delivered: boolean): Promise { + const events = live((produced) => { + const at = produced + 40; + if (at <= 400) return 400 + produced / 10; + if (at >= 1000 && at < 1300) return 1300 + (produced - 960) / 10; + return at; + }); + events[0][0] = 0; + if (delivered) { + for (let i = 0; i < 41; i++) events.push([5 + 10 * i, 100 + i, 90_000 + 20 * i]); + for (let s = 0; s < 15; s++) events.push([1000 + 20 * s, 141 + s, 99_700 + 20 * s]); + } + events.sort(order); + const [measured, oracle] = await play(events, (group) => group < 200); + expect(measured).toBe(oracle); + return measured; +} + +test("a join's backlog skipped below the cursor folds nothing", async () => { + const skipped = await join(true); + const unseen = await join(false); + expect(skipped).toBeLessThanOrEqual(80); + expect(unseen).toBeGreaterThanOrEqual(280); +}); + +test("late groups dropped from the start hold no join open", async () => { + // Every fifth frame arrives 300 ms late from the start: after the cursor passed it, so it is closed + // unread, but it was produced after the subscription began, so it is no backlog. The estimator + // measures the path from what it observes, as if the late frames never arrived. + const late = (group: number) => group > 200 && group % 5 === 0; + const events = live((produced) => produced + 40).map( + ([at, group, media]): Event => [at + (late(group) ? 300 : 0), group, media], + ); + events.sort(order); + const [dropped] = await play(events, () => false); + const [lost] = await play( + events.filter(([, group]) => !late(group)), + () => false, + ); + expect(dropped).toBe(lost); +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 43aaea5d42c..8aaa3d9e96b 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -241,6 +241,9 @@ export class Consumer { readonly buffered: Getter = this.#buffered; #jitter: Jitter; + // The publisher's largest group when the subscription began: one at or below it was produced + // before, so dropping it unread skips the join's backlog. + #joined?: bigint; #spread: Signal; /** * How much buffer late arrivals need, measured as frames come off the transport. @@ -270,6 +273,9 @@ export class Consumer { this.#spread = new Signal(Duration.millis(this.#jitter.measured)); this.spread = this.#spread; + this.#signals.spawn(async () => { + this.#joined = (await this.#track.largest())?.group; + }); this.#signals.spawn(this.#run.bind(this)); this.#signals.cleanup(() => { this.#track.close(); @@ -317,9 +323,11 @@ export class Consumer { // gap that moved the cursor, or was never read. Mirrors Rust's consumer. While the start // still waits for a group covering its reach, it starts at the lowest that arrived. if (!this.#starting && consumer.sequence < this.#active) { - // Its frames go unobserved, so the estimator learns when they arrived: a join's backlog - // still has to end before the join's intervals count. - this.#jitter.backlog(consumer.arrived.since(Instant.ZERO).asMillis()); + // Its frames go unobserved, so the estimator learns when backlog arrived: a join's + // intervals count only once it has. One produced since, such as an outage's, is no backlog. + if (this.#joined !== undefined && consumer.sequence <= this.#joined) { + this.#jitter.backlog(consumer.arrived.since(Instant.ZERO).asMillis()); + } consumer.close(); continue; } diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index fd6b5baa084..77cb4242268 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -123,21 +123,6 @@ describe("Jitter", () => { } }); - it("ends a join that skips hold open", () => { - // A skip reported ahead of every frame of `outage` keeps each interval measuring the join, but - // only for its first 2 s: the outages after it are still measured. - const outage = cases.find((c) => c.name === "outage") as Case; - const plain = new Jitter(); - const reported = new Jitter(); - for (let i = 0; i < outage.arrival.length; i++) { - plain.observe(outage.arrival[i], outage.media[i]); - reported.backlog(outage.arrival[i]); - reported.observe(outage.arrival[i], outage.media[i]); - } - expect(plain.measured).toBeGreaterThan(new Jitter().measured); - expect(reported.measured).toBe(plain.measured); - }); - it("does not follow the absolute delay", () => { // Only the spread of arrivals matters, not how far away the sender is. const near = new Jitter(); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index c21a9c6b2e4..63161c9f74a 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -23,13 +23,6 @@ const RESAMPLE = 500; /** The most weight one observation takes, so a lone one holds the quantile for one half-life at most. */ const FRESH_MAX = 0.1; -/** - * Milliseconds after the first arrival a skipped group can still extend the join. A skip carries no - * timestamp, so a path dropping a late group every interval from the start would otherwise hold the - * join open, and the estimate unmeasured, for as long as it lasts. - */ -const JOIN_MAX = 2000; - /** Idle intervals folded at once, purely to bound the work. */ const IDLE_MAX = 1024; @@ -235,9 +228,7 @@ export class Jitter { backlog(arrival: number): void { // Before any observation the join has not started, and starts later than this. // Once an interval measured the path the join is over, and a skip is an outage's. - if (this.#start !== undefined && this.#open && arrival < this.#start + JOIN_MAX) { - this.#joined = Math.max(this.#joined, arrival); - } + if (this.#start !== undefined && this.#open) this.#joined = Math.max(this.#joined, arrival); } /** diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 6c532ec9c8e..ff9062dfb2a 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -648,10 +648,11 @@ impl Consumer { Encoded::Gap { .. } if self.declared.take().is_some() => self.target.end(), _ => {} } - // A group the container dropped unread is never observed, so the estimator - // learns when it arrived instead: a join lasts until its skipped backlog has. - if let Some(skipped) = self.track.skipped() { - self.target.backlog(jitter::millis(skipped.duration_since(self.origin))); + // A backlog group the container dropped unread is never observed, so the + // estimator learns when it arrived instead: a join lasts until its skipped + // backlog has. + if let Some(backlog) = self.track.backlog() { + self.target.backlog(jitter::millis(backlog.duration_since(self.origin))); } // Every packet the container hands over is an arrival, including one the // decoder will refuse, so it is observed before anything can fail. @@ -2694,43 +2695,13 @@ mod tests { } } - /// Replay a join through the whole decode path, one PCM packet per group written at - /// its arrival, and return the delay once it has settled. Observed backlog opens the - /// join and the live edge queues behind it, then 15 stale groups land 1.0 to 1.3 s - /// in, after the cursor waited out their sequences, and the live packets produced - /// meanwhile queue behind them. Without `delivered`, those groups never reach this - /// node, though the live packets still queued behind them. - async fn join_delay(delivered: bool) -> f64 { + /// Write `events`, `(arrival, group, media)` in ms in arrival order, through the whole + /// decode path, one PCM packet per group, reading each one `read` admits as it lands. + /// Returns the delay once they have all landed, and whether the container reported a + /// backlog. + async fn play(events: &[(u64, u64, u64)], read: impl Fn(u64) -> bool) -> (f64, bool) { use moq_mux::container::Container as _; - // `(arrival, group, media)` in ms. - let mut events: Vec<(u64, u64, u64)> = Vec::new(); - for i in 0..=40 { - events.push((10 * i, 100 + i, 90_000 + 20 * i)); - } - if delivered { - // Produced before the subscription, within the transport's budget. - for s in 0..15 { - events.push((1000 + 20 * s, 141 + s, 99_700 + 20 * s)); - } - } - let (mut opening, mut held, mut last) = (0, 0, 0); - for k in 0..300 { - let mut at = 20 * k + 40; - if at <= 400 { - opening += 1; - at = 400 + 2 * opening; - } else if (1000..1300).contains(&at) { - held += 1; - at = 1300 + 2 * held; - } - // A queue drains in order, so nothing overtakes the packets still in it. - at = at.max(last); - last = at; - events.push((at, 156 + k, 100_000 + 20 * k)); - } - events.sort_by_key(|&(at, sequence, _)| (at, sequence)); - let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 8_000, 1); let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast @@ -2751,7 +2722,7 @@ mod tests { let container = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio); let payload: Bytes = vec![0u8; 160 * size_of::()].into(); let start = kio::clock::now(); - for (at, sequence, media) in events { + for &(at, sequence, media) in events { tokio::time::advance(Duration::from_millis(at).saturating_sub(start.elapsed())).await; let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); let frame = moq_mux::container::Frame { @@ -2762,21 +2733,100 @@ mod tests { }; container.write(&mut group, &[frame]).unwrap(); group.finish().unwrap(); - if !(141..156).contains(&sequence) { + if read(sequence) { next_frame(&mut consumer).await.unwrap().expect("one frame per packet"); } } - crate::jitter::millis(consumer.delay()) + ( + crate::jitter::millis(consumer.delay()), + consumer.track.backlog().is_some(), + ) + } + + /// The live edge, group 200 onward, a packet every 20 ms over a 40 ms path, its arrivals + /// moved by `delay`, which never lets one overtake another. + fn live(delay: impl Fn(u64) -> u64) -> Vec<(u64, u64, u64)> { + let mut last = 0; + (0..300) + .map(|k| { + let at = delay(20 * k).max(last); + last = at; + (at, 200 + k, 100_000 + 20 * k) + }) + .collect() } /// A join's backlog that arrives below the cursor is dropped unread, yet the target /// still learns when it arrived: the live packets queued behind it measured the join, - /// not the path, so they fold nothing rather than a 300 ms stall. + /// not the path, so they fold nothing rather than a 300 ms stall. The publisher stood at + /// group 200 as the subscription began; its backlog lands behind it, 41 groups over the + /// first 400 ms and 15 more 1.0 to 1.3 s in, with the live packets produced meanwhile + /// queued behind them. Without `delivered`, that backlog never reaches this node, though + /// the live packets still queue behind it. + async fn join_delay(delivered: bool) -> f64 { + let mut events = live(|produced| match produced + 40 { + at if at <= 400 => 400 + produced / 10, + at if (1000..1300).contains(&at) => 1300 + (produced - 960) / 10, + at => at, + }); + // Group 200 lands first, so the publisher's largest names it. + events[0].0 = 0; + if delivered { + for i in 0..41 { + events.push((5 + 10 * i, 100 + i, 90_000 + 20 * i)); + } + for s in 0..15 { + events.push((1000 + 20 * s, 141 + s, 99_700 + 20 * s)); + } + } + events.sort_by_key(|&(at, sequence, _)| (at, sequence)); + let (delay, backlog) = play(&events, |sequence| sequence >= 200).await; + assert_eq!(backlog, delivered); + + // The estimator fed the same arrivals directly, each skipped group reported. + let mut target = crate::jitter::Target::new(Duration::ZERO, true, None); + target.set_frame(20.0); + for &(at, sequence, media) in &events { + match sequence >= 200 { + true => target.observe(at as f64, media as f64), + false => target.backlog(at as f64), + } + } + assert_eq!(delay, target.millis()); + delay + } + #[tokio::test(start_paused = true)] async fn a_join_backlog_skipped_below_the_cursor_folds_nothing() { - // The path's own 40 ms plus the frame, where an unseen backlog folds the stall. - assert_eq!(join_delay(true).await, 60.0); - assert_eq!(join_delay(false).await, 340.0); + let skipped = join_delay(true).await; + let unseen = join_delay(false).await; + assert!( + skipped <= 100.0 && unseen >= 300.0, + "{skipped} ms skipped, {unseen} ms unseen" + ); + } + + /// A path that delivers every fifth packet 300 ms late from the start: each lands after + /// the cursor passed it and is dropped unread, but was produced after the subscription + /// began, so it is no backlog and holds no join open. The estimator measures the path + /// from what it observes, as if the late packets never arrived. + #[tokio::test(start_paused = true)] + async fn late_groups_dropped_from_the_start_hold_no_join_open() { + let late = |sequence: u64| sequence > 200 && sequence.is_multiple_of(5); + let mut events: Vec<_> = live(|produced| produced + 40) + .into_iter() + .map(|(at, sequence, media)| (at + if late(sequence) { 300 } else { 0 }, sequence, media)) + .collect(); + events.sort_by_key(|&(at, sequence, _)| (at, sequence)); + let (dropped, backlog) = play(&events, |sequence| !late(sequence)).await; + assert!(!backlog); + let never: Vec<_> = events + .iter() + .copied() + .filter(|&(_, sequence, _)| !late(sequence)) + .collect(); + let (lost, _) = play(&never, |_| true).await; + assert_eq!(dropped, lost); } /// A group lost between a declared end and what resumes leaves the end declared, as the diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs index ae0b8bb0425..dcd3c48210e 100644 --- a/rs/moq-audio/src/jitter.rs +++ b/rs/moq-audio/src/jitter.rs @@ -32,12 +32,6 @@ const RESAMPLE: f64 = 500.0; /// The most weight one observation takes, so a lone one holds the quantile for one half-life at most. const FRESH_MAX: f64 = 0.1; -/// Milliseconds after the first arrival a skipped group can still extend the join. A -/// skip carries no timestamp, so a path dropping a late group every interval from the -/// start would otherwise hold the join open, and the estimate unmeasured, for as long -/// as it lasts. -const JOIN_MAX: f64 = 2000.0; - /// Idle intervals folded at once, purely to bound the work. const IDLE_MAX: f64 = 1024.0; @@ -493,10 +487,7 @@ impl Estimator { fn backlog(&mut self, arrival: f64) { // Before any observation the join has not started, and starts later than this. Once // an interval measured the path the join is over, and a skip is an outage's. - if let Some(start) = self.start - && self.open - && arrival < start + JOIN_MAX - { + if self.start.is_some() && self.open { self.joined = self.joined.max(arrival); } } @@ -675,22 +666,6 @@ pub(crate) mod tests { } } - /// A skip reported ahead of every frame of `outage` keeps each interval measuring - /// the join, but only for its first 2 s: the outages after it are still measured. - #[test] - fn ends_a_join_that_skips_hold_open() { - let case = case("outage"); - let mut plain = Target::new(Duration::ZERO, true, None); - let mut reported = Target::new(Duration::ZERO, true, None); - for (&arrival, &media) in case.arrival.iter().zip(&case.media) { - plain.observe(arrival, media); - reported.backlog(arrival); - reported.observe(arrival, media); - } - assert!(plain.millis() > Target::new(Duration::ZERO, true, None).millis()); - assert_eq!(reported.millis(), plain.millis()); - } - /// The #3517 regression, without the corpus: a stale frame then paced audio far /// ahead of it must not carry the media-time gap into the target. #[test] diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 5bdcb6e71bd..040feec6760 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -206,8 +206,15 @@ pub struct Consumer { // When the wire frame behind the last frame read landed in this node's cache. arrived: Option, - // When the latest group dropped without delivering all of it landed. - skipped: Option, + // The newest group the publisher's largest named so far. + largest: Option, + + // The publisher's largest group when the subscription began: one at or below it was produced + // before, so dropping it unread skips the join's backlog. + joined: Option, + + // When the latest backlog group dropped without delivering all of it landed. + backlog: Option, } /// Whether a [`Consumer`] still reads its subscription. @@ -422,7 +429,9 @@ impl Consumer { next: None, group: 0, arrived: None, - skipped: None, + largest: None, + joined: None, + backlog: None, } } @@ -445,14 +454,16 @@ impl Consumer { self.arrived } - /// When the latest group this reader dropped without delivering all of it landed in this - /// node's cache, or `None` before the first. + /// When the latest backlog group this reader dropped without delivering all of it landed in + /// this node's cache, or `None` before the first. /// - /// Its frames were never read, so a caller timing [`arrival`](Self::arrival)s learns when a - /// join's backlog finished arriving even when the reader skipped it: one that arrived below - /// the cursor, or below the start, or that a skip passed over. - pub fn skipped(&self) -> Option { - self.skipped + /// Backlog is a group at or below the publisher's largest when the subscription began, so it + /// was produced before. Its frames were never read, so a caller timing + /// [`arrival`](Self::arrival)s learns when a join's backlog finished arriving even when the + /// reader skipped it: one that arrived below the cursor, or below the start, or that a skip + /// passed over. + pub fn backlog(&self) -> Option { + self.backlog } /// The group sequence of the last event read. @@ -558,6 +569,11 @@ impl Consumer { } fn poll_event_once(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + // Where the publisher stood as the subscription began, before any group can be dropped + // against it. Polled only until then, so later groups wake nothing extra. + if self.joined.is_none() { + self.poll_largest(waiter); + } // Grab any new groups from the track, recording whether the track is finished. let finished = self.poll_read_finish(waiter).is_ready(); @@ -581,11 +597,11 @@ impl Consumer { (floor, served) => floor.or(served), }; if let Some(start) = start { - let skipped = &mut self.skipped; + let (joined, backlog) = (self.joined, &mut self.backlog); self.pending.retain(|group| { let keep = group.sequence >= start; if !keep { - *skipped = (*skipped).max(Some(group.group.arrived())); + Self::skip(joined, backlog, group); } keep }); @@ -610,11 +626,9 @@ impl Consumer { // budget past them. let start = match matches!(self.startup, Startup::Newest) { true => { - let mut start = self.pending.back().expect("a group has a frame").sequence; - while let Poll::Ready(Some(largest)) = self.subscriber().poll_largest(waiter) { - start = start.max(largest.group); - } - start + self.poll_largest(waiter); + let back = self.pending.back().expect("a group has a frame").sequence; + back.max(self.largest.unwrap_or(back)) } // The start may still wait for a group earlier than any that arrived. false => match self.reach_start(waiter, finished) { @@ -1140,7 +1154,7 @@ impl Consumer { if sequence < self.current { tracing::debug!(old = ?sequence, current = ?self.current, "skipping old group"); - self.skipped = self.skipped.max(Some(reader.group.arrived())); + Self::skip(self.joined, &mut self.backlog, &reader); continue; } @@ -1182,6 +1196,23 @@ impl Consumer { self.memory.full() && self.pending.len() >= 2 } + // Note `group` dropped without delivering all of it: backlog when the publisher's largest at the + // subscription's start reached it. One above was produced since, such as an outage's. + fn skip(joined: Option, backlog: &mut Option, group: &GroupBuffer) { + if joined.is_some_and(|joined| group.sequence <= joined) { + *backlog = (*backlog).max(Some(group.group.arrived())); + } + } + + // Take every largest the publisher reported since the last poll; the first is the one the + // subscription began at. + fn poll_largest(&mut self, waiter: &kio::Waiter) { + while let Poll::Ready(Some(largest)) = self.subscriber().poll_largest(waiter) { + self.joined.get_or_insert(largest.group); + self.largest = self.largest.max(Some(largest.group)); + } + } + // Drop the pending groups `pick` selects without delivering them. A drop must not hide what // was already found, so the lowest one holding a fault, or with frames below the start of the // latest delivered group, ends the reader instead. What was never read stays unread: decoding @@ -1189,18 +1220,17 @@ impl Consumer { fn discard(&mut self, mut pick: impl FnMut(&mut GroupBuffer) -> bool) -> Result<(), F::Error> { let start = self.start; let mut fault = None; - let mut skipped = self.skipped; + let (joined, backlog) = (self.joined, &mut self.backlog); self.pending.retain_mut(|group| { if !pick(group) { return true; } - skipped = skipped.max(Some(group.group.arrived())); + Self::skip(joined, backlog, group); if fault.is_none() { fault = Self::discard_fault(start, group); } false }); - self.skipped = skipped; self.discarded(fault) } @@ -1210,7 +1240,7 @@ impl Consumer { let start = self.start; let mut fault = None; for mut group in self.pending.drain(..count) { - self.skipped = self.skipped.max(Some(group.group.arrived())); + Self::skip(self.joined, &mut self.backlog, &group); if fault.is_none() { fault = Self::discard_fault(start, &mut group); } @@ -2020,33 +2050,83 @@ mod tests { assert_eq!(reader.arrival(), Some(second), "an end carries no arrival of its own"); } - /// A group that arrives below the cursor is dropped unread, and the reader reports when it - /// landed, so a caller timing arrivals knows when the backlog it never saw arrived. + /// A group at or below the publisher's largest when the subscription began was produced + /// before it: dropped unread below the cursor, it is the join's backlog, and the reader + /// reports when it landed. One produced since, such as an outage's, is not. #[tokio::test(start_paused = true)] - async fn a_group_below_the_cursor_reports_when_it_landed() { + async fn a_backlog_group_below_the_cursor_reports_when_it_landed() { let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscriber = track.subscribe( moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))), ); let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Audio)); + reader.set_wait(Duration::from_millis(50)); + // The publisher stood at group 5 as the subscription began. write_group(&mut track, 5, &[ts(100_000)]); assert!(matches!( reader.read().await.unwrap(), Some(Read::Frame { group: 5, .. }) )); - assert_eq!(reader.skipped(), None, "nothing dropped yet"); + write_group(&mut track, 6, &[ts(120_000)]); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 6, .. }) + )); tokio::time::advance(Duration::from_millis(30)).await; - let below = kio::clock::now(); - write_group(&mut track, 2, &[ts(40_000)]); + write_group(&mut track, 4, &[ts(80_000)]); + write_group(&mut track, 7, &[ts(140_000)]); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 7, .. }) + )); + let landed = kio::clock::now(); + assert_eq!(reader.backlog(), Some(landed)); + + // Group 6 itself came after the subscription began, so a copy of it would not be backlog. tokio::time::advance(Duration::from_millis(30)).await; - write_group(&mut track, 6, &[ts(120_000)]); + write_group(&mut track, 8, &[ts(160_000)]); + write_group(&mut track, 10, &[ts(200_000)]); assert!(matches!( reader.read().await.unwrap(), - Some(Read::Frame { group: 6, .. }) + Some(Read::Frame { group: 8, .. }) + )); + assert!(matches!(reader.read().await.unwrap(), Some(Read::Gap { .. }))); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 10, .. }) + )); + write_group(&mut track, 9, &[ts(180_000)]); + write_group(&mut track, 11, &[ts(220_000)]); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 11, .. }) + )); + assert_eq!( + reader.backlog(), + Some(landed), + "group 9 was produced after the subscription" + ); + } + + /// The start drops the backlog below it unread, which still reports when it landed. + #[tokio::test(start_paused = true)] + async fn a_live_start_reports_the_backlog_it_drops() { + let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let landed = kio::clock::now(); + write_group(&mut track, 3, &[ts(0)]); + tokio::time::advance(Duration::from_millis(30)).await; + write_group(&mut track, 4, &[ts(20_000)]); + let subscriber = track.subscribe( + moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))), + ); + let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Audio)).live(); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 4, .. }) )); - assert_eq!(reader.skipped(), Some(below)); + assert_eq!(reader.backlog(), Some(landed)); } /// The holes a source left behind when it stopped expire together, a budget after the groups diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index e023f53ced7..75fad02e381 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows. Also on 2026-11-13, the estimator began counting one outage, the intervals its stalled frames span and the empty ones before them, as one observation, and every replay row was re-measured on top of the snap: only relay-mic-stall moved. Its single stall no longer raises the target, so its target p95 fell from 1540 to 80 ms and the trail the outage leaves is skipped onto the clock at once rather than through the target's falls: 2 skips a minute where it took 6.1, 5122.3 trimmed ms a minute where it trimmed 4678.2, and its A/V lag p95 from 118.6 to 19.7 ms. With 80 ms of buffer rather than 1540, more of the outage plays as silence: its stalled share rose from 0.067 to 0.092 and silence from 0.085 to 0.092. On 2026-11-14 every player began following the clock by its lag alone: a continuous WSOLA time stretch for any content, at a rate proportional to the lag and capped at 10%, which leaves a lag within half a 10 ms hop alone, and slows down while the audio received ahead of the playhead is about to run out, replacing NetEq's buffer-level decision and pitch-period splices. Every replay row was re-measured. Audio now rests within half a hop of the clock rather than within a pitch period: the A/V lag p50 fell from 10 to 0.2 ms on bursty-aac, whose p95 fell from 49.2 to 0.2, from 1.7 to -1.3 on local-aac, 2.2 to 1.8 on relay-mic and 0.9 to 0.6 on relay-mic-rtt40, and moved from -2 to 2.7 on relay-bbb and 2.2 to 3.1 on relay-mic-stall, each within the half hop the rule leaves alone. The p95 rose from 3.6 to 6.2 ms on relay-bbb, 4.5 to 6.9 on relay-mic, 2.9 to 6.9 on relay-mic-rtt40 and 19.7 to 32.1 on relay-mic-stall, and fell from 15.1 to 14.3 on local-aac: a lag under a hop is corrected only once its pace adds up to one. player_underruns_per_min rose from 4.4 to 6.6 on bursty-aac and 4 to 8.1 on local-aac, whose recorded outages outlast what slowing down at 10% bridges, and fell from 3.1 to 1 on relay-mic-rtt40, whose tap's underruns fell from 0.7 to 0.3 a minute; bursty-aac's silence rose from 0.007 to 0.008. Nothing more is trimmed on any row. The browser and native rows were measured on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row) against main the same day and left to the nightly runner: every browser row's A/V lag p50 now sits between 27 and 39 ms, where the fixed-250 control does on that host, where wide and bursty sat up to 58 ms behind, and every native row's within 15 ms of the clock. Also on 2026-11-14 the estimator stopped folding the intervals a subscription's join spans, those holding a frame due before its backlog finished arriving, and every replay row was re-measured: only relay-mic and relay-mic-stall moved. Their first frame arrives slower than the path after it, so their first interval folds nothing and the target leaves the prior 500 ms later, which moves where audio rests within the half hop the rule leaves alone: relay-mic's A/V lag p50 from 1.8 to -0.3 ms and p95 from 6.9 to 9, and relay-mic-stall's p50 from 3.1 to 1.7. relay-mic-rtt40 did not move: the browser consumer closes its join's stale groups below the cursor unread, so the estimator never sees when that backlog ended, and its target p95 holds at 320 ms. Later on 2026-11-14 both container consumers began reporting when each group they close unread arrived, which extends a join still open, and every replay row was re-measured: only relay-mic-rtt40 moved. Its 267 stale groups no longer leave the live frames queued behind them to fold as a 300 ms stall, so its target p95 fell from 320 to 120 ms, the 120 it settles at, and the 320 to 80 ms fall it skipped is gone: player_skips_per_min from 0.3 to 0 and trimmed_ms_per_min from 83 to 0. player_underruns_per_min fell from 1 to 0.7, while the tap's underruns rose from 0.3 to 0.7 a minute (0.2 to 0.8 ms), and its A/V lag p50 and p95 rose from 0.6 and 6.9 to 2.3 and 8.6 ms, within the half hop the rule leaves alone, since it holds 200 ms less audio from the join on.", + "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Later on 2026-11-11 the container consumer began waiting on a missing group only for the delay plus the lookahead, while still delivering groups that arrived until they age past the max delay, and every replay row was re-measured. relay-mic-rtt40's target p95 fell from 780 back to 320 ms, since the holes in its reordered join backlog no longer delay the frames behind them; its tap's missing audio rose from 0.9 to 1.1 ms a minute and its A/V lag p50 and p95 from 0.4 and 1.9 to 0.7 and 2.4 ms. bursty-aac's silence rose from 0.007 to 0.009 and its A/V lag p95 fell from 162.1 to 156.3 ms, and relay-bbb's lag p50 and p95 moved from -1.3 and 2.1 to -1.2 and 2.2 ms. relay-mic-stall did not move. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer delivers a late group that much longer before it ages out, and every replay row was re-measured: only relay-mic moved, its A/V lag p50 and p95 from 0.3 to 1.6 ms, under the 2.5 ms pitch period the stretch can move it by. Holding the constant max delay with the new bound reproduces the old values exactly. On 2026-11-12 the replays stopped playing a constant and began playing a seeded speech-like signal (syllables gliding a 90 to 220 Hz fundamental, with pauses held above the silence classifier), since the stretch's pitch search found only its shortest period in a constant, and the browser ring stopped zero-filling a hole of up to a millisecond and two samples between frames, as native stopped writing silence for one: the recorded AAC timestamps land up to 2.5 samples either side of a 1024-sample boundary, and on speech the 2-sample holes this filled ended 119.1 quanta a minute short on bursty-aac. Every replay row was re-measured. The tap's underruns fell to 0 on bursty-aac (from 3.3) and local-aac (from 2), and relay-mic-rtt40's missing audio from 1.1 to 0.3 ms a minute while its player underruns rose from 2.7 to 3.1. The A/V lag p95 fell from 156.3 to 104.8 ms on bursty-aac, 39.2 to 15.1 on local-aac, 2.2 to 1.4 on relay-bbb and 501.8 to 484.9 on relay-mic-stall, whose p50 fell from 25.3 to 1.1, and rose from 1.6 to 3.4 on relay-mic and 2.4 to 3.5 on relay-mic-rtt40. Where audio rests moved within one speech pitch period, which is as close as one stretch seam can steer it: the p50 is 11 ms on bursty-aac (from 10.4), -4.2 on relay-bbb (from -1.2), 1.1 on relay-mic (from 1.6), 0.9 on relay-mic-rtt40 (from 0.7) and 1.7 on local-aac (from 1.8). On 2026-11-13 the stretch default fell from 2 s to 200 ms and audio trailing the clock past it began skipping onto the clock rather than back to the bound, as video skips, with the audio subscription still reaching 2 s past the delay; every replay row was re-measured. A target fall past 200 ms now costs one skip rather than seconds of catch-up, and so does a trail past it, and a rise past it pads silence until the clock arrives rather than slowing down: relay-mic-stall skips the 1.46 s it trails after the outage and its 1540 to 1020 and 1020 to 80 ms falls, and pads the 0.5 s its 60 to 1540 ms rise leaves it ahead, where it slowed down for 4 s, so it trims 4678.2 ms a minute where it trimmed none and its stalled share rose from 0.05 to 0.067 and silence from 0.049 to 0.085. The frames it holds through the outage then trickle in at twice real time, so its low buffer slows it to 154 ms behind the clock, where it had sat ahead of it: its A/V lag p95 fell from 484.9 to 118.6 ms and its p50 rose from 1.1 to 2.2. relay-mic-rtt40 skips its 320 to 80 ms fall, 0.3 skips and 83.3 trimmed ms a minute, its p95 from 3.5 to 2.9 ms. bursty-aac skips a 225 ms trail 6 s in, 1.1 skips and 312.7 trimmed ms a minute, its p95 from 102.7 to 49.2 ms and p50 from 11 to 10. relay-mic trims nothing, but its tune-in's skips land on the clock rather than at the bound, which moves where it rests within a pitch period: its p50 from 1.1 to 2.2 ms and p95 from 3.4 to 4.5. Trimming at a target fall or a trail past 200 ms is the rule, not a regression. Also on 2026-11-13, for the same change, the congestion rows' trimmed_ms_per_min ceilings were raised by the same rule from two runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row): to 3800 on chromium AAC (worst 2740.3 ms/min), 3700 on chromium Opus (2697.2), 3900 on native AAC (2996.9) and 4600 on native Opus (3532.3), whose player_skips_per_min rose to 15 (9.8). The trim is the outage's audio skipped onto the clock rather than played seconds late: against main on the same host, chromium's A/V lag p95 fell from 1396.8 to 61.6 ms on AAC and 1424.8 to 64.3 on Opus, and native's from 1246.3 to 116.7 and 826.9 to 117.5. These four ceilings are provisional until the nightly re-measures the congestion rows. Also on 2026-11-13, the estimator began counting one outage, the intervals its stalled frames span and the empty ones before them, as one observation, and every replay row was re-measured on top of the snap: only relay-mic-stall moved. Its single stall no longer raises the target, so its target p95 fell from 1540 to 80 ms and the trail the outage leaves is skipped onto the clock at once rather than through the target's falls: 2 skips a minute where it took 6.1, 5122.3 trimmed ms a minute where it trimmed 4678.2, and its A/V lag p95 from 118.6 to 19.7 ms. With 80 ms of buffer rather than 1540, more of the outage plays as silence: its stalled share rose from 0.067 to 0.092 and silence from 0.085 to 0.092. On 2026-11-14 every player began following the clock by its lag alone: a continuous WSOLA time stretch for any content, at a rate proportional to the lag and capped at 10%, which leaves a lag within half a 10 ms hop alone, and slows down while the audio received ahead of the playhead is about to run out, replacing NetEq's buffer-level decision and pitch-period splices. Every replay row was re-measured. Audio now rests within half a hop of the clock rather than within a pitch period: the A/V lag p50 fell from 10 to 0.2 ms on bursty-aac, whose p95 fell from 49.2 to 0.2, from 1.7 to -1.3 on local-aac, 2.2 to 1.8 on relay-mic and 0.9 to 0.6 on relay-mic-rtt40, and moved from -2 to 2.7 on relay-bbb and 2.2 to 3.1 on relay-mic-stall, each within the half hop the rule leaves alone. The p95 rose from 3.6 to 6.2 ms on relay-bbb, 4.5 to 6.9 on relay-mic, 2.9 to 6.9 on relay-mic-rtt40 and 19.7 to 32.1 on relay-mic-stall, and fell from 15.1 to 14.3 on local-aac: a lag under a hop is corrected only once its pace adds up to one. player_underruns_per_min rose from 4.4 to 6.6 on bursty-aac and 4 to 8.1 on local-aac, whose recorded outages outlast what slowing down at 10% bridges, and fell from 3.1 to 1 on relay-mic-rtt40, whose tap's underruns fell from 0.7 to 0.3 a minute; bursty-aac's silence rose from 0.007 to 0.008. Nothing more is trimmed on any row. The browser and native rows were measured on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row) against main the same day and left to the nightly runner: every browser row's A/V lag p50 now sits between 27 and 39 ms, where the fixed-250 control does on that host, where wide and bursty sat up to 58 ms behind, and every native row's within 15 ms of the clock. Also on 2026-11-14 the estimator stopped folding the intervals a subscription's join spans, those holding a frame due before its backlog finished arriving, and every replay row was re-measured: only relay-mic and relay-mic-stall moved. Their first frame arrives slower than the path after it, so their first interval folds nothing and the target leaves the prior 500 ms later, which moves where audio rests within the half hop the rule leaves alone: relay-mic's A/V lag p50 from 1.8 to -0.3 ms and p95 from 6.9 to 9, and relay-mic-stall's p50 from 3.1 to 1.7. relay-mic-rtt40 did not move: the browser consumer closes its join's stale groups below the cursor unread, so the estimator never sees when that backlog ended, and its target p95 holds at 320 ms. Later on 2026-11-14 both container consumers began reporting when each group they close unread arrived, if it is at or below the publisher's largest group as the subscription began, which extends a join still open. The replay's track learns its largest from the first group to land, relay-mic-rtt40's stale tune-in frame, below the backlog it closes, so no replay row moved; with that backlog reported, its target p95 falls from 320 to 120 ms, which a recording carrying the subscribe answer's largest would measure.", "rows": [ { "runtime": "chromium", @@ -490,16 +490,16 @@ "codec": "opus", "rate": 48000, "profile": "relay-mic-rtt40", - "player_underruns_per_min": 0.7, - "player_skips_per_min": 0, - "underruns_per_min": 0.7, - "underrun_ms_per_min": 0.8, - "trimmed_ms_per_min": 0, + "player_underruns_per_min": 1, + "player_skips_per_min": 0.3, + "underruns_per_min": 0.3, + "underrun_ms_per_min": 0.2, + "trimmed_ms_per_min": 83, "stalled_share": 0, "silence_share": 0, - "target_ms_p95": 120, - "av_lag_ms_p50": 2.3, - "av_lag_ms_p95": 8.6 + "target_ms_p95": 320, + "av_lag_ms_p50": 0.6, + "av_lag_ms_p95": 6.9 }, { "runtime": "replay", From 13c43691e33257321215a2e84c0aec0e37d01ed7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 18:06:06 -0800 Subject: [PATCH 5/8] fix(hang,moq-mux): take the join's boundary once the publisher answered Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 7 ++- .../src/container/consumer.backlog.test.ts | 57 ++++++++++++++++++- js/hang/src/container/consumer.ts | 6 +- rs/moq-mux/src/container/consumer.rs | 22 +++---- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 0fb976195a1..7a632d13a13 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -331,9 +331,10 @@ which leaves the prior 500 ms later. A group the consumer skips unread carries no timestamp to test, so the consumer tells backlog by its sequence instead: one at or below the publisher's largest -group when the subscription began, the first largest the subscriber learns (from -moq-lite's subscribe answer, or the first group to land when the publisher -reports none), was produced before it. Only such a group's arrival is reported, +group when the subscription began, the largest the subscriber knows once the +publisher answered (moq-lite's subscribe answer carries it, and without one the +groups that landed stand in), was produced before it. Read any earlier, a cached +group racing the answer would set it. Only such a group's arrival is reported, with `backlog`, so a path that drops late groups from the start, all produced since, holds no join open. A report extends the join only while the join is **open**, before any interval has measured the path: after that, a skipped group diff --git a/js/hang/src/container/consumer.backlog.test.ts b/js/hang/src/container/consumer.backlog.test.ts index c49c41506e6..fd7c12a970a 100644 --- a/js/hang/src/container/consumer.backlog.test.ts +++ b/js/hang/src/container/consumer.backlog.test.ts @@ -2,7 +2,7 @@ // its cursor unread still tells the estimator when it arrived, but only when the publisher's largest // at the subscription's start shows it was produced before. -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; import { Duration, type Installed, Instant, now } from "@moq/clock"; import { Manual } from "@moq/clock/test"; import { Time, Track, Varint } from "@moq/net"; @@ -121,3 +121,58 @@ test("late groups dropped from the start hold no join open", async () => { ); expect(dropped).toBe(lost); }); + +test("the join's boundary is the publisher's answer, not a group that raced it", async () => { + // Group 150 lands from a cache before the publisher answers that it stood at group 200. Group 160 + // then arrives below the cursor: produced before the subscription, so it is backlog. + const track = new Track.Producer("audio"); + const subscriber = track.subscribe({ delay: { max: Duration.millis(2000) } }); + let answered = false; + let answer!: () => void; + const started = new Promise((resolve) => { + answer = resolve; + }); + const raced = new Proxy(subscriber, { + get(target, key) { + if (key === "start") return () => started.then(() => undefined); + if (key === "largest") return async () => ({ group: answered ? 200n : 150n, frame: 0 }); + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const reported: number[] = []; + const backlog = spyOn(Jitter.prototype, "backlog").mockImplementation((arrival: number) => { + reported.push(arrival); + }); + const consumer = new Consumer(raced, { + format: new Format("audio"), + delay: { max: Duration.millis(2000) }, + wait: Duration.millis(100), + }); + const write = (sequence: number, media: number) => { + const group = track.createGroup(BigInt(sequence)); + const payload = new Uint8Array([...Varint.encode(media * 1000), 1]); + group.writeFrame({ payload, timestamp: Time.Timestamp.fromMicros(media * 1000) }); + group.finish(); + }; + try { + write(150, 0); + await clock.advance(Duration.ZERO); + answered = true; + answer(); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 150n }); + write(201, 1020); + await clock.advance(Duration.millis(200)); + expect(await consumer.next()).toMatchObject({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 201n }); + write(160, 200); + write(202, 1040); + await clock.advance(Duration.ZERO); + expect(reported).toHaveLength(1); + } finally { + backlog.mockRestore(); + consumer.close(); + track.finish(); + track.close(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 8aaa3d9e96b..496653f4813 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -273,9 +273,6 @@ export class Consumer { this.#spread = new Signal(Duration.millis(this.#jitter.measured)); this.spread = this.#spread; - this.#signals.spawn(async () => { - this.#joined = (await this.#track.largest())?.group; - }); this.#signals.spawn(this.#run.bind(this)); this.#signals.cleanup(() => { this.#track.close(); @@ -316,6 +313,9 @@ export class Consumer { const floor = this.#track.floor(); const served = await this.#track.start(); const start = served !== undefined && (floor === undefined || served > floor) ? served : floor; + // Read once the start resolved, so the publisher's answer is in it rather than a group + // that raced it here. A group has arrived, so the largest already covers it. + this.#joined = (await this.#track.largest())?.group; this.#active = this.#startAt(start) ?? consumer.sequence; } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 040feec6760..d6defd44fa1 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -209,8 +209,8 @@ pub struct Consumer { // The newest group the publisher's largest named so far. largest: Option, - // The publisher's largest group when the subscription began: one at or below it was produced - // before, so dropping it unread skips the join's backlog. + // The publisher's largest group as the start was decided: one at or below it was produced + // before the subscription, so dropping it unread skips the join's backlog. joined: Option, // When the latest backlog group dropped without delivering all of it landed. @@ -569,11 +569,6 @@ impl Consumer { } fn poll_event_once(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { - // Where the publisher stood as the subscription began, before any group can be dropped - // against it. Polled only until then, so later groups wake nothing extra. - if self.joined.is_none() { - self.poll_largest(waiter); - } // Grab any new groups from the track, recording whether the track is finished. let finished = self.poll_read_finish(waiter).is_ready(); @@ -597,6 +592,7 @@ impl Consumer { (floor, served) => floor.or(served), }; if let Some(start) = start { + self.join(waiter); let (joined, backlog) = (self.joined, &mut self.backlog); self.pending.retain(|group| { let keep = group.sequence >= start; @@ -636,6 +632,7 @@ impl Consumer { None => return Poll::Pending, }, }; + self.join(waiter); // Nothing has been read, so discarding what lies below the start is no gap. self.discard(|group| group.sequence < start)?; self.current = start; @@ -1204,15 +1201,20 @@ impl Consumer { } } - // Take every largest the publisher reported since the last poll; the first is the one the - // subscription began at. + // Take every largest the publisher reported since the last poll. fn poll_largest(&mut self, waiter: &kio::Waiter) { while let Poll::Ready(Some(largest)) = self.subscriber().poll_largest(waiter) { - self.joined.get_or_insert(largest.group); self.largest = self.largest.max(Some(largest.group)); } } + // Where the publisher stood as the subscription began, taken as the start is decided: after its + // answer rather than from a group that raced it, and before anything is dropped against it. + fn join(&mut self, waiter: &kio::Waiter) { + self.poll_largest(waiter); + self.joined = self.largest; + } + // Drop the pending groups `pick` selects without delivering them. A drop must not hide what // was already found, so the lowest one holding a fault, or with frames below the start of the // latest delivered group, ends the reader instead. What was never read stays unread: decoding From 25d2770229780bb193359f94e297e4eab706f153 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 18:12:32 -0800 Subject: [PATCH 6/8] fix(hang,moq-mux): read the join's boundary off the delivery path Co-Authored-By: Claude Opus 5.5 --- .../src/container/consumer.backlog.test.ts | 24 +++++++++++++++++++ js/hang/src/container/consumer.ts | 7 ++++-- rs/moq-mux/src/container/consumer.rs | 4 ++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/js/hang/src/container/consumer.backlog.test.ts b/js/hang/src/container/consumer.backlog.test.ts index fd7c12a970a..88aa68fc1ff 100644 --- a/js/hang/src/container/consumer.backlog.test.ts +++ b/js/hang/src/container/consumer.backlog.test.ts @@ -176,3 +176,27 @@ test("the join's boundary is the publisher's answer, not a group that raced it", track.close(); } }); + +test("a range capped below the largest still delivers", async () => { + // The track holds groups 0 to 5, and the subscriber's range ends before 3, so it never reports a + // largest: the consumer reports no backlog, but delivers what it can read. + const track = new Track.Producer("audio"); + for (let sequence = 0; sequence <= 5; sequence++) { + const group = track.createGroup(BigInt(sequence)); + const media = sequence * 20; + group.writeFrame({ + payload: new Uint8Array([...Varint.encode(media * 1000), 1]), + timestamp: Time.Timestamp.fromMicros(media * 1000), + }); + group.finish(); + } + const subscriber = track.subscribe({ delay: { max: Duration.millis(2000) } }).withRange({ end: { excluded: 3n } }); + const consumer = new Consumer(subscriber, { format: new Format("audio"), delay: { max: Duration.millis(2000) } }); + try { + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 0n }); + } finally { + consumer.close(); + track.finish(); + track.close(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 496653f4813..68556c313b6 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -314,8 +314,11 @@ export class Consumer { const served = await this.#track.start(); const start = served !== undefined && (floor === undefined || served > floor) ? served : floor; // Read once the start resolved, so the publisher's answer is in it rather than a group - // that raced it here. A group has arrived, so the largest already covers it. - this.#joined = (await this.#track.largest())?.group; + // that raced it here, and off the delivery path: a range capped below the largest never + // reports one, which reports no backlog rather than stalling delivery. + this.#signals.spawn(async () => { + this.#joined = (await this.#track.largest())?.group; + }); this.#active = this.#startAt(start) ?? consumer.sequence; } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index d6defd44fa1..a76237ba631 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -569,6 +569,10 @@ impl Consumer { } fn poll_event_once(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + // A start decided before the largest was in range takes the first one that is. + if self.joined.is_none() && matches!(self.startup, Startup::Started) { + self.join(waiter); + } // Grab any new groups from the track, recording whether the track is finished. let finished = self.poll_read_finish(waiter).is_ready(); From 80c0ed66ef1c2e8e5d0909ab3e0cba5d72228f1d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 20:59:59 -0800 Subject: [PATCH 7/8] fix(hang,moq-mux): read the join's boundary as the publisher answers Live groups landing between the answer and the first group raised the largest past where the publisher stood, so a late live group counted as backlog. Both consumers now take it once the start resolves. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 5 +- .../src/container/consumer.backlog.test.ts | 54 ++++++++++++++++++ js/hang/src/container/consumer.ts | 14 +++-- rs/moq-mux/src/container/consumer.rs | 56 +++++++++++++++++-- 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 7a632d13a13..5dd9aa962cc 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -333,8 +333,9 @@ A group the consumer skips unread carries no timestamp to test, so the consumer tells backlog by its sequence instead: one at or below the publisher's largest group when the subscription began, the largest the subscriber knows once the publisher answered (moq-lite's subscribe answer carries it, and without one the -groups that landed stand in), was produced before it. Read any earlier, a cached -group racing the answer would set it. Only such a group's arrival is reported, +groups that landed stand in), was produced before it. It is read as the answer +arrives, not when the first group does: any earlier, a cached group racing the +answer would set it, and any later, live groups landing meanwhile would. Only such a group's arrival is reported, with `backlog`, so a path that drops late groups from the start, all produced since, holds no join open. A report extends the join only while the join is **open**, before any interval has measured the path: after that, a skipped group diff --git a/js/hang/src/container/consumer.backlog.test.ts b/js/hang/src/container/consumer.backlog.test.ts index 88aa68fc1ff..293c9c0fbe4 100644 --- a/js/hang/src/container/consumer.backlog.test.ts +++ b/js/hang/src/container/consumer.backlog.test.ts @@ -177,6 +177,60 @@ test("the join's boundary is the publisher's answer, not a group that raced it", } }); +test("the join's boundary is the publisher's answer, not the largest by the first group", async () => { + // The publisher answers that it stood at group 200, then groups up to 210 land before the first + // reaches this consumer. Group 205 then arrives below the cursor: produced after the subscription + // began, so it is no backlog, unlike group 160. + const track = new Track.Producer("audio"); + const subscriber = track.subscribe({ delay: { max: Duration.millis(2000) } }); + let advanced = false; + let answer!: () => void; + const started = new Promise((resolve) => { + answer = resolve; + }); + const raced = new Proxy(subscriber, { + get(target, key) { + if (key === "start") return () => started.then(() => undefined); + if (key === "largest") return async () => ({ group: advanced ? 210n : 200n, frame: 0 }); + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const reported: number[] = []; + const backlog = spyOn(Jitter.prototype, "backlog").mockImplementation((arrival: number) => { + reported.push(arrival); + }); + const consumer = new Consumer(raced, { + format: new Format("audio"), + delay: { max: Duration.millis(2000) }, + wait: Duration.millis(100), + }); + const write = (sequence: number, media: number) => { + const group = track.createGroup(BigInt(sequence)); + const payload = new Uint8Array([...Varint.encode(media * 1000), 1]); + group.writeFrame({ payload, timestamp: Time.Timestamp.fromMicros(media * 1000) }); + group.finish(); + }; + try { + answer(); + await clock.advance(Duration.ZERO); + advanced = true; + write(210, 200); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 210n }); + write(205, 100); + await clock.advance(Duration.ZERO); + expect(reported).toHaveLength(0); + write(160, 0); + await clock.advance(Duration.ZERO); + expect(reported).toHaveLength(1); + } finally { + backlog.mockRestore(); + consumer.close(); + track.finish(); + track.close(); + } +}); + test("a range capped below the largest still delivers", async () => { // The track holds groups 0 to 5, and the subscriber's range ends before 3, so it never reports a // largest: the consumer reports no backlog, but delivers what it can read. diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 68556c313b6..1fe6ed8156c 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -273,6 +273,14 @@ export class Consumer { this.#spread = new Signal(Duration.millis(this.#jitter.measured)); this.spread = this.#spread; + // Where the publisher stood as the subscription began, read as soon as its answer resolves the + // start: after it rather than from a cached group that raced it, and before live groups landing + // later raise the largest past it. Off the delivery path: a range capped below the largest never + // reports one, which reports no backlog rather than stalling delivery. + this.#signals.spawn(async () => { + await this.#track.start(); + this.#joined = (await this.#track.largest())?.group; + }); this.#signals.spawn(this.#run.bind(this)); this.#signals.cleanup(() => { this.#track.close(); @@ -313,12 +321,6 @@ export class Consumer { const floor = this.#track.floor(); const served = await this.#track.start(); const start = served !== undefined && (floor === undefined || served > floor) ? served : floor; - // Read once the start resolved, so the publisher's answer is in it rather than a group - // that raced it here, and off the delivery path: a range capped below the largest never - // reports one, which reports no backlog rather than stalling delivery. - this.#signals.spawn(async () => { - this.#joined = (await this.#track.largest())?.group; - }); this.#active = this.#startAt(start) ?? consumer.sequence; } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index a76237ba631..0cc1c909f1d 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -569,8 +569,12 @@ impl Consumer { } fn poll_event_once(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { - // A start decided before the largest was in range takes the first one that is. - if self.joined.is_none() && matches!(self.startup, Startup::Started) { + // Where the publisher stood as the subscription began, read as soon as its answer resolves + // the start: groups landing later raise the largest past it. A start decided first, or a + // largest not yet in range, takes the first one that is. + if self.joined.is_none() + && (matches!(self.startup, Startup::Started) || self.subscriber().poll_start(waiter).is_ready()) + { self.join(waiter); } // Grab any new groups from the track, recording whether the track is finished. @@ -1212,11 +1216,14 @@ impl Consumer { } } - // Where the publisher stood as the subscription began, taken as the start is decided: after its - // answer rather than from a group that raced it, and before anything is dropped against it. + // Where the publisher stood as the subscription began, taken once: after its answer rather than + // from a group that raced it, and at the latest as the start is decided, before anything is + // dropped against it. fn join(&mut self, waiter: &kio::Waiter) { self.poll_largest(waiter); - self.joined = self.largest; + if self.joined.is_none() { + self.joined = self.largest; + } } // Drop the pending groups `pick` selects without delivering them. A drop must not hide what @@ -2135,6 +2142,45 @@ mod tests { assert_eq!(reader.backlog(), Some(landed)); } + /// The join's boundary is where the publisher stood as it answered, not the newer largest by the + /// time a live start found a frame to start on. + #[tokio::test(start_paused = true)] + async fn the_join_is_the_largest_at_the_answer_not_at_the_start() { + let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let landed = kio::clock::now(); + let mut open = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + let subscriber = track.subscribe( + moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))), + ); + let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Audio)).live(); + // Group 3 holds no frame yet, so the start waits, though the publisher answered. + assert!( + kio::clock::timeout(Duration::from_millis(1), reader.read()) + .await + .is_err() + ); + + // Groups 4 and 5 were produced after the subscription began, so they are no backlog. + tokio::time::advance(Duration::from_millis(30)).await; + write_group(&mut track, 4, &[ts(20_000)]); + write_group(&mut track, 5, &[ts(40_000)]); + let frame = Frame { + timestamp: ts(0), + payload: Bytes::from_static(&[0xDE, 0xAD]), + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Audio) + .write(&mut open, &[frame]) + .unwrap(); + open.finish().unwrap(); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 5, .. }) + )); + assert_eq!(reader.backlog(), Some(landed), "group 4 was produced after the answer"); + } + /// The holes a source left behind when it stopped expire together, a budget after the groups /// past them arrived, rather than one budget each as the cursor reaches them. #[tokio::test(start_paused = true)] From 0dbae22ccab19f0dc74f83a325e6169d9e9fd64b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 21:08:31 -0800 Subject: [PATCH 8/8] fix(hang): read the join's boundary once the subscription is established start() can resolve before the wire layer has a served state, so wait for the track's info or its first group first, as start() asks. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.backlog.test.ts | 16 +++++++++++----- js/hang/src/container/consumer.ts | 9 +++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/js/hang/src/container/consumer.backlog.test.ts b/js/hang/src/container/consumer.backlog.test.ts index 293c9c0fbe4..39a430f245a 100644 --- a/js/hang/src/container/consumer.backlog.test.ts +++ b/js/hang/src/container/consumer.backlog.test.ts @@ -135,6 +135,7 @@ test("the join's boundary is the publisher's answer, not a group that raced it", const raced = new Proxy(subscriber, { get(target, key) { if (key === "start") return () => started.then(() => undefined); + if (key === "info") return () => started.then(() => ({})); if (key === "largest") return async () => ({ group: answered ? 200n : 150n, frame: 0 }); const value = Reflect.get(target, key, target); return typeof value === "function" ? value.bind(target) : value; @@ -178,11 +179,13 @@ test("the join's boundary is the publisher's answer, not a group that raced it", }); test("the join's boundary is the publisher's answer, not the largest by the first group", async () => { - // The publisher answers that it stood at group 200, then groups up to 210 land before the first - // reaches this consumer. Group 205 then arrives below the cursor: produced after the subscription - // began, so it is no backlog, unlike group 160. + // The start resolves before the subscription is established, while the largest still names cached + // group 150. The publisher answers that it stood at group 200, then groups up to 210 land before + // the first reaches this consumer. Group 205 then arrives below the cursor: produced after the + // subscription began, so it is no backlog, unlike group 160. const track = new Track.Producer("audio"); const subscriber = track.subscribe({ delay: { max: Duration.millis(2000) } }); + let answered = false; let advanced = false; let answer!: () => void; const started = new Promise((resolve) => { @@ -190,8 +193,9 @@ test("the join's boundary is the publisher's answer, not the largest by the firs }); const raced = new Proxy(subscriber, { get(target, key) { - if (key === "start") return () => started.then(() => undefined); - if (key === "largest") return async () => ({ group: advanced ? 210n : 200n, frame: 0 }); + if (key === "start") return async () => undefined; + if (key === "info") return () => started.then(() => ({})); + if (key === "largest") return async () => ({ group: advanced ? 210n : answered ? 200n : 150n, frame: 0 }); const value = Reflect.get(target, key, target); return typeof value === "function" ? value.bind(target) : value; }, @@ -212,6 +216,8 @@ test("the join's boundary is the publisher's answer, not the largest by the firs group.finish(); }; try { + await clock.advance(Duration.ZERO); + answered = true; answer(); await clock.advance(Duration.ZERO); advanced = true; diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 1fe6ed8156c..176e4aa29bd 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -1,6 +1,6 @@ import { Duration, Instant, now } from "@moq/clock"; import type * as Moq from "@moq/net"; -import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq/signals"; +import { Effect, type Getter, type GetterInit, getter, Once, race, Signal } from "@moq/signals"; import type { Format } from "./format"; import { Jitter } from "./jitter"; import type { BufferedRanges, Frame, ReceivedFrame } from "./types"; @@ -260,6 +260,8 @@ export class Consumer { #signals = new Effect(); #closed = new Once(); + // Set once the first group arrived, which establishes the subscription like its info. + #tuned = new Once(); /** Start consuming the given track, decoding frames with `props.format`. */ constructor(track: Moq.Track.Subscriber, props: ConsumerProps) { @@ -275,9 +277,11 @@ export class Consumer { // Where the publisher stood as the subscription began, read as soon as its answer resolves the // start: after it rather than from a cached group that raced it, and before live groups landing - // later raise the largest past it. Off the delivery path: a range capped below the largest never + // later raise the largest past it. The start says only once the info or a first group shows the + // subscription established. Off the delivery path: a range capped below the largest never // reports one, which reports no backlog rather than stalling delivery. this.#signals.spawn(async () => { + await race([this.#track.info().catch(() => undefined), this.#tuned]); await this.#track.start(); this.#joined = (await this.#track.largest())?.group; }); @@ -318,6 +322,7 @@ export class Consumer { // newest first, so the first arrival is not the oldest served; without either it stands // in. Mirrors Rust's `Consumer::poll_read`. if (this.#active === undefined) { + this.#tuned.set(true); const floor = this.#track.floor(); const served = await this.#track.start(); const start = served !== undefined && (floor === undefined || served > floor) ? served : floor;