From 8aa608b91e7e7f62e05e6d3e7163ec5eb3218f4c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 22:12:13 -0700 Subject: [PATCH 1/5] quest(m1): claim bufferbloat From 11f4bc70da091e2e7c7a5344819ca5458881aff6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 23:00:24 -0700 Subject: [PATCH 2/5] test(ts): judge the export on a congested path, not random loss `just test ts` gains --rate and --queue: moq-shaper's token bucket on the subscriber's leg behind a 2 s queue. Under --rate the export must stay T-STD compliant and continuous, shed only whole groups (holes.py --shed, matched by PTS), and still deliver the last group. The nightly --hrd and --passthrough lossy steps move to --rate 7000000; --loss stays for correctness. The relay drills, moq-quic's sim and BBR3's unit tests prove correctness and keep their random loss. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 19 +-- quest/m1/README.md | 1 - quest/m1/bufferbloat.md | 53 ------- quest/m1/random-loss-audit.md | 13 +- quest/m1/ts-reliability/grader-determinism.md | 2 +- rs/moq-relay/tests/drills.rs | 2 + test/drill/README.md | 9 ++ test/ts/README.md | 131 +++++++++++------- test/ts/holes.py | 122 ++++++++++++---- test/ts/holes.test.py | 36 ++++- test/ts/run.sh | 130 +++++++++++------ 11 files changed, 327 insertions(+), 191 deletions(-) delete mode 100644 quest/m1/bufferbloat.md diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 2e59dcbd686..05ba1742a3a 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -19,7 +19,7 @@ on: - "rs/moq-relay/**" - "rs/moq-loc/**" - "rs/moq-json/**" - # The lossy-path step's transport and shaper. + # The congested-path steps' transport and shaper. - "rs/moq-quic/**" - "rs/moq-shaper/**" # Workspace dependencies of moq-net, moq-tokio and moq-mux on the TS path. @@ -209,19 +209,20 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - # 1% random loss and 25 ms each way on the subscriber's leg (moq-dev/moq#4613), through - # moq-shaper: QUIC retransmission must keep the export T-STD compliant with no media lost. - # `--hrd`, since `--headroom` still loses audio just after the join at 1%. - - name: TS lossy path + # The subscriber's leg congested the way real paths are, through moq-shaper: 7 Mb/s under + # the 9 Mb/s video, behind a 2 s queue, 25 ms each way. The export must stay T-STD compliant + # and continuous, shed only whole groups, and keep its latency, so the last group arrives. + - name: TS congested path if: ${{ !cancelled() && github.event_name != 'pull_request' }} - run: nix develop --command just test ts --hrd --loss 1 --duration 120 + run: nix develop --command just test ts --hrd --rate 7000000 --duration 120 env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - # The passthrough over the same path: retransmission must leave no packet missing or moved. - - name: TS passthrough lossy path + # The passthrough over the same path, under its 10 Mb/s multiplex: what arrives must be the + # source's packets in place. + - name: TS passthrough congested path if: ${{ !cancelled() && github.event_name != 'pull_request' }} - run: nix develop --command just test ts --passthrough --loss 1 --duration 120 + run: nix develop --command just test ts --passthrough --rate 7000000 --duration 120 env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs diff --git a/quest/m1/README.md b/quest/m1/README.md index e80ea6efb2c..3f5ee2ca253 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -105,7 +105,6 @@ lands before the cut. - [Transcode finishes its group](/quest/m1/transcode-open-group.md) - `moq-transcode` never aborts a group a fetch's live loop opened - [Rust request credit](/quest/m1/rs-request-credit.md) - a moq-net request fails at once without stream credit instead of waiting - [qmux no-wait opens](/quest/m1/qmux-no-wait.md) - @moq/qmux rejects an over-limit create when waitUntilAvailable is false, like Chrome -- [Bufferbloat harness](/quest/m1/bufferbloat.md) - acceptance and performance checks impair paths with a rate limit and a deep queue, not random loss - [Random-loss audit](/quest/m1/random-loss-audit.md) - every decision random-loss gates drove is kept, reworked, or reverted on shaped paths - [Timed-out requests clean up](/quest/m1/legacy-request-timeout-cleanup.md) - on drafts 14-16, Rust and JS send PUBLISH_NAMESPACE_DONE or UNSUBSCRIBE_NAMESPACE for a timed-out request before its local reset - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide diff --git a/quest/m1/bufferbloat.md b/quest/m1/bufferbloat.md deleted file mode 100644 index c8c64d673dc..00000000000 --- a/quest/m1/bufferbloat.md +++ /dev/null @@ -1,53 +0,0 @@ -# [M] Acceptance and performance run on bufferbloat, not random loss - -## Goal - -Every acceptance or performance (QoS) check that impairs a network does it -with a rate limit below the media bitrate behind a deep but finite queue, so -it congests and tail-drops like a real path. Random loss remains only where -a test proves correctness, such as retransmits happening. - -## Plan - -Decided 2026-10-17 (maintainer), and recorded in the root `AGENTS.md`: -bufferbloat does double duty, since its overflow drops packets too, so it is -the impairment for anything judged on quality, latency, or throughput. -Random loss judged that way pushed decisions toward surviving a failure mode -real paths do not show (see the [audit](/quest/m1/random-loss-audit.md)). - -Classify each random-loss user, move the acceptance and performance ones to -shaping, and leave correctness checks alone: - -- TS harness: `just test ts` gains `--rate` and `--queue` beside `--loss` - (decided 2026-10-17), passed to `moq-shaper`'s existing token bucket; its - 100 ms default queue is shallow for bufferbloat. The nightly - `--hrd --loss 1` and `--passthrough --loss 1` steps in `ts.yml`, - `passthrough.py --loss-ok`, and the `test/ts/README.md` lossy-path section - grade output quality, so they move to `--rate`/`--queue`; `--loss` stays - for correctness runs. -- Relay drills: the 5% profile every `::impaired` lane and the cluster - drills use (`drills.rs`, `test/drill/README.md`). Decide per drill - whether it proves correctness or judges behavior under impairment. -- `moq-quic`'s sim (`Link.loss`, the lossy proptests) and BBR3's - deterministic periodic-drop unit tests prove recovery is correct; they - stay unless they assert throughput or latency. - -`moq-shaper --loss` stays for correctness use. Earlier in the same day the -plan was to delete it; the maintainer then scoped the rule to acceptance and -performance. - -The TS run's oracle under a path narrower than the muxrate, decided -2026-10-17: the export stays T-STD conformant and continuous (PCR, -continuity counters, no partial PES), sheds whole groups rather than -stalling, and end-to-end latency stays bounded instead of growing with the -queue. No loss-percentage or hole-count thresholds. Pick the shaped rate and -queue depth from what the run must exercise (the queue must overflow), and -say why in the harness. - -Public API: none. Wire: none. - -## Related - -- [Random-loss audit](/quest/m1/random-loss-audit.md) - re-judges what random-loss tests drove, on this harness -- [Relay burst drill](/quest/m1/test-flakes-2/relay-burst-impaired.md) - owns the impaired burst drill's flake -- [TS recovery](/quest/m1/ts-reliability/recovery.md) - MoQ content-loss schedules, separate from network impairment diff --git a/quest/m1/random-loss-audit.md b/quest/m1/random-loss-audit.md index c480ce3aa1b..cf8863de3ca 100644 --- a/quest/m1/random-loss-audit.md +++ b/quest/m1/random-loss-audit.md @@ -14,7 +14,14 @@ is not how real paths fail, so anything tuned to survive it as a quality or performance target may be wrong. Scope is every random-loss-driven decision, not only the 10% TS work. Small reverts land in the audit's PR; anything bigger becomes its own quest. Verdicts are measured on the -[bufferbloat harness](/quest/m1/bufferbloat.md), not argued. +bufferbloat harness (`just test ts --rate`, see `test/ts/README.md`), not +argued. + +Measured with that harness 2026-10-18: BBRv3 alone keeps about a round trip +queued, so a deep queue never overflows and congestion lands at the relay as +`Old` group evictions instead. A verdict that needs congestion loss, such as +BBR's loss response, needs a queue about a round trip deep or a loss-based +sender sharing the path. Leads found 2026-10-17 (no production default or congestion threshold was changed for random loss; `LOSS_THRESH` is still 2%): @@ -46,7 +53,3 @@ Record each verdict (decision, origin PR, keep/rework/revert, evidence) in the PR description. Public API: depends on the verdicts. Wire: none expected. - -## Required - -- [Bufferbloat harness](/quest/m1/bufferbloat.md) - verdicts are measured on shaped paths diff --git a/quest/m1/ts-reliability/grader-determinism.md b/quest/m1/ts-reliability/grader-determinism.md index 2a1b19dccbd..d1518987517 100644 --- a/quest/m1/ts-reliability/grader-determinism.md +++ b/quest/m1/ts-reliability/grader-determinism.md @@ -23,6 +23,6 @@ Two sources of run-to-run variance, found by #5810 and #5854: Decided 2026-10-17 (maintainer): one quest for the grader, with the CI path filter as a separate [m0 quest](/quest/m0/ci/ts-path-filter.md). The harness never gates on a random-loss percentage (see -[bufferbloat](/quest/m1/bufferbloat.md)). +the root `AGENTS.md`). Public API: none. Wire: none. diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index 84a3b118a04..6c7604db1c0 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -54,6 +54,8 @@ const TRACK: &str = "drill"; /// hostile one, and the drills' own budgets (a 2s idle timeout, 10s waits) hold /// under it unchanged. The loss is 5% because a drill only sends a few hundred /// datagrams, and at 2% that is a drop or two, which barely exercises recovery. +/// Random loss only proves correctness, which is all a drill asserts; nothing +/// here judges quality, latency or throughput. fn impairment() -> moq_shaper::Profile { moq_shaper::Profile { delay: Duration::from_millis(20), diff --git a/test/drill/README.md b/test/drill/README.md index 41e83ba43d7..b4e51600146 100644 --- a/test/drill/README.md +++ b/test/drill/README.md @@ -137,6 +137,15 @@ would turn this lane into a second loopback run that passes for free. There are no retries: a drill that only passes on loopback is a finding. +Random loss is not how real paths fail, so it may prove correctness but never +judge quality, latency or throughput; those run on a congested path (see +`test/ts/README.md`). Every drill here proves correctness: that a resource is +released, a failure surfaces, a route or publisher comes back, or every group is +recovered. The cluster drills' 10s bound on each group is a stall check, a third +of the 30s stalls of #4349, not a latency target. So the impaired lane keeps its +loss, which is what drops the `MAX_STREAMS` and `MAX_DATA` frames the cluster +drills' senders wait on. + The same shaper is a binary, for putting in front of a relay from another process. It applies one profile both ways, prints the seed at start and the counters at exit (Ctrl-C or SIGTERM), and exits nonzero if the profile never diff --git a/test/ts/README.md b/test/ts/README.md index 66d927979d7..421848edb6b 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -44,7 +44,8 @@ just test ts --leak # 1080p video with no HRD, filling its level's 2 just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --fragment-pes # legal headers split across packets, with FFmpeg/TSDuck grading just test ts --delay 2s # pass the exporter's --delay -just test ts --hrd --loss 1 # drop 1% of the relay's datagrams to the subscriber +just test ts --hrd --rate 7000000 # congest the subscriber's leg below the media rate +just test ts --loss 1 # drop 1% of the relay's datagrams to the subscriber just test ts --passthrough # carry the multiplex whole, grade it packet by packet just test ts --passthrough --mpts # the same with two programs on their own clocks just test ts-passthrough-capture # the same over an in-repo capture the T-STD model passes @@ -564,53 +565,79 @@ It counts the packets on each audio PID and the PCR-only packets on each PID, scaling the capture's counts by how much of the source's elementary stream on the PID it carries, and fails when their total exceeds the source's. -## Lossy path (`--loss`) - -`--loss ` puts [`moq-shaper`](../../rs/moq-shaper/README.md) between the -relay and the subscriber: it drops that share of the relay's datagrams at random -and delays each direction 25 ms, the path moq-dev/moq#4613 built with `netem` in -two network namespaces. The shaper needs no privileges and leaves the host's -network alone. `TSC_SEED` (default 1) fixes which datagrams are dropped, not when -the rest arrive. The run prints the shaper's counters beside the exporter's -`dropped` (missed their deadline), `trimmed` (before the output began) and `gaps` -(skipped upstream), and fails if the shaper never dropped anything. - -QUIC retransmits what the path drops, so a lossy path is not by itself an -omission of MoQ content, and every compliance check grades only the access units -that arrived: a capture missing most of its video still passes them. The generated -clips the decoded-content comparison does not cover (`--hrd`, `--headroom`, -`--leak`, `--full`) go through `holes.py` instead, which reads the capture's -timestamps against the source's, independently of what the exporter reports. Each -stream of a generated clip keeps one frame step, so a capture's step past one and -a half of its source's is a hole. The export keeps the source's timestamps, so every source stream -must also be in the capture and end within half a frame of the source's. It may -start up to 3 s late: the subscriber joins at the relay's latest group once the -catalog is out, which skips the first 1-2 s on a clean path. Omitting MoQ content -on purpose belongs to the [recovery tests](/quest/m1/ts-reliability/recovery.md). +## Impaired path (`--rate`, `--loss`) + +`--rate` and `--loss` put [`moq-shaper`](../../rs/moq-shaper/README.md) between +the relay and the subscriber, with 25 ms each way. Unlike `netem` in two network +namespaces, as moq-dev/moq#4613 built it, the shaper needs no privileges and +leaves the host's network alone. The run prints the shaper's counters beside the +exporter's `dropped` (missed their deadline), `trimmed` (before the output began) +and `gaps` (skipped upstream). + +### Congested path (`--rate`, `--queue`) + +`--rate ` carries the relay's datagrams to the subscriber at that rate +behind a queue `--queue` deep (2 s by default), and tail-drops what overflows it: +bufferbloat, which is how real paths fail. Below the media's rate the path cannot +carry it all, so the run judges what the export does instead. It must stay T-STD +conformant and continuous (`compliance.py`, `pcr-timing.py` and PCR accuracy gate +as on a clean path), shed whole groups rather than stall, and hold its latency +rather than let it grow with the queue. Nothing bounds how much it sheds. + +The generated clips go through `holes.py --shed`, which matches the capture's +frames to the source's by presentation timestamp, since the export derives each +picture's decode timestamp and a group cut short moves it. Of each source group, +the capture keeps a prefix in decode order (all, part, or none) and never resumes +mid-group. The source's last group must arrive whole: a stall, or a latency grown +past the exporter's 1 s delay, costs it. The decoded-content comparison owes every +unit, so it does not run, and `--open-gop`, which owes every leading picture, is +refused. A passthrough grades its packets in place and reports its gaps +(`passthrough.py --loss-ok`). + +The run fails unless the rate limit held datagrams back and the export shed +something: a rate above the media's never narrows the path. + +The default queue is twice the export's delay, so a sender that kept it full +would make every unit late. The delay-based default (BBRv3) does not: it keeps +about a round trip queued, so the backlog lands at the relay, which ages groups out +(`Old`), and the queue itself overflows only when it is about a round trip deep. +The loss-based `MOQ_QUIC_CONGESTION_CONTROL=loss` (CUBIC) fills it, latency +outgrows the delay, and the export refuses the run as a clock drift. + +Measured on 2026-10-18 with other test runs sharing the host: + +| Run | Queue | Duration | Result | +|---|---:|---:|---| +| `--hrd --rate 7000000` | 2 s | 120 s | passes; 37.6 s of video shed in whole groups, none overflowed | +| `--hrd --rate 7000000` | 150 ms - 1 s | 40-120 s | none overflowed | +| `--hrd --rate 7000000` | 100 ms | 40-120 s, four runs | 0, 0, 3 and 28 overflowed | +| `--hrd --rate 7000000` | 50 ms | 120 s | graders pass; 269 overflowed | +| `--hrd --rate 7000000 --live` | 2 s | 60 s | passes; 99.8 % of PCRs released on time, drift -28 ms | +| `--passthrough --rate 7000000` | 2 s | 120 s | passes; its gaps leave 463 continuity errors (reported) | +| `--hrd --rate 7000000`, CUBIC | 2 s | 60 s | fails: 918 overflowed, then the clock-drift refusal | ```bash -./holes.py capture.ts source.ts +./holes.py capture.ts source.ts [--shed] ``` -Measured on 2026-10-16 with about 30 other test runs sharing a 24-core host, so -timing is perturbed; nightly is the reference: +Without `--shed`, `holes.py` grades a path that should lose nothing: a capture's +step past one and a half of its source's is a hole, and every source stream must +be in the capture and end within half a frame of the source's. Each stream of a +generated clip keeps one frame step, so it reads the capture's timestamps against +the source's, independently of what the exporter reports. Either way a stream may +start up to 3 s late: the subscriber joins at the relay's latest group once the +catalog is out, which skips the first 1-2 s on a clean path. Omitting MoQ content +on purpose belongs to the [recovery tests](/quest/m1/ts-reliability/recovery.md). -| Clip | Loss | Duration | Result | -|---|---:|---:|---| -| default (H.264/AAC, low media rate) | 10 % | 40 s | passes, decoded content intact | -| `--hrd` (9 Mb/s video, 10 Mb/s mux) | 1 %, 3 % | 40 s, and 120 s at 3 % | passes, no holes | -| `--headroom` (12.5 Mb/s, four audio PIDs) | 1 % | 120 s, two seeds | T-STD passes; 0.2-0.4 s of MP2 holes in the first half second after the join | -| `--headroom` | 3 % | 120 s | T-STD passes; video stops 27 s in, AC-3 19 s in | -| `--hrd` | 5 %, 7 %, 10 % | 40-60 s | export fails: groups evicted `Old`, then a false clock-drift refusal | - -The ceiling is the congestion controller, not the export. The default `delay` -family is BBRv3, which takes a round with more than 2 % loss -(`LOSS_THRESH`) as congestion and cuts its inflight bound, so random loss above -that holds the subscriber's leg below the media rate, and the relay ages groups -out while they wait. CUBIC (`loss`) fared worse at 10 %: no output at all. With -`LOSS_THRESH` raised to 20 % as an experiment only, `--hrd --loss 10` passed T-STD -and PCR accuracy, though 1.56 s of video still missed the 1 s delay. The 10 % -case belongs to the [BBR media study](/quest/m2/quic-bbr-natural-drain.md). +### Lossy path (`--loss`) + +`--loss ` drops that share of the relay's datagrams at random. +`TSC_SEED` (default 1) fixes which, not when the rest arrive, and the run fails +if the shaper dropped none. Real paths do not lose datagrams independently of +their load, so this proves correctness only: QUIC retransmits what the path drops, +so a capture with a hole or a gap fails as on a clean path. It does not judge the +export's quality or latency, and no CI step runs it; combined with `--rate`, the +congested path's grading applies. ## Passthrough (`--passthrough`, `passthrough.py`) @@ -641,9 +668,10 @@ source in place: a group boundary promises no immediate decode for every program, and passthrough neither inserts tables nor extracts a program. A gap fails the run, `--loss` included, since QUIC retransmits what the path -drops. To grade a capture from a path that loses content, run the grader with +drops. A path that loses content, as `--rate` does, runs the grader with `--loss-ok`: gaps are reported and the checks run on the capture alone, since a -gap can take part of a PES or the next table with it. +gap can take part of a PES or the next table with it, and the source's packets +in place keep their continuity counters across it. ```bash ./passthrough.py source.ts capture.ts [--loss-ok] @@ -652,10 +680,7 @@ gap can take part of a PES or the next table with it. Release timing is the Rust tests' job (`rs/moq-mux/src/container/ts/release_test.rs`, on a paused clock), with the packet and schedule oracle in `oracle.rs` and the fixtures it cannot get from a -capture in `fixture.rs`. On a loaded host, `--passthrough --loss 5` left 77 % of -the packets missing in 30 gaps with one object counted late; the rest were -groups the relay aged out, the congestion ceiling described under -[Lossy path](#lossy-path---loss). +capture in `fixture.rs`. ## EIT fixtures @@ -806,11 +831,11 @@ open-GOP, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Eac All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, -`just test ts --live --duration 120`, `just test ts --hrd --loss 1 --duration 120` -and `just test ts --passthrough --loss 1 --duration 120`. +`just test ts --live --duration 120`, `just test ts --hrd --rate 7000000 --duration 120` +and `just test ts --passthrough --rate 7000000 --duration 120`. The pair exercises late-joining subscribers; the live window distinguishes ongoing -clock drift from initial buffer filling; the lossy paths prove T-STD and media -continuity through QUIC retransmission. +clock drift from initial buffer filling; the congested paths judge what the export +does when the path cannot carry the media. These are actual test invocations, not optional tools installed on a runner. Failed runs retain source/capture files, process logs and statuses, tool versions, diff --git a/test/ts/holes.py b/test/ts/holes.py index 7bec6d0bd3a..89bb130805c 100755 --- a/test/ts/holes.py +++ b/test/ts/holes.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Find the media a capture is missing, from its timestamps against its generated source. - ./holes.py capture.ts source.ts + ./holes.py capture.ts source.ts [--shed] Every audio and video stream of a generated clip has a constant frame duration, so the step between consecutive decode timestamps is the same throughout. A capture's step past one and a @@ -13,17 +13,28 @@ a generated clip on a clean path. The compliance checks grade only the access units that arrived, so a capture missing most of its video still passes them. +Under `--shed` the path was narrower than the media, so losing media is expected and reported, +but only by whole groups: of each group of the source, from a sync frame to the next, the +capture keeps a prefix in decode order (all of it, part of it, or none) and never resumes +mid-group. Frames are matched by presentation timestamp, since the export derives each +picture's decode timestamp from the frames around it and a group cut short moves it. The +source's last group still has to arrive whole, so a stream that stalled, or whose latency grew +past the exporter's delay, fails. + This reads the captures alone, independently of what the exporter reports, and needs only ffprobe. A real capture's streams need not keep a constant step, so it grades generated clips. -Exit status is 0 when every source stream is in the capture without a hole or a cut, 1 otherwise. +Exit status is 0 when every source stream is in the capture without a hole (or, under +`--shed`, losing only whole groups) or a cut, 1 otherwise. """ +import argparse import json import os import subprocess import sys from collections import Counter +from dataclasses import dataclass from fractions import Fraction # How much later than its source a stream may start, in seconds: the clean-path join plus a group. @@ -41,6 +52,34 @@ def holes(dts: list[int], frame: int) -> list[tuple[int, int]]: return [(a, b - a) for a, b in zip(dts, dts[1:]) if 2 * (b - a) > 3 * frame] +@dataclass +class Shed: + """Which of a source stream's frames a capture lost, after the first one it kept.""" + + missing: int # frames lost + mid_group: list[int] # PTS of each frame kept after its group lost an earlier one + tail: int # frames of the source's last group lost + + +def shed(pts: list[int], sync: list[bool], kept: set[int]) -> Shed: + """Grade the capture's `kept` PTS against the source's frames, in decode order with their sync flags.""" + result = Shed(missing=0, mid_group=[], tail=0) + joined = lost = False + for t, starts in zip(pts, sync): + if starts: + lost = False + result.tail = 0 + if t in kept: + joined = True + if lost: + result.mid_group.append(t) + elif joined: + lost = True + result.missing += 1 + result.tail += 1 + return result + + Span = tuple[Fraction, Fraction, Fraction] @@ -56,63 +95,90 @@ def cuts(source: Span, capture: Span, frame: Fraction) -> list[str]: return found -def probe(path: str) -> dict[str, tuple[dict, list[int], Span]]: - """Each audio and video stream by PID: the stream, its decode timestamps, and its span.""" +@dataclass +class Stream: + """An audio or video stream's frames, in decode order.""" + + info: dict # ffprobe's description + dts: list[int] + pts: list[int] + sync: list[bool] + span: Span # first PTS, last PTS and last DTS, in seconds + + +def probe(path: str) -> dict[str, Stream]: + """Each audio and video stream, by PID.""" # Probe the whole file, not FFmpeg's first 5 MB: a stream whose first frame comes after that # is never described, so the second frame of its first PES repeats the first's timestamp # and every later one in that PES runs a frame behind, which reads as a hole. report = json.loads(subprocess.run( ["ffprobe", "-v", "error", "-probesize", str(os.path.getsize(path)), "-analyzeduration", str(2**62), - "-show_entries", "stream=index,id,codec_type,codec_name,time_base:packet=stream_index,dts,pts", + "-show_entries", "stream=index,id,codec_type,codec_name,time_base:packet=stream_index,dts,pts,flags", "-of", "json", path], check=True, capture_output=True, text=True, ).stdout) streams = {s["index"]: s for s in report["streams"] if s.get("codec_type") in ("audio", "video")} dts: dict[int, list[int]] = {index: [] for index in streams} pts: dict[int, list[int]] = {index: [] for index in streams} + sync: dict[int, list[bool]] = {index: [] for index in streams} for packet in report.get("packets", []): index = packet["stream_index"] if index in dts and "dts" in packet: dts[index].append(int(packet["dts"])) pts[index].append(int(packet.get("pts", packet["dts"]))) + sync[index].append("K" in packet.get("flags", "")) return { - s["id"]: (s, dts[index], tuple(t * Fraction(s["time_base"]) for t in (min(pts[index]), max(pts[index]), dts[index][-1]))) + s["id"]: Stream(s, dts[index], pts[index], sync[index], + tuple(t * Fraction(s["time_base"]) for t in (min(pts[index]), max(pts[index]), dts[index][-1]))) for index, s in streams.items() if pts[index] } def main() -> None: - if len(sys.argv) != 3: - sys.exit(__doc__) - capture, source = probe(sys.argv[1]), probe(sys.argv[2]) + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("capture") + parser.add_argument("source") + parser.add_argument("--shed", action="store_true", help="allow losing whole groups, but not the last") + args = parser.parse_args() + capture, source = probe(args.capture), probe(args.source) if not source: - sys.exit(f"error: {sys.argv[2]} has no audio or video stream") + sys.exit(f"error: {args.source} has no audio or video stream") - print("### media holes (capture timestamps against the source)") + print("### media holes (capture timestamps against the source" + (", whole groups shed" if args.shed else "") + ")") failed = False - for pid, (stream, source_dts, span) in sorted(source.items()): - label = f"PID {pid} {stream['codec_name']}" + for pid, src in sorted(source.items()): + label = f"PID {pid} {src.info['codec_name']}" # The source's step, not the capture's: a capture that kept every other frame has a step of its own. - frame = step(source_dts) + frame = step(src.dts) if frame == 0: - sys.exit(f"error: {label} has fewer than two frames in {sys.argv[2]}") + sys.exit(f"error: {label} has fewer than two frames in {args.source}") if pid not in capture: print(f" FAIL {label}: not in the capture") failed = True continue - stream, dts, captured = capture[pid] - base = Fraction(stream["time_base"]) - found = holes(dts, frame) - cut = cuts(span, captured, frame * base) - missing = sum(gap - frame for _, gap in found) * base - verdict = "FAIL" if found or cut else "PASS" - failed |= bool(found or cut) - print(f" {verdict} {label}: {len(dts)} frames of {float(frame * base) * 1000:.2f} ms, " - f"{len(found)} holes, {float(missing) * 1000:.0f} ms missing, " - f"from {float(captured[0]):.3f}s to {float(captured[1]):.3f}s of {float(span[0]):.3f}s to {float(span[1]):.3f}s") - for at, gap in found[:5]: - print(f" at {float(at * base):.3f}s: {float(gap * base) * 1000:.0f} ms step") - for reason in cut: + got = capture[pid] + base = Fraction(got.info["time_base"]) + span = (f"from {float(got.span[0]):.3f}s to {float(got.span[1]):.3f}s " + f"of {float(src.span[0]):.3f}s to {float(src.span[1]):.3f}s") + if args.shed: + graded = shed(src.pts, src.sync, set(got.pts)) + reasons = [f"resumes mid-group at {float(t * base):.3f}s" for t in graded.mid_group[:5]] + if graded.tail: + reasons.append(f"lost {graded.tail} frames of the source's last group") + late = got.span[0] - src.span[0] + if late > JOIN: + reasons.append(f"starts {float(late):.3f}s after its source, past the {float(JOIN)}s join") + summary = f"{graded.missing} frames ({float(graded.missing * frame * base):.1f} s) shed" + else: + found = holes(got.dts, frame) + reasons = [f"at {float(at * base):.3f}s: {float(gap * base) * 1000:.0f} ms step" for at, gap in found[:5]] + reasons += cuts(src.span, got.span, frame * base) + missing = sum(gap - frame for _, gap in found) * base + summary = f"{len(found)} holes, {float(missing) * 1000:.0f} ms missing" + failed |= bool(reasons) + print(f" {'FAIL' if reasons else 'PASS'} {label}: {len(got.dts)} frames of " + f"{float(frame * base) * 1000:.2f} ms, {summary}, {span}") + for reason in reasons: print(f" {reason}") sys.exit(1 if failed else 0) diff --git a/test/ts/holes.test.py b/test/ts/holes.test.py index c6923e9172d..616b69850ea 100644 --- a/test/ts/holes.test.py +++ b/test/ts/holes.test.py @@ -36,6 +36,40 @@ def test_too_short(self): self.assertEqual(holes.step([0]), 0) +class Shed(unittest.TestCase): + # Four groups of 25 frames, each opening on a sync frame, with each P picture decoded ahead + # of the B picture it presents after. + PTS = [(i + (0 if i % 25 == 0 else 1 if i % 25 % 2 else -1)) * 3600 for i in range(100)] + SYNC = [i % 25 == 0 for i in range(100)] + + def grade(self, *lost: range) -> holes.Shed: + kept = {t for i, t in enumerate(self.PTS) if not any(i in r for r in lost)} + return holes.shed(self.PTS, self.SYNC, kept) + + def test_whole(self): + self.assertEqual(self.grade(), holes.Shed(missing=0, mid_group=[], tail=0)) + + def test_whole_group(self): + self.assertEqual(self.grade(range(25, 50)), holes.Shed(missing=25, mid_group=[], tail=0)) + + def test_rest_of_a_group(self): + # The rest of a group skipped once it went stale, resuming at the next one. + self.assertEqual(self.grade(range(40, 50)), holes.Shed(missing=10, mid_group=[], tail=0)) + + def test_late_join(self): + # Groups before the first kept frame are the join, not shed. + self.assertEqual(self.grade(range(0, 50)), holes.Shed(missing=0, mid_group=[], tail=0)) + + def test_resumes_mid_group(self): + graded = self.grade(range(40, 45)) + self.assertEqual(graded.mid_group, self.PTS[45:50]) + + def test_lost_tail(self): + # Shedding the last group means the export fell behind and never caught up. + self.assertEqual(self.grade(range(90, 100)).tail, 10) + self.assertEqual(self.grade(range(75, 100)).tail, 25) + + class Cuts(unittest.TestCase): # First PTS, last PTS, last DTS: the last pictures in decode order are B pictures presented before the P. SOURCE = (Fraction(1), Fraction(31), Fraction(30) + Fraction(23, 25)) @@ -74,7 +108,7 @@ def test_audio_starting_past_the_default_probe(self): "-c:a", "mp2", "-f", "mpegts", path, ], check=True) streams = holes.probe(path) - _, dts, _ = streams["0x101"] + dts = streams["0x101"].dts self.assertEqual(holes.holes(dts, holes.step(dts)), []) if __name__ == "__main__": diff --git a/test/ts/run.sh b/test/ts/run.sh index 47f8ef73c2b..aac6c3a4168 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -25,6 +25,7 @@ # ./run.sh --full # the same with two audio PIDs, filling its multiplex with no nulls # ./run.sh --fragment-pes # split generated clear PES headers across packets # ./run.sh --delay 2s # pass the exporter's --delay +# ./run.sh --rate 7000000 # bufferbloat: 7 Mb/s to the subscriber behind a deep queue (moq-shaper) # ./run.sh --loss 10 # drop 10% of the relay's datagrams to the subscriber (moq-shaper) # ./run.sh --passthrough # carry the multiplex whole (`import/export ts --passthrough`) # ./run.sh --passthrough --mpts # the same with two programs whose recovery points rarely line up @@ -82,7 +83,9 @@ HEADROOM="" # the same plus four audio PIDs, in a multiplex faster than the LEAK="" # publish video with no HRD at its level's leak, as a full-rate feed does FULL="" # the same with audio, in a multiplex the source fills with no nulls FRAGMENT_PES="" # use following null slots for fragmented PES headers -LOSS="" # percent of relay -> subscriber datagrams dropped by moq-shaper +LOSS="" # percent of relay -> subscriber datagrams dropped at random by moq-shaper +RATE="" # bits/s moq-shaper carries from the relay to the subscriber, behind QUEUE +QUEUE="" # how long a datagram may wait for RATE before the path tail-drops it PASSTHROUGH="" # carry the multiplex whole and grade the capture packet by packet against the source MPTS="" # publish two programs, which only --passthrough carries whole EXPORT=() # extra `export ts` flags (--delay) @@ -177,6 +180,14 @@ while [[ $# -gt 0 ]]; do LOSS="$2" shift 2 ;; + --rate) + RATE="$2" + shift 2 + ;; + --queue) + QUEUE="$2" + shift 2 + ;; --passthrough) PASSTHROUGH=1 shift @@ -248,24 +259,44 @@ if [[ -n "$OPEN_GOP" && -n "$ANALYZE_ONLY$LIVE$PAIR" ]]; then echo "error: --open-gop cannot be combined with --analyze-only, --live, or --pair" >&2 exit 1 fi +# open-gop.py owes every leading picture, which a path that sheds groups cannot deliver. +if [[ -n "$OPEN_GOP" && -n "$RATE" ]]; then + echo "error: --open-gop grades every leading picture and cannot be combined with --rate, which sheds groups" >&2 + exit 1 +fi if [[ -n "$FRAGMENT_PES" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$PAIR$LIVE" ]]; then echo "error: --fragment-pes requires the generated closed-GOP H.264/AAC profile" >&2 exit 1 fi -# The #4613 path: random loss on the relay's datagrams to the subscriber and 25 ms each way, -# through moq-shaper so it needs no privileges and leaves the host's network alone. -if [[ -n "$LOSS" ]]; then - if [[ -n "$ANALYZE_ONLY$PAIR" ]]; then - echo "error: --loss impairs the round-trip's one subscriber and cannot be combined with --analyze-only or --pair" >&2 - exit 1 - fi - if ! [[ "$LOSS" =~ ^[0-9]+(\.[0-9]+)?$ ]] || ! awk -v p="$LOSS" 'BEGIN { exit !(p > 0 && p < 100) }'; then - echo "error: --loss takes a percentage above 0 and below 100 (got '$LOSS')" >&2 - exit 1 - fi +# An impaired path to the subscriber, 25 ms each way through moq-shaper, so it needs no +# privileges and leaves the host's network alone. `--rate` congests it the way real paths +# fail and is what judges the export; `--loss` drops datagrams at random (moq-dev/moq#4613), +# which real paths do not, so it only proves QUIC retransmission keeps the output whole. +SHAPED="$LOSS$RATE" +if [[ -n "$SHAPED" && -n "$ANALYZE_ONLY$PAIR" ]]; then + echo "error: --loss and --rate impair the round-trip's one subscriber and cannot be combined with --analyze-only or --pair" >&2 + exit 1 fi +if [[ -n "$LOSS" ]] && { ! [[ "$LOSS" =~ ^[0-9]+(\.[0-9]+)?$ ]] || ! awk -v p="$LOSS" 'BEGIN { exit !(p > 0 && p < 100) }'; }; then + echo "error: --loss takes a percentage above 0 and below 100 (got '$LOSS')" >&2 + exit 1 +fi +if [[ -n "$RATE" ]] && ! [[ "$RATE" =~ ^[1-9][0-9]*$ ]]; then + echo "error: --rate takes bits per second (got '$RATE')" >&2 + exit 1 +fi +if [[ -n "$QUEUE" && -z "$RATE" ]]; then + echo "error: --queue is the depth behind --rate and needs it" >&2 + exit 1 +fi +# A bloated buffer, not the shaper's 100 ms default: twice the export's 1 s delay, so a +# sender that kept it full would make every unit late and lose the tail holes.py requires, +# while it still tail-drops what overflows it. Measured on --hrd at 7 Mb/s, BBRv3 never +# filled one deeper than about a round trip: 50 ms overflowed hundreds of datagrams, 100 ms +# a few or none, 150 ms and deeper none. +[[ -n "$RATE" && -z "$QUEUE" ]] && QUEUE=2s # The capture is the source's own packets, graded against it by passthrough.py; the arms that # grade a second capture or the live pipe grade the demultiplexed export's choices. @@ -348,7 +379,7 @@ TARGET_BASE=$(cargo metadata --format-version 1 --manifest-path "$WORKSPACE/Carg echo "### building moq-relay + moq-cli ($PROFILE)" flag=() [[ "$PROFILE" == "release" ]] && flag=(--release) -[[ -n "$LOSS" ]] && flag+=(-p moq-shaper) +[[ -n "$SHAPED" ]] && flag+=(-p moq-shaper) (cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli) RELAY="$TARGET_BASE/$PROFILE/moq-relay" MOQ="$TARGET_BASE/$PROFILE/moq" @@ -516,25 +547,26 @@ if ! harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"; then fi harness_endpoint relay "$URL" -# Subscribers dial the relay directly, or the shaper in front of it under --loss. The -# shaper carries only UDP, so the certificate's fingerprint comes from the relay itself. +# Subscribers dial the relay directly, or the shaper in front of it under --loss or --rate. +# The shaper carries only UDP, so the certificate's fingerprint comes from the relay itself. SUB_CONNECT=(--connect "$URL") -if [[ -n "$LOSS" ]]; then +if [[ -n "$SHAPED" ]]; then harness_port shaper SHAPER_PORT="$HARNESS_PORT" SEED="${TSC_SEED:-1}" - cat >"$HARNESS_RUN/shaper.toml" <"$HARNESS_RUN/shaper.toml" + echo "### starting moq-shaper on 127.0.0.1:${SHAPER_PORT}: ${PATH_DESC} to the subscriber, seed $SEED" harness_spawn shaper "$HARNESS_RUN/shaper.log" "$SHAPER" --listen "127.0.0.1:$SHAPER_PORT" \ --target "127.0.0.1:$PORT" --profile "$HARNESS_RUN/shaper.toml" --report "$HARNESS_RUN/shaper.json" SHAPER_PID="$HARNESS_PID" @@ -656,15 +688,29 @@ fi # profile never acted, which would turn the impaired run into an unimpaired pass. # shellcheck disable=SC2329 # invoked once the subscriber is done, below stop_shaper() { - [[ -n "$LOSS" ]] || return 0 + [[ -n "$SHAPED" ]] || return 0 kill -TERM -- -"$SHAPER_PID" 2>/dev/null || true ts_wait "$SHAPER_PID" shaper || { sed 's/^/ shaper: /' "$HARNESS_RUN/shaper.log" >&2 || true return 1 } - # The shaper accepts a silent profile while silence is still likely; a lossy run needs loss. - if ! python3 -c 'import json, sys; sys.exit(json.load(open(sys.argv[1]))["down"]["lost"] == 0)' "$HARNESS_RUN/shaper.json"; then - echo "error: moq-shaper dropped none of the relay's datagrams at ${LOSS}% loss; raise --loss or --duration" >&2 + # The shaper accepts a silent profile while silence is still likely, and never requires a + # rate limit to act; a lossy run needs loss, and a congested one a rate limit that held + # datagrams back. + local counter + for counter in ${LOSS:+lost} ${RATE:+throttled}; do + if ! python3 -c 'import json, sys; sys.exit(json.load(open(sys.argv[1]))["down"][sys.argv[2]] == 0)' \ + "$HARNESS_RUN/shaper.json" "$counter"; then + echo "error: moq-shaper ${counter} none of the relay's datagrams on a path of ${PATH_DESC}" >&2 + return 1 + fi + done + # A path narrower than the media congests the session, not the queue: the delay-based + # default (BBRv3) keeps its inflight near two round trips' worth, so a deep queue rarely + # overflows and the backlog lands at the relay, which sheds stale groups. So a congested + # run is one whose export shed something; one that kept everything was never narrower. + if [[ -n "$RATE" ]] && ! sed 's/\x1b\[[0-9;]*m//g' "$HARNESS_RUN/sub.log" | grep -E 'media skipped upstream|missed its deadline' >/dev/null; then + echo "error: the export shed nothing, so ${RATE} b/s never narrowed the path below the media; lower --rate" >&2 return 1 fi # What the path dropped beside what that cost the output: QUIC retransmits the former, @@ -760,10 +806,11 @@ echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') bytes -> analyzing" # Every captured packet must be the source's at its exact position from the first one on, and # compliance.py must grade the capture as it grades that interval of the source, whatever the # source's own verdict. QUIC retransmits what --loss drops, so a gap fails here as it does under -# holes.py; passthrough.py says whether packets went missing or were rewritten. +# holes.py; passthrough.py says whether packets went missing or were rewritten. A path narrower +# than the multiplex (--rate) has to shed, so there gaps are reported rather than failed. if [[ -n "$PASSTHROUGH" ]]; then echo - if ! python3 "$DIR/passthrough.py" "$SRC_TS" "$SUB_TS" --report-json "$HARNESS_RUN/passthrough.json"; then + if ! python3 "$DIR/passthrough.py" "$SRC_TS" "$SUB_TS" ${RATE:+--loss-ok} --report-json "$HARNESS_RUN/passthrough.json"; then echo >&2 echo "error: the capture is not the source's packets in place; the exporter reported:" >&2 sed 's/\x1b\[[0-9;]*m//g' "$HARNESS_RUN/sub.log" | @@ -772,7 +819,7 @@ if [[ -n "$PASSTHROUGH" ]]; then exit 1 fi echo - if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL$MPTS" ]]; then + if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL$MPTS$RATE" ]]; then python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode" || exit 1 else echo "decoded-content comparison: not covered for this profile" @@ -782,8 +829,9 @@ fi # The initial decoded-content profile is the generated closed-GOP H.264/AAC # pair. Other arms retain their specific structural/model oracles until their -# acquisition and decoder-priming contracts are covered by fixtures. -if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL" ]]; then +# acquisition and decoder-priming contracts are covered by fixtures. A path +# narrower than the media sheds groups, which only holes.py grades. +if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL$RATE" ]]; then if ! python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode"; then # The exporter counts what it dropped (late), trimmed (before its output began), # and lost upstream (gaps), and logs each loss, so the failure says which it was. @@ -799,10 +847,12 @@ fi # The other generated clips keep one frame step per stream, so media missing from the capture # shows in its timestamps even where its content is not compared. The compliance checks below -# grade only the access units that arrived. -if [[ -n "$HRD$HEADROOM$LEAK$FULL" ]]; then +# grade only the access units that arrived. Under --rate the path is narrower than the media, +# so the export has to shed: only whole groups, and never the last, which a stall or a +# latency that grew with the queue would cost. +if [[ -n "$HRD$HEADROOM$LEAK$FULL" || (-n "$RATE" && -z "$SOURCE") ]]; then echo - if ! python3 "$DIR/holes.py" "$SUB_TS" "$SRC_TS"; then + if ! python3 "$DIR/holes.py" "$SUB_TS" "$SRC_TS" ${RATE:+--shed}; then echo >&2 echo "error: the capture is missing media; the exporter reported:" >&2 sed 's/\x1b\[[0-9;]*m//g' "$HARNESS_RUN/sub.log" | From 98346204586e2a0da99963849d73596805d9d42c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 09:41:23 -0700 Subject: [PATCH 3/5] test(ts): match a shed capture's frames to the source in order holes.py --shed graded the captured PTS as a set, so a capture that reordered or repeated frames passed. Match the capture as an ordered subsequence of the source's decode-order PTS instead, and fail any frame past the last one that continues the source's order. Co-Authored-By: Claude Opus 5.5 --- test/ts/README.md | 2 +- test/ts/holes.py | 21 +++++++++++++++------ test/ts/holes.test.py | 22 +++++++++++++++++----- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/test/ts/README.md b/test/ts/README.md index 1243f8e3619..da18d70e523 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -588,7 +588,7 @@ The generated clips go through `holes.py --shed`, which matches the capture's frames to the source's by presentation timestamp, since the export derives each picture's decode timestamp and a group cut short moves it. Of each source group, the capture keeps a prefix in decode order (all, part, or none) and never resumes -mid-group. The source's last group must arrive whole: a stall, or a latency grown +mid-group, and a frame reordered, repeated, or not in the source fails. The source's last group must arrive whole: a stall, or a latency grown past the exporter's 1 s delay, costs it. The decoded-content comparison owes every unit, so it does not run, and `--open-gop`, which owes every leading picture, is refused. A passthrough grades its packets in place and reports its gaps diff --git a/test/ts/holes.py b/test/ts/holes.py index 89bb130805c..9a72a5b5a45 100755 --- a/test/ts/holes.py +++ b/test/ts/holes.py @@ -17,7 +17,8 @@ but only by whole groups: of each group of the source, from a sync frame to the next, the capture keeps a prefix in decode order (all of it, part of it, or none) and never resumes mid-group. Frames are matched by presentation timestamp, since the export derives each -picture's decode timestamp from the frames around it and a group cut short moves it. The +picture's decode timestamp from the frames around it and a group cut short moves it, and in +capture order: a frame reordered, repeated, or not in the source fails. The source's last group still has to arrive whole, so a stream that stalled, or whose latency grew past the exporter's delay, fails. @@ -59,17 +60,21 @@ class Shed: missing: int # frames lost mid_group: list[int] # PTS of each frame kept after its group lost an earlier one tail: int # frames of the source's last group lost + unmatched: list[int] # PTS of each captured frame past the last one in the source's order -def shed(pts: list[int], sync: list[bool], kept: set[int]) -> Shed: - """Grade the capture's `kept` PTS against the source's frames, in decode order with their sync flags.""" - result = Shed(missing=0, mid_group=[], tail=0) +def shed(pts: list[int], sync: list[bool], kept: list[int]) -> Shed: + """Grade the capture's `kept` PTS, in its order, against the source's frames, in decode order with their sync flags.""" + result = Shed(missing=0, mid_group=[], tail=0, unmatched=[]) joined = lost = False + i = 0 for t, starts in zip(pts, sync): if starts: lost = False result.tail = 0 - if t in kept: + # A generated stream's PTS are unique, so matching greedily finds the capture as a subsequence if it is one. + if i < len(kept) and kept[i] == t: + i += 1 joined = True if lost: result.mid_group.append(t) @@ -77,6 +82,7 @@ def shed(pts: list[int], sync: list[bool], kept: set[int]) -> Shed: lost = True result.missing += 1 result.tail += 1 + result.unmatched = kept[i:] return result @@ -161,8 +167,11 @@ def main() -> None: span = (f"from {float(got.span[0]):.3f}s to {float(got.span[1]):.3f}s " f"of {float(src.span[0]):.3f}s to {float(src.span[1]):.3f}s") if args.shed: - graded = shed(src.pts, src.sync, set(got.pts)) + graded = shed(src.pts, src.sync, got.pts) reasons = [f"resumes mid-group at {float(t * base):.3f}s" for t in graded.mid_group[:5]] + if graded.unmatched: + reasons.append(f"{len(graded.unmatched)} frames from {float(graded.unmatched[0] * base):.3f}s " + "are out of the source's order, repeated, or not in it") if graded.tail: reasons.append(f"lost {graded.tail} frames of the source's last group") late = got.span[0] - src.span[0] diff --git a/test/ts/holes.test.py b/test/ts/holes.test.py index 616b69850ea..8da00e6b34e 100644 --- a/test/ts/holes.test.py +++ b/test/ts/holes.test.py @@ -43,22 +43,34 @@ class Shed(unittest.TestCase): SYNC = [i % 25 == 0 for i in range(100)] def grade(self, *lost: range) -> holes.Shed: - kept = {t for i, t in enumerate(self.PTS) if not any(i in r for r in lost)} + kept = [t for i, t in enumerate(self.PTS) if not any(i in r for r in lost)] return holes.shed(self.PTS, self.SYNC, kept) def test_whole(self): - self.assertEqual(self.grade(), holes.Shed(missing=0, mid_group=[], tail=0)) + self.assertEqual(self.grade(), holes.Shed(missing=0, mid_group=[], tail=0, unmatched=[])) def test_whole_group(self): - self.assertEqual(self.grade(range(25, 50)), holes.Shed(missing=25, mid_group=[], tail=0)) + self.assertEqual(self.grade(range(25, 50)), holes.Shed(missing=25, mid_group=[], tail=0, unmatched=[])) def test_rest_of_a_group(self): # The rest of a group skipped once it went stale, resuming at the next one. - self.assertEqual(self.grade(range(40, 50)), holes.Shed(missing=10, mid_group=[], tail=0)) + self.assertEqual(self.grade(range(40, 50)), holes.Shed(missing=10, mid_group=[], tail=0, unmatched=[])) def test_late_join(self): # Groups before the first kept frame are the join, not shed. - self.assertEqual(self.grade(range(0, 50)), holes.Shed(missing=0, mid_group=[], tail=0)) + self.assertEqual(self.grade(range(0, 50)), holes.Shed(missing=0, mid_group=[], tail=0, unmatched=[])) + + def test_reordered(self): + kept = list(self.PTS) + kept[30], kept[31] = kept[31], kept[30] + self.assertTrue(holes.shed(self.PTS, self.SYNC, kept).unmatched) + + def test_repeated(self): + kept = self.PTS[:31] + self.PTS[30:] + self.assertTrue(holes.shed(self.PTS, self.SYNC, kept).unmatched) + + def test_not_in_the_source(self): + self.assertTrue(holes.shed(self.PTS, self.SYNC, self.PTS + [1000 * 3600]).unmatched) def test_resumes_mid_group(self): graded = self.grade(range(40, 45)) From 037b250ff19fdaeea109484b9a4d72db839c4a3f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 10:11:51 -0700 Subject: [PATCH 4/5] test(ts): add a nightly lane whose deep queue overflows BBRv3 never fills a queue deeper than about 1.5 round trips (100 ms overflowed in 3 of 6 runs, 150 ms and deeper never), so the 2 s congested lane covers relay shedding but no tail drop. The new lane runs the relay's loss-based CUBIC behind a 250 ms queue (five round trips), which overflowed 423-430 datagrams in each of four 120 s runs while every grader passed, and fails a loss-based run whose queue never overflowed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 12 +++++++++++- quest/m1/random-loss-audit.md | 3 ++- test/ts/README.md | 23 +++++++++++++++++++---- test/ts/run.sh | 10 ++++++---- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 7b300f585b4..33925471b61 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -226,7 +226,17 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - # The passthrough over the same path, under its 10 Mb/s multiplex: what arrives must be the + # The same path behind a 250 ms queue, with the relay's loss-based CUBIC: BBRv3 never fills a + # queue this deep, CUBIC overflows it every sawtooth. QUIC must recover what the queue drops + # inside the export's 1 s delay, and the run fails unless the queue overflowed. + - name: TS overflowing path + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + run: nix develop --command just test ts --hrd --rate 7000000 --queue 250ms --duration 120 + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + MOQ_QUIC_CONGESTION_CONTROL: loss + + # The passthrough over the congested path, under its 10 Mb/s multiplex: what arrives must be the # source's packets in place. - name: TS passthrough congested path if: ${{ !cancelled() && github.event_name != 'pull_request' }} diff --git a/quest/m1/random-loss-audit.md b/quest/m1/random-loss-audit.md index cf8863de3ca..caf37c02ffc 100644 --- a/quest/m1/random-loss-audit.md +++ b/quest/m1/random-loss-audit.md @@ -21,7 +21,8 @@ Measured with that harness 2026-10-18: BBRv3 alone keeps about a round trip queued, so a deep queue never overflows and congestion lands at the relay as `Old` group evictions instead. A verdict that needs congestion loss, such as BBR's loss response, needs a queue about a round trip deep or a loss-based -sender sharing the path. +sender sharing the path. The nightly overflowing path (2026-10-19) runs +CUBIC alone behind a 250 ms queue, which it overflows every run. Leads found 2026-10-17 (no production default or congestion threshold was changed for random loss; `LOSS_THRESH` is still 2%): diff --git a/test/ts/README.md b/test/ts/README.md index da18d70e523..82a760a94b7 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -604,16 +604,30 @@ about a round trip queued, so the backlog lands at the relay, which ages groups The loss-based `MOQ_QUIC_CONGESTION_CONTROL=loss` (CUBIC) fills it, latency outgrows the delay, and the export refuses the run as a clock drift. -Measured on 2026-10-18 with other test runs sharing the host: +So the queue's own tail drop runs as a lane of its own: CUBIC behind a 250 ms +queue, five round trips deep, which it overflows on every sawtooth, while a +retransmit behind a full queue (about twice 300 ms) still lands inside the 1 s +delay. A loss-based run fails unless the queue overflowed. + +```bash +MOQ_QUIC_CONGESTION_CONTROL=loss just test ts --hrd --rate 7000000 --queue 250ms +``` + +Measured on 2026-10-18 and 2026-10-19 with other test runs sharing the host: | Run | Queue | Duration | Result | |---|---:|---:|---| | `--hrd --rate 7000000` | 2 s | 120 s | passes; 37.6 s of video shed in whole groups, none overflowed | | `--hrd --rate 7000000` | 150 ms - 1 s | 40-120 s | none overflowed | -| `--hrd --rate 7000000` | 100 ms | 40-120 s, four runs | 0, 0, 3 and 28 overflowed | -| `--hrd --rate 7000000` | 50 ms | 120 s | graders pass; 269 overflowed | +| `--hrd --rate 7000000` | 100 ms | 40-120 s, six runs | 0, 0, 0, 3, 5 and 28 overflowed | +| `--hrd --rate 7000000` | 75 ms | 60 s, two runs | 70 and 103 overflowed | +| `--hrd --rate 7000000` | 50 ms | 60-120 s, two runs | graders pass; 111 and 269 overflowed | +| `--hrd --rate 3500000` | 2 s | 60 s | fails: none overflowed, 56 s of video shed and the last group lost | | `--hrd --rate 7000000 --live` | 2 s | 60 s | passes; 99.8 % of PCRs released on time, drift -28 ms | | `--passthrough --rate 7000000` | 2 s | 120 s | passes; its gaps leave 463 continuity errors (reported) | +| `--hrd --rate 7000000`, CUBIC | 250 ms | 120 s, four runs | passes; 423-430 overflowed, 18-42 units missed their deadline, 36.5-39.9 s shed | +| `--hrd --rate 7000000`, CUBIC | 500 ms | 60-120 s, three runs | passes; 541-625 overflowed, 79-229 units missed their deadline | +| `--hrd --rate 7000000`, CUBIC | 1 s | 120 s | fails: 919 overflowed, 107 s shed and the last group lost | | `--hrd --rate 7000000`, CUBIC | 2 s | 60 s | fails: 918 overflowed, then the clock-drift refusal | ```bash @@ -832,7 +846,8 @@ open-GOP, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Eac All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, -`just test ts --live --duration 120`, `just test ts --hrd --rate 7000000 --duration 120` +`just test ts --live --duration 120`, `just test ts --hrd --rate 7000000 --duration 120`, +the same with `--queue 250ms` and `MOQ_QUIC_CONGESTION_CONTROL=loss`, and `just test ts --passthrough --rate 7000000 --duration 120`. The pair exercises late-joining subscribers; the live window distinguishes ongoing clock drift from initial buffer filling; the congested paths judge what the export diff --git a/test/ts/run.sh b/test/ts/run.sh index aac6c3a4168..d0f2069ba16 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -59,7 +59,7 @@ source "$DIR/process.sh" # Captured before the parse below consumes it, so the rerun command carries the # source capture, mux rate, EIT fixture, and analyzer thresholds this run used, # whether they arrived as flags or as environment overrides. -RERUN="$(harness_env TSC_DURATION TSC_BITRATE TSC_PORT TSC_PROFILE TSC_SEED)just test ts$(harness_argv "$@")" +RERUN="$(harness_env TSC_DURATION TSC_BITRATE TSC_PORT TSC_PROFILE TSC_SEED MOQ_QUIC_CONGESTION_CONTROL)just test ts$(harness_argv "$@")" # Seamless session replacement needs an explicit publisher epoch. Keep this # opt-in local to the validation rig; epochless replacement is tested as refusal. @@ -696,9 +696,11 @@ stop_shaper() { } # The shaper accepts a silent profile while silence is still likely, and never requires a # rate limit to act; a lossy run needs loss, and a congested one a rate limit that held - # datagrams back. - local counter - for counter in ${LOSS:+lost} ${RATE:+throttled}; do + # datagrams back. A loss-based sender (MOQ_QUIC_CONGESTION_CONTROL=loss) grows until it + # loses, and the queue's tail drop is the only loss here, so its run must overflow too. + local counter overflow="" + [[ -n "$RATE" && "${MOQ_QUIC_CONGESTION_CONTROL:-}" == loss ]] && overflow=overflowed + for counter in ${LOSS:+lost} ${RATE:+throttled} $overflow; do if ! python3 -c 'import json, sys; sys.exit(json.load(open(sys.argv[1]))["down"][sys.argv[2]] == 0)' \ "$HARNESS_RUN/shaper.json" "$counter"; then echo "error: moq-shaper ${counter} none of the relay's datagrams on a path of ${PATH_DESC}" >&2 From 6050cddf486088bbf5a8f53c96299ffb044bea57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 10:25:44 -0700 Subject: [PATCH 5/5] test(ts): join on a sync frame under --shed, and keep the passthrough tail under --loss-ok holes.py --shed now counts frames a capture skipped before its first kept one against that group, so a capture opening mid-GOP fails as resuming mid-group. passthrough.py --loss-ok still reports internal gaps but fails a capture that ends before the source's last packet. Co-Authored-By: Claude Opus 5.5 --- test/ts/README.md | 8 +++++--- test/ts/holes.py | 10 ++++++---- test/ts/holes.test.py | 4 ++++ test/ts/passthrough.py | 19 +++++++++++++++---- test/ts/passthrough.test.py | 7 +++++++ test/ts/run.sh | 3 ++- 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/test/ts/README.md b/test/ts/README.md index 82a760a94b7..60f8fc23395 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -588,11 +588,12 @@ The generated clips go through `holes.py --shed`, which matches the capture's frames to the source's by presentation timestamp, since the export derives each picture's decode timestamp and a group cut short moves it. Of each source group, the capture keeps a prefix in decode order (all, part, or none) and never resumes -mid-group, and a frame reordered, repeated, or not in the source fails. The source's last group must arrive whole: a stall, or a latency grown +mid-group, so it also joins on a sync frame, and a frame reordered, repeated, or +not in the source fails. The source's last group must arrive whole: a stall, or a latency grown past the exporter's 1 s delay, costs it. The decoded-content comparison owes every unit, so it does not run, and `--open-gop`, which owes every leading picture, is refused. A passthrough grades its packets in place and reports its gaps -(`passthrough.py --loss-ok`). +(`passthrough.py --loss-ok`), but must still end on the source's last packet. The run fails unless the rate limit held datagrams back and the export shed something: a rate above the media's never narrows the path. @@ -685,7 +686,8 @@ A gap fails the run, `--loss` included, since QUIC retransmits what the path drops. A path that loses content, as `--rate` does, runs the grader with `--loss-ok`: gaps are reported and the checks run on the capture alone, since a gap can take part of a PES or the next table with it, and the source's packets -in place keep their continuity counters across it. +in place keep their continuity counters across it. The capture must still end on +the source's last packet, which a stall or a growing latency loses. ```bash ./passthrough.py source.ts capture.ts [--loss-ok] diff --git a/test/ts/holes.py b/test/ts/holes.py index 9a72a5b5a45..798c499bb6b 100755 --- a/test/ts/holes.py +++ b/test/ts/holes.py @@ -16,7 +16,7 @@ Under `--shed` the path was narrower than the media, so losing media is expected and reported, but only by whole groups: of each group of the source, from a sync frame to the next, the capture keeps a prefix in decode order (all of it, part of it, or none) and never resumes -mid-group. Frames are matched by presentation timestamp, since the export derives each +mid-group, so it also joins on a sync frame. Frames are matched by presentation timestamp, since the export derives each picture's decode timestamp from the frames around it and a group cut short moves it, and in capture order: a frame reordered, repeated, or not in the source fails. The source's last group still has to arrive whole, so a stream that stalled, or whose latency grew @@ -78,10 +78,12 @@ def shed(pts: list[int], sync: list[bool], kept: list[int]) -> Shed: joined = True if lost: result.mid_group.append(t) - elif joined: + else: + # Before the join too, so a capture that opens mid-group resumes mid-group. lost = True - result.missing += 1 - result.tail += 1 + if joined: + result.missing += 1 + result.tail += 1 result.unmatched = kept[i:] return result diff --git a/test/ts/holes.test.py b/test/ts/holes.test.py index 8da00e6b34e..d455f0944bb 100644 --- a/test/ts/holes.test.py +++ b/test/ts/holes.test.py @@ -76,6 +76,10 @@ def test_resumes_mid_group(self): graded = self.grade(range(40, 45)) self.assertEqual(graded.mid_group, self.PTS[45:50]) + def test_joins_mid_group(self): + # Opening on a frame whose group's sync frame was never captured is as undecodable as resuming there. + self.assertEqual(self.grade(range(0, 30)).mid_group, self.PTS[30:50]) + def test_lost_tail(self): # Shedding the last group means the export fell behind and never caught up. self.assertEqual(self.grade(range(90, 100)).tail, 10) diff --git a/test/ts/passthrough.py b/test/ts/passthrough.py index 21f5777fdeb..faccd6724ff 100644 --- a/test/ts/passthrough.py +++ b/test/ts/passthrough.py @@ -19,7 +19,8 @@ whatever the source is, so it is graded against the source, not against an absolute bar. Under `--loss-ok` gaps are reported rather than failed, and the checks report on the capture alone: a gap can take part of a PES or the next table with it, so the bytes being the source's says -nothing about whether the rest decodes. +nothing about whether the rest decodes. The capture must still end on the source's last packet: +a stall, or a latency grown past the exporter's delay, loses the tail. Exit status is 0 when the capture passes, 1 otherwise. """ @@ -104,6 +105,15 @@ def runs(i: int, after: int) -> list[int]: return placement +def shortfall(placement: Placement, loss_ok: bool) -> str | None: + """Why the source packets `placement` lacks fail the capture, or None when they do not.""" + if placement.tail: + return f"the capture ends {placement.tail} packets before its source" + if placement.gaps and not loss_ok: + return "the capture is missing source packets; --loss-ok reports gaps instead" + return None + + def verdicts(path: Path) -> dict[str, list[str]]: """Every compliance.py check's status and detail for `path`.""" import compliance @@ -115,7 +125,7 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("source", type=Path) parser.add_argument("capture", type=Path) - parser.add_argument("--loss-ok", action="store_true", help="report gaps instead of failing on them") + parser.add_argument("--loss-ok", action="store_true", help="report gaps instead of failing on them, but not a lost tail") parser.add_argument("--report-json", type=Path) args = parser.parse_args() @@ -165,8 +175,9 @@ def main() -> int: print("### compliance.py on the capture (report only: it has gaps)") for name, (status, detail) in report["compliance"]["capture"].items(): print(f" {name}: {status} {detail}") - if not args.loss_ok: - print("passthrough: FAIL: the capture is missing source packets; --loss-ok reports gaps instead", file=sys.stderr) + reason = shortfall(placement, args.loss_ok) + if reason: + print(f"passthrough: FAIL: {reason}", file=sys.stderr) code = 1 if args.report_json: diff --git a/test/ts/passthrough.test.py b/test/ts/passthrough.test.py index 22582dd89a5..f8a0ffe22dd 100644 --- a/test/ts/passthrough.test.py +++ b/test/ts/passthrough.test.py @@ -40,6 +40,13 @@ def test_a_missing_tail_is_counted(self): self.assertEqual((placement.gaps, placement.tail), ([], 40)) self.assertFalse(placement.complete) + def test_loss_ok_admits_gaps_but_not_a_lost_tail(self): + gapped = passthrough.place(SOURCE, SOURCE[10:60] + SOURCE[90:]) + self.assertIsNone(passthrough.shortfall(gapped, loss_ok=True)) + self.assertIsNotNone(passthrough.shortfall(gapped, loss_ok=False)) + truncated = passthrough.place(SOURCE, SOURCE[10:60] + SOURCE[90:-40]) + self.assertIsNotNone(passthrough.shortfall(truncated, loss_ok=True)) + def test_a_rewritten_packet_fails(self): capture = list(SOURCE) capture[50] = capture[50][:100] + b"\xff" + capture[50][101:] diff --git a/test/ts/run.sh b/test/ts/run.sh index d0f2069ba16..0fb1d4b656c 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -809,7 +809,8 @@ echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') bytes -> analyzing" # compliance.py must grade the capture as it grades that interval of the source, whatever the # source's own verdict. QUIC retransmits what --loss drops, so a gap fails here as it does under # holes.py; passthrough.py says whether packets went missing or were rewritten. A path narrower -# than the multiplex (--rate) has to shed, so there gaps are reported rather than failed. +# than the multiplex (--rate) has to shed, so there gaps are reported rather than failed, but +# the capture must still end on the source's last packet. if [[ -n "$PASSTHROUGH" ]]; then echo if ! python3 "$DIR/passthrough.py" "$SRC_TS" "$SUB_TS" ${RATE:+--loss-ok} --report-json "$HARNESS_RUN/passthrough.json"; then