diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index b00cf7a19af..1ac645ff8cb 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -141,6 +141,11 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: TS pair controls + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test ts-pair-controls + shell: bash -leo pipefail {0} + # Round-trips a PCR-paced TS through a relay and checks the subscriber's # `export ts` output with TSDuck (tsp/tsanalyze from the nix devShell). - name: TS compliance diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a98a76d8594..801b84d1712 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -217,6 +217,12 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just rs c-tests + - name: TS table alignment + if: ${{ !cancelled() }} + run: nix develop --command just test ts --pair + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The TS exporter's PCR release timing. Everything else about the stream # can be graded from a capture, but *when* the bytes were handed over # cannot: the grader has to sit on the exporter's stdout for the length of @@ -232,6 +238,15 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just test ts --live --duration 120 + - name: Upload TS pair failure artifacts + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: nightly-ts-pair + path: ${{ runner.temp }}/moq-test-runs + if-no-files-found: ignore + retention-days: 7 + tests: name: Tests (${{ matrix.recipe }}) runs-on: ubuntu-24.04-arm diff --git a/quest/m1/README.md b/quest/m1/README.md index e55b407d71c..18e9c19e8a4 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -67,7 +67,6 @@ session outcomes, and embedder refusals inside session outcomes. - [Untimed lite-07](/quest/m1/lite-untimed.md) - lite-07 carries an untimed track in both languages; lite-05/06 write send time - [A watch and publish release ships assets()](/quest/m1/assets-release.md) - the release that lets the sites host the worklets - [Dogfood hosted worklets](/quest/m1/dogfood-assets.md) - the moq.pro dashboard hosts the worklets and calls `assets()` after the release -- [TS pair acquisition](/quest/m1/ts-pair-acquisition.md) - fix the clean baseline and prove the grader's controls before #5328 enables its scheduled hard gate - [Initial catalog routing](/quest/m1/initial-catalog-routing.md) - a usable fallback does not strand initial catalog acquisition with a spurious Unroutable - [More tests under load](/quest/m1/test-flakes-2/README.md) - the second round of load-only failures, one quest per flake, fixed at the cause - [END_OF_TRACK placement](/quest/m1/ietf-end-of-track-placement.md) - END_OF_TRACK rides the upstream's Location, and Rust's header stops claiming END_OF_GROUP diff --git a/quest/m1/initial-catalog-routing.md b/quest/m1/initial-catalog-routing.md index a747ec0997e..1b26a0d9fef 100644 --- a/quest/m1/initial-catalog-routing.md +++ b/quest/m1/initial-catalog-routing.md @@ -59,4 +59,4 @@ decision rather than an invented retry or compatibility layer. - [Transport upgrade](/quest/m1/transport-upgrade/README.md) - existing handover and epoch contracts - [TS duration fidelity](/quest/m0/ts-duration-fidelity.md) - previously captured only a short suffix, not zero-packet initial acquisition - [Replacement catalog linger](/quest/m1/ts-follow-catalog-bound.md) - bounds a midstream stitch, not initial discovery -- [Pair acquisition](/quest/m1/ts-pair-acquisition.md) - independent grader/mux finding +- [Pair validation](/test/ts/README.md#table-anchoring-table-anchorpy) - independent grader/acquisition checks diff --git a/quest/m1/ts-pair-acquisition.md b/quest/m1/ts-pair-acquisition.md deleted file mode 100644 index c630b83afdc..00000000000 --- a/quest/m1/ts-pair-acquisition.md +++ /dev/null @@ -1,60 +0,0 @@ -# [M] TS pair validation distinguishes acquisition from persistent misalignment - -## Goal - -The late-joining pair test accepts supported clean acquisition and detects -persistent table misalignment without mistaking interleave for emission time. -Its positive baseline and negative controls run in CI before the scheduled -hard gate in #5328 is merged. - -## Plan - -Decided with the maintainer on 2026-10-11: fix the finding, not just diagnose -it. Keep this separate from initial catalog routing and the broader redundant -TS coordinate/API planning in #5327. Fix the baseline before #5328 merges; -do not temporarily make the pair test non-gating. - -The validation run in #5328 used `just test ts --pair`: two exports of one -45-second generated broadcast, the second joining five seconds late. Both -exporters and the publisher exited successfully. PAT/PMT agreement was 100%; -SDT had 20 common placements out of 23 in the union, or 86.96%, below the -existing 90% gate over 41 seconds of shared media. The mismatches were near -acquisition; 20 consecutive later SDT placements matched. These observations -do not establish permanent phase drift. - -`test/ts/table-anchor.py` tags a table with the next PES PTS and infers a -timer-driven phase difference from median cadence. Check those assumptions -against the current slot scheduler and the capture's actual table placement. -Correct the grader, the mux, or both as the evidence requires; do not assume -the exporter is defective because that diagnostic says so. - -Define and document the acquisition behavior being graded for the generated -profile. Any distinction between acquisition tables and recurring tables must -follow an observable stream condition with automated evidence. Do not ignore -an arbitrary number of seconds, reduce the agreement or coverage thresholds, -or shrink the shared window to the surviving table emissions. Keep missing -expected tables and inadequate evidence distinguishable from a passing check. - -Add deterministic controls for supported late acquisition and changed PES -interleave, plus persistent phase offset, drift and table loss within shared -media. The controls must fail for their intended reasons and must prevent a -grader that accepts everything from passing. Keep table carriage, cadence and -packet identity separate: this quest does not promise byte-identical TS legs, -introduce `--sync`, or claim ST 2022-7 compatibility. - -Run the short controls on relevant PRs and the real pair arm at least nightly, -using shared `just test` recipes. Preserve captures and reports on failure. -Require a passing clean pair baseline and failing negative controls before -unblocking #5328; a green PR run that skips the pair arm is not that evidence. -Update `test/ts/README.md` and misleading grader comments/diagnoses inline. - -Public API and wire: none expected. New shared-coordinate or synchronization -contracts remain with the broader alignment planning. - -## Related - -- [Validation PR #5328](https://github.com/moq-dev/moq/pull/5328) - its scheduled pair gate must wait for this fix -- [Pair gate review](https://github.com/moq-dev/moq/pull/5328#discussion_r4243103700) - a known-failing baseline would generate daily alerts -- [TS roadmap #5327](https://github.com/moq-dev/moq/pull/5327) - owns the broader redundant alignment planning -- [Deterministic TS legs](/quest/m2/ts-hitless.md) - packet identity requires a separate contract and proof -- [Initial catalog routing](/quest/m1/initial-catalog-routing.md) - distinct acquisition failure found by the same validation work diff --git a/test/justfile b/test/justfile index ad8a62bfa7f..98512d40741 100644 --- a/test/justfile +++ b/test/justfile @@ -121,6 +121,10 @@ drill-sensitivity *args: ts *args: ./ts/run.sh "$@" +# Table-placement grader: acquisition, interleave, phase, drift and missing-table controls. +ts-pair-controls: + python3 ./ts/table-anchor.test.py + # Round-trip the EIT fixtures: the sparse schedule must survive and # pending-version sections must be dropped at import. [positional-arguments] diff --git a/test/ts/README.md b/test/ts/README.md index 47e495ebe6c..1b1a5ebe592 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -389,76 +389,52 @@ checks to hard. ## Table anchoring (`table-anchor.py`) -Every other check here grades one stream in isolation, and no single stream can -answer the question this one asks: **are a table's emission points a property of -the broadcast, or of the exporter that happened to emit them?** - -It matters because two exporters of one broadcast are how redundancy is built. A -receiver merging two legs, or cutting from one to the other, needs them to agree -about where the tables sit. If each leg emits to its own clock the two outputs -are not interchangeable, and the divergence is permanent rather than something a -longer run settles. - -One stream cannot show this because a timer and an anchor look identical from -inside: both produce a table every so often. Only a second leg, joining at a -different moment, separates them. +Two exporters of the same broadcast must converge after acquisition. `--pair` +starts the second subscriber five seconds late and checks table placement on the +shared PCR timeline. This measures table alignment; it does not establish packet +identity, `--sync`, or ST 2022-7 compatibility. ```bash -just test ts --pair # round-trip twice, grade the pair -just test ts --pair --pair-join 20 --duration 60 # join the second leg 20s in -./table-anchor.py a.ts b.ts # grade two captures you already have +just test ts --pair +just test ts --pair --pair-join 20 --duration 60 +just test ts-pair-controls +./table-anchor.py a.ts b.ts --report-json pair.json ``` -The second subscriber joins late on purpose. Two exporters started together can -agree on a cadence by having started together, which is the confound; a leg that -joins mid-broadcast has to derive its emission points from the media, because it -has no shared history to derive them from. - -**What gets graded is the overlap, not the run**, so `--pair` defaults to a 45 s -duration rather than the 20 s the other arms use, and refuses a run leaving less -than 25 s of it. At 20 s with a 5 s join the legs share 15 s, which is about -seven SDT emissions against a floor of eight — the table this mode exists to -check would quietly drop to report-only. Raise `--duration` when you raise -`--pair-join`. `--pair` cannot be combined with `--live`: they grade different -things. - -The measurement is the PTS of the frame each table was emitted against. `export -ts` writes the tables that are due and then the frame's PES packets into one -buffer, so the first PES header after a table gives the PTS of the frame that -triggered it. **Agreement** is the emission points both legs used over those -either used, counted only inside the media time the two captures share, so a -late join costs nothing. 100 % means the emission points are a function of the -broadcast. +The CBR slot scheduler interleaves tables and PES, so the next PES timestamp does not +identify when a table was sent. The grader interpolates each table's packet +position between PCR samples. A match allows one packet of placement quantization +plus one PCR tick of rounding. Matching is one-to-one, so duplicates count. + +The comparison starts at the later leg's **second video random-access PES**, +after its first acquired group, matching the exporter's late-join convergence +regression. This boundary comes from media markers, never from the tables under +test or a fixed time allowance. The end is the earlier leg's final PCR. Tables +that disappear cannot shorten their own scoring window. Captures with multiple +program clocks, variable bitrate, midstream PCR discontinuities, missing acquisition markers, or +broken packet framing are refused. PCR wrap is supported. + +The default run lasts 45 seconds and requires 20 seconds of acquired shared +clock. PAT, declared PMTs, and every observed non-clock table must each have at +least eight emissions **on both legs**, with at least 90% agreement. Missing or +sparse tables fail rather than quietly becoming report-only. TDT/TOT remains +informational because its content follows wall-clock time; `--strict` grades it +as well. `--source` also requires the source's SI tables and PAT-declared PMTs, catching loss on both legs. These checks do not diagnose a process timer from cadence alone. | Option | Meaning | |---|---| -| `--min-agreement` | percent of shared emission points required per table (default 90) | -| `--min-emissions` | a table with fewer emissions in the overlap is reported, not graded (default 8) | -| `--min-window` | seconds of shared media required before any verdict is given (default 20) | -| `--strict` | fail on report-only tables too | -| `--report-json` | write the full report | - -`--min-window` exists because a capture that came up short is the commonest way -this grades clean: too little shared media puts the slower tables under the -emission floor, and a table reported without a verdict reads as a pass. The -window is taken from the media the two captures carry, **not** from the -emissions being scored — deriving it from the emissions is itself a false pass, -since a leg that stops emitting a table halfway through would pull the upper -bound back to its own last emission and score its desertion as 100 %. - -Two things are deliberately not graded. A table seen on only one leg is reported -without a verdict, because that is a carriage question rather than an anchoring -one. TDT/TOT is report-only whatever its agreement, since it carries wall-clock -time and is *supposed* to track a clock rather than the media. - -Where both legs run the same period at different phase, the report says so -rather than leaving a percentage to interpret: - -``` - FAIL SDT/BAT 10 10 19 1 5.26% - both legs emit every 2.000s, 0.480s out of phase - -> a timer started with the exporter, not an anchor in the media -``` +| `--min-agreement` | Required percentage of matched emissions (default 90) | +| `--min-emissions` | Required emissions per leg and table (default 8) | +| `--min-window` | Required seconds of acquired shared clock (default 20) | +| `--source` | Require the source's SI tables on both legs | +| `--strict` | Grade TDT/TOT too | +| `--report-json` | Save the window, tolerance, counts, and verdicts | + +`just test ts-pair-controls` runs deterministic transport fixtures through the +actual grader. Identical streams, different PES interleave, late acquisition with +bootstrap tables, and PCR wrap must pass. Persistent phase shifts, accumulating +drift, disappearing/absent/sparse tables, and insufficient shared media must fail. +Controls run in PR CI; the real pair runs nightly and on manual TS validation. ## Open GOP (`open-gop.py`) diff --git a/test/ts/run.sh b/test/ts/run.sh index 530449b453d..25222be2273 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -510,7 +510,8 @@ if [[ -n "$PAIR" ]]; then fi echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') + $(wc -c <"$SUB_B_TS" | tr -d ' ') bytes -> comparing table anchors" echo - if ! python3 "$DIR/table-anchor.py" "$SUB_TS" "$SUB_B_TS" $STRICT \ + if ! python3 "$DIR/table-anchor.py" "$SUB_TS" "$SUB_B_TS" --source "$SRC_TS" \ + --report-json "$HARNESS_RUN/table-anchor.json" $STRICT \ ${PASSTHRU[@]+"${PASSTHRU[@]}"}; then echo >&2 echo "error: table anchor analysis failed (see round-trip logs below)" >&2 diff --git a/test/ts/table-anchor.py b/test/ts/table-anchor.py index 58f1d3c9e1b..75dee3beef4 100644 --- a/test/ts/table-anchor.py +++ b/test/ts/table-anchor.py @@ -1,44 +1,31 @@ #!/usr/bin/env python3 -"""Grade whether a table's emission points belong to the broadcast or to the process. - - ./table-anchor.py a.ts b.ts - -`export ts` writes a frame by emitting any due tables and then that frame's PES -packets into one buffer, so the first PES header after a table belongs to the -frame that triggered it. Tagging each table emission with that frame's PTS says -what two exporters of one broadcast cannot otherwise be asked: whether they put -a table in the same place in *media* time, or merely at the same rate. - -That distinction is the whole property. A cadence driven by the exporter's own -clock still produces a correct stream on each leg and still emits at the right -average interval; it just emits at different points, so the legs are two -transports of one programme rather than one transport sent twice. Anything that -merges or switches between them - an ST 2022-7 receiver, a standby packager, a -late joiner expected to converge on a running peer - needs the second. - -Why not compare bytes. Two legs order the same frames differently, because the -exporter emits whichever track has a frame in hand and arrival order is a -property of the network path rather than of the broadcast. A packet-by-packet -walk desynchronises on that regardless of where the tables land, so it cannot -answer this question even in principle. Indexing by PTS is immune to it, which -isolates the cadence from the interleave. - -Comparison is restricted to the media-time window the two legs share, so a late -joiner is graded on the overlap rather than penalised for the start it missed. - -Exit status is 0 when every hard check passes, 1 otherwise. `--strict` promotes -the report-only (shape) checks to hard. +"""Compare table placement on the PCR timeline after a complete acquisition group. + +Tables and PES are interleaved by the slot scheduler. The next PES PTS is not the +emission time: interpolate packet position between PCRs instead. Start at the +second video random-access PES on each leg, when its first acquired group is out, +as the exporter's late-join convergence tests do. The end comes from the clock, +never from the tables, so disappearing tables cannot shorten their own test. + +This checks table placement, not packet identity or hitless switching. It supports +one continuous program clock and video carrying random_access_indicator; refuse +captures that cannot establish those conditions rather than guess a grace period. """ import argparse import json import sys +from bisect import bisect_right from collections import defaultdict +from dataclasses import dataclass +from importlib import import_module + +parse_pcr = import_module("pcr-timing").parse_pcr +CLOCK = 27_000_000 +WRAP = (1 << 33) * 300 PKT = 188 SYNC = 0x47 -HARD, SHAPE = "hard", "shape" -NULL_PID = 0x1FFF # PIDs whose table is fixed by the standard. A PMT sits on a PID the PAT chooses, # so it is discovered rather than listed. @@ -57,16 +44,16 @@ def packets(path): - """188-byte packets from a TS file, resynchronising on the first stable sync run.""" + """Read the exporter's complete 188-byte packets, refusing broken framing.""" with open(path, "rb") as f: data = f.read() - start = 0 - while start + PKT * 8 < len(data): - if all(data[start + i * PKT] == SYNC for i in range(8)): - break - start += 1 - end = len(data) - (len(data) - start) % PKT - return (data[i : i + PKT] for i in range(start, end, PKT)) + if not data or len(data) % PKT: + raise ValueError(f"{path}: empty or truncated transport") + for at in range(0, len(data), PKT): + pkt = data[at : at + PKT] + if pkt[0] != SYNC or pkt[1] & 0x80 or not pkt[3] & 0x30: + raise ValueError(f"{path}: malformed packet at {at}") + yield pkt def pid_of(pkt): @@ -97,13 +84,7 @@ def parse_pts(pkt): if len(p) < 14 or not (p[7] & 0x80): # PTS_DTS_flags return None b = p[9:14] - return ( - ((b[0] >> 1) & 0x07) << 30 - | b[1] << 22 - | ((b[2] >> 1) & 0x7F) << 15 - | b[3] << 7 - | ((b[4] >> 1) & 0x7F) - ) + return ((b[0] >> 1) & 0x07) << 30 | b[1] << 22 | ((b[2] >> 1) & 0x7F) << 15 | b[3] << 7 | ((b[4] >> 1) & 0x7F) def pmt_pids(path): @@ -111,7 +92,7 @@ def pmt_pids(path): Sections are reassembled across packets. A PAT carrying more than about forty programmes does not fit in one packet, and stopping at the packet boundary would - silently find only the programmes that happened to land in the first one — which + silently find only the programmes that happened to land in the first one; which on a full multiplex is a subset, with no error to say so. """ found = set() @@ -150,242 +131,174 @@ def pmt_pids(path): return found -def anchors(path, pmts): - """Frame PTS values each table was emitted at, and the capture's media bounds. +@dataclass +class Capture: + tables: dict + start: float + end: float + positions: list[int] + clocks: list[int] + + def rate(self, window): + """Packet duration in the shared window, refusing non-CBR interpolation.""" + lo = max(0, bisect_right(self.clocks, window[0]) - 1) + hi = bisect_right(self.clocks, window[1]) + clocks, positions = self.clocks[lo:hi], self.positions[lo:hi] + if len(clocks) < 2: + raise ValueError("insufficient shared PCR samples") + ticks = (clocks[-1] - clocks[0]) / (positions[-1] - positions[0]) + # Two integer PCR samples bound the rounding error in the fitted slope. + if any(abs(t - clocks[0] - (i - positions[0]) * ticks) > 2 for i, t in zip(positions, clocks)): + raise ValueError("pair grading requires a constant-rate shared PCR/packet timeline") + return ticks - The bounds are returned alongside because the scoring window has to come from - the media the capture covers, not from the emissions themselves — see - `agreement()`. - """ - out = defaultdict(list) - pending = [] - lo = hi = None + +def anchors(path, pmts): + """Table positions and the independently observed, acquired clock window.""" + clocks, positions = [], [] + tables, access = defaultdict(list), defaultdict(list) + clock_pid = None watched = set(WELL_KNOWN) | pmts - for pkt in packets(path): + for index, pkt in enumerate(packets(path)): pid = pid_of(pkt) - if pid == NULL_PID: - continue - if pid in watched: - if pkt[1] & 0x40: - pending.append(pid) - continue - pts = parse_pts(pkt) - if pts is None: - continue - lo = pts if lo is None or pts < lo else lo - hi = pts if hi is None or pts > hi else hi - if pending: - for table in pending: - out[table].append(pts) - pending = [] - return out, (lo, hi) + pcr = parse_pcr(pkt) + if pcr is not None: + if clock_pid is not None and pid != clock_pid: + raise ValueError("pair grading requires a single program clock") + clock_pid = pid + if clocks: + if pkt[5] & 0x80: + raise ValueError("PCR discontinuity inside pair capture") + delta = (pcr - clocks[-1]) % WRAP + if not 0 < delta < WRAP // 2: + raise ValueError("non-increasing program clock") + pcr = clocks[-1] + delta + clocks.append(pcr) + positions.append(index) + if pid in watched and pkt[1] & 0x40: + tables[pid].append(index) + off = payload_offset(pkt) + if parse_pts(pkt) is not None and 0xE0 <= pkt[off + 3] <= 0xEF: + # Record the PID even when every random-access flag is missing. + points = access[pid] + if pkt[3] & 0x20 and pkt[4] and pkt[5] & 0x40: + points.append(index) + if len(clocks) < 2 or not access or any(len(v) < 2 for v in access.values()): + raise ValueError("need PCR and two video random-access points per video PID") + + def at(index): + i = bisect_right(positions, index) - 1 + if i < 0 or i >= len(clocks) - 1: + raise ValueError("acquisition point is outside the measured PCR window") + return clocks[i] + (index - positions[i]) * (clocks[i + 1] - clocks[i]) / (positions[i + 1] - positions[i]) + + start = max(at(v[1]) for v in access.values()) + return Capture( + {pid: [at(i) for i in points if positions[0] <= i < positions[-1]] for pid, points in tables.items()}, + start, + clocks[-1], + positions, + clocks, + ) def name(pid, pmts): - if pid in WELL_KNOWN: - return WELL_KNOWN[pid] - return f"PMT {pid:#06x}" if pid in pmts else f"PID {pid:#06x}" - - -def agreement(a_pts, b_pts, window): - """Share of emission points the legs share, over the media time they both cover. + return WELL_KNOWN.get(pid, f"PMT {pid:#06x}" if pid in pmts else f"PID {pid:#06x}") - The window is the media both captures carry, and it must come from the captures - rather than from the emissions being scored. Deriving it from the emissions - instead is a false pass: a leg that stops emitting a table halfway through pulls - the upper bound back to its own last emission, so the partner's later emissions - fall outside the window and the desertion scores 100 %. - Returns None when the captures do not overlap in media time at all, which is a - different answer from "they overlap and disagree" and must not be scored as 0. - """ +def agreement(a, b, window, tolerance): + """One-to-one matches within one packet, retaining losses and duplicates.""" lo, hi = window - if lo is None or hi is None or lo > hi: - return None - oa = {t for t in a_pts if lo <= t <= hi} - ob = {t for t in b_pts if lo <= t <= hi} - union = oa | ob - if not union: - return None - shared = oa & ob - return len(oa), len(ob), len(union), len(shared), 100.0 * len(shared) / len(union) - - -def cadence(pts): - """The period a leg emits on, in seconds, or None if it keeps no steady one. - - "Steady" has to tolerate a few odd gaps rather than demand none: a leg that - re-phases once on joining, or drops an emission, is still running a timer, and a - spread test over the extremes would miss exactly the case worth catching. So the - period is the median gap and the leg counts as steady when most gaps sit on it. - """ - pts = sorted(pts) - if len(pts) < 4: - return None - gaps = sorted((pts[i + 1] - pts[i]) / 90000.0 for i in range(len(pts) - 1)) - period = gaps[len(gaps) // 2] - if period <= 0: - return None - on_period = sum(1 for g in gaps if abs(g - period) <= 0.05 * period) - return period if on_period >= 0.6 * len(gaps) else None - - -def phase(pts, period): - """Where a leg's grid sits inside one period, as a robust median. - - Taken relative to the first point so that a grid straddling the modulus does not - split into two clusters and median to a value in neither. - """ - base = min(pts) / 90000.0 - offs = sorted(((t / 90000.0 - base) % period) for t in pts) - return offs[len(offs) // 2] + base - - -def diagnose(a_pts, b_pts): - """Say why two legs disagree, when the shape of the disagreement is legible. - - The case worth naming is a table both legs emit on the same steady period but at - different phase: that is a timer started when the exporter started, and no amount - of running time will bring the two legs back together. - """ - pa, pb = cadence(a_pts), cadence(b_pts) - if pa is None or pb is None or abs(pa - pb) > 0.05 * max(pa, pb): - return None - offset = abs(phase(a_pts, pa) - phase(b_pts, pa)) % pa - offset = min(offset, pa - offset) - if offset <= 0.01 * pa: - return None - return ( - f"both legs emit every {pa:.3f}s, {offset:.3f}s out of phase " - f"-> a timer started with the exporter, not an anchor in the media" - ) + a = [t for t in a if lo <= t <= hi] + b = [t for t in b if lo <= t <= hi] + i = j = shared = 0 + while i < len(a) and j < len(b): + if abs(a[i] - b[j]) <= tolerance: + shared += 1 + i += 1 + j += 1 + elif a[i] < b[j]: + i += 1 + else: + j += 1 + union = len(a) + len(b) - shared + return len(a), len(b), union, shared, 100 * shared / union if union else 0 def main(): - ap = argparse.ArgumentParser(description="Grade table emission points across two exporters.") - ap.add_argument("a", help="TS captured from the first exporter") - ap.add_argument("b", help="TS captured from the second exporter") - ap.add_argument( - "--min-agreement", - type=float, - default=90.0, - help="percent of shared emission points required per table (default 90)", - ) - ap.add_argument( - "--min-emissions", - type=int, - default=8, - help="a table with fewer emissions in the overlap is reported, not graded (default 8)", - ) - ap.add_argument( - "--min-window", - type=float, - default=20.0, - help="seconds of shared media required before any verdict is given (default 20)", - ) - ap.add_argument("--strict", action="store_true", help="fail on shape checks too") - ap.add_argument("--report-json", help="write the full report here") + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("a") + ap.add_argument("b") + ap.add_argument("--min-agreement", type=float, default=90.0) + ap.add_argument("--min-emissions", type=int, default=8) + ap.add_argument("--min-window", type=float, default=20.0) + ap.add_argument("--source", help="require the source's non-clock SI tables on both legs") + ap.add_argument("--strict", action="store_true", help="also grade clock tables") + ap.add_argument("--report-json") args = ap.parse_args() - - pmts = pmt_pids(args.a) | pmt_pids(args.b) - a, (a_lo, a_hi) = anchors(args.a, pmts) - b, (b_lo, b_hi) = anchors(args.b, pmts) - if not a and not b: - print("error: neither capture carries a table on a known PID", file=sys.stderr) - return 1 - if None in (a_lo, a_hi, b_lo, b_hi): - print("error: a capture carries no PTS, so there is no media time to compare in", file=sys.stderr) - return 1 - # The media both captures carry. Every table is scored inside this one window, so a - # table one leg abandons is scored against the partner's emissions rather than - # silently shrinking the window to hide them. - window = (max(a_lo, b_lo), min(a_hi, b_hi)) - shared_s = (window[1] - window[0]) / 90000.0 - print(f"### shared media window: {shared_s:.1f}s") - # A window too short to carry the slower tables would put them under the emission - # floor and report them without a verdict, which reads as a pass. Refuse the run - # instead: a capture that came up short is the commonest way this grades clean. - if shared_s < args.min_window: - print( - f"error: the captures share only {shared_s:.1f}s of media, below the " - f"{args.min_window:.0f}s needed for a verdict", - file=sys.stderr, - ) - print(" run for longer, join earlier, or lower --min-window deliberately", file=sys.stderr) - return 1 - - rows = [] - for pid in sorted(set(a) | set(b)): - label = name(pid, pmts) - scored = agreement(a.get(pid, []), b.get(pid, []), window) - if scored is None: - rows.append((pid, label, len(a.get(pid, [])), len(b.get(pid, [])), None, None, "one leg only")) - continue - na, nb, union, shared, pct = scored - # A table emitted a handful of times cannot separate "anchored to the media" - # from "coincidence", so it is reported without a verdict rather than graded - # on a sample too small to mean anything. - severity = SHAPE if pid in NOT_GRADED or union < args.min_emissions else HARD - rows.append((pid, label, na, nb, union, shared, pct, severity)) - - width = max(len(r[1]) for r in rows) - print(f"### table anchor report - {len(rows)} table(s)") - print() - print(f" {'':4} {'table':<{width}} {'A':>7} {'B':>7} {'either':>7} {'both':>7} agreement") - failed = 0 - report = [] - for row in rows: - if len(row) == 7: - pid, label, na, nb, _, _, note = row - print(f" {'--':4} {label:<{width}} {na:>7,} {nb:>7,} {'-':>7} {'-':>7} {note}") - report.append({"pid": pid, "table": label, "a": na, "b": nb, "graded": False, "note": note}) - continue - pid, label, na, nb, union, shared, pct, severity = row - ok = pct >= args.min_agreement - if ok: - verdict = "PASS" - elif severity == HARD or args.strict: - verdict = "FAIL" - failed += 1 - else: - verdict = "WARN" - print( - f" {verdict:4} {label:<{width}} {na:>7,} {nb:>7,} {union:>7,} {shared:>7,} {pct:8.2f}%" - ) - why = None if ok else diagnose(a.get(pid, []), b.get(pid, [])) - if why: - print(f" {'':4} {'':<{width}} {why}") - report.append( - { - "pid": pid, - "table": label, - "a": na, - "b": nb, - "either": union, - "both": shared, - "agreement_pct": round(pct, 4), - "severity": severity, - "graded": True, - "ok": ok, - "diagnosis": why, - } + try: + pmts = pmt_pids(args.a) | pmt_pids(args.b) + if args.source: + pmts |= pmt_pids(args.source) + if not pmts: + raise ValueError("no PAT program mapping") + expected = {pid_of(pkt) for pkt in packets(args.source) if pid_of(pkt) in WELL_KNOWN} if args.source else set() + a, b = anchors(args.a, pmts), anchors(args.b, pmts) + # Captures may start on opposite sides of the 33-bit wrap. + shift = round((a.start - b.start) / WRAP) * WRAP + b.start += shift + b.end += shift + b.clocks = [t + shift for t in b.clocks] + b.tables = {pid: [t + shift for t in points] for pid, points in b.tables.items()} + window = max(a.start, b.start), min(a.end, b.end) + shared_s = (window[1] - window[0]) / CLOCK + if shared_s < args.min_window: + raise ValueError(f"only {shared_s:.3f}s of acquired shared clock; need {args.min_window}s") + tolerance = max(a.rate(window), b.rate(window)) + 1 + rows = [] + # PAT and its declared PMTs are mandatory, even when absent after acquisition. + for pid in sorted({0} | pmts | expected | set(a.tables) | set(b.tables)): + na, nb, union, shared, pct = agreement(a.tables.get(pid, []), b.tables.get(pid, []), window, tolerance) + graded = pid not in NOT_GRADED or args.strict + enough = min(na, nb) >= args.min_emissions + ok = enough and pct >= args.min_agreement + rows.append( + dict( + pid=pid, + table=name(pid, pmts), + a=na, + b=nb, + either=union, + both=shared, + agreement_pct=round(pct, 4), + graded=graded, + ok=ok, + note=None if enough else "insufficient emissions", + ) + ) + report = dict( + window_seconds=[t / CLOCK for t in window], + shared_seconds=shared_s, + tolerance_ticks=tolerance, + min_agreement=args.min_agreement, + tables=rows, ) - - print() - print( - "agreement = emission points both legs used, over those either used, inside the\n" - "media time they share. 100% means the emission points are a function of the\n" - "broadcast; a low figure with a healthy count on both legs means each leg is\n" - "emitting to its own clock." - ) - print() - print(f"{failed} check(s) failed" if failed else "all checks passed") - + print(f"### acquired shared clock: {shared_s:.3f}s") + for row in rows: + verdict = "PASS" if row["ok"] else "FAIL" if row["graded"] else "INFO" + print( + f"{verdict}: {row['table']}: {row['both']}/{row['either']} ({row['agreement_pct']:.2f}%)" + + (f"; {row['note']}" if row["note"] else "") + ) + failed = any(row["graded"] and not row["ok"] for row in rows) + except ValueError as error: + report = dict(error=str(error)) + print(f"error: {error}", file=sys.stderr) + failed = True if args.report_json: with open(args.report_json, "w") as f: - json.dump({"min_agreement": args.min_agreement, "tables": report}, f, indent=2) - - return 1 if failed else 0 + json.dump(report, f, indent=2) + return int(failed) if __name__ == "__main__": diff --git a/test/ts/table-anchor.test.py b/test/ts/table-anchor.test.py new file mode 100644 index 00000000000..23fe30b1778 --- /dev/null +++ b/test/ts/table-anchor.test.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Exercise the pair grader through complete synthetic transport captures.""" + +import json +import subprocess +import sys +import tempfile +import unittest +from importlib import import_module +from pathlib import Path + +fixtures = import_module("tstd-controls") +PKT, PMT, VIDEO = fixtures.packet, 0x1000, 0x100 +CLOCK = 27_000_000 + + +def capture( + path, *, start=0, phase=0, drift=0, loss=None, interleave=False, bootstrap=False, wrap=False, extra_program=False +): + pat = fixtures.section(0, 1, bytes.fromhex("0001f0000002f001" if extra_program else "0001f000")) + pmt = fixtures.section(2, 1, bytes.fromhex("e100f0001be100f000")) + sdt = fixtures.section(0x42, 1, bytes.fromhex("0001ff")) + # A packet per millisecond; PCR every 25 packets, independently of tables/PES. + events = {} + for at in range(7, 30000, 2000): + events[at + phase + (at // 2000) * drift] = 0x11 + if bootstrap: + events[start + 7] = 0x11 + with open(path, "wb") as out: + for at in range(start, 30001): + pid, payload, pusi, adaptation = 0x1FFF, b"", False, None + if at % 25 == 0: + ticks = (at * CLOCK // 1000 + (((1 << 33) * 300 - 10 * CLOCK) if wrap else 0)) % ((1 << 33) * 300) + base, ext = divmod(ticks, 300) + adaptation = b"\x10" + ((base << 15) | (0x3F << 9) | ext).to_bytes(6, "big") + pid = VIDEO + elif at % 500 == 2: + pid, payload, pusi = 0, b"\0" + pat, True + elif at % 500 == 3: + pid, payload, pusi = PMT, b"\0" + pmt, True + elif extra_program and at % 500 == 4: + pid, payload, pusi = 0x1001, b"\0" + fixtures.section(2, 2, bytes.fromhex("e100f0001be100f000")), True + elif at in events: + pid, payload, pusi = events[at], b"\0" + sdt, True + elif at % 20 == 10: + # Changing which PES follows a table does not change table placement. + audio = interleave and at % 1000 != 10 + pid = 0x101 if audio else VIDEO + pts = (at + (600 if audio else 200)) * 90 + payload = bytes.fromhex("000001") + bytes([0xC0 if audio else 0xE0]) + payload += bytes.fromhex("0000808005") + fixtures.stamp(2, pts) + b"media" + pusi = True + adaptation = b"\x40" if not audio and at % 1000 == 10 else None + if pid == 0x11 and (loss == "all" or loss == "tail" and at > 14000 or loss == "sparse" and at % 4000 != 7): + pid, payload, pusi = 0x1FFF, b"", False + out.write(PKT(pid, 0, payload, pusi, adaptation)) + + +class Controls(unittest.TestCase): + def test_source_declared_program(self): + with tempfile.TemporaryDirectory(prefix="ts-pair-source-") as directory: + directory = Path(directory) + source, output, report = directory / "source.ts", directory / "output.ts", directory / "report.json" + capture(source, extra_program=True) + capture(output) + result = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("table-anchor.py")), + str(output), + str(output), + "--source", + str(source), + "--report-json", + str(report), + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + missing = next(row for row in json.loads(report.read_text())["tables"] if row["pid"] == 0x1001) + self.assertTrue(missing["graded"]) + self.assertFalse(missing["ok"]) + self.assertEqual((missing["a"], missing["b"]), (0, 0)) + + def test_controls(self): + with tempfile.TemporaryDirectory(prefix="ts-pair-controls-") as directory: + directory = Path(directory) + a, b, report = directory / "a.ts", directory / "b.ts", directory / "report.json" + capture(a) + source = directory / "source.ts" + source.write_bytes(a.read_bytes()) + cases = [ + ("identical", {}, True), + ("different PES interleave", dict(interleave=True), True), + ("late acquisition and bootstrap", dict(start=3500, bootstrap=True, interleave=True), True), + ("persistent phase", dict(start=3500, phase=100), False), + ("drift", dict(start=3500, drift=10), False), + ("table disappears in shared media", dict(loss="tail"), False), + ("table absent", dict(loss="all"), False), + ("table absent on both legs", dict(loss="all"), False), + ("sparse emissions", dict(loss="sparse"), False), + ("sparse on both legs", dict(loss="sparse"), False), + ("insufficient shared media", dict(start=15000), False), + ] + for label, options, expected in cases: + with self.subTest(label=label): + capture( + a, + loss=options["loss"] if label in ("table absent on both legs", "sparse on both legs") else None, + ) + capture(b, **options) + result = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("table-anchor.py")), + str(a), + str(b), + "--source", + str(source), + "--report-json", + str(report), + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0 if expected else 1, result.stdout + result.stderr) + parsed = json.loads(report.read_text()) + if label == "insufficient shared media": + self.assertEqual(parsed["error"], "only 13.990s of acquired shared clock; need 20.0s") + else: + self.assertNotIn("error", parsed) + sdt = next(r for r in parsed["tables"] if r["pid"] == 0x11) + self.assertTrue(sdt["graded"]) + self.assertEqual(sdt["ok"], expected) + self.assertTrue(all(r["ok"] for r in parsed["tables"] if r["pid"] != 0x11)) + if label in ("persistent phase", "drift"): + self.assertEqual((sdt["a"], sdt["b"]), (12, 12)) + self.assertLess(sdt["agreement_pct"], 90) + elif not expected: + counts = { + "table disappears in shared media": (14, 6), + "table absent": (14, 0), + "table absent on both legs": (0, 0), + "sparse emissions": (14, 7), + "sparse on both legs": (7, 7), + } + self.assertEqual((sdt["a"], sdt["b"]), counts[label]) + print(f"PASS control: {label}") + capture(a, wrap=True) + capture(b, start=11500, wrap=True) + # A shorter requested evidence window isolates wrap alignment; normal pair + # runs retain the default 20-second minimum. + result = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("table-anchor.py")), + str(a), + str(b), + "--min-window", + "15", + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + print("PASS control: captures starting across PCR wrap") + + +if __name__ == "__main__": + unittest.main()