diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index 2776df94b7d..279bcb8ec25 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -59,6 +59,7 @@ on: - release-winget - rpm-repo - Swift + - TS - Video types: - completed diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index e74ed2fb484..f2520b9638f 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -3,8 +3,7 @@ name: Interop # Cross-language interop test built from THIS checkout (not the published # packages, unlike the moq-dev/smoke repo). Stands up a relay and runs the # publish x subscribe matrix across rust / python / go / browser / native-js / c, -# then browser-to-browser media QA and the TS/IRD compliance harness over the -# exporter's output. +# then browser-to-browser media QA. TS validation has its own workflow. on: # Manual on-demand runs. workflow_dispatch: @@ -20,10 +19,6 @@ on: - "test/max-age/**" - "test/portrait/**" - "rs/moq-tokio/tests/max_age.rs" - # The TS/IRD compliance harness runs in the same job (see below). - - "test/ts/**" - # The live TS round trip also checks stdin ingestion keeps up with its source. - - "rs/moq-cli/src/publish.rs" # The interop job also depends on the recipe that invokes it and the # workspace membership that links the JS clients to source. - "test/justfile" @@ -149,74 +144,6 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - name: TS pair controls - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts-pair-controls - shell: bash -leo pipefail {0} - - # Round-trips a PCR-paced TS through a relay and checks the subscriber's - # `export ts` output with TSDuck (tsp/tsanalyze from the nix devShell). - - name: TS compliance - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # The same round-trip at 2 Mb/s, where every keyframe outgrows a 25 ms PCR slot, - # so the export has to spread it over the slots before its DTS and still keep - # the constant-rate schedule. - - name: TS keyframe bursts - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts --bitrate 2000000 - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # 1080p video filling a broadcast-sized 9 Mbit CPB, at the default delay: the - # source sends pictures most of a second ahead of their decode time, and the - # export has to as well without overflowing the receiver's buffers. - - name: TS broadcast-sized CPB - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts --hrd - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # The same video beside four audio PIDs in a multiplex faster than its Rx: each slot - # has to interleave the video with the rest, or its transport buffer overflows. - - name: TS multiplex above the video's Rx - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts --headroom - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # The same round-trip over an open-GOP clip: every leading picture after a - # recovery point must come back, in decode order. - - name: TS open GOP - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts --open-gop - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # Builds the sparse-EPG and pending-version fixtures and round-trips - # them, asserting the schedule survives and pending sections are dropped. - - name: EIT fixtures - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts-eit - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # The T-STD model must pass a broadcast capture and fail the same capture - # delivered too slowly, too early, or in a burst. - - name: T-STD controls - if: ${{ !cancelled() && steps.setup.outcome == 'success' }} - run: nix develop --command just test ts-tstd - shell: bash -leo pipefail {0} - # A failing harness run keeps its directory, holding the process logs, the # relay config, and a Playwright trace of the failing page. Upload it for a # short while so a red cell is diagnosable; a passing run deletes its own. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1c1e1e6b0a7..f075ccb2a1b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -263,36 +263,6 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just rs c-tests - - name: TS table alignment - if: ${{ !cancelled() }} - run: nix develop --command just test ts --pair - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - - # The TS exporter's PCR release timing. Everything else about the stream - # can be graded from a capture, but *when* the bytes were handed over - # cannot: the grader has to sit on the exporter's stdout for the length of - # the run, stamping each packet. That is a real-time window no per-PR gate - # should pay for, so it lives here (see test/ts/README.md). - # - # 120 s rather than the 20 s default, because the drift term needs it. The - # exporter builds its mux buffer once and holds it, and a window that ends - # while it is still filling cannot tell that from a pipe running slow: the - # lag reaches ~480 ms over the first ~48 s and is flat after. Nightly is the - # one arm with the wall-clock time to see the flat part. - - name: TS timing - if: ${{ !cancelled() }} - run: nix develop --command just test ts --live --duration 120 - - - name: Upload TS pair failure artifacts - if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: nightly-ts-pair - path: ${{ runner.temp }}/moq-test-runs - if-no-files-found: ignore - retention-days: 7 - tests: name: Tests (${{ matrix.recipe }}) runs-on: ubuntu-24.04-arm diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml new file mode 100644 index 00000000000..85b01c848a6 --- /dev/null +++ b/.github/workflows/ts.yml @@ -0,0 +1,166 @@ +name: TS + +# Decoder, transport structure, buffer-model and release-timing gates are +# independent of browser interop. Decoder and transport gates have real-tool controls. +on: + workflow_dispatch: + schedule: + - cron: "0 8 * * *" + pull_request: + paths: + - "test/ts/**" + - "test/lib/**" + - "test/justfile" + - "rs/moq-mux/**" + - "rs/hang/**" + - "rs/moq-cli/**" + - "rs/moq-net/**" + - "rs/moq-tokio/**" + - "rs/moq-relay/**" + - "rs/moq-loc/**" + - "rs/moq-json/**" + # Workspace dependencies of moq-net, moq-tokio and moq-mux on the TS path. + - "rs/kio/**" + - "rs/moq-flate/**" + - "rs/moq-msf/**" + - "rs/moq-pattern/**" + - "rs/moq-sock/**" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "flake.nix" + - "flake.lock" + - ".github/actions/rust-cache/**" + - ".github/workflows/ts.yml" + +concurrency: + group: ts-${{ github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + ts: + name: TS validation + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + - name: Rust cache + uses: ./.github/actions/rust-cache + + - name: FFmpeg and TSDuck controls + if: ${{ !cancelled() }} + run: nix develop --command just test ts-controls + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + + - name: TS pair controls + if: ${{ !cancelled() }} + run: nix develop --command just test ts-pair-controls + + # Round-trips a PCR-paced TS through a relay and checks the subscriber's + # `export ts` output with TSDuck (tsp/tsanalyze from the nix devShell). + - name: TS compliance + if: ${{ !cancelled() }} + run: nix develop --command just test ts + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # The same round-trip at 2 Mb/s, where every keyframe outgrows a 25 ms PCR slot, + # so the export has to spread it over the slots before its DTS and still keep + # the constant-rate schedule. + - name: TS keyframe bursts + if: ${{ !cancelled() }} + run: nix develop --command just test ts --bitrate 2000000 + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # 1080p video filling a broadcast-sized 9 Mbit CPB, at the default delay: the + # source sends pictures most of a second ahead of their decode time, and the + # export has to as well without overflowing the receiver's buffers. + - name: TS broadcast-sized CPB + if: ${{ !cancelled() }} + run: nix develop --command just test ts --hrd + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # The same video beside four audio PIDs in a multiplex faster than its Rx: each slot + # has to interleave the video with the rest, or its transport buffer overflows. + - name: TS multiplex above the video's Rx + if: ${{ !cancelled() }} + run: nix develop --command just test ts --headroom + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # The same round-trip over an open-GOP clip: every leading picture after a + # recovery point must come back, in decode order. + - name: TS open GOP + if: ${{ !cancelled() }} + run: nix develop --command just test ts --open-gop + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # Builds the sparse-EPG and pending-version fixtures and round-trips + # them, asserting the schedule survives and pending sections are dropped. + - name: EIT fixtures + if: ${{ !cancelled() }} + run: nix develop --command just test ts-eit + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # The T-STD model must pass a broadcast capture and fail the same capture + # delivered too slowly, too early, or in a burst. + - name: T-STD controls + if: ${{ !cancelled() }} + run: nix develop --command just test ts-tstd + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + + # The pair needs enough overlap to grade the slower table cadences. + - name: TS table alignment + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + run: nix develop --command just test ts --pair + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + + # Live timing needs the real-time window to distinguish drift from startup. + - name: TS live release timing + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + run: nix develop --command just test ts --live --duration 120 + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + + - name: Upload failure artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ts-failure + path: ${{ runner.temp }}/moq-test-runs + retention-days: 7 + if-no-files-found: error diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index f1f6a1a06e5..92dcd32392e 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -55,7 +55,6 @@ remain with their current quests. ## Required -- [Automated TS validation](/quest/m1/ts-reliability/validation.md) - software decode, structure, timing, and controls run in CI - [Bounded PES assembly](/quest/m1/ts-reliability/pes.md) - split headers work and damaged input cannot grow staging indefinitely - [Plan: semantic fidelity](/quest/m1/ts-reliability/fidelity.md) - explicit profiles and PMT lifecycle contracts produce implementation quests - [Plan: PES timing and headers](/quest/m1/plan-ts-pes-untimed.md) - private PES semantics have a settled carriage and placement contract diff --git a/quest/m1/ts-reliability/fidelity.md b/quest/m1/ts-reliability/fidelity.md index 2cc629b868c..1db26d4d7f1 100644 --- a/quest/m1/ts-reliability/fidelity.md +++ b/quest/m1/ts-reliability/fidelity.md @@ -52,5 +52,5 @@ section/PES bytes only where the settled contract actually promises that. ## Related - [PES timing and headers](/quest/m1/plan-ts-pes-untimed.md) - separate placement, decode and presentation semantics -- [Automated validation](/quest/m1/ts-reliability/validation.md) - profile declarations link to executable evidence +- [Automated validation](/test/ts/README.md) - profile declarations link to executable evidence - [Typed SCTE-35](/quest/m3/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md) - application cue APIs follow basic opaque carriage diff --git a/quest/m1/ts-reliability/pes.md b/quest/m1/ts-reliability/pes.md index 66199ca6198..87c98177f83 100644 --- a/quest/m1/ts-reliability/pes.md +++ b/quest/m1/ts-reliability/pes.md @@ -40,5 +40,5 @@ retain its regression rather than duplicating that fix. ## Related -- [Automated validation](/quest/m1/ts-reliability/validation.md) - independent parsing and decoding of the reassembled output +- [Automated validation](/test/ts/README.md) - independent parsing and decoding of the reassembled output - [PES timing and headers](/quest/m1/plan-ts-pes-untimed.md) - defines semantics retained after assembly diff --git a/quest/m1/ts-reliability/recovery.md b/quest/m1/ts-reliability/recovery.md index b8285ff3761..edce1671009 100644 --- a/quest/m1/ts-reliability/recovery.md +++ b/quest/m1/ts-reliability/recovery.md @@ -40,10 +40,8 @@ Short deterministic arms run on relevant PRs. Long drift and impairment arms run nightly with failure artifacts. Keep byte-position PCR accuracy and measured release timing as separate checks. -## Required - -- [Automated validation](/quest/m1/ts-reliability/validation.md) - shared graders, profiles and known-bad controls ## Related +- [Automated validation](/test/ts/README.md) - shared graders, profiles and known-bad controls - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims after a gap - [Gradual recovery](/quest/m2/intra-refresh/README.md) - recovery counts and per-group warmup diff --git a/quest/m1/ts-reliability/validation.md b/quest/m1/ts-reliability/validation.md deleted file mode 100644 index eaa1cd96b1d..00000000000 --- a/quest/m1/ts-reliability/validation.md +++ /dev/null @@ -1,71 +0,0 @@ -# [L] TS validators run as automated tests - -## Goal - -TSDuck, FFmpeg, and the repository's TS graders assert the supported output -profiles in repeatable CI tests. A green run means the expected content was -decoded and the named timing and structure checks passed, not merely that a -tool parsed a file or returned zero. - -## Plan - -Required by the maintainer in the 2026-10-11 audit: every TS validation is -automated, including any additional validator introduced later. Use the pinned -Nix tools and share local `just test` recipes with CI. Reuse `test/ts` rather -than introducing a second harness. - -Current gaps: - -- `run.sh` uses FFmpeg to generate sources but never fully decodes the - exported audio/video. `ffprobe_opus_channels` checks metadata only. -- `just test ts --pair` and `table-anchor.py` exist but no workflow invokes - that arm. Add a scheduled gate at minimum. -- `interop.yml` triggers for the harness but not `rs/moq-mux/**`. Relevant - mux, catalog, and TS CLI changes need the short integration gates. -- Plain captured runs do not reject a failed publisher, and the subscriber - is reaped without its status contributing to the verdict. A partial file - can pass analysis after the pipeline has failed. - -Add a reusable full software-decode grader for supported clean profiles, -starting with the existing closed-GOP H.264/AAC generator. FFmpeg must fail -on decode errors, decode every expected stream, and produce nonzero video -frames and audio samples where the source has them. Preserve diagnostics and -per-stream counts. A successful probe or concealment-tolerant decode is not -the assertion. Prove the grader rejects malformed coded payload and a missing -expected stream with automated controls that fail for the intended reason. - -The first declared profile also compares decoded content against the source -over the admitted interval. Match video frames and aligned audio samples rather -than requiring identical demux packet boundaries. Account explicitly for -startup, decoder priming, and end trimming; never loosen an existing duration -check to accommodate missing content. A grader that only checks decode -acceptance and nonzero progress must report that narrower result until the -source comparison passes. Open-GOP acquisition expectations land -with the existing leading-pictures work. - -Keep TSDuck's structural checks and explicit PCR-accuracy threshold, the -byte-schedule grader, and live release timing as distinct assertions. A CBR -byte-position PCR check is not a wall-clock jitter measurement. VBR and MPTS -with independent clocks need their own applicable timing model before they -can be labelled covered. Retain process failures, reports, tool versions, -input fixtures or reproducible generation parameters, and failing output as -CI artifacts. - -The custom T-STD model currently excludes the PSI system buffer, refuses -several layouts/codecs, and borrows AAC's buffer size for Opus. Report these -limits by profile. Coordinate system-buffer controls with `tstd/psi-runs` -and MB/AAC controls with the existing m2 quest; do not duplicate them or call -the current model complete. Correct the blanket claim in the harness docs -that no maintained tool implements T-STD: commercial analyzers advertise -it, while no suitable maintained open-source replacement was established. - -Independent failure reporting landed in #5297; keep its workflow shape, and -ensure an earlier unrelated failure cannot suppress these tests. All new -controls have CI invocations too. - -## Related - -- [T-STD controls](/quest/m2/tstd-controls.md) - MB and AAC positive/negative coverage -- [PSI system buffer](/quest/m1/tstd/psi-runs.md) - system-buffer proof belongs with the PSI fix -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - acquisition trims change the decoder oracle -- [Loss and recovery](/quest/m1/ts-reliability/recovery.md) - extends these graders to declared omissions and recovery diff --git a/quest/m1/ts-same-instance-resume.md b/quest/m1/ts-same-instance-resume.md index fdade49da6a..9b1e9e4c05e 100644 --- a/quest/m1/ts-same-instance-resume.md +++ b/quest/m1/ts-same-instance-resume.md @@ -49,9 +49,14 @@ as acceptable. Every regression runs in CI, at least nightly for long integratio coverage. Completion requires the causal test and the relevant full checks, not a successful isolated rerun after a loaded failure. -Keep validation PR #5328 and catalog PR #5333 draft until their readiness -findings are resolved. Update affected test documentation and comments -inline. No new public API, wire format, or user guide is expected; a newly +Decided by the maintainer 2026-10-13: this quest no longer gates validation +PR #5328, and #5333 has merged. None of 137 runs reproduced the exit 1. With the +exporter's stderr kept in the assertion, 97 passed here: batches of 16 and 48 concurrent, 24 +concurrent runs of the whole `export` binary, 8 with the WebSocket race forced +(`MOQ_CONNECT_WEBSOCKET_DELAY=0s`), and a full `just check`, on `main` and the +merged validation branch. Another 40 of 40 passed in an earlier measurement on #5328. The +quest stays open on its own until a failure is captured with its stderr. Update +affected test documentation and comments inline. No new public API, wire format, or user guide is expected; a newly required behavior or API change returns to the maintainer for a decision. ## Related diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index 8fad2fd8577..f4e60e36e6d 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -57,5 +57,5 @@ MB/AAC controls. Do not relabel the current subset a full standards check. ## Related - [TS passthrough export](/quest/m2/ts-passthrough-export.md) - the passthrough lane named in the Goal -- [Automated validation](/quest/m1/ts-reliability/validation.md) - independent decode, profile coverage and CI ownership +- [Automated validation](/test/ts/README.md) - independent decode, profile coverage and CI ownership - [Loss and recovery](/quest/m1/ts-reliability/recovery.md) - actual object/group omissions and recovery diff --git a/quest/m2/ts-passthrough-export.md b/quest/m2/ts-passthrough-export.md index 6ee9738a281..8038e109afc 100644 --- a/quest/m2/ts-passthrough-export.md +++ b/quest/m2/ts-passthrough-export.md @@ -88,6 +88,6 @@ Public API: a new export mode; nothing existing breaks. - [Deterministic TS legs](/quest/m2/ts-hitless.md) - the demultiplexed lane's common packet identity under `--sync` - [T-STD TS export](/quest/m1/tstd/README.md) - the jitter buffer and clock recovery this release shares -- [TS validation](/quest/m1/ts-reliability/validation.md) - shared automatic graders and controls +- [TS validation](/test/ts/README.md) - shared automatic graders and controls - [Loss and recovery](/quest/m1/ts-reliability/recovery.md) - actual missing content and reacquisition proof - [Same-epoch importers](/quest/m1/hop-aligned-import.md) - identical common objects do not prove independent importers publish identical groups diff --git a/quest/m2/tstd-controls.md b/quest/m2/tstd-controls.md index 7c7002e9c1f..83a9d8b03ef 100644 --- a/quest/m2/tstd-controls.md +++ b/quest/m2/tstd-controls.md @@ -25,5 +25,5 @@ The only positive reference carries MPEG audio, but the export emits AAC. ## Related -- [TS validation](/quest/m1/ts-reliability/validation.md) - reports supported model coverage and assumptions without claiming full conformance +- [TS validation](/test/ts/README.md) - reports supported model coverage and assumptions without claiming full conformance - [PSI system buffer](/quest/m1/tstd/psi-runs.md) - owns the separate system-buffer assertion and regression diff --git a/test/justfile b/test/justfile index fdacef8cd45..d449ed53066 100644 --- a/test/justfile +++ b/test/justfile @@ -116,7 +116,7 @@ drill-sensitivity *args: ./drill/sensitivity.sh "$@" # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export -# ts` output with TSDuck plus a custom analyzer (the T-STD buffer model, PCR +# ts` output with FFmpeg, TSDuck and a custom analyzer (the T-STD buffer model, PCR # spacing and byte schedule). Flags: `--analyze-only file.ts` # (skip the round-trip), `--strict` (fail on shape warnings), `--source file.ts` # (use a real capture), `--live` (grade PCR release timing and byte position off @@ -144,7 +144,11 @@ ts-eit *args: # The T-STD model's controls: a broadcast capture must pass, and the same capture # delivered too slowly, too early, or in a burst must fail. Needs only TSDuck. ts-tstd: - ./ts/tstd-controls.py + bash ./ts/tstd-controls.sh + +# Prove FFmpeg decode/content and TSDuck structure/PCR assertions reject damaged TS. +ts-controls: + bash ./ts/controls.sh # A JS viewer watching a live track through a relay that drains (SIGTERM, after # a stand-in for DNS stops pointing at it) migrates to a second relay behind the diff --git a/test/ts/README.md b/test/ts/README.md index 583ac1a7223..48398d1ef2c 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -5,26 +5,25 @@ what an Integrated Receiver/Decoder (IRD) expects. It round-trips a stream through a relay (`import ts` -> relay -> `export ts`), captures the output, and runs [TSDuck](https://tsduck.io) plus a custom analyzer over it. -This is a diagnostic gate, not just a pass/fail: the exporter -([`rs/moq-mux/src/container/ts/export.rs`](../../rs/moq-mux/src/container/ts/export.rs)) -pads with null packets to the multiplex rate the source recorded (or `--mux-rate`), -leaves a source without one unpadded, never delays media to fit the rate, and -puts a PCR on its own packet every 25 ms of media time, so several -broadcast-shape checks are expected to flag. The report quantifies exactly where -and by how much. - -Four instruments live here. `compliance.py` (via `run.sh`) grades a captured file -against the IRD model. [`pcr-timing.py`](#pcr-timing-pcr-timingpy) grades a live -pipe, which is the only way to see *when* the exporter released each PCR. -[`table-anchor.py`](#table-anchoring-table-anchorpy) grades two exporters of one -broadcast against each other, which is the only way to see whether a table's -emission points come from the media or from the exporter's own clock. -[`open-gop.py`](#open-gop-open-goppy) grades an open-GOP capture against its -source, access unit by access unit. +The harness keeps software decoding, transport structure, decoder-buffer models, +and release timing as separate assertions. Passing them establishes the named +software profiles below; it does not certify a hardware IRD or every TS layout. + +- `decode.py` runs strict software FFmpeg decoding and compares captured content + with the source for generated closed-GOP H.264/AAC-LC streams. +- `compliance.py` uses TSDuck for structure and a custom elementary-stream T-STD + model for supported profiles. +- `accuracy.py` requires measured PCRs within 500 ns of their CBR byte position. +- `pcr-timing.py` checks PCR values and byte schedules from files, and release + timing from a live pipe. +- `table-anchor.py` compares table emission points between two subscribers. +- `open-gop.py` checks the coded access units and leading pictures of open GOPs. ## Running ```bash +just test ts-controls # real FFmpeg/TSDuck positive and negative controls +just test ts-tstd # buffer-model controls just test ts # generate a clip, round-trip, analyze just test ts --source cap.ts # round-trip a real capture instead just test ts --analyze-only x.ts # skip the round-trip, analyze a file @@ -52,10 +51,11 @@ the first PCR, failing on any PCR more than 500 ns off its byte position at the rate: the `PCR_accuracy_error` an IRD or a TR 101 290 probe reports, which `pcr-schedule`'s packet of slack lets through. -The live arm passes only when the grader's verdict *and* the publisher's exit status -are clean. The grader can only speak for what reached it, and the sample floor -rejects a window that came up short, so a publisher dying late in the run leaves -enough behind to pass every check: a broken round-trip reported as a good one. +Publisher and captured subscriber processes must exit successfully. Captures drain +through EOF within the existing process deadline; a nonempty partial file cannot +hide a failed pipeline or timeout. The live arm also requires its grader to pass; +after its measurement window, the reader drains through source EOF so neither a +broken pipe nor another exporter failure needs to be ignored. `--analyze-only` needs only TSDuck + Python, so you can point it at any captured subscriber output: @@ -66,7 +66,7 @@ moq --connect http://localhost:4443 --broadcast live.hang export ts > sub.ts ``` Requirements: `tsp`, `tsanalyze` and `tstables` (TSDuck) and `python3` for every mode; the -round-trip modes also need `cargo`, `ffmpeg`, `curl`, and `timeout`. +round-trip modes also need `cargo`, `ffmpeg`, `ffprobe`, `curl`, and `timeout`. ## Checks @@ -89,7 +89,7 @@ Severities: **hard** checks fail the run by default; **shape** checks report as | `pcr-monotonic` | hard | PCR strictly increases (one 33-bit wrap tolerated), except into a PCR that signals `discontinuity_indicator` | | `duration-fidelity` | hard | exported PCR span tracks the source's duration (round-trip only) | | `service-descriptors` | shape | an SDT naming the service is present | -| `tstd` | hard | the full T-STD buffer model: no TB, MB, EB or B overflow, no access unit incomplete at its decoding time (see [T-STD](#t-std)) | +| `tstd` | hard | the supported elementary-stream T-STD models: no TB, MB, EB or B overflow, no access unit incomplete at its decoding time (see [T-STD](#t-std)) | Every timing check reads the stream's own PCR, so a PCR emitted on the wrong clock rate stays internally consistent and passes them all. `duration-fidelity` @@ -105,7 +105,7 @@ machine-readable report. ## T-STD -`tstd` runs every audio and video stream through the ISO 13818-1 system target +`tstd` models supported audio and video streams using the ISO 13818-1 system target decoder (2.4.2; Rec. ITU-T H.222.0, whose 10/2014 edition is a free download), fed on the stream's own PCR clock: @@ -120,9 +120,9 @@ second, where an access unit is not wholly in EB/B at its decoding time AVC/HEVC). The report gives each stream's peak fill per buffer, how late the worst underflowed access unit finished arriving, and the longest any access unit waited. -No maintained tool implements this. TSDuck has no T-STD analyzer, and nothing else -in nixpkgs does either, so the model is hand-rolled and its parameters are -transcribed from the specs: H.264 Table A-1 and H.265 Table A.8 for the level, the +TSDuck supplies the transport analysis, not this decoder-buffer model. Commercial +analyzers advertise T-STD checks; no suitable maintained open-source replacement +was established for this harness. Its parameters are transcribed from the specs: H.264 Table A-1 and H.265 Table A.8 for the level, the ADTS and "other audio" rates and sizes in H.222.0 2.4.2.3, and ATSC A/52 and A/53 Part 5 for AC-3 and E-AC-3. TSDuck does the parsing: `tstables` decodes the PMT and `tsp -P pes --avc-access-unit` the SPS, AVC and HEVC alike. @@ -141,9 +141,9 @@ Opus is graded against ADTS's buffers for the same channel count: the Opus-in-TS draft gives Rx (2 Mb/s for 1-2 channels, matching ADTS) but leaves the buffer size unset, so that size is borrowed rather than specified. A stream with no parameters at all is refused by name rather than skipped, which fails the check: MPEG-1/2 -video, DVB E-AC-3, and HEVC beyond Main/Main 10. Sections and private data -(SCTE-35, teletext) have no elementary-stream buffers and are listed as not -graded. A signalled PCR discontinuity starts fresh buffers, since the timestamps +video, DVB E-AC-3, and HEVC beyond Main/Main 10. The PSI system buffer is not implemented. Sections and private data +(SCTE-35, teletext) are listed as not graded; passing elementary-stream buffers +does not establish conformance for them. A signalled PCR discontinuity starts fresh buffers, since the timestamps on either side of it are on different clocks. Every packet on an elementary stream's PID enters TB, including adaptation-only @@ -560,27 +560,64 @@ than by malfunction; the census makes that visible instead of leaving it to be inferred from the code. Counts differ for a table that *did* survive because the exporter re-emits SI on its own repetition cadence rather than the source's. +## Decoded content (`decode.py`) + +The generated closed-GOP H.264/AAC-LC profile, at both 10 and 2 Mb/s, fully +decodes every expected video/audio stream with software FFmpeg. Missing streams, +empty output, decoder errors (including error logs with a zero exit status), and +configuration changes fail. Video frame hashes must match an intact source +suffix through EOF; frame timestamps must keep one common rebase. + +Audio timestamps must remain contiguous, allowing the 90 kHz PTS quantization +and FFmpeg's conversion to the sample clock. Audio sample counts pin the admitted +source offset against the same video rebase, so a shifted audio track or a lost +tail fails. Exact PCM comparison uses a fresh decode of source AAC starting at +the same access unit: AAC overlap and perceptual-noise state differ between a +cold decoder and a decoder that saw earlier frames. No arbitrary warmup samples +are discarded. PES boundaries and TS packetization may change. + +`just test ts-controls` exercises unchanged input, remuxing with new timestamps +and PES sizes, timestamp quantization, and a cold join. Negative controls require +the intended failure for missing audio, changed pictures, changed audio, a lost +tail, A/V skew, video clock drift, an audio gap, corrupted coded payload, and +empty input. Real TSDuck controls remove PAT, break continuity, set TEI, and check +PCR against a wrong rate or an empty measurement. Process controls also require +nonzero/timeout exit codes to fail despite nonempty output. A missing tool or +an unexpected failure category is a failing test, not a successful negative. + +Other codecs, open-GOP acquisition, loss/recovery, raw carriage, and arbitrary +source captures do not yet have decoded-content comparisons. Their existing +specific checks remain active and the harness explicitly reports the narrower +coverage. VBR and independent-clock MPTS need applicable timing models before +claiming the CBR PCR-accuracy profile. + ## CI -`.github/workflows/interop.yml` runs `just test ts`, `just test ts --bitrate -2000000`, `just test ts --hrd`, `just test ts --headroom`, -`just test ts --open-gop`, `just test ts-eit`, and `just test ts-tstd` after the -interop matrix (nightly, on demand, and on PRs touching `test/ts/`). -`ts-eit` is `eit-roundtrip.sh`: it builds the sparse-schedule and -pending-version fixtures from a generated clip, round-trips them through a -relay, and censuses the capture, so a break in the generators or in the SI -carriage they pin fails a PR instead of landing silently. TSDuck comes from the -`nix develop` shell, so the run uses the same `tsp`/`tsanalyze` a local -developer would. - -`.github/workflows/nightly.yml` runs `just test ts --live --duration 120` as -well. It stays off the PR path because release timing needs a real-time window to -measure: the grader has to sit on the pipe for the length of the capture, which -no per-PR gate should be paying for, and a scheduled arm on a quiet runner is a -better place to notice the exporter drifting off its own clock anyway. The window -is 120 s rather than the 20 s default because that is what the drift term costs -to read: under a minute it catches the mux buffer still filling and cannot tell -that from a pipe running slow. +The round-trip rig explicitly negotiates `moq-lite-07-wip` on the relay, +publisher and subscribers. Its publisher epochs allow acquisition to continue +when WebSocket upgrades to QUIC. Production defaults remain unchanged: Lite-06 +cannot prove publisher identity across session replacement and must refuse a +pending acquisition. `rs/moq-net/tests/catalog_acquisition.rs` covers both +outcomes deterministically in Rust CI, without retries or implicit `--stitch`. +Actual transport teardown coverage belongs to the transport-upgrade tests; a +clean TS run alone does not prove that an upgrade occurred. + +[`.github/workflows/ts.yml`](../../.github/workflows/ts.yml) is independent of +browser interop. Relevant mux, Hang, CLI, transport, harness and toolchain PRs run +`just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, headroom, +open-GOP, EIT, and T-STD controls. Each step runs even after another test fails. +All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. + +Nightly and manual runs also invoke `just test ts --pair` and +`just test ts --live --duration 120`. The pair exercises late-joining subscribers; +the live window distinguishes ongoing clock drift from initial buffer filling. +These are actual test invocations, not optional tools installed on a runner. + +Failed runs retain source/capture files, process logs and statuses, tool versions, +decoder commands, frame hashes, PCM, and grader reports under the harness run +directory. CI uploads that directory for seven days. `MOQ_TEST_KEEP=1` preserves +successful local evidence too. `ts-eit` retains its sparse-schedule and pending +version fixtures; `ts-tstd` retains its existing model controls. ## Caveats diff --git a/test/ts/accuracy.py b/test/ts/accuracy.py new file mode 100644 index 00000000000..0c828286db9 --- /dev/null +++ b/test/ts/accuracy.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Assert TSDuck PCR accuracy against a known CBR byte rate.""" + +import argparse +import re +import subprocess +from pathlib import Path + + +def check(path, bitrate, skip, log): + command = [ + "tsp", + "-I", + "file", + str(path), + "-P", + "skip", + str(skip), + "-P", + "pcrverify", + "--bitrate", + str(bitrate), + "--absolute", + "--jitter-max", + "13", + "-O", + "drop", + ] + # 13 ticks at 27 MHz is the integral threshold within +/-500 ns. Do not + # rely on pcrverify's much looser default or treat no measured PCRs as a pass. + with log.open("w") as output: + result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, check=False) + if result.returncode: + raise RuntimeError(f"tsp exited {result.returncode}: {log}") + matches = re.findall(r"([\d,]+) PCR OK, ([\d,]+) with", log.read_text()) + if len(matches) != 1: + raise RuntimeError(f"expected one pcrverify verdict: {log}") + good, bad = (int(v.replace(",", "")) for v in matches[0]) + if not good or bad: + raise RuntimeError(f"PCR accuracy failed: {good} OK, {bad} outside threshold: {log}") + print(f"PCR accuracy: {good} OK, 0 outside +/-500 ns") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("ts", type=Path) + parser.add_argument("--bitrate", required=True, type=int) + parser.add_argument("--skip", default=0, type=int) + parser.add_argument("--log", required=True, type=Path) + args = parser.parse_args() + check(args.ts, args.bitrate, args.skip, args.log) diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 1a5651c94bd..67b8096ae3e 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -8,15 +8,15 @@ Division of labour: TSDuck does the transport-stream parsing (we shell out to `tsanalyze --json` for PSI/service/structure and `tstables` for the PMT), and this script does what TSDuck does not cover: the ISO 13818-1 T-STD buffer -model, which no maintained tool implements, and the PCR checks that need the +model for supported elementary streams, and the PCR checks that need the time-base discontinuities a 188/204-byte header scan here recovers. PCR value intervals, byte schedule and release timing live in `pcr-timing.py`. Checks split into two severities: - HARD (structural): fail the run by default. PAT/PMT, packet size, sync, - continuity counters, PSI CRC, PCR presence, PCR monotonicity. + continuity counters, PSI CRC, PCR presence, PCR monotonicity, T-STD buffers. - SHAPE (broadcast profile): reported as WARN and only fail the run under - `--strict`. Service descriptors (SDT), T-STD buffer model. + `--strict`. Service descriptors (SDT). Timing basis is the stream's own PCR clock (an IRD locks to PCR), so the harness needs no wall-clock capture and results are deterministic for a given file. @@ -77,7 +77,7 @@ def run_tsanalyze(ts_path: str) -> dict: text=True, check=False, ) - if proc.returncode != 0 and not proc.stdout: + if proc.returncode != 0: raise RuntimeError(f"tsanalyze failed: {proc.stderr.strip()}") data = json.loads(proc.stdout) data["_stderr"] = proc.stderr diff --git a/test/ts/controls.py b/test/ts/controls.py new file mode 100644 index 00000000000..79a23e79794 --- /dev/null +++ b/test/ts/controls.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Run real FFmpeg and TSDuck positive/negative controls without a relay.""" + +import json +import subprocess +import sys +from pathlib import Path + +import accuracy +import compliance +import decode + + +def command(args, log): + with log.open("w") as output: + subprocess.run(args, stdout=output, stderr=subprocess.STDOUT, check=True) + + +def main(root): + root.mkdir(parents=True, exist_ok=True) + source = root / "source.ts" + ffmpeg = ["ffmpeg", "-nostdin", "-y", "-hide_banner", "-v", "error"] + command( + ffmpeg + + [ + "-f", + "lavfi", + "-i", + "testsrc2=size=160x96:rate=25", + "-f", + "lavfi", + "-i", + "aevalsrc=sin(2*PI*(300*t+70*t*t)):s=48000", + "-t", + "3", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-threads", + "1", + "-x264-params", + "keyint=25:min-keyint=25:scenecut=0", + "-c:a", + "aac", + "-f", + "mpegts", + "-muxrate", + "1000000", + "-pcr_period", + "20", + "-pes_payload_size", + "0", + str(source), + ], + root / "generate.log", + ) + for tool, flag in [ + ("ffmpeg", "-version"), + ("ffprobe", "-version"), + ("tsp", "--version"), + ("tsanalyze", "--version"), + ]: + command([tool, flag], root / f"{tool}-version.txt") + + cases = [("unchanged", source, None)] + for name, options, expected in [ + ( + "remuxed", + ["-map", "0:v", "-map", "0:a", "-c", "copy", "-output_ts_offset", "5", "-pes_payload_size", "3000"], + None, + ), + ( + "audio-quantization", + ["-map", "0:v", "-map", "0:a", "-c", "copy", "-bsf:a", "setts=pts=PTS-mod(N\\,3):dts=DTS-mod(N\\,3)"], + None, + ), + ("missing-audio", ["-map", "0:v", "-c", "copy"], "streams"), + ( + "different-video", + ["-map", "0:v", "-map", "0:a", "-vf", "negate", "-c:v", "libx264", "-c:a", "copy"], + "video-content", + ), + ( + "different-audio", + ["-map", "0:v", "-map", "0:a", "-c:v", "copy", "-af", "atrim=start_sample=1024,volume=0", "-c:a", "aac"], + "audio-content", + ), + ("truncated-tail", ["-map", "0:v", "-map", "0:a", "-c", "copy", "-t", "2"], "video-content"), + ( + "audio-shift", + ["-map", "0:v", "-map", "0:a", "-c", "copy", "-bsf:a", "setts=pts=PTS+9000:dts=DTS+9000"], + "audio-length", + ), + ( + "video-rate", + ["-map", "0:v", "-map", "0:a", "-c", "copy", "-bsf:v", "setts=pts=PTS*1.01:dts=DTS*1.01"], + "video-timing", + ), + ( + "audio-gap", + [ + "-map", + "0:v", + "-map", + "0:a", + "-c", + "copy", + "-bsf:a", + "setts=pts=PTS+gte(N\\,60)*9000:dts=DTS+gte(N\\,60)*9000", + ], + "audio-timing", + ), + ]: + path = root / f"{name}.ts" + command(ffmpeg + ["-i", str(source)] + options + ["-f", "mpegts", str(path)], root / f"{name}.log") + cases.append((name, path, expected)) + + joined = root / "joined.ts" + command( + ffmpeg + ["-ss", "1", "-i", str(source), "-map", "0:v", "-map", "0:a", "-c", "copy", str(joined)], + root / "joined.log", + ) + cases.append(("joined", joined, None)) + + # Keep the transport/PES headers and parameter sets, damaging an actual + # coded slice mid-stream. This must reach the decoder, not just fail probing. + damaged = bytearray(source.read_bytes()) + changed = False + for off in range(len(damaged) // 2 // 188 * 188, len(damaged), 188): + pid = (damaged[off + 1] & 0x1F) << 8 | damaged[off + 2] + if pid != 0x100 or not damaged[off + 3] & 0x10 or damaged[off + 1] & 0x40: + continue + start = off + 4 + (1 + damaged[off + 4] if damaged[off + 3] & 0x20 else 0) + if start < off + 188: + damaged[start : off + 188] = b"\xff" * (off + 188 - start) + changed = True + break + assert changed, "fixture has no video continuation payload to corrupt" + corrupt = root / "corrupt-video.ts" + corrupt.write_bytes(damaged) + cases.append(("corrupt-video", corrupt, "decode")) + empty = root / "empty.ts" + empty.write_bytes(b"") + cases.append(("empty", empty, "decode")) + + results = [] + for name, path, expected in cases: + try: + report = decode.compare(source, path, root / name) + actual = None + except decode.Failure as error: + actual = error.check + report = {"check": actual, "detail": str(error)} + results.append({"case": name, "expected": expected, "actual": actual, "report": report}) + (root / "decode-controls.json").write_text(json.dumps(results, indent=2) + "\n") + assert actual == expected, f"{name}: expected {expected}, got {actual}: {report}" + if name == "joined": + assert report["startup_video_frames"] > 0 and report["cold_audio_reference"], report + print(f"PASS FFmpeg {name}: {actual or 'source content matches'}", flush=True) + + base = source.read_bytes() + variants = {"clean": base} + no_pat = bytearray(base) + for off in range(0, len(no_pat), 188): + if (no_pat[off + 1] & 0x1F) == 0 and no_pat[off + 2] == 0: + no_pat[off : off + 188] = b"\x47\x1f\xff\x10" + b"\xff" * 184 + variants["missing-pat"] = no_pat + cc = bytearray(base) + tei = bytearray(base) + off = len(base) // 2 // 188 * 188 + # Choose a media payload, since null PID continuity is intentionally ignored. + while ((base[off + 1] & 0x1F) << 8 | base[off + 2]) != 0x100 or not base[off + 3] & 0x10: + off += 188 + cc[off + 3] ^= 1 + tei[off + 1] |= 0x80 + variants.update(continuity=cc, transport_error=tei) + for name, data in variants.items(): + path = root / f"structure-{name}.ts" + path.write_bytes(data) + analysis = compliance.run_tsanalyze(str(path)) + checks = [ + *compliance.check_pat_pmt(analysis), + compliance.check_sync(analysis), + compliance.check_continuity(analysis), + ] + failed = {c.name for c in checks if c.status == compliance.Status.FAIL} + expected = {"missing-pat": "pat", "continuity": "continuity", "transport_error": "sync"}.get(name) + (root / f"structure-{name}.json").write_text(json.dumps(analysis, indent=2) + "\n") + assert (expected in failed) if expected else not failed, f"TSDuck {name}: {failed}" + print(f"PASS TSDuck {name}: {sorted(failed)}", flush=True) + + accuracy.check(source, 1_000_000, 0, root / "pcr-clean.log") + for name, rate, skip in [("wrong-rate", 2_000_000, 0), ("no-samples", 1_000_000, len(base) // 188)]: + try: + accuracy.check(source, rate, skip, root / f"pcr-{name}.log") + except RuntimeError as error: + assert "PCR accuracy failed" in str(error), str(error) + else: + raise AssertionError(f"pcrverify accepted {name}") + print(f"PASS TSDuck PCR {name}", flush=True) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/test/ts/controls.sh b/test/ts/controls.sh new file mode 100644 index 00000000000..0c62efc8c87 --- /dev/null +++ b/test/ts/controls.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Real software-validator controls, with evidence retained on failure. +set -euo pipefail +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin ts-controls "just test ts-controls" +python3 "$DIR/controls.py" "$HARNESS_RUN" +bash "$DIR/process.test.sh" diff --git a/test/ts/decode.py b/test/ts/decode.py new file mode 100644 index 00000000000..1d12aedc752 --- /dev/null +++ b/test/ts/decode.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Decode and compare a closed-GOP H.264/AAC TS round trip using software FFmpeg.""" + +import argparse +import json +import subprocess +import sys +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path + +import compliance + +TICK = Fraction(1, 90000) + + +class Failure(Exception): + """A named assertion failure, also used by the negative controls.""" + + def __init__(self, check, detail): + self.check = check + super().__init__(detail) + + +@dataclass +class Frame: + pts: Fraction + duration: int + size: int + digest: str + + +def run(command, log): + """Keep diagnostics even when an external validator fails.""" + with log.open("w") as stderr: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=stderr, check=False) + if result.returncode: + raise Failure("decode", f"{command[0]} exited {result.returncode}: {log}") + return result.stdout + + +def decode(path, output): + output.mkdir(parents=True, exist_ok=True) + probe = run(["ffprobe", "-v", "error", "-show_streams", "-of", "json", str(path)], output / "probe.log") + (output / "probe.json").write_bytes(probe) + streams = json.loads(probe)["streams"] + media = [s for s in streams if s["codec_type"] in ("video", "audio")] + video = [s for s in media if s["codec_type"] == "video"] + audio = [s for s in media if s["codec_type"] == "audio"] + if len(video) != 1 or len(audio) != 1: + raise Failure("streams", f"expected one video and one audio stream, got {len(video)} and {len(audio)}: {path}") + video, audio = video[0], audio[0] + if video["codec_name"] != "h264" or audio["codec_name"] != "aac" or audio.get("profile") != "LC": + raise Failure("profile", f"only the closed-GOP H.264/AAC-LC profile is covered: {path}") + + # Disable timestamp repair, frame duplication, hardware decode and tolerant + # error handling. PCM is also retained so comparison is sample-aligned, not + # tied to the framehash muxer's audio packet boundaries. + command = [ + "ffmpeg", + "-nostdin", + "-y", + "-hide_banner", + "-v", + "error", + "-xerror", + "-err_detect", + "explode", + "-abort_on", + "empty_output+empty_output_stream", + "-copyts", + "-hwaccel", + "none", + "-threads", + "1", + "-i", + str(path), + "-map", + f"0:{video['index']}", + "-map", + f"0:{audio['index']}", + "-c:v", + "rawvideo", + "-threads:v", + "1", + "-fps_mode", + "passthrough", + "-enc_time_base:v", + "1:90000", + "-c:a", + "pcm_s16le", + "-f", + "framehash", + str(output / "frames.sha256"), + "-map", + f"0:{audio['index']}", + "-c:a", + "pcm_s16le", + "-f", + "s16le", + str(output / "audio.pcm"), + "-map", + f"0:{audio['index']}", + "-c:a", + "copy", + "-f", + "adts", + str(output / "audio.aac"), + ] + (output / "command.json").write_text(json.dumps(command, indent=2) + "\n") + run(command, output / "decode.log") + # Some codec errors are logged without affecting the process exit status. + if (output / "decode.log").stat().st_size: + raise Failure("decode", f"FFmpeg logged errors: {output / 'decode.log'}") + frames = [[], []] + timebases = {} + for line in (output / "frames.sha256").read_text().splitlines(): + if line.startswith("#tb "): + index, value = line[4:].split(": ") + timebases[int(index)] = Fraction(value) + elif line and not line.startswith("#"): + index, _dts, pts, duration, size, digest = [x.strip() for x in line.split(",")] + index = int(index) + frames[index].append(Frame(int(pts) * timebases[index], int(duration), int(size), digest)) + if not all(frames): + raise Failure("decode", f"a declared stream decoded no frames: {path}") + return video, audio, frames, (output / "audio.pcm").read_bytes() + + +def audio_reference(output, samples, rate): + """Decode source audio with the same cold-start state as the subscriber.""" + data = (output / "source/audio.aac").read_bytes() + pos = count = 0 + while count < samples: + try: + size, duration = compliance.adts_frame(data, pos) + except compliance.Refused as error: + raise Failure("profile", str(error)) from error + if size < 7 or pos + size > len(data): + raise Failure("profile", "truncated source ADTS frame") + pos += size + count += round(duration * rate) + if count != samples: + raise Failure("audio-timing", "admitted audio does not start at a source AAC access-unit boundary") + reference = output / "reference.aac" + reference.write_bytes(data[pos:]) + log = output / "reference.log" + pcm = run( + [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-v", + "error", + "-xerror", + "-err_detect", + "explode", + "-threads", + "1", + "-i", + str(reference), + "-map", + "0:a:0", + "-c:a", + "pcm_s16le", + "-f", + "s16le", + "pipe:1", + ], + log, + ) + if log.stat().st_size: + raise Failure("decode", f"reference AAC decoder logged errors: {log}") + return pcm + + +def compare(source, capture, output): + """Compare the captured suffix, including its tail, with one shared A/V rebase.""" + sv, sa, sf, spcm = decode(source, output / "source") + cv, ca, cf, cpcm = decode(capture, output / "capture") + for keys, expected, actual in [ + (("width", "height", "pix_fmt"), sv, cv), + (("sample_rate", "channels", "channel_layout"), sa, ca), + ]: + if any(expected.get(k) != actual.get(k) for k in keys): + raise Failure("streams", f"stream configuration changed: {keys}") + + # Startup may admit a suffix. EOF is not a licence to lose the last frames: + # matching backwards pins the tail and detects missing/duplicated pictures. + start = len(sf[0]) - len(cf[0]) + if start < 0 or [f.digest for f in sf[0][start:]] != [f.digest for f in cf[0]]: + raise Failure("video-content", "decoded pictures are not an intact suffix of the source") + offset = cf[0][0].pts - sf[0][start].pts + if any(abs(c.pts - s.pts - offset) > TICK for s, c in zip(sf[0][start:], cf[0])): + raise Failure("video-timing", "video timing changed within the admitted interval") + + rate = int(sa["sample_rate"]) + stride = 2 * sa["channels"] + for label, rows, pcm in [("source", sf[1], spcm), ("capture", cf[1], cpcm)]: + samples = 0 + for frame in rows: + # PTS is quantized at 90 kHz, then FFmpeg rounds it onto the + # sample clock. Relative timestamps can differ by one sample plus + # one TS tick even when every AAC access unit is contiguous. + if abs(frame.pts - rows[0].pts - Fraction(samples, rate)) > TICK + Fraction(1, rate): + raise Failure("audio-timing", f"{label} has a gap or overlap in decoded audio") + if frame.size % stride: + raise Failure("audio-content", f"{label} has an incomplete PCM sample") + samples += frame.size // stride + if not samples or samples * stride != len(pcm): + raise Failure("audio-content", f"{label} PCM length disagrees with decoded frame sizes") + + audio_start = (cf[1][0].pts - offset - sf[1][0].pts) * rate + # PTS quantization and FFmpeg's sample timebase can differ by one sample. + # Content, length and the common video rebase must all agree on that offset. + candidates = [ + n + for n in range(round(audio_start) - 1, round(audio_start) + 2) + if n >= 0 and abs(Fraction(n, rate) - audio_start / rate) <= TICK + Fraction(1, rate) + ] + candidates = [n for n in candidates if len(spcm) - n * stride == len(cpcm)] + if not candidates: + raise Failure("audio-length", "audio does not reach the source tail at the shared A/V timestamp offset") + # Length pins a unique sample offset. AAC's MDCT overlap and perceptual + # noise state depend on earlier frames, so replay the admitted source AUs + # from a cold decoder too. No arbitrary warmup samples are excluded. + audio_start = candidates[0] + reference = audio_reference(output, audio_start, rate) if audio_start else spcm + if cpcm != reference: + raise Failure("audio-content", "aligned decoded PCM samples differ from the source") + return { + "profile": "closed-gop-h264-aac-lc", + "video_frames": len(cf[0]), + "source_video_frames": len(sf[0]), + "startup_video_frames": start, + "audio_samples": len(cpcm) // stride, + "startup_audio_samples": audio_start, + "cold_audio_reference": bool(audio_start), + "compared_audio_samples": len(cpcm) // stride, + "timestamp_offset_seconds": float(offset), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("capture", type=Path) + parser.add_argument("--output", required=True, type=Path, help="directory for decoder evidence and report.json") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + report = {"source": str(args.source), "capture": str(args.capture)} + try: + report.update(compare(args.source, args.capture, args.output)) + report["status"] = "PASS" + except Failure as error: + report.update(status="FAIL", check=error.check, detail=str(error)) + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + return int(report["status"] != "PASS") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/eit-roundtrip.sh b/test/ts/eit-roundtrip.sh index 1f6821f713b..35f4da7e903 100755 --- a/test/ts/eit-roundtrip.sh +++ b/test/ts/eit-roundtrip.sh @@ -17,8 +17,12 @@ set -euo pipefail DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -TMP=$(mktemp -d) -trap 'rm -rf "$TMP"' EXIT +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin ts-eit "$(harness_env TSC_EIT_DURATION)just test ts-eit$(harness_argv "$@")" +TMP="$HARNESS_RUN" +ffmpeg -version >"$TMP/ffmpeg-version.txt" +tsp --version >"$TMP/tsduck-version.txt" 2>&1 DURATION="${TSC_EIT_DURATION:-20}" diff --git a/test/ts/process.sh b/test/ts/process.sh new file mode 100644 index 00000000000..fe61f9863c5 --- /dev/null +++ b/test/ts/process.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# A capture is evidence only when its producer finished successfully. +ts_wait() { + local pid="$1" label="$2" status=0 + harness_wait "$pid" || status=$? + printf '%s: %s\n' "$label" "$status" >>"$HARNESS_RUN/process-status.log" + if [[ "$status" -ne 0 ]]; then + echo "error: $label exited $status; partial output is not a successful round trip" >&2 + return 1 + fi +} + +# A finite measurement must not break its producer's pipe before source EOF. +ts_drain() { + local status=0 + "$@" || status=$? + cat >/dev/null || return $? + return "$status" +} diff --git a/test/ts/process.test.sh b/test/ts/process.test.sh new file mode 100644 index 00000000000..06965ec56c5 --- /dev/null +++ b/test/ts/process.test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +# shellcheck source-path=SCRIPTDIR source=process.sh +source "$DIR/process.sh" +harness_begin ts-process "just test ts-controls" + +for code in 0 42 124; do + # shellcheck disable=SC2016 # $1 belongs to the child shell. + harness_spawn writer "$HARNESS_RUN/writer-$code.log" bash -c 'printf "captured content\n"; exit "$1"' _ "$code" + if ts_wait "$HARNESS_PID" "writer-$code"; then + [[ "$code" -eq 0 ]] || { + echo "accepted failed writer $code" >&2 + exit 1 + } + else + [[ "$code" -ne 0 ]] || { + echo "rejected successful writer" >&2 + exit 1 + } + fi + [[ -s "$HARNESS_RUN/writer-$code.log" ]] || { + echo "control produced no partial output" >&2 + exit 1 + } + # The grader may stop early, but the writer must finish and the grader's + # failure must survive draining. No sleeps or wall-clock timeouts are needed. + set +e + # shellcheck disable=SC2016 # $1 belongs to the child shell. + printf 'window\n%1000000s' '' | ts_drain bash -c 'read -r line; exit "$1"' _ "$code" + statuses=("${PIPESTATUS[@]}") + set -e + [[ "${statuses[0]}" -eq 0 && "${statuses[1]}" -eq "$code" ]] || { + echo "draining changed writer/grader statuses: ${statuses[*]}" >&2 + exit 1 + } +done +echo "PASS process status: partial output cannot hide a failure or timeout" diff --git a/test/ts/run.sh b/test/ts/run.sh index 5ac319a7c03..9e5b49a9ffd 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -29,7 +29,7 @@ # checks can see *when* the exporter handed the bytes over. pcr-timing.py reads # the subscriber's stdout a packet at a time and stamps each read, so it grades # release timing and byte position alongside the values. Nightly runs this arm -# (.github/workflows/nightly.yml); it is not a per-PR gate, because it needs a +# (.github/workflows/ts.yml); it is not a per-PR gate, because it needs a # real-time window to measure at all. # # `--pair` changes the rig rather than the analyzer: it subscribes twice to one @@ -46,12 +46,18 @@ WORKSPACE=$(cd "$DIR/../.." && pwd) # Run directory, reserved ports, and process-group ownership. See test/README.md. # shellcheck source-path=SCRIPTDIR source=../lib/harness.sh source "$DIR/../lib/harness.sh" +# shellcheck source-path=SCRIPTDIR source=process.sh +source "$DIR/process.sh" # Captured before the parse below consumes it, so the rerun command carries the # source capture, mux rate, EIT fixture, and analyzer thresholds this run used, # whether they arrived as flags or as environment overrides. RERUN="$(harness_env TSC_DURATION TSC_BITRATE TSC_PORT TSC_PROFILE)just test ts$(harness_argv "$@")" +# Seamless session replacement needs an explicit publisher epoch. Keep this +# opt-in local to the validation rig; epochless replacement is tested as refusal. +VERSION=moq-lite-07-wip + SOURCE="" # real capture to publish instead of a generated clip ANALYZE_ONLY="" # existing TS to analyze without a round-trip CAPTURE_OUT="" # keep the subscriber capture here (for callers asserting on it) @@ -196,7 +202,7 @@ require_tools() { done # ffmpeg + cargo are only needed for the round-trip, not for --analyze-only. if [[ -z "$ANALYZE_ONLY" ]]; then - for t in cargo ffmpeg curl timeout; do have "$t" || missing+=("$t"); done + for t in cargo ffmpeg ffprobe curl timeout; do have "$t" || missing+=("$t"); done fi # The EIT fixture reads the service triplet out of the stream and may need to pad a # stuffing-free clip to make room for the table. @@ -214,9 +220,10 @@ analyze() { # Single source of truth for the verdict: compliance.py runs the TSDuck # tools itself and prints the PASS/WARN/FAIL summary. A second argument is # the source TS, which enables the duration-fidelity check (round-trip only). - local ref=() + local ref=() report=() + [[ -n "${HARNESS_RUN:-}" ]] && report=(--report-json "$HARNESS_RUN/compliance.json") [[ -n "${2:-}" ]] && ref=(--reference "$2") - python3 "$DIR/compliance.py" --ts "$1" ${ref[@]+"${ref[@]}"} $STRICT ${PASSTHRU[@]+"${PASSTHRU[@]}"} + python3 "$DIR/compliance.py" --ts "$1" ${ref[@]+"${ref[@]}"} ${report[@]+"${report[@]}"} $STRICT ${PASSTHRU[@]+"${PASSTHRU[@]}"} } require_tools @@ -235,6 +242,9 @@ fi # Only the round-trip stands anything up, so `--analyze-only` above needs no run # directory, no reserved port, and nothing to reap. harness_begin ts "$RERUN" +ffmpeg -version >"$HARNESS_RUN/ffmpeg-version.txt" +ffprobe -version >"$HARNESS_RUN/ffprobe-version.txt" +tsp --version >"$HARNESS_RUN/tsduck-version.txt" 2>&1 TARGET_BASE=$(cargo metadata --format-version 1 --manifest-path "$WORKSPACE/Cargo.toml" --no-deps | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p') @@ -337,9 +347,10 @@ if harness_probe "$URL/certificate.sha256"; then exit 1 fi -echo "### starting relay on 127.0.0.1:${PORT}" +echo "### starting relay on 127.0.0.1:${PORT} ($VERSION)" +printf '%s\n' "$VERSION" >"$HARNESS_RUN/moq-version.txt" harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$DIR/relay.toml" \ - --listen "127.0.0.1:${PORT}" --web-http-listen "127.0.0.1:${PORT}" + --listen-version "$VERSION" --listen "127.0.0.1:${PORT}" --web-http-listen "127.0.0.1:${PORT}" if ! harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"; then echo "error: relay never became ready" >&2 sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true @@ -373,8 +384,8 @@ grade_live() { # status we are here to record. set +e timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} 2>"$HARNESS_RUN/sub.log" | - python3 "$DIR/pcr-timing.py" --live --seconds "$DURATION" --release-pct-max 1 $STRICT \ + "$MOQ" --connect "$URL" --connect-version "$VERSION" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} 2>"$HARNESS_RUN/sub.log" | + ts_drain python3 "$DIR/pcr-timing.py" --live --seconds "$DURATION" --release-pct-max 1 $STRICT \ ${SCHEDULE[@]+"${SCHEDULE[@]}"} ${PASSTHRU[@]+"${PASSTHRU[@]}"} >"$HARNESS_RUN/timing.out" 2>&1 printf '%s\n' "${PIPESTATUS[0]} ${PIPESTATUS[1]}" >"$HARNESS_RUN/timing.rc" } @@ -382,7 +393,7 @@ grade_live() { # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' capture() { timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} >"$SUB_TS" 2>"$HARNESS_RUN/sub.log" + "$MOQ" --connect "$URL" --connect-version "$VERSION" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} >"$SUB_TS" 2>"$HARNESS_RUN/sub.log" } # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' @@ -393,7 +404,7 @@ capture_b() { # the media, because it has no shared history to derive them from. sleep "$PAIR_JOIN" timeout -k 3 $((DURATION + 20)) \ - "$MOQ" --connect "$URL" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} >"$SUB_B_TS" 2>"$HARNESS_RUN/sub-b.log" + "$MOQ" --connect "$URL" --connect-version "$VERSION" --broadcast "$BROADCAST" export ts ${EXPORT[@]+"${EXPORT[@]}"} >"$SUB_B_TS" 2>"$HARNESS_RUN/sub-b.log" } if [[ -n "$LIVE" ]]; then @@ -425,27 +436,27 @@ sleep 1 echo "### publishing PCR-paced TS -> $BROADCAST" # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' publish() { - # shellcheck disable=SC2016 # $1..$4 are the child bash -c positionals, not ours. - timeout -k 3 $((DURATION + 20 + 30)) bash -c ' + # shellcheck disable=SC2016 # $1..$5 are the child bash -c positionals, not ours. + timeout -k 3 $((DURATION + 20 + 30)) bash -o pipefail -c ' tsp -I file "$1" -P regulate --pcr-synchronous --wait-min 5 | - "$2" --connect "$3" --broadcast "$4" import ts - ' _ "$SRC_TS" "$MOQ" "$URL" "$BROADCAST" + "$2" --connect "$3" --connect-version "$5" --broadcast "$4" import ts + ' _ "$SRC_TS" "$MOQ" "$URL" "$BROADCAST" "$VERSION" } harness_spawn pub "$HARNESS_RUN/pub.log" publish PUB_PID="$HARNESS_PID" -# Keep the publisher's exit status: `timeout` returns 124 when it had to kill a -# stalled `moq import ts`, non-zero/non-124 means the import itself errored. Both -# explain a truncated capture, so surface it alongside the logs on failure. -harness_wait "$PUB_PID" && PUB_RC=0 || PUB_RC=$? - # shellcheck disable=SC2329 # invoked from multiple failure paths below dump_logs() { - echo " publisher exit status: $PUB_RC" >&2 + cat "$HARNESS_RUN/process-status.log" >&2 || true sed 's/^/ pub: /' "$HARNESS_RUN/pub.log" >&2 || true sed 's/^/ sub: /' "$HARNESS_RUN/sub.log" >&2 || true } +if ! ts_wait "$PUB_PID" publisher; then + dump_logs + exit 1 +fi + # ── live: the grader owns the verdict ─────────────────────────────────────── if [[ -n "$LIVE" ]]; then harness_wait "$SUB_PID" || true @@ -457,16 +468,14 @@ if [[ -n "$LIVE" ]]; then read -r EXPORT_RC GRADE_RC <"$HARNESS_RUN/timing.rc" echo cat "$HARNESS_RUN/timing.out" - # 124 is `timeout` reaching the end of the window, which is how the exporter is - # meant to stop; SIGPIPE (141) is the grader closing the pipe on its own window. - # Anything else ended the stream under the grader, so say so: it graded a - # shorter window than it was asked for, and the report alone doesn't show that. - # Not fatal, because the grader's verdict on what it did see is still the - # verdict, and a short window can only make the sample smaller, not kinder. - if [[ "$EXPORT_RC" -ne 0 && "$EXPORT_RC" -ne 124 && "$EXPORT_RC" -ne 141 ]]; then + # The grader drains after its measurement window, so no broken pipe or + # timeout is an expected successful export. + printf 'exporter: %s\ngrader: %s\n' "$EXPORT_RC" "$GRADE_RC" >>"$HARNESS_RUN/process-status.log" + if [[ "$EXPORT_RC" -ne 0 ]]; then echo >&2 - echo "warning: the exporter exited $EXPORT_RC before the window closed" >&2 - sed 's/^/ sub: /' "$HARNESS_RUN/sub.log" >&2 || true + echo "error: the exporter exited $EXPORT_RC" >&2 + dump_logs + exit 1 fi if [[ "$GRADE_RC" -ne 0 ]]; then echo >&2 @@ -474,23 +483,15 @@ if [[ -n "$LIVE" ]]; then dump_logs exit "$GRADE_RC" fi - # The grader's verdict is not the whole run. It grades whatever reached it, and - # the sample floor only rejects a window that came up short: a publisher that - # dies late still leaves enough behind to pass every check. That is a broken - # round-trip reported as a good one, so the publisher's own status is a gate - # too. 124 is `timeout` killing a stalled `moq import ts`, which is a failure - # of the same kind rather than a clean end, so it is not excused here. - if [[ "$PUB_RC" -ne 0 ]]; then - echo >&2 - echo "error: the publisher exited $PUB_RC; the graded stream is not a whole round-trip" >&2 - dump_logs - exit 1 - fi exit 0 fi -sleep 3 -harness_reap "$SUB_PID" +# Let the exporter drain to EOF within its existing timeout. Killing it after a +# sleep can both truncate a valid tail and conceal an earlier process failure. +if ! ts_wait "$SUB_PID" exporter; then + dump_logs + exit 1 +fi if [[ ! -s "$SUB_TS" ]]; then echo "error: subscriber captured no data" >&2 @@ -500,7 +501,11 @@ fi # ── pair: the two captures are the measurement ────────────────────────────── if [[ -n "$PAIR" ]]; then - harness_reap "$SUB_B_PID" + if ! ts_wait "$SUB_B_PID" exporter-b; then + dump_logs + cat "$HARNESS_RUN/sub-b.log" >&2 + exit 1 + fi if [[ ! -s "$SUB_B_TS" ]]; then echo "error: the second subscriber captured no data" >&2 sed 's/^/ sub-b: /' "$HARNESS_RUN/sub-b.log" >&2 || true @@ -518,14 +523,6 @@ if [[ -n "$PAIR" ]]; then dump_logs exit 1 fi - # As in --live: a grader can only speak for what reached it, so a publisher that - # died mid-run must not be reported as a clean pair. - if [[ "$PUB_RC" -ne 0 ]]; then - echo >&2 - echo "error: the publisher exited $PUB_RC; the graded pair is not a whole round-trip" >&2 - dump_logs - exit 1 - fi exit 0 fi @@ -535,6 +532,15 @@ fi echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') bytes -> analyzing" +# The initial decoded-content profile is the generated closed-GOP H.264/AAC +# pair. Other arms retain their specific structural/model oracles until their +# acquisition and decoder-priming contracts are covered by fixtures. +if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM" ]]; then + python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode" +else + echo "decoded-content comparison: not covered for this profile" +fi + # Which SI survived the round-trip. Informational: the exporter rebuilds SI from the # catalog, so a PID the import path does not route simply is not there, and that is a # statement about SI_PIDS rather than a malformed stream. compliance.py grades the stream @@ -580,7 +586,7 @@ fi # capture goes through pcr-timing.py as well. Its hard checks gate here as they do under # --live, and so does pcr-schedule when the rate is known (SCHEDULE). echo -if ! python3 "$DIR/pcr-timing.py" "$SUB_TS" ${SCHEDULE[@]+"${SCHEDULE[@]}"} $STRICT; then +if ! python3 "$DIR/pcr-timing.py" "$SUB_TS" --report-json "$HARNESS_RUN/pcr-timing.json" ${SCHEDULE[@]+"${SCHEDULE[@]}"} $STRICT; then echo >&2 echo "error: PCR timing analysis failed (see round-trip logs below)" >&2 dump_logs @@ -593,11 +599,7 @@ fi if [[ ${#SCHEDULE[@]} -gt 0 ]]; then echo echo "### PCR accuracy (tsp -P pcrverify, +/-500 ns at ${BITRATE} b/s)" - tsp -I file "$SUB_TS" -P pcrverify --bitrate "$BITRATE" --absolute --jitter-max 13 \ - -O drop >"$HARNESS_RUN/pcrverify.log" 2>&1 || true - VERDICT=$(grep -E "PCR OK, " "$HARNESS_RUN/pcrverify.log" | tail -1) - echo " ${VERDICT:-no verdict}" - if [[ ! "$VERDICT" =~ ([0-9,]+)\ PCR\ OK,\ 0\ with ]]; then + if ! python3 "$DIR/accuracy.py" "$SUB_TS" --bitrate "$BITRATE" --log "$HARNESS_RUN/pcrverify.log"; then echo >&2 echo "error: PCRs stray from their byte position (see $HARNESS_RUN/pcrverify.log)" >&2 dump_logs diff --git a/test/ts/tstd-controls.py b/test/ts/tstd-controls.py index b1f94c558bb..aa336b4eaed 100755 --- a/test/ts/tstd-controls.py +++ b/test/ts/tstd-controls.py @@ -41,10 +41,10 @@ A model that passes everything, or fails everything, cannot tell these apart. """ +import json import os import subprocess import sys -import tempfile DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, DIR) @@ -276,30 +276,32 @@ def grade(path: str) -> dict[str, compliance.Check]: return {check.name: check for check in compliance.analyze(path)} -def main() -> int: +def main(tmp: str) -> int: """Run every case and exit non-zero if any verdict is not the expected one.""" failed = 0 - with tempfile.TemporaryDirectory() as tmp: - for n, (name, build, expected) in enumerate(CASES): - checks = grade(build(os.path.join(tmp, f"{n}.ts"))) - check = checks["tstd"] - streams = check.metrics.get("streams", {}) - seen = {v for s in streams.values() for v in s["violations"]} - if expected == "pass": - # Not vacuous: every stream must have had access units to grade. - ok = check.status == compliance.Status.PASS and all(s["access_units"] for s in streams.values()) - ok = ok and checks["pcr-presence"].status == compliance.Status.PASS - elif isinstance(expected, tuple): - check = checks[expected[1]] - ok = check.status == compliance.Status.FAIL - elif isinstance(expected, str): - ok = any(expected in why for why in check.metrics.get("refused", {}).values()) - else: - ok = expected <= seen - failed += not ok - want = " ".join(reversed(expected)) if isinstance(expected, tuple) else expected - want = want if isinstance(want, str) else ", ".join(sorted(want)) - print(f" {'ok ' if ok else 'FAIL'} {name:<18} want {want:<40} got {check.detail}") + for n, (name, build, expected) in enumerate(CASES): + checks = grade(build(os.path.join(tmp, f"{n}.ts"))) + with open(os.path.join(tmp, f"{n}.json"), "w") as report: + json.dump({key: {"status": value.status.value, "detail": value.detail, "metrics": value.metrics} + for key, value in checks.items()}, report, indent=2) + check = checks["tstd"] + streams = check.metrics.get("streams", {}) + seen = {v for s in streams.values() for v in s["violations"]} + if expected == "pass": + # Not vacuous: every stream must have had access units to grade. + ok = check.status == compliance.Status.PASS and all(s["access_units"] for s in streams.values()) + ok = ok and checks["pcr-presence"].status == compliance.Status.PASS + elif isinstance(expected, tuple): + check = checks[expected[1]] + ok = check.status == compliance.Status.FAIL + elif isinstance(expected, str): + ok = any(expected in why for why in check.metrics.get("refused", {}).values()) + else: + ok = expected <= seen + failed += not ok + want = " ".join(reversed(expected)) if isinstance(expected, tuple) else expected + want = want if isinstance(want, str) else ", ".join(sorted(want)) + print(f" {'ok ' if ok else 'FAIL'} {name:<18} want {want:<40} got {check.detail}") if failed: print(f"tstd controls: {failed} of {len(CASES)} cases wrong", file=sys.stderr) return 1 @@ -308,4 +310,4 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + sys.exit(main(sys.argv[1])) diff --git a/test/ts/tstd-controls.sh b/test/ts/tstd-controls.sh new file mode 100644 index 00000000000..6c667786d41 --- /dev/null +++ b/test/ts/tstd-controls.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Keep the model's synthetic fixtures and reports when any control fails. +set -euo pipefail +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin ts-tstd "just test ts-tstd" +tsp --version >"$HARNESS_RUN/tsduck-version.txt" 2>&1 +python3 "$DIR/tstd-controls.py" "$HARNESS_RUN"