From bef1494cf5b38c1f0b711536343fb798106c57c4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 22:12:26 -0700 Subject: [PATCH 1/7] quest(m1): claim ts-reliability/pes-import From fff662b5e3e0e94b9945533ed32c2e152e366063 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 23:18:59 -0700 Subject: [PATCH 2/7] feat(moq-mux): import private PES whole as pesPacket tracks ts::Import publishes every undecoded PES stream (KLV, DVB AC-3, SCTE private data) as a pesPacket track: each frame is the whole PES packet, placed by PCR byte-position interpolation and rebased in place, so a missing PTS stays missing and the optional header survives byte-exact. The verbatim `pes` framing and `Verbatim::stream_id` are removed; `verbatim` now carries sections only and `framing` is required. Refuses ESCR, pack_header_field, TREF, scrambled or unbounded private PES, and a program without a PCR. Adds unplaced and refused stream stats, a `--klv` harness clip graded against ffprobe, and a CI step. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 9 + doc/bin/cli.md | 10 +- doc/lib/rs/moq-mux.md | 4 +- doc/setup/upgrade.md | 2 +- drafts/draft-lcurley-moq-mpegts.md | 34 +- quest/m1/ts-reliability/README.md | 1 - quest/m1/ts-reliability/fidelity.md | 6 +- quest/m1/ts-reliability/pes-import.md | 98 --- quest/m1/ts-reliability/section-placement.md | 8 +- quest/m3/msfts-convergence.md | 2 +- rs/moq-cli/src/publish.rs | 43 +- rs/moq-mux/MIGRATE.md | 7 + rs/moq-mux/src/catalog/timebase.rs | 21 +- rs/moq-mux/src/container/ts/catalog.rs | 47 +- rs/moq-mux/src/container/ts/export.rs | 477 +----------- rs/moq-mux/src/container/ts/export_test.rs | 493 +----------- rs/moq-mux/src/container/ts/fixture.rs | 65 +- rs/moq-mux/src/container/ts/import.rs | 729 ++++++++++++------ rs/moq-mux/src/container/ts/mod.rs | 2 + rs/moq-mux/src/container/ts/passthrough.rs | 14 +- .../src/container/ts/pes_packet_test.rs | 636 +++++++++++++++ rs/moq-mux/src/container/ts/psi.rs | 72 +- rs/moq-mux/src/container/ts/stats.rs | 23 +- test/justfile | 1 + test/ts/README.md | 14 +- test/ts/klv.py | 458 +++++++++++ test/ts/klv.test.py | 139 ++++ test/ts/run.sh | 42 +- 28 files changed, 2115 insertions(+), 1342 deletions(-) delete mode 100644 quest/m1/ts-reliability/pes-import.md create mode 100644 rs/moq-mux/src/container/ts/pes_packet_test.rs create mode 100644 test/ts/klv.py create mode 100644 test/ts/klv.test.py diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 2e59dcbd686..e3ea3c0d10f 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -92,6 +92,15 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} + # KLV and DVB AC-3 the importer carries as whole PES: each PES comes back byte-equal + # outside its PTS and DTS, at its source placement, and the AC-3 passes the T-STD model. + - name: TS private PES + if: ${{ !cancelled() }} + run: nix develop --command just test ts --klv --strict + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + # `import/export ts --passthrough`: from its first packet the capture is the source's # packets in place, and every compliance check grades it as it grades that interval of # the source. The generated clip fails the T-STD model on its own; the capture must too. diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 703e4e07b0b..be8a51f9630 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -91,8 +91,14 @@ taking the wrap that lands it nearest the clock, so media that gets more than time) reads as a rewind. MPEG-TS import carries H.264/H.265 and AAC/MP2/AC-3/E-AC-3, and passes SCTE-35 -and subtitle PIDs through as tracks. SCTE-35 cues wait for a media timestamp; -input ending or declaring a clock break before that timestamp is refused. +and every other stream it does not decode (KLV, subtitles, DVB AC-3) through as +tracks. Each such PES goes through whole, at its place on the program clock +(PCR); one before the first PCR is dropped. A stream it cannot carry unaltered +is refused with a warning: any in a program with no PCR, and one whose PES carry +an ESCR, pack header or TREF, are scrambled, or leave a non-video PES unbounded. +SCTE-35 cues and private PES wait for a media timestamp; input ending or +declaring a clock break before that timestamp is refused, so a program of data +alone fails. PES and untimed section staging share a bounded budget, so a source that never supplies a PES boundary or clock fails rather than retaining bytes indefinitely. It takes one program: a multi-program diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 3983ed55a40..51aeb9868ec 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -14,7 +14,7 @@ back. This is what `moq import` / `export` and the gateways are built on. | Format | Import | Export | Notes | | --- | --- | --- | --- | | fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text and `stpp` TTML (IMSC timing profile) as `ttml` documents on the broadcast clock, preserving source language and label when present; other subtitle codecs are skipped. | -| MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 and subtitle PIDs as tracks; service tables round-trip. | +| MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 sections and every other undecoded PES stream (KLV, subtitles, DVB AC-3) as tracks; service tables round-trip. | | FLV / RTMP | yes | yes | Legacy H.264 + AAC + MP3, plus enhanced-RTMP HEVC, AV1, VP9, Opus, AC-3, E-AC-3, and multitrack. | | Matroska / WebM | yes | yes | Timed SRT and WebVTT subtitles import as WebVTT text renditions, preserving source language and label when present; other subtitle codecs are skipped. | | Annex-B (H.264, H.265) | yes | yes | Parameter sets extracted to the catalog or re-injected per keyframe. | @@ -28,7 +28,7 @@ A `catalog::Timebase` gives renditions on one timestamp base one shared shift. A timeline that rewinds ends the import. fMP4 export fixes its track set at the init segment, so a new rendition or a changed configuration ends the export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: -one program per broadcast, damaged packets, and feed checks. +one program per broadcast, private PES, damaged packets, and feed checks. Exporters take a catalog stream alongside the source. Choose its encoding and rendition selection before exporting; the same stream shape drives every container. diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index db894231a00..67967ee911f 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -33,7 +33,7 @@ ignored; its guide says what replaces it. These land with the next breaking release. - **Data-track breaks are an object with no bytes** (#5815). A hang data - track (TS verbatim PES, sections, `m2ts`, text) marks a discontinuity with a + track (TS private PES, sections, `m2ts`, text) marks a discontinuity with a group of one empty object instead of a skipped group. Older consumers fail to decode it, so upgrade exporters and players before publishers. - **Auth requests name `webtransport`** (#5293). Relays report a WebTransport diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index d8429b75c25..9f830e9f57c 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -93,7 +93,7 @@ What remains is media in a codec-neutral catalog, plus the signaling a demultipl | | mpeg2ts packaging | mpegts section | |:--------|:--------|:--------| | MoQ track | one per program or multiplex | one per elementary stream | -| Object payload | whole source packets | one media frame, or one verbatim PES payload, PES packet, or section | +| Object payload | whole source packets | one media frame, PES packet, or section | | Packaging | `mpeg2ts` | none defined; media keeps its own | | Catalog | `mpeg2ts*` members on each track | one `mpegts` root member | | Relay | caches and drops whole objects | drops and prioritizes per elementary stream | @@ -222,30 +222,24 @@ Descriptors are never parsed, so a consumer MUST re-emit a list in order and MUS # Verbatim Tracks {#verbatim} -An elementary stream the publisher does not decode rides its own MoQ track, byte-for-byte: +An elementary stream the publisher does not decode rides its own MoQ track, byte-for-byte. +A stream of sections is described by a `verbatim` record: ~~~ type Verbatim = { "streamType": number, - "framing": "pes" | "section", - "streamId": number | undefined, + "framing": "section", } ~~~ -`streamType` is the PMT `stream_type` to re-announce: 0x86 for SCTE-35 {{scte35}}, 0x06 for private PES, 0x05 for private sections. +`streamType` is the PMT `stream_type` to re-announce: 0x86 for SCTE-35 {{scte35}}, 0x05 for private sections. -`framing` says how the payload is framed, defaulting to `"pes"`. -With `"pes"` each frame is one complete PES payload, timestamped with its PTS, or 0 if it carried none. -With `"section"` each frame is one complete section, header and CRC included, timestamped with the media time it arrived at. -A consumer MUST refuse a catalog that names a `framing` it does not recognize, rather than guess. - -`streamId` is the original PES `stream_id`, for example 0xBD (`private_stream_1`) for teletext, DVB subtitles, and DVB AC-3. -It applies only to `"pes"` framing. -A publisher SHOULD record it, since strict demultiplexers reject a relabeled PES; a consumer with none falls back to `private_stream_1`. +`framing` is required, and `"section"` is its only value: each frame is one complete section, header and CRC included, timestamped with the media time it arrived at. +A consumer MUST refuse a catalog whose `verbatim` record names any other `framing` or none, rather than guess; in particular it MUST NOT read the `"pes"` framing of draft -00, or a record without `framing`, as sections. Each frame MUST be a keyframe in its own group: a verbatim payload is all-or-nothing. -A track can instead carry each PES whole, header included: +A stream of PES packets is described by a `pesPacket` record, and carries each PES whole, header included: ~~~ type PesPacket = { @@ -253,13 +247,20 @@ type PesPacket = { } ~~~ -`streamType` is the PMT `stream_type` to re-announce, as for `Verbatim`. +`streamType` is the PMT `stream_type` to re-announce, as for `Verbatim`: 0x06 for private data such as asynchronous KLV, teletext, DVB subtitles and DVB AC-3, 0x15 for synchronous metadata. Each frame MUST be exactly one PES packet, from its start code through its payload, with a `PES_packet_length` that matches the frame, or zero only where {{mpeg2}} lets a video stream leave it unbounded. The `stream_id`, the PTS and DTS when present, and every other header field ride in the frame, with the PTS and DTS on the catalog clock; a PES without a PTS carries none. The frame's timestamp is the packet's placement: the program clock at its first byte, on the same timeline. +Each frame MUST be a keyframe in its own group, as for `verbatim`. A consumer MUST refuse a frame that is not one well-formed PES packet. The record is a member beside `verbatim` rather than a `framing` value, so a consumer that predates it ignores the track instead of refusing the catalog. +A publisher demultiplexing a transport stream places each PES by the PCRs of its program ({{mpeg2}} Section 2.4.2.2): its first byte is interpolated between the reference bytes of the PCRs either side of it, and one past its timebase's last PCR is extrapolated from that timebase's last two. +Each PCR takes the 2^33 multiple of its base that lands it nearest the media's PTS, and the placement moves onto the catalog clock by the same shift as the media, so a PES's timestamp depends only on the input and that shift. +A PES before the first PCR the publisher reads, or past a lone one, has nothing to place it and is dropped. +The publisher moves each PTS and DTS by the media's shift modulo 2^33, keeping their prefix and marker bits, and leaves every other byte as received; the PES CRC covers the previous packet's data bytes, not its header, so it stays valid. +It MUST refuse to carry a stream rather than alter it: one in a program with no PCR, and one whose PES carry a clock the shift does not reach (an ESCR, a `pack_header_field`, or a TREF), PES-level scrambling, or a zero `PES_packet_length` on a stream that is not video. + A verbatim or whole-PES track has no codec, dimensions, or decoder configuration, so a publisher MUST NOT describe it as a media rendition and MUST describe it only by its `tracks` entry, and a consumer MUST refuse a catalog that describes one as both. Nothing there declares a container, so its frames use hang's `legacy` container ({{hang}}): a varint microsecond timestamp followed by the payload. @@ -365,7 +366,7 @@ A consumer rebuilding a transport stream: - MUST put the PMT on `program.pmtPid` and build the PAT and PMT from `program`. - MUST re-emit each track's `descriptors` as its ES-level descriptors, and `programDescriptors` as the PMT's `program_info`. - MUST re-emit each `si` entry's sections byte-for-byte on that entry's PID, reading them from its track ({{si-track}}), at least as often as its `interval` when declared. -- MUST repacketize each verbatim track per its `framing` and `streamType`, using `streamId` when recorded. +- MUST packetize each `verbatim` frame as one section on its track's PID. - MUST packetize each `pesPacket` frame unchanged on its track's PID, its first byte going out no earlier than the frame's timestamp, and SHOULD send it as close to that as the multiplex allows. - SHOULD pad the output with null packets to `muxRate` when present, so the rebuilt stream is constant-rate again; a source that exceeds the rate is passed through rather than delayed or dropped. @@ -442,6 +443,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - Added the `m2ts` section, a transport stream carried whole, with its mapping onto MSFTS; the comparison uses MSFTS's `mpeg2ts` names. - Added the `pesPacket` record: each frame one whole PES packet, timestamped with its placement. A consumer that predates it drops the track rather than refusing the catalog. +- `verbatim` carries sections only, breaking -00: `framing` is required and `"section"` its only value, and `streamId` is gone. A consumer refuses a -00 catalog with `"pes"` framing or a `verbatim` record without `framing`. A publisher carries an undecoded PES stream as `pesPacket`, and refuses one it cannot carry unaltered. - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. ## draft-lcurley-moq-mpegts-00 diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 2300c371410..0d0976ba929 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -101,7 +101,6 @@ remain with their current quests. - [Deterministic grading](/quest/m1/ts-reliability/grader-determinism.md) - the same input gets the same start and the same verdict - [Plan: semantic fidelity](/quest/m1/ts-reliability/fidelity.md) - explicit profiles and PMT lifecycle contracts produce implementation quests -- [Whole-PES import](/quest/m1/ts-reliability/pes-import.md) - private PES travels whole at its source placement, a missing PTS stays missing, and the `pes` framing is deleted - [Section placement](/quest/m1/ts-reliability/section-placement.md) - sections share the PCR placement clock instead of the video PTS - [Sequence holes get a time budget](/quest/m1/ts-reliability/hole-budget.md) - a hole gives up on max(wall, pts), so a break marker behind it on a paused track is delivered - [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks diff --git a/quest/m1/ts-reliability/fidelity.md b/quest/m1/ts-reliability/fidelity.md index 1a484c3106d..275a8b404b0 100644 --- a/quest/m1/ts-reliability/fidelity.md +++ b/quest/m1/ts-reliability/fidelity.md @@ -33,9 +33,8 @@ Audit findings to reproduce and scope: - Section export requires video, leaving audio-only programs with sections unsupported. Keep opaque SCTE-35 fidelity separate from the later typed browser cue API; basic carriage should not depend on that API existing. -- Private PES loses optional header semantics today; the - [whole-PES quests](/quest/m1/ts-reliability/pes-import.md) carry it - byte-exact. +- Private PES rides whole as a `pesPacket` track, byte-exact outside its + rebased PTS and DTS ([draft](/drafts/draft-lcurley-moq-mpegts.md)). The maintainer decides whether a supported profile handles live PMT changes in place with fresh track identities, replaces the broadcast, or initially @@ -52,6 +51,5 @@ section/PES bytes only where the settled contract actually promises that. ## Related -- [Whole-PES import](/quest/m1/ts-reliability/pes-import.md) - private PES keeps its header bytes and source placement - [Automated validation](/test/ts/README.md) - profile declarations link to executable evidence - [Typed SCTE-35](/quest/m3/2279-hang-typed-scte-35-ad-cue-signaling-carried-opaquely.md) - application cue APIs follow basic opaque carriage diff --git a/quest/m1/ts-reliability/pes-import.md b/quest/m1/ts-reliability/pes-import.md deleted file mode 100644 index 0f9128f9f68..00000000000 --- a/quest/m1/ts-reliability/pes-import.md +++ /dev/null @@ -1,98 +0,0 @@ -# [L] TS import carries private PES whole at its source placement - -## Goal - -`moq import ts` publishes every PES stream it does not decode (private data -such as KLV, teletext, DVB subtitles, DVB AC-3, unknown types) as a -`pesPacket` track: each frame the whole PES packet with its clock fields -moved onto the catalog clock, timestamped with its source placement. A PES -without a PTS keeps none, and no frame is stamped 0 or at the track's own -edge. Streams the profile cannot carry are refused and reported, never -silently altered. The legacy `pes` framing is gone from import, export and -the draft. - -## Plan - -Decided 2026-10-16 (see the [epic](/quest/m1/ts-reliability/README.md)). -Today `VerbatimStream::write` stamps a PES without a PTS at its own track's -`live_edge()`, or `Timestamp::ZERO` before any frame. A stream that never -carries a PTS (asynchronous KLV) therefore sits at 0 forever, below -`video_start`, and export drops all of it. Reproduce that with a failing -round-trip test first. - -- **Placement.** The frame timestamp is the program clock at the PES's first - byte, interpolated between the PCRs on either side by byte position - (ISO 13818-1 2.4.2.2): the PES start code's input offset against each - PCR's reference byte, not packet boundaries, reusing the byte-offset - clock in `passthrough.rs`, with the PCR unwrapped into the era of the - anchored PTS (the 2^33 multiple nearest it, since PTS leads PCR by well - under half a wrap) so a start near the wrap does not split them, shifted onto the catalog clock by the timebase - offset that shifts the PTS. A PES waits for the next PCR, within the - existing `Retained` bound; it never spans a PCR discontinuity. The value - depends only on the input bytes and the timebase offset, so it adds no - disagreement of its own; same-epoch importers agree once the - [epoch anchor](/quest/m2/epoch-anchor.md) fixes that offset. A program - with no PCR PID cannot place its PES, so its private PES streams are - refused. The timebase offset is anchored only by a PTS today, so a PES - also waits for that anchor within the same bound; a program whose PES - never carry a PTS (data only) is refused and tested, unless the PCR can - anchor the timebase without breaking same-epoch agreement. -- **Trailing PES.** A PES that starts after its timebase's last PCR (end of - input, or a PCR discontinuity before the next PCR) has no PCR on the far - side. Place it by extrapolating from that timebase's last two PCRs, which - is still deterministic, rather than hold or drop it. With fewer than two - PCRs it is dropped and counted in importer health. A PES before the - first PCR the importer sees is dropped and counted the same way, like - anything before acquisition: back-extrapolating would let importers - started apart disagree. A new timebase starts with its first PCR, so - this happens only at startup. Test each case. -- **Bytes.** The frame is the PES packet as received, start code through - payload. PTS and DTS are rewritten in place onto the catalog clock modulo - 2^33, exactly as the media PTS moves. Every other field stays byte-exact, - including flags, `data_alignment_indicator`, ES rate, trick mode, copy - info, the PES CRC (it covers the previous packet's data bytes, not its - header, so rebasing leaves it valid; assert that with a CRC fixture), extension data, stuffing, and the special stream-ID forms - that have no optional header. Each PES keeps its own stream_id. -- **Catalog.** The entry records `pid`, `descriptors` and the `pesPacket` - member that export reads, and - no `verbatim` record. A published exporter ignores such an entry and - silently leaves the stream out of its TS; that is the accepted cost. -- **Refusal.** A PES carrying a clock this does not rebase (ESCR, a - `pack_header_field`, TREF), PES-level scrambling, or a zero - `PES_packet_length` on a non-video payload, by export's rule - refuses its stream: - no track is advertised, or an advertised one ends with an error and loses - its catalog entry, with a warning and an importer health count. -- **Scope.** This covers every stream the importer carries undecoded as - PES, whatever its stream type, because nothing is inferred: a media - stream that splits an access unit across PES, leaving later ones without - a PTS, round-trips the same way. Decoded codecs keep their own timing. -- **Delete `pes`.** In the same change, so main never writes a catalog its - exporter refuses: drop `Framing::Pes` and `Verbatim::stream_id`, export's - `write_pes` path for verbatim tracks, and the DVB AC-3 repack - (`queue_ac3` and the per-frame PES it writes). A catalog naming - `framing: "pes"`, or a `verbatim` record with no `framing` (which meant - `pes` in -00), is refused: never reinterpret it as a section. Published - importers and existing recordings with private PES then fail to export, - loudly. Migrate the hand-built `pes` catalogs in tests, including - `rs/moq-cli/src/publish.rs`. The repack rewrites a multi-frame AC-3 PES - as one PES per sync frame; only that output rewrite goes. The PES PTS - covers only its first sync frame, so the export's AC-3 buffer model still - walks the sync frames in the unchanged bytes to remove each at its own - decode time, and the multi-frame AC-3 fixture below passes the strict - T-STD check. - -Tests: generated fixtures (extend `test/ts/fragment_pes.py` or add a -generator) for asynchronous KLV (stream_type 0x06, `KLVA` registration, no -PTS) and synchronous KLV (stream_type 0x15, metadata_stream 0xFC, PTS), -importer to track to exporter, asserting byte equality of each PES outside -its rebased PTS and DTS. Cover a start straddling the 33-bit wrap, PTS 0 versus no PTS, distinct PTS and DTS, -data before the first video and between media, optional fields, special -stream IDs, DVB AC-3 with several sync frames per PES, each refusal, and -the refused `pes` catalog. Add a `just test ts --klv` arm with FFmpeg and -TSDuck grading, run on PRs that touch `rs/moq-mux/src/container/ts`. - -Update `drafts/draft-lcurley-moq-mpegts.md` (the importer writes -`pesPacket` and states what it refuses; `framing` is `"section"` only and -required, `streamId` removed, changelog -01 noting the break with -00) and -the TS lines of `doc/bin/cli.md` and `doc/lib/rs/moq-mux.md`. diff --git a/quest/m1/ts-reliability/section-placement.md b/quest/m1/ts-reliability/section-placement.md index 2a6f06cff1f..a700d22d4f4 100644 --- a/quest/m1/ts-reliability/section-placement.md +++ b/quest/m1/ts-reliability/section-placement.md @@ -10,8 +10,8 @@ them, and a program without video has no clock for them at import. ## Plan -Reuse the placement the [importer](/quest/m1/ts-reliability/pes-import.md) -derives from the PCR in `SectionStream`, replacing `last_pts`/`start_pts` +Reuse the placement the [importer](/rs/moq-mux/src/container/ts/import.rs) +derives from the PCR for whole PES (`Placement`) in `SectionStream`, replacing `last_pts`/`start_pts` and the edge clamp in `SectionStream::emit`. SCTE-35 splice times live in the section bytes and keep their `pts_adjustment` rebasing. @@ -32,7 +32,3 @@ position and one with a cue beside the tune-in keyframe, and reword "the media time it arrived at" in the draft's {{verbatim}} to the shared placement definition. - -## Required - -- [Whole-PES import](/quest/m1/ts-reliability/pes-import.md) - the placement clock this reuses diff --git a/quest/m3/msfts-convergence.md b/quest/m3/msfts-convergence.md index e4f7f01182a..a8b70c7d83d 100644 --- a/quest/m3/msfts-convergence.md +++ b/quest/m3/msfts-convergence.md @@ -22,7 +22,7 @@ egress moved to msfts#37 (closed). Two differences remain: - **ES units.** MSFTS `es-packets` (draft-gregoire-moq-msfts-02) carry the TS packets of one PID. Here decoded media carries access units, and undecoded PES carries the whole PES packet at its source placement - ([whole-PES import](/quest/m1/ts-reliability/pes-import.md)). Document the + ([`pesPacket`](/drafts/draft-lcurley-moq-mpegts.md)). Document the mapping between them. Decided in the 2026-10-11 audit: keep the semantic lane and raw fallback diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 44c18fa97af..f466deca005 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -848,13 +848,13 @@ pub(crate) mod tests { 0x7f, 0xdf, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xad, 0x25, 0xe8, 0x39, ]; - /// Payload of an undecoded PES-framed stream (e.g. teletext/DVB AC-3 private data), - /// carried verbatim on its own PID with the original PES stream_id. - const PES_PAYLOAD: &[u8] = &[0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02]; + /// A whole PES of an undecoded stream (private_stream_1, no PTS), carried on its own PID. + const PES: &[u8] = &[ + 0x00, 0x00, 0x01, 0xBD, 0x00, 0x09, 0x84, 0x00, 0x00, 0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02, + ]; const SECTION_PID: u16 = 0x102; - const VERBATIM_PES_PID: u16 = 0x104; - const VERBATIM_PES_STREAM_ID: u8 = 0xC0; + const PES_PID: u16 = 0x104; async fn open_export( source: moq_mux::Source, @@ -882,7 +882,7 @@ pub(crate) mod tests { } /// Manufacture a TS feed carrying real video/audio plus one section-framed - /// verbatim stream (SCTE-35) and one PES-framed verbatim stream, by importing + /// verbatim stream (SCTE-35) and one whole-PES stream, by importing /// `bbb.ts` into a broadcast that also holds the two ancillary tracks and /// re-exporting with the `mpegts` catalog extension. /// Let the origin's driver run the fronts that requests and announcements @@ -933,15 +933,12 @@ pub(crate) mod tests { section_producer.cut(None).unwrap(); section_producer.finish().unwrap(); - // PES-framed verbatim stream (undecoded private data, stream_type 0x06), with - // an explicit PES stream_id to round-trip. + // Whole-PES stream (undecoded private data, stream_type 0x06). let pes = broadcast .unique_track(".data", hang::container::track_info(hang::catalog::PRIORITY.text)) .unwrap(); - let mut verbatim = tscat::Verbatim::new(0x06, tscat::Framing::Pes); - verbatim.stream_id = Some(VERBATIM_PES_STREAM_ID); - let mut pes_track = tscat::Track::new(VERBATIM_PES_PID); - pes_track.verbatim = Some(verbatim); + let mut pes_track = tscat::Track::new(PES_PID); + pes_track.pes_packet = Some(tscat::PesPacket::new(0x06)); catalog .modify() .unwrap() @@ -954,7 +951,7 @@ pub(crate) mod tests { .write(Frame { timestamp: Timestamp::from_millis(1450).unwrap(), duration: None, - payload: bytes::Bytes::from_static(PES_PAYLOAD), + payload: bytes::Bytes::from_static(PES), keyframe: true, }) .unwrap(); @@ -1002,8 +999,7 @@ pub(crate) mod tests { /// Full CLI round-trip: a TS feed with undecoded streams goes through `Publish` /// (which selects the `mpegts` catalog) and the subscribe-side catalog stream, - /// and the SCTE-35 section and the verbatim PES survive with their PIDs, framing, - /// PES stream_id, and byte-exact payloads. + /// and the SCTE-35 section and the whole PES survive with their PIDs and every byte. async fn ts_verbatim_round_trip(format: CatalogFormat) { // Paused time auto-advances when the exporter parks, so the `drain` timeouts // fire instantly instead of waiting on the wall clock. @@ -1070,16 +1066,9 @@ pub(crate) mod tests { .mpegts .tracks .iter() - .find(|(_, t)| t.verbatim.as_ref().is_some_and(|v| v.stream_type == 0x06)) - .expect("verbatim PES survived the round-trip"); - assert_eq!(pes.pid, VERBATIM_PES_PID, "verbatim PES PID preserved"); - let pes_verbatim = pes.verbatim.as_ref().unwrap(); - assert_eq!(pes_verbatim.framing, tscat::Framing::Pes, "PES framing preserved"); - assert_eq!( - pes_verbatim.stream_id, - Some(VERBATIM_PES_STREAM_ID), - "PES stream_id preserved" - ); + .find(|(_, t)| t.pes_packet.as_ref().is_some_and(|p| p.stream_type == 0x06)) + .expect("the PES track survived the round-trip"); + assert_eq!(pes.pid, PES_PID, "PES PID preserved"); let pes_name = pes_name.clone(); assert_eq!( @@ -1089,8 +1078,8 @@ pub(crate) mod tests { ); assert_eq!( read_frame(&consumer, &pes_name).await, - PES_PAYLOAD, - "verbatim PES payload round-trips byte-for-byte" + PES, + "the PES round-trips byte-for-byte" ); } diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index cf7adb6888e..424e61aea97 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### TS private PES rides `pesPacket` tracks; `verbatim` is sections only + +`ts::Framing::Pes` and `ts::Verbatim::stream_id` are gone, and `Verbatim::framing` no longer defaults: a catalog naming `"framing": "pes"`, or a `verbatim` record without `framing`, fails to parse. +`ts::Import` now publishes every undecoded PES stream as a `ts::PesPacket` track, each frame one whole PES packet placed by the PCR. +Write a hand-built PES track as `Track { pes_packet: Some(PesPacket::new(stream_type)), .. }` with each frame the whole PES, and a `verbatim` one only for sections, with `Framing::Section`. +Recordings published with `pes` framing no longer export; import their source again. + ### `fmp4::Wire::new` refuses a track that isn't audio or video (#5815) `container::fmp4::Wire::new` returns `Result`, as `Wire::from_init` already did, and refuses any other handler instead of treating it as data. diff --git a/rs/moq-mux/src/catalog/timebase.rs b/rs/moq-mux/src/catalog/timebase.rs index bb9786b0604..1ab00371ce8 100644 --- a/rs/moq-mux/src/catalog/timebase.rs +++ b/rs/moq-mux/src/catalog/timebase.rs @@ -86,17 +86,28 @@ impl Timebase { /// further ahead reads as a period behind, a rewind. pub(crate) fn nearest(&self, raw: u64, wrap: Wrap) -> crate::Result { let offset = self.anchor(moq_net::Timestamp::from_scale(raw, wrap.hz)?)?; - let micros = self.catalog.current_clock().now().as_micros() as i128 - offset.0 as i128; - let reading = (micros * wrap.hz as i128).div_euclid(1_000_000); - let period = wrap.period as i128; - let era = (reading - raw as i128 + period / 2).div_euclid(period); - u64::try_from(raw as i128 + era * period).map_err(|_| { + u64::try_from(self.era(raw, wrap, offset)).map_err(|_| { crate::Error::UnmappableTimestamp(format!( "{raw} at {} Hz lands before the broadcast clock's zero", wrap.hz )) }) } + + /// `raw` in the era [`Self::nearest`] gives it, without anchoring: `None` until the timebase + /// has an offset. Signed, since a value just behind the wrap the anchor sits past lands in + /// the era before zero, as a PCR leading a PTS just past the wrap does. + pub(crate) fn unwrap(&self, raw: u64, wrap: Wrap) -> Option { + Some(self.era(raw, wrap, self.offset()?)) + } + + fn era(&self, raw: u64, wrap: Wrap, offset: Offset) -> i128 { + let micros = self.catalog.current_clock().now().as_micros() as i128 - offset.0 as i128; + let reading = (micros * wrap.hz as i128).div_euclid(1_000_000); + let period = wrap.period as i128; + let era = (reading - raw as i128 + period / 2).div_euclid(period); + raw as i128 + era * period + } } /// A timestamp field that wraps every `period` ticks of a `hz` clock, as MPEG-TS's 33-bit PTS diff --git a/rs/moq-mux/src/container/ts/catalog.rs b/rs/moq-mux/src/container/ts/catalog.rs index 5a1c40d42a2..4d5b3487d9a 100644 --- a/rs/moq-mux/src/container/ts/catalog.rs +++ b/rs/moq-mux/src/container/ts/catalog.rs @@ -150,9 +150,11 @@ impl Mpegts { && self.mux_rate.is_none() } - /// Whether the section lists a track it carries itself: a verbatim stream or an SI table. + /// Whether the section lists a track it carries itself: an undecoded stream or an SI table. pub(crate) fn has_track(&self) -> bool { - self.tracks.values().any(|track| track.verbatim.is_some()) + self.tracks + .values() + .any(|track| track.verbatim.is_some() || track.pes_packet.is_some()) || self .si .values() @@ -314,7 +316,7 @@ pub struct Track { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub descriptors: Vec, - /// Present when the stream is carried verbatim (not decoded into `video`/`audio`). + /// Present when the stream is carried as sections (not decoded into `video`/`audio`). #[serde(default, skip_serializing_if = "Option::is_none")] pub verbatim: Option, @@ -344,27 +346,15 @@ pub struct Verbatim { /// Raw PMT `stream_type` to re-announce, including types the muxer does not recognize. pub stream_type: u8, - /// How the verbatim payload is framed, so export knows how to repacketize it. - #[serde(default)] + /// How the verbatim payload is framed, so export knows how to repacketize it. Required: + /// draft -00 let an absent one mean PES payloads, which nothing carries any more. pub framing: Framing, - - /// Original PES `stream_id` (e.g. 0xBD private_stream_1 for teletext/DVB - /// subtitles/DVB AC-3, 0xC0-0xDF audio). Preserved so export re-emits the PES - /// under its real id rather than relabeling it, which strict broadcast demuxers - /// and TR 101 290 analyzers reject. `None` for section framing or a non-TS - /// source; export then falls back to `private_stream_1`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stream_id: Option, } impl Verbatim { /// A new verbatim carriage record of the given `stream_type` and `framing`. pub fn new(stream_type: u8, framing: Framing) -> Self { - Self { - stream_type, - framing, - stream_id: None, - } + Self { stream_type, framing } } } @@ -392,16 +382,16 @@ impl PesPacket { } /// How a verbatim stream's payload is framed on the wire, so export can repacketize it. +/// +/// A PES stream rides [`PesPacket`] instead. The `"pes"` framing draft -00 defined is refused +/// rather than read as a section. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)] #[serde(rename_all = "camelCase")] #[non_exhaustive] pub enum Framing { - /// Packetized Elementary Stream: each frame is one PES payload (access unit), - /// timestamped by its PTS. Used by private PES, teletext, DVB subtitles, ... - #[default] - Pes, /// Private sections (table_id + section_length framing). Each frame is one /// complete section. Used by SCTE-35 and other private-section signaling. + #[default] Section, } @@ -663,6 +653,19 @@ mod test { ); } + /// Draft -00's PES framing, named or implied by an absent `framing`, is refused rather than + /// read as a section: its frames are bare PES payloads, which nothing can re-emit faithfully. + #[test] + fn legacy_pes_framing_is_refused() { + for verbatim in [ + r#"{ "streamType": 6, "framing": "pes", "streamId": 189 }"#, + r#"{ "streamType": 6 }"#, + ] { + let json = format!(r#"{{ "tracks": {{ "x.ts": {{ "pid": 500, "verbatim": {verbatim} }} }} }}"#); + serde_json::from_str::(&json).expect_err(&format!("{verbatim} must be refused")); + } + } + #[test] fn invalid_si_pid_key_is_refused() { let json = r#"{ "si": { "not-a-pid": { "66": { "track": "si" } } } }"#; diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index c3892e71c46..eb9764674b9 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -51,6 +51,7 @@ use crate::jitter::{self, Arrival, Push}; use super::adts; use super::catalog; +use super::psi; use super::schedule::{self, Buffer, Schedule}; /// PID of the single program's PMT. @@ -278,8 +279,6 @@ struct Track { timing: Timing, /// The receiver's buffers for an audio or verbatim track; video's come from [`Timing`]. buffer: Option, - /// What the last passed-through Dolby PES left for the next to complete. - dolby_carry: Option, /// The audio frames muxed into a PES that has yet to go out. pack: Option, /// The last frame's decode time, its jitter buffer key ([`Self::release_key`]) and the PES @@ -316,7 +315,6 @@ impl Track { keyframe: None, timing: Timing::default(), buffer: None, - dolby_carry: None, pack: None, released: None, } @@ -334,9 +332,6 @@ impl Track { self.restart = pending.restart; self.clock = DecodeClock::default(); } - if let Some(codec) = self.dolby() { - return self.queue_dolby(codec, name, pending, jitter); - } let Kind::Video(stream_type) = self.kind else { return self.push(name, pending, None, None, jitter); }; @@ -419,61 +414,6 @@ impl Track { Ok(()) } - /// Queue each Dolby access unit of a passed-through PES on its own, due at its own decode - /// time: a receiver's buffer holds and decodes each on its own, and the whole PES would - /// overflow it. A frame the last PES began decodes ahead of this PES, so the jitter buffer - /// has to order it on its own time, not the PES's. - fn queue_dolby( - &mut self, - codec: Dolby, - name: &str, - pending: Pending, - jitter: &mut jitter::Buffer, - ) -> anyhow::Result<()> { - let Pending { - frame, - restart, - skip, - cut, - arrived, - read, - } = pending; - let frames = dolby_units(codec, &mut self.dolby_carry, skip, frame.timestamp, &frame.payload)?; - for (timestamp, payload) in frames { - let frame = Frame { - timestamp, - duration: None, - payload, - // Only the first frame that begins in the PES has its PTS. - keyframe: frame.keyframe && timestamp == frame.timestamp, - }; - let pending = Pending { - frame, - restart, - skip, - cut, - arrived, - read, - }; - self.push(name, pending, None, None, jitter)?; - } - Ok(()) - } - - /// The Dolby audio the track passes through as DVB private data, whose PES may carry - /// several sync frames. - fn dolby(&self) -> Option { - let Kind::Verbatim { - stream_type: 0x06, - framing: catalog::Framing::Pes, - .. - } = self.kind - else { - return None; - }; - Dolby::described(&self.descriptors) - } - /// The samples an audio frame of `payload` lasts and their rate, if it may share a PES with /// the frames around it: one whole frame of audio whose receiver removes each frame from /// its buffer at the frame's own decode time. @@ -483,7 +423,6 @@ impl Track { Kind::Mp2 { .. } => crate::codec::mp2::parse_header, Kind::Ac3 => crate::codec::ac3::parse_header, Kind::Eac3 => crate::codec::eac3::parse_header, - _ if self.dolby() == Some(Dolby::Ac3) => crate::codec::ac3::parse_header, _ => return None, }; let header = parse(payload).ok()?; @@ -781,15 +720,9 @@ enum Kind { Ac3, /// E-AC-3 (ATSC stream_type 0x87), carried verbatim. Eac3, - /// An undecoded elementary stream carried verbatim (SCTE-35, private PES, - /// teletext, ...). Re-announced in the PMT with its recorded `stream_type` and - /// repacketized per its `framing`. `stream_id` is the original PES stream_id to - /// re-emit (PES framing only; `None` falls back to `private_stream_1`). - Verbatim { - stream_type: u8, - framing: catalog::Framing, - stream_id: Option, - }, + /// An undecoded elementary stream carried as private sections (SCTE-35, ...), + /// re-announced in the PMT with its recorded `stream_type`. + Verbatim { stream_type: u8 }, /// An undecoded elementary stream whose frames are whole PES packets, written unchanged /// and sent at their timestamp, the source's placement ([`catalog::PesPacket`]). PesPacket { stream_type: u8 }, @@ -844,7 +777,7 @@ struct PesUnit { /// (unwrapped) 90 kHz ticks (wrapped to the wire field in `write_pes`). `Some` only when /// it differs from the PTS; the PES then carries both PTS and DTS. dts: Option, - /// Explicit PES stream_id (verbatim PES); `None` derives it from `is_video`. + /// Explicit PES stream_id (Opus); `None` derives it from `is_video`. stream_id: Option, } @@ -1649,8 +1582,6 @@ where let kind = match (&entry.verbatim, &entry.pes_packet) { (Some(verbatim), _) => Kind::Verbatim { stream_type: verbatim.stream_type, - framing: verbatim.framing, - stream_id: verbatim.stream_id, }, (None, Some(pes)) => Kind::PesPacket { stream_type: pes.stream_type, @@ -2017,15 +1948,7 @@ where // Section-framed verbatim streams (SCTE-35, ...) are stamped on the video clock // and carry no PTS for the PCR, so they need a video track; audio alone would // leave them pinned to zero. - let needs_clock = tracks.iter().any(|t| { - matches!( - &t.kind, - Kind::Verbatim { - framing: catalog::Framing::Section, - .. - } - ) - }); + let needs_clock = tracks.iter().any(|t| matches!(&t.kind, Kind::Verbatim { .. })); let video = tracks.iter().find(|t| matches!(t.kind, Kind::Video(_))); anyhow::ensure!( !needs_clock || video.is_some(), @@ -2087,16 +2010,9 @@ where let program_info = if !self.program_descriptors.is_empty() { self.program_descriptors.clone() } else if tracks.iter().any(|t| { - // Only derive CUEI for section-framed 0x86 (SCTE-35); a PES-framed 0x86 + // Only derive CUEI for section-framed 0x86 (SCTE-35); a whole-PES 0x86 // (e.g. DTS audio) must not advertise SCTE-35 section signaling. - matches!( - &t.kind, - Kind::Verbatim { - stream_type: 0x86, - framing: catalog::Framing::Section, - .. - } - ) + matches!(&t.kind, Kind::Verbatim { stream_type: 0x86 }) }) { vec![registration(b"CUEI")] } else { @@ -2201,16 +2117,9 @@ where // Each moq Opus frame is one packet; prefix the Opus-in-TS control header. Kind::Opus { .. } => Some(opus_es_payload(&frame.payload)), // Legacy audio frames were ingested whole (framing header included), so - // they pass through untouched. PES-framed verbatim payloads likewise. + // they pass through untouched. Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 => Some(frame.payload.to_vec()), - Kind::Verbatim { - framing: catalog::Framing::Pes, - .. - } => Some(frame.payload.to_vec()), - Kind::Verbatim { - framing: catalog::Framing::Section, - .. - } => None, + Kind::Verbatim { .. } => None, // The frame is the PES already, written by [`Self::place`] below. Kind::PesPacket { .. } => None, }; @@ -2279,15 +2188,9 @@ where // carry the bytes verbatim. None => self.write_section(&mut out, pid, &frame.payload)?, Some(es_payload) => { - // Verbatim PES re-emits its original stream_id (falling back to - // private_stream_1 for an undecoded stream with none recorded); media + // Opus is private-data PES, carried under private_stream_1 like ffmpeg; media // derives it from is_video. - let stream_id = match &kind { - Kind::Verbatim { stream_id, .. } => Some(stream_id.unwrap_or(StreamId::PRIVATE_STREAM_1)), - // Opus is private-data PES, carried under private_stream_1 like ffmpeg. - Kind::Opus { .. } => Some(StreamId::PRIVATE_STREAM_1), - _ => None, - }; + let stream_id = matches!(kind, Kind::Opus { .. }).then_some(StreamId::PRIVATE_STREAM_1); let unit = PesUnit { pid, is_video, @@ -2873,7 +2776,8 @@ impl PesPacket { let stream_id = bytes[3]; let length = usize::from(u16::from_be_bytes([bytes[4], bytes[5]])); if length == 0 { - let video = matches!(stream_id, 0xe0..=0xef) || (stream_id == 0xfd && is_video_stream_type(stream_type)); + let video = + matches!(stream_id, 0xe0..=0xef) || (stream_id == 0xfd && psi::is_video_stream_type(stream_type)); anyhow::ensure!( video, "PES_packet_length 0 on stream_id {stream_id:#04x}, which only a video stream may leave unbounded" @@ -2887,7 +2791,7 @@ impl PesPacket { } // program_stream_map, padding, private_stream_2, ECM, EMM, directory, DSM-CC and // H.222.1 type E carry their payload straight after the length. - if matches!(stream_id, 0xbc | 0xbe | 0xbf | 0xf0 | 0xf1 | 0xf2 | 0xf8 | 0xff) { + if !psi::has_optional_header(stream_id) { return Ok(Self { pts: None, payload: PES_PREFIX_LEN, @@ -2966,15 +2870,6 @@ impl PesPacket { } } -/// Whether a PMT `stream_type` carries video (ISO 13818-1 Table 2-34, and VC-1's 0xEA), whose -/// PES alone may leave `PES_packet_length` unbounded. -fn is_video_stream_type(stream_type: u8) -> bool { - matches!( - stream_type, - 0x01 | 0x02 | 0x10 | 0x1b | 0x1e..=0x26 | 0x28..=0x2b | 0x31..=0x35 | 0xea - ) -} - /// A 33-bit PTS read off a placed PES, on the media timeline in nanoseconds: the value /// nearest its placement `at` that the wrapped field allows, since a PES decodes close to /// where it is placed. @@ -3290,17 +3185,11 @@ const EAC3_DESCRIPTOR: u8 = 0x7a; /// The DVB teletext descriptor's tag (ETSI EN 300 468 6.2.43). const TELETEXT_DESCRIPTOR: u8 = 0x56; -/// A verbatim stream's buffers, for the kinds its descriptors identify: Dolby audio as DVB +/// A whole-PES stream's buffers, for the kinds its descriptors identify: Dolby audio as DVB /// private data (ETSI TS 101 154 4.1.6.11), and teletext, whose 480-byte transport buffer drains at /// 6.75 Mb/s (ETSI EN 300 472 5). Any other stream has none the schedule knows. fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option { - let (Kind::Verbatim { - stream_type: 0x06, - framing: catalog::Framing::Pes, - .. - } - | Kind::PesPacket { stream_type: 0x06 }) = kind - else { + let Kind::PesPacket { stream_type: 0x06 } = kind else { return None; }; descriptors.iter().find_map(|descriptor| match descriptor.tag { @@ -3373,223 +3262,6 @@ fn ac3_core(data: &[u8]) -> bool { data.get(5).is_some_and(|byte| byte >> 3 <= 10) } -/// What a passed-through Dolby PES leaves for the next one. -struct DolbyCarry { - /// The playhead count it was read under ([`Pending::skip`]). - skip: u64, - /// When an access unit that the next sync frame begins is presented. - next: Timestamp, - /// The E-AC-3 access unit still open, which a substream at the next PES's start extends. - unit: Option, - /// The bytes so far of a sync frame left unfinished. - bytes: Vec, -} - -/// An access unit begun: when it is presented, and the blocks and rate its substreams share. -#[derive(Clone, Copy)] -struct DolbyUnit { - at: Timestamp, - samples: u64, - rate: u32, -} - -/// The Dolby access units a PES's `payload` completes or holds, each with its presentation -/// time; none when no sync frame can begin in it. An AC-3 access unit is one sync frame; an -/// E-AC-3 one is independent substream 0 with every substream after it (ETSI TS 102 366 -/// E.1.3.1.2), which share its presentation interval. -/// -/// A PES need not begin on a sync frame: a source that leaves `data_alignment_indicator` -/// clear splits frames, and even their headers, across PES. `carry` holds the frame the last -/// PES left unfinished, which this payload's first bytes complete, keeping its own time. The -/// PES's PTS, `start`, times the first access unit that begins in it (ISO 13818-1 2.4.3.7), the -/// rest count on from there, and a frame left unfinished at the end is carried into the next PES. -/// A carried frame is dropped when the playhead jumped since (`skip`: a skipped group, a -/// marker, a resubscription), or when it would not end where the next frame begins, by that -/// PTS and its sync word: either way it spans a gap in the source. So are bytes ahead of the -/// first sync frame with nothing carried. No decoder can use them. -/// -/// An E-AC-3 substream whose access unit went out with the last PES goes out on its own, at -/// that unit's time, whether its PES is timed by that unit or the next. One whose access unit -/// began before the join is dropped. -fn dolby_units( - codec: Dolby, - carry: &mut Option, - skip: u64, - start: Timestamp, - payload: &Bytes, -) -> anyhow::Result> { - let duration = |samples: u64, rate: u32| Timestamp::from_micros(samples * 1_000_000 / u64::from(rate.max(1))); - let joined = carry.as_ref().is_none_or(|head| head.skip != skip); - let mut units = Vec::new(); - let mut open = None; - let mut offset = None; - // When the next access unit to begin in this PES is presented. - let mut base = start; - match carry.take() { - None => {} - Some(head) if head.skip != skip => { - if !head.bytes.is_empty() { - tracing::debug!("dropped a Dolby sync frame cut by a jump in the playhead"); - } - } - Some(head) if head.bytes.is_empty() => { - // A unit left open extends into this PES only if this PES's units follow it. - if let Some(unit) = head.unit { - let length = duration(unit.samples, unit.rate)?; - let end = unit.at.checked_add(length)?; - let near = |at: Timestamp| at.as_micros().abs_diff(start.as_micros()) <= length.as_micros() / 2; - if near(end) { - open = Some(unit); - } else if near(unit.at) { - // Timed by the unit it extends, as a PES of only its substreams is: the - // next unit still begins where that one ends. - open = Some(unit); - base = end; - } else { - tracing::warn!(?end, ?start, "closed an E-AC-3 access unit at a gap between PES"); - } - } - } - Some(mut head) => { - let min = codec.min_header_len(); - let probe: Vec = head.bytes.iter().chain(payload.iter()).take(min).copied().collect(); - if probe.len() < min { - // Even the header runs on past this PES. - head.bytes.extend_from_slice(payload); - *carry = Some(head); - return Ok(units); - } - match codec.parse(&probe) { - Ok(header) => { - let need = header.len.saturating_sub(head.bytes.len()); - let Some((tail, after)) = payload.split_at_checked(need) else { - // The frame runs on past this PES as well. - head.bytes.extend_from_slice(payload); - *carry = Some(head); - return Ok(units); - }; - let at = match (codec.begins_unit(&probe), head.unit) { - (false, Some(unit)) => unit.at, - _ => head.next, - }; - let length = duration(header.samples, header.sample_rate)?; - let end = at.checked_add(length)?; - // Only a frame that begins here has a PTS to meet. - let next = after.is_empty() - || (end.as_micros().abs_diff(start.as_micros()) <= length.as_micros() / 2 - && ac3_sync_prefix(after)); - if next { - head.bytes.extend_from_slice(tail); - let frame = Bytes::from(head.bytes); - // A unit stays open only through a carried frame that completes without a gap. - open = head.unit; - dolby_place(codec, frame, &header, head.next, &mut open, &mut units)?; - offset = Some(need); - } else { - tracing::warn!(?end, ?start, "dropped a Dolby sync frame cut by a gap between PES"); - } - } - Err(_) => tracing::warn!("dropped the start of a Dolby sync frame whose header does not parse"), - } - } - } - let mut offset = match offset { - Some(offset) => offset, - None if payload.is_empty() => 0, - None => { - let Some(first) = dolby_sync(codec, payload) else { - // No decoder can use a PES without a sync frame either, so it goes nowhere. - match joined { - true => tracing::debug!(bytes = payload.len(), "dropped a PES begun before the join"), - false => tracing::warn!(bytes = payload.len(), "dropped a PES with no Dolby sync frame"), - } - return Ok(units); - }; - if first > 0 { - tracing::debug!( - bytes = first, - "dropped the end of a Dolby sync frame begun before the join" - ); - } - first - } - }; - let (mut samples, mut rate) = (0u64, 0u32); - let mut left = Vec::new(); - while offset < payload.len() { - let rest = &payload[offset..]; - let next = base.checked_add(duration(samples, rate)?)?; - let header = match codec.parse(rest) { - Ok(header) => header, - // A header cut short at the end, which the next PES completes. - Err(_) if rest.len() < codec.min_header_len() && ac3_sync_prefix(rest) => { - left = rest.to_vec(); - break; - } - Err(_) => { - tracing::warn!(bytes = rest.len(), "dropped the rest of a PES that lost Dolby sync"); - break; - } - }; - if rest.len() < header.len { - left = rest.to_vec(); - break; - } - let frame = payload.slice(offset..offset + header.len); - if dolby_place(codec, frame, &header, next, &mut open, &mut units)? { - samples += header.samples; - rate = header.sample_rate; - } - offset += header.len; - } - *carry = Some(DolbyCarry { - skip, - next: base.checked_add(duration(samples, rate)?)?, - unit: open.filter(|_| codec == Dolby::Eac3), - bytes: left, - }); - Ok(units) -} - -/// File the whole sync frame `frame` in `units`: as a new access unit presented at `next`, or -/// as a substream of the `open` one, joining it when it went out from this PES too. Returns -/// whether it began a unit. -fn dolby_place( - codec: Dolby, - frame: Bytes, - header: &crate::codec::legacy::Header, - next: Timestamp, - open: &mut Option, - units: &mut Vec<(Timestamp, Bytes)>, -) -> anyhow::Result { - if codec.begins_unit(&frame) { - *open = Some(DolbyUnit { - at: next, - samples: header.samples, - rate: header.sample_rate, - }); - units.push((next, frame)); - return Ok(true); - } - let Some(unit) = *open else { - tracing::debug!("dropped an E-AC-3 substream whose access unit began before the join"); - return Ok(false); - }; - anyhow::ensure!( - header.samples == unit.samples && header.sample_rate == unit.rate, - "E-AC-3 substream of {} samples at {} Hz in an access unit of {} at {} Hz", - header.samples, - header.sample_rate, - unit.samples, - unit.rate, - ); - match units.last_mut() { - Some((at, bytes)) if *at == unit.at => *bytes = [bytes.as_ref(), frame.as_ref()].concat().into(), - _ => units.push((unit.at, frame)), - } - Ok(false) -} - /// Where the first Dolby sync frame in `payload` begins: a sync word whose header parses and /// whose frame ends at the next sync word. Failing that, the first that may still begin one, /// for the next PES to confirm: a frame that runs to the payload's end or past it, or a @@ -3868,61 +3540,6 @@ mod tests { Timestamp::from_millis(value).unwrap() } - /// A PES that completes the AC-3 sync frame the last one began holds a frame decoding - /// ahead of its own PTS. Each sync frame reaches the jitter buffer at its own decode time, - /// so the buffer orders that frame against the other tracks on its own time, not the PES's. - #[tokio::test(start_paused = true)] - async fn ac3_frames_reach_the_jitter_buffer_at_their_own_decode_time() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let source = crate::source::announced(&broadcast.consume()).await; - let source = super::ExportSource::for_stream(&source, "audio", Duration::ZERO).unwrap(); - let mut track = super::Track::new( - source, - Kind::Verbatim { - stream_type: 0x06, - framing: super::catalog::Framing::Pes, - stream_id: None, - }, - ); - track.descriptors = vec![super::catalog::Descriptor { - tag: super::AC3_DESCRIPTOR, - data: bytes::Bytes::from_static(&[0x00]), - }]; - let mut jitter = super::jitter::Buffer::new(Duration::ZERO); - - // Three 32 ms sync frames, cut mid-frame into frames 0-1 and 1-2, timed by frames 0 and 2. - let mut frame = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; - frame.resize(128, 0x00); - let stream = bytes::Bytes::from(frame.repeat(3)); - // In microseconds, as every hang frame is. - let us = |ms: u64| Timestamp::from_micros(ms * 1_000).unwrap(); - let now = moq_clock::now(); - for (pts, payload) in [(us(0), stream.slice(..200)), (us(64), stream.slice(200..))] { - let frame = super::Frame { - timestamp: pts, - duration: None, - payload, - keyframe: true, - }; - let pending = super::Pending { - frame, - restart: 0, - skip: 0, - cut: 0, - arrived: now, - read: now, - }; - track.queue("audio", pending, &mut jitter).unwrap(); - } - - let waiter = kio::Waiter::noop(); - let mut decode = Vec::new(); - while let std::task::Poll::Ready(ready) = jitter.poll_next(&waiter) { - decode.push(ready.item.decode); - } - assert_eq!(decode, [us(0), us(32), us(64)]); - } - /// A six-block E-AC-3 sync frame of 512 bytes: `stream` is byte 2's top bits (strmtyp, /// substreamid) and `rate` byte 4 (fscod, numblkscod, acmod, lfeon). fn eac3(stream: u8, rate: u8) -> Vec { @@ -3931,70 +3548,12 @@ mod tests { frame } - /// Access units count on from the PES's PTS by the samples before them, so a 44.1 kHz - /// frame's 34,829.9 us does not round down once per frame. - #[test] - fn dolby_units_count_samples_not_rounded_periods() { - let payload = bytes::Bytes::from(eac3(0x00, 0x7f).repeat(30)); - let units = super::dolby_units( - super::Dolby::Eac3, - &mut None, - 0, - Timestamp::from_micros(0).unwrap(), - &payload, - ) - .unwrap(); - assert_eq!(units.len(), 30); - for (k, (at, _)) in units.iter().enumerate() { - assert_eq!(at.as_micros(), k as u128 * 1_536 * 1_000_000 / 44_100, "unit {k}"); - } - } - - /// An E-AC-3 stream may carry AC-3 as its independent substream 0, which begins each access - /// unit that the E-AC-3 substreams after it extend. - #[test] - fn dolby_units_begin_at_an_ac3_core() { - // 32 kb/s 48 kHz AC-3: 128 bytes, 1,536 samples, as six E-AC-3 blocks are. - let mut core = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; - core.resize(128, 0); - let unit = [core, eac3(0x40, 0x3f)].concat(); - let payload = bytes::Bytes::from(unit.repeat(2)); - let units = - super::dolby_units(Dolby::Eac3, &mut None, 0, Timestamp::from_micros(0).unwrap(), &payload).unwrap(); - let at: Vec<_> = units - .iter() - .map(|(at, bytes)| (at.as_micros(), bytes.to_vec())) - .collect(); - assert_eq!(at, [(0, unit.clone()), (32_000, unit)]); - } - - /// Substreams of one access unit share its blocks and rate (ETSI TS 102 366 E.1.3.1.2), so - /// one that differs is malformed rather than timed by a guess. - #[test] - fn dolby_units_refuse_a_substream_off_its_unit_interval() { - // A three-block dependent substream after a six-block independent one. - let payload = bytes::Bytes::from([eac3(0x00, 0x3f), eac3(0x40, 0x2f)].concat()); - let err = super::dolby_units( - super::Dolby::Eac3, - &mut None, - 0, - Timestamp::from_micros(0).unwrap(), - &payload, - ) - .unwrap_err(); - assert!(err.to_string().contains("E-AC-3 substream"), "{err}"); - } - /// Teletext passed through as DVB private data takes its transport buffer's rate from /// EN 300 472, so the schedule spreads it no faster than a receiver passes it on; private /// data nothing identifies has no buffers. #[test] fn teletext_drains_at_its_own_rate() { - let kind = Kind::Verbatim { - stream_type: 0x06, - framing: super::catalog::Framing::Pes, - stream_id: None, - }; + let kind = Kind::PesPacket { stream_type: 0x06 }; let descriptor = |tag| super::catalog::Descriptor { tag, data: bytes::Bytes::from_static(b"eng\x09\x00"), diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 252e0714140..c65809a3e4a 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1500,301 +1500,6 @@ async fn export_pcr_backfills_a_coarse_cadence() { } } -/// AC-3 passed through as DVB private data (stream_type 0x06 with the AC-3 descriptor) can -/// carry several sync frames in one PES, more than a receiver's 5,696-byte buffer holds at -/// once. The export splits it into its sync frames, each presented a frame after the last, so -/// each is due and decoded on its own. -#[tokio::test(start_paused = true)] -async fn export_splits_a_multi_frame_ac3_pes() { - let frames = exported_ac3(ac3_sync().repeat(3)).await; - assert_eq!(frames.len(), 3, "every sync frame: {frames:?}"); - for (k, &(pts, bytes)) in frames.iter().enumerate() { - assert_eq!(bytes, 128, "frame {k} carries one sync frame"); - assert_eq!( - pts, - frames[0].0 + k as u64 * 2_880, - "frame {k} is presented 32 ms after the last" - ); - } -} - -/// A sync frame cut short of the length its header gives, as a source stopped mid-frame -/// leaves at the end of its last PES, is no use to a decoder, so the export drops it rather -/// than pass the PES on whole: after whole frames, or alone. -#[tokio::test(start_paused = true)] -async fn export_drops_a_trailing_partial_ac3_frame() { - let mut payload = ac3_sync().repeat(2); - payload.extend_from_slice(&ac3_sync()[..60]); - let frames = exported_ac3(payload).await; - assert_eq!(frames.len(), 2, "the whole frames: {frames:?}"); - assert!(frames.iter().all(|&(_, bytes)| bytes == 128), "{frames:?}"); - - let frames = exported_ac3(ac3_sync()[..60].to_vec()).await; - assert!(frames.is_empty(), "a lone cut frame goes nowhere: {frames:?}"); -} - -/// A source that leaves `data_alignment_indicator` clear splits AC-3 sync frames across PES, -/// each PES's PTS timing the first frame that begins in it. The export puts each frame back -/// together, byte for byte and presented a frame after the last. -#[tokio::test(start_paused = true)] -async fn export_reassembles_ac3_frames_split_across_pes() { - let stream: Vec = (0..6).flat_map(ac3_frame).collect(); - // Cut mid-frame into frames 0-1, 1-3 and 3-5, timed by frames 0, 2 and 4. - let frames = exported_ac3_frames(vec![ - Ac3Write::Pes(0, stream[..200].to_vec()), - Ac3Write::Pes(64, stream[200..500].to_vec()), - Ac3Write::Pes(128, stream[500..].to_vec()), - ]) - .await; - assert_eq!(frames.len(), 6, "every sync frame"); - for (k, (pts, bytes)) in frames.iter().enumerate() { - assert_eq!(*bytes, ac3_frame(k as u8), "frame {k} whole and in order"); - assert_eq!( - *pts, - frames[0].0 + k as u64 * 2_880, - "frame {k} is presented 32 ms after the last" - ); - } -} - -/// A frame's header may itself straddle PES, across one boundary or two, and the frame still -/// goes out whole at the time of the PES it began in. -#[tokio::test(start_paused = true)] -async fn export_reassembles_an_ac3_header_split_across_pes() { - let stream: Vec = (0..2).flat_map(ac3_frame).collect(); - let cuts = [ - vec![ - Ac3Write::Pes(0, stream[..3].to_vec()), - Ac3Write::Pes(32, stream[3..].to_vec()), - ], - vec![ - Ac3Write::Pes(0, stream[..2].to_vec()), - Ac3Write::Pes(16, stream[2..5].to_vec()), - Ac3Write::Pes(32, stream[5..].to_vec()), - ], - ]; - for input in cuts { - let timed = exported_ac3_frames(input).await; - let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); - assert_eq!(frames, [&ac3_frame(0), &ac3_frame(1)]); - assert_eq!(timed[1].0 - timed[0].0, 2_880); - } -} - -/// The first PES read may begin mid-frame, and a PES may go missing. The bytes ahead of the -/// first sync frame, and a frame the gap cut, are no use to a decoder, so the export drops them -/// and carries on from the next sync frame at the time its PES gives it. -#[tokio::test(start_paused = true)] -async fn export_drops_ac3_fragments_at_a_join_or_a_gap() { - let stream: Vec = (0..6).flat_map(ac3_frame).collect(); - // Joined 60 bytes into frame 0; the PES carrying the rest of frame 2 and most of 3 is lost. - let timed = exported_ac3_frames(vec![ - Ac3Write::Pes(32, stream[60..300].to_vec()), - Ac3Write::Pes(128, stream[500..].to_vec()), - ]) - .await; - let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); - assert_eq!(frames, [&ac3_frame(1), &ac3_frame(4), &ac3_frame(5)]); - assert_eq!(timed[1].0 - timed[0].0, 3 * 2_880, "frame 4 at its PES's PTS"); - assert_eq!(timed[2].0 - timed[1].0, 2_880); -} - -/// A sync word in the bytes ahead of the first frame at a join can pass for a header whose -/// frame runs past the PES. The export takes it for a frame only when nothing better turns up, -/// and here takes the sync frames the next sync word confirms after it. -#[tokio::test(start_paused = true)] -async fn export_skips_an_unconfirmed_ac3_sync_word_at_a_join() { - // A 640 kb/s header, 2,560 bytes long, in the tail of a frame begun before the join. - let mut payload = vec![0x0b, 0x77, 0x00, 0x00, 0x25, 0x40, 0x40]; - payload.resize(40, 0x00); - payload.extend((1..3).flat_map(ac3_frame)); - let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, payload)]).await; - let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); - assert_eq!(frames, [&ac3_frame(1), &ac3_frame(2)]); -} - -/// A declared marker breaks the timeline even where the timestamps run on, so a frame left -/// unfinished before it is dropped rather than completed with the first bytes after it. The -/// marker follows a group that ended, so nothing held before the break is trimmed, and it is -/// declared, so it is no gap. -#[tokio::test(start_paused = true)] -async fn export_drops_an_ac3_frame_cut_by_a_marker() { - let stream: Vec = (0..4).flat_map(ac3_frame).collect(); - let (pes, stats) = exported_ac3_pes(vec![ - Ac3Write::Pes(0, stream[..200].to_vec()), - Ac3Write::Marker, - Ac3Write::Pes(64, stream[256..].to_vec()), - ]) - .await; - let timed = ac3_frames(pes); - let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); - assert_eq!(frames, [&ac3_frame(0), &ac3_frame(2), &ac3_frame(3)]); - assert_eq!(stats.gaps, 0, "the declared break is not media lost upstream"); -} - -/// AC-3 sync frames that run on share a PES, as many as DVB's 5,696-byte B holds with the -/// PES header (ISO 13818-1 2.4.2.3), and still decode in order 32 ms apart. -#[tokio::test(start_paused = true)] -async fn export_packs_ac3_frames_up_to_their_buffer() { - let stream: Vec = (0..100).flat_map(ac3_frame).collect(); - let (pes, _) = exported_ac3_pes(vec![Ac3Write::Pes(0, stream.clone())]).await; - // 44 of the 128-byte frames to a PES, and the first frame alone, sent as it starts. - assert!( - pes.len() <= 100usize.div_ceil(44) + 1, - "{} PES for 100 frames", - pes.len() - ); - for (_, payload) in &pes { - assert!(14 + payload.len() <= 5_696, "a {}-byte PES overflows B", payload.len()); - } - let bytes: Vec = pes.into_iter().flat_map(|(_, payload)| payload).collect(); - assert_eq!(bytes, stream, "every frame whole and in order"); - let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, stream)]).await; - assert!( - timed.windows(2).all(|pair| pair[1].0 - pair[0].0 == 2_880), - "32 ms apart" - ); -} - -/// One six-block 48 kHz E-AC-3 sync frame, with an optional dependent substream. -fn eac3_frame(dependent: bool) -> Vec { - let mut frame = vec![0x0b, 0x77, if dependent { 0x40 } else { 0 }, 0xff, 0x3f, 0x80]; - frame.resize(512, 0xaa); - frame -} - -#[tokio::test(start_paused = true)] -async fn export_splits_dvb_eac3_access_units() { - let frame = eac3_frame(false); - let (pes, _) = exported_dolby_pes(vec![Ac3Write::Pes(0, frame.repeat(3))], 0x7a).await; - assert_eq!(pes.len(), 3); - for (i, (pts, payload)) in pes.iter().enumerate() { - assert_eq!(*pts, pes[0].0 + i as u64 * 2_880); - assert_eq!(*payload, frame); - } -} - -#[tokio::test(start_paused = true)] -async fn export_keeps_dvb_eac3_dependent_substreams_on_one_interval() { - let unit = [eac3_frame(false), eac3_frame(true)].concat(); - let (pes, _) = exported_dolby_pes(vec![Ac3Write::Pes(0, unit.repeat(3))], 0x7a).await; - assert_eq!(pes.len(), 3); - for (i, (pts, payload)) in pes.iter().enumerate() { - assert_eq!(*pts, pes[0].0 + i as u64 * 2_880, "one 32 ms interval per access unit"); - assert_eq!(*payload, unit); - } -} - -#[tokio::test(start_paused = true)] -async fn export_drops_dvb_eac3_partial_frames() { - let frame = eac3_frame(false); - let payload = [frame.clone(), frame[..60].to_vec()].concat(); - let (pes, _) = exported_dolby_pes(vec![Ac3Write::Pes(0, payload)], 0x7a).await; - assert_eq!(pes.len(), 1); - assert_eq!(pes[0].1, frame); -} - -/// A source that leaves `data_alignment_indicator` clear splits E-AC-3 sync frames across PES, -/// as it does AC-3's, and each still goes out whole, a frame after the last. -#[tokio::test(start_paused = true)] -async fn export_reassembles_dvb_eac3_frames_split_across_pes() { - let frame = eac3_frame(false); - let stream = frame.repeat(6); - // Cut mid-frame into frames 0-1, 1-3 and 3-5, timed by frames 0, 2 and 4. - let (pes, _) = exported_dolby_pes( - vec![ - Ac3Write::Pes(0, stream[..800].to_vec()), - Ac3Write::Pes(64, stream[800..2000].to_vec()), - Ac3Write::Pes(128, stream[2000..].to_vec()), - ], - 0x7a, - ) - .await; - assert_eq!(pes.len(), 6, "every sync frame"); - for (k, (pts, payload)) in pes.iter().enumerate() { - assert_eq!(*payload, frame, "frame {k} whole"); - assert_eq!(*pts, pes[0].0 + k as u64 * 2_880, "frame {k} 32 ms after the last"); - } -} - -/// A dependent substream the next PES carries, whole or completing, goes out after its access -/// unit's independent substream at that unit's time, not the next unit's. -#[tokio::test(start_paused = true)] -async fn export_times_a_dvb_eac3_substream_by_its_access_unit_across_pes() { - let (independent, dependent) = (eac3_frame(false), eac3_frame(true)); - let unit = [independent.clone(), dependent.clone()].concat(); - let stream = unit.repeat(3); - // Cut between unit 0's substreams, then inside its dependent substream. - for cut in [512, 800] { - let (pes, _) = exported_dolby_pes( - vec![ - Ac3Write::Pes(0, stream[..cut].to_vec()), - Ac3Write::Pes(32, stream[cut..].to_vec()), - ], - 0x7a, - ) - .await; - let start = pes[0].0; - assert_eq!( - pes, - [ - (start, independent.clone()), - (start, dependent.clone()), - (start + 2_880, unit.clone()), - (start + 5_760, unit.clone()), - ], - "cut at {cut}" - ); - } -} - -/// A PES of only a dependent substream, timed by the access unit it extends as the export's -/// own output is, still joins that unit, and the next unit keeps its own time. -#[tokio::test(start_paused = true)] -async fn export_keeps_a_dvb_eac3_substream_pes_timed_by_its_unit() { - let (independent, dependent) = (eac3_frame(false), eac3_frame(true)); - let unit = [independent.clone(), dependent.clone()].concat(); - let (pes, _) = exported_dolby_pes( - vec![ - Ac3Write::Pes(0, independent.clone()), - Ac3Write::Pes(0, [dependent.clone(), unit.clone()].concat()), - ], - 0x7a, - ) - .await; - let start = pes[0].0; - assert_eq!(pes, [(start, independent), (start, dependent), (start + 2_880, unit)]); -} - -/// A PES without a sync frame holds nothing a decoder can use, so it goes nowhere rather -/// than out whole. -#[tokio::test(start_paused = true)] -async fn export_drops_a_dvb_eac3_pes_with_no_sync_frame() { - let frame = eac3_frame(false); - let (pes, _) = exported_dolby_pes( - vec![ - Ac3Write::Pes(0, frame.clone()), - Ac3Write::Pes(32, vec![0xee; 300]), - Ac3Write::Pes(64, frame.clone()), - ], - 0x7a, - ) - .await; - assert_eq!(pes.len(), 2, "{pes:?}"); - assert!(pes.iter().all(|(_, payload)| *payload == frame)); -} - -/// A dependent substream ahead of the first independent one belongs to an access unit that -/// began before the join, so it goes nowhere. -#[tokio::test(start_paused = true)] -async fn export_drops_dvb_eac3_substreams_before_the_join() { - let unit = [eac3_frame(false), eac3_frame(true)].concat(); - let payload = [eac3_frame(true), unit.clone()].concat(); - let (pes, _) = exported_dolby_pes(vec![Ac3Write::Pes(0, payload)], 0x7a).await; - assert_eq!(pes.len(), 1); - assert_eq!(pes[0].1, unit); -} - /// A 32 kb/s 48 kHz 2/0 AC-3 sync frame: 128 bytes, 1,536 samples (32 ms). fn ac3_sync() -> Vec { let mut sync = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; @@ -1809,141 +1514,6 @@ fn ac3_frame(k: u8) -> Vec { frame } -/// Export one PES of `payload` on an AC-3 passthrough PID beside bbb's media, and return the -/// PTS of every sync frame on that PID and how many bytes each carries. -async fn exported_ac3(payload: Vec) -> Vec<(u64, usize)> { - exported_ac3_frames(vec![Ac3Write::Pes(0, payload)]) - .await - .into_iter() - .map(|(pts, bytes)| (pts, bytes.len())) - .collect() -} - -/// What [`exported_ac3_pes`] writes to the AC-3 track. -enum Ac3Write { - /// A PES presented this many ms after the first. - Pes(u64, Vec), - /// A declared marker: a break in the timeline. - Marker, -} - -/// Export `input` on an AC-3 passthrough PID beside bbb's media, and return the PTS and -/// payload of every PES on that PID, and the export's stats. -async fn exported_ac3_pes(input: Vec) -> (Vec<(u64, Vec)>, super::stats::Export) { - exported_dolby_pes(input, 0x6a).await -} - -/// [`exported_ac3_pes`] for the Dolby audio `descriptor` tags. -async fn exported_dolby_pes(input: Vec, descriptor: u8) -> (Vec<(u64, Vec)>, super::stats::Export) { - const AC3_PID: u16 = 0x104; - let data = include_bytes!("test_data/bbb.ts"); - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let mut catalog = crate::catalog::Producer::new( - &mut broadcast, - crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), - ) - .unwrap(); - // Reserve before the hand-written track below, so its catalog edit is withheld and the - // import still places the clock, keeping bbb's PTS that track is stamped against. - let reserved = catalog.reserve(); - let ac3_track = broadcast - .unique_track(".ac3", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - { - let mut track = tscat::Track::new(AC3_PID); - track.verbatim = Some(tscat::Verbatim::new(0x06, tscat::Framing::Pes)); - track.descriptors = vec![tscat::Descriptor { - tag: descriptor, - data: Bytes::from_static(&[0x00]), - }]; - catalog - .modify() - .unwrap() - .ext - .mpegts - .tracks - .insert(ac3_track.name().to_string(), track); - } - let mut ac3 = Producer::new(ac3_track, HangContainer::Legacy(crate::container::Kind::Data)); - // Just after bbb's first keyframe at 1.4 s, so it survives the tune-in alignment. - for write in input { - match write { - Ac3Write::Pes(ms, payload) => ac3 - .write(Frame { - timestamp: Timestamp::from_millis(1_410 + ms).unwrap(), - duration: None, - payload: Bytes::from(payload), - keyframe: true, - }) - .unwrap(), - Ac3Write::Marker => ac3.discontinuity().unwrap(), - } - } - ac3.cut(None).unwrap(); - ac3.finish().unwrap(); - - let mut import = crate::container::ts::Import::new(broadcast, reserved); - import.decode(&BytesMut::from(&data[..])).unwrap(); - import.finish().unwrap(); - let (ts, stats) = drain_stats( - open_ts( - crate::source::announced(&consumer).await, - crate::catalog::CatalogFormat::Hang, - ) - .await - .unwrap(), - ) - .await; - - // The PTS of every PES on the AC-3 PID, and its payload. - let mut pes: Vec<(u64, Vec)> = Vec::new(); - for packet in ts.as_chunks::<188>().0 { - let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); - if pid != AC3_PID || packet[3] & 0x10 == 0 { - continue; - } - let start = 4 + if packet[3] & 0x20 != 0 { - usize::from(packet[4]) + 1 - } else { - 0 - }; - let body = &packet[start..]; - if packet[1] & 0x40 != 0 { - let b = &body[9..14]; - let pts = u64::from((b[0] >> 1) & 7) << 30 - | u64::from(b[1]) << 22 - | u64::from(b[2] >> 1) << 15 - | u64::from(b[3]) << 7 - | u64::from(b[4] >> 1); - pes.push((pts, body[9 + usize::from(body[8])..].to_vec())); - } else if let Some(last) = pes.last_mut() { - last.1.extend_from_slice(body); - } - } - (pes, stats) -} - -/// Export `input` as [`exported_ac3_pes`] does, and return each sync frame on the AC-3 PID, -/// timed by its PES's PTS and the frames ahead of it there. -async fn exported_ac3_frames(input: Vec) -> Vec<(u64, Vec)> { - ac3_frames(exported_ac3_pes(input).await.0) -} - -/// Each sync frame of `pes`, timed by its PES's PTS and the frames ahead of it there. -fn ac3_frames(pes: Vec<(u64, Vec)>) -> Vec<(u64, Vec)> { - let mut frames = Vec::new(); - for (mut pts, mut payload) in pes { - while !payload.is_empty() { - let header = crate::codec::ac3::parse_header(&payload).expect("the PES carries whole sync frames"); - let rest = payload.split_off(header.len); - frames.push((pts, std::mem::replace(&mut payload, rest))); - pts += header.samples * 90_000 / u64::from(header.sample_rate); - } - } - frames -} - /// Full SCTE-35 round-trip: import `bbb.ts` (real H.264 + AAC) into a broadcast /// that also carries a `.scte35` cue track, export to TS, re-import, and assert /// the splice_info_section came back byte-for-byte. The PMT must advertise the @@ -2084,15 +1654,16 @@ async fn export_scte35_roundtrip() { /// An opaque PMT byte is signalling, not a codec the exporter must recognize. #[tokio::test(start_paused = true)] async fn export_unknown_pes_stream_type() { - assert_unknown_stream_type(tscat::Framing::Pes).await; + assert_unknown_stream_type(true).await; } #[tokio::test(start_paused = true)] async fn export_unknown_section_stream_type() { - assert_unknown_stream_type(tscat::Framing::Section).await; + assert_unknown_stream_type(false).await; } -async fn assert_unknown_stream_type(framing: tscat::Framing) { +/// Export a track of stream type 0x99, carried as whole PES packets or as sections. +async fn assert_unknown_stream_type(pes: bool) { const PID: u16 = 0x104; let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); @@ -2106,7 +1677,16 @@ async fn assert_unknown_stream_type(framing: tscat::Framing) { .unique_track(".opaque", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); let mut entry = tscat::Track::new(PID); - entry.verbatim = Some(tscat::Verbatim::new(0x99, framing)); + let payload = match pes { + true => { + entry.pes_packet = Some(tscat::PesPacket::new(0x99)); + Bytes::from(pes_with(0xbd, 0x84, 0x00, &[], 0, b"opaque")) + } + false => { + entry.verbatim = Some(tscat::Verbatim::new(0x99, tscat::Framing::Section)); + Bytes::from_static(CUE) + } + }; catalog .modify() .unwrap() @@ -2119,7 +1699,7 @@ async fn assert_unknown_stream_type(framing: tscat::Framing) { .write(Frame { timestamp: Timestamp::from_millis(1410).unwrap(), duration: None, - payload: Bytes::from_static(CUE), + payload, keyframe: true, }) .unwrap(); @@ -2173,16 +1753,13 @@ async fn assert_unknown_stream_type(framing: tscat::Framing) { ); } -/// PES-framed verbatim round-trip: import `bbb.ts` (real H.264 + AAC, whose video -/// supplies the media clock the exporter needs) alongside a private PES-framed -/// stream (stream_type 0x06) carried verbatim, export to TS, then re-import and -/// assert the PID, framing, stream_id, and payload all survive. Exercises the -/// exporter's PES re-emit path; `private_pes_carried_verbatim` only covers import. +/// Whole-PES round-trip: import `bbb.ts` (real H.264 + AAC, whose video supplies the media +/// clock the exporter needs) alongside a `pesPacket` track (stream_type 0x06), export to TS, +/// then re-import and assert the PID, stream type, and every PES byte survive. #[tokio::test(start_paused = true)] -async fn export_pes_verbatim_roundtrip() { +async fn export_pes_packet_roundtrip() { const DATA_PID: u16 = 0x104; - const STREAM_ID: u8 = 0xc0; - const PAYLOAD: &[u8] = &[0xde, 0xad, 0xbe, 0xef, 0x01, 0x02]; + let pes = pes_with(0xbd, 0x84, 0x00, &[], 0, &[0xde, 0xad, 0xbe, 0xef, 0x01, 0x02]); let data = include_bytes!("test_data/bbb.ts"); @@ -2197,17 +1774,15 @@ async fn export_pes_verbatim_roundtrip() { // import still places the clock, keeping bbb's PTS that track is stamped against. let reserved = catalog.reserve(); - // Build the verbatim PES track BEFORE moving `broadcast` into `Import`; the + // Build the PES track BEFORE moving `broadcast` into `Import`; the // producer stays alive so the exporter can subscribe to the retained track. let data_track = broadcast .unique_track(".data", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); let data_name = data_track.name().to_string(); { - let mut verbatim = tscat::Verbatim::new(0x06, tscat::Framing::Pes); - verbatim.stream_id = Some(STREAM_ID); let mut track = tscat::Track::new(DATA_PID); - track.verbatim = Some(verbatim); + track.pes_packet = Some(tscat::PesPacket::new(0x06)); catalog .modify() .unwrap() @@ -2223,7 +1798,7 @@ async fn export_pes_verbatim_roundtrip() { .write(Frame { timestamp: Timestamp::from_millis(1410).unwrap(), duration: None, - payload: Bytes::from_static(PAYLOAD), + payload: Bytes::from(pes.clone()), keyframe: true, }) .unwrap(); @@ -2247,7 +1822,7 @@ async fn export_pes_verbatim_roundtrip() { .await; assert_packet_aligned(&ts); - // Re-import the exported TS and recover the verbatim PES stream. + // Re-import the exported TS and recover the PES stream. let mut broadcast2 = moq_net::broadcast::Info::new().produce(); let consumer2 = broadcast2.consume(); let catalog2 = crate::catalog::Producer::new( @@ -2265,12 +1840,9 @@ async fn export_pes_verbatim_roundtrip() { .mpegts .tracks .iter() - .find(|(_, t)| t.verbatim.as_ref().is_some_and(|v| v.stream_type == 0x06)) - .expect("verbatim PES survived the round-trip"); + .find(|(_, t)| t.pes_packet.as_ref().is_some_and(|p| p.stream_type == 0x06)) + .expect("the PES track survived the round-trip"); assert_eq!(track.pid, DATA_PID, "PES PID preserved"); - let verbatim = track.verbatim.as_ref().unwrap(); - assert_eq!(verbatim.framing, tscat::Framing::Pes, "PES framing preserved"); - assert_eq!(verbatim.stream_id, Some(STREAM_ID), "PES stream_id preserved"); let name = name.clone(); let track = consumer2 @@ -2284,12 +1856,8 @@ async fn export_pes_verbatim_roundtrip() { .read_frame() .await .unwrap() - .expect("no verbatim PES frame after round-trip"); - assert_eq!( - frame.payload.as_ref(), - PAYLOAD, - "verbatim PES payload round-trips byte-for-byte" - ); + .expect("no PES frame after round-trip"); + assert_eq!(frame.payload.as_ref(), &pes[..], "the PES round-trips byte-for-byte"); } // SCTE-35 cues are clocked on video, so the exporter rejects a cue program with no video @@ -10507,7 +10075,7 @@ async fn placed_pes_refuses_a_malformed_frame() { async fn placed_pes_refuses_an_ambiguous_entry() { let pes = || vec![(1_000, pes_with(0xbd, 0x84, 0x00, &[], 0, b"payload"))]; let mut both = Placed::new(pes()); - both.entry.verbatim = Some(tscat::Verbatim::new(0x06, tscat::Framing::Pes)); + both.entry.verbatim = Some(tscat::Verbatim::new(0x06, tscat::Framing::Section)); let err = export_placed(both).await.unwrap_err(); assert!(err.to_string().contains("names both"), "{err}"); @@ -10520,8 +10088,7 @@ async fn placed_pes_refuses_an_ambiguous_entry() { assert!(err.to_string().contains("media rendition"), "{err}"); } -/// DVB AC-3 carried whole keeps every sync frame of a PES in it, unsplit, where the legacy -/// `pes` framing cuts the PES into its frames. +/// DVB AC-3 carried whole keeps every sync frame of a PES in it, unsplit. #[tokio::test(start_paused = true)] async fn placed_ac3_pes_keeps_its_sync_frames_together() { let payload: Vec = (0..3).flat_map(ac3_frame).collect(); diff --git a/rs/moq-mux/src/container/ts/fixture.rs b/rs/moq-mux/src/container/ts/fixture.rs index e4c494e24e1..cc0605d69ba 100644 --- a/rs/moq-mux/src/container/ts/fixture.rs +++ b/rs/moq-mux/src/container/ts/fixture.rs @@ -128,6 +128,44 @@ impl Mux { self } + /// A whole PES on `pid`, split into as many packets as it needs, the last padded with + /// adaptation stuffing so the PES ends with it. Returns where its start code sits in the + /// first packet, and the packets, numbered on `pid` in order. + pub(super) fn split(&mut self, pid: u16, pes: &[u8]) -> (usize, Vec>) { + let mut start = 4; + let mut packets = Vec::new(); + for (i, chunk) in pes.chunks(184).enumerate() { + let cc = self.cc(pid); + let pusi = if i == 0 { 0x40 } else { 0 }; + let mut pkt = vec![0x47, pusi | (pid >> 8) as u8 & 0x1f, pid as u8]; + if chunk.len() == 184 { + pkt.push(0x10 | cc); + } else { + let field = 183 - chunk.len(); + pkt.extend_from_slice(&[0x30 | cc, field as u8]); + if field > 0 { + pkt.push(0x00); + pkt.resize(5 + field, 0xff); + } + } + if i == 0 { + start = pkt.len(); + } + pkt.extend_from_slice(chunk); + packets.push(pkt); + } + (start, packets) + } + + /// A whole PES on `pid`, as [`Self::split`] packetizes it. Returns the input offset of its + /// start code. + pub(super) fn whole(&mut self, pid: u16, pes: &[u8]) -> u64 { + let (start, packets) = self.split(pid, pes); + let at = (self.out.len() + start) as u64; + self.out.extend(packets.concat()); + at + } + /// A section on `pid`, split across as many packets as it needs. pub(super) fn section(&mut self, pid: u16, section: &[u8]) -> &mut Self { let mut payload = vec![0u8]; @@ -160,6 +198,22 @@ impl Mux { pcr_pid: u16, streams: &[(u8, u16)], info: usize, + ) -> &mut Self { + let streams: Vec<_> = streams + .iter() + .map(|&(stream_type, es)| (stream_type, es, &[][..])) + .collect(); + self.pmt_with(pid, program, pcr_pid, &streams, info) + } + + /// A [`Self::pmt`] whose streams each carry `(stream_type, pid, ES descriptors)`. + pub(super) fn pmt_with( + &mut self, + pid: u16, + program: u16, + pcr_pid: u16, + streams: &[(u8, u16, &[u8])], + info: usize, ) -> &mut Self { let mut descriptors = Vec::new(); while descriptors.len() < info { @@ -175,8 +229,15 @@ impl Mux { descriptors.len() as u8, ]; body.extend_from_slice(&descriptors); - for &(stream_type, es) in streams { - body.extend_from_slice(&[stream_type, 0xe0 | (es >> 8) as u8, es as u8, 0xf0, 0x00]); + for &(stream_type, es, info) in streams { + body.extend_from_slice(&[ + stream_type, + 0xe0 | (es >> 8) as u8, + es as u8, + 0xf0 | (info.len() >> 8) as u8, + info.len() as u8, + ]); + body.extend_from_slice(info); } self.section(pid, &psi::section(psi::PMT_TABLE_ID, program, 0, 0, 0, &body)) } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 2aaf418abe9..d99b0abee25 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -4,17 +4,18 @@ //! routes their payloads to the codec importers (H.264/H.265/AAC, plus the //! legacy MP2/AC-3/E-AC-3 verbatim path), which own their broadcast tracks and //! catalog entries. Elementary streams we don't decode are carried verbatim, one -//! MoQ track per PID, described in the `mpegts` catalog section: PES-framed streams -//! ride the normal PES reassembly, while section-framed streams (SCTE-35 and -//! other private sections, which are not PES) are reassembled as sections, as the -//! PAT and PMT are. TS adds PAT/PMT discovery, PES reassembly, the +//! MoQ track per PID, described in the `mpegts` catalog section: PES streams ride +//! the normal PES reassembly and travel as whole PES packets at their placement on +//! the program clock, while section-framed streams (SCTE-35 and other private +//! sections, which are not PES) are reassembled as sections, as the PAT and PMT +//! are. TS adds PAT/PMT discovery, PES reassembly, the //! private-section path, and the 90 kHz PTS conversion across its 33-bit wrap. The service //! layer is captured too: the identity parsed from the PAT as a //! [`Program`](catalog::Program) record in the `mpegts` catalog section, and the //! standalone SI PIDs as opaque sections on per-`(PID, table_id)` snapshot tracks //! (see [`si`](super::si)), so both survive the round-trip. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -25,6 +26,7 @@ use mpeg2ts::ts::{Pid, TsPacket}; use super::adts; use super::catalog; use super::health::{Continuation, Continuity, Health}; +use super::passthrough::{PCR, PCR_OFFSET, Sample, locate}; use super::psi::{self, PesStart, Pmt}; use super::stats; use crate::catalog::Offset; @@ -44,11 +46,23 @@ use moq_net::Timestamp; /// budget. Exceeding either limit refuses the input instead of retaining more bytes. /// /// Elementary streams we don't decode are carried verbatim, one MoQ track per -/// PID, when the catalog `E` carries the [`mpegts`](super::Mpegts) section: PES-framed -/// streams ride the normal PES reassembly, section-framed streams (SCTE-35, marked -/// by a program-level 'CUEI' registration descriptor, and other private sections) -/// are reassembled as sections. With a base `Catalog<()>` they're logged and dropped -/// instead. +/// PID, when the catalog `E` carries the [`mpegts`](super::Mpegts) section: section-framed +/// streams (SCTE-35, marked by a program-level 'CUEI' registration descriptor, and other +/// private sections) are reassembled as sections, and any other stream travels as whole PES +/// packets ([`PesPacket`](super::PesPacket)). With a base `Catalog<()>` they're logged and +/// dropped instead. +/// +/// A whole PES is timestamped with its placement: the program clock at its first byte, +/// interpolated between the PCRs either side of it (ISO 13818-1 2.4.2.2) and shifted onto the +/// catalog clock as the media is. Its PTS and DTS move by the same offset inside its bytes, and +/// every other byte is kept. So it waits for the PCR after it, and for a PTS on any stream to +/// anchor the timebase, within the staging bound below; one that starts past the last PCR before a +/// clock break or the end of input is extrapolated from the last two. One before the first PCR, +/// or with no interval to extrapolate from, is dropped and counted in +/// [`stats::Stream::unplaced`]. A stream whose PES carry a clock only the PTS and DTS of are +/// rebased (ESCR, a `pack_header_field`, a TREF), PES-level scrambling, or an unbounded non-video +/// PES is refused rather than altered, as is every such stream in a program with no PCR: +/// [`stats::Stream::refused`]. /// /// The PAT and every PMT are reassembled the same way, so a table that spans packets or /// sits behind a nonzero pointer_field is read whole. A PAT or PMT section whose CRC-32 @@ -111,8 +125,17 @@ pub struct Import { liveness: Liveness, /// In-progress PES reassembly, keyed by elementary PID. pending: HashMap, - /// Partial clear PES headers, at most 9 + 255 bytes per elementary PID. - headers: HashMap>, + /// Partial clear PES headers, at most 9 + 255 bytes per elementary PID, each with the input + /// offset of its start code. + headers: HashMap)>, + /// The input offset of the first byte in `scratch`, which places every packet in the input. + position: u64, + /// The program clock at each input byte, which places whole PES packets. + placement: Placement, + /// Whole PES dropped for want of a clock to place them, keyed by PID. + unplaced: BTreeMap, + /// Private PES streams refused rather than carried altered, keyed by PID. + refused: BTreeMap, /// One program-wide budget for PES payloads and sections awaiting their clock. staging: Arc, /// The TR 101 290 checks, and the per-PID continuity state routing reads. @@ -210,6 +233,10 @@ impl Import { liveness: Liveness::default(), pending: HashMap::new(), headers: HashMap::new(), + position: 0, + placement: Placement::default(), + unplaced: BTreeMap::new(), + refused: BTreeMap::new(), staging: Arc::default(), health: Health::default(), pcr_pid: None, @@ -278,6 +305,7 @@ impl Import { let mut off = 0; while let Some(at) = self.framer.next(&self.scratch, &mut off) { self.health.routed(&self.scratch, at); + let offset = self.position + at as u64; let pkt: [u8; TsPacket::SIZE] = self.scratch[at..at + TsPacket::SIZE].try_into().unwrap(); let pid = (((pkt[1] & 0x1f) as u16) << 8) | pkt[2] as u16; // Every packet paces the multiplex, null stuffing and retransmissions included. @@ -318,7 +346,12 @@ impl Import { if self.pcr_pid.is_some_and(|p| p.as_u16() == pid) && pkt[1] & 0x80 == 0 { if discontinuity_indicator(&pkt) { self.timebase_break()?; + // The new timebase starts with the PCR the flag rides, if it rides one. + if let Some(pcr) = pcr(&pkt) { + self.place_pcr(pcr, offset)?; + } } else if let Some(pcr) = pcr(&pkt) { + self.place_pcr(pcr, offset)?; self.liveness.pcr(pcr); if let Some(now) = self.liveness.now() { self.health.tick(now); @@ -388,13 +421,14 @@ impl Import { // so PES arrive before their PMT. match self.streams.get(&pid) { None | Some(Stream::Ignored) => {} - Some(_) => self.pes_packet(pid, &pkt)?, + Some(_) => self.pes_packet(pid, &pkt, offset)?, } } } self.health.drained(&self.scratch, off); self.scratch.drain(..off); + self.position += off as u64; // Cut the snapshot groups for whatever SI committed in this batch. Batching per // decode call (plus the store's own host-clock debounce) coalesces a junction's // burst of sub-table commits into few groups instead of one per commit. @@ -492,8 +526,9 @@ impl Import { Ok(()) } - /// Route one packet on an elementary stream PID into its PES reassembly. - fn pes_packet(&mut self, pid: Pid, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { + /// Route one packet on an elementary stream PID, at input offset `offset`, into its PES + /// reassembly. + fn pes_packet(&mut self, pid: Pid, pkt: &[u8; TsPacket::SIZE], offset: u64) -> anyhow::Result<()> { let payload = match payload(pkt) { Payload::None => return Ok(()), Payload::Bytes(payload) => payload, @@ -508,34 +543,37 @@ impl Import { if self.headers.remove(&pid).is_some() { self.damage(pid, &anyhow::anyhow!("PES header truncated at next start"))?; } + // The payload ends the packet, so this is where its start code sits in the input. + let at = offset + (TsPacket::SIZE - payload.len()) as u64; return match PesStart::header_len(payload) { Ok(len) if payload.len() >= len => match PesStart::parse(payload) { - Ok(pes) => self.handle_pes_start(pid, pes), + Ok(pes) => self.handle_pes_start(pid, pes, at), Err(err) => self.damage(pid, &err), }, - Ok(_) => self.pes_header(pid, payload.to_vec()), + Ok(_) => self.pes_header(pid, at, payload.to_vec()), Err(err) => self.damage(pid, &err), }; } - if let Some(mut header) = self.headers.remove(&pid) { + if let Some((at, mut header)) = self.headers.remove(&pid) { header.extend_from_slice(payload); - self.pes_header(pid, header) + self.pes_header(pid, at, header) } else { self.handle_pes_continuation(pid, payload) } } - fn pes_header(&mut self, pid: Pid, header: Vec) -> anyhow::Result<()> { + /// Continue a PES header split across packets, whose start code sits at input offset `at`. + fn pes_header(&mut self, pid: Pid, at: u64, header: Vec) -> anyhow::Result<()> { let len = match PesStart::header_len(&header) { Ok(len) => len, Err(err) => return self.damage(pid, &err), }; if header.len() < len { - self.headers.insert(pid, header); + self.headers.insert(pid, (at, header)); return Ok(()); } match PesStart::parse(&header) { - Ok(pes) => self.handle_pes_start(pid, pes), + Ok(pes) => self.handle_pes_start(pid, pes, at), Err(err) => self.damage(pid, &err), } } @@ -623,24 +661,35 @@ impl Import { } } Some(StreamType::Mpeg1Video | StreamType::Mpeg2Video) => Stream::Clock, - // A codec we don't decode, named or not. Carry it verbatim as PES when the + // A codec we don't decode, named or not. Carry it as whole PES packets when the // catalog supports the `mpegts` section. 0x86 is excluded: it's ambiguous (DTS // audio, or a non-conformant SCTE-35 mux without CUEI, which is sections a PES // parse would abort on), so drop it rather than risk PES-parsing sections. _ => { if self.supports_mpegts && stream_type != StreamType::Dts8ChannelLosslessAudio as u8 { - let descriptors = descriptors.to_vec(); - match VerbatimStream::new( - self.broadcast.clone(), - self.catalog.clone(), - pid.as_u16(), - stream_type, - descriptors, - ) { - Ok(stream) => Stream::Verbatim(Box::new(stream)), - Err(err) => { - tracing::warn!(?err, pid = pid.as_u16(), "failed to create verbatim stream, dropping"); - Stream::Ignored + if self.pcr_pid.is_none() { + // Nothing could place its PES, so no track is advertised. + tracing::warn!( + stream_type, + pid = pid.as_u16(), + "refused a private PES stream in a program with no PCR to place it by" + ); + *self.refused.entry(pid.as_u16()).or_default() += 1; + Stream::Ignored + } else { + let descriptors = descriptors.to_vec(); + match PesPacketStream::new( + self.broadcast.clone(), + self.catalog.clone(), + pid.as_u16(), + stream_type, + descriptors, + ) { + Ok(stream) => Stream::PesPacket(Box::new(stream)), + Err(err) => { + tracing::warn!(?err, pid = pid.as_u16(), "failed to create a PES stream, dropping"); + Stream::Ignored + } } } } else { @@ -733,7 +782,8 @@ impl Import { Some(stream) } - fn handle_pes_start(&mut self, pid: Pid, pes: PesStart) -> anyhow::Result<()> { + /// Open the PES whose header `pes` begins at input offset `at`. + fn handle_pes_start(&mut self, pid: Pid, pes: PesStart, at: u64) -> anyhow::Result<()> { // Finish the preceding PES before accepting another header. if self.pending.contains_key(&pid) { self.flush(pid)?; @@ -744,11 +794,25 @@ impl Import { return Ok(()); }; + let whole = match stream { + Stream::PesPacket(stream) => Some(stream.stream_type), + _ => None, + }; + if let Some(stream_type) = whole + && let Some(why) = refusal(&pes, stream_type) + { + return self.refuse(pid, why); + } + if pes.scrambled() { + return self.damage(pid, &anyhow::anyhow!("scrambled PES is not supported")); + } + let stream = &self.streams[&pid]; + let is_video = matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock); let is_clock = matches!(stream, Stream::Clock); // Verbatim streams with no cadence (SCTE-35, DVB subtitles) are left ungraded // rather than watched against an invented one. - if pes.pts.is_some() && !matches!(stream, Stream::Verbatim(_) | Stream::Ignored) { + if pes.pts.is_some() && !matches!(stream, Stream::PesPacket(_) | Stream::Ignored) { self.health.pts(pid.as_u16(), self.liveness.now()); } // The first PTS anchors the program, and each takes its era here: against the clock as the @@ -783,6 +847,8 @@ impl Import { } if pts.is_some() { self.release_sections(offset)?; + // A first PTS may have anchored the timebase whole PES were waiting on. + self.release_placed()?; } if is_clock { @@ -796,20 +862,44 @@ impl Import { return Ok(()); } + // A whole PES keeps its header, and is placed by the PCR before it and the one after. + let (header, data_len, span) = match whole { + Some(_) => { + let Some(before) = self.placement.last() else { + // Before the first PCR on this timebase nothing places it. A PTS it carried still + // anchored the timebase, so it releases the reservation like a flushed PES would. + if pts.is_some() { + self.initial_reservation = None; + } + *self.unplaced.entry(pid.as_u16()).or_default() += 1; + return Ok(()); + }; + let span = Span { + at, + before, + generation: self.placement.generation, + ticks: None, + }; + let len = pes.data_len.map(|len| pes.header.len() + len); + (pes.header, len, Some(span)) + } + None => (&[][..], pes.data_len, None), + }; let data = &pes.data[..pes.data_len.unwrap_or(pes.data.len()).min(pes.data.len())]; let mut retained = Retained::new(self.staging.clone()); - retained.grow(data.len(), 0)?; + retained.grow(header.len() + data.len(), 0)?; let mut pending = Pending { retained, pts, offset, dts: pes.dts, - stream_id: pes.stream_id, - data: Vec::with_capacity(data.len()), - data_len: pes.data_len, + data: Vec::with_capacity(header.len() + data.len()), + data_len, + span, }; + pending.data.extend_from_slice(header); pending.data.extend_from_slice(data); - let complete = matches!(pes.data_len, Some(len) if pending.data.len() >= len); + let complete = matches!(data_len, Some(len) if pending.data.len() >= len); self.pending.insert(pid, pending); if complete { @@ -900,12 +990,85 @@ impl Import { } self.published |= units > 0; self.liveness.delivered(pid.as_u16(), units); + if matches!(self.streams.get(&pid), Some(Stream::PesPacket(_))) { + self.release_placed()?; + } // Record the decoded media track's PID + PMT descriptors (language, ...) once // its lazily created track exists, so export can preserve them. self.record_media_track(pid) } + /// A PCR at input offset `offset`, its packet's, on the program's clock PID: it places every + /// whole PES that began since the last, and releases those complete. + fn place_pcr(&mut self, raw: u64, offset: u64) -> anyhow::Result<()> { + if self.placement.rewinds(raw) { + // The clock stepped back without saying so: what it placed so far is placed by the + // old timebase, and the new one starts here. + self.settle_placement()?; + } + let sample = self.placement.pcr(raw, offset + PCR_OFFSET); + let generation = self.placement.generation; + for pending in self.pending.values_mut() { + if let Some(span) = pending.span.as_mut() { + span.bracket(generation, sample); + } + } + for stream in self.streams.values_mut() { + if let Stream::PesPacket(stream) = stream { + stream.bracket(generation, sample); + } + } + self.release_placed() + } + + /// End the placement timebase: extrapolate every whole PES begun past its last PCR from its + /// last two, and publish what that places. + fn settle_placement(&mut self) -> anyhow::Result<()> { + for pending in self.pending.values_mut() { + if let Some(span) = pending.span.as_mut() { + span.settle(&self.placement); + } + } + for stream in self.streams.values_mut() { + if let Stream::PesPacket(stream) = stream { + stream.settle(&self.placement); + } + } + self.placement.reset(); + self.release_placed() + } + + /// Publish every held whole PES that is placed, once the timebase has its offset. + fn release_placed(&mut self) -> anyhow::Result<()> { + for (pid, stream) in &mut self.streams { + let Stream::PesPacket(stream) = stream else { + continue; + }; + let (units, unplaced) = stream.release(&self.timebase, self.placement.generation)?; + if unplaced > 0 { + *self.unplaced.entry(pid.as_u16()).or_default() += unplaced; + } + self.published |= units > 0; + self.liveness.delivered(pid.as_u16(), units); + } + Ok(()) + } + + /// Refuse the private PES stream on `pid` rather than carry it altered: its track ends with an + /// error and leaves the catalog, and the PID is dropped from here on. + fn refuse(&mut self, pid: Pid, why: &str) -> anyhow::Result<()> { + self.pending.remove(&pid); + self.headers.remove(&pid); + if let Some(stream) = self.retire_stream(pid) { + stream.abort(moq_net::Error::Unsupported); + } + self.streams.insert(pid, Stream::Ignored); + *self.refused.entry(pid.as_u16()).or_default() += 1; + tracing::warn!(pid = pid.as_u16(), why, "refused a private PES stream"); + Ok(()) + } + /// The packet chain on `pid` was cut: salvage the truncated PES, and treat the bytes /// after it as [lost](Stream::lost). fn broken(&mut self, pid: Pid) -> anyhow::Result<()> { @@ -958,6 +1121,12 @@ impl Import { for pid in self.streams.keys().copied().collect::>() { self.salvage(pid)?; } + // Whole PES the old clock began go out on its timeline, ahead of the marker. + self.settle_placement()?; + anyhow::ensure!( + self.streams.values().all(Stream::placed), + "TS private PES reached a clock break without a media timestamp" + ); for stream in self.streams.values_mut() { stream.desync(); stream.discontinuity(self.published)?; @@ -1143,6 +1312,11 @@ impl Import { self.sections.values().all(|section| section.held.is_empty()), "TS sections reached end of input without a media timestamp" ); + self.settle_placement()?; + anyhow::ensure!( + self.streams.values().all(Stream::placed), + "TS private PES reached end of input without a media timestamp" + ); // EOF releases an unmeasured short source as well as its decoder reservations. self.mux_rate_reservation = None; self.initial_reservation = None; @@ -1186,6 +1360,19 @@ impl Import { .or_insert_with(|| stats::Stream::new("", stats::Class::Data)) .damaged = damaged; } + // A stream refused before it had a track has no row of its own either. + for (&pid, &unplaced) in &self.unplaced { + streams + .entry(pid) + .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)) + .unplaced = unplaced; + } + for (&pid, &refused) in &self.refused { + streams + .entry(pid) + .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)) + .refused = refused; + } for (pid, stats) in &mut streams { (stats.units, stats.quiet) = self.liveness.stream(*pid); } @@ -1318,11 +1505,12 @@ struct Pending { /// Raw 90 kHz DTS, still wrapped. Present on reordered (B-frame) video; its distance below /// the PTS, modulo the wrap, is the reorder delay published as the catalog jitter. dts: Option, - /// PES stream_id, preserved for verbatim PES carriage. - stream_id: u8, + /// The payload, or the whole packet, header included, for a [`PesPacketStream`]. data: Vec, - /// Expected payload length for bounded PES, else `None` (unbounded video). + /// Expected length of `data` for bounded PES, else `None` (unbounded video). data_len: Option, + /// Where a whole PES sits on the program clock; `None` for every other stream. + span: Option, } impl Pending { @@ -1333,23 +1521,19 @@ impl Pending { pts: None, offset: Offset::default(), dts: None, - stream_id: 0, data: Vec::new(), data_len: None, + span: None, } } } -/// Create a verbatim track and record it in the `mpegts` catalog section as a -/// [`Track`](catalog::Track) with a `verbatim` carriage record. Shared by the -/// section- and PES-framed paths. +/// Create a track for an undecoded stream and record `entry`, its carriage record, under its +/// name in the `mpegts` catalog section. Shared by the section and whole-PES paths. fn register_verbatim( broadcast: &mut moq_net::broadcast::Producer, catalog: &mut crate::catalog::Producer, - pid: u16, - stream_type: u8, - framing: catalog::Framing, - descriptors: Vec, + entry: catalog::Track, ) -> anyhow::Result> { // Verbatim payloads ride the legacy container, which normalizes the per-frame // timestamp to microseconds on the wire (see `hang::container::Frame::encode`), @@ -1377,15 +1561,7 @@ fn register_verbatim( // catalog dropped the section since. anyhow::bail!("catalog extension no longer carries an mpegts section"); }; - mpegts.tracks.insert( - name, - catalog::Track { - pid, - descriptors, - verbatim: Some(catalog::Verbatim::new(stream_type, framing)), - pes_packet: None, - }, - ); + mpegts.tracks.insert(name, entry); drop(guard); Ok(media) @@ -1439,14 +1615,13 @@ impl SectionStream { stream_type: u8, descriptors: Vec, ) -> anyhow::Result { - let track = register_verbatim( - &mut broadcast, - &mut catalog, + let entry = catalog::Track { pid, - stream_type, - catalog::Framing::Section, descriptors, - )?; + verbatim: Some(catalog::Verbatim::new(stream_type, catalog::Framing::Section)), + pes_packet: None, + }; + let track = register_verbatim(&mut broadcast, &mut catalog, entry)?; let entry = VerbatimEntry { name: track.name().to_string(), catalog, @@ -1562,20 +1737,33 @@ fn adjust_splice(section: &mut [u8], offset: Offset) { section[body..].copy_from_slice(&crc.to_be_bytes()); } -/// Publishes whole reassembled PES payloads verbatim as frames on a track -/// described in the `mpegts` catalog section, for elementary streams we don't decode -/// (DTS audio, private PES, teletext, ...). +/// Publishes each PES of an elementary stream we don't decode (private data, teletext, DVB +/// subtitles, DVB AC-3, DTS, ...) whole, as one frame on a track described by a +/// [`PesPacket`](catalog::PesPacket) record in the `mpegts` catalog section. /// -/// Unlike [`SectionStream`], these ride the normal PES reassembly path, so this -/// type only stamps each PES payload with its PTS and writes it. -struct VerbatimStream { +/// A frame is the packet from its start code through its payload, its PTS and DTS moved onto the +/// catalog clock and every other byte as received, timestamped with its placement on the program +/// clock. A PES without a PTS keeps none. See [`Import`] for when one is held, dropped, or refuses +/// its stream. +struct PesPacketStream { track: crate::container::Producer, - entry: VerbatimEntry, - /// Whether the PES stream_id has been recorded into the catalog yet (once). - stream_id_recorded: bool, + /// Held for its `Drop`, which clears this track's catalog entry. + _entry: VerbatimEntry, + /// The PMT `stream_type`, which says whether an unbounded PES is video. + stream_type: u8, + /// Whole PES waiting for the PCR after them or the timebase's offset, in input order. + held: VecDeque, +} + +/// A whole PES waiting to be placed. +struct Held { + pes: Vec, + span: Span, + /// Held for its `Drop`, which returns the bytes to the staging budget. + _retained: Retained, } -impl VerbatimStream { +impl PesPacketStream { fn new( mut broadcast: moq_net::broadcast::Producer, mut catalog: crate::catalog::Producer, @@ -1583,54 +1771,82 @@ impl VerbatimStream { stream_type: u8, descriptors: Vec, ) -> anyhow::Result { - let track = register_verbatim( - &mut broadcast, - &mut catalog, + let entry = catalog::Track { pid, - stream_type, - catalog::Framing::Pes, descriptors, - )?; + verbatim: None, + pes_packet: Some(catalog::PesPacket::new(stream_type)), + }; + let track = register_verbatim(&mut broadcast, &mut catalog, entry)?; let entry = VerbatimEntry { name: track.name().to_string(), catalog, }; Ok(Self { track, - entry, - stream_id_recorded: false, + _entry: entry, + stream_type, + held: VecDeque::new(), }) } - /// Publish one reassembled PES payload verbatim, in its own group, stamped with - /// its PTS (or the live edge when the PES carried none). - fn write(&mut self, pending: Pending) -> anyhow::Result { - // Record the original PES stream_id once, from the first PES, so export - // re-emits the stream under its real id (e.g. 0xBD for teletext/DVB AC-3). - if !self.stream_id_recorded { - let name = self.track.name().to_string(); - if let Some(mpegts) = self.entry.catalog.modify()?.ext.mpegts_mut() - && let Some(verbatim) = mpegts.tracks.get_mut(&name).and_then(|t| t.verbatim.as_mut()) - { - verbatim.stream_id = Some(pending.stream_id); - } - self.stream_id_recorded = true; + /// Hold one complete PES until it can be placed: [`Self::release`] publishes it. + fn hold(&mut self, pending: Pending) { + let span = pending.span.expect("a whole PES is spanned at its start"); + self.held.push_back(Held { + pes: pending.data, + span, + _retained: pending.retained, + }); + } + + /// The PCR `after` arrived on timebase `generation`: every PES held since the last is + /// bracketed. + fn bracket(&mut self, generation: u64, after: Sample) { + for held in &mut self.held { + held.span.bracket(generation, after); } + } - let pts = match on_clock(pending.pts, pending.offset)? { - Some(pts) => pts, - // No clock of its own, so land on the edge. - None => self.track.live_edge().unwrap_or(Timestamp::ZERO), - }; - let frame = crate::container::Frame { - timestamp: pts, - duration: None, - payload: bytes::Bytes::from(pending.data), - keyframe: true, + /// The timebase of `placement` ends: extrapolate the PES held past its last PCR. + fn settle(&mut self, placement: &Placement) { + for held in &mut self.held { + held.span.settle(placement); + } + } + + /// Publish the placed PES at the front, each in its own group, once `timebase` has its + /// offset, and drop those no PCR on their own timebase can place: one ended before them, now + /// that `generation` is the timebase running. Returns how many went each way. + fn release(&mut self, timebase: &crate::catalog::Timebase, generation: u64) -> anyhow::Result<(u64, u64)> { + let Some(offset) = timebase.offset() else { + return Ok((0, 0)); }; - self.track.write(frame)?; - self.track.cut(None)?; - Ok(1) + let (mut published, mut unplaced) = (0, 0); + while let Some(held) = self.held.front() { + let timestamp = match held.span.ticks { + Some(ticks) => placement(timebase, offset, ticks), + // Its timebase ended without placing it. + None if held.span.generation != generation => None, + None => break, + }; + let mut held = self.held.pop_front().expect("a front"); + let Some(timestamp) = timestamp else { + unplaced += 1; + continue; + }; + rebase(&mut held.pes, offset); + let frame = crate::container::Frame { + timestamp, + duration: None, + payload: bytes::Bytes::from(held.pes), + keyframe: true, + }; + self.track.write(frame)?; + self.track.cut(None)?; + published += 1; + } + Ok((published, unplaced)) } fn discontinuity(&mut self, published: bool) -> anyhow::Result<()> { @@ -1655,6 +1871,143 @@ impl VerbatimStream { } } +/// Why a whole-PES track cannot carry `pes` unaltered, if it cannot: a clock field only the PTS +/// and DTS of are rebased, a scrambled payload, or an unbounded PES that is not video, which +/// export refuses. +fn refusal(pes: &PesStart, stream_type: u8) -> Option<&'static str> { + if let Some(clock) = pes.unrebased_clock() { + return Some(clock); + } + if pes.scrambled() { + return Some("PES-level scrambling"); + } + let video = + matches!(pes.stream_id, 0xe0..=0xef) || (pes.stream_id == 0xfd && psi::is_video_stream_type(stream_type)); + (pes.data_len.is_none() && !video).then_some("an unbounded PES on a stream that is not video") +} + +/// The catalog time of the program clock reading `ticks`, an unwrapped 27 MHz count, on a +/// timebase shifted by `offset`: its PCR in the era nearest the clock, floored to microseconds. +/// `None` for a reading before the broadcast clock's zero. +fn placement( + timebase: &crate::catalog::Timebase, + offset: Offset, + ticks: u64, +) -> Option { + let pcr = timebase.unwrap(ticks % PCR.period, PCR)?; + let micros = pcr.div_euclid(i128::from(PCR.hz / 1_000_000)) + offset.ticks(1_000_000); + Timestamp::from_micros(u64::try_from(micros).ok()?).ok() +} + +/// Move a whole PES's PTS and DTS onto the catalog clock in place, modulo 2^33 as a media PTS +/// moves, leaving their prefix and marker bits and every other byte as they were. The PES CRC +/// covers the previous packet's data bytes, not this header, so it stays valid. +fn rebase(pes: &mut [u8], offset: Offset) { + const FIELD: i128 = 1 << 33; + if offset == Offset::default() || !psi::has_optional_header(pes[3]) { + return; + } + let fields = match pes[7] >> 6 { + 0b10 => 1, + 0b11 => 2, + _ => 0, + }; + for field in pes[9..9 + 5 * fields].as_chunks_mut::<5>().0 { + let value = (u64::from(field[0] >> 1) & 0x07) << 30 + | u64::from(field[1]) << 22 + | u64::from(field[2] >> 1) << 15 + | u64::from(field[3]) << 7 + | u64::from(field[4] >> 1); + let value = (i128::from(value) + offset.ticks(90_000)).rem_euclid(FIELD) as u64; + field[0] = (field[0] & 0xf1) | ((value >> 29) & 0x0e) as u8; + field[1] = (value >> 22) as u8; + field[2] = (field[2] & 0x01) | ((value >> 14) & 0xfe) as u8; + field[3] = (value >> 7) as u8; + field[4] = (field[4] & 0x01) | ((value << 1) & 0xfe) as u8; + } +} + +/// The program clock at each input byte, from the PCRs of the current timebase, which places +/// whole PES: a byte is interpolated at the rate between the PCRs either side of it, as +/// [`Passthrough`](super::Passthrough) times its objects. +#[derive(Default)] +struct Placement { + /// The last two PCRs of the timebase, newest last, unwrapped from its first. + samples: VecDeque, + /// The last PCR, raw. + raw: u64, + /// Counts the timebases ended, so a PES never takes a PCR from one it did not start in. + generation: u64, +} + +impl Placement { + /// A PCR of `raw` 27 MHz ticks whose reference byte sits at input offset `at`. + fn pcr(&mut self, raw: u64, at: u64) -> Sample { + let ticks = match self.samples.back() { + Some(last) => last.ticks + (raw + PCR.period - self.raw) % PCR.period, + None => raw, + }; + let sample = Sample { at, ticks }; + self.samples.push_back(sample); + if self.samples.len() > 2 { + self.samples.pop_front(); + } + self.raw = raw; + sample + } + + /// Whether `raw` steps back from the last PCR, modulo the wrap. + fn rewinds(&self, raw: u64) -> bool { + !self.samples.is_empty() && (raw + PCR.period - self.raw) % PCR.period >= PCR.period / 2 + } + + /// The timebase's last PCR. + fn last(&self) -> Option { + self.samples.back().copied() + } + + /// The program clock at input offset `at`, past the last PCR, at the rate of the interval + /// before it; `None` without one. + fn extrapolate(&self, at: u64) -> Option { + (self.samples.len() == 2).then(|| locate(at, &self.samples)) + } + + /// End the timebase. + fn reset(&mut self) { + self.samples.clear(); + self.generation += 1; + } +} + +/// Where a whole PES starts in the input, and the program clock there once known. +#[derive(Clone, Copy)] +struct Span { + /// The input offset of its start code. + at: u64, + /// The last PCR before it. + before: Sample, + /// The [`Placement::generation`] it started in. + generation: u64, + /// The program clock at its first byte, in its timebase's unwrapped 27 MHz ticks. + ticks: Option, +} + +impl Span { + /// Interpolate with `after`, the first PCR past it on its timebase. + fn bracket(&mut self, generation: u64, after: Sample) { + if self.ticks.is_none() && self.generation == generation { + self.ticks = Some(locate(self.at, &VecDeque::from([self.before, after]))); + } + } + + /// Its timebase ended with no PCR after it: extrapolate from the last two. + fn settle(&mut self, placement: &Placement) { + if self.ticks.is_none() && self.generation == placement.generation { + self.ticks = placement.extrapolate(self.at); + } + } +} + /// Whether a packet's adaptation field sets `discontinuity_indicator`. /// /// What that declares depends on the PID: a continuity-counter break on an elementary @@ -2061,8 +2414,8 @@ enum Stream { Aac(Box>), Opus(Box), Legacy(Box>), - /// A codec we don't decode, carried verbatim as PES (DTS audio, private PES, ...). - Verbatim(Box>), + /// A codec we don't decode, carried as whole PES (DTS audio, private PES, ...). + PesPacket(Box>), /// MPEG-1/2 video we don't decode, kept only to advance the media clock. /// `is_video` counts it, so never reuse this variant for audio or data. Clock, @@ -2129,11 +2482,23 @@ impl Stream { Stream::Aac(stream) => stream.write(pending, batched), Stream::Opus(stream) => stream.write(pending), Stream::Legacy(stream) => stream.write(pending), - Stream::Verbatim(stream) => stream.write(pending), + // Published once placed: [`Import::release_placed`]. + Stream::PesPacket(stream) => { + stream.hold(pending); + Ok(0) + } Stream::Clock | Stream::Ignored => Ok(0), } } + /// Whether nothing waits to be placed on the program clock. + fn placed(&self) -> bool { + match self { + Stream::PesPacket(stream) => stream.held.is_empty(), + _ => true, + } + } + /// Whether a PES cut short by a break is still worth publishing. /// /// True where one PES carries many independently decodable units, so the ones ahead of @@ -2148,7 +2513,7 @@ impl Stream { // its control header promises is a parse error, and that error would travel up out // of `decode` and end the session. Losing the PES beats losing the broadcast. Stream::Opus(_) => false, - Stream::H264 { .. } | Stream::H265 { .. } | Stream::Verbatim(_) => false, + Stream::H264 { .. } | Stream::H265 { .. } | Stream::PesPacket(_) => false, Stream::Clock | Stream::Ignored => false, } } @@ -2162,7 +2527,7 @@ impl Stream { Stream::H265 { split, .. } => split.reset(), Stream::Aac(stream) => stream.desync(), Stream::Legacy(stream) => stream.desync(), - Stream::Opus(_) | Stream::Verbatim(_) | Stream::Clock | Stream::Ignored => {} + Stream::Opus(_) | Stream::PesPacket(_) | Stream::Clock | Stream::Ignored => {} } } @@ -2198,7 +2563,7 @@ impl Stream { Stream::Aac(stream) => stream.discontinuity(published), Stream::Opus(stream) => stream.discontinuity(published), Stream::Legacy(stream) => stream.discontinuity(published), - Stream::Verbatim(stream) => stream.discontinuity(published), + Stream::PesPacket(stream) => stream.discontinuity(published), Stream::Clock | Stream::Ignored => Ok(()), } } @@ -2216,7 +2581,7 @@ impl Stream { Stream::Aac(stream) => stream.seek(sequence), Stream::Opus(stream) => stream.seek(sequence), Stream::Legacy(stream) => stream.seek(sequence), - Stream::Verbatim(stream) => stream.seek(sequence), + Stream::PesPacket(stream) => stream.seek(sequence), Stream::Clock | Stream::Ignored => Ok(()), } } @@ -2230,7 +2595,7 @@ impl Stream { Stream::H264 { import, .. } => import.finish()?, Stream::H265 { import, .. } => import.finish()?, Stream::Opus(stream) => stream.finish()?, - Stream::Verbatim(stream) => stream.finish()?, + Stream::PesPacket(stream) => stream.finish()?, Stream::Clock | Stream::Ignored => {} } Ok(0) @@ -2243,7 +2608,7 @@ impl Stream { Stream::Aac(stream) => stream.abort(err), Stream::Opus(stream) => stream.abort(err), Stream::Legacy(stream) => stream.abort(err), - Stream::Verbatim(stream) => stream.abort(err), + Stream::PesPacket(stream) => stream.abort(err), Stream::Clock | Stream::Ignored => {} } } @@ -2257,7 +2622,7 @@ impl Stream { Stream::H264 { .. } => stats::Stream::new(".avc3", stats::Class::Video), Stream::H265 { .. } => stats::Stream::new(".hev1", stats::Class::Video), Stream::Opus(_) => stats::Stream::new(".opus", stats::Class::Audio), - Stream::Verbatim(_) => stats::Stream::new(".ts", stats::Class::Data), + Stream::PesPacket(_) => stats::Stream::new(".ts", stats::Class::Data), Stream::Clock => stats::Stream::new("", stats::Class::Video), Stream::Ignored => return None, }) @@ -2272,7 +2637,7 @@ impl Stream { Stream::Aac(stream) => stream.import.as_ref().map(|i| i.name().to_string()), Stream::Opus(stream) => Some(stream.import.name().to_string()), Stream::Legacy(stream) => stream.import.as_ref().map(|i| i.name().to_string()), - Stream::Verbatim(_) | Stream::Clock | Stream::Ignored => None, + Stream::PesPacket(_) | Stream::Clock | Stream::Ignored => None, } } } @@ -6140,73 +6505,9 @@ pub(super) mod test { assert_eq!(run(&[p1, p2, p3, p4]), vec![CUE.to_vec()]); } - // A PES-framed elementary stream we don't decode (private data, stream_type 0x06) - // is carried verbatim: cataloged in the `mpegts` section with its PID and framing, and - // its PES payload published byte-for-byte. - #[tokio::test(start_paused = true)] - async fn private_pes_carried_verbatim() { - use crate::catalog::hang::{Catalog, Container}; - use crate::container::Consumer; - use crate::container::ts::catalog::{Ext, Framing}; - - const VIDEO_PID: u16 = 0x0050; - const DATA_PID: u16 = 0x0052; - - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let catalog = crate::catalog::Producer::new( - &mut broadcast, - crate::catalog::Config::default().with_catalog(Catalog::::default()), - ) - .unwrap(); - let mut import = super::Import::new(broadcast, catalog.reserve()); - - let mut bytes = bytes::BytesMut::new(); - bytes.extend_from_slice(&synth_pmt( - &[ - (StreamType::Mpeg2Video, VIDEO_PID), - (StreamType::Mpeg2PacketizedData, DATA_PID), - ], - false, - )); - bytes.extend_from_slice(&pes_packet(VIDEO_PID, 90_000)); // video sets the media clock - let payload = [0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02]; - bytes.extend_from_slice(&audio_pes_packet(DATA_PID, 0, 90_000, &payload)); - import.decode(&bytes).unwrap(); - import.finish().unwrap(); - - let snap = catalog.snapshot(); - assert_eq!( - snap.ext.mpegts.tracks.len(), - 1, - "the private PES PID is carried verbatim" - ); - let (name, track) = snap.ext.mpegts.tracks.iter().next().unwrap(); - let verbatim = track.verbatim.as_ref().expect("a verbatim carriage record"); - assert_eq!(verbatim.stream_type, 0x06, "recorded the PMT stream_type"); - assert_eq!(verbatim.framing, Framing::Pes, "private PES is PES-framed"); - // `audio_pes_packet` uses stream_id 0xC0; it must be captured for faithful re-emit. - assert_eq!(verbatim.stream_id, Some(0xC0), "recorded the PES stream_id"); - assert_eq!(track.pid, DATA_PID, "recorded the original PID"); - - let track = consumer - .track(name.as_str()) - .unwrap() - .subscribe(moq_net::track::Subscription::default().with_max_delay(RECORDING_MAX_AGE)) - .await - .unwrap(); - let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) - .await - .expect("verbatim read timed out") - .unwrap() - .expect("a published verbatim frame"); - assert_eq!(&frame.payload[..], &payload[..], "verbatim PES payload round-trips"); - } - - /// A verbatim-only program retains the PES clock while waiting for rate measurement or EOF. + /// A private-data-only program retains the PES clock while waiting for rate measurement or EOF. #[tokio::test] - async fn verbatim_only_program_retains_its_pes_clock_until_catalog_release() { + async fn private_pes_only_program_retains_its_pes_clock_until_catalog_release() { use crate::catalog::hang::Catalog; use crate::container::ts::catalog::Ext; @@ -6903,7 +7204,9 @@ pub(super) mod test { let mut cc = 0; for part in [&header[..split], &header[split..], &data] { for chunk in part.chunks(183) { - import.pes_packet(pid, &pes_fragment(PID, cc == 0, cc, chunk)).unwrap(); + import + .pes_packet(pid, &pes_fragment(PID, cc == 0, cc, chunk), 0) + .unwrap(); cc += 1; } } @@ -6931,14 +7234,17 @@ pub(super) mod test { let header = [0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; for end in ["loss", "start", "eof"] { let mut import = ignored_pes_import(); - import.pes_packet(pid, &pes_fragment(0x50, true, 0, &header)).unwrap(); - assert_eq!(import.headers[&pid].len(), 9); + import + .pes_packet(pid, &pes_fragment(0x50, true, 0, &header), 0) + .unwrap(); + assert_eq!(import.headers[&pid].1.len(), 9); match end { "loss" => import.broken(pid).unwrap(), "start" => import .pes_packet( pid, &pes_fragment(0x50, true, 1, &[0, 0, 1, 0xe0, 0, 0, 0x80, 0, 0, 0xaa]), + 0, ) .unwrap(), _ => import.finish().unwrap(), @@ -6948,7 +7254,7 @@ pub(super) mod test { assert_eq!(import.pending[&pid].data, [0xaa]); } else { import - .pes_packet(pid, &pes_fragment(0x50, false, 1, &[0xff; 5])) + .pes_packet(pid, &pes_fragment(0x50, false, 1, &[0xff; 5]), 0) .unwrap(); assert!( !import.pending.contains_key(&pid), @@ -7134,12 +7440,11 @@ pub(super) mod test { let mut import = ignored_pes_import(); let header = [0, 0, 1, id, 0, 3]; import - .pes_packet(pid, &pes_fragment(0x50, true, 0, &header[..split])) + .pes_packet(pid, &pes_fragment(0x50, true, 0, &header[..split]), 0) .unwrap(); import - .pes_packet(pid, &pes_fragment(0x50, false, 1, &header[split..])) + .pes_packet(pid, &pes_fragment(0x50, false, 1, &header[split..]), 0) .unwrap(); - assert_eq!(import.pending[&pid].stream_id, id); assert_eq!(import.pending[&pid].data_len, Some(3)); assert_eq!(import.pending[&pid].pts, None); } @@ -7155,7 +7460,7 @@ pub(super) mod test { let pid = Pid::new(0x50).unwrap(); for bad in [&[1][..], &[0, 1][..], &[0, 0, 2][..]] { let mut import = ignored_pes_import(); - import.pes_packet(pid, &pes_fragment(0x50, true, 0, bad)).unwrap(); + import.pes_packet(pid, &pes_fragment(0x50, true, 0, bad), 0).unwrap(); assert!(import.headers.is_empty()); assert!(import.pending.is_empty()); } @@ -7169,6 +7474,7 @@ pub(super) mod test { .pes_packet( pid, &pes_fragment(0x50, true, 0, &[0, 0, 1, 0xe0, 0, 4, 0x80, 0x80, 255]), + 0, ) .unwrap(); assert!(import.headers.is_empty()); @@ -7190,7 +7496,7 @@ pub(super) mod test { pes.extend_from_slice(&data); for (cc, chunk) in pes.chunks(183).enumerate() { import - .pes_packet(pid, &pes_fragment(0x50, cc == 0, cc as u8, chunk)) + .pes_packet(pid, &pes_fragment(0x50, cc == 0, cc as u8, chunk), 0) .unwrap(); } if end == "eof" { @@ -7198,7 +7504,9 @@ pub(super) mod test { } else { let mut header = vec![0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; header.extend_from_slice(&pes_timestamp(2, 180_000)); - import.pes_packet(pid, &pes_fragment(0x50, true, 1, &header)).unwrap(); + import + .pes_packet(pid, &pes_fragment(0x50, true, 1, &header), 0) + .unwrap(); } assert_eq!( import.stats().streams[&0x50].units, @@ -7219,6 +7527,7 @@ pub(super) mod test { .pes_packet( pid, &pes_fragment(pid.as_u16(), true, 0, &[0, 0, 1, 0xe0, 0, 0, 0x80, 0, 0]), + 0, ) .unwrap(); import @@ -7317,13 +7626,13 @@ pub(super) mod test { let pid = Pid::new(raw).unwrap(); if fragmented { import - .pes_packet(pid, &pes_fragment(raw, true, 0, &header[..1])) + .pes_packet(pid, &pes_fragment(raw, true, 0, &header[..1]), 0) .unwrap(); import - .pes_packet(pid, &pes_fragment(raw, false, 1, &header[1..])) + .pes_packet(pid, &pes_fragment(raw, false, 1, &header[1..]), 0) .unwrap(); } else { - import.pes_packet(pid, &pes_fragment(raw, true, 0, &header)).unwrap(); + import.pes_packet(pid, &pes_fragment(raw, true, 0, &header), 0).unwrap(); } for chunk in data.chunks(184) { import @@ -8274,22 +8583,6 @@ pub(super) mod test { import_refused(&mux.out); } - /// A PES-framed stream we carry verbatim refuses a step back like any other. - #[tokio::test(start_paused = true)] - async fn verbatim_pes_below_the_edge_is_refused() { - const DATA_PID: u16 = 0x0052; - let mut mux = Mux { - out: synth_pmt(&[(StreamType::Mpeg2PacketizedData, DATA_PID)], false), - ..Default::default() - }; - for pts in [45 * 90_000, 46 * 90_000, 90_000] { - let cc = mux.cc(DATA_PID); - mux.out - .extend_from_slice(&audio_pes_packet(DATA_PID, cc, pts, &[0xDE, 0xAD])); - } - import_refused(&mux.out); - } - /// A cue takes its time from the video it arrives with, and a B-frame decoded after a /// P-frame presents earlier than it. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index ebcfd9ae97b..e7e1fe9d533 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -65,4 +65,6 @@ mod oracle; #[cfg(test)] mod passthrough_test; #[cfg(test)] +mod pes_packet_test; +#[cfg(test)] mod release_test; diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index 0743e08e1cc..557e75529b0 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -68,7 +68,7 @@ use super::psi::{self, Pmt}; const PCR_HZ: u64 = 27_000_000; /// The PCR field: a 33-bit base of 300 ticks each. -const PCR: crate::catalog::Wrap = crate::catalog::Wrap { +pub(super) const PCR: crate::catalog::Wrap = crate::catalog::Wrap { hz: PCR_HZ, period: PCR_WRAP, }; @@ -87,7 +87,7 @@ pub const GROUP_OBJECTS_MAX: usize = moq_net::group::MAX_GROUP_FRAMES / 2; /// The bytes a PCR sits into its packet: four of header, the adaptation field's length and flags, /// and the five bytes the 33-bit base ends in. -const PCR_OFFSET: u64 = 10; +pub(super) const PCR_OFFSET: u64 = 10; /// The packet sizes a sync byte can recur at: 188, BDAV's 192, and 204 with Reed-Solomon parity. const PACKET_SIZES: [usize; 3] = [TsPacket::SIZE, 192, 204]; @@ -262,12 +262,12 @@ struct Timebase { samples: VecDeque, } -#[derive(Clone, Copy)] -struct Sample { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Sample { /// Where the PCR's byte sits in the input. - at: u64, + pub(super) at: u64, /// The unwrapped PCR, in 27 MHz ticks. - ticks: u64, + pub(super) ticks: u64, } struct Object { @@ -795,7 +795,7 @@ fn random_access_indicator(pkt: &[u8; TsPacket::SIZE]) -> bool { /// Interpolated between the two PCRs either side of it; extrapolated at the rate of the nearest /// interval when it lies before the timebase's first PCR or after its last. With a single PCR /// there is no rate, and the byte takes that PCR's time. -fn locate(at: u64, samples: &VecDeque) -> u64 { +pub(super) fn locate(at: u64, samples: &VecDeque) -> u64 { let (first, last) = (samples[0], samples[samples.len() - 1]); let (a, b) = match samples.iter().zip(samples.iter().skip(1)).find(|(_, b)| at < b.at) { Some((a, b)) => (*a, *b), diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs new file mode 100644 index 00000000000..c8fee75b23f --- /dev/null +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -0,0 +1,636 @@ +//! Import of undecoded PES streams as `pesPacket` tracks: each frame a whole PES, every byte as +//! received but the PTS and DTS moved onto the catalog clock, timestamped with the program clock +//! at its first byte. + +use std::time::Duration; + +use super::catalog::Ext; +use super::fixture::{Flags, MS, Mux, WRAP}; + +const PMT_PID: u16 = 0x20; +const VIDEO_PID: u16 = 0x100; +const DATA_PID: u16 = 0x104; +const CLOCK_PID: u16 = 0x1ff; +/// The PMT `PCR_PID` of a program with no PCR. +const NO_PCR: u16 = 0x1fff; + +/// 2^33, where the PTS wraps. +const PTS_WRAP: u64 = 1 << 33; +/// The PCR leads each picture's PTS by this many 90 kHz ticks. +const LEAD: u64 = 9_000; + +/// `registration_descriptor` naming SMPTE RP 217 KLV ('KLVA'). +const KLVA: &[u8] = &[0x05, 0x04, b'K', b'L', b'V', b'A']; +/// `metadata_pointer`-free `metadata_descriptor` for synchronous KLV (ISO 13818-1 2.6.60). +const METADATA: &[u8] = &[0x26, 0x09, 0x01, 0x00, 0xff, b'K', b'L', b'V', b'A', 0x0f, 0x00]; +/// DVB `AC-3_descriptor` with no optional fields. +const AC3: &[u8] = &[0x6a, 0x01, 0x00]; + +/// A transport stream under construction: one program clocked by PCRs on [`CLOCK_PID`], an +/// MPEG-2 video stream read only for its clock on [`VIDEO_PID`], and the private stream on +/// [`DATA_PID`]. It remembers where each PCR and PES went, to work out the expected placement. +struct Feed { + mux: Mux, + /// Each PCR's reference byte and its unwrapped 27 MHz ticks. + pcrs: Vec<(u64, u64)>, + /// Each private PES's start code offset and its bytes. + pes: Vec<(u64, Vec)>, +} + +impl Feed { + fn new(stream_type: u8, descriptors: &[u8]) -> Self { + Self::with_pcr(stream_type, descriptors, CLOCK_PID) + } + + fn with_pcr(stream_type: u8, descriptors: &[u8], pcr_pid: u16) -> Self { + let mut mux = Mux::default(); + mux.pat(&[(1, PMT_PID)]).pmt_with( + PMT_PID, + 1, + pcr_pid, + &[(0x02, VIDEO_PID, &[]), (stream_type, DATA_PID, descriptors)], + 0, + ); + Self { + mux, + pcrs: Vec::new(), + pes: Vec::new(), + } + } + + /// A PCR of `ticks`, unwrapped: the packet carries them modulo the wrap. + fn pcr(&mut self, ticks: u64) -> &mut Self { + self.pcrs.push((self.mux.out.len() as u64 + 10, ticks)); + self.mux.packet(CLOCK_PID, Flags::pcr(ticks % WRAP)); + self + } + + /// A picture presenting `LEAD` after the program clock `ticks`, unwrapped. + fn picture(&mut self, ticks: u64) -> &mut Self { + self.mux + .pes(VIDEO_PID, Flags::default(), (ticks / 300 + LEAD) % PTS_WRAP); + self.mux.fill(VIDEO_PID, 2); + self + } + + fn data(&mut self, pes: Vec) -> &mut Self { + let at = self.mux.whole(DATA_PID, &pes); + self.pes.push((at, pes)); + self + } + + /// A PES whose packets straddle a PCR of `ticks`: its first `head` packets come before it. + fn data_across(&mut self, pes: Vec, head: usize, ticks: u64) -> &mut Self { + let (start, packets) = self.mux.split(DATA_PID, &pes); + self.pes.push(((self.mux.out.len() + start) as u64, pes)); + self.mux.out.extend(packets[..head].concat()); + self.pcr(ticks); + self.mux.out.extend(packets[head..].concat()); + self + } + + /// The program clock at input offset `at` in unwrapped 27 MHz ticks: interpolated between + /// the PCRs either side of it, extrapolated from the last two past the last. `None` before + /// the first PCR or past a lone one. + fn clock(&self, at: u64) -> Option { + let before = self.pcrs.iter().rposition(|&(pcr, _)| pcr <= at)?; + let (a, b) = match self.pcrs.get(before + 1) { + Some(&after) => (self.pcrs[before], after), + None if before > 0 => (self.pcrs[before - 1], self.pcrs[before]), + None => return None, + }; + Some(a.1 + (at - a.0) * (b.1 - a.1) / (b.0 - a.0)) + } + + fn bytes(&self) -> Vec { + self.mux.bytes() + } +} + +/// A PES with an optional header: `marker` is its first flags byte, `flags` its second, `fields` +/// the optional fields those flags name, then `stuffing` bytes of 0xFF. +fn pes_with(stream_id: u8, marker: u8, flags: u8, fields: &[u8], stuffing: usize, payload: &[u8]) -> Vec { + let header = fields.len() + stuffing; + let mut pes = vec![0, 0, 1, stream_id]; + pes.extend_from_slice(&((3 + header + payload.len()) as u16).to_be_bytes()); + pes.extend_from_slice(&[marker, flags, header as u8]); + pes.extend_from_slice(fields); + pes.resize(pes.len() + stuffing, 0xff); + pes.extend_from_slice(payload); + pes +} + +/// A PES of a `stream_id` with no optional header. +fn pes_bare(stream_id: u8, payload: &[u8]) -> Vec { + let mut pes = vec![0, 0, 1, stream_id]; + pes.extend_from_slice(&(payload.len() as u16).to_be_bytes()); + pes.extend_from_slice(payload); + pes +} + +/// A 33-bit PTS or DTS field behind its 4-bit `prefix`. +fn clock_field(prefix: u8, ticks: u64) -> [u8; 5] { + [ + prefix << 4 | ((ticks >> 29) & 0x0e) as u8 | 1, + (ticks >> 22) as u8, + ((ticks >> 14) & 0xfe) as u8 | 1, + (ticks >> 7) as u8, + ((ticks << 1) & 0xfe) as u8 | 1, + ] +} + +/// The value of a PTS or DTS field. +fn clock_value(field: &[u8]) -> u64 { + (u64::from(field[0] >> 1) & 0x07) << 30 + | u64::from(field[1]) << 22 + | u64::from(field[2] >> 1) << 15 + | u64::from(field[3]) << 7 + | u64::from(field[4] >> 1) +} + +/// An asynchronous KLV PES (private_stream_1, no PTS) of `len` payload bytes, filled with `fill`. +fn klv(fill: u8, len: usize) -> Vec { + // A SMPTE universal label, a BER length, then the value. + let mut payload = vec![ + 0x06, 0x0e, 0x2b, 0x34, 0x02, 0x0b, 0x01, 0x01, 0x0e, 0x01, 0x03, 0x01, 0x01, 0x00, 0x00, 0x00, + ]; + payload.extend_from_slice(&[0x82, (len >> 8) as u8, len as u8]); + payload.resize(payload.len() + len, fill); + pes_with(0xbd, 0x84, 0x00, &[], 0, &payload) +} + +/// A synchronous KLV PES (metadata stream, stream_id 0xFC) presenting at `pts`. +fn sync_klv(fill: u8, pts: u64) -> Vec { + // The metadata AU cell header (ISO 13818-1 2.12.4), then a KLV packet. + let mut payload = vec![0x00, 0x00, 0xdf, 0x00, 0x14]; + payload.extend_from_slice(&[0x06, 0x0e, 0x2b, 0x34, 0x01, 0x01, 0x01, 0x01, 0x0e, 0x01, 0x03, 0x01]); + payload.resize(payload.len() + 8, fill); + pes_with(0xfc, 0x84, 0x80, &clock_field(0x2, pts), 0, &payload) +} + +/// An imported feed: everything it published, and how it went. +struct Imported { + consumer: moq_net::broadcast::Consumer, + catalog: crate::catalog::Producer, + import: super::Import, + /// The catalog clock's shift of the source's timestamps, in microseconds. + offset: i128, +} + +/// Import `data`, onto a catalog clock already fixed when `captured` (so the timestamps shift), +/// or one the first PTS places (so they stay verbatim). +fn import(data: &[u8], captured: bool) -> anyhow::Result { + let mut import = start(captured); + import.import.decode(data)?; + import.import.finish()?; + Ok(import) +} + +fn start(captured: bool) -> Imported { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + // Paused, the clock reads the same until the first PTS anchors on it. + let offset = match captured { + true => catalog.clock().now().as_micros() as i128, + false => 0, + }; + let import = super::Import::new(broadcast, catalog.reserve()); + Imported { + consumer, + catalog, + import, + offset, + } +} + +impl Imported { + /// The catalog's `pesPacket` track on [`DATA_PID`]: its name and entry. + fn entry(&self) -> Option<(String, super::catalog::Track)> { + self.catalog + .snapshot() + .ext + .mpegts + .tracks + .into_iter() + .find(|(_, track)| track.pid == DATA_PID && track.pes_packet.is_some()) + } + + /// The [`DATA_PID`] row of the importer's stats. + fn stats(&self) -> super::stats::Stream { + self.import.stats().streams.get(&DATA_PID).cloned().unwrap_or_default() + } +} + +/// Every frame on track `name`, and the error that ended it if one did. +async fn read(consumer: &moq_net::broadcast::Consumer, name: &str) -> (Vec, Option) { + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); + let track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); + let mut reader = crate::container::Consumer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + let mut frames = Vec::new(); + loop { + match tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + Ok(Ok(Some(frame))) => frames.push(frame), + Ok(Ok(None)) | Err(_) => return (frames, None), + Ok(Err(err)) => return (frames, Some(err.to_string())), + } + } +} + +/// The microsecond timestamp the program clock `ticks` places a PES at, shifted by `offset`. +fn placed(ticks: u64, offset: i128) -> u64 { + u64::try_from(i128::from(ticks / 27) + offset).unwrap() +} + +/// Asynchronous KLV, with no PTS, takes the program clock at its first byte: the reproduction of +/// the PES that were stamped zero and then dropped by the export. It does so before the first +/// picture and between pictures, in a PES spanning packets and one straddling a PCR, and past the +/// last PCR by extrapolating from the last two. One before the first PCR has nothing to place it. +#[tokio::test(start_paused = true)] +async fn async_klv_takes_the_program_clock_at_its_first_byte() { + let mut feed = Feed::new(0x06, KLVA); + feed.data(klv(0, 8)); + let start = 2_000 * MS; + feed.pcr(start).data(klv(1, 8)); + for i in 0..10 { + let ticks = start + i * 40 * MS; + feed.picture(ticks); + match i { + 3 => feed.data(klv(2, 600)), + 5 => feed.data_across(klv(3, 400), 1, ticks + 20 * MS), + _ => feed.data(klv(i as u8 + 0x10, 32)), + }; + feed.mux.fill(VIDEO_PID, 3); + feed.pcr(ticks + 40 * MS); + } + feed.picture(start + 400 * MS).data(klv(4, 8)); + + let imported = import(&feed.bytes(), false).unwrap(); + let (name, entry) = imported.entry().expect("the KLV stream is in the catalog"); + assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, 0x06); + assert_eq!(entry.descriptors.len(), 1); + assert_eq!(entry.descriptors[0].tag, 0x05); + assert_eq!(&entry.descriptors[0].data[..], b"KLVA"); + + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + let expected: Vec<_> = feed.pes[1..] + .iter() + .map(|(at, pes)| (placed(feed.clock(*at).unwrap(), 0), pes.clone())) + .collect(); + let got: Vec<_> = frames + .iter() + .map(|frame| (frame.timestamp.as_micros() as u64, frame.payload.to_vec())) + .collect(); + assert_eq!(got.len(), expected.len()); + for (k, (got, expected)) in got.iter().zip(&expected).enumerate() { + assert_eq!(got.0, expected.0, "PES {k} timestamp"); + assert_eq!(got.1, expected.1, "PES {k} bytes"); + assert!(got.0 >= 2_000_000, "PES {k} is on the program clock"); + } + assert!(frames.iter().all(|frame| frame.keyframe)); + let stats = imported.stats(); + assert_eq!(stats.unplaced, 1, "the PES before the first PCR"); + assert_eq!(stats.refused, 0); + assert_eq!(stats.units, expected.len() as u64); +} + +/// Synchronous KLV starting just before the 33-bit wrap, onto a catalog clock already running: +/// each PTS and DTS moves by the timebase's shift modulo 2^33, keeping its prefix and markers, +/// PTS zero stays a PTS while no PTS stays none, and every other byte is untouched. +#[tokio::test(start_paused = true)] +async fn sync_klv_rebases_its_pts_and_dts_onto_the_catalog_clock() { + let mut feed = Feed::new(0x15, METADATA); + // The first picture presents one second before the PTS wraps. + let start = (PTS_WRAP - 90_000 - LEAD) * 300; + feed.pcr(start); + let mut wrapped = 0; + for i in 0..60 { + let ticks = start + i * 40 * MS; + feed.picture(ticks); + let pts = (ticks / 300 + LEAD) % PTS_WRAP; + match i { + // Its decode time a frame before its presentation. + 10 => { + let mut fields = clock_field(0x3, pts).to_vec(); + fields.extend_from_slice(&clock_field(0x1, pts - 3_600)); + feed.data(pes_with(0xfc, 0x84, 0xc0, &fields, 0, b"distinct")); + } + // No PTS at all. + 20 => { + feed.data(pes_with(0xfc, 0x84, 0x00, &[], 0, b"untimed")); + } + _ if pts < 3_600 => { + // The picture after the wrap: a PTS of zero. + wrapped += 1; + feed.data(sync_klv(0xee, 0)); + } + _ => { + feed.data(sync_klv(i as u8, pts)); + } + } + feed.pcr(ticks + 40 * MS); + } + assert_eq!(wrapped, 1, "one picture lands on PTS zero"); + + let imported = import(&feed.bytes(), true).unwrap(); + let (name, entry) = imported.entry().expect("the KLV stream is in the catalog"); + assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, 0x15); + // The first PTS anchored the timebase at the captured reading. + let first = (start / 300 + LEAD) * 1_000_000 / 90_000; + let offset = imported.offset - first as i128; + let shift = (offset * 9).div_euclid(100); + assert!(shift < -(1 << 32), "the shift carries PTS past the wrap: {shift}"); + + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), feed.pes.len()); + for (k, (frame, (at, pes))) in frames.iter().zip(&feed.pes).enumerate() { + assert_eq!( + frame.timestamp.as_micros() as u64, + placed(feed.clock(*at).unwrap(), offset), + "PES {k} timestamp" + ); + let got = &frame.payload[..]; + assert_eq!(got.len(), pes.len(), "PES {k} length"); + let fields = match pes[7] >> 6 { + 0b10 => 1, + 0b11 => 2, + _ => 0, + }; + assert_eq!(got[..9], pes[..9], "PES {k} header"); + for field in 0..fields { + let range = 9 + 5 * field..14 + 5 * field; + let raw = clock_value(&pes[range.clone()]); + let rebased = (i128::from(raw) + shift).rem_euclid(1 << 33) as u64; + assert_eq!( + got[range.clone()], + clock_field(pes[range.start] >> 4, rebased), + "PES {k} clock field {field}" + ); + } + assert_eq!(got[9 + 5 * fields..], pes[9 + 5 * fields..], "PES {k} after its clocks"); + } + // A PTS of zero is still a PTS, and none is still none. + let zero = feed.pes.iter().position(|(_, pes)| pes.ends_with(&[0xee; 8])).unwrap(); + assert_eq!(frames[zero].payload[7] >> 6, 0b10); + assert_eq!( + clock_value(&frames[zero].payload[9..14]), + shift.rem_euclid(1 << 33) as u64 + ); + let untimed = feed.pes.iter().position(|(_, pes)| pes.ends_with(b"untimed")).unwrap(); + assert_eq!(frames[untimed].payload[7] >> 6, 0b00); +} + +/// Every optional field a PES can carry without a clock of its own goes through as received, +/// as do `data_alignment_indicator` clear, stuffing, and the stream IDs with no optional header. +/// A PES CRC covers the previous PES's data bytes, so it still checks. +#[tokio::test(start_paused = true)] +async fn optional_fields_and_bare_stream_ids_go_through_untouched() { + let mut feed = Feed::new(0x06, &[]); + let start = 1_000 * MS; + feed.pcr(start); + + let previous = b"the data bytes the next PES's CRC covers".to_vec(); + let mut fields = clock_field(0x3, 99_000).to_vec(); + fields.extend_from_slice(&clock_field(0x1, 95_400)); + // ES_rate, DSM trick mode, additional_copy_info, previous_PES_packet_CRC. + fields.extend_from_slice(&[0x80, 0x10, 0x01]); + fields.push(0x1f); + fields.push(0x80 | 0x2a); + fields.extend_from_slice(&crc16(&previous).to_be_bytes()); + // The extension: private data, a sequence counter, P-STD, and extension 2 naming a + // stream_id_extension. + fields.push(0x80 | 0x20 | 0x10 | 0x0e | 0x01); + fields.extend_from_slice(&[0x5a; 16]); + fields.extend_from_slice(&[0x80 | 0x05, 0x80 | 0x02]); + fields.extend_from_slice(&[0x60, 0x20]); + fields.extend_from_slice(&[0x80 | 0x03, 0x71, 0xaa, 0xbb]); + let crc = pes_with(0xbd, 0x80, 0xc0 | 0x10 | 0x08 | 0x04 | 0x02 | 0x01, &fields, 3, b"crc"); + + let pes = [ + pes_with(0xbd, 0x80, 0x00, &[], 0, &previous), + crc, + pes_bare(0xbf, b"private_stream_2"), + pes_bare(0xbe, &[0xff; 20]), + pes_bare(0xf0, b"ECM"), + pes_bare(0xf1, b"EMM"), + pes_bare(0xf2, b"DSM-CC"), + pes_bare(0xf8, b"H.222.1 type E"), + pes_bare(0xff, b"directory"), + pes_with(0xbd, 0x80, 0x00, &[], 7, b"stuffed"), + ]; + for (i, pes) in pes.into_iter().enumerate() { + let ticks = start + i as u64 * 40 * MS; + feed.picture(ticks).data(pes).pcr(ticks + 40 * MS); + } + + let imported = import(&feed.bytes(), true).unwrap(); + let (name, _) = imported.entry().unwrap(); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), feed.pes.len()); + let first = (start / 300 + LEAD) * 1_000_000 / 90_000; + let shift = ((imported.offset - first as i128) * 9).div_euclid(100); + assert_ne!(shift, 0, "the timestamps shift"); + for (k, (frame, (_, pes))) in frames.iter().zip(&feed.pes).enumerate() { + match k { + // Only its PTS and DTS move. + 1 => { + let mut expected = pes.clone(); + for field in [9..14, 14..19] { + let raw = clock_value(&pes[field.clone()]); + let rebased = (i128::from(raw) + shift).rem_euclid(1 << 33) as u64; + expected[field.clone()].copy_from_slice(&clock_field(pes[field.start] >> 4, rebased)); + } + assert_eq!(&frame.payload[..], &expected[..], "PES {k}"); + } + _ => assert_eq!(&frame.payload[..], &pes[..], "PES {k}"), + } + } + let carried = &frames[1].payload[..]; + let at = 9 + 10 + 3 + 1 + 1; + assert_eq!( + u16::from_be_bytes([carried[at], carried[at + 1]]), + crc16(&frames[0].payload[9..]), + "the CRC still covers the previous PES's data bytes" + ); +} + +/// The CRC-16 a `previous_PES_packet_CRC` carries: polynomial 0x1021, registers preset to ones. +fn crc16(data: &[u8]) -> u16 { + let mut crc = 0xffffu16; + for &byte in data { + crc ^= u16::from(byte) << 8; + for _ in 0..8 { + crc = if crc & 0x8000 != 0 { crc << 1 ^ 0x1021 } else { crc << 1 }; + } + } + crc +} + +/// DVB AC-3 (private data with an AC-3 descriptor) is not decoded, so a PES of several sync +/// frames stays one frame, which the export writes back as one PES. +#[tokio::test(start_paused = true)] +async fn dvb_ac3_keeps_its_sync_frames_in_one_pes() { + let mut sync = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; + sync.resize(128, 0x00); + let mut feed = Feed::new(0x06, AC3); + let start = 1_000 * MS; + feed.pcr(start); + for i in 0..4 { + let ticks = start + i * 96 * MS; + let pts = ticks / 300 + LEAD; + let payload: Vec = (0..3).flat_map(|_| sync.clone()).collect(); + feed.picture(ticks) + .data(pes_with(0xbd, 0x84, 0x80, &clock_field(0x2, pts), 0, &payload)) + .pcr(ticks + 96 * MS); + } + + let imported = import(&feed.bytes(), false).unwrap(); + let (name, entry) = imported.entry().unwrap(); + assert_eq!(entry.descriptors[0].tag, 0x6a); + let (frames, _) = read(&imported.consumer, &name).await; + let got: Vec<_> = frames.iter().map(|frame| frame.payload.to_vec()).collect(); + let expected: Vec<_> = feed.pes.iter().map(|(_, pes)| pes.clone()).collect(); + assert_eq!(got, expected); +} + +/// A PES the track cannot carry unaltered refuses its stream: what was already placed stays +/// published, the track ends with an error and leaves the catalog, and the refusal is counted. +#[tokio::test(start_paused = true)] +async fn a_pes_with_a_field_no_rebase_reaches_refuses_its_stream() { + let cases: [(&str, Vec); 5] = [ + ( + "ESCR", + pes_with(0xbd, 0x80, 0x20, &[0x04, 0x00, 0x04, 0x00, 0x04, 0x01], 0, b"escr"), + ), + ( + "pack_header_field", + pes_with(0xbd, 0x80, 0x01, &[0x40, 0x01, 0xba], 0, b"pack"), + ), + ( + "TREF", + pes_with( + 0xbd, + 0x80, + 0x01, + &[0x01, 0x80 | 0x06, 0xfe, 0x21, 0x00, 0x01, 0x00, 0x01], + 0, + b"tref", + ), + ), + ("scrambling", pes_with(0xbd, 0x90, 0x00, &[], 0, b"scrambled")), + ("unbounded", { + let mut pes = pes_with(0xbd, 0x80, 0x00, &[], 0, b"unbounded"); + pes[4..6].fill(0); + pes + }), + ]; + for (why, bad) in cases { + let mut feed = Feed::new(0x06, &[]); + let start = 1_000 * MS; + feed.pcr(start).picture(start).data(klv(1, 8)).pcr(start + 40 * MS); + let head = feed.bytes(); + feed.picture(start + 40 * MS) + .data(bad) + .pcr(start + 80 * MS) + .picture(start + 80 * MS) + .data(klv(2, 8)) + .pcr(start + 120 * MS); + + let mut imported = start_import(&head); + let (name, _) = imported.entry().expect("the stream starts out carried"); + let reader = { + let consumer = imported.consumer.clone(); + let name = name.clone(); + tokio::spawn(async move { read(&consumer, &name).await }) + }; + for _ in 0..10 { + tokio::task::yield_now().await; + } + imported.import.decode(&feed.bytes()[head.len()..]).unwrap(); + imported.import.finish().unwrap(); + + let (frames, err) = reader.await.unwrap(); + assert_eq!(frames.len(), 1, "{why}: the PES placed before it stays published"); + assert!(err.is_some(), "{why}: the track ends with an error"); + assert!(imported.entry().is_none(), "{why}: the track left the catalog"); + assert_eq!(imported.stats().refused, 1, "{why}"); + } +} + +/// Begin importing `head` onto a clock the first PTS places. +fn start_import(head: &[u8]) -> Imported { + let mut imported = start(false); + imported.import.decode(head).unwrap(); + imported +} + +/// A program with no PCR has nothing to place a private PES by, so the stream is refused before +/// any track is advertised. +#[tokio::test(start_paused = true)] +async fn a_program_without_a_pcr_refuses_private_pes() { + let mut feed = Feed::with_pcr(0x06, KLVA, NO_PCR); + for i in 0..5 { + feed.picture(1_000 * MS + i * 40 * MS).data(klv(i as u8, 8)); + } + let imported = import(&feed.bytes(), false).unwrap(); + assert!(imported.entry().is_none(), "no track is advertised"); + assert_eq!(imported.stats().refused, 1); +} + +/// A PES after a lone PCR has no interval to extrapolate by, so it is dropped and counted. +#[tokio::test(start_paused = true)] +async fn a_pes_past_a_lone_pcr_is_unplaced() { + let mut feed = Feed::new(0x06, KLVA); + feed.pcr(1_000 * MS).picture(1_000 * MS).data(klv(0, 8)); + let imported = import(&feed.bytes(), false).unwrap(); + let (name, _) = imported.entry().unwrap(); + let (frames, _) = read(&imported.consumer, &name).await; + assert!(frames.is_empty()); + assert_eq!(imported.stats().unplaced, 1); +} + +/// A program of data alone never names a media time for the program clock to shift by, so its +/// PES are refused at the end of the input rather than published on a guessed clock. +#[test] +fn a_data_only_program_is_refused() { + let mut feed = Feed::new(0x06, KLVA); + for i in 0..10 { + feed.pcr(1_000 * MS + i * 40 * MS).data(klv(i as u8, 8)); + } + let err = import(&feed.bytes(), false).err().expect("the import is refused"); + assert!(err.to_string().contains("without a media timestamp"), "{err:#}"); +} + +/// A flagged clock break ends the placement timebase: the PES past its last PCR is extrapolated +/// on the old one, and those after the break are placed by the new one's PCRs. +#[tokio::test(start_paused = true)] +async fn a_clock_break_places_each_side_on_its_own_timebase() { + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + feed.pcr(start).picture(start).pcr(start + 40 * MS).data(klv(1, 8)); + let before = feed.clock(feed.pes[0].0).unwrap(); + let jump = 5_000 * MS; + feed.pcrs.clear(); + feed.mux.packet(CLOCK_PID, Flags::pcr(jump).discontinuity()); + feed.pcrs.push((feed.mux.out.len() as u64 - 188 + 10, jump)); + feed.picture(jump).data(klv(2, 8)).pcr(jump + 40 * MS); + + let imported = import(&feed.bytes(), false).unwrap(); + let (name, _) = imported.entry().unwrap(); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), 2); + assert_eq!(frames[0].timestamp.as_micros() as u64, placed(before, 0)); + assert!(frames[1].timestamp > frames[0].timestamp); + assert_eq!(&frames[1].payload[..], &feed.pes[1].1[..]); +} diff --git a/rs/moq-mux/src/container/ts/psi.rs b/rs/moq-mux/src/container/ts/psi.rs index 499e02e9565..f2b144c599c 100644 --- a/rs/moq-mux/src/container/ts/psi.rs +++ b/rs/moq-mux/src/container/ts/psi.rs @@ -272,6 +272,8 @@ fn descriptors(mut data: &[u8]) -> Option> { /// A complete PES header and the payload bytes that follow it. pub(super) struct PesStart<'a> { + /// The packet's header, from its start code through the optional fields and stuffing. + pub header: &'a [u8], pub stream_id: u8, /// Raw 90 kHz. pub pts: Option, @@ -294,14 +296,13 @@ impl<'a> PesStart<'a> { if payload.len() < 6 { return Ok(6); } - if matches!(payload[3], 0xbc | 0xbe | 0xbf | 0xf0 | 0xf1 | 0xf2 | 0xf8 | 0xff) { + if !has_optional_header(payload[3]) { return Ok(6); } if payload.len() < 9 { return Ok(9); } anyhow::ensure!(payload[6] & 0xc0 == 0x80, "unexpected PES header marker bits"); - anyhow::ensure!(payload[6] & 0x30 == 0, "scrambled PES is not supported"); anyhow::ensure!(payload[7] & 0xc0 != 0x40, "PES DTS without a PTS"); let len = usize::from(payload[8]); let packet_len = usize::from(u16::from_be_bytes([payload[4], payload[5]])); @@ -316,7 +317,8 @@ impl<'a> PesStart<'a> { /// /// Optional fields this demux has no use for (ESCR, ES rate, trick mode, copy info, the /// PES CRC, extensions, stuffing) are skipped by `PES_header_data_length`. Incomplete headers - /// must be assembled before parsing; malformed headers are refused. + /// must be assembled before parsing; malformed headers are refused. A scrambled one parses, + /// for its stream to refuse: [`Self::scrambled`]. pub fn parse(payload: &'a [u8]) -> anyhow::Result { let header_len = Self::header_len(payload)?; anyhow::ensure!(payload.len() >= header_len, "PES header truncated"); @@ -326,6 +328,7 @@ impl<'a> PesStart<'a> { // Streams that carry no optional header (ISO/IEC 13818-1 table 2-21). if header_len == 6 { return Ok(Self { + header: &payload[..6], stream_id, pts: None, dts: None, @@ -354,6 +357,7 @@ impl<'a> PesStart<'a> { ), }; Ok(Self { + header: &payload[..9 + header_len], stream_id, pts, dts, @@ -363,6 +367,68 @@ impl<'a> PesStart<'a> { } } +impl PesStart<'_> { + /// Whether `PES_scrambling_control` says the payload is scrambled. + pub fn scrambled(&self) -> bool { + self.header.get(6).is_some_and(|flags| flags & 0x30 != 0) + } + + /// The clock field in the optional header that a whole-PES track cannot carry, since only + /// the PTS and DTS are moved onto the catalog clock: ESCR, a `pack_header_field`, or a TREF + /// (ISO 13818-1 2.4.3.7). `None` for a header that carries none of them. + pub fn unrebased_clock(&self) -> Option<&'static str> { + let header = self.header; + let flags = *header.get(7)?; + if flags & 0x20 != 0 { + return Some("ESCR"); + } + if flags & 0x01 == 0 { + return None; + } + // Skip the PTS, DTS, ESCR, ES rate, trick mode, copy info and CRC to the extension. + let skip = match flags >> 6 { + 0b10 => 5, + 0b11 => 10, + _ => 0, + } + usize::from(flags & 0x10 != 0) * 3 + + usize::from(flags & 0x08 != 0) + + usize::from(flags & 0x04 != 0) + + usize::from(flags & 0x02 != 0) * 2; + let mut at = 9 + skip; + let extension = *header.get(at)?; + at += 1; + if extension & 0x40 != 0 { + return Some("pack_header_field"); + } + at += usize::from(extension & 0x80 != 0) * 16 + + usize::from(extension & 0x20 != 0) * 2 + + usize::from(extension & 0x10 != 0) * 2; + if extension & 0x01 == 0 { + return None; + } + // PES_extension_field_length, then stream_id_extension_flag; with it set, a clear + // tref_extension_flag says a TREF follows. + let field = *header.get(at + 1)?; + (field & 0x80 != 0 && field & 0x01 == 0).then_some("TREF") + } +} + +/// Whether a PES of `stream_id` carries the optional header: all but program_stream_map, +/// padding, private_stream_2, the ECM and EMM streams, DSMCC, H.222.1 type E and the program +/// stream directory (ISO 13818-1 table 2-21). +pub(super) fn has_optional_header(stream_id: u8) -> bool { + !matches!(stream_id, 0xbc | 0xbe | 0xbf | 0xf0 | 0xf1 | 0xf2 | 0xf8 | 0xff) +} + +/// Whether a PMT `stream_type` carries video (ISO 13818-1 Table 2-34, and VC-1's 0xEA), whose +/// PES alone may leave `PES_packet_length` unbounded. +pub(super) fn is_video_stream_type(stream_type: u8) -> bool { + matches!( + stream_type, + 0x01 | 0x02 | 0x10 | 0x1b | 0x1e..=0x26 | 0x28..=0x2b | 0x31..=0x35 | 0xea + ) +} + /// A 33-bit PTS or DTS behind its 4-bit prefix and three marker bits. fn timestamp(bytes: Option<&[u8]>, prefix: u8) -> anyhow::Result { let bytes = bytes.context("PES header truncated")?; diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index 92f26c2e166..84e14ce6eca 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -86,7 +86,7 @@ pub struct Stream { pub track: String, /// How this PID was classified. See [`Class`]. pub class: Class, - /// Access units the stream delivered: frames published for decoded media, PES payloads or + /// Access units the stream delivered: frames published for decoded media, whole PES or /// sections carried verbatim, and PES read on MPEG-1/2 video. At export, the PES or /// sections written for each frame. pub units: u64, @@ -110,6 +110,15 @@ pub struct Stream { pub unconfirmed: u64, /// Damaged packets, PES, or access units refused whole on this PID. Export stays zero. pub damaged: u64, + /// Whole PES dropped from a private PES track because the program clock could not place + /// them: before the first PCR, after the last with no interval to extrapolate from, or + /// before the broadcast clock's zero. Export stays zero. + pub unplaced: u64, + /// Times this PID's stream was refused rather than carried altered: a private PES stream + /// with a clock field only the PTS and DTS of are rebased (ESCR, a `pack_header_field`, a + /// TREF), PES-level scrambling, an unbounded non-video PES, or a program with no PCR to place + /// it by. Its track ends and leaves the catalog. Export stays zero. + pub refused: u64, /// This PID's share of [`Snapshot::continuity_count_error`]. pub continuity_count_error: u64, /// This PID's share of [`Snapshot::transport_error`]. @@ -145,7 +154,7 @@ impl Stream { /// only; a sparse data PID stays counted. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum Class { - /// Undecoded or sparse data (SCTE-35, ID3, private sections, verbatim PES), or a PCR PID + /// Undecoded or sparse data (SCTE-35, ID3, private sections, private PES), or a PCR PID /// that carries no elementary stream. A quiet second is not a stall. #[default] Data, @@ -299,6 +308,16 @@ impl Log { "damaged TS units refused" ); } + let dropped = |stream: &Stream| (stream.unplaced, stream.refused); + if previous.map_or((0, 0), dropped) != dropped(stream) { + tracing::info!( + pid = *pid, + track = stream.track, + unplaced = stream.unplaced, + refused = stream.refused, + "private PES not carried" + ); + } if previous.map_or((0, 0, 0), lost) == lost(stream) { continue; } diff --git a/test/justfile b/test/justfile index 4c972c5e43c..433ed4938ef 100644 --- a/test/justfile +++ b/test/justfile @@ -137,6 +137,7 @@ ts *args: python3 ./ts/open-gop.test.py python3 ./ts/holes.test.py python3 ./ts/passthrough.test.py + python3 ./ts/klv.test.py ./ts/run.sh "$@" # Carry a hardware encoder's capture, which the T-STD model passes, whole through the round-trip. diff --git a/test/ts/README.md b/test/ts/README.md index 66d927979d7..9ec4d538c59 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -43,6 +43,7 @@ just test ts --headroom # the same with four audio PIDs, muxed above the just test ts --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --fragment-pes # legal headers split across packets, with FFmpeg/TSDuck grading +just test ts --klv # KLV and DVB AC-3 carried as whole PES, graded against the source just test ts --delay 2s # pass the exporter's --delay just test ts --hrd --loss 1 # drop 1% of the relay's datagrams to the subscriber just test ts --passthrough # carry the multiplex whole, grade it packet by packet @@ -55,6 +56,17 @@ null slots after splitting the header. It cycles through header-byte boundaries, preserves packet count, PCR positions, payload bytes, and PID continuity, then runs the same independent decode/content and transport checks as the default arm. +`--klv` muxes DVB AC-3 beside the AAC, several sync frames a PES, and `klv.py` +fills null slots with asynchronous KLV (`stream_type` 0x06, `KLVA` registration, no +PTS) and synchronous KLV (`stream_type` 0x15, `metadata_stream` 0xFC, a PTS per +picture). The importer decodes none of them, so each PES rides its track whole. +`klv.py grade` holds each KLV stream in the capture to its source: the same PID, +`stream_type` and descriptors; every PES from the join to the end, in order and +byte-equal outside its PTS and DTS, which move by one shift; none out before its +source placement on the program clock, and none after its PTS; and FFmpeg reading +each as KLV with the same PES and PTS. `holes.py` and the T-STD model grade the +AC-3, whose buffer drains each sync frame at its own decode time. + `--live` swaps the analyzer, not the rig: the same round-trip runs, but the subscriber's stdout goes straight into `pcr-timing.py` instead of a capture file. That is the only arm that can see release timing at all, and it is what nightly @@ -802,7 +814,7 @@ outcomes deterministically in Rust CI, without retries or implicit `--stitch`. [`.github/workflows/ts.yml`](../../.github/workflows/ts.yml) is independent of browser interop. Relevant mux, Hang, CLI, transport, harness and toolchain PRs run `just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, full, headroom, -open-GOP, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. +open-GOP, KLV (strict), passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, diff --git a/test/ts/klv.py b/test/ts/klv.py new file mode 100644 index 00000000000..368a1567e84 --- /dev/null +++ b/test/ts/klv.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Inject KLV into a generated clip, and grade a round-trip's capture of it against the source. + + ./klv.py inject source.ts out.ts [--report klv.json] + ./klv.py grade source.ts capture.ts [--report-json klv.json] + +`inject` adds two private PES streams in the clip's null slots, leaving every other packet in +place: asynchronous KLV (stream_type 0x06 with a `KLVA` registration, private_stream_1, no PTS) +every 100 ms of program clock, and synchronous KLV (stream_type 0x15 with a metadata +descriptor, metadata_stream 0xFC) on each picture, presenting with it. Every PES is unique, so +a capture's can be found in the source. + +`grade` holds the capture to what the importer and exporter promise for a PES they do not +decode. Each stream keeps its PID, stream_type and descriptors. Its PES arrive in source order +with none missing, from within the join (holes.JOIN) of the source's first to the source's last, +each byte-equal to its source PES outside the PTS and DTS, which all move by one shift modulo +2^33. Each goes out no earlier than its source placement (the program clock at its first byte) +moved by that shift, to a packet; a PES with a PTS arrives by it. FFmpeg must read each stream as +SMPTE KLV, with the same PES and PTS. compliance.py grades the rest of the capture with TSDuck. + +Exit status is 0 when every check passes, 1 otherwise. +""" + +import argparse +import json +import subprocess +import sys +from dataclasses import dataclass, field +from fractions import Fraction +from pathlib import Path + +import compliance +import holes + +SIZE = 188 +NULL = 0x1FFF +ASYNC_PID = 0x110 +SYNC_PID = 0x111 +# PMT entries: (stream_type, PID, ES descriptors). +KLVA = bytes([0x05, 0x04]) + b"KLVA" +# metadata_descriptor (ISO 13818-1 2.6.60): application format 0xFFFF named by its identifier, +# format 0xFF named by 'KLVA', service 0, no decoder config. +METADATA = bytes([0x26, 0x0D, 0xFF, 0xFF]) + b"KLVA" + bytes([0xFF]) + b"KLVA" + bytes([0x00, 0x0F]) +STREAMS = [(0x06, ASYNC_PID, KLVA), (0x15, SYNC_PID, METADATA)] +# Program clock between asynchronous KLV, in 27 MHz ticks. +ASYNC_PERIOD = 27_000_000 // 10 +# Every tenth asynchronous KLV spans several packets. +LONG = 10 +PTS_WRAP = 1 << 33 +PCR_WRAP = PTS_WRAP * 300 +# A PCR's reference byte: the last of its base, in its packet. +PCR_OFFSET = 10 +# Universal key of a MISB ST 0601 local set. +KEY = bytes([0x06, 0x0E, 0x2B, 0x34, 0x02, 0x0B, 0x01, 0x01, 0x0E, 0x01, 0x03, 0x01, 0x01, 0x00, 0x00, 0x00]) + + +class Failure(Exception): + """A named grading failure.""" + + def __init__(self, check, detail): + self.check = check + super().__init__(f"{check}: {detail}") + + +def pid(packet): + return ((packet[1] & 0x1F) << 8) | packet[2] + + +def payload(packet): + control = (packet[3] >> 4) & 3 + if control == 1: + return packet[4:] + if control == 3 and 5 + packet[4] <= SIZE: + return packet[5 + packet[4] :] + return b"" + + +def pcr(packet): + if packet[3] & 0x20 and packet[4] >= 7 and packet[5] & 0x10: + base = int.from_bytes(packet[6:10], "big") << 1 | packet[10] >> 7 + return base * 300 + ((packet[10] & 1) << 8 | packet[11]) + return None + + +def clock_value(field): + return (field[0] >> 1 & 7) << 30 | field[1] << 22 | (field[2] >> 1) << 15 | field[3] << 7 | field[4] >> 1 + + +def clock_field(prefix, ticks): + return bytes([ + prefix << 4 | (ticks >> 29 & 0x0E) | 1, + ticks >> 22 & 0xFF, + (ticks >> 14 & 0xFE) | 1, + ticks >> 7 & 0xFF, + (ticks << 1 & 0xFE) | 1, + ]) + + +def packets(data): + if len(data) % SIZE or any(data[at] != 0x47 for at in range(0, len(data), SIZE)): + raise ValueError("expected aligned 188-byte TS packets") + return [data[at : at + SIZE] for at in range(0, len(data), SIZE)] + + +def split(route, pes, counter): + """`pes` on `route` in packets numbered from `counter`, the last padded with stuffing.""" + out = [] + for i, at in enumerate(range(0, len(pes), 184)): + chunk = pes[at : at + 184] + head = bytes([0x47, (0x40 if i == 0 else 0) | route >> 8, route & 0xFF]) + cc = (counter + i) & 15 + if len(chunk) == 184: + out.append(head + bytes([0x10 | cc]) + chunk) + continue + stuffing = 183 - len(chunk) + field = bytes([stuffing]) + (bytes([0]) + b"\xff" * (stuffing - 1) if stuffing else b"") + out.append(head + bytes([0x30 | cc]) + field + chunk) + return out + + +def pes_with(stream_id, flags, fields, data): + return bytes([0, 0, 1, stream_id]) + (3 + len(fields) + len(data)).to_bytes(2, "big") + bytes([0x84, flags, len(fields)]) + fields + data + + +def async_klv(index): + value = index.to_bytes(4, "big") + bytes([0xA5] * (400 if index % LONG == LONG - 1 else 24)) + return pes_with(0xBD, 0x00, b"", KEY + bytes([0x82]) + len(value).to_bytes(2, "big") + value) + + +def sync_klv(index, pts): + value = index.to_bytes(4, "big") + bytes([0x5A] * 12) + klv = KEY + bytes([len(value)]) + value + # The metadata AU cell (2.12.4): service 0, its sequence number, one whole cell with a random + # access point, then its length. + cell = bytes([0x00, index & 0xFF, 0xDF]) + len(klv).to_bytes(2, "big") + klv + return pes_with(0xFC, 0x80, clock_field(0x2, pts), cell) + + +@dataclass +class Program: + pmt_pid: int + pcr_pid: int + streams: dict = field(default_factory=dict) # PID -> (stream_type, descriptors) + + +def read_pmt(section): + """(PCR PID, {PID: (stream_type, descriptors)}) of a PMT section.""" + length = (section[1] & 0x0F) << 8 | section[2] + end = 3 + length - 4 + pcr_pid = (section[8] & 0x1F) << 8 | section[9] + at = 12 + ((section[10] & 0x0F) << 8 | section[11]) + streams = {} + while at + 5 <= end: + info = (section[at + 3] & 0x0F) << 8 | section[at + 4] + streams[(section[at + 1] & 0x1F) << 8 | section[at + 2]] = (section[at], bytes(section[at + 5 : at + 5 + info])) + at += 5 + info + return pcr_pid, streams + + +def single_section(packet): + """The PSI section a packet starts and holds whole, or None.""" + if not packet[1] & 0x40: + return None + body = payload(packet) + if not body: + return None + section = body[1 + body[0] :] + if len(section) < 3: + return None + length = 3 + ((section[1] & 0x0F) << 8 | section[2]) + return section[:length] if length <= len(section) else None + + +def program(data): + """The first program's PMT PID, PCR PID and streams.""" + pmt_pid = None + for packet in packets(data): + section = single_section(packet) + if section is None: + continue + if pid(packet) == 0 and section[0] == 0x00 and pmt_pid is None: + for at in range(8, len(section) - 4, 4): + if section[at] << 8 | section[at + 1]: + pmt_pid = (section[at + 2] & 0x1F) << 8 | section[at + 3] + break + elif pmt_pid is not None and pid(packet) == pmt_pid and section[0] == 0x02: + pcr_pid, streams = read_pmt(section) + return Program(pmt_pid, pcr_pid, streams) + raise ValueError("no PAT and PMT") + + +def add_streams(packet): + """A PMT packet with the KLV streams appended to its section, its CRC recomputed.""" + body = payload(packet) + start = len(packet) - len(body) + 1 + body[0] + section = bytearray(single_section(packet)) + entries = b"".join(bytes([kind, 0xE0 | route >> 8, route & 0xFF, 0xF0 | len(info) >> 8, len(info) & 0xFF]) + info for kind, route, info in STREAMS) + section = section[:-4] + entries + length = len(section) + 4 - 3 + section[1] = (section[1] & 0xF0) | length >> 8 + section[2] = length & 0xFF + section += compliance.crc32_mpeg(bytes(section)).to_bytes(4, "big") + if start + len(section) > SIZE: + raise ValueError("the PMT outgrows its packet") + out = bytearray(packet[:start]) + section + out += b"\xff" * (SIZE - len(out)) + return bytes(out) + + +def inject(data): + """`data` with the KLV streams in its null slots, and a report of what went in.""" + source = packets(data) + prog = program(data) + if ASYNC_PID in prog.streams or SYNC_PID in prog.streams: + raise ValueError("the clip already carries the KLV PIDs") + video = next(route for route, (kind, _) in prog.streams.items() if kind == 0x1B) + out = [] + queue = [] + counters = {ASYNC_PID: 0, SYNC_PID: 0} + counts = {ASYNC_PID: 0, SYNC_PID: 0} + deadline = None + last = None + + def enqueue(route, pes): + queue.extend(split(route, pes, counters[route])) + counters[route] += -(-len(pes) // 184) + counts[route] += 1 + + for packet in source: + route = pid(packet) + if route == prog.pmt_pid and (single_section(packet) or b"\0")[0] == 0x02: + packet = add_streams(packet) + if route == prog.pcr_pid and (now := pcr(packet)) is not None: + # Unwrapped, so the period runs on across a wrap. + now = now if last is None else last + (now - last % PCR_WRAP) % PCR_WRAP + last = now + deadline = deadline or now + while now >= deadline: + enqueue(ASYNC_PID, async_klv(counts[ASYNC_PID])) + deadline += ASYNC_PERIOD + body = payload(packet) + if route == video and packet[1] & 0x40 and body[:3] == b"\0\0\1" and body[7] & 0x80: + enqueue(SYNC_PID, sync_klv(counts[SYNC_PID], clock_value(body[9:14]))) + if route == NULL and queue: + packet = queue.pop(0) + out.append(packet) + if queue: + raise ValueError(f"{len(queue)} KLV packets found no null slot") + return b"".join(out), {hex(route): count for route, count in counts.items()} + + +@dataclass +class Pes: + at: int # input offset of its start code + data: bytes + + +def private_pes(data, routes): + """Every whole PES on each of `routes`, with where it starts.""" + found = {route: [] for route in routes} + open_ = {} + for index, packet in enumerate(packets(data)): + route = pid(packet) + if route not in found: + continue + body = payload(packet) + if packet[1] & 0x40: + open_[route] = Pes(index * SIZE + SIZE - len(body), bytearray()) + found[route].append(open_[route]) + if route in open_: + open_[route].data += body + for route, pes in found.items(): + for item in pes: + length = item.data[4] << 8 | item.data[5] + if length == 0 or len(item.data) < 6 + length: + raise Failure("pes", f"PID {route:#x}: a PES at byte {item.at} is truncated or unbounded") + item.data = bytes(item.data[: 6 + length]) + return found + + +def clock(data, pcr_pid): + """The program clock at an input offset, in unwrapped 27 MHz ticks: interpolated between the + PCRs either side of it, extrapolated from the nearest two outside them.""" + samples = [] + for index, packet in enumerate(packets(data)): + if pid(packet) == pcr_pid and (value := pcr(packet)) is not None: + if samples: + value = samples[-1][1] + (value - samples[-1][1] % PCR_WRAP) % PCR_WRAP + samples.append((index * SIZE + PCR_OFFSET, value)) + if len(samples) < 2: + raise Failure("pcr", "fewer than two PCRs") + + def at(offset): + lo, hi = 0, len(samples) - 1 + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if samples[mid][0] <= offset else (lo, mid) + (a, ta), (b, tb) = samples[lo], samples[hi] + return ta + Fraction((offset - a) * (tb - ta), b - a) + + rate = Fraction((samples[-1][0] - samples[0][0]) * 8 * 27_000_000, samples[-1][1] - samples[0][1]) + return at, rate + + +def clocks(pes): + """The offsets of `pes`'s PTS and DTS fields.""" + if pes[3] in (0xBC, 0xBE, 0xBF, 0xF0, 0xF1, 0xF2, 0xF8, 0xFF): + return [] + return [9 + 5 * k for k in range({0b10: 1, 0b11: 2}.get(pes[7] >> 6, 0))] + + +def masked(pes): + out = bytearray(pes) + for at in clocks(pes): + out[at : at + 5] = bytes(5) + return bytes(out) + + +def ffprobe_klv(path): + """FFmpeg's KLV streams by PID: their codec, and each packet's PTS (None without).""" + report = json.loads(subprocess.run( + ["ffprobe", "-v", "error", "-probesize", str(Path(path).stat().st_size), "-analyzeduration", str(2**62), + "-show_entries", "stream=index,id,codec_name:packet=stream_index,pts", "-of", "json", str(path)], + check=True, capture_output=True, text=True, + ).stdout) + streams = {s["index"]: (int(s["id"], 16), s.get("codec_name")) for s in report["streams"]} + found = {route: (codec, []) for route, codec in streams.values() if route in (ASYNC_PID, SYNC_PID)} + for packet in report.get("packets", []): + route, _ = streams[packet["stream_index"]] + if route in found: + pts = packet.get("pts") + found[route][1].append(None if pts is None else int(pts) % PTS_WRAP) + return found + + +def grade(source, capture, probe=None): + """Grade `capture` against `source`, returning a report; raises Failure on the first miss. + + `probe`, when given, returns FFmpeg's reading of the capture, as `ffprobe_klv` does. + """ + src_prog, cap_prog = program(source), program(capture) + report = {} + for kind, route, info in STREAMS: + if src_prog.streams.get(route) != (kind, info): + raise Failure("source", f"PID {route:#x} is not the injected stream") + if cap_prog.streams.get(route) != (kind, info): + raise Failure("pmt", f"PID {route:#x}: {cap_prog.streams.get(route)} in the capture, not {(kind, info)}") + routes = [route for _, route, _ in STREAMS] + src_pes, cap_pes = private_pes(source, routes), private_pes(capture, routes) + src_at, _ = clock(source, src_prog.pcr_pid) + cap_at, rate = clock(capture, cap_prog.pcr_pid) + # One packet at the capture's rate, in 27 MHz ticks: how finely a placement can be kept. + packet = SIZE * 8 * 27_000_000 / rate + + matched = {} + for route in routes: + index = {masked(pes.data): k for k, pes in enumerate(src_pes[route])} + found = [] + for pes in cap_pes[route]: + k = index.get(masked(pes.data)) + if k is None: + raise Failure("bytes", f"PID {route:#x}: a PES at byte {pes.at} matches none of the source's outside its PTS and DTS") + found.append(k) + if not found: + raise Failure("missing", f"PID {route:#x}: no PES in the capture") + first, last = found[0], len(src_pes[route]) - 1 + if found != list(range(first, last + 1)): + gaps = sorted(set(range(first, last + 1)) - set(found)) + raise Failure("missing", f"PID {route:#x}: not the source's PES {first}..{last} in order: missing {gaps[:5]}, got {len(found)}") + late = Fraction(src_at(src_pes[route][first].at) - src_at(src_pes[route][0].at), 27_000_000) + if late > holes.JOIN: + raise Failure("join", f"PID {route:#x}: starts {float(late):.3f}s after the source's first PES, past the {float(holes.JOIN)}s join") + matched[route] = [(pes, src_pes[route][k]) for pes, k in zip(cap_pes[route], found)] + + # Every PTS and DTS moves by the one shift the importer's timebase gives the media. + shift = None + for route, pairs in matched.items(): + for k, (pes, src) in enumerate(pairs): + for at in clocks(pes.data): + moved = (clock_value(pes.data[at : at + 5]) - clock_value(src.data[at : at + 5])) % PTS_WRAP + shift = moved if shift is None else shift + if moved != shift: + raise Failure("shift", f"PID {route:#x}: capture PES {k} moved {moved} ticks, the rest {shift}") + if shift is None: + raise Failure("shift", "no PES carries a PTS") + + for route, pairs in matched.items(): + lateness = [] + for k, (pes, src) in enumerate(pairs): + # Modulo the wrap, as each clock is unwrapped from its own first PCR. + ahead = cap_at(pes.at) - (src_at(src.at) + shift * 300) + ahead = (ahead + PCR_WRAP // 2) % PCR_WRAP - PCR_WRAP // 2 + if ahead < -packet: + raise Failure("early", f"PID {route:#x}: capture PES {k} goes out {float(-ahead) / 27_000:.3f} ms before its placement") + lateness.append(ahead) + if clocks(pes.data): + due = clock_value(pes.data[9:14]) * 300 - cap_at(pes.at) + if (due + PCR_WRAP // 2) % PCR_WRAP - PCR_WRAP // 2 < 0: + raise Failure("late", f"PID {route:#x}: capture PES {k} arrives after its PTS") + report[hex(route)] = { + "pes": len(pairs), + "source": len(src_pes[route]), + "first": len(src_pes[route]) - len(pairs), + "late_ms_max": float(max(lateness)) / 27_000, + } + + if probe is not None: + found = probe() + for kind, route, _ in STREAMS: + codec, pts = found.get(route, (None, [])) + if codec != "klv": + raise Failure("ffmpeg", f"PID {route:#x}: FFmpeg reads {codec}, not SMPTE KLV") + ours = [clock_value(pes.data[9:14]) if clocks(pes.data) else None for pes in cap_pes[route]] + if pts != ours: + raise Failure("ffmpeg", f"PID {route:#x}: FFmpeg reads {len(pts)} PES, the grader {len(ours)}, or their PTS differ") + report["shift"] = shift + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + add = sub.add_parser("inject") + add.add_argument("source", type=Path) + add.add_argument("output", type=Path) + add.add_argument("--report", type=Path) + check = sub.add_parser("grade") + check.add_argument("source", type=Path) + check.add_argument("capture", type=Path) + check.add_argument("--report-json", type=Path) + args = parser.parse_args() + + if args.command == "inject": + data, report = inject(args.source.read_bytes()) + args.output.write_bytes(data) + if args.report: + args.report.write_text(json.dumps(report, indent=2) + "\n") + print("injected KLV PES:", report) + return 0 + + print("### KLV (capture against the source)") + try: + report = grade(args.source.read_bytes(), args.capture.read_bytes(), lambda: ffprobe_klv(args.capture)) + except Failure as err: + print(f" FAIL {err}") + return 1 + if args.report_json: + args.report_json.write_text(json.dumps(report, indent=2) + "\n") + for route, row in report.items(): + if route != "shift": + print(f" PASS PID {route}: {row['pes']} PES of {row['source']} from #{row['first']}, " + f"at most {row['late_ms_max']:.3f} ms after placement") + print(f" PTS and DTS moved {report['shift']} ticks") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/klv.test.py b/test/ts/klv.test.py new file mode 100644 index 00000000000..ec8a280415d --- /dev/null +++ b/test/ts/klv.test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Controls for the KLV fixture and its grader: each miss it exists to catch fails it.""" + +import unittest + +import compliance +import klv + +VIDEO = 0x100 +PMT = 0x1000 +# The picture presents this long after the PCR its PES starts by, in 90 kHz ticks. +LEAD = 63_000 + + +def section(table_id, extension, body): + out = bytearray([table_id, 0xB0, 0, extension >> 8, extension & 0xFF, 0xC1, 0, 0]) + body + length = len(out) + 4 - 3 + out[1] |= length >> 8 + out[2] = length & 0xFF + return bytes(out) + compliance.crc32_mpeg(bytes(out)).to_bytes(4, "big") + + +def psi(route, table): + return bytes([0x47, 0x40 | route >> 8, route & 0xFF, 0x10, 0]) + table + b"\xff" * (183 - len(table)) + + +def picture(counter, pcr): + """A video packet carrying `pcr` and an unbounded PES presenting LEAD after it.""" + base, ext = pcr // 300, pcr % 300 + field = bytes([7, 0x10]) + (base >> 1).to_bytes(4, "big") + bytes([(base & 1) << 7 | 0x7E | ext >> 8, ext & 0xFF]) + pes = bytes([0, 0, 1, 0xE0, 0, 0, 0x80, 0x80, 5]) + klv.clock_field(0x2, (base + LEAD) % klv.PTS_WRAP) + body = pes + b"\x00" * (184 - len(field) - len(pes)) + return bytes([0x47, 0x40 | VIDEO >> 8, VIDEO & 0xFF, 0x30 | counter & 15]) + field + body + + +def null(): + return bytes([0x47, 0x1F, 0xFF, 0x10]) + b"\xff" * 184 + + +def media(pictures=50): + """A clip of `pictures` 40 ms apart, each followed by enough null slots for its KLV.""" + out = [ + psi(0, section(0x00, 1, bytes([0, 1, 0xE0 | PMT >> 8, PMT & 0xFF]))), + psi(PMT, section(0x02, 1, bytes([0xE0 | VIDEO >> 8, VIDEO & 0xFF, 0xF0, 0, 0x1B, 0xE0 | VIDEO >> 8, VIDEO & 0xFF, 0xF0, 0]))), + ] + for k in range(pictures): + out.append(picture(k, 27_000_000 + k * 40 * 27_000)) + out.extend(null() for _ in range(6)) + return b"".join(out) + + +def edit(data, offset, change): + out = bytearray(data) + change(out, offset) + return bytes(out) + + +def restamp(data, delta): + """`data` with every PCR moved by `delta` 27 MHz ticks.""" + out = bytearray(data) + for at in range(0, len(out), klv.SIZE): + packet = out[at : at + klv.SIZE] + value = klv.pcr(packet) + if value is None: + continue + value += delta + base, ext = value // 300, value % 300 + out[at + 6 : at + 10] = (base >> 1).to_bytes(4, "big") + out[at + 10] = (base & 1) << 7 | 0x7E | ext >> 8 + out[at + 11] = ext & 0xFF + return bytes(out) + + +class Klv(unittest.TestCase): + def setUp(self): + self.source, report = klv.inject(media()) + self.assertEqual(report, {"0x110": 20, "0x111": 50}) + self.pes = klv.private_pes(self.source, [klv.ASYNC_PID, klv.SYNC_PID]) + + def assertFails(self, check, capture): + with self.assertRaises(klv.Failure) as caught: + klv.grade(self.source, capture) + self.assertEqual(caught.exception.check, check, str(caught.exception)) + + def test_the_source_grades_itself(self): + report = klv.grade(self.source, self.source) + self.assertEqual(report["shift"], 0) + self.assertEqual(report["0x111"]["pes"], 50) + + def test_only_null_slots_change(self): + before, after = klv.packets(media()), klv.packets(self.source) + self.assertEqual(len(before), len(after)) + for old, new in zip(before, after): + if klv.pid(old) not in (klv.NULL, PMT): + self.assertEqual(old, new) + + def test_the_streams_and_their_pes(self): + streams = klv.program(self.source).streams + self.assertEqual(streams[klv.ASYNC_PID], (0x06, klv.KLVA)) + self.assertEqual(streams[klv.SYNC_PID], (0x15, klv.METADATA)) + sync = self.pes[klv.SYNC_PID][3].data + self.assertEqual(sync[3], 0xFC) + self.assertEqual(klv.clock_value(sync[9:14]), 27_000_000 // 300 + 3 * 3_600 + LEAD) + untimed = self.pes[klv.ASYNC_PID][0].data + self.assertEqual((untimed[3], untimed[7] >> 6), (0xBD, 0)) + + def test_a_shift_moves_every_clock_alike(self): + delta = 90_000 + capture = self.source + for pes in self.pes[klv.SYNC_PID]: + at = pes.at + 9 + value = (klv.clock_value(capture[at : at + 5]) + delta) % klv.PTS_WRAP + capture = edit(capture, at, lambda out, at, value=value: out.__setitem__(slice(at, at + 5), klv.clock_field(0x2, value))) + self.assertEqual(klv.grade(self.source, restamp(capture, delta * 300))["shift"], delta) + # One clock moved apart from the rest. + at = self.pes[klv.SYNC_PID][7].at + 9 + self.assertFails("shift", edit(capture, at, lambda out, at: out.__setitem__(slice(at, at + 5), klv.clock_field(0x2, 0)))) + + def test_a_dropped_pes_fails(self): + at = self.pes[klv.SYNC_PID][10].at // klv.SIZE * klv.SIZE + self.assertFails("missing", edit(self.source, at, lambda out, at: out.__setitem__(slice(at, at + klv.SIZE), null()))) + + def test_a_changed_byte_fails(self): + at = self.pes[klv.ASYNC_PID][1].at + 30 + self.assertFails("bytes", edit(self.source, at, lambda out, at: out.__setitem__(at, out[at] ^ 1))) + + def test_a_changed_descriptor_fails(self): + self.assertFails("pmt", self.source.replace(b"KLVA", b"KLVB", 1)) + + def test_a_pes_out_early_fails(self): + # Past one packet, which at this sparse clip's rate is about 6 ms. + self.assertFails("early", restamp(self.source, -20 * 27_000)) + + def test_a_pes_out_after_its_pts_fails(self): + self.assertFails("late", restamp(self.source, LEAD * 300 + 27_000)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/ts/run.sh b/test/ts/run.sh index 47f8ef73c2b..5bd431dced7 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -24,6 +24,7 @@ # ./run.sh --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak # ./run.sh --full # the same with two audio PIDs, filling its multiplex with no nulls # ./run.sh --fragment-pes # split generated clear PES headers across packets +# ./run.sh --klv # add KLV and DVB AC-3, carried as whole PES, and grade them # ./run.sh --delay 2s # pass the exporter's --delay # ./run.sh --loss 10 # drop 10% of the relay's datagrams to the subscriber (moq-shaper) # ./run.sh --passthrough # carry the multiplex whole (`import/export ts --passthrough`) @@ -82,6 +83,7 @@ HEADROOM="" # the same plus four audio PIDs, in a multiplex faster than the LEAK="" # publish video with no HRD at its level's leak, as a full-rate feed does FULL="" # the same with audio, in a multiplex the source fills with no nulls FRAGMENT_PES="" # use following null slots for fragmented PES headers +KLV="" # inject asynchronous and synchronous KLV beside DVB AC-3, and grade every PES LOSS="" # percent of relay -> subscriber datagrams dropped by moq-shaper PASSTHROUGH="" # carry the multiplex whole and grade the capture packet by packet against the source MPTS="" # publish two programs, which only --passthrough carries whole @@ -169,6 +171,10 @@ while [[ $# -gt 0 ]]; do FRAGMENT_PES=1 shift ;; + --klv) + KLV=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -254,6 +260,11 @@ if [[ -n "$FRAGMENT_PES" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN exit 1 fi +if [[ -n "$KLV" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAGMENT_PES$PAIR$LIVE$PASSTHROUGH$MPTS" ]]; then + echo "error: --klv generates its own clip and grades the demultiplexed export's PES" >&2 + exit 1 +fi + # The #4613 path: random loss on the relay's datagrams to the subscriber and 25 ms each way, # through moq-shaper so it needs no privileges and leaves the host's network alone. if [[ -n "$LOSS" ]]; then @@ -456,6 +467,22 @@ elif [[ -n "$FULL" ]]; then tsp -I file "$PADDED_TS" -P filter --negate --pid 0x1FFF \ -P pcradjust --bitrate "$BITRATE" --ignore-pts --ignore-dts -O file "$SRC_TS" echo " ${NULLS} null packets dropped of ${PACKETS}: ${BITRATE} b/s" +elif [[ -n "$KLV" ]]; then + echo "### generating ~${DURATION}s clip with DVB AC-3 and KLV" + # The importer decodes neither DVB AC-3 (system_b's private data with an AC-3 descriptor) + # nor KLV, so it carries each PES whole. FFmpeg's default PES size puts several AC-3 sync + # frames in each, whose PTS times only the first: the export has to remove each from the + # T-STD buffer at its own decode time. klv.py then fills null slots with KLV. + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc=size=1280x720:rate=25" \ + -f lavfi -i "sine=frequency=1000:sample_rate=48000" \ + -f lavfi -i "sine=frequency=500:sample_rate=48000" \ + -t "$DURATION" -map 0:v -map 1:a -map 2:a -ac 2 \ + -c:v libx264 -profile:v high -preset veryfast -pix_fmt yuv420p \ + -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 8M \ + -c:a:0 aac -b:a:0 128k -c:a:1 ac3 -b:a:1 192k \ + -f mpegts -mpegts_flags system_b -muxrate "$BITRATE" -pcr_period 20 "$HARNESS_RUN/media.ts" + python3 "$DIR/klv.py" inject "$HARNESS_RUN/media.ts" "$SRC_TS" --report "$HARNESS_RUN/klv-injected.json" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` @@ -783,7 +810,7 @@ fi # The initial decoded-content profile is the generated closed-GOP H.264/AAC # pair. Other arms retain their specific structural/model oracles until their # acquisition and decoder-priming contracts are covered by fixtures. -if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL" ]]; then +if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL$KLV" ]]; then if ! python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode"; then # The exporter counts what it dropped (late), trimmed (before its output began), # and lost upstream (gaps), and logs each loss, so the failure says which it was. @@ -800,7 +827,7 @@ fi # The other generated clips keep one frame step per stream, so media missing from the capture # shows in its timestamps even where its content is not compared. The compliance checks below # grade only the access units that arrived. -if [[ -n "$HRD$HEADROOM$LEAK$FULL" ]]; then +if [[ -n "$HRD$HEADROOM$LEAK$FULL$KLV" ]]; then echo if ! python3 "$DIR/holes.py" "$SUB_TS" "$SRC_TS"; then echo >&2 @@ -842,6 +869,17 @@ if [[ -n "$OPEN_GOP" ]]; then fi fi +# Each private PES is the source's, at the source's placement. +if [[ -n "$KLV" ]]; then + echo + if ! python3 "$DIR/klv.py" grade "$SRC_TS" "$SUB_TS" --report-json "$HARNESS_RUN/klv.json"; then + echo >&2 + echo "error: KLV analysis failed (see round-trip logs below)" >&2 + dump_logs + exit 1 + fi +fi + # The delay's lead over the source's hides packets the export spends beyond the source's # until it runs out, so they are counted outright. if [[ -n "$FULL" ]]; then From e580475f85819990c29e6526366f728811c3bc4c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 23:19:36 -0700 Subject: [PATCH 3/7] docs(moq-mux): link the private PES migration to its PR Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/MIGRATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index 424e61aea97..18c1a2f786a 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### TS private PES rides `pesPacket` tracks; `verbatim` is sections only +### TS private PES rides `pesPacket` tracks; `verbatim` is sections only (#5921) `ts::Framing::Pes` and `ts::Verbatim::stream_id` are gone, and `Verbatim::framing` no longer defaults: a catalog naming `"framing": "pes"`, or a `verbatim` record without `framing`, fails to parse. `ts::Import` now publishes every undecoded PES stream as a `ts::PesPacket` track, each frame one whole PES packet placed by the PCR. From fc5ae3939531c0a8035ffb214ec7f5dac98222e1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:12:39 -0700 Subject: [PATCH 4/7] fix(moq-mux): refuse at import every PES header export refuses The importer and the exporter now share one whole-PES header check, `psi::PesStart::whole`: a flagged optional field or PES extension that overruns PES_header_data_length, or a zero PES_packet_length on a non-video stream (with or without an optional header), refuses the stream at ingest instead of poisoning the export later. A PMT that moves the PCR PID now settles the placement timebase, so a whole PES is never interpolated between PCRs of two clocks. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- drafts/draft-lcurley-moq-mpegts.md | 3 +- rs/moq-mux/src/container/ts/export.rs | 107 ++---------------- rs/moq-mux/src/container/ts/import.rs | 33 +++--- .../src/container/ts/pes_packet_test.rs | 51 ++++++++- rs/moq-mux/src/container/ts/psi.rs | 96 ++++++++++------ rs/moq-mux/src/container/ts/stats.rs | 5 +- 7 files changed, 143 insertions(+), 155 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 6e4e757e83e..76ff9487109 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -95,7 +95,8 @@ and every other stream it does not decode (KLV, subtitles, DVB AC-3) through as tracks. Each such PES goes through whole, at its place on the program clock (PCR); one before the first PCR is dropped. A stream it cannot carry unaltered is refused with a warning: any in a program with no PCR, and one whose PES carry -an ESCR, pack header or TREF, are scrambled, or leave a non-video PES unbounded. +an ESCR, pack header or TREF, are scrambled, leave a non-video PES unbounded, +or have a malformed header. SCTE-35 cues and private PES wait for a media timestamp; input ending or declaring a clock break before that timestamp is refused, so a program of data alone fails. diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 4bcb628ece8..31ced5f8ecb 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -256,10 +256,11 @@ A consumer MUST refuse a frame that is not one well-formed PES packet. The record is a member beside `verbatim` rather than a `framing` value, so a consumer that predates it ignores the track instead of refusing the catalog. A publisher demultiplexing a transport stream places each PES by the PCRs of its program ({{mpeg2}} Section 2.4.2.2): its first byte is interpolated between the reference bytes of the PCRs either side of it, and one past its timebase's last PCR is extrapolated from that timebase's last two. +A timebase ends at a PCR discontinuity, and at a PMT that moves the `PCR_PID`. Each PCR takes the 2^33 multiple of its base that lands it nearest the media's PTS, and the placement moves onto the catalog clock by the same shift as the media, so a PES's timestamp depends only on the input and that shift. A PES before the first PCR the publisher reads, or past a lone one, has nothing to place it and is dropped. The publisher moves each PTS and DTS by the media's shift modulo 2^33, keeping their prefix and marker bits, and leaves every other byte as received; the PES CRC covers the previous packet's data bytes, not its header, so it stays valid. -It MUST refuse to carry a stream rather than alter it: one in a program with no PCR, and one whose PES carry a clock the shift does not reach (an ESCR, a `pack_header_field`, or a TREF), PES-level scrambling, or a zero `PES_packet_length` on a stream that is not video. +It MUST refuse to carry a stream rather than alter it: one in a program with no PCR, and one whose PES carry a clock the shift does not reach (an ESCR, a `pack_header_field`, or a TREF), PES-level scrambling, a zero `PES_packet_length` on a stream that is not video, or a header whose flagged optional fields overrun its `PES_header_data_length`, which a consumer refuses. A verbatim or whole-PES track has no codec, dimensions, or decoder configuration, so a publisher MUST NOT describe it as a media rendition and MUST describe it only by its `tracks` entry, and a consumer MUST refuse a catalog that describes one as both. Nothing there declares a container, so its frames use hang's `legacy` container ({{hang}}): a varint microsecond timestamp followed by the payload. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index eb9764674b9..72b9ed02c1c 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -2769,104 +2769,21 @@ impl PesPacket { /// where the `stream_id` has one (ISO 13818-1 2.4.3.6). Only video may leave the /// length zero, unbounded, as 2.4.3.7 permits. fn parse(bytes: &[u8], stream_type: u8) -> anyhow::Result { - anyhow::ensure!( - bytes.len() >= PES_PREFIX_LEN && bytes.starts_with(&[0, 0, 1]), - "no PES start code" - ); - let stream_id = bytes[3]; - let length = usize::from(u16::from_be_bytes([bytes[4], bytes[5]])); - if length == 0 { - let video = - matches!(stream_id, 0xe0..=0xef) || (stream_id == 0xfd && psi::is_video_stream_type(stream_type)); - anyhow::ensure!( - video, - "PES_packet_length 0 on stream_id {stream_id:#04x}, which only a video stream may leave unbounded" - ); - } else { + // The importer refuses with this same check, so a PES it published never fails here. + let pes = psi::PesStart::parse(bytes)?; + // The clock fields it refuses ride through unchanged. + pes.whole(stream_type)?; + if let Some(len) = pes.data_len { anyhow::ensure!( - length == bytes.len() - PES_PREFIX_LEN, - "PES_packet_length {length} where the frame holds {} bytes after it", - bytes.len() - PES_PREFIX_LEN + len == pes.data.len(), + "PES_packet_length declares {len} payload bytes where the frame holds {}", + pes.data.len() ); } - // program_stream_map, padding, private_stream_2, ECM, EMM, directory, DSM-CC and - // H.222.1 type E carry their payload straight after the length. - if !psi::has_optional_header(stream_id) { - return Ok(Self { - pts: None, - payload: PES_PREFIX_LEN, - }); - } - anyhow::ensure!( - bytes.len() >= 9 && bytes[6] & 0xc0 == 0x80, - "PES optional header missing or without its '10' marker" - ); - let flags = bytes[7]; - let payload = 9 + usize::from(bytes[8]); - let fields = bytes - .get(9..payload) - .context("PES_header_data_length runs past the packet")?; - let pts_dts = flags >> 6; - anyhow::ensure!(pts_dts != 0b01, "PTS_DTS_flags '01' is forbidden"); - // Each flagged field in order (Table 2-21): PTS, DTS, ESCR, ES_rate, DSM trick mode, - // additional_copy_info, previous_PES_CRC; stuffing fills the rest. - let mut at = [ - (0x80, 5), - (0x40, 5), - (0x20, 6), - (0x10, 3), - (0x08, 1), - (0x04, 1), - (0x02, 2), - ] - .iter() - .filter(|&&(flag, _)| flags & flag != 0) - .map(|&(_, len)| len) - .sum::(); - if flags & 0x01 != 0 { - let read = |at: usize| fields.get(at).copied().context("PES extension runs past its header"); - let extension = read(at)?; - at += 1; - // PES_private_data, then pack_header_field behind its own length byte. - if extension & 0x80 != 0 { - at += 16; - } - if extension & 0x40 != 0 { - at += 1 + usize::from(read(at)?); - } - // program_packet_sequence_counter and P-STD_buffer. - at += 2 * usize::from(extension & 0x20 != 0) + 2 * usize::from(extension & 0x10 != 0); - if extension & 0x01 != 0 { - at += 1 + usize::from(read(at)? & 0x7f); - } - } - anyhow::ensure!( - at <= fields.len(), - "PES optional fields need {at} bytes where PES_header_data_length gives {}", - fields.len() - ); - // Each clock field opens with its 4-bit prefix, PTS '0010' alone or '0011' beside a DTS - // '0001', and sets the marker bit after each of its three parts. - let clock = |field: &[u8], prefix: u8| { - anyhow::ensure!( - field[0] >> 4 == prefix && field[0] & field[2] & field[4] & 1 == 1, - "PES clock field without its '{prefix:04b}' prefix or marker bits" - ); - Ok(u64::from((field[0] >> 1) & 7) << 30 - | u64::from(field[1]) << 22 - | u64::from(field[2] >> 1) << 15 - | u64::from(field[3]) << 7 - | u64::from(field[4] >> 1)) - }; - let pts = match pts_dts { - 0b10 => Some(clock(&fields[..5], 0b0010)?), - 0b11 => { - clock(&fields[5..10], 0b0001)?; - Some(clock(&fields[..5], 0b0011)?) - } - _ => None, - }; - Ok(Self { pts, payload }) + Ok(Self { + pts: pes.pts, + payload: pes.header.len(), + }) } } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index d99b0abee25..b81fdc3a1eb 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -60,9 +60,10 @@ use moq_net::Timestamp; /// clock break or the end of input is extrapolated from the last two. One before the first PCR, /// or with no interval to extrapolate from, is dropped and counted in /// [`stats::Stream::unplaced`]. A stream whose PES carry a clock only the PTS and DTS of are -/// rebased (ESCR, a `pack_header_field`, a TREF), PES-level scrambling, or an unbounded non-video -/// PES is refused rather than altered, as is every such stream in a program with no PCR: -/// [`stats::Stream::refused`]. +/// rebased (ESCR, a `pack_header_field`, a TREF), PES-level scrambling, an unbounded non-video +/// PES, or a header export would refuse is refused rather than altered, as is every such stream +/// in a program with no PCR: [`stats::Stream::refused`]. A PMT that moves the PCR PID ends the +/// placement timebase, as a clock break does. /// /// The PAT and every PMT are reassembled the same way, so a table that spans packets or /// sits behind a nonzero pointer_field is read whole. A PAT or PMT section whose CRC-32 @@ -482,9 +483,11 @@ impl Import { if self.pcr_pid != pmt.pcr_pid { self.pcr_pid = pmt.pcr_pid; self.health.pcr_pid(pmt.pcr_pid.map(|pid| pid.as_u16())); - // A new clock: the intervals straddling the switch measure nothing. + // A new clock: the intervals straddling the switch measure nothing, and the whole PES + // the old one began are placed on its timebase, never between its PCRs and the new. self.liveness.discontinuity(); self.reset_mux_rate()?; + self.settle_placement()?; } // SCTE-35 is announced by a program-level registration descriptor with @@ -801,7 +804,7 @@ impl Import { if let Some(stream_type) = whole && let Some(why) = refusal(&pes, stream_type) { - return self.refuse(pid, why); + return self.refuse(pid, &why); } if pes.scrambled() { return self.damage(pid, &anyhow::anyhow!("scrambled PES is not supported")); @@ -1871,19 +1874,15 @@ impl PesPacketStream { } } -/// Why a whole-PES track cannot carry `pes` unaltered, if it cannot: a clock field only the PTS -/// and DTS of are rebased, a scrambled payload, or an unbounded PES that is not video, which -/// export refuses. -fn refusal(pes: &PesStart, stream_type: u8) -> Option<&'static str> { - if let Some(clock) = pes.unrebased_clock() { - return Some(clock); - } - if pes.scrambled() { - return Some("PES-level scrambling"); +/// Why a whole-PES track cannot carry `pes` unaltered, if it cannot: a header export refuses +/// ([`PesStart::whole`]), a clock field only the PTS and DTS of are rebased, or a scrambled +/// payload. +fn refusal(pes: &PesStart, stream_type: u8) -> Option { + match pes.whole(stream_type) { + Err(err) => Some(format!("{err:#}")), + Ok(Some(clock)) => Some(clock.to_string()), + Ok(None) => pes.scrambled().then(|| "PES-level scrambling".to_string()), } - let video = - matches!(pes.stream_id, 0xe0..=0xef) || (pes.stream_id == 0xfd && psi::is_video_stream_type(stream_type)); - (pes.data_len.is_none() && !video).then_some("an unbounded PES on a stream that is not video") } /// The catalog time of the program clock reading `ticks`, an unwrapped 27 MHz count, on a diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index c8fee75b23f..c63c693da52 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -503,11 +503,18 @@ async fn dvb_ac3_keeps_its_sync_frames_in_one_pes() { assert_eq!(got, expected); } -/// A PES the track cannot carry unaltered refuses its stream: what was already placed stays -/// published, the track ends with an error and leaves the catalog, and the refusal is counted. +/// A PES the track cannot carry unaltered, or whose header export refuses, refuses its stream: +/// what was already placed stays published, the track ends with an error and leaves the catalog, +/// and the refusal is counted. #[tokio::test(start_paused = true)] async fn a_pes_with_a_field_no_rebase_reaches_refuses_its_stream() { - let cases: [(&str, Vec); 5] = [ + let cases: [(&str, Vec); 8] = [ + // PES_extension_flag set with no header byte left for the extension. + ("truncated extension", pes_with(0xbd, 0x80, 0x01, &[], 0, b"ext")), + // ES_rate flagged with one of its three bytes inside PES_header_data_length. + ("overrunning ES_rate", pes_with(0xbd, 0x80, 0x10, &[0x80], 0, b"rate")), + // No optional header to carry, but PES_packet_length 0 on a stream that is not video. + ("unbounded without an optional header", pes_bare(0xbf, b"")), ( "ESCR", pes_with(0xbd, 0x80, 0x20, &[0x04, 0x00, 0x04, 0x00, 0x04, 0x01], 0, b"escr"), @@ -634,3 +641,41 @@ async fn a_clock_break_places_each_side_on_its_own_timebase() { assert!(frames[1].timestamp > frames[0].timestamp); assert_eq!(&frames[1].payload[..], &feed.pes[1].1[..]); } + +/// A PMT moving the PCR to another PID ends the placement timebase too: the PES past the old +/// clock's last PCR is extrapolated on it, never interpolated toward the new clock's first PCR, +/// and those after the switch are placed by the new clock alone. +#[tokio::test(start_paused = true)] +async fn a_pcr_pid_switch_places_each_side_on_its_own_timebase() { + const NEW_CLOCK_PID: u16 = 0x1fe; + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + feed.pcr(start).picture(start).pcr(start + 40 * MS).data(klv(1, 8)); + let before = feed.clock(feed.pes[0].0).unwrap(); + feed.mux.pmt_with( + PMT_PID, + 1, + NEW_CLOCK_PID, + &[(0x02, VIDEO_PID, &[]), (0x06, DATA_PID, KLVA)], + 0, + ); + // A clock far from the old one, so a PES placed across the two lands far from either. + let other = 5_000 * MS; + feed.pcrs.clear(); + for i in 0..3 { + feed.pcrs.push((feed.mux.out.len() as u64 + 10, other + i * 40 * MS)); + feed.mux.packet(NEW_CLOCK_PID, Flags::pcr(other + i * 40 * MS)); + if i == 1 { + feed.data(klv(2, 8)); + } + } + let after = feed.clock(feed.pes[1].0).unwrap(); + + let imported = import(&feed.bytes(), false).unwrap(); + let (name, _) = imported.entry().unwrap(); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + let got: Vec<_> = frames.iter().map(|frame| frame.timestamp.as_micros() as u64).collect(); + assert_eq!(got, [placed(before, 0), placed(after, 0)]); + assert_eq!(imported.stats().unplaced, 0); +} diff --git a/rs/moq-mux/src/container/ts/psi.rs b/rs/moq-mux/src/container/ts/psi.rs index f2b144c599c..d68a912eafc 100644 --- a/rs/moq-mux/src/container/ts/psi.rs +++ b/rs/moq-mux/src/container/ts/psi.rs @@ -373,43 +373,67 @@ impl PesStart<'_> { self.header.get(6).is_some_and(|flags| flags & 0x30 != 0) } - /// The clock field in the optional header that a whole-PES track cannot carry, since only - /// the PTS and DTS are moved onto the catalog clock: ESCR, a `pack_header_field`, or a TREF - /// (ISO 13818-1 2.4.3.7). `None` for a header that carries none of them. - pub fn unrebased_clock(&self) -> Option<&'static str> { - let header = self.header; - let flags = *header.get(7)?; - if flags & 0x20 != 0 { - return Some("ESCR"); - } - if flags & 0x01 == 0 { - return None; - } - // Skip the PTS, DTS, ESCR, ES rate, trick mode, copy info and CRC to the extension. - let skip = match flags >> 6 { - 0b10 => 5, - 0b11 => 10, - _ => 0, - } + usize::from(flags & 0x10 != 0) * 3 - + usize::from(flags & 0x08 != 0) - + usize::from(flags & 0x04 != 0) - + usize::from(flags & 0x02 != 0) * 2; - let mut at = 9 + skip; - let extension = *header.get(at)?; - at += 1; - if extension & 0x40 != 0 { - return Some("pack_header_field"); - } - at += usize::from(extension & 0x80 != 0) * 16 - + usize::from(extension & 0x20 != 0) * 2 - + usize::from(extension & 0x10 != 0) * 2; - if extension & 0x01 == 0 { - return None; + /// Check the header as a whole-PES track carries it, on a stream of `stream_type`, and return + /// the clock field it carries that only the PTS and DTS of are moved onto the catalog clock: + /// ESCR, a `pack_header_field`, or a TREF (ISO 13818-1 2.4.3.7). Refuses an unbounded PES + /// that is not video, and a flagged optional field that runs past `PES_header_data_length`. + pub fn whole(&self, stream_type: u8) -> anyhow::Result> { + let stream_id = self.stream_id; + let video = matches!(stream_id, 0xe0..=0xef) || (stream_id == 0xfd && is_video_stream_type(stream_type)); + anyhow::ensure!( + self.header[4..6] != [0, 0] || video, + "PES_packet_length 0 on stream_id {stream_id:#04x}, which only a video stream may leave unbounded" + ); + let Some(fields) = self.header.get(9..) else { + return Ok(None); + }; + let flags = self.header[7]; + // Each flagged field in order (Table 2-21): PTS, DTS, ESCR, ES_rate, DSM trick mode, + // additional_copy_info, previous_PES_CRC; stuffing fills the rest. + let mut at = [ + (0x80, 5), + (0x40, 5), + (0x20, 6), + (0x10, 3), + (0x08, 1), + (0x04, 1), + (0x02, 2), + ] + .iter() + .filter(|&&(flag, _)| flags & flag != 0) + .map(|&(_, len)| len) + .sum::(); + let mut clock = (flags & 0x20 != 0).then_some("ESCR"); + if flags & 0x01 != 0 { + let read = |at: usize| fields.get(at).copied().context("PES extension runs past its header"); + let extension = read(at)?; + at += 1; + // PES_private_data, then pack_header_field behind its own length byte. + if extension & 0x80 != 0 { + at += 16; + } + if extension & 0x40 != 0 { + clock = clock.or(Some("pack_header_field")); + at += 1 + usize::from(read(at)?); + } + // program_packet_sequence_counter and P-STD_buffer. + at += 2 * usize::from(extension & 0x20 != 0) + 2 * usize::from(extension & 0x10 != 0); + if extension & 0x01 != 0 { + let len = usize::from(read(at)? & 0x7f); + // A set stream_id_extension_flag with a clear tref_extension_flag says a TREF + // follows. + if len > 0 && read(at + 1)? & 0x81 == 0x80 { + clock = clock.or(Some("TREF")); + } + at += 1 + len; + } } - // PES_extension_field_length, then stream_id_extension_flag; with it set, a clear - // tref_extension_flag says a TREF follows. - let field = *header.get(at + 1)?; - (field & 0x80 != 0 && field & 0x01 == 0).then_some("TREF") + anyhow::ensure!( + at <= fields.len(), + "PES optional fields need {at} bytes where PES_header_data_length gives {}", + fields.len() + ); + Ok(clock) } } diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index 84e14ce6eca..21b1cf7e76e 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -116,8 +116,9 @@ pub struct Stream { pub unplaced: u64, /// Times this PID's stream was refused rather than carried altered: a private PES stream /// with a clock field only the PTS and DTS of are rebased (ESCR, a `pack_header_field`, a - /// TREF), PES-level scrambling, an unbounded non-video PES, or a program with no PCR to place - /// it by. Its track ends and leaves the catalog. Export stays zero. + /// TREF), PES-level scrambling, an unbounded non-video PES, a header export refuses, or a + /// program with no PCR to place it by. Its track ends and leaves the catalog. Export stays + /// zero. pub refused: u64, /// This PID's share of [`Snapshot::continuity_count_error`]. pub continuity_count_error: u64, From e032e164379c76e3e294e8672581fe0774155be4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:33:40 -0700 Subject: [PATCH 5/7] fix(moq-mux): count held whole PES against the staging unit bound Each whole PES waiting for a timestamp held a Vec, Span and Retained but reserved no staging unit, so a flood of tiny ones fit under the byte bound alone. Count each as one unit, as untimed sections are. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 10 +++++++--- rs/moq-mux/src/container/ts/pes_packet_test.rs | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index b81fdc3a1eb..282a843ffcc 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -890,7 +890,9 @@ impl Import { }; let data = &pes.data[..pes.data_len.unwrap_or(pes.data.len()).min(pes.data.len())]; let mut retained = Retained::new(self.staging.clone()); - retained.grow(header.len() + data.len(), 0)?; + // A whole PES is held as a unit until placed, so tiny ones count against the unit bound + // as sections do, not the byte bound alone. + retained.grow(header.len() + data.len(), usize::from(span.is_some()))?; let mut pending = Pending { retained, pts, @@ -1476,7 +1478,9 @@ impl Retained { ); if !reserve(&self.staging.units, units, STAGING_UNITS_MAX) { self.staging.bytes.fetch_sub(bytes, Ordering::Relaxed); - anyhow::bail!("TS staging exceeds {STAGING_UNITS_MAX} sections while waiting for a timestamp"); + anyhow::bail!( + "TS staging exceeds {STAGING_UNITS_MAX} sections and whole PES while waiting for a timestamp" + ); } self.bytes += bytes; self.units += units; @@ -7581,7 +7585,7 @@ pub(super) mod test { let err = import .decode(&packet(true, (super::STAGING_UNITS_MAX % 16) as u8, 0, &CUE)) .unwrap_err(); - assert!(err.to_string().contains("sections while waiting for a timestamp")); + assert!(err.to_string().contains("whole PES while waiting for a timestamp")); assert_eq!(import.sections[&0x21].held.len(), super::STAGING_UNITS_MAX); assert_eq!( import.staging.bytes.load(super::Ordering::Relaxed), diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index c63c693da52..37b6485bce4 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -679,3 +679,21 @@ async fn a_pcr_pid_switch_places_each_side_on_its_own_timebase() { assert_eq!(got, [placed(before, 0), placed(after, 0)]); assert_eq!(imported.stats().unplaced, 0); } + +/// Whole PES held for a timestamp count against the staging unit bound as sections do, so a +/// flood of tiny ones refuses the import long before their bookkeeping outgrows the byte bound. +#[test] +fn tiny_whole_pes_held_for_a_timestamp_are_bounded_by_count() { + let units = moq_net::group::MAX_GROUP_FRAMES / 2; + let mut feed = Feed::new(0x06, KLVA); + feed.pcr(1_000 * MS); + for _ in 0..=units { + feed.data(pes_with(0xbd, 0x80, 0x00, &[], 0, b"")); + } + let mut imported = start(false); + let err = imported.import.decode(&feed.bytes()).expect_err("the flood is refused"); + assert!( + err.to_string().contains("whole PES while waiting for a timestamp"), + "{err:#}" + ); +} From c84bee758000edc04679dc425ebef294ef8fe9db Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 11:55:54 -0700 Subject: [PATCH 6/7] fix(moq-mux): place split PES headers from their start and follow PCR_PID A PES header spanning packets took its 'before' PCR once the header was whole, so a PCR landing mid-header was used as the one before it, and a PES begun before the first PCR was published instead of unplaced. The span is now taken at the start code, and a PCR that lands mid-header brackets it. A private PES stream's route was settled only at creation. A PMT that drops the PCR now refuses a carried stream, and one that names a PCR again carries a stream refused only for want of one. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 76 +++++++++++++------ .../src/container/ts/pes_packet_test.rs | 64 ++++++++++++++++ 2 files changed, 117 insertions(+), 23 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 282a843ffcc..48d826472b3 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -63,7 +63,8 @@ use moq_net::Timestamp; /// rebased (ESCR, a `pack_header_field`, a TREF), PES-level scrambling, an unbounded non-video /// PES, or a header export would refuse is refused rather than altered, as is every such stream /// in a program with no PCR: [`stats::Stream::refused`]. A PMT that moves the PCR PID ends the -/// placement timebase, as a clock break does. +/// placement timebase, as a clock break does; one that drops the PCR refuses those streams, and +/// one that names it again carries them. /// /// The PAT and every PMT are reassembled the same way, so a table that spans packets or /// sits behind a nonzero pointer_field is read whole. A PAT or PMT section whose CRC-32 @@ -126,9 +127,9 @@ pub struct Import { liveness: Liveness, /// In-progress PES reassembly, keyed by elementary PID. pending: HashMap, - /// Partial clear PES headers, at most 9 + 255 bytes per elementary PID, each with the input - /// offset of its start code. - headers: HashMap)>, + /// Partial clear PES headers, at most 9 + 255 bytes per elementary PID, each with where its + /// start code sits on the program clock ([`Placement::span`]). + headers: HashMap, Vec)>, /// The input offset of the first byte in `scratch`, which places every packet in the input. position: u64, /// The program clock at each input byte, which places whole PES packets. @@ -137,6 +138,8 @@ pub struct Import { unplaced: BTreeMap, /// Private PES streams refused rather than carried altered, keyed by PID. refused: BTreeMap, + /// Private PES streams refused only for want of a PCR, carried once a PMT names one. + unclocked: HashSet, /// One program-wide budget for PES payloads and sections awaiting their clock. staging: Arc, /// The TR 101 290 checks, and the per-PID continuity state routing reads. @@ -238,6 +241,7 @@ impl Import { placement: Placement::default(), unplaced: BTreeMap::new(), refused: BTreeMap::new(), + unclocked: HashSet::new(), staging: Arc::default(), health: Health::default(), pcr_pid: None, @@ -548,35 +552,38 @@ impl Import { } // The payload ends the packet, so this is where its start code sits in the input. let at = offset + (TsPacket::SIZE - payload.len()) as u64; + // Taken here, not once the header is whole, so a PCR landing before the header's + // continuation is the one after it. + let span = self.placement.span(at); return match PesStart::header_len(payload) { Ok(len) if payload.len() >= len => match PesStart::parse(payload) { - Ok(pes) => self.handle_pes_start(pid, pes, at), + Ok(pes) => self.handle_pes_start(pid, pes, span), Err(err) => self.damage(pid, &err), }, - Ok(_) => self.pes_header(pid, at, payload.to_vec()), + Ok(_) => self.pes_header(pid, span, payload.to_vec()), Err(err) => self.damage(pid, &err), }; } - if let Some((at, mut header)) = self.headers.remove(&pid) { + if let Some((span, mut header)) = self.headers.remove(&pid) { header.extend_from_slice(payload); - self.pes_header(pid, at, header) + self.pes_header(pid, span, header) } else { self.handle_pes_continuation(pid, payload) } } - /// Continue a PES header split across packets, whose start code sits at input offset `at`. - fn pes_header(&mut self, pid: Pid, at: u64, header: Vec) -> anyhow::Result<()> { + /// Continue a PES header split across packets, placed by `span` from its start code. + fn pes_header(&mut self, pid: Pid, span: Option, header: Vec) -> anyhow::Result<()> { let len = match PesStart::header_len(&header) { Ok(len) => len, Err(err) => return self.damage(pid, &err), }; if header.len() < len { - self.headers.insert(pid, (at, header)); + self.headers.insert(pid, (span, header)); return Ok(()); } match PesStart::parse(&header) { - Ok(pes) => self.handle_pes_start(pid, pes, at), + Ok(pes) => self.handle_pes_start(pid, pes, span), Err(err) => self.damage(pid, &err), } } @@ -592,8 +599,19 @@ impl Import { self.pending.remove(&pid); self.headers.remove(&pid); } - if self.streams.contains_key(&pid) { - return Ok(()); + // A private PES stream follows the PMT's PCR_PID: refused while there is none to place + // its PES by, and carried again once there is. + match self.streams.get(&pid) { + Some(Stream::PesPacket(_)) if self.pcr_pid.is_none() => { + self.refuse(pid, "the program lost the PCR that placed it")?; + self.unclocked.insert(pid); + return Ok(()); + } + Some(Stream::Ignored) if self.pcr_pid.is_some() && self.unclocked.remove(&pid) => { + self.streams.remove(&pid); + } + Some(_) => return Ok(()), + None => {} } let stream = match StreamType::from_u8(stream_type).ok() { @@ -678,6 +696,7 @@ impl Import { "refused a private PES stream in a program with no PCR to place it by" ); *self.refused.entry(pid.as_u16()).or_default() += 1; + self.unclocked.insert(pid); Stream::Ignored } else { let descriptors = descriptors.to_vec(); @@ -775,6 +794,7 @@ impl Import { /// Remove a PES route while keeping the counters it accumulated for this importer. fn retire_stream(&mut self, pid: Pid) -> Option> { + self.unclocked.remove(&pid); let stream = self.streams.remove(&pid)?; if let Some(current) = stream.stats() { self.retired_stats @@ -785,8 +805,8 @@ impl Import { Some(stream) } - /// Open the PES whose header `pes` begins at input offset `at`. - fn handle_pes_start(&mut self, pid: Pid, pes: PesStart, at: u64) -> anyhow::Result<()> { + /// Open the PES whose header `pes` begins where `span` says, if any PCR came before it. + fn handle_pes_start(&mut self, pid: Pid, pes: PesStart, span: Option) -> anyhow::Result<()> { // Finish the preceding PES before accepting another header. if self.pending.contains_key(&pid) { self.flush(pid)?; @@ -868,7 +888,8 @@ impl Import { // A whole PES keeps its header, and is placed by the PCR before it and the one after. let (header, data_len, span) = match whole { Some(_) => { - let Some(before) = self.placement.last() else { + let generation = self.placement.generation; + let Some(mut span) = span.filter(|span| span.generation == generation) else { // Before the first PCR on this timebase nothing places it. A PTS it carried still // anchored the timebase, so it releases the reservation like a flushed PES would. if pts.is_some() { @@ -877,12 +898,10 @@ impl Import { *self.unplaced.entry(pid.as_u16()).or_default() += 1; return Ok(()); }; - let span = Span { - at, - before, - generation: self.placement.generation, - ticks: None, - }; + // A PCR that landed while the header spanned packets is the one after it. + if let Some(&after) = self.placement.samples.iter().find(|sample| sample.at > span.at) { + span.bracket(generation, after); + } let len = pes.data_len.map(|len| pes.header.len() + len); (pes.header, len, Some(span)) } @@ -1969,6 +1988,17 @@ impl Placement { self.samples.back().copied() } + /// A PES whose start code sits at input offset `at`, after the last PCR on this timebase; + /// `None` before the first. + fn span(&self, at: u64) -> Option { + Some(Span { + at, + before: self.last()?, + generation: self.generation, + ticks: None, + }) + } + /// The program clock at input offset `at`, past the last PCR, at the rate of the interval /// before it; `None` without one. fn extrapolate(&self, at: u64) -> Option { diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 37b6485bce4..abc566d176a 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -697,3 +697,67 @@ fn tiny_whole_pes_held_for_a_timestamp_are_bounded_by_count() { "{err:#}" ); } + +/// A PES whose header spans packets is placed from its start code, not from where the header +/// completes: one begun before the first PCR stays unplaced though a PCR lands mid-header, and +/// one begun after a PCR is bracketed by the PCR that lands mid-header. +#[tokio::test(start_paused = true)] +async fn a_pes_header_split_by_a_pcr_is_placed_from_its_start_code() { + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + // 9 + 200 header bytes, so each header ends in its PES's second packet. + let split = |fill: u8| pes_with(0xbd, 0x80, 0x00, &[], 200, &[fill; 8]); + feed.data_across(split(1), 1, start); + feed.picture(start).pcr(start + 40 * MS); + feed.data_across(split(2), 1, start + 80 * MS); + feed.pcr(start + 120 * MS); + let expected = placed(feed.clock(feed.pes[1].0).unwrap(), 0); + + let imported = import(&feed.bytes(), false).unwrap(); + let (name, _) = imported.entry().unwrap(); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].timestamp.as_micros() as u64, expected); + assert_eq!(&frames[0].payload[..], &feed.pes[1].1[..]); + assert_eq!(imported.stats().unplaced, 1, "the PES begun before the first PCR"); +} + +/// A later PMT that names a PCR carries the private PES stream refused for want of one, and one +/// that drops the PCR again refuses the stream it carried. +#[tokio::test(start_paused = true)] +async fn a_pmt_naming_or_dropping_the_pcr_reroutes_private_pes() { + let streams: &[(u8, u16, &[u8])] = &[(0x02, VIDEO_PID, &[]), (0x06, DATA_PID, KLVA)]; + let mut feed = Feed::with_pcr(0x06, KLVA, NO_PCR); + let start = 1_000 * MS; + feed.picture(start).data(klv(0, 8)); + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, streams, 0); + feed.pcr(start + 40 * MS) + .picture(start + 40 * MS) + .data(klv(1, 8)) + .pcr(start + 80 * MS); + let head = feed.bytes(); + feed.mux.pmt_with(PMT_PID, 1, NO_PCR, streams, 0); + feed.picture(start + 80 * MS).data(klv(2, 8)); + + let mut imported = start_import(&head); + let (name, _) = imported + .entry() + .expect("the stream is carried once the PMT names a PCR"); + let reader = { + let consumer = imported.consumer.clone(); + tokio::spawn(async move { read(&consumer, &name).await }) + }; + for _ in 0..10 { + tokio::task::yield_now().await; + } + imported.import.decode(&feed.bytes()[head.len()..]).unwrap(); + imported.import.finish().unwrap(); + + let (frames, err) = reader.await.unwrap(); + assert_eq!(frames.len(), 1, "the PES placed while the PCR was named"); + assert_eq!(&frames[0].payload[..], &feed.pes[1].1[..]); + assert!(err.is_some(), "the track ends with an error once the PCR goes"); + assert!(imported.entry().is_none(), "the track left the catalog"); + assert_eq!(imported.stats().refused, 2); +} From d0b19a3a8c3ad965d4f4951eaf0a1b07858ec6e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 12:41:27 -0700 Subject: [PATCH 7/7] fix(moq-mux): bracket staged PES headers per PCR and reroute retyped PIDs A PES header split across packets is now bracketed by place_pcr (and settled at a timebase end) while it assembles, like a pending PES, so the first PCR past its start code places it however many arrive before the header is whole. Placement keeps only two samples, so the lookup at header completion missed it after three. A PID refused as private PES for want of a PCR now routes again when a later PMT gives it another stream type, not only when a PCR appears. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 81 ++++++++++++++----- .../src/container/ts/pes_packet_test.rs | 27 +++++++ 2 files changed, 86 insertions(+), 22 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 48d826472b3..67f1b8babbc 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -138,8 +138,9 @@ pub struct Import { unplaced: BTreeMap, /// Private PES streams refused rather than carried altered, keyed by PID. refused: BTreeMap, - /// Private PES streams refused only for want of a PCR, carried once a PMT names one. - unclocked: HashSet, + /// The `stream_type` of each private PES stream refused only for want of a PCR, routed again + /// once a PMT names one or gives the PID another type. + unclocked: HashMap, /// One program-wide budget for PES payloads and sections awaiting their clock. staging: Arc, /// The TR 101 290 checks, and the per-PID continuity state routing reads. @@ -241,7 +242,7 @@ impl Import { placement: Placement::default(), unplaced: BTreeMap::new(), refused: BTreeMap::new(), - unclocked: HashSet::new(), + unclocked: HashMap::new(), staging: Arc::default(), health: Health::default(), pcr_pid: None, @@ -600,14 +601,21 @@ impl Import { self.headers.remove(&pid); } // A private PES stream follows the PMT's PCR_PID: refused while there is none to place - // its PES by, and carried again once there is. + // its PES by, and routed again once there is, or once the PID carries another type. match self.streams.get(&pid) { - Some(Stream::PesPacket(_)) if self.pcr_pid.is_none() => { + Some(Stream::PesPacket(stream)) if self.pcr_pid.is_none() => { + let carried = stream.stream_type; self.refuse(pid, "the program lost the PCR that placed it")?; - self.unclocked.insert(pid); + self.unclocked.insert(pid, carried); return Ok(()); } - Some(Stream::Ignored) if self.pcr_pid.is_some() && self.unclocked.remove(&pid) => { + Some(Stream::Ignored) + if self + .unclocked + .get(&pid) + .is_some_and(|&refused| self.pcr_pid.is_some() || refused != stream_type) => + { + self.unclocked.remove(&pid); self.streams.remove(&pid); } Some(_) => return Ok(()), @@ -696,7 +704,7 @@ impl Import { "refused a private PES stream in a program with no PCR to place it by" ); *self.refused.entry(pid.as_u16()).or_default() += 1; - self.unclocked.insert(pid); + self.unclocked.insert(pid, stream_type); Stream::Ignored } else { let descriptors = descriptors.to_vec(); @@ -888,8 +896,9 @@ impl Import { // A whole PES keeps its header, and is placed by the PCR before it and the one after. let (header, data_len, span) = match whole { Some(_) => { - let generation = self.placement.generation; - let Some(mut span) = span.filter(|span| span.generation == generation) else { + // A span whose timebase ended while its header assembled was settled with it, or + // is dropped as unplaced at release. + let Some(span) = span else { // Before the first PCR on this timebase nothing places it. A PTS it carried still // anchored the timebase, so it releases the reservation like a flushed PES would. if pts.is_some() { @@ -898,10 +907,6 @@ impl Import { *self.unplaced.entry(pid.as_u16()).or_default() += 1; return Ok(()); }; - // A PCR that landed while the header spanned packets is the one after it. - if let Some(&after) = self.placement.samples.iter().find(|sample| sample.at > span.at) { - span.bracket(generation, after); - } let len = pes.data_len.map(|len| pes.header.len() + len); (pes.header, len, Some(span)) } @@ -1033,10 +1038,11 @@ impl Import { } let sample = self.placement.pcr(raw, offset + PCR_OFFSET); let generation = self.placement.generation; - for pending in self.pending.values_mut() { - if let Some(span) = pending.span.as_mut() { - span.bracket(generation, sample); - } + // A PES still assembling its header is bracketed too, so the first PCR past its start + // code places it however many arrive before the header is whole. + let spans = self.pending.values_mut().filter_map(|pending| pending.span.as_mut()); + for span in spans.chain(self.headers.values_mut().filter_map(|(span, _)| span.as_mut())) { + span.bracket(generation, sample); } for stream in self.streams.values_mut() { if let Stream::PesPacket(stream) = stream { @@ -1049,10 +1055,9 @@ impl Import { /// End the placement timebase: extrapolate every whole PES begun past its last PCR from its /// last two, and publish what that places. fn settle_placement(&mut self) -> anyhow::Result<()> { - for pending in self.pending.values_mut() { - if let Some(span) = pending.span.as_mut() { - span.settle(&self.placement); - } + let spans = self.pending.values_mut().filter_map(|pending| pending.span.as_mut()); + for span in spans.chain(self.headers.values_mut().filter_map(|(span, _)| span.as_mut())) { + span.settle(&self.placement); } for stream in self.streams.values_mut() { if let Stream::PesPacket(stream) = stream { @@ -9345,4 +9350,36 @@ pub(super) mod test { let audio = catalog.audio.renditions.values().next().unwrap(); assert_eq!(audio.codec.to_string(), "mp2"); } + + /// A PID refused as private PES for want of a PCR routes again when a later PMT gives it a + /// type that needs no PCR, though the program still has none. + #[test] + fn an_unclocked_pid_reassigned_to_a_decoded_type_routes_again() { + use crate::container::ts::psi::{Es, Pmt}; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default() + .with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pid = Pid::new(0x104).unwrap(); + let pmt = |stream_type| Pmt { + program_number: 1, + pcr_pid: None, + program_info: Vec::new(), + streams: vec![Es { + stream_type, + pid, + descriptors: Vec::new(), + }], + }; + import.handle_pmt(pmt(0x06)).unwrap(); + assert!(matches!(import.streams[&pid], super::Stream::Ignored)); + import.handle_pmt(pmt(0x06)).unwrap(); + assert_eq!(import.stats().streams[&0x104].refused, 1, "refused once, not per PMT"); + import.handle_pmt(pmt(StreamType::H264 as u8)).unwrap(); + assert!(matches!(import.streams[&pid], super::Stream::H264 { .. })); + } } diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index abc566d176a..668565f45a5 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -723,6 +723,33 @@ async fn a_pes_header_split_by_a_pcr_is_placed_from_its_start_code() { assert_eq!(imported.stats().unplaced, 1, "the PES begun before the first PCR"); } +/// A header held past more PCRs than the placement keeps is still bracketed by the first one past +/// its start code. The PCRs run at an uneven rate, so any later pair would place it elsewhere. +#[tokio::test(start_paused = true)] +async fn a_pes_header_held_past_several_pcrs_takes_the_first_after_it() { + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + feed.pcr(start).picture(start).pcr(start + 40 * MS); + // 9 + 200 header bytes, so the header ends in the PES's second packet. + let pes = pes_with(0xbd, 0x80, 0x00, &[], 200, &[1; 8]); + let (head, packets) = feed.mux.split(DATA_PID, &pes); + feed.pes.push(((feed.mux.out.len() + head) as u64, pes)); + feed.mux.out.extend_from_slice(&packets[0]); + for ticks in [50, 300, 310] { + feed.pcr(start + ticks * MS); + } + feed.mux.out.extend(packets[1..].concat()); + feed.pcr(start + 320 * MS); + let expected = placed(feed.clock(feed.pes[0].0).unwrap(), 0); + + let imported = import(&feed.bytes(), false).unwrap(); + let (name, _) = imported.entry().unwrap(); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].timestamp.as_micros() as u64, expected); +} + /// A later PMT that names a PCR carries the private PES stream refused for want of one, and one /// that drops the PCR again refuses the stream it carried. #[tokio::test(start_paused = true)]