diff --git a/Cargo.lock b/Cargo.lock index 4387ae5f067..2b6a26a26a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4642,6 +4642,7 @@ dependencies = [ "mp4-atom", "mpeg2ts", "num_enum", + "roxmltree", "scuffle-av1", "scuffle-h265", "serde", @@ -7569,6 +7570,15 @@ dependencies = [ "portable-atomic-util", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rsa" version = "0.9.10" diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 2771d1485a5..3983ed55a40 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -13,7 +13,7 @@ back. This is what `moq import` / `export` and the gateways are built on. | Format | Import | Export | Notes | | --- | --- | --- | --- | -| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text, preserving source language and label when present. | +| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text and `stpp` TTML (IMSC timing profile) as `ttml` documents on the broadcast clock, preserving source language and label when present; other subtitle codecs are skipped. | | MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 and subtitle PIDs as tracks; service tables round-trip. | | FLV / RTMP | yes | yes | Legacy H.264 + AAC + MP3, plus enhanced-RTMP HEVC, AV1, VP9, Opus, AC-3, E-AC-3, and multitrack. | | Matroska / WebM | yes | yes | Timed SRT and WebVTT subtitles import as WebVTT text renditions, preserving source language and label when present; other subtitle codecs are skipped. | diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index fc3cd59b151..aae676df53f 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -359,7 +359,7 @@ The `renditions` field maps rendition IDs to text configurations, typically one The `format` field selects the cue serialization, and tells a consumer how to parse each frame's payload: - `vtt`: WebVTT ([W3C WebVTT](https://www.w3.org/TR/webvtt1/)). Each payload is a self-contained `WEBVTT` segment whose cues carry absolute timing. A cue's embedded start time MUST match its enclosing frame timestamp. -- `ttml`: TTML / IMSC1 ([W3C IMSC](https://www.w3.org/TR/ttml-imsc1.1/)) fragment (XML) with absolute timing. A cue's embedded start time MUST match its enclosing frame timestamp. +- `ttml`: TTML / IMSC1 ([W3C IMSC](https://www.w3.org/TR/ttml-imsc1.1/)). Each payload is one complete TTML document in the `media` time base with absolute timing, clipped to the frame. Its `body` begin MUST match the enclosing frame timestamp, and its `body` end bounds everything the document presents. - `utf8`: raw UTF-8 text with no embedded timing or styling. The cue is shown from its frame timestamp until the next cue; an empty payload clears it. A consumer MUST ignore a rendition whose `format` it does not recognize. diff --git a/js/hang/src/catalog/text.ts b/js/hang/src/catalog/text.ts index 2dc6bdf3f6a..d72a756f902 100644 --- a/js/hang/src/catalog/text.ts +++ b/js/hang/src/catalog/text.ts @@ -6,7 +6,8 @@ import { RelativeBroadcastSchema } from "./path"; /** * The cue serialization format of a text track, mirroring the Rust `TextFormat`. * - * Known values are `"vtt"` (WebVTT), `"ttml"` (TTML/IMSC1), and `"utf8"` (raw text). Any other + * Known values are `"vtt"` (WebVTT), `"ttml"` (one complete TTML/IMSC1 document per frame, clipped + * to the frame, whose `body` begins at the frame timestamp), and `"utf8"` (raw text). Any other * string is an unrecognized format the consumer should ignore. The frame timestamp always carries * the cue's start time, so a relay never parses the payload. */ diff --git a/quest/m2/README.md b/quest/m2/README.md index a305cea0aaa..25cf00badc8 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -77,7 +77,6 @@ ranked just below deterministic TS legs, which it requires. - [Android capture](/quest/m2/mobile-capture-android.md) - Rust captures through NDK/JNI on Android, reusing the existing codecs and setting the catalog rotation - [Time stretch](/quest/m2/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent - [Native audio quality](/quest/m2/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device -- [TTML caption import](/quest/m2/captions-ttml.md) - fMP4 `stpp` tracks within a supported TTML/IMSC profile import as `ttml` renditions on the shared clock instead of being skipped - [Bindings size profile](/quest/m2/ffi-size-profile.md) - a benchmark decides whether the moq-ffi builds ship at opt-level "s", which halves the dylib - [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md) - returned payloads reach the host without a copy, if the payload numbers justify it - [Namespaced external types](/quest/m2/uniffi-namespaces.md) - the UniFFI fork lets two crates export the same type name, reopening the per-layer crate split diff --git a/quest/m2/captions-ttml.md b/quest/m2/captions-ttml.md deleted file mode 100644 index ebdbec48489..00000000000 --- a/quest/m2/captions-ttml.md +++ /dev/null @@ -1,24 +0,0 @@ -# [M] TTML caption import - -## Goal - -Importing an fMP4 `stpp` track within a supported TTML/IMSC profile yields a `ttml` text -rendition whose cues stay on the shared media clock. Today those XML tracks are -skipped with a warning while SRT and WebVTT import as `vtt`. - -## Plan - -Copying XML bytes is not enough. MP4 `stpp` names an XML namespace, which is not -necessarily TTML, and the importer shifts the source clock, so every TTML time -expression must be rewritten onto the catalog clock. - -- Support a bounded TTML/IMSC timing profile: validate the namespace and refuse - anything outside the profile instead of relabelling arbitrary XML as TTML. -- Convert every time expression, including nested `begin`/`end`/`dur` and - `ttp:timeBase` variants the profile admits, after the clock shift. -- Keep skipping XML tracks the profile does not cover, one track at a time. - -Matroska has no registered TTML codec ID (the old `S_TEXT/TTML` claim was -removed), so confirm a real mapping before adding one. - -Cover a namespace mismatch, nested timing, and a shifted source clock. diff --git a/rs/hang/src/catalog/text/format.rs b/rs/hang/src/catalog/text/format.rs index edcaaf4aff0..8125fec686c 100644 --- a/rs/hang/src/catalog/text/format.rs +++ b/rs/hang/src/catalog/text/format.rs @@ -18,8 +18,8 @@ pub enum TextFormat { #[default] Vtt, - /// TTML / IMSC1 ([W3C IMSC]) fragment (XML). Each frame's payload is a document fragment with - /// absolute timing. + /// TTML / IMSC1 ([W3C IMSC]). Each frame's payload is one complete TTML document with absolute + /// timing, clipped to the frame: its `body` begins at the frame timestamp. /// /// [W3C IMSC]: https://www.w3.org/TR/ttml-imsc1.1/ #[display("ttml")] diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 2118ea0c732..927e12a3de9 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -29,6 +29,7 @@ moq-net = { workspace = true } mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" num_enum = { workspace = true } +roxmltree = "0.21.1" scuffle-av1 = { version = "0.1.4" } scuffle-h265 = { version = "0.2.2" } serde = { workspace = true } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index b066d4037f8..d1fe0a9f13c 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -27,7 +27,8 @@ use crate::catalog::Estimator; /// - Opus /// - FLAC /// -/// **Text:** WebVTT (`wvtt`), reconstructed as timed text; unsupported subtitle codecs are skipped. +/// **Text:** WebVTT (`wvtt`), reconstructed as timed text, and TTML (`stpp`) documents moved onto the +/// catalog clock; unsupported subtitle codecs, including other XML namespaces, are skipped. pub struct Import { /// The broadcast being produced broadcast: moq_net::broadcast::Producer, @@ -370,15 +371,22 @@ impl Import { } fn init_text(&mut self, trak: &Trak, moov: &Moov, reserved: &crate::catalog::Reserved) -> Result { - let [mp4_atom::Codec::Wvtt(codec)] = trak.mdia.minf.stbl.stsd.codecs.as_slice() else { - return Ok(false); + let (format, mut config) = match trak.mdia.minf.stbl.stsd.codecs.as_slice() { + [mp4_atom::Codec::Wvtt(codec)] => { + if codec.plaintext.data_reference_index != 1 { + return Err(Error::InvalidSubtitle("external subtitle data reference").into()); + } + let header = crate::container::text::header(&codec.config.config).map_err(Error::InvalidSubtitle)?; + let mut config = hang::catalog::TextConfig::new(hang::catalog::TextFormat::Vtt); + config.role = crate::container::text::role(&header); + (super::text::Format::Vtt { header }, config) + } + [mp4_atom::Codec::Unknown(kind, body)] if kind.as_ref() == b"stpp" && super::text::stpp(body)? => ( + super::text::Format::Ttml, + hang::catalog::TextConfig::new(hang::catalog::TextFormat::Ttml), + ), + _ => return Ok(false), }; - if codec.plaintext.data_reference_index != 1 { - return Err(Error::InvalidSubtitle("external subtitle data reference").into()); - } - let header = crate::container::text::header(&codec.config.config).map_err(Error::InvalidSubtitle)?; - let mut config = hang::catalog::TextConfig::new(hang::catalog::TextFormat::Vtt); - config.role = crate::container::text::role(&header); config.lang = Some(trak.mdia.mdhd.language.clone()); // The picker label is the HandlerBox name; `vlab` is an opaque source identifier. config.label = (!trak.mdia.hdlr.name.is_empty()).then(|| trak.mdia.hdlr.name.clone()); @@ -392,7 +400,7 @@ impl Import { trak.tkhd.track_id, super::text::Track { producer, - header, + format, defaults: sample::Defaults::from_moov(moov, trak.tkhd.track_id), timescale: moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?, end_decode_time: None, diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 2f7b0ee0876..02a88e0e6d0 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2087,3 +2087,148 @@ fn malformed_webvtt_origins_and_comments_are_refused() { assert!(importer.decode(&webvtt_input(payload, 1250)).is_err()); } } + +/// An `stpp` init segment. mp4-atom cannot encode an `stpp` entry, so encode a `metx` entry, whose +/// strings line up with `stpp`'s namespace, schema location and auxiliary MIME types, and rename it. +fn ttml_init(namespace: &str, mime: &str, timescale: u32) -> Vec { + let (_, moov) = decode_init(&webvtt_init("WEBVTT")); + let mut trak = moov.trak[0].clone(); + trak.mdia.hdlr.handler = b"subt".into(); + trak.mdia.mdhd.timescale = timescale; + trak.mdia.minf.stbl.stsd.codecs = vec![mp4_atom::Codec::Metx(mp4_atom::Metx { + metadata: mp4_atom::MetaData { + data_reference_index: 1, + }, + content_encoding: namespace.into(), + namespace: String::new(), + schema_location: mime.into(), + btrt: None, + })]; + let mut init = audio_init(trak); + let at = init.windows(4).position(|kind| kind == b"metx").unwrap(); + init[at..at + 4].copy_from_slice(b"stpp"); + init +} + +const TTML: &str = r#"

Hello

"#; + +/// Import one TTML sample lasting 4 s from source time 100 s, returning its rendition and frame. +async fn imported_ttml( + timebase: crate::catalog::Timebase, + broadcast: moq_net::broadcast::Producer, + catalog: &crate::catalog::Producer, + consumer: &moq_net::broadcast::Consumer, +) -> (hang::catalog::TextConfig, crate::container::Frame) { + let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 1000); + input.extend(webvtt_fragment(100_000, &[TTML.as_bytes().to_vec()], 4000)); + let mut importer = super::Import::new(broadcast, timebase.reserve()); + importer.decode(&input).unwrap(); + importer.finish().unwrap(); + let snapshot = catalog.snapshot(); + let (name, config) = snapshot.text.renditions.iter().next().unwrap(); + let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); + let mut reader = crate::container::Consumer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + (config.clone(), reader.read_frame().await.unwrap().unwrap()) +} + +#[tokio::test(start_paused = true)] +async fn ttml_samples_become_timed_text_with_source_metadata() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let (config, frame) = imported_ttml(catalog.timebase(), broadcast, &catalog, &consumer).await; + assert_eq!(config.format, hang::catalog::TextFormat::Ttml); + assert_eq!(config.lang.as_deref(), Some("eng")); + assert_eq!(config.label.as_deref(), Some("English captions")); + assert_eq!(frame.timestamp.as_micros(), 100_000_000); + assert_eq!( + &frame.payload[..], + br#"

Hello

"# + ); +} + +/// The source joins a catalog clock another input already fixed, so its TTML times move with it. +#[tokio::test(start_paused = true)] +async fn ttml_follows_a_shifted_source_clock() { + let wall = std::time::SystemTime::now(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + catalog + .timebase() + .place(moq_net::Timestamp::from_secs(10).unwrap(), wall) + .unwrap(); + let timebase = catalog.timebase(); + timebase + .place(moq_net::Timestamp::from_secs(100).unwrap(), wall) + .unwrap(); + let (_, frame) = imported_ttml(timebase, broadcast, &catalog, &consumer).await; + assert_eq!(frame.timestamp.as_micros(), 10_000_000); + assert_eq!( + &frame.payload[..], + br#"

Hello

"# + ); +} + +/// An `stpp` entry naming another XML namespace, or carrying image subsamples, skips only its track. +#[test] +fn unsupported_stpp_tracks_are_skipped() { + for (namespace, mime) in [("urn:example:not-ttml", ""), ("http://www.w3.org/ns/ttml", "image/png")] { + let mut input = ttml_init(namespace, mime, 1000); + input.extend(webvtt_fragment(0, &[TTML.as_bytes().to_vec()], 1000)); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = super::Import::new(broadcast, catalog.reserve()); + importer.decode(&input).unwrap(); + importer.finish().unwrap(); + assert!(catalog.snapshot().text.renditions.is_empty(), "{namespace} {mime}"); + } +} + +/// A document the profile refuses fails the import instead of publishing as TTML. +#[test] +fn ttml_documents_outside_the_profile_are_refused() { + for document in [ + r#""#, + r#""#, + ] { + let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 1000); + input.extend(webvtt_fragment(0, &[document.as_bytes().to_vec()], 1000)); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = super::Import::new(broadcast, catalog.reserve()); + assert!(importer.decode(&input).is_err(), "{document}"); + } +} + +/// A 90 kHz sample start is not a whole microsecond; the body still begins at the serialized +/// frame timestamp. +#[tokio::test(start_paused = true)] +async fn ttml_body_begins_at_the_serialized_frame_timestamp() { + let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 90_000); + input.extend(webvtt_fragment(9_000_001, &[TTML.as_bytes().to_vec()], 360_000)); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = super::Import::new(broadcast, catalog.reserve()); + importer.decode(&input).unwrap(); + importer.finish().unwrap(); + let snapshot = catalog.snapshot(); + let name = snapshot.text.renditions.keys().next().unwrap(); + let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); + let mut reader = crate::container::Consumer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + let frame = reader.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.timestamp.as_micros(), 100_000_011); + let payload = std::str::from_utf8(&frame.payload).unwrap(); + assert!( + payload.contains(r#""#), + "{payload}" + ); +} diff --git a/rs/moq-mux/src/container/fmp4/text.rs b/rs/moq-mux/src/container/fmp4/text.rs index 20396bb5ee4..25e816b2293 100644 --- a/rs/moq-mux/src/container/fmp4/text.rs +++ b/rs/moq-mux/src/container/fmp4/text.rs @@ -3,10 +3,38 @@ use mp4_atom::{Any, DecodeMaybe}; use super::{Error, sample}; use crate::container::text::{self, Cue}; +use crate::container::ttml; + +/// How a track's samples are serialized. +pub(super) enum Format { + /// WebVTT (`wvtt`) cue boxes, prefixed with the sample entry's header. + Vtt { header: String }, + /// One TTML document per sample (`stpp`). + Ttml, +} + +/// Whether an `stpp` sample entry body (ISO/IEC 14496-12 XMLSubtitleSampleEntry) carries TTML text. +/// +/// Image subsamples (auxiliary MIME types) and namespaces other than TTML are unsupported. +pub(super) fn stpp(body: &[u8]) -> Result { + let (header, strings) = body + .split_at_checked(8) + .ok_or(Error::InvalidSubtitle("truncated stpp sample entry"))?; + if header[6..] != [0, 1] { + return Err(Error::InvalidSubtitle("external subtitle data reference")); + } + // namespace, schema_location, auxiliary_mime_types; encoders may omit the trailing ones. + let mut strings = strings.split(|&byte| byte == 0); + let namespace = strings.next().unwrap_or_default(); + let namespace = std::str::from_utf8(namespace).map_err(|_| Error::InvalidSubtitle("invalid stpp namespace"))?; + let _schema_location = strings.next(); + let mime = strings.next().unwrap_or_default(); + Ok(namespace.split_whitespace().any(|name| name == ttml::NAMESPACE) && mime.is_empty()) +} pub(super) struct Track { pub producer: crate::container::Producer, - pub header: String, + pub format: Format, pub defaults: sample::Defaults, pub timescale: Timescale, /// The exclusive decode end of the previous fragment; the next one may not start before it. @@ -30,7 +58,7 @@ impl Track { let traf = input.traf; let mut dts = traf.tfdt.as_ref().ok_or(Error::MissingTfdt)?.base_media_decode_time; if self.end_decode_time.is_some_and(|end| dts < end) { - return Err(Error::InvalidSubtitle("WebVTT fragment overlaps the previous one").into()); + return Err(Error::InvalidSubtitle("subtitle fragment overlaps the previous one").into()); } if traf.trun.is_empty() { return Err(Error::MissingTrun.into()); @@ -61,14 +89,25 @@ impl Track { let pts = dts .checked_add_signed(entry.cts.unwrap_or_default() as i64) .ok_or(Error::PtsOverflow)?; - let timestamp = timebase.shift(Timestamp::new(pts, self.timescale)?)?; + let source = Timestamp::new(pts, self.timescale)?; + let timestamp = timebase.shift(source)?; let duration = Timestamp::new(duration as u64, self.timescale)?; let end = timestamp.checked_add(duration)?; let next = offset .checked_add(sample.size as usize) .ok_or(Error::InvalidDataOffset)?; let data = input.data.get(offset..next).ok_or(Error::InvalidDataOffset)?; - let payload = self.sample(data, timestamp, end)?; + let payload = match &self.format { + Format::Vtt { header } => vtt(header, data, timestamp, end)?, + // The frame timestamp is serialized in hang's scale, which the body's begin must match. + Format::Ttml => ttml::rewrite( + data, + source, + timestamp.convert(hang::container::TIMESCALE)?..end.convert(hang::container::TIMESCALE)?, + ) + .map_err(Error::InvalidSubtitle)? + .into(), + }; self.producer.write(crate::container::Frame { timestamp, duration: Some(duration), @@ -83,79 +122,78 @@ impl Track { self.end_decode_time = Some(dts); Ok(()) } +} - fn sample(&self, mut data: &[u8], start: Timestamp, end: Timestamp) -> crate::Result { - if data.is_empty() { - return Err(Error::InvalidSubtitle("empty WebVTT sample").into()); - } - let mut output = self.header.clone(); - while !data.is_empty() { - let atom = Any::decode_maybe(&mut data) - .map_err(Error::from)? - .ok_or(Error::InvalidSubtitle("truncated WebVTT sample box"))?; - let Any::Unknown(kind, body) = atom else { - return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()); - }; - match kind.as_ref() { - b"vtte" if body.is_empty() => {} - b"vttc" => { - let mut body = body.as_slice(); - let mut payload = None; - let mut id = None; - let mut settings = None; - let mut origin = None; - while !body.is_empty() { - let atom = Any::decode_maybe(&mut body) - .map_err(Error::from)? - .ok_or(Error::InvalidSubtitle("truncated WebVTT cue box"))?; - let Any::Unknown(kind, value) = atom else { - return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()); - }; - let field = match kind.as_ref() { - b"payl" => &mut payload, - b"iden" => &mut id, - b"sttg" => &mut settings, - b"ctim" => &mut origin, - _ => return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()), - }; - if field.is_some() { - return Err(Error::InvalidSubtitle("duplicate WebVTT cue box").into()); - } - *field = - Some(String::from_utf8(value).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?); - } - let payload = payload.ok_or(Error::InvalidSubtitle("missing WebVTT cue payload"))?; - // Inline timestamps sit on the source timeline, which only `ctim` anchors. - let payload = if let Some(origin) = origin { - let origin = text::timestamp(&origin).map_err(Error::InvalidSubtitle)?; - text::rebase(&payload, start..end, origin).map_err(Error::InvalidSubtitle)? - } else if text::has_timestamp(&payload) { - return Err(Error::InvalidSubtitle("inline WebVTT timestamp has no cue origin").into()); - } else { - payload +fn vtt(header: &str, mut data: &[u8], start: Timestamp, end: Timestamp) -> crate::Result { + if data.is_empty() { + return Err(Error::InvalidSubtitle("empty WebVTT sample").into()); + } + let mut output = header.to_string(); + while !data.is_empty() { + let atom = Any::decode_maybe(&mut data) + .map_err(Error::from)? + .ok_or(Error::InvalidSubtitle("truncated WebVTT sample box"))?; + let Any::Unknown(kind, body) = atom else { + return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()); + }; + match kind.as_ref() { + b"vtte" if body.is_empty() => {} + b"vttc" => { + let mut body = body.as_slice(); + let mut payload = None; + let mut id = None; + let mut settings = None; + let mut origin = None; + while !body.is_empty() { + let atom = Any::decode_maybe(&mut body) + .map_err(Error::from)? + .ok_or(Error::InvalidSubtitle("truncated WebVTT cue box"))?; + let Any::Unknown(kind, value) = atom else { + return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()); }; - text::append( - &mut output, - Cue { - start, - end, - body: &payload, - id: id.as_deref().unwrap_or_default(), - settings: settings.as_deref().unwrap_or_default(), - }, - ) - .map_err(Error::InvalidSubtitle)?; - } - b"vtta" => { - let value = - std::str::from_utf8(&body).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?; - if !value.is_empty() { - text::comments(&mut output, value).map_err(Error::InvalidSubtitle)?; + let field = match kind.as_ref() { + b"payl" => &mut payload, + b"iden" => &mut id, + b"sttg" => &mut settings, + b"ctim" => &mut origin, + _ => return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()), + }; + if field.is_some() { + return Err(Error::InvalidSubtitle("duplicate WebVTT cue box").into()); } + *field = + Some(String::from_utf8(value).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?); + } + let payload = payload.ok_or(Error::InvalidSubtitle("missing WebVTT cue payload"))?; + // Inline timestamps sit on the source timeline, which only `ctim` anchors. + let payload = if let Some(origin) = origin { + let origin = text::timestamp(&origin).map_err(Error::InvalidSubtitle)?; + text::rebase(&payload, start..end, origin).map_err(Error::InvalidSubtitle)? + } else if text::has_timestamp(&payload) { + return Err(Error::InvalidSubtitle("inline WebVTT timestamp has no cue origin").into()); + } else { + payload + }; + text::append( + &mut output, + Cue { + start, + end, + body: &payload, + id: id.as_deref().unwrap_or_default(), + settings: settings.as_deref().unwrap_or_default(), + }, + ) + .map_err(Error::InvalidSubtitle)?; + } + b"vtta" => { + let value = std::str::from_utf8(&body).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?; + if !value.is_empty() { + text::comments(&mut output, value).map_err(Error::InvalidSubtitle)?; } - _ => return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()), } + _ => return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()), } - Ok(output.into()) } + Ok(output.into()) } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 527ad033785..0d20178d039 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -22,6 +22,7 @@ mod source; #[cfg(test)] pub(crate) mod test_util; mod text; +mod ttml; pub mod flv; pub mod fmp4; diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs new file mode 100644 index 00000000000..c6ea7b9aaf5 --- /dev/null +++ b/rs/moq-mux/src/container/ttml.rs @@ -0,0 +1,600 @@ +//! TTML documents moved from a source's media timeline onto the catalog clock. +//! +//! Only well-formed XML in the timing profile IMSC1 admits is supported: a `tt` root in the TTML +//! namespace, no DTD, `ttp:timeBase="media"`, `par` time containers, and clock or offset time +//! expressions without sub-frames. Anything else is refused rather than relabelled. + +use moq_net::Timestamp; +use std::ops::Range; + +/// The TTML namespace, which a sample entry must list and a document's root must use. +pub(super) const NAMESPACE: &str = "http://www.w3.org/ns/ttml"; +const PARAMETER: &str = "http://www.w3.org/ns/ttml#parameter"; + +const NANOS: u128 = 1_000_000_000; + +/// Rewrite every time expression in a TTML `document` onto the catalog clock. +/// +/// The document's times are on the source track's media timeline (ISO/IEC 14496-30), where the +/// sample starts at `source`. The sample spans `sample` on the catalog clock, and the document only +/// presents within it, so every time is also clipped to it: the payload then renders on its own. +pub(super) fn rewrite(document: &[u8], source: Timestamp, sample: Range) -> Result { + let document = std::str::from_utf8(document).map_err(|_| "invalid TTML UTF-8")?; + // A strict parse: the output keeps every source byte it does not retime, so it is only as + // well-formed as the input. DTDs stay refused, since their entities and defaults could change + // what the attributes say. + let tree = roxmltree::Document::parse(document).map_err(|_| "malformed TTML document")?; + // The text was decoded as UTF-8, whatever the (already validated) declaration says. + let declaration = document + .trim_start_matches('\u{feff}') + .strip_prefix("")) + .map_or("", |(declaration, _)| declaration); + if let Some((_, encoding)) = declaration.split_once("encoding") { + let encoding = encoding.trim_start().trim_start_matches('=').trim_start(); + let quote = encoding.chars().next().unwrap_or('"'); + let name = encoding[quote.len_utf8()..].split(quote).next().unwrap_or_default(); + if !name.eq_ignore_ascii_case("UTF-8") { + return Err("TTML document is not UTF-8"); + } + } + let root = tree.root_element(); + if root.tag_name().namespace() != Some(NAMESPACE) || root.tag_name().name() != "tt" { + return Err("TTML root is not a tt element in the TTML namespace"); + } + let rates = Rates::from_root(root)?; + let clock = Clock { + offset: sample.start.as_nanos() as i128 - source.as_nanos() as i128, + start: sample.start.as_nanos(), + end: sample.end.as_nanos(), + }; + let mut edit = Edit { + document, + edits: Vec::new(), + }; + // Each element's scope, by node id, so a child finds its parent's; the root's parent begins at + // zero on both clocks. + let mut scopes = vec![Scope::default(); tree.descendants().count()]; + for element in root.descendants().filter(|node| node.is_element()) { + let parent = element + .parent_element() + .map_or(Scope::default(), |parent| scopes[parent.id().get_usize()]); + scopes[element.id().get_usize()] = if element.tag_name().namespace() == Some(NAMESPACE) { + timing(element, &rates, &clock, parent, &mut edit)? + } else { + // Foreign elements are not timed; their content inherits the enclosing begin. + parent + }; + } + Ok(edit.apply()) +} + +/// Splices replacement text into the original document, keeping every other byte verbatim. +struct Edit<'a> { + document: &'a str, + edits: Vec<(Range, String)>, +} + +impl Edit<'_> { + fn replace(&mut self, value: Range, nanos: u128) { + self.edits.push((value, clock_time(nanos))); + } + + /// Add an attribute right after the element's name. + fn insert(&mut self, element: roxmltree::Node, name: &str, nanos: u128) { + let start = element.range().start; + let tag = &self.document[start..]; + let at = start + + tag + .find(|c: char| c.is_ascii_whitespace() || c == '/' || c == '>') + .unwrap_or(tag.len()); + self.edits.push((at..at, format!(" {name}=\"{}\"", clock_time(nanos)))); + } + + fn apply(mut self) -> String { + self.edits.sort_by_key(|(range, _)| range.start); + let mut output = String::with_capacity(self.document.len()); + let mut copied = 0; + for (range, value) in self.edits { + output.push_str(&self.document[copied..range.start]); + output.push_str(&value); + copied = range.end; + } + output.push_str(&self.document[copied..]); + output + } +} + +/// An element's begin on both timelines, which its children's times are relative to. +#[derive(Clone, Copy, Default)] +struct Scope { + source: u128, + mapped: u128, + /// Where a child without its own begin starts on the catalog clock. Only the body differs from + /// `mapped`: it begins at the frame timestamp even when the source started it later. + implicit: u128, +} + +/// Maps a source media time onto the catalog clock, clipped to the sample. +struct Clock { + offset: i128, + start: u128, + end: u128, +} + +impl Clock { + /// Monotonic, so every relative time it rewrites stays non-negative. + fn map(&self, source: u128) -> u128 { + // A time past i128 is far past the sample, so saturating keeps the map monotonic. + i128::try_from(source) + .unwrap_or(i128::MAX) + .saturating_add(self.offset) + .clamp(self.start as i128, self.end as i128) as u128 + } +} + +/// Rewrite one TTML element's `begin`, `end`, and `dur`, returning the scope for its children. +/// +/// Each is resolved to an absolute source time, mapped, and written back relative to its own +/// reference: `begin` and `end` to the parent's begin, `dur` to the element's begin. The body always +/// begins at the frame timestamp and gains an end, so even untimed content stays inside the sample; +/// a later source begin moves onto its children instead. +fn timing( + element: roxmltree::Node, + rates: &Rates, + clock: &Clock, + parent: Scope, + edit: &mut Edit, +) -> Result { + let mut begin = None; + let mut end = None; + let mut dur = None; + for attribute in element.attributes().filter(|attribute| attribute.namespace().is_none()) { + // The raw range is spliced in place; the value, which may differ by character references, is + // what gets parsed. + let (raw, value) = (attribute.range_value(), attribute.value()); + match attribute.name() { + "begin" => begin = Some((raw, value)), + "end" => end = Some((raw, value)), + "dur" => dur = Some((raw, value)), + // `seq` would make each child's begin relative to its previous sibling's end. + "timeContainer" if value != "par" => return Err("unsupported TTML time container"), + _ => {} + } + } + // Timing on the root is not TTML, and would give the body a parent begin past its frame. + if element.tag_name().name() == "tt" && (begin.is_some() || end.is_some() || dur.is_some()) { + return Err("timing on the TTML root element"); + } + let body = element.tag_name().name() == "body"; + + let offset = begin.as_ref().map(|(_, value)| rates.time(value)).transpose()?; + let source = parent + .source + .checked_add(offset.unwrap_or_default()) + .ok_or("TTML time overflow")?; + let (mapped, implicit) = match (offset, body) { + (_, true) => (clock.start, clock.map(source)), + (None, false) => (parent.implicit, parent.implicit), + (Some(_), false) => (clock.map(source), clock.map(source)), + }; + // Only a body nested in timed content can begin before its parent. + let relative = mapped + .checked_sub(parent.mapped) + .ok_or("TTML body inside timed content")?; + match (begin, offset) { + (Some((raw, _)), Some(offset)) if offset != relative => edit.replace(raw, relative), + (None, _) if body || relative != 0 => edit.insert(element, "begin", relative), + _ => {} + } + + // A scope's mapped begin never exceeds the mapping of its source begin, so neither subtraction + // below underflows, even when an element ends before it begins. + let ends = end.is_some() || dur.is_some(); + if let Some((raw, value)) = end { + let offset = rates.time(value)?; + let at = parent.source.checked_add(offset).ok_or("TTML time overflow")?; + let relative = clock.map(at) - parent.mapped; + if relative != offset { + edit.replace(raw, relative); + } + } + if let Some((raw, value)) = dur { + let offset = rates.time(value)?; + let at = source.checked_add(offset).ok_or("TTML time overflow")?; + let relative = clock.map(at) - mapped; + if relative != offset { + edit.replace(raw, relative); + } + } + if body && !ends { + edit.insert(element, "end", clock.end - parent.mapped); + } + + Ok(Scope { + source, + mapped, + implicit, + }) +} + +/// The root's timing parameters, which give frame and tick expressions their length. +struct Rates { + /// The frame rate, when `ttp:frameRate` declares it. + frame: Option, + /// Ticks per second, when `ttp:tickRate` declares it. + tick: Option, +} + +impl Rates { + fn from_root(root: roxmltree::Node) -> Result { + let mut frame = None; + let mut multiplier = None; + let mut tick = None; + for attribute in root + .attributes() + .filter(|attribute| attribute.namespace() == Some(PARAMETER)) + { + let value = attribute.value(); + match attribute.name() { + "timeBase" if value != "media" => return Err("unsupported TTML time base"), + "frameRate" => frame = Some(positive(value)?), + "frameRateMultiplier" => { + let mut parts = value.split_whitespace(); + let (Some(num), Some(den), None) = (parts.next(), parts.next(), parts.next()) else { + return Err("invalid TTML frame rate multiplier"); + }; + multiplier = Some((positive(num)?, positive(den)?)); + } + "tickRate" => tick = Some(positive(value)?), + _ => {} + } + } + let frame = match (frame, multiplier) { + (Some(nominal), multiplier) => { + let (num, den) = multiplier.unwrap_or((1, 1)); + Some(FrameRate { + nominal, + num: nominal.checked_mul(num).ok_or("TTML time overflow")?, + den, + }) + } + (None, Some(_)) => return Err("TTML frame rate multiplier without a frame rate"), + (None, None) => None, + }; + Ok(Self { frame, tick }) + } + + /// Parse a time expression into nanoseconds, rounded to the nearest. + /// + /// IMSC1 requires the rate a frame or tick expression depends on to be declared, so the TTML + /// defaults are not assumed. + fn time(&self, value: &str) -> Result { + const INVALID: &str = "invalid TTML time expression"; + if value.contains(':') { + let mut parts = value.split(':'); + let (Some(hours), Some(minutes), Some(seconds)) = (parts.next(), parts.next(), parts.next()) else { + return Err(INVALID); + }; + let frames = parts.next(); + if parts.next().is_some() || hours.len() < 2 || minutes.len() != 2 { + return Err(INVALID); + } + let (whole, fraction) = seconds.split_once('.').map_or((seconds, None), |(a, b)| (a, Some(b))); + if whole.len() != 2 || (frames.is_some() && fraction.is_some()) { + return Err(INVALID); + } + let hours = integer(hours)?; + let minutes = integer(minutes)?; + let seconds = integer(whole)?; + if minutes >= 60 || seconds >= 60 { + return Err(INVALID); + } + let whole = hours + .checked_mul(3600) + .and_then(|value| value.checked_add(minutes * 60 + seconds)) + .and_then(|value| value.checked_mul(NANOS)) + .ok_or("TTML time overflow")?; + let part = match (fraction, frames) { + (Some(fraction), None) => decimal("0", Some(fraction), NANOS, 1)?, + (None, Some(frames)) => { + if frames.len() < 2 || frames.contains('.') { + return Err("unsupported TTML sub-frame time expression"); + } + let rate = self.frame.as_ref().ok_or("TTML frame time without ttp:frameRate")?; + let frames = integer(frames)?; + // The frame field counts nominal frames, whatever the multiplier. + if frames >= rate.nominal { + return Err(INVALID); + } + let unit = NANOS.checked_mul(rate.den).ok_or("TTML time overflow")?; + decimal_ratio(frames, 0, unit, rate.num)? + } + _ => 0, + }; + return whole.checked_add(part).ok_or("TTML time overflow"); + } + + let split = value.find(|c: char| !c.is_ascii_digit() && c != '.').ok_or(INVALID)?; + let (count, metric) = value.split_at(split); + let (whole, fraction) = count.split_once('.').map_or((count, None), |(a, b)| (a, Some(b))); + let (num, den) = match metric { + "h" => (3600 * NANOS, 1), + "m" => (60 * NANOS, 1), + "s" => (NANOS, 1), + "ms" => (NANOS / 1000, 1), + "f" => { + let rate = self.frame.as_ref().ok_or("TTML frame time without ttp:frameRate")?; + (NANOS.checked_mul(rate.den).ok_or("TTML time overflow")?, rate.num) + } + "t" => (NANOS, self.tick.ok_or("TTML tick time without ttp:tickRate")?), + _ => return Err(INVALID), + }; + decimal(whole, fraction, num, den) + } +} + +/// `ttp:frameRate` and the effective rate its multiplier gives, `num / den` frames per second. +struct FrameRate { + nominal: u128, + num: u128, + den: u128, +} + +/// `whole.fraction` times `num / den`, rounded to the nearest. +fn decimal(whole: &str, fraction: Option<&str>, num: u128, den: u128) -> Result { + let (places, fraction) = match fraction { + Some(fraction) => (fraction.len() as u32, integer(fraction)?), + None => (0, 0), + }; + let mantissa = 10u128 + .checked_pow(places) + .and_then(|scale| integer(whole).ok()?.checked_mul(scale)) + .and_then(|value| value.checked_add(fraction)) + .ok_or("invalid TTML time expression")?; + decimal_ratio(mantissa, places, num, den) +} + +/// `mantissa / 10^places` times `num / den`, rounded to the nearest. +fn decimal_ratio(mantissa: u128, places: u32, num: u128, den: u128) -> Result { + let den = 10u128 + .checked_pow(places) + .and_then(|scale| scale.checked_mul(den)) + .ok_or("TTML time overflow")?; + let product = mantissa.checked_mul(num).ok_or("TTML time overflow")?; + Ok(product / den + u128::from(product % den >= den - den / 2)) +} + +fn integer(value: &str) -> Result { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err("invalid TTML time expression"); + } + value.parse().map_err(|_| "TTML time overflow") +} + +fn positive(value: &str) -> Result { + match integer(value) { + Ok(0) | Err(_) => Err("invalid TTML timing parameter"), + Ok(value) => Ok(value), + } +} + +/// A TTML clock-time expression, `hh:mm:ss` with only the fraction it needs. +fn clock_time(nanos: u128) -> String { + let seconds = nanos / NANOS; + let mut output = format!("{:02}:{:02}:{:02}", seconds / 3600, seconds / 60 % 60, seconds % 60); + let fraction = nanos % NANOS; + if fraction != 0 { + output.push_str(format!(".{fraction:09}").trim_end_matches('0')); + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secs(value: u64) -> Timestamp { + Timestamp::from_secs(value).unwrap() + } + + const ROOT: &str = r#" +{ROOT} ttp:frameRate="25" ttp:tickRate="10000000"> + +

Hello world

Clipped

"# + ); + let output = rewrite(document.as_bytes(), secs(100), secs(10)..secs(14)).unwrap(); + assert_eq!( + output, + format!( + r#" +{ROOT} ttp:frameRate="25" ttp:tickRate="10000000"> + +

Hello world

Clipped

"# + ) + ); + } + + /// The body begins at the frame timestamp; a later source begin moves onto its untimed children. + #[test] + fn body_begins_at_the_frame_and_clips_to_it() { + let document = format!(r#"{ROOT}>

Hi

"#); + let output = rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(); + assert_eq!( + output, + format!(r#"{ROOT}>

Hi

"#) + ); + // Character references are parsed by value, and an element may end before it begins. + let document = format!(r#"{ROOT}>

Hi

"#); + assert_eq!( + rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + format!( + r#"{ROOT}>

Hi

"# + ) + ); + // Predefined entities and character references resolve without a DTD. + let entities = format!(r#"{ROOT}>

< &

"#); + assert_eq!( + rewrite(entities.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + entities + ); + // A time past i128 clips to the sample's end rather than wrapping before its start. + let far = format!( + r#"{ROOT}>

"#, + i128::MAX as u128 / NANOS + 1 + ); + assert!( + rewrite(far.as_bytes(), secs(1), secs(1)..secs(3)) + .unwrap() + .contains(r#"

"#) + ); + let declared = format!(r#"{entities}"#); + assert_eq!( + rewrite(declared.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + declared + ); + // A byte order mark is not text outside the root. + let marked = format!("\u{feff}{document}"); + assert_eq!( + rewrite(marked.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + format!( + "\u{feff}{}", + rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap() + ) + ); + // An empty document clears the captions. + let empty = format!("{ROOT}/>"); + assert_eq!(rewrite(empty.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), empty); + } + + #[test] + fn time_expressions() { + let rates = Rates { + frame: Some(FrameRate { + nominal: 30, + num: 30_000, + den: 1001, + }), + tick: Some(90_000), + }; + for (value, nanos) in [ + ("1.5h", 5_400 * NANOS), + ("2m", 120 * NANOS), + ("0.25s", 250_000_000), + ("1500ms", 1_500_000_000), + ("90000t", NANOS), + ("1t", 11_111), + ("30f", 1_001_000_000), + ("00:00:01.25", 1_250_000_000), + ("100:00:00", 360_000 * NANOS), + ("00:00:01:15", 1_500_500_000), + ] { + assert_eq!(rates.time(value), Ok(nanos), "{value}"); + } + for value in [ + "", + "1", + "s", + ".5s", + "1.s", + "1x", + "-1s", + " 1s", + "0:00:01", + "00:0:01", + "00:60:00", + "00:00:60", + "00:00:01:30", + "00:00:01:01.1", + "00:00:01.5:01", + "00:00:01:1", + ] { + assert!(rates.time(value).is_err(), "{value}"); + } + let huge = Rates { + frame: Some(FrameRate { + nominal: 100, + num: u128::MAX, + den: u128::MAX, + }), + tick: None, + }; + assert!(huge.time("00:00:00:99").is_err()); + // The frame field is bounded by the nominal rate, not the effective one. + let halved = Rates { + frame: Some(FrameRate { + nominal: 30, + num: 30, + den: 2, + }), + tick: None, + }; + assert_eq!(halved.time("00:00:00:20"), Ok(1_333_333_333)); + let doubled = Rates { + frame: Some(FrameRate { + nominal: 30, + num: 60, + den: 1, + }), + tick: None, + }; + assert!(doubled.time("00:00:00:45").is_err()); + let unset = Rates { + frame: None, + tick: None, + }; + assert!(unset.time("1f").is_err()); + assert!(unset.time("00:00:00:01").is_err()); + assert!(unset.time("1t").is_err()); + } + + #[test] + fn clock_time_trims_the_fraction() { + assert_eq!(clock_time(0), "00:00:00"); + assert_eq!(clock_time(3_723_500_000_000), "01:02:03.5"); + assert_eq!(clock_time(360_000 * NANOS + 1), "100:00:00.000000001"); + } + + #[test] + fn documents_outside_the_profile_are_refused() { + for document in [ + String::new(), + "".into(), + r#""#.into(), + r#"

"#.into(), + format!(r#"{ROOT} ttp:timeBase="smpte"/>"#), + format!(r#"{ROOT} ttp:frameRateMultiplier="1000 1001"/>"#), + format!(r#"{ROOT} ttp:frameRate="0"/>"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT} begin="1s">"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}/>{ROOT}/>"#), + format!(r#"{ROOT}/>trailing"#), + format!("{ROOT}/>"), + format!(r#"{ROOT}>

a b

"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}/>"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}/>"#), + format!(r#"{ROOT}/>"#), + ] { + assert!( + rewrite(document.as_bytes(), secs(1), secs(1)..secs(2)).is_err(), + "{document}" + ); + } + assert!(rewrite(b"\xff", secs(1), secs(1)..secs(2)).is_err()); + } +}