From 4f31d5da61eba45ca7f98ac51041edef7d4ee2db Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:16:55 -0700 Subject: [PATCH 01/10] quest: claim captions-ttml From 7472a6fa8bf43eb7c275c6e44c16a241d4a331b5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:30:38 -0700 Subject: [PATCH 02/10] feat(mux): import fMP4 TTML captions onto the catalog clock Import `stpp` tracks that list the TTML namespace as `ttml` text renditions. Each sample's TTML document has every begin/end/dur resolved on the source media timeline, shifted onto the catalog clock, clipped to the sample, and written back relative to its parent, so nested timing survives a shifted clock. Documents outside the IMSC timing profile are refused; stpp entries naming another namespace or carrying image subsamples skip their track. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + doc/lib/rs/moq-mux.md | 2 +- quest/m2/README.md | 1 - quest/m2/captions-ttml.md | 24 - rs/moq-mux/Cargo.toml | 1 + rs/moq-mux/src/container/fmp4/import.rs | 28 +- rs/moq-mux/src/container/fmp4/import_test.rs | 115 ++++ rs/moq-mux/src/container/fmp4/text.rs | 177 ++++--- rs/moq-mux/src/container/mod.rs | 1 + rs/moq-mux/src/container/ttml.rs | 520 +++++++++++++++++++ 10 files changed, 762 insertions(+), 108 deletions(-) delete mode 100644 quest/m2/captions-ttml.md create mode 100644 rs/moq-mux/src/container/ttml.rs diff --git a/Cargo.lock b/Cargo.lock index 4387ae5f067..0f40fd61943 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4642,6 +4642,7 @@ dependencies = [ "mp4-atom", "mpeg2ts", "num_enum", + "quick-xml 0.42.0", "scuffle-av1", "scuffle-h265", "serde", diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 2771d1485a5..3983ed55a40 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -13,7 +13,7 @@ back. This is what `moq import` / `export` and the gateways are built on. | Format | Import | Export | Notes | | --- | --- | --- | --- | -| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text, preserving source language and label when present. | +| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text and `stpp` TTML (IMSC timing profile) as `ttml` documents on the broadcast clock, preserving source language and label when present; other subtitle codecs are skipped. | | MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 and subtitle PIDs as tracks; service tables round-trip. | | FLV / RTMP | yes | yes | Legacy H.264 + AAC + MP3, plus enhanced-RTMP HEVC, AV1, VP9, Opus, AC-3, E-AC-3, and multitrack. | | Matroska / WebM | yes | yes | Timed SRT and WebVTT subtitles import as WebVTT text renditions, preserving source language and label when present; other subtitle codecs are skipped. | diff --git a/quest/m2/README.md b/quest/m2/README.md index a305cea0aaa..25cf00badc8 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -77,7 +77,6 @@ ranked just below deterministic TS legs, which it requires. - [Android capture](/quest/m2/mobile-capture-android.md) - Rust captures through NDK/JNI on Android, reusing the existing codecs and setting the catalog rotation - [Time stretch](/quest/m2/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent - [Native audio quality](/quest/m2/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device -- [TTML caption import](/quest/m2/captions-ttml.md) - fMP4 `stpp` tracks within a supported TTML/IMSC profile import as `ttml` renditions on the shared clock instead of being skipped - [Bindings size profile](/quest/m2/ffi-size-profile.md) - a benchmark decides whether the moq-ffi builds ship at opt-level "s", which halves the dylib - [Zero-copy byte returns](/quest/m2/ffi-bytes-return.md) - returned payloads reach the host without a copy, if the payload numbers justify it - [Namespaced external types](/quest/m2/uniffi-namespaces.md) - the UniFFI fork lets two crates export the same type name, reopening the per-layer crate split diff --git a/quest/m2/captions-ttml.md b/quest/m2/captions-ttml.md deleted file mode 100644 index ebdbec48489..00000000000 --- a/quest/m2/captions-ttml.md +++ /dev/null @@ -1,24 +0,0 @@ -# [M] TTML caption import - -## Goal - -Importing an fMP4 `stpp` track within a supported TTML/IMSC profile yields a `ttml` text -rendition whose cues stay on the shared media clock. Today those XML tracks are -skipped with a warning while SRT and WebVTT import as `vtt`. - -## Plan - -Copying XML bytes is not enough. MP4 `stpp` names an XML namespace, which is not -necessarily TTML, and the importer shifts the source clock, so every TTML time -expression must be rewritten onto the catalog clock. - -- Support a bounded TTML/IMSC timing profile: validate the namespace and refuse - anything outside the profile instead of relabelling arbitrary XML as TTML. -- Convert every time expression, including nested `begin`/`end`/`dur` and - `ttp:timeBase` variants the profile admits, after the clock shift. -- Keep skipping XML tracks the profile does not cover, one track at a time. - -Matroska has no registered TTML codec ID (the old `S_TEXT/TTML` claim was -removed), so confirm a real mapping before adding one. - -Cover a namespace mismatch, nested timing, and a shifted source clock. diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 2118ea0c732..278a4f74a1d 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -28,6 +28,7 @@ moq-msf = { workspace = true } moq-net = { workspace = true } mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" +quick-xml = "0.42.0" num_enum = { workspace = true } scuffle-av1 = { version = "0.1.4" } scuffle-h265 = { version = "0.2.2" } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index b066d4037f8..d1fe0a9f13c 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -27,7 +27,8 @@ use crate::catalog::Estimator; /// - Opus /// - FLAC /// -/// **Text:** WebVTT (`wvtt`), reconstructed as timed text; unsupported subtitle codecs are skipped. +/// **Text:** WebVTT (`wvtt`), reconstructed as timed text, and TTML (`stpp`) documents moved onto the +/// catalog clock; unsupported subtitle codecs, including other XML namespaces, are skipped. pub struct Import { /// The broadcast being produced broadcast: moq_net::broadcast::Producer, @@ -370,15 +371,22 @@ impl Import { } fn init_text(&mut self, trak: &Trak, moov: &Moov, reserved: &crate::catalog::Reserved) -> Result { - let [mp4_atom::Codec::Wvtt(codec)] = trak.mdia.minf.stbl.stsd.codecs.as_slice() else { - return Ok(false); + let (format, mut config) = match trak.mdia.minf.stbl.stsd.codecs.as_slice() { + [mp4_atom::Codec::Wvtt(codec)] => { + if codec.plaintext.data_reference_index != 1 { + return Err(Error::InvalidSubtitle("external subtitle data reference").into()); + } + let header = crate::container::text::header(&codec.config.config).map_err(Error::InvalidSubtitle)?; + let mut config = hang::catalog::TextConfig::new(hang::catalog::TextFormat::Vtt); + config.role = crate::container::text::role(&header); + (super::text::Format::Vtt { header }, config) + } + [mp4_atom::Codec::Unknown(kind, body)] if kind.as_ref() == b"stpp" && super::text::stpp(body)? => ( + super::text::Format::Ttml, + hang::catalog::TextConfig::new(hang::catalog::TextFormat::Ttml), + ), + _ => return Ok(false), }; - if codec.plaintext.data_reference_index != 1 { - return Err(Error::InvalidSubtitle("external subtitle data reference").into()); - } - let header = crate::container::text::header(&codec.config.config).map_err(Error::InvalidSubtitle)?; - let mut config = hang::catalog::TextConfig::new(hang::catalog::TextFormat::Vtt); - config.role = crate::container::text::role(&header); config.lang = Some(trak.mdia.mdhd.language.clone()); // The picker label is the HandlerBox name; `vlab` is an opaque source identifier. config.label = (!trak.mdia.hdlr.name.is_empty()).then(|| trak.mdia.hdlr.name.clone()); @@ -392,7 +400,7 @@ impl Import { trak.tkhd.track_id, super::text::Track { producer, - header, + format, defaults: sample::Defaults::from_moov(moov, trak.tkhd.track_id), timescale: moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?, end_decode_time: None, diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 2f7b0ee0876..a21a10e59b1 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2087,3 +2087,118 @@ fn malformed_webvtt_origins_and_comments_are_refused() { assert!(importer.decode(&webvtt_input(payload, 1250)).is_err()); } } + +/// An `stpp` init segment. mp4-atom cannot encode an `stpp` entry, so encode a `metx` entry, whose +/// strings line up with `stpp`'s namespace, schema location and auxiliary MIME types, and rename it. +fn ttml_init(namespace: &str, mime: &str) -> Vec { + let (_, moov) = decode_init(&webvtt_init("WEBVTT")); + let mut trak = moov.trak[0].clone(); + trak.mdia.hdlr.handler = b"subt".into(); + trak.mdia.minf.stbl.stsd.codecs = vec![mp4_atom::Codec::Metx(mp4_atom::Metx { + metadata: mp4_atom::MetaData { + data_reference_index: 1, + }, + content_encoding: namespace.into(), + namespace: String::new(), + schema_location: mime.into(), + btrt: None, + })]; + let mut init = audio_init(trak); + let at = init.windows(4).position(|kind| kind == b"metx").unwrap(); + init[at..at + 4].copy_from_slice(b"stpp"); + init +} + +const TTML: &str = r#"

Hello

"#; + +/// Import one TTML sample lasting 4 s from source time 100 s, returning its rendition and frame. +async fn imported_ttml( + timebase: crate::catalog::Timebase, + broadcast: moq_net::broadcast::Producer, + catalog: &crate::catalog::Producer, + consumer: &moq_net::broadcast::Consumer, +) -> (hang::catalog::TextConfig, crate::container::Frame) { + let mut input = ttml_init("http://www.w3.org/ns/ttml", ""); + input.extend(webvtt_fragment(100_000, &[TTML.as_bytes().to_vec()], 4000)); + let mut importer = super::Import::new(broadcast, timebase.reserve()); + importer.decode(&input).unwrap(); + importer.finish().unwrap(); + let snapshot = catalog.snapshot(); + let (name, config) = snapshot.text.renditions.iter().next().unwrap(); + let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); + let mut reader = crate::container::Consumer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + (config.clone(), reader.read_frame().await.unwrap().unwrap()) +} + +#[tokio::test(start_paused = true)] +async fn ttml_samples_become_timed_text_with_source_metadata() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let (config, frame) = imported_ttml(catalog.timebase(), broadcast, &catalog, &consumer).await; + assert_eq!(config.format, hang::catalog::TextFormat::Ttml); + assert_eq!(config.lang.as_deref(), Some("eng")); + assert_eq!(config.label.as_deref(), Some("English captions")); + assert_eq!(frame.timestamp.as_micros(), 100_000_000); + assert_eq!( + &frame.payload[..], + br#"

Hello

"# + ); +} + +/// The source joins a catalog clock another input already fixed, so its TTML times move with it. +#[tokio::test(start_paused = true)] +async fn ttml_follows_a_shifted_source_clock() { + let wall = std::time::SystemTime::now(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + catalog + .timebase() + .place(moq_net::Timestamp::from_secs(10).unwrap(), wall) + .unwrap(); + let timebase = catalog.timebase(); + timebase + .place(moq_net::Timestamp::from_secs(100).unwrap(), wall) + .unwrap(); + let (_, frame) = imported_ttml(timebase, broadcast, &catalog, &consumer).await; + assert_eq!(frame.timestamp.as_micros(), 10_000_000); + assert_eq!( + &frame.payload[..], + br#"

Hello

"# + ); +} + +/// An `stpp` entry naming another XML namespace, or carrying image subsamples, skips only its track. +#[test] +fn unsupported_stpp_tracks_are_skipped() { + for (namespace, mime) in [("urn:example:not-ttml", ""), ("http://www.w3.org/ns/ttml", "image/png")] { + let mut input = ttml_init(namespace, mime); + input.extend(webvtt_fragment(0, &[TTML.as_bytes().to_vec()], 1000)); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = super::Import::new(broadcast, catalog.reserve()); + importer.decode(&input).unwrap(); + importer.finish().unwrap(); + assert!(catalog.snapshot().text.renditions.is_empty(), "{namespace} {mime}"); + } +} + +/// A document the profile refuses fails the import instead of publishing as TTML. +#[test] +fn ttml_documents_outside_the_profile_are_refused() { + for document in [ + r#""#, + r#""#, + ] { + let mut input = ttml_init("http://www.w3.org/ns/ttml", ""); + input.extend(webvtt_fragment(0, &[document.as_bytes().to_vec()], 1000)); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = super::Import::new(broadcast, catalog.reserve()); + assert!(importer.decode(&input).is_err(), "{document}"); + } +} diff --git a/rs/moq-mux/src/container/fmp4/text.rs b/rs/moq-mux/src/container/fmp4/text.rs index 20396bb5ee4..c6064aaa2c0 100644 --- a/rs/moq-mux/src/container/fmp4/text.rs +++ b/rs/moq-mux/src/container/fmp4/text.rs @@ -3,10 +3,38 @@ use mp4_atom::{Any, DecodeMaybe}; use super::{Error, sample}; use crate::container::text::{self, Cue}; +use crate::container::ttml; + +/// How a track's samples are serialized. +pub(super) enum Format { + /// WebVTT (`wvtt`) cue boxes, prefixed with the sample entry's header. + Vtt { header: String }, + /// One TTML document per sample (`stpp`). + Ttml, +} + +/// Whether an `stpp` sample entry body (ISO/IEC 14496-12 XMLSubtitleSampleEntry) carries TTML text. +/// +/// Image subsamples (auxiliary MIME types) and namespaces other than TTML are unsupported. +pub(super) fn stpp(body: &[u8]) -> Result { + let (header, strings) = body + .split_at_checked(8) + .ok_or(Error::InvalidSubtitle("truncated stpp sample entry"))?; + if header[6..] != [0, 1] { + return Err(Error::InvalidSubtitle("external subtitle data reference")); + } + // namespace, schema_location, auxiliary_mime_types; encoders may omit the trailing ones. + let mut strings = strings.split(|&byte| byte == 0); + let namespace = strings.next().unwrap_or_default(); + let namespace = std::str::from_utf8(namespace).map_err(|_| Error::InvalidSubtitle("invalid stpp namespace"))?; + let _schema_location = strings.next(); + let mime = strings.next().unwrap_or_default(); + Ok(namespace.split_whitespace().any(|name| name == ttml::NAMESPACE) && mime.is_empty()) +} pub(super) struct Track { pub producer: crate::container::Producer, - pub header: String, + pub format: Format, pub defaults: sample::Defaults, pub timescale: Timescale, /// The exclusive decode end of the previous fragment; the next one may not start before it. @@ -30,7 +58,7 @@ impl Track { let traf = input.traf; let mut dts = traf.tfdt.as_ref().ok_or(Error::MissingTfdt)?.base_media_decode_time; if self.end_decode_time.is_some_and(|end| dts < end) { - return Err(Error::InvalidSubtitle("WebVTT fragment overlaps the previous one").into()); + return Err(Error::InvalidSubtitle("subtitle fragment overlaps the previous one").into()); } if traf.trun.is_empty() { return Err(Error::MissingTrun.into()); @@ -61,14 +89,20 @@ impl Track { let pts = dts .checked_add_signed(entry.cts.unwrap_or_default() as i64) .ok_or(Error::PtsOverflow)?; - let timestamp = timebase.shift(Timestamp::new(pts, self.timescale)?)?; + let source = Timestamp::new(pts, self.timescale)?; + let timestamp = timebase.shift(source)?; let duration = Timestamp::new(duration as u64, self.timescale)?; let end = timestamp.checked_add(duration)?; let next = offset .checked_add(sample.size as usize) .ok_or(Error::InvalidDataOffset)?; let data = input.data.get(offset..next).ok_or(Error::InvalidDataOffset)?; - let payload = self.sample(data, timestamp, end)?; + let payload = match &self.format { + Format::Vtt { header } => vtt(header, data, timestamp, end)?, + Format::Ttml => ttml::rewrite(data, source, timestamp..end) + .map_err(Error::InvalidSubtitle)? + .into(), + }; self.producer.write(crate::container::Frame { timestamp, duration: Some(duration), @@ -83,79 +117,78 @@ impl Track { self.end_decode_time = Some(dts); Ok(()) } +} - fn sample(&self, mut data: &[u8], start: Timestamp, end: Timestamp) -> crate::Result { - if data.is_empty() { - return Err(Error::InvalidSubtitle("empty WebVTT sample").into()); - } - let mut output = self.header.clone(); - while !data.is_empty() { - let atom = Any::decode_maybe(&mut data) - .map_err(Error::from)? - .ok_or(Error::InvalidSubtitle("truncated WebVTT sample box"))?; - let Any::Unknown(kind, body) = atom else { - return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()); - }; - match kind.as_ref() { - b"vtte" if body.is_empty() => {} - b"vttc" => { - let mut body = body.as_slice(); - let mut payload = None; - let mut id = None; - let mut settings = None; - let mut origin = None; - while !body.is_empty() { - let atom = Any::decode_maybe(&mut body) - .map_err(Error::from)? - .ok_or(Error::InvalidSubtitle("truncated WebVTT cue box"))?; - let Any::Unknown(kind, value) = atom else { - return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()); - }; - let field = match kind.as_ref() { - b"payl" => &mut payload, - b"iden" => &mut id, - b"sttg" => &mut settings, - b"ctim" => &mut origin, - _ => return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()), - }; - if field.is_some() { - return Err(Error::InvalidSubtitle("duplicate WebVTT cue box").into()); - } - *field = - Some(String::from_utf8(value).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?); - } - let payload = payload.ok_or(Error::InvalidSubtitle("missing WebVTT cue payload"))?; - // Inline timestamps sit on the source timeline, which only `ctim` anchors. - let payload = if let Some(origin) = origin { - let origin = text::timestamp(&origin).map_err(Error::InvalidSubtitle)?; - text::rebase(&payload, start..end, origin).map_err(Error::InvalidSubtitle)? - } else if text::has_timestamp(&payload) { - return Err(Error::InvalidSubtitle("inline WebVTT timestamp has no cue origin").into()); - } else { - payload +fn vtt(header: &str, mut data: &[u8], start: Timestamp, end: Timestamp) -> crate::Result { + if data.is_empty() { + return Err(Error::InvalidSubtitle("empty WebVTT sample").into()); + } + let mut output = header.to_string(); + while !data.is_empty() { + let atom = Any::decode_maybe(&mut data) + .map_err(Error::from)? + .ok_or(Error::InvalidSubtitle("truncated WebVTT sample box"))?; + let Any::Unknown(kind, body) = atom else { + return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()); + }; + match kind.as_ref() { + b"vtte" if body.is_empty() => {} + b"vttc" => { + let mut body = body.as_slice(); + let mut payload = None; + let mut id = None; + let mut settings = None; + let mut origin = None; + while !body.is_empty() { + let atom = Any::decode_maybe(&mut body) + .map_err(Error::from)? + .ok_or(Error::InvalidSubtitle("truncated WebVTT cue box"))?; + let Any::Unknown(kind, value) = atom else { + return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()); }; - text::append( - &mut output, - Cue { - start, - end, - body: &payload, - id: id.as_deref().unwrap_or_default(), - settings: settings.as_deref().unwrap_or_default(), - }, - ) - .map_err(Error::InvalidSubtitle)?; - } - b"vtta" => { - let value = - std::str::from_utf8(&body).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?; - if !value.is_empty() { - text::comments(&mut output, value).map_err(Error::InvalidSubtitle)?; + let field = match kind.as_ref() { + b"payl" => &mut payload, + b"iden" => &mut id, + b"sttg" => &mut settings, + b"ctim" => &mut origin, + _ => return Err(Error::InvalidSubtitle("unsupported WebVTT cue box").into()), + }; + if field.is_some() { + return Err(Error::InvalidSubtitle("duplicate WebVTT cue box").into()); } + *field = + Some(String::from_utf8(value).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?); + } + let payload = payload.ok_or(Error::InvalidSubtitle("missing WebVTT cue payload"))?; + // Inline timestamps sit on the source timeline, which only `ctim` anchors. + let payload = if let Some(origin) = origin { + let origin = text::timestamp(&origin).map_err(Error::InvalidSubtitle)?; + text::rebase(&payload, start..end, origin).map_err(Error::InvalidSubtitle)? + } else if text::has_timestamp(&payload) { + return Err(Error::InvalidSubtitle("inline WebVTT timestamp has no cue origin").into()); + } else { + payload + }; + text::append( + &mut output, + Cue { + start, + end, + body: &payload, + id: id.as_deref().unwrap_or_default(), + settings: settings.as_deref().unwrap_or_default(), + }, + ) + .map_err(Error::InvalidSubtitle)?; + } + b"vtta" => { + let value = std::str::from_utf8(&body).map_err(|_| Error::InvalidSubtitle("invalid WebVTT UTF-8"))?; + if !value.is_empty() { + text::comments(&mut output, value).map_err(Error::InvalidSubtitle)?; } - _ => return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()), } + _ => return Err(Error::InvalidSubtitle("unsupported WebVTT sample box").into()), } - Ok(output.into()) } + Ok(output.into()) } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 527ad033785..0d20178d039 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -22,6 +22,7 @@ mod source; #[cfg(test)] pub(crate) mod test_util; mod text; +mod ttml; pub mod flv; pub mod fmp4; diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs new file mode 100644 index 00000000000..4c1c8e3eba3 --- /dev/null +++ b/rs/moq-mux/src/container/ttml.rs @@ -0,0 +1,520 @@ +//! TTML documents moved from a source's media timeline onto the catalog clock. +//! +//! Only the timing profile IMSC1 admits is supported: a `tt` root in the TTML namespace, +//! `ttp:timeBase="media"`, `par` time containers, and clock or offset time expressions without +//! sub-frames. Anything else is refused rather than relabelled. + +use moq_net::Timestamp; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::name::{Namespace, ResolveResult}; +use std::borrow::Cow; +use std::ops::Range; + +/// The TTML namespace, which a sample entry must list and a document's root must use. +pub(super) const NAMESPACE: &str = "http://www.w3.org/ns/ttml"; +const PARAMETER: &str = "http://www.w3.org/ns/ttml#parameter"; + +const NANOS: u128 = 1_000_000_000; + +/// Rewrite every time expression in a TTML `document` onto the catalog clock. +/// +/// The document's times are on the source track's media timeline (ISO/IEC 14496-30), where the +/// sample starts at `source`. The sample spans `sample` on the catalog clock, and the document only +/// presents within it, so every time is also clipped to it: the payload then renders on its own. +pub(super) fn rewrite(document: &[u8], source: Timestamp, sample: Range) -> Result { + let document = std::str::from_utf8(document).map_err(|_| "invalid TTML UTF-8")?; + let clock = Clock { + offset: sample.start.as_nanos() as i128 - source.as_nanos() as i128, + start: sample.start.as_nanos(), + end: sample.end.as_nanos(), + }; + let mut reader = quick_xml::NsReader::from_str(document); + let mut rates = None; + // The enclosing elements' begins; the root's is zero on both clocks. + let mut scopes = vec![Scope::default()]; + let mut edit = Edit { + document, + edits: Vec::new(), + }; + loop { + let (namespace, event) = reader.read_resolved_event().map_err(|_| "malformed TTML document")?; + let ttml = matches!(namespace, ResolveResult::Bound(Namespace(NAMESPACE))); + let (element, open) = match event { + Event::Start(element) => (element, true), + Event::Empty(element) => (element, false), + Event::End(_) => { + scopes.pop(); + continue; + } + Event::Decl(decl) => { + if let Some(encoding) = decl.encoding() { + let encoding = encoding.map_err(|_| "malformed TTML declaration")?; + if !encoding.eq_ignore_ascii_case("UTF-8") { + return Err("TTML document is not UTF-8"); + } + } + continue; + } + // A DTD can declare entities and defaults that change what the attributes say. + Event::DocType(_) => return Err("TTML document type declarations are unsupported"), + Event::Text(text) if scopes.len() == 1 && !text.trim().is_empty() => { + return Err("text outside the TTML root element"); + } + Event::Eof => break, + _ => continue, + }; + if rates.is_some() && scopes.len() == 1 { + return Err("multiple TTML root elements"); + } + let parent = *scopes.last().ok_or("malformed TTML document")?; + let rates = match &mut rates { + Some(rates) => rates, + slot => { + if !ttml || element.local_name().as_ref() != "tt" { + return Err("TTML root is not a tt element in the TTML namespace"); + } + slot.insert(Rates::from_root(&reader, &element)?) + } + }; + let scope = if ttml { + timing(&reader, &element, rates, &clock, parent, &mut edit)? + } else { + // Foreign elements are not timed; their content inherits the enclosing begin. + parent + }; + if open { + scopes.push(scope); + } + } + if rates.is_none() { + return Err("missing TTML root element"); + } + Ok(edit.apply()) +} + +/// Splices replacement text into the original document, keeping every other byte verbatim. +struct Edit<'a> { + document: &'a str, + edits: Vec<(Range, String)>, +} + +impl Edit<'_> { + /// The byte range of `part`, which borrows from the document. + fn span(&self, part: &str) -> Range { + let start = part.as_ptr() as usize - self.document.as_ptr() as usize; + debug_assert!(start + part.len() <= self.document.len()); + start..start + part.len() + } + + fn replace(&mut self, part: &str, nanos: u128) { + let range = self.span(part); + self.edits.push((range, clock_time(nanos))); + } + + fn insert(&mut self, element: &BytesStart, name: &str, nanos: u128) { + let at = self.span(element.name().0).end; + self.edits.push((at..at, format!(" {name}=\"{}\"", clock_time(nanos)))); + } + + fn apply(mut self) -> String { + self.edits.sort_by_key(|(range, _)| range.start); + let mut output = String::with_capacity(self.document.len()); + let mut copied = 0; + for (range, value) in self.edits { + output.push_str(&self.document[copied..range.start]); + output.push_str(&value); + copied = range.end; + } + output.push_str(&self.document[copied..]); + output + } +} + +/// An element's begin on both timelines, which its children's times are relative to. +#[derive(Clone, Copy, Default)] +struct Scope { + source: u128, + mapped: u128, +} + +/// Maps a source media time onto the catalog clock, clipped to the sample. +struct Clock { + offset: i128, + start: u128, + end: u128, +} + +impl Clock { + /// Monotonic, so every relative time it rewrites stays non-negative. + fn map(&self, source: u128) -> u128 { + (source as i128) + .saturating_add(self.offset) + .clamp(self.start as i128, self.end as i128) as u128 + } +} + +/// Rewrite one TTML element's `begin`, `end`, and `dur`, returning the scope for its children. +/// +/// Each is resolved to an absolute source time, mapped, and written back relative to its own +/// reference: `begin` and `end` to the parent's begin, `dur` to the element's begin. The body also +/// gains a begin and an end, so even untimed content stays inside the sample. +fn timing( + reader: &quick_xml::NsReader, + element: &BytesStart<'_>, + rates: &Rates, + clock: &Clock, + parent: Scope, + edit: &mut Edit, +) -> Result { + let mut begin = None; + let mut end = None; + let mut dur = None; + for attribute in element.attributes() { + let attribute = attribute.map_err(|_| "malformed TTML attribute")?; + let (ResolveResult::Unbound, name) = reader.resolver().resolve_attribute(attribute.key) else { + continue; + }; + // Attribute values borrow from the document, so they can be spliced in place. + let Cow::Borrowed(value) = attribute.value else { + return Err("malformed TTML attribute"); + }; + match name.as_ref() { + "begin" => begin = Some(value), + "end" => end = Some(value), + "dur" => dur = Some(value), + // `seq` would make each child's begin relative to its previous sibling's end. + "timeContainer" if value != "par" => return Err("unsupported TTML time container"), + _ => {} + } + } + let body = element.local_name().as_ref() == "body"; + + let offset = begin.map(|value| rates.time(value)).transpose()?; + let source = parent + .source + .checked_add(offset.unwrap_or_default()) + .ok_or("TTML time overflow")?; + let mapped = match (offset, body) { + (None, false) => parent.mapped, + _ => clock.map(source), + }; + let relative = mapped - parent.mapped; + match (begin, offset) { + (Some(value), Some(offset)) if offset != relative => edit.replace(value, relative), + (None, _) if body => edit.insert(element, "begin", relative), + _ => {} + } + + if let Some(value) = end { + let offset = rates.time(value)?; + let at = parent.source.checked_add(offset).ok_or("TTML time overflow")?; + let relative = clock.map(at) - parent.mapped; + if relative != offset { + edit.replace(value, relative); + } + } + if let Some(value) = dur { + let offset = rates.time(value)?; + let at = source.checked_add(offset).ok_or("TTML time overflow")?; + let relative = clock.map(at) - mapped; + if relative != offset { + edit.replace(value, relative); + } + } + if body && end.is_none() && dur.is_none() { + edit.insert(element, "end", clock.end - parent.mapped); + } + + Ok(Scope { source, mapped }) +} + +/// The root's timing parameters, which give frame and tick expressions their length. +struct Rates { + /// Frames per second as a fraction, when `ttp:frameRate` declares it. + frame: Option<(u128, u128)>, + /// Ticks per second, when `ttp:tickRate` declares it. + tick: Option, +} + +impl Rates { + fn from_root(reader: &quick_xml::NsReader, root: &BytesStart<'_>) -> Result { + let mut frame = None; + let mut multiplier = None; + let mut tick = None; + for attribute in root.attributes() { + let attribute = attribute.map_err(|_| "malformed TTML attribute")?; + let (ResolveResult::Bound(Namespace(PARAMETER)), name) = reader.resolver().resolve_attribute(attribute.key) + else { + continue; + }; + let value = attribute.value; + match name.as_ref() { + "timeBase" if value != "media" => return Err("unsupported TTML time base"), + "frameRate" => frame = Some(positive(&value)?), + "frameRateMultiplier" => { + let mut parts = value.split_whitespace(); + let (Some(num), Some(den), None) = (parts.next(), parts.next(), parts.next()) else { + return Err("invalid TTML frame rate multiplier"); + }; + multiplier = Some((positive(num)?, positive(den)?)); + } + "tickRate" => tick = Some(positive(&value)?), + _ => {} + } + } + let frame = match (frame, multiplier) { + (Some(rate), Some((num, den))) => Some((rate.checked_mul(num).ok_or("TTML time overflow")?, den)), + (Some(rate), None) => Some((rate, 1)), + (None, Some(_)) => return Err("TTML frame rate multiplier without a frame rate"), + (None, None) => None, + }; + Ok(Self { frame, tick }) + } + + /// Parse a time expression into nanoseconds, rounded to the nearest. + /// + /// IMSC1 requires the rate a frame or tick expression depends on to be declared, so the TTML + /// defaults are not assumed. + fn time(&self, value: &str) -> Result { + const INVALID: &str = "invalid TTML time expression"; + if value.contains(':') { + let mut parts = value.split(':'); + let (Some(hours), Some(minutes), Some(seconds)) = (parts.next(), parts.next(), parts.next()) else { + return Err(INVALID); + }; + let frames = parts.next(); + if parts.next().is_some() || hours.len() < 2 || minutes.len() != 2 { + return Err(INVALID); + } + let (whole, fraction) = seconds.split_once('.').map_or((seconds, None), |(a, b)| (a, Some(b))); + if whole.len() != 2 || (frames.is_some() && fraction.is_some()) { + return Err(INVALID); + } + let hours = integer(hours)?; + let minutes = integer(minutes)?; + let seconds = integer(whole)?; + if minutes >= 60 || seconds >= 60 { + return Err(INVALID); + } + let whole = hours + .checked_mul(3600) + .and_then(|value| value.checked_add(minutes * 60 + seconds)) + .and_then(|value| value.checked_mul(NANOS)) + .ok_or("TTML time overflow")?; + let part = match (fraction, frames) { + (Some(fraction), None) => decimal("0", Some(fraction), NANOS, 1)?, + (None, Some(frames)) => { + if frames.len() < 2 || frames.contains('.') { + return Err("unsupported TTML sub-frame time expression"); + } + let (rate, den) = self.frame.ok_or("TTML frame time without ttp:frameRate")?; + let frames = integer(frames)?; + if frames * den >= rate { + return Err(INVALID); + } + decimal_ratio(frames, 0, NANOS * den, rate)? + } + _ => 0, + }; + return whole.checked_add(part).ok_or("TTML time overflow"); + } + + let split = value.find(|c: char| !c.is_ascii_digit() && c != '.').ok_or(INVALID)?; + let (count, metric) = value.split_at(split); + let (whole, fraction) = count.split_once('.').map_or((count, None), |(a, b)| (a, Some(b))); + let (num, den) = match metric { + "h" => (3600 * NANOS, 1), + "m" => (60 * NANOS, 1), + "s" => (NANOS, 1), + "ms" => (NANOS / 1000, 1), + "f" => { + let (rate, den) = self.frame.ok_or("TTML frame time without ttp:frameRate")?; + (NANOS.checked_mul(den).ok_or("TTML time overflow")?, rate) + } + "t" => (NANOS, self.tick.ok_or("TTML tick time without ttp:tickRate")?), + _ => return Err(INVALID), + }; + decimal(whole, fraction, num, den) + } +} + +/// `whole.fraction` times `num / den`, rounded to the nearest. +fn decimal(whole: &str, fraction: Option<&str>, num: u128, den: u128) -> Result { + let (places, fraction) = match fraction { + Some(fraction) => (fraction.len() as u32, integer(fraction)?), + None => (0, 0), + }; + let mantissa = 10u128 + .checked_pow(places) + .and_then(|scale| integer(whole).ok()?.checked_mul(scale)) + .and_then(|value| value.checked_add(fraction)) + .ok_or("invalid TTML time expression")?; + decimal_ratio(mantissa, places, num, den) +} + +/// `mantissa / 10^places` times `num / den`, rounded to the nearest. +fn decimal_ratio(mantissa: u128, places: u32, num: u128, den: u128) -> Result { + let den = 10u128 + .checked_pow(places) + .and_then(|scale| scale.checked_mul(den)) + .ok_or("TTML time overflow")?; + let product = mantissa.checked_mul(num).ok_or("TTML time overflow")?; + Ok(product / den + u128::from(product % den >= den - den / 2)) +} + +fn integer(value: &str) -> Result { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err("invalid TTML time expression"); + } + value.parse().map_err(|_| "TTML time overflow") +} + +fn positive(value: &str) -> Result { + match integer(value) { + Ok(0) | Err(_) => Err("invalid TTML timing parameter"), + Ok(value) => Ok(value), + } +} + +/// A TTML clock-time expression, `hh:mm:ss` with only the fraction it needs. +fn clock_time(nanos: u128) -> String { + let seconds = nanos / NANOS; + let mut output = format!("{:02}:{:02}:{:02}", seconds / 3600, seconds / 60 % 60, seconds % 60); + let fraction = nanos % NANOS; + if fraction != 0 { + output.push_str(format!(".{fraction:09}").trim_end_matches('0')); + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secs(value: u64) -> Timestamp { + Timestamp::from_secs(value).unwrap() + } + + const ROOT: &str = r#" +{ROOT} ttp:frameRate="25" ttp:tickRate="10000000"> + +

Hello world

Clipped

"# + ); + let output = rewrite(document.as_bytes(), secs(100), secs(10)..secs(14)).unwrap(); + assert_eq!( + output, + format!( + r#" +{ROOT} ttp:frameRate="25" ttp:tickRate="10000000"> + +

Hello world

Clipped

"# + ) + ); + } + + #[test] + fn body_timing_is_clipped_and_kept_when_unchanged() { + let document = format!(r#"{ROOT}>

Hi

"#); + let output = rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(); + assert_eq!( + output, + format!(r#"{ROOT}>

Hi

"#) + ); + // A byte order mark is not text outside the root. + let marked = format!("\u{feff}{document}"); + assert!(rewrite(marked.as_bytes(), secs(1), secs(1)..secs(3)).is_ok()); + // An empty document clears the captions. + let empty = format!("{ROOT}/>"); + assert_eq!(rewrite(empty.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), empty); + } + + #[test] + fn time_expressions() { + let rates = Rates { + frame: Some((30_000, 1001)), + tick: Some(90_000), + }; + for (value, nanos) in [ + ("1.5h", 5_400 * NANOS), + ("2m", 120 * NANOS), + ("0.25s", 250_000_000), + ("1500ms", 1_500_000_000), + ("90000t", NANOS), + ("1t", 11_111), + ("30f", 1_001_000_000), + ("00:00:01.25", 1_250_000_000), + ("100:00:00", 360_000 * NANOS), + ("00:00:01:15", 1_500_500_000), + ] { + assert_eq!(rates.time(value), Ok(nanos), "{value}"); + } + for value in [ + "", + "1", + "s", + ".5s", + "1.s", + "1x", + "-1s", + " 1s", + "0:00:01", + "00:0:01", + "00:60:00", + "00:00:60", + "00:00:01:30", + "00:00:01:01.1", + "00:00:01.5:01", + "00:00:01:1", + ] { + assert!(rates.time(value).is_err(), "{value}"); + } + let unset = Rates { + frame: None, + tick: None, + }; + assert!(unset.time("1f").is_err()); + assert!(unset.time("00:00:00:01").is_err()); + assert!(unset.time("1t").is_err()); + } + + #[test] + fn clock_time_trims_the_fraction() { + assert_eq!(clock_time(0), "00:00:00"); + assert_eq!(clock_time(3_723_500_000_000), "01:02:03.5"); + assert_eq!(clock_time(360_000 * NANOS + 1), "100:00:00.000000001"); + } + + #[test] + fn documents_outside_the_profile_are_refused() { + for document in [ + String::new(), + "".into(), + r#""#.into(), + r#"

"#.into(), + format!(r#"{ROOT} ttp:timeBase="smpte"/>"#), + format!(r#"{ROOT} ttp:frameRateMultiplier="1000 1001"/>"#), + format!(r#"{ROOT} ttp:frameRate="0"/>"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}/>{ROOT}/>"#), + format!(r#"{ROOT}/>trailing"#), + format!("{ROOT}/>"), + format!(r#"{ROOT}/>"#), + ] { + assert!( + rewrite(document.as_bytes(), secs(1), secs(1)..secs(2)).is_err(), + "{document}" + ); + } + assert!(rewrite(b"\xff", secs(1), secs(1)..secs(2)).is_err()); + } +} From 097e89c13f072a9a558f3b522d3396c6a5614187 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:51:14 -0700 Subject: [PATCH 03/10] chore(mux): sort the quick-xml dependency Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 278a4f74a1d..d65c3d28bf0 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -28,8 +28,8 @@ moq-msf = { workspace = true } moq-net = { workspace = true } mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" -quick-xml = "0.42.0" num_enum = { workspace = true } +quick-xml = "0.42.0" scuffle-av1 = { version = "0.1.4" } scuffle-h265 = { version = "0.2.2" } serde = { workspace = true } From 8b8088ed9e8bb773c15f45028c608033ea5f1a2e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:53:49 -0700 Subject: [PATCH 04/10] docs(hang): define a ttml frame as one document clipped to the frame Each ttml payload is one complete TTML document in the media time base whose body begins at the frame timestamp and whose body end bounds what it presents, matching what the fMP4 importer now emits. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-hang.md | 2 +- js/hang/src/catalog/text.ts | 3 ++- rs/hang/src/catalog/text/format.rs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index fc3cd59b151..aae676df53f 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -359,7 +359,7 @@ The `renditions` field maps rendition IDs to text configurations, typically one The `format` field selects the cue serialization, and tells a consumer how to parse each frame's payload: - `vtt`: WebVTT ([W3C WebVTT](https://www.w3.org/TR/webvtt1/)). Each payload is a self-contained `WEBVTT` segment whose cues carry absolute timing. A cue's embedded start time MUST match its enclosing frame timestamp. -- `ttml`: TTML / IMSC1 ([W3C IMSC](https://www.w3.org/TR/ttml-imsc1.1/)) fragment (XML) with absolute timing. A cue's embedded start time MUST match its enclosing frame timestamp. +- `ttml`: TTML / IMSC1 ([W3C IMSC](https://www.w3.org/TR/ttml-imsc1.1/)). Each payload is one complete TTML document in the `media` time base with absolute timing, clipped to the frame. Its `body` begin MUST match the enclosing frame timestamp, and its `body` end bounds everything the document presents. - `utf8`: raw UTF-8 text with no embedded timing or styling. The cue is shown from its frame timestamp until the next cue; an empty payload clears it. A consumer MUST ignore a rendition whose `format` it does not recognize. diff --git a/js/hang/src/catalog/text.ts b/js/hang/src/catalog/text.ts index 2dc6bdf3f6a..d72a756f902 100644 --- a/js/hang/src/catalog/text.ts +++ b/js/hang/src/catalog/text.ts @@ -6,7 +6,8 @@ import { RelativeBroadcastSchema } from "./path"; /** * The cue serialization format of a text track, mirroring the Rust `TextFormat`. * - * Known values are `"vtt"` (WebVTT), `"ttml"` (TTML/IMSC1), and `"utf8"` (raw text). Any other + * Known values are `"vtt"` (WebVTT), `"ttml"` (one complete TTML/IMSC1 document per frame, clipped + * to the frame, whose `body` begins at the frame timestamp), and `"utf8"` (raw text). Any other * string is an unrecognized format the consumer should ignore. The frame timestamp always carries * the cue's start time, so a relay never parses the payload. */ diff --git a/rs/hang/src/catalog/text/format.rs b/rs/hang/src/catalog/text/format.rs index edcaaf4aff0..8125fec686c 100644 --- a/rs/hang/src/catalog/text/format.rs +++ b/rs/hang/src/catalog/text/format.rs @@ -18,8 +18,8 @@ pub enum TextFormat { #[default] Vtt, - /// TTML / IMSC1 ([W3C IMSC]) fragment (XML). Each frame's payload is a document fragment with - /// absolute timing. + /// TTML / IMSC1 ([W3C IMSC]). Each frame's payload is one complete TTML document with absolute + /// timing, clipped to the frame: its `body` begins at the frame timestamp. /// /// [W3C IMSC]: https://www.w3.org/TR/ttml-imsc1.1/ #[display("ttml")] From 6ce105ab454f122c673ab800eb3952bcbff97838 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:58:44 -0700 Subject: [PATCH 05/10] fix(mux): parse TTML times by value and check frame products Character references in timing attributes are normalized before parsing, while the raw span is still what gets spliced. Clock-with-frames products use checked multiplication, so a huge frame rate multiplier errors instead of wrapping. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ttml.rs | 59 +++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs index 4c1c8e3eba3..8e8e958c058 100644 --- a/rs/moq-mux/src/container/ttml.rs +++ b/rs/moq-mux/src/container/ttml.rs @@ -174,14 +174,18 @@ fn timing( let (ResolveResult::Unbound, name) = reader.resolver().resolve_attribute(attribute.key) else { continue; }; - // Attribute values borrow from the document, so they can be spliced in place. - let Cow::Borrowed(value) = attribute.value else { + // The raw value borrows from the document, so it can be spliced in place; the normalized one, + // which may differ by character references, is what gets parsed. + let Cow::Borrowed(raw) = attribute.value else { return Err("malformed TTML attribute"); }; + let value = attribute + .normalized_value(quick_xml::XmlVersion::Implicit1_0) + .map_err(|_| "malformed TTML attribute")?; match name.as_ref() { - "begin" => begin = Some(value), - "end" => end = Some(value), - "dur" => dur = Some(value), + "begin" => begin = Some((raw, value)), + "end" => end = Some((raw, value)), + "dur" => dur = Some((raw, value)), // `seq` would make each child's begin relative to its previous sibling's end. "timeContainer" if value != "par" => return Err("unsupported TTML time container"), _ => {} @@ -189,7 +193,7 @@ fn timing( } let body = element.local_name().as_ref() == "body"; - let offset = begin.map(|value| rates.time(value)).transpose()?; + let offset = begin.as_ref().map(|(_, value)| rates.time(value)).transpose()?; let source = parent .source .checked_add(offset.unwrap_or_default()) @@ -200,28 +204,31 @@ fn timing( }; let relative = mapped - parent.mapped; match (begin, offset) { - (Some(value), Some(offset)) if offset != relative => edit.replace(value, relative), + (Some((raw, _)), Some(offset)) if offset != relative => edit.replace(raw, relative), (None, _) if body => edit.insert(element, "begin", relative), _ => {} } - if let Some(value) = end { - let offset = rates.time(value)?; + // A scope's mapped begin never exceeds the mapping of its source begin, so neither subtraction + // below underflows, even when an element ends before it begins. + let ends = end.is_some() || dur.is_some(); + if let Some((raw, value)) = end { + let offset = rates.time(&value)?; let at = parent.source.checked_add(offset).ok_or("TTML time overflow")?; let relative = clock.map(at) - parent.mapped; if relative != offset { - edit.replace(value, relative); + edit.replace(raw, relative); } } - if let Some(value) = dur { - let offset = rates.time(value)?; + if let Some((raw, value)) = dur { + let offset = rates.time(&value)?; let at = source.checked_add(offset).ok_or("TTML time overflow")?; let relative = clock.map(at) - mapped; if relative != offset { - edit.replace(value, relative); + edit.replace(raw, relative); } } - if body && end.is_none() && dur.is_none() { + if body && !ends { edit.insert(element, "end", clock.end - parent.mapped); } @@ -247,7 +254,9 @@ impl Rates { else { continue; }; - let value = attribute.value; + let value = attribute + .normalized_value(quick_xml::XmlVersion::Implicit1_0) + .map_err(|_| "malformed TTML attribute")?; match name.as_ref() { "timeBase" if value != "media" => return Err("unsupported TTML time base"), "frameRate" => frame = Some(positive(&value)?), @@ -309,10 +318,13 @@ impl Rates { } let (rate, den) = self.frame.ok_or("TTML frame time without ttp:frameRate")?; let frames = integer(frames)?; - if frames * den >= rate { + let (Some(scaled), Some(unit)) = (frames.checked_mul(den), NANOS.checked_mul(den)) else { + return Err("TTML time overflow"); + }; + if scaled >= rate { return Err(INVALID); } - decimal_ratio(frames, 0, NANOS * den, rate)? + decimal_ratio(frames, 0, unit, rate)? } _ => 0, }; @@ -427,6 +439,14 @@ mod tests { output, format!(r#"{ROOT}>

Hi

"#) ); + // Character references are parsed by value, and an element may end before it begins. + let document = format!(r#"{ROOT}>

Hi

"#); + assert_eq!( + rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + format!( + r#"{ROOT}>

Hi

"# + ) + ); // A byte order mark is not text outside the root. let marked = format!("\u{feff}{document}"); assert!(rewrite(marked.as_bytes(), secs(1), secs(1)..secs(3)).is_ok()); @@ -475,6 +495,11 @@ mod tests { ] { assert!(rates.time(value).is_err(), "{value}"); } + let huge = Rates { + frame: Some((u128::MAX, u128::MAX)), + tick: None, + }; + assert!(huge.time("00:00:00:99").is_err()); let unset = Rates { frame: None, tick: None, From 9cc521d05c5b758217b0b1a8a3b8d09bbf1f044a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:48:28 -0700 Subject: [PATCH 06/10] fix(mux): begin every TTML body at its frame timestamp A source body that began after its sample kept that begin, breaking the hang rule that a ttml body begins at the frame timestamp. The body now always begins there, and its untimed children gain the later begin so nothing they present moves. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ttml.rs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs index 8e8e958c058..05ad7599eee 100644 --- a/rs/moq-mux/src/container/ttml.rs +++ b/rs/moq-mux/src/container/ttml.rs @@ -135,6 +135,9 @@ impl Edit<'_> { struct Scope { source: u128, mapped: u128, + /// Where a child without its own begin starts on the catalog clock. Only the body differs from + /// `mapped`: it begins at the frame timestamp even when the source started it later. + implicit: u128, } /// Maps a source media time onto the catalog clock, clipped to the sample. @@ -156,8 +159,9 @@ impl Clock { /// Rewrite one TTML element's `begin`, `end`, and `dur`, returning the scope for its children. /// /// Each is resolved to an absolute source time, mapped, and written back relative to its own -/// reference: `begin` and `end` to the parent's begin, `dur` to the element's begin. The body also -/// gains a begin and an end, so even untimed content stays inside the sample. +/// reference: `begin` and `end` to the parent's begin, `dur` to the element's begin. The body always +/// begins at the frame timestamp and gains an end, so even untimed content stays inside the sample; +/// a later source begin moves onto its children instead. fn timing( reader: &quick_xml::NsReader, element: &BytesStart<'_>, @@ -198,14 +202,15 @@ fn timing( .source .checked_add(offset.unwrap_or_default()) .ok_or("TTML time overflow")?; - let mapped = match (offset, body) { - (None, false) => parent.mapped, - _ => clock.map(source), + let (mapped, implicit) = match (offset, body) { + (_, true) => (clock.start, clock.map(source)), + (None, false) => (parent.implicit, parent.implicit), + (Some(_), false) => (clock.map(source), clock.map(source)), }; let relative = mapped - parent.mapped; match (begin, offset) { (Some((raw, _)), Some(offset)) if offset != relative => edit.replace(raw, relative), - (None, _) if body => edit.insert(element, "begin", relative), + (None, _) if body || relative != 0 => edit.insert(element, "begin", relative), _ => {} } @@ -232,7 +237,11 @@ fn timing( edit.insert(element, "end", clock.end - parent.mapped); } - Ok(Scope { source, mapped }) + Ok(Scope { + source, + mapped, + implicit, + }) } /// The root's timing parameters, which give frame and tick expressions their length. @@ -431,13 +440,14 @@ mod tests { ); } + /// The body begins at the frame timestamp; a later source begin moves onto its untimed children. #[test] - fn body_timing_is_clipped_and_kept_when_unchanged() { + fn body_begins_at_the_frame_and_clips_to_it() { let document = format!(r#"{ROOT}>

Hi

"#); let output = rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(); assert_eq!( output, - format!(r#"{ROOT}>

Hi

"#) + format!(r#"{ROOT}>

Hi

"#) ); // Character references are parsed by value, and an element may end before it begins. let document = format!(r#"{ROOT}>

Hi

"#); From ea613eee599d976c04fd58740f2036e2935ba28b Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:01:38 -0700 Subject: [PATCH 07/10] fix(mux): bound TTML frame fields by the nominal rate, refuse root timing A clock-time frame field counts nominal frames, so validate it against ttp:frameRate rather than the multiplied rate. Timing on the tt root, or a body nested in timed content, is refused instead of underflowing the body's relative begin. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ttml.rs | 78 ++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs index 05ad7599eee..8860d99bff2 100644 --- a/rs/moq-mux/src/container/ttml.rs +++ b/rs/moq-mux/src/container/ttml.rs @@ -195,6 +195,10 @@ fn timing( _ => {} } } + // Timing on the root is not TTML, and would give the body a parent begin past its frame. + if element.local_name().as_ref() == "tt" && (begin.is_some() || end.is_some() || dur.is_some()) { + return Err("timing on the TTML root element"); + } let body = element.local_name().as_ref() == "body"; let offset = begin.as_ref().map(|(_, value)| rates.time(value)).transpose()?; @@ -207,7 +211,10 @@ fn timing( (None, false) => (parent.implicit, parent.implicit), (Some(_), false) => (clock.map(source), clock.map(source)), }; - let relative = mapped - parent.mapped; + // Only a body nested in timed content can begin before its parent. + let relative = mapped + .checked_sub(parent.mapped) + .ok_or("TTML body inside timed content")?; match (begin, offset) { (Some((raw, _)), Some(offset)) if offset != relative => edit.replace(raw, relative), (None, _) if body || relative != 0 => edit.insert(element, "begin", relative), @@ -246,8 +253,8 @@ fn timing( /// The root's timing parameters, which give frame and tick expressions their length. struct Rates { - /// Frames per second as a fraction, when `ttp:frameRate` declares it. - frame: Option<(u128, u128)>, + /// The frame rate, when `ttp:frameRate` declares it. + frame: Option, /// Ticks per second, when `ttp:tickRate` declares it. tick: Option, } @@ -281,8 +288,14 @@ impl Rates { } } let frame = match (frame, multiplier) { - (Some(rate), Some((num, den))) => Some((rate.checked_mul(num).ok_or("TTML time overflow")?, den)), - (Some(rate), None) => Some((rate, 1)), + (Some(nominal), multiplier) => { + let (num, den) = multiplier.unwrap_or((1, 1)); + Some(FrameRate { + nominal, + num: nominal.checked_mul(num).ok_or("TTML time overflow")?, + den, + }) + } (None, Some(_)) => return Err("TTML frame rate multiplier without a frame rate"), (None, None) => None, }; @@ -325,15 +338,14 @@ impl Rates { if frames.len() < 2 || frames.contains('.') { return Err("unsupported TTML sub-frame time expression"); } - let (rate, den) = self.frame.ok_or("TTML frame time without ttp:frameRate")?; + let rate = self.frame.as_ref().ok_or("TTML frame time without ttp:frameRate")?; let frames = integer(frames)?; - let (Some(scaled), Some(unit)) = (frames.checked_mul(den), NANOS.checked_mul(den)) else { - return Err("TTML time overflow"); - }; - if scaled >= rate { + // The frame field counts nominal frames, whatever the multiplier. + if frames >= rate.nominal { return Err(INVALID); } - decimal_ratio(frames, 0, unit, rate)? + let unit = NANOS.checked_mul(rate.den).ok_or("TTML time overflow")?; + decimal_ratio(frames, 0, unit, rate.num)? } _ => 0, }; @@ -349,8 +361,8 @@ impl Rates { "s" => (NANOS, 1), "ms" => (NANOS / 1000, 1), "f" => { - let (rate, den) = self.frame.ok_or("TTML frame time without ttp:frameRate")?; - (NANOS.checked_mul(den).ok_or("TTML time overflow")?, rate) + let rate = self.frame.as_ref().ok_or("TTML frame time without ttp:frameRate")?; + (NANOS.checked_mul(rate.den).ok_or("TTML time overflow")?, rate.num) } "t" => (NANOS, self.tick.ok_or("TTML tick time without ttp:tickRate")?), _ => return Err(INVALID), @@ -359,6 +371,13 @@ impl Rates { } } +/// `ttp:frameRate` and the effective rate its multiplier gives, `num / den` frames per second. +struct FrameRate { + nominal: u128, + num: u128, + den: u128, +} + /// `whole.fraction` times `num / den`, rounded to the nearest. fn decimal(whole: &str, fraction: Option<&str>, num: u128, den: u128) -> Result { let (places, fraction) = match fraction { @@ -468,7 +487,11 @@ mod tests { #[test] fn time_expressions() { let rates = Rates { - frame: Some((30_000, 1001)), + frame: Some(FrameRate { + nominal: 30, + num: 30_000, + den: 1001, + }), tick: Some(90_000), }; for (value, nanos) in [ @@ -506,10 +529,33 @@ mod tests { assert!(rates.time(value).is_err(), "{value}"); } let huge = Rates { - frame: Some((u128::MAX, u128::MAX)), + frame: Some(FrameRate { + nominal: 100, + num: u128::MAX, + den: u128::MAX, + }), tick: None, }; assert!(huge.time("00:00:00:99").is_err()); + // The frame field is bounded by the nominal rate, not the effective one. + let halved = Rates { + frame: Some(FrameRate { + nominal: 30, + num: 30, + den: 2, + }), + tick: None, + }; + assert_eq!(halved.time("00:00:00:20"), Ok(1_333_333_333)); + let doubled = Rates { + frame: Some(FrameRate { + nominal: 30, + num: 60, + den: 1, + }), + tick: None, + }; + assert!(doubled.time("00:00:00:45").is_err()); let unset = Rates { frame: None, tick: None, @@ -537,6 +583,8 @@ mod tests { format!(r#"{ROOT} ttp:frameRateMultiplier="1000 1001"/>"#), format!(r#"{ROOT} ttp:frameRate="0"/>"#), format!(r#"{ROOT}>"#), + format!(r#"{ROOT} begin="1s">"#), + format!(r#"{ROOT}>"#), format!(r#"{ROOT}>

"#), format!(r#"{ROOT}>

"#), format!(r#"{ROOT}>"#), From b87393a591bd9243019da47d0ccfe8c7a8dd2b26 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:14:38 -0700 Subject: [PATCH 08/10] fix(mux): clip TTML to the frame's serialized microseconds Text frames carry hang's microsecond timestamps, so a 90 kHz sample start was floored on the wire while the TTML body began at the exact nanosecond. Clip to the microsecond-normalized span so the body begin equals the frame timestamp. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import_test.rs | 38 +++++++++++++++++--- rs/moq-mux/src/container/fmp4/text.rs | 11 ++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index a21a10e59b1..02a88e0e6d0 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2090,10 +2090,11 @@ fn malformed_webvtt_origins_and_comments_are_refused() { /// An `stpp` init segment. mp4-atom cannot encode an `stpp` entry, so encode a `metx` entry, whose /// strings line up with `stpp`'s namespace, schema location and auxiliary MIME types, and rename it. -fn ttml_init(namespace: &str, mime: &str) -> Vec { +fn ttml_init(namespace: &str, mime: &str, timescale: u32) -> Vec { let (_, moov) = decode_init(&webvtt_init("WEBVTT")); let mut trak = moov.trak[0].clone(); trak.mdia.hdlr.handler = b"subt".into(); + trak.mdia.mdhd.timescale = timescale; trak.mdia.minf.stbl.stsd.codecs = vec![mp4_atom::Codec::Metx(mp4_atom::Metx { metadata: mp4_atom::MetaData { data_reference_index: 1, @@ -2118,7 +2119,7 @@ async fn imported_ttml( catalog: &crate::catalog::Producer, consumer: &moq_net::broadcast::Consumer, ) -> (hang::catalog::TextConfig, crate::container::Frame) { - let mut input = ttml_init("http://www.w3.org/ns/ttml", ""); + let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 1000); input.extend(webvtt_fragment(100_000, &[TTML.as_bytes().to_vec()], 4000)); let mut importer = super::Import::new(broadcast, timebase.reserve()); importer.decode(&input).unwrap(); @@ -2176,7 +2177,7 @@ async fn ttml_follows_a_shifted_source_clock() { #[test] fn unsupported_stpp_tracks_are_skipped() { for (namespace, mime) in [("urn:example:not-ttml", ""), ("http://www.w3.org/ns/ttml", "image/png")] { - let mut input = ttml_init(namespace, mime); + let mut input = ttml_init(namespace, mime, 1000); input.extend(webvtt_fragment(0, &[TTML.as_bytes().to_vec()], 1000)); let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); @@ -2194,7 +2195,7 @@ fn ttml_documents_outside_the_profile_are_refused() { r#""#, r#""#, ] { - let mut input = ttml_init("http://www.w3.org/ns/ttml", ""); + let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 1000); input.extend(webvtt_fragment(0, &[document.as_bytes().to_vec()], 1000)); let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); @@ -2202,3 +2203,32 @@ fn ttml_documents_outside_the_profile_are_refused() { assert!(importer.decode(&input).is_err(), "{document}"); } } + +/// A 90 kHz sample start is not a whole microsecond; the body still begins at the serialized +/// frame timestamp. +#[tokio::test(start_paused = true)] +async fn ttml_body_begins_at_the_serialized_frame_timestamp() { + let mut input = ttml_init("http://www.w3.org/ns/ttml", "", 90_000); + input.extend(webvtt_fragment(9_000_001, &[TTML.as_bytes().to_vec()], 360_000)); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = super::Import::new(broadcast, catalog.reserve()); + importer.decode(&input).unwrap(); + importer.finish().unwrap(); + let snapshot = catalog.snapshot(); + let name = snapshot.text.renditions.keys().next().unwrap(); + let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); + let mut reader = crate::container::Consumer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + let frame = reader.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.timestamp.as_micros(), 100_000_011); + let payload = std::str::from_utf8(&frame.payload).unwrap(); + assert!( + payload.contains(r#""#), + "{payload}" + ); +} diff --git a/rs/moq-mux/src/container/fmp4/text.rs b/rs/moq-mux/src/container/fmp4/text.rs index c6064aaa2c0..25e816b2293 100644 --- a/rs/moq-mux/src/container/fmp4/text.rs +++ b/rs/moq-mux/src/container/fmp4/text.rs @@ -99,9 +99,14 @@ impl Track { let data = input.data.get(offset..next).ok_or(Error::InvalidDataOffset)?; let payload = match &self.format { Format::Vtt { header } => vtt(header, data, timestamp, end)?, - Format::Ttml => ttml::rewrite(data, source, timestamp..end) - .map_err(Error::InvalidSubtitle)? - .into(), + // The frame timestamp is serialized in hang's scale, which the body's begin must match. + Format::Ttml => ttml::rewrite( + data, + source, + timestamp.convert(hang::container::TIMESCALE)?..end.convert(hang::container::TIMESCALE)?, + ) + .map_err(Error::InvalidSubtitle)? + .into(), }; self.producer.write(crate::container::Frame { timestamp, From 62b89e6ec534eef337a150b260d181560a0b36da Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:42:02 -0700 Subject: [PATCH 09/10] fix(mux): refuse undeclared TTML entities, saturate far-future times Without a DTD only character references and XML's predefined entities resolve, so a document using another one is refused instead of published unparseable. A time past i128 now saturates to the sample end rather than wrapping negative onto its start. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ttml.rs | 39 +++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs index 8860d99bff2..27b6af20a82 100644 --- a/rs/moq-mux/src/container/ttml.rs +++ b/rs/moq-mux/src/container/ttml.rs @@ -60,9 +60,26 @@ pub(super) fn rewrite(document: &[u8], source: Timestamp, sample: Range { return Err("text outside the TTML root element"); } + // Without a DTD, only character references and XML's predefined entities resolve. + Event::GeneralRef(reference) => { + let resolved = match reference.resolve_char_ref() { + Ok(found) => found.is_some(), + Err(_) => false, + } || quick_xml::escape::resolve_predefined_entity(&reference.xml10_content()).is_some(); + if !resolved { + return Err("unresolved TTML entity reference"); + } + continue; + } Event::Eof => break, _ => continue, }; + for attribute in element.attributes() { + let attribute = attribute.map_err(|_| "malformed TTML attribute")?; + attribute + .normalized_value(quick_xml::XmlVersion::Implicit1_0) + .map_err(|_| "unresolved TTML entity reference")?; + } if rates.is_some() && scopes.len() == 1 { return Err("multiple TTML root elements"); } @@ -150,7 +167,9 @@ struct Clock { impl Clock { /// Monotonic, so every relative time it rewrites stays non-negative. fn map(&self, source: u128) -> u128 { - (source as i128) + // A time past i128 is far past the sample, so saturating keeps the map monotonic. + i128::try_from(source) + .unwrap_or(i128::MAX) .saturating_add(self.offset) .clamp(self.start as i128, self.end as i128) as u128 } @@ -476,6 +495,22 @@ mod tests { r#"{ROOT}>

Hi

"# ) ); + // Predefined entities and character references resolve without a DTD. + let entities = format!(r#"{ROOT}>

< &

"#); + assert_eq!( + rewrite(entities.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + entities + ); + // A time past i128 clips to the sample's end rather than wrapping before its start. + let far = format!( + r#"{ROOT}>

"#, + i128::MAX as u128 / NANOS + 1 + ); + assert!( + rewrite(far.as_bytes(), secs(1), secs(1)..secs(3)) + .unwrap() + .contains(r#"

"#) + ); // A byte order mark is not text outside the root. let marked = format!("\u{feff}{document}"); assert!(rewrite(marked.as_bytes(), secs(1), secs(1)..secs(3)).is_ok()); @@ -591,6 +626,8 @@ mod tests { format!(r#"{ROOT}/>{ROOT}/>"#), format!(r#"{ROOT}/>trailing"#), format!("{ROOT}/>"), + format!(r#"{ROOT}>

a b

"#), + format!(r#"{ROOT}>

"#), format!(r#"{ROOT}/>"#), ] { assert!( From 66a96367304ce69f4b16321bf9bdc8f825edfa81 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 02:17:37 -0700 Subject: [PATCH 10/10] refactor(mux): parse TTML with a strict XML parser quick-xml streams without validating, so each well-formedness gap (comments, CDATA outside the root, undeclared prefixes or entities) needed its own check. roxmltree refuses all of them, and DTDs, by construction, and still exposes attribute value ranges for splicing, so the rewrite walks its tree instead. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 11 +- rs/moq-mux/Cargo.toml | 2 +- rs/moq-mux/src/container/ttml.rs | 204 +++++++++++++------------------ 3 files changed, 93 insertions(+), 124 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f40fd61943..2b6a26a26a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4642,7 +4642,7 @@ dependencies = [ "mp4-atom", "mpeg2ts", "num_enum", - "quick-xml 0.42.0", + "roxmltree", "scuffle-av1", "scuffle-h265", "serde", @@ -7570,6 +7570,15 @@ dependencies = [ "portable-atomic-util", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rsa" version = "0.9.10" diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index d65c3d28bf0..927e12a3de9 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -29,7 +29,7 @@ moq-net = { workspace = true } mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" num_enum = { workspace = true } -quick-xml = "0.42.0" +roxmltree = "0.21.1" scuffle-av1 = { version = "0.1.4" } scuffle-h265 = { version = "0.2.2" } serde = { workspace = true } diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs index 27b6af20a82..c6ea7b9aaf5 100644 --- a/rs/moq-mux/src/container/ttml.rs +++ b/rs/moq-mux/src/container/ttml.rs @@ -1,13 +1,10 @@ //! TTML documents moved from a source's media timeline onto the catalog clock. //! -//! Only the timing profile IMSC1 admits is supported: a `tt` root in the TTML namespace, -//! `ttp:timeBase="media"`, `par` time containers, and clock or offset time expressions without -//! sub-frames. Anything else is refused rather than relabelled. +//! Only well-formed XML in the timing profile IMSC1 admits is supported: a `tt` root in the TTML +//! namespace, no DTD, `ttp:timeBase="media"`, `par` time containers, and clock or offset time +//! expressions without sub-frames. Anything else is refused rather than relabelled. use moq_net::Timestamp; -use quick_xml::events::{BytesStart, Event}; -use quick_xml::name::{Namespace, ResolveResult}; -use std::borrow::Cow; use std::ops::Range; /// The TTML namespace, which a sample entry must list and a document's root must use. @@ -23,88 +20,51 @@ const NANOS: u128 = 1_000_000_000; /// presents within it, so every time is also clipped to it: the payload then renders on its own. pub(super) fn rewrite(document: &[u8], source: Timestamp, sample: Range) -> Result { let document = std::str::from_utf8(document).map_err(|_| "invalid TTML UTF-8")?; + // A strict parse: the output keeps every source byte it does not retime, so it is only as + // well-formed as the input. DTDs stay refused, since their entities and defaults could change + // what the attributes say. + let tree = roxmltree::Document::parse(document).map_err(|_| "malformed TTML document")?; + // The text was decoded as UTF-8, whatever the (already validated) declaration says. + let declaration = document + .trim_start_matches('\u{feff}') + .strip_prefix("")) + .map_or("", |(declaration, _)| declaration); + if let Some((_, encoding)) = declaration.split_once("encoding") { + let encoding = encoding.trim_start().trim_start_matches('=').trim_start(); + let quote = encoding.chars().next().unwrap_or('"'); + let name = encoding[quote.len_utf8()..].split(quote).next().unwrap_or_default(); + if !name.eq_ignore_ascii_case("UTF-8") { + return Err("TTML document is not UTF-8"); + } + } + let root = tree.root_element(); + if root.tag_name().namespace() != Some(NAMESPACE) || root.tag_name().name() != "tt" { + return Err("TTML root is not a tt element in the TTML namespace"); + } + let rates = Rates::from_root(root)?; let clock = Clock { offset: sample.start.as_nanos() as i128 - source.as_nanos() as i128, start: sample.start.as_nanos(), end: sample.end.as_nanos(), }; - let mut reader = quick_xml::NsReader::from_str(document); - let mut rates = None; - // The enclosing elements' begins; the root's is zero on both clocks. - let mut scopes = vec![Scope::default()]; let mut edit = Edit { document, edits: Vec::new(), }; - loop { - let (namespace, event) = reader.read_resolved_event().map_err(|_| "malformed TTML document")?; - let ttml = matches!(namespace, ResolveResult::Bound(Namespace(NAMESPACE))); - let (element, open) = match event { - Event::Start(element) => (element, true), - Event::Empty(element) => (element, false), - Event::End(_) => { - scopes.pop(); - continue; - } - Event::Decl(decl) => { - if let Some(encoding) = decl.encoding() { - let encoding = encoding.map_err(|_| "malformed TTML declaration")?; - if !encoding.eq_ignore_ascii_case("UTF-8") { - return Err("TTML document is not UTF-8"); - } - } - continue; - } - // A DTD can declare entities and defaults that change what the attributes say. - Event::DocType(_) => return Err("TTML document type declarations are unsupported"), - Event::Text(text) if scopes.len() == 1 && !text.trim().is_empty() => { - return Err("text outside the TTML root element"); - } - // Without a DTD, only character references and XML's predefined entities resolve. - Event::GeneralRef(reference) => { - let resolved = match reference.resolve_char_ref() { - Ok(found) => found.is_some(), - Err(_) => false, - } || quick_xml::escape::resolve_predefined_entity(&reference.xml10_content()).is_some(); - if !resolved { - return Err("unresolved TTML entity reference"); - } - continue; - } - Event::Eof => break, - _ => continue, - }; - for attribute in element.attributes() { - let attribute = attribute.map_err(|_| "malformed TTML attribute")?; - attribute - .normalized_value(quick_xml::XmlVersion::Implicit1_0) - .map_err(|_| "unresolved TTML entity reference")?; - } - if rates.is_some() && scopes.len() == 1 { - return Err("multiple TTML root elements"); - } - let parent = *scopes.last().ok_or("malformed TTML document")?; - let rates = match &mut rates { - Some(rates) => rates, - slot => { - if !ttml || element.local_name().as_ref() != "tt" { - return Err("TTML root is not a tt element in the TTML namespace"); - } - slot.insert(Rates::from_root(&reader, &element)?) - } - }; - let scope = if ttml { - timing(&reader, &element, rates, &clock, parent, &mut edit)? + // Each element's scope, by node id, so a child finds its parent's; the root's parent begins at + // zero on both clocks. + let mut scopes = vec![Scope::default(); tree.descendants().count()]; + for element in root.descendants().filter(|node| node.is_element()) { + let parent = element + .parent_element() + .map_or(Scope::default(), |parent| scopes[parent.id().get_usize()]); + scopes[element.id().get_usize()] = if element.tag_name().namespace() == Some(NAMESPACE) { + timing(element, &rates, &clock, parent, &mut edit)? } else { // Foreign elements are not timed; their content inherits the enclosing begin. parent }; - if open { - scopes.push(scope); - } - } - if rates.is_none() { - return Err("missing TTML root element"); } Ok(edit.apply()) } @@ -116,20 +76,18 @@ struct Edit<'a> { } impl Edit<'_> { - /// The byte range of `part`, which borrows from the document. - fn span(&self, part: &str) -> Range { - let start = part.as_ptr() as usize - self.document.as_ptr() as usize; - debug_assert!(start + part.len() <= self.document.len()); - start..start + part.len() + fn replace(&mut self, value: Range, nanos: u128) { + self.edits.push((value, clock_time(nanos))); } - fn replace(&mut self, part: &str, nanos: u128) { - let range = self.span(part); - self.edits.push((range, clock_time(nanos))); - } - - fn insert(&mut self, element: &BytesStart, name: &str, nanos: u128) { - let at = self.span(element.name().0).end; + /// Add an attribute right after the element's name. + fn insert(&mut self, element: roxmltree::Node, name: &str, nanos: u128) { + let start = element.range().start; + let tag = &self.document[start..]; + let at = start + + tag + .find(|c: char| c.is_ascii_whitespace() || c == '/' || c == '>') + .unwrap_or(tag.len()); self.edits.push((at..at, format!(" {name}=\"{}\"", clock_time(nanos)))); } @@ -181,9 +139,8 @@ impl Clock { /// reference: `begin` and `end` to the parent's begin, `dur` to the element's begin. The body always /// begins at the frame timestamp and gains an end, so even untimed content stays inside the sample; /// a later source begin moves onto its children instead. -fn timing( - reader: &quick_xml::NsReader, - element: &BytesStart<'_>, +fn timing( + element: roxmltree::Node, rates: &Rates, clock: &Clock, parent: Scope, @@ -192,20 +149,11 @@ fn timing( let mut begin = None; let mut end = None; let mut dur = None; - for attribute in element.attributes() { - let attribute = attribute.map_err(|_| "malformed TTML attribute")?; - let (ResolveResult::Unbound, name) = reader.resolver().resolve_attribute(attribute.key) else { - continue; - }; - // The raw value borrows from the document, so it can be spliced in place; the normalized one, - // which may differ by character references, is what gets parsed. - let Cow::Borrowed(raw) = attribute.value else { - return Err("malformed TTML attribute"); - }; - let value = attribute - .normalized_value(quick_xml::XmlVersion::Implicit1_0) - .map_err(|_| "malformed TTML attribute")?; - match name.as_ref() { + for attribute in element.attributes().filter(|attribute| attribute.namespace().is_none()) { + // The raw range is spliced in place; the value, which may differ by character references, is + // what gets parsed. + let (raw, value) = (attribute.range_value(), attribute.value()); + match attribute.name() { "begin" => begin = Some((raw, value)), "end" => end = Some((raw, value)), "dur" => dur = Some((raw, value)), @@ -215,10 +163,10 @@ fn timing( } } // Timing on the root is not TTML, and would give the body a parent begin past its frame. - if element.local_name().as_ref() == "tt" && (begin.is_some() || end.is_some() || dur.is_some()) { + if element.tag_name().name() == "tt" && (begin.is_some() || end.is_some() || dur.is_some()) { return Err("timing on the TTML root element"); } - let body = element.local_name().as_ref() == "body"; + let body = element.tag_name().name() == "body"; let offset = begin.as_ref().map(|(_, value)| rates.time(value)).transpose()?; let source = parent @@ -244,7 +192,7 @@ fn timing( // below underflows, even when an element ends before it begins. let ends = end.is_some() || dur.is_some(); if let Some((raw, value)) = end { - let offset = rates.time(&value)?; + let offset = rates.time(value)?; let at = parent.source.checked_add(offset).ok_or("TTML time overflow")?; let relative = clock.map(at) - parent.mapped; if relative != offset { @@ -252,7 +200,7 @@ fn timing( } } if let Some((raw, value)) = dur { - let offset = rates.time(&value)?; + let offset = rates.time(value)?; let at = source.checked_add(offset).ok_or("TTML time overflow")?; let relative = clock.map(at) - mapped; if relative != offset { @@ -279,22 +227,18 @@ struct Rates { } impl Rates { - fn from_root(reader: &quick_xml::NsReader, root: &BytesStart<'_>) -> Result { + fn from_root(root: roxmltree::Node) -> Result { let mut frame = None; let mut multiplier = None; let mut tick = None; - for attribute in root.attributes() { - let attribute = attribute.map_err(|_| "malformed TTML attribute")?; - let (ResolveResult::Bound(Namespace(PARAMETER)), name) = reader.resolver().resolve_attribute(attribute.key) - else { - continue; - }; - let value = attribute - .normalized_value(quick_xml::XmlVersion::Implicit1_0) - .map_err(|_| "malformed TTML attribute")?; - match name.as_ref() { + for attribute in root + .attributes() + .filter(|attribute| attribute.namespace() == Some(PARAMETER)) + { + let value = attribute.value(); + match attribute.name() { "timeBase" if value != "media" => return Err("unsupported TTML time base"), - "frameRate" => frame = Some(positive(&value)?), + "frameRate" => frame = Some(positive(value)?), "frameRateMultiplier" => { let mut parts = value.split_whitespace(); let (Some(num), Some(den), None) = (parts.next(), parts.next(), parts.next()) else { @@ -302,7 +246,7 @@ impl Rates { }; multiplier = Some((positive(num)?, positive(den)?)); } - "tickRate" => tick = Some(positive(&value)?), + "tickRate" => tick = Some(positive(value)?), _ => {} } } @@ -511,9 +455,20 @@ mod tests { .unwrap() .contains(r#"

"#) ); + let declared = format!(r#"{entities}"#); + assert_eq!( + rewrite(declared.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + declared + ); // A byte order mark is not text outside the root. let marked = format!("\u{feff}{document}"); - assert!(rewrite(marked.as_bytes(), secs(1), secs(1)..secs(3)).is_ok()); + assert_eq!( + rewrite(marked.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), + format!( + "\u{feff}{}", + rewrite(document.as_bytes(), secs(1), secs(1)..secs(3)).unwrap() + ) + ); // An empty document clears the captions. let empty = format!("{ROOT}/>"); assert_eq!(rewrite(empty.as_bytes(), secs(1), secs(1)..secs(3)).unwrap(), empty); @@ -628,7 +583,12 @@ mod tests { format!("{ROOT}/>"), format!(r#"{ROOT}>

a b

"#), format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}>"#), + format!(r#"{ROOT}>

"#), + format!(r#"{ROOT}/>"#), + format!(r#"{ROOT}>"#), format!(r#"{ROOT}/>"#), + format!(r#"{ROOT}/>"#), ] { assert!( rewrite(document.as_bytes(), secs(1), secs(1)..secs(2)).is_err(),