From 98aa7a57631e08dfce3756c7562697a5a7ba5d78 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 21 Nov 2026 09:12:19 +0530 Subject: [PATCH 1/2] fix(msf): bound catalog parsing before initRef expansion --- rs/moq-msf/src/lib.rs | 234 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 233 insertions(+), 1 deletion(-) diff --git a/rs/moq-msf/src/lib.rs b/rs/moq-msf/src/lib.rs index 83087ca2016..7c2993cde52 100644 --- a/rs/moq-msf/src/lib.rs +++ b/rs/moq-msf/src/lib.rs @@ -27,6 +27,13 @@ use serde_with::DurationMilliSecondsWithFrac; /// The default track name for the MSF catalog. pub const DEFAULT_NAME: &str = "catalog"; +// Bound catalog parsing before initRef expansion. +/// Maximum number of tracks or initDataList entries accepted during parsing. +pub const MAX_TRACKS: usize = 64; + +/// Maximum serialized catalog bytes and aggregate resolved initData string bytes. +pub const MAX_CATALOG_BYTES: usize = 32 * 1024 * 1024; + /// A snapshot of an MSF catalog: the tracks currently in a broadcast. /// /// This is a version-agnostic view. The on-wire details (the catalog `version` @@ -223,6 +230,12 @@ impl Catalog { /// Deserialize an MSF catalog from a JSON string. #[allow(clippy::should_implement_trait)] pub fn from_str(s: &str) -> Result { + // Reject oversized wire input before allocating any catalog fields. + if s.len() > MAX_CATALOG_BYTES { + return Err(serde::de::Error::custom(format!( + "MSF catalog exceeds {MAX_CATALOG_BYTES} serialized bytes" + ))); + } serde_json::from_str(s) } } @@ -320,6 +333,26 @@ impl<'de, E: CatalogExt> Deserialize<'de> for Catalog { // of tracks rather than tracks x entries. let by_id: HashMap<&str, &InitData> = init_data_list.iter().map(|e| (e.id.as_str(), e)).collect(); + // Preflight the complete resolved size before ANY + // initRef payload is cloned. Inline data is already owned and counts once, + // including when it also carries a matching reference. Missing references + // and conflicting data retain the upstream errors in the resolution below. + let mut remaining = MAX_CATALOG_BYTES; + for track in &wire.tracks { + let data = track.init_data.as_ref().or_else(|| { + track + .init_ref + .as_ref() + .and_then(|id| by_id.get(id.as_str()).map(|entry| &entry.data)) + }); + let len = data.map_or(0, String::len); + remaining = remaining.checked_sub(len).ok_or_else(|| { + ::custom(format!( + "MSF catalog exceeds {MAX_CATALOG_BYTES} resolved initData bytes" + )) + })?; + } + let tracks = wire .tracks .into_iter() @@ -381,8 +414,9 @@ struct Wire { generated_at: Option, #[serde(default, skip_serializing_if = "is_false")] is_complete: bool, - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_bounded_entries")] tracks: Vec, + #[serde(default, deserialize_with = "deserialize_bounded_init_data_list")] init_data_list: Option>, /// The extension's own root members, written flat alongside the ones above. @@ -390,6 +424,58 @@ struct Wire { ext: E, } +// Do not let an untrusted sequence size hint reserve +// arbitrary capacity, or deserialize a 65th entry before rejecting the catalog. +fn deserialize_bounded_entries<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + struct EntriesVisitor(std::marker::PhantomData); + struct RejectExtraEntry; + + impl<'de> serde::de::DeserializeSeed<'de> for RejectExtraEntry { + type Value = (); + + fn deserialize>(self, _: D) -> Result<(), D::Error> { + Err(serde::de::Error::custom(format!( + "MSF catalog exceeds {MAX_TRACKS} entries" + ))) + } + } + + impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for EntriesVisitor { + type Value = Vec; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "an array of at most {MAX_TRACKS} entries") + } + + fn visit_seq>(self, mut seq: A) -> Result, A::Error> { + let mut entries = Vec::new(); + while entries.len() < MAX_TRACKS { + match seq.next_element()? { + Some(entry) => entries.push(entry), + None => return Ok(entries), + } + } + let _ = seq.next_element_seed(RejectExtraEntry)?; + Ok(entries) + } + } + + deserializer.deserialize_seq(EntriesVisitor(std::marker::PhantomData)) +} + +fn deserialize_bounded_init_data_list<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result>, D::Error> { + #[derive(Deserialize)] + struct Entries(#[serde(deserialize_with = "deserialize_bounded_entries")] Vec); + + Option::::deserialize(deserializer).map(|entries| entries.map(|entries| entries.0)) +} + /// Wire encoding of the catalog version. Deserialization accepts draft-00's /// number `1` or any draft-01 `"draft-XX"` string; serialization always emits /// [`CURRENT_VERSION`], so callers never deal with the version on the wire. @@ -620,6 +706,152 @@ impl<'de> Deserialize<'de> for Role { mod test { use super::*; + // Bound regression tests: exercise the public string parser + // and serde's Value path, which must share structural/expansion guards. + fn assert_bound_error_both(value: serde_json::Value, expected: &str) { + let json = serde_json::to_string(&value).unwrap(); + let error = Catalog::<()>::from_str(&json).expect_err("string parser must reject catalog"); + assert_eq!(error.classify(), serde_json::error::Category::Data); + assert!(error.to_string().contains(expected), "unexpected string error: {error}"); + let error = serde_json::from_value::>(value).expect_err("Value parser must reject catalog"); + assert_eq!(error.classify(), serde_json::error::Category::Data); + assert!(error.to_string().contains(expected), "unexpected Value error: {error}"); + } + + #[test] + fn tracks_limit_rejects_before_deserializing_65th_entry() { + let mut tracks = vec![serde_json::json!({"name": "v", "packaging": "loc"}); MAX_TRACKS]; + // A normal Vec deserializer would report this invalid Track instead. + tracks.push(serde_json::json!(false)); + assert_bound_error_both( + serde_json::json!({"version": "draft-01", "tracks": tracks}), + "MSF catalog exceeds 64 entries", + ); + } + + #[test] + fn init_data_limit_rejects_before_deserializing_65th_entry() { + let mut entries = vec![serde_json::json!({"id": "i", "type": "inline", "data": ""}); MAX_TRACKS]; + // Reject before asking serde to turn this invalid item into InitData. + entries.push(serde_json::json!(false)); + assert_bound_error_both( + serde_json::json!({"version": "draft-01", "initDataList": entries}), + "MSF catalog exceeds 64 entries", + ); + } + + #[test] + fn track_and_init_entry_exact_limits_are_accepted() { + let tracks: Vec<_> = (0..MAX_TRACKS) + .map(|i| serde_json::json!({"name": format!("v{i}"), "packaging": "loc", "initRef": format!("i{i}")})) + .collect(); + let entries: Vec<_> = (0..MAX_TRACKS) + .map(|i| serde_json::json!({"id": format!("i{i}"), "type": "inline", "data": "AQID"})) + .collect(); + let value = serde_json::json!({"version": "draft-01", "tracks": tracks, "initDataList": entries}); + let json = serde_json::to_string(&value).unwrap(); + let from_string = Catalog::<()>::from_str(&json).unwrap(); + let from_value = serde_json::from_value::>(value).unwrap(); + assert_eq!(from_string, from_value); + assert_eq!(from_string.tracks.len(), MAX_TRACKS); + assert!( + from_string + .tracks + .iter() + .all(|track| track.init_data.as_deref() == Some("AQID")) + ); + } + + #[test] + fn aggregate_reference_limit_precedes_resolution_errors() { + // If reference resolution starts before the complete size preflight, the + // first missing reference wins instead of the size error. This also makes + // a late/post-expansion guard fail without requiring a huge allocation. + let mut tracks = vec![serde_json::json!({"name": "missing", "packaging": "loc", "initRef": "absent"})]; + tracks.extend( + (0..17).map(|i| serde_json::json!({"name": format!("v{i}"), "packaging": "loc", "initRef": "shared"})), + ); + assert_bound_error_both( + serde_json::json!({ + "version": "draft-01", + "tracks": tracks, + "initDataList": [{"id": "shared", "type": "inline", "data": "A".repeat(2 * 1024 * 1024)}], + }), + "MSF catalog exceeds 33554432 resolved initData bytes", + ); + } + + #[test] + fn aggregate_limit_counts_inline_and_referenced_data() { + let mut tracks = vec![serde_json::json!({ + "name": "inline", "packaging": "loc", "initData": "B".repeat(1024 * 1024), + })]; + tracks.extend( + (0..16).map(|i| serde_json::json!({"name": format!("v{i}"), "packaging": "loc", "initRef": "shared"})), + ); + assert_bound_error_both( + serde_json::json!({ + "version": "draft-01", + "tracks": tracks, + "initDataList": [{"id": "shared", "type": "inline", "data": "A".repeat(2 * 1024 * 1024)}], + }), + "MSF catalog exceeds 33554432 resolved initData bytes", + ); + } + + #[test] + fn shared_references_accept_exact_resolved_byte_limit() { + let payload = "A".repeat(MAX_CATALOG_BYTES / MAX_TRACKS); + let mut tracks: Vec<_> = (0..MAX_TRACKS) + .map(|i| serde_json::json!({"name": format!("v{i}"), "packaging": "loc", "initRef": "shared"})) + .collect(); + // Matching inline + reference counts once, preserving upstream semantics. + tracks[0]["initData"] = serde_json::json!(payload); + let value = serde_json::json!({ + "version": "draft-01", + "tracks": tracks, + "initDataList": [{"id": "shared", "type": "inline", "data": payload}], + }); + let json = serde_json::to_string(&value).unwrap(); + for catalog in [ + Catalog::<()>::from_str(&json).unwrap(), + serde_json::from_value::>(value).unwrap(), + ] { + assert_eq!(catalog.tracks.len(), MAX_TRACKS); + assert_eq!( + catalog + .tracks + .iter() + .map(|track| track.init_data.as_ref().unwrap().len()) + .sum::(), + MAX_CATALOG_BYTES, + ); + assert!( + catalog + .tracks + .iter() + .all(|track| track.init_data.as_deref() == Some(payload.as_str())) + ); + assert!(catalog.tracks.iter().all(|track| track.init_ref.is_none())); + } + } + + #[test] + fn serialized_catalog_byte_limit_is_exact_and_precedes_parsing() { + let mut json = r#"{"version":"draft-01","tracks":[]}"#.to_string(); + json.extend(std::iter::repeat_n(' ', MAX_CATALOG_BYTES - json.len())); + assert!(Catalog::<()>::from_str(&json).unwrap().tracks.is_empty()); + // Invalid trailing JSON proves that the byte guard runs before parsing. + json.push('!'); + let error = Catalog::<()>::from_str(&json).unwrap_err(); + assert_eq!(error.classify(), serde_json::error::Category::Data); + assert!( + error + .to_string() + .contains("MSF catalog exceeds 33554432 serialized bytes") + ); + } + fn video_track() -> Track { Track { name: "video0".to_string(), From 78bc75ebbb40fb95e941fd80f6a5e58c2afc97b0 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 21 Nov 2026 19:02:57 +0530 Subject: [PATCH 2/2] fix(msf): preserve catalogs with more than 64 entries Keep the serialized and resolved init data byte limits. Restore ordinary Serde sequence deserialization so the patch release does not add a separate track or init data entry count limit. The 65-entry compatibility test failed with the old count limit and now passes through both public parsing paths. All 36 moq-msf nextest cases pass, and scoped formatting and Clippy checks pass. Co-Authored-By: GPT-6.2 Sol --- rs/moq-msf/src/lib.rs | 100 +++++------------------------------------- 1 file changed, 11 insertions(+), 89 deletions(-) diff --git a/rs/moq-msf/src/lib.rs b/rs/moq-msf/src/lib.rs index 7c2993cde52..1268576c817 100644 --- a/rs/moq-msf/src/lib.rs +++ b/rs/moq-msf/src/lib.rs @@ -27,10 +27,6 @@ use serde_with::DurationMilliSecondsWithFrac; /// The default track name for the MSF catalog. pub const DEFAULT_NAME: &str = "catalog"; -// Bound catalog parsing before initRef expansion. -/// Maximum number of tracks or initDataList entries accepted during parsing. -pub const MAX_TRACKS: usize = 64; - /// Maximum serialized catalog bytes and aggregate resolved initData string bytes. pub const MAX_CATALOG_BYTES: usize = 32 * 1024 * 1024; @@ -414,9 +410,8 @@ struct Wire { generated_at: Option, #[serde(default, skip_serializing_if = "is_false")] is_complete: bool, - #[serde(default, deserialize_with = "deserialize_bounded_entries")] + #[serde(default)] tracks: Vec, - #[serde(default, deserialize_with = "deserialize_bounded_init_data_list")] init_data_list: Option>, /// The extension's own root members, written flat alongside the ones above. @@ -424,58 +419,6 @@ struct Wire { ext: E, } -// Do not let an untrusted sequence size hint reserve -// arbitrary capacity, or deserialize a 65th entry before rejecting the catalog. -fn deserialize_bounded_entries<'de, D, T>(deserializer: D) -> Result, D::Error> -where - D: serde::Deserializer<'de>, - T: Deserialize<'de>, -{ - struct EntriesVisitor(std::marker::PhantomData); - struct RejectExtraEntry; - - impl<'de> serde::de::DeserializeSeed<'de> for RejectExtraEntry { - type Value = (); - - fn deserialize>(self, _: D) -> Result<(), D::Error> { - Err(serde::de::Error::custom(format!( - "MSF catalog exceeds {MAX_TRACKS} entries" - ))) - } - } - - impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for EntriesVisitor { - type Value = Vec; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "an array of at most {MAX_TRACKS} entries") - } - - fn visit_seq>(self, mut seq: A) -> Result, A::Error> { - let mut entries = Vec::new(); - while entries.len() < MAX_TRACKS { - match seq.next_element()? { - Some(entry) => entries.push(entry), - None => return Ok(entries), - } - } - let _ = seq.next_element_seed(RejectExtraEntry)?; - Ok(entries) - } - } - - deserializer.deserialize_seq(EntriesVisitor(std::marker::PhantomData)) -} - -fn deserialize_bounded_init_data_list<'de, D: serde::Deserializer<'de>>( - deserializer: D, -) -> Result>, D::Error> { - #[derive(Deserialize)] - struct Entries(#[serde(deserialize_with = "deserialize_bounded_entries")] Vec); - - Option::::deserialize(deserializer).map(|entries| entries.map(|entries| entries.0)) -} - /// Wire encoding of the catalog version. Deserialization accepts draft-00's /// number `1` or any draft-01 `"draft-XX"` string; serialization always emits /// [`CURRENT_VERSION`], so callers never deal with the version on the wire. @@ -706,8 +649,7 @@ impl<'de> Deserialize<'de> for Role { mod test { use super::*; - // Bound regression tests: exercise the public string parser - // and serde's Value path, which must share structural/expansion guards. + // Both the string parser and serde's Value path enforce the resolved byte limit. fn assert_bound_error_both(value: serde_json::Value, expected: &str) { let json = serde_json::to_string(&value).unwrap(); let error = Catalog::<()>::from_str(&json).expect_err("string parser must reject catalog"); @@ -719,33 +661,12 @@ mod test { } #[test] - fn tracks_limit_rejects_before_deserializing_65th_entry() { - let mut tracks = vec![serde_json::json!({"name": "v", "packaging": "loc"}); MAX_TRACKS]; - // A normal Vec deserializer would report this invalid Track instead. - tracks.push(serde_json::json!(false)); - assert_bound_error_both( - serde_json::json!({"version": "draft-01", "tracks": tracks}), - "MSF catalog exceeds 64 entries", - ); - } - - #[test] - fn init_data_limit_rejects_before_deserializing_65th_entry() { - let mut entries = vec![serde_json::json!({"id": "i", "type": "inline", "data": ""}); MAX_TRACKS]; - // Reject before asking serde to turn this invalid item into InitData. - entries.push(serde_json::json!(false)); - assert_bound_error_both( - serde_json::json!({"version": "draft-01", "initDataList": entries}), - "MSF catalog exceeds 64 entries", - ); - } - - #[test] - fn track_and_init_entry_exact_limits_are_accepted() { - let tracks: Vec<_> = (0..MAX_TRACKS) + fn more_than_64_tracks_and_init_entries_are_accepted_within_the_byte_limits() { + const ENTRIES: usize = 65; + let tracks: Vec<_> = (0..ENTRIES) .map(|i| serde_json::json!({"name": format!("v{i}"), "packaging": "loc", "initRef": format!("i{i}")})) .collect(); - let entries: Vec<_> = (0..MAX_TRACKS) + let entries: Vec<_> = (0..ENTRIES) .map(|i| serde_json::json!({"id": format!("i{i}"), "type": "inline", "data": "AQID"})) .collect(); let value = serde_json::json!({"version": "draft-01", "tracks": tracks, "initDataList": entries}); @@ -753,7 +674,7 @@ mod test { let from_string = Catalog::<()>::from_str(&json).unwrap(); let from_value = serde_json::from_value::>(value).unwrap(); assert_eq!(from_string, from_value); - assert_eq!(from_string.tracks.len(), MAX_TRACKS); + assert_eq!(from_string.tracks.len(), ENTRIES); assert!( from_string .tracks @@ -801,8 +722,9 @@ mod test { #[test] fn shared_references_accept_exact_resolved_byte_limit() { - let payload = "A".repeat(MAX_CATALOG_BYTES / MAX_TRACKS); - let mut tracks: Vec<_> = (0..MAX_TRACKS) + const TRACKS: usize = 64; + let payload = "A".repeat(MAX_CATALOG_BYTES / TRACKS); + let mut tracks: Vec<_> = (0..TRACKS) .map(|i| serde_json::json!({"name": format!("v{i}"), "packaging": "loc", "initRef": "shared"})) .collect(); // Matching inline + reference counts once, preserving upstream semantics. @@ -817,7 +739,7 @@ mod test { Catalog::<()>::from_str(&json).unwrap(), serde_json::from_value::>(value).unwrap(), ] { - assert_eq!(catalog.tracks.len(), MAX_TRACKS); + assert_eq!(catalog.tracks.len(), TRACKS); assert_eq!( catalog .tracks