From cc6a0f291f9d01a26514dd71b866817935cc2afa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 15:14:57 -0700 Subject: [PATCH 1/6] chore: claim compressed IETF signaling quest Co-Authored-By: GPT-6 From 233bc1021d58c37e700049f1ed87e8251a3eb86c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 15:22:03 -0700 Subject: [PATCH 2/6] wip(moq-net): decode the IETF compression setup offer Checkpoint only: no compression capability is advertised and track/object behavior is unchanged. Four focused codec/peer tests pass through just rs test; full CI, drafts and interop remain for the complete implementation. Blocked on policy for compression declarations inside Immutable Properties: mutable-only declarations preserve unconditional plain fallback; allowing sealed declarations requires refusing or weakening fallback. Do not implement either until the maintainer decides. Co-Authored-By: GPT-6 --- rs/moq-net/src/client.rs | 1 + rs/moq-net/src/ietf/flate.rs | 61 +++++++++++++++++++++++++++++++ rs/moq-net/src/ietf/mod.rs | 1 + rs/moq-net/src/ietf/parameters.rs | 3 ++ rs/moq-net/src/ietf/peer.rs | 5 +++ rs/moq-net/src/ietf/publisher.rs | 1 + rs/moq-net/src/ietf/session.rs | 1 + rs/moq-net/src/server.rs | 1 + 8 files changed, 74 insertions(+) create mode 100644 rs/moq-net/src/ietf/flate.rs diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index fa06045ed8b..c17afa57103 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -462,6 +462,7 @@ impl Client { hidden: ietf::hidden::from_setup(¶meters, v), max_age: ietf::max_age::from_setup(¶meters, v), active_count: ietf::active_count::from_setup(¶meters, v), + flate: ietf::flate::from_setup(¶meters, v)?, ..Default::default() }; diff --git a/rs/moq-net/src/ietf/flate.rs b/rs/moq-net/src/ietf/flate.rs new file mode 100644 index 00000000000..c3eb85c8959 --- /dev/null +++ b/rs/moq-net/src/ietf/flate.rs @@ -0,0 +1,61 @@ +//! Negotiation for DEFLATE-compressed tracks (draft-lcurley-moq-flate). + +use crate::coding::DecodeError; + +use super::{ParameterVarInt, Parameters, Version}; + +/// FLATE Setup Option, whose value 1 offers the group-scoped DEFLATE format. +pub(super) const OPTION: u64 = 0x40B6A; + +pub(crate) fn supported(version: Version) -> bool { + !matches!(version, Version::Draft14 | Version::Draft15) +} + +#[cfg(test)] +fn into_setup(parameters: &mut Parameters, version: Version) { + if supported(version) { + parameters.set_varint(ParameterVarInt::Flate, 1); + } +} + +pub(crate) fn from_setup(parameters: &Parameters, version: Version) -> Result { + if !supported(version) { + return Ok(false); + } + match parameters.get_varint(ParameterVarInt::Flate) { + None => Ok(false), + Some(1) => Ok(true), + Some(_) => Err(DecodeError::InvalidValue), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn negotiation_requires_an_offer_on_draft16_or_later() { + for version in Version::ALL { + let mut parameters = Parameters::default(); + assert!(!from_setup(¶meters, version).unwrap()); + into_setup(&mut parameters, version); + assert_eq!(from_setup(¶meters, version).unwrap(), supported(version)); + use crate::coding::{Decode, Encode}; + let encoded = parameters.encode_bytes(version).unwrap(); + let (decoded, consumed) = Parameters::decode_slice(&encoded, version).unwrap(); + assert_eq!(consumed, encoded.len()); + assert_eq!(from_setup(&decoded, version).unwrap(), supported(version)); + for invalid in [0, 2, u64::MAX >> 2] { + parameters.set_varint(ParameterVarInt::Flate, invalid); + assert_eq!(from_setup(¶meters, version).is_err(), supported(version)); + } + } + } + + #[test] + fn setup_code_point_is_even_and_not_grease() { + assert_eq!(OPTION % 2, 0); + assert_ne!((OPTION - 0x9d) % 0x7f, 0); + const { assert!(OPTION >= 0x4000) }; + } +} diff --git a/rs/moq-net/src/ietf/mod.rs b/rs/moq-net/src/ietf/mod.rs index 6b08cb6ea84..af0902c68a0 100644 --- a/rs/moq-net/src/ietf/mod.rs +++ b/rs/moq-net/src/ietf/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod epoch; pub(crate) mod error; mod fetch; mod filter; +pub(crate) mod flate; mod goaway; mod group; pub mod hidden; diff --git a/rs/moq-net/src/ietf/parameters.rs b/rs/moq-net/src/ietf/parameters.rs index a662500e4b8..976a4c05e6e 100644 --- a/rs/moq-net/src/ietf/parameters.rs +++ b/rs/moq-net/src/ietf/parameters.rs @@ -35,6 +35,8 @@ pub enum ParameterVarInt { MaxAge = super::max_age::OPTION, /// EPOCH, from the MoQ Epoch extension. Epoch = super::epoch::EPOCH, + /// DEFLATE-compressed tracks, from the MoQ Flate extension. + Flate = super::flate::OPTION, #[num_enum(catch_all)] Unknown(u64), } @@ -98,6 +100,7 @@ impl Decode for Parameters { ParameterVarInt::MaxRequestUpdates if !super::request_update::supported(version) => { ParameterVarInt::Unknown(kind) } + ParameterVarInt::Flate if !super::flate::supported(version) => ParameterVarInt::Unknown(kind), kind => kind, }; if !matches!(kind, ParameterVarInt::Unknown(_)) && params.get_varint(kind).is_some() { diff --git a/rs/moq-net/src/ietf/peer.rs b/rs/moq-net/src/ietf/peer.rs index 05f7ec9fb59..0f742ad69a3 100644 --- a/rs/moq-net/src/ietf/peer.rs +++ b/rs/moq-net/src/ietf/peer.rs @@ -33,6 +33,9 @@ pub(crate) struct Peer { /// MoQ Epoch: whether both sides offered it, so advertisements and requests carry the /// publisher's epoch in both directions. pub epoch: bool, + + /// Whether this draft-16+ peer offered DEFLATE-compressed tracks. + pub flate: bool, } /// Shared slot for [`Peer`], filled when the peer's SETUP is read. @@ -118,6 +121,7 @@ mod tests { auth: false, active_count: false, epoch: false, + flate: false, }; let slot = PeerSetup::default(); @@ -133,6 +137,7 @@ mod tests { auth: true, active_count: true, epoch: true, + flate: true, }); assert_eq!(slot.get().await, first); diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index e7107564ede..8789a766890 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -14266,6 +14266,7 @@ mod tests { auth: false, active_count: false, epoch: false, + flate: false, }); slot } diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index 8c7b01bd03e..c9e57dd050e 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -758,6 +758,7 @@ fn peer_from_params(params: &ietf::Parameters, version: Version) -> Result Date: Wed, 21 Oct 2026 17:21:52 -0700 Subject: [PATCH 3/6] wip(net): carry negotiated IETF compression and sealed declarations Implement the approved sealed-property refusal policy and bounded stored-form object admission. Focused compression regressions pass across negotiated IETF and lite relay paths. Draft16 TRACK_STATUS cannot carry compression metadata: forcing SUBSCRIBE regresses existing fetch-only guarantees, so its query signaling needs a maintainer decision before publication. No new wire shape is chosen for that gap. Co-Authored-By: GPT-6 --- doc/concept/standard.md | 3 +- drafts/draft-lcurley-moq-flate.md | 45 ++++++- rs/moq-net/benches/session.rs | 23 +++- rs/moq-net/src/client.rs | 1 + rs/moq-net/src/ietf/flate.rs | 32 ++++- rs/moq-net/src/ietf/object.rs | 105 ++++++++++++++++- rs/moq-net/src/ietf/properties.rs | 27 ++++- rs/moq-net/src/ietf/publish.rs | 2 + rs/moq-net/src/ietf/publisher.rs | 144 ++++++++++++++++++++--- rs/moq-net/src/ietf/session.rs | 1 + rs/moq-net/src/ietf/subscribe.rs | 3 + rs/moq-net/src/ietf/subscriber.rs | 45 +++++-- rs/moq-net/src/ietf/track.rs | 1 + rs/moq-net/src/ietf/version.rs | 1 + rs/moq-net/src/lite/publisher.rs | 2 + rs/moq-net/src/model/track.rs | 7 +- rs/moq-net/src/server.rs | 1 + rs/moq-net/src/test_interop.rs | 1 + rs/moq-net/tests/integration/compress.rs | 142 +++++++++++++++++++--- 19 files changed, 529 insertions(+), 57 deletions(-) diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 59605edbf21..90e80089aa3 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -26,7 +26,8 @@ maps everything else to "not supported" or a harmless equivalent. The subset drops. What a peer actually observes against this implementation: - **Pull.** Subscribers ask. Single-track `PUBLISH` offers are declined; announce a namespace and serve the resulting subscriptions. Announcements go out unsolicited, and we also ask for every prefix we may discover. The solicit `SETUP` option makes us wait to be asked. Without it, a draft 16+ peer also gets each match as a `NAMESPACE` on its `SUBSCRIBE_NAMESPACE` stream, so it hears the namespace twice. We never send `PUBLISH`: on drafts 16 and 17 a `SUBSCRIBE_NAMESPACE` asking only for `PUBLISH` is refused, one asking for both gets only `NAMESPACE`, and any other Subscribe Options value closes the session. -- **History is one group.** In Rust, Object ID gaps must be marked by Prior Object ID Gap properties for a group to be fetchable. A `FETCH` returns one group from the cache, or on drafts 14 through 19 the saved prefix of the group a new subscription just joined. A range of groups is refused, as is a draft 20+ `LOCATION_FILTER` that ends at Largest Object rather than an absolute End Location. On draft 20 and later, `FETCH_OK`'s End Location is the requested end, capped at Largest Object: a range holding no objects gets an empty fetch stream, and only a start past Largest Object is refused, with `INVALID_RANGE`. A relay's copy with no upstream subscription can't know Largest Object short of the track's end, so it echoes the requested end instead. JavaScript publishing refuses every `FETCH`. Datagrams are never fetchable. A Rust reader that only fetches learns the track from `TRACK_STATUS` instead of subscribing, so a finished track stays fetchable; draft 17 still subscribes, since its `TRACK_STATUS` answer cannot say whether the track is timed, and so does a reader whose publisher refuses `TRACK_STATUS` as `NOT_SUPPORTED`. +- **History is one group.** In Rust, Object ID gaps must be marked by Prior Object ID Gap properties for a group to be fetchable. A `FETCH` returns one group from the cache, or on drafts 14 through 19 the saved prefix of the group a new subscription just joined. A range of groups is refused, as is a draft 20+ `LOCATION_FILTER` that ends at Largest Object rather than an absolute End Location. On draft 20 and later, `FETCH_OK`'s End Location is the requested end, capped at Largest Object: a range holding no objects gets an empty fetch stream, and only a start past Largest Object is refused, with `INVALID_RANGE`. A relay's copy with no upstream subscription can't know Largest Object short of the track's end, so it echoes the requested end instead. JavaScript publishing refuses every `FETCH`. Datagrams are never fetchable. A Rust reader that only fetches learns the track from `TRACK_STATUS` instead of subscribing, so a finished track stays fetchable; draft 17 still subscribes, since its `TRACK_STATUS` answer cannot say whether the track is timed, and draft 16 does when compression is negotiated, since its answer cannot declare that, and so does a reader whose publisher refuses `TRACK_STATUS` as `NOT_SUPPORTED`. +- **Compression.** Rust peers on draft 16 and later exchange compressed tracks when both offer the [Flate extension](/draft/moq-flate). Older peers receive one shared inflated copy. A publisher that seals the compression declaration makes that fallback unavailable: an incompatible peer is refused, preserving the sealed metadata. JavaScript does not offer the IETF extension. - **Timing.** A track whose `SUBSCRIBE_OK` declares no `TIMESCALE` is untimed, as is every track on drafts 14 through 16. A Rust reader that never subscribed takes the units from `TRACK_STATUS` on draft 18 and later. - **Strict SETUP and parameters.** Repeated unknown `SETUP` options, GREASE included, are accepted if well-formed; a repeated known option is rejected. On draft 17 and later, a `GROUP_ORDER` other than Ascending (1) or Descending (2) closes the session. Message parameters follow the negotiated draft's lists: drafts 14 and 15 ignore an unknown or misplaced parameter, draft 16 ignores one defined only for another message but closes the session on an unknown one, and draft 17 and later close it on both. Other malformed control input closes the session with the draft's code too, such as a server `SETUP` carrying `PATH` or `AUTHORITY`, a request cut short by FIN, or a second `GOAWAY` on one stream; a first `GOAWAY` on a request stream is legal. - **One `SETUP` credential**, forwarded to the [auth server](/bin/relay/auth#the-contract) unverified. A second `SETUP` token closes the session. We advertise no token cache, so an alias reference in `SETUP` closes it with `PROTOCOL_VIOLATION`, one on a request with `UNKNOWN_AUTH_TOKEN_ALIAS`, and a registration on a request with `AUTH_TOKEN_CACHE_OVERFLOW`. Where the Rust side authorizes the peer or verifies tokens, a token on a `SUBSCRIBE` or `PUBLISH_NAMESPACE` is verified like an in-band session token and authorizes that one request alone, even where that authorization already covers it; it never widens the session. It is refused with `NOT_SUPPORTED` when nothing verifies it, when it is empty, or when its Token Type is anything but 0 (out of band). A side that does neither, JavaScript included, ignores the token. Rust refuses a request carrying more than one distinct token with `NOT_SUPPORTED`, since it verifies one; JavaScript, which verifies none, ignores them all. A token-authorized `SUBSCRIBE` still stays within the grant the peer gave us. A token-authorized `PUBLISH_NAMESPACE` needs a grant covering everything beneath the namespace, and the tracks we subscribe under it rely on that announce instead of our limit, ending with it. An update renews the token, except on a drafts 14 and 15 `PUBLISH_NAMESPACE`, which has no update, and a refused renewal ends only that request. Rust and JavaScript send the token a route names (`Route::auth`) on its `PUBLISH_NAMESPACE`, including the one drafts 14 and 15 answer a solicited `SUBSCRIBE_NAMESPACE` with, and swapping it renews the request with `REQUEST_UPDATE` from draft 16, or ends and re-sends it before; the inline `NAMESPACE` that answers a solicited `SUBSCRIBE_NAMESPACE` from draft 16 carries no token. The token belongs to the session that added it, so a reconnect means adding it again and re-announcing. A token on any other request is decoded and grants nothing, and a request repeating one is refused with `MALFORMED_AUTH_TOKEN`. `SUBSCRIBE_TRACKS` reads its namespace before refusal; its token is not decoded. On draft 17 and later, the negotiated [MoQ Auth extension](/concept/moq-lite#authorization) accepts additional session tokens in band. diff --git a/drafts/draft-lcurley-moq-flate.md b/drafts/draft-lcurley-moq-flate.md index 29828b52e54..025db944059 100644 --- a/drafts/draft-lcurley-moq-flate.md +++ b/drafts/draft-lcurley-moq-flate.md @@ -84,9 +84,39 @@ A consumer MUST treat an object that inflates to more or less than its declared A relay forwards the compressed objects as received to a peer whose transport declares compression. For any other peer it inflates them, once per subgroup however many such peers it serves, and sends the original objects without the declaration. +If a declaration is sealed inside Immutable Properties, the relay MUST preserve those bytes and MUST refuse delivery when it cannot remove the declaration for an inflated copy. A peer is never sent compressed bytes it was not told about. +# MoQ Transport Signaling {#signaling} +This extension applies to MoQ Transport draft-16 and later, whose Track Properties were called Track Extensions in draft-16. +A peer offers it with the FLATE Setup Option, key `0x40B6A`, an integer with value 1. +Other values are invalid. +Both endpoints MUST offer FLATE before either sends a compressed track. +Draft-14, draft-15, and a peer that did not offer FLATE receive inflated payloads without these declarations, subject to the sealed-property refusal in {{declare}}. + +## Compression Track Property {#property} +COMPRESSION, property type `0x40B6C`, has Track scope and integer value 1 for the format in {{stream}}. +Absence means plain payloads; other values are invalid. +A receiver MUST refuse a compressed track unless FLATE was negotiated. +The property can appear in the mutable list or Immutable Properties, but MUST NOT appear more than once across them. +A relay recognizes it rather than retaining a second mutable copy among unknown properties. +The declaration applies even when a subscriber omits Track Properties from an answer using INCLUDE_PROPERTIES=0: the subscriber already knows the track's properties. + +## Inflated Size Object Property {#size} +INFLATED_SIZE, property type `0x40B6E`, has Object scope and an integer value equal to the payload's size before compression. +Every Normal Object on a compressed track MUST carry it, including an empty payload, whose size is zero. +A plain track or a status object MUST NOT carry it. +It can appear in the mutable list or Immutable Properties, but MUST NOT appear more than once across them. +The base Object Payload Length remains the size of the compressed bytes. +The same rules apply to subgroup streams, FETCH streams, and datagrams. + +A receiver MUST validate the declared size against the same resource limits as an uncompressed payload before admitting it, and MUST NOT allocate a buffer solely from that declaration. +Incremental inflate output is charged to the receiver's cache budget and shared across its plain readers. +A relay restates the size beside forwarded compressed bytes and removes its mutable declaration beside inflated bytes. +A sealed declaration MUST match the stored object's inflated size; a relay MUST refuse forwarding when that cannot be preserved. + + # Security Considerations A shared window is a side channel. The compressed size of one payload reveals how much it has in common with the payloads before it, enough to recover a secret an attacker can partially guess, as CRIME and BREACH demonstrated against TLS and HTTP compression. @@ -99,8 +129,18 @@ Each open subgroup also holds a window of up to 32 KiB, so a consumer SHOULD bou # IANA Considerations -This document requests no registrations. -The format lives inside object payloads, and the declaration in the transport that carries them. +This document requests the following entry in the MOQT Setup Options registry: + +| Value | Name | Specification | +|:------|:-----|:--------------| +| 0x40B6A | FLATE | This Document | + +It also requests these entries in the MOQ Properties registry, outside the application-specific and Mandatory Track Property ranges: + +| Value | Name | Scope | Specification | +|:------|:-----|:------|:--------------| +| 0x40B6C | COMPRESSION | Track | This Document | +| 0x40B6E | INFLATED_SIZE | Object | This Document | --- back @@ -109,6 +149,7 @@ The format lives inside object payloads, and the declaration in the transport th ## Since draft-lcurley-moq-flate-00 (in progress) +- Negotiated FLATE signaling adds COMPRESSION and INFLATED_SIZE properties to MoQ Transport draft-16 and later; sealed declarations are forwarded unchanged or refused when incompatible. - Compression is declared by the transport instead of the application: {{moql}} carries a Compression codec in TRACK_INFO and each object's Inflated Size on FRAME and datagrams. - A relay inflates for a peer whose transport does not declare compression. - A datagram is compressed alone, as a stream of its own. diff --git a/rs/moq-net/benches/session.rs b/rs/moq-net/benches/session.rs index 972772fd4c0..44d5c2b4ad6 100644 --- a/rs/moq-net/benches/session.rs +++ b/rs/moq-net/benches/session.rs @@ -117,6 +117,8 @@ struct Shape { window: usize, /// Group streams a viewer's relay may have open toward it at once. credit: Option, + /// Publish compressed groups and exercise negotiated versus plain fallback egress. + compress: bool, /// Read each group via a fresh FETCH rather than a live subscription. fetch: bool, } @@ -138,6 +140,7 @@ impl Shape { window: 1024 * 1024, credit: None, fetch: false, + compress: false, }; /// What each viewer subscribes with: a burst is a backlog, which the default @@ -373,7 +376,11 @@ impl Cluster { let broadcast = origin .publish(path(publisher * shape.broadcasts + index), Default::default()) .unwrap(); - this.tracks.push(broadcast.create_track(TRACK, timed()).unwrap()); + this.tracks.push( + broadcast + .create_track(TRACK, timed().with_compress(shape.compress)) + .unwrap(), + ); this._broadcasts.push(broadcast); } @@ -1205,6 +1212,20 @@ fn session(c: &mut Criterion) { } let base = Shape::BASE; + delivery( + c, + "compression", + [1, 16].into_iter().flat_map(|publishers| { + [1, 16].map(move |viewers| Shape { + publishers, + viewers, + compress: true, + frame: 1024, + ..base + }) + }), + ); + delivery( c, "publishers", diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index c17afa57103..ab4b6c0598d 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -404,6 +404,7 @@ impl Client { ietf::hidden::into_setup(&mut parameters, ietf_encoding); ietf::max_age::into_setup(&mut parameters, ietf_encoding); ietf::active_count::into_setup(&mut parameters, ietf_encoding); + ietf::flate::into_setup(&mut parameters, ietf_encoding); let parameters = parameters.encode_bytes(ietf_encoding)?; let client = setup::Client { diff --git a/rs/moq-net/src/ietf/flate.rs b/rs/moq-net/src/ietf/flate.rs index c3eb85c8959..4f1e8279a94 100644 --- a/rs/moq-net/src/ietf/flate.rs +++ b/rs/moq-net/src/ietf/flate.rs @@ -6,13 +6,16 @@ use super::{ParameterVarInt, Parameters, Version}; /// FLATE Setup Option, whose value 1 offers the group-scoped DEFLATE format. pub(super) const OPTION: u64 = 0x40B6A; +/// DEFLATE track codec, in the MOQ Properties registry. +pub(super) const COMPRESSION: u64 = 0x40B6C; +/// Inflated Size object property, in the MOQ Properties registry. +pub(super) const INFLATED: u64 = 0x40B6E; pub(crate) fn supported(version: Version) -> bool { !matches!(version, Version::Draft14 | Version::Draft15) } -#[cfg(test)] -fn into_setup(parameters: &mut Parameters, version: Version) { +pub(crate) fn into_setup(parameters: &mut Parameters, version: Version) { if supported(version) { parameters.set_varint(ParameterVarInt::Flate, 1); } @@ -29,13 +32,36 @@ pub(crate) fn from_setup(parameters: &Parameters, version: Version) -> Result Result<(), crate::Error> { + if !compression + && info + .properties + .as_ref() + .is_some_and(|p| p.sealed().any(|id| id == COMPRESSION)) + { + return Err(crate::Error::Unsupported); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; #[test] fn negotiation_requires_an_offer_on_draft16_or_later() { - for version in Version::ALL { + for version in [ + Version::Draft14, + Version::Draft15, + Version::Draft16, + Version::Draft17, + Version::Draft18, + Version::Draft19, + Version::Draft20, + Version::Draft21, + Version::Draft22, + ] { let mut parameters = Parameters::default(); assert!(!from_setup(¶meters, version).unwrap()); into_setup(&mut parameters, version); diff --git a/rs/moq-net/src/ietf/object.rs b/rs/moq-net/src/ietf/object.rs index ed38d40734f..c37af3c138a 100644 --- a/rs/moq-net/src/ietf/object.rs +++ b/rs/moq-net/src/ietf/object.rs @@ -71,6 +71,7 @@ pub(crate) struct Decoded { pub properties: Properties, pub timestamp: Option, pub gap: Option, + pub inflated: Option, } /// The largest properties block an object may carry, which a relay forwards unchanged. @@ -105,6 +106,8 @@ struct List { time: Option, gap: Option, sealed_time: Option, + inflated: Option, + sealed_inflated: Option, immutable: bool, } @@ -118,6 +121,8 @@ fn list(data: &Bytes, version: Version, immutable: bool, lift: bool) -> Result Result Result, timescale: Option) -> Result, version: Version, inflated: Option) -> Result, Error> { + if data.is_empty() && inflated.is_none() { + return Ok(data); + } + let mut parsed = list(&Bytes::from(data), version, false, false)?; + if let Some(sealed) = parsed.sealed_inflated { + return match inflated == Some(sealed) { + true => serialize(&parsed.values, version), + false => Err(Error::Unsupported), + }; + } + parsed.values.retain(|p| p.id != super::flate::INFLATED); + if let Some(size) = inflated { + parsed.values.push(Property { + id: super::flate::INFLATED, + value: Value::Int(size), + }); + } + let out = serialize(&parsed.values, version)?; + if out.len() > MAX_BLOCK { + return Err(Error::Unsupported); + } + Ok(out) +} + +/// A compressed frame must declare its inflated size; a plain frame must not. +pub(super) fn inflated(size: Option, compressed: bool) -> Result, Error> { + if size.is_some() != compressed { + return Err(Error::MalformedTrack); + } + Ok(size) +} + #[cfg(test)] pub(crate) fn listed(properties: &Properties, version: Version) -> Vec { properties @@ -284,7 +332,10 @@ pub(crate) fn unframed(data: &Bytes, version: Version, timescale: Option, framed: bool) -> Result { let lifted = timescale.filter(|_| framed); - let parsed = list(data, version, false, lifted.is_some())?; + let mut parsed = list(data, version, false, lifted.is_some())?; + if framed { + parsed.values.retain(|p| p.id != super::flate::INFLATED); + } let properties = Properties { sealed_time: timed(parsed.sealed_time, timescale)?, block: block(&parsed.values, version)?, @@ -296,6 +347,7 @@ fn decode(data: &Bytes, version: Version, timescale: Option, framed: properties, timestamp: timed(parsed.time, timescale)?, gap: parsed.gap, + inflated: parsed.inflated, }) } @@ -809,3 +861,54 @@ mod tests { )); } } + +#[cfg(test)] +mod compression_tests { + use super::*; + fn property(id: u64, value: u64) -> Property { + Property { + id, + value: Value::Int(value), + } + } + #[test] + fn sizes_are_known_and_unknown_properties_survive() { + for version in [Version::Draft16, Version::Draft18, Version::Draft22] { + let unknown = property(0x8800, 7); + let wire = compression(serialize([&unknown], version).unwrap(), version, Some(123)).unwrap(); + let decoded = read(&wire.into(), version, None).unwrap(); + assert_eq!(decoded.inflated, Some(123)); + assert_eq!(listed(&decoded.properties, version), [unknown]); + assert_eq!(inflated(Some(0), true).unwrap(), Some(0)); + assert!(inflated(None, true).is_err()); + assert!(inflated(Some(0), false).is_err()); + } + } + #[test] + fn sealed_sizes_are_forwarded_or_refused_without_rewriting() { + for version in [Version::Draft16, Version::Draft18, Version::Draft22] { + let sealed = Property { + id: 0x0b, + value: Value::Bytes( + serialize([&property(super::super::flate::INFLATED, 23)], version) + .unwrap() + .into(), + ), + }; + let wire = serialize([&sealed], version).unwrap(); + let decoded = read(&wire.clone().into(), version, None).unwrap(); + assert_eq!(decoded.inflated, Some(23)); + assert_eq!(compression(wire.clone(), version, Some(23)).unwrap(), wire); + assert!(matches!( + compression(wire.clone(), version, None), + Err(Error::Unsupported) + )); + assert!(matches!( + compression(wire.clone(), version, Some(24)), + Err(Error::Unsupported) + )); + let repeated = serialize([&sealed, &property(super::super::flate::INFLATED, 23)], version).unwrap(); + assert!(read(&repeated.into(), version, None).is_err()); + } + } +} diff --git a/rs/moq-net/src/ietf/properties.rs b/rs/moq-net/src/ietf/properties.rs index d724f0775ea..f02aad4c62e 100644 --- a/rs/moq-net/src/ietf/properties.rs +++ b/rs/moq-net/src/ietf/properties.rs @@ -73,6 +73,9 @@ pub struct Properties { /// Whether the track takes NEW_GROUP_REQUEST (DYNAMIC_GROUPS = 1), mutable or sealed. pub dynamic_groups: bool, + /// Whether each subgroup carries the negotiated group-scoped DEFLATE codec. + pub compress: bool, + /// Everything else, Immutable Properties included, forwarded as the publisher sent it, /// serialized as the message's draft serializes them. pub(crate) unknown: Vec, @@ -100,6 +103,7 @@ impl Properties { group_order: Some(GroupOrder::Descending), // A track opted in to refresh answers a NEW_GROUP_REQUEST as a join. dynamic_groups: info.refresh.is_some(), + compress: info.compress, unknown: Vec::new(), } } @@ -146,6 +150,10 @@ impl Properties { known.push(int(DYNAMIC_GROUPS, 1)); } + if self.compress { + known.push(int(super::flate::COMPRESSION, 1)); + } + let mut properties: Vec<_> = known .iter() .filter(|property| !sealed_keys(&self.unknown, version).any(|id| id == property.id)) @@ -263,6 +271,12 @@ impl Properties { }; } DYNAMIC_GROUPS => self.dynamic_groups = dynamic_groups(value)?, + super::flate::COMPRESSION => { + if value != 1 { + return Err(DecodeError::InvalidValue); + } + self.compress = true; + } _ => return Ok(false), } if seen.contains(&id) { @@ -275,12 +289,13 @@ impl Properties { /// The Track Properties moq-net applies itself, which a relay restates from the track's /// info rather than forwarding the publisher's. -const UNDERSTOOD: [u64; 5] = [ +const UNDERSTOOD: [u64; 6] = [ MAX_CACHE_DURATION, TIMESCALE, DEFAULT_PUBLISHER_PRIORITY, DEFAULT_PUBLISHER_GROUP_ORDER, DYNAMIC_GROUPS, + super::flate::COMPRESSION, ]; /// Whether moq-net applies the Track Property `id` itself. @@ -384,6 +399,7 @@ impl TrackProperties { // Advisory: a relay serves its own order and forwards the publisher's preference. DEFAULT_PUBLISHER_GROUP_ORDER => false, DYNAMIC_GROUPS => sealed.dynamic_groups != restated.dynamic_groups, + super::flate::COMPRESSION => sealed.compress != restated.compress, _ => unreachable!("only the understood properties are seen"), }) } @@ -489,6 +505,7 @@ mod tests { priority: Some(37), group_order: Some(GroupOrder::Descending), dynamic_groups: true, + compress: false, }; let mut buf = Vec::new(); @@ -558,6 +575,7 @@ mod tests { let properties = Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: Some(Timescale::MILLI), priority: None, @@ -580,6 +598,7 @@ mod tests { let properties = Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: None, priority: None, @@ -637,6 +656,7 @@ mod tests { let mut buf = Vec::new(); Properties { dynamic_groups: true, + compress: false, ..Default::default() } .encode(&mut Encoder::new(&mut buf, Version::Draft22.into()), Version::Draft22) @@ -647,6 +667,7 @@ mod tests { // (draft 15 carries DYNAMIC_GROUPS as a SUBSCRIBE_OK parameter instead). let both = Properties { dynamic_groups: true, + compress: false, timescale: Some(Timescale::MILLI), ..Default::default() }; @@ -782,6 +803,7 @@ mod tests { priority: Some(1), group_order: Some(GroupOrder::Descending), dynamic_groups: true, + compress: false, unknown: unknown(MAX_FORWARDED).values, }; assert!( @@ -900,7 +922,8 @@ mod tests { TIMESCALE, DEFAULT_PUBLISHER_PRIORITY, DEFAULT_PUBLISHER_GROUP_ORDER, - DYNAMIC_GROUPS + DYNAMIC_GROUPS, + super::super::flate::COMPRESSION ] ); } diff --git a/rs/moq-net/src/ietf/publish.rs b/rs/moq-net/src/ietf/publish.rs index 231fe8f1688..4304d4f170f 100644 --- a/rs/moq-net/src/ietf/publish.rs +++ b/rs/moq-net/src/ietf/publish.rs @@ -901,6 +901,7 @@ mod tests { properties: Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: None, priority: None, @@ -939,6 +940,7 @@ mod tests { properties: Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: None, priority: None, diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index b4d93e2fbef..8c68f2c4153 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -49,7 +49,17 @@ fn serving_subscription(subscriber_priority: u8) -> Subscription { /// Fails with [`Error::Unsupported`] for Immutable Properties the draft cannot carry: /// rewriting them for its encoding would break them, and forwarding them unchanged would /// change their meaning. -fn track_properties(info: &track::Info, wanted: bool, version: Version) -> Result { +fn track_properties( + info: &track::Info, + wanted: bool, + version: Version, + negotiated: bool, +) -> Result { + let compressed = info.compress && negotiated && super::flate::supported(version); + super::flate::egress(info, compressed)?; + let mut plain = info.clone(); + plain.compress = compressed; + let info = &plain; if !wanted { return Ok(ietf::Properties::default()); } @@ -123,6 +133,7 @@ struct FetchObjectInfo<'a> { object: u64, prior: FetchPrior, timestamp: Option, + inflated: Option, timescale: Option, version: Version, fields: Option<&'a crate::model::object::Fields>, @@ -172,6 +183,7 @@ async fn read_fetch( skip: u64, until: Option, priority: u8, + compression: bool, ) -> Result { let info = track.query().await?; let cached = track.peek_group(sequence); @@ -198,6 +210,9 @@ async fn read_fetch( }, ) .await?; + if compression { + group = group.compressed(); + } let mut first = skip; let mut frames = Vec::new(); // Reading an object past the range proves the range missed the group's end. @@ -1284,7 +1299,12 @@ where .info() .timescale .filter(|_| ietf::Properties::sends_timescale(self.version)); - let mut properties = match track_properties(track.info(), msg.properties_wanted, self.version) { + let mut properties = match track_properties( + track.info(), + msg.properties_wanted, + self.version, + self.peer_setup.get().await.flate, + ) { Ok(properties) => properties, Err(err) => return Ok(Subscribed::Refused(err.to_string(), err)), }; @@ -1299,6 +1319,7 @@ where // it carries. let mut track_serve = TrackServe::new(self.session.clone(), track, request_id, self.version, range, timescale) + .compression(self.peer_setup.get().await.flate) .ranked(self.priority.clone()); // A fixed start is no join, so its NEW_GROUP_REQUEST asks as an update's would. if fixed.start.is_some() @@ -1865,6 +1886,11 @@ where }, ) .await?; + let group = if self.peer_setup.get().await.flate { + group.compressed() + } else { + group + }; Self::write_fetch_group(&mut stream, group, sequence, skip, until, timescale, self.version).await } .await; @@ -1956,6 +1982,7 @@ where object: index, prior: fetch_prior(version, prior, index)?, timestamp: frame.timestamp, + inflated: frame.inflated, timescale, version, fields: frame.fields.as_deref(), @@ -2003,6 +2030,7 @@ where object: index, prior: fetch_prior(version, prior, index)?, timestamp: frame.timestamp, + inflated: frame.inflated, timescale, version, fields: frame.fields.as_deref(), @@ -2056,6 +2084,7 @@ where object, prior, timestamp, + inflated, timescale, version, fields, @@ -2085,6 +2114,8 @@ where } }; + let properties = super::object::compression(properties.unwrap_or_default(), version, inflated)?; + let properties = (!properties.is_empty()).then_some(properties); if version == Version::Draft14 { let properties = properties.unwrap_or_default(); stream.buffer_varint(sequence)?; @@ -2421,7 +2452,14 @@ where // The subscriber cancelling is the only other way this ends early, and is owed // nothing. let group = { - let mut read = std::pin::pin!(read_fetch(&track, start.group, start.object, until, priority)); + let mut read = std::pin::pin!(read_fetch( + &track, + start.group, + start.object, + until, + priority, + self.peer_setup.get().await.flate + )); let mut finished = false; let mut messages = super::request_stream::Messages::new(self.session.clone(), self.control.client()); kio::wait(|waiter| { @@ -2466,7 +2504,12 @@ where return self.reject_fetch(stream, msg.request_id, &err, &err.to_string()).await; } } - let properties = match track_properties(&group.info, msg.properties_wanted, self.version) { + let properties = match track_properties( + &group.info, + msg.properties_wanted, + self.version, + self.peer_setup.get().await.flate, + ) { Ok(properties) => properties, Err(err) => return self.reject_fetch(stream, msg.request_id, &err, &err.to_string()).await, }; @@ -2599,6 +2642,7 @@ where object, prior: position, timestamp: frame.timestamp, + inflated: frame.inflated, timescale, version: self.version, fields: frame.fields.as_deref(), @@ -2616,6 +2660,7 @@ where object: end.id, prior: fetch_prior(self.version, prior, end.id)?, timestamp: None, + inflated: None, timescale, version: self.version, fields: Some(end), @@ -2809,11 +2854,13 @@ where }) .await }; - let properties = match info.map(|info| track_properties(&info?, msg.properties_wanted, self.version)) { - Some(Ok(properties)) => properties, - Some(Err(err)) => return self.reject_track_status(stream, request_id, &err).await, - None => return Ok(()), - }; + let negotiated = self.peer_setup.get().await.flate; + let properties = + match info.map(|info| track_properties(&info?, msg.properties_wanted, self.version, negotiated)) { + Some(Ok(properties)) => properties, + Some(Err(err)) => return self.reject_track_status(stream, request_id, &err).await, + None => return Ok(()), + }; // The answer stays gated until the transport takes it: one parked on stream credit // when the grant narrows is reset rather than delivered. @@ -4190,6 +4237,7 @@ struct TrackServe { /// Serve the track's datagrams too, as OBJECT_DATAGRAMs. Off when the transport has no /// datagrams; there is no stream fallback. datagrams: bool, + compression: bool, /// A track that always has another group ready still lets the session's other tasks run. budget: kio::coop::Budget, } @@ -4229,6 +4277,7 @@ impl TrackServe { Self { datagrams, + compression: false, session, track, request_id, @@ -4247,6 +4296,11 @@ impl TrackServe { } } + fn compression(mut self, negotiated: bool) -> Self { + self.compression = negotiated && super::flate::supported(self.version) && self.track.info().compress; + self + } + /// Rank this subscription's group streams in `queue`, shared with the session's other /// subscriptions. Without it they are only ranked against each other. fn ranked(mut self, queue: PriorityQueue) -> Self { @@ -4372,6 +4426,7 @@ impl TrackServe { } match self.track.poll_recv_group(waiter) { Poll::Ready(Ok(Some(group))) => { + let group = if self.compression { group.compressed() } else { group }; let sequence = group.sequence; tracing::debug!(subscribe = %self.request_id, track = %self.track.name(), sequence, "serving group"); @@ -4402,7 +4457,8 @@ impl TrackServe { // Object Property) only when TIMESCALE was declared for this version. // Drafts 14-16 cannot, so the flag stays clear. flags: ietf::GroupFlags { - has_extensions: self.timescale.is_some() + has_extensions: self.compression + || self.timescale.is_some() || wire.as_ref().is_some_and(|wire| wire.extensions), first_object: wire .as_ref() @@ -4546,6 +4602,9 @@ impl TrackServe { ) } }; + let (payload, inflated) = datagram.sent(self.compression); + let properties = super::object::compression(properties.unwrap_or_default(), self.version, inflated)?; + let properties = (!properties.is_empty()).then_some(properties); let body = ietf::ObjectDatagram { track_alias: self.request_id.0, group_id: sequence, @@ -4555,7 +4614,7 @@ impl TrackServe { properties, body: match status { Some(status) => ietf::DatagramBody::Status(status), - None => ietf::DatagramBody::Payload(datagram.datagram.payload), + None => ietf::DatagramBody::Payload(payload.clone()), }, }; let body = match body.encode_bytes(self.version) { @@ -5022,6 +5081,7 @@ impl GroupServe { delta, timestamp: batched.timestamp, size: batched.payload.len() as u64, + inflated: batched.inflated, status: 0, fields: batched.fields.as_deref(), }, @@ -5069,6 +5129,7 @@ impl GroupServe { delta: object_delta(self.prior_object, id)?, timestamp: next.timestamp, size: next.size, + inflated: next.inflated, status: 0, fields: next.fields.as_deref(), }, @@ -5100,6 +5161,7 @@ impl GroupServe { delta: object_delta(self.prior_object, end.id)?, timestamp: None, size: 0, + inflated: None, status: 3, fields: Some(&end), }, @@ -5197,6 +5259,7 @@ struct Object<'a> { delta: u64, timestamp: Option, size: u64, + inflated: Option, status: u64, fields: Option<&'a crate::model::object::Fields>, } @@ -5262,6 +5325,7 @@ fn buffer_object( ext } }; + let ext = super::object::compression(ext, encoding.version, object.inflated)?; writer.buffer_varint(ext.len() as u64)?; writer.buffer_raw(&ext); } @@ -6293,7 +6357,7 @@ mod subscribe_cursor_test { open.write_frame(crate::Timestamp::ZERO, b"a".to_vec()).unwrap(); let consumer = track.consume(); - let read = moq_clock::sim::spawn(async move { read_fetch(&consumer, 0, 0, None, 0).await.map(|_| ()) }); + let read = moq_clock::sim::spawn(async move { read_fetch(&consumer, 0, 0, None, 0, false).await.map(|_| ()) }); moq_clock::sim::yield_now().await; // The track runs a full second past the group's successor. for ms in [1000, 2000] { @@ -8234,7 +8298,7 @@ mod serve_tests { Version::Draft16 | Version::Draft17 => ietf::Properties::default(), // Only draft-20 can carry the opt-out. _ if !wanted && Filter::is_draft20(version) => ietf::Properties::default(), - _ => track_properties(&track::Info::timed(), true, version).unwrap(), + _ => track_properties(&track::Info::timed(), true, version, true).unwrap(), }; assert_eq!( ok, @@ -9279,7 +9343,7 @@ mod serve_tests { } let properties = match ietf::Properties::sends_timescale(version) { false => ietf::Properties::default(), - true => track_properties(&track::Info::timed(), true, version).unwrap(), + true => track_properties(&track::Info::timed(), true, version, true).unwrap(), }; assert_eq!(ok.properties, properties, "{version}: wrong properties"); assert_eq!( @@ -9884,7 +9948,7 @@ mod serve_tests { let expected = match ietf::Properties::sends_timescale(version) { // Drafts 14-16 write no properties, as their SUBSCRIBE_OK does not. false => ietf::Properties::default(), - true => track_properties(&info, wanted, version).unwrap(), + true => track_properties(&info, wanted, version, true).unwrap(), }; assert_eq!(ok.properties, expected, "{version} wanted={wanted}"); @@ -10083,20 +10147,23 @@ mod serve_tests { }; assert_eq!(mutable.ietf(), Some(Version::Draft18), "hop={hop}"); assert_eq!( - ids(track_properties(&mutable, true, Version::Draft18).unwrap()), + ids(track_properties(&mutable, true, Version::Draft18, true).unwrap()), [0x06, 0x3801], "hop={hop}" ); assert_eq!( - ids(track_properties(&mutable, true, Version::Draft19).unwrap()), + ids(track_properties(&mutable, true, Version::Draft19, true).unwrap()), [0x3801], "hop={hop}" ); - assert!(track_properties(&sealed, true, Version::Draft18).is_ok(), "hop={hop}"); + assert!( + track_properties(&sealed, true, Version::Draft18, true).is_ok(), + "hop={hop}" + ); assert!( matches!( - track_properties(&sealed, true, Version::Draft19), + track_properties(&sealed, true, Version::Draft19, true), Err(Error::Unsupported) ), "hop={hop}" @@ -16480,3 +16547,42 @@ mod fill_range_tests { ); } } + +#[cfg(test)] +mod compression_tests { + use super::*; + #[test] + fn sealed_compression_refuses_incompatible_egress() { + for version in [Version::Draft16, Version::Draft18, Version::Draft22] { + let mut sealed = Vec::new(); + let mut encoder = Encoder::new(&mut sealed, version.into()); + encoder.varint(super::super::flate::COMPRESSION).unwrap(); + encoder.varint(1).unwrap(); + let info = track::Info { + compress: true, + properties: Some(Arc::new(crate::model::object::TrackProperties { + version, + values: vec![crate::model::object::Property { + id: 0x0b, + value: crate::model::object::Value::Bytes(sealed.into()), + }], + })), + ..Default::default() + }; + assert!(track_properties(&info, true, version, true).unwrap().compress); + for wanted in [false, true] { + assert!(matches!( + track_properties(&info, wanted, version, false), + Err(Error::Unsupported) + )); + assert!(matches!( + track_properties(&info, wanted, Version::Draft14, true), + Err(Error::Unsupported) + )); + } + let plain = track::Info::default().with_compress(true); + assert!(!track_properties(&plain, true, version, false).unwrap().compress); + assert!(!track_properties(&plain, true, Version::Draft15, true).unwrap().compress); + } + } +} diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index c9e57dd050e..b5bd8c77f2e 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -812,6 +812,7 @@ async fn run_setup( epoch::into_setup(&mut parameters, version); } request_update::into_setup(&mut parameters, version); + super::flate::into_setup(&mut parameters, version); let parameters = parameters.encode_bytes(version)?; writer diff --git a/rs/moq-net/src/ietf/subscribe.rs b/rs/moq-net/src/ietf/subscribe.rs index 3967895258c..51973791038 100644 --- a/rs/moq-net/src/ietf/subscribe.rs +++ b/rs/moq-net/src/ietf/subscribe.rs @@ -1390,6 +1390,7 @@ mod tests { properties: Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: None, priority: None, @@ -1421,6 +1422,7 @@ mod tests { properties: Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: None, priority: None, @@ -1451,6 +1453,7 @@ mod tests { largest: None, properties: Properties { dynamic_groups: true, + compress: false, ..Default::default() }, }; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 06b43df0a2c..5386843b8ff 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -470,6 +470,7 @@ impl Target { /// metadata differs. A sealed TIMESCALE travels downstream unchanged, so it must too. fn accepted_info(properties: &ietf::Properties, version: Version) -> track::Info { let mut info = track::Info::default() + .with_compress(properties.compress) .with_timescale(properties.timescale) .with_max_age(properties.max_cache_duration) .with_priority(super::priority::from_wire(properties.priority.unwrap_or(128))) @@ -485,7 +486,10 @@ fn accepted_info(properties: &ietf::Properties, version: Version) -> track::Info /// Why a track a `version` publisher answered with `properties` is neither processed nor /// forwarded, if it is not. -fn refused(properties: &ietf::Properties, version: Version) -> Option { +fn refused(properties: &ietf::Properties, version: Version, negotiated: bool) -> Option { + if properties.compress && !(negotiated && super::flate::supported(version)) { + return Some(Error::UnsupportedExtension); + } let info = accepted_info(properties, version); info.properties.as_deref()?.refused(&info) } @@ -2647,8 +2651,11 @@ where // track from TRACK_STATUS rather than a SUBSCRIBE: a finished track refuses one, and // one made only to learn the track races the fetches it was for. Draft-17 still // SUBSCRIBEs: its TRACK_STATUS answer cannot say whether the track is timed. + // Negotiated compression also needs SUBSCRIBE_OK on draft-16: TRACK_STATUS has no properties. let mut target = match request.subscription() { - None if ietf::TrackStatusOk::describes_track(self.version) => { + None if ietf::TrackStatusOk::describes_track(self.version) + && !(self.version == Version::Draft16 && self.peer_setup.get().await.flate) => + { let Some(idle) = self.track_status(&broadcast_path, &track_name, request).await else { return; }; @@ -2938,8 +2945,11 @@ where // A track that needs a property we do not understand, or carries more than we // could forward, is neither processed nor forwarded: the subscription is // cancelled and the subscribers refused. - Ok(Some(accepted)) if refused(&accepted.properties, self.version).is_some() => { - let err = refused(&accepted.properties, self.version).expect("checked by the guard"); + Ok(Some(accepted)) + if refused(&accepted.properties, self.version, self.peer_setup.get().await.flate).is_some() => + { + let err = refused(&accepted.properties, self.version, self.peer_setup.get().await.flate) + .expect("checked by the guard"); tracing::debug!(broadcast = %self.origin.absolute(broadcast_path), track = %track_name, %err, "subscribe refused"); // Refuse before waiting on the peer to acknowledge the cancellation. if let Some(pending) = self.take_pending(request_id) { @@ -3873,7 +3883,7 @@ where let _ = stream.writer.finish(); let ok = match response { // A track that needs a property we do not understand is not forwarded. - Ok(ok) => match refused(&ok.properties, self.version) { + Ok(ok) => match refused(&ok.properties, self.version, self.peer_setup.get().await.flate) { Some(err) => Err(err), None => Ok(ok), }, @@ -4134,21 +4144,26 @@ where // timed track without a Timestamp is refused by the model. let properties = bytes::Bytes::from(datagram.properties.unwrap_or_default()); // A status object has no frame to hold its time, so its block keeps it. - let (timestamp, properties) = match status.is_some_and(|status| status != 0) { + let (timestamp, properties, inflated) = match status.is_some_and(|status| status != 0) { true => ( None, super::object::unframed(&properties, self.version, track.timescale)?, + None, ), false => { let decoded = super::object::read(&properties, self.version, track.timescale)?; super::object::check_frame(&decoded.properties, track.timescale)?; - (decoded.timestamp, decoded.properties) + (decoded.timestamp, decoded.properties, decoded.inflated) } }; let Some(producer) = track.producer.as_mut() else { return Ok(()); }; - producer.insert_wire_datagram(crate::model::datagram::Stored { + let inflated = match status.is_some_and(|s| s != 0) { + true => None, + false => super::object::inflated(inflated, producer.info().compress)?, + }; + let stored = crate::model::datagram::Stored { datagram: crate::Datagram { sequence, timestamp, @@ -4162,7 +4177,11 @@ where properties, })), deflated: None, - })?; + }; + match inflated { + Some(size) => producer.insert_deflated_datagram(stored, size)?, + None => producer.insert_wire_datagram(stored)?, + } Ok(()) } } @@ -4443,6 +4462,7 @@ struct GroupIngest { /// The property list `properties` was read from, for a status object, which has no frame. raw: bytes::Bytes, gap: Option, + inflated: Option, /// The object being read is one the group started past, so it is read and dropped. dropping: bool, phase: IngestPhase, @@ -4484,6 +4504,7 @@ impl GroupIngest { properties: Default::default(), raw: bytes::Bytes::new(), gap: None, + inflated: None, dropping: false, phase: IngestPhase::Delta, budget: subscriber.frames.clone(), @@ -4866,7 +4887,7 @@ where let header = frame::Header { info: frame::Info { size, timestamp }, fields, - inflated: None, + inflated: super::object::inflated(decoded.inflated, producer.compressed())?, }; let mut frame = producer.create_frame_owned(header, &self.frames)?; if let Err(err) = std::future::poll_fn(|cx| stream.poll_read_frame(cx, &mut frame)).await { @@ -5603,7 +5624,7 @@ impl GroupIngest { Ok(frame::Header { info, fields, - inflated: None, + inflated: super::object::inflated(self.inflated, group.compressed())?, }) } @@ -5628,6 +5649,7 @@ impl GroupIngest { self.properties = Default::default(); self.raw = bytes::Bytes::new(); self.gap = None; + self.inflated = None; self.dropping = self.start.is_some() && id < group.frame_count() as u64; self.phase = match self.has_extensions { true => IngestPhase::ExtSize, @@ -5645,6 +5667,7 @@ impl GroupIngest { self.properties = decoded.properties; self.raw = ext; self.gap = decoded.gap; + self.inflated = decoded.inflated; self.phase = IngestPhase::Size { timestamp }; } IngestPhase::Size { timestamp } => { diff --git a/rs/moq-net/src/ietf/track.rs b/rs/moq-net/src/ietf/track.rs index fbbcf8b40e6..abc97b75ce6 100644 --- a/rs/moq-net/src/ietf/track.rs +++ b/rs/moq-net/src/ietf/track.rs @@ -392,6 +392,7 @@ mod tests { let properties = Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: Some(std::time::Duration::from_secs(30)), timescale: Some(crate::Timescale::MILLI), priority: Some(7), diff --git a/rs/moq-net/src/ietf/version.rs b/rs/moq-net/src/ietf/version.rs index 0e29f184548..0688350ea0c 100644 --- a/rs/moq-net/src/ietf/version.rs +++ b/rs/moq-net/src/ietf/version.rs @@ -82,6 +82,7 @@ mod tests { let properties = Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: None, timescale: Some(Timescale::new(90_000).unwrap()), priority: Some(64), diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 5f318539282..2d20ebfebb7 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1955,6 +1955,7 @@ impl Request for TrackInfoServe { ) -> Poll, Error>> { let ietf = ready!(shared.peer_setup.poll_ietf(waiter)); let info = ready!(self.querying.poll_ok(waiter))?; + crate::ietf::flate::egress(&info, shared.version.has_compression())?; // TRACK_INFO only flows on Lite05+ (the encode errors otherwise). writer.buffer(&lite::TrackInfo { @@ -2125,6 +2126,7 @@ impl Request for SubscribeServe { Self::Confirm { subscribing, .. } => { let ietf = ready!(shared.peer_setup.poll_ietf(waiter)); let mut track = ready!(subscribing.poll_ok(waiter))?; + crate::ietf::flate::egress(track.info(), shared.version.has_compression())?; let Self::Confirm { msg, latest, update, .. } = std::mem::replace( diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 51e5894a85c..a6c9d43c6e3 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -141,9 +141,10 @@ pub struct Info { /// earlier ones in the same group. Defaults to `false`. /// /// Writers and readers see plain frames either way: the producer compresses each - /// frame as it is written and the cache stores the compressed form. A moq-lite-07 peer - /// receives the compressed bytes and the flag; every other session receives the - /// frames inflated, from one inflated copy per group shared by every plain reader. + /// frame as it is written and the cache stores the compressed form. A moq-lite-07 peer or a + /// moq-transport draft-16+ peer negotiating the Flate extension receives the compressed + /// bytes and the flag. Other sessions receive one shared inflated copy per group, + /// unless a sealed compression declaration makes that fallback unsupported. /// Worth it for small, repetitive payloads like JSON; wasted on media. pub compress: bool, /// The ratio of a group's bytes after its first frame to the first frame past which a diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 80e210534ce..4da53934384 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -721,6 +721,7 @@ where ietf::hidden::into_setup(&mut parameters, v); ietf::max_age::into_setup(&mut parameters, v); ietf::active_count::into_setup(&mut parameters, v); + ietf::flate::into_setup(&mut parameters, v); parameters.encode_bytes(v)? } Version::Lite(v) => lite::Parameters::default().encode_bytes(v)?, diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 232f6cdfd13..c555f07c2f2 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -439,6 +439,7 @@ fn ietf_track_status_interop() { _ => ietf::Properties { unknown: Default::default(), dynamic_groups: false, + compress: false, max_cache_duration: Some(std::time::Duration::from_millis(5000)), timescale: Some(crate::Timescale::MILLI), // The JS track's priority 2, lower-first on the wire. diff --git a/rs/moq-net/tests/integration/compress.rs b/rs/moq-net/tests/integration/compress.rs index e1aa55702d4..07f454abc7e 100644 --- a/rs/moq-net/tests/integration/compress.rs +++ b/rs/moq-net/tests/integration/compress.rs @@ -2,9 +2,8 @@ //! A compressed track crosses a relay to every kind of subscriber. //! -//! The publisher hop always speaks moq-lite 07, so the relay caches the DEFLATE slices as -//! received. A moq-lite 07 subscriber gets them as stored and inflates them itself; every -//! other version gets the plain frames the relay inflates once for all of them. +//! Negotiated IETF and lite-07 hops keep DEFLATE slices as received. Peers that cannot +//! signal compression receive the plain frames the relay inflates once for all of them. use crate::support; @@ -15,8 +14,18 @@ use support::harness::{MockConnectOptions, MockPair, connect_mock}; const TIMEOUT: Duration = Duration::from_secs(10); -/// Every downstream wire: one that carries compression, and ones that do not. -const VERSIONS: &[&str] = &["moq-lite-07-wip", "moq-lite-06", "moq-lite-03", "moq-transport-17"]; +/// Negotiated IETF and lite-07 wires, plus versions that require plain fallback. +const VERSIONS: &[&str] = &[ + "moq-lite-07-wip", + "moq-lite-06", + "moq-lite-03", + "moq-transport-14", + "moq-transport-15", + "moq-transport-16", + "moq-transport-17", + "moq-transport-18", + "moq-transport-22", +]; fn compressed() -> moq_net::track::Info { moq_net::track::Info::default() @@ -43,16 +52,15 @@ struct Fixture { _origins: Vec, } -/// A compressed track published over moq-lite 07 to a relay, and subscribed from the relay -/// over `downstream`. -async fn connect(downstream: &str) -> Fixture { +/// A compressed track crosses a relay between the requested protocols. +async fn connect(upstream: &str, downstream: &str) -> Fixture { let publisher = produce_origin(1); let broadcast = publisher.create_broadcast("bcast").unwrap(); let track = broadcast.create_track("data", compressed()).unwrap(); broadcast.announce(Default::default()).unwrap(); let relay = produce_origin(2); - let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); + let mut options = MockConnectOptions::new(upstream.parse::().unwrap()); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(relay.clone()); let upstream = connect_mock(options).await; @@ -79,15 +87,18 @@ async fn connect(downstream: &str) -> Fixture { } /// Whole and chunked frames on a compressed track read back as written on every version, -/// and only a moq-lite 07 subscriber learns the track is compressed. +/// and only a peer that can signal it learns the track is compressed. #[moq_clock::test] async fn relay_serves_every_version() { for &version in VERSIONS { moq_clock::timeout(TIMEOUT, async { - let mut fixture = connect(version).await; + let mut fixture = connect("moq-lite-07-wip", version).await; assert_eq!( fixture.subscriber.info().compress, - version == "moq-lite-07-wip", + !matches!( + version, + "moq-lite-06" | "moq-lite-03" | "moq-transport-14" | "moq-transport-15" + ), "{version}: compression declared" ); @@ -110,6 +121,9 @@ async fn relay_serves_every_version() { frame.write(bytes::Bytes::copy_from_slice(chunk)).unwrap(); } frame.finish().unwrap(); + group + .write_frame(Timestamp::from_millis(3).unwrap(), bytes::Bytes::new()) + .unwrap(); group.finish().unwrap(); let mut group = fixture.subscriber.recv_group().await.unwrap().expect("group"); @@ -117,6 +131,7 @@ async fn relay_serves_every_version() { let frame = group.read_frame().await.unwrap().expect("frame"); assert_eq!(&frame.payload[..], &payload(index)[..], "{version}: frame {index}"); } + assert!(group.read_frame().await.unwrap().unwrap().payload.is_empty()); assert!(group.read_frame().await.unwrap().is_none(), "{version}: extra frame"); }) .await @@ -128,9 +143,9 @@ async fn relay_serves_every_version() { /// compressed or inflated. #[moq_clock::test] async fn datagrams_serve_every_version() { - for &version in &["moq-lite-07-wip", "moq-lite-06"] { + for &version in &["moq-lite-07-wip", "moq-lite-06", "moq-transport-16", "moq-transport-22"] { moq_clock::timeout(TIMEOUT, async { - let mut fixture = connect(version).await; + let mut fixture = connect("moq-lite-07-wip", version).await; fixture .track .insert_datagram(0, Timestamp::from_millis(0).unwrap(), payload(0)) @@ -215,3 +230,102 @@ async fn datagrams_meter_the_bytes_on_the_wire() { .unwrap_or_else(|_| panic!("{version}: timed out")); } } + +/// Negotiated IETF ingress stays compressed across a relay to IETF or lite-07 egress. +#[moq_clock::test] +async fn ietf_ingress_stays_compressed_across_protocols() { + for upstream in [ + "moq-transport-16", + "moq-transport-17", + "moq-transport-18", + "moq-transport-22", + ] { + for downstream in [ + "moq-lite-07-wip", + "moq-transport-16", + "moq-transport-22", + "moq-transport-14", + ] { + moq_clock::timeout(TIMEOUT, async { + let mut fixture = connect(upstream, downstream).await; + assert_eq!( + fixture.subscriber.info().compress, + downstream != "moq-transport-14", + "{upstream} -> {downstream}" + ); + let mut group = fixture.track.append_group().unwrap(); + for index in 0..3 { + group + .write_frame(Timestamp::from_millis(index).unwrap(), payload(index)) + .unwrap(); + } + group.finish().unwrap(); + let mut received = fixture.subscriber.recv_group().await.unwrap().unwrap(); + for index in 0..3 { + assert_eq!( + received.read_frame().await.unwrap().unwrap().payload.as_ref(), + payload(index) + ); + } + assert!(received.read_frame().await.unwrap().is_none()); + fixture + .track + .insert_datagram(1, Timestamp::from_millis(3).unwrap(), payload(3)) + .unwrap(); + assert_eq!( + fixture + .subscriber + .recv_datagram() + .await + .unwrap() + .unwrap() + .payload + .as_ref(), + payload(3) + ); + }) + .await + .unwrap_or_else(|_| panic!("{upstream} -> {downstream}: timed out")); + } + } +} + +/// A finished compressed track remains fetchable without subscribing to its live feed. +#[moq_clock::test] +async fn ietf_fetch_keeps_the_stored_form() { + for version in ["moq-transport-18", "moq-transport-22"] { + moq_clock::timeout(TIMEOUT, async { + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("bcast").unwrap(); + let track = broadcast.create_track("data", compressed()).unwrap(); + let mut group = track.append_group().unwrap(); + for index in 0..3 { + group + .write_frame(Timestamp::from_millis(index).unwrap(), payload(index)) + .unwrap(); + } + group.finish().unwrap(); + track.finish().unwrap(); + broadcast.announce(Default::default()).unwrap(); + let origin = produce_origin(2); + let mut options = MockConnectOptions::new(version.parse().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(origin.clone()); + let _pair = connect_mock(options).await; + origin.consume().routed("bcast").await.unwrap(); + let remote = origin.consume().request_broadcast("bcast", None).await.unwrap(); + let consumer = remote.track("data").unwrap(); + assert!(consumer.query().await.unwrap().compress); + let mut fetched = consumer.fetch_group(0, moq_net::group::Fetch::default()).await.unwrap(); + for index in 0..3 { + assert_eq!( + fetched.read_frame().await.unwrap().unwrap().payload.as_ref(), + payload(index) + ); + } + assert!(fetched.read_frame().await.unwrap().is_none()); + }) + .await + .unwrap_or_else(|_| panic!("{version}: timed out")); + } +} From 26891196a9943fbc63ae74532f23a2e2f45fa348 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 17:31:03 -0700 Subject: [PATCH 4/6] wip(net): verify compression bounds and preserve pending query decision Focused compression tests: 43 passed. Draft rendering and cross-language interop --all passed. Full moq-net: 2870 passed, two draft16 fetch-only failures, eight skipped. CI stops at the unchanged runner-image user-namespace restriction. The publisher/viewer benchmark grid completed. The temporary base fixture overlay is restored and excluded. Keep this checkpoint local until the maintainer chooses a negotiated Track Properties parameter, trailing block, or support beginning at draft17. Co-Authored-By: GPT-6 --- quest/m1/compress/ietf.md | 8 ++++++++ rs/moq-net/src/ietf/flate.rs | 25 ++++++++++++++----------- rs/moq-net/src/ietf/object.rs | 26 ++++++++++++++++++++++++++ rs/moq-net/src/ietf/publisher.rs | 21 ++++++++++++++++++++- rs/moq-net/src/ietf/subscriber.rs | 31 +++++++++++++++++++++++++++++++ rs/moq-net/src/lite/publisher.rs | 26 ++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 12 deletions(-) diff --git a/quest/m1/compress/ietf.md b/quest/m1/compress/ietf.md index 8fdff029543..ba98802a784 100644 --- a/quest/m1/compress/ietf.md +++ b/quest/m1/compress/ietf.md @@ -14,6 +14,14 @@ compressed on lite-07. See the [epic](/quest/m1/compress/README.md) for the decisions. +- The maintainer permits compression declarations inside Immutable Properties. + Preserve them unchanged and refuse plain fallback whenever that would remove a + sealed track declaration or object size. +- Draft-16's TRACK_STATUS answer has no Track Extensions block. Before publishing, + settle whether mutual FLATE adds a length-prefixed Track Properties parameter + using the existing codec (recommended), adds a trailing block to REQUEST_OK, or + compression starts at draft-17. Forcing a live SUBSCRIBE to learn metadata + regresses existing fetch-only behavior, including finished tracks. - A setup parameter is required because receivers discard unknown track properties: without it an old peer would read compressed bytes as plain. - Rewrite `drafts/draft-lcurley-moq-flate.md` as the MoQT extension: the diff --git a/rs/moq-net/src/ietf/flate.rs b/rs/moq-net/src/ietf/flate.rs index 4f1e8279a94..b48987c0f37 100644 --- a/rs/moq-net/src/ietf/flate.rs +++ b/rs/moq-net/src/ietf/flate.rs @@ -45,23 +45,26 @@ pub(crate) fn egress(info: &crate::track::Info, compression: bool) -> Result<(), Ok(()) } +/// A sealed object size cannot describe a plain fallback on another protocol either. +pub(crate) fn egress_object( + properties: &crate::model::object::Properties, + source: Option, + compression: bool, +) -> Result<(), crate::Error> { + let Some(block) = properties.block.as_ref().filter(|b| !compression && b.immutable) else { + return Ok(()); + }; + super::object::compression(block.bytes.to_vec(), source.ok_or(crate::Error::MalformedTrack)?, None)?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; #[test] fn negotiation_requires_an_offer_on_draft16_or_later() { - for version in [ - Version::Draft14, - Version::Draft15, - Version::Draft16, - Version::Draft17, - Version::Draft18, - Version::Draft19, - Version::Draft20, - Version::Draft21, - Version::Draft22, - ] { + for version in crate::Versions::all().iter().filter_map(|v| Version::try_from(*v).ok()) { let mut parameters = Parameters::default(); assert!(!from_setup(¶meters, version).unwrap()); into_setup(&mut parameters, version); diff --git a/rs/moq-net/src/ietf/object.rs b/rs/moq-net/src/ietf/object.rs index c37af3c138a..2ddf31913cd 100644 --- a/rs/moq-net/src/ietf/object.rs +++ b/rs/moq-net/src/ietf/object.rs @@ -865,6 +865,27 @@ mod tests { #[cfg(test)] mod compression_tests { use super::*; + #[test] + fn inflated_size_is_admitted_before_any_inflate() { + let broadcast = crate::broadcast::Info::default().produce(); + let track = broadcast + .create_track("data", crate::track::Info::default().with_compress(true)) + .unwrap(); + let group = track.append_group().unwrap(); + let header = crate::frame::Header { + info: crate::frame::Info { + size: 1, + timestamp: None, + }, + fields: None, + inflated: inflated(Some(crate::group::MAX_CACHE_BYTES + 1), true).unwrap(), + }; + assert!(matches!( + group.create_frame_owned(header, &crate::frame::Budget::default()), + Err(Error::FrameTooLarge) + )); + } + fn property(id: u64, value: u64) -> Property { Property { id, @@ -898,6 +919,11 @@ mod compression_tests { let wire = serialize([&sealed], version).unwrap(); let decoded = read(&wire.clone().into(), version, None).unwrap(); assert_eq!(decoded.inflated, Some(23)); + assert!(super::super::flate::egress_object(&decoded.properties, Some(version), true).is_ok()); + assert!(matches!( + super::super::flate::egress_object(&decoded.properties, Some(version), false), + Err(Error::Unsupported) + )); assert_eq!(compression(wire.clone(), version, Some(23)).unwrap(), wire); assert!(matches!( compression(wire.clone(), version, None), diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 8c68f2c4153..81b8975170e 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -16569,7 +16569,26 @@ mod compression_tests { })), ..Default::default() }; - assert!(track_properties(&info, true, version, true).unwrap().compress); + let outgoing = track_properties(&info, true, version, true).unwrap(); + assert!(outgoing.compress); + let mut bytes = Vec::new(); + outgoing + .encode(&mut Encoder::new(&mut bytes, version.into()), version) + .unwrap(); + let decoded = + ietf::Properties::decode(&mut crate::coding::Decoder::new(&bytes, version.into()), version).unwrap(); + assert!(decoded.compress); + assert_eq!(decoded.unknown, outgoing.unknown); + let mut contradiction = info.clone(); + contradiction.compress = false; + assert!( + contradiction + .properties + .as_ref() + .unwrap() + .refused(&contradiction) + .is_some() + ); for wanted in [false, true] { assert!(matches!( track_properties(&info, wanted, version, false), diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 5386843b8ff..0a9be072d9c 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -16528,3 +16528,34 @@ mod track_property_tests { } } } + +#[cfg(test)] +mod compression_negotiation_tests { + use super::*; + #[test] + fn a_compression_declaration_requires_mutual_negotiation() { + let properties = ietf::Properties { + compress: true, + ..Default::default() + }; + for version in [ + Version::Draft14, + Version::Draft15, + Version::Draft16, + Version::Draft18, + Version::Draft22, + ] { + assert!(matches!( + refused(&properties, version, false), + Some(Error::UnsupportedExtension) + )); + assert_eq!( + refused(&properties, version, true).is_none(), + super::super::flate::supported(version) + ); + } + let info = accepted_info(&properties, Version::Draft22); + assert!(info.compress); + assert!(info.properties.unwrap().values.is_empty()); + } +} diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 2d20ebfebb7..fd7245a6552 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -4094,6 +4094,16 @@ impl TrackRun { /// Send a datagram, recording its sequence once it went out. fn serve_datagram(&mut self, datagram: crate::model::datagram::Stored) { + if let Some(wire) = &datagram.wire + && crate::ietf::flate::egress_object( + &wire.properties, + self.track.info().ietf(), + self.ctx.version.has_compression(), + ) + .is_err() + { + return; + } let sequence = datagram.sequence; // Metered in the form the wire carries. let (payload, _) = datagram.sent(self.ctx.version.has_compression()); @@ -4505,6 +4515,14 @@ impl GroupServe { } else if *batch_pos < batch.len() { let index = self.group.index() - batch.len() as u64 + *batch_pos as u64; let batched = &mut batch[*batch_pos]; + if let Some(fields) = &batched.fields + && let Err(err) = crate::ietf::flate::egress_object( + &fields.properties, + self.group.ietf(), + self.ctx.version.has_compression(), + ) { + break 'serve Err(err); + } let buffered = records .as_mut() .map_or(Ok(()), |records| { @@ -4551,6 +4569,14 @@ impl GroupServe { match self.group.poll_next_frame(waiter) { Poll::Ready(Ok(Some(next))) => { let index = self.group.index() - 1; + if let Some(fields) = &next.fields + && let Err(err) = crate::ietf::flate::egress_object( + &fields.properties, + self.group.ietf(), + self.ctx.version.has_compression(), + ) { + break 'serve Err(err); + } let buffered = records .as_mut() .map_or(Ok(()), |records| { From 02774a2ee463aa1cae0163cdf057097f27efe4c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 17:48:52 -0700 Subject: [PATCH 5/6] feat(net): negotiate IETF compression from draft17 Preserve legacy query and plain fallback behavior, refuse sealed declarations that cannot be preserved, and verify the real JS offer and subscriber consume inflated fallback. Co-Authored-By: GPT-6 --- doc/concept/standard.md | 4 +- drafts/draft-lcurley-moq-flate.md | 6 +- quest/m1/compress/README.md | 12 ++-- quest/m1/compress/ietf.md | 41 ------------ rs/moq-net/src/ietf/flate.rs | 16 ++++- rs/moq-net/src/ietf/object.rs | 32 +++++++-- rs/moq-net/src/ietf/peer.rs | 2 +- rs/moq-net/src/ietf/properties.rs | 4 +- rs/moq-net/src/ietf/publisher.rs | 58 ++++++++++++++++- rs/moq-net/src/ietf/subscriber.rs | 6 +- rs/moq-net/src/lite/subscriber.rs | 2 +- rs/moq-net/src/model/track.rs | 2 +- rs/moq-net/src/test_interop.rs | 2 +- rs/moq-net/tests/integration/compress.rs | 23 +++---- test/interop/ietf-compression.ts | 82 ++++++++++++++++++++++++ test/justfile | 2 +- 16 files changed, 212 insertions(+), 82 deletions(-) delete mode 100644 quest/m1/compress/ietf.md create mode 100644 test/interop/ietf-compression.ts diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 90e80089aa3..7fe39a8402b 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -26,8 +26,8 @@ maps everything else to "not supported" or a harmless equivalent. The subset drops. What a peer actually observes against this implementation: - **Pull.** Subscribers ask. Single-track `PUBLISH` offers are declined; announce a namespace and serve the resulting subscriptions. Announcements go out unsolicited, and we also ask for every prefix we may discover. The solicit `SETUP` option makes us wait to be asked. Without it, a draft 16+ peer also gets each match as a `NAMESPACE` on its `SUBSCRIBE_NAMESPACE` stream, so it hears the namespace twice. We never send `PUBLISH`: on drafts 16 and 17 a `SUBSCRIBE_NAMESPACE` asking only for `PUBLISH` is refused, one asking for both gets only `NAMESPACE`, and any other Subscribe Options value closes the session. -- **History is one group.** In Rust, Object ID gaps must be marked by Prior Object ID Gap properties for a group to be fetchable. A `FETCH` returns one group from the cache, or on drafts 14 through 19 the saved prefix of the group a new subscription just joined. A range of groups is refused, as is a draft 20+ `LOCATION_FILTER` that ends at Largest Object rather than an absolute End Location. On draft 20 and later, `FETCH_OK`'s End Location is the requested end, capped at Largest Object: a range holding no objects gets an empty fetch stream, and only a start past Largest Object is refused, with `INVALID_RANGE`. A relay's copy with no upstream subscription can't know Largest Object short of the track's end, so it echoes the requested end instead. JavaScript publishing refuses every `FETCH`. Datagrams are never fetchable. A Rust reader that only fetches learns the track from `TRACK_STATUS` instead of subscribing, so a finished track stays fetchable; draft 17 still subscribes, since its `TRACK_STATUS` answer cannot say whether the track is timed, and draft 16 does when compression is negotiated, since its answer cannot declare that, and so does a reader whose publisher refuses `TRACK_STATUS` as `NOT_SUPPORTED`. -- **Compression.** Rust peers on draft 16 and later exchange compressed tracks when both offer the [Flate extension](/draft/moq-flate). Older peers receive one shared inflated copy. A publisher that seals the compression declaration makes that fallback unavailable: an incompatible peer is refused, preserving the sealed metadata. JavaScript does not offer the IETF extension. +- **History is one group.** In Rust, Object ID gaps must be marked by Prior Object ID Gap properties for a group to be fetchable. A `FETCH` returns one group from the cache, or on drafts 14 through 19 the saved prefix of the group a new subscription just joined. A range of groups is refused, as is a draft 20+ `LOCATION_FILTER` that ends at Largest Object rather than an absolute End Location. On draft 20 and later, `FETCH_OK`'s End Location is the requested end, capped at Largest Object: a range holding no objects gets an empty fetch stream, and only a start past Largest Object is refused, with `INVALID_RANGE`. A relay's copy with no upstream subscription can't know Largest Object short of the track's end, so it echoes the requested end instead. JavaScript publishing refuses every `FETCH`. Datagrams are never fetchable. A Rust reader that only fetches learns the track from `TRACK_STATUS` instead of subscribing, so a finished track stays fetchable; draft 17 still subscribes, since its `TRACK_STATUS` answer cannot say whether the track is timed, and so does a reader whose publisher refuses `TRACK_STATUS` as `NOT_SUPPORTED`. +- **Compression.** Rust peers on draft 17 and later exchange compressed tracks when both offer the [Flate extension](/draft/moq-flate). Older peers receive one shared inflated copy. A publisher that seals the compression declaration makes that fallback unavailable: an incompatible peer is refused, preserving the sealed metadata. JavaScript does not offer the IETF extension. - **Timing.** A track whose `SUBSCRIBE_OK` declares no `TIMESCALE` is untimed, as is every track on drafts 14 through 16. A Rust reader that never subscribed takes the units from `TRACK_STATUS` on draft 18 and later. - **Strict SETUP and parameters.** Repeated unknown `SETUP` options, GREASE included, are accepted if well-formed; a repeated known option is rejected. On draft 17 and later, a `GROUP_ORDER` other than Ascending (1) or Descending (2) closes the session. Message parameters follow the negotiated draft's lists: drafts 14 and 15 ignore an unknown or misplaced parameter, draft 16 ignores one defined only for another message but closes the session on an unknown one, and draft 17 and later close it on both. Other malformed control input closes the session with the draft's code too, such as a server `SETUP` carrying `PATH` or `AUTHORITY`, a request cut short by FIN, or a second `GOAWAY` on one stream; a first `GOAWAY` on a request stream is legal. - **One `SETUP` credential**, forwarded to the [auth server](/bin/relay/auth#the-contract) unverified. A second `SETUP` token closes the session. We advertise no token cache, so an alias reference in `SETUP` closes it with `PROTOCOL_VIOLATION`, one on a request with `UNKNOWN_AUTH_TOKEN_ALIAS`, and a registration on a request with `AUTH_TOKEN_CACHE_OVERFLOW`. Where the Rust side authorizes the peer or verifies tokens, a token on a `SUBSCRIBE` or `PUBLISH_NAMESPACE` is verified like an in-band session token and authorizes that one request alone, even where that authorization already covers it; it never widens the session. It is refused with `NOT_SUPPORTED` when nothing verifies it, when it is empty, or when its Token Type is anything but 0 (out of band). A side that does neither, JavaScript included, ignores the token. Rust refuses a request carrying more than one distinct token with `NOT_SUPPORTED`, since it verifies one; JavaScript, which verifies none, ignores them all. A token-authorized `SUBSCRIBE` still stays within the grant the peer gave us. A token-authorized `PUBLISH_NAMESPACE` needs a grant covering everything beneath the namespace, and the tracks we subscribe under it rely on that announce instead of our limit, ending with it. An update renews the token, except on a drafts 14 and 15 `PUBLISH_NAMESPACE`, which has no update, and a refused renewal ends only that request. Rust and JavaScript send the token a route names (`Route::auth`) on its `PUBLISH_NAMESPACE`, including the one drafts 14 and 15 answer a solicited `SUBSCRIBE_NAMESPACE` with, and swapping it renews the request with `REQUEST_UPDATE` from draft 16, or ends and re-sends it before; the inline `NAMESPACE` that answers a solicited `SUBSCRIBE_NAMESPACE` from draft 16 carries no token. The token belongs to the session that added it, so a reconnect means adding it again and re-announcing. A token on any other request is decoded and grants nothing, and a request repeating one is refused with `MALFORMED_AUTH_TOKEN`. `SUBSCRIBE_TRACKS` reads its namespace before refusal; its token is not decoded. On draft 17 and later, the negotiated [MoQ Auth extension](/concept/moq-lite#authorization) accepts additional session tokens in band. diff --git a/drafts/draft-lcurley-moq-flate.md b/drafts/draft-lcurley-moq-flate.md index 025db944059..87a9087d007 100644 --- a/drafts/draft-lcurley-moq-flate.md +++ b/drafts/draft-lcurley-moq-flate.md @@ -89,11 +89,11 @@ A peer is never sent compressed bytes it was not told about. # MoQ Transport Signaling {#signaling} -This extension applies to MoQ Transport draft-16 and later, whose Track Properties were called Track Extensions in draft-16. +This extension applies to MoQ Transport draft-17 and later. A peer offers it with the FLATE Setup Option, key `0x40B6A`, an integer with value 1. Other values are invalid. Both endpoints MUST offer FLATE before either sends a compressed track. -Draft-14, draft-15, and a peer that did not offer FLATE receive inflated payloads without these declarations, subject to the sealed-property refusal in {{declare}}. +Draft-14 through draft-16, and a peer that did not offer FLATE receive inflated payloads without these declarations, subject to the sealed-property refusal in {{declare}}. ## Compression Track Property {#property} COMPRESSION, property type `0x40B6C`, has Track scope and integer value 1 for the format in {{stream}}. @@ -149,7 +149,7 @@ It also requests these entries in the MOQ Properties registry, outside the appli ## Since draft-lcurley-moq-flate-00 (in progress) -- Negotiated FLATE signaling adds COMPRESSION and INFLATED_SIZE properties to MoQ Transport draft-16 and later; sealed declarations are forwarded unchanged or refused when incompatible. +- Negotiated FLATE signaling adds COMPRESSION and INFLATED_SIZE properties to MoQ Transport draft-17 and later; sealed declarations are forwarded unchanged or refused when incompatible. - Compression is declared by the transport instead of the application: {{moql}} carries a Compression codec in TRACK_INFO and each object's Inflated Size on FRAME and datagrams. - A relay inflates for a peer whose transport does not declare compression. - A datagram is compressed alone, as a stream of its own. diff --git a/quest/m1/compress/README.md b/quest/m1/compress/README.md index 173f88607ce..d375809f106 100644 --- a/quest/m1/compress/README.md +++ b/quest/m1/compress/README.md @@ -46,11 +46,12 @@ Every interview prompt and the selected answer (✅) are in the planning PR. built on first demand, shared across readers, billed to the cache pool, and dropped with its group. A thousand old-lite viewers cost one inflate. - **Signaling:** lite-07 TrackInfo carries the codec id and the frame header the - size; IETF draft-16+ carries them only - after both peers advertise a setup parameter - ([IETF](/quest/m1/compress/ietf.md)), because unknown track properties are - discarded. Lite-01..06, IETF draft-14/15, and IETF peers without the - parameter get the inflated copy, with no flag. + size; IETF draft-17+ carries them only + after both peers advertise a setup parameter, because unknown track + properties are discarded. Lite-01..06, IETF draft-14/15/16, and IETF peers without the + parameter get the inflated copy, with no flag. A sealed compression + declaration is preserved, so delivery is refused when plain fallback would + remove it. - **Ranges.** A range starting at frame N of a compressed group is served from N. The requester only asks from N because it holds 0..N-1, so it inflates from its own cached prefix. Reading frame N without 0..N-1 is a @@ -80,7 +81,6 @@ This README owns the end-to-end proof and the guide once every child lands: ## Required -- [IETF extension](/quest/m1/compress/ietf.md) - moq-transport draft-16+ peers that negotiate the extension exchange compressed tracks - [JavaScript](/quest/m1/compress/js.md) - js/net publishes and consumes compressed tracks on lite-07 - [Migrate](/quest/m1/compress/migrate.md) - catalogs, stats, and timelines publish one compressed track; moq-flate, @moq/flate, and every `.z` name are gone diff --git a/quest/m1/compress/ietf.md b/quest/m1/compress/ietf.md deleted file mode 100644 index ba98802a784..00000000000 --- a/quest/m1/compress/ietf.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] Compressed tracks over moq-transport - -## Goal - -Two moq-transport draft-16+ peers that both advertise the compression setup -parameter exchange a compressed track as compressed bytes, with a track -property flagging it and an object extension declaring each object's -inflated size. An IETF peer without the parameter, and draft-14/15, get the -inflated copy as today. moq-relay -accepts a compressed track from a negotiated IETF publisher and forwards it -compressed on lite-07. - -## Plan - -See the [epic](/quest/m1/compress/README.md) for the decisions. - -- The maintainer permits compression declarations inside Immutable Properties. - Preserve them unchanged and refuse plain fallback whenever that would remove a - sealed track declaration or object size. -- Draft-16's TRACK_STATUS answer has no Track Extensions block. Before publishing, - settle whether mutual FLATE adds a length-prefixed Track Properties parameter - using the existing codec (recommended), adds a trailing block to REQUEST_OK, or - compression starts at draft-17. Forcing a live SUBSCRIBE to learn metadata - regresses existing fetch-only behavior, including finished tracks. -- A setup parameter is required because receivers discard unknown track - properties: without it an old peer would read compressed bytes as plain. -- Rewrite `drafts/draft-lcurley-moq-flate.md` as the MoQT extension: the - codec as today, plus the setup parameter, the track property, and the - per-object size extension. Pick values outside the ranges the base draft - reserves; validate with `just drafts check`. -- The property is known to moq-net, so it stays out of the unknown property - list a relay forwards (`track::Info::properties`), and egress to a peer - without the parameter drops it. -- Docs inline in `doc/concept/standard.md`. - -Tests: a negotiated IETF publisher -> moq-relay -> negotiated IETF and -lite-07 subscribers stays compressed; a non-negotiated IETF subscriber -reads plain. Run `just test interop --all`. - -Public API: none beyond Core. Wire: a new setup parameter, track property, -and object extension on draft-16+. diff --git a/rs/moq-net/src/ietf/flate.rs b/rs/moq-net/src/ietf/flate.rs index b48987c0f37..1c4ac1c833f 100644 --- a/rs/moq-net/src/ietf/flate.rs +++ b/rs/moq-net/src/ietf/flate.rs @@ -12,7 +12,7 @@ pub(super) const COMPRESSION: u64 = 0x40B6C; pub(super) const INFLATED: u64 = 0x40B6E; pub(crate) fn supported(version: Version) -> bool { - !matches!(version, Version::Draft14 | Version::Draft15) + !matches!(version, Version::Draft14 | Version::Draft15 | Version::Draft16) } pub(crate) fn into_setup(parameters: &mut Parameters, version: Version) { @@ -63,7 +63,7 @@ mod tests { use super::*; #[test] - fn negotiation_requires_an_offer_on_draft16_or_later() { + fn negotiation_requires_an_offer_on_draft17_or_later() { for version in crate::Versions::all().iter().filter_map(|v| Version::try_from(*v).ok()) { let mut parameters = Parameters::default(); assert!(!from_setup(¶meters, version).unwrap()); @@ -81,6 +81,18 @@ mod tests { } } + #[test] + fn legacy_setup_is_unchanged() { + for version in [Version::Draft14, Version::Draft15, Version::Draft16] { + let mut parameters = Parameters::default(); + into_setup(&mut parameters, version); + assert_eq!(parameters.get_varint(ParameterVarInt::Flate), None); + } + let mut parameters = Parameters::default(); + into_setup(&mut parameters, Version::Draft17); + assert_eq!(parameters.get_varint(ParameterVarInt::Flate), Some(1)); + } + #[test] fn setup_code_point_is_even_and_not_grease() { assert_eq!(OPTION % 2, 0); diff --git a/rs/moq-net/src/ietf/object.rs b/rs/moq-net/src/ietf/object.rs index 2ddf31913cd..2445f7b7ffc 100644 --- a/rs/moq-net/src/ietf/object.rs +++ b/rs/moq-net/src/ietf/object.rs @@ -266,7 +266,6 @@ fn timed(value: Option, timescale: Option) -> Result, version: Version, inflated: Option) -> Result, Error> { if data.is_empty() && inflated.is_none() { @@ -301,6 +300,7 @@ pub(super) fn inflated(size: Option, compressed: bool) -> Result Vec { properties @@ -327,7 +327,11 @@ pub(crate) fn read(data: &Bytes, version: Version, timescale: Option) /// on a track in `timescale`. No frame holds its time, so a mutable Timestamp stays in the /// block, as on an untimed track. pub(crate) fn unframed(data: &Bytes, version: Version, timescale: Option) -> Result { - Ok(decode(data, version, timescale, false)?.properties) + let decoded = decode(data, version, timescale, false)?; + if decoded.inflated.is_some() { + return Err(Error::MalformedTrack); + } + Ok(decoded.properties) } fn decode(data: &Bytes, version: Version, timescale: Option, framed: bool) -> Result { @@ -894,7 +898,7 @@ mod compression_tests { } #[test] fn sizes_are_known_and_unknown_properties_survive() { - for version in [Version::Draft16, Version::Draft18, Version::Draft22] { + for version in [Version::Draft17, Version::Draft18, Version::Draft22] { let unknown = property(0x8800, 7); let wire = compression(serialize([&unknown], version).unwrap(), version, Some(123)).unwrap(); let decoded = read(&wire.into(), version, None).unwrap(); @@ -905,9 +909,29 @@ mod compression_tests { assert!(inflated(Some(0), false).is_err()); } } + #[test] + fn a_status_cannot_declare_an_inflated_size() { + for version in [Version::Draft17, Version::Draft18, Version::Draft22] { + let size = property(super::super::flate::INFLATED, 0); + for value in [ + size.clone(), + Property { + id: 0x0b, + value: Value::Bytes(serialize([&size], version).unwrap().into()), + }, + ] { + let wire = serialize([&value], version).unwrap(); + assert!(matches!( + unframed(&wire.into(), version, None), + Err(Error::MalformedTrack) + )); + } + } + } + #[test] fn sealed_sizes_are_forwarded_or_refused_without_rewriting() { - for version in [Version::Draft16, Version::Draft18, Version::Draft22] { + for version in [Version::Draft17, Version::Draft18, Version::Draft22] { let sealed = Property { id: 0x0b, value: Value::Bytes( diff --git a/rs/moq-net/src/ietf/peer.rs b/rs/moq-net/src/ietf/peer.rs index 0f742ad69a3..a992a1e721a 100644 --- a/rs/moq-net/src/ietf/peer.rs +++ b/rs/moq-net/src/ietf/peer.rs @@ -34,7 +34,7 @@ pub(crate) struct Peer { /// publisher's epoch in both directions. pub epoch: bool, - /// Whether this draft-16+ peer offered DEFLATE-compressed tracks. + /// Whether this draft-17+ peer offered DEFLATE-compressed tracks. pub flate: bool, } diff --git a/rs/moq-net/src/ietf/properties.rs b/rs/moq-net/src/ietf/properties.rs index f02aad4c62e..093a4bb17d3 100644 --- a/rs/moq-net/src/ietf/properties.rs +++ b/rs/moq-net/src/ietf/properties.rs @@ -8,7 +8,7 @@ /// have no count and are read until the end of the message payload. /// /// MAX_CACHE_DURATION, TIMESCALE, DEFAULT_PUBLISHER_PRIORITY, DEFAULT_PUBLISHER_GROUP_ORDER, -/// and DYNAMIC_GROUPS are understood; the rest are kept so a relay can forward them. +/// DYNAMIC_GROUPS, and COMPRESSION are understood; the rest are kept so a relay can forward them. use std::time::Duration; use bytes::Bytes; @@ -150,7 +150,7 @@ impl Properties { known.push(int(DYNAMIC_GROUPS, 1)); } - if self.compress { + if self.compress && super::flate::supported(version) { known.push(int(super::flate::COMPRESSION, 1)); } diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 81b8975170e..72c984b51db 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -16553,7 +16553,7 @@ mod compression_tests { use super::*; #[test] fn sealed_compression_refuses_incompatible_egress() { - for version in [Version::Draft16, Version::Draft18, Version::Draft22] { + for version in [Version::Draft17, Version::Draft18, Version::Draft22] { let mut sealed = Vec::new(); let mut encoder = Encoder::new(&mut sealed, version.into()); encoder.varint(super::super::flate::COMPRESSION).unwrap(); @@ -16595,7 +16595,7 @@ mod compression_tests { Err(Error::Unsupported) )); assert!(matches!( - track_properties(&info, wanted, Version::Draft14, true), + track_properties(&info, wanted, Version::Draft16, true), Err(Error::Unsupported) )); } @@ -16604,4 +16604,58 @@ mod compression_tests { assert!(!track_properties(&plain, true, Version::Draft15, true).unwrap().compress); } } + + /// The actual JS SETUP offer selects the plain egress of a compressed Rust track. + #[moq_clock::test] + #[ignore = "requires Bun; run by just test interop"] + async fn ietf_compression_fallback_interop() { + use crate::coding::Decode; + use crate::lite::test_transport::{Log, SinkSession}; + for (version, draft) in [(Version::Draft17, 17), (Version::Draft22, 22)] { + let input = serde_json::json!({"draft": draft}); + let bytes: Vec = + serde_json::from_slice(&crate::test_interop::bun("ietf-compression.ts", input)).unwrap(); + let (parameters, consumed) = ietf::Parameters::decode_slice(&bytes, version).unwrap(); + assert_eq!(consumed, bytes.len()); + let negotiated = super::super::flate::from_setup(¶meters, version).unwrap(); + assert!(!negotiated, "JS offered an unsupported codec"); + let track = track::Producer::new( + Arc::new(crate::broadcast::Info::default()), + "data", + track::Info::timed().with_compress(true), + ); + let subscriber = track.subscribe(None); + let frames = [vec![b'a'; 1024], vec![b'b'; 1024], Vec::new()]; + let mut group = track.append_group().unwrap(); + for payload in &frames { + group.write_frame(crate::Timestamp::ZERO, payload.clone()).unwrap(); + } + group.finish().unwrap(); + track.finish().unwrap(); + let response = ietf::SubscribeOk { + request_id: None, + track_alias: 0, + largest: None, + properties: track_properties(track.info(), true, version, negotiated).unwrap(), + } + .encode_bytes(version) + .unwrap(); + let log = Log::default(); + let mut serve = TrackServe::new( + SinkSession::new(log.clone()), + subscriber, + RequestId(0), + version, + ServeRange::default(), + Some(Timescale::MILLI), + ) + .compression(negotiated); + kio::wait(|waiter| serve.poll(waiter)).await.unwrap(); + let written = log.writes.lock().unwrap().clone(); + let input = serde_json::json!({"draft": draft, "response": response.to_vec(), "group": written}); + let received: Vec> = + serde_json::from_slice(&crate::test_interop::bun("ietf-compression.ts", input)).unwrap(); + assert_eq!(received, frames, "draft{draft}: plain JS fallback"); + } + } } diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 0a9be072d9c..833b605e98a 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2651,11 +2651,8 @@ where // track from TRACK_STATUS rather than a SUBSCRIBE: a finished track refuses one, and // one made only to learn the track races the fetches it was for. Draft-17 still // SUBSCRIBEs: its TRACK_STATUS answer cannot say whether the track is timed. - // Negotiated compression also needs SUBSCRIBE_OK on draft-16: TRACK_STATUS has no properties. let mut target = match request.subscription() { - None if ietf::TrackStatusOk::describes_track(self.version) - && !(self.version == Version::Draft16 && self.peer_setup.get().await.flate) => - { + None if ietf::TrackStatusOk::describes_track(self.version) => { let Some(idle) = self.track_status(&broadcast_path, &track_name, request).await else { return; }; @@ -16542,6 +16539,7 @@ mod compression_negotiation_tests { Version::Draft14, Version::Draft15, Version::Draft16, + Version::Draft17, Version::Draft18, Version::Draft22, ] { diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 1ae7f9305b0..8fa8173f847 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -3233,7 +3233,7 @@ mod tests { }; let broadcast = crate::broadcast::Info::new().produce(); let request = broadcast.reserve_track("video").unwrap(); - let serving = ServeLoop::new(&serve, request, crate::track::Info::timed(), Some(Timescale::MILLI)); + let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); let mut arriving = serving.serving.create_group(group::Info { sequence: 2 }).unwrap(); arriving.write_frame(crate::Timestamp::ZERO, b"2".as_slice()).unwrap(); let mut reader = broadcast diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index a6c9d43c6e3..0ec194da4d2 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -142,7 +142,7 @@ pub struct Info { /// /// Writers and readers see plain frames either way: the producer compresses each /// frame as it is written and the cache stores the compressed form. A moq-lite-07 peer or a - /// moq-transport draft-16+ peer negotiating the Flate extension receives the compressed + /// moq-transport draft-17+ peer negotiating the Flate extension receives the compressed /// bytes and the flag. Other sessions receive one shared inflated copy per group, /// unless a sealed compression declaration makes that fallback unsupported. /// Worth it for small, repetitive payloads like JSON; wasted on media. diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index c555f07c2f2..5bf9412188f 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -5,7 +5,7 @@ use crate::coding::{Decoder, Encode, Encoder, Form, varint}; use crate::{ietf, lite}; // Runs a script with a JSON argument, returning its stdout. -fn bun(script: &str, input: serde_json::Value) -> Vec { +pub(crate) fn bun(script: &str, input: serde_json::Value) -> Vec { let output = std::process::Command::new("bun") .arg(format!("{}/../../test/interop/{script}", env!("CARGO_MANIFEST_DIR"))) .arg(input.to_string()) diff --git a/rs/moq-net/tests/integration/compress.rs b/rs/moq-net/tests/integration/compress.rs index 07f454abc7e..52166fd4c5d 100644 --- a/rs/moq-net/tests/integration/compress.rs +++ b/rs/moq-net/tests/integration/compress.rs @@ -97,7 +97,7 @@ async fn relay_serves_every_version() { fixture.subscriber.info().compress, !matches!( version, - "moq-lite-06" | "moq-lite-03" | "moq-transport-14" | "moq-transport-15" + "moq-lite-06" | "moq-lite-03" | "moq-transport-14" | "moq-transport-15" | "moq-transport-16" ), "{version}: compression declared" ); @@ -143,7 +143,13 @@ async fn relay_serves_every_version() { /// compressed or inflated. #[moq_clock::test] async fn datagrams_serve_every_version() { - for &version in &["moq-lite-07-wip", "moq-lite-06", "moq-transport-16", "moq-transport-22"] { + for &version in &[ + "moq-lite-07-wip", + "moq-lite-06", + "moq-transport-16", + "moq-transport-17", + "moq-transport-22", + ] { moq_clock::timeout(TIMEOUT, async { let mut fixture = connect("moq-lite-07-wip", version).await; fixture @@ -234,12 +240,7 @@ async fn datagrams_meter_the_bytes_on_the_wire() { /// Negotiated IETF ingress stays compressed across a relay to IETF or lite-07 egress. #[moq_clock::test] async fn ietf_ingress_stays_compressed_across_protocols() { - for upstream in [ - "moq-transport-16", - "moq-transport-17", - "moq-transport-18", - "moq-transport-22", - ] { + for upstream in ["moq-transport-17", "moq-transport-18", "moq-transport-22"] { for downstream in [ "moq-lite-07-wip", "moq-transport-16", @@ -250,7 +251,7 @@ async fn ietf_ingress_stays_compressed_across_protocols() { let mut fixture = connect(upstream, downstream).await; assert_eq!( fixture.subscriber.info().compress, - downstream != "moq-transport-14", + !matches!(downstream, "moq-transport-14" | "moq-transport-16"), "{upstream} -> {downstream}" ); let mut group = fixture.track.append_group().unwrap(); @@ -293,7 +294,7 @@ async fn ietf_ingress_stays_compressed_across_protocols() { /// A finished compressed track remains fetchable without subscribing to its live feed. #[moq_clock::test] async fn ietf_fetch_keeps_the_stored_form() { - for version in ["moq-transport-18", "moq-transport-22"] { + for version in ["moq-transport-16", "moq-transport-18", "moq-transport-22"] { moq_clock::timeout(TIMEOUT, async { let publisher = produce_origin(1); let broadcast = publisher.create_broadcast("bcast").unwrap(); @@ -315,7 +316,7 @@ async fn ietf_fetch_keeps_the_stored_form() { origin.consume().routed("bcast").await.unwrap(); let remote = origin.consume().request_broadcast("bcast", None).await.unwrap(); let consumer = remote.track("data").unwrap(); - assert!(consumer.query().await.unwrap().compress); + assert_eq!(consumer.query().await.unwrap().compress, version != "moq-transport-16"); let mut fetched = consumer.fetch_group(0, moq_net::group::Fetch::default()).await.unwrap(); for index in 0..3 { assert_eq!( diff --git a/test/interop/ietf-compression.ts b/test/interop/ietf-compression.ts new file mode 100644 index 00000000000..236178391ea --- /dev/null +++ b/test/interop/ietf-compression.ts @@ -0,0 +1,82 @@ +/** JS does not offer IETF FLATE and consumes Rust's inflated fallback through its subscriber. */ +import assert from "node:assert/strict"; +import { offered } from "../../js/net/src/connection/extensions.ts"; +import { exchangeSetup } from "../../js/net/src/connection/handshake.ts"; +import { NativeSession } from "../../js/net/src/ietf/adapter.ts"; +import { Group } from "../../js/net/src/ietf/object.ts"; +import { Setup } from "../../js/net/src/ietf/setup.ts"; +import { Subscribe, SubscribeOk } from "../../js/net/src/ietf/subscribe.ts"; +import { Subscriber } from "../../js/net/src/ietf/subscriber.ts"; +import type { IetfVersion } from "../../js/net/src/ietf/version.ts"; +import { createMockTransportPair } from "../../js/net/src/mock.ts"; +import * as Path from "../../js/net/src/path.ts"; +import { Reader, Stream, Writer } from "../../js/net/src/stream.ts"; + +console.debug = console.error; +const input: { draft: number; response?: number[]; group?: number[] } = JSON.parse(process.argv[2]); +const version = (0xff000000 + input.draft) as IetfVersion; +const pair = createMockTransportPair(`moqt-${input.draft}`); + +try { + if (input.response === undefined) { + // Exercise the real offer builder rather than constructing a reduced test offer. + const exchanging = exchangeSetup(pair.client, version, "compression interop", { + client: true, + extensions: offered(), + }); + const remote = new Writer(await pair.server.createUnidirectionalStream(), version); + await remote.u53(Setup.id); + await new Setup().encode(remote, version); + const incoming = pair.server.incomingUnidirectionalStreams.getReader(); + const next = await incoming.read(); + assert(next.value); + incoming.releaseLock(); + const reader = new Reader(next.value, undefined, version); + assert.equal(await reader.u53(), Setup.id); + const setup = await Setup.decode(reader, version); + await exchanging; + const bytes: number[] = []; + const writer = new Writer( + new WritableStream({ + write: (chunk) => { + bytes.push(...chunk); + }, + }), + version, + ); + await setup.parameters.encode(writer, version); + await writer.close(); + console.log(JSON.stringify(bytes)); + } else { + const subscriber = new Subscriber({ session: new NativeSession(pair.client, version, true) }); + const track = subscriber.consume(Path.from("room")).track("data").subscribe(); + try { + const request = await Stream.accept(pair.server, version); + assert(request); + assert.equal(await request.reader.u53(), Subscribe.id); + await Subscribe.decode(request.reader, version); + await request.writer.u53(SubscribeOk.id); + await request.writer.write(Uint8Array.from(input.response)); + await track.info(); + assert(input.group); + const reader = new Reader(undefined, Uint8Array.from(input.group), version); + await subscriber.handleGroup(await Group.decode(reader, version), reader); + const group = await track.recvGroup(); + assert(group); + const frames: number[][] = []; + for (;;) { + const frame = await group.readFrame(); + if (!frame) break; + frames.push([...frame.payload]); + } + console.log(JSON.stringify(frames)); + group.close(); + } finally { + subscriber.close(); + track.close(); + } + } +} finally { + pair.client.close(); + pair.server.close(); +} diff --git a/test/justfile b/test/justfile index 9f16a7112be..87bf69f974b 100644 --- a/test/justfile +++ b/test/justfile @@ -45,7 +45,7 @@ harness: # draft, and Lite AUTH_ERROR NOT_SUPPORTED after a grant in both directions. [positional-arguments] interop *args: - just rs test -p moq-net --lib varint_interop ietf_datagram_interop ietf_track_status_interop auth_unsupported_interop --run-ignored all + just rs test -p moq-net --lib varint_interop ietf_datagram_interop ietf_track_status_interop ietf_compression_fallback_interop auth_unsupported_interop --run-ignored all ./interop/interop.sh "$@" # The harnesses append one record per case to MOQ_TEST_RESULTS; flags link the page to its From 7f2371589f6a7fb5ded71f36d08a1ed45e1eb226 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 23:31:55 -0700 Subject: [PATCH 6/6] fix(net): compress SUBSCRIBE_TRACKS publishes when FLATE is negotiated The SUBSCRIBE_TRACKS publisher from main declares a listed track's properties and serves resumed PUBLISHes; both now follow the negotiated FLATE option, so a compressed track is declared and sent compressed only to a peer that offered it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/publisher/tracks.rs | 65 +++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/ietf/publisher/tracks.rs b/rs/moq-net/src/ietf/publisher/tracks.rs index 888637e605e..947fe1103cc 100644 --- a/rs/moq-net/src/ietf/publisher/tracks.rs +++ b/rs/moq-net/src/ietf/publisher/tracks.rs @@ -512,6 +512,7 @@ where ) { let namespace = request.prefix.join(&suffix).to_owned(); let absolute = self.origin.absolute(&namespace).to_owned(); + let flate = self.peer_setup.get().await.flate; // Resolving may wait on a route for the broadcast, which its going ends. let resolved = { @@ -655,7 +656,7 @@ where } // A track this draft cannot describe is skipped rather than PUBLISHed with // its properties rewritten. - let Ok(properties) = track_properties(&info, request.properties_wanted, self.version) else { + let Ok(properties) = track_properties(&info, request.properties_wanted, self.version, flate) else { skip(name); continue; }; @@ -1121,6 +1122,7 @@ where range, timescale, ) + .compression(self.peer_setup.get().await.flate) .ranked(self.priority.clone()); serve.pausable = true; serve.dynamic = dynamic; @@ -1247,10 +1249,10 @@ mod tests { } fn harness(version: Version) -> Harness { - harness_with(version, memory::Config::default()) + harness_with(version, memory::Config::default(), peer::Peer::default()) } - fn harness_with(version: Version, config: memory::Config) -> Harness { + fn harness_with(version: Version, config: memory::Config, declared: peer::Peer) -> Harness { let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let broadcast = origin.publish("room/cam", crate::origin::Route::default()).unwrap(); let tracks = ["video", "audio"] @@ -1259,7 +1261,7 @@ mod tests { .collect(); let (session, peer) = memory::pair(config); let setup = peer::PeerSetup::default(); - setup.set(peer::Peer::default()); + setup.set(declared); let publisher = Publisher::new( crate::time::sim(), session.clone(), @@ -1489,6 +1491,59 @@ mod tests { } } + /// A listed compressed track is declared and served compressed only to a peer that + /// negotiated FLATE; any other peer gets the plain copy with no declaration. + #[moq_clock::test] + async fn a_publish_compresses_only_when_negotiated() { + for version in DRAFTS { + for flate in [false, true] { + let declared = peer::Peer { + flate, + ..Default::default() + }; + let mut h = harness_with(version, memory::Config::default(), declared); + let data = h + .broadcast + .create_track("data", crate::track::Info::timed().with_compress(true)) + .unwrap(); + + let mut request = h.request(&listing(false)).await; + assert_eq!(next(&mut request).await.0, ietf::RequestOk::ID, "{version}"); + let mut stream = loop { + let mut stream = timeout(Stream::accept(&mut h.peer, version)).await.unwrap(); + assert_eq!(stream.reader.varint().await.unwrap(), ietf::Publish::ID); + let body: ietf::Body = stream.reader.decode().await.unwrap(); + let publish: ietf::Publish = body.message(version).unwrap(); + if &*publish.track_name == "data" { + assert_eq!(publish.properties.compress, flate, "{version} flate={flate}"); + break stream; + } + }; + publish_ok(&mut stream).await; + send(&mut stream, &update(2, true)).await; + assert_eq!(next(&mut stream).await.0, ietf::RequestOk::ID, "{version}"); + + let payload = vec![b'a'; 1024]; + let mut group = data.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, payload.clone()).unwrap(); + group.finish().unwrap(); + let recv = timeout(std::future::poll_fn(|cx| h.peer.poll_accept_uni(cx))) + .await + .unwrap(); + let mut reader = crate::coding::Reader::new(recv, version); + let mut bytes = Vec::new(); + while let Some(chunk) = timeout(std::future::poll_fn(|cx| reader.poll_read_chunk(cx, 1024))) + .await + .unwrap() + { + bytes.extend_from_slice(&chunk); + } + let plain = bytes.windows(payload.len()).any(|w| w == payload); + assert_eq!(plain, !flate, "{version} flate={flate}: {} bytes", bytes.len()); + } + } + } + /// FORWARD=1 asks for every track's objects, which we never push unasked: from /// draft-19 each track is skipped, and draft-18, with no way to skip, refuses it. #[moq_clock::test] @@ -2182,7 +2237,7 @@ mod tests { stream_buffer: 32, ..Default::default() }; - let mut h = harness_with(version, config); + let mut h = harness_with(version, config, peer::Peer::default()); // The REQUEST_OK stays unread, leaving no room for the REQUEST_ERROR. let mut request = h.request(&listing(false)).await; let (mut stream, ..) = h.publish().await;