From 9da199931225ba4b524c4d1725980a87a9dd061a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 05:50:50 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m2/archive-v2.md Quest-Claim: 7dc3d531ed172890 From 3bdb445b848eeface1c3e5cc553cf45af2923d0b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 06:08:46 -0800 Subject: [PATCH 2/2] feat(archive)!: the archive format ships as version 2, the only one read The reader accepts exactly version 2 and names any other version it finds. The unpublished layouts main called 2, 3, and 4 are gone: no legacy entry form, no version 2 priority byte. Object keys drop the unused timestamp field, `.info` and `end` ignore unknown members, and `Info` loses its version field. Co-Authored-By: Claude Opus 5.5 --- doc/concept/archives.md | 9 +- quest/m2/README.md | 1 - quest/m2/archive-v2.md | 84 ------------ quest/m2/time-addressing/consumers.md | 1 - rs/moq-archive/src/consumer/index.rs | 10 +- rs/moq-archive/src/consumer/mod.rs | 2 +- rs/moq-archive/src/consumer/tests.rs | 20 +++ rs/moq-archive/src/error.rs | 9 +- rs/moq-archive/src/info.rs | 164 ++++++++++-------------- rs/moq-archive/src/layout.rs | 164 ++++++++---------------- rs/moq-archive/src/lib.rs | 35 +++-- rs/moq-archive/src/producer/backends.rs | 26 +--- rs/moq-archive/src/producer/resume.rs | 8 +- rs/moq-archive/src/producer/tests.rs | 52 +++----- rs/moq-archive/src/producer/track.rs | 2 +- rs/moq-archive/src/store.rs | 16 +-- rs/moq-cli/src/archive.rs | 2 +- 17 files changed, 209 insertions(+), 396 deletions(-) delete mode 100644 quest/m2/archive-v2.md diff --git a/doc/concept/archives.md b/doc/concept/archives.md index 752c1cfe453..9e86b811ff4 100644 --- a/doc/concept/archives.md +++ b/doc/concept/archives.md @@ -67,8 +67,8 @@ Tracks have separate directories, with each track name percent-encoded as one segment. Within one broadcast prefix: - Each track's `.info` stores its timescale, when timed, and priority. -- Data objects are keyed by their first group's start, frame index, and - presentation timestamp; untimed tracks omit the timestamp. The sorted key listing is the +- Data objects are keyed by their first group's start and frame index. On a + timed track a group's start is its time, so the sorted key listing is the index, with no manifest or timeline object. - Each track's `end` records its final position or stream error. The broadcast's `.end` marks the recording complete, after its track markers are written. @@ -82,7 +82,10 @@ window and grace; the newest object of each track stays. Pruning runs only while the recorder runs: once a recording completes or its recorder stops, objects not yet due stay until something else deletes them, such as a store lifecycle rule. -The [moq-archive rustdoc](https://docs.rs/moq-archive) specifies the format for +The store format is versioned, and a reader accepts only its own version: a +prefix written in another one is refused with the version it found. Version 1 +shipped first; the current format is version 2, which cannot read version 1 +recordings. The [moq-archive rustdoc](https://docs.rs/moq-archive) specifies the format for the published version. This page follows the [layout documented in current source](https://github.com/moq-dev/moq/blob/main/rs/moq-archive/src/lib.rs#L8), which can differ from an installed package; use the reference matching your version. diff --git a/quest/m2/README.md b/quest/m2/README.md index 42df4d4cfa6..60e5eece453 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -57,7 +57,6 @@ removal, a rename, or a wire, catalog, or format change that is free until - **Catalog, drafts, and formats.** [hang-04](/quest/m2/hang04/README.md) (the `sync` section, a version per entry, no video `warmup`, `m2ts` under `mpegts`, one strict rule); - [the archive as version 2](/quest/m2/archive-v2.md); [declared ends](/quest/m2/time-addressing/consumers.md), back from m3; [finish](/quest/m2/finish-sweep.md), [TS outcomes](/quest/m2/ts-typed-outcomes.md), and [moq-json writes](/quest/m2/json-write-at.md). diff --git a/quest/m2/archive-v2.md b/quest/m2/archive-v2.md deleted file mode 100644 index d674381facb..00000000000 --- a/quest/m2/archive-v2.md +++ /dev/null @@ -1,84 +0,0 @@ -# [M] The archive format ships as version 2, and readers accept only it - -## Goal - -The stored format `moq-archive` writes at the cut is version 2, the -successor to the version 1 that `release` ships, and it is one format: the -reader accepts exactly version 2 and names the version it refused. The code -for the layouts main called 2, 3, and 4, none of which was ever published, -is gone, and `rs/moq-archive/src/lib.rs`'s format description matches what -the reader uses. - -## Plan - -Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): - -- ✅ Ship as version 2, and the reader accepts exactly it (rejected: - bumping to 5 as the one version at the cut; keeping 4 and deleting only - the legacy paths; freezing as is). The number counts published formats: - 2, 3, and 4 only ever existed on main. - -Facts found by the audit: - -- `rs/moq-archive/src/lib.rs` has `VERSION = 4` and `VERSION_MIN = 2`; - `origin/release` has `VERSION = 1`, a different layout that main already - cannot read. -- The reader still carries the unpublished layouts: - `rs/moq-archive/src/layout.rs` (`LEGACY_GROUP`, `legacy = version < 4`, - the three-field entry) and `rs/moq-archive/src/info.rs` (the version 2 - priority byte, and the `VERSION_MIN..=VERSION` range checks in both - files). -- [moq-dev/moq#8638](https://github.com/moq-dev/moq/pull/8638) changed - what version 4 means without bumping it: the entry sequence and the - key's group field became the group's start - (`rs/moq-archive/src/producer/track.rs`, `let sequence = group.start`). - A version 4 archive written before it and one written after disagree. -- The object key is `--`. Since #8638 the consumer - drops the third field: `rs/moq-archive/src/consumer/index.rs` keeps - `(start.group, start.frame)` and says "a group's sequence is its start, - so these sort by time too", while `lib.rs` still says the listing - "answers group and time lookups". The producer reads `pts` back to - resume (`producer/track.rs`). -- `.info` and the layout structs use `deny_unknown_fields` - (`info.rs`, `layout.rs` twice), so no field can be added without a - version bump. - -Recommendations for the PR (not decided): - -- The key's ``: use it or remove it. A field every implementer must - write and validate, and no reader uses, is what a freeze makes - permanent. Removing it needs the producer's resume to recover the - timestamp another way (the object's first frame); if that costs a GET - per resume, keeping the field and making the consumer's time lookup use - it is the better end. Say which in the PR. -- Tolerate unknown fields in `.info` and the layout records, so version 2 - can gain an optional field without becoming version 3. Keep - `deny_unknown_fields` only where an ignored field would change what the - bytes mean. -- Decide on purpose whether per-frame timestamps stay absolute while - sequences are delta coded; the audit had low confidence it matters. - -For the PR: - -- First check moq.pro for stores written by a main build (its staging - recorder): those archives say 2, 3, or 4 and mean something else. The - format is unpublished, so they are rewritten or discarded, not read; - the Impact section says so. A deployed store that must stay readable is - the one thing that reopens the decision. -- A version 1 archive stays unreadable, as on main today; the error names - the version found and the one supported, so nobody mistakes a release - archive for a corrupt one. -- Update `doc/concept/archives.md` and the crate docs to the one format. - -Tests: a version 2 archive round-trips through producer and consumer; an -`.info` or layout that says 1, 3, or 4 is refused with the typed version -error; resume after a restart finds the last object; if unknown fields are -tolerated, a record with an extra field reads. - -Public API: none expected beyond the version constants. Wire: the stored -archive format; archives written by any main build before this are not -read. - -## Related - -- [Media consumers address groups by start](/quest/m2/time-addressing/consumers.md) - the model that made the key's group field a start diff --git a/quest/m2/time-addressing/consumers.md b/quest/m2/time-addressing/consumers.md index 3634505149a..115b23a3c3d 100644 --- a/quest/m2/time-addressing/consumers.md +++ b/quest/m2/time-addressing/consumers.md @@ -141,5 +141,4 @@ them. - [Writing a group needs no invented start, and its end has one word](/quest/m2/group-api.md) - names the call that declares a group's end - [The catalog's clock and join interval live under a sync section](/quest/m2/hang04/sync.md) - renames the interval the HLS grid reads -- [The archive format ships as version 2](/quest/m2/archive-v2.md) - freezes the keys this made starts - [A group's frames present before the next group starts](/quest/m3/groups-never-overlap.md) - tightens the group rule a declared end relies on diff --git a/rs/moq-archive/src/consumer/index.rs b/rs/moq-archive/src/consumer/index.rs index 681d90ffc26..e578f410ae9 100644 --- a/rs/moq-archive/src/consumer/index.rs +++ b/rs/moq-archive/src/consumer/index.rs @@ -134,13 +134,7 @@ impl Index { /// read its `end` when `end` is set. `.info` sorts before every object key and `end` after, /// so listing after `.info` lists everything else. Object stores promise no listing order, so /// it is sorted here. -pub(super) async fn list( - store: &S, - dir: &Path, - cursor: &Path, - timed: bool, - end: bool, -) -> Result { +pub(super) async fn list(store: &S, dir: &Path, cursor: &Path, end: bool) -> Result { let metas: Vec<_> = store.list_with_offset(Some(dir), cursor).try_collect().await?; let mut listed = Listed::default(); let mut found = None; @@ -153,7 +147,7 @@ pub(super) async fn list( .location .filename() .ok_or_else(|| Error::Path(meta.location.to_string()))?; - match Name::parse(filename, timed)? { + match Name::parse(filename)? { Name::Info => {} Name::End => found = Some(meta.location), Name::Object(start) => listed.objects.push(((start.group, start.frame), meta.location)), diff --git a/rs/moq-archive/src/consumer/mod.rs b/rs/moq-archive/src/consumer/mod.rs index a14fccfc9ed..166b7c3c4ec 100644 --- a/rs/moq-archive/src/consumer/mod.rs +++ b/rs/moq-archive/src/consumer/mod.rs @@ -325,7 +325,7 @@ async fn refresh(shared: Arc>, mut tracks: Vec/.info`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(try_from = "Stored")] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Info { - /// Recording format version: [`VERSION`], or 2 or 3 for an older recording, which only a - /// reader accepts. - pub version: u64, /// The publisher's track priority, higher first. pub priority: i8, /// Units per second for this track's timestamps, 1 through 2^53 - 1, or `None` when the track /// is untimed and its frames store no timestamps. - #[serde(default, skip_serializing_if = "Option::is_none")] pub timescale: Option, } -/// [`Info`] as stored, before its priority is read in its version's encoding. -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] +/// [`Info`] as stored, with the format version. Unknown members are ignored, so a later writer +/// can add one a reader may skip. +#[derive(Serialize, Deserialize)] struct Stored { version: u64, - priority: i64, - #[serde(default)] + priority: i8, + #[serde(default, skip_serializing_if = "Option::is_none")] timescale: Option, } -impl TryFrom for Info { - type Error = String; - - fn try_from(stored: Stored) -> std::result::Result { - let priority = match stored.version { - // Version 2 stored the moq-lite byte, `priority + 128`. - 2 => u8::try_from(stored.priority).map(|byte| (byte ^ 0x80) as i8), - _ => i8::try_from(stored.priority), - } - .map_err(|_| format!("priority {} is out of range", stored.priority))?; - Ok(Self { - version: stored.version, - priority, - timescale: stored.timescale, - }) - } -} - impl Info { - /// The current format version with `priority` and `timescale`, `None` for an untimed track. + /// The properties `priority` and `timescale`, `None` for an untimed track. pub fn new(priority: i8, timescale: Option) -> Result { - let info = Self { - version: VERSION, - priority, - timescale, - }; + let info = Self { priority, timescale }; info.validate()?; Ok(info) } - /// Encode as compact JSON, refusing any version but the current one. + /// Encode as compact JSON in the current format version. pub fn encode(&self) -> Result { - if self.version != VERSION { - return Err(Error::Version(self.version)); - } self.validate()?; - Ok(Bytes::from(serde_json::to_vec(self)?)) + let stored = Stored { + version: VERSION, + priority: self.priority, + timescale: self.timescale, + }; + Ok(Bytes::from(serde_json::to_vec(&stored)?)) } - /// Decode JSON and refuse an unknown version or invalid timescale. + /// Decode JSON, refusing any version but [`VERSION`] or an invalid timescale. pub fn decode(bytes: &[u8]) -> Result { - let info: Self = serde_json::from_slice(bytes)?; + // Read the version alone first, so another version's members are never misread as ours. + #[derive(Deserialize)] + struct Version { + version: u64, + } + let Version { version } = serde_json::from_slice(bytes)?; + if version != VERSION { + return Err(Error::Version(version)); + } + let stored: Stored = serde_json::from_slice(bytes)?; + let info = Self { + priority: stored.priority, + timescale: stored.timescale, + }; info.validate()?; Ok(info) } fn validate(&self) -> Result<()> { - if !(VERSION_MIN..=VERSION).contains(&self.version) { - return Err(Error::Version(self.version)); - } if let Some(timescale) = self.timescale { if timescale == 0 { return Err(Error::Timescale(timescale)); @@ -98,7 +83,7 @@ mod tests { fn roundtrip() { let info = Info::new(0, Some(1_000_000)).unwrap(); let bytes = info.encode().unwrap(); - assert_eq!(bytes.as_ref(), br#"{"version":4,"priority":0,"timescale":1000000}"#); + assert_eq!(bytes.as_ref(), br#"{"version":2,"priority":0,"timescale":1000000}"#); assert_eq!(Info::decode(&bytes).unwrap(), info); } @@ -106,31 +91,30 @@ mod tests { fn an_untimed_track_omits_its_timescale() { let info = Info::new(3, None).unwrap(); let bytes = info.encode().unwrap(); - assert_eq!(bytes.as_ref(), br#"{"version":4,"priority":3}"#); + assert_eq!(bytes.as_ref(), br#"{"version":2,"priority":3}"#); assert_eq!(Info::decode(&bytes).unwrap(), info); } #[test] fn whitespace_and_member_order_do_not_affect_equality() { - let info = Info::decode(br#"{"timescale":1000,"priority":7,"version":4}"#).unwrap(); + let info = Info::decode(br#"{"timescale":1000,"priority":7,"version":2}"#).unwrap(); assert_eq!( info, - Info::decode(br#"{ "version": 4, "priority": 7, "timescale": 1000 }"#).unwrap() + Info::decode(br#"{ "version": 2, "priority": 7, "timescale": 1000 }"#).unwrap() ); - assert_eq!(info.priority, 7); - assert_eq!(info.timescale, Some(1000)); + assert_eq!(info, Info::new(7, Some(1000)).unwrap()); } #[test] fn priority_is_stored_signed() { for priority in [i8::MIN, -1, 0, i8::MAX] { let info = Info::new(priority, None).unwrap(); - let json = format!(r#"{{"version":4,"priority":{priority}}}"#); + let json = format!(r#"{{"version":2,"priority":{priority}}}"#); assert_eq!(info.encode().unwrap().as_ref(), json.as_bytes()); assert_eq!(Info::decode(json.as_bytes()).unwrap(), info); } for priority in [-129, 128] { - let json = format!(r#"{{"version":4,"priority":{priority}}}"#); + let json = format!(r#"{{"version":2,"priority":{priority}}}"#); assert!( matches!(Info::decode(json.as_bytes()), Err(Error::Json(_))), "{priority}" @@ -138,35 +122,25 @@ mod tests { } } - /// Version 2 stored the priority as its moq-lite byte, `priority + 128`, so an old - /// recording keeps its meaning; nothing writes version 2 any more. - #[test] - fn a_version_2_info_reads_its_lite_byte() { - for (byte, priority) in [(0, i8::MIN), (127, -1), (128, 0), (255, i8::MAX)] { - let json = format!(r#"{{"version":2,"priority":{byte},"timescale":1000}}"#); - let info = Info::decode(json.as_bytes()).unwrap(); - assert_eq!((info.version, info.priority, info.timescale), (2, priority, Some(1000))); - assert!(matches!(info.encode(), Err(Error::Version(2)))); - } - for byte in [-1, 256] { - let json = format!(r#"{{"version":2,"priority":{byte}}}"#); - assert!(matches!(Info::decode(json.as_bytes()), Err(Error::Json(_))), "{byte}"); - } - } - + /// Release wrote version 1, and main builds before the format shipped wrote 3 and 4, each + /// meaning something else: every one is refused by its version, whatever its other members. #[test] - fn a_version_3_info_reads_its_signed_priority() { - let info = Info::decode(br#"{"version":3,"priority":-1,"timescale":1000}"#).unwrap(); - assert_eq!((info.version, info.priority, info.timescale), (3, -1, Some(1000))); - assert!(matches!(info.encode(), Err(Error::Version(3)))); - } - - #[test] - fn unknown_version_is_refused() { - for version in [1, VERSION + 1] { - let json = format!(r#"{{"version":{version},"priority":0,"timescale":1000}}"#); - assert!(matches!(Info::decode(json.as_bytes()), Err(Error::Version(v)) if v == version)); + fn another_version_is_refused_by_its_number() { + for json in [ + r#"{"version":1,"priority":255,"timescale":1000}"#, + r#"{"version":3,"priority":-1,"timescale":1000}"#, + r#"{"version":4,"priority":0}"#, + r#"{"version":3,"epoch":"x"}"#, + ] { + let Err(Error::Version(version)) = Info::decode(json.as_bytes()) else { + panic!("{json} decoded"); + }; + assert_ne!(version, VERSION); } + assert_eq!( + Error::Version(4).to_string(), + "archive format version 4 is not supported, only version 2" + ); } #[test] @@ -175,21 +149,23 @@ mod tests { assert!(Info::new(0, Some(ID_MAX)).is_ok()); assert!(matches!(Info::new(0, Some(0)), Err(Error::Timescale(0)))); assert!(matches!(Info::new(0, Some(ID_MAX + 1)), Err(Error::Timescale(_)))); + assert!(matches!( + Info::decode(br#"{"version":2,"priority":0,"timescale":0}"#), + Err(Error::Timescale(0)) + )); } + /// A member a later writer adds is skipped, so the format can grow without a new version. #[test] - fn unknown_members_and_missing_fields_are_refused() { - assert!(matches!( - Info::decode(br#"{"version":2,"priority":128,"timescale":1000,"epoch":"x"}"#), - Err(Error::Json(_)) - )); - assert!(matches!( - Info::decode(br#"{"version":4,"priority":0,"timescale":1,"extra":true}"#), - Err(Error::Json(_)) - )); - assert!(matches!( - Info::decode(br#"{"version":4,"timescale":1}"#), - Err(Error::Json(_)) - )); + fn unknown_members_are_ignored() { + let info = Info::decode(br#"{"version":2,"priority":1,"timescale":90000,"label":{"en":"x"}}"#).unwrap(); + assert_eq!(info, Info::new(1, Some(90000)).unwrap()); + } + + #[test] + fn missing_members_are_refused() { + for json in [r#"{"version":2,"timescale":1}"#, r#"{"priority":0}"#, "[]"] { + assert!(matches!(Info::decode(json.as_bytes()), Err(Error::Json(_))), "{json}"); + } } } diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs index f3030f11247..fb0f9c76491 100644 --- a/rs/moq-archive/src/layout.rs +++ b/rs/moq-archive/src/layout.rs @@ -9,7 +9,7 @@ use object_store::path::Path; use serde::{Deserialize, Serialize}; use crate::path::{check_id, decode_track, encode_track, push}; -use crate::{Error, ID_WIDTH, Result, VERSION, VERSION_MIN}; +use crate::{Error, ID_WIDTH, Result, VERSION}; /// The broadcast's completion marker, beside its track segments. pub(crate) const BROADCAST_END: &str = ".end"; @@ -36,11 +36,11 @@ pub(crate) enum Name { impl Name { /// Parse the last segment of a key under a track's directory, refusing anything else. - pub(crate) fn parse(name: &str, timed: bool) -> Result { + pub(crate) fn parse(name: &str) -> Result { match name { TRACK_INFO => Ok(Self::Info), TRACK_END => Ok(Self::End), - name => Ok(Self::Object(Start::parse(name, timed)?)), + name => Ok(Self::Object(Start::parse(name)?)), } } } @@ -50,42 +50,27 @@ pub(crate) fn parse_track(segment: &str) -> Result { decode_track(segment) } -/// Where an object's content starts, which is its key: the first sequence, the first frame's -/// index within it, and, on a timed track, that frame's timestamp. +/// Where an object's content starts, which is its key: its first group's start, and the index +/// of the first frame within that group. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct Start { pub group: u64, pub frame: u64, - pub pts: Option, } impl Start { - /// `--` on a timed track, `-` on an untimed one, - /// zero-padded so keys sort in content order. + /// `-`, zero-padded so keys sort in content order. pub(crate) fn name(&self) -> Result { - let (group, frame) = (id(self.group)?, id(self.frame)?); - match self.pts { - Some(pts) => Ok(format!("{group}-{frame}-{}", id(pts)?)), - None => Ok(format!("{group}-{frame}")), - } + Ok(format!("{}-{}", id(self.group)?, id(self.frame)?)) } - fn parse(name: &str, timed: bool) -> Result { - let fields: Vec<&str> = name.split('-').collect(); + fn parse(name: &str) -> Result { let invalid = || Error::Path(name.to_string()); - match (timed, fields.as_slice()) { - (true, [group, frame, pts]) => Ok(Self { - group: parse_id(group).ok_or_else(invalid)??, - frame: parse_id(frame).ok_or_else(invalid)??, - pts: Some(parse_id(pts).ok_or_else(invalid)??), - }), - (false, [group, frame]) => Ok(Self { - group: parse_id(group).ok_or_else(invalid)??, - frame: parse_id(frame).ok_or_else(invalid)??, - pts: None, - }), - _ => Err(invalid()), - } + let (group, frame) = name.split_once('-').ok_or_else(invalid)?; + Ok(Self { + group: parse_id(group).ok_or_else(invalid)??, + frame: parse_id(frame).ok_or_else(invalid)??, + }) } } @@ -106,9 +91,10 @@ fn parse_id(field: &str) -> Option> { ) } -/// How a track's last run ended, stored in its `end` once the broadcast ended. +/// How a track's last run ended, stored in its `end` once the broadcast ended. An unknown kind +/// is refused, since skipping it would misreport how the track ended. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase", deny_unknown_fields)] +#[serde(rename_all = "lowercase")] pub(crate) enum End { /// A clean end at this exclusive position. Finish(At), @@ -116,9 +102,8 @@ pub(crate) enum End { Error(u32), } -/// A [`Position`] as stored. +/// A [`Position`] as stored, ignoring members it does not know. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub(crate) struct At { pub group: u64, pub frame: u64, @@ -167,9 +152,6 @@ pub(crate) struct Object { pub entries: Vec, } -/// The form field version 2 and 3 entries carry: 0 for a group, 1 for a datagram. -const LEGACY_GROUP: u64 = 0; - impl Object { /// The key this object is stored under. pub(crate) fn start(&self) -> Result { @@ -177,10 +159,14 @@ impl Object { Ok(Start { group: first.sequence, frame: self.frame_start, - pts: first.frames.first().ok_or(Error::Empty)?.timestamp, }) } + /// The first stored frame's timestamp, `None` when empty or untimed. + pub(crate) fn timestamp(&self) -> Option { + self.entries.first()?.frames.first()?.timestamp + } + /// Its payload bytes plus a small per-frame overhead, for a byte-bounded cache. pub(crate) fn weight(&self) -> u64 { let frames = self.entries.iter().flat_map(|entry| &entry.frames); @@ -234,18 +220,15 @@ impl Object { } /// Decode a whole table before slicing any payload; `timed` is whether the track's `.info` - /// declares a timescale, which says whether each frame stores a timestamp. A version 2 or 3 - /// object's entries each carry a form, and one holding a datagram is refused: an archive - /// replays groups only, and skipping the entry would read as a gap the recording never had. + /// declares a timescale, which says whether each frame stores a timestamp. pub(crate) fn decode(mut buf: impl Buf, timed: bool) -> Result { let version = read(&mut buf)?; - if !(VERSION_MIN..=VERSION).contains(&version) { + if version != VERSION { return Err(Error::Version(version)); } - let legacy = version < 4; let frame_start = read(&mut buf)?; - // Each entry is at least a sequence and a frame count, plus a form when legacy. - let count = read_count(&mut buf, if legacy { 3 } else { 2 })?; + // Each entry is at least a sequence and a frame count. + let count = read_count(&mut buf, 2)?; let per_frame = if timed { 2 } else { 1 }; let mut lengths = Vec::new(); @@ -262,12 +245,6 @@ impl Object { }; check_id(sequence)?; prev = Some(sequence); - if legacy { - let form = read(&mut buf)?; - if form != LEGACY_GROUP { - return Err(Error::Form(form)); - } - } let frames = read_count(&mut buf, per_frame)?; let mut timestamps = Vec::with_capacity(frames); for _ in 0..frames { @@ -370,23 +347,15 @@ mod tests { } #[test] - fn timed_keys_sort_in_content_order() { - let names: Vec = [(0, 0, 9), (0, 3, 2), (1, 0, 0), (10, 0, 0)] + fn keys_sort_in_content_order() { + let names: Vec = [(0, 0), (0, 3), (1, 0), (10, 0), (10, 11)] .into_iter() - .map(|(group, frame, pts)| { - Start { - group, - frame, - pts: Some(pts), - } - .name() - .unwrap() - }) + .map(|(group, frame)| Start { group, frame }.name().unwrap()) .collect(); let mut sorted = names.clone(); sorted.sort(); assert_eq!(names, sorted); - assert_eq!(names[1], "0000000000000000000-0000000000000000003-0000000000000000002"); + assert_eq!(names[1], "0000000000000000000-0000000000000000003"); assert!( names .iter() @@ -395,36 +364,29 @@ mod tests { } #[test] - fn names_roundtrip_and_refuse_the_other_timing() { - let timed = Start { - group: 5, + fn names_roundtrip() { + let start = Start { + group: ID_MAX, frame: 2, - pts: Some(ID_MAX), }; - let name = timed.name().unwrap(); - assert_eq!(Name::parse(&name, true).unwrap(), Name::Object(timed)); - assert!(matches!(Name::parse(&name, false), Err(Error::Path(_)))); + let name = start.name().unwrap(); + assert_eq!(Name::parse(&name).unwrap(), Name::Object(start)); - let untimed = Start { - group: 7, - frame: 2, - pts: None, - }; - let name = untimed.name().unwrap(); - assert_eq!(name, "0000000000000000007-0000000000000000002"); - assert_eq!(Name::parse(&name, false).unwrap(), Name::Object(untimed)); - assert!(matches!(Name::parse(&name, true), Err(Error::Path(_)))); - - assert_eq!(Name::parse(".info", true).unwrap(), Name::Info); - assert_eq!(Name::parse("end", false).unwrap(), Name::End); + assert_eq!(Name::parse(".info").unwrap(), Name::Info); + assert_eq!(Name::parse("end").unwrap(), Name::End); for bad in [ "7", "segments", "0000000000000000007", "0000000000000000007-x", "0009007199254740992-0000000000000000000", + // Main builds before the format shipped added the first frame's timestamp. + "0000000000000000007-0000000000000000002-0000000000000000009", ] { - assert!(Name::parse(bad, false).is_err(), "{bad}"); + assert!( + matches!(Name::parse(bad), Err(Error::Path(_)) | Err(Error::Id(_))), + "{bad}" + ); } } @@ -440,6 +402,12 @@ mod tests { assert_eq!(error.encode().unwrap().as_ref(), br#"{"error":52}"#); assert_eq!(End::decode(br#"{"error":52}"#).unwrap(), error); assert!(End::decode(br#"{"finish":{"group":5}}"#).is_err()); + assert!(End::decode(br#"{"pause":{"group":5,"frame":3}}"#).is_err()); + // A member a later writer adds to the position is skipped. + assert_eq!( + End::decode(br#"{"finish":{"group":5,"frame":3,"at":1}}"#).unwrap(), + finish + ); } #[test] @@ -460,14 +428,8 @@ mod tests { let bytes = object.encode(true).unwrap(); assert_eq!(Object::decode(bytes.clone(), true).unwrap(), object); assert!(Object::decode(bytes, false).is_err()); - assert_eq!( - object.start().unwrap(), - Start { - group: 3, - frame: 4, - pts: Some(10) - } - ); + assert_eq!(object.start().unwrap(), Start { group: 3, frame: 4 }); + assert_eq!(object.timestamp(), Some(10)); assert_eq!(object.end().unwrap(), Position::new(7, 1)); let split = Object { @@ -493,25 +455,6 @@ mod tests { assert!(matches!(object.encode(true), Err(Error::Timing))); } - /// Version 2 and 3 entries carry a form: a group reads as today's entry, and a datagram - /// refuses the whole object rather than reading as a gap. - #[test] - fn older_objects_read_groups_and_refuse_datagrams() { - let object = Object { - frame_start: 0, - entries: vec![Entry { - sequence: 2, - frames: vec![frame(None, b"x"), frame(None, b"yz")], - }], - }; - for version in [2, 3] { - let group = [version, 0, 1, 2, 0, 2, 1, 2, b'x', b'y', b'z']; - assert_eq!(Object::decode(&group[..], false).unwrap(), object); - let datagram = [version, 0, 1, 2, 1, 1, b'x']; - assert!(matches!(Object::decode(&datagram[..], false), Err(Error::Form(1)))); - } - } - #[test] fn malformed_objects_are_refused() { let empty = Object::default(); @@ -559,10 +502,11 @@ mod tests { Object::decode(&[VERSION as u8, 0, 0x7f][..], false), Err(Error::Table) )); - for version in [1, VERSION as u8 + 1] { + // Release wrote version 1, and main builds before the format shipped wrote 3 and 4. + for version in [1, 3, 4] { assert!(matches!( Object::decode(&[version, 0, 0][..], false), - Err(Error::Version(_)) + Err(Error::Version(v)) if v == version as u64 )); } } diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 46d31819983..57af1649279 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -8,16 +8,16 @@ //! //! A [`Producer`] writes, under its prefix: //! -//! - `/.info`: the track's properties as JSON, `{"version":4,"priority":0,"timescale":1000}`, +//! - `/.info`: the track's properties as JSON, `{"version":2,"priority":0,"timescale":1000}`, //! written before its first object. `timescale` is absent on an untimed track. Properties never //! change for a track name. The source's max age is not stored: it bounds the live cache, not //! the archive. -//! - `/--` on a timed track, `/-` on an untimed -//! one: an object, keyed by its first sequence, the index of its first frame within it, and -//! that frame's timestamp in the track's timescale. Each field is 19 zero-padded decimal digits, -//! so a sorted listing is content order and answers group and time lookups; there is no index -//! object. A group may split between frames across objects, which the frame index keeps unique -//! and ordered even where timestamps repeat or go backward. Every object is written once. +//! - `/-`: an object, keyed by its first group's start and the index of its +//! first frame within that group. Each field is 19 zero-padded decimal digits, so a sorted +//! listing is content order; on a timed track a group's start is its time in the track's +//! timescale, so the same listing answers time lookups, and there is no index object. A group +//! may split between frames across objects, which the frame index keeps unique and ordered. +//! Every object is written once. //! - `/end`: how the track's last run ended, `{"finish":{"group":5,"frame":0}}` for a //! clean end at that exclusive position or `{"error":52}` for the stream error code. It sorts //! after every object key, and DVR pruning never deletes it. @@ -32,7 +32,7 @@ //! //! ```text //! Object { -//! Version (i) = 4, +//! Version (i) = 2, //! Frame Start (i), // the first group's first frame index //! Entry Count (i), //! Entry (..) ..., @@ -40,17 +40,21 @@ //! } //! //! Entry { -//! Sequence (i), // absolute for the first entry, then current - previous - 1 +//! Group Start (i), // absolute for the first entry, then current - previous - 1 //! Frame Count (i), // at least 1 //! Frame (..) ..., //! } //! //! Frame { -//! [Timestamp (i)], // exactly when the track's .info declares a timescale +//! [Timestamp (i)], // absolute, exactly when the track's .info declares a timescale //! Length (i), //! } //! ``` //! +//! JSON readers ignore members they do not know, so a later writer may add a member a reader can +//! safely skip without a new version; anything that changes what existing bytes mean takes one. +//! A reader accepts exactly [`VERSION`] and refuses any other with [`Error::Version`]. +//! //! An archive records stream tracks only. A datagram track is live-only, so a broadcast's //! archive holds none of its datagram tracks. @@ -74,15 +78,10 @@ pub use info::Info; pub use producer::Producer; pub use store::Store; -/// Recording format version written into `.info` and the binary envelope. -pub const VERSION: u64 = 4; - -/// The oldest recording format version a [`Consumer`] accepts. Version 2 stored `.info`'s -/// priority as its moq-lite byte, and versions 2 and 3 give each object entry a form, which -/// a reader accepts only as a group. -pub(crate) const VERSION_MIN: u64 = 2; +/// The archive format version, written into `.info` and every object, and the only one read. +pub const VERSION: u64 = 2; -/// Largest sequence, frame index, or timestamp value the format allows (`2^53 - 1`). +/// Largest group start, frame index, or timestamp value the format allows (`2^53 - 1`). pub const ID_MAX: u64 = (1 << 53) - 1; /// Decimal width of each object key field. diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs index 50430993924..01d4d37603b 100644 --- a/rs/moq-archive/src/producer/backends.rs +++ b/rs/moq-archive/src/producer/backends.rs @@ -101,16 +101,8 @@ async fn record(store: &Store) { stored( store, &[ - path( - store, - VIDEO, - "0000000000000000000-0000000000000000000-0000000000000000000", - ), - path( - store, - VIDEO, - "0000000000000000002-0000000000000000000-0000000000000002000", - ), + path(store, VIDEO, "0000000000000000000-0000000000000000000"), + path(store, VIDEO, "0000000000000000002-0000000000000000000"), path(store, CHAT, "0000000000000000000-0000000000000000000"), ], ) @@ -119,7 +111,7 @@ async fn record(store: &Store) { assert_eq!( decode( &first_objects, - "video%2F1080p/0000000000000000002-0000000000000000000-0000000000000002000", + "video%2F1080p/0000000000000000002-0000000000000000000", true, ), [ @@ -188,9 +180,9 @@ async fn recordings_are_byte_identical_on_every_backend() { "chat%2Ejson/0000000000000000000-0000000000000000001", "chat%2Ejson/end", "video%2F1080p/.info", - "video%2F1080p/0000000000000000000-0000000000000000000-0000000000000000000", - "video%2F1080p/0000000000000000002-0000000000000000000-0000000000000002000", - "video%2F1080p/0000000000000000004-0000000000000000000-0000000000000004000", + "video%2F1080p/0000000000000000000-0000000000000000000", + "video%2F1080p/0000000000000000002-0000000000000000000", + "video%2F1080p/0000000000000000004-0000000000000000000", "video%2F1080p/end", ] ); @@ -204,11 +196,7 @@ async fn recordings_are_byte_identical_on_every_backend() { "the resumed group continues in its own object" ); assert_eq!( - decode( - &expected, - "video%2F1080p/0000000000000000004-0000000000000000000-0000000000000004000", - true - ), + decode(&expected, "video%2F1080p/0000000000000000004-0000000000000000000", true), [ (4, vec!["v4.0".into(), "v4.1".into()]), (5, vec!["v5.0".into(), "v5.1".into()]) diff --git a/rs/moq-archive/src/producer/resume.rs b/rs/moq-archive/src/producer/resume.rs index 18bfeb8e7e1..80207bff60c 100644 --- a/rs/moq-archive/src/producer/resume.rs +++ b/rs/moq-archive/src/producer/resume.rs @@ -9,7 +9,7 @@ use object_store::path::Path; use object_store::{ObjectStore, ObjectStoreExt}; use crate::layout::{self, End, Name, Object, Start}; -use crate::{Error, Info, Result, Store, VERSION}; +use crate::{Error, Info, Result, Store}; /// One track as stored. pub(super) struct Stored { @@ -49,10 +49,6 @@ async fn load_track(store: &Store, dir: &Path) -> Result = store.inner().list(Some(dir)).try_collect().await?; let info_path = crate::path::push(dir, layout::TRACK_INFO)?; let info = Info::decode(&store.inner().get(&info_path).await?.bytes().await?)?; - // A reader still reads an older format, but nothing appends to it. - if info.version != VERSION { - return Err(Error::Version(info.version)); - } let timed = info.timescale.is_some(); let mut objects: Vec<(Start, Path)> = Vec::new(); @@ -66,7 +62,7 @@ async fn load_track(store: &Store, dir: &Path) -> Result {} Name::End => end = Some(End::decode(&store.inner().get(&entry.location).await?.bytes().await?)?), Name::Object(start) => objects.push((start, entry.location)), diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index c0d44718551..62d2c7cb932 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -140,11 +140,7 @@ fn contents(objects: &BTreeMap) -> Vec)>> .collect() } -fn key(group: u64, frame: u64, pts: u64) -> String { - format!("{group:019}-{frame:019}-{pts:019}") -} - -fn untimed(group: u64, frame: u64) -> String { +fn key(group: u64, frame: u64) -> String { format!("{group:019}-{frame:019}") } @@ -176,7 +172,7 @@ async fn timed_objects_cut_at_group_boundaries_by_duration() { let objects = objects(&store, "video", true).await; assert_eq!( objects.keys().cloned().collect::>(), - [key(0, 0, 0), key(2, 0, 2000), key(4, 0, 4000)] + [key(0, 0), key(2, 0), key(4, 0)] ); assert_eq!( contents(&objects)[0], @@ -209,10 +205,7 @@ async fn a_late_group_is_a_gap_that_never_stalls_later_groups() { // Nothing waits on 1, so every group is stored within the flush. sleep(SECOND + SECOND / 2).await; let stored = objects(&store, "video", true).await; - assert_eq!( - stored.keys().cloned().collect::>(), - [key(0, 0, 0), key(2, 0, 2000)] - ); + assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0), key(2, 0)]); assert_eq!( contents(&stored)[1], [(2, vec!["2000/c".to_string()]), (3, vec!["3000/d".to_string()]),] @@ -254,7 +247,7 @@ async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { let objects = objects(&store, "video", true).await; assert_eq!( objects.keys().cloned().collect::>(), - [key(0, 0, 0), key(0, 2, 2000), key(0, 5, 4000)] + [key(0, 0), key(0, 2), key(0, 5)] ); let stored: Vec = contents(&objects) .into_iter() @@ -280,7 +273,7 @@ async fn an_open_untimed_group_is_cut_at_each_flush() { open.write_frame(moq_net::Timestamp::unscaled(0), "a").unwrap(); settle().await; sleep(SECOND).await; - let head = untimed(0, 0); + let head = key(0, 0); let first = get(&store, &format!("data/{head}")).await.unwrap(); open.write_frame(moq_net::Timestamp::unscaled(0), "b").unwrap(); @@ -288,7 +281,7 @@ async fn an_open_untimed_group_is_cut_at_each_flush() { sleep(SECOND).await; assert_eq!(get(&store, &format!("data/{head}")).await.unwrap(), first); let stored = objects(&store, "data", false).await; - assert_eq!(stored.keys().cloned().collect::>(), [head, untimed(0, 1)]); + assert_eq!(stored.keys().cloned().collect::>(), [head, key(0, 1)]); assert_eq!( contents(&stored), [[(0, vec!["a".to_string()])], [(0, vec!["b".to_string()])]] @@ -333,7 +326,7 @@ async fn an_untimed_group_splits_by_wall_clock() { let stored = objects(&store, "data", false).await; assert_eq!( stored.keys().cloned().collect::>(), - [untimed(0, 0), untimed(0, 4), untimed(1, 0)] + [key(0, 0), key(0, 4), key(1, 0)] ); let strings = |frames: &[&str]| frames.iter().map(|frame| frame.to_string()).collect::>(); assert_eq!( @@ -362,7 +355,7 @@ async fn an_idle_track_is_flushed_within_the_bound() { sleep(SECOND / 2).await; assert_eq!( objects(&store, "video", true).await.keys().cloned().collect::>(), - [key(0, 0, 0)] + [key(0, 0)] ); // The open group continues in the next object. @@ -374,7 +367,7 @@ async fn an_idle_track_is_flushed_within_the_bound() { finished(task).await; assert_eq!( objects(&store, "video", true).await.keys().cloned().collect::>(), - [key(0, 0, 0), key(0, 1, 100)] + [key(0, 0), key(0, 1)] ); } @@ -418,10 +411,7 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { finished(task).await; let stored = objects(&store, "video", true).await; - assert_eq!( - stored.keys().cloned().collect::>(), - [key(0, 0, 0), key(0, 2, 200)] - ); + assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0), key(0, 2)]); let frames: Vec = contents(&stored) .into_iter() .flatten() @@ -431,10 +421,7 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { // The untimed group continues in its own object too. let stored = objects(&store, "data", false).await; - assert_eq!( - stored.keys().cloned().collect::>(), - [untimed(0, 0), untimed(0, 2)] - ); + assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0), key(0, 2)]); let frames: Vec = contents(&stored) .into_iter() .flatten() @@ -476,7 +463,7 @@ async fn a_partial_group_resumes_after_its_last_stored_frame() { let stored = objects(&store, "video", true).await; assert_eq!( stored.keys().cloned().collect::>(), - [key(0, 0, 0), key(1, 2, 500), key(1, 3, 600)] + [key(0, 0), key(1, 2), key(1, 3)] ); let frames: Vec = contents(&stored) .into_iter() @@ -524,18 +511,19 @@ async fn changed_properties_fail_the_resume() { ); } -/// A reader still reads a version 2 `.info`, but a producer never appends to one. +/// A prefix in another format version, such as one a main build wrote before the format shipped, +/// is refused before anything is written to it. #[tokio::test(start_paused = true)] -async fn an_older_version_fails_the_resume() { +async fn another_version_fails_the_resume() { let store = store(); let info = object_store::path::Path::from("rec/video/.info"); - let v2 = br#"{"version":2,"priority":128,"timescale":1000}"#; - store.inner().put(&info, v2.as_ref().into()).await.unwrap(); + let v4 = br#"{"version":4,"priority":0,"timescale":1000}"#; + store.inner().put(&info, v4.as_ref().into()).await.unwrap(); let broadcast = broadcast::Info::new().produce(); let _video = broadcast.create_track("video", timed()).unwrap(); let err = spawn(&store, &broadcast, config()).await.unwrap().unwrap_err(); - assert!(matches!(err, Error::Version(2)), "{err}"); - assert_eq!(get(&store, "video/.info").await.as_deref(), Some(&v2[..])); + assert!(matches!(err, Error::Version(4)), "{err}"); + assert_eq!(get(&store, "video/.info").await.as_deref(), Some(&v4[..])); } #[tokio::test(start_paused = true)] @@ -561,7 +549,7 @@ async fn track_names_stay_one_segment_through_a_resume() { let stored = keys(&store).await; for dir in ["a%2Fb", "%2Ehidden", "end"] { - assert!(stored.contains(&format!("{dir}/{}", untimed(0, 0))), "{stored:?}"); + assert!(stored.contains(&format!("{dir}/{}", key(0, 0))), "{stored:?}"); assert!(stored.contains(&format!("{dir}/.info")), "{stored:?}"); } diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 204d0a362ce..b6736f4e580 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -320,7 +320,7 @@ impl Track { || match (self.timescale, frame.timestamp) { (Some(timescale), Some(pts)) => { let duration = units(config.duration, timescale); - let first = building.object.start()?.pts.expect("a timed object starts timed"); + let first = building.object.timestamp().expect("a timed object starts timed"); match boundary { true => pts.saturating_sub(first) >= duration, // An oversized group splits once its own run here reaches the duration. diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 81ad5b2d62e..c07533cc841 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -34,8 +34,7 @@ impl Store { /// Create a track's `.info`, or accept an existing object with the same parsed properties. /// - /// An existing recording in an older format is refused with [`Error::Version`]: a - /// [`Consumer`](crate::Consumer) still reads it, but nothing appends to it. + /// An existing `.info` in another format version is refused with [`Error::Version`]. pub(crate) async fn put_info(&self, track: &str, info: &Info) -> Result<()> { let path = self.info_path(track)?; let bytes = info.encode()?; @@ -47,9 +46,6 @@ impl Store { Ok(_) => Ok(()), Err(object_store::Error::AlreadyExists { .. }) => { let existing = Info::decode(&self.get_bytes(&path).await?)?; - if existing.version != info.version { - return Err(Error::Version(existing.version)); - } if existing.priority != info.priority { return Err(Error::Priority { existing: existing.priority, @@ -105,7 +101,7 @@ mod tests { .inner() .put( &path, - br#"{ "timescale": 1000, "priority": 1, "version": 4 }"#.as_ref().into(), + br#"{ "timescale": 1000, "priority": 1, "version": 2 }"#.as_ref().into(), ) .await .unwrap(); @@ -116,7 +112,7 @@ mod tests { .bytes() .await .unwrap(); - assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 4 }"#); + assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 2 }"#); } #[tokio::test] @@ -129,11 +125,11 @@ mod tests { &br#"{"version":1,"priority":0,"timescale":1000}"#[..], (|err| matches!(err, Error::Version(1))) as fn(&Error) -> bool, ), - ("v2", br#"{"version":2,"priority":128,"timescale":1000}"#, |err| { - matches!(err, Error::Version(2)) + ("v4", br#"{"version":4,"priority":0,"timescale":1000}"#, |err| { + matches!(err, Error::Version(4)) }), ("junk", b"not json", |err| matches!(err, Error::Json(_))), - ("zero", br#"{"version":4,"priority":0,"timescale":0}"#, |err| { + ("zero", br#"{"version":2,"priority":0,"timescale":0}"#, |err| { matches!(err, Error::Timescale(0)) }), ] { diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 246ca566960..9d196cbf447 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -445,7 +445,7 @@ mod tests { } assert!( files.contains(&format!( - "{path}/@{instance}/video/0000000000000000000-0000000000000000000-0000000000000000000" + "{path}/@{instance}/video/0000000000000000000-0000000000000000000" )), "{files:?}" );