diff --git a/Cargo.lock b/Cargo.lock index a51cdadcec9..584ede7c50d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4538,7 +4538,6 @@ dependencies = [ "kio 0.6.2", "moq-clock", "moq-json", - "moq-mux", "moq-net", "object_store", "percent-encoding", @@ -4736,7 +4735,6 @@ dependencies = [ "moq-clock", "moq-hls", "moq-json", - "moq-msf", "moq-mux", "moq-relay", "moq-rtc", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 0d03ae49fa1..0aed2e4eb97 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -24,7 +24,7 @@ Docker; see [Install](/setup/install). | `export` | `fmp4`, `mkv`, `ts`, `flv`, `h264`, `h265` | Write a container to stdout. | | `export` | `hls --listen` | Serve the broadcast as HLS over HTTP. | | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | -| `export` | `archive ` | Record the broadcast into an object store. | +| `export` | `archive ` | Record every broadcast under the prefix into an object store. | | `play` | | Decode and play in a native window with sound. | | `transcode` | | Publish a just-in-time rendition ladder next to a broadcast. | | `announced` | `[prefix]` | Follow the broadcasts announced on a relay. | @@ -298,26 +298,31 @@ equivalents. ## Archive ```bash -# Record a broadcast until it ends -moq --connect https://relay.example.com/anon --broadcast event.hang export archive s3://recordings/event +# Record every broadcast announced under `event` +moq --connect https://relay.example.com/anon --broadcast event export archive s3://recordings -# Replay it under another name +# Replay a timeline recording under another name moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event ``` -`export archive` records one broadcast with -[moq-archive](https://docs.rs/moq-archive), cutting each track into 2-10 s -spans at group boundaries. An [untimed](/concept/moq-lite#subscriptions) track, -as every track is over moq-lite before 05 and moq-transport 14-16, is stored -one group per span and replays untimed. A store that already holds the recording is -continued, unless the source now announces another -[epoch](/concept/moq-lite#publisher-epochs): that is a restart, so the export -fails; record it under a new prefix. `--retention 1h` keeps only the last -hour, a DVR, which refuses an untimed track since it has no time to measure. - -`import archive` republishes a recording as live tracks, fetching spans on -request. It ends where the recording ends; `--follow 2s` keeps polling a -recording still being made. +`export archive` records every broadcast announced at or under `--broadcast` +with [moq-archive](https://docs.rs/moq-archive), each publisher instance under +`/@` in the store, until the connection ends. Like +`announced`, it leaves out a broadcast whose name starts with `.` unless +`--broadcast` names it. A broadcast path segment that starts with `@` is +stored as `%40`, so no recording lands inside another. Every track is stored as its own objects, cut about every 2 s +of media (or wall clock for an [untimed](/concept/moq-lite#subscriptions) +track) and at least every 4 s, splitting a long group between frames. A broadcast restarted under a new +[epoch](/concept/moq-lite#publisher-epochs) records under its own key; a route +without an epoch gets a fresh one each time it announces, so it never resumes +or stitches into an older recording. Any recording that fails ends the export. +`--retention 1h` keeps only the last hour, a DVR, deleting older objects +`--retention-grace` (default 30 s) after they leave the window. + +`import archive` republishes a timeline recording made by an older `moq` as +live tracks, fetching spans on request. It ends where the recording ends; +`--follow 2s` keeps polling a recording still being made. It does not yet read +the per-track layout that `export archive` writes. Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`, with credentials from the @@ -332,7 +337,7 @@ over one connection: moq --connect https://relay.example.com/anon \ import --broadcast event.hang srt --listen 0.0.0.0:9000 \ -- export --broadcast event.hang hls --listen 0.0.0.0:8080 \ - -- export --broadcast event.hang archive file:///recordings/event + -- export --broadcast event.hang archive file:///recordings ``` ## Publisher runs diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index e0127248255..e9205196a6f 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -21,7 +21,7 @@ links cover crates awaiting their first release. | [moq-uring](https://docs.rs/moq-uring) | Experimental Linux io\_uring server for moq-lite. | | [hang](/lib/rs/hang) | The media layer: catalog, containers, ordered frame delivery. | | [moq-mux](/lib/rs/moq-mux) | Import and export fMP4/CMAF, MPEG-TS, Matroska, FLV, and Annex-B. | -| [moq-archive](https://docs.rs/moq-archive) | Record and replay hang broadcasts on any `object_store` backend. | +| [moq-archive](https://docs.rs/moq-archive) | Record and replay MoQ broadcasts on any `object_store` backend. | | [moq-video](/lib/rs/moq-video) | Native capture, hardware encode/decode (Apple, Windows, NVIDIA, VAAPI, V4L2, Android), and GPU rendering. | | [moq-v4l](https://docs.rs/moq-v4l) | Safe Video4Linux 2 bindings. | | [moq-audio](/lib/rs/moq-audio) | Microphone and speaker, Opus/PCM/AAC codecs, echo cancellation. | diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 7ec39a1aa6e..5c0f33d14c8 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -988,10 +988,10 @@ A wildcard replay path names no generation. The `replayEpoch` field, if present, names the publisher instance of `replay`, with the meaning and rules of a rendition's `epoch` ({{field-epoch}}). -The `store` field, if present, is the object-store URL the recording objects ({{recording}}) live under. +The `store` field, if present, is the object-store URL the recording objects live under, in the layout the moq-archive crate defines. Authorization for `replay` and `store` is external. -The `version` field is the recording object format version ({{recording-track}}). +The `version` field is the version of that recording object format. It is 2 for this specification. A publisher that exposes a store MUST set it; a publisher that does not MUST omit it. @@ -1147,199 +1147,6 @@ This is what makes a time table proxyable: each tick republishes a small snapsho Carrying the source's time rather than synthesizing one keeps the clock consistent with the EPG (event times are expressed on the source's clock) and preserves TOT's local-time-offset descriptors, which are operator policy a re-multiplexer cannot invent. -# Recording {#recording} -A live broadcast is bounded history: moq-lite {{moql}} serves the present with SUBSCRIBE and the recent past with FETCH, both ending at the publisher's cache. -A *recording* is the persistent tier, writing a broadcast to a filesystem or object store so it can be served back long after the live session ended. - -A recording is addressed by record. -Each recorded track has its own timeline ({{timeline}}), and each of its records is stored as one object named by the record's `sequence`. -A reader that wants a record of a track issues one whole-object GET, with no range header and no second request, which is what both an HLS or DASH origin and a player seeking a VOD recording need. - -A recording covers the tracks its writer selects, including the catalog, plus each one's timeline. - -## Layout {#recording-layout} -A recording is a set of objects under a common prefix: - -~~~ -//.info -//segments/ -~~~ - -The common prefix is application-defined and is not interpreted by this format. - -`` is the track's UTF-8 name with every byte outside `A-Z a-z 0-9 _ -` percent-encoded as `%` followed by two uppercase hexadecimal digits. -For example, `catalog.json` becomes `catalog%2Ejson`. -An encoded name never contains `/` or begins with `.`, so a track cannot address anything outside the prefix. -`.info` and `segments` are reserved within a track directory. - -A recording names each track's timeline by appending `.timeline.z` to the track's name, so a recording is recoverable from its objects alone; a recorded track's own name MUST NOT end in `.timeline.z`. -For a recorded track, `segments/` holds the frames of record `sequence` of its timeline. -For a timeline track, `segments/` holds the complete Window groups ({{timeline-framing}}) closed while committing record `sequence` of the track it indexes, at least one per object. - -`` is written as exactly 19 decimal digits, padded with leading zeros, so lexical and numeric order agree. -Recorded sequences, group IDs, and frame indices MUST be integers in the range 0 through 9007199254740991 (2^53 - 1), so timeline JSON preserves them exactly. -Writers and readers MUST reject IDs outside this range, including reconstructed group IDs in the binary table. -After its first record, a timeline MUST commit consecutive sequences. -A writer MUST stop before allocating a sequence beyond the limit; sequences never wrap. -Each track has its own objects so a reader can fetch one rendition without downloading the others. - -## Track Objects {#recording-track} -`/.info` is a UTF-8 JSON object containing the immutable properties needed to interpret and replay the track: - -~~~ -{ - "version": 2, - "priority": 0, - "timescale": 1000000 -} -~~~ - -`version` and `priority` are required integers. -`version` identifies this recording format and MUST be 2. -`priority` and `timescale` have the meanings of moq-lite `TRACK_INFO` {{moql}}: `priority` is in the range 0 through 255, and `timescale` is in the range 1 through 9007199254740991, so JSON consumers can preserve it exactly. -`timescale` is absent exactly when the track is untimed: its frames store no timestamps ({{recording-segments}}) and its records carry no `pts` or `duration` ({{timeline-records}}). -A timeline track's object declares a `timescale` exactly when the track it indexes does. -A reader MUST preserve integer values exactly. -An optional `epoch` string is the Epoch of the route the source was read through {{moql}}, in its canonical UUID text, and is omitted when that route had none. -`Publisher Max Age` is not stored; a reader supplies its own serving policy. - -The track object MUST be durable before its first segment object is stored, including for a timeline track. -It is immutable for the lifetime of the recording. -A reader MUST refuse an unknown version or invalid track properties. -On an existing `.info`, a writer MUST validate and compare the parsed `version`, `priority`, `timescale`, and `epoch` values, an absent `timescale` or `epoch` matching only an absent one; JSON whitespace and member order do not affect equality. -Different property values MUST fail enrollment, and the existing object MUST NOT be rewritten. -A different `epoch` means the source restarted, so the writer starts a new recording rather than continuing this one. - -## Segment Objects {#recording-segments} -A segment object holds one record's frames: - -~~~ -Segment Object { - Version (i) = 2 - Frame Start (i) - Group Count (i) - Group Table { - Sequence Delta (i) - Frame Count (i) - Frame Table { - [Timestamp (i)] - Payload Offset (i) - Payload Length (i) - } ... - } ... - Payload Bytes (..) -} -~~~ - -Fields annotated `(i)` are variable-length integers using the QUIC encoding ({{!RFC9000}}, Section 16). -`Frame Start` is the index within its group of the first group's first stored frame; every later group starts at frame 0. -`Group Count` gives the number of group entries; each `Frame Count` gives the number of frame entries in that group. -The first `Sequence Delta` is the absolute group sequence number. -Each subsequent value is the group sequence number minus the previous sequence number minus one; zero therefore means the next consecutive group. -Groups MUST have strictly ascending sequence numbers; reconstruction MUST reject values exceeding the recording ID limit ({{recording-layout}}). -Frame entries appear in their original order within each group. -`Timestamp` is the frame's absolute timestamp in the track's `timescale` units, not a delta. -It MUST be in the range 0 through 9007199254740991 (2^53 - 1), so browser replay preserves it exactly; writers and readers MUST reject larger values. -`Timestamp` is present exactly when the track object declares a `timescale` ({{recording-track}}); an untimed track's frame entries omit it, and a reader replays those frames untimed. - -`Payload Offset` is relative to the start of `Payload Bytes`, immediately after the last table entry; `Payload Length` is the frame's payload size in bytes. -Payloads are concatenated in table order, without gaps, overlap, or trailing bytes. -They are the original frame payloads, including any group-scoped compression, without moq-lite FRAME headers. -A reader can locate a group or a frame index directly from the table without parsing earlier payloads. - -A decoder MUST validate the complete table before accessing any payload. -Counts and entries MUST fit in the retrieved object, and every offset and length MUST describe a range within its payload bytes, with overflow checked before arithmetic. -An unknown version or malformed table is a missing record, not an invalid recording. - -A recorded track's object MUST hold exactly the frames of its record's span ({{timeline-records}}): its first group and `Frame Start` match `start`, its groups are consecutive and none is empty, and its last group ends at `end`. -A group may span several objects when a record was split between frames; a reader stitches it from consecutive records. -Segment objects are immutable once written and are parseable with their track object alone, without the timeline. - -The timeline objects use the same envelope with a `Frame Start` of zero, each frame stamped with the `pts` of the record it was committed with, or unstamped for an untimed track. -Their own groups are discovered by listing, not indexed by a record, avoiding a record that must index itself. -A reader replays their checkpoints and operations in group order, preserving group boundaries and their independent DEFLATE windows. -Neither timeline nor media objects are appended to or rewritten. -One writer owns a recording prefix. An existing segment object key MAY be reused only for identical object bytes; a conflicting create MUST fail. - -## Writer Behavior {#recording-writer} -A writer subscribes to the broadcast and buffers each track's open span independently of the publisher or relay cache. -It cuts each track's timeline itself ({{timeline-cutting}}) and commits each track independently: a track whose span is complete does not wait for any other track. -The recording contract MUST NOT depend on a relay retaining those groups while storage catches up. - -A writer MUST accept new groups for each track in strictly increasing sequence order and MUST refuse a duplicate or decreasing sequence; it does not reorder arrivals. -Groups already accepted MAY complete in any order; a group's frames are indexed once every earlier accepted group has finished, and a group still open is indexed frame by frame as it grows. -A writer MUST NOT insert frames before the end of a record it already committed. -A frame whose timestamp is missing on a timed track, or present on an untimed one, is malformed. - -To commit record N of a track, the writer makes the track's `segments/N` object durable, publishes the record through the track's timeline encoder, applies any retention pops, closes the timeline's current Window group, and stores the complete groups under the timeline track's `segments/N` key. -It MUST make that timeline object durable before committing the track's next record or deleting expired objects. -If a track's object cannot be made durable, the writer drops the record: its frames are lost, and the next record takes sequence N. -If the timeline object cannot be made durable, the recording stops at the preceding durable timeline object. -The recording owns its timeline encoders so they publish only what was stored. - -On a clean end, the writer MUST close each track's final record, finish its timeline track, and make its final complete groups durable. -There is no completion marker; object listing alone does not distinguish a clean end from an interruption. - -## Retention {#recording-retention} -A recording has one of two retention modes: - -- An *archive* is unbounded and retains every record until explicitly deleted. -- A *DVR* retains a configured duration of each track's records and expires each track's oldest records as newer ones become durable. - -An application offering DVR without an explicit retention value SHOULD default to at least 30 seconds. -The writer removes a track's oldest record only while the track's remaining records still span the configured duration, measured from the second-oldest record's `pts` to the newest record's end. -A track's newest record is never removed, so a catalog that never changes outlives the media recorded with it. -Retention is measured from timeline records, not from wall-clock arrival or relay cache state. -An untimed track's records have no duration, so a DVR writer MUST refuse to record an untimed track. - -Trimming occurs only as part of committing a new record of the same track, before that record's timeline object is finalized ({{recording-writer}}). -A recording MUST NOT trim after a track's final record is committed. -Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired records' objects. -The writer MUST retain each timeline's latest object and enough earlier timeline groups to recover the retained window from a checkpoint. -It MAY delete a timeline's earlier objects, oldest first, so its remaining record IDs stay consecutive. -The index therefore never advertises media the retention process has already deleted, although media objects MAY temporarily outlive the index. -Relay cache eviction does not change the recording timeline. - -## Bootstrap and Recovery {#recording-recovery} -A reader or restarting writer lists each timeline track's `segments/` prefix and replays its objects in numeric sequence order, including retention operations, from a retained checkpoint. -The catalog supplies the timeline names ({{archive-catalog}}); a restarting writer finds them by their suffix ({{recording-layout}}). -The recovered records determine the committed object keys; a missing or malformed referenced object MUST NOT be served. -Objects not referenced by a recovered timeline do not advertise content on their own. -Listing is not an atomic snapshot across tracks; absence from an earlier listing MUST NOT override a successful GET of an object referenced by a later durable timeline record. - -Before accepting new groups, a restarting writer MUST recover each retained timeline under exclusive ownership of the recording prefix. -A track's objects at or past its next sequence were never referenced by a timeline, so the writer deletes them before reusing their keys. -It resumes each track after the end of its newest committed record, including partway through a group that record split. -A restarting DVR writer also lists every recorded track's `segments/` prefix and, after waiting the configured deletion grace period from successful recovery, deletes objects whose keys are absent from the recovered retained records, including expired objects left by interrupted expiration. -It MUST NOT perform this cleanup if timeline recovery or listing fails or is incomplete, or while another writer can create or commit objects. -This completes previously committed expiration; it does not pop additional records after a final record. -Timeline objects needed for checkpoint recovery and `.info` objects are not candidates for this cleanup. - -After replaying a timeline's record N below the recording ID limit, a reader follows an active recording by GET of that timeline's `segments/N+1`, without refreshing media listings. -At the limit there is no next key; this does not establish a clean recording end. -Not Found does not distinguish a pending commit, an expired DVR record, or an interrupted recording. -A reader that has fallen behind retention MUST bootstrap again from a retained checkpoint. -Alternatively, a reader MAY list timeline keys after its last replayed key, consume all pages, sort the results, and replay them in order. -A continuation token is used only within that enumeration; the next refresh starts from the last replayed key. -A reader MUST NOT advance its replay cursor past a missing object without recovering from a retained checkpoint. -Retention operations evict expired records from the reader's cached index; incremental listing alone does not report deletions. - -## Reader Behavior {#recording-reader} -Reading record N of track T is a GET of T's `segments/N`; the table MUST hold exactly the frames the record names. -Nothing on that path reads media the consumer did not ask for, and nothing requires a second request. - -A reader MAY serve moq-lite FETCH from a recording. -Given a group, a reader finds the retained records whose spans hold it, in order, and serves their frames from the requested `frame_start` {{moql}}. -A group whose earliest retained record starts after the requested frame is not served. -A group whose records do not reach its end yet, such as an append-log still being recorded, grows as later records commit, and ends where the recording does once the reader learns it ended. -The reader reconstructs moq-lite FRAME headers from the stored timestamps and lengths, preserving the original payload bytes. -A group absent from the recording is a normal FETCH failure. -Recording catalog groups does not establish which catalog update applies to a media group; this format does not define that correlation. - -A reader deriving a presentation-ordered format renders it from the timelines ({{timeline-derived}}) and transmuxes record objects on demand. -Nothing derived needs to be stored: the playlist or manifest is a function of the timelines, and a media segment is a function of the records its span covers. - - # Rooms A room is a broadcast path prefix. A participant publishes camera and microphone @@ -1368,12 +1175,6 @@ Clamping a reference that escapes above that root would silently redirect the su TODO Security -A consumer parsing a recording ({{recording}}) is parsing data at rest that it did not necessarily write. -It MUST validate the group/frame table against the bytes actually retrieved before allocating from its counts or accessing payloads ({{recording-segments}}), and treat a malformed object as a missing record rather than letting it invalidate the recording. -It MUST reject a track object with a zero `timescale`. -Varint fields are subject to the same limits as moq-lite {{moql}}. -A recording inherits the confidentiality and integrity properties of the storage holding it; encryption at rest is transparent to the format and out of scope. - # IANA Considerations @@ -1392,6 +1193,7 @@ This document has no IANA actions. - Added the root `version` ({{catalog-version}}): the oldest draft that reads the catalog, which a consumer refuses when above its own. A `track` other than the entry's ID or a video `warmup` needs 4. Consumers MUST ignore unknown fields. - A data track marks a discontinuity with a group of one object with no bytes, since its empty payload is data. An object with no bytes on an audio or video track, or sharing its group, is malformed. - Each archive timeline declares its own fixed nominal record duration, omitted for an untimed track; removed the broadcast-wide duration bound. +- Removed the Recording section: the store layout is a moq-archive format, documented with that crate, not part of this draft. - Added the optional video `preference` field: a consumer selects automatically among the highest preference it can decode. - Added an optional `epoch` beside every rendition and data track `broadcast` reference, and `replayEpoch` beside the archive's `replay` ({{field-epoch}}). A consumer reads each referenced path from the first broadcast it resolves for a catalog instance; a present `epoch` fails a rendition proven to resolve another instance, and is best effort over a route without an Epoch. - Derived HLS and DASH exports hand disabled references to enabled renditions at the first record numbered past the advertised ones, keeping the record `sequence` as the HLS media sequence and starting a DASH period. diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 1d940249f59..d528cdb82c3 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -42,8 +42,8 @@ archive (per-track `.timeline.z` records named by the catalog's root stall. Anything younger is still in the live cache. - **Per-track objects, keyed by position.** Groups coalesce in order into objects cut by duration, keyed - `//--` (`` - for an untimed track, `` percent-encoded as one + `//--` + (`-` for an untimed track, `` percent-encoded as one segment), beside a small per-track `.info` holding its timescale and priority. Each stored sequence keeps its form, group or datagram. A track's `end` records how it ended, clean or with its error code, written @@ -55,7 +55,7 @@ archive (per-track `.timeline.z` records named by the catalog's root stored keys and another epoch is another prefix. An optional DVR mode prunes old objects. Rejected: an object per whole-broadcast window (every read pulls every rendition) and size-based cuts. The - [producer](/quest/m1/archive/producer.md) records the rest. + `archive::Producer` records the rest. - **Hosts serve archives, relays do not.** `archive::Consumer` serves a stored prefix through `broadcast::Dynamic` and `track::Dynamic`, the archived catalog included as an ordinary track, and a host publishes it at a path of @@ -85,7 +85,6 @@ from a host. Each child fixes the stale docs it touches. ## Required - [TRACKS in js/net](/quest/m1/archive/tracks-js.md) - `@moq/net` lists a broadcast's tracks over lite-07's Tracks Stream and answers listings of its own broadcasts, mirroring Rust -- [Archive producer](/quest/m1/archive/producer.md) - `archive::Producer` stores every matching broadcast as duration-cut per-track objects keyed by group and pts, recording gaps instead of stalling, with an optional DVR mode - [Archive consumer](/quest/m1/archive/consumer.md) - `archive::Consumer` serves a stored prefix as a broadcast from its key listing, and `moq export/import archive` use the new pair - [Catalog HLS field](/quest/m1/archive/catalog-hls.md) - the root `archive` section is gone; an opt-in per-rendition `hls` field marks each aligned HLS segment's first group and pts and declares its nominal duration - [SUBSCRIBE_TRACKS](/quest/m1/archive/subscribe-tracks.md) - moq-transport SUBSCRIBE_TRACKS is answered through the TRACKS hook, and an IETF peer's tracks are learned with FORWARD=0, instead of NOT_SUPPORTED diff --git a/quest/m1/archive/consumer.md b/quest/m1/archive/consumer.md index a71663eee6f..a356218ea04 100644 --- a/quest/m1/archive/consumer.md +++ b/quest/m1/archive/consumer.md @@ -5,9 +5,10 @@ `archive::Consumer` serves one broadcast's stored key prefix as an ordinary broadcast: every stored track, the archived catalog included, with its original groups, frames, timescale, and priority. A host publishes it at a path of its choosing; -moq-relay never uses it. `moq export archive` records through -`archive::Producer` and `moq import archive` serves through -`archive::Consumer`. It replaces `moq_archive::Reader`. +moq-relay never uses it. `moq export archive` already records through +`archive::Producer`; `moq import archive` serves through +`archive::Consumer`. It replaces `moq_archive::Reader`, and with it goes +`.info`'s tolerance of the `epoch` an older `Writer` stored. ## Plan @@ -20,8 +21,8 @@ Decided by the maintainer on 2026-10-17 (see the track with its own cursor, a LIST of that track's key prefix StartAfter its newest known key, since one prefix-wide cursor skips a new key under a lexically earlier track. A delimiter LIST of the broadcast prefix finds - tracks added since. It also re-reads a track's newest object, which the - producer may rewrite in place while an untimed group is open. + tracks added since. Every object is written once, an open group continuing + in the next object (decided 2026-10-21). - Each served track advertises no Publisher Max Age limit (TRACK_INFO's encoded `0`), not the source's: the store's retention bounds what it holds, and DVR pruning is the publisher dropping a group early, which TRACK_INFO @@ -81,7 +82,3 @@ and re-added under its name replaying both runs in sequence, a growing archive with no markers staying open, and an empty or unrecognized prefix refused loudly. Public API: `archive::Consumer` replaces `Reader`. Wire: none. - -## Required - -- [Archive producer](/quest/m1/archive/producer.md) - the layout this serves diff --git a/quest/m1/archive/producer.md b/quest/m1/archive/producer.md deleted file mode 100644 index bac4caf7453..00000000000 --- a/quest/m1/archive/producer.md +++ /dev/null @@ -1,157 +0,0 @@ -# [XL] Archive producer - -## Goal - -`archive::Producer` takes a path prefix and stores every track of every -broadcast under it into an `object_store::ObjectStore`, -each broadcast under its own key prefix. It holds what the publisher's cache -stops serving: a lost group becomes a gap and never stalls the archive. -Datagrams are lossy and never cached, so a track's datagrams are recorded -from when its subscription takes effect; earlier ones are not. An -optional DVR mode prunes old objects. It replaces `moq_archive::Writer` and -its timelines and recovery machinery. - -## Plan - -Decided by the maintainer on 2026-10-17 (see the -[epic](/quest/m1/archive/README.md)): - -- **Input.** One producer records every broadcast under a prefix as it - appears: ANNOUNCE finds each broadcast announced at its own path, - TRACKS (`broadcast::Consumer::tracks`) lists its tracks, and each track - records through its own SUBSCRIBE. A covering-prefix route is capability, - not inventory, so broadcasts served under one on request without their own - announcement are not recorded, and a TRACKS refused with NOT_FOUND on an - announced path means it is not a broadcast; any other refusal, such as - UNAUTHORIZED, is that broadcast's error and reaches the caller, never a - silent skip - (decided 2026-10-19, replacing one SUBSCRIBE_ALL). The application maps - each broadcast (path and epoch) to its key prefix, so a different epoch is - a different prefix and the producer stores no epoch. Open since discovery - became per broadcast: keep the prefix input, or take one broadcast and leave - the ANNOUNCE loop to the host. Recommended: keep the prefix, so every host - does not repeat that loop. -- **Ordering.** Each track records through an ascending subscription with max - delay = min(the publisher's max age from TRACK_INFO, a configured cap). A - group still missing at the cap is skipped and stays absent from the store, - which is how a gap is recorded. Anything younger is still in the live cache. -- **Objects.** Groups coalesce in order into per-track objects, cut at a group - boundary once a configured duration passes. An oversized timed group splits - between frames. An untimed track (no timescale) cuts by wall clock at group - boundaries. Each object preserves group sequences, frame boundaries, - timestamps, and payloads. Rejected: whole-broadcast objects per window and - size-based cuts. -- **Datagrams keep their form.** A track's groups and datagrams share one - sequence space, so each stored sequence records its forwarding form, group - or datagram, and a datagram is stored as its one payload and timestamp in - sequence order beside the groups. The consumer replays a datagram as a - datagram, never as a cached one-frame group, so the replay keeps the - source's lossy contract. A datagram is never split and counts as one - sequence for keys, cuts, and resume. Datagrams sent before the track's - SUBSCRIBE took effect are not recorded, as above. -- **Keys.** `//--` for a - timed track and `//` for an untimed one, - zero-padded so keys sort in content order. The frame index keeps the - fragments of an oversized timed group unique and ordered, since they share - the group and pts can repeat or go backward in decode order; pts stays for - time lookups. `` is the name in the existing canonical percent - encoding (`rs/moq-archive/src/path.rs`, everything but `A-Z a-z 0-9 _ -` - escaped), so a name holding `/` stays one segment and no name collides with - `.info` or `.end`; a consumer refuses a non-canonical segment. A sorted LIST answers - group and time lookups; there is no index object or manifest. Rejected for - untimed: `-`, since an untimed group is never - split. -- **Untimed groups never split.** While an untimed group is open, the bounded - flush rewrites its object in place under the same key, so keys stay unique - without a frame suffix; the consumer re-reads the newest object of a growing - track. Folded from the untimed quest (#5720 landed untimed in the old writer). -- **Track properties.** A small `//.info` holds the source - properties the stored media depends on (timescale, priority), written before - the track's first object, so a consumer knows them before reading media. - Properties never change for a track name, so a resume that finds different - ones fails loud. The source's max age is not stored: it bounds the live - cache, not the archive, and replaying it would expire archived history - (timestamp age measures a group against the newest one), so the consumer - advertises its own retention instead. Rejected: properties only in each - object's header. -- **Completion markers**, decided by the maintainer on 2026-10-19 ("record - the error"). A track has ended once its ordered groups and its datagrams - (`poll_recv_datagram`) have both reached their end, so no received - datagram is lost. How it ended is decided then: a clean end at its - declared exclusive `Position` (`track::Subscriber::poll_finished`; group - and frame, the shape `track::Producer::finish_at` takes, so a mid-group - end survives), or the stream error code it ended with - (`StreamError::to_code`). The markers wait for the broadcast's listing to - end with the instance, since until then a track removed and re-added - under its name continues its sequences and a written marker would be - wrong. Then, with every frame and datagram stored, the producer writes - each track's small `//end`, recording how its last run - ended, followed by `/.end`, which says no track will be added. - Until then an ended track just stops growing. The names are chosen for the consumer's - refresh: a track's `end` sorts after its zero-padded numeric keys, so - the track's `StartAfter` LIST reaches it with no extra request, and it can - never be an object key; the broadcast's `.end` sits beside the track - segments, where the delimiter LIST that finds added tracks sees it, and - no canonical track name starts with `.`. The consumer ends each track as - its marker says and otherwise treats the archive as live. Only a crash, or a producer - stopped before the end, leaves no marker, so its archive reads as still - growing. A producer resuming mid-broadcast follows every stored track as - live, including one that ended before the restart; it writes markers only - for tracks whose end it saw, so a stored track that ended unseen leaves - the broadcast without `.end`, as a stop before the end does. A resume - under a broadcast that already has its `.end` records nothing. DVR - pruning never deletes a marker. -- **Bounded flush** (folded from the idle-flush quest): an object holding a - frame is stored within a bounded wall-clock delay of that frame's arrival, - even when its track goes idle. The bound covers closing the object, not - upload latency. -- **No manual `flush()` or `cut()`** until a caller needs one. -- **Resume.** Under a key prefix that already holds objects, each track - continues at the frame after its last stored one: its subscription starts - at `Position(last group, stored frame count)`, so a group split across - objects or still open at the restart loses nothing. A timed track stores - the rest as a new object keyed by its first frame's index and pts. An untimed track - reads its newest object back and keeps rewriting it in place with the - appended frames. A completed group yields no more frames, so this is the - same rule whether or not the restart landed mid-group; if the source has - dropped the rest, the stored frames are all the archive holds of it. The - start is per track: each track's SUBSCRIBE carries its own floor. - Rejected: resuming after the - last stored group (loses a split or open group's tail) and re-recording the - last group from frame 0 (fails once the source has dropped its head). -- **DVR mode** stays in the crate, for hosts without bucket lifecycle rules: - it deletes a track's objects older than a configured window after a grace - period, so readers in flight finish, and always keeps each track's newest - object so a static catalog outlives the media around it. Rejected: no - retention in the crate. -- **Layout docs.** Delete the hang draft's Recording section - (`drafts/draft-lcurley-moq-hang.md`): the layout is a `moq-archive` format, - not a hang one. Document it in moq-archive rustdoc and the epic's Archives - doc page. - -Test on mock time with the in-memory store: a lost group recorded as a gap -without stalling later groups, duration and wall-clock cuts, a split oversized -timed group whose fragments get unique keys in frame order even when pts -repeats or goes backward, an open untimed group rewritten in place, an idle track flushed -within the bound, resume after a restart (including mismatched properties, -and a restart mid-group for both a split timed group and an open untimed -group, with no frame lost or duplicated), a track name holding `/` or -starting with `.` round-tripping as one segment, -DVR pruning with grace, a track added mid-recording, two broadcasts under -the prefix landing under their own key prefixes, a track interleaving groups -and datagrams stored with each sequence's form, a clean end writing no -marker until the listing ends and then each track's `end` (including a -mid-group end) followed by the broadcast's `.end`, a track removed and -re-added under its name before the broadcast ends whose replay holds both -runs in sequence and whose marker records the second run's end, a track -whose groups and datagrams interleave and end at once storing every -received datagram before its `end`, a track ending with an error writing an -error marker while its broadcast still gets `.end`, and a producer stopped -mid-recording writing no marker. - -Public API: `archive::Producer` and its config replace `Writer`, `Control`, -and the timeline-driven types. Wire: none. - -## Related - -- [Archive consumer](/quest/m1/archive/consumer.md) - reads this layout back diff --git a/quest/m1/resolved-demand.md b/quest/m1/resolved-demand.md index a00ee625d33..6c945e1b0e2 100644 --- a/quest/m1/resolved-demand.md +++ b/quest/m1/resolved-demand.md @@ -174,6 +174,5 @@ max age or the range. ## Related -- [Archive producer](/quest/m1/archive/producer.md) - its resume floor crosses relays that viewers may have warmed - [Drop set](/quest/m1/drop-set.md) - closes the gap this leaves for groups already reported dropped - [JS unbounded groups](/quest/m2/unbounded-groups/js.md) - the JS byte bound for groups and the cursor runs between them diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index 533f73963c1..e2ceac06336 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -22,8 +22,6 @@ and fix the rest the same way: streams in `moq-rtmp`, `moq-srt`, `moq-rtc`) and `--connect` (one push). SRT already rides the TS follower (`--linger`, `--stitch`); confirm its listen side does too. FLV and WebRTC need a decision. -- `export archive`: a restart already fails a continued store under a new - epoch; decide what a mid-recording restart does. - `transcode`: its source broadcast; following means restarting the ladder on the new instance. diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 3a9ad66bc6e..a0b47661cbc 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "moq-archive" -description = "Versioned hang recording objects on any object_store::ObjectStore" +description = "MoQ broadcast archives on any object_store::ObjectStore" authors = ["Luke Curley "] repository = "https://github.com/moq-dev/moq" license = "MIT OR Apache-2.0" @@ -19,7 +19,6 @@ hang = { workspace = true } kio = { workspace = true } moq-clock = { workspace = true } moq-json = { workspace = true } -moq-mux = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } @@ -41,5 +40,5 @@ tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "net", "rt", "test-util"] } [[bench]] -name = "writer" +name = "producer" harness = false diff --git a/rs/moq-archive/MIGRATE.md b/rs/moq-archive/MIGRATE.md index 93e45670400..35fa7a38048 100644 --- a/rs/moq-archive/MIGRATE.md +++ b/rs/moq-archive/MIGRATE.md @@ -4,11 +4,20 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Producer` replaces `Writer` (#6160) + +`Writer`, its `Control`, and `writer::{Config, Retention}` are removed; record with `Producer::new(store, broadcast, producer::Config::default()).run().await` instead. +A producer records every track of one broadcast, found with TRACKS rather than a catalog, and needs no timeline: the layout is per-track objects keyed by group, frame, and timestamp, documented in the crate docs. +Run one producer per broadcast under a key prefix you pick, such as `/@`; it stores no epoch, so `Info::epoch` and `Info::with_epoch` are removed. +`writer::Retention` becomes `producer::Dvr`, set with `Config::with_dvr`, and works on untimed tracks too. +`Error::{EpochMismatch, Enrolled, Untimed, Timeline, Closed}` are removed; a failed source listing is `Error::Source`. +The new layout is not readable by `Reader`, which still replays timeline recordings. + ### Untimed tracks record untimed (#5720) `Info::new` and `Info::timescale` take and return `Option`, and `Error::TimescaleMismatch` carries `Option` fields. `segment::Frame::timestamp` is `Option`. `Object::decode(buf, timed)` and `Store::get_segments(track, segment, timed)` take whether the track is timed. -`Error::Untimed` now means a DVR cannot record an untimed track; mismatched timing is `Error::Timing`. +Mismatched timing is `Error::Timing`. Untimed recordings use the version 2 format, which an older version 2 reader refuses. Pass `Some(timescale)` to `Info::new`, and `info.timescale.is_some()` as `timed`. diff --git a/rs/moq-archive/README.md b/rs/moq-archive/README.md index a357d1f807b..af917e9a37f 100644 --- a/rs/moq-archive/README.md +++ b/rs/moq-archive/README.md @@ -4,25 +4,26 @@ # moq-archive -Versioned [hang](https://docs.rs/hang) recordings on any +MoQ broadcast archives on any [`object_store::ObjectStore`](https://docs.rs/object_store). -The crate owns the portable layout and codecs: percent-encoded track names, -per-track `.info` JSON, the binary segment envelope, and put/get/list/delete. -`Store` wraps a store under a recording prefix and stays generic, so a caller -that needs runtime dispatch passes `Arc`. A broadcast -advertises its recording through the catalog's -[`archive`](https://doc.moq.dev/concept/hang) entry. +`Producer` records every track of one broadcast under a key prefix, with no +catalog dependency: it lists the tracks with TRACKS and subscribes to each, +storing duration-cut objects keyed by group, frame, and timestamp. A lost +group becomes a gap instead of a stall, a restarted producer resumes each +track where it stopped, and an optional DVR window prunes old objects. The +host picks the prefix and runs one producer per broadcast. ```bash cargo add moq-archive ``` -Group bounds are finite inclusive ranges in first-to-last order: - ```rust -let key = moq_archive::Key::groups("video", 5..=7)?; -assert_eq!(key.track(), "video"); +use moq_archive::object_store::memory::InMemory; + +// `broadcast` is a `moq_net::broadcast::Consumer`. +let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +moq_archive::Producer::new(store, broadcast, Default::default()).run().await?; ``` See [docs.rs/moq-archive](https://docs.rs/moq-archive) for the object layout. diff --git a/rs/moq-archive/benches/producer.rs b/rs/moq-archive/benches/producer.rs new file mode 100644 index 00000000000..12e41f8fb7e --- /dev/null +++ b/rs/moq-archive/benches/producer.rs @@ -0,0 +1,90 @@ +// Host time measures benchmark throughput. +#![allow(clippy::disallowed_methods, clippy::disallowed_types)] + +//! `Producer::run` recording every track of a broadcast into memory, swept over the number of +//! tracks and the groups each one holds. Throughput is per frame, so a per-event cost that grows +//! with the track table shows up as a slope across the track counts. +//! +//! Run with `cargo bench -p moq-archive --bench producer`. + +use std::time::{Duration, Instant}; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::{Producer, Store, producer}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; + +const FRAMES_PER_GROUP: u64 = 2; + +/// A broadcast of `tracks` tracks, each already holding `groups` one-second groups. +fn setup(tracks: usize, groups: u64) -> (broadcast::Producer, Vec) { + let source = broadcast::Info::new().produce(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let tracks = (0..tracks) + .map(|index| { + let track = source.create_track(format!("track{index}"), info.clone()).unwrap(); + for sequence in 0..groups { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for frame in 0..FRAMES_PER_GROUP { + let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); + group.write_frame(pts, vec![0u8; 64]).unwrap(); + } + group.finish().unwrap(); + } + track + }) + .collect(); + (source, tracks) +} + +/// Record the broadcast until every track ends, from the producer's start. +async fn record(source: broadcast::Producer, tracks: Vec) -> Duration { + let config = producer::Config::default() + .with_duration(Duration::from_secs(2)) + .with_max_delay(Duration::from_secs(3600)); + let store = Store::new(InMemory::new(), "bench"); + let start = Instant::now(); + let run = tokio::spawn(Producer::new(store, source.consume(), config).run()); + for track in &tracks { + while track.subscription().is_none() { + tokio::task::yield_now().await; + } + track.finish().unwrap(); + } + source.close(); + run.await.unwrap().unwrap(); + start.elapsed() +} + +fn producer(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + let mut bench = c.benchmark_group("producer"); + bench.sample_size(10); + bench.warm_up_time(Duration::from_secs(1)); + bench.measurement_time(Duration::from_secs(2)); + for tracks in [1, 10, 100] { + for groups in [10, 100] { + bench.throughput(Throughput::Elements(tracks as u64 * groups * FRAMES_PER_GROUP)); + bench.bench_function(BenchmarkId::new(format!("{groups} groups"), tracks), |b| { + b.iter_custom(|iters| { + (0..iters) + .map(|_| { + let (source, tracks) = setup(tracks, groups); + runtime.block_on(record(source, tracks)) + }) + .sum() + }); + }); + } + } + bench.finish(); +} + +criterion_group!(benches, producer); +criterion_main!(benches); diff --git a/rs/moq-archive/benches/writer.rs b/rs/moq-archive/benches/writer.rs deleted file mode 100644 index d342d9b7c1f..00000000000 --- a/rs/moq-archive/benches/writer.rs +++ /dev/null @@ -1,81 +0,0 @@ -// Host time measures benchmark throughput. -#![allow(clippy::disallowed_methods, clippy::disallowed_types)] - -//! `Writer::run` recording every enrolled track into memory, swept over the number of tracks and -//! the groups each one publishes. Throughput is per frame, so a per-event cost that grows with the -//! track table shows up as a slope across the track counts. -//! -//! Run with `cargo bench -p moq-archive --bench writer`. - -use std::time::{Duration, Instant}; - -use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_archive::object_store::memory::InMemory; -use moq_archive::{Store, Writer, writer}; -use moq_net::{Timescale, Timestamp, broadcast, group, track}; - -const FRAMES_PER_GROUP: u64 = 2; - -/// A writer with `tracks` enrolled tracks, each already holding `groups` one-second groups. -async fn setup(tracks: usize, groups: u64) -> Writer { - let source = broadcast::Info::new().produce(); - let store = Store::new(InMemory::new(), "bench"); - let writer = Writer::new(store, source.consume(), writer::Config::default()) - .await - .unwrap(); - let control = writer.control(); - let config = moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)); - - for index in 0..tracks { - let name = format!("track{index}"); - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - let track = source.create_track(name.as_str(), info).unwrap(); - control.track(&name, config.clone()).await.unwrap(); - for sequence in 0..groups { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for frame in 0..FRAMES_PER_GROUP { - let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); - group.write_frame(pts, vec![0u8; 64]).unwrap(); - } - group.finish().unwrap(); - } - track.finish().unwrap(); - } - source.close(); - writer -} - -fn writer(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - let mut bench = c.benchmark_group("writer"); - bench.sample_size(10); - bench.warm_up_time(Duration::from_secs(1)); - bench.measurement_time(Duration::from_secs(2)); - for tracks in [1, 10, 100] { - for groups in [10, 100] { - bench.throughput(Throughput::Elements(tracks as u64 * groups * FRAMES_PER_GROUP)); - bench.bench_function(BenchmarkId::new(format!("{groups} groups"), tracks), |b| { - b.iter_custom(|iters| { - let mut total = Duration::ZERO; - for _ in 0..iters { - let writer = runtime.block_on(setup(tracks, groups)); - let start = Instant::now(); - runtime.block_on(writer.run()).unwrap(); - total += start.elapsed(); - } - total - }); - }); - } - } - bench.finish(); -} - -criterion_group!(benches, writer); -criterion_main!(benches); diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index e33439b030a..6bbd956dd41 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -40,20 +40,6 @@ pub enum Error { intended: Option, }, - /// An existing `.info` was recorded from another publisher instance: the source - /// restarted, so the caller starts a new prefix. - #[error( - "source epoch {} does not match the recorded {}", - epoch_text(intended), - epoch_text(existing) - )] - EpochMismatch { - /// The epoch already recorded. - existing: Option, - /// The epoch of the source being recorded now. - intended: Option, - }, - /// The object has no groups. #[error("empty object")] Empty, @@ -102,14 +88,6 @@ pub enum Error { #[error("moq: {0}")] Moq(String), - /// The track was already enrolled, or is the recording's own timeline. - #[error("track already enrolled: {0}")] - Enrolled(String), - - /// The track is untimed, so a DVR has no content time to measure its retention by. - #[error("untimed track: {0}")] - Untimed(String), - /// A frame's timestamp does not match its track: missing on a timed track, or present on an /// untimed one. #[error("frame timing does not match the track")] @@ -118,14 +96,6 @@ pub enum Error { /// The source broadcast or one of its tracks failed. #[error("source: {0}")] Source(String), - - /// The recording's timeline could not be recovered, segmented, or published. - #[error("timeline: {0}")] - Timeline(String), - - /// The writer stopped accepting commands. - #[error("writer closed")] - Closed, } impl From for Error { @@ -151,10 +121,3 @@ impl From for Error { /// A [`Result`](std::result::Result) using this crate's [`Error`]. pub type Result = std::result::Result; - -fn epoch_text(epoch: &Option) -> String { - match epoch { - Some(epoch) => epoch.to_string(), - None => "none".to_string(), - } -} diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index cfd546005b9..ff90254914d 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -6,7 +6,7 @@ use crate::{Error, Result, VERSION}; /// Immutable track properties stored at `/.info`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[serde(from = "Stored")] pub struct Info { /// Recording format version; must be [`VERSION`]. pub version: u64, @@ -16,9 +16,29 @@ pub struct Info { /// is untimed and its frames store no timestamps. #[serde(default, skip_serializing_if = "Option::is_none")] pub timescale: Option, - /// The source's publisher epoch, when its route had one. A resume from another refuses. - #[serde(default, skip_serializing_if = "Option::is_none", with = "epoch")] - pub epoch: Option, +} + +/// [`Info`] as stored, also taking the `epoch` an older `Writer` recorded, so [`Reader`](crate::Reader) +/// still replays those recordings; it goes when `Reader` does. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Stored { + version: u64, + priority: u8, + #[serde(default)] + timescale: Option, + #[serde(default, rename = "epoch")] + _epoch: Option, +} + +impl From for Info { + fn from(stored: Stored) -> Self { + Self { + version: stored.version, + priority: stored.priority, + timescale: stored.timescale, + } + } } impl Info { @@ -28,18 +48,11 @@ impl Info { version: VERSION, priority, timescale, - epoch: None, }; info.validate()?; Ok(info) } - /// Record `epoch` as the source's publisher epoch. - pub fn with_epoch(mut self, epoch: Option) -> Self { - self.epoch = epoch; - self - } - /// Encode as compact JSON. pub fn encode(&self) -> Result { self.validate()?; @@ -67,25 +80,6 @@ impl Info { } } -/// The epoch as its UUID text. -mod epoch { - use serde::{Deserialize, Deserializer, Serializer, de::Error}; - - pub fn serialize(epoch: &Option, serializer: S) -> Result { - match epoch { - Some(epoch) => serializer.serialize_str(epoch.as_str()), - None => serializer.serialize_none(), - } - } - - pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { - let Some(text) = Option::::deserialize(deserializer)? else { - return Ok(None); - }; - text.parse().map(Some).map_err(D::Error::custom) - } -} - #[cfg(test)] mod tests { use super::*; @@ -126,22 +120,6 @@ mod tests { } } - #[test] - fn epoch_roundtrip() { - let epoch: moq_net::Epoch = "01900000-0000-7000-8000-000000000001".parse().unwrap(); - let info = Info::new(0, Some(1000)).unwrap().with_epoch(Some(epoch)); - let bytes = info.encode().unwrap(); - assert_eq!( - bytes.as_ref(), - br#"{"version":2,"priority":0,"timescale":1000,"epoch":"01900000-0000-7000-8000-000000000001"}"# - ); - assert_eq!(Info::decode(&bytes).unwrap(), info); - assert!(matches!( - Info::decode(br#"{"version":2,"priority":0,"timescale":1000,"epoch":"not-a-uuid"}"#), - Err(Error::Json(_)) - )); - } - #[test] fn timescale_bounds() { assert!(Info::new(0, Some(1)).is_ok()); @@ -150,6 +128,16 @@ mod tests { assert!(matches!(Info::new(0, Some(ID_MAX + 1)), Err(Error::Timescale(_)))); } + #[test] + fn an_older_writers_epoch_is_ignored() { + let info = Info::decode(br#"{"version":2,"priority":0,"timescale":1000,"epoch":"x"}"#).unwrap(); + assert_eq!(info, Info::new(0, Some(1000)).unwrap()); + assert_eq!( + info.encode().unwrap().as_ref(), + br#"{"version":2,"priority":0,"timescale":1000}"# + ); + } + #[test] fn unknown_members_and_missing_fields_are_refused() { assert!(matches!( diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs new file mode 100644 index 00000000000..a2c44366129 --- /dev/null +++ b/rs/moq-archive/src/layout.rs @@ -0,0 +1,597 @@ +//! The keys and objects of the store layout a [`Producer`](crate::Producer) writes. +//! +//! See the [crate docs](crate#layout) for the layout itself. + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use moq_net::track::Position; +use moq_net::varint; +use object_store::path::Path; +use serde::{Deserialize, Serialize}; + +use crate::path::{check_id, decode_track, encode_track, push}; +use crate::segment::Frame; +use crate::{Error, ID_WIDTH, Result, VERSION}; + +/// The broadcast's completion marker, beside its track segments. +pub(crate) const BROADCAST_END: &str = ".end"; +/// A track's properties, which sort before every object key. +pub(crate) const TRACK_INFO: &str = ".info"; +/// A track's completion marker, which sorts after every object key. +pub(crate) const TRACK_END: &str = "end"; + +/// A track's directory under a broadcast prefix: its name as one canonical segment. +pub(crate) fn track_dir(prefix: &Path, track: &str) -> Result { + push(prefix, &encode_track(track)?) +} + +/// What one entry under a track's directory is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Name { + /// `.info` + Info, + /// `end` + End, + /// An object, named by where its content starts. + Object(Start), +} + +impl Name { + /// Parse the last segment of a key under a track's directory, refusing anything else. + pub(crate) fn parse(name: &str, timed: bool) -> Result { + match name { + TRACK_INFO => Ok(Self::Info), + TRACK_END => Ok(Self::End), + name => Ok(Self::Object(Start::parse(name, timed)?)), + } + } +} + +/// Decode a track directory's segment back into its name, refusing a non-canonical one. +pub(crate) fn parse_track(segment: &str) -> Result { + decode_track(segment) +} + +/// Where an object's content starts, which is its key: the first sequence, the first frame's +/// index within it, and, on a timed track, that frame's timestamp. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct Start { + pub group: u64, + pub frame: u64, + pub pts: Option, +} + +impl Start { + /// `--` on a timed track, `-` on an untimed one, + /// zero-padded so keys sort in content order. + pub(crate) fn name(&self) -> Result { + let (group, frame) = (id(self.group)?, id(self.frame)?); + match self.pts { + Some(pts) => Ok(format!("{group}-{frame}-{}", id(pts)?)), + None => Ok(format!("{group}-{frame}")), + } + } + + fn parse(name: &str, timed: bool) -> Result { + let fields: Vec<&str> = name.split('-').collect(); + let invalid = || Error::Path(name.to_string()); + match (timed, fields.as_slice()) { + (true, [group, frame, pts]) => Ok(Self { + group: parse_id(group).ok_or_else(invalid)??, + frame: parse_id(frame).ok_or_else(invalid)??, + pts: Some(parse_id(pts).ok_or_else(invalid)??), + }), + (false, [group, frame]) => Ok(Self { + group: parse_id(group).ok_or_else(invalid)??, + frame: parse_id(frame).ok_or_else(invalid)??, + pts: None, + }), + _ => Err(invalid()), + } + } +} + +fn id(value: u64) -> Result { + Ok(format!("{:0width$}", check_id(value)?, width = ID_WIDTH)) +} + +/// `None` when the field is not exactly [`ID_WIDTH`] digits. +fn parse_id(field: &str) -> Option> { + if field.len() != ID_WIDTH || !field.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + Some( + field + .parse() + .map_err(|_| Error::Path(field.to_string())) + .and_then(check_id), + ) +} + +/// How a track's last run ended, stored in its `end` once the broadcast ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase", deny_unknown_fields)] +pub(crate) enum End { + /// A clean end at this exclusive position. + Finish(At), + /// The stream error code it ended with. + Error(u32), +} + +/// A [`Position`] as stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct At { + pub group: u64, + pub frame: u64, +} + +impl From for At { + fn from(position: Position) -> Self { + Self { + group: position.group, + frame: position.frame, + } + } +} + +impl End { + pub(crate) fn encode(&self) -> Result { + Ok(Bytes::from(serde_json::to_vec(self)?)) + } + + pub(crate) fn decode(bytes: &[u8]) -> Result { + Ok(serde_json::from_slice(bytes)?) + } +} + +/// One stored sequence, in the form it was forwarded in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Entry { + /// A group's frames: all of them, or the run of them this object holds. + Group { sequence: u64, frames: Vec }, + /// A datagram: one payload and timestamp, never split. + Datagram { sequence: u64, frame: Frame }, +} + +impl Entry { + pub(crate) fn sequence(&self) -> u64 { + match self { + Self::Group { sequence, .. } | Self::Datagram { sequence, .. } => *sequence, + } + } + + fn first(&self) -> &Frame { + match self { + Self::Group { frames, .. } => &frames[0], + Self::Datagram { frame, .. } => frame, + } + } +} + +/// One object: entries in ascending sequence order, the first starting at `frame_start`. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct Object { + /// The index of the first group's first stored frame; zero when the first entry is a datagram. + pub frame_start: u64, + pub entries: Vec, +} + +const GROUP: u64 = 0; +const DATAGRAM: u64 = 1; + +impl Object { + /// The key this object is stored under. + pub(crate) fn start(&self) -> Result { + let first = self.entries.first().ok_or(Error::Empty)?; + Ok(Start { + group: first.sequence(), + frame: self.frame_start, + pts: first.first().timestamp, + }) + } + + /// The position just past the last stored frame or datagram. + pub(crate) fn end(&self) -> Result { + let last = self.entries.last().ok_or(Error::Empty)?; + Ok(match last { + Entry::Datagram { sequence, .. } => Position::group(sequence.checked_add(1).ok_or(Error::Overflow)?), + Entry::Group { sequence, frames } => { + let first = match self.entries.len() { + 1 => self.frame_start, + _ => 0, + }; + Position { + group: *sequence, + frame: first.checked_add(frames.len() as u64).ok_or(Error::Overflow)?, + } + } + }) + } + + /// The version, the first frame index, then each entry's sequence (the first absolute, the + /// rest `current - previous - 1`), form, and frame table, followed by every payload in order. + pub(crate) fn encode(&self, timed: bool) -> Result { + self.validate(timed)?; + + let mut table = BytesMut::new(); + write(&mut table, VERSION)?; + write(&mut table, self.frame_start)?; + write(&mut table, self.entries.len() as u64)?; + + let mut payload = BytesMut::new(); + let mut frame = |table: &mut BytesMut, frame: &Frame| -> Result<()> { + if let Some(timestamp) = frame.timestamp { + write(table, timestamp)?; + } + write(table, frame.payload.len() as u64)?; + payload.extend_from_slice(&frame.payload); + Ok(()) + }; + + let mut prev: Option = None; + for entry in &self.entries { + let sequence = entry.sequence(); + let delta = match prev { + None => sequence, + Some(prev) => sequence - prev - 1, + }; + prev = Some(sequence); + write(&mut table, delta)?; + match entry { + Entry::Group { frames, .. } => { + write(&mut table, GROUP)?; + write(&mut table, frames.len() as u64)?; + for f in frames { + frame(&mut table, f)?; + } + } + Entry::Datagram { frame: f, .. } => { + write(&mut table, DATAGRAM)?; + frame(&mut table, f)?; + } + } + } + + table.extend_from_slice(&payload); + Ok(table.freeze()) + } + + /// Decode a whole table before slicing any payload; `timed` is whether the track's `.info` + /// declares a timescale, which says whether each frame stores a timestamp. + pub(crate) fn decode(mut buf: impl Buf, timed: bool) -> Result { + let version = read(&mut buf)?; + if version != VERSION { + return Err(Error::Version(version)); + } + let frame_start = read(&mut buf)?; + // Each entry is at least a sequence, a form, and a frame count or length. + let count = read_count(&mut buf, 3)?; + + let per_frame = if timed { 2 } else { 1 }; + let mut lengths = Vec::new(); + let mut read_frame = |buf: &mut dyn Buf| -> Result> { + let timestamp = match timed { + true => Some(check_id(read(buf)?)?), + false => None, + }; + lengths.push(read(buf)?); + Ok(timestamp) + }; + + enum Raw { + Group(u64, Vec>), + Datagram(u64, Option), + } + let mut raw = Vec::with_capacity(count); + let mut prev: Option = None; + for _ in 0..count { + let delta = read(&mut buf)?; + let sequence = match prev { + None => delta, + Some(prev) => prev + .checked_add(1) + .and_then(|next| next.checked_add(delta)) + .ok_or(Error::Overflow)?, + }; + check_id(sequence)?; + prev = Some(sequence); + match read(&mut buf)? { + GROUP => { + let frames = read_count(&mut buf, per_frame)?; + let mut timestamps = Vec::with_capacity(frames); + for _ in 0..frames { + timestamps.push(read_frame(&mut buf)?); + } + raw.push(Raw::Group(sequence, timestamps)); + } + DATAGRAM => raw.push(Raw::Datagram(sequence, read_frame(&mut buf)?)), + _ => return Err(Error::Table), + } + } + + let mut payload = buf.copy_to_bytes(buf.remaining()); + let total = lengths + .iter() + .try_fold(0u64, |sum, len| sum.checked_add(*len)) + .ok_or(Error::Overflow)?; + if total != payload.len() as u64 { + return Err(Error::Table); + } + + let mut lengths = lengths.into_iter(); + let mut take = |timestamp: Option| -> Frame { + let len = lengths.next().expect("one length per frame") as usize; + Frame { + timestamp, + payload: payload.split_to(len), + } + }; + let entries = raw + .into_iter() + .map(|raw| match raw { + Raw::Group(sequence, timestamps) => Entry::Group { + sequence, + frames: timestamps.into_iter().map(&mut take).collect(), + }, + Raw::Datagram(sequence, timestamp) => Entry::Datagram { + sequence, + frame: take(timestamp), + }, + }) + .collect(); + + let object = Self { frame_start, entries }; + object.validate(timed)?; + Ok(object) + } + + fn validate(&self, timed: bool) -> Result<()> { + let first = self.entries.first().ok_or(Error::Empty)?; + if matches!(first, Entry::Datagram { .. }) && self.frame_start != 0 { + return Err(Error::Table); + } + let mut prev = None; + for entry in &self.entries { + let sequence = check_id(entry.sequence())?; + if prev.is_some_and(|prev| sequence <= prev) { + return Err(Error::Sequence); + } + prev = Some(sequence); + let frames = match entry { + Entry::Group { frames, .. } if frames.is_empty() => return Err(Error::Empty), + Entry::Group { frames, .. } => frames.as_slice(), + Entry::Datagram { frame, .. } => std::slice::from_ref(frame), + }; + for frame in frames { + match frame.timestamp { + Some(timestamp) if timed => { + check_id(timestamp)?; + } + None if !timed => {} + _ => return Err(Error::Timing), + } + } + } + Ok(()) + } +} + +fn write(buf: &mut impl BufMut, value: u64) -> Result<()> { + varint::encode_quic(value, buf).map_err(|_| Error::Overflow) +} + +fn read(mut buf: &mut (impl Buf + ?Sized)) -> Result { + varint::decode_quic(&mut buf).map_err(|_| Error::Table) +} + +/// A count of entries at least `min` bytes each, refused when the rest of the buffer can't hold it. +fn read_count(buf: &mut impl Buf, min: usize) -> Result { + let count = usize::try_from(read(buf)?).map_err(|_| Error::Overflow)?; + if count > buf.remaining() / min { + return Err(Error::Table); + } + Ok(count) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ID_MAX; + + fn frame(timestamp: Option, payload: &'static [u8]) -> Frame { + Frame { + timestamp, + payload: Bytes::from_static(payload), + } + } + + #[test] + fn timed_keys_sort_in_content_order() { + let names: Vec = [(0, 0, 9), (0, 3, 2), (1, 0, 0), (10, 0, 0)] + .into_iter() + .map(|(group, frame, pts)| { + Start { + group, + frame, + pts: Some(pts), + } + .name() + .unwrap() + }) + .collect(); + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!(names, sorted); + assert_eq!(names[1], "0000000000000000000-0000000000000000003-0000000000000000002"); + assert!( + names + .iter() + .all(|name| name.as_str() < TRACK_END && name.as_str() > TRACK_INFO) + ); + } + + #[test] + fn names_roundtrip_and_refuse_the_other_timing() { + let timed = Start { + group: 5, + frame: 2, + pts: Some(ID_MAX), + }; + let name = timed.name().unwrap(); + assert_eq!(Name::parse(&name, true).unwrap(), Name::Object(timed)); + assert!(matches!(Name::parse(&name, false), Err(Error::Path(_)))); + + let untimed = Start { + group: 7, + frame: 2, + pts: None, + }; + let name = untimed.name().unwrap(); + assert_eq!(name, "0000000000000000007-0000000000000000002"); + assert_eq!(Name::parse(&name, false).unwrap(), Name::Object(untimed)); + assert!(matches!(Name::parse(&name, true), Err(Error::Path(_)))); + + assert_eq!(Name::parse(".info", true).unwrap(), Name::Info); + assert_eq!(Name::parse("end", false).unwrap(), Name::End); + for bad in [ + "7", + "segments", + "0000000000000000007", + "0000000000000000007-x", + "0009007199254740992-0000000000000000000", + ] { + assert!(Name::parse(bad, false).is_err(), "{bad}"); + } + } + + #[test] + fn end_markers_are_json() { + let finish = End::Finish(At { group: 5, frame: 3 }); + assert_eq!( + finish.encode().unwrap().as_ref(), + br#"{"finish":{"group":5,"frame":3}}"# + ); + assert_eq!(End::decode(&finish.encode().unwrap()).unwrap(), finish); + let error = End::Error(0x34); + assert_eq!(error.encode().unwrap().as_ref(), br#"{"error":52}"#); + assert_eq!(End::decode(br#"{"error":52}"#).unwrap(), error); + assert!(End::decode(br#"{"finish":{"group":5}}"#).is_err()); + } + + #[test] + fn groups_and_datagrams_keep_their_form() { + let object = Object { + frame_start: 4, + entries: vec![ + Entry::Group { + sequence: 3, + frames: vec![frame(Some(10), b"a"), frame(Some(9), b"bb")], + }, + Entry::Datagram { + sequence: 4, + frame: frame(Some(11), b"d"), + }, + Entry::Group { + sequence: 7, + frames: vec![frame(Some(12), b"ccc")], + }, + ], + }; + let bytes = object.encode(true).unwrap(); + assert_eq!(Object::decode(bytes.clone(), true).unwrap(), object); + assert!(Object::decode(bytes, false).is_err()); + assert_eq!( + object.start().unwrap(), + Start { + group: 3, + frame: 4, + pts: Some(10) + } + ); + assert_eq!(object.end().unwrap(), Position { group: 7, frame: 1 }); + + let split = Object { + frame_start: 4, + entries: vec![object.entries[0].clone()], + }; + assert_eq!(split.end().unwrap(), Position { group: 3, frame: 6 }); + + let datagram = Object { + frame_start: 0, + entries: vec![object.entries[1].clone()], + }; + assert_eq!(datagram.end().unwrap(), Position::group(5)); + } + + #[test] + fn an_untimed_object_stores_no_timestamps() { + let object = Object { + frame_start: 0, + entries: vec![Entry::Group { + sequence: 2, + frames: vec![frame(None, b"x"), frame(None, b"yz")], + }], + }; + let bytes = object.encode(false).unwrap(); + // version, frame start, one entry: sequence 2, a group of two frames, then the payloads. + assert_eq!(bytes.as_ref(), &[VERSION as u8, 0, 1, 2, 0, 2, 1, 2, b'x', b'y', b'z']); + assert_eq!(Object::decode(bytes, false).unwrap(), object); + assert!(matches!(object.encode(true), Err(Error::Timing))); + } + + #[test] + fn malformed_objects_are_refused() { + let empty = Object::default(); + assert!(matches!(empty.encode(true), Err(Error::Empty))); + + let descending = Object { + frame_start: 0, + entries: vec![ + Entry::Datagram { + sequence: 5, + frame: frame(None, b"a"), + }, + Entry::Datagram { + sequence: 5, + frame: frame(None, b"b"), + }, + ], + }; + assert!(matches!(descending.encode(false), Err(Error::Sequence))); + + let datagram_mid_group = Object { + frame_start: 1, + entries: vec![Entry::Datagram { + sequence: 5, + frame: frame(None, b"a"), + }], + }; + assert!(matches!(datagram_mid_group.encode(false), Err(Error::Table))); + + let good = Object { + frame_start: 0, + entries: vec![Entry::Group { + sequence: 1, + frames: vec![frame(None, b"abc")], + }], + } + .encode(false) + .unwrap(); + // Truncated payload, trailing bytes, an unknown form, and an absurd count. + assert!(Object::decode(&good[..good.len() - 1], false).is_err()); + let mut trailing = good.to_vec(); + trailing.push(0); + assert!(Object::decode(&trailing[..], false).is_err()); + let mut form = good.to_vec(); + form[4] = 2; + assert!(matches!(Object::decode(&form[..], false), Err(Error::Table))); + assert!(matches!( + Object::decode(&[VERSION as u8, 0, 0x7f][..], false), + Err(Error::Table) + )); + assert!(matches!( + Object::decode(&[VERSION as u8 + 1, 0, 0][..], false), + Err(Error::Version(_)) + )); + } +} diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 94ec612132b..2871111019b 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -1,17 +1,59 @@ -//! Versioned hang recording objects on any [`object_store::ObjectStore`]. +//! Broadcast archives on any [`object_store::ObjectStore`]. //! -//! The crate owns the portable layout and codecs: percent-encoded track names, `.info` JSON, -//! the binary segment envelope, and put/get/list/delete. A [`Writer`] records selected tracks of -//! a broadcast into those objects, and a [`Reader`] serves a recording back through a `moq_net` -//! broadcast. Callers that need runtime dispatch -//! supply `Arc`; the archive API itself stays generic. +//! A [`Producer`] stores every track of one broadcast under a key prefix the host picks, and a +//! [`Reader`] serves a recording back through a `moq_net` broadcast. Callers that need runtime +//! dispatch supply `Arc`; the archive API itself stays generic. //! -//! Every object is keyed by its track and a record sequence: -//! ``` -//! let key = moq_archive::Key::segments("video", 7)?; -//! assert_eq!(key.track(), "video"); -//! # Ok::<(), moq_archive::Error>(()) +//! # Layout +//! +//! A [`Producer`] writes, under its prefix: +//! +//! - `/.info`: the track's properties as JSON, `{"version":2,"priority":127,"timescale":1000}`, +//! written before its first object. `timescale` is absent on an untimed track. Properties never +//! change for a track name. The source's max age is not stored: it bounds the live cache, not +//! the archive. +//! - `/--` on a timed track, `/-` on an untimed +//! one: an object, keyed by its first sequence, the index of its first frame within it, and +//! that frame's timestamp in the track's timescale. Each field is 19 zero-padded decimal digits, +//! so a sorted listing is content order and answers group and time lookups; there is no index +//! object. A group may split between frames across objects, which the frame index keeps unique +//! and ordered even where timestamps repeat or go backward. Every object is written once. +//! - `/end`: how the track's last run ended, `{"finish":{"group":5,"frame":0}}` for a +//! clean end at that exclusive position or `{"error":52}` for the stream error code. It sorts +//! after every object key, and DVR pruning never deletes it. +//! - `.end`, empty: the broadcast ended and no track will be added. It is written after every +//! track's `end`; without it the archive is still growing, or its producer stopped early. +//! +//! `` is the track name percent-encoded as one segment: everything but `A-Z a-z 0-9 _ -` +//! is escaped with uppercase hex, so a name holding `/` stays one segment and none starts with +//! `.`. A reader refuses a segment that is not in this canonical form. +//! +//! An object is a table followed by every payload in order, each table field a QUIC varint: +//! +//! ```text +//! Object { +//! Version (i) = 2, +//! Frame Start (i), // the first group's first frame index; 0 when it starts with a datagram +//! Entry Count (i), +//! Entry (..) ..., +//! Payload (..), // every frame's bytes, in table order +//! } +//! +//! Entry { +//! Sequence (i), // absolute for the first entry, then current - previous - 1 +//! Form (i), // 0 for a group, 1 for a datagram +//! [Frame Count (i)], // a group only, at least 1 +//! Frame (..) ..., // a datagram has exactly one +//! } +//! +//! Frame { +//! [Timestamp (i)], // exactly when the track's .info declares a timescale +//! Length (i), +//! } //! ``` +//! +//! A track's groups and datagrams share one sequence space, so each entry keeps its form, and a +//! datagram replays as a datagram, never as a cached one-frame group. #![warn(missing_docs)] @@ -20,27 +62,24 @@ pub use object_store; mod error; /// The `.info` object holding a track's immutable properties. pub mod info; +mod layout; #[cfg(test)] mod mock; mod path; -#[cfg(test)] -#[allow(clippy::disallowed_methods, clippy::disallowed_types)] -mod proof; +pub mod producer; pub mod reader; -mod recover; /// The binary segment object holding a record's groups and frames. pub mod segment; /// Put, get, list, and delete recording objects on an object store. pub mod store; -pub mod writer; pub use error::{Error, Result}; pub use info::Info; pub use path::Key; +pub use producer::Producer; pub use reader::Reader; pub use segment::{Frame, Group, Object}; pub use store::Store; -pub use writer::Writer; /// Recording format version written into `.info` and the binary envelope. pub const VERSION: u64 = 2; diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs new file mode 100644 index 00000000000..0a2218d83a6 --- /dev/null +++ b/rs/moq-archive/src/producer/backends.rs @@ -0,0 +1,278 @@ +//! The same recording on every backend, including the S3 HTTP adapter rather than a listing fake. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::http::{Method, Uri}; +use bytes::Bytes; +use futures::TryStreamExt; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut}; +use object_store::local::LocalFileSystem; +use object_store::memory::InMemory; +use object_store::path::Path; +use s3s::service::S3ServiceBuilder; + +use super::*; + +const VIDEO: &str = "video/1080p"; +const CHAT: &str = "chat.json"; + +/// Real time, so the S3 client's own timers behave: a flush short enough to store the untimed +/// group while the test waits, and long enough that no put outlasts it. +fn config() -> Config { + Config::default() + .with_duration(Duration::from_secs(2)) + .with_flush(Duration::from_millis(200)) + .with_max_delay(Duration::from_secs(10)) +} + +/// A source with `groups` one-second video groups, and the chat's groups as `(sequence, frames)`. +fn source(groups: u64, chat: &[(u64, &[&str])]) -> (broadcast::Producer, [track::Producer; 2]) { + let source = broadcast::Info::new().produce(); + let video = source + .create_track(VIDEO, track::Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + for sequence in 0..groups { + let mut group = video.create_group(group::Info { sequence }).unwrap(); + for frame in 0..2 { + let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); + group.write_frame(pts, format!("v{sequence}.{frame}")).unwrap(); + } + group.finish().unwrap(); + } + let text = source.create_track(CHAT, track::Info::default()).unwrap(); + for (sequence, frames) in chat { + let mut group = text.create_group(group::Info { sequence: *sequence }).unwrap(); + for frame in *frames { + group.write_frame(None, frame.to_string()).unwrap(); + } + group.finish().unwrap(); + } + (source, [video, text]) +} + +fn path(store: &Store, track: &str, name: &str) -> Path { + Path::parse(format!( + "{}/{}/{name}", + store.prefix(), + crate::path::encode_track(track).unwrap() + )) + .unwrap() +} + +/// Wait until every one of `paths` is stored. +async fn stored(store: &Store, paths: &[Path]) { + moq_clock::timeout(Duration::from_secs(10), async { + for path in paths { + while store.inner().head(path).await.is_err() { + moq_clock::sleep(Duration::from_millis(10)).await; + } + } + }) + .await + .expect("the producer stores the objects"); +} + +/// Record in two runs: the first is stopped once it stored a split timed track and the head of +/// an untimed group, and the second resumes both, continuing that group in a new object. +async fn record(store: &Store) { + let (first, _tracks) = source(4, &[(0, &["c0"])]); + let run = tokio::spawn(Producer::new(store.clone(), first.consume(), config()).run()); + stored( + store, + &[ + path( + store, + VIDEO, + "0000000000000000000-0000000000000000000-0000000000000000000", + ), + path( + store, + VIDEO, + "0000000000000000002-0000000000000000000-0000000000000002000", + ), + path(store, CHAT, "0000000000000000000-0000000000000000000"), + ], + ) + .await; + run.abort(); + assert!(run.await.unwrap_err().is_cancelled()); + + let (second, tracks) = source(6, &[(0, &["c0", "c1"]), (1, &["c2"])]); + let run = tokio::spawn(Producer::new(store.clone(), second.consume(), config()).run()); + for track in &tracks { + while track.subscription().is_none() { + moq_clock::sleep(Duration::from_millis(1)).await; + } + track.finish().unwrap(); + } + second.close(); + moq_clock::timeout(Duration::from_secs(10), run) + .await + .expect("the producer ends with its broadcast") + .unwrap() + .unwrap(); +} + +/// Every object under the store's prefix, by path relative to it. +async fn objects(store: &Store) -> BTreeMap { + let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); + let mut objects = BTreeMap::new(); + for meta in metas { + let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); + let key = meta.location.as_ref().strip_prefix("rec/").unwrap().to_string(); + objects.insert(key, bytes); + } + objects +} + +fn decode(objects: &BTreeMap, key: &str, timed: bool) -> Vec<(u64, Vec)> { + let object = crate::layout::Object::decode(objects[key].clone(), timed).unwrap(); + object + .entries + .iter() + .map(|entry| match entry { + crate::layout::Entry::Group { sequence, frames } => ( + *sequence, + frames + .iter() + .map(|frame| String::from_utf8(frame.payload.to_vec()).unwrap()) + .collect(), + ), + crate::layout::Entry::Datagram { .. } => panic!("no datagrams were sent"), + }) + .collect() +} + +#[tokio::test] +async fn recordings_are_byte_identical_on_every_backend() { + let memory = Store::new(InMemory::new(), "rec"); + record(&memory).await; + let expected = objects(&memory).await; + assert_eq!( + expected.keys().map(String::as_str).collect::>(), + [ + ".end", + "chat%2Ejson/.info", + "chat%2Ejson/0000000000000000000-0000000000000000000", + "chat%2Ejson/0000000000000000000-0000000000000000001", + "chat%2Ejson/end", + "video%2F1080p/.info", + "video%2F1080p/0000000000000000000-0000000000000000000-0000000000000000000", + "video%2F1080p/0000000000000000002-0000000000000000000-0000000000000002000", + "video%2F1080p/0000000000000000004-0000000000000000000-0000000000000004000", + "video%2F1080p/end", + ] + ); + assert_eq!( + decode(&expected, "chat%2Ejson/0000000000000000000-0000000000000000000", false), + [(0, vec!["c0".into()])], + ); + assert_eq!( + decode(&expected, "chat%2Ejson/0000000000000000000-0000000000000000001", false), + [(0, vec!["c1".into()]), (1, vec!["c2".into()])], + "the resumed group continues in its own object" + ); + assert_eq!( + decode( + &expected, + "video%2F1080p/0000000000000000004-0000000000000000000-0000000000000004000", + true + ), + [ + (4, vec!["v4.0".into(), "v4.1".into()]), + (5, vec!["v5.0".into(), "v5.1".into()]) + ], + "the resumed timed track continues after its last stored group" + ); + + let dir = tempfile::tempdir().unwrap(); + let local = Store::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap(), "rec"); + record(&local).await; + assert_eq!(objects(&local).await, expected, "local disk"); + + let server = Server::new().await; + record(&server.store).await; + assert_eq!(objects(&server.store).await, expected, "S3"); + let requests = server.requests.lock().unwrap(); + assert!( + requests + .iter() + .any(|request| request.method == Method::PUT && request.conditional), + "timed objects are created conditionally" + ); + assert!( + requests + .iter() + .any(|request| request.uri.path().contains("chat%252Ejson")), + "a percent-encoded key survives URL encoding" + ); +} + +struct Request { + method: Method, + uri: Uri, + conditional: bool, +} + +/// An S3 endpoint over a temporary directory, recording each request it serves. +struct Server { + store: Store, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + _dir: tempfile::TempDir, +} + +impl Server { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("archive")).unwrap(); + let service = S3ServiceBuilder::new(s3s_fs::FileSystem::new(dir.path()).unwrap()).build(); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let app = axum::Router::new().fallback(move |request: axum::extract::Request| { + let service = service.clone(); + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(Request { + method: request.method().clone(), + uri: request.uri().clone(), + conditional: request.headers().get("if-none-match").is_some_and(|value| value == "*"), + }); + service + .call(request.map(s3s::Body::http_body_unsync)) + .await + .unwrap() + .map(Body::new) + } + }); + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let store = AmazonS3Builder::new() + .with_bucket_name("archive") + .with_region("us-east-1") + .with_endpoint(endpoint) + .with_allow_http(true) + .with_skip_signature(true) + .with_conditional_put(S3ConditionalPut::ETagMatch) + .build() + .unwrap(); + Self { + store: Store::new(store, "rec"), + requests, + task, + _dir: dir, + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs new file mode 100644 index 00000000000..ae7551ecc7a --- /dev/null +++ b/rs/moq-archive/src/producer/mod.rs @@ -0,0 +1,243 @@ +//! Record every track of one broadcast into a [`Store`]. +//! +//! A [`Producer`] lists the broadcast's tracks (TRACKS, [`broadcast::Consumer::tracks`]) and +//! records each through its own ascending SUBSCRIBE, so it holds what the publisher's cache +//! stops serving: a group still missing once the subscription's max delay passes is a gap, never +//! a stall. Datagrams are never cached, so a track's are recorded from when its subscription +//! takes effect. The host picks the key prefix, per broadcast path and epoch say, and runs one +//! producer per broadcast; the producer stores no epoch. See the [crate docs](crate#layout) for +//! what it writes. +//! +//! A producer started under a prefix that already holds objects resumes each track at the frame +//! after its last stored one, and one under a prefix holding `.end` records nothing. +//! +//! ```no_run +//! # async fn example(broadcast: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { +//! use moq_archive::object_store::memory::InMemory; +//! +//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +//! moq_archive::Producer::new(store, broadcast, Default::default()).run().await?; +//! # Ok(()) +//! # } +//! ``` + +mod ordered; +mod resume; +mod track; + +#[cfg(test)] +mod backends; +#[cfg(test)] +mod tests; + +use std::collections::HashMap; +use std::sync::Arc; +use std::task::Poll; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{FutureExt, StreamExt}; +use moq_net::broadcast::TracksEvent; +use moq_net::{StreamError, broadcast}; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; + +use crate::layout::{self, End}; +use crate::path::push; +use crate::{Error, Result, Store}; + +/// How a [`Producer`] cuts, flushes, and prunes its objects. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Config { + /// How much content an object holds before it is cut at the next group boundary: media time + /// on a timed track, wall-clock time on an untimed one. A group whose own run in an object + /// reaches it splits between frames. + pub duration: Duration, + /// The longest a frame waits in memory before its object is stored, however idle its track. + /// The object is cut there, mid-group if need be. Keep it above [`Self::duration`], or + /// objects are cut by it instead. + pub flush: Duration, + /// The longest a missing group is waited for, capping the publisher's max age. A group still + /// missing then is a gap, and the frames behind it wait for that verdict. A track with nothing + /// stored waits this long from its first arrival for a lower sequence, unless that one is 0. + pub max_delay: Duration, + /// Prune old objects (a DVR), or keep everything when `None` (an archive). + pub dvr: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + duration: Duration::from_secs(2), + flush: Duration::from_secs(4), + max_delay: Duration::from_secs(10), + dvr: None, + } + } +} + +impl Config { + /// Set [`duration`](Self::duration). + pub fn with_duration(mut self, duration: Duration) -> Self { + self.duration = duration; + self + } + + /// Set [`flush`](Self::flush). + pub fn with_flush(mut self, flush: Duration) -> Self { + self.flush = flush; + self + } + + /// Set [`max_delay`](Self::max_delay). + pub fn with_max_delay(mut self, max_delay: Duration) -> Self { + self.max_delay = max_delay; + self + } + + /// Set [`dvr`](Self::dvr). + pub fn with_dvr(mut self, dvr: impl Into>) -> Self { + self.dvr = dvr.into(); + self + } +} + +/// A DVR's pruning, for hosts without bucket lifecycle rules. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Dvr { + /// How long an object stays after a newer one of its track supersedes it. + pub window: Duration, + /// How much longer it stays, so readers that already listed it finish their GETs. + pub grace: Duration, +} + +impl Dvr { + /// Delete each superseded object `window + grace` after it was superseded. + pub fn new(window: Duration, grace: Duration) -> Self { + Self { window, grace } + } +} + +/// Records every track of one broadcast until the broadcast ends. +pub struct Producer { + shared: Arc>, +} + +struct Shared { + store: Store, + broadcast: broadcast::Consumer, + config: Config, +} + +/// What one track's recorder reports once the broadcast's listing ended and it stored everything. +type Recorded = Result<(String, Option)>; + +impl Producer { + /// Record `broadcast` under `store`'s prefix. + pub fn new(store: Store, broadcast: broadcast::Consumer, config: Config) -> Self { + Self { + shared: Arc::new(Shared { + store, + broadcast, + config, + }), + } + } + + /// Record until the broadcast ends, then write each track's `end` and the broadcast's `.end`. + /// + /// Returns at once, recording nothing, when the prefix already holds `.end`, or when TRACKS + /// is refused with not found, since the path names no broadcast. Any other refusal, a store + /// failure, or a track whose properties differ from the stored ones fails. Dropping the + /// future stops recording without markers, so the archive reads as still growing. + pub async fn run(self) -> Result<()> { + let shared = self.shared; + let Some(mut stored) = resume::load(&shared.store).await? else { + return Ok(()); + }; + + let mut listing = Some(shared.broadcast.tracks().map_err(source)?); + let mut queues: HashMap> = HashMap::new(); + let mut recorders: FuturesUnordered> = FuturesUnordered::new(); + let mut ends: HashMap> = HashMap::new(); + + enum Event { + Listed(moq_net::Result>), + Recorded(Recorded), + Done, + } + + loop { + let event = kio::wait(|waiter| { + if let Poll::Ready(Some(recorded)) = recorders.poll_next_unpin(&mut waiter.context()) { + return Poll::Ready(Event::Recorded(recorded)); + } + match listing.as_mut() { + Some(listing) => listing.poll_next(waiter).map(Event::Listed), + None if recorders.is_empty() => Poll::Ready(Event::Done), + None => Poll::Pending, + } + }) + .await; + + match event { + Event::Listed(Ok(Some(TracksEvent::Added { name, info }))) => { + let queue = queues.entry(name.to_string()).or_insert_with(|| { + let queue = kio::Queue::new(); + let record = track::record( + shared.clone(), + name.to_string(), + queue.clone(), + stored.remove(name.as_ref()), + ); + recorders.push(record.boxed()); + queue + }); + // Only the newest info matters, so one still queued is replaced: a blocked + // recorder holds at most one, however often its track is re-added. + let _ = queue.try_pop(); + // Only fails once closed, which happens when the listing ends. + let _ = queue.try_push(info); + } + Event::Listed(Ok(Some(TracksEvent::Synced | TracksEvent::Removed { .. }))) => {} + Event::Listed(Ok(None)) => { + listing = None; + for queue in queues.values() { + queue.close(); + } + } + Event::Listed(Err(err)) if queues.is_empty() && StreamError::from(&err) == StreamError::NotFound => { + tracing::debug!(broadcast = %shared.broadcast.info().path, "not a broadcast"); + return Ok(()); + } + Event::Listed(Err(err)) => return Err(source(err)), + Event::Recorded(recorded) => { + let (name, end) = recorded?; + ends.insert(name, end); + } + Event::Done => break, + } + } + + let store = &shared.store; + for (name, end) in &ends { + if let Some(end) = end { + let path = push(&layout::track_dir(store.prefix(), name)?, layout::TRACK_END)?; + store.inner().put(&path, PutPayload::from(end.encode()?)).await?; + } + } + // A stored track that ended unseen, or was never listed again, may still grow. + let complete = ends.values().all(Option::is_some) && stored.values().all(|track| track.end.is_some()); + if complete { + let path = push(store.prefix(), layout::BROADCAST_END)?; + store.inner().put(&path, PutPayload::new()).await?; + } + Ok(()) + } +} + +fn source(err: moq_net::Error) -> Error { + Error::Source(err.to_string()) +} diff --git a/rs/moq-archive/src/producer/ordered.rs b/rs/moq-archive/src/producer/ordered.rs new file mode 100644 index 00000000000..7479b78e551 --- /dev/null +++ b/rs/moq-archive/src/producer/ordered.rs @@ -0,0 +1,246 @@ +//! A track's groups and datagrams in sequence order, waiting a bounded time for a late one. +//! +//! `moq_net::track::Ordered` seeks past a missing group at once and orders groups alone, so it +//! fits a player, not an archive that must keep every group and datagram it can in key order. + +use std::collections::{BTreeMap, VecDeque}; +use std::pin::Pin; +use std::task::Poll; +use std::time::Duration; + +use moq_clock::Instant; +use moq_net::track::{Position, Subscriber}; +use moq_net::{Datagram, group}; + +/// One sequence, in the form it was forwarded in. +pub(super) enum Item { + // Boxed, since a group consumer dwarfs a datagram and waits in `Ordered::pending`. + Group(Box), + Datagram(Datagram), +} + +/// Hands out each sequence once every lower one was handed out or given up: at once when it is +/// the next, otherwise once `max_delay` passes from the oldest arrival still waiting, which is +/// when the gap below it was first seen. While no start is known, anything but 0 waits the same +/// way, since arrival is not sequence order. Once both channels end, everything received is +/// handed out without waiting. +pub(super) struct Ordered { + subscriber: Subscriber, + max_delay: Duration, + /// The next sequence to hand out; `None` until a start is known or the first wait ends. + next: Option, + /// Received sequences waiting on a lower one. + pending: BTreeMap, + /// Each pending sequence with when it arrived, oldest first. The front is always pending, + /// and the ones handed out behind it go once it does, so this holds at most what arrived + /// within a max delay. The oldest dates the gap, so a later, lower arrival never extends it. + arrivals: VecDeque<(u64, Instant)>, + timer: Option>>, + groups_done: bool, + datagrams_done: bool, +} + +impl Ordered { + /// Order `subscriber` from `next`, where a resume continues; otherwise from the subscription's + /// floor when it names one. + pub(super) fn new(subscriber: Subscriber, max_delay: Duration, next: Option) -> Self { + let next = next.or_else(|| subscriber.floor()); + Self { + subscriber, + max_delay, + next, + pending: BTreeMap::new(), + arrivals: VecDeque::new(), + timer: None, + groups_done: false, + datagrams_done: false, + } + } + + /// Take every arrival so far, registering `waiter` for the next. + pub(super) fn poll_arrivals(&mut self, waiter: &kio::Waiter) { + let now = moq_clock::now(); + while !self.groups_done { + match self.subscriber.poll_recv_group(waiter) { + Poll::Ready(Ok(Some(group))) => self.arrive(group.sequence, Item::Group(Box::new(group)), now), + // How the track ended is read from `poll_finished`. + Poll::Ready(_) => self.groups_done = true, + Poll::Pending => break, + } + } + while !self.datagrams_done { + match self.subscriber.poll_recv_datagram(waiter) { + Poll::Ready(Ok(Some(datagram))) => self.arrive(datagram.sequence, Item::Datagram(datagram), now), + Poll::Ready(_) => self.datagrams_done = true, + Poll::Pending => break, + } + } + } + + fn arrive(&mut self, sequence: u64, item: Item, now: Instant) { + // A sequence names one group, so a copy of one already here or handed out is dropped. + if self.next.is_none_or(|next| sequence >= next) + && let std::collections::btree_map::Entry::Vacant(entry) = self.pending.entry(sequence) + { + entry.insert(item); + self.arrivals.push_back((sequence, now)); + } + } + + /// The next sequence in order, or `None` once both channels ended and nothing is left. + pub(super) fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + self.poll_arrivals(waiter); + let draining = self.groups_done && self.datagrams_done; + loop { + let Some(entry) = self.pending.first_entry() else { + return match draining { + true => Poll::Ready(None), + false => Poll::Pending, + }; + }; + let sequence = *entry.key(); + let gap = match self.next { + Some(next) if sequence < next => { + entry.remove(); + self.trim(); + continue; + } + Some(next) => sequence > next, + // Nothing is below 0. + None => sequence > 0, + }; + if gap && !draining { + // The oldest arrival still pending is when this gap was first seen. + let (_, since) = self.arrivals.front().expect("a pending sequence arrived"); + let deadline = *since + self.max_delay; + if moq_clock::now() < deadline { + let timer = self + .timer + .get_or_insert_with(|| Box::pin(moq_clock::sleep_until(deadline))); + timer.as_mut().reset(deadline); + if waiter.poll_future(timer.as_mut()).is_pending() { + return Poll::Pending; + } + } + } + let item = self.pending.remove(&sequence).expect("the lowest pending sequence"); + self.trim(); + self.next = Some(sequence.saturating_add(1)); + return Poll::Ready(Some((sequence, item))); + } + } + + /// Drop arrivals handed out, up to the oldest one still pending. + fn trim(&mut self) { + while self + .arrivals + .front() + .is_some_and(|(sequence, _)| !self.pending.contains_key(sequence)) + { + self.arrivals.pop_front(); + } + } + + /// How the track ended, once it has. + pub(super) fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { + self.subscriber.poll_finished(waiter) + } +} + +#[cfg(test)] +mod tests { + use moq_net::track::{Order, Range, Subscription}; + use moq_net::{broadcast, track}; + + use super::*; + + const MAX_DELAY: Duration = Duration::from_secs(10); + + fn group(track: &track::Producer, sequence: u64) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, "x").unwrap(); + group.finish().unwrap(); + } + + fn next(ordered: &mut Ordered) -> Option { + match ordered.poll_next(&kio::Waiter::noop()) { + Poll::Ready(Some((sequence, _))) => Some(sequence), + _ => None, + } + } + + async fn ordered(broadcast: &broadcast::Producer, next: Option) -> Ordered { + let subscription = Subscription::default() + .with_order(Order::Ascending) + .with_max_delay(MAX_DELAY) + .with_ranges([Range::oldest()]); + let subscriber = broadcast + .consume() + .track("t") + .unwrap() + .subscribe(subscription) + .await + .unwrap(); + Ordered::new(subscriber, MAX_DELAY, next) + } + + /// The next sequence is handed out at once, a datagram in turn with the groups, and one past a + /// missing sequence only once the max delay from its arrival passes. + #[tokio::test(start_paused = true)] + async fn only_a_missing_sequence_waits_the_max_delay() { + let broadcast = broadcast::Info::new().produce(); + let mut track = broadcast.create_track("t", track::Info::default()).unwrap(); + let mut ordered = ordered(&broadcast, None).await; + + group(&track, 0); + assert_eq!(next(&mut ordered), Some(0)); + + group(&track, 2); + assert_eq!(next(&mut ordered), None); + track.insert_datagram(1, None, "d").unwrap(); + assert_eq!(next(&mut ordered), Some(1)); + assert_eq!(next(&mut ordered), Some(2)); + + group(&track, 4); + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(MAX_DELAY - Duration::from_millis(1)).await; + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(Duration::from_millis(1)).await; + assert_eq!(next(&mut ordered), Some(4)); + + // Too late for its turn. + group(&track, 3); + track.finish().unwrap(); + assert!(matches!(ordered.poll_next(&kio::Waiter::noop()), Poll::Ready(None))); + } + + /// A gap's wait runs from when it was first seen, so ever lower arrivals above it can't extend it. + #[tokio::test(start_paused = true)] + async fn lower_arrivals_never_extend_a_gap() { + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("t", track::Info::default()).unwrap(); + let mut ordered = ordered(&broadcast, Some(0)).await; + + group(&track, 10); + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(MAX_DELAY - Duration::from_millis(1)).await; + group(&track, 9); + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(Duration::from_millis(1)).await; + assert_eq!(next(&mut ordered), Some(9)); + assert_eq!(next(&mut ordered), Some(10)); + } + + /// A gap-free track keeps no arrival it handed out. + #[tokio::test(start_paused = true)] + async fn handed_out_arrivals_are_dropped() { + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("t", track::Info::default()).unwrap(); + let mut ordered = ordered(&broadcast, None).await; + for sequence in 0..100 { + group(&track, sequence); + assert_eq!(next(&mut ordered), Some(sequence)); + } + assert!(ordered.arrivals.is_empty()); + } +} diff --git a/rs/moq-archive/src/producer/resume.rs b/rs/moq-archive/src/producer/resume.rs new file mode 100644 index 00000000000..4d8ca41958b --- /dev/null +++ b/rs/moq-archive/src/producer/resume.rs @@ -0,0 +1,90 @@ +//! What a prefix already holds, so a restarted producer continues each track after its last +//! stored frame. + +use std::collections::HashMap; + +use futures::TryStreamExt; +use moq_net::track::Position; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::layout::{self, End, Name, Object, Start}; +use crate::{Error, Info, Result, Store}; + +/// One track as stored. +pub(super) struct Stored { + pub info: Info, + /// Every object's key, oldest first. + pub objects: Vec, + /// Where the track continues: just past its newest object's last frame. + pub next: Option, + /// How its last run ended, when the producer that stored it saw that. + pub end: Option, +} + +/// Every stored track by name, or `None` when the broadcast's `.end` says it is complete. +pub(super) async fn load(store: &Store) -> Result>> { + let prefix = store.prefix(); + let scope = (!prefix.as_ref().is_empty()).then_some(prefix); + let listed = store.inner().list_with_delimiter(scope).await?; + + if let Some(object) = listed.objects.first() { + match object.location.filename() { + Some(layout::BROADCAST_END) => return Ok(None), + _ => return Err(Error::Path(object.location.to_string())), + } + } + + let mut tracks = HashMap::new(); + for dir in listed.common_prefixes { + let segment = dir.filename().ok_or_else(|| Error::Path(dir.to_string()))?; + let name = layout::parse_track(segment)?; + let track = load_track(store, &dir).await?; + tracks.insert(name, track); + } + Ok(Some(tracks)) +} + +async fn load_track(store: &Store, dir: &Path) -> Result { + let entries: Vec<_> = store.inner().list(Some(dir)).try_collect().await?; + let info_path = crate::path::push(dir, layout::TRACK_INFO)?; + let info = Info::decode(&store.inner().get(&info_path).await?.bytes().await?)?; + let timed = info.timescale.is_some(); + + let mut objects: Vec<(Start, Path)> = Vec::new(); + let mut end = None; + for entry in entries { + // A flat listing only returns objects directly under the track; anything deeper is foreign. + if entry.location.prefix_match(dir).is_none_or(|parts| parts.count() != 1) { + return Err(Error::Path(entry.location.to_string())); + } + let filename = entry + .location + .filename() + .ok_or_else(|| Error::Path(entry.location.to_string()))?; + match Name::parse(filename, timed)? { + Name::Info => {} + Name::End => end = Some(End::decode(&store.inner().get(&entry.location).await?.bytes().await?)?), + Name::Object(start) => objects.push((start, entry.location)), + } + } + objects.sort(); + + let next = match objects.last() { + Some((start, path)) => { + let object = Object::decode(store.inner().get(path).await?.bytes().await?, timed)?; + if object.start()? != *start { + return Err(Error::Path(path.to_string())); + } + Some(object.end()?) + } + None => None, + }; + + Ok(Stored { + info, + objects: objects.into_iter().map(|(_, path)| path).collect(), + next, + end, + }) +} diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs new file mode 100644 index 00000000000..ad06608f94e --- /dev/null +++ b/rs/moq-archive/src/producer/tests.rs @@ -0,0 +1,949 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use futures::TryStreamExt; +use moq_clock::sleep; +use moq_net::track::Position; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStoreExt; +use tokio::task::JoinHandle; + +use super::*; +use crate::Info; +use crate::layout::{At, Entry, Object}; +use crate::mock::Mock; + +const SECOND: Duration = Duration::from_secs(1); + +fn store() -> Store { + Store::new(Mock::memory(), "rec") +} + +/// No flush or gap verdict lands unless a test waits for it. +fn config() -> Config { + Config::default() + .with_duration(2 * SECOND) + .with_flush(100 * SECOND) + .with_max_delay(10 * SECOND) +} + +fn spawn(store: &Store, broadcast: &broadcast::Producer, config: Config) -> JoinHandle> { + tokio::spawn(Producer::new(store.clone(), broadcast.consume(), config).run()) +} + +fn timed() -> track::Info { + track::Info::default().with_timescale(Timescale::MILLI) +} + +fn ms(ms: u64) -> Timestamp { + Timestamp::from_millis(ms).unwrap() +} + +/// Write a whole group of `(pts, payload)` frames. +fn group(track: &track::Producer, sequence: u64, frames: &[(u64, &str)]) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for (pts, payload) in frames { + group.write_frame(ms(*pts), payload.to_string()).unwrap(); + } + group.finish().unwrap(); +} + +/// Let the producer catch up without moving time far. +async fn settle() { + for _ in 0..10 { + sleep(Duration::from_millis(1)).await; + } +} + +/// Wait for the producer to subscribe to `track`. +async fn subscribed(track: &track::Producer) { + while track.subscription().is_none() { + sleep(Duration::from_millis(1)).await; + } +} + +async fn finished(task: JoinHandle>) { + moq_clock::timeout(Duration::from_secs(1000), task) + .await + .expect("the producer ends with its broadcast") + .unwrap() + .expect("the producer succeeds"); +} + +/// Every stored key under the prefix, relative to it. +async fn keys(store: &Store) -> Vec { + let mut keys: Vec = store + .inner() + .list(Some(store.prefix())) + .map_ok(|meta| meta.location.as_ref().strip_prefix("rec/").unwrap().to_string()) + .try_collect() + .await + .unwrap(); + keys.sort(); + keys +} + +async fn get(store: &Store, key: &str) -> Option { + let path = object_store::path::Path::parse(format!("rec/{key}")).unwrap(); + match store.inner().get(&path).await { + Ok(got) => Some(got.bytes().await.unwrap()), + Err(object_store::Error::NotFound { .. }) => None, + Err(err) => panic!("{err}"), + } +} + +/// A track's objects by key name, in key order. +async fn objects(store: &Store, track: &str, timed: bool) -> BTreeMap { + let dir = format!("{}/", crate::path::encode_track(track).unwrap()); + let mut objects = BTreeMap::new(); + for key in keys(store).await { + let Some(name) = key.strip_prefix(&dir) else { + continue; + }; + if name == layout::TRACK_INFO || name == layout::TRACK_END { + continue; + } + let object = Object::decode(get(store, &key).await.unwrap(), timed).unwrap(); + objects.insert(name.to_string(), object); + } + objects +} + +/// Each object's entries as `(sequence, form, frames)`, where a frame is `pts/payload`. +fn contents(objects: &BTreeMap) -> Vec)>> { + objects + .values() + .map(|object| { + object + .entries + .iter() + .map(|entry| { + let (form, frames) = match entry { + Entry::Group { frames, .. } => ("group", frames.as_slice()), + Entry::Datagram { frame, .. } => ("datagram", std::slice::from_ref(frame)), + }; + let frames = frames + .iter() + .map(|frame| { + let payload = String::from_utf8(frame.payload.to_vec()).unwrap(); + match frame.timestamp { + Some(pts) => format!("{pts}/{payload}"), + None => payload, + } + }) + .collect(); + (entry.sequence(), form, frames) + }) + .collect() + }) + .collect() +} + +fn key(group: u64, frame: u64, pts: u64) -> String { + format!("{group:019}-{frame:019}-{pts:019}") +} + +fn untimed(group: u64, frame: u64) -> String { + format!("{group:019}-{frame:019}") +} + +async fn end(store: &Store, track: &str) -> Option { + let key = format!("{}/end", crate::path::encode_track(track).unwrap()); + get(store, &key).await.map(|bytes| End::decode(&bytes).unwrap()) +} + +async fn complete(store: &Store) -> bool { + get(store, layout::BROADCAST_END).await.is_some() +} + +#[tokio::test(start_paused = true)] +async fn timed_objects_cut_at_group_boundaries_by_duration() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + for sequence in 0..5 { + let pts = sequence * 1000; + group(&video, sequence, &[(pts, "a"), (pts + 500, "b")]); + } + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let objects = objects(&store, "video", true).await; + assert_eq!( + objects.keys().cloned().collect::>(), + [key(0, 0, 0), key(2, 0, 2000), key(4, 0, 4000)] + ); + assert_eq!( + contents(&objects)[0], + [ + (0, "group", vec!["0/a".to_string(), "500/b".to_string()]), + (1, "group", vec!["1000/a".to_string(), "1500/b".to_string()]), + ] + ); + let info = Info::decode(&get(&store, "video/.info").await.unwrap()).unwrap(); + assert_eq!(info.timescale, Some(1000)); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 5, frame: 0 }))); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let config = config().with_flush(SECOND).with_max_delay(2 * SECOND); + let task = spawn(&store, &broadcast, config); + subscribed(&video).await; + + group(&video, 0, &[(0, "a")]); + group(&video, 2, &[(2000, "c")]); + group(&video, 3, &[(3000, "d")]); + settle().await; + // Group 0 is flushed alone; 2 and 3 wait on the missing 1. + sleep(SECOND + SECOND / 2).await; + assert_eq!( + objects(&store, "video", true).await.keys().cloned().collect::>(), + [key(0, 0, 0)] + ); + + // Once the max delay passes, the gap is given up and the rest follow within the flush. + sleep(2 * SECOND).await; + let stored = objects(&store, "video", true).await; + assert_eq!( + stored.keys().cloned().collect::>(), + [key(0, 0, 0), key(2, 0, 2000)] + ); + assert_eq!( + contents(&stored)[1], + [ + (2, "group", vec!["2000/c".to_string()]), + (3, "group", vec!["3000/d".to_string()]), + ] + ); + + // The lost group arriving too late stays a gap. + group(&video, 1, &[(1000, "b")]); + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + let stored = objects(&store, "video", true).await; + assert_eq!(stored.len(), 2); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 4, frame: 0 }))); +} + +/// Arrival is not sequence order, so a fresh track's first arrival waits for a lower sequence. +#[tokio::test(start_paused = true)] +async fn a_fresh_track_waits_for_a_group_below_its_first_arrival() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let config = config().with_flush(SECOND).with_max_delay(2 * SECOND); + let task = spawn(&store, &broadcast, config); + subscribed(&video).await; + + group(&video, 2, &[(2000, "c")]); + settle().await; + group(&video, 1, &[(1000, "b")]); + settle().await; + // Nothing is stored until the wait from the first arrival gives up on group 0. + sleep(SECOND + SECOND / 2).await; + assert!(objects(&store, "video", true).await.is_empty()); + + sleep(2 * SECOND).await; + let stored = objects(&store, "video", true).await; + assert_eq!(stored.keys().cloned().collect::>(), [key(1, 0, 1000)]); + assert_eq!( + contents(&stored)[0], + [ + (1, "group", vec!["1000/b".to_string()]), + (2, "group", vec!["2000/c".to_string()]), + ] + ); + + // Group 0 arriving after that is too late. + group(&video, 0, &[(0, "a")]); + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(objects(&store, "video", true).await.len(), 1); +} + +#[tokio::test(start_paused = true)] +async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + // Timestamps repeat and go backward, as decode order allows. + let frames = [ + (0, "0"), + (1000, "1"), + (2000, "2"), + (2000, "3"), + (1500, "4"), + (4000, "5"), + (4000, "6"), + ]; + group(&video, 0, &frames); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let objects = objects(&store, "video", true).await; + assert_eq!( + objects.keys().cloned().collect::>(), + [key(0, 0, 0), key(0, 2, 2000), key(0, 5, 4000)] + ); + let stored: Vec = contents(&objects) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + let expected: Vec = frames.iter().map(|(pts, payload)| format!("{pts}/{payload}")).collect(); + assert_eq!(stored, expected); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 1, frame: 0 }))); +} + +/// An open untimed group is cut at each flush, so it never holds more than a flush of frames in +/// memory, and a stored object is never rewritten. +#[tokio::test(start_paused = true)] +async fn an_open_untimed_group_is_cut_at_each_flush() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let data = broadcast.create_track("data", track::Info::default()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&data).await; + + let mut open = data.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(None, "a").unwrap(); + settle().await; + sleep(SECOND).await; + let head = untimed(0, 0); + let first = get(&store, &format!("data/{head}")).await.unwrap(); + + open.write_frame(None, "b").unwrap(); + settle().await; + sleep(SECOND).await; + assert_eq!(get(&store, &format!("data/{head}")).await.unwrap(), first); + let stored = objects(&store, "data", false).await; + assert_eq!(stored.keys().cloned().collect::>(), [head, untimed(0, 1)]); + assert_eq!( + contents(&stored), + [ + [(0, "group", vec!["a".to_string()])], + [(0, "group", vec!["b".to_string()])] + ] + ); + + open.finish().unwrap(); + data.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(end(&store, "data").await, Some(End::Finish(At { group: 1, frame: 0 }))); + let info = Info::decode(&get(&store, "data/.info").await.unwrap()).unwrap(); + assert_eq!(info.timescale, None); +} + +/// Without a flush due, an untimed group splits once its own run in an object lasts the duration +/// of wall clock, and the next group starts the next object past the duration. +#[tokio::test(start_paused = true)] +async fn an_untimed_group_splits_by_wall_clock() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let data = broadcast.create_track("data", track::Info::default()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&data).await; + + let mut open = data.create_group(group::Info { sequence: 0 }).unwrap(); + for payload in ["a", "b", "c", "d", "e"] { + open.write_frame(None, payload).unwrap(); + settle().await; + sleep(SECOND / 2).await; + } + open.finish().unwrap(); + settle().await; + sleep(2 * SECOND).await; + let mut next = data.create_group(group::Info { sequence: 1 }).unwrap(); + next.write_frame(None, "f").unwrap(); + next.finish().unwrap(); + settle().await; + data.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "data", false).await; + assert_eq!( + stored.keys().cloned().collect::>(), + [untimed(0, 0), untimed(0, 4), untimed(1, 0)] + ); + let strings = |frames: &[&str]| frames.iter().map(|frame| frame.to_string()).collect::>(); + assert_eq!( + contents(&stored), + [ + vec![(0, "group", strings(&["a", "b", "c", "d"]))], + vec![(0, "group", strings(&["e"]))], + vec![(1, "group", strings(&["f"]))], + ] + ); +} + +#[tokio::test(start_paused = true)] +async fn an_idle_track_is_flushed_within_the_bound() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + + let mut open = video.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ms(0), "a").unwrap(); + settle().await; + sleep(SECOND / 2).await; + assert!(objects(&store, "video", true).await.is_empty()); + sleep(SECOND / 2).await; + assert_eq!( + objects(&store, "video", true).await.keys().cloned().collect::>(), + [key(0, 0, 0)] + ); + + // The open group continues in the next object. + open.write_frame(ms(100), "b").unwrap(); + open.finish().unwrap(); + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!( + objects(&store, "video", true).await.keys().cloned().collect::>(), + [key(0, 0, 0), key(0, 1, 100)] + ); +} + +#[tokio::test(start_paused = true)] +async fn a_restart_mid_group_loses_and_duplicates_nothing() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let data = broadcast.create_track("data", track::Info::default()).unwrap(); + let config = config().with_flush(SECOND); + let task = spawn(&store, &broadcast, config.clone()); + subscribed(&video).await; + subscribed(&data).await; + + let mut timed_group = video.create_group(group::Info { sequence: 0 }).unwrap(); + let mut untimed_group = data.create_group(group::Info { sequence: 0 }).unwrap(); + for (pts, payload) in [(0, "a"), (100, "b")] { + timed_group.write_frame(ms(pts), payload).unwrap(); + untimed_group.write_frame(None, payload).unwrap(); + } + settle().await; + sleep(SECOND).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + + // Both groups grow while nothing records them. + timed_group.write_frame(ms(200), "c").unwrap(); + untimed_group.write_frame(None, "c").unwrap(); + let task = spawn(&store, &broadcast, config); + settle().await; + timed_group.write_frame(ms(300), "d").unwrap(); + untimed_group.write_frame(None, "d").unwrap(); + timed_group.finish().unwrap(); + untimed_group.finish().unwrap(); + settle().await; + video.finish().unwrap(); + data.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "video", true).await; + assert_eq!( + stored.keys().cloned().collect::>(), + [key(0, 0, 0), key(0, 2, 200)] + ); + let frames: Vec = contents(&stored) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "100/b", "200/c", "300/d"]); + + // The untimed group continues in its own object too. + let stored = objects(&store, "data", false).await; + assert_eq!( + stored.keys().cloned().collect::>(), + [untimed(0, 0), untimed(0, 2)] + ); + let frames: Vec = contents(&stored) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["a", "b", "c", "d"]); + assert!(complete(&store).await); +} + +/// A later group that starts past frame 0, its head held elsewhere, opens its own object: an +/// entry after the first has no frame start, so a resume would otherwise rewind into it. +#[tokio::test(start_paused = true)] +async fn a_partial_group_resumes_after_its_last_stored_frame() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let config = config().with_flush(SECOND); + let task = spawn(&store, &broadcast, config.clone()); + subscribed(&video).await; + + group(&video, 0, &[(0, "a")]); + let mut partial = video.create_group(group::Info { sequence: 1 }).unwrap(); + partial.start_at(2).unwrap(); + partial.write_frame(ms(500), "c").unwrap(); + settle().await; + sleep(SECOND).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + + let task = spawn(&store, &broadcast, config); + settle().await; + partial.write_frame(ms(600), "d").unwrap(); + partial.finish().unwrap(); + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "video", true).await; + assert_eq!( + stored.keys().cloned().collect::>(), + [key(0, 0, 0), key(1, 2, 500), key(1, 3, 600)] + ); + let frames: Vec = contents(&stored) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "500/c", "600/d"]); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 2, frame: 0 }))); +} + +#[tokio::test(start_paused = true)] +async fn changed_properties_fail_the_resume() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + group(&video, 0, &[(0, "a")]); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + video.finish().unwrap(); + drop(video); + broadcast.close(); + finished(task).await; + + // Another instance, recorded under the same prefix, which only a missing `.end` allows. + let store = Store::new(store.inner().fork(), "rec"); + store + .inner() + .delete(&object_store::path::Path::from("rec/.end")) + .await + .unwrap(); + let broadcast = broadcast::Info::new().produce(); + let _video = broadcast + .create_track("video", track::Info::default().with_timescale(Timescale::MICRO)) + .unwrap(); + let err = spawn(&store, &broadcast, config()).await.unwrap().unwrap_err(); + assert!( + matches!( + err, + Error::TimescaleMismatch { + existing: Some(1000), + intended: Some(1_000_000) + } + ), + "{err}" + ); +} + +#[tokio::test(start_paused = true)] +async fn track_names_stay_one_segment_through_a_resume() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let names = ["a/b", ".hidden", "end"]; + let tracks: Vec<_> = names + .iter() + .map(|name| broadcast.create_track(*name, track::Info::default()).unwrap()) + .collect(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + for track in &tracks { + subscribed(track).await; + let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(None, "x").unwrap(); + group.finish().unwrap(); + } + settle().await; + sleep(SECOND).await; + task.abort(); + let _ = task.await; + + let stored = keys(&store).await; + for dir in ["a%2Fb", "%2Ehidden", "end"] { + assert!(stored.contains(&format!("{dir}/{}", untimed(0, 0))), "{stored:?}"); + assert!(stored.contains(&format!("{dir}/.info")), "{stored:?}"); + } + + // The resume parses each one back to its name and continues after its stored group. + let task = spawn(&store, &broadcast, config()); + for track in &tracks { + subscribed(track).await; + track.finish().unwrap(); + } + broadcast.close(); + finished(task).await; + for name in names { + assert_eq!(end(&store, name).await, Some(End::Finish(At { group: 1, frame: 0 }))); + assert_eq!(objects(&store, name, false).await.len(), 1); + } +} + +#[tokio::test(start_paused = true)] +async fn a_dvr_prunes_superseded_objects_after_its_window_and_grace() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let dvr = Dvr::new(2 * SECOND, SECOND); + let task = spawn(&store, &broadcast, config().with_duration(SECOND).with_dvr(dvr)); + subscribed(&video).await; + + // Each group is its own object, closed when the next group starts. + for sequence in 0..4 { + group(&video, sequence, &[(sequence * 1000, "x")]); + settle().await; + sleep(SECOND).await; + } + // Object 0 was superseded when object 1 was stored, at 2s: it stays through 5s. + let names = + |objects: BTreeMap| objects.values().map(|o| o.start().unwrap().group).collect::>(); + assert_eq!(names(objects(&store, "video", true).await), [0, 1, 2]); + sleep(SECOND).await; + settle().await; + assert_eq!(names(objects(&store, "video", true).await), [1, 2]); + sleep(3 * SECOND).await; + settle().await; + // The newest stored object is always kept. + assert_eq!(names(objects(&store, "video", true).await), [2]); + + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(names(objects(&store, "video", true).await), [2, 3]); + assert!(end(&store, "video").await.is_some()); +} + +#[tokio::test(start_paused = true)] +async fn a_track_added_mid_recording_is_recorded() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + group(&video, 0, &[(0, "v")]); + settle().await; + + let audio = broadcast.create_track("audio", timed()).unwrap(); + subscribed(&audio).await; + group(&audio, 0, &[(0, "a")]); + settle().await; + video.finish().unwrap(); + audio.finish().unwrap(); + broadcast.close(); + finished(task).await; + + assert_eq!(objects(&store, "audio", true).await.len(), 1); + assert_eq!(objects(&store, "video", true).await.len(), 1); + assert!(end(&store, "audio").await.is_some()); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn groups_and_datagrams_keep_their_form_through_an_abrupt_end() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let mut track = broadcast.create_track("mixed", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&track).await; + + group(&track, 0, &[(0, "g0")]); + track.insert_datagram(1, ms(100), "d1").unwrap(); + group(&track, 2, &[(200, "g2")]); + track.insert_datagram(3, ms(300), "d3").unwrap(); + track.insert_datagram(4, ms(400), "d4").unwrap(); + // Ends at once: every received datagram is stored before the end. + track.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "mixed", true).await; + assert_eq!( + contents(&stored), + [[ + (0, "group", vec!["0/g0".to_string()]), + (1, "datagram", vec!["100/d1".to_string()]), + (2, "group", vec!["200/g2".to_string()]), + (3, "datagram", vec!["300/d3".to_string()]), + (4, "datagram", vec!["400/d4".to_string()]), + ]] + ); + assert_eq!(end(&store, "mixed").await, Some(End::Finish(At { group: 5, frame: 0 }))); +} + +#[tokio::test(start_paused = true)] +async fn markers_wait_for_the_listing_to_end() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let mut video = broadcast.create_track("video", timed()).unwrap(); + let audio = broadcast.create_track("audio", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + subscribed(&audio).await; + + group(&video, 0, &[(0, "a")]); + let mut last = video.create_group(group::Info { sequence: 1 }).unwrap(); + last.write_frame(ms(1000), "b").unwrap(); + last.write_frame(ms(1100), "c").unwrap(); + // A mid-group end: frames 0 and 1 of group 1. + video.finish_at(Position { group: 1, frame: 2 }).unwrap(); + last.finish().unwrap(); + settle().await; + sleep(SECOND).await; + // Ended, but a track re-added under the name would continue it. + assert_eq!(end(&store, "video").await, None); + assert!(!complete(&store).await); + + audio.abort(moq_net::Error::App(7)).unwrap(); + settle().await; + assert_eq!(end(&store, "audio").await, None); + + broadcast.close(); + finished(task).await; + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 1, frame: 2 }))); + // An error end is recorded, and the broadcast still completes. + assert_eq!(end(&store, "audio").await, Some(End::Error(64 + 7))); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_re_added_track_continues_and_its_marker_records_the_last_run() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let first = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&first).await; + group(&first, 0, &[(0, "a")]); + group(&first, 1, &[(1000, "b")]); + first.finish().unwrap(); + settle().await; + drop(first); + settle().await; + + let second = broadcast.create_track("video", timed()).unwrap(); + subscribed(&second).await; + group(&second, 2, &[(2000, "c")]); + settle().await; + second.abort(moq_net::Error::App(3)).unwrap(); + settle().await; + broadcast.close(); + finished(task).await; + + let frames: Vec = contents(&objects(&store, "video", true).await) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "1000/b", "2000/c"]); + assert_eq!(end(&store, "video").await, Some(End::Error(64 + 3))); +} + +/// A track re-added with other properties after it was listed is refused before any of its +/// frames is stored under the listed properties. +#[tokio::test(start_paused = true)] +async fn a_re_added_track_is_stored_under_its_own_properties() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let first = broadcast.create_track("video", timed().with_priority(1)).unwrap(); + store.inner().hold_puts("video/.info"); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + settle().await; + drop(first); + settle().await; + + let second = broadcast.create_track("video", timed().with_priority(2)).unwrap(); + group(&second, 0, &[(0, "x")]); + store.inner().release(); + settle().await; + sleep(SECOND).await; + settle().await; + assert!(objects(&store, "video", true).await.is_empty()); + + second.finish().unwrap(); + broadcast.close(); + let err = moq_clock::timeout(Duration::from_secs(1000), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!( + matches!( + err, + Error::Priority { + existing: 1, + intended: 2 + } + ), + "{err}" + ); + assert!(objects(&store, "video", true).await.is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn a_stopped_producer_writes_no_marker() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + group(&video, 0, &[(0, "a")]); + settle().await; + sleep(SECOND).await; + task.abort(); + let _ = task.await; + video.finish().unwrap(); + broadcast.close(); + + assert_eq!(objects(&store, "video", true).await.len(), 1); + assert_eq!(end(&store, "video").await, None); + assert!(!complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_completed_broadcast_records_nothing() { + let store = store(); + store + .inner() + .put(&object_store::path::Path::from("rec/.end"), Default::default()) + .await + .unwrap(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + group(&video, 0, &[(0, "a")]); + finished(spawn(&store, &broadcast, config())).await; + assert_eq!(keys(&store).await, [".end"]); + assert!(video.subscription().is_none()); +} + +#[tokio::test(start_paused = true)] +async fn a_track_that_ended_unseen_leaves_the_broadcast_open() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let audio = broadcast.create_track("audio", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + subscribed(&audio).await; + group(&audio, 0, &[(0, "a")]); + settle().await; + sleep(SECOND).await; + task.abort(); + let _ = task.await; + + // Audio ends while nothing records, so the resume never sees how. + audio.finish().unwrap(); + drop(audio); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert!(end(&store, "video").await.is_some()); + assert_eq!(end(&store, "audio").await, None); + assert!(!complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_put_fails_the_recording() { + let store = store(); + store.inner().fail_puts("video/0"); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + group(&video, 0, &[(0, "a")]); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + let err = task.await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Store(_)), "{err}"); + assert_eq!(keys(&store).await, ["video/.info"]); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_resume_listing_writes_nothing() { + let store = store(); + store + .put_info("video", &Info::new(0, Some(1000)).unwrap()) + .await + .unwrap(); + store.inner().fail_lists(); + let broadcast = broadcast::Info::new().produce(); + let _video = broadcast.create_track("video", timed()).unwrap(); + store.inner().take(); + let err = spawn(&store, &broadcast, config()).await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Store(_)), "{err}"); + let ops = store.inner().take(); + assert!(!ops.iter().any(|op| matches!(op, crate::mock::Op::Put(_))), "{ops:?}"); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_prune_fails_the_dvr() { + let store = store(); + store.inner().fail_deletes("video/"); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let dvr = Dvr::new(SECOND, Duration::ZERO); + let task = spawn(&store, &broadcast, config().with_duration(SECOND).with_dvr(dvr)); + subscribed(&video).await; + for sequence in 0..3 { + group(&video, sequence, &[(sequence * 1000, "x")]); + } + settle().await; + sleep(2 * SECOND).await; + let err = task.await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Store(_)), "{err}"); +} + +#[tokio::test(start_paused = true)] +async fn a_stalled_put_holds_only_its_track() { + let store = store(); + store.inner().hold_puts("video/0"); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let audio = broadcast.create_track("audio", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + subscribed(&audio).await; + group(&video, 0, &[(0, "v")]); + group(&audio, 0, &[(0, "a")]); + settle().await; + sleep(SECOND).await; + settle().await; + assert_eq!(objects(&store, "audio", true).await.len(), 1); + assert!(objects(&store, "video", true).await.is_empty()); + + store.inner().release(); + video.finish().unwrap(); + audio.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(objects(&store, "video", true).await.len(), 1); + assert!(complete(&store).await); +} diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs new file mode 100644 index 00000000000..7059a8619c7 --- /dev/null +++ b/rs/moq-archive/src/producer/track.rs @@ -0,0 +1,472 @@ +//! One track's recorder: its runs, in order, into duration-cut objects. + +use std::collections::VecDeque; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Poll; +use std::time::Duration; + +use bytes::Bytes; +use moq_clock::Instant; +use moq_net::track::{Anchor, Order, Position, Range, Subscription}; +use moq_net::{StreamError, Timescale, group}; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; + +use super::Shared; +use super::ordered::{Item, Ordered}; +use super::resume::Stored; +use crate::layout::{self, End, Entry, Object}; +use crate::segment::Frame; +use crate::{Error, Info, Result}; + +/// Record each run of the track `name` reported through `queue`, until the queue closes with the +/// broadcast's listing. Returns how the last run ended, or the stored marker when no run did. +pub(super) async fn record( + shared: Arc>, + name: String, + queue: kio::Queue, + stored: Option, +) -> Result<(String, Option)> { + let mut track = Track::new(shared, &name, stored)?; + while let Some(info) = track.idle(&queue).await? { + let end = track.run(&info).await?; + track.end = Some(end); + } + Ok((name, track.end)) +} + +struct Track { + shared: Arc>, + name: String, + dir: Path, + /// The stored properties, once checked or written. + info: Option, + /// Whether `info` is known to be stored. + info_stored: bool, + timescale: Option, + /// The first position not stored yet, unknown until something is. + next: Option, + /// The object being filled. + object: Option, + /// The newest stored object, which DVR pruning always keeps. + newest: Option, + /// Objects a newer one superseded, oldest first, with when. + superseded: VecDeque<(Instant, Path)>, + /// Writes owed to the store, in order. + puts: Vec<(Path, Bytes)>, + end: Option, +} + +struct Building { + object: Object, + /// When it was opened, which cuts an untimed object by wall clock. + opened: Instant, + /// The arrival of its oldest frame, which bounds the flush. + unstored: Instant, + /// Where its last entry's run in this object starts: the timestamp on a timed track, the + /// arrival on an untimed one, which splits an oversized group. + run_pts: u64, + run_at: Instant, +} + +enum Event { + Frame(u64, moq_net::Result>), + Next(Option<(u64, Item)>), + Timer, +} + +impl Track { + fn new(shared: Arc>, name: &str, stored: Option) -> Result { + let dir = layout::track_dir(shared.store.prefix(), name)?; + let mut track = Self { + shared, + name: name.to_string(), + dir, + info: None, + info_stored: false, + timescale: None, + next: None, + object: None, + newest: None, + superseded: VecDeque::new(), + puts: Vec::new(), + end: None, + }; + let Some(stored) = stored else { + return Ok(track); + }; + + track.timescale = timescale(&stored.info)?; + track.info = Some(stored.info); + track.end = stored.end; + track.next = stored.next; + let mut objects = stored.objects; + track.newest = objects.pop(); + if track.shared.config.dvr.is_some() { + // Their age is unknown, so their window starts now. + let now = moq_clock::now(); + track.superseded = objects.into_iter().map(|path| (now, path)).collect(); + } + Ok(track) + } + + /// Wait for the next run, pruning meanwhile: the queued track info, which the listing keeps + /// to the newest, or `None` once the listing ended. + async fn idle(&mut self, queue: &kio::Queue) -> Result> { + loop { + let mut timer = self.prune_at().map(|at| Box::pin(moq_clock::sleep_until(at))); + let popped = kio::wait(|waiter| { + if let Poll::Ready(popped) = queue.poll_pop(waiter) { + return Poll::Ready(Some(popped)); + } + ready_at(&mut timer, waiter).map(|()| None) + }) + .await; + match popped { + Some(Ok(info)) => return Ok(Some(info)), + Some(Err(kio::Closed)) => return Ok(None), + None => self.prune().await?, + } + } + } + + /// Store the track's properties before its first object, refusing ones that changed. + async fn properties(&mut self, source: &moq_net::track::Info) -> Result<()> { + let info = Info::new(source.priority, source.timescale.map(Timescale::as_u64))?; + if let Some(existing) = &self.info { + if existing.priority != info.priority { + return Err(Error::Priority { + existing: existing.priority, + intended: info.priority, + }); + } + if existing.timescale != info.timescale { + return Err(Error::TimescaleMismatch { + existing: existing.timescale, + intended: info.timescale, + }); + } + } + if !self.info_stored { + self.shared.store.put_info(&self.name, &info).await?; + self.info_stored = true; + } + self.timescale = source.timescale; + self.info = Some(info); + Ok(()) + } + + /// Record one subscription until both its groups and its datagrams end, returning how. + async fn run(&mut self, info: &moq_net::track::Info) -> Result { + let shared = self.shared.clone(); + let config = &shared.config; + let range = match self.next { + Some(next) => Range { + start: Anchor::Absolute(next), + end: None, + }, + None => Range::oldest(), + }; + // The cap, since `info` may describe an older instance than the one subscribed, and a + // backlog a smaller budget skipped is not sent again. The wait for a gap is narrowed below. + let subscription = Subscription::default() + .with_order(Order::Ascending) + .with_max_delay(config.max_delay) + .with_ranges([range]); + + let subscribed = match self.shared.broadcast.track(&self.name) { + Ok(track) => track.subscribe(subscription).await, + Err(err) => Err(err), + }; + let subscriber = match subscribed { + Ok(subscriber) => subscriber, + Err(err) => { + // The track keeps its `.info` beside the `end` this run leaves. + self.properties(info).await?; + return Ok(error(&err)); + } + }; + // The listing's info may describe an instance the name no longer resolves to, so the + // subscribed one's properties are what its frames are stored under. + self.properties(subscriber.info()).await?; + let max_delay = delay(config, subscriber.info()); + + let mut ordered = Ordered::new(subscriber, max_delay, self.next.map(|next| next.group)); + // The group being stored, frame by frame. + let mut current: Option> = None; + loop { + self.flush_due(moq_clock::now())?; + self.write().await?; + self.prune().await?; + + let mut timer = [self.flush_at(), self.prune_at()] + .into_iter() + .flatten() + .min() + .map(|at| Box::pin(moq_clock::sleep_until(at))); + let event = kio::wait(|waiter| { + match current.as_mut() { + Some(current) => { + let index = current.index(); + if let Poll::Ready(frame) = current.poll_read_frame(waiter) { + return Poll::Ready(Event::Frame(index, frame)); + } + // Arrival times decide the waits, so arrivals are taken meanwhile. + ordered.poll_arrivals(waiter); + } + None => { + if let Poll::Ready(next) = ordered.poll_next(waiter) { + return Poll::Ready(Event::Next(next)); + } + } + } + ready_at(&mut timer, waiter).map(|()| Event::Timer) + }) + .await; + + let now = moq_clock::now(); + match event { + Event::Frame(index, read) => self.frame(&mut current, index, read, now)?, + Event::Next(Some((sequence, Item::Datagram(datagram)))) => { + let frame = Frame { + timestamp: self.stamp(datagram.timestamp)?, + payload: datagram.payload, + }; + self.push(sequence, 0, frame, true, now)?; + self.next = Some(Position::group(sequence + 1)); + } + Event::Next(Some((sequence, Item::Group(mut group)))) => { + // A resumed group continues after its stored frames. The clamp also starts a + // group whose head is held elsewhere at its first frame, rather than lagging + // behind it. + let stored = self + .next + .filter(|next| next.group == sequence) + .map_or(0, |next| next.frame); + group.set_frames(stored..); + self.next = Some(Position { + group: sequence, + frame: group.index(), + }); + current = Some(group); + } + Event::Next(None) => break, + Event::Timer => {} + } + } + + let end = match kio::wait(|waiter| ordered.poll_finished(waiter)).await { + Ok(end) => End::Finish(end.into()), + Err(err) => error(&err), + }; + self.close()?; + self.write().await?; + Ok(end) + } + + /// Store the frame read at `index` of the group being stored, or move past that group once it + /// ended, cleanly or not (aborted, or past the budget), keeping what it stored. + fn frame( + &mut self, + current: &mut Option>, + index: u64, + read: moq_net::Result>, + now: Instant, + ) -> Result<()> { + let sequence = current.as_ref().expect("reading a group").sequence; + match read { + Ok(Some(frame)) => { + let frame = Frame { + timestamp: self.stamp(frame.timestamp)?, + payload: frame.payload, + }; + self.push(sequence, index, frame, false, now)?; + self.next = Some(Position { + group: sequence, + frame: index + 1, + }); + } + ended => { + if let Err(err) = ended { + tracing::debug!(track = %self.name, sequence, %err, "group ended early"); + } + *current = None; + self.next = Some(Position::group(sequence + 1)); + } + } + Ok(()) + } + + /// A frame's timestamp in the track's timescale, which it carries exactly when timed. + fn stamp(&self, timestamp: Option) -> Result> { + match (timestamp, self.timescale) { + (Some(timestamp), Some(timescale)) => { + Ok(Some(timestamp.convert(timescale).map_err(|_| Error::Timing)?.value())) + } + (None, None) => Ok(None), + _ => Err(Error::Timing), + } + } + + /// Append one frame of `sequence`, or a datagram, cutting the object before it when due. + fn push(&mut self, sequence: u64, index: u64, frame: Frame, datagram: bool, now: Instant) -> Result<()> { + let config = &self.shared.config; + if let Some(building) = &self.object { + let last = building.object.entries.last().map(Entry::sequence); + let boundary = last != Some(sequence); + // Only an object's first entry carries a frame start, so a group starting past frame 0 + // opens its own object. + let cut = (boundary && index != 0) + || match (self.timescale, frame.timestamp) { + (Some(timescale), Some(pts)) => { + let duration = units(config.duration, timescale); + let first = building.object.start()?.pts.expect("a timed object starts timed"); + match boundary { + true => pts.saturating_sub(first) >= duration, + // An oversized group splits once its own run here reaches the duration. + false => pts.saturating_sub(building.run_pts) >= duration, + } + } + _ => { + let since = match boundary { + true => building.opened, + false => building.run_at, + }; + now.saturating_duration_since(since) >= config.duration + } + }; + if cut { + self.close()?; + } + } + + let building = self.object.get_or_insert_with(|| Building { + object: Object { + frame_start: index, + entries: Vec::new(), + }, + opened: now, + unstored: now, + run_pts: 0, + run_at: now, + }); + let pts = frame.timestamp.unwrap_or(0); + match building.object.entries.last_mut() { + Some(Entry::Group { sequence: last, frames }) if *last == sequence && !datagram => frames.push(frame), + _ => { + building.run_pts = pts; + building.run_at = now; + building.object.entries.push(match datagram { + true => Entry::Datagram { sequence, frame }, + false => Entry::Group { + sequence, + frames: vec![frame], + }, + }); + } + } + Ok(()) + } + + /// When the object's oldest frame is due in the store. + fn flush_at(&self) -> Option { + Some(self.object.as_ref()?.unstored + self.shared.config.flush) + } + + /// Store the object once its oldest frame is due, so its group continues in the next. + fn flush_due(&mut self, now: Instant) -> Result<()> { + if self.flush_at().is_some_and(|at| at <= now) { + self.close()?; + } + Ok(()) + } + + /// Store the object for good and start the next one at the following frame. + fn close(&mut self) -> Result<()> { + let Some(building) = self.object.take() else { + return Ok(()); + }; + let bytes = building.object.encode(self.timescale.is_some())?; + let path = push(&self.dir, &building.object.start()?.name()?)?; + if let Some(superseded) = self.newest.replace(path.clone()) + && self.shared.config.dvr.is_some() + { + self.superseded.push_back((moq_clock::now(), superseded)); + } + self.puts.push((path, bytes)); + Ok(()) + } + + /// Perform the owed writes in order. + async fn write(&mut self) -> Result<()> { + let store = self.shared.store.inner(); + for (path, bytes) in std::mem::take(&mut self.puts) { + let payload = PutPayload::from(bytes.clone()); + match store.put_opts(&path, payload, PutMode::Create.into()).await { + Ok(_) => {} + // Written already, by an earlier attempt that failed to report success. + Err(object_store::Error::AlreadyExists { .. }) => { + let existing = store.get(&path).await?.bytes().await?; + if existing != bytes { + return Err(Error::Conflict(path.to_string())); + } + } + Err(err) => return Err(err.into()), + } + } + Ok(()) + } + + fn prune_at(&self) -> Option { + let dvr = self.shared.config.dvr.as_ref()?; + let (superseded, _) = self.superseded.front()?; + Some(*superseded + dvr.window + dvr.grace) + } + + /// Delete the superseded objects whose window and grace have passed. + async fn prune(&mut self) -> Result<()> { + let now = moq_clock::now(); + while self.prune_at().is_some_and(|at| at <= now) { + let (_, path) = self.superseded.pop_front().expect("a superseded object is due"); + match self.shared.store.inner().delete(&path).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => {} + Err(err) => return Err(err.into()), + } + } + Ok(()) + } +} + +fn push(dir: &Path, name: &str) -> Result { + crate::path::push(dir, name) +} + +fn timescale(info: &Info) -> Result> { + info.timescale + .map(|timescale| Timescale::new(timescale).map_err(|_| Error::Timescale(timescale))) + .transpose() +} + +/// How long a missing group is waited for: the configured cap, or the publisher's max age below it. +fn delay(config: &super::Config, info: &moq_net::track::Info) -> Duration { + info.max_age.map_or(config.max_delay, |age| age.min(config.max_delay)) +} + +/// `duration` in `timescale` units, saturating. +fn units(duration: Duration, timescale: Timescale) -> u64 { + let units = duration.as_nanos() * u128::from(timescale.as_u64()) / 1_000_000_000; + u64::try_from(units).unwrap_or(u64::MAX) +} + +fn error(err: &moq_net::Error) -> End { + End::Error(StreamError::from(err).to_code()) +} + +/// Poll an optional timer, pending forever without one. +fn ready_at(timer: &mut Option>>, waiter: &kio::Waiter) -> Poll<()> { + match timer.as_mut() { + Some(timer) => waiter.poll_future(timer.as_mut()), + None => Poll::Pending, + } +} diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs deleted file mode 100644 index d72b8ab66c8..00000000000 --- a/rs/moq-archive/src/proof.rs +++ /dev/null @@ -1,577 +0,0 @@ -//! End to end: one multi-rendition broadcast recorded by a [`Writer`], stored byte-identically on -//! every backend, and replayed exactly through a [`Reader`]. - -use std::collections::BTreeMap; -use std::time::Duration; - -use bytes::Bytes; -use futures::TryStreamExt; -use moq_net::{Timescale, Timestamp, broadcast, group, track}; -use object_store::local::LocalFileSystem; -use object_store::path::Path; -use object_store::{ObjectStore, ObjectStoreExt}; - -use crate::mock::{Mock, Op}; -use crate::reader::Config as ReaderConfig; -use crate::writer::{Config, Retention}; -use crate::{Reader, Store, Writer}; - -const RENDITIONS: [&str; 2] = ["video/1080p", "video/360p"]; -const MEDIA: [&str; 3] = ["video/1080p", "video/360p", "audio"]; -const TRACKS: [&str; 5] = ["video/1080p", "video/360p", "audio", "catalog.json", "chat"]; - -/// Frames of one source group: (millisecond timestamp, payload). -type Frames = Vec<(u64, Bytes)>; - -/// Source segments: three 2s segments, then a one-frame tail. -const SEGMENTS: u64 = 4; - -/// The source groups of one segment: (track, sequence, frames). -/// -/// Video renditions share a 2s GOP and audio has four 500ms groups per segment, which its own -/// timeline packs into 2s records by the default minimum. The catalog and a non-media chat track publish sparsely, -/// including a sequence skip and an empty payload. -fn plan(segment: u64) -> Vec<(&'static str, u64, Frames)> { - let pts = segment * 2000; - let (frames, audio) = if segment + 1 < SEGMENTS { (4, 4) } else { (1, 1) }; - let mut groups = Vec::new(); - for track in RENDITIONS { - let frames = (0..frames) - .map(|i| (pts + i * 500, Bytes::from(format!("{track} {segment}.{i}")))) - .collect(); - groups.push((track, segment, frames)); - } - for sequence in segment * 4..segment * 4 + audio { - let frames = (0..2) - .map(|i| (sequence * 500 + i * 250, Bytes::from(format!("audio {sequence}.{i}")))) - .collect(); - groups.push(("audio", sequence, frames)); - } - let catalog = |sequence: u64| Bytes::from(format!(r#"{{"version":{sequence}}}"#)); - let chat = |text: &'static str| Bytes::from_static(text.as_bytes()); - match segment { - 0 => { - groups.push(("catalog.json", 0, vec![(0, catalog(0))])); - groups.push(("chat", 0, vec![(1200, chat("hello")), (1300, Bytes::new())])); - } - 2 => { - groups.push(("catalog.json", 1, vec![(4000, catalog(1))])); - groups.push(("chat", 3, vec![(4100, chat("skip"))])); - groups.push(("chat", 5, vec![(5100, chat("bye"))])); - } - _ => {} - } - groups -} - -fn ms(value: u64) -> Timestamp { - Timestamp::new(value, Timescale::MILLI).unwrap() -} - -fn write(track: &track::Producer, sequence: u64, frames: &Frames) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for (timestamp, payload) in frames { - group.write_frame(ms(*timestamp), payload.clone()).unwrap(); - } - group.finish().unwrap(); -} - -/// Let the writer read and report everything already published. Reads never touch the store. -async fn settle() { - for _ in 0..256 { - tokio::task::yield_now().await; - } -} - -/// Media cut at 1s, so one-second test groups get a record each. -fn media() -> moq_mux::timeline::Config { - moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)) -} - -fn timeline(track: &str) -> String { - hang::timeline::default_name(track) -} - -/// Every recorded track's timeline, as the catalog's `archive` entry names them. -fn config() -> ReaderConfig { - ReaderConfig::new( - TRACKS - .iter() - .map(|track| (track.to_string(), timeline(track))) - .collect(), - ) -} - -/// Record every segment of [`plan`] into `store`. -async fn record(store: &Store) { - let source = broadcast::Info::new().produce(); - let tracks: BTreeMap<&str, track::Producer> = TRACKS - .iter() - .map(|&name| { - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - (name, source.create_track(name, info).unwrap()) - }) - .collect(); - - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - for name in TRACKS { - let config = match MEDIA.contains(&name) { - // Keep this storage-layout proof at one record per 2s source segment. The default - // timeline minimum is tested in moq-mux and is not part of this fixture's contract. - true => moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(2)), - false => moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO), - }; - control.track(name, config).await.unwrap(); - } - let run = tokio::spawn(writer.run()); - - // Each track cuts its own timeline, so the interleave across tracks changes nothing stored. - for segment in 0..SEGMENTS { - for (name, sequence, frames) in plan(segment) { - write(&tracks[name], sequence, &frames); - } - settle().await; - } - for track in tracks.values() { - track.finish().unwrap(); - } - source.close(); - run.await.unwrap().unwrap(); -} - -/// Every object under the store's prefix, by path relative to the store root. -async fn objects(store: &Store) -> BTreeMap { - let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); - let mut objects = BTreeMap::new(); - for meta in metas { - let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); - objects.insert(meta.location.to_string(), bytes); - } - objects -} - -fn id(value: u64) -> String { - format!("{value:019}") -} - -/// Each track's record count: a GOP per video record, four audio groups per record, and a group per -/// sparse record. -const RECORDS: [(&str, u64); 5] = [ - ("video%2F1080p", 4), - ("video%2F360p", 4), - ("audio", 4), - ("catalog%2Ejson", 2), - ("chat", 3), -]; - -/// The exact layout [`plan`] produces: no manifest, index, or completion marker. -fn layout() -> Vec { - let mut keys = Vec::new(); - for (track, records) in RECORDS { - for directory in [track.to_string(), format!("{track}%2Etimeline%2Ez")] { - keys.push(format!("rec/{directory}/.info")); - keys.extend((0..records).map(|record| format!("rec/{directory}/segments/{}", id(record)))); - } - } - keys.sort(); - keys -} - -/// FETCH one whole group, returning its frames. -async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> moq_net::Result { - let track = broadcast.consume().track(track)?; - let mut group = track.fetch_group(sequence, group::Fetch::default()).await?; - let mut frames = Vec::new(); - while let Some(frame) = group.read_frame().await? { - frames.push(( - frame.timestamp.unwrap().convert(Timescale::MILLI).unwrap().value(), - frame.payload, - )); - } - Ok(frames) -} - -#[tokio::test] -async fn recordings_are_byte_identical_on_every_backend() { - let memory = Store::new(Mock::memory(), "rec"); - record(&memory).await; - let expected = objects(&memory).await; - assert_eq!(expected.keys().cloned().collect::>(), layout()); - assert_eq!( - &expected["rec/video%2F360p/.info"][..], - br#"{"version":2,"priority":127,"timescale":1000}"# - ); - - let dir = tempfile::tempdir().unwrap(); - let local = Store::new(Mock::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap()), "rec"); - record(&local).await; - assert_eq!(objects(&local).await, expected, "local disk"); - - // A backend with no listing order, and a sibling recording sharing the prefix's stem. - let unordered = Mock::memory(); - unordered.unordered(); - let sibling = Store::new(unordered.clone(), "rec-other"); - record(&sibling).await; - let unordered = Store::new(unordered, "rec"); - record(&unordered).await; - assert_eq!(objects(&unordered).await, expected, "unordered listing"); - assert_eq!(objects(&sibling).await.len(), expected.len()); - - // The media objects hold exactly the source groups, in order, byte for byte. - let mut stored: BTreeMap<(String, u64), Frames> = BTreeMap::new(); - for (path, bytes) in &expected { - let key = crate::Key::parse(&Path::from("rec"), &Path::parse(path).unwrap()).unwrap(); - if let crate::Key::Segments { track, .. } = key - && !track.ends_with(hang::timeline::SUFFIX) - { - let object = crate::Object::decode(bytes.clone(), true).unwrap(); - assert_eq!(object.frame_start, 0, "no source group outlived the maximum"); - for group in object.groups { - let frames = group - .frames - .into_iter() - .map(|f| (f.timestamp.unwrap(), f.payload)) - .collect(); - stored.insert((track.clone(), group.sequence), frames); - } - } - } - let source: BTreeMap<(String, u64), Frames> = (0..SEGMENTS) - .flat_map(plan) - .map(|(track, sequence, frames)| ((track.to_string(), sequence), frames)) - .collect(); - assert_eq!(stored, source); -} - -#[tokio::test] -async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - record(&store).await; - - let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(store, &broadcast, config()).await.unwrap(); - tokio::spawn(reader.serve()); - mock.take(); - - // Low-rendition playback never downloads the 1080p object. - for sequence in 0..SEGMENTS { - let expected = plan(sequence) - .into_iter() - .find(|(name, ..)| *name == "video/360p") - .unwrap() - .2; - assert_eq!(fetch(&broadcast, "video/360p", sequence).await.unwrap(), expected); - } - let object = |track: &str, record: u64| format!("rec/{track}/segments/{}", id(record)); - // Each track request also GETs that track's `.info`, but nothing of another track. - let media = |gets: Vec, track: &str| { - let prefix = format!("rec/{track}/"); - assert!(gets.iter().all(|path| path.starts_with(&prefix)), "{gets:?}"); - gets.into_iter() - .filter(|path| path.contains("/segments/")) - .collect::>() - }; - assert_eq!( - media(mock.gets(), "video%2F360p"), - (0..4).map(|record| object("video%2F360p", record)).collect::>() - ); - - // Audio-only playback: four adjacent groups per GET, the rest from the cache. - for segment in 0..SEGMENTS { - for (_, sequence, frames) in plan(segment).into_iter().filter(|(name, ..)| *name == "audio") { - assert_eq!(fetch(&broadcast, "audio", sequence).await.unwrap(), frames); - } - } - assert_eq!( - media(mock.gets(), "audio"), - (0..4).map(|record| object("audio", record)).collect::>() - ); - - // Every other enrolled group replays its original sequence, timestamps, and payloads. - for segment in 0..SEGMENTS { - for (name, sequence, frames) in plan(segment) { - assert_eq!( - fetch(&broadcast, name, sequence).await.unwrap(), - frames, - "{name} {sequence}" - ); - } - } - for (track, sequence) in [("chat", 1), ("chat", 4), ("chat", 6), ("audio", 13), ("video/1080p", 4)] { - assert!( - matches!(fetch(&broadcast, track, sequence).await, Err(moq_net::Error::NotFound)), - "{track} {sequence} was never recorded" - ); - } - assert!( - !mock.take().iter().any(|op| matches!(op, Op::List { .. })), - "a group request resolves its object without listing" - ); -} - -#[tokio::test] -async fn an_offline_reader_follows_dvr_expiry() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - let source = broadcast::Info::new().produce(); - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - let video = source.create_track("video", info).unwrap(); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - let frames = |sequence: u64| vec![(sequence * 1000, Bytes::from(format!("video {sequence}")))]; - let committed = |segment: u64| { - let store = store.clone(); - async move { - while store.get_segments(&timeline("video"), segment, true).await.is_err() { - tokio::time::sleep(Duration::from_millis(5)).await; - } - } - }; - - for sequence in 0..3 { - write(&video, sequence, &frames(sequence)); - } - committed(1).await; - - let broadcast = broadcast::Info::new().produce(); - let config = ReaderConfig::new([("video".to_string(), timeline("video"))].into()); - let mut reader = Reader::open(store.clone(), &broadcast, config).await.unwrap(); - tokio::spawn(reader.serve()); - assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), frames(0)); - - // The reader is offline while the DVR commits and expires several records. - for sequence in 3..10 { - write(&video, sequence, &frames(sequence)); - } - committed(8).await; - // Records 7 and 8 hold the 2s window, and the newest timeline object restates both. - while store.get_segments("video", 6, true).await.is_ok() - || store.get_segments(&timeline("video"), 7, true).await.is_ok() - { - tokio::time::sleep(Duration::from_millis(5)).await; - } - - mock.take(); - reader.refresh().await.unwrap(); - let ops = mock.take(); - let (list, gets) = ops.split_first().unwrap(); - assert_eq!( - list, - &Op::List { - prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), - offset: Some(format!("rec/video%2Etimeline%2Ez/segments/{}", id(1))), - } - ); - assert_eq!( - gets, - &[Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{}", id(8)))], - "only the retained timeline key is read" - ); - - // Expired groups are gone from the index, so they cost no media GET. - for sequence in 0..7 { - assert!( - matches!( - fetch(&broadcast, "video", sequence).await, - Err(moq_net::Error::NotFound) - ), - "group {sequence} expired" - ); - } - let gets = mock.gets(); - assert!( - !gets.iter().any(|path| path.starts_with("rec/video/segments/")), - "{gets:?}" - ); - for sequence in 7..9 { - assert_eq!(fetch(&broadcast, "video", sequence).await.unwrap(), frames(sequence)); - } - - video.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); -} - -/// Exercise the S3 HTTP adapter, rather than an ObjectStore listing fake. -mod s3 { - use std::num::NonZeroUsize; - use std::sync::{Arc, Mutex}; - - use axum::body::Body; - use axum::http::{Method, Uri}; - use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut}; - use object_store::{PutMode, PutPayload}; - use s3s::service::S3ServiceBuilder; - - use super::*; - - struct Request { - method: Method, - uri: Uri, - conditional: bool, - } - - struct Server { - store: Store, - requests: Arc>>, - task: tokio::task::JoinHandle<()>, - _dir: tempfile::TempDir, - } - - impl Server { - async fn new() -> Self { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir(dir.path().join("archive-proof")).unwrap(); - let service = S3ServiceBuilder::new(s3s_fs::FileSystem::new(dir.path()).unwrap()).build(); - let requests = Arc::new(Mutex::new(Vec::new())); - let seen = requests.clone(); - let app = axum::Router::new().fallback(move |request: axum::extract::Request| { - let service = service.clone(); - let seen = seen.clone(); - async move { - seen.lock().unwrap().push(Request { - method: request.method().clone(), - uri: request.uri().clone(), - conditional: request.headers().get("if-none-match").is_some_and(|value| value == "*"), - }); - service - .call(request.map(s3s::Body::http_body_unsync)) - .await - .unwrap() - .map(Body::new) - } - }); - let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) - .await - .unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - let store = AmazonS3Builder::new() - .with_bucket_name("archive-proof") - .with_region("us-east-1") - .with_endpoint(endpoint) - .with_allow_http(true) - .with_skip_signature(true) - .with_conditional_put(S3ConditionalPut::ETagMatch) - .build() - .unwrap(); - Self { - store: Store::new(store, "rec"), - requests, - task, - _dir: dir, - } - } - } - - impl Drop for Server { - fn drop(&mut self) { - self.task.abort(); - } - } - - #[tokio::test] - async fn archive_proof_crosses_s3_http() { - let memory = Store::new(Mock::memory(), "rec"); - record(&memory).await; - let expected = objects(&memory).await; - let server = Server::new().await; - record(&server.store).await; - assert_eq!(objects(&server.store).await, expected); - let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(server.store.clone(), &broadcast, config()).await.unwrap(); - let task = tokio::spawn(reader.serve()); - for segment in 0..SEGMENTS { - for (name, sequence, frames) in plan(segment) { - assert_eq!(fetch(&broadcast, name, sequence).await.unwrap(), frames); - } - } - task.abort(); - assert!( - server - .requests - .lock() - .unwrap() - .iter() - .any(|request| request.uri.path().contains("catalog%252Ejson")), - "a percent-encoded recording key must survive URL encoding" - ); - } - - #[tokio::test] - async fn s3_create_offset_and_continuation_are_enforced() { - let server = Server::new().await; - record(&server.store).await; - let expected = objects(&server.store).await; - let (path, original) = expected.first_key_value().unwrap(); - let path = Path::parse(path).unwrap(); - let collision = server - .store - .inner() - .put_opts(&path, PutPayload::from_static(b"different"), PutMode::Create.into()) - .await; - assert!( - matches!(collision, Err(object_store::Error::AlreadyExists { .. })), - "{collision:?}" - ); - assert_eq!( - server.store.inner().get(&path).await.unwrap().bytes().await.unwrap(), - *original - ); - - let mut query = crate::store::list::Query::new().page_size(NonZeroUsize::new(2).unwrap()); - let mut listed = Vec::new(); - loop { - let page = server.store.list_paginated(&query).await.unwrap(); - assert!(page.entries.len() <= 2); - listed.extend( - page.entries - .iter() - .map(|entry| server.store.path(&entry.key).unwrap().to_string()), - ); - let Some(next) = page.next else { break }; - query = next; - } - listed.sort(); - assert_eq!(listed, expected.keys().cloned().collect::>()); - - let after = crate::Key::segments("video/1080p", 0).unwrap(); - let query = crate::store::list::Query::segments("video/1080p") - .unwrap() - .after(&after) - .unwrap(); - let later: Vec<_> = server.store.list(&query).try_collect().await.unwrap(); - assert_eq!(later.len(), (SEGMENTS - 1) as usize); - assert!(later.iter().all( - |entry| matches!(&entry.key, crate::Key::Segments { track, segment } if track == "video/1080p" && *segment > 0) - )); - let requests = server.requests.lock().unwrap(); - assert!( - requests - .iter() - .any(|request| request.method == Method::PUT && request.conditional) - ); - assert!(requests.iter().any(|request| { - request - .uri - .query() - .is_some_and(|query| query.contains("continuation-token=")) - })); - assert!( - requests - .iter() - .any(|request| request.uri.query().is_some_and(|query| query.contains("start-after="))) - ); - } -} diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index 3cf641605e3..d3bd64b6adb 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -701,67 +701,6 @@ async fn timeline_tracks_are_republished_and_finished_on_request() { assert_eq!(records, vec![0, 1]); } -/// An untimed recording replays untimed: no media or timeline frame gains a timestamp. -#[tokio::test] -async fn an_untimed_recording_replays_untimed() { - let store = Store::new(Mock::memory(), "rec"); - let source = broadcast::Info::new().produce(); - let info = moq_net::track::Info::default().with_max_age(Duration::from_secs(3600)); - let data = source.create_track("data", info).unwrap(); - let writer = crate::Writer::new(store.clone(), source.consume(), Default::default()) - .await - .unwrap(); - writer - .control() - .track("data", moq_mux::timeline::Config::default()) - .await - .unwrap(); - for sequence in 0..2 { - let mut group = data.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, format!("data/{sequence}")).unwrap(); - group.finish().unwrap(); - } - data.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let broadcast = broadcast::Info::new().produce(); - let config = Config::new([("data".to_string(), timeline("data"))].into()); - let reader = Reader::open(store, &broadcast, config).await.unwrap(); - tokio::spawn(reader.serve()); - - let track = broadcast.consume().track("data").unwrap(); - let mut group = track.fetch_group(1, group::Fetch::default()).await.unwrap(); - assert_eq!(group.timescale(), None); - let frame = group.read_frame().await.unwrap().unwrap(); - assert_eq!((frame.timestamp, frame.payload), (None, Bytes::from("data/1"))); - - let subscriber = broadcast - .consume() - .track(&timeline("data")) - .unwrap() - .subscribe(None) - .await - .unwrap(); - assert_eq!(subscriber.info().timescale, None); - let mut timeline = - window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); - reader.finish().unwrap(); - let mut records = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let window::Event::Push { value, .. } = event { - records.push(value); - } - } - assert_eq!( - records, - vec![ - Record::untimed(0, Position::group(0), Position::group(1)), - Record::untimed(1, Position::group(1), Position::group(2)), - ] - ); -} - /// A record whose timing contradicts its timeline is not indexed, though its object is stored. #[tokio::test] async fn a_record_contradicting_its_timeline_is_not_indexed() { diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs deleted file mode 100644 index 188e36c9043..00000000000 --- a/rs/moq-archive/src/recover.rs +++ /dev/null @@ -1,210 +0,0 @@ -//! Recover a recording so a restarted [`Writer`](crate::Writer) continues it. - -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::ops::{Range, RangeInclusive}; - -use futures::TryStreamExt; -use hang::timeline::{Position, Record}; -use moq_json::window::{self, Checkpoint}; -use object_store::ObjectStore; - -use crate::store::list::Query; -use crate::{Error, Key, Object, Result, Store}; - -/// What a restarted writer continues from. -#[derive(Default)] -pub(crate) struct Recovery { - /// Each recorded track's retained timeline, keyed by the recorded track's name. - pub tracks: HashMap, - /// Committed objects no retained record references, such as an interrupted expiration. Only - /// collected for a DVR, which deletes them after its grace. - pub orphans: Vec, - /// Objects at or past a track's next record: uploads a crash left before their commit. No - /// timeline ever referenced them, and the resumed writer reuses their keys. - pub uncommitted: Vec, - /// Each timeline's stored segments older than any recovery reads. Only collected for a DVR. - pub stale: Vec<(String, Range)>, -} - -/// One track's retained timeline. -pub(crate) struct Resume { - /// The retained window. - pub checkpoint: Checkpoint, - /// The next stored timeline group sequence, so the stored numbering keeps increasing. - pub sequence: u64, - /// The timeline objects recovery replayed, oldest first, with the first record index each - /// one's opening checkpoint restates. Only collected for a DVR. - pub checkpoints: VecDeque<(u64, u64)>, -} - -impl Resume { - /// Where the newest committed record ended: the resumed track refuses anything before it. - pub fn floor(&self) -> Option { - self.checkpoint.records.last().map(|record| record.end) - } -} - -/// List the whole recording, then replay each track's timeline from a retained checkpoint. -/// -/// A DVR (`complete`) replays far enough back to recover every retained record and reports the -/// unreferenced objects; an archive only needs the newest checkpoint. Any listing, GET, or decode -/// failure fails recovery, so a caller never acts on a partial view. -pub(crate) async fn recover(store: &Store, complete: bool) -> Result { - let entries: Vec<_> = store.list(&Query::new()).try_collect().await?; - let mut segments: BTreeMap> = BTreeMap::new(); - for entry in entries { - if let Key::Segments { track, segment } = entry.key { - segments.entry(track).or_default().push(segment); - } - } - - let mut recovery = Recovery::default(); - for (timeline, ids) in &mut segments { - let Some(track) = timeline.strip_suffix(hang::timeline::SUFFIX) else { - continue; - }; - ids.sort_unstable(); - let (first, last) = (ids[0], ids[ids.len() - 1]); - if ids.len() as u64 != last - first + 1 { - return Err(Error::Timeline(format!( - "{timeline} segments {first}..={last} are not contiguous" - ))); - } - let resume = replay(store, timeline, first..=last, complete).await?; - // The newest object is segment `last`, so the window must end on the next one. A shorter - // window would resume onto that segment and collide with it. - let next = last.checked_add(1).ok_or(Error::Overflow)?; - if resume.checkpoint.range.end != next { - return Err(Error::Timeline(format!( - "{timeline} window ends at {}, not segment {next}", - resume.checkpoint.range.end - ))); - } - if let Some(&(needed, _)) = resume.checkpoints.front() - && first < needed - { - recovery.stale.push((timeline.clone(), first..needed)); - } - recovery.tracks.insert(track.to_string(), resume); - } - - for (track, ids) in segments { - if track.ends_with(hang::timeline::SUFFIX) { - continue; - } - let resume = recovery.tracks.get(&track); - let next = resume.map_or(0, |resume| resume.checkpoint.range.end); - let retained: HashSet = resume - .iter() - .flat_map(|resume| &resume.checkpoint.records) - .map(|record| record.sequence) - .collect(); - for id in ids { - let key = Key::segments(track.clone(), id)?; - if id >= next { - recovery.uncommitted.push(key); - } else if complete && !retained.contains(&id) { - recovery.orphans.push(key); - } - } - } - - Ok(recovery) -} - -/// Replay `segments` from the newest checkpoint that restates every record `complete` needs. -async fn replay( - store: &Store, - timeline: &str, - segments: RangeInclusive, - complete: bool, -) -> Result { - // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, - // so walk back until a checkpoint restates from there. The writer prunes by the same rule. - let timed = store.get_info(timeline).await?.timescale.is_some(); - let mut objects = VecDeque::new(); - let mut checkpoints = VecDeque::new(); - let mut needed = None; - for segment in segments.rev() { - let object = store.get_segments(timeline, segment, timed).await?; - let (offset, start) = checkpoint(&object)?; - let needed = *needed.get_or_insert(offset); - objects.push_front(object); - if complete { - checkpoints.push_front((segment, start)); - } - if !complete || start <= needed { - break; - } - } - - let mut decoder = decoder(); - let mut records = BTreeMap::new(); - for object in &objects { - for stored in &object.groups { - let mut group = decoder.group(); - for frame in &stored.frames { - group.decode(&frame.payload).map_err(json_error)?; - } - } - while let Some(event) = decoder.next_event() { - match event { - window::Event::Push { index, value } => { - records.insert(index, value); - } - window::Event::Pop(range) | window::Event::Skip(range) => { - let tail = records.split_off(&range.end); - records.retain(|index, _| *index < range.start); - records.extend(tail); - } - _ => {} - } - } - } - - // The records must be a contiguous suffix of the window, and all of it for a DVR. - let range = decoder.range(); - let start = records.keys().next().copied().unwrap_or(range.end); - if records.len() as u64 != range.end - start || (complete && start != range.start) { - return Err(Error::Timeline(format!( - "cannot recover {timeline} window {range:?} from the retained checkpoints" - ))); - } - - let last = objects.back().and_then(|object| object.groups.last()); - let sequence = last.map_or(Ok(0), |group| group.sequence.checked_add(1).ok_or(Error::Overflow))?; - let checkpoint = Checkpoint { - range, - records: records.into_values().collect(), - }; - Ok(Resume { - checkpoint, - sequence, - checkpoints, - }) -} - -/// The retained offset and first restated index of the checkpoint opening `object`. -pub(crate) fn checkpoint(object: &Object) -> Result<(u64, u64)> { - let frame = object - .groups - .first() - .and_then(|group| group.frames.first()) - .ok_or_else(|| Error::Timeline("timeline object has no checkpoint".into()))?; - let mut decoder = decoder(); - decoder.group().decode(&frame.payload).map_err(json_error)?; - let offset = decoder.range().start; - let start = match decoder.next_event() { - Some(window::Event::Skip(skipped)) => skipped.end, - _ => offset, - }; - Ok((offset, start)) -} - -fn decoder() -> window::Decoder { - window::Decoder::new(window::ConsumerConfig::default().with_compression(true)) -} - -fn json_error(err: moq_json::Error) -> Error { - Error::Timeline(err.to_string()) -} diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 355f338c372..d3d7cee6809 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -167,12 +167,6 @@ impl Store { intended: info.timescale, }); } - if parsed.epoch != info.epoch { - return Err(Error::EpochMismatch { - existing: parsed.epoch, - intended: info.epoch.clone(), - }); - } Ok(key) } } @@ -198,15 +192,13 @@ impl Store { Object::decode(self.get_bytes(&path).await?, timed) } - /// Each recorded track's timeline track, found by listing the track directories a [`Writer`] - /// creates: a timeline is named by [`hang::timeline::default_name`]. One listing of the + /// Each recorded track's timeline track, found by listing the recording's track directories: + /// a timeline is named by [`hang::timeline::default_name`]. One listing of the /// prefix's directories plus a HEAD per timeline, so following a long recording doesn't list /// its objects. A timeline whose `.info` isn't stored yet, such as one still enrolling, is left /// out until it is. /// /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. - /// - /// [`Writer`]: crate::Writer pub async fn timelines(&self) -> Result> { let listed = self.inner.list_with_delimiter(self.list_path(None).as_ref()).await?; let mut timelines = std::collections::BTreeMap::new(); diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs deleted file mode 100644 index 013d5cc8e62..00000000000 --- a/rs/moq-archive/src/writer.rs +++ /dev/null @@ -1,2305 +0,0 @@ -//! Record explicitly selected tracks of a [`broadcast::Consumer`] into a [`Store`]. -//! -//! The application enrolls each track through a [`Control`]; the writer never parses a catalog. -//! Every track has its own timeline: frames feed the track's segmenter as they arrive, and for -//! each closed record the writer stores the track's object, commits the record through the -//! track's own timeline encoder, and stores that record's timeline groups before committing the -//! track's next one. A timeline therefore only advertises durable objects, and no track waits for -//! another. -//! -//! An untimed track, such as any track over moq-transport without TIMESCALE, is recorded untimed: -//! each group is its own record with no `pts` or `duration`, and its objects store no timestamps. -//! -//! A writer started on a prefix that already holds a recording resumes it: see [`Writer::new`]. -//! -//! ```no_run -//! # async fn example(source: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { -//! use moq_archive::object_store::memory::InMemory; -//! -//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); -//! let writer = moq_archive::Writer::new(store, source, Default::default()).await?; -//! let control = writer.control(); -//! let media = moq_mux::timeline::Config::default(); -//! control.track("video", media.clone()).await?; -//! control.track("audio", media.clone()).await?; -//! // Sparse data: each catalog group is its own record. -//! control.track("catalog.json", media.with_duration_min(std::time::Duration::ZERO)).await?; -//! writer.run().await?; -//! # Ok(()) -//! # } -//! ``` - -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::future::Future; -use std::ops::Range; -use std::sync::{Arc, Mutex}; -use std::task::Poll; -use std::time::Duration; - -use futures::future::BoxFuture; -use futures::stream::FuturesUnordered; -use futures::{FutureExt, StreamExt}; -use hang::timeline::{Position, Record}; -use moq_clock::Instant; -use moq_mux::timeline::{self, Segmenter}; -use moq_net::{Timescale, Timestamp, broadcast, group, track}; -use object_store::ObjectStore; -use tokio::sync::{mpsc, watch}; - -use crate::recover::{Resume, checkpoint, recover}; -use crate::segment::{Frame, Group, Object}; -use crate::{Error, Info, Key, Result, Store}; - -/// Subscribers ask for every cached group; the publisher clamps this to its own max age. -const REPLAY: Duration = Duration::from_secs(u32::MAX as u64); - -/// How a [`Writer`] retains its recording. Each track's cutting is chosen when it enrolls. -#[derive(Clone, Debug, Default)] -#[non_exhaustive] -pub struct Config { - /// Expire old records (a DVR), or keep everything when `None` (an archive). - pub retention: Option, -} - -impl Config { - /// Set [`retention`](Self::retention). - pub fn with_retention(mut self, retention: impl Into>) -> Self { - self.retention = retention.into(); - self - } -} - -/// A DVR retention policy. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct Retention { - /// Keep at least this much of each track's content, measured from its timeline records. - pub window: Duration, - /// How long expired objects outlive the timeline that stopped advertising them, so readers - /// holding an older timeline can finish their GETs. - pub grace: Duration, -} - -impl Retention { - /// Keep `window` of content and delete expired objects `grace` after they leave the timeline. - pub fn new(window: Duration, grace: Duration) -> Self { - Self { window, grace } - } -} - -/// Records enrolled tracks until the source broadcast closes. -/// -/// Enroll tracks through [`control`](Self::control), then drive the recording with -/// [`run`](Self::run). -pub struct Writer { - control: Control, - commands: mpsc::UnboundedReceiver>, - /// Deadlines for deleting expired or orphaned objects, oldest first. - deletions: VecDeque<(Instant, Expired)>, -} - -/// Enrolls, cuts, and removes tracks on a [`Writer`]. -pub struct Control { - shared: Arc>, - commands: mpsc::UnboundedSender>, -} - -impl Clone for Control { - fn clone(&self) -> Self { - Self { - shared: self.shared.clone(), - commands: self.commands.clone(), - } - } -} - -struct Shared { - store: Store, - source: broadcast::Consumer, - retention: Option, - /// Owns the recording's timeline tracks, read back to store their groups. - timelines: broadcast::Producer, - /// Each track's retained timeline from a resumed recording, taken when it enrolls again. - recovered: Mutex>, - /// Every name ever enrolled. A name is never reused, so its records stay consecutive. - enrolled: Mutex>, - /// Enrollments between their first check and their command, so the recording does not end - /// under one that is about to arrive. - enrolling: watch::Sender, -} - -/// One [`Control::track`] in flight: drops its count, and its name unless it enrolled, however -/// the call ends. -struct Enrolling<'a, S> { - shared: &'a Shared, - name: &'a str, - enrolled: bool, -} - -impl Drop for Enrolling<'_, S> { - fn drop(&mut self) { - self.shared.enrolling.send_modify(|count| *count -= 1); - if !self.enrolled { - self.shared.enrolled.lock().unwrap().remove(self.name); - } - } -} - -enum Command { - Enroll(Box>), - Cut(Timestamp), - Remove(String), - Stop, -} - -impl Writer { - /// Start a recording under `store`'s prefix, reading tracks from `source`, or resume the one - /// already there. - /// - /// Resuming replays each track's retained timeline, and a re-enrolled track continues after - /// its newest committed record, skipping what the source replays from before it. A source - /// announced under another epoch than the recording is a restart: enrolling its tracks fails - /// with [`Error::EpochMismatch`], and the caller starts a new prefix. Objects a crash left - /// before their commit are deleted now, since the resumed records reuse their keys. A DVR - /// also deletes, one grace period after recovery, every object its retained records do not - /// reference, such as interrupted expirations, and every timeline object older than the - /// checkpoint it recovered from. The writer must own the prefix exclusively. Fails, deleting - /// nothing, when the recording cannot be listed or a timeline cannot be replayed. - pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { - let recovery = recover(&store, config.retention.is_some()).await?; - for key in &recovery.uncommitted { - store.delete(key).await?; - } - - let mut deletions = VecDeque::new(); - let stale = Expired { - objects: recovery.orphans, - timelines: recovery.stale, - }; - if let Some(retention) = &config.retention - && !stale.is_empty() - { - deletions.push_back((Instant::now() + retention.grace, stale)); - } - - let shared = Arc::new(Shared { - store, - source, - retention: config.retention, - timelines: broadcast::Info::new().produce(), - recovered: Mutex::new(recovery.tracks), - enrolled: Mutex::new(HashSet::new()), - enrolling: watch::Sender::new(0), - }); - let (commands, receiver) = mpsc::unbounded_channel(); - Ok(Self { - control: Control { shared, commands }, - commands: receiver, - deletions, - }) - } - - /// A handle that enrolls, cuts, and removes tracks, before or during [`run`](Self::run). - pub fn control(&self) -> Control { - self.control.clone() - } - - /// Record until the source broadcast closes and every enrolled track ends, or [`Control::stop`]. - /// - /// Then flush each track's final record and finish its timeline. Fails when a timeline cannot - /// be committed or stored: the recording stops at each track's last durable timeline object. - /// Also fails when an enrolled track delivers a frame the recording cannot represent, but - /// only after stopping every track and committing what each already reported, so the - /// recording stays readable up to the bad frame. That includes a resumed track whose first - /// new group starts before the recorded end, the one restart a source without an epoch - /// shows. Returns the source's error, after finishing, when the broadcast aborted. - pub async fn run(self) -> Result<()> { - let Self { - control, - mut commands, - mut deletions, - } = self; - let shared = control.shared.clone(); - let source = shared.source.clone(); - let grace = shared.retention.as_ref().map(|retention| retention.grace); - let mut enrolling = shared.enrolling.subscribe(); - drop(control); - // Each timeline's oldest segment that may still be stored; pruning deletes upward from it. - let mut pruned = HashMap::new(); - - let mut tracks: HashMap>> = HashMap::new(); - let mut reads = FuturesUnordered::new(); - let mut commits: FuturesUnordered> = FuturesUnordered::new(); - let mut closed = false; - // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. - let mut accepting = true; - // Every track was stopped, by `Control::stop` or malformed input, so `run` ends once flushed. - let mut stopped = false; - // Malformed source input, returned once every track flushed. - let mut failed: Option = None; - // The tracks the last event touched. An event touches at most one, except a failure, so the - // loop never scans the whole table in steady state. - let mut touched: Vec = Vec::new(); - - loop { - for name in std::mem::take(&mut touched) { - let Some(track) = tracks.get_mut(&name) else { - continue; - }; - if let Some(commit) = track.commit()? { - commits.push(commit); - } - if track.finish() { - tracks.remove(&name); - } - } - - let idle = (closed || stopped) && tracks.is_empty() && commits.is_empty(); - if idle && *enrolling.borrow_and_update() == 0 { - // Refuse late commands, so an enrollment racing the end fails instead of vanishing. - commands.close(); - if !accepting { - break; - } - } - - let deadline = deletions.front().map(|(deadline, _)| *deadline); - tokio::select! { - biased; - command = commands.recv(), if accepting => match command { - None => accepting = false, - Some(Command::Enroll(mut track)) => { - let subscriber = track.subscriber.take().expect("an enrolling track carries its subscriber"); - // A stopped recording takes no new content; the track just finishes its timeline. - if stopped { - track.remove(); - } - reads.push(guard(track.cancelled.clone(), recv(track.name.clone(), subscriber)).boxed()); - touched.push(track.name.clone()); - tracks.insert(track.name.clone(), track); - } - Some(Command::Cut(pts)) => { - for track in tracks.values_mut() { - track.segmenter.cut(pts); - } - } - Some(Command::Remove(name)) => { - if let Some(track) = tracks.get_mut(&name) { - track.remove(); - touched.push(name); - } - } - Some(Command::Stop) => { - stopped = true; - touched.extend(stop(&mut tracks)); - commands.close(); - } - }, - Some(read) = reads.next(), if !reads.is_empty() => match handle(read, &mut tracks, &mut reads) { - Ok(name) => touched.extend(name), - Err(err) => { - tracing::error!(%err, "malformed source input; flushing every track and stopping"); - failed.get_or_insert(err); - stopped = true; - touched.extend(stop(&mut tracks)); - commands.close(); - } - }, - Some((name, committer, result)) = commits.next(), if !commits.is_empty() => { - if let Some(track) = tracks.get_mut(&name) { - track.committer = Some(committer); - touched.push(name); - } - let expired = result?; - if let Some(grace) = grace && !expired.is_empty() { - deletions.push_back((Instant::now() + grace, expired)); - } - } - _ = source.closed(), if !closed => closed = true, - // Also once commands closed: an enrollment in flight still settles its count. - _ = enrolling.changed(), if idle => {} - _ = moq_clock::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { - let (_, expired) = deletions.pop_front().unwrap(); - delete(&shared.store, &mut pruned, expired).await; - } - } - } - - // Complete expirations already committed to the timelines; nothing new expires after the - // final records. - for (deadline, expired) in deletions { - moq_clock::sleep_until(deadline).await; - delete(&shared.store, &mut pruned, expired).await; - } - - if let Some(err) = failed { - return Err(err); - } - if stopped { - return Ok(()); - } - // A broadcast end carries no cause. - source.closed().await; - Ok(()) - } -} - -impl Control { - /// Enroll the track `name`, its records cut by `config`. - /// - /// Media keeps the default minimum; sparse data such as a catalog uses zero, so each group is - /// stored as soon as it finishes. An untimed track ignores `config`: each group is its own - /// record, stored once it finishes. Subscribes to the track and creates its `.info` and its - /// timeline's before accepting any group. Fails when the name was already enrolled, names a - /// timeline, or `.info` conflicts, and with [`Error::Untimed`] when a DVR enrolls an untimed - /// track, whose retention it cannot measure. - pub async fn track(&self, name: &str, config: timeline::Config) -> Result<()> { - if name.ends_with(hang::timeline::SUFFIX) || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { - return Err(Error::Enrolled(name.to_string())); - } - self.shared.enrolling.send_modify(|count| *count += 1); - // Settles the count and the name even if this future is dropped mid-enrollment. - let mut enrolling = Enrolling { - shared: &self.shared, - name, - enrolled: false, - }; - self.subscribe(name, config).await?; - enrolling.enrolled = true; - Ok(()) - } - - /// Stop recording every track: store what already arrived, finish each timeline, and end - /// [`run`](Writer::run) without waiting for the source. Later commands are refused. - pub fn stop(&self) -> Result<()> { - self.send(Command::Stop) - } - - /// Declare a boundary at `pts` on every enrolled track; see [`Segmenter::cut`]. - pub fn cut(&self, pts: Timestamp) -> Result<()> { - self.send(Command::Cut(pts)) - } - - /// Stop recording `name`, storing what already arrived and dropping groups that could not be - /// indexed yet. The name cannot be enrolled again. - pub fn remove(&self, name: &str) -> Result<()> { - self.send(Command::Remove(name.to_string())) - } - - async fn subscribe(&self, name: &str, config: timeline::Config) -> Result<()> { - let shared = &self.shared; - let replay = track::Subscription::default().with_max_delay(REPLAY); - let subscriber = shared - .source - .track(name) - .map_err(source_error)? - .subscribe(replay.clone()) - .await - .map_err(source_error)?; - let timescale = subscriber.info().timescale; - if timescale.is_none() && shared.retention.is_some() { - return Err(Error::Untimed(name.to_string())); - } - let info = Info::new(subscriber.info().priority, timescale.map(Timescale::as_u64))? - .with_epoch(shared.source.info().epoch.clone()); - shared.store.put_info(name, &info).await?; - - let timeline = hang::timeline::default_name(name); - let output = shared - .timelines - .create_track(timeline.as_str(), timeline::Producer::info()) - .map_err(timeline_error_net)?; - let groups = shared - .timelines - .consume() - .track(&timeline) - .map_err(timeline_error_net)? - .subscribe(replay) - .await - .map_err(timeline_error_net)? - .ordered(); - // The stored timeline is timed exactly when its track is: an untimed track's records have no - // content time to stamp their timeline frames with. - let timeline_scale = timescale.and(groups.info().timescale); - let info = Info::new(groups.info().priority, timeline_scale.map(Timescale::as_u64))?; - shared.store.put_info(&timeline, &info).await?; - - let resume = shared.recovered.lock().unwrap().remove(name); - let (output, segmenter, window, checkpoints, offset, floor, recorded) = match resume { - Some(resume) => { - let floor = resume.floor(); - let recorded = recorded_end(&resume, timescale)?; - let output = timeline::Producer::resume(output, &resume.checkpoint).map_err(timeline_error)?; - let segmenter = Segmenter::new(config).with_sequence(resume.checkpoint.range.end); - let window = resume.checkpoint.records.into(); - ( - output, - segmenter, - window, - resume.checkpoints, - resume.sequence, - floor, - recorded, - ) - } - None => ( - timeline::Producer::new(output), - Segmenter::new(config), - VecDeque::new(), - VecDeque::new(), - 0, - None, - None, - ), - }; - - let (cancel, cancelled) = watch::channel(()); - let track = Track { - name: name.to_string(), - timescale, - segmenter, - floor, - largest: None, - // A new group must not start before the recorded end, or it would overlap it. - reported: recorded, - accepted: BTreeMap::new(), - frames: VecDeque::new(), - records: VecDeque::new(), - subscriber: Some(Box::new(subscriber)), - subscribed: true, - closed: false, - committer: Some(Committer { - shared: shared.clone(), - name: name.to_string(), - timeline, - output, - groups, - window, - checkpoints, - offset, - }), - cancel: Some(cancel), - cancelled, - }; - self.send(Command::Enroll(Box::new(track))) - } - - fn send(&self, command: Command) -> Result<()> { - self.commands.send(command).map_err(|_| Error::Closed) - } -} - -/// A group accepted from the source, not yet fully reported. -#[derive(Default)] -struct Incoming { - /// The index of the next frame to arrive. - next: u64, - /// Frames that arrived but were not reported, with their indices. - pending: Vec<(u64, Frame)>, - /// No more frames will arrive. - finished: bool, -} - -/// One enrolled track: its source reads, its segmenter, and its committer. -struct Track { - name: String, - /// `None` when the track is untimed. - timescale: Option, - segmenter: Segmenter, - /// A resumed track's committed end; earlier content is already stored. - floor: Option, - /// The newest accepted group; later groups must exceed it. - largest: Option, - /// The first-frame timestamp of the newest reported group; later groups must not precede it. - /// A resumed track starts at the recorded end. - reported: Option, - /// Accepted groups in sequence order. Reported front first, so a later group's frames wait - /// for every earlier accepted group to finish. - accepted: BTreeMap, - /// Reported frames waiting for the record that holds them. - frames: VecDeque<(Position, Frame)>, - /// Closed records waiting to be committed, oldest first. - records: VecDeque, - /// Moved into the first read once enrolled. - subscriber: Option>, - /// The subscription is still delivering groups. - subscribed: bool, - /// The segmenter was closed: every remaining record is queued. - closed: bool, - /// `None` while a commit is in flight. - committer: Option>, - /// Dropping the sender cancels every read for this track. - cancel: Option>, - cancelled: watch::Receiver<()>, -} - -impl Track { - /// Report every frame the segmenter can place: the front group's arrivals, then each later - /// group once the ones before it finish. - /// - /// Fails on a group that starts before the previous one, which the timeline cannot place. - fn report(&mut self) -> Result<()> { - while let Some(mut entry) = self.accepted.first_entry() { - let sequence = *entry.key(); - let incoming = entry.get_mut(); - for (index, frame) in incoming.pending.drain(..) { - if index == 0 - && let Some(timestamp) = frame.timestamp - { - if let Some(reported) = self.reported - && timestamp < reported - { - return Err(malformed( - &self.name, - sequence, - format!("timestamp {timestamp} precedes {reported}"), - )); - } - self.reported = Some(timestamp); - } - // Frame timestamps were validated against the track while reading, so this conversion - // succeeds, and an untimed track reports no timestamps. - let pts = frame - .timestamp - .zip(self.timescale) - .map(|(timestamp, timescale)| { - Timestamp::new(timestamp, timescale).map_err(|_| Error::Id(timestamp)) - }) - .transpose()?; - let position = Position::new(sequence, index); - self.segmenter - .frame(position, pts, index == 0) - .map_err(|err| malformed(&self.name, sequence, err))?; - self.frames.push_back((position, frame)); - } - if !incoming.finished { - break; - } - entry.remove(); - self.segmenter.finish_group(sequence); - } - Ok(()) - } - - /// Stop reading: drop the groups that were never reported and close the segmenter. - fn remove(&mut self) { - self.cancel = None; - self.subscribed = false; - self.accepted.clear(); - } - - /// Start committing the next closed record, if the committer is idle. - fn commit(&mut self) -> Result>> { - if !self.subscribed && self.accepted.is_empty() && !self.closed { - self.segmenter.close(); - self.closed = true; - } - self.records.extend(self.segmenter.by_ref()); - if self.committer.is_none() { - return Ok(None); - } - let Some(record) = self.records.pop_front() else { - return Ok(None); - }; - - let mut groups: Vec = Vec::new(); - while let Some((position, _)) = self.frames.front() - && *position < record.end - { - let (position, frame) = self.frames.pop_front().expect("checked above"); - match groups.last_mut() { - Some(group) if group.sequence == position.group => group.frames.push(frame), - _ => groups.push(Group { - sequence: position.group, - frames: vec![frame], - }), - } - } - let object = Object { - frame_start: record.start.frame, - groups, - }; - object.check_span(record.start, record.end)?; - - let committer = self.committer.take().expect("checked above"); - Ok(Some(committer.commit(record, object).boxed())) - } - - /// Finish the timeline once the track ended and every record is durable, returning whether the - /// track is done. - fn finish(&mut self) -> bool { - if !self.closed || !self.records.is_empty() { - return false; - } - let Some(committer) = self.committer.as_mut() else { - return false; - }; - if let Err(err) = committer.output.finish() { - tracing::warn!(track = %self.name, %err, "failed to finish the timeline"); - } - true - } -} - -// Each read is already boxed as a future, so the variant sizes cost nothing extra. -#[allow(clippy::large_enum_variant)] -enum Read { - Group { - name: String, - subscriber: Box, - result: moq_net::Result>, - }, - Frame { - name: String, - group: Box, - result: moq_net::Result>, - }, - Cancelled, -} - -async fn recv(name: String, mut subscriber: Box) -> Read { - let result = subscriber.recv_group().await; - Read::Group { - name, - subscriber, - result, - } -} - -async fn frame(name: String, mut group: Box) -> Read { - let result = group.read_frame().await; - Read::Frame { name, group, result } -} - -/// Stop reading every track, returning their names so each flushes and finishes. -fn stop(tracks: &mut HashMap>>) -> Vec { - tracks - .iter_mut() - .map(|(name, track)| { - track.remove(); - name.clone() - }) - .collect() -} - -/// Resolve to [`Read::Cancelled`] once the track stops reading. -async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { - tokio::select! { - read = read => read, - _ = cancelled.changed() => Read::Cancelled, - } -} - -/// Returns the track that may have records to commit or be finished. -/// -/// Fails on malformed source input; a group the network aborted keeps the frames that arrived. -fn handle( - read: Read, - tracks: &mut HashMap>>, - reads: &mut FuturesUnordered>, -) -> Result> { - match read { - Read::Cancelled => Ok(None), - Read::Group { - name, - subscriber, - result, - } => { - let Some(track) = tracks.get_mut(&name) else { - return Ok(None); - }; - match result { - Ok(Some(group)) => { - let sequence = group.sequence; - let stored = track.floor.is_some_and(|floor| Position::group(sequence + 1) <= floor); - if track.largest.is_some_and(|largest| sequence <= largest) || stored { - tracing::debug!(track = %name, sequence, "refusing a duplicate, decreasing, or stored group"); - } else { - crate::path::check_id(sequence).map_err(|err| malformed(&name, sequence, err))?; - track.largest = Some(sequence); - track.accepted.insert(sequence, Incoming::default()); - reads.push(guard(track.cancelled.clone(), frame(name.clone(), Box::new(group))).boxed()); - } - reads.push(guard(track.cancelled.clone(), recv(name, subscriber)).boxed()); - // An accepted group holds no frames yet. - Ok(None) - } - Ok(None) => { - track.subscribed = false; - Ok(Some(name)) - } - Err(err) => { - tracing::warn!(track = %name, %err, "track ended"); - track.subscribed = false; - Ok(Some(name)) - } - } - } - Read::Frame { name, group, result } => { - let Some(track) = tracks.get_mut(&name) else { - return Ok(None); - }; - let sequence = group.sequence; - let Some(incoming) = track.accepted.get_mut(&sequence) else { - return Ok(None); - }; - let touched = name.clone(); - match result { - Ok(Some(frame)) => { - let index = incoming.next; - incoming.next += 1; - // A track is all timed or all untimed. - let timestamp = match (track.timescale, frame.timestamp) { - (Some(timescale), Some(timestamp)) => { - let timestamp = timestamp - .convert(timescale) - .map_err(|_| malformed(&name, sequence, Error::Overflow))? - .value(); - Some(crate::path::check_id(timestamp).map_err(|err| malformed(&name, sequence, err))?) - } - (None, None) => None, - _ => return Err(malformed(&name, sequence, Error::Timing)), - }; - // A resumed track already stored the head of the group it stopped inside. - if track.floor.is_none_or(|floor| Position::new(sequence, index) >= floor) { - let frame = Frame { - timestamp, - payload: frame.payload, - }; - incoming.pending.push((index, frame)); - } - reads.push(guard(track.cancelled.clone(), self::frame(name, group)).boxed()); - } - Ok(None) => incoming.finished = true, - Err(err) => { - tracing::warn!(track = %name, sequence, %err, "keeping the frames of an incomplete group"); - incoming.finished = true; - } - } - track.report()?; - Ok(Some(touched)) - } - } -} - -/// An in-flight record commit, returning the track, its committer, and the expired objects. -type Commit = BoxFuture<'static, (String, Committer, Result)>; - -/// Objects the timelines stopped needing, deleted once the grace elapses. -#[derive(Default)] -struct Expired { - /// Media objects no retained record references. - objects: Vec, - /// Timeline segments no recovery reads, by timeline. - timelines: Vec<(String, Range)>, -} - -impl Expired { - fn is_empty(&self) -> bool { - self.objects.is_empty() && self.timelines.is_empty() - } -} - -/// Commits one track's records in order: the object, the record, retention, then the timeline -/// object. -struct Committer { - shared: Arc>, - name: String, - /// The track's timeline track. - timeline: String, - output: timeline::Producer, - /// The timeline track read back, to store its complete groups. - groups: track::Ordered, - /// Committed records still in the timeline window, oldest first. - window: VecDeque, - /// A DVR's stored timeline segments that recovery still reads, oldest first, each with the - /// first record index its opening checkpoint restates. - checkpoints: VecDeque<(u64, u64)>, - /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. - offset: u64, -} - -impl Committer { - /// Commit `record` with its `object`. - async fn commit(mut self, record: Record, object: Object) -> (String, Self, Result) { - let result = self.commit_inner(record, object).await; - (self.name.clone(), self, result) - } - - async fn commit_inner(&mut self, mut record: Record, object: Object) -> Result { - let shared = self.shared.clone(); - let store = &shared.store; - - // Number by the window rather than the segmenter, so a record dropped below leaves no hole. - record.sequence = self.output.range().end; - if let Err(err) = store.put_segments(&self.name, record.sequence, &object).await { - // The timeline only advertises durable objects, so this span is lost. The key may have - // landed anyway; clear it so the next record can take it. - tracing::warn!(track = %self.name, sequence = record.sequence, %err, "dropping a record that was not stored"); - let _ = store.delete(&Key::segments(self.name.clone(), record.sequence)?).await; - return Ok(Expired::default()); - } - - self.output.push(&record).map_err(timeline_error)?; - let pts = record - .pts - .map(|pts| Timestamp::new(pts, TIMESCALE).map_err(|_| Error::Id(pts))) - .transpose()?; - self.window.push_back(record.clone()); - - let expired = self.expire(); - if !expired.is_empty() { - self.output.pop(expired.len() as u64).map_err(timeline_error)?; - } - self.output.flush().map_err(timeline_error)?; - - let object = self.read_timeline(pts)?; - store.put_segments(&self.timeline, record.sequence, &object).await?; - - let objects = expired - .iter() - .map(|record| Key::segments(self.name.clone(), record.sequence)) - .collect::>()?; - let segments = self.prune(record.sequence, &object)?; - let timelines = match segments.is_empty() { - true => Vec::new(), - false => vec![(self.timeline.clone(), segments)], - }; - Ok(Expired { objects, timelines }) - } - - /// Track the newly stored timeline `object`, returning the older segments no recovery reads. - /// - /// Recovery walks back from the newest object to the first whose checkpoint restates the - /// newest one's offset, so every object before that one is no longer needed. - fn prune(&mut self, segment: u64, object: &Object) -> Result> { - if self.shared.retention.is_none() { - return Ok(segment..segment); - } - let (offset, start) = checkpoint(object)?; - self.checkpoints.push_back((segment, start)); - // Checkpoint starts never decrease, so a binary search finds the last one at or before - // `offset` without walking a long DVR window on every commit. - let keep = self - .checkpoints - .partition_point(|&(_, start)| start <= offset) - .saturating_sub(1); - let first = self.checkpoints[0].0; - self.checkpoints.drain(..keep); - Ok(first..self.checkpoints[0].0) - } - - /// Pop the oldest records while the rest still cover the retention window. The newest record - /// always stays, such as a catalog that never changes. - fn expire(&mut self) -> Vec { - let Some(retention) = self.shared.retention.as_ref().map(|retention| retention.window) else { - return Vec::new(); - }; - let target = (retention.as_micros() * TIMESCALE.as_u64() as u128).div_ceil(1_000_000); - let mut expired = Vec::new(); - while self.window.len() > 1 { - // A DVR refuses untimed tracks, so every record here is timed. - let newest = self.window.back().expect("non-empty"); - let end = newest - .pts - .zip(newest.duration) - .map(|(pts, duration)| pts as u128 + duration as u128); - let covered = end - .zip(self.window[1].pts) - .map(|(end, pts)| end.saturating_sub(pts as u128)); - if covered.is_none_or(|covered| covered < target) { - break; - } - expired.push(self.window.pop_front().expect("non-empty")); - } - expired - } - - /// Collect the timeline groups completed since the last record, stamped at `pts`, or unstamped - /// for an untimed track. - /// - /// The live timeline track stamps frames with the wall clock; storing the record's content time - /// instead keeps a recording's bytes a function of its content alone. - fn read_timeline(&mut self, pts: Option) -> Result { - let waiter = kio::Waiter::noop(); - let timestamp = match pts { - Some(pts) => { - let timescale = self - .groups - .info() - .timescale - .ok_or_else(|| Error::Timeline("untimed timeline track".to_string()))?; - let timestamp = pts.convert(timescale).map_err(|_| Error::Overflow)?.value(); - Some(crate::path::check_id(timestamp)?) - } - None => None, - }; - let mut groups = Vec::new(); - while let Poll::Ready(result) = self.groups.poll_next_group(&waiter) { - let Some(mut group) = result.map_err(timeline_error_net)? else { - break; - }; - let mut frames = Vec::new(); - loop { - match group.poll_read_frame(&waiter) { - Poll::Ready(Ok(Some(frame))) => frames.push(Frame { - timestamp, - payload: frame.payload, - }), - Poll::Ready(Ok(None)) => break, - Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), - // Flushing closed every group, so an open one is a bug. - Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), - } - } - let sequence = group.sequence.checked_add(self.offset).ok_or(Error::Overflow)?; - groups.push(Group { - sequence: crate::path::check_id(sequence)?, - frames, - }); - } - Ok(Object::new(groups)) - } -} - -/// The timescale of record `pts` and `duration`, as the catalog section advertises it. -const TIMESCALE: Timescale = Timescale::MILLI; - -/// Delete `expired`'s media objects, and each timeline's objects from its `pruned` cursor up to -/// its expired segments' end. -/// -/// Timeline objects go oldest first and stop at the first failure, so the stored segments stay -/// consecutive, as recovery requires, and a later deletion retries from there. -async fn delete(store: &Store, pruned: &mut HashMap, expired: Expired) { - let objects = async { - let deletes = expired - .objects - .iter() - .map(|key| async move { (key, store.delete(key).await) }); - for (key, result) in futures::future::join_all(deletes).await { - // An object that outlives its expiry advertises nothing; recovery cleans it up. - if let Err(err) = result { - tracing::warn!(?key, %err, "failed to delete an expired object"); - } - } - }; - let timelines = async { - for (timeline, segments) in expired.timelines { - let cursor = pruned.entry(timeline.clone()).or_insert(segments.start); - while *cursor < segments.end { - let result = match Key::segments(timeline.clone(), *cursor) { - Ok(key) => store.delete(&key).await, - Err(err) => Err(err), - }; - match result { - Ok(()) | Err(Error::NotFound(_)) => *cursor += 1, - Err(err) => { - tracing::warn!(%timeline, segment = *cursor, %err, "failed to delete a pruned timeline object"); - break; - } - } - } - } - }; - futures::join!(objects, timelines); -} - -fn timeline_error(err: moq_mux::Error) -> Error { - Error::Timeline(err.to_string()) -} - -fn timeline_error_net(err: moq_net::Error) -> Error { - Error::Timeline(err.to_string()) -} - -fn source_error(err: moq_net::Error) -> Error { - Error::Source(err.to_string()) -} - -fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { - Error::Source(format!("track {track} group {sequence}: {err}")) -} - -/// The content time the newest committed record reached, in `timescale`, or `None` when nothing -/// was recorded or the track is untimed. -/// -/// Fails when any retained record's timing disagrees with its track's, so a resume never extends a -/// mixed timeline. -fn recorded_end(resume: &Resume, timescale: Option) -> Result> { - let records = &resume.checkpoint.records; - if let Some(record) = records - .iter() - .find(|record| record.pts.is_some() != timescale.is_some() || record.duration.is_some() != timescale.is_some()) - { - return Err(Error::Timeline(format!( - "record {} timing does not match its track", - record.sequence - ))); - } - let (Some(record), Some(timescale)) = (records.last(), timescale) else { - return Ok(None); - }; - let (Some(pts), Some(duration)) = (record.pts, record.duration) else { - unreachable!("checked above"); - }; - let millis = pts.checked_add(duration).ok_or(Error::Overflow)?; - let end = Timestamp::new(millis, TIMESCALE).map_err(|_| Error::Id(millis))?; - Ok(Some(end.convert(timescale).map_err(|_| Error::Overflow)?.value())) -} - -#[cfg(test)] -#[allow(clippy::disallowed_methods, clippy::disallowed_types)] -mod tests { - use futures::TryStreamExt; - use object_store::memory::InMemory; - - use super::*; - use crate::mock::Mock; - use crate::store::list::Query; - - fn ms(v: u64) -> Timestamp { - Timestamp::from_millis(v).unwrap() - } - - /// Media cut at 1s, so one-second test groups get a record each. - fn media() -> timeline::Config { - timeline::Config::default().with_duration_min(Duration::from_secs(1)) - } - - /// Sparse data: every group is its own record. - fn sparse() -> timeline::Config { - timeline::Config::default().with_duration_min(Duration::ZERO) - } - - /// An append-only log with a three-second nominal duration and nine-second safety ceiling. - fn append_only() -> timeline::Config { - sparse().with_duration_hint(Duration::from_secs(3)) - } - - fn timeline(track: &str) -> String { - hang::timeline::default_name(track) - } - - /// A millisecond track that keeps an hour of history, so enrollment sees every group. - fn track(broadcast: &broadcast::Producer, name: &str) -> track::Producer { - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - broadcast.create_track(name, info).unwrap() - } - - /// Write one complete group with a frame at each timestamp. - fn group(track: &track::Producer, sequence: u64, timestamps: &[u64]) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for ×tamp in timestamps { - group - .write_frame(ms(timestamp), format!("{sequence}@{timestamp}")) - .unwrap(); - } - group.finish().unwrap(); - } - - /// The stored timeline segments of `track`, ascending. - async fn segments(store: &Store, track: &str) -> Vec { - let entries: Vec<_> = store - .list(&Query::segments(&timeline(track)).unwrap()) - .try_collect() - .await - .unwrap(); - let mut segments: Vec = entries - .into_iter() - .map(|entry| match entry.key { - Key::Segments { segment, .. } => segment, - key => panic!("unexpected {key:?}"), - }) - .collect(); - segments.sort(); - segments - } - - /// Replay every stored timeline object of `track`, returning the records still in its window. - async fn window(store: &Store, track: &str) -> Vec { - let timeline = timeline(track); - let segments = segments(store, track).await; - if let (Some(first), Some(last)) = (segments.first(), segments.last()) { - assert_eq!( - segments, - (*first..=*last).collect::>(), - "timeline objects are consecutive" - ); - } - - let config = moq_json::window::ConsumerConfig::default().with_compression(true); - let mut decoder = moq_json::window::Decoder::::new(config); - let mut records = BTreeMap::new(); - for segment in segments { - let object = store - .get_segments(&timeline, segment, timed(store, &timeline).await) - .await - .unwrap(); - for stored in object.groups { - let mut group = decoder.group(); - for frame in stored.frames { - group.decode(&frame.payload).unwrap(); - } - } - while let Some(event) = decoder.next_event() { - match event { - moq_json::window::Event::Push { index, value } => { - assert_eq!(index, value.sequence); - records.insert(index, value); - } - moq_json::window::Event::Pop(range) | moq_json::window::Event::Skip(range) => { - for index in range { - records.remove(&index); - } - } - _ => unreachable!(), - } - } - } - records.into_values().collect() - } - - /// Each record's groups, as `(first, last)`. - fn groups(records: &[Record]) -> Vec<(u64, u64)> { - records - .iter() - .map(|record| (*record.groups().start(), *record.groups().end())) - .collect() - } - - fn sequences(records: &[Record]) -> Vec { - records.iter().map(|record| record.sequence).collect() - } - - /// Whether `track`'s stored `.info` declares a timescale. - async fn timed(store: &Store, track: &str) -> bool { - store.get_info(track).await.unwrap().timescale.is_some() - } - - /// Every record resolves to one stored object holding exactly its frames. - async fn check_objects(store: &Store, track: &str, records: &[Record]) { - let timed = timed(store, track).await; - for record in records { - let object = store.get_segments(track, record.sequence, timed).await.unwrap(); - object.check_span(record.start, record.end).unwrap(); - } - } - - /// Every stored media object, across all tracks. - async fn stored(store: &Store) -> HashSet { - store - .list(&Query::new()) - .try_filter_map(|entry| async move { - let media = - matches!(&entry.key, Key::Segments { track, .. } if !track.ends_with(hang::timeline::SUFFIX)); - Ok(media.then_some(entry.key)) - }) - .try_collect() - .await - .unwrap() - } - - /// The media objects the retained records reference. - fn referenced(track: &str, records: &[Record]) -> HashSet { - records - .iter() - .map(|record| Key::segments(track, record.sequence).unwrap()) - .collect() - } - - #[tokio::test] - async fn records_selected_tracks() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let catalog = track(&source, "catalog.json"); - let _ignored = track(&source, "ignored"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - control.track("catalog.json", sparse()).await.unwrap(); - - group(&catalog, 0, &[0]); - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - catalog.finish().unwrap(); - source.close(); - - tokio::spawn(writer.run()).await.unwrap().unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - let catalog = window(&store, "catalog.json").await; - assert_eq!(groups(&catalog), vec![(0, 0)]); - check_objects(&store, "catalog.json", &catalog).await; - assert!(window(&store, "ignored").await.is_empty()); - - let object = store.get_segments("video", 2, true).await.unwrap(); - assert_eq!(object.groups[0].frames[1].timestamp, Some(2500)); - assert_eq!(object.groups[0].frames[1].payload, "2@2500"); - assert_eq!( - store.get_info("video").await.unwrap(), - Info::new(127, Some(1000)).unwrap() - ); - store.get_info(&timeline("video")).await.unwrap(); - assert!(store.get_info("ignored").await.is_err()); - } - - /// An untimed track that keeps an hour of history, so enrollment sees every group. - fn untimed(broadcast: &broadcast::Producer, name: &str) -> track::Producer { - let info = track::Info::default() - .with_timescale(None) - .with_max_age(Duration::from_secs(3600)); - broadcast.create_track(name, info).unwrap() - } - - /// Write one complete untimed group of `frames` frames. - fn untimed_group(track: &track::Producer, sequence: u64, frames: u64) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for index in 0..frames { - group.write_frame(None, format!("{sequence}/{index}")).unwrap(); - } - group.finish().unwrap(); - } - - /// An untimed track records each group as its own record, with no timing anywhere. - #[tokio::test] - async fn an_untimed_track_is_recorded_untimed() { - let source = broadcast::Info::new().produce(); - let data = untimed(&source, "data"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - // The minimum measures time, so it cannot pack untimed groups. - writer.control().track("data", media()).await.unwrap(); - - untimed_group(&data, 0, 2); - untimed_group(&data, 1, 1); - untimed_group(&data, 3, 3); - data.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let records = window(&store, "data").await; - assert_eq!( - records, - vec![ - Record::untimed(0, Position::group(0), Position::group(1)), - Record::untimed(1, Position::group(1), Position::group(2)), - Record::untimed(2, Position::group(3), Position::group(4)), - ] - ); - check_objects(&store, "data", &records).await; - let object = store.get_segments("data", 2, false).await.unwrap(); - assert!(object.groups[0].frames.iter().all(|frame| frame.timestamp.is_none())); - assert_eq!(object.groups[0].frames[2].payload, "3/2"); - - assert_eq!(store.get_info("data").await.unwrap(), Info::new(127, None).unwrap()); - let timeline = timeline("data"); - assert_eq!(store.get_info(&timeline).await.unwrap().timescale, None); - let stored = store.get_segments(&timeline, 0, false).await.unwrap(); - assert!(stored.groups[0].frames.iter().all(|frame| frame.timestamp.is_none())); - } - - /// A restarted writer resumes an untimed track after its newest record. - #[tokio::test] - async fn an_untimed_track_resumes() { - let store = Store::new(InMemory::new(), "rec"); - for groups in [0..2, 0..4] { - let source = broadcast::Info::new().produce(); - let data = untimed(&source, "data"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("data", sparse()).await.unwrap(); - // The source's cache replays groups the recording already holds. - for sequence in groups { - untimed_group(&data, sequence, 1); - } - data.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - } - - let records = window(&store, "data").await; - assert_eq!(sequences(&records), (0..4).collect::>()); - assert_eq!(groups(&records), (0..4).map(|s| (s, s)).collect::>()); - check_objects(&store, "data", &records).await; - } - - /// A resume refuses a retained record whose timing disagrees with its track's, at any age. - #[test] - fn a_resume_refuses_mixed_timing() { - let resume = |records: Vec| Resume { - checkpoint: moq_json::window::Checkpoint { - range: 0..records.len() as u64, - records, - }, - sequence: 0, - checkpoints: VecDeque::new(), - }; - let timed = Record::new(1, 1_000, 1_000, Position::group(1), Position::group(2)); - let untimed = Record::untimed(0, Position::group(0), Position::group(1)); - - let mixed = resume(vec![untimed.clone(), timed.clone()]); - assert!(matches!( - recorded_end(&mixed, Some(Timescale::MILLI)), - Err(Error::Timeline(_)) - )); - assert!(matches!(recorded_end(&mixed, None), Err(Error::Timeline(_)))); - - assert_eq!(recorded_end(&resume(vec![untimed]), None).unwrap(), None); - assert_eq!( - recorded_end(&resume(vec![timed]), Some(Timescale::MILLI)).unwrap(), - Some(2_000) - ); - } - - /// A DVR measures retention in content time, which an untimed track has none of. - #[tokio::test] - async fn a_dvr_refuses_an_untimed_track() { - let source = broadcast::Info::new().produce(); - let _data = untimed(&source, "data"); - - let store = Store::new(InMemory::new(), "rec"); - let retention = Retention::new(Duration::from_secs(30), Duration::ZERO); - let config = Config::default().with_retention(retention); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - assert_eq!( - writer.control().track("data", sparse()).await, - Err(Error::Untimed("data".into())) - ); - assert!(store.get_info("data").await.is_err(), "nothing is recorded"); - } - - #[tokio::test] - async fn tracks_cut_independently() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let audio = track(&source, "audio"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", timeline::Config::default()).await.unwrap(); - control.track("audio", timeline::Config::default()).await.unwrap(); - - // Two second GOPs, 400ms audio groups: with the default 4s minimum, audio packs by its own - // boundaries, not video's GOPs. - for sequence in 0..3 { - group(&video, sequence, &[sequence * 2000, sequence * 2000 + 1000]); - } - for sequence in 0..15 { - group(&audio, sequence, &[sequence * 400]); - } - video.finish().unwrap(); - audio.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let video = window(&store, "video").await; - assert_eq!(groups(&video), vec![(0, 1), (2, 2)]); - let audio = window(&store, "audio").await; - assert_eq!(groups(&audio), vec![(0, 9), (10, 14)]); - check_objects(&store, "audio", &audio).await; - } - - #[tokio::test] - async fn an_append_only_group_spans_several_objects() { - let source = broadcast::Info::new().produce(); - let log = track(&source, "log"); - - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default(); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("log", append_only()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - // One group that never closes, a frame a second. - let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); - for second in 0..19 { - open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); - } - // The group is still open, yet its first eighteen seconds are already durable. - while window(&store, "log").await.len() < 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - let records = window(&store, "log").await; - assert_eq!( - records - .iter() - .map(|record| (record.start, record.end)) - .collect::>(), - vec![ - (Position::new(0, 0), Position::new(0, 9)), - (Position::new(0, 9), Position::new(0, 18)), - ] - ); - check_objects(&store, "log", &records).await; - assert_eq!(store.get_segments("log", 1, true).await.unwrap().frame_start, 9); - - open.finish().unwrap(); - log.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); - - let records = window(&store, "log").await; - assert_eq!(records.last().unwrap().start, Position::new(0, 18)); - assert_eq!(records.last().unwrap().end, Position::group(1)); - check_objects(&store, "log", &records).await; - } - - #[tokio::test] - async fn a_failed_put_omits_only_that_track() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let audio = track(&source, "audio"); - - let mock = Mock::memory(); - mock.fail_puts("/audio/segments/"); - let store = Store::new(mock, "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - control.track("audio", media()).await.unwrap(); - - for sequence in 0..3 { - group(&video, sequence, &[sequence * 1000]); - group(&audio, sequence, &[sequence * 1000]); - } - video.finish().unwrap(); - audio.finish().unwrap(); - source.close(); - - writer.run().await.unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); - check_objects(&store, "video", &records).await; - assert!(window(&store, "audio").await.is_empty()); - } - - #[tokio::test] - async fn retention_pops_then_deletes_expired_objects() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let catalog = track(&source, "catalog.json"); - - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - writer.control().track("catalog.json", sparse()).await.unwrap(); - - // A static catalog published once, then six seconds of video. - group(&catalog, 0, &[0]); - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - catalog.finish().unwrap(); - source.close(); - - writer.run().await.unwrap(); - - // Records 3 and 4 hold two seconds; the partial final record 5 is always kept. - let records = window(&store, "video").await; - assert_eq!(sequences(&records), vec![3, 4, 5]); - check_objects(&store, "video", &records).await; - - // The catalog's newest record outlives the video it was published with. - let catalog = window(&store, "catalog.json").await; - assert_eq!(sequences(&catalog), vec![0]); - check_objects(&store, "catalog.json", &catalog).await; - - let expected = &referenced("video", &records) | &referenced("catalog.json", &catalog); - assert_eq!(stored(&store).await, expected); - // Every checkpoint restates the whole window, so only the newest timeline object is needed. - assert_eq!(segments(&store, "video").await, vec![5]); - assert_eq!(segments(&store, "catalog.json").await, vec![0]); - } - - #[tokio::test] - async fn commands_are_refused_once_the_recording_ends() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - // A long grace keeps `run` waiting on deletions after the timeline finishes. - let config = - Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000]); - } - video.finish().unwrap(); - source.close(); - - let run = tokio::spawn(writer.run()); - while window(&store, "video").await.last().map(|record| record.sequence) != Some(5) { - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!run.is_finished()); - assert_eq!(control.remove("video"), Err(Error::Closed)); - run.abort(); - } - - #[tokio::test] - async fn accepted_groups_may_complete_out_of_order() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - let mut first = video.create_group(group::Info { sequence: 0 }).unwrap(); - first.write_frame(ms(0), "0@0").unwrap(); - group(&video, 1, &[1000]); - group(&video, 2, &[2000]); - tokio::time::sleep(Duration::from_millis(50)).await; - assert!(window(&store, "video").await.is_empty(), "group 1 waits for group 0"); - - first.write_frame(ms(500), "0@500").unwrap(); - first.finish().unwrap(); - video.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); - check_objects(&store, "video", &records).await; - let object = store.get_segments("video", 0, true).await.unwrap(); - let frames: Vec<_> = object.groups[0].frames.iter().map(|f| f.timestamp).collect(); - assert_eq!(frames, vec![Some(0), Some(500)]); - } - - #[tokio::test] - async fn decreasing_arrivals_are_refused() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - - group(&video, 0, &[0]); - group(&video, 2, &[1000]); - group(&video, 1, &[500]); - group(&video, 3, &[2000]); - video.finish().unwrap(); - source.close(); - - writer.run().await.unwrap(); - - // The skipped sequence closes a record, so no record holds a gap. - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (2, 2), (3, 3)]); - check_objects(&store, "video", &records).await; - } - - #[tokio::test] - async fn an_unrepresentable_group_fails_the_recording() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - - group(&video, 0, &[0]); - // One past the recording's largest group ID. - group(&video, 1 << 53, &[1000]); - video.finish().unwrap(); - source.close(); - - assert_eq!( - writer.run().await, - Err(Error::Source(format!( - "track video group {}: {}", - 1u64 << 53, - Error::Id(1 << 53) - ))) - ); - } - - /// Stopping a recording whose source is still live finishes every timeline and ends `run`. - #[tokio::test] - async fn stopping_ends_without_the_source() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - group(&video, 0, &[0]); - control.stop().unwrap(); - - writer.run().await.unwrap(); - let records = window(&store, "video").await; - check_objects(&store, "video", &records).await; - assert!( - control.track("audio", media()).await.is_err(), - "a stopped recording refuses enrollment" - ); - drop(source); - } - - #[tokio::test] - async fn a_stalled_track_holds_only_itself() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let audio = track(&source, "audio"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - control.track("audio", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - group(&audio, 0, &[0]); - // The audio group never completes, so its record never closes on its own. - let mut stalled = audio.create_group(group::Info { sequence: 1 }).unwrap(); - stalled.write_frame(ms(1000), "stalled").unwrap(); - group(&audio, 2, &[2000]); - for sequence in 0..4 { - group(&video, sequence, &[sequence * 1000]); - } - video.finish().unwrap(); - source.close(); - - // Video commits without waiting for the stalled audio. - while window(&store, "video").await.len() < 4 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!run.is_finished(), "a stalled track holds the recording open"); - - control.remove("audio").unwrap(); - run.await.unwrap().unwrap(); - - // What arrived of the stalled group is stored; the group queued behind it is dropped. - let records = window(&store, "audio").await; - assert_eq!( - records - .iter() - .map(|record| (record.start, record.end)) - .collect::>(), - vec![ - (Position::group(0), Position::group(1)), - (Position::group(1), Position::new(1, 1)) - ] - ); - check_objects(&store, "audio", &records).await; - drop(stalled); - } - - #[tokio::test] - async fn enrollment_is_refused_for_duplicates_and_timelines() { - let source = broadcast::Info::new().produce(); - let _video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - assert_eq!( - control.track("video", sparse()).await, - Err(Error::Enrolled("video".into())) - ); - let timeline = timeline("video"); - assert_eq!(control.track(&timeline, media()).await, Err(Error::Enrolled(timeline))); - } - - #[tokio::test] - async fn a_conflicting_info_fails_enrollment() { - let source = broadcast::Info::new().produce(); - let _video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - store - .put_info("video", &Info::new(7, Some(1000)).unwrap()) - .await - .unwrap(); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - let control = writer.control(); - assert_eq!( - control.track("video", media()).await, - Err(Error::Priority { - existing: 7, - intended: 127 - }) - ); - } - - /// Record `video` groups `sequences`, one per second, until the source ends. - async fn record(store: &Store, config: Config, sequences: std::ops::Range) { - let source = broadcast::Info::new().produce(); - record_into(store, config, source, sequences).await - } - - /// [`record`] from a source announced under `epoch`. - async fn record_from( - store: &Store, - epoch: Option, - sequences: std::ops::Range, - ) { - record_into(store, Config::default(), source(epoch), sequences).await - } - - async fn record_into( - store: &Store, - config: Config, - source: broadcast::Producer, - sequences: std::ops::Range, - ) { - let video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - for sequence in sequences { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - } - - /// A source broadcast resolved through a route with `epoch`. - fn source(epoch: Option) -> broadcast::Producer { - let mut info = broadcast::Info::new(); - info.epoch = epoch; - info.produce() - } - - fn epoch(n: u8) -> moq_net::Epoch { - format!("01900000-0000-7000-8000-0000000000{n:02x}").parse().unwrap() - } - - fn orphan(sequence: u64) -> Object { - Object::new(vec![Group { - sequence, - frames: vec![Frame { - timestamp: Some(sequence * 1000), - payload: "orphan".into(), - }], - }]) - } - - #[tokio::test] - async fn a_restarted_writer_resumes_the_recording() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - // The source's cache replays groups the recording already holds. - record(&store, Config::default(), 0..6).await; - - let records = window(&store, "video").await; - assert_eq!(sequences(&records), (0..6).collect::>()); - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - - // The resumed timeline groups continue the stored numbering. - let mut numbers = Vec::new(); - for segment in 0..6 { - let object = store.get_segments(&timeline("video"), segment, true).await.unwrap(); - numbers.extend(object.groups.iter().map(|group| group.sequence)); - } - assert!(numbers.windows(2).all(|pair| pair[0] < pair[1]), "{numbers:?}"); - } - - /// The group sequence continued, but content time jumped back before the recorded end. - #[tokio::test] - async fn a_backward_timestamp_is_refused() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - // Group 3 is the next one. Its timestamp is behind the last frame, at 2500. - group(&video, 3, &[500]); - video.finish().unwrap(); - source.close(); - - assert_eq!( - writer.run().await, - Err(Error::Source("track video group 3: timestamp 500 precedes 2500".into())) - ); - let records = window(&store, "video").await; - assert_eq!( - groups(&records), - (0..3).map(|sequence| (sequence, sequence)).collect::>() - ); - check_objects(&store, "video", &records).await; - } - - /// A source announced under another epoch restarted, whatever its groups say: enrolling it - /// fails and nothing more is recorded. - #[tokio::test] - async fn another_epoch_is_refused() { - let store = Store::new(InMemory::new(), "rec"); - record_from(&store, Some(epoch(1)), 0..3).await; - - for other in [Some(epoch(2)), None] { - let source = source(other.clone()); - let _video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - assert_eq!( - writer.control().track("video", media()).await, - Err(Error::EpochMismatch { - existing: Some(epoch(1)), - intended: other, - }) - ); - } - - // And a recording without an epoch refuses a source with one. - let plain = Store::new(InMemory::new(), "plain"); - record(&plain, Config::default(), 0..3).await; - let source = source(Some(epoch(1))); - let _video = track(&source, "video"); - let writer = Writer::new(plain.clone(), source.consume(), Config::default()) - .await - .unwrap(); - assert_eq!( - writer.control().track("video", media()).await, - Err(Error::EpochMismatch { - existing: None, - intended: Some(epoch(1)), - }) - ); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..3).map(|s| (s, s)).collect::>()); - } - - /// The same epoch resumes, skipping the groups its cache replays. - #[tokio::test] - async fn the_same_epoch_resumes() { - let store = Store::new(InMemory::new(), "rec"); - record_from(&store, Some(epoch(1)), 0..3).await; - record_from(&store, Some(epoch(1)), 0..6).await; - - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - assert_eq!(store.get_info("video").await.unwrap().epoch, Some(epoch(1))); - } - - /// A source that jumped forward, in both group sequence and content time, keeps recording. - #[tokio::test] - async fn a_forward_restart_keeps_recording() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - group(&video, 10, &[10_000, 10_500]); - video.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2), (10, 10)]); - check_objects(&store, "video", &records).await; - } - - #[tokio::test] - async fn a_resumed_append_only_group_continues_mid_group() { - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default(); - - // The first run stores frames 0..18 of a group that never closes, then stops. - let source = broadcast::Info::new().produce(); - let log = track(&source, "log"); - let writer = Writer::new(store.clone(), source.consume(), config.clone()) - .await - .unwrap(); - writer.control().track("log", append_only()).await.unwrap(); - let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); - for second in 0..19 { - open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); - } - let run = tokio::spawn(writer.run()); - while window(&store, "log").await.len() < 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - run.abort(); - let _ = run.await; - drop(open); - - // The restarted source replays the whole group; the stored head is skipped. - let source = broadcast::Info::new().produce(); - let log = track(&source, "log"); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("log", append_only()).await.unwrap(); - let mut replayed = log.create_group(group::Info { sequence: 0 }).unwrap(); - for second in 0..20 { - replayed - .write_frame(ms(second * 1000), format!("line {second}")) - .unwrap(); - } - replayed.finish().unwrap(); - log.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let records = window(&store, "log").await; - assert_eq!( - records - .iter() - .map(|record| (record.start, record.end)) - .collect::>(), - vec![ - (Position::new(0, 0), Position::new(0, 9)), - (Position::new(0, 9), Position::new(0, 18)), - (Position::new(0, 18), Position::group(1)), - ] - ); - check_objects(&store, "log", &records).await; - } - - #[tokio::test] - async fn a_restarted_dvr_deletes_unreferenced_objects() { - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - record(&store, config.clone(), 0..6).await; - - // An interrupted expiration, an uncommitted upload, and a track no timeline names. - store.put_segments("video", 1, &orphan(1)).await.unwrap(); - store.put_segments("video", 6, &orphan(7)).await.unwrap(); - store.put_segments("audio", 0, &orphan(0)).await.unwrap(); - - record(&store, config, 6..10).await; - - let records = window(&store, "video").await; - assert_eq!(sequences(&records), vec![7, 8, 9]); - assert_eq!(groups(&records), vec![(7, 7), (8, 8), (9, 9)]); - check_objects(&store, "video", &records).await; - assert_eq!(stored(&store).await, referenced("video", &records)); - store.get_info("video").await.unwrap(); - store.get_info(&timeline("video")).await.unwrap(); - assert_eq!(segments(&store, "video").await, vec![9]); - } - - #[tokio::test] - async fn a_dvr_crash_between_pop_and_delete_is_cleaned_on_restart() { - let store = Store::new(InMemory::new(), "rec"); - - // A grace longer than the test keeps every expired object past the crash. - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let config = - Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - // Record 5 stays open, so the newest durable timeline object is record 4. - while window(&store, "video").await.last().map(|record| record.sequence) != Some(4) { - tokio::time::sleep(Duration::from_millis(10)).await; - } - run.abort(); - let _ = run.await; - - let retained = window(&store, "video").await; - assert_eq!(sequences(&retained), vec![3, 4]); - let expired: HashSet<_> = (0..3).map(|s| Key::segments("video", s).unwrap()).collect(); - assert_eq!(stored(&store).await, &referenced("video", &retained) | &expired); - assert_eq!(segments(&store, "video").await, (0..=4).collect::>()); - // An upload the crash left uncommitted. - store.put_segments("video", 5, &orphan(5)).await.unwrap(); - - let grace = Duration::from_millis(200); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let started = Instant::now(); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - assert!( - stored(&store).await.is_superset(&expired), - "expired objects outlive the grace, for readers holding the old timeline" - ); - assert!( - !stored(&store).await.contains(&Key::segments("video", 5).unwrap()), - "the uncommitted upload is cleared before its key is reused" - ); - assert_eq!(segments(&store, "video").await, (0..=4).collect::>()); - - writer.control().track("video", media()).await.unwrap(); - // The source replays groups the recording already holds; they are refused. - for sequence in 4..9 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - assert!(started.elapsed() >= grace); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(6, 6), (7, 7), (8, 8)]); - check_objects(&store, "video", &records).await; - assert_eq!(stored(&store).await, referenced("video", &records)); - assert_eq!(segments(&store, "video").await, vec![8]); - } - - #[tokio::test] - async fn an_archive_restart_clears_uncommitted_objects() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - // Media stored after the last timeline commit: a crash before its record. - store.put_segments("video", 3, &orphan(3)).await.unwrap(); - - record(&store, Config::default(), 3..6).await; - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - assert_eq!(stored(&store).await, referenced("video", &records)); - } - - #[tokio::test] - async fn a_failed_recovery_deletes_nothing() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - record(&store, config.clone(), 0..6).await; - store.put_segments("video", 1, &orphan(1)).await.unwrap(); - let before = stored(&store).await; - - let failing = mock.fork(); - failing.fail_lists(); - let source = broadcast::Info::new().produce(); - let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; - assert!(matches!(result, Err(Error::Store(_)))); - - // A gap in the timeline keys leaves the retained window unrecoverable. - let newest = store.get_segments(&timeline("video"), 5, true).await.unwrap(); - store.put_segments(&timeline("video"), 3, &newest).await.unwrap(); - let result = Writer::new(store.clone(), source.consume(), config).await; - assert!(matches!(result, Err(Error::Timeline(_)))); - assert_eq!(segments(&store, "video").await, vec![3, 5]); - - assert_eq!(stored(&store).await, before); - } - - #[tokio::test] - async fn a_dvr_window_longer_than_one_checkpoint_is_recovered() { - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(280), Duration::ZERO)); - record(&store, config, 0..300).await; - - let recovery = recover(&store, true).await.unwrap(); - let resume = &recovery.tracks["video"]; - let records = window(&store, "video").await; - assert!(records.len() > 256, "the window outgrows one checkpoint"); - assert_eq!(resume.checkpoint.records, records); - assert_eq!(resume.checkpoint.range.end, 300); - assert!(recovery.orphans.is_empty()); - assert!(recovery.uncommitted.is_empty()); - assert_eq!(resume.floor(), Some(Position::group(300))); - - // The oldest stored timeline object is exactly the one recovery starts from. - let stored = segments(&store, "video").await; - assert!(stored.len() > 1, "the newest checkpoint omits part of the window"); - assert!(stored[0] > 0, "earlier timeline objects are pruned"); - assert_eq!(resume.checkpoints.front().unwrap().0, stored[0]); - assert!(recovery.stale.is_empty()); - } - - /// Yield to the writer until `check` holds. - async fn until>(mut check: impl FnMut() -> F) { - while !check().await { - tokio::task::yield_now().await; - } - } - - #[tokio::test(start_paused = true)] - async fn a_dvr_prunes_timeline_objects_one_grace_after_they_are_unneeded() { - let store = Store::new(InMemory::new(), "rec"); - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let grace = Duration::from_secs(10); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - for sequence in 0..4 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - // Record 3 stays open, so record 2 is the newest timeline object. - until(|| async { segments(&store, "video").await.last() == Some(&2) }).await; - assert_eq!( - segments(&store, "video").await, - vec![0, 1, 2], - "unneeded timeline objects outlive the grace, for readers that listed them" - ); - - tokio::time::sleep(grace).await; - until(|| async { segments(&store, "video").await == [2] }).await; - - for sequence in 4..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); - - assert_eq!(segments(&store, "video").await, vec![5]); - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(3, 3), (4, 4), (5, 5)]); - check_objects(&store, "video", &records).await; - let recovery = recover(&store, true).await.unwrap(); - assert_eq!(recovery.tracks["video"].checkpoint.records, records); - } - - #[tokio::test] - async fn a_failed_timeline_delete_keeps_the_keys_consecutive() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - let failed = Key::segments(timeline("video"), 1).unwrap(); - mock.fail_deletes(store.path(&failed).unwrap().as_ref()); - record(&store, config.clone(), 0..6).await; - - // Pruning stops at the failed object instead of deleting past it. - assert_eq!(segments(&store, "video").await, (1..=5).collect::>()); - window(&store, "video").await; - - // A restart retries from the oldest stored object. - mock.heal(); - record(&store, config, 6..8).await; - assert_eq!(segments(&store, "video").await, vec![7]); - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(5, 5), (6, 6), (7, 7)]); - check_objects(&store, "video", &records).await; - } - - #[tokio::test] - async fn a_timeline_that_does_not_end_at_the_next_segment_fails_recovery() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..6).await; - let before = stored(&store).await; - - // `segments/5` still decodes, but it restates an earlier window, so resuming would write the - // next record on top of it. - let older = store.get_segments(&timeline("video"), 0, true).await.unwrap(); - store - .delete(&Key::segments(timeline("video"), 5).unwrap()) - .await - .unwrap(); - store.put_segments(&timeline("video"), 5, &older).await.unwrap(); - - let source = broadcast::Info::new().produce(); - match Writer::new(store.clone(), source.consume(), Config::default()).await { - Err(Error::Timeline(message)) => assert!(message.contains("not segment 6"), "{message}"), - Err(err) => panic!("expected a timeline error, got {err}"), - Ok(_) => panic!("expected recovery to fail"), - } - assert_eq!(stored(&store).await, before); - } - - /// An enrollment dropped mid-flight still settles its count, so stopping the recording ends it. - #[tokio::test(start_paused = true)] - async fn a_cancelled_enrollment_does_not_block_the_end() { - let source = broadcast::Info::new().produce(); - let mut audio = track(&source, "audio"); - - let mock = Mock::memory(); - mock.hold_puts("audio/.info"); - let store = Store::new(mock.clone(), "rec"); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - let control = writer.control(); - let run = tokio::spawn(writer.run()); - let enrolling = { - let control = control.clone(); - tokio::spawn(async move { control.track("audio", media()).await }) - }; - assert!( - audio.subscription_changed().await.unwrap().is_some(), - "enrollment subscribed" - ); - enrolling.abort(); - let _ = enrolling.await; - - control.stop().unwrap(); - tokio::time::timeout(Duration::from_secs(5), run) - .await - .expect("the recording ends") - .unwrap() - .unwrap(); - // The cancelled name was never enrolled, so it isn't taken. - mock.release(); - drop(source); - } - - /// The publisher already has the subscriber, and the broadcast ends while `.info` is written. - /// - /// The enroll command is queued only after that write. The recording stays up across the gap, - /// so the track is kept instead of failing with the writer closed. - #[tokio::test] - async fn a_track_that_ends_during_enrollment_is_kept() { - let source = broadcast::Info::new().produce(); - let mut audio = track(&source, "audio"); - group(&audio, 0, &[0, 500]); - - let mock = Mock::memory(); - mock.hold_puts("audio/.info"); - let store = Store::new(mock.clone(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - let run = tokio::spawn(writer.run()); - let enrolling = tokio::spawn(async move { control.track("audio", media()).await }); - - let subscription = audio.subscription_changed().await.unwrap(); - assert!(subscription.is_some(), "enrollment subscribed"); - audio.finish().unwrap(); - source.close(); - mock.release(); - - enrolling.await.unwrap().unwrap(); - run.await.unwrap().unwrap(); - - let records = window(&store, "audio").await; - assert_eq!(groups(&records), vec![(0, 0)]); - } -} diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 0abb35dde87..2970c6dec35 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -120,8 +120,6 @@ moq-clock = { workspace = true } moq-hls = { workspace = true, features = ["server"] } # Writes and reads the stats and feedback snapshots. moq-json = { workspace = true } -# Names the MSF catalog track `export archive` records. -moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the # CLI's own feature set (iroh, quic, cluster-lan) is what selects them. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 959f12a33b4..c52537abb08 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,13 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### `export archive` records a prefix (#6160) + +`moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//@`, until the connection ends, instead of one broadcast at ``. +Point `` at the root you want recordings under, and drop `--catalog-format`, which it now refuses. +Untimed tracks record, and `--retention` works on them. +`import archive` does not read the new layout yet; it still replays recordings made by older `moq` versions. + ### `--cluster-lan-secret` is a file path only (#5910) `--cluster-lan-secret`, `MOQ_CLUSTER_LAN_SECRET`, and `[cluster.lan] secret` no longer accept 64 hex characters inline. @@ -62,10 +69,6 @@ A restarted publisher exits 1 unless `--stitch` follows it as a program switch. moq export ts --linger 10s --stitch ... ``` -### `moq export archive` refuses untimed tracks - -Recording a track with no timestamps fails with `Untimed`, so an archive over moq-lite before 05 or moq-transport drafts 14 to 16 fails instead of recording arrival times. - ### `moq export fmp4` fixes its track set at the init segment A rendition added, changed, or left undescribed after the init segment ends the export with an error instead of writing a track missing from the moov. diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 16a4611efd8..4c8aa754b54 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -1,20 +1,19 @@ -//! Archive endpoints: record one broadcast into an object store (export), or -//! republish a recording from one (import), through `moq-archive`. +//! Archive endpoints: record every broadcast announced under a prefix into an object store +//! (export), or republish a recording from one (import), through `moq-archive`. -use std::collections::HashSet; +use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use anyhow::Context; use hang::moq_net; -use moq_archive::writer::{Control, Retention}; -use moq_mux::catalog::{CatalogFormat, Stream as _}; use object_store::ObjectStore; +use object_store::path::{Path, PathPart}; use url::Url; use crate::moq::notify_ready; -/// How long expired objects outlive the timeline that dropped them, unless -/// `--retention-grace` says otherwise. +/// How long a superseded object outlives its DVR window, unless `--retention-grace` says otherwise. const GRACE: Duration = Duration::from_secs(30); /// `export archive` args. @@ -22,14 +21,15 @@ const GRACE: Duration = Duration::from_secs(30); #[usage(unknown_flags = "error", args_override_self = false)] pub struct ExportArgs { /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. + /// Each broadcast records under `/@` beneath it. pub store: Url, - /// Keep only this much recent content (a DVR), deleting older records. Unset keeps everything. + /// Keep only this much recent content (a DVR), deleting older objects. Unset keeps everything. #[usage(long)] pub retention: Option, - /// How long expired objects outlive the timeline that dropped them, so readers can finish - /// their downloads. Defaults to 30s; needs `--retention`. + /// How long a deleted object outlives the retention, so readers can finish their downloads. + /// Defaults to 30s; needs `--retention`. #[usage(long)] pub retention_grace: Option, } @@ -38,7 +38,7 @@ pub struct ExportArgs { #[derive(usage_rs::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] pub struct ImportArgs { - /// The recording to replay, as `export archive` wrote it. + /// The timeline recording to replay. pub store: Url, /// Keep following a recording that is still growing, checking for new segments at this @@ -47,59 +47,196 @@ pub struct ImportArgs { pub follow: Option, } -/// Record the broadcast `name` into `args.store` until the broadcast ends. +/// One publisher instance being recorded: its path, and its epoch when its route has one. +type Instance = (moq_net::PathOwned, Option); + +/// Record every broadcast announced at or under `prefix` into `args.store`, each publisher +/// instance under `/@`, until the announcements end. /// -/// Reads the broadcast's own catalog: every track gets its own timeline, cut at group boundaries -/// between 2s and 10s, while the catalog and data tracks use a zero minimum and record each group -/// as it finishes. -pub async fn export( - origin: moq_net::origin::Consumer, - name: String, - format: CatalogFormat, - args: ExportArgs, -) -> anyhow::Result<()> { - let retention = match (args.retention, args.retention_grace) { - (Some(window), grace) => Some(Retention::new( +/// A route without an epoch names no instance, so each one it announces records under an epoch +/// minted here: a restarted export never resumes it, and never stitches two instances together. +pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: ExportArgs) -> anyhow::Result<()> { + let dvr = match (args.retention, args.retention_grace) { + (Some(window), grace) => Some(moq_archive::producer::Dvr::new( window.into_std(), grace.map_or(GRACE, crate::duration::Duration::into_std), )), (None, Some(_)) => anyhow::bail!("`--retention-grace` needs `--retention`"), (None, None) => None, }; - let catalog_track = catalog_track(format)?; - let store = open(&args.store)?; + let config = moq_archive::producer::Config::default().with_dvr(dvr); + let (store, root) = open_store(&args.store)?; + + // Announcements leave hidden routes out unless asked, and a prefix naming one asks. + let hidden = prefix.split('/').any(|segment| segment.starts_with('.')); + let prefix = moq_net::Path::new(&prefix).to_owned(); + let mut announced = origin.clone().with_hidden(hidden).announced(); + let mut recordings = tokio::task::JoinSet::new(); + let mut running = Running::default(); + let claims = Claims::default(); + + tracing::info!(%prefix, store = %args.store, "recording"); + notify_ready(); - let broadcast = origin - .routed_broadcast(&name) - .await - .with_context(|| format!("broadcast `{name}` is unavailable"))?; - let config = moq_archive::writer::Config::default().with_retention(retention); - let writer = moq_archive::Writer::new(store, broadcast.clone(), config) - .await - .with_context(|| format!("failed to start recording into {}", args.store))?; - let control = writer.control(); - control.track(catalog_track, sparse()).await?; - let catalog = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; + loop { + let instance = tokio::select! { + event = announced.next() => { + let announce = match event.context("announcements ended")? { + // Another publisher instance, or the same one announced again. + moq_net::announce::Event::Start(announce) | moq_net::announce::Event::Restart(announce) => announce, + _ => continue, + }; + if !announce.prefix.has_prefix(&prefix) { + continue; + } + let instance: Instance = (announce.prefix, announce.route.epoch); + if !running.start(&instance) { + continue; + } + instance + } + Some(done) = recordings.join_next() => { + let (instance, result) = done.context("recording panicked")?; + result?; + if !running.done(&instance) { + continue; + } + instance + } + }; + let key = key(&root, &instance)?; + let store = moq_archive::Store::new(store.clone(), key); + // Requested now, so an announcement resolves the instance serving it when read, not + // whichever one serves by the time its task runs. + let request = origin.request_broadcast(&instance.0, instance.1.clone()); + recordings.spawn(record(request, instance, store, config.clone(), claims.clone())); + } +} - tracing::info!(%name, store = %args.store, "recording"); - notify_ready(); +/// The broadcasts being recorded. Announcements read after another instance took over all +/// resolve that one, so it is recorded once. +#[derive(Clone, Default)] +struct Claims(Arc>>>); + +impl Claims { + /// Claim `broadcast` for recording, or `None` when it is recorded already. + fn claim(&self, path: &moq_net::PathOwned, broadcast: &moq_net::broadcast::Consumer) -> Option { + let mut claims = self.0.lock().unwrap(); + let claimed = claims.entry(path.clone()).or_default(); + if claimed.iter().any(|other| other.is_clone(broadcast)) { + return None; + } + claimed.push(broadcast.clone()); + Some(Claim { + claims: self.clone(), + path: path.clone(), + broadcast: broadcast.clone(), + }) + } +} - // The writer decides when the recording is over; the catalog only feeds it tracks. - let run = writer.run(); - tokio::pin!(run); - tokio::select! { - result = &mut run => result.context("recording failed"), - Err(err) = enroll(control.clone(), catalog) => { - // Store what already arrived before failing, so the recording stays readable. - control.stop()?; - if let Err(stopped) = run.await { - tracing::warn!(%stopped, "recording failed while stopping"); +/// A broadcast being recorded, released on drop. +struct Claim { + claims: Claims, + path: moq_net::PathOwned, + broadcast: moq_net::broadcast::Consumer, +} + +impl Drop for Claim { + fn drop(&mut self) { + let mut claims = self.claims.0.lock().unwrap(); + if let Some(claimed) = claims.get_mut(&self.path) { + claimed.retain(|other| !other.is_clone(&self.broadcast)); + if claimed.is_empty() { + claims.remove(&self.path); + } + } + } +} + +/// The epoch-pinned instances recording, so each records under its key once at a time, and again +/// when it was announced anew meanwhile. +#[derive(Default)] +struct Running(HashMap); + +impl Running { + /// Whether to record `instance` now that it was announced. An epochless one is a new instance + /// each time, under a fresh key. + fn start(&mut self, instance: &Instance) -> bool { + if instance.1.is_none() { + return true; + } + match self.0.get_mut(instance) { + Some(again) => { + *again = true; + false + } + None => { + self.0.insert(instance.clone(), false); + true + } + } + } + + /// Whether to record `instance` again, now that its recording ended. + fn done(&mut self, instance: &Instance) -> bool { + match self.0.get_mut(instance) { + Some(again) if *again => { + *again = false; + true + } + _ => { + self.0.remove(instance); + false } - Err(err) } } } +/// Record one publisher instance until its broadcast ends, unless another recording has it. +async fn record( + request: impl Future>, + instance: Instance, + store: moq_archive::Store>, + config: moq_archive::producer::Config, + claims: Claims, +) -> (Instance, anyhow::Result<()>) { + let result = async { + let path = &instance.0; + let broadcast = request + .await + .with_context(|| format!("broadcast `{path}` is unavailable"))?; + let Some(_claim) = claims.claim(path, &broadcast) else { + return Ok(()); + }; + tracing::info!(%path, key = %store.prefix(), "recording broadcast"); + moq_archive::Producer::new(store, broadcast, config) + .run() + .await + .with_context(|| format!("recording `{path}` failed")) + } + .await; + (instance, result) +} + +/// The key prefix one instance records under: its path's segments, then `@`. +/// +/// A path segment's key never starts with `@`, since a leading one is percent-encoded like `%` +/// always is, so no instance's prefix lies inside another's, whatever paths a publisher picks. +fn key(root: &Path, (path, epoch): &Instance) -> anyhow::Result { + let epoch = epoch.clone().unwrap_or_else(moq_net::Epoch::mint); + let mut key = root.clone(); + for segment in path.as_str().split('/').filter(|segment| !segment.is_empty()) { + let part = PathPart::from(segment); + key = match part.as_ref().strip_prefix('@') { + Some(rest) => key.join(PathPart::parse(&format!("%40{rest}"))?), + None => key.join(part), + }; + } + let epoch = format!("@{epoch}"); + Ok(key.join(PathPart::parse(&epoch).context("an epoch is one key segment")?)) +} + /// Republish the recording at `args.store` as the broadcast `name`, announced under `epoch`. /// /// Each track's timeline replays as a live track and every other track's groups are served on @@ -156,373 +293,206 @@ pub async fn import( } /// Open the object store `url` names, taking cloud credentials from the `AWS_*`, `GOOGLE_*`, -/// and `AZURE_*` environment variables. -fn open(url: &Url) -> anyhow::Result>> { +/// and `AZURE_*` environment variables, with the prefix the URL names. +fn open_store(url: &Url) -> anyhow::Result<(Arc, Path)> { let credentials = std::env::vars_os() .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) .filter(|(key, _)| ["AWS_", "GOOGLE_", "AZURE_"].iter().any(|cloud| key.starts_with(cloud))); let (store, prefix) = object_store::parse_url_opts(url, credentials).with_context(|| format!("unsupported archive store {url}"))?; - Ok(moq_archive::Store::new(store, prefix)) + Ok((Arc::from(store), prefix)) } -/// How sparse data, such as a catalog, is cut: every group is its own record. -fn sparse() -> moq_mux::timeline::Config { - moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO) -} - -/// The track carrying the catalog `format` reads. -fn catalog_track(format: CatalogFormat) -> anyhow::Result<&'static str> { - Ok(match format { - CatalogFormat::Hang => hang::Catalog::DEFAULT_NAME, - CatalogFormat::HangZ => hang::Catalog::COMPRESSED_NAME, - CatalogFormat::Msf => moq_msf::DEFAULT_NAME, - other => anyhow::bail!("`export archive` cannot record a {other:?} catalog"), - }) -} - -/// Enroll each track as the catalog lists it, until the catalog ends. -async fn enroll( - control: Control, - mut catalog: moq_mux::catalog::Consumer<()>, -) -> anyhow::Result<()> { - let mut tracks = Tracks::default(); - while let Some(snapshot) = catalog.next().await? { - for change in tracks.update(&snapshot)? { - match change { - Change::Media(name) => control.track(&name, moq_mux::timeline::Config::default()).await?, - Change::Sparse(name) => control.track(&name, sparse()).await?, - Change::Remove(name) => control.remove(&name)?, - } - } - } - Ok(()) -} - -/// A change to the recorded track set. -#[derive(Debug, PartialEq, Eq)] -enum Change { - /// Record a rendition, its records cut by duration. - Media(String), - /// Record a data track, each group its own record. - Sparse(String), - /// Stop recording a track the catalog dropped. - Remove(String), -} - -/// The catalog tracks being recorded, diffed against each new snapshot. -#[derive(Default)] -struct Tracks { - /// Every name ever enrolled. The writer never takes a name back. - enrolled: HashSet, - /// The names the latest snapshot listed. - listed: HashSet, -} - -impl Tracks { - /// The changes that bring the recording in line with `catalog`. - /// - /// Refuses a track served from another broadcast, since an archive holds exactly one, and a - /// track that returns after the catalog dropped it, since the writer cannot resume one. - fn update(&mut self, catalog: &hang::Catalog) -> anyhow::Result> { - use hang::catalog::Reference; - - // Recorded by track name, which is what the archive's timelines are keyed by. - fn entry<'a, C: Reference>( - (id, config): (&'a String, &'a C), - media: bool, - ) -> (&'a str, Option<&'a moq_net::path::RelativeOwned>, bool) { - (config.track(id), config.broadcast(), media) - } - let video = catalog.video.renditions.iter().map(|e| entry(e, true)); - let audio = catalog.audio.renditions.iter().map(|e| entry(e, true)); - let text = catalog.text.renditions.iter().map(|e| entry(e, false)); - let json = catalog.json.tracks.iter().map(|e| entry(e, false)); - let binary = catalog.binary.tracks.iter().map(|e| entry(e, false)); - - let mut listed = HashSet::new(); - let mut changes = Vec::new(); - for (name, broadcast, media) in video.chain(audio).chain(text).chain(json).chain(binary) { - if let Some(broadcast) = broadcast { - anyhow::bail!( - "track `{name}` is served from broadcast `{broadcast}`; an archive records one broadcast" - ); - } - if !listed.insert(name.to_string()) || self.listed.contains(name) { - continue; - } - anyhow::ensure!( - self.enrolled.insert(name.to_string()), - "track `{name}` returned to the catalog after it was dropped; the recording cannot resume it" - ); - changes.push(match media { - true => Change::Media(name.to_string()), - false => Change::Sparse(name.to_string()), - }); - } - - let mut dropped: Vec<_> = self.listed.difference(&listed).cloned().collect(); - dropped.sort(); - changes.extend(dropped.into_iter().map(Change::Remove)); - self.listed = listed; - Ok(changes) - } +/// [`open_store`] as one recording. +fn open(url: &Url) -> anyhow::Result>> { + let (store, prefix) = open_store(url)?; + Ok(moq_archive::Store::new(store, prefix)) } #[cfg(test)] mod tests { use super::*; - use hang::catalog::{AudioCodec, AudioConfig, JsonConfig, Mode}; use moq_net::{Timescale, Timestamp, group, track}; - fn audio() -> AudioConfig { - AudioConfig::new(AudioCodec::Opus, 48_000, 2) - } - - /// Renditions are media, data tracks are sparse, and a later snapshot only adds what is new. - #[test] - fn renditions_are_media_and_the_rest_sparse() { - let mut tracks = Tracks::default(); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("audio".into(), audio()); - catalog - .json - .tracks - .insert("chat".into(), JsonConfig::new(Mode::Snapshot)); - - assert_eq!( - tracks.update(&catalog).unwrap(), - [Change::Media("audio".into()), Change::Sparse("chat".into())] - ); - - catalog.audio.renditions.insert("audio2".into(), audio()); - assert_eq!(tracks.update(&catalog).unwrap(), [Change::Media("audio2".into())]); - assert_eq!(tracks.update(&catalog).unwrap(), []); - } - - /// A rendition keyed apart from its track records the track. - #[test] - fn renditions_record_their_track() { - let mut rendition = audio(); - rendition.track = Some("audio".into()); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("main".into(), rendition); - - assert_eq!( - Tracks::default().update(&catalog).unwrap(), - [Change::Media("audio".into())] - ); - } - - /// A dropped rendition stops recording, and coming back is refused rather than silently lost. - #[test] - fn a_dropped_track_is_removed_for_good() { - let mut tracks = Tracks::default(); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("audio".into(), audio()); - tracks.update(&catalog).unwrap(); - - let removed = catalog.audio.renditions.remove("audio").unwrap(); - assert_eq!(tracks.update(&catalog).unwrap(), [Change::Remove("audio".into())]); - - catalog.audio.renditions.insert("audio".into(), removed); - let err = tracks.update(&catalog).unwrap_err().to_string(); - assert!(err.contains("cannot resume"), "{err}"); - } - - /// A rendition living in a sibling broadcast would be missing from the recording. - #[test] - fn another_broadcast_is_refused() { - let mut rendition = audio(); - rendition.broadcast = Some(moq_net::path::RelativeOwned::new("./source")); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("audio".into(), rendition); - - let err = Tracks::default().update(&catalog).unwrap_err().to_string(); - assert!(err.contains("broadcast `source`"), "{err}"); - } - - /// Wait until the export's subscription has reached `track`. - /// - /// `.info` is written after that, while the enroll command is still being queued. - /// Finishing on the file races that command and the recording ends with "writer closed". - async fn subscribed(track: &mut track::Producer, recording: &mut tokio::task::JoinHandle>) { - tokio::select! { - subscription = track.subscription_changed() => { - let subscription = subscription.expect("the rendition stays open until the export subscribes"); - assert!(subscription.is_some(), "the export subscribed"); + /// Every file under `dir`, relative to it. + fn files(dir: &std::path::Path) -> Vec { + let mut files = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(next) = pending.pop() { + for entry in std::fs::read_dir(next).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.push(path.strip_prefix(dir).unwrap().to_string_lossy().into_owned()); + } } - result = &mut *recording => panic!("the export ended before it subscribed: {result:?}"), } + files.sort(); + files } - /// What `export archive` records, `import archive` serves back group for group, and the - /// export ends cleanly with its broadcast. + /// `export archive` records each broadcast announced under its prefix into its own + /// `/@`, and leaves out what is announced elsewhere. #[tokio::test] - async fn a_recording_replays_its_groups() { - replays_its_groups(Some(Timescale::MILLI)).await; - } - - /// An untimed track, as every track is over moq-transport without TIMESCALE, records and - /// replays untimed. - #[tokio::test] - async fn an_untimed_recording_replays_untimed() { - replays_its_groups(None).await; - } - - async fn replays_its_groups(timescale: Option) { - // `open` reads the credential variables. + async fn each_broadcast_records_under_its_own_key_prefix() { + // `open_store` reads the credential variables. let _env = crate::test_env::EnvGuard::clear(&[]); let dir = tempfile::tempdir().unwrap(); let url = Url::from_directory_path(dir.path()).unwrap(); let origin = moq_tokio::origin::spawn(); - let mut broadcast = origin.create_broadcast("live.hang").unwrap(); - let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); - let info = track::Info::default() - .with_timescale(timescale) - .with_max_age(Duration::from_secs(3600)); - let mut track = broadcast.create_track("audio", info).unwrap(); - catalog - .mutate(|catalog| { - catalog.audio.renditions.insert("audio".into(), audio()); - }) - .unwrap(); - broadcast.announce(Default::default()).unwrap(); - let args = ExportArgs { - store: url.clone(), + store: url, retention: None, retention_grace: None, }; - let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + let recording = tokio::spawn(export(origin.consume(), "live".into(), args)); - for sequence in 0..3 { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for offset in [0, 500] { - let timestamp = timescale.map(|_| Timestamp::from_millis(sequence * 1000 + offset).unwrap()); - group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); - } + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let mut epochs = Vec::new(); + let mut broadcasts = Vec::new(); + for path in ["live/a", "live/b", "other"] { + let broadcast = origin.create_broadcast(path).unwrap(); + let track = broadcast.create_track("video", info.clone()).unwrap(); + let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::from_millis(0).unwrap(), path).unwrap(); group.finish().unwrap(); + let epoch = moq_net::Epoch::mint(); + broadcast + .announce(moq_net::origin::Route::default().with_epoch(epoch.clone())) + .unwrap(); + epochs.push(epoch); + broadcasts.push((broadcast, track)); } - subscribed(&mut track, &mut recording).await; - track.finish().unwrap(); - catalog.finish().unwrap(); - broadcast.close(); - - tokio::time::timeout(Duration::from_secs(10), recording) - .await - .expect("the export ends with its broadcast") - .unwrap() - .expect("the recording succeeds"); + // Each recording ends with its broadcast, once it subscribed. + for (broadcast, mut track) in broadcasts.drain(..2) { + tokio::time::timeout(Duration::from_secs(10), track.subscription_changed()) + .await + .expect("the export subscribes") + .unwrap(); + track.finish().unwrap(); + broadcast.close(); + } + let complete = [ + format!("live/a/@{}/.end", epochs[0]), + format!("live/b/@{}/.end", epochs[1]), + ]; + tokio::time::timeout(Duration::from_secs(10), async { + while !complete.iter().all(|end| dir.path().join(end).exists()) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both recordings complete"); + recording.abort(); - let replay = moq_tokio::origin::spawn(); - let args = ImportArgs { - store: url, - follow: None, - }; - let serving = tokio::spawn(import( - replay.clone(), - "replay.hang".into(), - args, - moq_net::Epoch::mint(), - )); - - let consumer = replay.consume().routed_broadcast("replay.hang").await.unwrap(); - let audio = consumer.track("audio").unwrap(); - for sequence in 0..3 { - let mut group = audio.fetch_group(sequence, None).await.unwrap(); - for offset in [0, 500] { - let frame = group.read_frame().await.unwrap().expect("a frame"); - assert_eq!( - frame.timestamp.map(|timestamp| timestamp.as_millis()), - timescale.map(|_| u128::from(sequence * 1000 + offset)) - ); - assert_eq!(frame.payload, format!("{sequence}+{offset}")); + let files = files(dir.path()); + for (path, epoch) in [("live/a", &epochs[0]), ("live/b", &epochs[1])] { + for file in [".end", "video/.info", "video/end"] { + assert!(files.contains(&format!("{path}/@{epoch}/{file}")), "{files:?}"); } - assert!(group.read_frame().await.unwrap().is_none()); + assert!( + files.contains(&format!( + "{path}/@{epoch}/video/0000000000000000000-0000000000000000000-0000000000000000000" + )), + "{files:?}" + ); } + assert!(!files.iter().any(|file| file.starts_with("other")), "{files:?}"); + } - serving.abort(); + /// An epochless route announced again while its last recording drains is a new recording, and + /// an epoch-pinned one records again once its recording ends. + #[test] + fn a_re_announced_instance_is_recorded() { + let mut running = Running::default(); + let epochless = (moq_net::Path::new("live/a").to_owned(), None); + assert!(running.start(&epochless)); + assert!(running.start(&epochless)); + assert!(!running.done(&epochless)); + + let pinned = (moq_net::Path::new("live/b").to_owned(), Some(moq_net::Epoch::mint())); + assert!(running.start(&pinned)); + assert!(!running.start(&pinned)); + assert!(running.done(&pinned)); + assert!(!running.done(&pinned)); + assert!(running.start(&pinned)); } - /// Renditions already live in the opening catalog are all recorded from their first group. - /// - /// Groups are published before export starts, so the writer can drain the first rendition - /// while the next subscription is still in flight. + /// Announcements that resolve the same broadcast record it once at a time, while another + /// instance at the path records beside it. #[tokio::test] - async fn an_opening_snapshot_records_every_rendition() { - let _env = crate::test_env::EnvGuard::clear(&[]); - let dir = tempfile::tempdir().unwrap(); - let url = Url::from_directory_path(dir.path()).unwrap(); - + async fn a_resolved_broadcast_is_recorded_once() { let origin = moq_tokio::origin::spawn(); - let mut broadcast = origin.create_broadcast("live.hang").unwrap(); - let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - let mut first = broadcast.create_track("audio", info.clone()).unwrap(); - let mut second = broadcast.create_track("audio2", info).unwrap(); - for track in [&first, &second] { - for sequence in 0..3 { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for offset in [0, 500] { - let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); - group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); - } - group.finish().unwrap(); - } - } - catalog - .mutate(|catalog| { - catalog.audio.renditions.insert("audio".into(), audio()); - catalog.audio.renditions.insert("audio2".into(), audio()); - }) - .unwrap(); - broadcast.announce(Default::default()).unwrap(); + let path = moq_net::Path::new("live/a").to_owned(); + let newest = origin.create_broadcast(&path).unwrap(); + let older = origin.create_broadcast(&path).unwrap(); + + let claims = Claims::default(); + let claim = claims.claim(&path, &newest.consume()).expect("first claim"); + assert!(claims.claim(&path, &newest.consume()).is_none()); + let beside = claims.claim(&path, &older.consume()).expect("another instance"); + + drop(claim); + assert!(claims.claim(&path, &newest.consume()).is_some()); + drop(beside); + assert!(claims.0.lock().unwrap().is_empty()); + } - let args = ExportArgs { - store: url.clone(), - retention: None, - retention_grace: None, - }; - let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + /// A route without an epoch records under one minted per instance, so two instances never + /// share a key prefix. + #[test] + fn an_epochless_instance_gets_a_fresh_key() { + let instance = (moq_net::Path::new("live/a").to_owned(), None); + let first = key(&Path::from("rec"), &instance).unwrap(); + let second = key(&Path::from("rec"), &instance).unwrap(); + assert_ne!(first, second); + assert!(first.as_ref().starts_with("rec/live/a/"), "{first}"); + + let epoch = moq_net::Epoch::mint(); + let pinned = (moq_net::Path::new("live/a").to_owned(), Some(epoch.clone())); + assert_eq!( + key(&Path::from("rec"), &pinned).unwrap().as_ref(), + format!("rec/live/a/@{epoch}") + ); + } - subscribed(&mut first, &mut recording).await; - subscribed(&mut second, &mut recording).await; - first.finish().unwrap(); - second.finish().unwrap(); - catalog.finish().unwrap(); - broadcast.close(); + /// A path segment's key never starts with `@`, so a path that spells out another instance's + /// key prefix records beside it, never inside it. + #[test] + fn keys_are_prefix_free() { + let epoch = moq_net::Epoch::mint(); + let victim = key( + &Path::from("rec"), + &(moq_net::Path::new("live/a").to_owned(), Some(epoch.clone())), + ) + .unwrap(); + let hostile = format!("live/a/@{epoch}"); + let nested = key( + &Path::from("rec"), + &(moq_net::Path::new(&hostile).to_owned(), Some(moq_net::Epoch::mint())), + ) + .unwrap(); + assert!(nested.prefix_match(&victim).is_none(), "{nested} is under {victim}"); + assert!( + nested.as_ref().starts_with(&format!("rec/live/a/%40{epoch}/@")), + "{nested}" + ); - tokio::time::timeout(Duration::from_secs(10), recording) - .await - .expect("the export ends with its broadcast") - .unwrap() - .expect("the recording succeeds"); - - // Each rendition's own timeline opens at its first group. - let store = super::open(&url).unwrap(); - for track in ["audio", "audio2"] { - let object = store - .get_segments(&hang::timeline::default_name(track), 0, true) - .await - .expect("record 0"); - let config = moq_json::window::ConsumerConfig::default().with_compression(true); - let mut decoder = moq_json::window::Decoder::::new(config); - for stored in object.groups { - let mut group = decoder.group(); - for frame in stored.frames { - group.decode(&frame.payload).unwrap(); - } - } - let Some(moq_json::window::Event::Push { value, .. }) = decoder.next_event() else { - panic!("{track} record 0 is pushed first"); - }; - assert_eq!(value.start, hang::timeline::Position::group(0), "{track}"); - } + // `%` is escaped too, so an escaped-looking segment keys apart from the `@` one. + let literal = key( + &Path::from("rec"), + &( + moq_net::Path::new(&format!("live/a/%40{epoch}")).to_owned(), + Some(epoch), + ), + ) + .unwrap(); + assert!(literal.as_ref().starts_with("rec/live/a/%2540"), "{literal}"); } } diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 02fb827d2a9..81d8c719aa9 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -335,6 +335,12 @@ impl Invocation { "`export {sink}` reads no media to report on; remove --echo" ); } + if matches!(export.sink, ExportSink::Archive(_)) { + anyhow::ensure!( + export.catalog_format.is_none(), + "`export archive` records every track without reading a catalog, so it takes no --catalog-format" + ); + } if export.echo.is_some() { if export.sink.stdout().is_none() && let Some(sink) = export.sink.ignores_selection() @@ -1001,7 +1007,7 @@ pub enum ExportSink { Srt(crate::srt::ExportArgs), /// WebRTC: WHIP client pushing to a remote (`--connect`) or WHEP server serving plays (`--listen`). Rtc(crate::rtc::Args), - /// Record the broadcast into an object store until it ends. + /// Record every broadcast announced under `--broadcast` into an object store. Archive(crate::archive::ExportArgs), /// Write the broadcast's stats snapshots to stdout as JSON lines. Stats, diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 3b1f4da3d3c..196863afe11 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -884,12 +884,7 @@ fn spawn_export( } } ExportSink::Archive(args) => { - let format = export - .catalog_format - .map(Into::into) - .or_else(|| moq_mux::catalog::CatalogFormat::detect(&name)) - .unwrap_or_default(); - tasks.spawn(archive::export(origin.consume(), name, format, args)); + tasks.spawn(archive::export(origin.consume(), name, args)); } ExportSink::Stats => { tasks.spawn(report::print_stats(origin.consume(), name));