From bbf6dd309890ca4ef7d8c8530835ffd79a5ac87f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 23 Sep 2026 09:56:02 -0700 Subject: [PATCH 1/4] quest: claim cli-fetch From e4e69cb612a18ac88f2e9d15ca5894c0062f219b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 23 Sep 2026 09:56:16 -0700 Subject: [PATCH 2/4] quest: move cli fetch under main --- quest/main/README.md | 1 + quest/{next/cli-inspect/fetch.md => main/cli-fetch.md} | 0 quest/next/cli-inspect/README.md | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename quest/{next/cli-inspect/fetch.md => main/cli-fetch.md} (100%) diff --git a/quest/main/README.md b/quest/main/README.md index 4d654664a11..ab06807e5c4 100644 --- a/quest/main/README.md +++ b/quest/main/README.md @@ -91,6 +91,7 @@ do not add another media abstraction or a renderer crate during stabilization. - [Binding parity](/quest/main/binding-parity.md) - every wrapper reaches every moq-ffi method in its own idiom with moq-net's verbs - [Binary stats](/quest/main/stats-binary/README.md) - an allocation-free stats tick and an on-demand FlatBuffers `.fb.z` flavor with a checked-in schema - [Audio jitter target](/quest/main/audio-jitter-target/README.md) - the audio playout target is a measured estimate of arrival timing in both languages, not a round-trip guess +- [CLI fetch](/quest/main/cli-fetch.md) - `moq fetch` writes one group of a track to stdout, raw or as JSON lines ## Related diff --git a/quest/next/cli-inspect/fetch.md b/quest/main/cli-fetch.md similarity index 100% rename from quest/next/cli-inspect/fetch.md rename to quest/main/cli-fetch.md diff --git a/quest/next/cli-inspect/README.md b/quest/next/cli-inspect/README.md index 5aceb92dc35..4b0a3c3a2f7 100644 --- a/quest/next/cli-inspect/README.md +++ b/quest/next/cli-inspect/README.md @@ -22,4 +22,3 @@ This README owns the guide, written once both verbs land: a new - [Caught up](/quest/next/cli-inspect/caught-up.md) - moq-net's announce consumer says when the initial set has landed, and shell completion drops its settle timer - [ls](/quest/next/cli-inspect/ls.md) - `moq ls` prints the live set and exits, or follows changes, as paths or JSON lines -- [Fetch](/quest/next/cli-inspect/fetch.md) - `moq fetch` writes one group of a track to stdout, raw or as JSON lines From a65f102e4dc360d84ec6c5ef6534564086dc0d6e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 23 Sep 2026 10:26:00 -0700 Subject: [PATCH 3/4] feat(cli): add `moq fetch` to read one group of a track `moq --connect --broadcast fetch [--group N] [--json]` is the MoQ counterpart of the relay's HTTP `/fetch`. The "latest" lookup moves into `moq_relay::fetch_group`, shared by both. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + doc/bin/cli.md | 25 ++- quest/main/README.md | 1 - quest/main/cli-fetch.md | 31 ---- rs/moq-cli/Cargo.toml | 3 + rs/moq-cli/src/args.rs | 72 +++++--- rs/moq-cli/src/fetch.rs | 360 ++++++++++++++++++++++++++++++++++++++ rs/moq-cli/src/main.rs | 5 + rs/moq-relay/src/fetch.rs | 21 +++ rs/moq-relay/src/lib.rs | 2 + rs/moq-relay/src/web.rs | 30 +--- 11 files changed, 471 insertions(+), 80 deletions(-) delete mode 100644 quest/main/cli-fetch.md create mode 100644 rs/moq-cli/src/fetch.rs create mode 100644 rs/moq-relay/src/fetch.rs diff --git a/Cargo.lock b/Cargo.lock index 5754bf2a982..6758707bdc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4287,6 +4287,7 @@ dependencies = [ "anyhow", "axum", "axum-server", + "base64 0.23.1", "bytes", "hang", "humantime", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index e626cfb3a23..e5cb190d9af 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -23,7 +23,8 @@ or Docker; see [Install](/setup/install). | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | | `play` | | Decode and play in a native window with sound. | | `transcode` | | Publish a just-in-time rendition ladder next to a broadcast. | -| `token` | | Generate, sign, and verify relay JWTs. | +| `fetch` | `` | Write one group of a track to stdout. | +| `auth` | | Generate, sign, and verify relay JWTs. | | `devices` | | List capture sources and their ids. | ## Grammar @@ -32,6 +33,7 @@ or Docker; see [Install](/setup/install). moq import [options] moq export [options] moq play [options] +moq fetch [options] ``` The **MoQ side** goes first and attaches the process to the network: @@ -153,6 +155,27 @@ heights and bitrates must then increase strictly together. Duplicate heights or bitrates, inverted rankings, and zero-sized or zero-bitrate rungs are rejected before connecting. +## Fetch + +```bash +moq --connect https://relay.example.com/anon --broadcast my-stream.hang fetch catalog.json | jq +moq ... fetch video/hd --group 42 --json +``` + +Writes one group of a track to stdout over MoQ, with the session's own auth: +the counterpart of the relay's HTTP `/fetch//?group=N`. Without +`--group` it reads the newest group. By default stdout carries the frame +payloads back to back, byte for byte what `curl` gets from `/fetch`. `--json` +prints one line per frame instead: +`{"group": 42, "frame": 0, "size": 1234, "payload": ""}`, with a +zero-based `frame` and padded standard base64. + +`` is the literal track name. `/fetch` splits its path on the last `/`, +so the two agree only for names without one. Fetch only dials `--connect`, and +refuses a listener or cluster flag. It gives up after 30 seconds, as `/fetch` +does, and exits non-zero when the broadcast or group is not found, the relay +refuses, or the deadline passes. + ## Multiple stages Separate stages with `--` to bridge several broadcasts, or both directions, diff --git a/quest/main/README.md b/quest/main/README.md index ab06807e5c4..4d654664a11 100644 --- a/quest/main/README.md +++ b/quest/main/README.md @@ -91,7 +91,6 @@ do not add another media abstraction or a renderer crate during stabilization. - [Binding parity](/quest/main/binding-parity.md) - every wrapper reaches every moq-ffi method in its own idiom with moq-net's verbs - [Binary stats](/quest/main/stats-binary/README.md) - an allocation-free stats tick and an on-demand FlatBuffers `.fb.z` flavor with a checked-in schema - [Audio jitter target](/quest/main/audio-jitter-target/README.md) - the audio playout target is a measured estimate of arrival timing in both languages, not a round-trip guess -- [CLI fetch](/quest/main/cli-fetch.md) - `moq fetch` writes one group of a track to stdout, raw or as JSON lines ## Related diff --git a/quest/main/cli-fetch.md b/quest/main/cli-fetch.md deleted file mode 100644 index 064e740381b..00000000000 --- a/quest/main/cli-fetch.md +++ /dev/null @@ -1,31 +0,0 @@ -# [S] The fetch verb prints one group - -## Goal - -`moq --connect --broadcast fetch [--group N]` writes one -group of a track to stdout, the MoQ equivalent of the relay's HTTP -`/fetch//?group=N`. With no `--group` it fetches the latest -group, as the endpoint does. By default stdout carries the frame payloads -concatenated, byte-identical to `curl /fetch`, so a JSON catalog pipes into -`jq`. `` is the literal track name, never split on `/`, so the two -agree only for names without a slash; the endpoint splits its wildcard itself. -`--json` instead prints one -`{"group": , "frame": , "size": , "payload": ""}` per -frame, with a zero-based `frame` and padded standard base64 `payload`. - -## Plan - -- A MoQ verb like `ls`: a subscriber-only session that never publishes. One - 30-second deadline spans route discovery, group lookup, and the final frame - read, mirroring `serve_fetch`; it exits zero after the group's last frame and - non-zero on not found, refusal, or timeout, with the reason on stderr. -- Reuse the endpoint's semantics rather than copy its code: "latest" needs a - subscription to learn the newest sequence, then fetches it so an evicted - group is retrieved from upstream (`serve_fetch` in - `rs/moq-relay/src/web.rs`). If both need the helper, move it into a library - both depend on rather than keeping two copies. -- Update `doc/bin/cli.md` beside `ls`. -- Test: against an in-process relay, fetching a known sequence prints its - exact bytes, the default fetches the newest group, a missing sequence exits - non-zero, a lookup timeout and a frame-read timeout exit non-zero, and each - `--json` line parses to the full record and decodes to the same bytes. diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 07b2f87c0ca..353c879e4f4 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -89,6 +89,7 @@ pulseaudio = ["moq-audio?/pulseaudio"] anyhow = { workspace = true, features = ["backtrace"] } axum = { workspace = true } axum-server = { workspace = true } +base64 = { workspace = true } bytes = { workspace = true } hang = { workspace = true } humantime = { workspace = true } @@ -127,6 +128,8 @@ sd-notify = { workspace = true } # the wrong catalog track apart from one reading the right one: `moq-mux`'s producer # publishes hang and MSF from the same source, so both answer for an ordinary broadcast. moq-msf = { workspace = true } +# `test_relay` stands up a real relay for the `fetch` tests to read through. +moq-relay = { workspace = true, features = ["test-support"] } tempfile = { workspace = true } # `test-util` enables `tokio::time::pause` so the TS round-trip test simulates the # exporter drain timeouts instantly instead of waiting on the wall clock. diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 963d8b429c8..3fbb392b0de 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -2,7 +2,7 @@ //! //! Grammar: `moq [-- ]...`, where a stage is //! ` [endpoint opts]`, plus `moq play` for -//! native playback. +//! native playback and `moq fetch ` to read one group. //! //! - The MoQ side (`--connect`, the `--listen*` transport binds, `--cluster-lan`, //! and `--cluster-connect` / `--cluster-connect-api`; all optional, at least @@ -181,6 +181,12 @@ impl Invocation { self.typed.reject(command) } + /// Refuse every MoQ-side flag but `--connect` and `--broadcast`, on a verb that + /// only reads from a relay. Answered from the command line, like [`Self::reject`]. + pub fn dial_only(&self, command: &str) -> anyhow::Result<()> { + self.typed.dial_only(command) + } + /// Split `argv` on `--` and run each chunk through a real parser. pub fn try_parse_from(argv: I) -> Result where @@ -248,8 +254,8 @@ impl Invocation { } // Only `import` and `export` share an Origin. The rest own the process: `play` - // drives a window on the main thread, `transcode` builds its own Origin, and - // `token` / `devices` never touch the network at all. + // drives a window on the main thread, `transcode` builds its own Origin, `fetch` + // opens its own session, and `auth` / `devices` never touch the network at all. if let Some(command) = self.stages.iter().find(|command| !command.is_stageable()) { anyhow::bail!( "`{}` must be the only verb; it can't share a process with another `--` stage", @@ -505,6 +511,25 @@ impl MoqSide { /// the same reason it used to be out of it -- an ambient `MOQ_HOP` no longer /// reaches here, so a typed one can be refused like the rest. fn reject(&self, command: &str) -> anyhow::Result<()> { + if let Some(flag) = self.given().next() { + anyhow::bail!("`{command}` runs locally and takes no MoQ side; drop {flag}"); + } + Ok(()) + } + + /// Refuse every MoQ-side flag except the dial, on a verb that only reads from a + /// relay: a listener or cluster it would never serve is not silently ignored. + /// Private for the same reason as [`Self::reject`], and reached through + /// [`Invocation::dial_only`]. + fn dial_only(&self, command: &str) -> anyhow::Result<()> { + if let Some(flag) = self.given().find(|flag| !matches!(*flag, "--connect" | "--broadcast")) { + anyhow::bail!("`{command}` only dials a relay with --connect; drop {flag}"); + } + Ok(()) + } + + /// The MoQ-side flags this side was given. + fn given(&self) -> impl Iterator { #[cfg(feature = "cluster-lan")] let cluster_secret = self.cluster.lan.secret.is_some(); #[cfg(not(feature = "cluster-lan"))] @@ -516,7 +541,7 @@ impl MoqSide { // A legacy `--client-connect` must be rejected here too; the fold has already // landed it in `url`. - let ignored = [ + let flags = [ ("--connect", self.client.url.is_some()), ("--listen", self.server.bind.is_some()), ("--listen-tcp-bind", self.server.tcp.bind.is_some()), @@ -535,26 +560,24 @@ impl MoqSide { ("--broadcast", self.broadcast.is_some()), ("--hop", self.hop.is_some()), ]; - let ignored = ignored.into_iter().find(|(_, given)| *given).map(|(flag, _)| flag); #[cfg(unix)] - let ignored = ignored - .or_else(|| self.server.unix.bind.is_some().then_some("--listen-unix-bind")) - .or_else(|| { - let allow = &self.server.unix.allow; - [ - ("--listen-unix-allow-uid", !allow.uid.is_empty()), - ("--listen-unix-allow-gid", !allow.gid.is_empty()), - ("--listen-unix-allow-pid", !allow.pid.is_empty()), - ] - .into_iter() - .find_map(|(flag, given)| given.then_some(flag)) - }); - - if let Some(flag) = ignored { - anyhow::bail!("`{command}` runs locally and takes no MoQ side; drop {flag}"); - } - - Ok(()) + let unix = { + let allow = &self.server.unix.allow; + [ + ("--listen-unix-bind", self.server.unix.bind.is_some()), + ("--listen-unix-allow-uid", !allow.uid.is_empty()), + ("--listen-unix-allow-gid", !allow.gid.is_empty()), + ("--listen-unix-allow-pid", !allow.pid.is_empty()), + ] + }; + #[cfg(not(unix))] + let unix = []; + + flags + .into_iter() + .chain(unix) + .filter(|(_, given)| *given) + .map(|(flag, _)| flag) } } @@ -572,6 +595,8 @@ pub enum Command { /// The released spelling of [`Self::Export`]. #[usage(hide = true)] Subscribe(Export), + /// Write one group of a track to stdout. + Fetch(crate::fetch::Args), /// Play a broadcast in a native window and speaker. #[cfg(feature = "play")] Play(crate::play::Args), @@ -629,6 +654,7 @@ impl Command { match self { Self::Import(_) | Self::Publish(_) => "import", Self::Export(_) | Self::Subscribe(_) => "export", + Self::Fetch(_) => "fetch", #[cfg(feature = "play")] Self::Play(_) => "play", #[cfg(feature = "transcode")] diff --git a/rs/moq-cli/src/fetch.rs b/rs/moq-cli/src/fetch.rs new file mode 100644 index 00000000000..11cfc902584 --- /dev/null +++ b/rs/moq-cli/src/fetch.rs @@ -0,0 +1,360 @@ +//! `moq fetch`: write one group of a track to stdout, the MoQ counterpart of the +//! relay's HTTP `/fetch//?group=N`. + +use std::time::Duration; + +use anyhow::Context; +use base64::Engine; +use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio::time::{Instant, timeout_at}; + +use crate::args::MoqSide; + +/// How long discovery, lookup, and every frame read may take in total, matching +/// the relay's HTTP `/fetch`. +const TIMEOUT: Duration = Duration::from_secs(30); + +/// Read one group of a track and write it to stdout. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct Args { + /// The track name, taken literally: a `/` in it is part of the name. + pub track: String, + + /// The group sequence to read; the newest group when omitted. + #[usage(long)] + pub group: Option, + + /// Print one JSON object per frame instead of the concatenated payloads. + #[usage(long)] + pub json: bool, +} + +/// One frame of `--json` output. +#[derive(serde::Serialize)] +struct Frame { + group: u64, + frame: u64, + size: u64, + /// Padded standard base64. + payload: String, +} + +/// Fetch the group `args` names from the broadcast `--broadcast` names and write it to stdout. +pub async fn run(moq: MoqSide, args: Args, net: crate::Net) -> anyhow::Result<()> { + let mut stdout = tokio::io::stdout(); + fetch(&moq, &args, &net, Instant::now() + TIMEOUT, &mut stdout).await +} + +async fn fetch( + moq: &MoqSide, + args: &Args, + net: &crate::Net, + deadline: Instant, + out: &mut (impl AsyncWrite + Unpin), +) -> anyhow::Result<()> { + let url = moq + .client + .url + .clone() + .context("`fetch` dials a relay: pass --connect ")?; + let broadcast = moq.broadcast.clone().unwrap_or_default(); + + // Subscribe-only: this session reads and never publishes. + let origin = moq_tokio::origin::spawn(); + let client = net + .client(moq.client.clone())? + .with_subscriber(origin.clone()) + .with_reconnect(false); + + let result = timeout_at(deadline, async { + // Held until the last frame is read; dropping it closes the session. + let _connection = client.connect(url).established().await.context("failed to connect")?; + + // Wait for a covering route rather than asking on the spot: the announcement + // is still in flight right after connecting. + let broadcast = origin + .consume() + .routed_broadcast(broadcast.as_str()) + .await + .with_context(|| format!("broadcast `{}` not found", crate::display_name(&broadcast)))?; + let track = broadcast.track(&args.track)?; + let mut group = moq_relay::fetch_group(&track, args.group) + .await + .with_context(|| match args.group { + Some(sequence) => format!("group {sequence} of `{}` not found", args.track), + None => format!("no group of `{}` found", args.track), + })?; + + // A missing sequence can resolve to a group that fails on its first read. + let sequence = group.sequence; + let mut index = 0; + while let Some(frame) = group + .read_frame() + .await + .with_context(|| format!("failed to read group {sequence} of `{}`", args.track))? + { + match args.json { + true => { + let mut line = serde_json::to_vec(&Frame { + group: sequence, + frame: index, + size: frame.payload.len() as u64, + payload: base64::engine::general_purpose::STANDARD.encode(&frame.payload), + })?; + line.push(b'\n'); + out.write_all(&line).await?; + } + false => out.write_all(&frame.payload).await?, + } + index += 1; + } + out.flush().await?; + anyhow::Ok(()) + }) + .await; + + result.unwrap_or_else(|_| anyhow::bail!("fetch timed out")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::args::{Command, Invocation}; + use crate::test_env::EnvGuard; + use hang::moq_net; + + /// The frames of each finished group on the `data` track. + fn frames(sequence: u64) -> [Vec; 2] { + [ + format!("g{sequence}f0").into_bytes(), + format!("g{sequence}f1").into_bytes(), + ] + } + + /// A running relay with a publisher attached, and the flags that reach it. + struct Fixture { + /// `--connect` and the TLS pin for the relay's generated certificate. + connect: [String; 4], + /// The relay's HTTP listener, for comparing against `/fetch`. + http: std::net::SocketAddr, + /// Kept so the published broadcast, its tracks, and the open group stay live. + _publisher: ( + moq_tokio::Connection, + moq_net::broadcast::Producer, + Vec, + moq_net::group::Producer, + ), + } + + impl Fixture { + /// Publish `demo` with three finished groups on `data`, no group on `empty`, + /// and one frame of a group that never finishes on `live`. + async fn new() -> Self { + let _ = moq_tokio::crypto::install_default(); + let fixture = moq_relay::test_relay().await.expect("test relay"); + let ready = fixture.relay.ready(); + tokio::spawn(fixture.relay.run()); + ready.wait().await.expect("relay ready"); + + let connect = [ + "--connect".to_string(), + fixture.url.to_string(), + "--connect-tls-fingerprint".to_string(), + fixture.fingerprint.clone(), + ]; + + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast("demo").expect("broadcast"); + broadcast.announce(Default::default()).expect("announce"); + + let data = broadcast.create_track("data", None).expect("data track"); + for sequence in 0..3 { + let mut group = data.append_group().expect("group"); + for frame in frames(sequence) { + group.write_frame(moq_net::Timestamp::ZERO, frame).expect("frame"); + } + group.finish().expect("finish"); + } + let empty = broadcast.create_track("empty", None).expect("empty track"); + let live = broadcast.create_track("live", None).expect("live track"); + let mut open = live.append_group().expect("open group"); + open.write_frame(moq_net::Timestamp::ZERO, b"first".as_ref()) + .expect("frame"); + + let (moq, _) = parse(&connect, &[]); + let connection = net() + .client(moq.client.clone()) + .expect("client") + .with_publisher(origin.consume()) + .with_reconnect(false) + .connect(fixture.url.clone()) + .established() + .await + .expect("publisher connects"); + + Self { + connect, + http: fixture.http, + _publisher: (connection, broadcast, vec![data, empty, live], open), + } + } + + /// Run `moq --broadcast demo fetch ` with `timeout`, returning + /// the outcome and what it wrote. + async fn fetch(&self, args: &[&str], timeout: Duration) -> (anyhow::Result<()>, Vec) { + let (moq, args) = parse(&self.connect, args); + let mut out = Vec::new(); + let result = super::fetch(&moq, &args, &net(), Instant::now() + timeout, &mut out).await; + (result, out) + } + + /// The relay's HTTP `/fetch` body for the same group. + async fn curl(&self, query: &str) -> Vec { + let response = reqwest::get(format!("http://{}/fetch/demo/data{query}", self.http)) + .await + .expect("HTTP fetch"); + assert_eq!(response.status(), 200); + response.bytes().await.expect("HTTP body").to_vec() + } + } + + /// Parse a fetch invocation the way `main` does. + fn parse(connect: &[String], args: &[&str]) -> (MoqSide, Args) { + let argv = ["moq"] + .into_iter() + .chain(connect.iter().map(String::as_str)) + .chain(["--broadcast", "demo", "fetch"]) + .chain(args.iter().copied()) + .chain(args.is_empty().then_some("data")); + let mut cli = Invocation::try_parse_from(argv).expect("parse"); + cli.dial_only("fetch").expect("only the dial"); + match cli.stages.remove(0) { + Command::Fetch(args) => (cli.moq, args), + _ => unreachable!("parsed a fetch"), + } + } + + fn net() -> crate::Net { + crate::Net { + quic: Default::default(), + #[cfg(feature = "iroh")] + iroh: None, + } + } + + const ENV: &[&str] = &["MOQ_CONNECT", "MOQ_HOP"]; + + /// A known sequence prints its frames back to back, the same bytes as `/fetch`. + #[tokio::test] + async fn a_sequence_prints_its_exact_bytes() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + + let (result, out) = fixture.fetch(&["data", "--group", "1"], TIMEOUT).await; + result.expect("fetch"); + assert_eq!(out, frames(1).concat()); + assert_eq!(out, fixture.curl("?group=1").await); + } + + /// No `--group` reads the newest group, as `/fetch` does by default. + #[tokio::test] + async fn the_default_is_the_newest_group() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + + let (result, out) = fixture.fetch(&["data"], TIMEOUT).await; + result.expect("fetch"); + assert_eq!(out, frames(2).concat()); + assert_eq!(out, fixture.curl("").await); + } + + #[tokio::test] + async fn a_missing_sequence_fails() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + + let (result, out) = fixture.fetch(&["data", "--group", "99"], TIMEOUT).await; + let err = result.expect_err("group 99 does not exist"); + let err = format!("{err:#}"); + assert!(err.contains("group 99") && err.contains("not found"), "{err}"); + assert!(out.is_empty()); + } + + /// A track with no group never resolves "newest", so the deadline ends it. + #[tokio::test] + async fn a_lookup_times_out() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + + let (result, out) = fixture.fetch(&["empty"], Duration::from_millis(500)).await; + let err = result.expect_err("no group ever arrives"); + assert!(err.to_string().contains("timed out"), "{err:#}"); + assert!(out.is_empty()); + } + + /// A group that never finishes is cut off by the same deadline, after the frames + /// that did arrive. + #[tokio::test] + async fn a_frame_read_times_out() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + + let (result, out) = fixture + .fetch(&["live", "--group", "0"], Duration::from_millis(500)) + .await; + let err = result.expect_err("the group never finishes"); + assert!(err.to_string().contains("timed out"), "{err:#}"); + assert_eq!(out, b"first"); + } + + /// Each `--json` line is the whole record and decodes to the frame's bytes. + #[tokio::test] + async fn json_lines_decode_to_the_frames() { + let _env = EnvGuard::clear(ENV); + let fixture = Fixture::new().await; + + let (result, out) = fixture.fetch(&["data", "--group", "1", "--json"], TIMEOUT).await; + result.expect("fetch"); + + let lines: Vec = out + .split(|&byte| byte == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice(line).expect("a JSON line")) + .collect(); + assert_eq!(lines.len(), 2); + for (index, (line, frame)) in lines.iter().zip(frames(1)).enumerate() { + let payload = base64::engine::general_purpose::STANDARD + .decode(line["payload"].as_str().expect("payload")) + .expect("base64"); + assert_eq!( + *line, + serde_json::json!({ + "group": 1, + "frame": index, + "size": frame.len(), + "payload": line["payload"], + }) + ); + assert_eq!(payload, frame); + } + } + + /// A listener or cluster flag is refused rather than silently never served. + #[test] + fn only_the_dial_is_accepted() { + let _env = EnvGuard::clear(ENV); + let cli = Invocation::try_parse_from([ + "moq", + "--connect", + "http://relay", + "--listen-tcp-bind", + "127.0.0.1:0", + "fetch", + "data", + ]) + .expect("parse"); + let err = cli.dial_only("fetch").unwrap_err().to_string(); + assert!(err.contains("--listen-tcp-bind"), "{err}"); + } +} diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 927745de406..0cd29566c4c 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -10,6 +10,7 @@ mod complete; #[cfg(feature = "capture")] mod devices; mod duration; +mod fetch; mod hls; mod moq; mod play; @@ -316,6 +317,9 @@ async fn main() -> anyhow::Result<()> { } } + if let [Command::Fetch(_)] = stages.as_slice() { + cli.dial_only("fetch")?; + } cli.moq.validate()?; let net = Net { @@ -334,6 +338,7 @@ async fn main() -> anyhow::Result<()> { // lone one of those runs by itself; everything else is a list of stages. if stages.len() == 1 && !stages[0].is_stageable() { match stages.remove(0) { + Command::Fetch(args) => return fetch::run(cli.moq, args, net).await, #[cfg(feature = "play")] Command::Play(args) => return run_play(cli.moq, args, net).await, #[cfg(feature = "transcode")] diff --git a/rs/moq-relay/src/fetch.rs b/rs/moq-relay/src/fetch.rs new file mode 100644 index 00000000000..94dfb063ca8 --- /dev/null +++ b/rs/moq-relay/src/fetch.rs @@ -0,0 +1,21 @@ +/// Fetch one group of `track`: the one at `sequence`, or the newest when `None`. +/// +/// The newest needs a live subscription to learn its sequence, since a fetch can +/// only retrieve a sequence already known. Once known it is fetched rather than +/// read off the subscription, so an evicted group is retrieved from upstream +/// instead of waited on forever. Returns [`moq_net::Error::NotFound`] when the +/// group can never be served, including a track that ends before any group. +pub async fn fetch_group( + track: &moq_net::track::Consumer, + sequence: Option, +) -> moq_net::Result { + if let Some(sequence) = sequence { + return track.fetch_group(sequence, None).await; + } + + let mut subscriber = track.subscribe(None).await?; + match subscriber.latest() { + Some(sequence) => track.fetch_group(sequence, None).await, + None => subscriber.recv_group().await?.ok_or(moq_net::Error::NotFound), + } +} diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index 0998cba3db4..96dd2afbf3a 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -16,6 +16,7 @@ pub mod cluster; mod config; mod connection; mod duration; +mod fetch; mod http_client; pub mod internal; mod listener; @@ -52,6 +53,7 @@ fn configured_tier(label: Option) -> moq_net::stats::Tier { pub use config::*; pub use connection::*; +pub use fetch::*; pub use relay::*; #[cfg(feature = "test-support")] pub use test_support::*; diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 8b89b5ff874..f948c49fc04 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -920,31 +920,13 @@ async fn serve_fetch( // freshly-connected subscribers don't get a spurious 404 before gossip arrives. let consumer = origin.consume(); let broadcast = consumer.routed_broadcast("").await.map_err(|_| StatusCode::NOT_FOUND)?; - let group = match params.group { - // "latest" needs a live subscription to learn the newest sequence, since a - // fetch can only retrieve a sequence you already know. Once it's known, fetch - // it rather than reading it off the subscription, so an evicted latest is - // re-retrieved from upstream instead of waited on forever. - FetchGroup::Latest => { - async { - let consumer = broadcast.track(&track)?; - let mut sub = consumer.subscribe(None).await?; - match sub.latest() { - Some(sequence) => consumer.fetch_group(sequence, None).await.map(Some), - None => sub.recv_group().await, - } - } - .await - } - // A one-shot fetch, no subscription required. - FetchGroup::Num(sequence) => async { broadcast.track(&track)?.fetch_group(sequence, None).await } - .await - .map(Some), + let sequence = match params.group { + FetchGroup::Num(sequence) => Some(sequence), + FetchGroup::Latest => None, }; - - let group = match group { - Ok(Some(group)) => group, - Ok(None) | Err(moq_net::Error::NotFound) => return Err(StatusCode::NOT_FOUND), + let group = match async { crate::fetch_group(&broadcast.track(&track)?, sequence).await }.await { + Ok(group) => group, + Err(moq_net::Error::NotFound) => return Err(StatusCode::NOT_FOUND), Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), }; From 5102cc24a0e4b31c7f16413b44bc169662e8db57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 15:18:15 -0700 Subject: [PATCH 4/4] fix(moq-cli): skip validating ambient listener settings for `moq fetch` `fetch` only dials, so an unrelated MOQ_LISTEN or cluster setting in the environment should not refuse it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 0cd29566c4c..27ee2604512 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -317,10 +317,13 @@ async fn main() -> anyhow::Result<()> { } } + // `fetch` only dials, so an ambient listener or cluster setting it never uses + // is not validated either. if let [Command::Fetch(_)] = stages.as_slice() { cli.dial_only("fetch")?; + } else { + cli.moq.validate()?; } - cli.moq.validate()?; let net = Net { quic: cli.moq.quic.clone(),