diff --git a/js/net/src/broadcast.test.ts b/js/net/src/broadcast.test.ts index efbbe4411ad..6251563cf24 100644 --- a/js/net/src/broadcast.test.ts +++ b/js/net/src/broadcast.test.ts @@ -1,7 +1,7 @@ import { expect, setSystemTime, test } from "bun:test"; import { Manual, Milli, Nano } from "@moq/clock"; import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; -import { GroupTooLarge, NotFound, TimestampMismatch } from "./error.ts"; +import { GroupTooLarge, NotFound, ProtocolViolation, TimestampMismatch } from "./error.ts"; import { Producer as GroupProducer, MAX_GROUP_FRAMES } from "./group.ts"; import { TrackSequences } from "./internal.ts"; import { textFrame } from "./mock.ts"; @@ -123,6 +123,55 @@ test("a replacement producer refuses a rewind past its predecessor", () => { next.close(); }); +// A replacement that closes before writing anything still ends past its predecessor's groups: +// an end below them would contradict the ones subscribers already received. +test("an empty replacement ends past its predecessor's groups", async () => { + const broadcast = new BroadcastProducer(); + const first = broadcast.createTrack("media", { timescale: Timescale.MILLI }); + first.appendGroup().close(); + first.close(); + + const second = broadcast.createTrack("media", { timescale: Timescale.MILLI }); + const subscriber = second.subscribe(); + expect(() => second.finishAt({ group: 0, frame: 0 })).toThrow(ProtocolViolation); + expect(() => second.finishAt({ group: 0, frame: 1 })).toThrow(ProtocolViolation); + second.close(); + expect(await subscriber.finished()).toEqual({ group: 1, frame: 0 }); + broadcast.close(); +}); + +test("a removed producer writes nothing once its replacement claims the name", async () => { + const broadcast = new BroadcastProducer(); + const first = broadcast.createTrack("media", { timescale: Timescale.MILLI }); + first.appendGroup().close(); + broadcast.removeTrack("media"); + + const second = broadcast.createTrack("media", { timescale: Timescale.MILLI }); + const subscriber = second.subscribe(); + second.close(); + expect(() => first.appendGroup()).toThrow("replaced"); + expect(() => first.writeGroup(new GroupProducer(1))).toThrow("replaced"); + expect(() => first.appendDatagram(Timestamp.fromMillis(0), new Uint8Array())).toThrow("replaced"); + expect(await subscriber.finished()).toEqual({ group: 1, frame: 0 }); + first.close(); + broadcast.close(); +}); + +test("a removed producer's abort ignores its replacement's groups", async () => { + const broadcast = new BroadcastProducer(); + const first = broadcast.createTrack("media", { timescale: Timescale.MILLI }); + first.finishAt({ group: 2, frame: 0 }); + broadcast.removeTrack("media"); + + const second = broadcast.createTrack("media", { timescale: Timescale.MILLI }); + for (let i = 0; i < 3; i++) second.appendGroup().close(); + const abort = new Error("cut off"); + first.close(abort); + expect(first.closed.peek()).toBe(abort); + second.close(); + broadcast.close(); +}); + // A name keeps one sequence namespace across producer replacements. The application owns where a // replacement starts: an append that would reuse an earlier producer's sequence throws. test("a replacement producer refuses to append a sequence the name already wrote", async () => { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 0c5764c540f..b9b558e0785 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -209,6 +209,8 @@ export interface TrackSequence { next: number; /** Open producers writing through this namespace. */ holders: number; + /** The open producer the name's newest claim went to; a later claim supersedes it. */ + owner?: Producer; /** * The highest-sequence checked group start any producer of the name wrote, which bounds a * replacement's groups the way the eviction watermark bounds a track's own. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 8d1a6b2f085..c9c6eae1211 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -763,6 +763,10 @@ export class Producer { // `max_sequence`, and where the next append lands. The namespace above can run ahead of // it: earlier producers of the same broadcast track advanced it too. #received = 0; + // Where the namespace stood when a replacement claimed the name. From then on it is this + // producer's inherited edge, so the replacement's groups never count as its progress, and + // it writes nothing more, so it can't land a group past the replacement's end. + #handoff?: number; // Recently written source groups, retained for replay to late subscribers and // pruned once idle for longer than the cache window. Each entry tracks the mirror @@ -839,6 +843,11 @@ export class Producer { bindProducerSequence = (producer, sequence) => { const own = producer.#sequence; if (own === sequence) return; + if (producer.#state.closed.peek() === undefined) { + const owner = sequence.owner; + if (owner && owner.#state.closed.peek() === undefined) owner.#handoff = sequence.next; + sequence.owner = producer; + } sequence.next = Math.max(sequence.next, producer.#received); // What earlier producers of the name wrote is gone from this track's cache, so their // highest checked start is its eviction watermark. @@ -1448,6 +1457,7 @@ export class Producer { // Refuse a write once the track is closed, or at or past its declared end. #writable(sequence: number): void { if (this.#state.closed.peek() !== undefined) throw new Error("track is closed"); + if (this.#handoff !== undefined) throw new Error(`track ${this.name} was replaced`); const end = this.#state.end.peek(); if (end !== undefined && sequence >= finalSequence(end)) { throw new Error(`sequence ${sequence} is at or past the track's end ${end.group}/${end.frame}`); @@ -1599,9 +1609,20 @@ export class Producer { * at or past it is refused. Unlike {@link close} it is not terminal: call `close()` once * the remaining groups are written. Readers end once the live edge reaches the end's * group, independently of closure. Throws if the track is closed, already has an end, or - * `end` would orphan a group or frame already produced. + * `end` falls before or inside a group already produced, including one an earlier producer + * of the name wrote, which a replacement continues. */ finishAt(end: Location): void { + // A replacement continues its name's timeline, so it can't end before or inside a group + // an earlier producer wrote; only its own last group can be cut mid-way. + const ownLast = end.frame !== 0 && this.#received === end.group + 1; + if (end.group < this.#inherited() && !ownLast) { + throw new ProtocolViolation(`track end ${end.group}/${end.frame} is inside the name's earlier groups`); + } + this.#finishAt(end); + } + + #finishAt(end: Location): void { if (this.#state.closed.peek() !== undefined) throw new Error("track is closed"); const declared = this.#state.end.peek(); if (declared !== undefined) throw new Error(`track already ends at ${declared.group}/${declared.frame}`); @@ -1609,11 +1630,12 @@ export class Producer { } // The wire's end: repeating it is a no-op, and a mid-group end may refine the group - // boundary it falls just before. Mirrors the Rust `finish_at` on a `received()` copy. + // boundary it falls just before. Mirrors the Rust `finish_at` on a `received()` copy, + // which answers to its publisher's end rather than the name's local writes. #endPending(end: Location): void { const declared = this.#state.end.peek(); if (declared === undefined) { - this.finishAt(end); + this.#finishAt(end); } else if (declared.group !== end.group || declared.frame !== end.frame) { if (declared.frame !== 0 || end.frame === 0 || finalSequence(end) !== declared.group) { throw new ProtocolViolation(`conflicting track end: ${end.group}/${end.frame}`); @@ -1672,6 +1694,18 @@ export class Producer { return !this.#arrivals.covers(Math.min(floor, final), final); } + // One past the highest sequence produced: where the live edge stands against the end. A + // replacement starts at its name's, since what earlier producers wrote never arrives here. + // Mirrors the Rust `edge`. + #liveEdge(): number { + return Math.max(this.#received, this.#inherited()); + } + + // One past what the name's producers wrote, up to the replacement that superseded this one. + #inherited(): number { + return this.#handoff ?? this.#sequence.next; + } + #declareEnd(end: Location): void { const { group, frame } = end; if (!Number.isSafeInteger(group) || group < 0 || !Number.isSafeInteger(frame) || frame < 0) { @@ -1703,7 +1737,7 @@ export class Producer { #endLargest(): void { if (this.#state.largestEnded.peek() || this.#largestOpen) return; const end = this.#state.end.peek(); - if (end === undefined || this.#received < finalSequence(end)) return; + if (end === undefined || this.#liveEdge() < finalSequence(end)) return; for (const state of [this.#state, ...this.#sinks]) state.largestEnded.set(true); } @@ -1722,14 +1756,17 @@ export class Producer { // A gap past the grace was lost before the abort rather than cut off by it. if (abort && this.#tailPending) this.#arrivals.expire(now(), this.#grace()); if (abort && this.#settled()) abort = undefined; + // A replacement continues its name's timeline: ending below an earlier producer's + // groups would contradict the ones subscribers already received. if (abort === undefined && this.#state.end.peek() === undefined) { - this.#declareEnd({ group: this.#received, frame: 0 }); + this.#declareEnd({ group: this.#liveEdge(), frame: 0 }); } // Nobody will finish these, so a subscriber that has not taken one yet never sees it. // Not evicted: a reader already holding one keeps its frames and sees the abort. const open = abort ? this.#cache.filter((entry) => entry.group.closed.peek() === undefined) : []; closeTrackState(this.#state, abort); this.#sequence.holders--; + if (this.#sequence.owner === this) this.#sequence.owner = undefined; for (const { group } of this.#cache) group.close(abort); for (const entry of open) { this.#unlink(entry); @@ -1752,7 +1789,7 @@ export class Producer { const end = this.#state.end.peek(); if (end === undefined) return false; const final = finalSequence(end); - if (this.#received < final || this.#owes(final)) return false; + if (this.#liveEdge() < final || this.#owes(final)) return false; return this.#cache.every(({ group }) => group.sequence >= final || group.closed.peek() === null); } diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 07e34532d8a..f4b0af63950 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -1860,6 +1860,29 @@ mod test { ); } + /// A replacement can't declare an end before or inside a group an earlier producer of the + /// name wrote: a subscriber that received that group would read it as a contradiction. + #[test] + fn replacement_refuses_an_end_below_its_predecessor() { + let producer = Info::new().produce(); + let first = producer.create_track("media", crate::track::Info::timed()).unwrap(); + first.append_group().unwrap().finish().unwrap(); + first.finish().unwrap(); + drop(first); + + let mut second = producer.create_track("media", crate::track::Info::timed()).unwrap(); + for end in [ + crate::track::Position::group(0), + crate::track::Position { group: 0, frame: 1 }, + ] { + assert!( + matches!(second.finish_at(end), Err(Error::ProtocolViolation)), + "{end:?}" + ); + } + second.finish_at(crate::track::Position::group(1)).unwrap(); + } + /// A replacement continues the name's timeline, so its groups can't start before the /// highest one an earlier producer checked, while a new broadcast starts over. #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 5a0f04c654d..c9e915a45e6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2198,8 +2198,11 @@ impl TrackState { } /// One past the highest sequence produced: where the live edge stands against the end. + /// A replacement starts at its name's, since what earlier producers wrote never arrives here. fn edge(&self) -> u64 { - self.max_sequence.map_or(0, |max| max.saturating_add(1)) + self.max_sequence + .map_or(0, |max| max.saturating_add(1)) + .max(self.sequence.next()) } /// Whether a group below `fin` the wire subscription still owes has yet to arrive. @@ -2924,7 +2927,8 @@ impl Producer { /// Mark the track as finished after the last appended group. /// - /// Ends the track at the head of the group after the current max_sequence. + /// Ends the track at the head of the group after the largest one written to its name, + /// including by an earlier producer this one replaces. /// No new groups at or above it can be appended. /// NOTE: Old groups with lower sequence numbers can still arrive. pub fn finish(&self) -> Result<()> { @@ -2933,6 +2937,9 @@ impl Producer { Some(max) => max.checked_add(1).ok_or(coding::BoundsExceeded)?, None => 0, }; + // A replacement continues its name's timeline: ending below an earlier producer's + // groups would contradict the ones subscribers already received. + let final_sequence = final_sequence.max(state.sequence.next()); state.set_end(Position::group(final_sequence)) } @@ -2953,13 +2960,21 @@ impl Producer { /// /// Repeating the declared end is a no-op, and a mid-group end may refine the group /// boundary it falls just before (a peer may name only the boundary first). Any other - /// second end is [`Error::ProtocolViolation`]. + /// second end is [`Error::ProtocolViolation`], and so is one before or inside a group an + /// earlier producer of the name wrote, which a replacement continues. /// /// On a [received](Self::received) copy, groups below the end may still be in flight: /// readers wait for a missing one until a grace passes, the subscription's max delay /// or one second without one, before ending at the boundary. pub fn finish_at(&mut self, end: Position) -> Result<()> { let mut state = self.modify()?; + // A replacement continues its name's timeline, so it can't end before or inside a group + // an earlier producer wrote; only its own last group can be cut mid-way. A received + // copy answers to its publisher's end, which the session judges. + let own_last = end.frame != 0 && state.max_sequence == Some(end.group); + if !state.received && end.group < state.sequence.next() && !own_last { + return Err(Error::ProtocolViolation); + } match state.end { Some(declared) if declared == end => return Ok(()), Some(declared) if declared.frame == 0 && end.frame != 0 && end.final_sequence() == Some(declared.group) => { diff --git a/rs/moq-net/tests/integration/empty_replacement.rs b/rs/moq-net/tests/integration/empty_replacement.rs new file mode 100644 index 00000000000..8c34a501ea8 --- /dev/null +++ b/rs/moq-net/tests/integration/empty_replacement.rs @@ -0,0 +1,124 @@ +#![cfg(feature = "async")] + +//! A dynamic handler serves a track, the relay's copy goes idle, and the next request gets a +//! replacement producer that finishes before writing anything. The relay still caches the +//! first producer's group, so the replacement's end must not fall below it: an end there +//! contradicts a received group, which the relay treats as a protocol violation and closes +//! the whole session. + +use crate::support; + +use std::time::Duration; + +use moq_net::{Hop, Timestamp, Version, broadcast, track}; +use support::harness::{MockConnectOptions, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(5); + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +/// Subscribe to `name` through `remote`, answering the request with a new producer. +async fn serve( + remote: &broadcast::Consumer, + dynamic: &mut broadcast::Dynamic, + name: &str, +) -> (track::Producer, track::Subscriber) { + let answer = async { + let request = moq_clock::timeout(TIMEOUT, dynamic.requested_track()) + .await + .expect("the request never reached the publisher") + .unwrap(); + request.accept(track::Info::default().with_timescale(moq_net::Timescale::MILLI)) + }; + let subscribing = moq_clock::timeout(TIMEOUT, remote.track(name).unwrap().subscribe(None)); + let (producer, subscriber) = futures::future::join(answer, subscribing).await; + let subscriber = subscriber.expect("subscribe hung").expect("subscribe failed"); + (producer, subscriber) +} + +async fn empty_replacement_ends_after_the_cached_group(version: &str) { + let version: Version = version.parse().unwrap(); + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("room").unwrap(); + let mut dynamic = broadcast.dynamic(); + broadcast.announce(Default::default()).unwrap(); + + let relay = produce_origin(2); + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + let pair = connect_mock(options).await; + + let consumer = relay.consume(); + moq_clock::timeout(TIMEOUT, consumer.routed("room")) + .await + .unwrap() + .unwrap(); + let remote = moq_clock::timeout(TIMEOUT, consumer.request_broadcast("room", None)) + .await + .unwrap() + .unwrap(); + + // The first producer writes group 0, which the relay caches. + let (first, mut subscriber) = serve(&remote, &mut dynamic, "feed").await; + let mut group = first.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, &b"first"[..]).unwrap(); + group.finish().unwrap(); + let group = moq_clock::timeout(TIMEOUT, subscriber.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(group.sequence, 0, "{version:?}"); + drop((group, subscriber)); + + // The relay's copy goes idle and cancels upstream; the handler finishes and lets go. + moq_clock::timeout(TIMEOUT, support::unwanted(first.demand())) + .await + .unwrap() + .unwrap(); + first.finish().unwrap(); + drop(first); + + // The idle copy, still holding group 0, subscribes again, and the replacement finishes + // before writing anything. + let (second, mut subscriber) = serve(&remote, &mut dynamic, "feed").await; + second.finish().unwrap(); + assert_eq!(second.end(), Some(track::Position::group(1)), "{version:?}"); + + // Its own readers end at once: the earlier producer's groups never arrive here. + let mut local = second.subscribe(None); + let ended = moq_clock::timeout(TIMEOUT, local.recv_group()) + .await + .expect("a local reader of the replacement never ended"); + assert!(matches!(ended, Ok(None)), "{version:?}"); + + // The relay's copy ends after the group it cached, and its session stays up. + loop { + match moq_clock::timeout(TIMEOUT, subscriber.recv_group()) + .await + .expect("the replacement never ended") + { + Ok(Some(_)) => {} + Ok(None) => break, + Err(err) => panic!("{version:?}: the replacement's end failed the subscription: {err}"), + } + } + if let Ok(err) = moq_clock::timeout(TIMEOUT, pair.client.closed()).await { + panic!("{version:?}: the relay's session closed: {err}"); + } +} + +#[moq_clock::test] +async fn empty_replacement_lite_06() { + empty_replacement_ends_after_the_cached_group("moq-lite-06").await; +} + +#[moq_clock::test] +async fn empty_replacement_lite_07() { + empty_replacement_ends_after_the_cached_group("moq-lite-07-wip").await; +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 1001d50ea54..62d2576a9f6 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -18,6 +18,7 @@ mod datagram; mod dial_split_horizon; mod drained_claim; mod empty_group; +mod empty_replacement; mod extensions; mod fetch_cancel; mod fetch_caps;