Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 86 additions & 1 deletion js/net/src/lite/subscriber.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { type AnnounceBroadcast, AnnounceInit, AnnounceOk, encodeAnnounceBroadca
import { Group } from "./group.ts";
import { Probe } from "./probe.ts";
import { StreamId } from "./stream.ts";
import { encodeSubscribeResponse, SubscribeEnd } from "./subscribe.ts";
import { SUBSCRIBE_SETUP_TIMEOUT_MS, Subscriber } from "./subscriber.ts";
import { Track, TrackInfo } from "./track.ts";
import { Version } from "./version.ts";
Expand Down Expand Up @@ -862,6 +863,8 @@ interface FakeStream {
written: Uint8Array[];
// Hands the stream to the subscriber, for an open the session was told to park.
release: () => void;
// Fails every later write with `reason`, as a peer's STOP_SENDING does. Resolves on the first.
stop: (reason: unknown) => Promise<void>;
}

// A session whose streams the test answers by hand and that never fails them on its own, so
Expand All @@ -884,6 +887,7 @@ function fakeSession(park: number[] = [], stall: number[] = []) {
let onFinish!: () => void;
let onWrite!: () => void;
let release!: () => void;
let stopped: { reason: unknown; refused: () => void } | undefined;
const reading = new Promise<void>((resolve) => (onRead = resolve));
const aborted = new Promise<unknown>((resolve) => (onAbort = resolve));
const finished = new Promise<void>((resolve) => (onFinish = resolve));
Expand All @@ -902,6 +906,10 @@ function fakeSession(park: number[] = [], stall: number[] = []) {
const written: Uint8Array[] = [];
const writable = new WritableStream<Uint8Array>({
write: (chunk) => {
if (stopped) {
stopped.refused();
return Promise.reject(stopped.reason);
}
written.push(new Uint8Array(chunk));
onWrite();
return new Promise<void>((resolve) => {
Expand All @@ -921,7 +929,11 @@ function fakeSession(park: number[] = [], stall: number[] = []) {
});
const opened = new Promise((resolve) => (release = () => resolve({ readable, writable })));
if (!park.includes(streams.length)) release();
streams.push({ inbound, reading, aborted, finished, writing, written, release });
const stop = (reason: unknown) =>
new Promise<void>((refused) => {
stopped = { reason, refused };
});
streams.push({ inbound, reading, aborted, finished, writing, written, release, stop });
count.set(streams.length);
return opened;
},
Expand Down Expand Up @@ -1699,6 +1711,79 @@ test("a publisher FIN without SUBSCRIBE_END still violates an open subscription"
}
});

// A publisher that finishes the track stops reading its subscribe stream, so a SUBSCRIBE_UPDATE
// written then fails. The track still ends with the publisher's finish; a reset still fails it.
test.each([
["lite-05 finish", Version.DRAFT_05, "finish"],
["lite-06 finish", Version.DRAFT_06, "finish"],
["lite-07 finish", Version.DRAFT_07, "finish"],
["lite-07 reset", Version.DRAFT_07, "reset"],
] as const)("an update the publisher stopped reading defers to its %s", async (_, version, end) => {
const { quic, streams, opened } = fakeSession();
const subscriber = new Subscriber(quic, version, HopSchema.parse(1n));
const broadcast = subscriber.consume(Path.from("room"));
const track = broadcast.track("video").subscribe();
try {
await opened(0);
await answerTrackInfo(streams[0], version);
const stream = await opened(1);
await stream.reading;

const stopped = Object.assign(new Error("The stream was aborted by the remote server"), {
source: "stream",
streamErrorCode: StreamCode.Cancel,
});
const refused = stream.stop(stopped);
track.update({ priority: 1 });
await refused;
await drainReactions();
expect(track.closed.peek()).toBeUndefined();

if (end === "finish") {
const chunks: Uint8Array[] = [];
const writer = new Writer(
new WritableStream<Uint8Array>({ write: (chunk) => void chunks.push(new Uint8Array(chunk)) }),
version,
);
await encodeSubscribeResponse(writer, { end: new SubscribeEnd(0) }, version);
for (const chunk of chunks) stream.inbound.enqueue(chunk);
stream.inbound.close();
expect(await track.closed).toBeNull();
} else {
stream.inbound.error(
Object.assign(new Error("reset"), { source: "stream", streamErrorCode: StreamCode.NotFound }),
);
const err = await track.closed;
expect(err).toBeInstanceOf(StreamError);
expect((err as StreamError).code).toBe(StreamCode.NotFound);
}
} finally {
track.close();
broadcast.close();
subscriber.close();
}
});

// Only a failed write defers to the responses: an update the wire can't carry still fails the
// track. Lite-05 sends the start group plus one, past the largest u53 here.
test("an update whose bound the wire can't carry fails the track", async () => {
const { quic, streams, opened } = fakeSession();
const subscriber = new Subscriber(quic, Version.DRAFT_05, HopSchema.parse(1n));
const broadcast = subscriber.consume(Path.from("room"));
const track = broadcast.track("video").subscribe();
try {
await opened(0);
await answerTrackInfo(streams[0], Version.DRAFT_05);
await (await opened(1)).reading;
track.update({ groups: { start: { included: Number.MAX_SAFE_INTEGER } } });
expect(await track.closed).toBeInstanceOf(RangeError);
} finally {
track.close();
broadcast.close();
subscriber.close();
}
});

test("a retired subscribe response type closes the session", async () => {
const { quic, streams, opened, closes } = fakeSession();
const subscriber = new Subscriber(quic, Version.DRAFT_07, HopSchema.parse(1n));
Expand Down
20 changes: 15 additions & 5 deletions js/net/src/lite/subscriber.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import { Cost, type Hop, MAX_HOPS, type Route, routesEqual, UNKNOWN_HOP } from "
import { groupBounds, hiddenBelow, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts";
import * as Path from "../path.ts";
import { sendOrder } from "../priority.ts";
import { type OpenOptions, type Reader, Stream } from "../stream.ts";
import { type OpenOptions, type Reader, Stream, Writer } from "../stream.ts";
import { TAIL_GRACE_MS, Tail } from "../tail.ts";
import * as Time from "../time.ts";
import type * as track from "../track.ts";
Expand Down Expand Up @@ -768,8 +768,8 @@ export class Subscriber {

// Terminal conditions (stream end, track close, a failed subscription update) settle at most
// once; race them into one stable promise so the demand loop doesn't re-subscribe each pass.
// Updates stop quietly at the FIN, which can land before the responses ahead of it are
// decoded, so only their failure is terminal on its own.
// Updates stop quietly at the FIN or a failed write, either of which can land before the
// responses ahead of it are decoded, so only a refused update is terminal on its own.
const terminal: PromiseLike<unknown>[] = [closed, producer.closed];
if (subscriptionUpdates !== undefined) terminal.push(subscriptionUpdates.then(() => closed));
const done = race(terminal);
Expand Down Expand Up @@ -1237,7 +1237,7 @@ export class Subscriber {
/**
* Send SUBSCRIBE_UPDATE messages whenever the track's aggregate subscription changes.
*
* Resolves cleanly when the stream or track closes, so the caller can include
* Resolves cleanly when the stream or track closes, or a write fails, so the caller can include
* this in a race without leaving a dangling pending write that would
* become an unhandled rejection if the user calls update after close.
*
Expand Down Expand Up @@ -1293,7 +1293,17 @@ export class Subscriber {
startGroup: current.groups?.start === undefined ? undefined : bounds.start,
endGroup: inclusiveGroupEnd(bounds.end),
});
await update.encode(stream.writer, this.version);
// Encoded apart from the write, so a bound the wire can't carry still fails the track.
const encoded: Uint8Array[] = [];
const buffer = new WritableStream<Uint8Array>({ write: (chunk) => void encoded.push(chunk.slice()) });
await update.encode(new Writer(buffer, this.version), this.version);
// A failed write means the publisher stopped reading, as it does once it finishes the
// track. The responses decide how the track ends, cleanly at the FIN or with a reset's error.
try {
for (const chunk of encoded) await stream.writer.write(chunk);
} catch {
return;
Comment on lines +1304 to +1305

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Do not swallow SUBSCRIBE_UPDATE encoding errors

On pre-Lite-06 connections, an update whose start group is Number.MAX_SAFE_INTEGER reaches this catch as a local RangeError, not a stopped-stream error: groupBounds accepts that safe integer, but encodeStartGroup adds one for the older wire format and Writer.u53 rejects the resulting value. Returning here leaves the track open while silently ignoring its requested bound; restrict suppression to transport write failures, or reject the unencodable bound before this catch, so codec errors still surface.

AGENTS.md reference: AGENTS.md:L17-L17

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, fixed in 074066a. The update is now encoded into memory before the stream write, and only a failed write defers to the responses. An unencodable bound (your lite-05 MAX_SAFE_INTEGER start) fails the track again, covered by an update whose bound the wire can't carry fails the track. Rust already keeps buffer_update (encode, propagated with ?) apart from the flush, so it needed no change.

(Written by Claude Opus 5.5)

Comment on lines +1302 to +1305

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Restore unsent JS subscription state before settling

When the failed update lowers the start after SUBSCRIBE_START, this return defers to the publisher's END/FIN even though entry.tail.demand(...) and the producer's current subscription already reflect the update that the publisher never received. #settleTail then treats those newly requested groups as owed and waits for its grace, which can be an arbitrarily large maxDelay, before closing an otherwise finished track. Preserve the last successfully written bounds and delay, or roll back the unsent update state before deferring to the responses.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Real, but I'm keeping it out of this PR. It predates the failed-write path: the owed range comes from the current demand, not what the publisher acknowledged. So a lowered-start update that crosses SUBSCRIBE_END on the wire, with a write that succeeds, already makes the tail wait out its grace the same way. It only affects lite-05/06 (lite-07 counts streams), and the track still ends cleanly, at worst one grace late. Before this PR, the same race failed the track outright. Rolling back state only on a failed write would cover one of the two crossings. The fix belongs in how the tail bounds its debt once the publisher has ended (demand made after its END owes nothing), in both languages. I've suggested that as a follow-up.

(Written by Claude Opus 5.5)

}
lastSent = { ...current };
console.debug(`subscribe update: id=${id} broadcast=${broadcast} track=${track.name}`);
}
Expand Down
1 change: 0 additions & 1 deletion quest/m1/test-flakes-2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ Public API: none. Wire: none.

## Required

- [Update after finish](/quest/m1/test-flakes-2/update-after-finish.md) - a subscribe update racing a publisher's finish ends the track cleanly in JS and Rust, not with the write's error
- [Auth refresh mocks time](/quest/m1/test-flakes-2/auth-refresh-mock-time.md) - moq-relay's auth refresh and fixed-lease tests measure a grant's life on one mocked clock
- [uring teardown close](/quest/m1/test-flakes-2/uring-teardown-close.md) - moq-uring's published-close teardown test holds under load, fixed at its cause
- [WASM IETF read](/quest/m1/test-flakes-2/wasm-ietf-read.md) - the WASM suite's IETF read case reads whole groups on a loaded runner, fixed at the burst or the assertion as the evidence says
Expand Down
39 changes: 0 additions & 39 deletions quest/m1/test-flakes-2/update-after-finish.md

This file was deleted.

13 changes: 10 additions & 3 deletions rs/moq-net/src/coding/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,19 @@ impl<S: crate::transport::poll::SendStream, V: StreamCodes> Writer<S, V> {
}

/// Poll until the write buffer has fully hit the stream.
///
/// A failed stream never takes another byte, so a write error drops the buffer too.
pub fn poll_flush(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
while self.flushed < self.buffer.len() {
let stream = self.stream.as_mut().unwrap();
let n = ready!(stream.poll_write(cx, &self.buffer[self.flushed..]))
.map_err(|err| self.version.transport_error(err))?;
self.flushed += n;
match ready!(stream.poll_write(cx, &self.buffer[self.flushed..])) {
Ok(n) => self.flushed += n,
Err(err) => {
self.buffer.clear();
self.flushed = 0;
return Poll::Ready(Err(self.version.transport_error(err)));
}
}
}
self.buffer.clear();
self.flushed = 0;
Expand Down
82 changes: 77 additions & 5 deletions rs/moq-net/src/lite/subscriber.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2883,6 +2883,79 @@ mod tests {
));
}

/// A publisher that finishes the track stops reading its subscribe stream, so a
/// SUBSCRIBE_UPDATE written then fails. The track still ends with the publisher's
/// finish; a reset still hands it back with the reset's error.
#[moq_net_sim::test]
async fn an_update_the_publisher_stopped_reading_defers_to_its_end() {
use crate::transport::poll::{RecvStream as _, SendStream as _, Session as _};
for reset in [false, true] {
let version = Version::Lite06;
let (session, mut peer) = crate::transport::memory::pair(Default::default());
let subscriber = Subscriber::new(SubscriberConfig {
runtime: crate::time::Clock::sim(),
session,
origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(),
recv_bandwidth: None,
version,
peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Binding::InBand),
peer_hop: None,
cost: None,
going_away: Default::default(),
auth: crate::auth::Handle::new(false),
});
let serve = TrackServe {
validity: Default::default(),
subscriber,
path: Path::new("room").to_owned(),
epoch: None,
name: "video".into(),
};
let broadcast = crate::broadcast::Info::new().produce();
let mut serving = ServeLoop::new(
&serve,
broadcast.reserve_track("video").unwrap(),
Default::default(),
None,
);
let mut reader = serving.serving.subscribe(None);
assert!(serving.poll(&serve, &kio::Waiter::noop()).is_pending());
assert!(matches!(serving.sub, Sub::Active(_)), "the SUBSCRIBE is on the wire");

// The publisher ends the subscription and stops reading the stream.
let (mut send, mut recv) = std::future::poll_fn(|cx| peer.poll_accept_bi(cx)).await.unwrap();
recv.stop(crate::StreamError::Cancel.to_code());
if reset {
send.reset(crate::StreamError::NotFound.to_code());
} else {
let mut end = Vec::new();
lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0, streams: 0 })
.encode(&mut crate::coding::Encoder::new(&mut end, version.into()), version)
.unwrap();
std::future::poll_fn(|cx| send.poll_write(cx, &end)).await.unwrap();
send.finish().unwrap();
}

// The update races it, and its write fails.
reader.update(Subscription::default().with_priority(1)).unwrap();
let gate = serve.gate();
let mut running = TrackServeRun {
serve,
state: TrackRunState::Serve(serving),
gate,
};
assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_ready());
let end = reader.recv_group().now_or_never().expect("the track ended").map(|_| ());
match reset {
false => assert!(matches!(end, Ok(())), "{end:?}"),
true => assert!(
matches!(end, Err(Error::Stream(crate::StreamError::NotFound))),
"{end:?}"
),
}
}
}

/// Mid-group demand: a subscriber resuming at frame 3 of group 5, capped by a later
/// bound in the same group.
fn mid_group_demand() -> Subscription {
Expand Down Expand Up @@ -5530,12 +5603,11 @@ impl<S: crate::transport::poll::Session> ServeLoop<S> {
if let Sub::Active(active) = &mut self.sub {
match active.stream.writer.poll_flush(&mut cx) {
Poll::Ready(Ok(())) => {}
// The publisher stopped reading, as it does once it finishes the
// track. The update is dropped, and the responses decide how the
// subscription ends: cleanly at the FIN, or with a reset's error.
Poll::Ready(Err(err)) => {
// The stream is broken; drop it (the writer resets) and
// hand the track back.
serve.subscriber.remove_subscribe(active.id);
self.sub = Sub::None;
return Poll::Ready(ServeEnd::GiveBack(err));
tracing::debug!(track = %serve.name, %err, "subscribe update dropped")
}
Comment on lines 5609 to 5611

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Restore unsent Rust subscription state before settling

When a failed update lowered the floor after SUBSCRIBE_START, begin_subscription has already changed active.start, restarted the tail debt, and applied the new grace before this error is ignored. The subsequent END/FIN therefore enters ServeMode::Tail with groups the publisher never agreed to send and can delay a clean track end for the full, caller-controlled max-delay grace. Commit this state only after a successful flush, or restore the previously sent state on this error before processing responses.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Real, but I'm keeping it out of this PR. It predates the failed-write path: the owed range comes from the current demand, not what the publisher acknowledged. So a lowered-start update that crosses SUBSCRIBE_END on the wire, with a write that succeeds, already makes the tail wait out its grace the same way. It only affects lite-05/06 (lite-07 counts streams), and the track still ends cleanly, at worst one grace late. Before this PR, the same race failed the track outright. Rolling back state only on a failed write would cover one of the two crossings. The fix belongs in how the tail bounds its debt once the publisher has ended (demand made after its END owes nothing), in both languages. I've suggested that as a follow-up.

(Written by Claude Opus 5.5)

Poll::Pending => return Poll::Pending,
}
Expand Down
Loading