Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions js/net/src/internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,8 @@ export const hooks: {
* `setGroups`, which never rewinds.
*/
replaceGroups: (subscriber: Subscriber, groups: Groups) => void;
/** The lowest group sequence a serving cursor delivers, which datagrams bypass. */
groupFloor: (subscriber: Subscriber) => number;
/** Return a group's first timestamp, retained even after its first frame is read. */
groupTimestamp: (group: GroupConsumer) => Timestamp | undefined;
groupLatest: (group: GroupConsumer) => Timestamp | undefined;
Expand Down Expand Up @@ -200,6 +202,9 @@ export const hooks: {
replaceGroups: () => {
throw new Error("track.ts not loaded");
},
groupFloor: () => {
throw new Error("track.ts not loaded");
},
groupTimestamp: () => {
throw new Error("group.ts not loaded");
},
Expand Down
88 changes: 88 additions & 0 deletions js/net/src/lite/publisher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { Reader, Stream, Writer } from "../stream.ts";
import { Milli, Timestamp } from "../time.ts";
import { DEFAULT_MAX_AGE_MS } from "../track.ts";
import { AnnounceRequest } from "./announce.ts";
import { Datagram as DatagramMessage } from "./datagram.ts";
import { Fetch } from "./fetch.ts";
import { Group as GroupMessage } from "./group.ts";
import { sendOrder } from "./priority.ts";
Expand Down Expand Up @@ -574,6 +575,7 @@ async function servedSubscription(
const serving = publisher.runSubscribe(msg, server);

const opened = pair.client.incomingUnidirectionalStreams.getReader();
const datagrams = pair.client.datagrams.readable.getReader();

return {
client,
Expand All @@ -587,6 +589,9 @@ async function servedSubscription(
group.close();
track.writeGroup(group);
},
datagram(sequence: number, bytes = 1) {
track.insertDatagram(sequence, Timestamp.fromMillis(sequence), new Uint8Array(bytes));
},
// The next group stream the publisher opened, drained, or undefined once it has gone
// idle. A group the publisher dropped then fails an assertion instead of hanging the
// test on a stream that will never arrive.
Expand Down Expand Up @@ -615,9 +620,21 @@ async function servedSubscription(
async servedSequence(): Promise<number | undefined> {
return (await this.servedGroup())?.sequence;
},
// The next datagram's sequence, or undefined once the publisher has gone idle.
async sentDatagram(): Promise<number | undefined> {
let timer: ReturnType<typeof setTimeout> | undefined;
const idle = new Promise<undefined>((resolve) => {
timer = setTimeout(() => resolve(undefined), IDLE_MS);
});
const next = await Promise.race([datagrams.read(), idle]);
clearTimeout(timer);
if (!next || next.done) return undefined;
return (await DatagramMessage.decode(next.value, version)).sequence;
},
async close() {
// Settles any pending read as well as dropping the stream.
await opened.cancel();
await datagrams.cancel();
publisher.close();
client.close();
},
Expand Down Expand Up @@ -668,6 +685,77 @@ test("lite draft-05: a straggler below the announced start group is not served",
}
});

// A datagram too large for the transport is never sent, so it must not resolve the start:
// that would drop an older group below it for nothing.
test("lite draft-05: a dropped datagram does not resolve the start", async () => {
const debug = spyOn(console, "debug");
const sub = await servedSubscription();
try {
sub.datagram(10, 2000);
while (!debug.mock.calls.some(([msg]) => String(msg).startsWith("dropping oversize datagram"))) await flush();

sub.serve(5);
expect(await sub.servedSequence()).toBe(5);
const resp = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_05);
if (!("start" in resp)) throw new Error("expected SUBSCRIBE_START");
expect(resp.start.group).toBe(5);
} finally {
debug.mockRestore();
await sub.close();
}
});

// SUBSCRIBE_START promises nothing below the start, so a datagram below it is dropped like a
// group there would be.
test("lite draft-05: a datagram below the announced start is not sent", async () => {
const sub = await servedSubscription();
try {
sub.serve(5);
expect(await sub.servedSequence()).toBe(5);
const resp = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_05);
if (!("start" in resp)) throw new Error("expected SUBSCRIBE_START");
expect(resp.start.group).toBe(5);

sub.datagram(3);
sub.datagram(6);
expect(await sub.sentDatagram()).toBe(6);
} finally {
await sub.close();
}
});

// The datagram and group loops race to claim SUBSCRIBE_START, and the START write runs a few
// turns after the claim. Whichever loop wins, no group goes out below the start it announced,
// even one popped before that write ran. Sweeping the group's arrival across those turns
// covers the window without depending on how many microtasks each loop takes.
test("lite draft-05: no group goes out below a datagram's start", async () => {
const debug = spyOn(console, "debug");
try {
for (let turns = 0; turns < 16; turns++) {
debug.mockClear();
const sub = await servedSubscription();
try {
// Both loops are parked, so the datagram and the groups race from here.
while (!debug.mock.calls.some(([msg]) => String(msg).startsWith("publish ok"))) await flush();
await flush();

sub.datagram(10);
for (let i = 0; i < turns; i++) await Promise.resolve();
sub.serve(5);
sub.serve(11);

const resp = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_05);
if (!("start" in resp)) throw new Error("expected SUBSCRIBE_START");
expect(await sub.servedSequence()).toBeGreaterThanOrEqual(resp.start.group);
} finally {
await sub.close();
}
}
} finally {
debug.mockRestore();
}
});

// A group pop is the linearization point. Once the publisher reaches the SUBSCRIBE_START write,
// the group and its range have already been decided, so an update queued during the write applies
// to the next pop rather than reaching backward into this one.
Expand Down
41 changes: 29 additions & 12 deletions js/net/src/lite/publisher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -267,18 +267,24 @@ class SubscriptionControls {
*/
class SubscribeResponses {
#controls: SubscriptionControls;
#start: (sequence: number) => Promise<void>;
#start: (sequence: number) => () => Promise<void>;
#writes: Promise<boolean> = Promise.resolve(true);
#started?: Promise<boolean>;

constructor(controls: SubscriptionControls, start: (sequence: number) => Promise<void>) {
/** `start` claims the start synchronously and returns the SUBSCRIBE_START write. */
constructor(controls: SubscriptionControls, start: (sequence: number) => () => Promise<void>) {
this.#controls = controls;
this.#start = start;
}

/** Sends SUBSCRIBE_START at `sequence`, once; false when peer departure superseded it. */
/**
* Sends SUBSCRIBE_START at `sequence`, once; false when peer departure superseded it.
*
* The first call claims the start in the same turn, so whatever either serving loop pops
* next already sees it, even while the write waits its turn.
*/
start(sequence: number): Promise<boolean> {
this.#started ??= this.#write(() => this.#start(sequence));
this.#started ??= this.#write(this.#start(sequence));
return this.#started;
}

Expand Down Expand Up @@ -667,17 +673,20 @@ export class Publisher {
endGroup: msg.endGroup,
endFrame: msg.endFrame,
};
const responses = new SubscribeResponses(controls, async (sequence) => {
const responses = new SubscribeResponses(controls, (sequence) => {
// SUBSCRIBE_START promises nothing below this sequence will be delivered.
// Arrival-order serving could later surface a straggler below the first
// group, so pin the floor to what was announced.
// Arrival-order serving could later surface a straggler below it, so pin the
// floor to what is announced now, not when the write runs: a group popped in
// between would otherwise go out below it.
hooks.replaceGroups(track, {
start: { included: sequence },
end: bounds.endGroup === undefined ? undefined : { included: bounds.endGroup },
});
// JS publishes only what it produces, so the origin serving it is ours.
const start = new SubscribeStart(sequence, this.#origin);
await encodeSubscribeResponse(stream.writer, { start }, this.version);
return async () => {
// JS publishes only what it produces, so the origin serving it is ours.
const start = new SubscribeStart(sequence, this.#origin);
await encodeSubscribeResponse(stream.writer, { start }, this.version);
};
});

// Serve datagrams concurrently with groups whenever the transport carries them
Expand Down Expand Up @@ -1015,18 +1024,26 @@ export class Publisher {
for (;;) {
const datagram = await track.recvDatagram();
if (!datagram) return; // Track finished; #runTrack tears the subscription down.
if (!(await responses.start(datagram.sequence))) return;

// Below the floor, as a group there is: SUBSCRIBE_START promised nothing below
// the start.
if (datagram.sequence < hooks.groupFloor(track)) {
console.debug(`dropping datagram below the start: sub=${sub} sequence=${datagram.sequence}`);
continue;
}

// Convert the timestamp to the track's advertised timescale, matching #serveGroup.
const ts = Math.round(datagram.timestamp.as(timescale));
const body = new DatagramMessage(sub, datagram.sequence, ts, datagram.payload).encode(this.version);

// No group fallback: drop anything that doesn't fit a single datagram.
// No group fallback: drop anything that doesn't fit a single datagram. It is never
// sent, so it must not resolve the start either.
if (body.byteLength > maxSize) {
console.debug(`dropping oversize datagram: sub=${sub} size=${body.byteLength} max=${maxSize}`);
continue;
}

if (!(await responses.start(datagram.sequence))) return;
await writer.ready;
await writer.write(body);
}
Expand Down
1 change: 1 addition & 0 deletions js/net/src/track.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,7 @@ export class Subscriber {
subscriber.#enforceLatency = false;
};
hooks.replaceGroups = (subscriber, groups) => subscriber.#replaceGroups(groups);
hooks.groupFloor = (subscriber) => subscriber.#cursor.peek().start;
// The sequence cursor lives here (it shares the buffer and the drift anchor with
// the arrival cursor); `Ordered` is the handle that reaches it.
ordered_ = {
Expand Down
7 changes: 1 addition & 6 deletions quest/m0/wildcard/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Wildcard advertisements
# [S] Wildcard advertisements

## Goal

Expand Down Expand Up @@ -189,11 +189,6 @@ announcement shadows it. A claim names no generation, so a client that must
distinguish recording generations reads the catalog's archive entry
([archive](/quest/m1/archive/README.md)) rather than announce state.

## Required

- [Datagrams behind SUBSCRIBE_START](/quest/m0/wildcard/datagram-start.md) - no datagram crosses a subscription's start in either direction, in Rust or JS
- [JS origin granularity](/quest/m0/wildcard/js-origin.md) - `@moq/net` tracks a reply's origin at the same granularity as Rust

## Related

- [archive](/quest/m1/archive/README.md) - an archive claims the root, and its
Expand Down
53 changes: 0 additions & 53 deletions quest/m0/wildcard/datagram-start.md

This file was deleted.

Loading
Loading