diff --git a/js/net/src/fill.ts b/js/net/src/fill.ts index a395002abb..ba9b06e34f 100644 --- a/js/net/src/fill.ts +++ b/js/net/src/fill.ts @@ -735,11 +735,12 @@ export class FillGroups { /** * Insert group `key` into the track's cache, handing it to the readers waiting on it, - * and return the producer to write its frames into. `info` commits the track's {@link Info} - * if it has none yet. A key from the {@link floor} up to the start answers the start with that - * group. A cached copy missing its first frames, which a reader asked whole, is replaced. Returns - * `undefined` if the group was already answered or cached whole. Throws a `RangeError` outside - * the fill, and an error if the track is closed. + * and return the producer to write its frames into. A live subscription covering it receives + * it too. `info` commits the track's {@link Info} if it has none yet. A key from the + * {@link floor} up to the start answers the start with that group. A cached copy missing its + * first frames, which a reader asked whole, is replaced. Returns `undefined` if the group was + * already answered or cached whole. Throws a `RangeError` outside the fill, and an error if the + * track is closed. */ accept(start: bigint, info?: Info): GroupProducer | undefined { checkKey(start); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index ab99b26ec0..15d459f976 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -522,6 +522,8 @@ export interface ResumeSource { resolved(): (Resolved | undefined)[]; /** The runs of `[start, end)` the serving copy reads, undefined without one. */ runs(start: bigint, end: bigint): Run[] | undefined; + /** Whether a time start the serving copy still waits to place holds back a gap; see `hooks.gapHeld`. */ + gapHeld(at: bigint, end: bigint): boolean; /** Whether a route's copy serves the reader now. */ serving(): boolean; /** Where the run of keys from `key` the serving copy says no group arrives at ends; see `hooks.holeEnd`. */ @@ -711,6 +713,8 @@ export const hooks: { subscriberEndError: (subscriber: Subscriber) => Error | undefined; /** What this subscriber resolves its ranges against, for a test. Assigned by `track.ts`. */ subscriberHoldings: (subscriber: Subscriber) => Holdings; + /** The times a received copy's feed accounting may place, for a test. Assigned by `track.ts`. */ + landingSpan: (producer: Producer) => Run | undefined; /** * How many keys just below `key` the publisher declared it never produced: the skip run * ending at `key`, which a moq-transport Prior Group ID Gap names. Mirrors the Rust @@ -750,11 +754,17 @@ export const hooks: { startsAtKeys: (producer: Producer) => void; /** * Record where the upstream publisher started each of a lite-07 copy's live wire ranges, as - * `[time, start]` pairs by the time each asked from, replacing the last set: a reader's time - * start the copy cannot place among its own groups resolves to it. Mirrors the Rust + * `[time, start, through]` by the time each asked from (0 for the oldest group), `start` + * undefined for one with no group to deliver, replacing the last set: a reader's time start the + * copy cannot place among its own groups resolves to it. An empty answer holds for a reader + * within `budget`, the narrowest the subscription asked in milliseconds. Mirrors the Rust * `track::Producer::time_starts`; assigned by `track.ts`. */ - timeStarts: (producer: Producer, starts: Iterable) => void; + timeStarts: ( + producer: Producer, + starts: Iterable, + budget: number, + ) => void; /** * Tell a lite-07 copy where its publisher started each live range counting back, as * `[back, start, latest]` triples, replacing the last set: the copy holds only part of the @@ -822,6 +832,11 @@ export const hooks: { * Assigned by `track.ts`. */ servingRuns: (subscriber: Subscriber, start: bigint, end: bigint) => Run[]; + /** + * Whether a time start still waiting below `end` holds back a gap an ordered read at `at` would + * return. Assigned by `track.ts`. + */ + gapHeld: (subscriber: Subscriber, at: bigint, end: bigint) => boolean; /** * The exclusive end past every group a subscriber reads, or undefined while one of its ranges * stays open. Assigned by `track.ts`. @@ -1169,6 +1184,9 @@ export const hooks: { subscriberHoldings: () => { throw new Error("track.ts not loaded"); }, + landingSpan: () => { + throw new Error("track.ts not loaded"); + }, skippedBelow: () => { throw new Error("track.ts not loaded"); }, @@ -1190,6 +1208,9 @@ export const hooks: { servingRuns: () => { throw new Error("track.ts not loaded"); }, + gapHeld: () => { + throw new Error("track.ts not loaded"); + }, servingEnd: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 02077e4971..b00764842c 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -1663,7 +1663,7 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07]) { try { hooks.receiveTrack(sub.track); // A lite-07 source reports where it started the time asked for. - if (hasRanges(version)) hooks.timeStarts(sub.track, [[2n, source]]); + if (hasRanges(version)) hooks.timeStarts(sub.track, [[2n, source]], NO_MAX_DELAY); hooks.startTrack(sub.track, { start: source, requested: 2n }); sub.serve(6n); expect(await sub.servedSequence()).toBe(6n); @@ -1691,7 +1691,7 @@ test("lite draft-07: a time start a copy's oldest range covers starts where that try { hooks.receiveTrack(sub.track); await flush(); - hooks.timeStarts(sub.track, [[0n, 6n]]); + hooks.timeStarts(sub.track, [[0n, 6n]], NO_MAX_DELAY); hooks.startTrack(sub.track, { start: 6n }); sub.serve(6n); expect(await sub.servedSequence()).toBe(6n); @@ -1714,7 +1714,7 @@ for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { hooks.startTrack(sub.track, { requested: 2n }); sub.serve(6n); await flush(); - if (hasRanges(version)) hooks.timeStarts(sub.track, [[2n, 3n]]); + if (hasRanges(version)) hooks.timeStarts(sub.track, [[2n, 3n]], NO_MAX_DELAY); hooks.startTrack(sub.track, { start: 3n, requested: 2n }); const response = await decodeSubscribeResponse(sub.client.reader, version); if (!("start" in response)) throw new Error("expected SUBSCRIBE_START"); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index f8b086a45f..372abf3475 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -23,7 +23,7 @@ import { import type { Consumer as GroupConsumer } from "../group.ts"; import { type Hop, isAnonymous, MAX_HOPS, toHop, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; -import { hooks } from "../internal.ts"; +import { hooks, NO_MAX_DELAY } from "../internal.ts"; import { GRACE } from "../ledger.ts"; import * as Path from "../path.ts"; import { Reader, Writer } from "../stream.ts"; @@ -2972,6 +2972,51 @@ test("a widened range's SUBSCRIBE_STARTED resolves the feed's start", async () = } }); +// A range the publisher answers with no group to deliver lands none of its groups, so the copy is +// told, and an ordered read holds no later group back for its time. Only one whose end takes in its +// time says so: one ending by a count back may still have skipped the group containing it. +test("a lite-07 range answered empty is reported only for a range reaching its time", async () => { + const version = Version.DRAFT_07; + const { quic, streams, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, toHop(1n)); + const broadcast = subscriber.consume(Path.from("room")); + const reported: (readonly [bigint, bigint | undefined, (bigint | undefined)?])[][] = []; + const timeStarts = hooks.timeStarts; + const budgets: number[] = []; + hooks.timeStarts = (producer, starts, budget) => { + reported.push([...starts]); + budgets.push(budget); + timeStarts(producer, starts, budget); + }; + const track = broadcast.track("video").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 20n } }, + { start: { time: 30n }, end: { relative: 0n } }, + { start: { time: 66n } }, + ], + }); + try { + await opened(0); + await answerTrackInfo(streams[0], version); + const request = await opened(1); + await request.reading; + await respond(request, version, [{ start: new SubscribeStart([undefined, undefined, 66n]) }]); + await drainReactions(); + expect(reported.at(-1)).toEqual([ + [10n, undefined, 20n], + [66n, 66n, undefined], + ]); + // Under the narrowest budget the subscription asked with, here none. + expect(budgets.at(-1)).toEqual(NO_MAX_DELAY); + } finally { + hooks.timeStarts = timeStarts; + track.close(); + broadcast.close(); + subscriber.close(); + } +}); + // A lite-07 stale (OLD) drop accounts for its groups for good, whichever of it and a group's stream // lands first: an update owes none of them again, and a group still arriving, such as one another // subscription sharing the copy delivers, is never aborted by the drop. diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index ee4ec341af..34e1d0f362 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -89,6 +89,7 @@ import { Subscribe, SubscribeUpdate, toWireRanges, + type WireAnchor, type WireRange, } from "./subscribe.ts"; import { TrackInfo, Track as TrackMessage } from "./track.ts"; @@ -2870,10 +2871,10 @@ export class Subscriber { // requested floor, pending, while SUBSCRIBE_START or a range's SUBSCRIBE_STARTED has yet to // say where one starts. Mirrors the Rust `SubStream::declare`. #declare(entry: SubscribeEntry): void { - // Where the publisher started each live range asked from a group's start, which a reader's - // time start there resolves to when the copy holds no group at it, and one asked from the - // oldest group as asked from 0. Mirrors the Rust `SubStream::report_starts`. - const times: [bigint, bigint][] = []; + // Where the publisher started each live range asked from a group's start or the oldest group + // (time 0), with the last time it reaches, which a reader's time start there or folded into it + // resolves to when the copy holds no group at it. Mirrors the Rust `SubStream::report_starts`. + const times: [bigint, bigint | undefined, bigint | undefined][] = []; // And each one counting back, which the copy, holding only part of the track, can't count, // with the latest group it counted back from. const relative: [bigint, bigint, bigint | undefined][] = []; @@ -2881,13 +2882,19 @@ export class Subscriber { const range = entry.live.get(id); const time = range?.start?.time ?? range?.start?.absolute; const back = range?.start?.relative; + if (!entry.starts.has(id)) continue; + // Undefined for a range with no group to deliver. const at = entry.starts.get(id); - if (at === undefined) continue; - if (time !== undefined) times.push([time, at]); - else if (back !== undefined) relative.push([back, at, entry.edges.get(id)]); - else if (range !== undefined && range.start === undefined) times.push([0n, at]); + if (back !== undefined) { + if (at !== undefined) relative.push([back, at, entry.edges.get(id)]); + } else if (time !== undefined || range?.start === undefined) { + const asked = time ?? 0n; + if (at !== undefined || reaches(range?.end, asked)) times.push([asked, at, through(range?.end, asked)]); + } } - hooks.timeStarts(entry.track, times); + // The wire does not say which budget a range resolved under, so an empty one holds only for the + // narrowest asked. + hooks.timeStarts(entry.track, times, entry.staleBudget); hooks.relativeStarts(entry.track, relative); let start: bigint | null = null; @@ -3427,3 +3434,23 @@ class ConsumeBroadcast extends broadcast.Consumer { return new ConsumeBroadcast(this.#subscriber, this.#path, this.#consumed, this.shareState()); } } + +// The last time a lite-07 range ending at `end` reaches, undefined while it follows the live edge: +// an exclusive group end stops at the time before it, and a relative end, unknown here, at the +// range's own time `asked`. Mirrors the Rust `SubStream::report_starts`. +// Whether a lite-07 range with no group to deliver says its publisher held none at or before its +// reach: only one whose end takes in the group containing its time, which a range that ends before +// its time, or by a count back, may not. Mirrors the Rust `SubStream::report_starts`. +function reaches(end: WireAnchor | undefined, asked: bigint): boolean { + if (end === undefined) return false; + if (end.time !== undefined) return end.time >= asked; + if (end.absolute !== undefined) return end.absolute > asked; + return false; +} + +function through(end: WireAnchor | undefined, asked: bigint): bigint | undefined { + if (end === undefined) return undefined; + if (end.time !== undefined) return end.time; + if (end.absolute !== undefined) return end.absolute > asked ? end.absolute - 1n : asked; + return asked; +} diff --git a/js/net/src/ordered.test.ts b/js/net/src/ordered.test.ts index 4b7072b804..86d3bcfd01 100644 --- a/js/net/src/ordered.test.ts +++ b/js/net/src/ordered.test.ts @@ -4,7 +4,8 @@ import { expect, test } from "bun:test"; import { Duration, now, sleep } from "@moq/clock"; import { Manual } from "@moq/clock/test"; import { StreamCode, StreamError } from "./error.ts"; -import { hooks, staleHole } from "./internal.ts"; +import { rejectGroups } from "./fill.ts"; +import { hooks, NO_MAX_DELAY, staleHole } from "./internal.ts"; import { manual, settlesWithin, textFrame } from "./mock.ts"; import { Timescale, Timestamp } from "./time.ts"; import { type Ordered, Producer as TrackProducer } from "./track.ts"; @@ -436,6 +437,264 @@ test("a read returning only gaps keeps its handed-out record compact", () => { expect(hooks.deliveredRuns(sub)).toBe(1); }); +test("an ordered read hands out nothing past a time start still waiting to be placed", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + const ordered = producer.subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }).ordered(); + expect(items(ordered)).toEqual([]); + // A session's tail holds the read at the answered start's group until it lands, and says + // nothing lies between it and 2000. + hooks.timeStarts(producer, [[1500n, 1000n]], NO_MAX_DELAY); + put(producer, 1000n, false); + producer.skipGroups(1001n, 2000n); + expect(items(ordered)).toEqual(["group 1000", "group 2000"]); +}); + +test("a time start folded upstream into an earlier range places from the range covering it", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + const subscriber = producer.subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }); + // A range asked from 1000 that ends at 1200 doesn't reach the time. + hooks.timeStarts(producer, [[1000n, 1000n, 1200n]], NO_MAX_DELAY); + expect(hooks.servingResolved(subscriber)).toEqual([undefined]); + // The oldest group's range, answered at 2000, served nothing at or before it. + hooks.timeStarts( + producer, + [ + [1000n, 1000n, 1200n], + [0n, 2000n, undefined], + ], + NO_MAX_DELAY, + ); + expect(hooks.servingResolved(subscriber)).toEqual([{ start: 0n, end: undefined, floored: false }]); +}); + +test("a folded time start wakes its read when the grace for a gap below it passes", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const producer = receivedTrack(); + // An oldest-group reader holds the feed open from its answered start, 1000. + const oldest = producer.subscribe({ delay: { max: Duration.secs(1) } }); + hooks.startTrack(producer, { start: 1000n }); + hooks.timeStarts(producer, [[0n, 1000n]], NO_MAX_DELAY); + // Group 1000's stream reset before its header: only 1800, 2000, and the skips around them + // land, 1800 last, so the read's own budget for the time runs out after the gap's grace. + producer.skipGroups(1001n, 1800n); + producer.skipGroups(1801n, 2000n); + put(producer, 2000n, false); + clock.jump(Duration.millis(10)); + put(producer, 1800n, false); + const ordered = producer + .subscribe({ delay: { max: Duration.secs(1) }, ranges: [{ start: { time: 1500n } }] }) + .ordered(); + expect(items(ordered)).toEqual([]); + let read: bigint | undefined; + const next = ordered.nextGroup().then((group) => { + read = group?.start; + }); + // Let the read settle into its wait, so only a timer can wake it. + await flush(); + clock.jump(Duration.millis(989)); + await flush(); + expect(read).toBeUndefined(); + clock.jump(Duration.millis(1)); + await flush(); + // 1800 came a full budget after 2000 superseded it. + expect(read).toBe(2000n); + await next; + oldest.close(); + } finally { + installed.close(); + } +}); + +test("a waiting time start holds back a gap above its group", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + drop(producer, 1500n, 1600n); + const ordered = producer.subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }).ordered(); + expect(items(ordered)).toEqual([]); + hooks.timeStarts(producer, [[1500n, 1000n]], NO_MAX_DELAY); + put(producer, 1000n, false); + producer.skipGroups(1001n, 1500n); + producer.skipGroups(1600n, 2000n); + expect(items(ordered)).toEqual(["group 1000", "gap 1500..1600 (delivery timeout)", "group 2000"]); +}); + +test("a waiting time start holds an ordered read only within its budget", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const producer = receivedTrack(); + put(producer, 2000n, false); + const ordered = producer + .subscribe({ delay: { max: Duration.millis(100) }, ranges: [{ start: { time: 1500n } }] }) + .ordered(); + expect(items(ordered)).toEqual([]); + let read: bigint | undefined; + const next = ordered.nextGroup().then((group) => { + read = group?.start; + }); + await flush(); + clock.jump(Duration.millis(99)); + await flush(); + expect(read).toBeUndefined(); + clock.jump(Duration.millis(1)); + await flush(); + expect(read).toBe(2000n); + await next; + } finally { + installed.close(); + } +}); + +test("a waiting time start below an ordered read's own floor holds nothing", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + const ordered = producer + .subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }) + .withRange({ start: { included: 2000n } }) + .ordered(); + expect(items(ordered)).toEqual(["group 2000"]); +}); + +test("a waiting time start holds no gap once nothing from the read's floor through it may land", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + const ordered = () => + producer + .subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }) + .withRange({ start: { included: 1500n }, end: { excluded: 1600n } }) + .ordered(); + drop(producer, 1501n, 1600n); + const waiting = ordered(); + expect(items(waiting)).toEqual([]); + // The group containing the time, if still to come, lies below the read's floor. + drop(producer, 1500n, 1501n); + expect(items(waiting)[0]).toStartWith("gap 1500.."); + expect(items(ordered())[0]).toStartWith("gap 1500.."); +}); + +test("a stale verdict narrower than an ordered read's budget leaves its time start waiting", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + hooks.dropTrackGroups(producer, 1000n, 1500n, staleHole(100)); + drop(producer, 1500n, 1600n); + const ordered = producer + .subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }) + .withRange({ start: { included: 1000n }, end: { excluded: 1600n } }) + .ordered(); + // The group containing 1500 may still land for this budget. + expect(items(ordered)).toEqual([]); + put(producer, 1200n, false); + expect(items(ordered)).toEqual(["group 1200"]); + hooks.timeStarts(producer, [[1500n, 1200n]], NO_MAX_DELAY); + producer.skipGroups(1201n, 1500n); + expect(items(ordered)).toEqual(["gap 1500..1600 (delivery timeout)"]); +}); + +test("a stale verdict a capped ordered read rejects leaves its time start waiting", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + hooks.dropTrackGroups(producer, 1000n, 1500n, staleHole(10_000)); + drop(producer, 1500n, 1600n); + const ordered = producer + .subscribe({ delay: { max: Duration.secs(1) }, ranges: [{ start: { time: 1500n } }] }) + .withRange({ start: { included: 1000n }, end: { excluded: 1600n } }) + .ordered(); + // Within the verdict's budget, but the cap keeps the read from the groups it rests on. + expect(items(ordered)).toEqual([]); + hooks.timeStarts(producer, [[1500n, 1500n]], NO_MAX_DELAY); + expect(items(ordered)).toEqual(["gap 1500..1600 (delivery timeout)"]); +}); + +test("a refusal judged stale under a narrower budget leaves a widened read's time start waiting", async () => { + const producer = receivedTrack(); + const requests = producer.requests(); + put(producer, 2000n, false); + hooks.dropTrackGroups(producer, 1000n, 1500n, staleHole(10_000)); + drop(producer, 1500n, 1600n); + const subscriber = producer + .subscribe({ delay: { max: Duration.millis(100) } }) + .withRange({ start: { included: 1000n }, end: { excluded: 1600n } }); + const ordered = subscriber.ordered(); + // Capped, so the stale run does not settle it: it asks the run again. + expect(items(ordered)).toEqual([]); + const fill = await requests.requestedFill(); + expect([fill?.start, fill?.end]).toEqual([1000n, 1500n]); + if (fill) rejectGroups(fill, 1000n, 1500n, new StreamError(StreamCode.Old), { budget: 100, everyone: false }); + // Widened past the refusal's budget, with a time start nothing held places. + subscriber.update({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }); + expect(items(ordered)).toEqual([]); + requests.close(); +}); + +test("an empty answer never extends the reach of a start reported for its time", () => { + for (const emptyFirst of [false, true]) { + const producer = receivedTrack(); + const starts: [bigint, bigint | undefined, bigint | undefined][] = [ + [1000n, 1200n, 1300n], + [1000n, undefined, 1800n], + ]; + if (emptyFirst) starts.reverse(); + hooks.timeStarts(producer, starts, NO_MAX_DELAY); + drop(producer, 1200n, 1201n); + producer.skipGroups(1201n, 1600n); + const at = (time: bigint) => + hooks.servingResolved(producer.subscribe({ delay: { max: null }, ranges: [{ start: { time } }] })); + expect(at(1250n)).toEqual([{ start: 1200n, end: undefined, floored: true }]); + // No answer reaches 1500. + expect(at(1500n)).toEqual([undefined]); + } +}); + +test("a skip is no group to place a waiting time start from", () => { + const producer = receivedTrack(); + // GROUP 2000 says the group before it is 1000, which is still on its way. + producer.skipGroups(1001n, 2000n); + put(producer, 2000n, false); + const ordered = producer.subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }).ordered(); + expect(items(ordered)).toEqual([]); + put(producer, 1000n, false); + expect(items(ordered)).toEqual(["group 1000", "group 2000"]); +}); + +test("a folded time start never places below the range covering it", () => { + const producer = receivedTrack(); + put(producer, 500n, false); + hooks.timeStarts(producer, [[1000n, 1200n, 1500n]], NO_MAX_DELAY); + // The covering range's first group was given up, and nothing lies between it and the time. + drop(producer, 1200n, 1201n); + producer.skipGroups(1201n, 1600n); + const subscriber = producer.subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }); + expect(hooks.servingResolved(subscriber)).toEqual([{ start: 1200n, end: undefined, floored: true }]); +}); + +test("a range answered empty holds back no later group within its budget, though it places nothing", () => { + const producer = receivedTrack(); + put(producer, 2000n, false); + // Asked with a one second budget: the answer says nothing about a group a wider reader still wants. + hooks.timeStarts(producer, [[1000n, undefined, 1800n]], 1000); + const at = (time: bigint) => + hooks.servingResolved(producer.subscribe({ delay: { max: null }, ranges: [{ start: { time } }] })); + expect(at(1000n)).toEqual([undefined]); + expect(at(1500n)).toEqual([undefined]); + // GROUP 2000 says nothing lies between the time and it. + producer.skipGroups(1000n, 2000n); + const read = (time: bigint, max: Duration | null) => + items(producer.subscribe({ delay: { max }, ranges: [{ start: { time } }] }).ordered()); + for (const time of [1000n, 1500n]) { + expect(read(time, Duration.secs(1))).toEqual(["group 2000"]); + expect(read(time, null)).toEqual([]); + } +}); + +// Run every pending microtask and the ones they queue, as a timer firing would. +async function flush() { + for (let i = 0; i < 20; i++) await Promise.resolve(); +} + // Mirrors the Rust `a_time_end_between_*` tests in `track.rs`. const between = { ranges: [{ start: { time: 1500n }, end: { time: 1999n } }], delay: { max: Duration.millis(30_000) } }; diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 3cb20d55f8..e84bcb2d10 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -11,7 +11,7 @@ import type { Session as IetfSession } from "./ietf/session.ts"; import { Subscriber as IetfSubscriber } from "./ietf/subscriber.ts"; import { Version as IetfVersion } from "./ietf/version.ts"; import * as Instance from "./instance.ts"; -import { hooks, staleHole } from "./internal.ts"; +import { hooks, NO_MAX_DELAY, staleHole } from "./internal.ts"; import type { Lister } from "./listing.ts"; import { Producer } from "./origin.ts"; import * as Path from "./path.ts"; @@ -2105,6 +2105,46 @@ test("a replacement spliced before the first copy answers is still checked again b.close(); }); +// A front's key-order reader holds back a gap above a time start its serving copy still waits to +// place, as a plain reader does: the group containing it may land below the gap. +test("a front's ordered reader holds a gap above a waiting time start", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + hooks.receiveTrack(ta.producer); + front.serve(a.consume()); + ta.write(2000n); + hooks.dropTrackGroups(ta.producer, 1500n, 1600n, { error: new StreamError(StreamCode.NotFound) }); + const ordered = front.consumer + .track("seq") + .subscribe({ delay: { max: null }, ranges: [{ start: { time: 1500n } }] }) + .ordered(); + const read = () => { + const out: string[] = []; + for (let item = ordered.tryNextItem(); item; item = ordered.tryNextItem()) { + if (item.kind === "group") out.push(`group ${item.group.start}`); + else out.push(`gap ${item.start}..${item.end}`); + } + return out; + }; + await settle(); + expect(read()).toEqual([]); + // A parked read wakes once the start is placed. + const next = ordered.nextItem(); + expect(await isPending(next)).toBe(true); + hooks.timeStarts(ta.producer, [[1500n, 1000n]], NO_MAX_DELAY); + ta.write(1000n); + ta.producer.skipGroups(1001n, 1500n); + ta.producer.skipGroups(1600n, 2000n); + const first = await next; + expect(first?.kind === "group" ? first.group.start : first).toBe(1000n); + expect(read()).toEqual(["gap 1500..1600", "group 2000"]); + + ordered.close(); + front.consumer.close(); + a.close(); +}); + test("an ordered reader takes a replaced copy's buffered group before the new copy's later one", async () => { const front = new Front(); const a = new BroadcastProducer(); @@ -2416,6 +2456,8 @@ for (const resolution of ["arrival", "drop", "other copy", "budget"] as const) { expect((await ordered.nextGroup())?.start).toBe(4n); ta.write(6n); tb.write(7n); + // The session reports where the range the front asked from time 4 started, as with its START. + hooks.timeStarts(tb.producer, [[4n, 7n]], NO_MAX_DELAY); hooks.startTrack(tb.producer, { start: 7n, requested: 4n }); front.serve(b.consume()); await settle(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 80ca887065..2eb13e4f20 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1692,6 +1692,14 @@ class Reader implements ResumeSource { return serving && hooks.servingRuns(serving.sub, start, end); } + // The serving copy's runs leave out the keys below a time start it still waits to place, so its + // barrier holds a gap the group containing that start may lie below. + gapHeld(at: bigint, end: bigint): boolean { + this.sync(); + const serving = this.#serving(); + return serving !== undefined && hooks.gapHeld(serving.sub, at, end); + } + servingEnd(): bigint | undefined { this.sync(); const serving = this.#serving(); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index a99ec62ca9..579a6c8b30 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -6,7 +6,7 @@ import { DEFAULT_EXPIRY } from "./cache.ts"; import { ProtocolViolation, Skipped, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; import type { Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { Version as IetfVersion } from "./ietf/version.ts"; -import { hooks, maxDelayOf, skipHole, staleHole } from "./internal.ts"; +import { hooks, maxDelayOf, NO_MAX_DELAY, skipHole, staleHole } from "./internal.ts"; import { textFrame } from "./mock.ts"; import { rescale, rescaleIn } from "./rescale.ts"; import { Timescale, Timestamp } from "./time.ts"; @@ -5653,7 +5653,8 @@ test("finishing returns handles that write nothing new but fill a late group", a // Mirrors moq-net's `unplaced_holds_only_what_containing_cant_place`: a run holds only times // `containing` can't place, but those a reported start places directly, and leaves out only -// times below the newest start reported past an earlier range's time. +// times below the newest start reported past an earlier range's time, or those the feed's +// accounting may place. test("track: a reader's unplaced times are only times its holdings can't place", () => { let seed = 5; const random = (below: number) => { @@ -5665,23 +5666,31 @@ test("track: a reader's unplaced times are only times its holdings can't place", for (let round = 0; round < 400; round++) { const received = random(2) === 0; const producer = received ? receivedTrack() : new TrackProducer("test").accept({ maxAge: null }); - const starts: [bigint, bigint][] = []; + const starts: [bigint, bigint, undefined][] = []; if (received) { - for (let n = random(3); n > 0; n--) starts.push([BigInt(random(48)), BigInt(random(48))]); - hooks.timeStarts(producer, starts); + for (let n = random(3); n > 0; n--) starts.push([BigInt(random(48)), BigInt(random(48)), undefined]); + hooks.timeStarts(producer, starts, NO_MAX_DELAY); } const reported = new Set(starts.map(([time]) => time)); let past = 0n; - for (const [asked, start] of new Map(starts)) if (asked < start && start > past) past = start; + const lowest = new Map(); + for (const [asked, start] of starts) { + const held = lowest.get(asked); + if (held === undefined || start < held) lowest.set(asked, start); + } + for (const [asked, start] of lowest) if (asked < start && start > past) past = start; const reader = producer.subscribe(); for (let step = random(16); step >= 0; step--) { const holdings = hooks.subscriberHoldings(reader); const unplaced = holdings.unplaced?.(); expect(unplaced).toBeDefined(); + const landing = received ? hooks.landingSpan(producer) : undefined; for (let at = 0n; at < 64n; at++) { const open = unplaced?.runs.some((run) => run.start <= at && at < run.end) ?? false; const unknown = holdings.containing(at) === "unknown"; if (reported.has(at)) expect([at, unknown]).toEqual([at, false]); + // The window tries the times the feed's accounting may place on each change. + else if (landing && landing.start <= at && at < landing.end) expect([at, open]).toEqual([at, false]); else if (open) expect([at, unknown]).toEqual([at, true]); else if (unknown) expect([at, received && at < past]).toEqual([at, true]); } diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 5383643d5e..040caf533b 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1010,14 +1010,79 @@ let keysOf: (producer: Producer) => readonly bigint[]; // producer's static block. See `hooks.startsAtKeys` and `hooks.timeStarts`. let timeAnchors: (producer: Producer) => { atKeys: boolean; - starts: ReadonlyMap; - /** The newest of `starts` past the time its range asked from, 0 when none is. */ + starts: ReadonlyMap; + /** The newest start in `starts` past the time its range asked from, 0 when none is. */ past: bigint; relative: ReadonlyMap; /** Bumped whenever the others, or whether the producer is a received copy, change. */ placing: number; }; +/** + * How a received copy's publisher answered a range asked from a time: the group it starts at, or + * `empty`, the budget in milliseconds under which it had no group to deliver (the publisher starts + * at the oldest group its budget has not expired, so a wider reader may still get one). `through` + * is the last time it reaches, undefined while it follows the live edge. Mirrors the Rust `TimeStart`. + */ +type TimeStart = ({ start: bigint; empty?: never } | { start?: never; empty: number }) & { through?: bigint }; + +// Whether a received copy's range asked from `time`, or the one upstream folded it into, had no +// group to deliver to a reader within `budget`: the live feed brings it none of the range's groups. +// Upstream judges a range by its terms on the wire, never a reader's local bounds, so a reader +// within the budget it was asked with gets the same answer from it. Assigned in the producer's +// static block. Mirrors the Rust `TrackState::answered_empty`. +let answeredEmpty: (producer: Producer, time: bigint, budget: number) => boolean; + +// Where a received copy's live feed landed every key it could through `time`, counting from +// `covering`, where a range covering the time started, or else its first arrival at or below +// `time`: the group containing the time starts there or past it. Undefined while a key may still +// land, or with neither to count from. Assigned in the producer's static block. Mirrors the Rust +// `TrackState::landing_from` and `TrackState::landed_through`. +let landedFrom: (producer: Producer, time: bigint, covering: bigint | undefined) => bigint | undefined; + +// The lowest key of `[start, end)` a received copy's live feed has not accounted for a reader, with +// when its gap's grace folds; see `Producer.#unaccounted`. `settled` gives the runs of a gap that +// reader has a verdict on. Unlike {@link mayLand}, a key below where the feed starts counts. Mirrors +// the Rust `TrackState::unaccounted`. +let unaccounted: ( + producer: Producer, + start: bigint, + end: bigint, + settled: (start: bigint, end: bigint) => Run[], +) => { folds?: Instant } | undefined; + +// When a received copy's time start at `time`, waiting to be placed, next may place itself with no +// change to wake its readers: the grace of the lowest gap {@link landedFrom} still waits on +// folds. Mirrors the Rust `TrackState::time_start_folds`. +let timeStartFolds: (producer: Producer, time: bigint) => Instant | undefined; + +// The times {@link landedFrom} may place on a received copy, wherever its groups and drops leave +// them: from the lowest time a first arrival or a reported range reaches, up to the farthest key +// one of them may still land. Mirrors the Rust `TrackState::landing_span`. +let landingSpan: (producer: Producer) => Run | undefined; + +// `runs`, sorted and disjoint, less what of them `cut` covers. +function withoutRun(runs: readonly Run[], cut: Run): Run[] { + const out: Run[] = []; + for (const run of runs) { + if (run.start < cut.start) out.push({ start: run.start, end: min(run.end, cut.start) }); + if (run.end > cut.end) out.push({ start: max(run.start, cut.end), end: run.end }); + } + return out; +} + +// The range asked from an earlier time that reaches `time`, which upstream folded a range asked from +// `time` into, so it reports nothing for this time itself. Mirrors the Rust +// `TrackState::covering_start`. +function coveringStart(starts: ReadonlyMap, time: bigint): TimeStart | undefined { + let covering: [bigint, TimeStart] | undefined; + for (const [asked, reported] of starts) { + if (asked >= time || (reported.through !== undefined && reported.through < time)) continue; + if (covering === undefined || asked > covering[0]) covering = [asked, reported]; + } + return covering?.[1]; +} + /** The groups a reader reads, and the exclusive end it stops at; undefined before its ranges resolve. */ type Coverage = { runs: readonly Run[]; cap?: bigint } | undefined; @@ -1282,7 +1347,10 @@ export class Producer { #startsAtKeys = false; // Where the upstream publisher started each of a lite-07 copy's live wire ranges, by the time // it asked from. Mirrors the Rust `TrackState::time_starts`. - #timeStarts = new Map(); + #timeStarts = new Map(); + // The lowest time and highest start in `#timeStarts`, so the times its ranges may place are + // bounded without a walk of them all. + #timeStartReach: { lowest: bigint; highest?: bigint } | undefined; // The newest start in `#timeStarts` past the time its range asked from, kept as it is // replaced: a reader's `unplaced` reads it on every settle. Mirrors the Rust `starts_past`. #startsPast = 0n; @@ -1338,6 +1406,10 @@ export class Producer { // subscription's grace after a later arrival opened it, as the session's tail does, so this // holds only the gaps still worth waiting for. #arrivals = new Ledger(); + // The lowest key the live feed delivered a group at. Unlike the arrivals' first key, never one + // a drop, a skip, or a forgotten run accounted, so a group starts there. Mirrors the Rust + // `first_arrival`. + #firstArrival: bigint | undefined; // What this track's own publisher produced since its last append, which skips every // key below it that never arrived and was not given up: the producer moved on. The // arrivals cannot say so, since a gap folds into them once nobody waits for it. Only @@ -1535,13 +1607,81 @@ export class Producer { producer.#startsAtKeys = true; producer.#placing++; }; - hooks.timeStarts = (producer, starts) => { - producer.#timeStarts = new Map(starts); + hooks.timeStarts = (producer, starts, budget) => { + const merged = new Map(); + for (const [time, start, through] of starts) { + // An open range never resolves empty, so such an answer says nothing. + if (start === undefined && through === undefined) continue; + const held = merged.get(time); + if (held === undefined) { + merged.set(time, start === undefined ? { empty: budget, through } : { start, through }); + continue; + } + // Two ranges asked from one time: the lower start and the farther reach hold for both. + // A start reaches only as far as the answers placing it, never an empty one's. + if (held.start !== undefined && start === undefined) continue; + if (held.start === undefined && start !== undefined) { + merged.set(time, { start, through }); + continue; + } + const reach = + held.through === undefined || through === undefined ? undefined : max(held.through, through); + merged.set( + time, + held.start === undefined || start === undefined + ? { empty: budget, through: reach } + : { start: min(held.start, start), through: reach }, + ); + } + producer.#timeStarts = merged; + let reach: { lowest: bigint; highest?: bigint } | undefined; + for (const [asked, reported] of merged) { + const start = reported.start; + const highest = + reach?.highest === undefined || start === undefined + ? (reach?.highest ?? start) + : max(reach.highest, start); + reach = { lowest: reach ? min(reach.lowest, asked) : asked, highest }; + } + producer.#timeStartReach = reach; producer.#startsPast = 0n; - for (const [asked, start] of producer.#timeStarts) { - if (asked < start && start > producer.#startsPast) producer.#startsPast = start; + for (const [asked, reported] of merged) { + const start = reported.start; + if (start !== undefined && asked < start && start > producer.#startsPast) producer.#startsPast = start; } producer.#placing++; + // The answer may place a time start an ordered read waits at. + for (const state of [producer.#state, ...producer.#sinks]) state.timelineChanged.update((rev) => rev + 1); + }; + landedFrom = (producer, time, covering) => { + const from = producer.#landingFrom(time, covering); + if (from === undefined || producer.#landingThrough(time, from) !== undefined) return undefined; + return from; + }; + landingSpan = (producer) => { + const first = producer.#firstArrival; + const reach = producer.#timeStartReach; + const asked = reach && reach.lowest + 1n; + const start = first === undefined ? asked : asked === undefined ? first : min(first, asked); + if (start === undefined) return undefined; + // The first key landable from a place only rises with it, so the highest bounds them all. + const from = first === undefined ? (reach?.highest ?? start) : max(first, reach?.highest ?? first); + const landable = producer.#landing({ start: from, end: UNBOUNDED, floor: true, grace: producer.#grace() }); + const next = landable.next(); + const end = next.done ? UNBOUNDED : next.value.start; + return end > start ? { start, end } : undefined; + }; + answeredEmpty = (producer, time, budget) => { + const reported = producer.#timeStarts.get(time) ?? coveringStart(producer.#timeStarts, time); + return reported?.empty !== undefined && budget <= reported.empty; + }; + timeStartFolds = (producer, time) => { + // A reported start places it. An empty answer places nothing, as for `containing`, so a + // reader wider than its budget still waits on folds. + if (!producer.#wire || producer.#timeStarts.get(time)?.start !== undefined) return undefined; + const covering = coveringStart(producer.#timeStarts, time); + const from = producer.#landingFrom(time, covering?.start); + return from === undefined ? undefined : producer.#landingThrough(time, from)?.folds; }; hooks.relativeStarts = (producer, starts) => { producer.#relativeStarts = new Map([...starts].map(([back, start, latest]) => [back, { start, latest }])); @@ -1631,6 +1771,7 @@ export class Producer { } }; mayLand = (producer, land) => producer.#landable(land) !== undefined; + unaccounted = (producer, start, end, settled) => producer.#unaccounted(start, end, settled); wireFed = (producer) => producer.#wire !== undefined; feedFloor = (producer) => { if (!producer.#wire) return undefined; @@ -1653,6 +1794,21 @@ export class Producer { }; } + // Where the group containing `time` starts at the earliest: `covering`, or else the feed's first + // arrival at or below `time`. Each is a group; a skip or a forgotten run names none. Mirrors the + // Rust `TrackState::landing_from`. + #landingFrom(time: bigint, covering: bigint | undefined): bigint | undefined { + const first = this.#firstArrival; + return covering ?? (first !== undefined && first <= time ? first : undefined); + } + + // Undefined once the live feed landed every key it could from `from` through `time`, else the + // lowest gap it still waits on, with when its grace folds, if it ever does. + #landingThrough(time: bigint, from: bigint): { folds?: Instant } | undefined { + const landing = this.#landing({ start: from, end: time + 1n, floor: true, grace: this.#grace() }).next(); + return landing.done ? undefined : { folds: landing.value.folds }; + } + // The first group that may still land from the live feed, by the rule a pending tail is judged // by; see {@link Land}. Mirrors the Rust `TrackState::may_land`. #landable(land: Land): { key: bigint; folds?: Instant } | undefined { @@ -1688,6 +1844,28 @@ export class Producer { } } + // The lowest key of `[start, end)` the live feed has not accounted for a reader: not arrived, + // outside the runs `settled` gives it a verdict on, before the track's end, and in a gap whose + // grace has not folded. Unlike #landing, a key below where the feed starts counts. Mirrors the + // Rust `TrackState::unaccounted`. + #unaccounted( + start: bigint, + end: bigint, + settled: (start: bigint, end: bigint) => Run[], + ): { folds?: Instant } | undefined { + if (this.#state.closed.peek() !== undefined) return undefined; + const declared = this.#state.end.peek()?.at; + if (declared !== undefined) end = min(end, finalKey(declared)); + const at = now(); + const limit = this.#grace(); + for (const gap of this.#arrivals.gaps(start, end)) { + const folds = gap.since?.saturatingAdd(limit); + if (folds !== undefined && at.ge(folds)) continue; + if (runGaps(settled(gap.start, gap.end), gap.start, gap.end).length > 0) return { folds }; + } + return undefined; + } + // Record `[start, end)` given up as `hole` says, returning the runs recorded. A skip is final // and the most authoritative verdict, so a drop never replaces one. An upstream drop (`open`) // also ends the groups still arriving in it with its error, so their readers settle now @@ -2328,7 +2506,10 @@ export class Producer { insertKey(this.#keys, group.start); this.#fills.arrived(group.start); this.#advance(group.start, !fetched); - if (!fetched) this.#arrive(group.start); + if (!fetched) { + this.#firstArrival = this.#firstArrival === undefined ? group.start : min(this.#firstArrival, group.start); + this.#arrive(group.start); + } this.#produce(group.start); // A newer group demotes the live edge to an evictable one. const edge = this.#edgeEntry(); @@ -3664,23 +3845,30 @@ export class Subscriber { // `TrackState::containing_received`. if (wire) { if (below === time) return { group: time }; - const start = anchors.starts.get(time); + const reported = anchors.starts.get(time); // The publisher serves nothing at or before it, as for a time older than every group. - if (start !== undefined) return start <= time ? { group: start } : "before"; + if (reported?.start !== undefined) + return reported.start <= time ? { group: reported.start } : "before"; + // No group to deliver holds only under the budget it was asked with, which may have left + // out a group a wider reader still gets: it places nothing. + const covering = coveringStart(anchors.starts, time)?.start; if (below !== undefined && (drops.covers(below + 1n, time + 1n) || index < keys.length)) { return { group: below }; } - if (below === undefined) { - if (drops.covers(0n, time + 1n)) return "before"; - // A range asked from an earlier time, or from the oldest group (as 0), that the - // publisher started past it says the same, while the copy holds no group there. - if (anchors.past > time) { - for (const [asked, start] of anchors.starts) { - if (asked < time && start > time) return "before"; - } - } - } - return "unknown"; + if (below === undefined && drops.covers(0n, time + 1n)) return "before"; + // The live feed accounted for every key it could land from a group at `from` through the + // time, so the group containing it came already, at `from` or past it: the newest one held + // there, or `from` itself when nothing after it through the time was a group. A group held + // below a `from` at or before the time is not it, since `from` is a group itself. One held + // below a covering range that started past the time may be one the publisher expired under + // the budget it was asked with, so it may still contain the time, which waits. Else it was + // evicted, and reaching back to the oldest group looks it up. + const from = producer && landedFrom(producer, time, covering); + if (from === undefined) return "unknown"; + if (below !== undefined && below >= from) return { group: below }; + if (below !== undefined && from > time) return "unknown"; + if (from <= time && drops.covers(from + 1n, time + 1n)) return { group: from }; + return "before"; } if (below !== undefined) return { group: below }; // Every key at or before it is gone, so whichever group held it will never arrive. @@ -3712,7 +3900,9 @@ export class Subscriber { const above = uncovered(last + 1n); if (above) runs.push(above); } - return { runs, mark }; + // The times the live feed's accounting places, which no drop or group need cover. + const landing = wire && producer ? landingSpan(producer) : undefined; + return { runs: landing ? withoutRun(runs, landing) : runs, mark }; }, nextStart: (from) => keys[keyIndex(keys, from)], // A lite-07 copy holds only part of the track, so it starts where its publisher did, @@ -4322,6 +4512,7 @@ export class Subscriber { hooks.subscriberEnd = (subscriber) => subscriber.#resume ? subscriber.#resume.end() : subscriber.#state.end.peek()?.at; hooks.subscriberHoldings = (subscriber) => subscriber.#holdings(); + hooks.landingSpan = (producer) => landingSpan(producer); hooks.subscriberEndError = (subscriber) => subscriber.#resume ? subscriber.#resume.endError() : subscriber.#state.end.peek()?.error; hooks.askSubscriber = (subscriber, ask) => { @@ -4359,6 +4550,7 @@ export class Subscriber { hooks.tryDrop = (subscriber) => subscriber.#tryDrop(); // A reader following route copies leaves its ranges to the serving copy, which resolves // their anchors against the groups it holds, then narrows them to its own local bounds. + hooks.gapHeld = (subscriber, at, end) => subscriber.#gapHeld(at, end); hooks.servingRuns = (subscriber, start, end) => { const cursor = subscriber.#cursor.peek(); const from = max(start, cursor.start); @@ -5308,11 +5500,23 @@ export class Subscriber { if (!lowest) return { kind: "idle" }; const owed = this.#nextInRange(items.at); const ready = lowest.span.start <= items.at || owed === undefined || owed >= lowest.span.start; - return ready ? this.#takeGap(items, lowest.index, lowest.span, lowest.span.end) : { kind: "idle" }; + if (!ready || this.#gapHeld(items.at, lowest.span.end)) return { kind: "idle" }; + return this.#takeGap(items, lowest.index, lowest.span, lowest.span.end); } } } + // Whether a time start still waiting below `end` holds back a gap an ordered read at `at` would + // return, as #waitTimeStart holds a group: the group containing it may lie below the gap, and an + // ordered read never goes back. A resumed reader asks its serving copy, whose runs it reads. Mirrors + // the Rust `Cursor::poll_gap_held`. + #gapHeld(at: bigint, end: bigint): boolean { + if (this.#resume) return this.#resume.gapHeld(at, end); + this.#settle(); + const start = max(max(this.#cursor.peek().start, this.#nextKey), at); + return this.#waitTimeStart(start, end, this.#drift()); + } + // The group held behind a gap, while the bounds still cover it, else the cursor's next. A group // read is settled, so no later cut or drop of it is a gap. #readHeld(items: Items): Recv { @@ -5495,6 +5699,62 @@ export class Subscriber { if (this.#dropsSeen) this.#pending = insertDrop(this.#pending, run.start, run.end, hole); } + // The runs of `[start, end)` this reader has a verdict on: a drop that settles it, or what its + // own fills answered, a group or a refusal within its budget, or it gave up. A stale verdict that does not settle it + // is none: the reader asks the group again, and only that answer counts. Mirrors the Rust + // `Misses::unanswered`. + #verdicts(start: bigint, end: bigint, drift: Drift, coverage: Coverage): Run[] { + const runs: Run[] = []; + for (const drop of this.#state.drops.overlapping(start, end)) { + if (settles({ ...drop, start: max(drop.start, start) }, drift.budget, coverage)) { + insertRun(runs, drop.start, drop.end); + } + } + // A refusal judged stale under a narrower budget is asked again once this reader widened. + for (const run of this.#state.refused.peek()) { + if (run.stale !== undefined && drift.budget > run.stale) continue; + if (run.start < end && start < run.end) insertRun(runs, run.start, run.end); + } + for (const run of intersect(this.#givenUp, start, end)) insertRun(runs, run.start, run.end); + const timeline = this.#state.timeline; + for (let i = timelineIndex(timeline, start); i < timeline.length && timeline[i].start < end; i++) { + const group = timeline[i]; + if (!hooks.groupFailed(group)) insertRun(runs, group.start, group.start + 1n); + } + return runs; + } + + // In order, the group a waiting time start resolves to comes first: it starts at or before that + // time, below `key`, and may still land at or past the read's floor `start`, until the budget + // gives up on it as on an owed group. Its landing, the answer placing the start, the grace for a + // gap below it folding, or that verdict wakes the read; a closed track places the start. + // Mirrors the Rust `Cursor::poll_time_start`. + #waitTimeStart(start: bigint, key: bigint, drift: Drift): boolean { + const producer = this.#state.producer; + let waits = false; + const coverage = this.#coverage(); + for (const time of this.#window.waitingIn(start, key)) { + if (producer && wireFed(producer)) { + // With every key from the floor through the time accounted for, a group still to come + // containing it lies below the floor, which the read never hands out. + const open = unaccounted(producer, start, time + 1n, (from, to) => + this.#verdicts(from, to, drift, coverage), + ); + if (open === undefined) continue; + if (open.folds !== undefined) this.#wakeAt(open.folds.since(Instant.ZERO).asNanos()); + } + // Its range had no group to deliver, so the feed lands none of it. Its successor is this + // group's or one below: the group containing the time is judged as an owed group at it would be. + if (producer && answeredEmpty(producer, time, drift.budget)) continue; + if (this.#staleAt(time, drift, true)) continue; + const folds = producer && timeStartFolds(producer, time); + if (folds !== undefined) this.#wakeAt(folds.since(Instant.ZERO).asNanos()); + waits = true; + } + if (waits) this.#state.waitingOrdered = true; + return waits; + } + // A missing group the subscription still owes holds sequence order until it lands, // is accounted for, or the same drift budget as a buffered group gives up on it. #waitOwed(start: bigint, end: bigint, drift: Drift): boolean { @@ -5530,6 +5790,7 @@ export class Subscriber { if (group.start < start || !this.#window.contains(group.start)) continue; if (cursor.end !== undefined && group.start >= cursor.end) return undefined; if (!this.#delivered.has(group.start) && !this.#staleAt(group.start, drift)) { + if (ordered && this.#waitTimeStart(start, group.start, drift)) return undefined; return group.start; } } @@ -5592,6 +5853,7 @@ export class Subscriber { group.close(); continue; } + if (this.#waitTimeStart(start, group.start, drift)) return { kind: "idle" }; if (this.#waitOwed(start, group.start, drift)) return { kind: "idle" }; // A group missing its head comes from the fill asked for it, in its turn. if (this.#lacksHead(group)) { diff --git a/js/net/src/window.ts b/js/net/src/window.ts index 7285fa8c39..4317d1d194 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -654,6 +654,17 @@ export class Window { this.#count(key, span); } + /** + * The times in `[start, end)` of the time starts still waiting on the groups held. The group + * containing each starts at or before its time, so one at or past a read's floor and below a + * group may still land between them. Mirrors the Rust `Window::waiting_in`. + */ + waitingIn(start: bigint, end: bigint): bigint[] { + const times: bigint[] = []; + for (const entry of this.#waiting.in(start, end)) times.push(entry.time); + return times; + } + /** Whether group `key` is in range. Everything is while a relative anchor waits. */ contains(key: bigint): boolean { return this.pending || this.#groups.contains(key); @@ -933,6 +944,14 @@ class ByTime { if (first) yield* this.#entries.from(first.time); } + /** Every entry whose time is in `[start, end)`, lowest first. */ + *in(start: bigint, end: bigint): Generator { + for (const entry of this.#entries.from(start)) { + if (entry.time >= end) return; + yield entry; + } + } + /** Add the span keyed `key` under `time`. */ add(time: bigint, key: bigint): void { const entry = this.#entries.ceil(time); diff --git a/quest/m1/time-start-asks-upstream.md b/quest/m1/time-start-asks-upstream.md new file mode 100644 index 0000000000..ea07340d7e --- /dev/null +++ b/quest/m1/time-start-asks-upstream.md @@ -0,0 +1,26 @@ +# [L] A relay copy asks upstream for every time start it cannot prove + +## Goal + +A lite-07 relay copy places a reader's time start only where it can prove what upstream would answer that reader, as `AGENTS.md` requires of a cache. For any other time start, it updates its upstream subscription and waits for the answer. +Two cases fail today, on `main` and with [moq-dev/moq#9172](https://github.com/moq-dev/moq/pull/9172) alike: + +- A reader whose time falls inside a range another reader already asks for adds no range of its own (`wire_ranges` folds it), so upstream never answers for its time. The copy infers a placement from the covering range's start (#9172), from an earlier range that started past the time ([moq-dev/moq#9174](https://github.com/moq-dev/moq/pull/9174)), or waits out the arrival grace. +- A range answer is judged under the `Subscriber Max Age` in effect when the publisher resolved it. A range never resolves again (draft: "A Range resolves once"). A reader wider than that budget, or capped below the live edge, is never answered for its own terms: an empty answer (Range Start 0) leaves it waiting out the grace, and a start (`Answer::At`) places its time for every reader, though the publisher may have started past a group it expired under the narrower budget. + +Done when a viewer behind a relay reads the same groups for a time start as it would reading the publisher directly, with any budget, joining before or after a narrower viewer. + +## Plan + +- No new wire message. A copy adds a reader's `Time` range to its upstream subscription as a range of its own in SUBSCRIBE_UPDATE whenever it cannot prove that reader's placement: a time no answer covers for the reader's budget and coverage, including one folded into another range, or one answered under a narrower budget. A new Range ID resolves fresh under the budget in effect then. Groups the subscription already delivered or dropped stay as they are. A dropped group the reader still wants comes on a request of its own, the miss path #9172's release step already relies on. +- Record each range's answer under the narrowest budget asked since that range was added (today `SubStream::stale_budget` is one value for the whole subscription). Then a range re-asked after a widening holds for every reader no wider than the new budget, as the maintainer noted on #9172. +- Accepted (maintainer, [moq-dev/moq#9172](https://github.com/moq-dev/moq/pull/9172#issuecomment-7054040452)): one more wire range per distinct unprovable reader time. These ranges are bounded by the copy's readers and retired with them. They replace the folding that keeps the range list short today, and that folding is what leaves those times unanswered: a partial hit always asks upstream for the miss. +- Then remove the inference no answer proves: #9172's covering-range placement (`landing_from` with a covering start, `Answer::Empty` releasing the barrier), #9174's earlier-range `Before`, and the hole placement in `TrackState::containing_received` (and its js/net twin in `track.ts`), which places a time between held groups whenever a higher group is held, with no Previous Start link. With groups 10, 15, and 20 at the publisher, a copy holding only 10 and 20 resolves `Time(18)` to 10 where the publisher answers 15. Keep only what the copy proves from held groups (#8638's Previous Start placement). +- Mirror in js/net (`lite/subscriber.ts` `#declare` and its demand ranges), and run `just test interop --all`. +- Reproduce first with the fixture in `rs/moq-net/tests/integration/waiting_time_start.rs` (publisher, relay, viewer). One case: an unlimited-budget ordered reader asks `Time(10)..Time(20)` and `Time(66)..` while the publisher holds only group 66. Through a relay this reads nothing, on `main` too: the relay's wire publisher waits in `Subscriber::poll_start` on the unplaced `Time(10)`, which ignores the empty answer, so it never answers its own subscriber or forwards 66. The fix covers that path as well as `Ordered`. A second: a narrow viewer joins first, then a wider one asks from a time whose group the narrow budget expired. +- [moq-dev/moq#9228](https://github.com/moq-dev/moq/pull/9228) made a time start an ordinary group fill with a `floor`, but a received copy never asks one: its readers' time ranges go upstream on the demand subscription, which is what this quest extends. +- [no-relative-anchors](/quest/m2/time-addressing/no-relative-anchors.md) needs the same ask-and-wait for a Min Age live join, which it resolves as a time start. Whichever lands first builds the mechanism. + +## Related + +- [no-relative-anchors](/quest/m2/time-addressing/no-relative-anchors.md) - needs the same ask-and-wait for a Min Age live join diff --git a/quest/m1/waiting-time-start-reads.md b/quest/m1/waiting-time-start-reads.md deleted file mode 100644 index 2879aca4dd..0000000000 --- a/quest/m1/waiting-time-start-reads.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] A waiting time start reads its owed group, and a fill reaches subscriptions - -## Goal - -A Rust reader whose time start is still waiting on the groups held never hands -out a later group ahead of the one the start resolves to, and a group a -publisher accepts from a fill is delivered on its live subscriptions the way -`@moq/net` delivers it, or the difference is documented as intended. - -## Plan - -Both were seen while chasing the interop hang that -[moq-dev/moq#9127](https://github.com/moq-dev/moq/pull/9127) fixed, through a -lite-07 publisher, a relay, and two viewers, all in one simulated test: - -- A viewer that fetched group 67, then subscribed from `Time(66)` before 66 - was written, read 67 from an `Ordered` reader at once, then reported 66 as an - `Old` gap once 66 landed, though its budget was unlimited. Its own copy held - 67 and placed nothing for the time, so the waiting span read from 66, and the - ordered read did not wait for the group the start would resolve to. -- With the fetch on the other viewer instead, the subscribing viewer got 66 and - never 67: neither the Rust publisher nor the relay served the fill-accepted - group on a subscription, while the `@moq/net` publisher serves one there - (the interop relay received 69 over its subscription in every passing run). - -Start from a test in `rs/moq-net/tests/integration` with that topology -(`MockConnectOptions` on `moq-lite-07-wip`, the publisher's `Requests` serving -67 only). Decide the second from the draft's Subscribe section: a fill below -the live edge is history, but one at or past it is the live feed. - -Public API: none expected. Wire: none. diff --git a/quest/m1/widened-reader-asks-stale-again.md b/quest/m1/widened-reader-asks-stale-again.md new file mode 100644 index 0000000000..5a34661212 --- /dev/null +++ b/quest/m1/widened-reader-asks-stale-again.md @@ -0,0 +1,15 @@ +# [M] A widened Rust ordered reader asks its own stale give-ups again + +## Goal + +When a Rust `track::Ordered` reader widens its `Delay::max`, a run its own fill gave up as stale (`Error::Old`) under the narrower budget is no verdict for it anymore, unless the reader has already returned it as a gap. The reader asks that run again, and the publisher answers under the wider budget, as js/net already does. +This is maintainer decision 1 on [moq-dev/moq#9172](https://github.com/moq-dev/moq/pull/9172#issuecomment-7026835526) applied to a reader's own refusals: a drop judged under a narrower budget never settles a wider reader. + +Today a Rust reader keeps the give-up. Reproduce on a received copy holding 900 and 2000, a stale drop of 1000..1500 under 10s, and keys 901..1000 still owed. An ordered reader of 900..1600 with a 1s budget reads 900 and asks a fill for 1000..1500, and the fill refuses it with `Old`. The reader then widens to `None`, and 901..1000 is skipped. The reader returns `Gap(1000..1500)` and never asks again. js/net, given the same steps, holds and asks again for 901..1500. + +## Plan + +- The give-up lives in three places, and each needs a stale budget to reverse it: `Misses::given_up` (`track/miss.rs`, which `give_up_stale` fills with the asked budget), the record's `reported` runs (`record.rs`, moved there when `Ordered::find_gap` polls the drop), and `Ordered::gaps`, which holds a gap found but not returned yet. +- On a budget that widens past a run's budget, withdraw it from all three: the queued gap, its `Report::Dropped`, and the `given_up` run along with its `judged` keys. The next judgement then asks for it. A gap already returned stands, since a read in key order never goes back. +- Keep #9172's time-start barrier (`Misses::unanswered`) in step: it counts a given-up run as accounted, which is right only while that run stays given up. +- Mirror check: js/net's `refused` runs carry their stale budget, and `#verdicts` skips a narrower one. Port the reproduction to `ordered.test.ts` so both stay pinned. diff --git a/rs/moq-net/proptest-regressions/model/track.txt b/rs/moq-net/proptest-regressions/model/track.txt new file mode 100644 index 0000000000..33a0a5c116 --- /dev/null +++ b/rs/moq-net/proptest-regressions/model/track.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc c443c74044d087822aa18f1202b5106aec905b52dbe5fc307ae900b041b18609 # shrinks to ops = [(2, 0, 1)], wire = true, starts = [(0, 2, None)] +cc 9f25beb5f5012b3975089a5fa29ffdf0eafecf7b9a39677b6a40249575717545 # shrinks to ops = [], wire = true, starts = [(0, None, None)] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 336cb783f9..9e0a3c0dc4 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -3299,6 +3299,60 @@ mod tests { assert_eq!(active.never_serves(None), None, "nothing to go live at"); } + /// A lite-07 range answered with no group to deliver lands none of its groups, so the copy's + /// ordered reads hold no later group back for its time. Only one whose end takes in its + /// time says so: one ending before it may still have skipped the group containing it. + #[kio::test] + async fn an_empty_answer_releases_only_a_range_reaching_its_time() { + let tail = kio::Producer::new(Tail::default()); + let Sub::Active(mut active) = feed_sub(&tail).await else { + unreachable!("feed_sub is active") + }; + // Times 10 through 20, time 30 ending at group 25, below it, and time 66 onward. + active.live = lite::LiveRanges::new( + Version::Lite07, + &[ + lite::Range::span(Some(10), 0, Some(20), None), + lite::Range::span(Some(30), 0, Some(24), None), + lite::Range::span(Some(66), 0, None, None), + ], + ); + active.starts = [(0, None), (1, None), (2, Some(66))].into(); + // Asked with no limit, so an empty answer holds for every reader. + active.stale_budget = Duration::MAX; + let info = track::Info::default().with_timescale(None).with_max_age(None); + let serving = track::Producer::new(Arc::new(crate::broadcast::Info::default()), "video", info); + serving.received(); + serving.anchored(); + serving + .create_group(group::Info { start: 66 }) + .unwrap() + .finish() + .unwrap(); + active.report_starts(&serving); + + let read = |time: u64| { + let mut ordered = serving + .subscribe( + track::Subscription::default() + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered(); + match ordered.next_item().now_or_never() { + Some(Ok(Some(track::Item::Group(group)))) => Some(group.start), + None => None, + Some(res) => panic!("not a group: {:?}", res.map(|item| item.is_some())), + } + }; + assert_eq!(read(10), Some(66), "its range reached 20, past it"); + assert_eq!(read(15), Some(66), "folded into that range"); + assert_eq!(read(30), None, "its range ended before it"); + } + /// QUIC does not order streams, so group 2's header can reach the relay before group 1's. /// A fetch held for group 1 keeps waiting through the grace and is served when it lands. #[kio::test] @@ -10229,12 +10283,11 @@ impl SubStream { Some((start, self.started.is_none())) } - /// Tell the copy where the publisher started each live range: one asked from a group's - /// start, which a reader's time start there resolves to when the copy holds no group at - /// it, and one counting back, which the copy, holding only part of the track, can't count. - /// One asked from the oldest group counts as asked from time 0, so its start places any - /// earlier time start a reader joining it asks. Lite-07 only, where each range's start is - /// reported. + /// Tell the copy where the publisher started each live range: one asked from a group, or + /// from the oldest group as time 0, with how far it reaches, which a reader's time start + /// there or folded into it resolves to when the copy holds no group at it, and one + /// counting back, which the copy, holding only part of the track, can't count. Lite-07 + /// only, where each range's start is reported. fn report_starts(&self, serving: &track::Producer) { if !self.live.reports_starts() { return; @@ -10242,21 +10295,45 @@ impl SubStream { let mut times = Vec::new(); let mut relative = Vec::new(); for (id, range) in self.live.iter() { - let Some(Some(start)) = self.starts.get(&id) else { + let Some(&start) = self.starts.get(&id) else { continue; }; - match range.start { - Some(lite::Anchor::Absolute(time)) => times.push((time.group, *start)), - None => times.push((0, *start)), + // The last time the range reaches: an exclusive group end stops at the time before + // it, and a relative end, unknown here, at the range's own time. + let through = |asked: u64| match range.end { + None => None, + Some(lite::Anchor::Absolute(end)) if end.frame == 0 => Some(end.group.saturating_sub(1).max(asked)), + Some(lite::Anchor::Absolute(end)) => Some(end.group), + Some(lite::Anchor::Time(end)) => Some(end), + Some(lite::Anchor::Relative(_)) => Some(asked), + }; + // Whether a range with no group to deliver says its publisher held none at or before + // its reach: only one whose end takes in the group containing its time, which a + // range that ends before its time, or by a count back, may not. + let reaches = |asked: u64| match range.end { + Some(lite::Anchor::Absolute(end)) if end.frame == 0 => end.group > asked, + Some(lite::Anchor::Absolute(end)) => end.group >= asked, + Some(lite::Anchor::Time(end)) => end >= asked, + None | Some(lite::Anchor::Relative(_)) => false, + }; + let asked = match range.start { + None => 0, + Some(lite::Anchor::Absolute(time)) => time.group, Some(lite::Anchor::Relative(back)) => { - if let Some(Some(latest)) = self.edges.get(&id) { - relative.push((back, *start, *latest)); + if let (Some(start), Some(Some(latest))) = (start, self.edges.get(&id)) { + relative.push((back, start, *latest)); } + continue; } - _ => {} + _ => continue, + }; + if start.is_some() || reaches(asked) { + times.push((asked, start, through(asked))); } } - serving.time_starts(times); + // The wire does not say which budget a range resolved under, so an empty one holds only + // for the narrowest asked. + serving.time_starts(times, self.stale_budget); serving.relative_starts(relative); } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 41d4ca92b5..167e8967bb 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -6690,6 +6690,49 @@ mod test { } } + /// A front's key-order reader holds back a gap above a time start its serving copy still + /// waits to place, as a plain reader does: the group containing it may land below the gap. + #[test] + fn a_front_holds_a_gap_above_a_waiting_time_start() { + let routes = Producer::new(); + let logical = logical(&routes); + let serving = received_copy(); + serving.anchored(); + routes.serve(serving.consume()); + write(&serving, 2000); + serving.drop_groups(1500..1600, StreamError::NotFound); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([track::Range { + start: Anchor::Time(1500), + end: None, + }]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap() + .ordered(); + let waiter = kio::Waiter::noop(); + let mut read = || -> Vec { + let mut items = Vec::new(); + while let Poll::Ready(Ok(Some(item))) = sub.poll_next_item(&waiter) { + items.push(match item { + track::Item::Group(group) => format!("group {}", group.start), + track::Item::Gap { range, .. } => format!("gap {range:?}"), + }); + } + items + }; + assert_eq!(read(), Vec::::new(), "the gap waits on the start"); + serving.time_starts([(1500, Some(1000), None)], Duration::MAX); + write(&serving, 1000); + serving.skip_groups(1001..1500); + serving.skip_groups(1600..2000); + assert_eq!(read(), ["group 1000", "gap 1500..1600", "group 2000"]); + } + /// A front's key-order reader reports a group that lands on its copy after the read passed /// its key, as old, and never one it handed out. #[test] diff --git a/rs/moq-net/src/model/subscription/window.rs b/rs/moq-net/src/model/subscription/window.rs index 649330cc8b..62d64ae5eb 100644 --- a/rs/moq-net/src/model/subscription/window.rs +++ b/rs/moq-net/src/model/subscription/window.rs @@ -133,11 +133,12 @@ pub(crate) enum Found { Unanswered, } -/// The times a hop's [`Holdings::containing`] can't place, as up to two runs: every time in -/// them is [`Containing::Unknown`], except the few that `mark` places another way. +/// The times a hop's [`Holdings::containing`] can't place, as up to four runs, sorted: every +/// time in them is [`Containing::Unknown`], except the few that `mark` places another way. +/// They may leave out some unknown times too, which a window then tries each time. #[derive(Clone, Debug, PartialEq, Eq)] pub(in crate::model) struct Unplaced { - pub runs: [ops::RangeInclusive; 2], + pub runs: [ops::RangeInclusive; 4], /// Changes whenever a time inside the runs may have come to be placed by something other /// than a group arriving or a drop, so a window that saw another mark tries every start. pub mark: u64, @@ -146,6 +147,22 @@ pub(in crate::model) struct Unplaced { impl Unplaced { /// An empty run, for a hop that places every time on one side. pub const NONE: ops::RangeInclusive = ops::RangeInclusive::new(1, 0); + + /// `run` less what of it `cut` covers: the part below it, then the part above it. + pub fn without(run: ops::RangeInclusive, cut: &ops::RangeInclusive) -> [ops::RangeInclusive; 2] { + if run.is_empty() || cut.is_empty() { + return [run, Self::NONE]; + } + let below = match cut.start().checked_sub(1) { + Some(end) => *run.start()..=(*run.end()).min(end), + None => Self::NONE, + }; + let above = match cut.end().checked_add(1) { + Some(start) => start.max(*run.start())..=*run.end(), + None => Self::NONE, + }; + [below, above] + } } /// A hop that holds no groups of its own, such as a reader following route copies that @@ -351,6 +368,21 @@ impl Window { self.waiting.last().is_some_and(|&(time, _)| time > floor) } + /// The times in `times` of the time starts still waiting on the groups held. The group + /// containing each starts at or before its time, so one at or past a read's floor and + /// below a group may still land between them. + pub fn waiting_in + 'static>(&self, times: R) -> impl Iterator + '_ { + let from = match times.start_bound() { + ops::Bound::Included(&start) => start, + ops::Bound::Excluded(&start) => start.saturating_add(1), + ops::Bound::Unbounded => 0, + }; + self.waiting + .range((from, 0)..) + .map(|&(time, _)| time) + .take_while(move |time| times.contains(time)) + } + /// Resolve what the groups now held settle: relative anchors once `latest` exists, and /// time starts once a group arrives. Returns whether the groups in range moved. pub fn settle(&mut self, latest: Option, holdings: &impl Holdings) -> bool { @@ -1014,7 +1046,7 @@ mod tests { false => Unplaced::NONE, }; Some(Unplaced { - runs: [unknown, Unplaced::NONE], + runs: [unknown, Unplaced::NONE, Unplaced::NONE, Unplaced::NONE], mark: 0, }) } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8ac17d86bd..3525776a07 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -605,6 +605,34 @@ impl Anchors { } } +/// How a received copy's publisher answered a range asked from a time, and the last time the +/// range reaches, `None` while it follows the live edge. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct TimeStart { + answer: Answer, + through: Option, +} + +/// Where a publisher started a range, as its answer says. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Answer { + /// The group the range starts at. + At(u64), + /// No group to deliver, under at most this budget: the publisher starts at the oldest group + /// its budget has not expired, so a wider reader may still get one. + Empty(Duration), +} + +impl TimeStart { + /// The group the range starts at, `None` when it had none to deliver. + fn start(&self) -> Option { + match self.answer { + Answer::At(start) => Some(start), + Answer::Empty(_) => None, + } + } +} + /// Where a received copy's publisher started a range counting back, and the latest group it /// counted from. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -734,6 +762,9 @@ pub(crate) struct TrackState { // away a subscription's grace after a later arrival opened it, as the session's tail // does, so this holds only the gaps still worth waiting for. arrivals: Ledger, + // The lowest key the live feed delivered a group at. Unlike the arrivals' first key, never + // one a forgotten drop or skip accounted, so a group starts there. + first_arrival: Option, // What `arrivals` bills to the pool, [`ledger::RUN_OVERHEAD`] per gap. Paying eviction // debt folds its oldest gaps early, so an unbounded grace holds no more gaps than the // pool would. @@ -756,10 +787,14 @@ pub(crate) struct TrackState { // of the track cannot tell which earlier group contains the time. starts_at_keys: bool, // Where the upstream publisher started each of a lite-07 copy's live wire ranges, by the - // time it asked from: a copy holding part of the track cannot tell which group contains - // a time it holds no group at, but its publisher reports where it resolved one. Replaced - // whole as the ranges change, so it holds at most one entry per live range. - time_starts: BTreeMap, + // time it asked from (0 for the oldest group): a copy holding part of the track cannot tell + // which group contains a time it holds no group at, but its publisher reports where it + // resolved one. Replaced whole as the ranges change, so it holds at most one entry per + // time asked from. + time_starts: BTreeMap, + // The highest start in `time_starts`, so the times its ranges may place are bounded + // without a walk of them all. + highest_time_start: Option, // The newest start in `time_starts` past the time its range asked from, kept as it is // replaced: [`Holdings::unplaced`] reads it on every settle of every reader. starts_past: Option, @@ -1162,7 +1197,7 @@ impl Holdings for TrackState { Some(run) => run.end..=u64::MAX, None => from..=u64::MAX, }; - let runs = if self.starts_at_keys { + let [below, above] = if self.starts_at_keys { [none.clone(), none] } else { match (self.lookup.first_key_value(), self.lookup.last_key_value()) { @@ -1186,8 +1221,15 @@ impl Holdings for TrackState { } } }; + // The times the live feed's accounting places, which no drop or group need cover. + let landing = match self.anchors == Anchors::Wire && !self.starts_at_keys { + true => self.landing_span(), + false => Unplaced::NONE, + }; + let [below_low, below_high] = Unplaced::without(below, &landing); + let [above_low, above_high] = Unplaced::without(above, &landing); Some(Unplaced { - runs, + runs: [below_low, below_high, above_low, above_high], mark: self.placing, }) } @@ -1378,12 +1420,24 @@ impl TrackState { return Containing::Group(time); } match self.time_starts.get(&time) { - Some(&start) if start <= time => return Containing::Group(start), + Some(&TimeStart { + answer: Answer::At(start), + .. + }) if start <= time => return Containing::Group(start), // The publisher serves nothing at or before it, as it does for a time older than // every group it holds. - Some(_) => return Containing::Before, - None => {} + Some(TimeStart { + answer: Answer::At(_), .. + }) => return Containing::Before, + // No group to deliver holds only under the budget it was asked with, which may + // have left out a group a wider reader still gets: it places nothing. + Some(TimeStart { + answer: Answer::Empty(_), + .. + }) + | None => {} } + let covering = self.covering_start(time).and_then(|reported| reported.start()); match self.lookup.range(..=time).next_back() { Some((&key, _)) if self.dropped_cover(key.saturating_add(1)..time.saturating_add(1)) @@ -1391,16 +1445,124 @@ impl TrackState { { Containing::Group(key) } - None if self.dropped_cover(0..time.saturating_add(1)) - || (self.starts_past.is_some_and(|past| past > time) - && self.time_starts.range(..time).any(|(_, &start)| start > time)) => - { - Containing::Before - } - _ => Containing::Unknown, + None if self.dropped_cover(0..time.saturating_add(1)) => Containing::Before, + // The live feed accounted for every key it could land from a group at `from` + // through the time, so the group containing it came already, at `from` or past it: + // the newest one held there, or `from` itself when nothing after it through the + // time was a group. A group held below a `from` at or before the time is not it, + // since `from` is a group itself. One held below a covering range that started + // past the time may be one the publisher expired under the budget it was asked + // with, so it may still contain the time, which waits. Else it was evicted, and + // reaching back to the oldest group, as on the publisher's own track, looks it up. + held => match self.landing_from(time, covering) { + Some(from) if self.landed_through(time, from) => match held { + Some((&key, _)) if key >= from => Containing::Group(key), + Some(_) if from > time => Containing::Unknown, + _ if from <= time && self.dropped_cover(from.saturating_add(1)..time.saturating_add(1)) => { + Containing::Group(from) + } + _ => Containing::Before, + }, + _ => Containing::Unknown, + }, } } + /// Whether the range asked from `time`, or the one upstream folded it into, had no group to + /// deliver to a reader within `budget`: the live feed brings it none of the range's groups. + /// Upstream judges a range by its terms on the wire, never a reader's local bounds, so a + /// reader within the budget it was asked with gets the same answer from it. + fn answered_empty(&self, time: u64, budget: Duration) -> bool { + let reported = self + .time_starts + .get(&time) + .copied() + .or_else(|| self.covering_start(time)); + matches!(reported, Some(TimeStart { answer: Answer::Empty(asked), .. }) if budget <= asked) + } + + /// The range asked from an earlier time that reaches `time`, which upstream folded a range + /// asked from `time` into, so it reports nothing for this time itself. + fn covering_start(&self, time: u64) -> Option { + self.time_starts + .range(..time) + .rev() + .find(|(_, reported)| reported.through.is_none_or(|through| through >= time)) + .map(|(_, reported)| *reported) + } + + /// The times [`Self::landed_through`] may place, wherever the groups and drops leave them: + /// from the lowest time a first arrival or a reported range reaches, through the key below + /// the first one the highest place it counts from may still land. The first key landable + /// from a place only rises with it, so that one bounds them all. Empty when it places none. + fn landing_span(&self) -> ops::RangeInclusive { + if !self.readable() { + return Unplaced::NONE; + } + let first = self.first_arrival; + let asked = self.time_starts.keys().next().map(|asked| asked.saturating_add(1)); + let Some(lowest) = first.into_iter().chain(asked).min() else { + return Unplaced::NONE; + }; + let from = first.into_iter().chain(self.highest_time_start).max().unwrap_or(lowest); + let grace = self.arrival_grace(); + match self.find_landable(from..u64::MAX, true, grace, |open, _| Some(open.start)) { + Some(key) => match key.checked_sub(1) { + Some(last) => lowest..=last, + None => Unplaced::NONE, + }, + None => lowest..=u64::MAX, + } + } + + /// When a received copy's time start at `time`, waiting to be placed, next may place + /// itself with no change to wake its readers: the arrival grace of the lowest gap + /// [`Self::landed_through`] still waits on folds. + fn time_start_folds(&self, time: u64) -> Option { + // A reported start places it. An empty answer places nothing, as for + // [`Self::containing_received`], so a reader wider than its budget still waits on folds. + if !self.anchors.received() + || self + .time_starts + .get(&time) + .is_some_and(|reported| reported.start().is_some()) + { + return None; + } + let covering = self.covering_start(time).and_then(|reported| reported.start()); + let from = self.landing_from(time, covering)?; + self.landing_through(time, from).flatten() + } + + /// Where the group containing `time` starts at the earliest, which [`Self::landed_through`] + /// counts from: `covering`, where a range covering the time started, or else the feed's + /// first arrival at or below `time`. Each is a group, so the one containing the time is + /// there or past it, and a range added below changes nothing of it. A skip or a forgotten + /// drop names no group. Arrival order is not key order, so with neither, a lower key may + /// still land. + fn landing_from(&self, time: u64, covering: Option) -> Option { + covering.or_else(|| self.first_arrival.filter(|&first| first <= time)) + } + + /// Whether the live feed may land nothing more at or below `time` that could contain it: + /// every key from `from` (see [`Self::landing_from`]) through `time` arrived, was dropped, + /// or is no longer waited for. A covering range that started past the time served nothing + /// at or before it. + fn landed_through(&self, time: u64, from: u64) -> bool { + self.landing_through(time, from).is_none() + } + + /// `None` once [`Self::landed_through`] holds, else `Some` of when the arrival grace of the + /// lowest gap it still waits on folds, if it ever does. + fn landing_through(&self, time: u64, from: u64) -> Option> { + if !self.readable() { + return Some(None); + } + self.find_landable(from..time.saturating_add(1), true, self.arrival_grace(), |_, folds| { + Some(folds) + }) + } + /// Whether drops, for whatever reason, cover every key in `keys`. An empty run is covered. fn dropped_cover(&self, keys: ops::Range) -> bool { keys.is_empty() || self.dropped.get(&keys.start).is_some_and(|run| run.end >= keys.end) @@ -2863,6 +3025,7 @@ impl TrackState { /// Record that the live feed delivered `key`, then fold the gaps that aged out. fn record_arrival(&mut self, key: u64, now: crate::time::Instant) { self.arrivals.arrive(key, now); + self.first_arrival = Some(self.first_arrival.map_or(key, |first| first.min(key))); if !self.anchors.received() && self.appended.is_none_or(|appended| key >= appended) && let Some(end) = key.checked_add(1) @@ -3427,6 +3590,37 @@ impl TrackState { None } + /// When the grace folds for the lowest key of `keys` the live feed has not accounted for + /// a reader under `drift`: not arrived, without a verdict settling it, before the track's + /// end, and in a gap whose grace has not folded. A stale verdict that does not settle the + /// reader is no verdict: it asks the group again, and only the answer, in `misses`, counts. + /// `None` with none left. Unlike [`Self::find_landable`], a key below where the feed starts + /// counts. + fn unaccounted( + &self, + keys: ops::Range, + drift: &Drift, + grace: Duration, + misses: Option<&miss::Misses>, + ) -> Option> { + let end = keys.end.min(self.final_key().unwrap_or(u64::MAX)); + let now = self.cache.pool().now_wall(); + for (gap, opened) in self.arrivals.gaps(keys.start..end) { + let folds = opened.and_then(|opened| opened.checked_add(grace)); + if folds.is_some_and(|folds| now >= folds) { + continue; + } + let open = match misses { + Some(misses) => misses.unanswered(self, gap, drift), + None => miss::unsettled(self, gap, drift), + }; + if !open.is_empty() { + return Some(folds); + } + } + None + } + /// The lowest group in `groups` and `window` a key-order reader still waits for: /// one only a subscription owes, so a received copy's live feed may still land it. Not a /// group accounted for by its arrivals (since evicted or aborted), dropped, @@ -4070,9 +4264,20 @@ impl TrackState { // A head holding frame 0 gives a group received past it a start. self.fetched += 1; } - // Invisible to arrival-order subscribers: fetched on demand, not produced - // live by the publisher. - None => self.insert_group(&group.handle(), false), + // Below the live edge, history: invisible to arrival-order subscribers, served by + // key to whoever misses it. A whole group past every one the live publisher + // produced is the live feed, which a subscription delivers like any group it + // produces. A received copy's feed is its upstream's subscription, which delivers + // the group itself, and a track nobody accepted has no live feed yet. + None => { + let live = !self.anchors.received() + && self.minted() + && self.feed.is_live() + && frames.start == 0 + && frames.end.is_none() + && self.newest_produced.is_none_or(|newest| key > newest); + self.insert_group(&group.handle(), live) + } } self.evict_expired(); Ok(Some(group)) @@ -4920,18 +5125,48 @@ impl Producer { } /// Record where the upstream publisher started each of this lite-07 copy's live wire - /// ranges, as `(time, start)` pairs by the time each asked from, replacing the last set. - /// A reader's time start this copy cannot place among its own groups resolves to it. + /// ranges, as `(time, start, through)` by the time each asked from (0 for the oldest group), + /// `None` for one with no group to deliver, and the last time it reaches (`None` while + /// open), replacing the last set. A reader's time start this copy cannot place among its + /// own groups resolves to the range asked from it, or to one reaching it that upstream + /// folded it into. An empty answer holds for a reader within `budget`, the narrowest the + /// subscription asked with, since the publisher resolves a start by the budget it asked. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn time_starts(&self, starts: impl IntoIterator) { - let starts: BTreeMap = starts.into_iter().collect(); + pub(crate) fn time_starts( + &self, + starts: impl IntoIterator, Option)>, + budget: Duration, + ) { + let mut merged: BTreeMap = BTreeMap::new(); + for (time, start, through) in starts { + // An open range never resolves empty, so such an answer says nothing. + if start.is_none() && through.is_none() { + continue; + } + let answer = start.map_or(Answer::Empty(budget), Answer::At); + // Two ranges asked from one time: the lower start and the farther reach hold for both. + // A start reaches only as far as the answers placing it, never an empty one's. + merged + .entry(time) + .and_modify(|held| { + let reach = held.through.zip(through).map(|(a, b)| a.max(b)); + (held.answer, held.through) = match (held.answer, answer) { + (Answer::At(held), Answer::At(start)) => (Answer::At(held.min(start)), reach), + (Answer::At(_), Answer::Empty(_)) => (held.answer, held.through), + (Answer::Empty(_), Answer::At(_)) => (answer, through), + (Answer::Empty(_), Answer::Empty(_)) => (held.answer, reach), + }; + }) + .or_insert(TimeStart { answer, through }); + } + let starts = merged; if let Ok(mut state) = self.modify() && state.time_starts != starts { + state.highest_time_start = starts.values().filter_map(TimeStart::start).max(); state.starts_past = starts .iter() - .filter(|&(asked, start)| asked < start) - .map(|(_, &start)| start) + .filter_map(|(&asked, reported)| reported.start().filter(|&start| asked < start)) .max(); state.time_starts = starts; state.placing += 1; @@ -9654,6 +9889,85 @@ impl Cursor { }) } + /// Pending while a time start in `times` waits to be placed and the group containing it, + /// at or before its time and so below `times.end`, may still land at or past the read's + /// floor `times.start` within the drift budget. Its landing, the answer placing the start, + /// the grace for a gap below it folding, or the budget giving up on it wakes us; a closed + /// track places it. `Ready(true)` when a change placed one meanwhile, so the caller looks + /// again. + fn poll_time_start(&mut self, times: ops::Range, drift: &Drift, waiter: &kio::Waiter) -> Poll { + if self.filter.window.is_settled() || self.filter.window.waiting_in(times.clone()).next().is_none() { + return Poll::Ready(false); + } + let window = &self.filter.window; + let waited = std::cell::RefCell::new(Vec::new()); + let closed = self + .poll(waiter, |state| { + waited.borrow_mut().clear(); + // An unjudgeable old gap cannot hold a finite read, as for an owed group. + let floor = times.start.max(state.cache_floor(drift.budget)); + let grace = state.arrival_grace(); + for time in window.waiting_in(floor..times.end) { + // With every key from the floor through the time accounted for, a group still + // to come containing it lies below the floor, which the read never hands out. + let unaccounted = match state.anchors.received() { + true => state.unaccounted(floor..time.saturating_add(1), drift, grace, self.misses.as_deref()), + false => Some(None), + }; + let Some(unaccounted) = unaccounted else { + continue; + }; + if let Some(folds) = unaccounted { + state.cache.watch_wall(folds, waiter); + } + // Its range had no group to deliver, so the feed lands none of it. Its + // successor is this group's or one below: the group containing the time is + // judged as an owed group at it would be. + if state.answered_empty(time, drift.budget) + || state.poll_drifted(time, &drift.edge.servable, drift.budget, None, waiter) + { + continue; + } + if let Some(folds) = state.time_start_folds(time) { + state.cache.watch_wall(folds, waiter); + } + waited.borrow_mut().push(time); + } + Poll::>::Pending + }) + .is_ready(); + let waited = waited.into_inner(); + if closed || waited.is_empty() { + return Poll::Ready(false); + } + // Registered before syncing again, so a change placing one meanwhile is not missed. + self.sync(); + let window = &self.filter.window; + if waited + .iter() + .any(|&time| window.waiting_in(time..=time).next().is_none()) + { + return Poll::Ready(true); + } + Poll::Pending + } + + /// Pending while a time start below `end` holds a gap there as [`Self::poll_time_start`] + /// holds a group: the group containing it may lie below the gap, and an ordered read never + /// goes back. `from` is where the ordered read stands. + fn poll_gap_held(&mut self, from: u64, end: u64, waiter: &kio::Waiter) -> Poll<()> { + self.sync(); + let drift = match ready!(self.poll_drift(&self.servable, waiter)) { + Ok(drift) => drift, + // The read reports the error. + Err(_) => return Poll::Ready(()), + }; + let from = from.max(self.next_key).max(self.floor()); + // One placed meanwhile changed the runs, which the caller looks at again. + while ready!(self.poll_time_start(from..end, &drift, waiter)) {} + Poll::Ready(()) + } + /// Whether the drift budget says to skip `group`, against a [`Drift`] already resolved /// for this poll. fn poll_stale(&self, group: &group::Consumer, drift: &Drift, waiter: &kio::Waiter) -> Poll> { @@ -10365,6 +10679,12 @@ impl Cursor { continue; } + // In order means the group a waiting time start resolves to comes first, until the + // budget gives up on it. + if ready!(self.poll_time_start(from..group.start, &drift, waiter)) { + continue; + } + // In order means a group the live feed still owes below this one comes first, // until the budget gives up on it. Its landing, its drop, or that verdict wakes us. ready!(self.poll_owed(from..group.start, &drift, waiter))?; @@ -11068,6 +11388,17 @@ impl Subscriber { record.tell(keys, Report::Settled); } + /// Pending while a waiting time start below `end` holds back a gap an [`Ordered`] read at + /// `from` would return; see [`Cursor::poll_gap_held`]. A resumed reader's own window owes + /// every key from the read until a copy places its time starts, so [`Self::next_in_range`] + /// already holds such a gap. + fn poll_gap_held(&mut self, from: u64, end: u64, waiter: &kio::Waiter) -> Poll<()> { + match &mut self.inner { + Inner::Plain(cursor) => cursor.poll_gap_held(from, end, waiter), + Inner::Resume(..) => Poll::Ready(()), + } + } + /// The key cursor behind [`Ordered`], which owns the only public door to it. pub(super) fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Poll>> { self.poll_group(true, waiter) @@ -11118,9 +11449,21 @@ impl Subscriber { let start = ready!(Consumer::poll_state_start(&cursor.state, waiter)); cursor.sync(); if cursor.filter.window.is_waiting_above(cursor.min_key) { - // Registered before syncing again, so a change placing it after this wakes it. A - // closed track places it at once, since nothing more can arrive. - let closed = cursor.state.poll(waiter, |_| Poll::<()>::Pending).is_ready(); + // Registered before syncing again, so a change placing it after this wakes it, as + // does the grace for a gap below one folding. A closed track places it at once, + // since nothing more can arrive. + let (window, floor) = (&cursor.filter.window, cursor.min_key); + let closed = cursor + .state + .poll(waiter, |state| { + for time in window.waiting_in(floor.saturating_add(1)..) { + if let Some(folds) = state.time_start_folds(time) { + state.cache.watch_wall(folds, waiter); + } + } + Poll::<()>::Pending + }) + .is_ready(); cursor.sync(); if !closed && cursor.filter.window.is_waiting_above(cursor.min_key) { return Poll::Pending; @@ -12096,7 +12439,7 @@ impl Ordered { } } // What the read still owes lies above the gap, so the gap goes first. - Poll::Pending => match self.next_gap() { + Poll::Pending => match self.next_gap(waiter) { Some(item) => Poll::Ready(Ok(Some(item))), None => Poll::Pending, }, @@ -12213,8 +12556,13 @@ impl Ordered { } } - /// The lowest gap, once nothing the read still owes lies below it. - fn next_gap(&mut self) -> Option { + /// The lowest gap, once nothing the read still owes lies below it, and no time start still + /// waiting below its end may resolve to a group below it. + fn next_gap(&mut self, waiter: &kio::Waiter) -> Option { + let (_, span) = self.lowest_gap()?; + if self.inner.poll_gap_held(self.at, span.end, waiter).is_pending() { + return None; + } let (at, span) = self.lowest_gap()?; let ready = span.start <= self.at || self.inner.next_in_range(self.at).is_none_or(|owed| owed >= span.start); let resume = span.end; @@ -12726,28 +13074,44 @@ mod test { /// for the ones a copy's reported time starts place, after each step: through groups /// arriving and leaving, and every way keys are given up. A time it leaves out lies below /// the newest start reported past an earlier range's time, which may place it once no held - /// group precedes it. + /// group precedes it, or where a lite-07 copy's feed accounting may place it, which the + /// window tries each time. #[test] fn unplaced_holds_only_what_containing_cant_place( ops in proptest::collection::vec((0u8..6, 0u64..40, 1u64..8), 0..24), wire: bool, - starts in proptest::collection::vec((0u64..48, 0u64..48), 0..3), + starts in proptest::collection::vec( + (0u64..48, proptest::option::of(0u64..48), proptest::option::of(0u64..56)), + 0..3 + ), ) { let producer = track_producer("test", Info::default()); if wire { + producer.received(); producer.anchored(); - producer.time_starts(starts.clone()); + producer.time_starts(starts.clone(), Duration::MAX); } - let reported: BTreeSet = starts.iter().map(|(time, _)| *time).collect(); - let past = starts.iter().filter(|(asked, start)| asked < start).map(|(_, start)| *start).max(); + // A range answered with no group to deliver places nothing. + let reported: BTreeSet = starts + .iter() + .filter(|(_, start, _)| start.is_some()) + .map(|(time, ..)| *time) + .collect(); + let past = starts + .iter() + .filter_map(|&(asked, start, _)| start.filter(|&start| asked < start)) + .max(); let check = |producer: &Producer| -> std::result::Result<(), proptest::test_runner::TestCaseError> { let state = producer.state.read(); let unplaced = state.unplaced().expect("a track always tells"); + let landing = state.landing_span(); for time in (0..64).chain([u64::MAX - 1, u64::MAX]) { let inside = unplaced.runs.iter().any(|run| run.contains(&time)); let unknown = state.containing(time) == Containing::Unknown; if wire && reported.contains(&time) { proptest::prop_assert!(!unknown, "a reported start places {}", time); + } else if wire && landing.contains(&time) { + proptest::prop_assert!(!inside, "the runs leave {} to be tried", time); } else if inside { proptest::prop_assert!(unknown, "{} is unplaced but placed", time); } else if unknown { @@ -16196,7 +16560,7 @@ mod test { // The publisher started the range asked from 1700 at 1000, and the one from 500 at 1000 // too, past it: nothing at or before 500 is served. - producer.time_starts([(1700, 1000), (500, 1000)]); + producer.time_starts([(1700, Some(1000), None), (500, Some(1000), None)], Duration::MAX); assert_eq!(placed(&mut answered), Some((1000, true))); assert_eq!(placed(&mut before), Some((0, false))); assert_eq!(placed(&mut later), None); @@ -16214,6 +16578,658 @@ mod test { assert_eq!(drain(&mut past), vec![1000, 3000]); } + /// An ordered read of a lite-07 copy hands out no group past a time start still waiting to + /// be placed, since the group containing the time may still land below it: it reads that + /// group first, with no gap, once the publisher's answer and the group arrive. + #[test] + #[cfg(feature = "async")] + fn an_ordered_read_waits_for_its_time_start_to_be_placed() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered(); + let mut next = || match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Group(group)))) => Some(Ok(group.start)), + Some(Ok(Some(Item::Gap { range, .. }))) => Some(Err(range)), + Some(res) => panic!("ended: {:?}", res.map(|_| ())), + }; + assert_eq!(next(), None, "2000 waits on the start"); + // A session's tail holds the read at the answered start's group until it lands, and + // says nothing lies between it and 2000. + producer.time_starts([(1500, Some(1000), None)], Duration::MAX); + write_group(&producer, 1000); + producer.skip_groups(1001..2000); + assert_eq!(next(), Some(Ok(1000))); + assert_eq!(next(), Some(Ok(2000))); + } + + /// A gap at a time start still waiting goes after the group the start resolves to, as a + /// later group does: that group may lie below the gap, and an ordered read never goes back. + #[test] + #[cfg(feature = "async")] + fn a_waiting_time_start_holds_back_a_gap_above_its_group() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + producer.drop_groups(1500..1600, StreamError::NotFound); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered(); + let mut next = || match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Group(group)))) => Some(Ok(group.start)), + Some(Ok(Some(Item::Gap { range, .. }))) => Some(Err(range)), + Some(res) => panic!("ended: {:?}", res.map(|_| ())), + }; + assert_eq!(next(), None, "the gap waits on the start"); + producer.time_starts([(1500, Some(1000), None)], Duration::MAX); + write_group(&producer, 1000); + producer.skip_groups(1001..1500); + producer.skip_groups(1600..2000); + assert_eq!(next(), Some(Ok(1000))); + assert_eq!(next(), Some(Err(1500..1600))); + assert_eq!(next(), Some(Ok(2000))); + } + + /// A range its publisher answered with no group to deliver, under a budget, lands none of + /// its groups for a reader within that budget, so that reader's ordered read holds no later + /// group back for the time it asks from, or one folded into it. A wider reader may still + /// get one once the budget widens, so it keeps waiting, and the answer places neither time. + #[test] + #[cfg(feature = "async")] + fn an_empty_range_holds_back_no_later_group_within_its_budget() { + let budget = Duration::from_millis(100); + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + producer.time_starts([(1000, None, Some(1800))], budget); + { + let state = producer.state.read(); + assert_eq!(state.containing(1000), Containing::Unknown); + assert_eq!(state.containing(1500), Containing::Unknown, "folded into it"); + } + let read = |time: u64, max: Option| { + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(time), + end: None, + }]) + .with_delay(Delay::default().with_max(max)), + ) + .ordered(); + match ordered.next_item().now_or_never() { + Some(Ok(Some(Item::Group(group)))) => Some(group.start), + None => None, + Some(res) => panic!("not a group: {:?}", res.map(|item| item.is_some())), + } + }; + assert_eq!(read(1000, Some(budget)), Some(2000)); + assert_eq!(read(1500, Some(budget)), Some(2000), "folded into it"); + assert_eq!(read(1500, None), None, "a wider reader may still get 1000"); + } + + /// A range answered empty releases a reader within its budget whatever that reader's own + /// bounds: the range's wire end, or none past it, leaves it no group to wait for. Here the + /// publisher holds only 66, so a reader asking `Time(10)..Time(20)` and `Time(66)..` reads it. + #[test] + #[cfg(feature = "async")] + fn an_empty_bounded_range_holds_back_no_later_range() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 66); + producer.time_starts([(10, None, Some(20)), (66, Some(66), None)], Duration::MAX); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([ + Range { + start: Anchor::Time(10), + end: Some(Anchor::Time(20)), + }, + Range { + start: Anchor::Time(66), + end: None, + }, + ]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered(); + match ordered.next_item().now_or_never() { + Some(Ok(Some(Item::Group(group)))) => assert_eq!(group.start, 66), + None => panic!("the empty range holds back 66"), + Some(res) => panic!("not a group: {:?}", res.map(|item| item.is_some())), + } + } + + /// A lite-07 copy places a time start that upstream folded into an earlier range by that + /// range's answer, which reports nothing for the time itself: one that started past the + /// time served nothing at or before it, so an ordered read goes on. A range ending before + /// the time says nothing of it. + #[test] + #[cfg(feature = "async")] + fn a_folded_time_start_places_from_the_range_covering_it() { + let ordered = |producer: &Producer| { + producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered() + }; + let next = |ordered: &mut Ordered| match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Group(group)))) => Some(group.start), + Some(res) => panic!("not a group: {:?}", res.map(|item| item.is_some())), + }; + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + + // A range asked from 1000 that ends at 1200 doesn't reach the time. + producer.time_starts([(1000, Some(1000), Some(1200))], Duration::MAX); + let mut bounded = ordered(&producer); + assert_eq!(next(&mut bounded), None, "the start still waits"); + + // The oldest group's range, answered at 2000, reaches it. + producer.time_starts([(1000, Some(1000), Some(1200)), (0, Some(2000), None)], Duration::MAX); + assert_eq!(next(&mut bounded), Some(2000)); + assert_eq!(next(&mut ordered(&producer)), Some(2000)); + } + + /// A lite-07 copy whose live feed accounted for every key through a time start's time + /// places it though the group containing it was since evicted, as when its range was folded + /// into a wider one upstream, which answers nothing for the time: an ordered read goes on + /// rather than wait for a group that came and went. + #[test] + #[cfg(feature = "async")] + fn an_evicted_start_group_still_places_a_time_start() { + let (mut producer, _pool) = pooled_producer(2_000); + producer.received(); + producer.anchored(); + producer.start_resolved(1000).unwrap(); + let mut big = producer.create_group(group::Info { start: 1000 }).unwrap(); + big.write_frame( + Timestamp::from_millis(1000).unwrap(), + bytes::Bytes::from(vec![0u8; 10_000]), + ) + .unwrap(); + big.finish().unwrap(); + for (key, below) in [(2000, 1001), (3000, 2001)] { + producer.skip_groups(below..key); + write_group(&producer, key); + } + assert!(producer.consume().peek_group(1000).is_none(), "1000 is evicted"); + + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered(); + let mut groups = Vec::new(); + while let Some(item) = ordered.next_item().now_or_never() { + if let Item::Group(group) = item.unwrap().expect("an item") { + groups.push(group.start); + } + } + assert_eq!(groups.last(), Some(&3000), "the read goes on past the evicted group"); + } + + /// A time start folded into an open range waits out the grace of a gap below its time, + /// where the group containing it may still land, and that grace folding alone wakes an + /// ordered read parked on it, with no change to the track: the group after the time + /// arrived later than the gap opened, so the read's own budget runs out after it. + #[kio::test] + #[cfg(feature = "async")] + async fn a_folded_time_start_wakes_its_read_when_a_gap_below_folds() { + let grace = replay().budget(); + let mut producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + elapse(&producer, Duration::ZERO); + producer.start_resolved(1000).unwrap(); + producer.time_starts([(0, Some(1000), None)], Duration::MAX); + // Group 1000's stream reset before its header: only 1800, 2000, and the skips around + // them land, 1800 last. + producer.skip_groups(1001..1800); + producer.skip_groups(1801..2000); + write_group(&producer, 2000); + elapse(&producer, Duration::from_millis(10)); + write_group(&producer, 1800); + let grace = grace - Duration::from_millis(10); + let mut ordered = producer + .subscribe(replay().with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }])) + .ordered(); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); + assert!(ordered.poll_next_item(&waiter).is_pending(), "1000 may still land"); + elapse(&producer, grace - Duration::from_millis(1)); + assert!(!woken.load(Ordering::SeqCst), "woken before the grace"); + assert!(ordered.poll_next_item(&waiter).is_pending()); + elapse(&producer, Duration::from_millis(1)); + assert!(woken.load(Ordering::SeqCst), "the grace never woke the read"); + let mut read = Vec::new(); + while let Poll::Ready(item) = ordered.poll_next_item(&waiter) { + match item.unwrap().expect("an item") { + Item::Group(group) => { + read.push(group.start); + break; + } + Item::Gap { .. } => {} + } + } + // 1800 came a full budget after 2000 superseded it. + assert_eq!(read, [2000]); + } + + /// A start [`Subscriber::poll_start`] waits on, folded into an open range, wakes it when the + /// grace of a gap below its time folds, with no change to the track. + #[kio::test] + #[cfg(feature = "async")] + async fn a_folded_time_start_wakes_poll_start_when_a_gap_below_folds() { + let grace = replay().budget(); + let mut producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + elapse(&producer, Duration::ZERO); + producer.start_resolved(1000).unwrap(); + producer.time_starts([(0, Some(1000), None)], Duration::MAX); + producer.skip_groups(1001..2000); + write_group(&producer, 2000); + let mut subscriber = producer.subscribe(replay().with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }])); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); + assert!(subscriber.poll_start(&waiter).is_pending(), "1000 may still land"); + elapse(&producer, grace - Duration::from_millis(1)); + assert!(!woken.load(Ordering::SeqCst), "woken before the grace"); + assert!(subscriber.poll_start(&waiter).is_pending()); + elapse(&producer, Duration::from_millis(1)); + assert!(woken.load(Ordering::SeqCst), "the grace never woke the start"); + assert!(subscriber.poll_start(&waiter).is_ready()); + } + + /// A time start its own range answered empty, under a budget narrower than the reader's, + /// places nothing, so the reader still waits on the feed placing it, and the grace of a gap + /// below folding wakes it as it does any start the copy places from its own groups. + #[kio::test] + #[cfg(feature = "async")] + async fn a_time_start_answered_empty_wakes_poll_start_when_a_gap_below_folds() { + let grace = replay().budget(); + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + elapse(&producer, Duration::ZERO); + producer.time_starts([(1500, None, Some(1800))], Duration::from_millis(1)); + // The feed's first group, cut since, and a later one with the gap between still folding. + let cut = producer.create_group(group::Info { start: 1000 }).unwrap(); + cut.abort(Error::Cancel).unwrap(); + producer.modify().unwrap().reclaim_aborted(1000); + write_group(&producer, 2000); + let mut subscriber = producer.subscribe(replay().with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }])); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); + assert!( + subscriber.poll_start(&waiter).is_pending(), + "a group above 1000 may still land" + ); + elapse(&producer, grace); + assert!(woken.load(Ordering::SeqCst), "the grace never woke the start"); + assert!(subscriber.poll_start(&waiter).is_ready()); + } + + /// A finite budget gives up on the group a waiting time start resolves to as it would on an + /// owed one: once the later group it holds is a full budget old, an ordered read hands it + /// out rather than wait for an answer that may never come. + #[kio::test] + #[cfg(feature = "async")] + async fn a_waiting_time_start_holds_a_read_only_within_its_budget() { + let budget = Duration::from_millis(100); + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + elapse(&producer, Duration::ZERO); + write_group(&producer, 2000); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(budget)), + ) + .ordered(); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); + assert!( + ordered.poll_next_item(&waiter).is_pending(), + "the group at 1500 may land" + ); + elapse(&producer, budget - Duration::from_millis(1)); + assert!(!woken.load(Ordering::SeqCst), "woken before the budget"); + assert!(ordered.poll_next_item(&waiter).is_pending()); + elapse(&producer, Duration::from_millis(1)); + assert!(woken.load(Ordering::SeqCst), "the budget never woke the read"); + let mut read = Vec::new(); + while let Poll::Ready(item) = ordered.poll_next_item(&waiter) { + if let Item::Group(group) = item.unwrap().expect("an item") { + read.push(group.start); + break; + } + } + assert_eq!(read, [2000]); + } + + /// A waiting time start below an ordered read's own floor holds nothing: the group + /// containing its time starts at or before it, which the read never hands out. + #[test] + #[cfg(feature = "async")] + fn a_waiting_time_start_below_the_read_floor_holds_nothing() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .with_range(2000..) + .ordered(); + match ordered.next_item().now_or_never() { + Some(Ok(Some(Item::Group(group)))) => assert_eq!(group.start, 2000), + other => panic!("2000 waited: {:?}", other.map(|res| res.map(|item| item.is_some()))), + } + } + + /// A waiting time start whose time the read's floor reaches holds no gap once every key from + /// the floor through the time is accounted for: the group containing it lies below the floor, + /// which the read never hands out. + #[test] + #[cfg(feature = "async")] + fn a_waiting_time_start_holds_no_gap_once_nothing_past_the_floor_may_contain_it() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + let ordered = |producer: &Producer| { + producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .with_range(1500..1600) + .ordered() + }; + let next = |ordered: &mut Ordered| match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Gap { range, .. }))) => Some(range), + Some(res) => panic!("not a gap: {:?}", res.map(|item| item.is_some())), + }; + producer.drop_groups(1501..1600, StreamError::NotFound); + let mut waiting = ordered(&producer); + assert_eq!(next(&mut waiting), None, "1500 may still land"); + producer.drop_groups(1500..1501, StreamError::NotFound); + assert_eq!(next(&mut waiting).map(|gap| gap.start), Some(1500)); + assert_eq!(next(&mut ordered(&producer)).map(|gap| gap.start), Some(1500)); + } + + /// A stale verdict under a narrower budget than an ordered read's settles none of its keys for + /// it, so the group containing a waiting time start may still land there, and a gap above + /// the time waits for it. + #[test] + #[cfg(feature = "async")] + fn a_narrower_stale_verdict_leaves_a_time_start_waiting() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + producer.give_up(1000..1500, Hole::stale(Duration::from_millis(100))); + producer.drop_groups(1500..1600, StreamError::NotFound); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .with_range(1000..1600) + .ordered(); + let mut next = || match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Group(group)))) => Some(Ok(group.start)), + Some(Ok(Some(Item::Gap { range, .. }))) => Some(Err(range)), + Some(res) => panic!("ended: {:?}", res.map(|_| ())), + }; + assert_eq!(next(), None, "the group containing 1500 may still land"); + write_group(&producer, 1200); + assert_eq!(next(), Some(Ok(1200))); + // Passed as a gap, so nothing the read may still take can contain 1500. + assert_eq!(next(), Some(Err(1201..1500))); + assert_eq!(next(), Some(Err(1500..1600))); + } + + /// A stale verdict within a capped read's budget settles none of its keys either: the cap + /// keeps it from the newer groups the verdict rests on. The read asks the run again, and a + /// gap above a waiting time start holds until that answer comes. + #[test] + #[cfg(feature = "async")] + fn a_capped_read_asks_a_stale_run_again_before_passing_a_time_start() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + let dynamic = producer.requests(); + write_group(&producer, 900); + write_group(&producer, 2000); + producer.skip_groups(901..1000); + producer.give_up(1000..1500, Hole::stale(Duration::from_secs(10))); + producer.drop_groups(1500..1600, StreamError::NotFound); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(Some(Duration::from_secs(1)))), + ) + .with_range(900..1600) + .ordered(); + let mut read = || { + let mut items = Vec::new(); + loop { + match ordered.next_item().now_or_never() { + None => return items, + Some(Ok(Some(Item::Group(group)))) => items.push(Ok(group.start)), + Some(Ok(Some(Item::Gap { range, .. }))) => items.push(Err(range)), + Some(res) => panic!("ended: {:?}", res.map(|_| ())), + } + } + }; + assert_eq!(read(), [Ok(900)], "the read asks the stale run again"); + let fill = dynamic.requested_fill().now_or_never().expect("asked").unwrap(); + assert_eq!(fill.start().group..fill.end(), 1000..1500); + fill.accept(1200, None) + .unwrap() + .expect("a fresh group") + .finish() + .unwrap(); + fill.refuse_groups(1000..1200, Error::Old); + fill.refuse_groups(1201..1500, Error::Old); + assert_eq!(read(), [Err(1000..1200), Ok(1200), Err(1201..1500), Err(1500..1600)]); + } + + /// Below a time start placed by nothing held, a stale verdict a capped read rejects leaves its + /// keys to the upstream's answer for the time: the gap above it waits for that answer. + #[test] + #[cfg(feature = "async")] + fn a_stale_verdict_a_capped_read_rejects_leaves_a_time_start_waiting() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 2000); + producer.give_up(1000..1500, Hole::stale(Duration::from_secs(10))); + producer.drop_groups(1500..1600, StreamError::NotFound); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(Some(Duration::from_secs(1)))), + ) + .with_range(1000..1600) + .ordered(); + let mut next = || match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Group(group)))) => Some(Ok(group.start)), + Some(Ok(Some(Item::Gap { range, .. }))) => Some(Err(range)), + Some(res) => panic!("ended: {:?}", res.map(|_| ())), + }; + assert_eq!(next(), None, "the group containing 1500 may still land"); + producer.time_starts([(1500, Some(1500), None)], Duration::MAX); + assert_eq!(next(), Some(Err(1500..1600))); + } + + /// A range answered empty reaches no time for the start another range asked from the same + /// time reported: that start covers only the times its own answer reaches. + #[test] + fn an_empty_answer_never_extends_the_reach_of_a_start() { + for empty_first in [false, true] { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 500); + let mut starts = vec![(1000, Some(1200), Some(1300)), (1000, None, Some(1800))]; + if empty_first { + starts.reverse(); + } + producer.time_starts(starts, Duration::MAX); + let cut = producer.create_group(group::Info { start: 1200 }).unwrap(); + cut.abort(Error::Cancel).unwrap(); + producer.modify().unwrap().reclaim_aborted(1200); + producer.skip_groups(1201..1600); + let state = producer.state.read(); + assert_eq!(state.containing(1250), Containing::Group(1200)); + assert_eq!(state.containing(1500), Containing::Unknown, "no answer reaches it"); + } + } + + /// A skip, even one forgotten into the feed's accounting, names no group, so it can't stand + /// in for the feed's first arrival a waiting time start counts from: the group containing + /// the time may still land below it. + #[test] + #[cfg(feature = "async")] + fn a_forgotten_skip_is_no_group_to_place_a_time_start_from() { + let mut producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + // Asked from the live edge, so the feed's accounting starts wherever it lands. + producer.request_start(None).unwrap(); + // GROUP 2000 says the group before it is 1000, which is still on its way. + producer.skip_groups(1001..2000); + write_group(&producer, 2000); + producer.forget_drops(1001..2000); + assert_eq!( + producer.state.read().arrivals.first(), + Some(1001), + "the skip is accounted" + ); + assert_eq!(producer.state.read().containing(1500), Containing::Unknown); + let mut ordered = producer + .subscribe( + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(1500), + end: None, + }]) + .with_delay(Delay::default().with_max(None)), + ) + .ordered(); + let mut next = || match ordered.next_item().now_or_never() { + None => None, + Some(Ok(Some(Item::Group(group)))) => Some(group.start), + Some(res) => panic!("not a group: {:?}", res.map(|item| item.is_some())), + }; + assert_eq!(next(), None, "1000 may still land"); + write_group(&producer, 1000); + assert_eq!(next(), Some(1000)); + assert_eq!(next(), Some(2000)); + } + + /// A time start folded into a range that started at a group since cut places at that + /// range's start, never at an older group held below it, which the range doesn't serve. + #[test] + fn a_folded_time_start_never_places_below_its_covering_range() { + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 500); + producer.time_starts([(1000, Some(1200), Some(1500))], Duration::MAX); + let cut = producer.create_group(group::Info { start: 1200 }).unwrap(); + cut.abort(Error::Cancel).unwrap(); + producer.modify().unwrap().reclaim_aborted(1200); + producer.skip_groups(1201..1600); + assert!(!producer.state.read().lookup.contains_key(&1200), "1200 is cut"); + assert_eq!(producer.state.read().containing(1500), Containing::Group(1200)); + } + /// A reader joining a lite-07 copy that another reader already asks from further back adds /// no range of its own, so its publisher reports no start for the reader's time. The start /// of a range asked from earlier, or from the oldest group, places it: past the time, no @@ -16244,11 +17260,11 @@ mod test { assert!(before.poll_start(&kio::Waiter::noop()).is_pending()); // Only a range asked from a later time: says nothing of 500. - producer.time_starts([(700, 1000)]); + producer.time_starts([(700, Some(1000), None)], Duration::MAX); assert_eq!(placed(&mut before), None); // The range asked from the oldest group started at 1000. - producer.time_starts([(0, 1000)]); + producer.time_starts([(0, Some(1000), None)], Duration::MAX); assert_eq!(placed(&mut before), Some((0, false))); assert!(before.poll_start(&kio::Waiter::noop()).is_ready()); assert_eq!(drain(&mut before), vec![1000]); @@ -16277,7 +17293,7 @@ mod test { write_group(&producer, 2000); let mut reader = producer.subscribe(from(2500)); - producer.time_starts([(0, 3000)]); + producer.time_starts([(0, Some(3000), None)], Duration::MAX); assert_eq!(placed(&mut reader), None, "the held 2000 may contain 2500"); write_group(&producer, 3000); @@ -16308,7 +17324,7 @@ mod test { write_group(&producer, 2000); let mut reader = producer.subscribe(from(2500)); - producer.time_starts([(0, 3000)]); + producer.time_starts([(0, Some(3000), None)], Duration::MAX); assert_eq!(placed(&mut reader), None, "the held 2000 may contain 2500"); producer.evict(2000); @@ -17957,12 +18973,14 @@ mod test { assert!(!consumer.is_live(), "a track whose producer left is not live"); } - /// A one-shot fetch populates the shared cache but never the live arrival cursor, - /// so it cannot make content available to a subscription look stale. + /// A received copy's one-shot fetch populates the shared cache but never the live arrival + /// cursor, which only its upstream's subscription feeds, so it cannot make content available + /// to a subscription look stale. #[kio::test] #[cfg(feature = "async")] async fn fetched_group_is_not_a_live_drift_edge() { let mut producer = track_producer("test", Info::timed()); + producer.received(); let dynamic = producer.requests(); let consumer = producer.consume(); append_at(&mut producer, 0); @@ -17988,6 +19006,49 @@ mod test { groups.assert_no_group(); } + /// A publisher's fill answer past every group it produced is its live feed: a subscription + /// delivers it in arrival order, as a group the publisher produces. One below the live edge + /// stays history, served by key to whoever misses it. + #[kio::test] + #[cfg(feature = "async")] + async fn a_fill_past_the_live_edge_is_the_live_feed() { + let producer = track_producer("test", Info::timed()); + let dynamic = producer.requests(); + let consumer = producer.consume(); + let mut subscriber = producer.subscribe(replay()); + for key in [0, 1] { + producer + .create_group(group::Info { start: key }) + .unwrap() + .finish() + .unwrap(); + } + assert_eq!(drain(&mut subscriber), vec![0, 1]); + + let fill = |key: u64| { + let pending = consumer.fetch_group(key, None); + let fill = dynamic + .requested_fill_group() + .now_or_never() + .expect("the fetch asks at once") + .unwrap(); + let mut group = fill.accept(None).unwrap().expect("a fresh group"); + group + .write_frame(Timestamp::from_millis(key).unwrap(), b"x".as_slice()) + .unwrap(); + group.finish().unwrap(); + pending + }; + fill(3).await.unwrap(); + assert_eq!(drain(&mut subscriber), vec![3], "past the edge, live"); + let mut live = producer.subscribe(replay().with_ranges([Range::live()])); + assert_eq!(drain(&mut live), vec![3], "the live edge"); + // 3 moved the live edge, so 2 lands below it: the fetch gets it, no subscription does. + assert_eq!(fill(2).await.unwrap().start, 2); + assert!(drain(&mut subscriber).is_empty(), "below the edge, history"); + assert!(drain(&mut live).is_empty(), "below the edge, history"); + } + /// A failed fill answer is no verdict on a gap the feed owes: an ordered reader neither takes /// the failed copy nor moves past it, and gets the feed's group once it lands. Mirrors JS /// `fill: a failed answer to a gap the feed owes holds the end and the ordered reader`. @@ -24940,6 +26001,7 @@ mod test { #[cfg(feature = "async")] async fn the_feed_takes_a_group_a_fetch_cached_first() { let (producer, _pool) = pooled_producer(1 << 40); + producer.received(); let dynamic = producer.requests(); let consumer = producer.consume(); let mut subscriber = producer.subscribe(replay()); @@ -27972,12 +29034,13 @@ mod test { assert_eq!(drain(&mut live), vec![0, 1]); } - /// Fetched backfill past the newest produced group moves no relative start, whether the - /// track had a produced group at subscribe or not. + /// A received copy's fetched backfill past the newest group its feed delivered moves no + /// relative start, whether the copy had a group at subscribe or not. #[kio::test] #[cfg(feature = "async")] async fn a_live_range_ignores_a_fetched_group_past_the_newest_group() { let producer = track_producer("test", Info::timed()); + producer.received(); let dynamic = producer.requests(); let consumer = producer.consume(); let mut early = producer.subscribe(replay().with_ranges([Range::live()])); diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index eaada91ab8..3af51ec769 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -1013,6 +1013,10 @@ impl FillGroups { /// Insert group `key` into the track's cache, handing it to the readers waiting on /// it, and return a [`group::Producer`] to fill. /// + /// On a track this publisher accepted, a group past every one it produced is its live feed, + /// which its subscriptions deliver too; one below is history, served only to whoever misses + /// it. A relay's copy leaves that to its upstream's subscription. + /// /// The group starts at [`start`](Self::start)'s frame when it is the first group, so the /// frames written keep the indices they have in the group. A key from the /// [`floor`](Self::floor) up to the start's answers the start with that group, from its diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index a8ad8d36f0..edfc9dca7f 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -280,6 +280,21 @@ impl Deferred { } } +/// The part of `hole` the track's drops leave open for a reader under `drift`: a received +/// stale run that does not settle it is no verdict for it (see `Drift::settles`), so it asks +/// the group again. +pub(super) fn unsettled(state: &TrackState, hole: ops::Range, drift: &Drift) -> RangeSet { + let mut left = RangeSet::new(); + left.insert(hole.clone()); + // Only what overlaps the hole, so judging every run costs each one the same. + for (run, gone) in state.drops.overlapping(&hole) { + if drift.settles(run.start.max(hole.start), gone) { + left.remove(run.clone()); + } + } + left +} + impl Misses { /// The lowest key still to come from a fill: asked, or answered and not handed out. pub fn lowest(&self) -> Option { @@ -1334,23 +1349,24 @@ impl Misses { /// The part of `hole` nobody has a verdict on yet: a received stale run that does not /// settle this reader is no verdict for it (see `Drift::settles`). fn unclaimed(&self, state: &TrackState, hole: ops::Range, drift: &Drift) -> Vec> { - let mut left = RangeSet::new(); - left.insert(hole.clone()); - // Only what overlaps the hole, so judging every run costs each one the same. - for (run, gone) in state.drops.overlapping(&hole) { - if drift.settles(run.start.max(hole.start), gone) { - left.remove(run.clone()); - } + let mut left = self.unanswered(state, hole.clone(), drift); + for run in self.wanted.overlapping(&hole) { + left.remove(run.clone()); } - for claimed in [&self.given_up, &self.wanted] { - for run in claimed.overlapping(&hole) { - left.remove(run.clone()); - } + left.iter().cloned().collect() + } + + /// The part of `hole` with no verdict for this reader and no answer from its fills: still + /// to ask, or asked and not answered. + pub(super) fn unanswered(&self, state: &TrackState, hole: ops::Range, drift: &Drift) -> RangeSet { + let mut left = unsettled(state, hole.clone(), drift); + for run in self.given_up.overlapping(&hole) { + left.remove(run.clone()); } for key in self.filled.range(hole) { left.remove(*key..*key + 1); } - left.iter().cloned().collect() + left } /// Take in a fill's refusal of `groups`: final where the feed gave up on them too, diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index c8de57db0a..a7efb47ccd 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -106,6 +106,7 @@ mod tracks_ietf; mod unannounce_release; mod untimed; mod viewer_target; +mod waiting_time_start; mod warm_backfill; mod wide_key; mod widened_max_age; diff --git a/rs/moq-net/tests/integration/waiting_time_start.rs b/rs/moq-net/tests/integration/waiting_time_start.rs new file mode 100644 index 0000000000..f599cfcefa --- /dev/null +++ b/rs/moq-net/tests/integration/waiting_time_start.rs @@ -0,0 +1,183 @@ +#![cfg(feature = "async")] + +//! A lite-07 publisher, a relay, and two viewers, the way the interop replacement round +//! drives them: the publisher writes group 66, a viewer fetches group 67, which the publisher +//! serves only from a fill, and a viewer then subscribes from time 66. + +use crate::support; + +use std::time::Duration; + +use moq_net::track::{Anchor, Delay, Item, Order, Range, Subscription}; +use moq_net::{Timestamp, Version}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(5); + +/// The group only a fill serves. +const FILLED: u64 = 67; +/// The group the time start resolves to, written live. +const LIVE: u64 = 66; + +fn origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = + moq_net::origin::Producer::new(moq_net::origin::Config::new(std::num::NonZeroU64::new(hop).unwrap())); + support::harness::spawn(driver); + producer.with_proxy(true) +} + +async fn link(from: &moq_net::origin::Producer, to: &moq_net::origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + connect_mock(options).await +} + +fn write(mut group: moq_net::group::Producer) { + let key = group.start; + group + .write_frame(Timestamp::unscaled(key), key.to_string().into_bytes()) + .unwrap(); + group.finish().unwrap(); +} + +struct Fixture { + track: moq_net::track::Producer, + viewers: [moq_net::broadcast::Consumer; 2], + serving: kio::clock::sim::JoinHandle<()>, + _broadcast: moq_net::origin::Route, + _links: Vec, + _origins: Vec, +} + +impl Fixture { + /// A publisher whose scale-less track holds nothing, serving [`FILLED`] from a fill and + /// dropping every other fill, a relay, and two viewers of it. + async fn new() -> Self { + let publisher = origin(1); + let broadcast = publisher + .route("bcast", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let track = broadcast + .broadcast() + .create_track("value", moq_net::track::Info::default().with_max_age(None)) + .unwrap(); + broadcast.announce().unwrap(); + let requests = track.requests(); + let serving = kio::clock::sim::spawn(async move { + while let Ok(fill) = requests.requested_fill().await { + if (fill.start().group..fill.end()).contains(&FILLED) + && let Ok(Some(group)) = fill.accept(FILLED, None) + { + write(group); + } + } + }); + + let relay = origin(2); + let mut links = vec![link(&publisher, &relay).await]; + let mut viewers = Vec::new(); + let mut origins = vec![publisher, relay.clone()]; + for hop in 3..5 { + let viewer = origin(hop); + links.push(link(&relay, &viewer).await); + let remote = kio::clock::timeout(TIMEOUT, viewer.consume().follow("bcast", None).unwrap().available()) + .await + .expect("resolve timeout") + .expect("broadcast resolves") + .unwrap(); + viewers.push(remote); + origins.push(viewer); + } + Self { + track, + viewers: viewers.try_into().map_err(drop).unwrap(), + serving, + _broadcast: broadcast, + _links: links, + _origins: origins, + } + } + + /// Fetch [`FILLED`] on viewer `index` and read it whole. + async fn fetch(&self, index: usize) { + let mut fetched = kio::clock::timeout( + TIMEOUT, + self.viewers[index].track("value").unwrap().fetch_group(FILLED, None), + ) + .await + .expect("fetch timeout") + .unwrap(); + let frame = kio::clock::timeout(TIMEOUT, fetched.read_frame()) + .await + .expect("read timeout") + .unwrap() + .expect("a frame"); + assert_eq!(&frame.payload[..], FILLED.to_string().as_bytes()); + } + + /// An ordered reader on viewer `index` from time [`LIVE`], with no budget. + async fn subscribe(&self, index: usize) -> moq_net::track::Ordered { + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_order(Order::Ascending) + .with_ranges([Range { + start: Anchor::Time(LIVE), + end: None, + }]); + kio::clock::timeout( + TIMEOUT, + self.viewers[index].track("value").unwrap().subscribe(subscription), + ) + .await + .expect("subscribe timeout") + .unwrap() + .ordered() + } + + /// Write [`LIVE`] at the publisher. + fn write_live(&self) { + write(self.track.create_group(moq_net::group::Info { start: LIVE }).unwrap()); + } +} + +/// The keys of the next `count` items `reader` returns, a gap as its range. +async fn read(reader: &mut moq_net::track::Ordered, count: usize) -> Vec>> { + let mut items = Vec::new(); + while items.len() < count { + let item = kio::clock::timeout(TIMEOUT, reader.next_item()) + .await + .unwrap_or_else(|_| panic!("timed out after {items:?}")) + .unwrap() + .expect("an item"); + items.push(match item { + Item::Group(group) => Ok(group.start), + Item::Gap { range, .. } => Err(range), + }); + } + items +} + +/// A viewer that holds [`FILLED`] from its own fetch reads the group its time start resolves +/// to first, though its copy can't place the start until the publisher answers. +#[kio::test] +async fn a_waiting_time_start_reads_its_group_before_a_held_later_one() { + let fixture = Fixture::new().await; + fixture.write_live(); + fixture.fetch(0).await; + let mut reader = fixture.subscribe(0).await; + assert_eq!(read(&mut reader, 2).await, [Ok(LIVE), Ok(FILLED)]); + fixture.serving.abort(); +} + +/// A group the publisher accepted from another viewer's fetch, past the live edge, reaches a +/// subscription covering it. +#[kio::test] +async fn a_filled_group_past_the_live_edge_reaches_a_subscription() { + let fixture = Fixture::new().await; + fixture.write_live(); + fixture.fetch(1).await; + let mut reader = fixture.subscribe(0).await; + assert_eq!(read(&mut reader, 2).await, [Ok(LIVE), Ok(FILLED)]); + fixture.serving.abort(); +}