diff --git a/js/net/bench/track.ts b/js/net/bench/track.ts index 9fe3d8d3b87..ce7d312ebc7 100644 --- a/js/net/bench/track.ts +++ b/js/net/bench/track.ts @@ -1,6 +1,7 @@ /** Sweep retained groups and subscribers for one group published through a track and received. */ import { Milli } from "@moq/clock"; +import { DEFAULT_EXPIRY } from "../src/cache.ts"; import type { Consumer as GroupConsumer, Producer as GroupProducer } from "../src/group.ts"; import { Timescale, Timestamp } from "../src/time.ts"; import { Producer, type Subscriber } from "../src/track.ts"; @@ -16,8 +17,6 @@ const holds = [ ] as const; type Hold = (typeof holds)[number]; const groups = 100; -// Matches the private idle window and Rust cache::DEFAULT_EXPIRY. -const CACHE_WINDOW_MS = 30_000; const reps = 7; // Publishing a group must not cost more as the retained window grows, or a guard or cache pass // has gone back to scanning every retained group. The sweep spans 60x, so a scan lands well past @@ -61,7 +60,7 @@ async function measure( let sequence = 0; // Inserted by sequence, the way the wire hands a subscribed track its groups. const publish = (close: boolean) => { - clock += held > 0 ? 1 : CACHE_WINDOW_MS / retained; + clock += held > 0 ? 1 : DEFAULT_EXPIRY / retained; const group = producer.createGroup(BigInt(sequence)); group.writeFrame({ payload, timestamp: Timestamp.fromMillis(sequence++) }); if (close) group.finish(); @@ -114,7 +113,7 @@ async function measure( producer.finish(); producer.close(); // Ending the track releases the newest group too; idle cleanup reclaims the cache. - clock += CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8 + 1; + clock += 2 * DEFAULT_EXPIRY; const idleStart = now(); const replay = producer.subscribe({ delay: { max: window } }); if (replay.tryRecvGroup()) throw new Error("idle cache entries were retained"); diff --git a/js/net/src/broadcast.test.ts b/js/net/src/broadcast.test.ts index 0215b901ad0..b8511cf68c4 100644 --- a/js/net/src/broadcast.test.ts +++ b/js/net/src/broadcast.test.ts @@ -513,10 +513,9 @@ test("a stalled consumer does not pin evicted groups", async () => { producer.writeFrame(textFrame("old")); - // Advance past the cache window, plus the scan slice a prune may lag by, and write - // again to trigger a prune. The old (closed, aged-out) group is dropped from the - // stalled sink, not retained. - clock.jump(Nano.fromMilli(Milli(30_000 + 30_000 / 8))); + // Advance past the cache window and write again, which scans for idle groups. The old + // (closed, aged-out) group is dropped from the stalled sink, not retained. + clock.jump(Nano.fromMilli(Milli(30_000 + 1_000))); producer.writeFrame(textFrame("fresh")); // The next group in arrival order is the fresh one (seq 1): the old (seq 0) group diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 39cd0104c7f..b01c1e53592 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -4,6 +4,7 @@ * @module */ import { type GetPromise, Once, Signal } from "@moq/signals"; +import * as cache from "./cache.ts"; import * as datagram from "./datagram.ts"; import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; @@ -54,6 +55,16 @@ class BroadcastState { unheld = false; // Run synchronously once the broadcast closes, before `closed` notifies its observers. closers = new Set<() => void>(); + // The cache pool every track of the broadcast caches into, defaulting as Rust's + // `broadcast::Info` does: no byte capacity, and the default idle window. + pool = new cache.Pool({ expiry: cache.DEFAULT_EXPIRY }); +} + +// A new track of the broadcast, caching into its pool. +function createProducer(state: BroadcastState, name: string): track.Producer { + const producer = new track.Producer(name); + hooks.poolTrack(producer, state.pool); + return producer; } // Datagram tracks never enter the stream listing or cache. @@ -204,7 +215,7 @@ function logical(state: BroadcastState, name: string): { producer: track.Produce const existing = lookup(state, name); if (existing) return { producer: existing, requested: false }; - const producer = new track.Producer(name); + const producer = createProducer(state, name); if (!state.served) { // Answer through the track rather than throwing, so a peer's subscribe is refused with // the code instead of an unhandled error. @@ -398,7 +409,7 @@ export class Producer { const state = broadcast.#state; if (state.closed.peek() !== undefined) throw new Error("broadcast is closed"); if (lookup(state, name)) throw new Error("duplicate reserved track"); - const producer = new track.Producer(name); + const producer = createProducer(state, name); const lookups = new Lookups(); state.lookups.set(producer, lookups); state.requests.set(name, producer); @@ -462,7 +473,7 @@ export class Producer { throw new Error(`duplicate track: ${name}`); } - const producer = new track.Producer(name).accept(info); + const producer = createProducer(this.#state, name).accept(info); adopt(this.#state, producer, track.authoredInfo(info)); return producer; } diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts new file mode 100644 index 00000000000..7885e62b300 --- /dev/null +++ b/js/net/src/cache.test.ts @@ -0,0 +1,276 @@ +import { expect, test } from "bun:test"; +import { Manual, Milli, Nano, yieldNow } from "@moq/clock"; +import * as cache from "./cache.ts"; +import { StreamCode, TooFarBehind } from "./error.ts"; +import { hooks } from "./internal.ts"; +import { Producer as TrackProducer } from "./track.ts"; + +// Run `fn` on a manual clock, stepped in milliseconds. +async function withClock(fn: (clock: { advance: (ms: number) => Promise }) => Promise | void) { + const clock = new Manual(); + const installed = clock.install(); + try { + await fn({ advance: (ms) => clock.advance(Nano.fromMilli(Milli(ms))) }); + } finally { + installed.close(); + } +} + +// A track caching into `pool`, as a broadcast's tracks do. +function pooled(pool: cache.Pool, name = "test"): TrackProducer { + const producer = new TrackProducer(name); + hooks.poolTrack(producer, pool); + return producer.accept({ maxAge: null }); +} + +function finished(producer: TrackProducer, size: number): bigint { + const group = producer.appendGroup(); + group.writeFrame({ payload: new Uint8Array(size) }); + group.finish(); + return group.sequence; +} + +// The sequences a fresh subscriber is handed: what the track still caches. Its mirrors are +// released once its close settles. +async function cached(producer: TrackProducer): Promise { + const subscriber = producer.subscribe({ delay: { max: null } }); + const sequences: bigint[] = []; + for (let group = subscriber.tryRecvGroup(); group; group = subscriber.tryRecvGroup()) + sequences.push(group.sequence); + subscriber.close(); + await yieldNow(); + return sequences; +} + +// What one finished group of `size` payload bytes bills: its frame, the frame's own +// bookkeeping (group.ts `FRAME_BYTES`), and the group's. +const billed = (size: number) => size + 256 + cache.ENTRY_OVERHEAD; + +// While the pool is over capacity, every append accrues debt and pays it by evicting the +// track's own oldest groups, so the newest content survives. Mirrors Rust `debt_evicts_oldest_group`. +test("debt evicts a track's oldest groups", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: 2 * billed(10_000) }); + const producer = pooled(pool); + for (let i = 0; i < 4; i++) finished(producer, 10_000); + + const left = await cached(producer); + expect(left).not.toContain(0n); + expect(left).toContain(3n); + // The protected live edge plus the group just demoted, since debt is charged before it. + expect(pool.used).toBeLessThanOrEqual(2 * billed(10_000)); + }); +}); + +// Far too small for one group, the pool still keeps the newest group whole. Mirrors Rust +// `latest_group_never_evicted`. +test("the newest group survives any capacity", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: 100 }); + const producer = pooled(pool); + for (let i = 0; i < 3; i++) finished(producer, 1_000); + const left = await cached(producer); + expect(left).not.toContain(0n); + expect(left).toContain(2n); + const newest = producer.subscribe({ delay: { max: null }, ranges: [{ start: { absolute: 2n } }] }); + expect((await newest.tryRecvGroup()?.readFrame())?.payload.length).toBe(1_000); + }); +}); + +// A reader holding an evicted group's unread frames is told so, rather than hanging or +// reading a truncated clean end. Mirrors Rust `eviction_aborts_readers`. +test("eviction fails a reader's unread group with Evicted", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: billed(10_000) + cache.MIRROR_OVERHEAD }); + const producer = pooled(pool); + const subscriber = producer.subscribe({ delay: { max: null } }); + finished(producer, 10_000); // seq 0 + const held = subscriber.tryRecvGroup(); + finished(producer, 10_000); // seq 1: demotes seq 0 + finished(producer, 10_000); // seq 2: pays by evicting seq 0 + await expect(held?.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); + subscriber.close(); + }); +}); + +// Debt too small to evict an old group carries over rather than shedding the live group's +// frames, which its reader has not taken yet. Mirrors Rust `carried_debt_keeps_the_live_group`. +test("carried debt keeps the live group", async () => { + await withClock(async () => { + const unit = 100 * cache.ENTRY_OVERHEAD; + const pool = new cache.Pool({ capacity: 22 * unit }); + const producer = pooled(pool); + finished(producer, 20 * unit); + + const group = producer.appendGroup(); + const reader = group.consume(); + for (let i = 0; i < 3; i++) group.writeFrame({ payload: new Uint8Array(unit) }); + group.finish(); + for (let i = 0; i < 3; i++) expect((await reader.readFrame())?.payload.length).toBe(unit); + expect(await reader.readFrame()).toBeUndefined(); + }); +}); + +// A read lifts a group's access above the pool's mean, so the eviction walk passes over it +// and evicts a never-read group instead, even one that arrived later. Mirrors Rust +// `fetch_refresh_survives_eviction`. +test("a read group outlives a later unread one", async () => { + await withClock(async (clock) => { + // Each group is mirrored to the reader too. + const pool = new cache.Pool({ capacity: Math.floor(3.5 * (billed(3_000) + cache.MIRROR_OVERHEAD)) }); + const producer = pooled(pool); + const reader = producer.subscribe({ delay: { max: null } }); + + finished(producer, 3_000); // seq 0 + await clock.advance(1_000); + finished(producer, 3_000); // seq 1 + await clock.advance(1_000); + finished(producer, 3_000); // seq 2 + await clock.advance(500); + const first = reader.tryRecvGroup(); + expect(first?.sequence).toBe(0n); + expect((await first?.readFrame())?.payload.length).toBe(3_000); + await clock.advance(500); + + finished(producer, 3_000); // seq 3 + await clock.advance(1_000); + finished(producer, 3_000); // seq 4 + reader.close(); + + const left = await cached(producer); + expect(left).toContain(0n); + expect(left).not.toContain(1n); + }); +}); + +// Taking a group is a cache access: a reader handed one just inside the window can still read +// it almost a window later. Mirrors Rust `delivery_restarts_the_expiry_clock`. +test("delivery restarts a group's idle window", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(10_000) }); + const producer = pooled(pool); + const subscriber = producer.subscribe({ delay: { max: null } }); + finished(producer, 1); // seq 0 + finished(producer, 1); // seq 1 takes the live edge, so seq 0 can expire + + await clock.advance(9_000); + const reading = subscriber.tryRecvGroup(); + expect(reading?.sequence).toBe(0n); + // Far beyond the write, inside the delivery. The new group runs the expiry scan. + await clock.advance(9_000); + finished(producer, 1); + expect((await reading?.readFrame())?.payload.length).toBe(1); + subscriber.close(); + }); +}); + +// Every track of a broadcast caches into one pool, and the mean access across all of them +// orders eviction: a track writing past the capacity pays with its own groups, but only those +// accessed no later than the mean, so a fresher track's groups survive its own debt while a +// staler track's write evicts its oldest at double rate. +test("a shared capacity spares a fresher track and evicts a staler one's oldest", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ capacity: 6 * billed(10_000) }); + const open = (producer: TrackProducer) => { + const group = producer.appendGroup(); + group.writeFrame({ payload: new Uint8Array(10_000) }); + return group; + }; + const staler = pooled(pool, "staler"); + const old = [open(staler), open(staler), open(staler), open(staler)]; + + await clock.advance(10_000); + const fresher = pooled(pool, "fresher"); + const reader = fresher.subscribe({ delay: { max: null } }); + const read = []; + for (let i = 0; i < 4; i++) { + read.push(open(fresher)); + expect((await reader.tryRecvGroup()?.readFrame())?.payload.length).toBe(10_000); + } + expect(pool.used).toBeGreaterThan(6 * billed(10_000)); + for (const group of read) expect(group.closed.peek()).toBeUndefined(); + + open(staler); + // One write's worth of debt, doubled: two groups. + expect(old[0].closed.peek()).toMatchObject({ code: StreamCode.Evicted }); + expect(old[1].closed.peek()).toMatchObject({ code: StreamCode.Evicted }); + expect(old[2].closed.peek()).toBeUndefined(); + for (const group of read) expect(group.closed.peek()).toBeUndefined(); + reader.close(); + }); +}); + +// The pool's sweep reclaims every track's idle groups with no write behind it, the live edges +// excepted, and leaves only what those still bill. Mirrors Rust `pool_sweep_expires_without_a_write`. +test("the sweep expires idle groups across tracks without a write", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const tracks = [pooled(pool, "a"), pooled(pool, "b")]; + const stalled = tracks.map((track) => { + const group = track.appendGroup(); // left open + group.writeFrame({ payload: new Uint8Array(1) }); + finished(track, 1); // the live edge + return group; + }); + + await clock.advance(1_000); + for (const group of stalled) expect(group.closed.peek()).toBeUndefined(); + await clock.advance(1_000 + 500); + for (const group of stalled) expect(group.closed.peek()).toBeInstanceOf(TooFarBehind); + for (const track of tracks) expect(await cached(track)).toEqual([1n]); + expect(pool.used).toBe(2 * billed(1)); + }); +}); + +// Every byte a group bills, its subscribers' mirrors included, goes back once it leaves. +test("the pool is billed only for what is cached", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const producer = pooled(pool); + const subscriber = producer.subscribe({ delay: { max: null } }); + finished(producer, 100); + expect(pool.used).toBe(billed(100) + cache.MIRROR_OVERHEAD); + subscriber.close(); + await yieldNow(); + expect(pool.used).toBe(billed(100)); + + producer.finish(); + producer.close(); + await clock.advance(3_000); + expect(pool.used).toBe(0); + }); +}); + +// The group's own tail limit still sheds its front, and the pool is credited for it. +test("a group's tail limit releases what it sheds from the pool", async () => { + await withClock(() => { + const pool = new cache.Pool(); + const producer = pooled(pool); + const group = producer.appendGroup(); + const frame = new Uint8Array(1024 * 1024); + for (let i = 0; i < 40; i++) group.writeFrame({ payload: frame }); + expect(hooks.groupCacheBytes(group)).toBeLessThanOrEqual(32 * 1024 * 1024); + expect(pool.used).toBe(hooks.groupCacheBytes(group) + cache.ENTRY_OVERHEAD); + }); +}); + +// With nothing evictable, the debt sheds the live edge's front, keeping the newest frame, so +// an open group a track only appends frames to stays near the capacity. +test("the debt sheds the front of an open live edge", async () => { + await withClock(async () => { + const capacity = 1024 * 1024; + const pool = new cache.Pool({ capacity }); + const producer = pooled(pool); + const reader = producer.subscribe({ delay: { max: null } }); + const group = producer.appendGroup(); + const behind = reader.tryRecvGroup(); + const frame = new Uint8Array(64 * 1024); + for (let i = 0; i < 64; i++) group.writeFrame({ payload: frame }); + + expect(group.closed.peek()).toBeUndefined(); + // What a frame write may add before it settles, on top of the capacity. + expect(pool.used).toBeLessThanOrEqual(capacity + 512 * 1024); + // A reader that never read lost the shed front. + await expect(behind?.readFrame()).rejects.toThrow(); + }); +}); diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts new file mode 100644 index 00000000000..de7b9b80bb1 --- /dev/null +++ b/js/net/src/cache.ts @@ -0,0 +1,560 @@ +/** + * A shared byte budget for cached groups, repaid by write-time eviction. Mirrors the Rust + * `cache` module. + * + * Every cached group bills its frames and a fixed overhead through a {@link Charge} to its + * track's {@link Track} account, and every track of a broadcast caches into one {@link Pool}. + * The pool never evicts on its own: while it is over capacity, each track accrues eviction debt + * as it writes, sized to what it wrote, and pays it by evicting its own oldest groups. A group + * accessed more recently than the mean access of the evictable groups is never evicted, so fresh + * content in one track outlives stale content in another, and a track whose oldest group is + * staler than the mean accrues at double rate. + * + * The pool also owns the idle window: a group nobody reads or writes for that long is reclaimed, + * except a live track's latest. Writes scan for idle groups as they go, and the pool sweeps + * every track at half the window for the ones nothing writes to. + * + * Package-internal: not exported from the entrypoint. + * + * @module + */ +import { Instant, Milli, Nano, now, sleepUntil } from "@moq/clock"; + +/** How long a group nobody reads or writes stays cached by default, matching Rust `cache::DEFAULT_EXPIRY`. */ +export const DEFAULT_EXPIRY = Milli(30_000); + +// Bun 1.4's heap (bun:jsc `heapStats`) grows about 2.9 KB per cached single-frame group with no +// subscriber, and 3.9 KB more per subscriber, whose sink holds its own mirror of every cached +// group. Billing payload alone would let a track of tiny groups cache far more than its budget. +/** Bookkeeping billed per cached group on top of its frames. */ +export const ENTRY_OVERHEAD = 3 * 1024; +/** Bookkeeping billed per subscriber mirror a cached group holds. */ +export const MIRROR_OVERHEAD = 4 * 1024; + +// The coarse clock behind access stamps, as Rust's: the expiry window rounds up to it. +const TICK = Nano.fromMilli(Milli(100)); + +// Sub-tick boosts break ties within one tick: a write outranks a merely inserted group, and a +// read outranks both, so a same-tick access still reads as newer than the mean of weaker ones. +const WRITE_BOOST = 1; +const READ_BOOST = 2; +const BOOSTS = 4; + +// Bytes a track writes before a frame write settles its debt, so a track appending frames to +// open groups, never inserting another, still pays. +const WRITE_CHARGE_THRESHOLD = 256 * 1024; + +// Ticks between the idle scans frame writes run, so a low-bitrate track still ages out. +const EXPIRY_SCAN_TICKS = 10; + +// Live candidates one debt payment examines, and entries one write-driven idle scan does: enough +// to step over a few fresh groups, few enough that a write never walks a long queue. +const EVICT_SCAN = 4; + +// Dead entries the eviction order tolerates beyond twice the live ones before it is rebuilt. +const EVICT_SLACK = 64; + +/** The initial policy for a {@link Pool}. */ +export interface Config { + /** The byte target usage converges toward; unbounded when omitted. */ + capacity?: number; + /** How long a group nobody reads or writes stays cached; never reclaimed for idleness when omitted. */ + expiry?: Milli; +} + +/** + * A shared byte budget and idle window for the cached groups of every track charging into it. + * + * The capacity is a target usage converges toward, not a hard limit: carried debt, capped + * payments, and the always protected live edge all let usage exceed it for a while. + */ +export class Pool { + #capacity: number; + // The idle window in ticks. + readonly #expiry?: number; + #used = 0; + // The first instant the pool read, and the latest tick since it, never moving back. + #epoch?: Instant; + #tick = 0; + // The sum and count of the evictable groups' access stamps, for their mean. + #sum = 0; + #count = 0; + // The accounts holding a group that may expire, which the idle sweep visits. + #tracks = new Set>(); + #sweep?: AbortController; + + constructor(config: Config = {}) { + this.#capacity = config.capacity ?? Number.POSITIVE_INFINITY; + if (config.expiry !== undefined) this.#expiry = Math.max(1, Math.ceil(Nano.fromMilli(config.expiry) / TICK)); + } + + /** The byte target, or undefined when unbounded. */ + get capacity(): number | undefined { + return Number.isFinite(this.#capacity) ? this.#capacity : undefined; + } + + /** Bytes billed across every cached group. */ + get used(): number { + return this.#used; + } + + /** The idle window, rounded up to the pool's 100 ms tick, or undefined when idle groups stay. */ + get expiry(): Milli | undefined { + return this.#expiry === undefined ? undefined : Nano.toMilli(Nano(this.#expiry * TICK)); + } + + /** Change the byte target; undefined makes it unbounded. Writes pay a shrink down as they go. */ + resize(capacity?: number): void { + this.#capacity = capacity ?? Number.POSITIVE_INFINITY; + } + + /** Ticks since the pool first read the clock. */ + now(): number { + const at = now(); + this.#epoch ??= at; + this.#tick = Math.max(this.#tick, Math.floor((at - this.#epoch) / TICK)); + return this.#tick; + } + + /** The idle window in ticks, or undefined when idle groups stay. */ + get expiryTicks(): number | undefined { + return this.#expiry; + } + + /** The mean access stamp of the evictable groups, or undefined with none. */ + average(): number | undefined { + return this.#count === 0 ? undefined : this.#sum / this.#count; + } + + /** + * The eviction debt writing `written` bytes accrues, or undefined while the pool is within + * capacity: `written * used / capacity`, so paying it frees a little more than was written + * and the overshoot decays toward the capacity. + */ + accrue(written: number): number | undefined { + if (this.#used <= this.#capacity) return undefined; + return Math.floor((written * this.#used) / Math.max(1, this.#capacity)); + } + + /** Bill `n` more bytes. */ + add(n: number): void { + this.#used += n; + } + + /** Release `n` billed bytes. */ + sub(n: number): void { + this.#used -= n; + } + + /** The current tick as an access stamp, with `boost` breaking ties within it. */ + stamp(boost: number): number { + return this.now() * BOOSTS + boost; + } + + /** A group with access stamp `stamp` became evictable. */ + accessInsert(stamp: number): void { + this.#sum += stamp; + this.#count++; + } + + /** A group with access stamp `stamp` stopped being evictable. */ + accessRemove(stamp: number): void { + this.#sum -= stamp; + this.#count--; + } + + /** An evictable group's access stamp moved from `old` to `stamp`. */ + accessRefresh(old: number, stamp: number): void { + this.#sum += stamp - old; + } + + /** Visit `track` on each idle sweep, if the pool has an idle window, while it holds a group that may expire. */ + register(track: Track): void { + if (this.#expiry === undefined) return; + this.#tracks.add(track); + this.#arm(); + } + + /** Stop visiting `track`, which holds no group that may expire. */ + unregister(track: Track): void { + this.#tracks.delete(track); + } + + /** Expire the idle groups of every registered track. */ + sweep(): void { + for (const track of [...this.#tracks]) track.sweep(); + } + + // Sweep again in half the idle window while any track holds a group that may expire. Never a + // reason to hold a Node or Bun process open. + #arm(): void { + if (this.#sweep !== undefined || this.#expiry === undefined || this.#tracks.size === 0) return; + const sweep = new AbortController(); + this.#sweep = sweep; + const at = Instant.add(now(), Nano((this.#expiry * TICK) / 2)); + void sleepUntil(at, { signal: sweep.signal, ref: false }).then( + () => { + if (this.#sweep !== sweep) return; + this.#sweep = undefined; + this.sweep(); + this.#arm(); + }, + () => {}, + ); + } +} + +/** A cached group as its track's account orders it for eviction. */ +export interface Entry { + /** What the group bills. */ + readonly charge: Charge; +} + +// A group's place in an account's eviction order, emptied once the group leaves the cache. +interface Slot { + entry: E | undefined; +} + +/** How an account reaches the track whose groups it evicts. */ +export interface Owner { + /** Whether `entry` is the live edge, which neither eviction nor expiry takes. */ + protects(entry: E): boolean; + /** The live edge while protected, whose front pays the debt when no group can go whole. */ + edge(): E | undefined; + /** Take `entry` out of the cache and end it, for its bytes (`evicted`) or for idling (`expired`). */ + evict(entry: E, reason: "evicted" | "expired"): void; + /** Shed whole frames from the front of `entry` toward `debt`, freeing about `cap` at most, and return the bytes freed. */ + shed(entry: E, debt: number, cap: number): number; + /** Something was evicted: settle what the cache's contents decide. */ + settle(): void; +} + +/** + * One track's account against a {@link Pool}: the bytes its groups wrote, the debt they owe, and + * the order its groups are evicted in. Mirrors the Rust `cache::Track` with the eviction order + * and debt of the Rust `TrackState`. + */ +export class Track { + /** The pool this track caches into. */ + readonly pool: Pool; + #owner: Owner; + // Bytes billed since the last debt accrual. + #written = 0; + // The tick a frame write may next run an idle scan at. + #nextExpiry = 0; + // Every cached group with its slot in the eviction order, oldest first. A group leaving the + // cache empties its slot, so the order never pins its frames; a scan drops empty slots. The + // order starts at `#head`, since `Array.shift` moves every slot behind it in V8. + #live = new Map>(); + #order: Slot[] = []; + #head = 0; + // Debt carried over from earlier payments. + #debt = 0; + // Where the next bounded idle scan starts, rotating so fresh groups at the front do not + // starve the ones behind them. + #cursor = 0; + + constructor(pool: Pool, owner: Owner) { + this.pool = pool; + this.#owner = owner; + } + + /** Open the charge for a new group, billing its fixed overhead. */ + charge(): Charge { + return makeCharge(this as Track); + } + + /** Count `n` billed bytes toward the next debt accrual. */ + wrote(n: number): void { + this.#written += n; + } + + /** Whether the account caches any group. */ + get empty(): boolean { + return this.#live.size === 0; + } + + /** Cache `entry` at the back of the eviction order, evictable now unless it is the live edge. */ + insert(entry: E): void { + const slot = { entry }; + this.#live.set(entry, slot); + this.#order.push(slot); + if (!this.#owner.protects(entry)) this.demote(entry); + } + + /** Make `entry` evictable once the live edge no longer protects it. */ + demote(entry: E): void { + if (!this.#live.has(entry)) return; + entry.charge.demote(); + this.pool.register(this as Track); + } + + /** Forget `entry`, which left the cache, releasing what it billed. */ + remove(entry: E): void { + const slot = this.#live.get(entry); + if (slot === undefined) return; + this.#live.delete(entry); + slot.entry = undefined; + entry.charge.clear(); + if (this.#live.size === 0) { + this.#order = []; + this.#head = 0; + this.#debt = 0; + this.pool.unregister(this as Track); + } + } + + /** + * Accrue and pay the eviction debt of everything written since the last accrual. Run before a + * new group is cached, so the write that creates it can never evict it. Doubles when the + * track's oldest group is staler than the pool's mean. Returns whether it evicted a group. + */ + chargeDebt(): boolean { + const written = this.#written; + this.#written = 0; + const accrued = this.pool.accrue(written); + // Under capacity, a stale debt would only make a burst at the next pressure spike. + if (accrued === undefined) { + this.#debt = 0; + return false; + } + const debt = this.#oldestIsStale() ? 2 * accrued : accrued; + // What is used bounds what eviction could ever free, so a track that cannot pay + // (everything protected) does not hoard a stale schedule. + this.#debt = Math.min(this.#debt + debt, this.pool.used); + // One write never dumps a deep backlog at once: the remainder carries over. + return this.#payDebt(2 * written); + } + + // Whether the oldest evictable group was accessed no later than the pool's mean. A dead entry + // at the front reads as not stale until a scan drops it. + #oldestIsStale(): boolean { + const average = this.pool.average(); + const front = this.#order[this.#head]?.entry; + if (average === undefined || front === undefined) return false; + return !this.#owner.protects(front) && front.charge.accessed <= average; + } + + // Evict the stalest groups until the debt is paid or `cap` bytes are freed, examining a few + // candidates from the front of the order. A group accessed after the pool's mean rotates to + // the back instead, leaving the debt to tracks holding staler content. A group larger than + // the debt left stays, carrying the debt over. With no group able to go whole, the fresh + // ones and the live edge shed their fronts, so neither grows without bound. + #payDebt(cap: number): boolean { + const average = this.pool.average() ?? 0; + let paid = 0; + let scanned = 0; + let evicted = false; + const fresh: E[] = []; + for (let i = this.#order.length - this.#head; i > 0; i--) { + if (this.#debt === 0 || paid >= cap || scanned >= EVICT_SCAN) break; + // An eviction that emptied the account reset the order. + if (this.#head >= this.#order.length) break; + const slot = this.#order[this.#head++]; + const entry = slot.entry; + if (entry === undefined) continue; + if (this.#owner.protects(entry)) { + this.#order.push(slot); + continue; + } + scanned++; + if (entry.charge.accessed > average) { + this.#order.push(slot); + fresh.push(entry); + continue; + } + const size = entry.charge.size; + if (size > this.#debt) { + // Back to the front, where its slot still is. + this.#head--; + this.#compact(); + return evicted; + } + this.#debt -= size; + paid += size; + evicted = true; + this.#owner.evict(entry, "evicted"); + } + const edge = this.#owner.edge(); + if (edge) fresh.push(edge); + for (const entry of fresh) { + if (this.#debt === 0 || paid >= cap) break; + if (!this.#live.has(entry)) continue; + const freed = this.#owner.shed(entry, this.#debt, cap - paid); + this.#debt -= Math.min(freed, this.#debt); + paid += freed; + } + this.#compact(); + return evicted; + } + + /** + * Evict the groups idle past the pool's window, never the live edge: every one when `full`, + * or else a bounded scan from a rotating cursor. Returns whether it evicted a group. + */ + evictExpired(full: boolean): boolean { + const expiry = this.pool.expiryTicks; + const order = this.#order; + const head = this.#head; + const len = order.length - head; + if (expiry === undefined || len === 0) return false; + const now = this.pool.now(); + const start = full ? 0 : this.#cursor % len; + const width = full ? len : Math.min(len, EVICT_SCAN); + if (!full) this.#cursor = start + width; + const expired: E[] = []; + for (let step = 0; step < width; step++) { + const entry = order[head + ((start + step) % len)].entry; + if (entry === undefined || this.#owner.protects(entry)) continue; + if (now - Math.floor(entry.charge.accessed / BOOSTS) > expiry) expired.push(entry); + } + for (const entry of expired) this.#owner.evict(entry, "expired"); + this.#compact(); + return expired.length > 0; + } + + // Skip the empty slots at the front of the order, and rebuild it once the slots before the + // front and the empty ones behind it clearly outnumber the live ones. + #compact(): void { + const order = this.#order; + while (this.#head < order.length && order[this.#head].entry === undefined) this.#head++; + if (order.length > 2 * this.#live.size + EVICT_SLACK) { + this.#order = order.slice(this.#head).filter((slot) => slot.entry !== undefined); + this.#head = 0; + } + } + + /** + * Settle from a frame write: pay the debt once enough was written, and scan for idle groups + * once per second, so a track that only appends frames to open groups still pays and ages. + */ + settle(): void { + const debt = this.#written >= WRITE_CHARGE_THRESHOLD; + const expiry = this.#expiryDue(); + let evicted = false; + if (debt) evicted = this.chargeDebt(); + if (expiry) evicted = this.evictExpired(false) || evicted; + if (evicted) this.#owner.settle(); + } + + /** Expire every idle group, from the pool's sweep. */ + sweep(): void { + if (this.evictExpired(true)) this.#owner.settle(); + // A live edge alone never expires, so it needs no sweep to keep its track alive for. + for (const entry of this.#live.keys()) if (!this.#owner.protects(entry)) return; + this.pool.unregister(this as Track); + } + + // Claim the next write-driven idle scan once its time comes. + #expiryDue(): boolean { + const expiry = this.pool.expiryTicks; + if (expiry === undefined) return false; + const now = this.pool.now(); + if (now < this.#nextExpiry) return false; + this.#nextExpiry = now + Math.min(expiry, EXPIRY_SCAN_TICKS); + return true; + } +} + +let makeCharge: (track: Track) => Charge; + +/** + * What one cached group bills to its track's account: its bytes, overhead included, and its + * access stamp, which the pool's mean samples while the group is evictable. Cleared once the + * group leaves the cache, after which every operation is a no-op. + */ +export class Charge { + #track?: Track; + #bytes = 0; + #stamp: number; + // Whether the stamp is a sample in the pool's mean: the group is evictable. + #counted = false; + + private constructor(track: Track) { + this.#track = track; + this.#stamp = track.pool.stamp(0); + this.bill(ENTRY_OVERHEAD); + } + + static { + makeCharge = (track) => new Charge(track); + } + + /** Bill `n` more bytes of a write, which is an access too: it restarts the idle window. */ + add(n: number): void { + this.bill(n); + this.#touch(WRITE_BOOST); + } + + /** Bill `n` more bytes nobody accessed, such as a subscriber's mirror. */ + bill(n: number): void { + const track = this.#track; + if (!track) return; + track.pool.add(n); + track.wrote(n); + this.#bytes += n; + } + + /** Release `n` bytes the group dropped while still cached. */ + sub(n: number): void { + const track = this.#track; + if (!track) return; + n = Math.min(n, this.#bytes); + track.pool.sub(n); + this.#bytes -= n; + } + + /** Every byte billed, overhead included. */ + get size(): number { + return this.#bytes; + } + + /** The last access stamp: its tick times four, plus the boost of the access that set it. */ + get accessed(): number { + return this.#stamp; + } + + /** Record a read: a fetched group or a frame read. */ + refresh(): void { + this.#touch(READ_BOOST); + } + + /** Record a write that bills no new bytes. */ + recordWrite(): void { + this.#touch(WRITE_BOOST); + } + + /** Make the group evictable, sampling its stamp into the pool's mean. Idempotent. */ + demote(): void { + const track = this.#track; + if (!track || this.#counted) return; + track.pool.accessInsert(this.#stamp); + this.#counted = true; + } + + /** Settle the account's debt and idle scan after a frame write. */ + settle(): void { + this.#track?.settle(); + } + + /** Release everything billed and the mean's sample, and detach from the account. Idempotent. */ + clear(): void { + const track = this.#track; + if (!track) return; + track.pool.sub(this.#bytes); + this.#bytes = 0; + if (this.#counted) track.pool.accessRemove(this.#stamp); + this.#counted = false; + this.#track = undefined; + } + + // Advance the stamp to the current tick with `boost`, never moving it back. + #touch(boost: number): void { + const track = this.#track; + if (!track) return; + const stamp = track.pool.stamp(boost); + if (stamp <= this.#stamp) return; + if (this.#counted) track.pool.accessRefresh(this.#stamp, stamp); + this.#stamp = stamp; + } +} diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index fe1e558ed50..d4d99aa3b4e 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -949,11 +949,42 @@ test.each(["eviction", "expiry"] as const)("a group end wait observes reader %s" const expired = new Signal(false); hooks.expireGroup(reader, { expired: () => expired.peek(), changed: [expired] }); const end = hooks.groupEnd(reader); - if (failure === "eviction") hooks.evictGroup(reader); + if (failure === "eviction") hooks.evictGroup(reader, new StreamError(StreamCode.Evicted)); else expired.set(true); await expect(end).rejects.toMatchObject({ - code: failure === "eviction" ? StreamCode.TooFarBehind : StreamCode.Old, + code: failure === "eviction" ? StreamCode.Evicted : StreamCode.Old, }); source.finish(); reader.close(); }); + +// Rust's `GroupState::shed` keeps an open group's newest committed frame even while a newer +// frame is still being written, since that partial frame is not readable yet. +test("a debt shed keeps the newest committed frame behind one being written", async () => { + const producer = new Producer(0n); + producer.writeFrame({ payload: new Uint8Array(1024) }); + const reader = producer.consume(); + producer.createFrame({ size: 10 }).write(new Uint8Array(5)); + + expect(hooks.shedGroup(producer, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, [])).toBe(0); + expect((await reader.readFrame())?.payload.byteLength).toBe(1024); +}); + +// A finished group no longer tees into its copies, so shedding it ends the copies its track +// holds that had not taken the shed frames, rather than crediting bytes they still hold. +test("a debt shed of a finished group ends the copies that lost a frame", async () => { + const producer = new Producer(0n); + producer.writeFrame({ payload: new Uint8Array(1024) }); + producer.writeFrame({ payload: new Uint8Array(1024) }); + producer.finish(); + const behind = producer.consume(); + const drained = producer.consume(); + await drained.readFrame(); + await drained.readFrame(); + + expect( + hooks.shedGroup(producer, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, [behind, drained]), + ).toBeGreaterThan(0); + await expect(behind.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); + expect(await drained.readFrame()).toBeUndefined(); +}); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 4785a3df392..dd354f2b8a2 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -5,7 +5,8 @@ */ import { type Instant, now } from "@moq/clock"; import { type Dispose, type GetPromise, type Getter, Once, race, Signal } from "@moq/signals"; -import { FrameTooLarge, ProtocolViolation, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; +import type { Charge } from "./cache.ts"; +import { FrameTooLarge, ProtocolViolation, StreamCode, StreamError, TimestampMismatch } from "./error.ts"; import { Compression, encodable, stored } from "./flate/group.ts"; import { type Expiry, type GroupRecover, hooks, type ReadGroupFrame, type Replacement } from "./internal.ts"; import { Queue } from "./queue.ts"; @@ -211,6 +212,9 @@ class GroupState { offset = 0; cacheBytes = 0; + // What the source copy bills its track's cache pool, while cached. A mirror shares its + // source's, so its reads count as accesses. + charge?: Charge; compressed?: boolean; flate?: Compression; // The owning track's info, which every frame's timestamp must `rescale` to: a timed track's frames @@ -341,10 +345,6 @@ export class Producer { // past the track's end is refused rather than served as content that does not exist. #limit?: number; - // When the group was created or last written. Retention ages a group from this, - // not from its creation, so a group still being filled is never reclaimed. - #activity = now(); - // The frame being written in pieces, if any; no other frame is written until it ends. #open?: FrameProducer; @@ -378,7 +378,10 @@ export class Producer { state.flate = flate; const frames = Array.from(state.frames.peek(), (frame) => flate.push(frame)); state.frames.set(new Queue(frames)); - state.cacheBytes = frames.reduce((bytes, frame) => bytes + frameBytes(frame), 0); + const bytes = frames.reduce((bytes, frame) => bytes + frameBytes(frame), 0); + if (bytes > state.cacheBytes) state.charge?.bill(bytes - state.cacheBytes); + else state.charge?.sub(state.cacheBytes - bytes); + state.cacheBytes = bytes; for (const mirror of group.#mirrors?.keys() ?? []) { mirror.frames.set(new Queue(frames.slice(mirror.start - state.start))); mirror.flate = flate; @@ -430,6 +433,12 @@ export class Producer { const bytes = state.flate?.stored ?? state.cacheBytes - state.frames.peek().length * FRAME_BYTES; return bytes - head > head * ratio; }; + hooks.chargeGroup = (group, charge) => { + const state = group.#state; + state.charge = charge; + charge?.bill(state.cacheBytes); + }; + hooks.shedGroup = (group, debt, cap, readers) => group.#shed(debt, cap, readers); hooks.limitGroup = (group, frames) => { // A closed group takes no more frames, but one already past the cap still // contradicts the end. @@ -440,15 +449,6 @@ export class Producer { }; } - /** - * When the group was created or last written, on the installed clock. - * - * @internal Idle cache eviction only. - */ - get activity(): Instant { - return this.#activity; - } - /** * Settles once the group closes: `null` on a clean close, or the abort {@link Error}. * Peek it synchronously (`undefined` while open), observe it reactively, or `await` it. @@ -476,6 +476,7 @@ export class Producer { dst.latest = this.#state.latest; dst.flate = this.#state.flate; dst.compressed = this.#state.compressed; + dst.charge = this.#state.charge; // A track restamps its own mirror with when the group joined it. dst.arrived = this.#state.arrived; if (this.#state.start === this.#state.offset && !headShed(this.#state.frames.peek().first)) { @@ -585,20 +586,22 @@ export class Producer { const commits = this.#admit(frame); const entry = new Chunked(info); - this.#append(entry, now(), commits); + this.#append(entry, commits); const open = makeFrameProducer(info, { written: () => entry.written.peek(), write: (chunk) => { if (this.#open !== open) throw new Error("group is closed"); - this.#activity = now(); - this.#state.cacheBytes += entry.write(chunk); + const bytes = entry.write(chunk); + this.#state.cacheBytes += bytes; + this.#state.charge?.add(bytes); this.#trim(); + this.#state.charge?.settle(); }, finish: () => { if (this.#open !== open) throw new Error("group is closed"); this.#open = undefined; entry.arrived = now(); - this.#activity = entry.arrived; + this.#state.charge?.recordWrite(); entry.done.set(null); // Wake whole-frame readers waiting for it to complete. this.#state.frames.mutate(() => {}); @@ -626,7 +629,7 @@ export class Producer { if (this.#state.flate) { frame = this.#state.flate.push(frame, inflated); } - this.#append(frame, arrived, commits); + this.#append(frame, commits); } // Check that the next frame may be written, returning what records its admission. @@ -649,9 +652,10 @@ export class Producer { return undefined; } - #append(frame: Frame, arrived: Instant, commits: (() => void)[] | undefined) { - this.#activity = arrived; - this.#state.cacheBytes += frameBytes(frame); + #append(frame: Frame, commits: (() => void)[] | undefined) { + const bytes = frameBytes(frame); + this.#state.cacheBytes += bytes; + this.#state.charge?.add(bytes); appendFrame(this.#state, frame); if (commits) { this.#admits = undefined; @@ -663,13 +667,15 @@ export class Producer { else appendFrame(mirror, frame); } this.#trim(); + // Past the frame's own bookkeeping: paying the track's debt may evict other groups. + this.#state.charge?.settle(); } + // Shed the front past the live tail budget. #trim(): void { const state = this.#state; const frames = state.frames.peek(); - const start = state.start; - let shed = false; + const bytes = state.cacheBytes; while (state.cacheBytes > TAIL_BYTES) { const front = frames.first; if (!front) break; @@ -677,17 +683,58 @@ export class Producer { const freed = front instanceof Chunked ? front.shed() : 0; if (freed > 0) { state.cacheBytes -= freed; - shed = true; continue; } // The newest frame stays readable: its writer holds it whole anyway. if (frames.length === 1) break; - frames.shift(); - state.cacheBytes -= frameBytes(front); - if (front instanceof Chunked) front.release(); - state.start++; + this.#shift(); } - if (state.start === start && !shed) return; + this.#shedMirrors(bytes); + } + + // Shed whole frames from the front toward the cache pool's `debt`, freeing about `cap` at + // most, for a group the debt cannot evict whole: one being read, or the live edge. A frame + // larger than what is left of the debt stays, with everything after it. An open group keeps + // its newest committed frame, for the reader about to take it, and any frame still being + // written. A finished group no longer tees into its mirrors, so it ends `readers`, the copies + // its track still holds, that had yet to take a shed frame. Returns the bytes freed. Mirrors + // Rust's `GroupState::shed`. + #shed(debt: number, cap: number, readers: Iterable): number { + const state = this.#state; + const closed = state.closed.peek(); + if (closed instanceof Error) return 0; + const frames = state.frames.peek(); + const last = frames.last; + const writing = last instanceof Chunked && last.done.peek() === undefined; + const keep = closed === null ? 0 : writing ? 2 : 1; + const bytes = state.cacheBytes; + while (frames.length > keep && bytes - state.cacheBytes < cap) { + const front = frames.first; + if (!front || (front instanceof Chunked && front.done.peek() === undefined)) break; + if (frameBytes(front) > debt - (bytes - state.cacheBytes)) break; + this.#shift(); + } + this.#shedMirrors(bytes); + if (closed === null) for (const reader of readers) shedReader(reader, state.start); + return bytes - state.cacheBytes; + } + + // Drop the front frame. + #shift(): void { + const state = this.#state; + const front = state.frames.peek().shift() as Frame; + state.cacheBytes -= frameBytes(front); + if (front instanceof Chunked) front.release(); + state.start++; + } + + // Release what the front shed since it held `bytes`, and end the mirrors that can no longer + // read from their start. + #shedMirrors(bytes: number): void { + const state = this.#state; + const frames = state.frames.peek(); + if (state.cacheBytes === bytes) return; + state.charge?.sub(bytes - state.cacheBytes); state.flate?.shed(state.start); const lost = headShed(frames.first); for (const mirror of this.#mirrors?.keys() ?? []) { @@ -915,6 +962,8 @@ export class FrameConsumer { } let makeConsumer: (state: GroupState) => Consumer; +// End a track-held copy of a finished group that had yet to take frame `start - 1`, now shed. +let shedReader: (group: Consumer, start: number) => void; // A replacement copy asked for on behalf of one consumer, settled or still out. interface Attempt { @@ -1228,7 +1277,12 @@ export class Consumer { group.#closed = undefined; }; hooks.readGroupFrame = (group, compressed, from = 0) => group.#readFrame(from, compressed, true); - hooks.evictGroup = (group) => group.#evict(); + hooks.evictGroup = (group, lost) => group.#evict(lost); + shedReader = (group, start) => { + // A copy that took every shed frame keeps reading; one that had not lost them. + if (group.#state === group.#mirrored && group.#state.start < start) group.#evict(evicted()); + }; + hooks.refreshGroup = (group) => group.#state.charge?.refresh(); hooks.onShedRead = (group, notify) => { group.#state.shedRead = notify; }; @@ -1288,17 +1342,17 @@ export class Consumer { return true; } - #evict(): boolean { + #evict(lost: Error): boolean { // A group read on from another copy is that copy's to reclaim. if (this.#state !== this.#mirrored) return false; const frames = this.#bufferedFrames(); if (frames.length > 0) { // Unread content is going away either way. If the budget had already given up // on this group that is the more specific reason, so ask it first; otherwise - // the reader simply lagged behind the cache. + // the cache reclaimed it before the reader got to it. this.#expire(); if (!this.#terminal) { - this.#terminal = new TooFarBehind(); + this.#terminal = lost; if (this.#verdict.peek() === undefined) this.#verdict.set(this.#terminal); } } @@ -1444,6 +1498,7 @@ export class Consumer { const sequence = this.#state.start++; this.#streamed = front; this.#closed?.refresh(); + this.#state.charge?.refresh(); return { sequence, frame: front, chunks: makeFrameConsumer(sequence, front) }; } if (front.done.peek() === undefined) return undefined; @@ -1467,6 +1522,7 @@ export class Consumer { if (frames.length === 0 && this.#state.closed.peek() !== undefined) this.#state.flate = undefined; const sequence = this.#state.start++; this.#closed?.refresh(); + this.#state.charge?.refresh(); return { sequence, frame, inflated: slice?.inflated }; } diff --git a/js/net/src/ietf/fetch.test.ts b/js/net/src/ietf/fetch.test.ts index a34db56c948..bf0cbc5b5fd 100644 --- a/js/net/src/ietf/fetch.test.ts +++ b/js/net/src/ietf/fetch.test.ts @@ -2,10 +2,10 @@ import { expect, test } from "bun:test"; import { Manual, Milli, Nano } from "@moq/clock"; import { Signal } from "@moq/signals"; import type { Grant } from "../auth.ts"; +import { DEFAULT_EXPIRY } from "../cache.ts"; import { accept as acceptSession, connect as connectSession } from "../connection/index.ts"; import { error } from "../error.ts"; import { hooks } from "../internal.ts"; -import { CACHE_EXPIRY } from "../ledger.ts"; import * as Lite from "../lite/index.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; @@ -436,7 +436,7 @@ test("a FETCH fills never-cached and evicted history, then reuses the cache", as using fx = serve(version, undefined, undefined, null); if (evicted) { writeGroup(fx.track, 1n, ["1-0", "1-1", "1-2"]); - await clock.advance(ms(CACHE_EXPIRY * 2)); + await clock.advance(ms(DEFAULT_EXPIRY * 2)); } writeGroup(fx.track, 3n, ["live"]); const cache = fx.track.subscribe({ ranges: [] }); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 8495da60ff0..6c2de73d94d 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -8,6 +8,7 @@ import { type Instant, Milli } from "@moq/clock"; import type { Dispose, GetPromise, Getter } from "@moq/signals"; import type { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; +import type { Charge, Pool } from "./cache.ts"; import type * as datagram from "./datagram.ts"; import type { Delivered } from "./delivered.ts"; import { StreamCode, StreamError, toStreamCode } from "./error.ts"; @@ -526,6 +527,8 @@ export const hooks: { * as a dead route. Assigned by `track.ts`. */ releaseTrack: (producer: Producer) => void; + /** Cache a new track's groups into its broadcast's `pool`, before it caches any. Assigned by `track.ts`. */ + poolTrack: (producer: Producer, pool: Pool) => void; /** * Settles once the producer is released (closed or aborted), unlike its public `closed`, * which also settles on a reached end while the producer keeps serving. Assigned by `track.ts`. @@ -769,6 +772,13 @@ export const hooks: { writeGroupFrame: (group: GroupProducer, frame: Frame, inflated?: number) => void; /** Payload footprint including a shared inflated copy, for measurements. */ groupCacheBytes: (group: GroupProducer) => number; + /** Bill a cached group's bytes to `charge` from now on, or stop billing them with undefined. */ + chargeGroup: (group: GroupProducer, charge: Charge | undefined) => void; + /** + * Shed whole frames from a group's front toward a cache `debt`, freeing about `cap` at most, + * and end the `readers` its track holds that lost a shed frame; returns the bytes freed. + */ + shedGroup: (group: GroupProducer, debt: number, cap: number, readers: Iterable) => number; /** Frame count without creating a read handle. */ groupFrameCount: (group: GroupProducer) => number; /** @@ -824,10 +834,13 @@ export const hooks: { /** Notify the owning track when a reader reaches its shed front. */ onShedRead: (group: GroupConsumer, notify: () => void) => void; /** - * Make an evicted mirror terminal while its track timeline still contains it. Whether the - * mirror is released too: not when its group reads on, or may yet, from another copy. + * Make an evicted mirror terminal while its track timeline still contains it, failing its + * unread content with `lost`. Whether the mirror is released too: not when its group reads + * on, or may yet, from another copy. */ - evictGroup: (group: GroupConsumer) => boolean; + evictGroup: (group: GroupConsumer, lost: Error) => boolean; + /** Record a cache access on a group a reader takes, restarting its idle window. */ + refreshGroup: (group: GroupConsumer) => void; /** Call `fn` synchronously when an open broadcast closes, from any handle. */ onClosed: (producer: BroadcastProducer, fn: () => void) => Dispose; /** Reserve an on-demand track without listing it or enabling a request handler. */ @@ -881,6 +894,9 @@ export const hooks: { releaseTrack: () => { throw new Error("track.ts not loaded"); }, + poolTrack: () => { + throw new Error("track.ts not loaded"); + }, trackReleased: () => { throw new Error("track.ts not loaded"); }, @@ -1085,6 +1101,12 @@ export const hooks: { groupCacheBytes: () => { throw new Error("group.ts not loaded"); }, + chargeGroup: () => { + throw new Error("group.ts not loaded"); + }, + shedGroup: () => { + throw new Error("group.ts not loaded"); + }, groupFrameCount: () => { throw new Error("group.ts not loaded"); }, @@ -1154,6 +1176,9 @@ export const hooks: { evictGroup: () => { throw new Error("group.ts not loaded"); }, + refreshGroup: () => { + throw new Error("group.ts not loaded"); + }, onClosed: () => { throw new Error("broadcast.ts not loaded"); }, diff --git a/js/net/src/ledger.ts b/js/net/src/ledger.ts index ddf7617ecc9..0044625ba3e 100644 --- a/js/net/src/ledger.ts +++ b/js/net/src/ledger.ts @@ -18,12 +18,6 @@ const compare = (a: bigint, b: bigint): number => (a < b ? -1 : a > b ? 1 : 0); */ export const GRACE = Milli(1000); -/** - * How long idle cached content is kept, matching Rust `cache::DEFAULT_EXPIRY`: a group older - * than this is evicted, so nothing waits for one longer. - */ -export const CACHE_EXPIRY = Milli(30_000); - /** The grace for a subscription with `delay.max`: how long it was willing to wait for a late group anyway, or {@link GRACE} without one. */ export function grace(maxDelay: Milli): Milli { return maxDelay > 0 ? maxDelay : GRACE; diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index 135ea477131..6d1978d8c3c 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -3,10 +3,10 @@ // rule, and inside the copy, once it evicted what it received. import { expect, test } from "bun:test"; import { Elapsed, Manual, Milli, Nano, sleep, timeout } from "@moq/clock"; +import { DEFAULT_EXPIRY } from "../cache.ts"; import { accept as acceptSession, connect as connectSession } from "../connection/index.ts"; import type { Consumer as GroupConsumer } from "../group.ts"; import { hooks } from "../internal.ts"; -import { CACHE_EXPIRY } from "../ledger.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; @@ -204,7 +204,7 @@ test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( // Both caches age out what nobody touched, a prune slice past the window; only the // live edge stays. - await clock.advance(Nano.fromMilli(Milli(CACHE_EXPIRY + 10_000))); + await clock.advance(Nano.fromMilli(Milli(DEFAULT_EXPIRY + 10_000))); write(peer.track.createGroup(4n), 4n, "live"); expect((await first.recvGroup())?.sequence).toBe(4n); const before = peer.fetches(); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index ffe91c753f1..993d7db4921 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4,6 +4,7 @@ import { type Getter, Signal } from "@moq/signals"; import type { Grant } from "../auth.ts"; import type { AuthSession } from "../auth_session.ts"; import { Viewers } from "../bandwidth.ts"; +import { DEFAULT_EXPIRY } from "../cache.ts"; import type { Probe as ProbeStats } from "../connection/stats.ts"; import { error, @@ -21,7 +22,7 @@ import type { Consumer as GroupConsumer } from "../group.ts"; import { HopSchema, isAnonymous, MAX_HOPS, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; import { hooks } from "../internal.ts"; -import { CACHE_EXPIRY, GRACE } from "../ledger.ts"; +import { GRACE } from "../ledger.ts"; import * as Path from "../path.ts"; import { Reader, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; @@ -2827,7 +2828,7 @@ test("a finished group wins over a late stale drop once evicted on lite-07", asy new Reader(new ReadableStream({ start: (controller) => controller.close() }), undefined, version), ); // Everything below the live edge ages out of the cache. - const evict = () => clock.advance(Nano.fromMilli(Milli(2 * CACHE_EXPIRY))); + const evict = () => clock.advance(Nano.fromMilli(Milli(2 * DEFAULT_EXPIRY))); let priority = 0; const update = async () => { track.update({ priority: ++priority, delay: { max: Milli(500) } }); diff --git a/js/net/src/parked-expiry.test.ts b/js/net/src/parked-expiry.test.ts index 624901a6005..62b7b6547dd 100644 --- a/js/net/src/parked-expiry.test.ts +++ b/js/net/src/parked-expiry.test.ts @@ -113,7 +113,7 @@ for (const protocol of protocols) { await clock.advance(Nano.fromMilli(Milli(budget - 1))); expect(ended).toBe(false); await clock.advance(Nano.fromMilli(Milli(1))); - } else await clock.advance(Nano.fromMilli(Milli(33_750))); + } else await clock.advance(Nano.fromMilli(Milli(45_000))); expect(ended).toBe(true); if (protocol.kind === "lite") { for (;;) { @@ -221,7 +221,7 @@ for (const protocol of protocols) { try { await blocked.promise; write(1n, 1); - await clock.advance(Nano.fromMilli(Milli(33_750))); + await clock.advance(Nano.fromMilli(Milli(45_000))); // The write already handed to the stream finishes, and the stream is reset after it. credit.resolve(); await clock.advance(Nano.zero); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 7944ed61e46..f7d33a77a94 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2,6 +2,7 @@ import { expect, spyOn, test } from "bun:test"; import { Manual, Milli, Nano, sleep, Wall, yieldNow } from "@moq/clock"; import { hostTime } from "@moq/clock/test"; import { race } from "@moq/signals"; +import { DEFAULT_EXPIRY } from "./cache.ts"; import { ProtocolViolation, Skipped, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; import type { Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { Version as IetfVersion } from "./ietf/version.ts"; @@ -14,8 +15,9 @@ import { unwanted } from "./util/demand.ts"; const enc = new TextEncoder(); const dec = new TextDecoder(); -// Matches the private idle cache window and Rust cache::DEFAULT_EXPIRY. -const CACHE_WINDOW_MS = 30_000; +const CACHE_WINDOW_MS = DEFAULT_EXPIRY; +// How late the pool's sweep may reclaim a group nothing writes to: it runs every half window. +const SWEEP_LAG_MS = CACHE_WINDOW_MS / 2; test("a cached transport tail covers only positions it actually holds", async () => { const producer = new TrackProducer("test"); @@ -1899,7 +1901,7 @@ test("an abandoned open group ages out with no further write", async () => { if (!group) throw new Error("missing group"); expect(await group.readString()).toBe("first"); const read = group.readFrame().catch((err: unknown) => err); - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); expect(await read).toBeInstanceOf(TooFarBehind); } finally { producer.finish(); @@ -1917,10 +1919,10 @@ test("publisher maxAge does not change the idle cache deadline", async () => { idle.writeFrame(textFrame("cached")); producer.appendGroup(); // Eviction ends the idle open group a cache window after its last write, give or take - // the one slice a scan can lag. + // the lag of the pool's sweep. await clock.advance(10_000 + CACHE_WINDOW_MS - 1); expect(idle.closed.peek()).toBeUndefined(); - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); expect(idle.closed.peek()).toBeInstanceOf(TooFarBehind); } finally { producer.finish(); @@ -2921,8 +2923,15 @@ test("no publisher limit still ages idle groups out", async () => { expect(await (await within.recvGroup())?.readString()).toBe("old"); within.close(); - // A scan runs at most once per slice of the window, so step past the next one. - clock.set(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + // The read was an access, as in Rust: past the write's window, the group stays. Taking it + // again is another access. + clock.set(10_000 + CACHE_WINDOW_MS + 1_000); + const read = producer.subscribe({ delay: { max: Milli(100_000) } }); + expect(read.tryRecvGroup()?.sequence).toBe(0n); + read.close(); + + // Past that delivery's window, a subscribe's own scan reclaims it. + clock.set(10_000 + 2 * CACHE_WINDOW_MS + 2_000); const after = producer.subscribe({ delay: { max: Milli(100_000) } }); expect(after.tryRecvGroup()?.sequence).toBe(1n); expect(after.tryRecvGroup()).toBeUndefined(); @@ -3095,7 +3104,7 @@ test("closed settles once the cache evicts the last open group below the end", a await settle(); expect(producer.closed.peek()).toBeUndefined(); - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); await settle(); expect(open.closed.peek()).toBeInstanceOf(TooFarBehind); expect(producer.closed.peek()).toBeNull(); @@ -3160,8 +3169,8 @@ test("an ended track's buffered groups age out for a stale subscriber", async () producer.close(); } - // Nothing writes after close; the scheduled cleanup can lag by one scan slice. - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + // Nothing writes after close, so the pool's sweep reclaims it. + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); if (abort) expect(() => stale.tryRecvGroup()).toThrow(abort); else expect(stale.tryRecvGroup()).toBeUndefined(); } finally { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 1d6f689f9d4..9fe8c1eff9c 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -6,6 +6,7 @@ import { Instant, Milli, Nano, now, sleepUntil } from "@moq/clock"; import { Derived, type Dispose, type GetPromise, type Getter, Once, Signal } from "@moq/signals"; +import * as cache from "./cache.ts"; import { Delivered } from "./delivered.ts"; import { GroupTooLarge, @@ -43,7 +44,7 @@ import { staleHole, type TrackRequestOptions, } from "./internal.ts"; -import { CACHE_EXPIRY, grace, Ledger } from "./ledger.ts"; +import { grace, Ledger } from "./ledger.ts"; import { rescale } from "./rescale.ts"; import { Timescale, type Timestamp } from "./time.ts"; import { checkSequence, max, min, UNBOUNDED } from "./util/sequence.ts"; @@ -77,12 +78,6 @@ function taken(read: () => Recv, ended: () => undefined): GroupConsumer | undefi } } -// Media delay.max only bounds how far behind the live edge a reader may fall. -const CACHE_WINDOW = Nano.fromMilli(CACHE_EXPIRY); - -// The cache scans at most this many times per idle cache window. -const PRUNE_SLICES = 8; - // The unset priority, the midpoint of the signed range: moq-lite carries it as 128 and // moq-transport as 127, one urgency on both. const DEFAULT_PRIORITY = 0; @@ -793,12 +788,13 @@ function after(a: Position, b: Position): boolean { } // A source group retained in the producer cache, with the mirror handed to each sink -// so eviction can drop them together. +// so eviction can drop them together, and what it all bills the cache pool. // `arrived` is when the group joined this track, which its mirrors age a predecessor from. // `bound` is its first frame's timestamp once checked to fit between its neighbors', which // bounds theirs; `unverified` marks one accepted below the eviction watermark instead. type CachedGroup = { group: GroupProducer; + charge: cache.Charge; mirrors: Map; arrived: Instant; bound?: Timestamp; @@ -994,11 +990,12 @@ export class Producer { // must not see them. The wire's aggregate, `#state.update`, keeps them. #live = new Signal(undefined); - // Recently written source groups, retained for replay to late subscribers and - // pruned once idle for longer than the cache window. Each entry tracks the mirror - // it handed to every sink so eviction can drop them too: otherwise a slow consumer - // that never reads would pin old groups (and their frame bytes) forever. - #cache: CachedGroup[] = []; + // Recently written source groups, in arrival order, retained for replay to late subscribers + // until the cache pool evicts them, for its budget or once idle. A set, so evicting a run of + // them is linear. Each entry tracks the mirror it handed to every sink so eviction can drop + // them too: otherwise a slow consumer that never reads would pin old groups (and their frame + // bytes) forever. + #cache = new Set(); // The same entries by sequence, so a write finds a duplicate without a scan. #cached = new Map(); // The cached groups' checked starts, sorted by sequence, so a first frame finds its @@ -1008,16 +1005,9 @@ export class Producer { // group above it is checked against it as well as its cached neighbors; one at or below // it may have lost its lower bound. One entry, so nothing grows with the cache. #evicted?: Start; - // When the cache was last scanned. See #prune. - #pruned = Number.NEGATIVE_INFINITY; - // When the track closed, which restarts the former newest group's idle window. See #prune. - #closedAt = Number.NEGATIVE_INFINITY; - - // Wakeup for the next entry due to age out. Writes settle retention inline, but a - // publisher that stalls stops writing, so without this an abandoned group (and any - // reader parked in it) would wait for a write that never comes. - #pruneTimer?: AbortController; - #pruneTimerAt = 0; + // This track's account against its broadcast's cache pool (a pool of its own when it has no + // broadcast), which bills the cached groups and evicts them under its budget and idle window. + #account: cache.Track; // One independent downstream state per live subscriber. #sinks = new Set(); @@ -1078,9 +1068,31 @@ export class Producer { closed: () => this.#state.closed.peek() !== undefined, released: this.#state.closed, }); + this.#account = this.#openAccount(new cache.Pool({ expiry: cache.DEFAULT_EXPIRY })); + } + + // Open this track's account against `pool`, evicting through this producer's cache. + #openAccount(pool: cache.Pool): cache.Track { + return new cache.Track(pool, { + protects: (entry) => this.#protects(entry), + edge: () => { + const entry = this.#edgeEntry(); + return entry && this.#protects(entry) ? entry : undefined; + }, + evict: (entry, reason) => { + this.#evict(entry, reason); + this.#uncache(entry, true); + }, + shed: (entry, debt, cap) => hooks.shedGroup(entry.group, debt, cap, entry.mirrors.values()), + settle: () => this.#settle(), + }); } static { + hooks.poolTrack = (producer, pool) => { + if (!producer.#account.empty) throw new Error("track already caches groups"); + producer.#account = producer.#openAccount(pool); + }; hooks.subscribing = (producer) => producer.#subscribing; hooks.requested = (producer) => producer.#state.update; resubscribe = (producer) => producer.#updateSubscription(); @@ -1178,6 +1190,8 @@ export class Producer { return undefined; } hooks.arriveGroup(mirror, entry.arrived); + // A cache hit is an access: the group outranks those nobody read. + entry.charge.refresh(); return mirror; }; reofferCached = (producer, sink, start, end, offer) => { @@ -1461,6 +1475,7 @@ export class Producer { // A group its recovery moved to another copy no longer reads this one. if (hooks.readsMirrored(mirror)) hooks.releaseGroupReader(mirror, abort); entry.mirrors.delete(sink); + entry.charge.sub(cache.MIRROR_OVERHEAD); } } for (const group of sink.groups.peek()) hooks.releaseGroupReader(group, abort); @@ -1468,7 +1483,8 @@ export class Producer { }); } - this.#prune(); + // Full, unlike a write's bounded scan, so the subscriber gets no group already idled out. + this.#evictExpired(true); for (const entry of this.#cache) this.#mirror(entry, sink); sink.drops.set(this.#state.drops.peek()); sink.judged = this.#state.judged; @@ -1506,6 +1522,7 @@ export class Producer { const dst = entry.group.mirror(); hooks.arriveGroup(dst, entry.arrived); if (entry.unverified) hooks.unverifyGroup(dst); + if (!entry.mirrors.has(sink)) entry.charge.bill(cache.MIRROR_OVERHEAD); entry.mirrors.set(sink, dst); if (sink.reads?.(dst.sequence) === false) hooks.groupReads(dst, false); timelineInsert(sink.timeline, dst); @@ -1515,18 +1532,20 @@ export class Producer { sink.groups.mutate((groups) => bufferInsert(groups, dst)); } - // Drop a cached group's mirror from every sink so no consumer can pin it. - #evict(entry: CachedGroup): void { + // Drop a cached group's mirror from every sink so no consumer can pin it, the cache pool + // having reclaimed it for its bytes (`evicted`) or for idling (`expired`). + #evict(entry: CachedGroup, reason: "evicted" | "expired" = "expired"): void { // Reclaiming a group the publisher never closed ends it as the gap it is, before // the mirrors below would otherwise report a clean finish to a reader that had // drained it. The usual case, an already-closed group aging out, keeps its own // terminal state. - if (!entry.group.isClosed) { - const evicted = new TooFarBehind(); - EVICTED.add(evicted); - entry.group.abort(evicted); - } - const mirrors = [...entry.mirrors.values()].filter((mirror) => hooks.evictGroup(mirror)); + const lost = + reason === "evicted" + ? new StreamError(StreamCode.Evicted, { message: "group evicted from the cache" }) + : new TooFarBehind(); + EVICTED.add(lost); + if (!entry.group.isClosed) entry.group.abort(lost); + const mirrors = [...entry.mirrors.values()].filter((mirror) => hooks.evictGroup(mirror, lost)); this.#unlink(entry); for (const mirror of mirrors) mirror.close(); } @@ -1542,14 +1561,17 @@ export class Producer { const index = timelineIndex(sink.timeline, mirror.sequence); if (sink.timeline[index] === mirror) sink.timeline.splice(index, 1); } + entry.charge.sub(entry.mirrors.size * cache.MIRROR_OVERHEAD); entry.mirrors.clear(); } - // Take a cached group out of the cache lookups. + // Take a cached group out of the cache lookups, releasing what it billed the pool. #uncache(entry: CachedGroup, evicted = false): void { - this.#cache.splice(this.#cache.indexOf(entry), 1); + this.#cache.delete(entry); this.#cached.delete(entry.group.sequence); this.#unbound(entry, evicted); + this.#account.remove(entry); + hooks.chargeGroup(entry.group, undefined); } // The starts a first frame of group `sequence` must fall between: the nearest checked @@ -1600,14 +1622,21 @@ export class Producer { } } + // The cached group at the newest sequence written, if still cached. + #edgeEntry(): CachedGroup | undefined { + const latest = this.#state.latest; + return latest === undefined ? undefined : this.#cached.get(latest); + } + // The newest group remains replayable while the track is live, even after the group // closes: a catalog can publish one snapshot for the broadcast's whole lifetime. An // aborted one holds nothing worth replaying, so it ages out like any other. - #edge(): GroupProducer | undefined { - const latest = this.#state.latest; - if (latest === undefined) return undefined; - const group = this.#cached.get(latest)?.group; - return group?.closed.peek() instanceof Error ? undefined : group; + #protects(entry: CachedGroup): boolean { + return ( + this.#state.closed.peek() === undefined && + entry.group.sequence === this.#state.latest && + !(entry.group.closed.peek() instanceof Error) + ); } // The newest frame in the cache, as {@link Subscriber.largest} reads it from a sink: @@ -1647,86 +1676,33 @@ export class Producer { return this.#state.largest.peek(); } - // Evict cached groups idle for longer than the cache window. Idle means nothing - // written, so an abandoned open group ages out instead of pinning its buffer (and - // any reader parked in it) forever. - // - // Scans at most once per slice of the window, so a track publishing faster than that - // evicts a run of groups per scan instead of scanning everything to evict one per - // write. A group can outlive the window by up to one slice. - #prune(): void { - const at = now(); - const slice = CACHE_WINDOW / PRUNE_SLICES; - if (at < this.#pruned + slice) { - // Something may have come due since the last scan, so make sure another follows. - this.#wake(this.#pruned + slice); - return; - } - this.#pruned = at; - - const cutoff = at - CACHE_WINDOW; - const edge = this.#edge(); - const live = this.#state.closed.peek() === undefined; - - let oldest: number | undefined; - const retained: CachedGroup[] = []; - for (const entry of this.#cache) { - // Once the track closes, the former edge ages from the close instead, so close() - // never reclaims a group its subscribers have yet to drain. - const activity = - entry.group === edge ? Math.max(entry.group.activity, this.#closedAt) : entry.group.activity; - if (entry.group === edge && live) { - retained.push(entry); - } else if (activity >= cutoff) { - retained.push(entry); - if (oldest === undefined || activity < oldest) oldest = activity; - } else { - this.#cached.delete(entry.group.sequence); - this.#unbound(entry, true); - this.#evict(entry); - } - } - const evicted = retained.length < this.#cache.length; - this.#cache = retained; - // An evicted group below the end no longer holds it open, as in Rust. - if (evicted) this.#settle(); - - // Replace the wakeup with one for the next entry due to age out. Writes settle - // retention inline, so this only has to cover the case no write follows. Cheap to - // over-arm: an entry written since is retained and re-armed. - this.#pruneTimer?.abort(); - this.#pruneTimer = undefined; - if (oldest !== undefined) this.#wake(Math.max(oldest + CACHE_WINDOW, at + slice)); + // Accrue and pay the cache pool's eviction debt for what this track wrote. + #chargeDebt(): void { + if (this.#account.chargeDebt()) this.#settle(); } - // Arm the prune wakeup for `at`, unless one is already armed sooner. Kept after a close, - // until the cache empties, so what the track left behind still ages out. - #wake(at: number): void { - if (this.#pruneTimer !== undefined && this.#pruneTimerAt <= at) return; - this.#pruneTimer?.abort(); - - const timer = new AbortController(); - // A cache prune is never a reason to hold a Node/Bun process open. - void sleepUntil(Instant(at), { signal: timer.signal, ref: false }).then( - () => { - // The wakeup can already be queued when a sooner one replaces it. - if (timer.signal.aborted) return; - this.#pruneTimer = undefined; - this.#prune(); - }, - () => {}, - ); - this.#pruneTimer = timer; - this.#pruneTimerAt = at; + // Evict the groups idle past the pool's window: every one when `full`, else a bounded scan. + #evictExpired(full: boolean): void { + if (this.#account.evictExpired(full)) this.#settle(); } // Retain a source group and fan it out to every live sink; its frame 0 is checked // against this track when it lands. #publish(group: GroupProducer): void { + // Before the group is cached, so the write creating it can never evict it. + this.#chargeDebt(); this.#settle(); - const entry: CachedGroup = { group, mirrors: new Map(), arrived: now() }; + const entry: CachedGroup = { + group, + charge: this.#account.charge(), + mirrors: new Map(), + arrived: now(), + }; + hooks.chargeGroup(group, entry.charge); this.#undrop(group.sequence); group.closed.then((closed) => { + // An aborted live edge is no longer protected, so it may expire like any other. + if (closed instanceof Error) this.#account.demote(entry); if (!(closed instanceof Error) || HOLES.has(toStreamCode(closed))) return; // A re-created group already replaced this one. const current = this.#cached.get(group.sequence); @@ -1742,13 +1718,17 @@ export class Producer { return () => this.#record(entry, start, late); }); } - this.#cache.push(entry); + this.#cache.add(entry); this.#cached.set(group.sequence, entry); this.#fills.arrived(group.sequence); this.#advance(group.sequence, true); this.#arrive(group.sequence); this.#produce(group.sequence); + // A newer group demotes the live edge to an evictable one. + const edge = this.#edgeEntry(); this.#state.latest = max(this.#state.latest ?? 0n, group.sequence); + if (edge && edge !== entry && !this.#protects(edge)) this.#account.demote(edge); + this.#account.insert(entry); for (const sink of this.#sinks) this.#mirror(entry, sink); // Follow the group's frames as they land, past the head `#advance` already named. const stop = hooks.groupProgress(group, (total) => { @@ -1765,7 +1745,7 @@ export class Producer { }); // Give held mirrors the new live edge before pruning their timeline entry, // so their latency guard can preserve a terminal expiry verdict. - this.#prune(); + this.#evictExpired(false); } // Record a written sequence. It raises the largest too, at the head of an empty group, so @@ -2176,7 +2156,7 @@ export class Producer { if (abort && (this.#state.outcome.peek() === null || this.#settled())) abort = undefined; // Nobody will finish these, so a subscriber that has not taken one yet never sees it. // Not evicted: a reader already holding one keeps its frames and sees the abort. - const open = this.#cache.filter((entry) => entry.group.closed.peek() === undefined); + const open = [...this.#cache].filter((entry) => entry.group.closed.peek() === undefined); closeTrackState(this.#state, abort); if (this.#state.outcome.peek() === undefined) this.#state.outcome.set(abort ?? this.#state.endError ?? null); // Closing a copy never supplies the missing tail of a group. Already finished groups @@ -2194,8 +2174,14 @@ export class Producer { closeTrackState(sink, abort); } this.#sinks.clear(); - this.#closedAt = now(); - this.#prune(); + // The former live edge becomes evictable too, its idle window restarted, so the close + // never reclaims a group its subscribers have yet to drain. + const edge = this.#edgeEntry(); + if (edge) { + edge.charge.recordWrite(); + this.#account.demote(edge); + } + this.#evictExpired(true); } // Whether the declared end was reached, the pending tail owes nothing below it, and every @@ -2207,7 +2193,8 @@ export class Producer { if (end === undefined) return false; const final = finalSequence(end); if (!reaches(this.#received, this.#state.drops.peek(), final) || this.#owes(final)) return false; - return this.#cache.every(({ group }) => group.sequence >= final || group.closed.peek() === null); + for (const { group } of this.#cache) if (group.sequence < final && group.closed.peek() !== null) return false; + return true; } /** Append a frame as its own single-frame group. */ @@ -2535,8 +2522,10 @@ export class Subscriber { } #guard(group: GroupConsumer): GroupConsumer { - // Handed out, it is read, whatever this reader judged it while buffered. + // Handed out, it is read, whatever this reader judged it while buffered. Taking it is the + // cache access buffering it is not, as in Rust: it restarts the group's idle window. hooks.groupReads(group, true); + hooks.refreshGroup(group); const held: Superseded = {}; hooks.onShedRead(group, () => { if (this.#state.latest === group.sequence && this.#state.producer) hooks.joinProducer(this.#state.producer); diff --git a/quest/m1/README.md b/quest/m1/README.md index d3032313f7a..32e19d8deb4 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -31,7 +31,6 @@ moves what remains of them once their claims merge. ## Required -- [JS cache pool](/quest/m1/js-cache-pool.md) - `js/net` bills cached frames and drop-set runs to a per-broadcast byte pool ported from Rust - [Drop set](/quest/m1/drop-set.md) - the drop set's track and cursor runs are billed to the cache pool, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` - [Media fuzz findings](/quest/m1/fuzz-findings/README.md) - the nightly media fuzz targets run clean: mp4-atom, MKV block, TS step, SPS overflow, and catalog encoder bugs they found are fixed - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 3e4e3e60843..e32605f983b 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -36,9 +36,9 @@ setting. Reason: the 2026-11-02 audit found the existing 32 MiB limit in `js/net/src/group.ts` bounds only one group's frame tail, and a mock-time probe retained 64, 256, and 1,024 disjoint OLD runs with zero cached groups; a shared pool bounds drop-only tracks the same way in both languages. Not -chosen: a per-track run cap; upstream backpressure. The pool port itself is -[JS cache pool](/quest/m1/js-cache-pool.md), split out on 2026-11-03 since it -lands on its own; this quest bills the runs to it. +chosen: a per-track run cap; upstream backpressure. The pool port itself +landed on its own as `js/net/src/cache.ts` (split out on 2026-11-03), billing +cached groups; this quest bills the runs to it. Decided 2026-11-03 (maintainer, /quest-unblock), renamed and narrowed by the `/quest-complete` decisions below: range terms carry an @@ -280,7 +280,7 @@ starts. sequence order. - **Billed to the pool, never guessed.** Each track run and each cursor run is billed through a `Charge` at a fixed per-run cost (in JS, through the - [JS cache pool](/quest/m1/js-cache-pool.md)). Under pressure the pool + pool in `js/net/src/cache.ts`). Under pressure the pool evicts the oldest track runs as it evicts groups, and a run older than the cache's retention window is forgotten. An evicted track run's sequences are unknown again: a reader judges them by `drifted` or fills them by range. @@ -351,10 +351,6 @@ Tests, on mock time through a relay, in Rust and mirrored in `js/net`: Public API: none. Wire: none. -## Required - -- [JS cache pool](/quest/m1/js-cache-pool.md) - the byte account the JS runs bill - ## Related - [Subscribe ranges](/quest/m2/subscribe-ranges/README.md) - fills the cache miss an evicted run leaves diff --git a/quest/m1/js-cache-pool.md b/quest/m1/js-cache-pool.md deleted file mode 100644 index b9bab0d69c1..00000000000 --- a/quest/m1/js-cache-pool.md +++ /dev/null @@ -1,52 +0,0 @@ -# [L] JS cache pool - -## Goal - -`js/net` bounds what a broadcast caches in bytes, as Rust does: a port of -Rust's per-broadcast `cache::Pool` (`rs/moq-net/src/model/cache.rs`) bills -every cached group's frames and fixed bookkeeping, and the drop set's track -and cursor runs at a fixed cost per run, and evicts under it the way Rust -does. Rust and JS mirror each other under Rust's default budget, with no new -setting. - -## Plan - -Decided 2026-11-03 (maintainer, /quest-unblock, recorded in -[Drop set](/quest/m1/drop-set.md)): port Rust's per-broadcast `cache::Pool` -to JS. Shared byte accounting per broadcast bills frames and drop-set runs (a -fixed cost per run), under Rust's default budget, with eviction and the -fold-into-floor or `TooFarBehind` behavior the drop set plans. Not chosen: a -per-track run cap; upstream backpressure. Split out of the drop set on -2026-11-03, since the pool lands on its own: its first consumer is the group -cache, and the drop set's run billing follows on top of it. - -What JS has today: `js/net/src/group.ts` caps each group's frame tail at -`TAIL_BYTES` (32 MiB) and evicts that group's oldest frames past it, while -the track cache (`#prune` in `js/net/src/track.ts`) is purely time-based on -`CACHE_EXPIRY` (30 s). Nothing bills bytes across groups or tracks, and -`TrackState.drops` and each cursor's runs grow without bound under a peer -that fragments them. - -Port, mirroring Rust's names and shape: - -- A pool per broadcast, defaulting as `broadcast::Info::default` does in - Rust: no byte capacity and `DEFAULT_EXPIRY` (30 s) idle expiry. -- A per-track account and a per-group charge billing payload plus a fixed - per-group overhead, and a fixed per-run cost for drop-set runs. -- Write-time eviction debt: a track over capacity pays by evicting its own - oldest groups with EVICTED, the latest live group protected, and the - access-time mean ordering eviction across tracks. -- The idle-expiry sweep on the pool's clock, replacing `#prune`'s own timer. - -Test on mock time: a capacity bounds cached groups across tracks of one -broadcast, evicting the oldest first and sparing a fresher track; idle groups -expire; a drop-only flood with no cached groups stays within the pool once -[Drop set](/quest/m1/drop-set.md) bills its runs; the per-group frame-tail -limit still applies. - -Public API: none beyond what Rust already exposes, mirrored if a JS consumer -needs it. Wire: none. - -## Related - -- [Drop set](/quest/m1/drop-set.md) - bills its track and cursor runs to this pool diff --git a/test/relay-fill/relay.ts b/test/relay-fill/relay.ts index a1b5123c682..81cdaad706c 100644 --- a/test/relay-fill/relay.ts +++ b/test/relay-fill/relay.ts @@ -9,8 +9,8 @@ import assert from "node:assert/strict"; import { createInterface } from "node:readline"; import { Manual, Milli, Nano } from "@moq/clock"; import Session from "@moq/qmux"; +import { DEFAULT_EXPIRY } from "../../js/net/src/cache.ts"; import { connect } from "../../js/net/src/connection/connect.ts"; -import { CACHE_EXPIRY } from "../../js/net/src/ledger.ts"; import { Producer } from "../../js/net/src/origin.ts"; import * as Path from "../../js/net/src/path.ts"; import { wireOf } from "../../js/net/src/wire.ts"; @@ -71,8 +71,9 @@ try { continue; } assert.equal(line, "evict", `unknown command: ${line}`); - // A prune slice past the cache window, so the next scan evicts what nobody touched. - await clock.advance(Nano.fromMilli(Milli(CACHE_EXPIRY + 10_000))); + // Past the cache window and the sweep that trails it by up to half a window, so the + // copy has evicted what nobody touched. + await clock.advance(Nano.fromMilli(Milli(2 * DEFAULT_EXPIRY))); console.log("evicted"); } } finally {