From f90826fa510c06b87685f3fe5d73e14e5594f673 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 09:58:02 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m1/js-cache-pool.md Quest-Claim: 60c78b3f0d59a6ba From 78eff1f927b763fea888c62040c99657d6c24552 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 10:52:38 -0800 Subject: [PATCH 2/8] feat(net): js/net caches into a per-broadcast byte pool ported from Rust Port Rust's `cache::Pool` to `js/net/src/cache.ts`. Every cached group bills its frames, a fixed per-group overhead, and each subscriber mirror it holds to its track's account, and every track of a broadcast shares one pool, defaulting as Rust's `broadcast::Info` does: no byte capacity and a 30 s idle window. Over capacity, a track accrues eviction debt as it writes and pays it by evicting its own oldest groups with Evicted, never the live edge, ordered by the mean access across the pool. Reads count as accesses. The pool's sweep, every half window, replaces each track's own prune timer. Finishes quest/m1/js-cache-pool.md. Co-Authored-By: Claude Opus 5.5 --- js/net/src/broadcast.test.ts | 7 +- js/net/src/broadcast.ts | 17 +- js/net/src/cache.test.ts | 255 ++++++++++++++ js/net/src/cache.ts | 543 +++++++++++++++++++++++++++++ js/net/src/group.test.ts | 4 +- js/net/src/group.ts | 113 ++++-- js/net/src/ietf/fetch.test.ts | 4 +- js/net/src/internal.ts | 23 +- js/net/src/ledger.ts | 6 - js/net/src/lite/copy_fill.test.ts | 4 +- js/net/src/lite/subscriber.test.ts | 5 +- js/net/src/parked-expiry.test.ts | 4 +- js/net/src/track.test.ts | 28 +- js/net/src/track.ts | 201 +++++------ quest/m1/README.md | 1 - quest/m1/drop-set.md | 12 +- quest/m1/js-cache-pool.md | 52 --- 17 files changed, 1039 insertions(+), 240 deletions(-) create mode 100644 js/net/src/cache.test.ts create mode 100644 js/net/src/cache.ts delete mode 100644 quest/m1/js-cache-pool.md diff --git a/js/net/src/broadcast.test.ts b/js/net/src/broadcast.test.ts index 0215b901ad0..b8511cf68c4 100644 --- a/js/net/src/broadcast.test.ts +++ b/js/net/src/broadcast.test.ts @@ -513,10 +513,9 @@ test("a stalled consumer does not pin evicted groups", async () => { producer.writeFrame(textFrame("old")); - // Advance past the cache window, plus the scan slice a prune may lag by, and write - // again to trigger a prune. The old (closed, aged-out) group is dropped from the - // stalled sink, not retained. - clock.jump(Nano.fromMilli(Milli(30_000 + 30_000 / 8))); + // Advance past the cache window and write again, which scans for idle groups. The old + // (closed, aged-out) group is dropped from the stalled sink, not retained. + clock.jump(Nano.fromMilli(Milli(30_000 + 1_000))); producer.writeFrame(textFrame("fresh")); // The next group in arrival order is the fresh one (seq 1): the old (seq 0) group diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 39cd0104c7f..b01c1e53592 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -4,6 +4,7 @@ * @module */ import { type GetPromise, Once, Signal } from "@moq/signals"; +import * as cache from "./cache.ts"; import * as datagram from "./datagram.ts"; import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; @@ -54,6 +55,16 @@ class BroadcastState { unheld = false; // Run synchronously once the broadcast closes, before `closed` notifies its observers. closers = new Set<() => void>(); + // The cache pool every track of the broadcast caches into, defaulting as Rust's + // `broadcast::Info` does: no byte capacity, and the default idle window. + pool = new cache.Pool({ expiry: cache.DEFAULT_EXPIRY }); +} + +// A new track of the broadcast, caching into its pool. +function createProducer(state: BroadcastState, name: string): track.Producer { + const producer = new track.Producer(name); + hooks.poolTrack(producer, state.pool); + return producer; } // Datagram tracks never enter the stream listing or cache. @@ -204,7 +215,7 @@ function logical(state: BroadcastState, name: string): { producer: track.Produce const existing = lookup(state, name); if (existing) return { producer: existing, requested: false }; - const producer = new track.Producer(name); + const producer = createProducer(state, name); if (!state.served) { // Answer through the track rather than throwing, so a peer's subscribe is refused with // the code instead of an unhandled error. @@ -398,7 +409,7 @@ export class Producer { const state = broadcast.#state; if (state.closed.peek() !== undefined) throw new Error("broadcast is closed"); if (lookup(state, name)) throw new Error("duplicate reserved track"); - const producer = new track.Producer(name); + const producer = createProducer(state, name); const lookups = new Lookups(); state.lookups.set(producer, lookups); state.requests.set(name, producer); @@ -462,7 +473,7 @@ export class Producer { throw new Error(`duplicate track: ${name}`); } - const producer = new track.Producer(name).accept(info); + const producer = createProducer(this.#state, name).accept(info); adopt(this.#state, producer, track.authoredInfo(info)); return producer; } diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts new file mode 100644 index 00000000000..9c311dec6ec --- /dev/null +++ b/js/net/src/cache.test.ts @@ -0,0 +1,255 @@ +import { expect, test } from "bun:test"; +import { Manual, Milli, Nano, yieldNow } from "@moq/clock"; +import * as cache from "./cache.ts"; +import { StreamCode, TooFarBehind } from "./error.ts"; +import { hooks } from "./internal.ts"; +import { Producer as TrackProducer } from "./track.ts"; + +// Run `fn` on a manual clock, stepped in milliseconds. +async function withClock(fn: (clock: { advance: (ms: number) => Promise }) => Promise | void) { + const clock = new Manual(); + const installed = clock.install(); + try { + await fn({ advance: (ms) => clock.advance(Nano.fromMilli(Milli(ms))) }); + } finally { + installed.close(); + } +} + +// A track caching into `pool`, as a broadcast's tracks do. +function pooled(pool: cache.Pool, name = "test"): TrackProducer { + const producer = new TrackProducer(name); + hooks.poolTrack(producer, pool); + return producer.accept({ maxAge: null }); +} + +function finished(producer: TrackProducer, size: number): bigint { + const group = producer.appendGroup(); + group.writeFrame({ payload: new Uint8Array(size) }); + group.finish(); + return group.sequence; +} + +// The sequences a fresh subscriber is handed: what the track still caches. Its mirrors are +// released once its close settles. +async function cached(producer: TrackProducer): Promise { + const subscriber = producer.subscribe({ delay: { max: null } }); + const sequences: bigint[] = []; + for (let group = subscriber.tryRecvGroup(); group; group = subscriber.tryRecvGroup()) + sequences.push(group.sequence); + subscriber.close(); + await yieldNow(); + return sequences; +} + +// What one finished group of `size` payload bytes bills: its frame, the frame's own +// bookkeeping (group.ts `FRAME_BYTES`), and the group's. +const billed = (size: number) => size + 256 + cache.ENTRY_OVERHEAD; + +// While the pool is over capacity, every append accrues debt and pays it by evicting the +// track's own oldest groups, so the newest content survives. Mirrors Rust `debt_evicts_oldest_group`. +test("debt evicts a track's oldest groups", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: 2 * billed(10_000) }); + const producer = pooled(pool); + for (let i = 0; i < 4; i++) finished(producer, 10_000); + + const left = await cached(producer); + expect(left).not.toContain(0n); + expect(left).toContain(3n); + // The protected live edge plus the group just demoted, since debt is charged before it. + expect(pool.used).toBeLessThanOrEqual(2 * billed(10_000)); + }); +}); + +// Far too small for one group, the pool still keeps the newest group whole. Mirrors Rust +// `latest_group_never_evicted`. +test("the newest group survives any capacity", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: 100 }); + const producer = pooled(pool); + for (let i = 0; i < 3; i++) finished(producer, 1_000); + const left = await cached(producer); + expect(left).not.toContain(0n); + expect(left).toContain(2n); + const newest = producer.subscribe({ delay: { max: null }, ranges: [{ start: { absolute: 2n } }] }); + expect((await newest.tryRecvGroup()?.readFrame())?.payload.length).toBe(1_000); + }); +}); + +// A reader holding an evicted group's unread frames is told so, rather than hanging or +// reading a truncated clean end. Mirrors Rust `eviction_aborts_readers`. +test("eviction fails a reader's unread group with Evicted", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: billed(10_000) + cache.MIRROR_OVERHEAD }); + const producer = pooled(pool); + const subscriber = producer.subscribe({ delay: { max: null } }); + finished(producer, 10_000); // seq 0 + const held = subscriber.tryRecvGroup(); + finished(producer, 10_000); // seq 1: demotes seq 0 + finished(producer, 10_000); // seq 2: pays by evicting seq 0 + await expect(held?.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); + subscriber.close(); + }); +}); + +// Debt too small to evict an old group carries over rather than shedding the live group's +// frames, which its reader has not taken yet. Mirrors Rust `carried_debt_keeps_the_live_group`. +test("carried debt keeps the live group", async () => { + await withClock(async () => { + const unit = 100 * cache.ENTRY_OVERHEAD; + const pool = new cache.Pool({ capacity: 22 * unit }); + const producer = pooled(pool); + finished(producer, 20 * unit); + + const group = producer.appendGroup(); + const reader = group.consume(); + for (let i = 0; i < 3; i++) group.writeFrame({ payload: new Uint8Array(unit) }); + group.finish(); + for (let i = 0; i < 3; i++) expect((await reader.readFrame())?.payload.length).toBe(unit); + expect(await reader.readFrame()).toBeUndefined(); + }); +}); + +// A read lifts a group's access above the pool's mean, so the eviction walk passes over it +// and evicts a never-read group instead, even one that arrived later. Mirrors Rust +// `fetch_refresh_survives_eviction`. +test("a read group outlives a later unread one", async () => { + await withClock(async (clock) => { + // Each group is mirrored to the reader too. + const pool = new cache.Pool({ capacity: Math.floor(3.5 * (billed(3_000) + cache.MIRROR_OVERHEAD)) }); + const producer = pooled(pool); + const reader = producer.subscribe({ delay: { max: null } }); + + finished(producer, 3_000); // seq 0 + await clock.advance(1_000); + finished(producer, 3_000); // seq 1 + await clock.advance(1_000); + finished(producer, 3_000); // seq 2 + await clock.advance(500); + const first = reader.tryRecvGroup(); + expect(first?.sequence).toBe(0n); + expect((await first?.readFrame())?.payload.length).toBe(3_000); + await clock.advance(500); + + finished(producer, 3_000); // seq 3 + await clock.advance(1_000); + finished(producer, 3_000); // seq 4 + reader.close(); + + const left = await cached(producer); + expect(left).toContain(0n); + expect(left).not.toContain(1n); + }); +}); + +// Every track of a broadcast caches into one pool, and the mean access across all of them +// orders eviction: a track writing past the capacity pays with its own groups, but only those +// accessed no later than the mean, so a fresher track's groups survive its own debt while a +// staler track's write evicts its oldest at double rate. +test("a shared capacity spares a fresher track and evicts a staler one's oldest", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ capacity: 6 * billed(10_000) }); + const open = (producer: TrackProducer) => { + const group = producer.appendGroup(); + group.writeFrame({ payload: new Uint8Array(10_000) }); + return group; + }; + const staler = pooled(pool, "staler"); + const old = [open(staler), open(staler), open(staler), open(staler)]; + + await clock.advance(10_000); + const fresher = pooled(pool, "fresher"); + const reader = fresher.subscribe({ delay: { max: null } }); + const read = []; + for (let i = 0; i < 4; i++) { + read.push(open(fresher)); + expect((await reader.tryRecvGroup()?.readFrame())?.payload.length).toBe(10_000); + } + expect(pool.used).toBeGreaterThan(6 * billed(10_000)); + for (const group of read) expect(group.closed.peek()).toBeUndefined(); + + open(staler); + // One write's worth of debt, doubled: two groups. + expect(old[0].closed.peek()).toMatchObject({ code: StreamCode.Evicted }); + expect(old[1].closed.peek()).toMatchObject({ code: StreamCode.Evicted }); + expect(old[2].closed.peek()).toBeUndefined(); + for (const group of read) expect(group.closed.peek()).toBeUndefined(); + reader.close(); + }); +}); + +// The pool's sweep reclaims every track's idle groups with no write behind it, the live edges +// excepted, and leaves only what those still bill. Mirrors Rust `pool_sweep_expires_without_a_write`. +test("the sweep expires idle groups across tracks without a write", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const tracks = [pooled(pool, "a"), pooled(pool, "b")]; + const stalled = tracks.map((track) => { + const group = track.appendGroup(); // left open + group.writeFrame({ payload: new Uint8Array(1) }); + finished(track, 1); // the live edge + return group; + }); + + await clock.advance(1_000); + for (const group of stalled) expect(group.closed.peek()).toBeUndefined(); + await clock.advance(1_000 + 500); + for (const group of stalled) expect(group.closed.peek()).toBeInstanceOf(TooFarBehind); + for (const track of tracks) expect(await cached(track)).toEqual([1n]); + expect(pool.used).toBe(2 * billed(1)); + }); +}); + +// Every byte a group bills, its subscribers' mirrors included, goes back once it leaves. +test("the pool is billed only for what is cached", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const producer = pooled(pool); + const subscriber = producer.subscribe({ delay: { max: null } }); + finished(producer, 100); + expect(pool.used).toBe(billed(100) + cache.MIRROR_OVERHEAD); + subscriber.close(); + await yieldNow(); + expect(pool.used).toBe(billed(100)); + + producer.finish(); + producer.close(); + await clock.advance(3_000); + expect(pool.used).toBe(0); + }); +}); + +// The group's own tail limit still sheds its front, and the pool is credited for it. +test("a group's tail limit releases what it sheds from the pool", async () => { + await withClock(() => { + const pool = new cache.Pool(); + const producer = pooled(pool); + const group = producer.appendGroup(); + const frame = new Uint8Array(1024 * 1024); + for (let i = 0; i < 40; i++) group.writeFrame({ payload: frame }); + expect(hooks.groupCacheBytes(group)).toBeLessThanOrEqual(32 * 1024 * 1024); + expect(pool.used).toBe(hooks.groupCacheBytes(group) + cache.ENTRY_OVERHEAD); + }); +}); + +// With nothing evictable, the debt sheds the live edge's front, keeping the newest frame, so +// an open group a track only appends frames to stays near the capacity. +test("the debt sheds the front of an open live edge", async () => { + await withClock(async () => { + const capacity = 1024 * 1024; + const pool = new cache.Pool({ capacity }); + const producer = pooled(pool); + const reader = producer.subscribe({ delay: { max: null } }); + const group = producer.appendGroup(); + const behind = reader.tryRecvGroup(); + const frame = new Uint8Array(64 * 1024); + for (let i = 0; i < 64; i++) group.writeFrame({ payload: frame }); + + expect(group.closed.peek()).toBeUndefined(); + // What a frame write may add before it settles, on top of the capacity. + expect(pool.used).toBeLessThanOrEqual(capacity + 512 * 1024); + // A reader that never read lost the shed front. + await expect(behind?.readFrame()).rejects.toThrow(); + }); +}); diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts new file mode 100644 index 00000000000..32ff64c8d3a --- /dev/null +++ b/js/net/src/cache.ts @@ -0,0 +1,543 @@ +/** + * A shared byte budget for cached groups, repaid by write-time eviction. Mirrors the Rust + * `cache` module. + * + * Every cached group bills its frames and a fixed overhead through a {@link Charge} to its + * track's {@link Track} account, and every track of a broadcast caches into one {@link Pool}. + * The pool never evicts on its own: while it is over capacity, each track accrues eviction debt + * as it writes, sized to what it wrote, and pays it by evicting its own oldest groups. A group + * accessed more recently than the mean access of the evictable groups is never evicted, so fresh + * content in one track outlives stale content in another, and a track whose oldest group is + * staler than the mean accrues at double rate. + * + * The pool also owns the idle window: a group nobody reads or writes for that long is reclaimed, + * except a live track's latest. Writes scan for idle groups as they go, and the pool sweeps + * every track at half the window for the ones nothing writes to. + * + * Package-internal: not exported from the entrypoint. + * + * @module + */ +import { Instant, Milli, Nano, now, sleepUntil } from "@moq/clock"; + +/** How long a group nobody reads or writes stays cached by default, matching Rust `cache::DEFAULT_EXPIRY`. */ +export const DEFAULT_EXPIRY = Milli(30_000); + +// Bun 1.4's heap (bun:jsc `heapStats`) grows about 2.9 KB per cached single-frame group with no +// subscriber, and 3.9 KB more per subscriber, whose sink holds its own mirror of every cached +// group. Billing payload alone would let a track of tiny groups cache far more than its budget. +/** Bookkeeping billed per cached group on top of its frames. */ +export const ENTRY_OVERHEAD = 3 * 1024; +/** Bookkeeping billed per subscriber mirror a cached group holds. */ +export const MIRROR_OVERHEAD = 4 * 1024; + +// The coarse clock behind access stamps, as Rust's: the expiry window rounds up to it. +const TICK = Nano.fromMilli(Milli(100)); + +// Sub-tick boosts break ties within one tick: a write outranks a merely inserted group, and a +// read outranks both, so a same-tick access still reads as newer than the mean of weaker ones. +const WRITE_BOOST = 1; +const READ_BOOST = 2; +const BOOSTS = 4; + +// Bytes a track writes before a frame write settles its debt, so a track appending frames to +// open groups, never inserting another, still pays. +const WRITE_CHARGE_THRESHOLD = 256 * 1024; + +// Ticks between the idle scans frame writes run, so a low-bitrate track still ages out. +const EXPIRY_SCAN_TICKS = 10; + +// Live candidates one debt payment examines, and entries one write-driven idle scan does: enough +// to step over a few fresh groups, few enough that a write never walks a long queue. +const EVICT_SCAN = 4; + +// Dead entries the eviction order tolerates beyond twice the live ones before it is rebuilt. +const EVICT_SLACK = 64; + +/** The initial policy for a {@link Pool}. */ +export interface Config { + /** The byte target usage converges toward; unbounded when omitted. */ + capacity?: number; + /** How long a group nobody reads or writes stays cached; never reclaimed for idleness when omitted. */ + expiry?: Milli; +} + +/** + * A shared byte budget and idle window for the cached groups of every track charging into it. + * + * The capacity is a target usage converges toward, not a hard limit: carried debt, capped + * payments, and the always protected live edge all let usage exceed it for a while. + */ +export class Pool { + #capacity: number; + // The idle window in ticks. + readonly #expiry?: number; + #used = 0; + // The first instant the pool read, and the latest tick since it, never moving back. + #epoch?: Instant; + #tick = 0; + // The sum and count of the evictable groups' access stamps, for their mean. + #sum = 0; + #count = 0; + // The accounts holding a group that may expire, which the idle sweep visits. + #tracks = new Set>(); + #sweep?: AbortController; + + constructor(config: Config = {}) { + this.#capacity = config.capacity ?? Number.POSITIVE_INFINITY; + if (config.expiry !== undefined) this.#expiry = Math.max(1, Math.ceil(Nano.fromMilli(config.expiry) / TICK)); + } + + /** The byte target, or undefined when unbounded. */ + get capacity(): number | undefined { + return Number.isFinite(this.#capacity) ? this.#capacity : undefined; + } + + /** Bytes billed across every cached group. */ + get used(): number { + return this.#used; + } + + /** The idle window, rounded up to the pool's 100 ms tick, or undefined when idle groups stay. */ + get expiry(): Milli | undefined { + return this.#expiry === undefined ? undefined : Nano.toMilli(Nano(this.#expiry * TICK)); + } + + /** Change the byte target; undefined makes it unbounded. Writes pay a shrink down as they go. */ + resize(capacity?: number): void { + this.#capacity = capacity ?? Number.POSITIVE_INFINITY; + } + + /** Ticks since the pool first read the clock. */ + now(): number { + const at = now(); + this.#epoch ??= at; + this.#tick = Math.max(this.#tick, Math.floor((at - this.#epoch) / TICK)); + return this.#tick; + } + + /** The idle window in ticks, or undefined when idle groups stay. */ + get expiryTicks(): number | undefined { + return this.#expiry; + } + + /** The mean access stamp of the evictable groups, or undefined with none. */ + average(): number | undefined { + return this.#count === 0 ? undefined : this.#sum / this.#count; + } + + /** + * The eviction debt writing `written` bytes accrues, or undefined while the pool is within + * capacity: `written * used / capacity`, so paying it frees a little more than was written + * and the overshoot decays toward the capacity. + */ + accrue(written: number): number | undefined { + if (this.#used <= this.#capacity) return undefined; + return Math.floor((written * this.#used) / Math.max(1, this.#capacity)); + } + + /** Bill `n` more bytes. */ + add(n: number): void { + this.#used += n; + } + + /** Release `n` billed bytes. */ + sub(n: number): void { + this.#used -= n; + } + + /** The current tick as an access stamp, with `boost` breaking ties within it. */ + stamp(boost: number): number { + return this.now() * BOOSTS + boost; + } + + /** A group with access stamp `stamp` became evictable. */ + accessInsert(stamp: number): void { + this.#sum += stamp; + this.#count++; + } + + /** A group with access stamp `stamp` stopped being evictable. */ + accessRemove(stamp: number): void { + this.#sum -= stamp; + this.#count--; + } + + /** An evictable group's access stamp moved from `old` to `stamp`. */ + accessRefresh(old: number, stamp: number): void { + this.#sum += stamp - old; + } + + /** Visit `track` on each idle sweep, if the pool has an idle window, while it holds a group that may expire. */ + register(track: Track): void { + if (this.#expiry === undefined) return; + this.#tracks.add(track); + this.#arm(); + } + + /** Stop visiting `track`, which holds no group that may expire. */ + unregister(track: Track): void { + this.#tracks.delete(track); + } + + /** Expire the idle groups of every registered track. */ + sweep(): void { + for (const track of [...this.#tracks]) track.sweep(); + } + + // Sweep again in half the idle window while any track holds a group that may expire. Never a + // reason to hold a Node or Bun process open. + #arm(): void { + if (this.#sweep !== undefined || this.#expiry === undefined || this.#tracks.size === 0) return; + const sweep = new AbortController(); + this.#sweep = sweep; + const at = Instant.add(now(), Nano((this.#expiry * TICK) / 2)); + void sleepUntil(at, { signal: sweep.signal, ref: false }).then( + () => { + if (this.#sweep !== sweep) return; + this.#sweep = undefined; + this.sweep(); + this.#arm(); + }, + () => {}, + ); + } +} + +/** A cached group as its track's account orders it for eviction. */ +export interface Entry { + /** What the group bills. */ + readonly charge: Charge; +} + +/** How an account reaches the track whose groups it evicts. */ +export interface Owner { + /** Whether `entry` is the live edge, which neither eviction nor expiry takes. */ + protects(entry: E): boolean; + /** The live edge while protected, whose front pays the debt when no group can go whole. */ + edge(): E | undefined; + /** Take `entry` out of the cache and end it, for its bytes (`evicted`) or for idling (`expired`). */ + evict(entry: E, reason: "evicted" | "expired"): void; + /** Shed whole frames from the front of `entry` toward `debt`, freeing about `cap` at most, and return the bytes freed. */ + shed(entry: E, debt: number, cap: number): number; + /** Something was evicted: settle what the cache's contents decide. */ + settle(): void; +} + +/** + * One track's account against a {@link Pool}: the bytes its groups wrote, the debt they owe, and + * the order its groups are evicted in. Mirrors the Rust `cache::Track` with the eviction order + * and debt of the Rust `TrackState`. + */ +export class Track { + /** The pool this track caches into. */ + readonly pool: Pool; + #owner: Owner; + // Bytes billed since the last debt accrual. + #written = 0; + // The tick a frame write may next run an idle scan at. + #nextExpiry = 0; + // Every cached group, and the eviction order, oldest first, which keeps hints of groups no + // longer cached until a scan reaches them. + #live = new Set(); + #order: E[] = []; + // Debt carried over from earlier payments. + #debt = 0; + // Where the next bounded idle scan starts, rotating so fresh groups at the front do not + // starve the ones behind them. + #cursor = 0; + + constructor(pool: Pool, owner: Owner) { + this.pool = pool; + this.#owner = owner; + } + + /** Open the charge for a new group, billing its fixed overhead. */ + charge(): Charge { + return makeCharge(this as Track); + } + + /** Count `n` billed bytes toward the next debt accrual. */ + wrote(n: number): void { + this.#written += n; + } + + /** Whether the account caches any group. */ + get empty(): boolean { + return this.#live.size === 0; + } + + /** Cache `entry` at the back of the eviction order, evictable now unless it is the live edge. */ + insert(entry: E): void { + this.#live.add(entry); + this.#order.push(entry); + if (!this.#owner.protects(entry)) this.demote(entry); + } + + /** Make `entry` evictable once the live edge no longer protects it. */ + demote(entry: E): void { + if (!this.#live.has(entry)) return; + entry.charge.demote(); + this.pool.register(this as Track); + } + + /** Forget `entry`, which left the cache, releasing what it billed. */ + remove(entry: E): void { + if (!this.#live.delete(entry)) return; + entry.charge.clear(); + if (this.#live.size === 0) { + this.#order = []; + this.#debt = 0; + this.pool.unregister(this as Track); + } + } + + /** + * Accrue and pay the eviction debt of everything written since the last accrual. Run before a + * new group is cached, so the write that creates it can never evict it. Doubles when the + * track's oldest group is staler than the pool's mean. Returns whether it evicted a group. + */ + chargeDebt(): boolean { + const written = this.#written; + this.#written = 0; + const accrued = this.pool.accrue(written); + // Under capacity, a stale debt would only make a burst at the next pressure spike. + if (accrued === undefined) { + this.#debt = 0; + return false; + } + const debt = this.#oldestIsStale() ? 2 * accrued : accrued; + // What is used bounds what eviction could ever free, so a track that cannot pay + // (everything protected) does not hoard a stale schedule. + this.#debt = Math.min(this.#debt + debt, this.pool.used); + // One write never dumps a deep backlog at once: the remainder carries over. + return this.#payDebt(2 * written); + } + + // Whether the oldest evictable group was accessed no later than the pool's mean. A dead entry + // at the front reads as not stale until a scan drops it. + #oldestIsStale(): boolean { + const average = this.pool.average(); + const front = this.#order[0]; + if (average === undefined || front === undefined || !this.#live.has(front)) return false; + return !this.#owner.protects(front) && front.charge.accessed <= average; + } + + // Evict the stalest groups until the debt is paid or `cap` bytes are freed, examining a few + // candidates from the front of the order. A group accessed after the pool's mean rotates to + // the back instead, leaving the debt to tracks holding staler content. A group larger than + // the debt left stays, carrying the debt over. With no group able to go whole, the fresh + // ones and the live edge shed their fronts, so neither grows without bound. + #payDebt(cap: number): boolean { + const average = this.pool.average() ?? 0; + let paid = 0; + let scanned = 0; + let evicted = false; + const fresh: E[] = []; + for (let i = this.#order.length; i > 0; i--) { + if (this.#debt === 0 || paid >= cap || scanned >= EVICT_SCAN) break; + const entry = this.#order.shift(); + if (entry === undefined) break; + if (!this.#live.has(entry)) continue; + if (this.#owner.protects(entry)) { + this.#order.push(entry); + continue; + } + scanned++; + if (entry.charge.accessed > average) { + this.#order.push(entry); + fresh.push(entry); + continue; + } + const size = entry.charge.size; + if (size > this.#debt) { + this.#order.unshift(entry); + return evicted; + } + this.#debt -= size; + paid += size; + evicted = true; + this.#owner.evict(entry, "evicted"); + } + const edge = this.#owner.edge(); + if (edge) fresh.push(edge); + for (const entry of fresh) { + if (this.#debt === 0 || paid >= cap) break; + if (!this.#live.has(entry)) continue; + const freed = this.#owner.shed(entry, this.#debt, cap - paid); + this.#debt -= Math.min(freed, this.#debt); + paid += freed; + } + return evicted; + } + + /** + * Evict the groups idle past the pool's window, never the live edge: every one when `full`, + * or else a bounded scan from a rotating cursor. Returns whether it evicted a group. + */ + evictExpired(full: boolean): boolean { + const expiry = this.pool.expiryTicks; + const order = this.#order; + const len = order.length; + if (expiry === undefined || len === 0) return false; + const now = this.pool.now(); + const start = full ? 0 : this.#cursor % len; + const width = full ? len : Math.min(len, EVICT_SCAN); + if (!full) this.#cursor = start + width; + const expired: E[] = []; + for (let step = 0; step < width; step++) { + const entry = order[(start + step) % len]; + if (!this.#live.has(entry) || this.#owner.protects(entry)) continue; + if (now - Math.floor(entry.charge.accessed / BOOSTS) > expiry) expired.push(entry); + } + for (const entry of expired) this.#owner.evict(entry, "expired"); + this.#compact(); + return expired.length > 0; + } + + // Drop the dead entries at the front of the order, and rebuild it once dead entries behind + // a live front clearly outnumber the live ones. + #compact(): void { + const order = this.#order; + let dead = 0; + while (dead < order.length && !this.#live.has(order[dead])) dead++; + if (dead > 0) order.splice(0, dead); + if (order.length > 2 * this.#live.size + EVICT_SLACK) { + this.#order = order.filter((entry) => this.#live.has(entry)); + } + } + + /** + * Settle from a frame write: pay the debt once enough was written, and scan for idle groups + * once per second, so a track that only appends frames to open groups still pays and ages. + */ + settle(): void { + const debt = this.#written >= WRITE_CHARGE_THRESHOLD; + const expiry = this.#expiryDue(); + let evicted = false; + if (debt) evicted = this.chargeDebt(); + if (expiry) evicted = this.evictExpired(false) || evicted; + if (evicted) this.#owner.settle(); + } + + /** Expire every idle group, from the pool's sweep. */ + sweep(): void { + if (this.evictExpired(true)) this.#owner.settle(); + // A live edge alone never expires, so it needs no sweep to keep its track alive for. + for (const entry of this.#live) if (!this.#owner.protects(entry)) return; + this.pool.unregister(this as Track); + } + + // Claim the next write-driven idle scan once its time comes. + #expiryDue(): boolean { + const expiry = this.pool.expiryTicks; + if (expiry === undefined) return false; + const now = this.pool.now(); + if (now < this.#nextExpiry) return false; + this.#nextExpiry = now + Math.min(expiry, EXPIRY_SCAN_TICKS); + return true; + } +} + +let makeCharge: (track: Track) => Charge; + +/** + * What one cached group bills to its track's account: its bytes, overhead included, and its + * access stamp, which the pool's mean samples while the group is evictable. Cleared once the + * group leaves the cache, after which every operation is a no-op. + */ +export class Charge { + #track?: Track; + #bytes = 0; + #stamp: number; + // Whether the stamp is a sample in the pool's mean: the group is evictable. + #counted = false; + + private constructor(track: Track) { + this.#track = track; + this.#stamp = track.pool.stamp(0); + this.bill(ENTRY_OVERHEAD); + } + + static { + makeCharge = (track) => new Charge(track); + } + + /** Bill `n` more bytes of a write, which is an access too: it restarts the idle window. */ + add(n: number): void { + this.bill(n); + this.#touch(WRITE_BOOST); + } + + /** Bill `n` more bytes nobody accessed, such as a subscriber's mirror. */ + bill(n: number): void { + const track = this.#track; + if (!track) return; + track.pool.add(n); + track.wrote(n); + this.#bytes += n; + } + + /** Release `n` bytes the group dropped while still cached. */ + sub(n: number): void { + const track = this.#track; + if (!track) return; + n = Math.min(n, this.#bytes); + track.pool.sub(n); + this.#bytes -= n; + } + + /** Every byte billed, overhead included. */ + get size(): number { + return this.#bytes; + } + + /** The last access stamp: its tick times four, plus the boost of the access that set it. */ + get accessed(): number { + return this.#stamp; + } + + /** Record a read: a fetched group or a frame read. */ + refresh(): void { + this.#touch(READ_BOOST); + } + + /** Record a write that bills no new bytes. */ + recordWrite(): void { + this.#touch(WRITE_BOOST); + } + + /** Make the group evictable, sampling its stamp into the pool's mean. Idempotent. */ + demote(): void { + const track = this.#track; + if (!track || this.#counted) return; + track.pool.accessInsert(this.#stamp); + this.#counted = true; + } + + /** Settle the account's debt and idle scan after a frame write. */ + settle(): void { + this.#track?.settle(); + } + + /** Release everything billed and the mean's sample, and detach from the account. Idempotent. */ + clear(): void { + const track = this.#track; + if (!track) return; + track.pool.sub(this.#bytes); + this.#bytes = 0; + if (this.#counted) track.pool.accessRemove(this.#stamp); + this.#counted = false; + this.#track = undefined; + } + + // Advance the stamp to the current tick with `boost`, never moving it back. + #touch(boost: number): void { + const track = this.#track; + if (!track) return; + const stamp = track.pool.stamp(boost); + if (stamp <= this.#stamp) return; + if (this.#counted) track.pool.accessRefresh(this.#stamp, stamp); + this.#stamp = stamp; + } +} diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index fe1e558ed50..db9efd788c3 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -949,10 +949,10 @@ test.each(["eviction", "expiry"] as const)("a group end wait observes reader %s" const expired = new Signal(false); hooks.expireGroup(reader, { expired: () => expired.peek(), changed: [expired] }); const end = hooks.groupEnd(reader); - if (failure === "eviction") hooks.evictGroup(reader); + if (failure === "eviction") hooks.evictGroup(reader, new StreamError(StreamCode.Evicted)); else expired.set(true); await expect(end).rejects.toMatchObject({ - code: failure === "eviction" ? StreamCode.TooFarBehind : StreamCode.Old, + code: failure === "eviction" ? StreamCode.Evicted : StreamCode.Old, }); source.finish(); reader.close(); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 4785a3df392..0d9f8a2c74e 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -5,7 +5,8 @@ */ import { type Instant, now } from "@moq/clock"; import { type Dispose, type GetPromise, type Getter, Once, race, Signal } from "@moq/signals"; -import { FrameTooLarge, ProtocolViolation, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; +import type { Charge } from "./cache.ts"; +import { FrameTooLarge, ProtocolViolation, StreamCode, StreamError, TimestampMismatch } from "./error.ts"; import { Compression, encodable, stored } from "./flate/group.ts"; import { type Expiry, type GroupRecover, hooks, type ReadGroupFrame, type Replacement } from "./internal.ts"; import { Queue } from "./queue.ts"; @@ -211,6 +212,9 @@ class GroupState { offset = 0; cacheBytes = 0; + // What the source copy bills its track's cache pool, while cached. A mirror shares its + // source's, so its reads count as accesses. + charge?: Charge; compressed?: boolean; flate?: Compression; // The owning track's info, which every frame's timestamp must `rescale` to: a timed track's frames @@ -341,10 +345,6 @@ export class Producer { // past the track's end is refused rather than served as content that does not exist. #limit?: number; - // When the group was created or last written. Retention ages a group from this, - // not from its creation, so a group still being filled is never reclaimed. - #activity = now(); - // The frame being written in pieces, if any; no other frame is written until it ends. #open?: FrameProducer; @@ -378,7 +378,10 @@ export class Producer { state.flate = flate; const frames = Array.from(state.frames.peek(), (frame) => flate.push(frame)); state.frames.set(new Queue(frames)); - state.cacheBytes = frames.reduce((bytes, frame) => bytes + frameBytes(frame), 0); + const bytes = frames.reduce((bytes, frame) => bytes + frameBytes(frame), 0); + if (bytes > state.cacheBytes) state.charge?.bill(bytes - state.cacheBytes); + else state.charge?.sub(state.cacheBytes - bytes); + state.cacheBytes = bytes; for (const mirror of group.#mirrors?.keys() ?? []) { mirror.frames.set(new Queue(frames.slice(mirror.start - state.start))); mirror.flate = flate; @@ -430,6 +433,12 @@ export class Producer { const bytes = state.flate?.stored ?? state.cacheBytes - state.frames.peek().length * FRAME_BYTES; return bytes - head > head * ratio; }; + hooks.chargeGroup = (group, charge) => { + const state = group.#state; + state.charge = charge; + charge?.bill(state.cacheBytes); + }; + hooks.shedGroup = (group, debt, cap) => group.#shed(debt, cap); hooks.limitGroup = (group, frames) => { // A closed group takes no more frames, but one already past the cap still // contradicts the end. @@ -440,15 +449,6 @@ export class Producer { }; } - /** - * When the group was created or last written, on the installed clock. - * - * @internal Idle cache eviction only. - */ - get activity(): Instant { - return this.#activity; - } - /** * Settles once the group closes: `null` on a clean close, or the abort {@link Error}. * Peek it synchronously (`undefined` while open), observe it reactively, or `await` it. @@ -476,6 +476,7 @@ export class Producer { dst.latest = this.#state.latest; dst.flate = this.#state.flate; dst.compressed = this.#state.compressed; + dst.charge = this.#state.charge; // A track restamps its own mirror with when the group joined it. dst.arrived = this.#state.arrived; if (this.#state.start === this.#state.offset && !headShed(this.#state.frames.peek().first)) { @@ -585,20 +586,22 @@ export class Producer { const commits = this.#admit(frame); const entry = new Chunked(info); - this.#append(entry, now(), commits); + this.#append(entry, commits); const open = makeFrameProducer(info, { written: () => entry.written.peek(), write: (chunk) => { if (this.#open !== open) throw new Error("group is closed"); - this.#activity = now(); - this.#state.cacheBytes += entry.write(chunk); + const bytes = entry.write(chunk); + this.#state.cacheBytes += bytes; + this.#state.charge?.add(bytes); this.#trim(); + this.#state.charge?.settle(); }, finish: () => { if (this.#open !== open) throw new Error("group is closed"); this.#open = undefined; entry.arrived = now(); - this.#activity = entry.arrived; + this.#state.charge?.recordWrite(); entry.done.set(null); // Wake whole-frame readers waiting for it to complete. this.#state.frames.mutate(() => {}); @@ -626,7 +629,7 @@ export class Producer { if (this.#state.flate) { frame = this.#state.flate.push(frame, inflated); } - this.#append(frame, arrived, commits); + this.#append(frame, commits); } // Check that the next frame may be written, returning what records its admission. @@ -649,9 +652,10 @@ export class Producer { return undefined; } - #append(frame: Frame, arrived: Instant, commits: (() => void)[] | undefined) { - this.#activity = arrived; - this.#state.cacheBytes += frameBytes(frame); + #append(frame: Frame, commits: (() => void)[] | undefined) { + const bytes = frameBytes(frame); + this.#state.cacheBytes += bytes; + this.#state.charge?.add(bytes); appendFrame(this.#state, frame); if (commits) { this.#admits = undefined; @@ -663,13 +667,15 @@ export class Producer { else appendFrame(mirror, frame); } this.#trim(); + // Past the frame's own bookkeeping: paying the track's debt may evict other groups. + this.#state.charge?.settle(); } + // Shed the front past the live tail budget. #trim(): void { const state = this.#state; const frames = state.frames.peek(); - const start = state.start; - let shed = false; + const bytes = state.cacheBytes; while (state.cacheBytes > TAIL_BYTES) { const front = frames.first; if (!front) break; @@ -677,17 +683,52 @@ export class Producer { const freed = front instanceof Chunked ? front.shed() : 0; if (freed > 0) { state.cacheBytes -= freed; - shed = true; continue; } // The newest frame stays readable: its writer holds it whole anyway. if (frames.length === 1) break; - frames.shift(); - state.cacheBytes -= frameBytes(front); - if (front instanceof Chunked) front.release(); - state.start++; + this.#shift(); + } + this.#shedMirrors(bytes); + } + + // Shed whole frames from the front toward the cache pool's `debt`, freeing about `cap` at + // most, for a group the debt cannot evict whole: one being read, or the live edge. A frame + // larger than what is left of the debt stays, with everything after it. An open group keeps + // its newest frame, for the reader about to take it. Returns the bytes freed. Mirrors Rust's + // `GroupState::shed`. + #shed(debt: number, cap: number): number { + const state = this.#state; + if (state.closed.peek() instanceof Error) return 0; + const frames = state.frames.peek(); + const keep = state.closed.peek() === undefined ? 1 : 0; + const bytes = state.cacheBytes; + while (frames.length > keep && bytes - state.cacheBytes < cap) { + const front = frames.first; + if (!front || (front instanceof Chunked && front.done.peek() === undefined)) break; + if (frameBytes(front) > debt - (bytes - state.cacheBytes)) break; + this.#shift(); } - if (state.start === start && !shed) return; + this.#shedMirrors(bytes); + return bytes - state.cacheBytes; + } + + // Drop the front frame. + #shift(): void { + const state = this.#state; + const front = state.frames.peek().shift() as Frame; + state.cacheBytes -= frameBytes(front); + if (front instanceof Chunked) front.release(); + state.start++; + } + + // Release what the front shed since it held `bytes`, and end the mirrors that can no longer + // read from their start. + #shedMirrors(bytes: number): void { + const state = this.#state; + const frames = state.frames.peek(); + if (state.cacheBytes === bytes) return; + state.charge?.sub(bytes - state.cacheBytes); state.flate?.shed(state.start); const lost = headShed(frames.first); for (const mirror of this.#mirrors?.keys() ?? []) { @@ -1228,7 +1269,7 @@ export class Consumer { group.#closed = undefined; }; hooks.readGroupFrame = (group, compressed, from = 0) => group.#readFrame(from, compressed, true); - hooks.evictGroup = (group) => group.#evict(); + hooks.evictGroup = (group, lost) => group.#evict(lost); hooks.onShedRead = (group, notify) => { group.#state.shedRead = notify; }; @@ -1288,17 +1329,17 @@ export class Consumer { return true; } - #evict(): boolean { + #evict(lost: Error): boolean { // A group read on from another copy is that copy's to reclaim. if (this.#state !== this.#mirrored) return false; const frames = this.#bufferedFrames(); if (frames.length > 0) { // Unread content is going away either way. If the budget had already given up // on this group that is the more specific reason, so ask it first; otherwise - // the reader simply lagged behind the cache. + // the cache reclaimed it before the reader got to it. this.#expire(); if (!this.#terminal) { - this.#terminal = new TooFarBehind(); + this.#terminal = lost; if (this.#verdict.peek() === undefined) this.#verdict.set(this.#terminal); } } @@ -1444,6 +1485,7 @@ export class Consumer { const sequence = this.#state.start++; this.#streamed = front; this.#closed?.refresh(); + this.#state.charge?.refresh(); return { sequence, frame: front, chunks: makeFrameConsumer(sequence, front) }; } if (front.done.peek() === undefined) return undefined; @@ -1467,6 +1509,7 @@ export class Consumer { if (frames.length === 0 && this.#state.closed.peek() !== undefined) this.#state.flate = undefined; const sequence = this.#state.start++; this.#closed?.refresh(); + this.#state.charge?.refresh(); return { sequence, frame, inflated: slice?.inflated }; } diff --git a/js/net/src/ietf/fetch.test.ts b/js/net/src/ietf/fetch.test.ts index a34db56c948..bf0cbc5b5fd 100644 --- a/js/net/src/ietf/fetch.test.ts +++ b/js/net/src/ietf/fetch.test.ts @@ -2,10 +2,10 @@ import { expect, test } from "bun:test"; import { Manual, Milli, Nano } from "@moq/clock"; import { Signal } from "@moq/signals"; import type { Grant } from "../auth.ts"; +import { DEFAULT_EXPIRY } from "../cache.ts"; import { accept as acceptSession, connect as connectSession } from "../connection/index.ts"; import { error } from "../error.ts"; import { hooks } from "../internal.ts"; -import { CACHE_EXPIRY } from "../ledger.ts"; import * as Lite from "../lite/index.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; @@ -436,7 +436,7 @@ test("a FETCH fills never-cached and evicted history, then reuses the cache", as using fx = serve(version, undefined, undefined, null); if (evicted) { writeGroup(fx.track, 1n, ["1-0", "1-1", "1-2"]); - await clock.advance(ms(CACHE_EXPIRY * 2)); + await clock.advance(ms(DEFAULT_EXPIRY * 2)); } writeGroup(fx.track, 3n, ["live"]); const cache = fx.track.subscribe({ ranges: [] }); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 8495da60ff0..5c8e0682095 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -8,6 +8,7 @@ import { type Instant, Milli } from "@moq/clock"; import type { Dispose, GetPromise, Getter } from "@moq/signals"; import type { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; +import type { Charge, Pool } from "./cache.ts"; import type * as datagram from "./datagram.ts"; import type { Delivered } from "./delivered.ts"; import { StreamCode, StreamError, toStreamCode } from "./error.ts"; @@ -526,6 +527,8 @@ export const hooks: { * as a dead route. Assigned by `track.ts`. */ releaseTrack: (producer: Producer) => void; + /** Cache a new track's groups into its broadcast's `pool`, before it caches any. Assigned by `track.ts`. */ + poolTrack: (producer: Producer, pool: Pool) => void; /** * Settles once the producer is released (closed or aborted), unlike its public `closed`, * which also settles on a reached end while the producer keeps serving. Assigned by `track.ts`. @@ -769,6 +772,10 @@ export const hooks: { writeGroupFrame: (group: GroupProducer, frame: Frame, inflated?: number) => void; /** Payload footprint including a shared inflated copy, for measurements. */ groupCacheBytes: (group: GroupProducer) => number; + /** Bill a cached group's bytes to `charge` from now on, or stop billing them with undefined. */ + chargeGroup: (group: GroupProducer, charge: Charge | undefined) => void; + /** Shed whole frames from a group's front toward a cache `debt`, freeing about `cap` at most; returns the bytes freed. */ + shedGroup: (group: GroupProducer, debt: number, cap: number) => number; /** Frame count without creating a read handle. */ groupFrameCount: (group: GroupProducer) => number; /** @@ -824,10 +831,11 @@ export const hooks: { /** Notify the owning track when a reader reaches its shed front. */ onShedRead: (group: GroupConsumer, notify: () => void) => void; /** - * Make an evicted mirror terminal while its track timeline still contains it. Whether the - * mirror is released too: not when its group reads on, or may yet, from another copy. + * Make an evicted mirror terminal while its track timeline still contains it, failing its + * unread content with `lost`. Whether the mirror is released too: not when its group reads + * on, or may yet, from another copy. */ - evictGroup: (group: GroupConsumer) => boolean; + evictGroup: (group: GroupConsumer, lost: Error) => boolean; /** Call `fn` synchronously when an open broadcast closes, from any handle. */ onClosed: (producer: BroadcastProducer, fn: () => void) => Dispose; /** Reserve an on-demand track without listing it or enabling a request handler. */ @@ -881,6 +889,9 @@ export const hooks: { releaseTrack: () => { throw new Error("track.ts not loaded"); }, + poolTrack: () => { + throw new Error("track.ts not loaded"); + }, trackReleased: () => { throw new Error("track.ts not loaded"); }, @@ -1085,6 +1096,12 @@ export const hooks: { groupCacheBytes: () => { throw new Error("group.ts not loaded"); }, + chargeGroup: () => { + throw new Error("group.ts not loaded"); + }, + shedGroup: () => { + throw new Error("group.ts not loaded"); + }, groupFrameCount: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/ledger.ts b/js/net/src/ledger.ts index ddf7617ecc9..0044625ba3e 100644 --- a/js/net/src/ledger.ts +++ b/js/net/src/ledger.ts @@ -18,12 +18,6 @@ const compare = (a: bigint, b: bigint): number => (a < b ? -1 : a > b ? 1 : 0); */ export const GRACE = Milli(1000); -/** - * How long idle cached content is kept, matching Rust `cache::DEFAULT_EXPIRY`: a group older - * than this is evicted, so nothing waits for one longer. - */ -export const CACHE_EXPIRY = Milli(30_000); - /** The grace for a subscription with `delay.max`: how long it was willing to wait for a late group anyway, or {@link GRACE} without one. */ export function grace(maxDelay: Milli): Milli { return maxDelay > 0 ? maxDelay : GRACE; diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index 135ea477131..6d1978d8c3c 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -3,10 +3,10 @@ // rule, and inside the copy, once it evicted what it received. import { expect, test } from "bun:test"; import { Elapsed, Manual, Milli, Nano, sleep, timeout } from "@moq/clock"; +import { DEFAULT_EXPIRY } from "../cache.ts"; import { accept as acceptSession, connect as connectSession } from "../connection/index.ts"; import type { Consumer as GroupConsumer } from "../group.ts"; import { hooks } from "../internal.ts"; -import { CACHE_EXPIRY } from "../ledger.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; @@ -204,7 +204,7 @@ test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( // Both caches age out what nobody touched, a prune slice past the window; only the // live edge stays. - await clock.advance(Nano.fromMilli(Milli(CACHE_EXPIRY + 10_000))); + await clock.advance(Nano.fromMilli(Milli(DEFAULT_EXPIRY + 10_000))); write(peer.track.createGroup(4n), 4n, "live"); expect((await first.recvGroup())?.sequence).toBe(4n); const before = peer.fetches(); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index ffe91c753f1..993d7db4921 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4,6 +4,7 @@ import { type Getter, Signal } from "@moq/signals"; import type { Grant } from "../auth.ts"; import type { AuthSession } from "../auth_session.ts"; import { Viewers } from "../bandwidth.ts"; +import { DEFAULT_EXPIRY } from "../cache.ts"; import type { Probe as ProbeStats } from "../connection/stats.ts"; import { error, @@ -21,7 +22,7 @@ import type { Consumer as GroupConsumer } from "../group.ts"; import { HopSchema, isAnonymous, MAX_HOPS, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; import { hooks } from "../internal.ts"; -import { CACHE_EXPIRY, GRACE } from "../ledger.ts"; +import { GRACE } from "../ledger.ts"; import * as Path from "../path.ts"; import { Reader, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; @@ -2827,7 +2828,7 @@ test("a finished group wins over a late stale drop once evicted on lite-07", asy new Reader(new ReadableStream({ start: (controller) => controller.close() }), undefined, version), ); // Everything below the live edge ages out of the cache. - const evict = () => clock.advance(Nano.fromMilli(Milli(2 * CACHE_EXPIRY))); + const evict = () => clock.advance(Nano.fromMilli(Milli(2 * DEFAULT_EXPIRY))); let priority = 0; const update = async () => { track.update({ priority: ++priority, delay: { max: Milli(500) } }); diff --git a/js/net/src/parked-expiry.test.ts b/js/net/src/parked-expiry.test.ts index 624901a6005..62b7b6547dd 100644 --- a/js/net/src/parked-expiry.test.ts +++ b/js/net/src/parked-expiry.test.ts @@ -113,7 +113,7 @@ for (const protocol of protocols) { await clock.advance(Nano.fromMilli(Milli(budget - 1))); expect(ended).toBe(false); await clock.advance(Nano.fromMilli(Milli(1))); - } else await clock.advance(Nano.fromMilli(Milli(33_750))); + } else await clock.advance(Nano.fromMilli(Milli(45_000))); expect(ended).toBe(true); if (protocol.kind === "lite") { for (;;) { @@ -221,7 +221,7 @@ for (const protocol of protocols) { try { await blocked.promise; write(1n, 1); - await clock.advance(Nano.fromMilli(Milli(33_750))); + await clock.advance(Nano.fromMilli(Milli(45_000))); // The write already handed to the stream finishes, and the stream is reset after it. credit.resolve(); await clock.advance(Nano.zero); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 7944ed61e46..ade6ba81f9c 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2,6 +2,7 @@ import { expect, spyOn, test } from "bun:test"; import { Manual, Milli, Nano, sleep, Wall, yieldNow } from "@moq/clock"; import { hostTime } from "@moq/clock/test"; import { race } from "@moq/signals"; +import { DEFAULT_EXPIRY } from "./cache.ts"; import { ProtocolViolation, Skipped, StreamCode, StreamError, TimestampMismatch, TooFarBehind } from "./error.ts"; import type { Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { Version as IetfVersion } from "./ietf/version.ts"; @@ -14,8 +15,9 @@ import { unwanted } from "./util/demand.ts"; const enc = new TextEncoder(); const dec = new TextDecoder(); -// Matches the private idle cache window and Rust cache::DEFAULT_EXPIRY. -const CACHE_WINDOW_MS = 30_000; +const CACHE_WINDOW_MS = DEFAULT_EXPIRY; +// How late the pool's sweep may reclaim a group nothing writes to: it runs every half window. +const SWEEP_LAG_MS = CACHE_WINDOW_MS / 2; test("a cached transport tail covers only positions it actually holds", async () => { const producer = new TrackProducer("test"); @@ -1899,7 +1901,7 @@ test("an abandoned open group ages out with no further write", async () => { if (!group) throw new Error("missing group"); expect(await group.readString()).toBe("first"); const read = group.readFrame().catch((err: unknown) => err); - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); expect(await read).toBeInstanceOf(TooFarBehind); } finally { producer.finish(); @@ -1917,10 +1919,10 @@ test("publisher maxAge does not change the idle cache deadline", async () => { idle.writeFrame(textFrame("cached")); producer.appendGroup(); // Eviction ends the idle open group a cache window after its last write, give or take - // the one slice a scan can lag. + // the lag of the pool's sweep. await clock.advance(10_000 + CACHE_WINDOW_MS - 1); expect(idle.closed.peek()).toBeUndefined(); - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); expect(idle.closed.peek()).toBeInstanceOf(TooFarBehind); } finally { producer.finish(); @@ -2921,8 +2923,14 @@ test("no publisher limit still ages idle groups out", async () => { expect(await (await within.recvGroup())?.readString()).toBe("old"); within.close(); - // A scan runs at most once per slice of the window, so step past the next one. - clock.set(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + // The read was an access, as in Rust: past the write's window, the group stays. + clock.set(10_000 + CACHE_WINDOW_MS + 1_000); + const read = producer.subscribe({ delay: { max: Milli(100_000) } }); + expect(read.tryRecvGroup()?.sequence).toBe(0n); + read.close(); + + // Past the read's window, a subscribe's own scan reclaims it. + clock.set(10_000 + 2 * CACHE_WINDOW_MS + 1_000); const after = producer.subscribe({ delay: { max: Milli(100_000) } }); expect(after.tryRecvGroup()?.sequence).toBe(1n); expect(after.tryRecvGroup()).toBeUndefined(); @@ -3095,7 +3103,7 @@ test("closed settles once the cache evicts the last open group below the end", a await settle(); expect(producer.closed.peek()).toBeUndefined(); - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); await settle(); expect(open.closed.peek()).toBeInstanceOf(TooFarBehind); expect(producer.closed.peek()).toBeNull(); @@ -3160,8 +3168,8 @@ test("an ended track's buffered groups age out for a stale subscriber", async () producer.close(); } - // Nothing writes after close; the scheduled cleanup can lag by one scan slice. - await clock.advance(10_000 + CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8); + // Nothing writes after close, so the pool's sweep reclaims it. + await clock.advance(10_000 + CACHE_WINDOW_MS + SWEEP_LAG_MS); if (abort) expect(() => stale.tryRecvGroup()).toThrow(abort); else expect(stale.tryRecvGroup()).toBeUndefined(); } finally { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 1d6f689f9d4..2315d29985d 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -6,6 +6,7 @@ import { Instant, Milli, Nano, now, sleepUntil } from "@moq/clock"; import { Derived, type Dispose, type GetPromise, type Getter, Once, Signal } from "@moq/signals"; +import * as cache from "./cache.ts"; import { Delivered } from "./delivered.ts"; import { GroupTooLarge, @@ -43,7 +44,7 @@ import { staleHole, type TrackRequestOptions, } from "./internal.ts"; -import { CACHE_EXPIRY, grace, Ledger } from "./ledger.ts"; +import { grace, Ledger } from "./ledger.ts"; import { rescale } from "./rescale.ts"; import { Timescale, type Timestamp } from "./time.ts"; import { checkSequence, max, min, UNBOUNDED } from "./util/sequence.ts"; @@ -77,12 +78,6 @@ function taken(read: () => Recv, ended: () => undefined): GroupConsumer | undefi } } -// Media delay.max only bounds how far behind the live edge a reader may fall. -const CACHE_WINDOW = Nano.fromMilli(CACHE_EXPIRY); - -// The cache scans at most this many times per idle cache window. -const PRUNE_SLICES = 8; - // The unset priority, the midpoint of the signed range: moq-lite carries it as 128 and // moq-transport as 127, one urgency on both. const DEFAULT_PRIORITY = 0; @@ -793,12 +788,13 @@ function after(a: Position, b: Position): boolean { } // A source group retained in the producer cache, with the mirror handed to each sink -// so eviction can drop them together. +// so eviction can drop them together, and what it all bills the cache pool. // `arrived` is when the group joined this track, which its mirrors age a predecessor from. // `bound` is its first frame's timestamp once checked to fit between its neighbors', which // bounds theirs; `unverified` marks one accepted below the eviction watermark instead. type CachedGroup = { group: GroupProducer; + charge: cache.Charge; mirrors: Map; arrived: Instant; bound?: Timestamp; @@ -994,8 +990,8 @@ export class Producer { // must not see them. The wire's aggregate, `#state.update`, keeps them. #live = new Signal(undefined); - // Recently written source groups, retained for replay to late subscribers and - // pruned once idle for longer than the cache window. Each entry tracks the mirror + // Recently written source groups, retained for replay to late subscribers until the cache + // pool evicts them, for its budget or once idle. Each entry tracks the mirror // it handed to every sink so eviction can drop them too: otherwise a slow consumer // that never reads would pin old groups (and their frame bytes) forever. #cache: CachedGroup[] = []; @@ -1008,16 +1004,9 @@ export class Producer { // group above it is checked against it as well as its cached neighbors; one at or below // it may have lost its lower bound. One entry, so nothing grows with the cache. #evicted?: Start; - // When the cache was last scanned. See #prune. - #pruned = Number.NEGATIVE_INFINITY; - // When the track closed, which restarts the former newest group's idle window. See #prune. - #closedAt = Number.NEGATIVE_INFINITY; - - // Wakeup for the next entry due to age out. Writes settle retention inline, but a - // publisher that stalls stops writing, so without this an abandoned group (and any - // reader parked in it) would wait for a write that never comes. - #pruneTimer?: AbortController; - #pruneTimerAt = 0; + // This track's account against its broadcast's cache pool (a pool of its own when it has no + // broadcast), which bills the cached groups and evicts them under its budget and idle window. + #account: cache.Track; // One independent downstream state per live subscriber. #sinks = new Set(); @@ -1078,9 +1067,31 @@ export class Producer { closed: () => this.#state.closed.peek() !== undefined, released: this.#state.closed, }); + this.#account = this.#openAccount(new cache.Pool({ expiry: cache.DEFAULT_EXPIRY })); + } + + // Open this track's account against `pool`, evicting through this producer's cache. + #openAccount(pool: cache.Pool): cache.Track { + return new cache.Track(pool, { + protects: (entry) => this.#protects(entry), + edge: () => { + const entry = this.#edgeEntry(); + return entry && this.#protects(entry) ? entry : undefined; + }, + evict: (entry, reason) => { + this.#evict(entry, reason); + this.#uncache(entry, true); + }, + shed: (entry, debt, cap) => hooks.shedGroup(entry.group, debt, cap), + settle: () => this.#settle(), + }); } static { + hooks.poolTrack = (producer, pool) => { + if (!producer.#account.empty) throw new Error("track already caches groups"); + producer.#account = producer.#openAccount(pool); + }; hooks.subscribing = (producer) => producer.#subscribing; hooks.requested = (producer) => producer.#state.update; resubscribe = (producer) => producer.#updateSubscription(); @@ -1178,6 +1189,8 @@ export class Producer { return undefined; } hooks.arriveGroup(mirror, entry.arrived); + // A cache hit is an access: the group outranks those nobody read. + entry.charge.refresh(); return mirror; }; reofferCached = (producer, sink, start, end, offer) => { @@ -1461,6 +1474,7 @@ export class Producer { // A group its recovery moved to another copy no longer reads this one. if (hooks.readsMirrored(mirror)) hooks.releaseGroupReader(mirror, abort); entry.mirrors.delete(sink); + entry.charge.sub(cache.MIRROR_OVERHEAD); } } for (const group of sink.groups.peek()) hooks.releaseGroupReader(group, abort); @@ -1468,7 +1482,8 @@ export class Producer { }); } - this.#prune(); + // Full, unlike a write's bounded scan, so the subscriber gets no group already idled out. + this.#evictExpired(true); for (const entry of this.#cache) this.#mirror(entry, sink); sink.drops.set(this.#state.drops.peek()); sink.judged = this.#state.judged; @@ -1506,6 +1521,7 @@ export class Producer { const dst = entry.group.mirror(); hooks.arriveGroup(dst, entry.arrived); if (entry.unverified) hooks.unverifyGroup(dst); + if (!entry.mirrors.has(sink)) entry.charge.bill(cache.MIRROR_OVERHEAD); entry.mirrors.set(sink, dst); if (sink.reads?.(dst.sequence) === false) hooks.groupReads(dst, false); timelineInsert(sink.timeline, dst); @@ -1515,18 +1531,20 @@ export class Producer { sink.groups.mutate((groups) => bufferInsert(groups, dst)); } - // Drop a cached group's mirror from every sink so no consumer can pin it. - #evict(entry: CachedGroup): void { + // Drop a cached group's mirror from every sink so no consumer can pin it, the cache pool + // having reclaimed it for its bytes (`evicted`) or for idling (`expired`). + #evict(entry: CachedGroup, reason: "evicted" | "expired" = "expired"): void { // Reclaiming a group the publisher never closed ends it as the gap it is, before // the mirrors below would otherwise report a clean finish to a reader that had // drained it. The usual case, an already-closed group aging out, keeps its own // terminal state. - if (!entry.group.isClosed) { - const evicted = new TooFarBehind(); - EVICTED.add(evicted); - entry.group.abort(evicted); - } - const mirrors = [...entry.mirrors.values()].filter((mirror) => hooks.evictGroup(mirror)); + const lost = + reason === "evicted" + ? new StreamError(StreamCode.Evicted, { message: "group evicted from the cache" }) + : new TooFarBehind(); + EVICTED.add(lost); + if (!entry.group.isClosed) entry.group.abort(lost); + const mirrors = [...entry.mirrors.values()].filter((mirror) => hooks.evictGroup(mirror, lost)); this.#unlink(entry); for (const mirror of mirrors) mirror.close(); } @@ -1542,14 +1560,17 @@ export class Producer { const index = timelineIndex(sink.timeline, mirror.sequence); if (sink.timeline[index] === mirror) sink.timeline.splice(index, 1); } + entry.charge.sub(entry.mirrors.size * cache.MIRROR_OVERHEAD); entry.mirrors.clear(); } - // Take a cached group out of the cache lookups. + // Take a cached group out of the cache lookups, releasing what it billed the pool. #uncache(entry: CachedGroup, evicted = false): void { this.#cache.splice(this.#cache.indexOf(entry), 1); this.#cached.delete(entry.group.sequence); this.#unbound(entry, evicted); + this.#account.remove(entry); + hooks.chargeGroup(entry.group, undefined); } // The starts a first frame of group `sequence` must fall between: the nearest checked @@ -1600,14 +1621,21 @@ export class Producer { } } + // The cached group at the newest sequence written, if still cached. + #edgeEntry(): CachedGroup | undefined { + const latest = this.#state.latest; + return latest === undefined ? undefined : this.#cached.get(latest); + } + // The newest group remains replayable while the track is live, even after the group // closes: a catalog can publish one snapshot for the broadcast's whole lifetime. An // aborted one holds nothing worth replaying, so it ages out like any other. - #edge(): GroupProducer | undefined { - const latest = this.#state.latest; - if (latest === undefined) return undefined; - const group = this.#cached.get(latest)?.group; - return group?.closed.peek() instanceof Error ? undefined : group; + #protects(entry: CachedGroup): boolean { + return ( + this.#state.closed.peek() === undefined && + entry.group.sequence === this.#state.latest && + !(entry.group.closed.peek() instanceof Error) + ); } // The newest frame in the cache, as {@link Subscriber.largest} reads it from a sink: @@ -1647,86 +1675,33 @@ export class Producer { return this.#state.largest.peek(); } - // Evict cached groups idle for longer than the cache window. Idle means nothing - // written, so an abandoned open group ages out instead of pinning its buffer (and - // any reader parked in it) forever. - // - // Scans at most once per slice of the window, so a track publishing faster than that - // evicts a run of groups per scan instead of scanning everything to evict one per - // write. A group can outlive the window by up to one slice. - #prune(): void { - const at = now(); - const slice = CACHE_WINDOW / PRUNE_SLICES; - if (at < this.#pruned + slice) { - // Something may have come due since the last scan, so make sure another follows. - this.#wake(this.#pruned + slice); - return; - } - this.#pruned = at; - - const cutoff = at - CACHE_WINDOW; - const edge = this.#edge(); - const live = this.#state.closed.peek() === undefined; - - let oldest: number | undefined; - const retained: CachedGroup[] = []; - for (const entry of this.#cache) { - // Once the track closes, the former edge ages from the close instead, so close() - // never reclaims a group its subscribers have yet to drain. - const activity = - entry.group === edge ? Math.max(entry.group.activity, this.#closedAt) : entry.group.activity; - if (entry.group === edge && live) { - retained.push(entry); - } else if (activity >= cutoff) { - retained.push(entry); - if (oldest === undefined || activity < oldest) oldest = activity; - } else { - this.#cached.delete(entry.group.sequence); - this.#unbound(entry, true); - this.#evict(entry); - } - } - const evicted = retained.length < this.#cache.length; - this.#cache = retained; - // An evicted group below the end no longer holds it open, as in Rust. - if (evicted) this.#settle(); - - // Replace the wakeup with one for the next entry due to age out. Writes settle - // retention inline, so this only has to cover the case no write follows. Cheap to - // over-arm: an entry written since is retained and re-armed. - this.#pruneTimer?.abort(); - this.#pruneTimer = undefined; - if (oldest !== undefined) this.#wake(Math.max(oldest + CACHE_WINDOW, at + slice)); + // Accrue and pay the cache pool's eviction debt for what this track wrote. + #chargeDebt(): void { + if (this.#account.chargeDebt()) this.#settle(); } - // Arm the prune wakeup for `at`, unless one is already armed sooner. Kept after a close, - // until the cache empties, so what the track left behind still ages out. - #wake(at: number): void { - if (this.#pruneTimer !== undefined && this.#pruneTimerAt <= at) return; - this.#pruneTimer?.abort(); - - const timer = new AbortController(); - // A cache prune is never a reason to hold a Node/Bun process open. - void sleepUntil(Instant(at), { signal: timer.signal, ref: false }).then( - () => { - // The wakeup can already be queued when a sooner one replaces it. - if (timer.signal.aborted) return; - this.#pruneTimer = undefined; - this.#prune(); - }, - () => {}, - ); - this.#pruneTimer = timer; - this.#pruneTimerAt = at; + // Evict the groups idle past the pool's window: every one when `full`, else a bounded scan. + #evictExpired(full: boolean): void { + if (this.#account.evictExpired(full)) this.#settle(); } // Retain a source group and fan it out to every live sink; its frame 0 is checked // against this track when it lands. #publish(group: GroupProducer): void { + // Before the group is cached, so the write creating it can never evict it. + this.#chargeDebt(); this.#settle(); - const entry: CachedGroup = { group, mirrors: new Map(), arrived: now() }; + const entry: CachedGroup = { + group, + charge: this.#account.charge(), + mirrors: new Map(), + arrived: now(), + }; + hooks.chargeGroup(group, entry.charge); this.#undrop(group.sequence); group.closed.then((closed) => { + // An aborted live edge is no longer protected, so it may expire like any other. + if (closed instanceof Error) this.#account.demote(entry); if (!(closed instanceof Error) || HOLES.has(toStreamCode(closed))) return; // A re-created group already replaced this one. const current = this.#cached.get(group.sequence); @@ -1748,7 +1723,11 @@ export class Producer { this.#advance(group.sequence, true); this.#arrive(group.sequence); this.#produce(group.sequence); + // A newer group demotes the live edge to an evictable one. + const edge = this.#edgeEntry(); this.#state.latest = max(this.#state.latest ?? 0n, group.sequence); + if (edge && edge !== entry && !this.#protects(edge)) this.#account.demote(edge); + this.#account.insert(entry); for (const sink of this.#sinks) this.#mirror(entry, sink); // Follow the group's frames as they land, past the head `#advance` already named. const stop = hooks.groupProgress(group, (total) => { @@ -1765,7 +1744,7 @@ export class Producer { }); // Give held mirrors the new live edge before pruning their timeline entry, // so their latency guard can preserve a terminal expiry verdict. - this.#prune(); + this.#evictExpired(false); } // Record a written sequence. It raises the largest too, at the head of an empty group, so @@ -2194,8 +2173,14 @@ export class Producer { closeTrackState(sink, abort); } this.#sinks.clear(); - this.#closedAt = now(); - this.#prune(); + // The former live edge becomes evictable too, its idle window restarted, so the close + // never reclaims a group its subscribers have yet to drain. + const edge = this.#edgeEntry(); + if (edge) { + edge.charge.recordWrite(); + this.#account.demote(edge); + } + this.#evictExpired(true); } // Whether the declared end was reached, the pending tail owes nothing below it, and every diff --git a/quest/m1/README.md b/quest/m1/README.md index d3032313f7a..32e19d8deb4 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -31,7 +31,6 @@ moves what remains of them once their claims merge. ## Required -- [JS cache pool](/quest/m1/js-cache-pool.md) - `js/net` bills cached frames and drop-set runs to a per-broadcast byte pool ported from Rust - [Drop set](/quest/m1/drop-set.md) - the drop set's track and cursor runs are billed to the cache pool, so fragmenting drops costs what the pool holds; eviction folds final runs or ends the cursor with `TooFarBehind` - [Media fuzz findings](/quest/m1/fuzz-findings/README.md) - the nightly media fuzz targets run clean: mp4-atom, MKV block, TS step, SPS overflow, and catalog encoder bugs they found are fixed - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate diff --git a/quest/m1/drop-set.md b/quest/m1/drop-set.md index 3e4e3e60843..e32605f983b 100644 --- a/quest/m1/drop-set.md +++ b/quest/m1/drop-set.md @@ -36,9 +36,9 @@ setting. Reason: the 2026-11-02 audit found the existing 32 MiB limit in `js/net/src/group.ts` bounds only one group's frame tail, and a mock-time probe retained 64, 256, and 1,024 disjoint OLD runs with zero cached groups; a shared pool bounds drop-only tracks the same way in both languages. Not -chosen: a per-track run cap; upstream backpressure. The pool port itself is -[JS cache pool](/quest/m1/js-cache-pool.md), split out on 2026-11-03 since it -lands on its own; this quest bills the runs to it. +chosen: a per-track run cap; upstream backpressure. The pool port itself +landed on its own as `js/net/src/cache.ts` (split out on 2026-11-03), billing +cached groups; this quest bills the runs to it. Decided 2026-11-03 (maintainer, /quest-unblock), renamed and narrowed by the `/quest-complete` decisions below: range terms carry an @@ -280,7 +280,7 @@ starts. sequence order. - **Billed to the pool, never guessed.** Each track run and each cursor run is billed through a `Charge` at a fixed per-run cost (in JS, through the - [JS cache pool](/quest/m1/js-cache-pool.md)). Under pressure the pool + pool in `js/net/src/cache.ts`). Under pressure the pool evicts the oldest track runs as it evicts groups, and a run older than the cache's retention window is forgotten. An evicted track run's sequences are unknown again: a reader judges them by `drifted` or fills them by range. @@ -351,10 +351,6 @@ Tests, on mock time through a relay, in Rust and mirrored in `js/net`: Public API: none. Wire: none. -## Required - -- [JS cache pool](/quest/m1/js-cache-pool.md) - the byte account the JS runs bill - ## Related - [Subscribe ranges](/quest/m2/subscribe-ranges/README.md) - fills the cache miss an evicted run leaves diff --git a/quest/m1/js-cache-pool.md b/quest/m1/js-cache-pool.md deleted file mode 100644 index b9bab0d69c1..00000000000 --- a/quest/m1/js-cache-pool.md +++ /dev/null @@ -1,52 +0,0 @@ -# [L] JS cache pool - -## Goal - -`js/net` bounds what a broadcast caches in bytes, as Rust does: a port of -Rust's per-broadcast `cache::Pool` (`rs/moq-net/src/model/cache.rs`) bills -every cached group's frames and fixed bookkeeping, and the drop set's track -and cursor runs at a fixed cost per run, and evicts under it the way Rust -does. Rust and JS mirror each other under Rust's default budget, with no new -setting. - -## Plan - -Decided 2026-11-03 (maintainer, /quest-unblock, recorded in -[Drop set](/quest/m1/drop-set.md)): port Rust's per-broadcast `cache::Pool` -to JS. Shared byte accounting per broadcast bills frames and drop-set runs (a -fixed cost per run), under Rust's default budget, with eviction and the -fold-into-floor or `TooFarBehind` behavior the drop set plans. Not chosen: a -per-track run cap; upstream backpressure. Split out of the drop set on -2026-11-03, since the pool lands on its own: its first consumer is the group -cache, and the drop set's run billing follows on top of it. - -What JS has today: `js/net/src/group.ts` caps each group's frame tail at -`TAIL_BYTES` (32 MiB) and evicts that group's oldest frames past it, while -the track cache (`#prune` in `js/net/src/track.ts`) is purely time-based on -`CACHE_EXPIRY` (30 s). Nothing bills bytes across groups or tracks, and -`TrackState.drops` and each cursor's runs grow without bound under a peer -that fragments them. - -Port, mirroring Rust's names and shape: - -- A pool per broadcast, defaulting as `broadcast::Info::default` does in - Rust: no byte capacity and `DEFAULT_EXPIRY` (30 s) idle expiry. -- A per-track account and a per-group charge billing payload plus a fixed - per-group overhead, and a fixed per-run cost for drop-set runs. -- Write-time eviction debt: a track over capacity pays by evicting its own - oldest groups with EVICTED, the latest live group protected, and the - access-time mean ordering eviction across tracks. -- The idle-expiry sweep on the pool's clock, replacing `#prune`'s own timer. - -Test on mock time: a capacity bounds cached groups across tracks of one -broadcast, evicting the oldest first and sparing a fresher track; idle groups -expire; a drop-only flood with no cached groups stays within the pool once -[Drop set](/quest/m1/drop-set.md) bills its runs; the per-group frame-tail -limit still applies. - -Public API: none beyond what Rust already exposes, mirrored if a JS consumer -needs it. Wire: none. - -## Related - -- [Drop set](/quest/m1/drop-set.md) - bills its track and cursor runs to this pool From 30994dc882c0ceff0b7a8d3dae6a27caa357c141 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 10:57:23 -0800 Subject: [PATCH 3/8] fix(net): point the relay-fill harness and track bench at cache.ts's expiry Co-Authored-By: Claude Opus 5.5 --- js/net/bench/track.ts | 7 +++---- test/relay-fill/relay.ts | 7 ++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/js/net/bench/track.ts b/js/net/bench/track.ts index 9fe3d8d3b87..ce7d312ebc7 100644 --- a/js/net/bench/track.ts +++ b/js/net/bench/track.ts @@ -1,6 +1,7 @@ /** Sweep retained groups and subscribers for one group published through a track and received. */ import { Milli } from "@moq/clock"; +import { DEFAULT_EXPIRY } from "../src/cache.ts"; import type { Consumer as GroupConsumer, Producer as GroupProducer } from "../src/group.ts"; import { Timescale, Timestamp } from "../src/time.ts"; import { Producer, type Subscriber } from "../src/track.ts"; @@ -16,8 +17,6 @@ const holds = [ ] as const; type Hold = (typeof holds)[number]; const groups = 100; -// Matches the private idle window and Rust cache::DEFAULT_EXPIRY. -const CACHE_WINDOW_MS = 30_000; const reps = 7; // Publishing a group must not cost more as the retained window grows, or a guard or cache pass // has gone back to scanning every retained group. The sweep spans 60x, so a scan lands well past @@ -61,7 +60,7 @@ async function measure( let sequence = 0; // Inserted by sequence, the way the wire hands a subscribed track its groups. const publish = (close: boolean) => { - clock += held > 0 ? 1 : CACHE_WINDOW_MS / retained; + clock += held > 0 ? 1 : DEFAULT_EXPIRY / retained; const group = producer.createGroup(BigInt(sequence)); group.writeFrame({ payload, timestamp: Timestamp.fromMillis(sequence++) }); if (close) group.finish(); @@ -114,7 +113,7 @@ async function measure( producer.finish(); producer.close(); // Ending the track releases the newest group too; idle cleanup reclaims the cache. - clock += CACHE_WINDOW_MS + CACHE_WINDOW_MS / 8 + 1; + clock += 2 * DEFAULT_EXPIRY; const idleStart = now(); const replay = producer.subscribe({ delay: { max: window } }); if (replay.tryRecvGroup()) throw new Error("idle cache entries were retained"); diff --git a/test/relay-fill/relay.ts b/test/relay-fill/relay.ts index a1b5123c682..81cdaad706c 100644 --- a/test/relay-fill/relay.ts +++ b/test/relay-fill/relay.ts @@ -9,8 +9,8 @@ import assert from "node:assert/strict"; import { createInterface } from "node:readline"; import { Manual, Milli, Nano } from "@moq/clock"; import Session from "@moq/qmux"; +import { DEFAULT_EXPIRY } from "../../js/net/src/cache.ts"; import { connect } from "../../js/net/src/connection/connect.ts"; -import { CACHE_EXPIRY } from "../../js/net/src/ledger.ts"; import { Producer } from "../../js/net/src/origin.ts"; import * as Path from "../../js/net/src/path.ts"; import { wireOf } from "../../js/net/src/wire.ts"; @@ -71,8 +71,9 @@ try { continue; } assert.equal(line, "evict", `unknown command: ${line}`); - // A prune slice past the cache window, so the next scan evicts what nobody touched. - await clock.advance(Nano.fromMilli(Milli(CACHE_EXPIRY + 10_000))); + // Past the cache window and the sweep that trails it by up to half a window, so the + // copy has evicted what nobody touched. + await clock.advance(Nano.fromMilli(Milli(2 * DEFAULT_EXPIRY))); console.log("evicted"); } } finally { From e46ff983584c57031de0d6b218d443ae31172f27 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 11:06:54 -0800 Subject: [PATCH 4/8] perf(net): evict a run of cached groups in linear time The track's cache was an array searched and spliced per eviction, so a sweep expiring N groups shifted the retained tail N times. An insertion-ordered set removes each in constant time. Co-Authored-By: Claude Opus 5.5 --- js/net/src/track.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 2315d29985d..e9b134d78bc 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -990,11 +990,12 @@ export class Producer { // must not see them. The wire's aggregate, `#state.update`, keeps them. #live = new Signal(undefined); - // Recently written source groups, retained for replay to late subscribers until the cache - // pool evicts them, for its budget or once idle. Each entry tracks the mirror - // it handed to every sink so eviction can drop them too: otherwise a slow consumer - // that never reads would pin old groups (and their frame bytes) forever. - #cache: CachedGroup[] = []; + // Recently written source groups, in arrival order, retained for replay to late subscribers + // until the cache pool evicts them, for its budget or once idle. A set, so evicting a run of + // them is linear. Each entry tracks the mirror it handed to every sink so eviction can drop + // them too: otherwise a slow consumer that never reads would pin old groups (and their frame + // bytes) forever. + #cache = new Set(); // The same entries by sequence, so a write finds a duplicate without a scan. #cached = new Map(); // The cached groups' checked starts, sorted by sequence, so a first frame finds its @@ -1566,7 +1567,7 @@ export class Producer { // Take a cached group out of the cache lookups, releasing what it billed the pool. #uncache(entry: CachedGroup, evicted = false): void { - this.#cache.splice(this.#cache.indexOf(entry), 1); + this.#cache.delete(entry); this.#cached.delete(entry.group.sequence); this.#unbound(entry, evicted); this.#account.remove(entry); @@ -1717,7 +1718,7 @@ export class Producer { return () => this.#record(entry, start, late); }); } - this.#cache.push(entry); + this.#cache.add(entry); this.#cached.set(group.sequence, entry); this.#fills.arrived(group.sequence); this.#advance(group.sequence, true); @@ -2155,7 +2156,7 @@ export class Producer { if (abort && (this.#state.outcome.peek() === null || this.#settled())) abort = undefined; // Nobody will finish these, so a subscriber that has not taken one yet never sees it. // Not evicted: a reader already holding one keeps its frames and sees the abort. - const open = this.#cache.filter((entry) => entry.group.closed.peek() === undefined); + const open = [...this.#cache].filter((entry) => entry.group.closed.peek() === undefined); closeTrackState(this.#state, abort); if (this.#state.outcome.peek() === undefined) this.#state.outcome.set(abort ?? this.#state.endError ?? null); // Closing a copy never supplies the missing tail of a group. Already finished groups @@ -2192,7 +2193,8 @@ export class Producer { if (end === undefined) return false; const final = finalSequence(end); if (!reaches(this.#received, this.#state.drops.peek(), final) || this.#owes(final)) return false; - return this.#cache.every(({ group }) => group.sequence >= final || group.closed.peek() === null); + for (const { group } of this.#cache) if (group.sequence < final && group.closed.peek() !== null) return false; + return true; } /** Append a frame as its own single-frame group. */ From b5f1e1141f4a9118cb4dacf1c8e3aaccff157ce5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 11:15:11 -0800 Subject: [PATCH 5/8] feat(net): taking a cached group restarts its idle window, as in Rust Rust stamps a cache access when a subscriber takes a group, not only when it reads a frame. Mirror it from the guard every handed-out group passes through, and port `delivery_restarts_the_expiry_clock`. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 21 +++++++++++++++++++++ js/net/src/group.ts | 1 + js/net/src/internal.ts | 5 +++++ js/net/src/track.test.ts | 7 ++++--- js/net/src/track.ts | 4 +++- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 9c311dec6ec..7885e62b300 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -143,6 +143,27 @@ test("a read group outlives a later unread one", async () => { }); }); +// Taking a group is a cache access: a reader handed one just inside the window can still read +// it almost a window later. Mirrors Rust `delivery_restarts_the_expiry_clock`. +test("delivery restarts a group's idle window", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(10_000) }); + const producer = pooled(pool); + const subscriber = producer.subscribe({ delay: { max: null } }); + finished(producer, 1); // seq 0 + finished(producer, 1); // seq 1 takes the live edge, so seq 0 can expire + + await clock.advance(9_000); + const reading = subscriber.tryRecvGroup(); + expect(reading?.sequence).toBe(0n); + // Far beyond the write, inside the delivery. The new group runs the expiry scan. + await clock.advance(9_000); + finished(producer, 1); + expect((await reading?.readFrame())?.payload.length).toBe(1); + subscriber.close(); + }); +}); + // Every track of a broadcast caches into one pool, and the mean access across all of them // orders eviction: a track writing past the capacity pays with its own groups, but only those // accessed no later than the mean, so a fresher track's groups survive its own debt while a diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 0d9f8a2c74e..9563223d955 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -1270,6 +1270,7 @@ export class Consumer { }; hooks.readGroupFrame = (group, compressed, from = 0) => group.#readFrame(from, compressed, true); hooks.evictGroup = (group, lost) => group.#evict(lost); + hooks.refreshGroup = (group) => group.#state.charge?.refresh(); hooks.onShedRead = (group, notify) => { group.#state.shedRead = notify; }; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 5c8e0682095..be243f5ca2f 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -836,6 +836,8 @@ export const hooks: { * on, or may yet, from another copy. */ evictGroup: (group: GroupConsumer, lost: Error) => boolean; + /** Record a cache access on a group a reader takes, restarting its idle window. */ + refreshGroup: (group: GroupConsumer) => void; /** Call `fn` synchronously when an open broadcast closes, from any handle. */ onClosed: (producer: BroadcastProducer, fn: () => void) => Dispose; /** Reserve an on-demand track without listing it or enabling a request handler. */ @@ -1171,6 +1173,9 @@ export const hooks: { evictGroup: () => { throw new Error("group.ts not loaded"); }, + refreshGroup: () => { + throw new Error("group.ts not loaded"); + }, onClosed: () => { throw new Error("broadcast.ts not loaded"); }, diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index ade6ba81f9c..f7d33a77a94 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2923,14 +2923,15 @@ test("no publisher limit still ages idle groups out", async () => { expect(await (await within.recvGroup())?.readString()).toBe("old"); within.close(); - // The read was an access, as in Rust: past the write's window, the group stays. + // The read was an access, as in Rust: past the write's window, the group stays. Taking it + // again is another access. clock.set(10_000 + CACHE_WINDOW_MS + 1_000); const read = producer.subscribe({ delay: { max: Milli(100_000) } }); expect(read.tryRecvGroup()?.sequence).toBe(0n); read.close(); - // Past the read's window, a subscribe's own scan reclaims it. - clock.set(10_000 + 2 * CACHE_WINDOW_MS + 1_000); + // Past that delivery's window, a subscribe's own scan reclaims it. + clock.set(10_000 + 2 * CACHE_WINDOW_MS + 2_000); const after = producer.subscribe({ delay: { max: Milli(100_000) } }); expect(after.tryRecvGroup()?.sequence).toBe(1n); expect(after.tryRecvGroup()).toBeUndefined(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index e9b134d78bc..0957ce0e069 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2522,8 +2522,10 @@ export class Subscriber { } #guard(group: GroupConsumer): GroupConsumer { - // Handed out, it is read, whatever this reader judged it while buffered. + // Handed out, it is read, whatever this reader judged it while buffered. Taking it is the + // cache access buffering it is not, as in Rust: it restarts the group's idle window. hooks.groupReads(group, true); + hooks.refreshGroup(group); const held: Superseded = {}; hooks.onShedRead(group, () => { if (this.#state.latest === group.sequence && this.#state.producer) hooks.joinProducer(this.#state.producer); From e93ce5c38a86e038edfe9e1f0cb854feea6f347d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 16:03:32 -0800 Subject: [PATCH 6/8] fix(net): a debt shed keeps what readers still need, and pays in constant time An open group keeps its newest committed frame behind one still being written, as Rust's `GroupState::shed` does. Shedding a finished group ends the copies its track holds that had not taken the shed frames, since a finished group no longer tees into them. The account's eviction order reads from a head index, because `Array.shift` moves the whole array in V8. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.ts | 35 +++++++++++++++++++++-------------- js/net/src/group.test.ts | 31 +++++++++++++++++++++++++++++++ js/net/src/group.ts | 24 ++++++++++++++++++------ js/net/src/internal.ts | 7 +++++-- js/net/src/track.ts | 2 +- 5 files changed, 76 insertions(+), 23 deletions(-) diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index 32ff64c8d3a..d5ec6b79f8d 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -238,9 +238,11 @@ export class Track { // The tick a frame write may next run an idle scan at. #nextExpiry = 0; // Every cached group, and the eviction order, oldest first, which keeps hints of groups no - // longer cached until a scan reaches them. + // longer cached until a scan reaches them. The order starts at `#head`, since `Array.shift` + // moves every entry behind it in V8. #live = new Set(); #order: E[] = []; + #head = 0; // Debt carried over from earlier payments. #debt = 0; // Where the next bounded idle scan starts, rotating so fresh groups at the front do not @@ -287,6 +289,7 @@ export class Track { entry.charge.clear(); if (this.#live.size === 0) { this.#order = []; + this.#head = 0; this.#debt = 0; this.pool.unregister(this as Track); } @@ -318,7 +321,7 @@ export class Track { // at the front reads as not stale until a scan drops it. #oldestIsStale(): boolean { const average = this.pool.average(); - const front = this.#order[0]; + const front = this.#order[this.#head]; if (average === undefined || front === undefined || !this.#live.has(front)) return false; return !this.#owner.protects(front) && front.charge.accessed <= average; } @@ -334,10 +337,11 @@ export class Track { let scanned = 0; let evicted = false; const fresh: E[] = []; - for (let i = this.#order.length; i > 0; i--) { + for (let i = this.#order.length - this.#head; i > 0; i--) { if (this.#debt === 0 || paid >= cap || scanned >= EVICT_SCAN) break; - const entry = this.#order.shift(); - if (entry === undefined) break; + // An eviction that emptied the account reset the order. + if (this.#head >= this.#order.length) break; + const entry = this.#order[this.#head++]; if (!this.#live.has(entry)) continue; if (this.#owner.protects(entry)) { this.#order.push(entry); @@ -351,7 +355,9 @@ export class Track { } const size = entry.charge.size; if (size > this.#debt) { - this.#order.unshift(entry); + // Its slot still holds it. + this.#head--; + this.#compact(); return evicted; } this.#debt -= size; @@ -368,6 +374,7 @@ export class Track { this.#debt -= Math.min(freed, this.#debt); paid += freed; } + this.#compact(); return evicted; } @@ -378,7 +385,8 @@ export class Track { evictExpired(full: boolean): boolean { const expiry = this.pool.expiryTicks; const order = this.#order; - const len = order.length; + const head = this.#head; + const len = order.length - head; if (expiry === undefined || len === 0) return false; const now = this.pool.now(); const start = full ? 0 : this.#cursor % len; @@ -386,7 +394,7 @@ export class Track { if (!full) this.#cursor = start + width; const expired: E[] = []; for (let step = 0; step < width; step++) { - const entry = order[(start + step) % len]; + const entry = order[head + ((start + step) % len)]; if (!this.#live.has(entry) || this.#owner.protects(entry)) continue; if (now - Math.floor(entry.charge.accessed / BOOSTS) > expiry) expired.push(entry); } @@ -395,15 +403,14 @@ export class Track { return expired.length > 0; } - // Drop the dead entries at the front of the order, and rebuild it once dead entries behind - // a live front clearly outnumber the live ones. + // Skip the dead entries at the front of the order, and rebuild it once the slots before the + // front and the dead entries behind it clearly outnumber the live ones. #compact(): void { const order = this.#order; - let dead = 0; - while (dead < order.length && !this.#live.has(order[dead])) dead++; - if (dead > 0) order.splice(0, dead); + while (this.#head < order.length && !this.#live.has(order[this.#head])) this.#head++; if (order.length > 2 * this.#live.size + EVICT_SLACK) { - this.#order = order.filter((entry) => this.#live.has(entry)); + this.#order = order.slice(this.#head).filter((entry) => this.#live.has(entry)); + this.#head = 0; } } diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index db9efd788c3..d4d99aa3b4e 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -957,3 +957,34 @@ test.each(["eviction", "expiry"] as const)("a group end wait observes reader %s" source.finish(); reader.close(); }); + +// Rust's `GroupState::shed` keeps an open group's newest committed frame even while a newer +// frame is still being written, since that partial frame is not readable yet. +test("a debt shed keeps the newest committed frame behind one being written", async () => { + const producer = new Producer(0n); + producer.writeFrame({ payload: new Uint8Array(1024) }); + const reader = producer.consume(); + producer.createFrame({ size: 10 }).write(new Uint8Array(5)); + + expect(hooks.shedGroup(producer, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, [])).toBe(0); + expect((await reader.readFrame())?.payload.byteLength).toBe(1024); +}); + +// A finished group no longer tees into its copies, so shedding it ends the copies its track +// holds that had not taken the shed frames, rather than crediting bytes they still hold. +test("a debt shed of a finished group ends the copies that lost a frame", async () => { + const producer = new Producer(0n); + producer.writeFrame({ payload: new Uint8Array(1024) }); + producer.writeFrame({ payload: new Uint8Array(1024) }); + producer.finish(); + const behind = producer.consume(); + const drained = producer.consume(); + await drained.readFrame(); + await drained.readFrame(); + + expect( + hooks.shedGroup(producer, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, [behind, drained]), + ).toBeGreaterThan(0); + await expect(behind.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); + expect(await drained.readFrame()).toBeUndefined(); +}); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 9563223d955..dd354f2b8a2 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -438,7 +438,7 @@ export class Producer { state.charge = charge; charge?.bill(state.cacheBytes); }; - hooks.shedGroup = (group, debt, cap) => group.#shed(debt, cap); + hooks.shedGroup = (group, debt, cap, readers) => group.#shed(debt, cap, readers); hooks.limitGroup = (group, frames) => { // A closed group takes no more frames, but one already past the cap still // contradicts the end. @@ -695,13 +695,18 @@ export class Producer { // Shed whole frames from the front toward the cache pool's `debt`, freeing about `cap` at // most, for a group the debt cannot evict whole: one being read, or the live edge. A frame // larger than what is left of the debt stays, with everything after it. An open group keeps - // its newest frame, for the reader about to take it. Returns the bytes freed. Mirrors Rust's - // `GroupState::shed`. - #shed(debt: number, cap: number): number { + // its newest committed frame, for the reader about to take it, and any frame still being + // written. A finished group no longer tees into its mirrors, so it ends `readers`, the copies + // its track still holds, that had yet to take a shed frame. Returns the bytes freed. Mirrors + // Rust's `GroupState::shed`. + #shed(debt: number, cap: number, readers: Iterable): number { const state = this.#state; - if (state.closed.peek() instanceof Error) return 0; + const closed = state.closed.peek(); + if (closed instanceof Error) return 0; const frames = state.frames.peek(); - const keep = state.closed.peek() === undefined ? 1 : 0; + const last = frames.last; + const writing = last instanceof Chunked && last.done.peek() === undefined; + const keep = closed === null ? 0 : writing ? 2 : 1; const bytes = state.cacheBytes; while (frames.length > keep && bytes - state.cacheBytes < cap) { const front = frames.first; @@ -710,6 +715,7 @@ export class Producer { this.#shift(); } this.#shedMirrors(bytes); + if (closed === null) for (const reader of readers) shedReader(reader, state.start); return bytes - state.cacheBytes; } @@ -956,6 +962,8 @@ export class FrameConsumer { } let makeConsumer: (state: GroupState) => Consumer; +// End a track-held copy of a finished group that had yet to take frame `start - 1`, now shed. +let shedReader: (group: Consumer, start: number) => void; // A replacement copy asked for on behalf of one consumer, settled or still out. interface Attempt { @@ -1270,6 +1278,10 @@ export class Consumer { }; hooks.readGroupFrame = (group, compressed, from = 0) => group.#readFrame(from, compressed, true); hooks.evictGroup = (group, lost) => group.#evict(lost); + shedReader = (group, start) => { + // A copy that took every shed frame keeps reading; one that had not lost them. + if (group.#state === group.#mirrored && group.#state.start < start) group.#evict(evicted()); + }; hooks.refreshGroup = (group) => group.#state.charge?.refresh(); hooks.onShedRead = (group, notify) => { group.#state.shedRead = notify; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index be243f5ca2f..6c2de73d94d 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -774,8 +774,11 @@ export const hooks: { groupCacheBytes: (group: GroupProducer) => number; /** Bill a cached group's bytes to `charge` from now on, or stop billing them with undefined. */ chargeGroup: (group: GroupProducer, charge: Charge | undefined) => void; - /** Shed whole frames from a group's front toward a cache `debt`, freeing about `cap` at most; returns the bytes freed. */ - shedGroup: (group: GroupProducer, debt: number, cap: number) => number; + /** + * Shed whole frames from a group's front toward a cache `debt`, freeing about `cap` at most, + * and end the `readers` its track holds that lost a shed frame; returns the bytes freed. + */ + shedGroup: (group: GroupProducer, debt: number, cap: number, readers: Iterable) => number; /** Frame count without creating a read handle. */ groupFrameCount: (group: GroupProducer) => number; /** diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 0957ce0e069..9fe8c1eff9c 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1083,7 +1083,7 @@ export class Producer { this.#evict(entry, reason); this.#uncache(entry, true); }, - shed: (entry, debt, cap) => hooks.shedGroup(entry.group, debt, cap), + shed: (entry, debt, cap) => hooks.shedGroup(entry.group, debt, cap, entry.mirrors.values()), settle: () => this.#settle(), }); } From 73ab959681746f9e15fd01b1e7b56421766c79b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 16:06:34 -0800 Subject: [PATCH 7/8] fix(net): a group leaving the cache leaves no reference in the eviction order Each cached group holds a slot in its account's order, which removal empties, so neither the slots behind the head index nor the dead hints a scan has yet to reach pin an expired group's frames. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 20 +++++++++++++++ js/net/src/cache.ts | 54 ++++++++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 7885e62b300..702f816b73a 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -274,3 +274,23 @@ test("the debt sheds the front of an open live edge", async () => { await expect(behind?.readFrame()).rejects.toThrow(); }); }); + +// A group leaving the cache leaves nothing in the eviction order holding its frames, so a track +// that stops publishing does not pin what already expired. +test("expired groups are not retained by the eviction order", async () => { + await withClock(async (clock) => { + const pool = new cache.Pool({ expiry: Milli(1_000) }); + const producer = pooled(pool); + const refs: WeakRef[] = []; + for (let i = 0; i < 8; i++) { + const group = producer.appendGroup(); + group.writeFrame({ payload: new Uint8Array(1) }); + group.finish(); + refs.push(new WeakRef(group)); + } + await clock.advance(3_000); + expect(await cached(producer)).toEqual([7n]); + Bun.gc(true); + expect(refs.slice(0, 7).filter((ref) => ref.deref() !== undefined)).toEqual([]); + }); +}); diff --git a/js/net/src/cache.ts b/js/net/src/cache.ts index d5ec6b79f8d..de7b9b80bb1 100644 --- a/js/net/src/cache.ts +++ b/js/net/src/cache.ts @@ -210,6 +210,11 @@ export interface Entry { readonly charge: Charge; } +// A group's place in an account's eviction order, emptied once the group leaves the cache. +interface Slot { + entry: E | undefined; +} + /** How an account reaches the track whose groups it evicts. */ export interface Owner { /** Whether `entry` is the live edge, which neither eviction nor expiry takes. */ @@ -237,11 +242,11 @@ export class Track { #written = 0; // The tick a frame write may next run an idle scan at. #nextExpiry = 0; - // Every cached group, and the eviction order, oldest first, which keeps hints of groups no - // longer cached until a scan reaches them. The order starts at `#head`, since `Array.shift` - // moves every entry behind it in V8. - #live = new Set(); - #order: E[] = []; + // Every cached group with its slot in the eviction order, oldest first. A group leaving the + // cache empties its slot, so the order never pins its frames; a scan drops empty slots. The + // order starts at `#head`, since `Array.shift` moves every slot behind it in V8. + #live = new Map>(); + #order: Slot[] = []; #head = 0; // Debt carried over from earlier payments. #debt = 0; @@ -271,8 +276,9 @@ export class Track { /** Cache `entry` at the back of the eviction order, evictable now unless it is the live edge. */ insert(entry: E): void { - this.#live.add(entry); - this.#order.push(entry); + const slot = { entry }; + this.#live.set(entry, slot); + this.#order.push(slot); if (!this.#owner.protects(entry)) this.demote(entry); } @@ -285,7 +291,10 @@ export class Track { /** Forget `entry`, which left the cache, releasing what it billed. */ remove(entry: E): void { - if (!this.#live.delete(entry)) return; + const slot = this.#live.get(entry); + if (slot === undefined) return; + this.#live.delete(entry); + slot.entry = undefined; entry.charge.clear(); if (this.#live.size === 0) { this.#order = []; @@ -321,8 +330,8 @@ export class Track { // at the front reads as not stale until a scan drops it. #oldestIsStale(): boolean { const average = this.pool.average(); - const front = this.#order[this.#head]; - if (average === undefined || front === undefined || !this.#live.has(front)) return false; + const front = this.#order[this.#head]?.entry; + if (average === undefined || front === undefined) return false; return !this.#owner.protects(front) && front.charge.accessed <= average; } @@ -341,21 +350,22 @@ export class Track { if (this.#debt === 0 || paid >= cap || scanned >= EVICT_SCAN) break; // An eviction that emptied the account reset the order. if (this.#head >= this.#order.length) break; - const entry = this.#order[this.#head++]; - if (!this.#live.has(entry)) continue; + const slot = this.#order[this.#head++]; + const entry = slot.entry; + if (entry === undefined) continue; if (this.#owner.protects(entry)) { - this.#order.push(entry); + this.#order.push(slot); continue; } scanned++; if (entry.charge.accessed > average) { - this.#order.push(entry); + this.#order.push(slot); fresh.push(entry); continue; } const size = entry.charge.size; if (size > this.#debt) { - // Its slot still holds it. + // Back to the front, where its slot still is. this.#head--; this.#compact(); return evicted; @@ -394,8 +404,8 @@ export class Track { if (!full) this.#cursor = start + width; const expired: E[] = []; for (let step = 0; step < width; step++) { - const entry = order[head + ((start + step) % len)]; - if (!this.#live.has(entry) || this.#owner.protects(entry)) continue; + const entry = order[head + ((start + step) % len)].entry; + if (entry === undefined || this.#owner.protects(entry)) continue; if (now - Math.floor(entry.charge.accessed / BOOSTS) > expiry) expired.push(entry); } for (const entry of expired) this.#owner.evict(entry, "expired"); @@ -403,13 +413,13 @@ export class Track { return expired.length > 0; } - // Skip the dead entries at the front of the order, and rebuild it once the slots before the - // front and the dead entries behind it clearly outnumber the live ones. + // Skip the empty slots at the front of the order, and rebuild it once the slots before the + // front and the empty ones behind it clearly outnumber the live ones. #compact(): void { const order = this.#order; - while (this.#head < order.length && !this.#live.has(order[this.#head])) this.#head++; + while (this.#head < order.length && order[this.#head].entry === undefined) this.#head++; if (order.length > 2 * this.#live.size + EVICT_SLACK) { - this.#order = order.slice(this.#head).filter((entry) => this.#live.has(entry)); + this.#order = order.slice(this.#head).filter((slot) => slot.entry !== undefined); this.#head = 0; } } @@ -431,7 +441,7 @@ export class Track { sweep(): void { if (this.evictExpired(true)) this.#owner.settle(); // A live edge alone never expires, so it needs no sweep to keep its track alive for. - for (const entry of this.#live) if (!this.#owner.protects(entry)) return; + for (const entry of this.#live.keys()) if (!this.#owner.protects(entry)) return; this.pool.unregister(this as Track); } From b351e2401e7c31ecf101c98a2fd6264841fd5e01 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 16:54:21 -0800 Subject: [PATCH 8/8] test(net): drop the GC-dependent eviction-order retention test JSC scans the stack conservatively, so whether a WeakRef clears depends on what ran before it; the test failed when cache.test.ts ran beside bandwidth.test.ts. Co-Authored-By: Claude Opus 5.5 --- js/net/src/cache.test.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 702f816b73a..7885e62b300 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -274,23 +274,3 @@ test("the debt sheds the front of an open live edge", async () => { await expect(behind?.readFrame()).rejects.toThrow(); }); }); - -// A group leaving the cache leaves nothing in the eviction order holding its frames, so a track -// that stops publishing does not pin what already expired. -test("expired groups are not retained by the eviction order", async () => { - await withClock(async (clock) => { - const pool = new cache.Pool({ expiry: Milli(1_000) }); - const producer = pooled(pool); - const refs: WeakRef[] = []; - for (let i = 0; i < 8; i++) { - const group = producer.appendGroup(); - group.writeFrame({ payload: new Uint8Array(1) }); - group.finish(); - refs.push(new WeakRef(group)); - } - await clock.advance(3_000); - expect(await cached(producer)).toEqual([7n]); - Bun.gc(true); - expect(refs.slice(0, 7).filter((ref) => ref.deref() !== undefined)).toEqual([]); - }); -});