diff --git a/bench/README.md b/bench/README.md index 7d28c3cb4d1..e976bc84dc2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -409,12 +409,13 @@ Crashes, wrong bytes, missing frames, a window that presents nothing, or an unavailable CPU clock fail the run; timing never does. The nightly runs `--smoke` and keeps its run directory as the `bench-browser` artifact. -The in-memory frame, retained-history, and gap ledger paths also run directly in Chromium: +The in-memory frame, retained-history, gap ledger, and resume-history paths also run directly in Chromium: ```bash nix develop --command bun bench/browser/micro.ts --bench frames --out .tmp/frames.json base=/path/to/release current=. aa=. nix develop --command bun bench/browser/micro.ts --bench drift --out .tmp/drift.json before=/path/to/main current=. aa=. nix develop --command bun bench/browser/micro.ts --bench ledger --out .tmp/ledger.json before=/path/to/main current=. aa=. +nix develop --command bun bench/browser/micro.ts --bench resume-history --out .tmp/resume-history.json before=/path/to/main current=. aa=. ``` Each variant uses its own workspace source and a persistent page, with one full @@ -432,7 +433,22 @@ Its `time_mode` column is 0 for untimed, 1 for millisecond timestamps, and 2 for mixed millisecond/microsecond timestamps, which exercise exact cross-scale comparison. -The browser smoke lane preserves one measured round of all three fixtures, and drift +`resume-history` moves long-lived logical fronts across lite sessions over the +in-memory transport, make-before-break as a session draining after GOAWAY does. +It sweeps fronts, names read per route, and route count, with `stable` names +(the same ones on every route) against `churn` (new names first answered by each +route), each read by info lookup, fetch, or subscription and then closed. Once a +manual clock ages out every cache, full collections from Chromium's exposed `gc` +let weak references count the retired routes' sessions, transports, broadcast +handles, and delivered payload buffers still reachable. These counts are exact, +so any nonzero one fails the run once every sample is written, leaving a retaining +`before` revision's counts in the JSON. A concluded control must release everything, +or the fixture fails. `size_bytes` is the JS heap the fronts and their serving +route hold, from Chromium's precise memory info; `serve_us` (clamped to the +page's coarse timer) and `read_us` include no setup. Bun's collector scans its +stack conservatively, so the fixture refuses to run there. + +The browser smoke lane preserves one measured round of all four fixtures, and drift and ledger also run in the nightly JS benchmark leg. Timing is observational, with the A/A spread showing the host's noise floor. diff --git a/bench/browser/micro.ts b/bench/browser/micro.ts index 1bf4d138034..7988efd7f79 100644 --- a/bench/browser/micro.ts +++ b/bench/browser/micro.ts @@ -14,8 +14,8 @@ const { values, positionals } = parseArgs({ }, }); const benchmark = values.bench; -if (benchmark !== "frames" && benchmark !== "drift" && benchmark !== "ledger") - throw new Error("--bench must be frames, drift, or ledger"); +const expectedRows = ({ frames: 26, drift: 36, ledger: 12, "resume-history": 72 } as Record)[benchmark]; +if (expectedRows === undefined) throw new Error("--bench must be frames, drift, ledger, or resume-history"); const rounds = Number(values.rounds); if (!Number.isSafeInteger(rounds) || rounds < 1) throw new Error("--rounds must be a positive integer"); const roots = new Map(); @@ -39,11 +39,13 @@ for (const [name, root] of roots) { { name: "repeat-microbenchmark", setup(build) { - // The ledger is internal, so select it explicitly instead of a package export. - build.onResolve({ filter: /^\.\.\/src\/ledger\.ts$/ }, () => ({ - path: resolve(root, "js/net/src/ledger.ts"), - })); - build.onLoad({ filter: /\/bench\/(frames|drift|ledger)\.ts$/ }, async ({ path }) => { + // Internal modules have no package export, so select each variant's own explicitly. + build.onResolve({ filter: /^\.\.\/src\// }, ({ path, importer }) => + importer.endsWith(`/js/net/bench/${benchmark}.ts`) + ? { path: resolve(root, "js/net/bench", path) } + : undefined, + ); + build.onLoad({ filter: /\/bench\/(frames|drift|ledger|resume-history)\.ts$/ }, async ({ path }) => { const source = await Bun.file(path).text(); if (benchmark !== "frames") return { loader: "ts", contents: `${source}\nglobalThis.measure = run;` }; @@ -81,7 +83,12 @@ const server = Bun.serve({ }); }, }); -const browser = await chromium.launch({ headless: true, executablePath: process.env.BROWSER_BIN }); +const browser = await chromium.launch({ + headless: true, + executablePath: process.env.BROWSER_BIN, + // Retention is counted after full collections the page asks for, against an unrounded heap size. + args: benchmark === "resume-history" ? ["--js-flags=--expose-gc", "--enable-precise-memory-info"] : [], +}); const samples: { round: number; variant: string; rows: string[] }[] = []; try { const pages = new Map(); @@ -109,8 +116,7 @@ try { rows.set(variant, []); await pages.get(variant)?.evaluate("globalThis.measure()"); const measured = rows.get(variant); - if (measured?.length !== (benchmark === "frames" ? 26 : benchmark === "drift" ? 36 : 12)) - throw new Error(`incomplete rows: ${variant}`); + if (measured?.length !== expectedRows) throw new Error(`incomplete rows: ${variant}`); samples.push({ round, variant, rows: measured }); await Bun.write( values.out, @@ -123,6 +129,22 @@ try { console.log(`measured ${round} ${variant}`); } } + // Reachability is exact here, so a retired route still reachable fails the run, once every + // variant's samples are written for comparison. + if (benchmark === "resume-history") { + const retaining = samples.filter(({ rows }) => + rows.some((row) => + row + .split(",") + .slice(5, 9) + .some((retired) => retired !== "0"), + ), + ); + if (retaining.length) + throw new Error( + `retired routes stayed reachable: ${[...new Set(retaining.map((s) => s.variant))].join(", ")}`, + ); + } } finally { await browser.close(); server.stop(true); diff --git a/bench/browser/run.sh b/bench/browser/run.sh index 6c16ba98a76..8ac635ddc72 100755 --- a/bench/browser/run.sh +++ b/bench/browser/run.sh @@ -88,7 +88,7 @@ bun install --frozen-lockfile cd "$HERE" # The in-memory paths share Chromium installation and artifact retention with this lane. -for benchmark in frames drift ledger; do +for benchmark in frames drift ledger resume-history; do bun micro.ts --bench "$benchmark" --rounds 1 --out "$HARNESS_RUN/$benchmark.json" \ current="$WORKSPACE" aa="$WORKSPACE" done diff --git a/js/net/bench/resume-history.ts b/js/net/bench/resume-history.ts new file mode 100644 index 00000000000..05e076e711e --- /dev/null +++ b/js/net/bench/resume-history.ts @@ -0,0 +1,301 @@ +/** + * Sweep fronts, first-seen track names, and route replacements for long-lived logical fronts, + * then count what each retired route leaves reachable once its readers closed, its session ended, + * and every cache aged out. + * + * Every route is a real lite session over the in-memory transport, so a retired route that is + * still reachable keeps its session, subscriber, transport, and broadcast handles. `stable` reads + * the same names on every route, so only their first route ever records them; `churn` reads names + * first answered by each route. Each case reads its names one way (info lookup, group fetch, or + * subscription) and closes every reader. A route is replaced make-before-break, as a session + * draining after GOAWAY is: the fronts serve the new route, then the old session ends. Only the + * serving route may stay reachable. + * + * Counts come from weak references after full collections, so they are exact only under a precise + * collector: run it in Chromium with `bench/browser/micro.ts --bench resume-history`, which + * exposes `gc` and fails once a retired route stays reachable. Bun scans its stack conservatively, + * so a dead reference there can survive a collection. + */ +import { Manual, Milli, Nano } from "@moq/clock"; +import type { Producer as BroadcastProducer } from "../src/broadcast.ts"; +import { accept } from "../src/connection/accept.ts"; +import { connect } from "../src/connection/connect.ts"; +import * as Epoch from "../src/epoch.ts"; +import { ALPN_06 } from "../src/lite/version.ts"; +import { createMockTransportPair } from "../src/mock.ts"; +import { Producer as OriginProducer } from "../src/origin.ts"; +import * as Path from "../src/path.ts"; +import { Front } from "../src/resume.ts"; +import { wireOf } from "../src/wire.ts"; + +const PAYLOAD = 4096; +const MAX_DELAY = Milli(1000); +/** Past every cache's age-out, including the 30 s window a closed track's cache drains over. */ +const EXPIRY = Nano.fromMilli(Milli(60_000)); +const url = new URL("https://localhost/resume-history"); +type History = "stable" | "churn"; +type Access = "lookup" | "fetch" | "subscribe"; + +/** Full collections, each from a fresh task so no frame of the caller keeps a target. */ +async function collect(): Promise { + const gc = (globalThis as { gc?: () => void }).gc; + if (!gc) throw new Error("resume-history needs a precise collector: run it with bench/browser/micro.ts"); + for (let pass = 0; pass < 3; pass++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } +} + +/** + * Age out every cache and collect until what is reachable stops changing: teardown runs on host + * tasks and clock timers, and either may arm the other. + */ +async function drain(clock: Manual, probes: Probe[]): Promise { + let last = ""; + for (let round = 0; round < 8; round++) { + await clock.advance(EXPIRY); + await collect(); + const reachable = JSON.stringify(count(probes)); + if (reachable === last) return; + last = reachable; + } + throw new Error(`reachable routes never settled: ${last}`); +} + +/** The JS heap in use, exact under Chromium's `--enable-precise-memory-info`. */ +function heap(): number { + const memory = (performance as { memory?: { usedJSHeapSize: number } }).memory; + if (!memory) throw new Error("resume-history needs performance.memory"); + return memory.usedJSHeapSize; +} + +/** What one route leaves behind, observed only weakly. */ +interface Probe { + session: WeakRef; + transport: WeakRef; + broadcasts: WeakRef[]; + payloads: WeakRef[]; +} + +interface Retained { + sessions: number; + transports: number; + broadcasts: number; + /** Delivered payload backing stores still reachable, each counted once. */ + bytes: number; +} + +function count(probes: Probe[]): Retained { + const buffers = new Set(); + let [sessions, transports, broadcasts] = [0, 0, 0]; + for (const probe of probes) { + if (probe.session.deref()) sessions++; + if (probe.transport.deref()) transports++; + for (const broadcast of probe.broadcasts) if (broadcast.deref()) broadcasts++; + for (const payload of probe.payloads) { + const buffer = payload.deref(); + if (buffer) buffers.add(buffer); + } + } + let bytes = 0; + for (const buffer of buffers) bytes += buffer.byteLength; + return { sessions, transports, broadcasts, bytes }; +} + +const names = (history: History, route: number, count: number) => + Array.from({ length: count }, (_, index) => (history === "stable" ? `t${index}` : `r${route}t${index}`)); + +/** Read every name through `front` the chosen way, closing each reader, and keep the payloads weakly. */ +async function read(front: Front, tracks: string[], access: Access, payloads: WeakRef[]): Promise { + for (const name of tracks) { + const track = front.consumer.track(name); + if (access === "lookup") { + await track.info(); + continue; + } + const subscriber = access === "subscribe" ? track.subscribe({ maxDelay: MAX_DELAY }) : undefined; + const group = subscriber ? await subscriber.recvGroup() : await track.fetchGroup(0); + const frame = await group?.readFrame(); + group?.close(); + subscriber?.close(); + if (frame?.payload.byteLength !== PAYLOAD || frame.payload.some((byte) => byte !== 7)) + throw new Error(`${name}: wrong payload`); + payloads.push(new WeakRef(frame.payload.buffer as ArrayBuffer)); + } +} + +interface Case { + fronts: number; + names: number; + routes: number; + history: History; + access: Access; +} + +interface Result { + /** Microseconds in `Front.serve` per front per route. */ + serveUs: number; + /** Microseconds per read, including the session round trips it waits on. */ + readUs: number; + retired: Retained; + serving: Retained; + /** JS heap the fronts and their serving route kept, against the same heap once concluded. */ + heldBytes: number; + concluded: Retained; +} + +/** + * The only strong hold on the serving route, so dropping it leaves that route to the fronts. + * Nothing about a route is returned: a suspended caller's dead locals can still pin a value. + */ +interface Routes { + /** Ends the serving route's session and drops the table's handles on it. */ + close?: () => void; + serveMs: number; + readMs: number; + probes: Probe[]; +} + +/** Connect a route, serve every front from it, end the previous one, and read through the fronts. */ +async function open( + routes: Routes, + publisher: OriginProducer, + paths: Path.Valid[], + fronts: Front[], + read: (front: Front, payloads: WeakRef[]) => Promise, +): Promise { + const pair = createMockTransportPair(ALPN_06); + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client }), + accept({ url, transport: pair.server, publish: publisher.consume() }), + ]); + const handles = paths.map((path) => wireOf(client).consume(path)); + const probe: Probe = { + session: new WeakRef(client), + transport: new WeakRef(pair.client), + broadcasts: handles.map((handle) => new WeakRef(handle)), + payloads: [], + }; + routes.probes.push(probe); + let start = performance.now(); + for (let index = 0; index < fronts.length; index++) fronts[index].serve(handles[index]); + routes.serveMs += performance.now() - start; + routes.close?.(); + routes.close = () => { + for (const handle of handles) handle.close(); + client.abort(); + server.abort(); + }; + start = performance.now(); + for (const front of fronts) await read(front, probe.payloads); + routes.readMs += performance.now() - start; +} + +/** Close every front and the serving route, in a frame of its own so no temporary outlives it. */ +function conclude(routes: Routes, fronts: Front[]): void { + for (const front of fronts.splice(0)) front.consumer.close(); + routes.close?.(); + routes.close = undefined; +} + +async function measure(shape: Case): Promise { + // Cache age-out runs on the installed clock, so moving it expires whatever only waited on it. + const clock = new Manual(); + const installed = clock.install(); + try { + return await measureOn(clock, shape); + } finally { + installed.close(); + } +} + +async function measureOn(clock: Manual, { fronts, names: perRoute, routes, history, access }: Case): Promise { + // One publisher instance per front: every route carries the same broadcasts and tracks. + const publisher = new OriginProducer(); + const paths = Array.from({ length: fronts }, (_, index) => Path.from(`room/${index}`)); + const broadcasts: BroadcastProducer[] = paths.map((path) => { + const publication = publisher.createBroadcast(path); + publication.announce({ epoch: Epoch.mint() }); + return publication.broadcast; + }); + const payload = new Uint8Array(PAYLOAD).fill(7); + for (const broadcast of broadcasts) { + const all = new Set(Array.from({ length: routes }, (_, route) => names(history, route, perRoute)).flat()); + for (const name of all) { + const track = broadcast.createTrack(name); + const group = track.appendGroup(); + group.writeFrame({ payload }); + group.finish(); + } + } + + const logical = paths.map(() => new Front()); + const state: Routes = { serveMs: 0, readMs: 0, probes: [] }; + const probes = state.probes; + for (let route = 0; route < routes; route++) { + const tracks = names(history, route, perRoute); + await open(state, publisher, paths, logical, (front, payloads) => read(front, tracks, access, payloads)); + } + + await drain(clock, probes); + const retired = count(probes.slice(0, -1)); + const serving = count(probes.slice(-1)); + const held = heap(); + + // Control: concluding the fronts and ending the last session releases everything, so a + // harness that could not see a release fails here rather than report zero. + conclude(state, logical); + for (const broadcast of broadcasts) broadcast.close(); + publisher.close(); + await drain(clock, probes); + const concluded = count(probes); + return { + serveUs: (state.serveMs * 1000) / (fronts * routes), + readUs: (state.readMs * 1000) / (fronts * routes * perRoute), + retired, + serving, + heldBytes: Math.max(0, held - heap()), + concluded, + }; +} + +/** Print every case; a reachable retired route shows in its row, for the harness to judge. */ +export async function run(): Promise { + // Keep protocol diagnostics out of the measured CSV. + console.debug = () => {}; + console.log( + "fronts,names,routes,history,access,retired_sessions,retired_transports,retired_broadcasts,retired_payload_bytes,serving_sessions,serve_us,read_us,size_bytes", + ); + for (const access of ["lookup", "fetch", "subscribe"] as const) + for (const history of ["stable", "churn"] as const) + for (const fronts of [1, 8]) + for (const perRoute of [1, 8]) + for (const routes of [2, 8, 32]) { + const result = await measure({ fronts, names: perRoute, routes, history, access }); + const { retired, serving, concluded } = result; + const label = `${fronts} fronts, ${perRoute} names, ${routes} routes, ${history}, ${access}`; + if (concluded.sessions || concluded.transports || concluded.broadcasts || concluded.bytes) + throw new Error(`${label}: the control kept ${JSON.stringify(concluded)} after concluding`); + if (serving.sessions !== 1 || serving.broadcasts !== fronts) + throw new Error(`${label}: the serving route was not observed`); + console.log( + [ + fronts, + perRoute, + routes, + history, + access, + retired.sessions, + retired.transports, + retired.broadcasts, + retired.bytes, + serving.sessions, + result.serveUs.toFixed(2), + result.readUs.toFixed(2), + result.heldBytes, + ].join(","), + ); + } +} + +if (import.meta.main) await run(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index b7450b95376..7d28429123a 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -138,8 +138,15 @@ function ask(source: Source, name: string, retired: AbortSignal, hold?: AbortSig }); } +let sources = 0; + /** One route's broadcast, held while the front serves it or a reader still reads a copy of it. */ class Source { + /** + * Names this source where it outlives its route, as a front's record of who said what a + * track is: a reference there would keep a retired route's session reachable. + */ + readonly id = sources++; readonly wire: BroadcastWire; /** The table's handle this was served from, so serving it again is a no-op. */ readonly from: broadcast.Consumer; @@ -1516,8 +1523,11 @@ export class Front { readonly consumer: broadcast.Consumer; #source?: Source; #routes = new Map(); - /** Each track's properties as first committed, and the source that said so: every later copy must match. */ - #info = new Map(); + /** + * Each track's properties as first committed, and the {@link Source.id} of the source that said + * so: every later copy must match. Kept for the front's lifetime, so it holds no route. + */ + #info = new Map(); /** Bumped whenever the source changes, so an answer from a replaced one is dropped. */ #changed = new Signal(0); /** Aborted once the current source is replaced, retiring what was asked of it. */ @@ -1721,7 +1731,7 @@ export class Front { const previous = route.served; // The first copy of a track has nothing to match, and the source that said what it is // matches itself. - if ((!recorded && !previous) || recorded?.source === source) { + if ((!recorded && !previous) || recorded?.source === source.id) { route.serve(source); return; } @@ -1787,7 +1797,7 @@ export class Front { // never reads a copy a subscription would refuse. async #verify(name: string, source: Source): Promise { const recorded = this.#info.get(name); - if (recorded?.source === source) return; + if (recorded?.source === source.id) return; let verified = this.#verified.get(source); if (verified?.has(name)) return; const retired = this.#retire.signal; @@ -1818,7 +1828,7 @@ export class Front { } #record(name: string, info: track.Info, source: Source): void { - if (!this.#info.has(name)) this.#info.set(name, { info, source }); + if (!this.#info.has(name)) this.#info.set(name, { info, source: source.id }); } subscribe(name: string, options?: track.Subscription): track.Subscriber { diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 7739cd159e9..7c5faa7e3d6 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -106,6 +106,14 @@ An operation-count probe imported the actual JS `bandwidth.allocate`, counted The bandwidth child must retain a reproducible benchmark, including the complete allocator and Rust consumers, before selecting a fix. +[Resume identity history](https://github.com/moq-dev/moq/pull/7514) found, in +Chromium after cache age-out, that a JS front's first-seen track record kept +every route that first answered a name reachable: 32 routes of churned names +left 31 sessions, transports and broadcast handles per front (7.7 MB of JS heap +for 8 fronts reading 8 names by lookup). Recording the answering source by id +releases them; the `resume-history` browser fixture now fails on any retired +route still reachable. + `python3 sh/gh/nightly.test.py NightlyTest.test_every_cargo_bench_target_runs_nightly` reproduced the missing `moq-archive/producer` selection. The existing [CI audit PR](https://github.com/moq-dev/moq/pull/6266) already fixes it. @@ -186,7 +194,6 @@ No duplicate quest or workflow edit belongs here. - [Buffered browser groups](/quest/m2/performance-survey/hang-buffered.md) - per-frame scans and bulk skip work - [Idle browser stats](/quest/m2/performance-survey/browser-idle-stats.md) - local transport polling without active publishing demand - [Transport preference history](/quest/m2/performance-survey/transport-history.md) - full URL retention across retired sessions -- [Resume identity history](/quest/m2/performance-survey/resume-history.md) - historical source graphs held by first-seen track metadata - [UDP socket queues](/quest/m2/performance-survey/socket-queues.md) - drop attribution and queue-size tradeoffs across workers and load - [Session datagrams](/quest/m2/performance-survey/datagram-delivery.md) - complete per-object delivery across publishers and viewers - [Announcement snapshots](/quest/m2/performance-survey/announced-snapshots.md) - HTTP polling cost versus result size and unrelated routes diff --git a/quest/m2/performance-survey/resume-history.md b/quest/m2/performance-survey/resume-history.md deleted file mode 100644 index ef5bc47230e..00000000000 --- a/quest/m2/performance-survey/resume-history.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] Resume identity retains only necessary state - -## Goal - -Long-lived browser fronts preserve track identity across routes without -unnecessarily retaining historical source object graphs. - -## Plan - -[Front metadata](/js/net/src/resume.ts) records the first `{info, source}` for -each track. A `Source` owns broadcast handles and a wire view; releasing its -last hold closes the cloned consumer but leaves those strong references. -This establishes a retained JS path, not that a closed socket or its native -buffers remain live. A fixed track set pins first-answer identities, not a -new source for every reconnect. Track-name churn plus route turnover is the -important adverse case. - -Sweep fronts × distinct first-seen track names, independently varying route -replacements. Compare one stable track across many routes against new names -first answered by each route. Close readers after lookup, fetch and subscribe; -after cache expiry and browser GC, inspect heap dominators, retained source/ -subscriber/transport objects and payload bytes. Measure transition CPU too. - -Keep the metadata proof: the same name must still mean compatible content. -Preserve incompatibility refusal, concurrent cancellation and current-source -reuse. Coordinate with [route health](/quest/m2/route-health/README.md) before -changing identity representation; do not delete history just to reduce heap. - -Follow the [survey benchmark contract](/quest/m2/performance-survey/README.md): -baseline first, paired evidence, equivalent work and nightly coverage. A -measured no-win is valid. No API or wire change is expected; report actual impact. diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index d38d68f7341..a3e88f90fe2 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -193,6 +193,13 @@ def test_every_js_bench_runs_nightly(self): continue for path in recipes[args[2]]: entries.add(ROOT / "js" / path.replace("{{ name }}", args[-1])) + # The browser smoke lane runs the in-memory net fixtures in Chromium. + self.assertIn("recipe: bench-browser --smoke", workflow) + smoke = re.search( + r"^for benchmark in (.+); do\n\s+bun micro\.ts ", (ROOT / "bench/browser/run.sh").read_text(), re.M + ) + self.assertIsNotNone(smoke) + entries |= {ROOT / "js/net/bench" / f"{name}.ts" for name in smoke[1].split()} # Bundled helpers only run through the entry that names them. helpers = { path for entry in entries for path in entry.parent.glob("*.ts") if f"/{path.name}" in entry.read_text()