From 2fa00c663de18fd620a69e743b47f22abb6fea0f Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Wed, 9 Sep 2026 14:32:28 -0400 Subject: [PATCH] GC unreferenced declared captures via the manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A declared-lane ledger row was only reclaimable when superseded (a newer generation under the same capture identity) or when its source was tombstoned. A slot that is renamed, deleted, or re-specced mints a new specHash, so its old row met neither arm and leaked its object permanently — and authors iterating on declarations mint these constantly. The sweep gains an 'unreferenced' arm: a declared-lane row whose spec hash no live prerendered_html row's screenshots manifest names is collectible. The manifest is the correct liveness set — ?name= serving is pinned to it, carry-forwards keep their hash in it across generations, and error renders preserve the last-known-good manifest — so a row it doesn't reference is unreachable by any serve. Two guards keep the arm conservative: a live prerendered row must exist (a source mid-first-index has none, and absence must not read as an empty roster), and every live row's rendered_at must be older than min-age, so a manifest published moments ago (mid-iteration churn, a capture failure the retry lane is still working) collects nothing until the state has held for a full window. Realm-copied rows answer to their own realm's manifest, which the copy keeps consistent. Co-Authored-By: Claude Fable 5 --- .../realm-server/tests/media-cache-gc-test.ts | 241 ++++++++++++++++++ packages/runtime-common/media-cache.ts | 74 +++++- .../runtime-common/tasks/media-cache-gc.ts | 4 +- 3 files changed, 314 insertions(+), 5 deletions(-) diff --git a/packages/realm-server/tests/media-cache-gc-test.ts b/packages/realm-server/tests/media-cache-gc-test.ts index 14aaa71203a..0e093a26367 100644 --- a/packages/realm-server/tests/media-cache-gc-test.ts +++ b/packages/realm-server/tests/media-cache-gc-test.ts @@ -135,6 +135,59 @@ module(basename(import.meta.filename), function (hooks) { await seedIndexRow({ url: sourceURL, realmURL, isDeleted: true }); } + // A live production prerendered row whose `screenshots` manifest is the + // unreferenced arm's liveness set. `manifestSpecHashes` seeds one entry per + // hash (names are irrelevant to the arm); null seeds a manifest-less row. + async function seedPrerenderedRow({ + url, + fileAlias = url, + realmURL = 'http://test-realm/a/', + type = 'instance', + manifestSpecHashes, + renderedAt, + }: { + url: string; + fileAlias?: string; + realmURL?: string; + type?: 'instance' | 'file'; + manifestSpecHashes: string[] | null; + renderedAt: number; + }) { + let screenshots = + manifestSpecHashes === null + ? null + : Object.fromEntries( + manifestSpecHashes.map((specHash, i) => [ + `slot-${i}`, + { + specHash, + objectKey: `object-for-${specHash}`, + contentType: 'image/png', + width: 170, + height: 250, + deviceScaleFactor: 2, + }, + ]), + ); + let { nameExpressions, valueExpressions } = asExpressions( + { + url, + file_alias: fileAlias, + realm_url: realmURL, + type, + generation: 1, + is_deleted: false, + rendered_at: renderedAt, + screenshots, + }, + { jsonFields: ['screenshots'] }, + ); + await query( + dbAdapter, + insert('prerendered_html', nameExpressions, valueExpressions), + ); + } + async function ledgerRows(): Promise< { source_generation: number; object_key: string }[] > { @@ -319,6 +372,194 @@ module(basename(import.meta.filename), function (hooks) { ); }); + test('reclaims a declared row the current manifest no longer references', async function (assert) { + let now = Date.now(); + // The slot was re-specced: the current manifest carries only the new + // hash, so nothing ever supersedes the old row (a new hash is a new + // capture identity) and this arm is its only reclamation path. + await seedLedgerRow({ + captureSpecHash: 'spec-old', + sourceGeneration: 1, + objectKey: 'old-spec-object', + createdAt: now - 3 * DAY, + }); + await seedLedgerRow({ + captureSpecHash: 'spec-new', + sourceGeneration: 2, + objectKey: 'new-spec-object', + createdAt: now - 2 * DAY, + }); + await seedPrerenderedRow({ + url: 'http://test-realm/a/card-1.json', + fileAlias: 'http://test-realm/a/card-1', + manifestSpecHashes: ['spec-new'], + renderedAt: now - 2 * DAY, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 1); + assert.deepEqual(adapter.deleted, ['old-spec-object']); + assert.deepEqual( + (await ledgerRows()).map((row) => row.object_key), + ['new-spec-object'], + 'the manifest-referenced capture survives', + ); + }); + + test('a manifest-less live row reclaims all declared captures of its source', async function (assert) { + let now = Date.now(); + // Every slot was deleted from the declaration: the source's current row + // publishes no manifest at all, so nothing references the old capture. + await seedLedgerRow({ + captureSpecHash: 'spec-deleted-slot', + sourceGeneration: 1, + objectKey: 'deleted-slot-object', + createdAt: now - 3 * DAY, + }); + await seedPrerenderedRow({ + url: 'http://test-realm/a/card-1.json', + fileAlias: 'http://test-realm/a/card-1', + manifestSpecHashes: null, + renderedAt: now - 2 * DAY, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 1); + assert.deepEqual(adapter.deleted, ['deleted-slot-object']); + }); + + test('a freshly published manifest collects nothing until it has held for min-age', async function (assert) { + let now = Date.now(); + // The manifest dropped the hash moments ago — an author mid-iteration, + // or a capture failure the retry lane is still working. The stability + // guard waits a full min-age window before believing it. + await seedLedgerRow({ + captureSpecHash: 'spec-old', + sourceGeneration: 1, + objectKey: 'maybe-orphaned-object', + createdAt: now - 3 * DAY, + }); + await seedPrerenderedRow({ + url: 'http://test-realm/a/card-1.json', + fileAlias: 'http://test-realm/a/card-1', + manifestSpecHashes: ['spec-new'], + renderedAt: now - 1 * HOUR, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 0, 'nothing reclaimed yet'); + assert.deepEqual(adapter.deleted, []); + }); + + test('a carried-forward capture survives: its hash stays in the manifest across generations', async function (assert) { + let now = Date.now(); + // A file-content-keyed capture is never re-persisted while the bytes are + // unchanged: its ledger row stays at the old generation while the + // manifest (republished at each new generation) keeps naming its hash. + // No newer ledger row exists, so the superseded arm can't touch it — and + // the manifest reference is exactly what keeps this arm off it too. + await seedLedgerRow({ + captureSpecHash: 'spec-carried', + sourceGeneration: 1, + objectKey: 'carried-forward-object', + createdAt: now - 60 * DAY, + }); + await seedPrerenderedRow({ + url: 'http://test-realm/a/card-1.json', + fileAlias: 'http://test-realm/a/card-1', + manifestSpecHashes: ['spec-carried'], + renderedAt: now - 2 * DAY, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 0); + assert.ok(adapter.objects.has('carried-forward-object')); + }); + + test('a source with no prerendered row keeps its declared captures', async function (assert) { + let now = Date.now(); + // No live row means no manifest to consult — a source mid-first-index, + // or a realm whose prerender pass hasn't landed. Absence of evidence + // must not read as an empty roster. + await seedLedgerRow({ + captureSpecHash: 'spec-1', + sourceGeneration: 1, + objectKey: 'unjudgeable-object', + createdAt: now - 60 * DAY, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 0); + assert.ok(adapter.objects.has('unjudgeable-object')); + }); + + test('each realm-copied capture answers to its own realm manifest', async function (assert) { + let now = Date.now(); + // The capture was realm-copied along with its prerendered row; the + // source realm then re-specced the slot while the destination kept it. + // Only the source realm's copy is reclaimed. + for (let realmURL of ['http://test-realm/a/', 'http://test-realm/b/']) { + await seedLedgerRow({ + realmURL, + sourceURL: `${realmURL}card-1`, + captureSpecHash: 'spec-shared', + sourceGeneration: 1, + objectKey: `object-${realmURL.endsWith('a/') ? 'a' : 'b'}`, + createdAt: now - 3 * DAY, + }); + } + await seedPrerenderedRow({ + url: 'http://test-realm/a/card-1.json', + fileAlias: 'http://test-realm/a/card-1', + realmURL: 'http://test-realm/a/', + manifestSpecHashes: ['spec-respecced'], + renderedAt: now - 2 * DAY, + }); + await seedPrerenderedRow({ + url: 'http://test-realm/b/card-1.json', + fileAlias: 'http://test-realm/b/card-1', + realmURL: 'http://test-realm/b/', + manifestSpecHashes: ['spec-shared'], + renderedAt: now - 2 * DAY, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 1); + assert.deepEqual(adapter.deleted, ['object-a']); + assert.ok(adapter.objects.has('object-b')); + }); + + test('the manifest arm never touches on-demand captures', async function (assert) { + let now = Date.now(); + // An on-demand capture's spec hash is naturally absent from any declared + // manifest — that lane lives and dies by its access TTL alone. + await seedLedgerRow({ + captureSpecHash: 'spec-dsl', + sourceGeneration: 1, + objectKey: 'active-dsl-object', + lane: 'on-demand', + createdAt: now - 60 * DAY, + lastAccessedAt: now - 1 * DAY, + }); + await seedPrerenderedRow({ + url: 'http://test-realm/a/card-1.json', + fileAlias: 'http://test-realm/a/card-1', + manifestSpecHashes: ['spec-declared'], + renderedAt: now - 2 * DAY, + }); + + let result = await runGc(); + + assert.strictEqual(result.rowsDeleted, 0); + assert.ok(adapter.objects.has('active-dsl-object')); + }); + test('an object still referenced by a surviving row keeps its bytes', async function (assert) { let now = Date.now(); // Two captures produced identical bytes (dedupe): the superseded row is diff --git a/packages/runtime-common/media-cache.ts b/packages/runtime-common/media-cache.ts index 6acd2278778..ad747923689 100644 --- a/packages/runtime-common/media-cache.ts +++ b/packages/runtime-common/media-cache.ts @@ -410,7 +410,11 @@ export const MEDIA_CACHE_GC_MIN_AGE_MS = 24 * 60 * 60 * 1000; // a declared screenshot, which never ages out. export const MEDIA_CACHE_ON_DEMAND_TTL_MS = 30 * 24 * 60 * 60 * 1000; -export type MediaCacheGcReason = 'tombstoned' | 'superseded' | 'expired'; +export type MediaCacheGcReason = + | 'tombstoned' + | 'superseded' + | 'unreferenced' + | 'expired'; export interface MediaCacheGcCandidate extends MediaCacheEntryKey { objectKey: string; @@ -427,11 +431,27 @@ export interface MediaCacheGcCandidate extends MediaCacheEntryKey { // - 'superseded': a newer-generation row exists for the same capture // identity, and has for at least the min-age (so a serve that resolved // the old row just before the swap can still finish streaming). +// - 'unreferenced': a declared-lane row whose capture spec hash no live +// `prerendered_html` row's `screenshots` manifest names — the slot was +// renamed, deleted, or re-specced (any identity change mints a new +// hash), so nothing supersedes the old row and no serve can reach its +// object (`?name=` serving is pinned to the manifest). Two guards keep +// this arm honest: it requires a live prerendered row to exist (no +// evidence is not evidence of absence — a source mid-first-index has no +// row yet), and it requires every live row's `rendered_at` to be older +// than min-age, so a manifest published moments ago — an author mid- +// iteration, a transient capture failure the retry lane is still +// working — collects nothing until the state has held for a full +// window. Carry-forwards need no special case: a carried-forward entry +// keeps its spec hash in the current manifest, which is exactly what +// protects its older-generation row. Error renders keep the last-known- +// good manifest, so a failing source protects its captures the same +// way. // - 'expired': an on-demand capture idle past the TTL. // Every arm additionally requires the row itself to be older than min-age. -// The jsonb-free SQL here is still Postgres-shaped (row-value EXISTS, -// bigint arithmetic); like the reconcile scans, the GC task runs solely -// behind the Postgres queue. +// The SQL here is Postgres-only (row-value EXISTS, bigint arithmetic, +// `jsonb_each` over the manifest); like the reconcile scans, the GC task +// runs solely behind the Postgres queue. // The tombstone arm, verbatim in both the reason CASE and the WHERE below — // one string so the two can't drift. `IN ('instance', 'file')` covers both @@ -453,6 +473,46 @@ const GC_TOMBSTONED_PREDICATE = ` AND (i.is_deleted = FALSE OR i.is_deleted IS NULL) )`; +// A live prerendered row for the ledger row's source, in the row's own realm +// — realm-copied captures are duplicated per realm along with their +// prerendered rows, so each copy answers to its own realm's manifest. The +// ledger spelling (`screenshotLedgerSourceURL`) matches the row's `url` for +// files and its `file_alias` for instances, the same double match the +// tombstone arm uses; a matching row of either type participates, so an +// alias collision errs toward protecting bytes. +const GC_LIVE_PRERENDERED_ROW = ` + FROM prerendered_html p + WHERE (p.url = r.source_url OR p.file_alias = r.source_url) + AND p.realm_url = r.realm_url + AND p.type IN ('instance', 'file') + AND (p.is_deleted = FALSE OR p.is_deleted IS NULL)`; + +// The unreferenced arm, once in the reason CASE and once in the WHERE via +// this helper so the two can't drift. Carries the min-age cutoff for the +// manifest-stability guard, so unlike the tombstone arm it is an Expression +// rather than a bare string. The `jsonb_typeof` guard keeps a malformed +// manifest (anything but an object) from erroring the whole sweep — +// `jsonb_each` refuses scalars — and a NULL manifest simply protects +// nothing, which is the point: a live row with no manifest says the source +// currently declares no captures at all. +function gcUnreferencedPredicate(minAgeCutoff: number): Expression { + return [ + `r.lane = 'declared' + AND EXISTS (SELECT 1 ${GC_LIVE_PRERENDERED_ROW}) + AND NOT EXISTS (SELECT 1 ${GC_LIVE_PRERENDERED_ROW} AND p.rendered_at >=`, + param(minAgeCutoff), + `) + AND NOT EXISTS ( + SELECT 1 ${GC_LIVE_PRERENDERED_ROW} + AND jsonb_typeof(p.screenshots) = 'object' + AND EXISTS ( + SELECT 1 FROM jsonb_each(p.screenshots) AS slot + WHERE slot.value->>'specHash' = r.capture_spec_hash + ) + )`, + ]; +} + export async function findMediaCacheGcCandidates( dbAdapter: DBAdapter, { @@ -475,6 +535,9 @@ export async function findMediaCacheGcCandidates( AND n.created_at <`, param(minAgeCutoff), `) THEN 'superseded' + WHEN (`, + ...gcUnreferencedPredicate(minAgeCutoff), + `) THEN 'unreferenced' ELSE 'expired' END AS reason FROM media_cache_ledger r @@ -489,6 +552,9 @@ export async function findMediaCacheGcCandidates( AND n.source_generation > r.source_generation AND n.created_at <`, param(minAgeCutoff), + `) + OR (`, + ...gcUnreferencedPredicate(minAgeCutoff), `) OR (r.lane = 'on-demand' AND r.last_accessed_at <`, param(idleCutoff), diff --git a/packages/runtime-common/tasks/media-cache-gc.ts b/packages/runtime-common/tasks/media-cache-gc.ts index e66a5471238..a429de3ee65 100644 --- a/packages/runtime-common/tasks/media-cache-gc.ts +++ b/packages/runtime-common/tasks/media-cache-gc.ts @@ -53,7 +53,9 @@ registerQueueJobDefinition({ // Reconcile-style GC for the MediaCache: reclaims ledger rows whose capture // is superseded by a newer generation, whose source instance is tombstoned, -// or (on-demand lane) idle past the TTL — then deletes each object whose +// whose declared slot no current manifest references (renamed, deleted, or +// re-specced), or (on-demand lane) idle past the TTL — then deletes each +// object whose // last ledger reference those rows held. Objects are deleted before their // rows so a sweep that dies mid-way leaves rows behind for the next sweep to // re-find, never bytes the ledger no longer knows about (the ledger is the