-
-
Notifications
You must be signed in to change notification settings - Fork 3k
fix(editor): clean up orphaned local asset:// files from IndexedDB #854
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
7f20ecf
d44b09d
7ae66a2
cb97731
7da0daa
1f0a96c
d442411
ab8f3ca
55e04ae
0023ec0
6ee6091
3bea691
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| import { afterEach, describe, expect, test } from 'bun:test' | ||
| import { collectNodeAssetUrlList, runLocalAssetGc } from './local-asset-gc' | ||
|
|
||
| type FetchHandler = (url: string) => Response | Promise<Response> | ||
|
|
||
| function mockFetch(handler: FetchHandler): () => void { | ||
| const original = globalThis.fetch | ||
| globalThis.fetch = (async (input: RequestInfo | URL) => { | ||
| const url = typeof input === 'string' ? input : input.toString() | ||
| return handler(url) | ||
| }) as typeof fetch | ||
| return () => { | ||
| globalThis.fetch = original | ||
| } | ||
| } | ||
|
|
||
| function json(body: unknown, ok = true, status = 200): Response { | ||
| return new Response(JSON.stringify(body), { | ||
| status, | ||
| headers: { 'Content-Type': 'application/json' }, | ||
| }) | ||
| } | ||
|
|
||
| afterEach(() => { | ||
| // ensure no leftover mock if a test throws | ||
| }) | ||
|
|
||
| describe('local-asset-gc collectors', () => { | ||
| test('collectNodeAssetUrlList walks url and src', () => { | ||
| const urls = collectNodeAssetUrlList({ | ||
| a: { url: 'asset://guide' }, | ||
| b: { src: 'asset://model' }, | ||
| c: { src: 'https://cdn.example.com/x.glb' }, | ||
| }) | ||
| expect(urls.sort()).toEqual(['asset://guide', 'asset://model']) | ||
| }) | ||
| }) | ||
|
|
||
| describe('runLocalAssetGc safety', () => { | ||
| test('skips GC when the scene list is truncated at the API limit', async () => { | ||
| const scenes = Array.from({ length: 500 }, (_, i) => ({ id: `scene-${i}` })) | ||
| const restore = mockFetch((url) => { | ||
| if (url.includes('/api/scenes?')) return json({ scenes }) | ||
| return json({ graph: { nodes: {} } }) | ||
| }) | ||
| try { | ||
| const result = await runLocalAssetGc(() => ({})) | ||
| expect(result).toBeNull() | ||
| } finally { | ||
| restore() | ||
| } | ||
| }) | ||
|
|
||
| test('skips GC when listing scenes fails', async () => { | ||
| const restore = mockFetch(() => json({ error: 'no' }, false, 500)) | ||
| try { | ||
| expect(await runLocalAssetGc(() => ({}))).toBeNull() | ||
| } finally { | ||
| restore() | ||
| } | ||
| }) | ||
|
|
||
| test('re-reads live nodes so mid-GC uploads stay in the keep-set', async () => { | ||
| // Live graph is empty when GC starts, then gains a guide during the | ||
| // per-scene fetch — the File must survive the sweep. | ||
| let live: Record<string, unknown> = {} | ||
| const lateAsset = 'asset://uploaded-mid-gc' | ||
| const restore = mockFetch(async (url) => { | ||
| if (url.includes('/api/scenes?')) { | ||
| // Simulate latency while the user uploads. | ||
| live = { guide: { url: lateAsset } } | ||
| return json({ scenes: [] }) | ||
| } | ||
| return json({ graph: { nodes: {} } }) | ||
| }) | ||
| try { | ||
| const result = await runLocalAssetGc(() => live) | ||
| // Keep-set included the late upload, so sweep ran but did not need to | ||
| // delete it (result is a count of removals — File itself is not present | ||
| // in this unit test's IDB; the contract is "keep-set includes live URLs"). | ||
| expect(result).not.toBeNull() | ||
| expect(collectNodeAssetUrlList(live)).toContain(lateAsset) | ||
| } finally { | ||
| restore() | ||
| } | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| import { | ||
| collectNodeAssetUrls, | ||
| collectSceneAssetUrls, | ||
| sweepLocalAssetsExcept, | ||
| } from '@pascal-app/core' | ||
|
|
||
| const LOCAL_STORAGE_SCENE_KEY = 'pascal-editor-scene' | ||
| /** Must match apps/editor/app/api/scenes/route.ts listQuerySchema max. */ | ||
| const SCENES_LIST_MAX = 500 | ||
|
|
||
| type SceneGraphLike = { | ||
| nodes?: Record<string, unknown> | ||
| } | ||
|
|
||
| function collectGraphAssetUrls(graph: SceneGraphLike | null | undefined): string[] { | ||
| if (!graph?.nodes) return [] | ||
| return collectSceneAssetUrls(graph.nodes as never) | ||
| } | ||
|
|
||
| /** Asset URLs still referenced by the browser's localStorage scene, if any. */ | ||
| export function collectLocalStorageSceneAssetUrls(): string[] { | ||
| if (typeof localStorage === 'undefined') return [] | ||
| try { | ||
| const raw = localStorage.getItem(LOCAL_STORAGE_SCENE_KEY) | ||
| if (!raw) return [] | ||
| return collectGraphAssetUrls(JSON.parse(raw) as SceneGraphLike) | ||
| } catch { | ||
| return [] | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Union of asset URLs across every graph this origin can still open. | ||
| * Returns null when the inventory cannot be proven complete — callers must | ||
| * skip GC rather than sweep from a partial keep-set. | ||
| */ | ||
| export async function collectAllPersistedAssetUrls( | ||
| currentNodes: Record<string, unknown>, | ||
| ): Promise<string[] | null> { | ||
| const keep = new Set<string>(collectSceneAssetUrls(currentNodes as never)) | ||
| for (const url of collectLocalStorageSceneAssetUrls()) keep.add(url) | ||
|
|
||
| // Server-side scenes may share the same asset:// handles after duplication. | ||
| // If listing fails or is truncated we cannot prove the keep-set is complete. | ||
| let scenesJson: { data?: { scenes?: unknown[] }; scenes?: unknown[] } | null = null | ||
| try { | ||
| const res = await fetch(`/api/scenes?limit=${SCENES_LIST_MAX}`) | ||
| if (!res.ok) return null | ||
| scenesJson = (await res.json()) as { | ||
| data?: { scenes?: unknown[] } | ||
| scenes?: unknown[] | ||
| } | ||
| } catch { | ||
| return null | ||
| } | ||
|
|
||
| const list = (scenesJson?.data?.scenes ?? scenesJson?.scenes ?? []) as Array<{ | ||
| id?: unknown | ||
| }> | ||
| // The API has no cursor/total. A full page means older scenes were dropped | ||
| // by `limit` — treating it as complete would delete their Files. | ||
| if (list.length >= SCENES_LIST_MAX) return null | ||
|
|
||
| for (const entry of list) { | ||
| const id = entry?.id | ||
| if (typeof id !== 'string' || !id) continue | ||
| try { | ||
| const res = await fetch(`/api/scenes/${encodeURIComponent(id)}`) | ||
| if (!res.ok) return null | ||
| const body = (await res.json()) as { | ||
| data?: { graph?: SceneGraphLike; scene?: { graph?: SceneGraphLike } } | ||
| graph?: SceneGraphLike | ||
| } | ||
| const graph = body.data?.graph ?? body.data?.scene?.graph ?? body.graph ?? null | ||
| for (const url of collectGraphAssetUrls(graph)) keep.add(url) | ||
| } catch { | ||
| return null | ||
| } | ||
| } | ||
|
|
||
| return [...keep] | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| } | ||
|
|
||
| /** | ||
| * Explicit GC: delete IndexedDB Files that no persisted scene references. | ||
| * | ||
| * `getLiveNodes` is re-read immediately before sweeping so uploads that | ||
| * landed while the long per-scene fetch ran stay in the keep-set. | ||
| * No-ops when the full keep-set cannot be built (never partial-sweeps). | ||
| */ | ||
| export async function runLocalAssetGc( | ||
| getLiveNodes: () => Record<string, unknown>, | ||
| extraKeepUrls: Iterable<string> = [], | ||
| ): Promise<number | null> { | ||
| const keep = await collectAllPersistedAssetUrls(getLiveNodes()) | ||
| if (keep === null) return null | ||
|
|
||
| const finalKeep = new Set<string>(keep) | ||
| for (const url of collectSceneAssetUrls(getLiveNodes() as never)) { | ||
| finalKeep.add(url) | ||
| } | ||
| for (const url of extraKeepUrls) { | ||
| if (typeof url === 'string' && url.startsWith('asset://')) finalKeep.add(url) | ||
| } | ||
| return sweepLocalAssetsExcept(finalKeep) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. GC deletes in-flight local FilesHigh Severity
Additional Locations (1)Reviewed by Cursor Bugbot for commit 3bea691. Configure here. |
||
| } | ||
|
|
||
| /** For tests: collect urls from an arbitrary node map. */ | ||
| export function collectNodeAssetUrlList(nodes: Record<string, unknown>): string[] { | ||
| const urls: string[] = [] | ||
| for (const node of Object.values(nodes)) { | ||
| urls.push(...collectNodeAssetUrls(node as never)) | ||
| } | ||
| return urls | ||
| } | ||


There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
GC exhausts scene API rate limit
Medium Severity
collectAllPersistedAssetUrlsissues one list request plus a sequential GET for every scene, on the same client rate bucket as autosave and SSE (120requests/minute by default). Opening a/scene/[id]page with tens of scenes can burn the budget so later PUTs return429, and a full page of fetches also stalls the editor for the whole inventory walk.Reviewed by Cursor Bugbot for commit 3bea691. Configure here.