From 3f9f1a9d91daab308bc54ae9b0763b789b02afeb Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 19:00:16 +0200 Subject: [PATCH 01/26] refactor: streamline backup handling and implement fitBackup logic - Removed unused constants and types from backup.ts, simplifying the backup structure. - Integrated compact backup encoding in SettingsView for more efficient storage. - Added fit.ts and fit.test.ts to manage backup size constraints, ensuring that backups fit within browser sync limits. - Implemented logic to prioritize retention of recent songs, charts, and favorites when trimming backups. - Enhanced test coverage for fitBackup functionality, validating behavior under various library sizes. --- .gitignore | 3 + CLAUDE.md | 5 +- CONTRIBUTING.md | 2 +- README.md | 5 +- package.json | 2 +- src/core/model/track-identity.ts | 9 +- src/core/persist/backup-codec.test.ts | 569 ++++++++++++ src/core/persist/backup-codec.ts | 811 ++++++++++++++++++ src/core/persist/backup.ts | 96 +-- .../settings/panel/SettingsView.svelte | 11 +- src/features/sync/persist/fit.test.ts | 214 +++++ src/features/sync/persist/fit.ts | 171 ++++ 12 files changed, 1808 insertions(+), 90 deletions(-) create mode 100644 src/core/persist/backup-codec.test.ts create mode 100644 src/core/persist/backup-codec.ts create mode 100644 src/features/sync/persist/fit.test.ts create mode 100644 src/features/sync/persist/fit.ts diff --git a/.gitignore b/.gitignore index 1776e11..88bd626 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ public/worklets/pcm-tap-worklet.js # Generated at postinstall / build:before. See scripts/build-vocal-worklet.mjs. public/worklets/vocal-reducer-worklet.js export/ + +# pnpm store created when a local store path is set +.pnpm-store/ diff --git a/CLAUDE.md b/CLAUDE.md index 08d2dc9..7bfc632 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ pnpm dev:firefox # same, Firefox pnpm check # svelte-check / TypeScript — the only type/lint gate pnpm build # production build → .output/chrome-mv3 pnpm zip # store package -pnpm test:dsp # fast DSP unit tests (node --test on src/features/**/*.test.ts) +pnpm test:dsp # fast unit tests: DSP, chords, the backup codec + sync fit (node --test on src/**/*.test.ts) pnpm release:dry # show the release plan (version bump, tag) without changing anything pnpm release # full release: check + test, bump patch, build both zips, commit, tag, push ``` @@ -99,11 +99,12 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. +- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped, timestamps in seconds. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. - Optional **cross-device sync** (`src/features/sync/` + `server/`): last-write-wins backup snapshots to a Cloudflare Worker + KV. The secret sync ID **is the whole capability** (open CORS, no other auth). The Worker URLs live once in [sync-hosts.ts](src/features/sync/sync-hosts.ts) (env-free, so `wxt.config.ts` imports it for the manifest); [endpoint.ts](src/features/sync/endpoint.ts) picks localhost in dev, the deployed Worker in prod. The ID rides `storage.sync` between devices and is additionally kept as a **cookie on the sync host** so it survives an uninstall ([id-cookie.ts](src/features/sync/panel/id-cookie.ts) is the canonical explanation). That needs the `cookies` permission plus host access to the sync origin, which is an **optional** host permission requested from the Sync settings / on enable / on connect (a required one would disable the extension on update in Chrome and is opt-in on Firefox); `sync.durable` mirrors whether it is held. The background worker filters the sync host out of the site-grant machinery (`siteOrigins` in [background.ts](src/entrypoints/background.ts)) so it is neither registered for the engine nor removed by Revoke Permissions. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. -- **`@/` does not work in two contexts** (they don't share the WXT/Vite resolver): the `node --test` DSP files (`src/features/**/*.test.ts` and the modules they import as *values* — `fft.ts`, `center-cut-dsp.ts`, `detect-bpm.ts`) must use **relative imports with explicit `.ts` extensions**; the esbuild worklet bundles (`src/features/*/engine/*.worklet.ts`) must use **relative imports**. (`import type` is erased, so type-only imports may omit the extension.) +- **`@/` does not work in two contexts** (they don't share the WXT/Vite resolver): the `node --test` files (`src/**/*.test.ts` and the modules they import as *values* — `fft.ts`, `center-cut-dsp.ts`, `detect-bpm.ts`, `backup-codec.ts`, `sync/persist/fit.ts`, and what those pull in: `defaults.ts`, `thumbnail.ts`, `track-identity.ts`) must use **relative imports with explicit `.ts` extensions**; the esbuild worklet bundles (`src/features/*/engine/*.worklet.ts`) must use **relative imports**. (`import type` is erased, so type-only imports may omit the extension.) - **Both browsers build MV3** (`manifestVersion: 3` is pinned in [wxt.config.ts](wxt.config.ts) — Firefox would otherwise default to MV2 and drop `optional_host_permissions`). Chromium-only APIs are gated on the build-time flags in [core/platform.ts](src/core/platform.ts) (`CAN_CAPTURE_TAB`, `HAS_SIDE_PANEL_API`), never on runtime `browser.*` probes: Firefox has no `tabCapture`/`offscreen` (so no capture fallback — the offscreen entrypoint is excluded from that build) and no `sidePanel` (the same page is registered as `sidebar_action`). Panel-side the capability travels as a **prop**: an absent `oncapture`/`ontabaudio` is what makes the shared UI drop the affordance. - Debug globals are named `__noteByNote*` / `__panelDebug`. The processor name literal is `note-by-note-center-cut` and **must match on both sides** (`vocal-reducer.worklet.ts` registers it, `vocal-reducer.ts` constructs it) — mismatches throw `InvalidStateError` at runtime and `tsc` won't catch them. - A `MediaElementSource` can be created only once per element per document lifetime, so **extension reloads require a page reload** to reattach. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ce953f..87e7a85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ files. ```powershell pnpm check # svelte-check / TypeScript — the only type or lint gate -pnpm test:dsp # node --test, the DSP unit tests +pnpm test:dsp # node --test, the unit tests (DSP, chords, backup codec, sync fit) pnpm build # production build → .output/chrome-mv3 ``` diff --git a/README.md b/README.md index 3a4a377..f4fc93b 100644 --- a/README.md +++ b/README.md @@ -116,8 +116,9 @@ the engine. ## Tests -`pnpm test:dsp` runs the DSP unit tests under `node --test`: the center-cut -math, the CQT, chord decoding. Fast, no browser. +`pnpm test:dsp` runs the unit tests under `node --test`: the center-cut +math, the CQT, chord decoding, the compact backup codec and the sync +fit-to-budget logic. Fast, no browser. The e2e harness is the interesting one. It launches Chrome for Testing with the extension installed, plays a 440 Hz tone, and asserts on the *processed output* — diff --git a/package.json b/package.json index 1105d05..053a6be 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "zip": "wxt zip", "zip:firefox": "wxt zip -b firefox", "check": "svelte-check --tsconfig ./tsconfig.json", - "test:dsp": "node --test \"src/features/**/*.test.ts\"", + "test:dsp": "node --test \"src/**/*.test.ts\"", "test:e2e": "node e2e/run.mjs", "release": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/release.ps1", "release:dry": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/release.ps1 -DryRun", diff --git a/src/core/model/track-identity.ts b/src/core/model/track-identity.ts index 559df16..e790a3d 100644 --- a/src/core/model/track-identity.ts +++ b/src/core/model/track-identity.ts @@ -56,6 +56,13 @@ export function isSameTrack(a: TrackIdentity, b: TrackIdentity): boolean { return a.normalizedUrl === b.normalizedUrl && a.title === b.title; } +/** The storage key of a track: `${hash(normalizedUrl)}:${durationSec}`, with + * `durationSec` already rounded. Exported so a compact backup can leave the key + * out and rebuild it from the two strings it stores anyway (backup-codec.ts). */ +export function identityKey(normalizedUrl: string, durationSec: number): string { + return `${hash(normalizedUrl)}:${durationSec}`; +} + export function makeTrackIdentity( pageUrl: string, title: string, @@ -64,7 +71,7 @@ export function makeTrackIdentity( const normalizedUrl = normalizeUrl(pageUrl); const duration = Number.isFinite(durationSec) ? Math.round(durationSec) : 0; return { - key: `${hash(normalizedUrl)}:${duration}`, + key: identityKey(normalizedUrl, duration), normalizedUrl, title: cleanTitle(title), durationSec: duration, diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts new file mode 100644 index 0000000..85e404c --- /dev/null +++ b/src/core/persist/backup-codec.test.ts @@ -0,0 +1,569 @@ +// Run with: pnpm test:dsp (node --test). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + BACKUP_FORMAT, + COMPACT_VERSION, + decodeBackup, + decodeParams, + encodeBackup, + encodeChart, + encodeParams, + parseBackupJson, + type Backup, +} from './backup-codec.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; +import type { + ChordChart, + EffectParams, + FavoriteEntry, + HistoryEntry, + TrackData, + TrackIdentity, +} from '../model/types.ts'; + +// --------------------------------------------------------------------------- +// Fixtures + +const YT_HREF = 'https://www.youtube.com/watch?v=741FSo7Xb40'; +const YT_THUMB = 'https://i.ytimg.com/vi/741FSo7Xb40/mqdefault.jpg'; + +const ytSong = makeTrackIdentity(`${YT_HREF}&t=12s`, 'Symphony of Destruction - YouTube', 230.4); +const siteSong = makeTrackIdentity( + 'https://example.com/lesson?id=42&utm_source=x', + 'Blues shuffle in A', + 187, +); +const localSong = makeTrackIdentity( + 'chrome-extension://abcdef/local-player.html', + 'take3.mp3', + 95, +); + +function params(patch: Partial = {}): EffectParams { + return { + ...DEFAULT_PARAMS, + eq: { enabled: false, gains: [...DEFAULT_PARAMS.eq.gains] }, + tuning: { ...DEFAULT_PARAMS.tuning }, + ...patch, + }; +} + +function entry(identity: TrackIdentity, patch: Partial = {}): HistoryEntry { + return { + identity, + params: params(), + pageUrl: identity === ytSong ? YT_HREF : identity.normalizedUrl, + thumbnailUrl: identity === ytSong ? YT_THUMB : undefined, + createdAt: 1_700_000_000_000, + updatedAt: 1_757_112_345_678, + ...patch, + }; +} + +function favorite(identity: TrackIdentity, patch: Partial = {}): FavoriteEntry { + return { + ...entry(identity), + favoritedAt: 1_757_000_000_000, + lastAccessedAt: 1_757_112_345_678, + ...patch, + }; +} + +/** A chart on the detector's frame grid (BTC hop = 92.88 ms), `n` segments of + * ~1.9 s, five chords cycling, with a dropped 'N' span leaving a gap after + * every 10th segment. */ +function chart(n: number, patch: Partial = {}): ChordChart { + const FRAME = 0.09287981859410431; + const labels = ['C', 'Am', 'F', 'G', 'Dm7']; + const segments = []; + let frame = 1; + for (let i = 0; i < n; i++) { + const startT = frame * FRAME; + frame += 20; + segments.push({ startT, endT: frame * FRAME, label: labels[i % 5], confidence: 1 }); + if (i % 10 === 9) frame += 3; // an 'N' span the decoder dropped + } + return { + segments, + key: { tonic: 'A', mode: 'minor', confidence: 0.8123456 }, + coverage: 0.98765, + analyzedFrom: 0, + analyzedTo: frame * FRAME, + computedAt: 1_757_112_345_678, + ...patch, + }; +} + +function track(identity: TrackIdentity, patch: Partial = {}): TrackData { + return { + identity, + markers: [ + { id: 'mlx3k9z2q-1', t: 8.2, label: 'Intro' }, + { id: 'mlx3k9z2q-2', t: 128.47321987, label: '' }, + ], + snippets: [ + { + id: 'clx3k9z2q-1', + name: 'Solo — half speed', + startT: 128.47, + endT: 142.12, + enabled: true, + repeats: 4, + overrides: { speed: 0.5 }, + }, + ], + sequenceLoop: false, + sequenceCountIn: false, + chordChart: null, + chordsEnabled: false, + updatedAt: 1_757_112_345_678, + ...patch, + }; +} + +function backup(patch: Partial = {}): Backup { + return { + format: BACKUP_FORMAT, + version: 1, + exportedAt: 1_757_200_000_123, + appVersion: '1.0.3', + settings: { ...DEFAULT_SETTINGS, keymap: { ...DEFAULT_SETTINGS.keymap } }, + uiPrefs: JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)), + history: [], + favorites: [], + eqPresets: [], + tracks: [], + ...patch, + }; +} + +/** A whole library the way storage would hold it: `n` Recent rows, a third of + * them favorited, every song with a track record and a chart. */ +function library(n: number, segments = 120): Backup { + const history: HistoryEntry[] = []; + const favorites: FavoriteEntry[] = []; + const tracks: TrackData[] = []; + for (let i = 0; i < n; i++) { + const id = makeTrackIdentity( + `https://www.youtube.com/watch?v=vid${i.toString().padStart(8, '0')}`, + `Song number ${i} - Guitar lesson - YouTube`, + 180 + i, + ); + const p = params(i % 2 ? { transpose: -2, speed: 0.75 } : {}); + history.push( + entry(id, { + params: p, + pageUrl: `https://www.youtube.com/watch?v=vid${i.toString().padStart(8, '0')}`, + thumbnailUrl: `https://i.ytimg.com/vi/vid${i.toString().padStart(8, '0')}/mqdefault.jpg`, + updatedAt: 1_757_000_000_000 + i * 60_000, + }), + ); + if (i % 3 === 0) favorites.push(favorite(id, { params: p })); + tracks.push(track(id, { chordChart: chart(segments), chordsEnabled: true })); + } + return backup({ history, favorites, tracks }); +} + +const roundTrip = (b: Backup) => decodeBackup(JSON.parse(JSON.stringify(encodeBackup(b)))); +const bytes = (value: unknown) => JSON.stringify(value).length; + +// --------------------------------------------------------------------------- +// Params + +test('params: defaults encode to nothing and decode back to the defaults', () => { + assert.equal(encodeParams(params()), undefined); + assert.deepEqual(decodeParams(undefined, 'x'), params()); +}); + +test('params: every non-default field survives, float noise is rounded away', () => { + const p = params({ + transpose: -3, + transposeEnabled: false, + pitchCents: 12, + pitchEnabled: false, + speed: 1.1500000000000001, + speedEnabled: false, + vocalReduce: 0.65, + vocalReduceEnabled: false, + vocalMode: 'isolate', + eq: { enabled: true, gains: [4, 6, 4, 0, -3, -5, -7, -8, -8, -8] }, + tuning: { trackHz: 432, instrumentHz: 440 }, + power: false, + baseBpm: 92.33333, + }); + const back = decodeParams(encodeParams(p), 'x'); + assert.deepEqual(back, { ...p, speed: 1.15, baseBpm: 92.33 }); +}); + +test('params: eq travels when gains are set but the band is off; not otherwise', () => { + const off = encodeParams(params({ eq: { enabled: false, gains: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] } })); + assert.deepEqual(off, { e: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] }); + const on = encodeParams(params({ eq: { enabled: true, gains: DEFAULT_PARAMS.eq.gains } })); + assert.deepEqual(on, { e: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }); +}); + +test('params: a row from before the switches existed reads as defaults', () => { + const old = { transpose: 2, pitchCents: 0, speed: 1, vocalReduce: 0, power: true }; + const back = decodeParams(encodeParams(old as unknown as EffectParams), 'x'); + assert.deepEqual(back, params({ transpose: 2 })); +}); + +// --------------------------------------------------------------------------- +// Settings / UI prefs / EQ presets + +test('settings: only what differs from the defaults is written', () => { + const b = backup(); + assert.deepEqual(encodeBackup(b).s, {}); + assert.deepEqual(encodeBackup(b).u, {}); + b.settings = { + ...b.settings, + theme: 'dark', + countInBpm: 120, + keymap: { ...b.settings.keymap, addMarker: 'x' }, + lastUsedParams: params({ transpose: 1 }), + }; + b.uiPrefs = { + ...b.uiPrefs, + accentHue: 30, + collapsed: { ...b.uiPrefs.collapsed, chords: false }, + boundaryLabels: { start: 'A', end: '' }, + }; + const enc = encodeBackup(b); + assert.deepEqual(enc.s, { + theme: 'dark', + countInBpm: 120, + keymap: { addMarker: 'x' }, + lp: { t: 1 }, + }); + assert.deepEqual(enc.u, { + accentHue: 30, + collapsed: { chords: false }, + boundaryLabels: { start: 'A' }, + }); + const back = roundTrip(b); + assert.deepEqual(back.settings, b.settings); + assert.deepEqual(back.uiPrefs, b.uiPrefs); +}); + +test('settings: a keymap saved before an action existed is backfilled', () => { + const b = backup(); + const { zoomFit: _dropped, ...keymap } = b.settings.keymap; + b.settings = { ...b.settings, keymap: keymap as typeof b.settings.keymap }; + assert.equal(roundTrip(b).settings.keymap.zoomFit, DEFAULT_SETTINGS.keymap.zoomFit); +}); + +test('eq presets round-trip as tuples', () => { + const b = backup({ eqPresets: [{ name: 'Mine', gains: [1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0] }] }); + assert.deepEqual(encodeBackup(b).eq, [['Mine', 1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0]]); + assert.deepEqual(roundTrip(b).eqPresets, b.eqPresets); +}); + +// --------------------------------------------------------------------------- +// Recent / Favorites + +test('recent: a plain YouTube row is an index and a timestamp', () => { + const b = backup({ history: [entry(ytSong)] }); + const enc = encodeBackup(b); + assert.deepEqual(enc.songs, [['yt:741FSo7Xb40', 'Symphony of Destruction', 230]]); + assert.deepEqual(enc.h, [{ i: 0, at: 1757112346 }]); + const back = roundTrip(b).history[0]; + assert.deepEqual(back.identity, ytSong); + assert.equal(back.pageUrl, YT_HREF); + assert.equal(back.thumbnailUrl, YT_THUMB); + assert.equal(back.updatedAt, 1757112346000); + assert.equal(back.createdAt, back.updatedAt); + assert.deepEqual(back.params, params()); +}); + +test('recent: a playlist href and a non-YouTube poster are kept verbatim', () => { + const playlist = `${YT_HREF}&list=PL123&index=4`; + const b = backup({ + history: [ + entry(ytSong, { pageUrl: playlist }), + entry(siteSong, { thumbnailUrl: 'https://example.com/poster.jpg' }), + entry(localSong), + ], + }); + const enc = encodeBackup(b); + assert.equal(enc.h[0].url, playlist); + assert.equal(enc.h[0].th, undefined); + assert.equal(enc.h[1].url, undefined); + assert.equal(enc.h[1].th, 'https://example.com/poster.jpg'); + assert.equal(enc.songs[1][0], 'https://example.com/lesson?id=42'); + assert.equal(enc.songs[2][0], localSong.normalizedUrl); + const back = roundTrip(b).history; + assert.equal(back[0].pageUrl, playlist); + assert.equal(back[1].thumbnailUrl, 'https://example.com/poster.jpg'); + assert.equal(back[1].pageUrl, siteSong.normalizedUrl); + assert.equal(back[2].thumbnailUrl, undefined); + assert.deepEqual(back[2].identity, localSong); +}); + +test('favorites keep their order and their two extra clocks', () => { + const b = backup({ + favorites: [ + favorite(siteSong, { favoritedAt: 3_000, lastAccessedAt: 9_000 }), + favorite(ytSong, { favoritedAt: 1_000, lastAccessedAt: 2_000, params: params({ speed: 0.5 }) }), + ], + }); + const back = roundTrip(b).favorites; + assert.deepEqual( + back.map((f) => [f.identity.key, f.favoritedAt, f.lastAccessedAt, f.params.speed]), + [ + [siteSong.key, 3_000, 9_000, 1], + [ytSong.key, 1_000, 2_000, 0.5], + ], + ); +}); + +// --------------------------------------------------------------------------- +// Tracks + +test('tracks: markers, snippets and flags round-trip; ids are regenerated', () => { + const b = backup({ + tracks: [ + track(ytSong, { + snippets: [ + ...track(ytSong).snippets, + { + id: 'x', + name: 'Forever', + startT: 1, + endT: 2, + enabled: false, + repeats: null as unknown as number, // Infinity after a storage round-trip + overrides: {}, + }, + { id: 'y', name: 'Plain', startT: 3, endT: 4, enabled: true, repeats: 1, overrides: {} }, + { id: 'z', name: 'Twice', startT: 3, endT: 4, enabled: true, repeats: 2, overrides: {} }, + ], + sequenceLoop: true, + sequenceCountIn: true, + chordsEnabled: undefined, + }), + ], + }); + const enc = encodeBackup(b).t[0]; + assert.deepEqual(enc.m, [ + [8200, 'Intro'], + [128473], + ]); + assert.deepEqual(enc.s, [ + ['Solo — half speed', 128470, 142120, 4, 1, { s: 0.5 }], + ['Forever', 1000, 2000, 0, 0], + ['Plain', 3000, 4000], + ['Twice', 3000, 4000, 2], + ]); + assert.equal(enc.L, 1); + assert.equal(enc.C, 1); + assert.equal(enc.ce, undefined); + assert.equal(enc.ch, undefined); + + const back = roundTrip(b).tracks[0]; + assert.deepEqual( + back.markers.map((m) => [m.id, m.t, m.label]), + [ + ['m1', 8.2, 'Intro'], + ['m2', 128.473, ''], + ], + ); + assert.deepEqual( + back.snippets.map((s) => [s.id, s.name, s.enabled, s.repeats, s.overrides]), + [ + ['c1', 'Solo — half speed', true, 4, { speed: 0.5 }], + ['c2', 'Forever', false, Infinity, {}], + ['c3', 'Plain', true, 1, {}], + ['c4', 'Twice', true, 2, {}], + ], + ); + assert.equal(back.sequenceLoop, true); + assert.equal(back.sequenceCountIn, true); + assert.equal('chordsEnabled' in back, false); + assert.equal(back.chordChart, null); +}); + +test('tracks: an empty chart is not written; chordsEnabled keeps false/true', () => { + const b = backup({ + tracks: [ + track(ytSong, { chordChart: chart(0), chordsEnabled: false }), + track(siteSong, { chordChart: undefined, chordsEnabled: true }), + ], + }); + const enc = encodeBackup(b).t; + assert.equal(enc.every((t) => t.ch === undefined), true); + const back = roundTrip(b).tracks; + const byKey = new Map(back.map((t) => [t.identity.key, t])); + assert.equal(byKey.get(ytSong.key)?.chordsEnabled, false); + assert.equal(byKey.get(siteSong.key)?.chordsEnabled, true); + assert.equal(byKey.get(ytSong.key)?.chordChart, null); +}); + +test('chart: times land within half a centisecond, gaps and key survive', () => { + const c = chart(120); + const enc = encodeChart(c); + assert.equal(enc.d.length, 120); + assert.equal(enc.i.length, 120); + assert.deepEqual(enc.l, ['C', 'Am', 'F', 'G', 'Dm7']); + assert.ok(enc.g, 'the dropped spans leave gaps'); + assert.deepEqual(enc.k, ['A', 1, 0.812]); + const back = roundTrip(backup({ tracks: [track(ytSong, { chordChart: c })] })).tracks[0] + .chordChart!; + assert.equal(back.segments.length, 120); + back.segments.forEach((seg, n) => { + assert.ok(Math.abs(seg.startT - c.segments[n].startT) <= 0.005, `start ${n}`); + assert.ok(Math.abs(seg.endT - c.segments[n].endT) <= 0.005, `end ${n}`); + assert.equal(seg.label, c.segments[n].label); + assert.equal(seg.confidence, 1); + }); + assert.deepEqual(back.key, { tonic: 'A', mode: 'minor', confidence: 0.812 }); + assert.equal(back.coverage, 0.988); + assert.ok(Math.abs(back.analyzedTo - c.analyzedTo) <= 0.005); + assert.equal(back.computedAt, 1757112346000); +}); + +test('chart: contiguous segments need no gap array; unsorted input is sorted', () => { + const c = chart(10, { key: null }); + c.segments = c.segments.map((s, n) => ({ ...s, startT: n * 2, endT: n * 2 + 2 })).reverse(); + const enc = encodeChart(c); + assert.equal(enc.g, undefined); + assert.equal(enc.k, undefined); + assert.equal(enc.t0, 0); + assert.deepEqual(enc.d, Array(10).fill(200)); + const back = roundTrip(backup({ tracks: [track(ytSong, { chordChart: c })] })).tracks[0] + .chordChart!; + assert.deepEqual( + back.segments.map((s) => s.startT), + [0, 2, 4, 6, 8, 10, 12, 14, 16, 18], + ); + assert.equal(back.key, null); +}); + +// --------------------------------------------------------------------------- +// Identity + +test('identity: the key is rebuilt from the URL and duration, never stored', () => { + const b = library(6); + const enc = encodeBackup(b); + assert.ok(enc.songs.every((row) => row.length === 3)); + const back = roundTrip(b); + assert.deepEqual( + back.history.map((h) => h.identity), + b.history.map((h) => h.identity), + ); + assert.deepEqual( + back.tracks.map((t) => t.identity.key).sort(), + b.tracks.map((t) => t.identity.key).sort(), + ); +}); + +test('identity: a key that cannot be rebuilt travels explicitly', () => { + const odd = { ...ytSong, key: 'legacy:230' }; + const b = backup({ history: [entry(odd)] }); + const enc = encodeBackup(b); + assert.equal(enc.songs[0][3], 'legacy:230'); + assert.equal(roundTrip(b).history[0].identity.key, 'legacy:230'); +}); + +test('identity: one song is one row; a duration that drifted is another', () => { + const drifted = makeTrackIdentity(YT_HREF, ytSong.title, 231); + const b = backup({ + history: [entry(ytSong)], + favorites: [favorite(ytSong)], + tracks: [track(ytSong), track(drifted)], + }); + const enc = encodeBackup(b); + assert.equal(enc.songs.length, 2); + assert.equal(enc.h[0].i, enc.f[0].i); + const back = roundTrip(b); + assert.deepEqual( + back.tracks.map((t) => t.identity.key).sort(), + [ytSong.key, drifted.key].sort(), + ); +}); + +// --------------------------------------------------------------------------- +// Whole file + +test('encode is idempotent past the first pass and JSON-clean', () => { + const b = library(12); + b.history[0].params.speed = 1.1500000000000001; + b.tracks[0].markers[1].t = 128.47321987; + b.settings.lastUsedParams = params({ baseBpm: 92.33333 }); + const first = encodeBackup(b); + const again = encodeBackup(decodeBackup(JSON.parse(JSON.stringify(first)))); + assert.deepEqual(again, first); + assert.deepEqual(JSON.parse(JSON.stringify(first)), first); + const text = JSON.stringify(decodeBackup(first)); + assert.ok(!text.includes('null,') || true); // nulls are legitimate (chordChart, baseBpm) + assert.ok(!/NaN/.test(text)); +}); + +test('encode is deterministic regardless of track enumeration order', () => { + const b = library(9); + const shuffled = backup({ ...b, tracks: [...b.tracks].reverse() }); + assert.deepEqual(encodeBackup(shuffled), encodeBackup(b)); +}); + +test('exportedAt is kept to the second; appVersion is dropped', () => { + const back = roundTrip(backup()); + assert.equal(back.exportedAt, 1_757_200_000_000); + assert.equal(back.appVersion, ''); + assert.equal(back.version, COMPACT_VERSION); +}); + +test('a verbose v1 file still parses and is backfilled', () => { + const v1 = JSON.parse(JSON.stringify(backup({ history: [entry(ytSong)] }))); + delete v1.settings.countInBeep; + delete v1.uiPrefs.accentHue; + const back = parseBackupJson(v1); + assert.equal(back.version, 1); + assert.equal(back.settings.countInBeep, DEFAULT_SETTINGS.countInBeep); + assert.equal(back.uiPrefs.accentHue, DEFAULT_UI_PREFS.accentHue); + assert.deepEqual(back.history, [entry(ytSong)]); + assert.equal(back.appVersion, '1.0.3'); +}); + +test('a v2 file routes through the codec; anything newer or foreign is refused', () => { + const v2 = JSON.parse(JSON.stringify(encodeBackup(library(2)))); + assert.equal(parseBackupJson(v2).history.length, 2); + assert.throws(() => parseBackupJson({ ...v2, version: 3 }), /newer version/); + assert.throws(() => parseBackupJson({ ...v2, format: 'other' }), /isn't a Note by Note backup/); + assert.throws(() => parseBackupJson('nope'), /isn't a Note by Note backup/); + assert.throws( + () => parseBackupJson({ format: BACKUP_FORMAT, version: 1, settings: {} }), + /"history" list is missing or damaged/, + ); +}); + +test('damaged v2 input is refused with the section named', () => { + const good = JSON.parse(JSON.stringify(encodeBackup(library(2)))); + const mutate = (fn: (raw: any) => void) => { + const raw = JSON.parse(JSON.stringify(good)); + fn(raw); + return () => decodeBackup(raw); + }; + assert.throws(mutate((r) => delete r.songs), /"songs" list is missing or damaged/); + assert.throws(mutate((r) => r.songs[0][0] = 'yt:bad/id'), /"songs" list is damaged/); + assert.throws(mutate((r) => r.h[0].i = 99), /"history" list is damaged/); + assert.throws(mutate((r) => r.h[0].at = 'now'), /"history" list is damaged/); + assert.throws(mutate((r) => r.f[0].fa = null), /"favorites" list is damaged/); + assert.throws(mutate((r) => r.t[0].m = [[1, 2, 3]]), /"tracks" list is damaged/); + assert.throws(mutate((r) => r.t[0].s = [['only name']]), /"tracks" list is damaged/); + assert.throws(mutate((r) => r.t[0].ch.i.pop()), /"tracks" list is damaged/); + assert.throws(mutate((r) => r.t[0].ch.i[0] = 42), /"tracks" list is damaged/); + assert.throws(mutate((r) => r.eq = [[]]), /"eqPresets" list is damaged/); + assert.throws(mutate((r) => r.h[0].p = { e: [1, 2] }), /"history" list is damaged/); +}); + +test('size: a library shrinks by more than 5×, a chart to under 1.5 KB', () => { + const b = library(50); + const raw = bytes(b); + const compact = bytes(encodeBackup(b)); + assert.ok(compact * 5 < raw, `${compact} vs ${raw}`); + const c = bytes(encodeChart(chart(120))); + assert.ok(c < 1500, `chart is ${c} bytes`); + const row = bytes(encodeBackup(backup({ history: [entry(ytSong, { params: params({ transpose: 2, speed: 0.75 }) })] })).h[0]); + assert.ok(row < 150, `recent row is ${row} bytes`); +}); diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts new file mode 100644 index 0000000..c1b89f7 --- /dev/null +++ b/src/core/persist/backup-codec.ts @@ -0,0 +1,811 @@ +import { + DEFAULT_KEYMAP, + DEFAULT_PARAMS, + DEFAULT_SETTINGS, + DEFAULT_UI_PREFS, +} from '../model/defaults.ts'; +import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; +import { identityKey } from '../model/track-identity.ts'; +import type { + ChordChart, + ChordSegment, + EffectParams, + EqPreset, + FavoriteEntry, + HistoryEntry, + Marker, + Settings, + Snippet, + SnippetOverrides, + TrackData, + TrackIdentity, + UiPrefs, +} from '../model/types'; + +/** + * The backup file format — the verbose in-memory `Backup` (v1) and its + * compact serialization (v2), which is what the export writes and what will + * ride the browser's sync storage. Nothing in `storage.local` changes: the + * codec shrinks the wire shape only, and `decodeBackup` hands back today's + * types. + * + * Pure and DOM-free so it runs under `node --test`; hence relative `.ts` + * imports and no `#imports` (see CLAUDE.md). + * + * Where the bytes go, and what v2 does about it: identities were repeated in + * Recent, Favorites and the track record (one `songs` table, referenced by + * index); every Recent row carried the full 13-field params object (a delta + * against the defaults, usually empty); chord charts spelled out four keys and + * 17-digit floats per segment (parallel arrays on a centisecond grid, labels + * through a per-chart table); settings/UI prefs carried every default (deep + * diff). Things derivable from what is kept are dropped: the identity key, + * YouTube thumbnails, the plain watch-page URL, marker/snippet ids. + * + * Every rounding is idempotent — `encode(decode(encode(x)))` deep-equals + * `encode(x)` — which is what will let two devices compare content hashes of + * data that both went through this. + */ + +/** Marks a file as ours, so a stray JSON can be rejected on sight. */ +export const BACKUP_FORMAT = 'note-by-note-backup'; + +/** The verbose shape: what `createBackup` builds and what exports were + * before the compact format. Still accepted on import. */ +export const BACKUP_VERSION = 1; + +/** The compact shape below. `parseBackupJson` rejects anything newer. */ +export const COMPACT_VERSION = 2; + +/** Everything a user owns, in one file. Host permissions are deliberately out: + * they live in the browser's permission store, and only a prompt can grant + * them — a backup that listed origins would restore access it can't give. */ +export interface Backup { + format: typeof BACKUP_FORMAT; + version: number; + exportedAt: number; + appVersion: string; + settings: Settings; + uiPrefs: UiPrefs; + history: HistoryEntry[]; + favorites: FavoriteEntry[]; + eqPresets: EqPreset[]; + /** Per-track markers and snippets, one entry per saved track. */ + tracks: TrackData[]; +} + +// --------------------------------------------------------------------------- +// Compact shape + +/** Effect params as a delta against `DEFAULT_PARAMS`; absent = default. */ +export interface CompactParams { + /** transpose (semitones) */ + t?: number; + /** transposeEnabled off */ + te?: 0; + /** pitchCents */ + c?: number; + /** pitchEnabled off */ + ce?: 0; + /** speed */ + s?: number; + /** speedEnabled off */ + se?: 0; + /** vocalReduce */ + v?: number; + /** vocalReduceEnabled off */ + ve?: 0; + /** vocalMode 'isolate' */ + vm?: 1; + /** eq: [enabled, ...gains] — present when enabled or any gain is non-zero */ + e?: number[]; + /** tuning: [trackHz, instrumentHz] — present when not 440/440 */ + tu?: [number, number]; + /** power off */ + pw?: 0; + /** baseBpm */ + b?: number; +} + +/** `[normalizedUrl, title, durationSec, key?]`. YouTube watch URLs are + * shortened to `yt:`. `key` appears only when it can't be rebuilt from the + * other two — a safety net, never the case for keys this build made. */ +export type CompactSong = [string, string, number] | [string, string, number, string]; + +export interface CompactEntry { + /** Index into `songs`. */ + i: number; + /** updatedAt, seconds. */ + at: number; + p?: CompactParams; + /** pageUrl, when not the song's plain page. */ + url?: string; + /** thumbnailUrl, when not derivable from the page URL. */ + th?: string; +} + +export interface CompactFavorite extends CompactEntry { + /** favoritedAt, seconds. */ + fa: number; + /** lastAccessedAt, seconds. */ + la: number; +} + +/** `[t_ms, label?]` — label omitted when empty. */ +export type CompactMarker = [number] | [number, string]; + +/** `[name, start_ms, end_ms, repeats?, enabled?, overrides?]`, trailing + * defaults omitted (`1`, `1`, `{}`). `repeats` 0 stands for Infinity. */ +export type CompactSnippet = [string, number, number, number?, number?, CompactOverrides?]; + +export interface CompactOverrides { + s?: number; + t?: number; + v?: number; +} + +/** Segments as parallel arrays on a centisecond grid. */ +export interface CompactChart { + /** First segment start. */ + t0: number; + /** Durations. */ + d: number[]; + /** Gap before each segment (index 0 is always 0); omitted when all zero. */ + g?: number[]; + /** Label table, first-appearance order. */ + l: string[]; + /** Label index per segment. */ + i: number[]; + /** Key signature: [tonic, minor, confidence]; omitted when none. */ + k?: [string, 0 | 1, number]; + cov: number; + a0: number; + a1: number; + /** computedAt, seconds. */ + c: number; +} + +export interface CompactTrack { + i: number; + /** updatedAt, seconds. */ + at: number; + m?: CompactMarker[]; + s?: CompactSnippet[]; + /** sequenceLoop */ + L?: 1; + /** sequenceCountIn */ + C?: 1; + /** chordsEnabled, only when the record has the switch at all. */ + ce?: 0 | 1; + ch?: CompactChart; +} + +export interface CompactBackup { + format: typeof BACKUP_FORMAT; + version: typeof COMPACT_VERSION; + /** exportedAt, seconds. */ + at: number; + /** Settings that differ from the defaults (`lastUsedParams` as `lp`). */ + s: Record; + /** UI prefs that differ from the defaults. */ + u: Record; + /** `[name, ...gains]` per saved EQ preset. */ + eq: (string | number)[][]; + songs: CompactSong[]; + h: CompactEntry[]; + f: CompactFavorite[]; + t: CompactTrack[]; +} + +// --------------------------------------------------------------------------- +// Shared helpers + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function damaged(section: string): Error { + return new Error(`This backup's "${section}" list is damaged.`); +} + +export function requireArray(value: unknown, field: string): unknown[] { + if (!Array.isArray(value)) { + throw new Error(`This backup's "${field}" list is missing or damaged.`); + } + return value; +} + +/** Entries are keyed by `identity.key`; without one they can't be stored or + * matched back to a track, so a file carrying them is not usable. */ +export function requireKeyedArray(value: unknown, field: string): T[] { + const list = requireArray(value, field); + const keyed = list.every( + (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.key === 'string', + ); + if (!keyed) throw damaged(field); + return list as T[]; +} + +const roundTo = (dp: number) => { + const f = 10 ** dp; + return (x: number) => Math.round(x * f) / f; +}; +const round2 = roundTo(2); +const round3 = roundTo(3); +const millis = (seconds: number) => Math.round(seconds * 1000); +const centis = (seconds: number) => Math.round(seconds * 100); +const secs = (ms: number) => Math.round((Number.isFinite(ms) ? ms : 0) / 1000); + +function num(value: unknown, section: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) throw damaged(section); + return value; +} + +function str(value: unknown, section: string): string { + if (typeof value !== 'string') throw damaged(section); + return value; +} + +function arr(value: unknown, section: string): unknown[] { + if (!Array.isArray(value)) throw damaged(section); + return value; +} + +function rec(value: unknown, section: string): Record { + if (!isRecord(value)) throw damaged(section); + return value; +} + +function jsonEqual(a: unknown, b: unknown): boolean { + return JSON.stringify(a) === JSON.stringify(b); +} + +/** Keys of `value` whose (JSON) value differs from `defaults`, recursing into + * plain objects. Keys unknown to `defaults` are kept verbatim. */ +function diffPlain( + value: Record, + defaults: Record, +): Record { + const out: Record = {}; + for (const [key, v] of Object.entries(value)) { + if (v === undefined) continue; + const d = defaults[key]; + if (isRecord(v) && isRecord(d)) { + const nested = diffPlain(v, d); + if (Object.keys(nested).length) out[key] = nested; + } else if (!(key in defaults) || !jsonEqual(v, d)) { + out[key] = v; + } + } + return out; +} + +/** The inverse of `diffPlain`: a deep clone of `defaults` with `diff` laid + * over it. */ +function mergePlain( + defaults: Record, + diff: Record, +): Record { + const out: Record = JSON.parse(JSON.stringify(defaults)); + for (const [key, v] of Object.entries(diff)) { + if (v === undefined) continue; + const d = out[key]; + out[key] = isRecord(v) && isRecord(d) ? mergePlain(d, v) : v; + } + return out; +} + +// --------------------------------------------------------------------------- +// Effect params + +/** Old rows may predate a field (the switches, tuning, vocalMode, baseBpm); + * a missing one reads as its default, which is how the UI treats it too. */ +export function encodeParams(p: EffectParams): CompactParams | undefined { + const out: CompactParams = {}; + const transpose = round3(p.transpose ?? 0); + if (transpose !== 0) out.t = transpose; + if (p.transposeEnabled === false) out.te = 0; + const cents = round3(p.pitchCents ?? 0); + if (cents !== 0) out.c = cents; + if (p.pitchEnabled === false) out.ce = 0; + const speed = round3(p.speed ?? 1); + if (speed !== 1) out.s = speed; + if (p.speedEnabled === false) out.se = 0; + const vocal = round3(p.vocalReduce ?? 0); + if (vocal !== 0) out.v = vocal; + if (p.vocalReduceEnabled === false) out.ve = 0; + if (p.vocalMode === 'isolate') out.vm = 1; + const eq = p.eq ?? DEFAULT_PARAMS.eq; + const gains = (eq.gains ?? DEFAULT_PARAMS.eq.gains).map(round2); + if (eq.enabled || gains.some((g) => g !== 0)) out.e = [eq.enabled ? 1 : 0, ...gains]; + const tuning = p.tuning ?? DEFAULT_PARAMS.tuning; + const trackHz = round2(tuning.trackHz ?? 440); + const instrumentHz = round2(tuning.instrumentHz ?? 440); + if (trackHz !== 440 || instrumentHz !== 440) out.tu = [trackHz, instrumentHz]; + if (p.power === false) out.pw = 0; + if (typeof p.baseBpm === 'number' && Number.isFinite(p.baseBpm)) out.b = round2(p.baseBpm); + return Object.keys(out).length ? out : undefined; +} + +function defaultParams(): EffectParams { + return { + ...DEFAULT_PARAMS, + eq: { enabled: false, gains: [...DEFAULT_PARAMS.eq.gains] }, + tuning: { ...DEFAULT_PARAMS.tuning }, + }; +} + +export function decodeParams(raw: unknown, section: string): EffectParams { + const p = defaultParams(); + if (raw === undefined) return p; + const c = rec(raw, section); + if (c.t !== undefined) p.transpose = num(c.t, section); + if (c.te !== undefined) p.transposeEnabled = false; + if (c.c !== undefined) p.pitchCents = num(c.c, section); + if (c.ce !== undefined) p.pitchEnabled = false; + if (c.s !== undefined) p.speed = num(c.s, section); + if (c.se !== undefined) p.speedEnabled = false; + if (c.v !== undefined) p.vocalReduce = num(c.v, section); + if (c.ve !== undefined) p.vocalReduceEnabled = false; + if (c.vm !== undefined) p.vocalMode = 'isolate'; + if (c.e !== undefined) { + const e = arr(c.e, section); + if (e.length !== 1 + p.eq.gains.length) throw damaged(section); + p.eq = { enabled: num(e[0], section) === 1, gains: e.slice(1).map((g) => num(g, section)) }; + } + if (c.tu !== undefined) { + const tu = arr(c.tu, section); + if (tu.length !== 2) throw damaged(section); + p.tuning = { trackHz: num(tu[0], section), instrumentHz: num(tu[1], section) }; + } + if (c.pw !== undefined) p.power = false; + if (c.b !== undefined) p.baseBpm = num(c.b, section); + return p; +} + +// --------------------------------------------------------------------------- +// Settings / UI prefs + +const SETTINGS_DEFAULTS: Record = { + ...DEFAULT_SETTINGS, + keymap: { ...DEFAULT_KEYMAP }, +}; + +export function encodeSettings(settings: Settings): Record { + const { lastUsedParams, ...rest } = settings; + const out = diffPlain(rest, SETTINGS_DEFAULTS); + if (lastUsedParams) out.lp = encodeParams(lastUsedParams) ?? {}; + return out; +} + +export function decodeSettings(raw: unknown): Settings { + const diff = raw === undefined ? {} : rec(raw, 'settings'); + const { lp, ...rest } = diff; + const settings = mergePlain(SETTINGS_DEFAULTS, rest) as unknown as Settings; + if (lp !== undefined) settings.lastUsedParams = decodeParams(lp, 'settings'); + return settings; +} + +export function encodeUiPrefs(uiPrefs: UiPrefs): Record { + return diffPlain( + uiPrefs as unknown as Record, + DEFAULT_UI_PREFS as unknown as Record, + ); +} + +export function decodeUiPrefs(raw: unknown): UiPrefs { + const diff = raw === undefined ? {} : rec(raw, 'uiPrefs'); + return mergePlain( + DEFAULT_UI_PREFS as unknown as Record, + diff, + ) as unknown as UiPrefs; +} + +// --------------------------------------------------------------------------- +// Songs (identities) + +const YT_WATCH = 'https://youtube.com/watch?v='; +const YT_ID_RE = /^[\w-]+$/; + +function shortUrl(normalizedUrl: string): string { + if (normalizedUrl.startsWith(YT_WATCH)) { + const id = normalizedUrl.slice(YT_WATCH.length); + if (YT_ID_RE.test(id)) return `yt:${id}`; + } + return normalizedUrl; +} + +function longUrl(short: string): string { + if (short.startsWith('yt:')) { + const id = short.slice(3); + if (!YT_ID_RE.test(id)) throw damaged('songs'); + return YT_WATCH + id; + } + return short; +} + +/** The page a song is opened at when the entry carries no `url` of its own: + * the engine records `location.href`, which on YouTube is the `www.` form of + * the watch page — so that, not the normalized URL, is the default there. */ +function defaultPageUrl(normalizedUrl: string): string { + if (normalizedUrl.startsWith(YT_WATCH)) { + return `https://www.youtube.com/watch?v=${normalizedUrl.slice(YT_WATCH.length)}`; + } + return normalizedUrl; +} + +/** One row per distinct (url, title, duration) — not per URL: the duration is + * part of the key, and a song whose duration drifted legitimately has two. */ +class SongTable { + rows: CompactSong[] = []; + #index = new Map(); + + add(identity: TrackIdentity): number { + const url = identity.normalizedUrl ?? ''; + const title = identity.title ?? ''; + const duration = Number.isFinite(identity.durationSec) ? identity.durationSec : 0; + const key = identity.key ?? identityKey(url, duration); + const tableKey = `${url}\n${title}\n${duration}\n${key}`; + const existing = this.#index.get(tableKey); + if (existing !== undefined) return existing; + const row: CompactSong = + key === identityKey(url, duration) + ? [shortUrl(url), title, duration] + : [shortUrl(url), title, duration, key]; + this.rows.push(row); + this.#index.set(tableKey, this.rows.length - 1); + return this.rows.length - 1; + } +} + +function decodeSongs(raw: unknown): TrackIdentity[] { + return requireArray(raw, 'songs').map((row) => { + const r = arr(row, 'songs'); + if (r.length < 3 || r.length > 4) throw damaged('songs'); + const normalizedUrl = longUrl(str(r[0], 'songs')); + const title = str(r[1], 'songs'); + const durationSec = num(r[2], 'songs'); + const key = r.length === 4 ? str(r[3], 'songs') : identityKey(normalizedUrl, durationSec); + return { key, normalizedUrl, title, durationSec }; + }); +} + +function songAt(songs: TrackIdentity[], index: unknown, section: string): TrackIdentity { + if (typeof index !== 'number' || !Number.isInteger(index) || index < 0 || index >= songs.length) { + throw damaged(section); + } + return songs[index]; +} + +// --------------------------------------------------------------------------- +// Recent / Favorites + +function encodeEntry(entry: HistoryEntry, songs: SongTable): CompactEntry { + const out: CompactEntry = { i: songs.add(entry.identity), at: secs(entry.updatedAt) }; + const params = encodeParams(entry.params ?? DEFAULT_PARAMS); + if (params) out.p = params; + const pageUrl = entry.pageUrl ?? ''; + if (pageUrl !== defaultPageUrl(entry.identity.normalizedUrl)) out.url = pageUrl; + if (entry.thumbnailUrl && entry.thumbnailUrl !== youtubeThumbnailUrl(pageUrl)) { + out.th = entry.thumbnailUrl; + } + return out; +} + +function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): HistoryEntry { + const c = rec(raw, section); + const identity = songAt(songs, c.i, section); + const updatedAt = num(c.at, section) * 1000; + const pageUrl = c.url === undefined ? defaultPageUrl(identity.normalizedUrl) : str(c.url, section); + const thumbnailUrl = c.th === undefined ? youtubeThumbnailUrl(pageUrl) : str(c.th, section); + const entry: HistoryEntry = { + identity: { ...identity }, + params: decodeParams(c.p, section), + pageUrl, + createdAt: updatedAt, + updatedAt, + }; + if (thumbnailUrl !== undefined) entry.thumbnailUrl = thumbnailUrl; + return entry; +} + +function encodeFavorite(entry: FavoriteEntry, songs: SongTable): CompactFavorite { + return { + ...encodeEntry(entry, songs), + fa: secs(entry.favoritedAt), + la: secs(entry.lastAccessedAt), + }; +} + +function decodeFavorite(raw: unknown, songs: TrackIdentity[]): FavoriteEntry { + const c = rec(raw, 'favorites'); + return { + ...decodeEntry(c, songs, 'favorites'), + favoritedAt: num(c.fa, 'favorites') * 1000, + lastAccessedAt: num(c.la, 'favorites') * 1000, + }; +} + +// --------------------------------------------------------------------------- +// Tracks + +function encodeMarker(marker: Marker): CompactMarker { + const t = millis(marker.t); + return marker.label ? [t, marker.label] : [t]; +} + +function decodeMarker(raw: unknown, index: number): Marker { + const r = arr(raw, 'tracks'); + if (r.length < 1 || r.length > 2) throw damaged('tracks'); + return { + id: `m${index + 1}`, + t: num(r[0], 'tracks') / 1000, + label: r.length === 2 ? str(r[1], 'tracks') : '', + }; +} + +function encodeOverrides(overrides: SnippetOverrides | undefined): CompactOverrides { + const out: CompactOverrides = {}; + if (!overrides) return out; + if (typeof overrides.speed === 'number') out.s = round3(overrides.speed); + if (typeof overrides.transpose === 'number') out.t = round3(overrides.transpose); + if (typeof overrides.vocalReduce === 'number') out.v = round3(overrides.vocalReduce); + return out; +} + +function decodeOverrides(raw: unknown): SnippetOverrides { + const c = rec(raw, 'tracks'); + const out: SnippetOverrides = {}; + if (c.s !== undefined) out.speed = num(c.s, 'tracks'); + if (c.t !== undefined) out.transpose = num(c.t, 'tracks'); + if (c.v !== undefined) out.vocalReduce = num(c.v, 'tracks'); + return out; +} + +function encodeSnippet(snippet: Snippet): CompactSnippet { + // `repeats: Infinity` is `null` once it has been through JSON (storage); + // both mean "forever", written as 0 — real counts start at 1. + const repeats = + typeof snippet.repeats === 'number' && Number.isFinite(snippet.repeats) ? snippet.repeats : 0; + const overrides = encodeOverrides(snippet.overrides); + const out: CompactSnippet = [ + snippet.name ?? '', + millis(snippet.startT), + millis(snippet.endT), + repeats, + snippet.enabled === false ? 0 : 1, + overrides, + ]; + if (Object.keys(overrides).length === 0) { + out.pop(); + if (out[4] === 1) { + out.pop(); + if (out[3] === 1) out.pop(); + } + } + return out; +} + +function decodeSnippet(raw: unknown, index: number): Snippet { + const r = arr(raw, 'tracks'); + if (r.length < 3 || r.length > 6) throw damaged('tracks'); + const repeats = r.length > 3 ? num(r[3], 'tracks') : 1; + return { + id: `c${index + 1}`, + name: str(r[0], 'tracks'), + startT: num(r[1], 'tracks') / 1000, + endT: num(r[2], 'tracks') / 1000, + enabled: r.length > 4 ? num(r[4], 'tracks') === 1 : true, + repeats: repeats === 0 ? Infinity : repeats, + overrides: r.length > 5 ? decodeOverrides(r[5]) : {}, + }; +} + +export function encodeChart(chart: ChordChart): CompactChart { + const segments = [...chart.segments].sort((a, b) => a.startT - b.startT); + const d: number[] = []; + const g: number[] = []; + const l: string[] = []; + const i: number[] = []; + const labelIndex = new Map(); + let t0 = 0; + let prevEnd = 0; + let anyGap = false; + segments.forEach((seg, n) => { + const start = centis(seg.startT); + const end = Math.max(start, centis(seg.endT)); + if (n === 0) { + t0 = start; + g.push(0); + } else { + const gap = start - prevEnd; + if (gap !== 0) anyGap = true; + g.push(gap); + } + d.push(end - start); + let li = labelIndex.get(seg.label); + if (li === undefined) { + li = l.length; + l.push(seg.label); + labelIndex.set(seg.label, li); + } + i.push(li); + prevEnd = end; + }); + const out: CompactChart = { + t0, + d, + l, + i, + cov: round3(chart.coverage ?? 0), + a0: centis(chart.analyzedFrom ?? 0), + a1: centis(chart.analyzedTo ?? 0), + c: secs(chart.computedAt), + }; + if (anyGap) out.g = g; + if (chart.key) { + out.k = [chart.key.tonic, chart.key.mode === 'minor' ? 1 : 0, round3(chart.key.confidence)]; + } + return out; +} + +export function decodeChart(raw: unknown): ChordChart { + const c = rec(raw, 'tracks'); + const d = arr(c.d, 'tracks'); + const l = arr(c.l, 'tracks').map((label) => str(label, 'tracks')); + const i = arr(c.i, 'tracks'); + const g = c.g === undefined ? undefined : arr(c.g, 'tracks'); + if (i.length !== d.length || (g !== undefined && g.length !== d.length)) throw damaged('tracks'); + const segments: ChordSegment[] = []; + let acc = num(c.t0, 'tracks'); + for (let n = 0; n < d.length; n++) { + const li = num(i[n], 'tracks'); + if (!Number.isInteger(li) || li < 0 || li >= l.length) throw damaged('tracks'); + const start = acc + (g === undefined ? 0 : num(g[n], 'tracks')); + const end = start + num(d[n], 'tracks'); + segments.push({ startT: start / 100, endT: end / 100, label: l[li], confidence: 1 }); + acc = end; + } + let key: ChordChart['key'] = null; + if (c.k !== undefined) { + const k = arr(c.k, 'tracks'); + if (k.length !== 3) throw damaged('tracks'); + key = { + tonic: str(k[0], 'tracks'), + mode: num(k[1], 'tracks') === 1 ? 'minor' : 'major', + confidence: num(k[2], 'tracks'), + }; + } + return { + segments, + key, + coverage: num(c.cov, 'tracks'), + analyzedFrom: num(c.a0, 'tracks') / 100, + analyzedTo: num(c.a1, 'tracks') / 100, + computedAt: num(c.c, 'tracks') * 1000, + }; +} + +function encodeTrack(track: TrackData, songs: SongTable): CompactTrack { + const out: CompactTrack = { i: songs.add(track.identity), at: secs(track.updatedAt) }; + if (track.markers?.length) out.m = track.markers.map(encodeMarker); + if (track.snippets?.length) out.s = track.snippets.map(encodeSnippet); + if (track.sequenceLoop) out.L = 1; + if (track.sequenceCountIn) out.C = 1; + if (track.chordsEnabled !== undefined) out.ce = track.chordsEnabled ? 1 : 0; + if (track.chordChart && track.chordChart.segments?.length) { + out.ch = encodeChart(track.chordChart); + } + return out; +} + +function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { + const c = rec(raw, 'tracks'); + const track: TrackData = { + identity: { ...songAt(songs, c.i, 'tracks') }, + markers: c.m === undefined ? [] : arr(c.m, 'tracks').map(decodeMarker), + snippets: c.s === undefined ? [] : arr(c.s, 'tracks').map(decodeSnippet), + sequenceLoop: c.L !== undefined, + sequenceCountIn: c.C !== undefined, + chordChart: c.ch === undefined ? null : decodeChart(c.ch), + updatedAt: num(c.at, 'tracks') * 1000, + }; + if (c.ce !== undefined) track.chordsEnabled = num(c.ce, 'tracks') === 1; + return track; +} + +// --------------------------------------------------------------------------- +// Whole backup + +function encodeEqPreset(preset: EqPreset): (string | number)[] { + return [preset.name ?? '', ...(preset.gains ?? []).map(round2)]; +} + +function decodeEqPreset(raw: unknown): EqPreset { + const r = arr(raw, 'eqPresets'); + if (r.length < 1) throw damaged('eqPresets'); + return { name: str(r[0], 'eqPresets'), gains: r.slice(1).map((g) => num(g, 'eqPresets')) }; +} + +const byKey = (a: { identity: TrackIdentity }, b: { identity: TrackIdentity }) => + a.identity.key < b.identity.key ? -1 : a.identity.key > b.identity.key ? 1 : 0; + +/** Deterministic for equal input: tracks are sorted by key (their storage + * enumeration order is arbitrary) and the song table is filled in Recent, + * Favorites, tracks order. */ +export function encodeBackup(backup: Backup): CompactBackup { + const songs = new SongTable(); + const h = backup.history.map((e) => encodeEntry(e, songs)); + const f = backup.favorites.map((e) => encodeFavorite(e, songs)); + const t = [...backup.tracks].sort(byKey).map((track) => encodeTrack(track, songs)); + return { + format: BACKUP_FORMAT, + version: COMPACT_VERSION, + at: secs(backup.exportedAt), + s: encodeSettings(backup.settings), + u: encodeUiPrefs(backup.uiPrefs), + eq: backup.eqPresets.map(encodeEqPreset), + songs: songs.rows, + h, + f, + t, + }; +} + +/** Reads a compact (v2) backup, or throws an `Error` whose message is safe to + * show the user. Unknown keys are ignored so the format can grow. */ +export function decodeBackup(raw: unknown): Backup { + if (!isRecord(raw) || raw.format !== BACKUP_FORMAT || raw.version !== COMPACT_VERSION) { + throw new Error("That file isn't a Note by Note backup."); + } + const songs = decodeSongs(raw.songs); + return { + format: BACKUP_FORMAT, + version: COMPACT_VERSION, + exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at * 1000 : 0, + appVersion: '', + settings: decodeSettings(raw.s), + uiPrefs: decodeUiPrefs(raw.u), + history: requireArray(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), + favorites: requireArray(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), + eqPresets: requireArray(raw.eq, 'eqPresets').map(decodeEqPreset), + tracks: requireArray(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), + }; +} + +/** The verbose v1 file: today's in-memory shape, written out as is. Objects + * are backfilled from the defaults so a file from an older build gains any + * setting added since. */ +function normalizeV1(raw: Record): Backup { + return { + format: BACKUP_FORMAT, + version: BACKUP_VERSION, + exportedAt: typeof raw.exportedAt === 'number' ? raw.exportedAt : 0, + appVersion: typeof raw.appVersion === 'string' ? raw.appVersion : '', + settings: { + ...DEFAULT_SETTINGS, + ...(isRecord(raw.settings) ? raw.settings : {}), + } as Settings, + uiPrefs: { + ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), + ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), + }, + history: requireKeyedArray(raw.history, 'history'), + favorites: requireKeyedArray(raw.favorites, 'favorites'), + eqPresets: requireArray(raw.eqPresets, 'eqPresets') as EqPreset[], + tracks: requireKeyedArray(raw.tracks, 'tracks'), + }; +} + +/** + * Reads a parsed backup file (any version this build knows) into a `Backup`, + * or throws an `Error` whose message is safe to show the user. + */ +export function parseBackupJson(raw: unknown): Backup { + if (!isRecord(raw) || raw.format !== BACKUP_FORMAT) { + throw new Error("That file isn't a Note by Note backup."); + } + if (typeof raw.version !== 'number' || raw.version > COMPACT_VERSION) { + throw new Error('That backup was made by a newer version of Note by Note.'); + } + return raw.version === COMPACT_VERSION ? decodeBackup(raw) : normalizeV1(raw); +} diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index a20e809..0bbea2b 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -1,12 +1,10 @@ -import { DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults'; -import type { - EqPreset, - FavoriteEntry, - HistoryEntry, - Settings, - TrackData, - UiPrefs, -} from '../model/types'; +import type { TrackData } from '../model/types'; +import { + BACKUP_FORMAT, + BACKUP_VERSION, + parseBackupJson, + type Backup, +} from './backup-codec'; import { eqPresetsItem, favoritesItem, @@ -17,33 +15,9 @@ import { uiPrefsItem, } from './storage'; -/** Marks a file as ours, so a stray JSON can be rejected on sight. */ -const FORMAT = 'note-by-note-backup'; - -/** Bump only for changes older files can't be read as; `parseBackup` accepts - * anything up to this and backfills what a lower version lacked. */ -const VERSION = 1; - -/** Everything a user owns, in one file. Host permissions are deliberately out: - * they live in the browser's permission store, and only a prompt can grant - * them — a backup that listed origins would restore access it can't give. */ -export interface Backup { - format: typeof FORMAT; - version: number; - exportedAt: number; - appVersion: string; - settings: Settings; - uiPrefs: UiPrefs; - history: HistoryEntry[]; - favorites: FavoriteEntry[]; - eqPresets: EqPreset[]; - /** Per-track markers and snippets, one entry per saved track. */ - tracks: TrackData[]; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} +/** The file shape and its compact codec live in `backup-codec.ts` (pure, so + * they run under `node --test`); this module is the storage side. */ +export type { Backup }; /** Raw storage keys have no `local:` prefix — see `trackDataKey`. */ async function loadAllTrackData(): Promise { @@ -64,8 +38,8 @@ export async function createBackup(): Promise { loadAllTrackData(), ]); return { - format: FORMAT, - version: VERSION, + format: BACKUP_FORMAT, + version: BACKUP_VERSION, exportedAt: Date.now(), appVersion: browser.runtime.getManifest().version, settings, @@ -83,28 +57,12 @@ export function backupFilename(exportedAt: number): string { return `note-by-note-backup-${day}.json`; } -function requireArray(value: unknown, field: string): unknown[] { - if (!Array.isArray(value)) { - throw new Error(`This backup's "${field}" list is missing or damaged.`); - } - return value; -} - -/** Entries are keyed by `identity.key`; without one they can't be stored or - * matched back to a track, so a file carrying them is not usable. */ -function requireKeyedArray(value: unknown, field: string): T[] { - const list = requireArray(value, field); - const keyed = list.every( - (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.key === 'string', - ); - if (!keyed) throw new Error(`This backup's "${field}" list is damaged.`); - return list as T[]; -} - /** * Reads a backup file's text into a `Backup`, or throws an `Error` whose - * message is safe to show the user. Objects are backfilled from the defaults - * so a file from an older build gains any setting added since. + * message is safe to show the user. Accepts the compact format the export + * writes and the verbose one older builds wrote; either way objects are + * backfilled from the defaults so a file from an older build gains any setting + * added since. */ export function parseBackup(text: string): Backup { let raw: unknown; @@ -113,27 +71,7 @@ export function parseBackup(text: string): Backup { } catch { throw new Error("That file isn't valid JSON."); } - if (!isRecord(raw) || raw.format !== FORMAT) { - throw new Error("That file isn't a Note by Note backup."); - } - if (typeof raw.version !== 'number' || raw.version > VERSION) { - throw new Error('That backup was made by a newer version of Note by Note.'); - } - return { - format: FORMAT, - version: raw.version, - exportedAt: typeof raw.exportedAt === 'number' ? raw.exportedAt : 0, - appVersion: typeof raw.appVersion === 'string' ? raw.appVersion : '', - settings: { ...DEFAULT_SETTINGS, ...(isRecord(raw.settings) ? raw.settings : {}) }, - uiPrefs: { - ...structuredClone(DEFAULT_UI_PREFS), - ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), - }, - history: requireKeyedArray(raw.history, 'history'), - favorites: requireKeyedArray(raw.favorites, 'favorites'), - eqPresets: requireArray(raw.eqPresets, 'eqPresets') as EqPreset[], - tracks: requireKeyedArray(raw.tracks, 'tracks'), - }; + return parseBackupJson(raw); } /** diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index 1c74f86..5c8877e 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -18,6 +18,7 @@ parseBackup, restoreBackup, } from '@/core/persist/backup'; + import { encodeBackup } from '@/core/persist/backup-codec'; import { history } from '@/features/library/panel/history.svelte'; import { applyTheme, settings } from '@/features/settings/panel/settings.svelte'; import { session } from '@/core/state/session.svelte'; @@ -136,9 +137,10 @@ notice = null; try { const backup = await createBackup(); - const blob = new Blob([JSON.stringify(backup, null, 2)], { - type: 'application/json', - }); + // The compact form — a fraction of the verbose one and the shape that + // will ride the browser's sync storage; import reads both. + const text = JSON.stringify(encodeBackup(backup)); + const blob = new Blob([text], { type: 'application/json' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; @@ -148,7 +150,8 @@ // outlive this task. setTimeout(() => URL.revokeObjectURL(url), 0); const songs = backup.history.length + backup.favorites.length; - notice = { ok: true, text: `Saved ${link.download} (${songs} songs).` }; + const kb = Math.max(1, Math.round(new TextEncoder().encode(text).length / 1024)); + notice = { ok: true, text: `Saved ${link.download} (${songs} songs, ${kb} KB).` }; } catch (err) { notice = { ok: false, text: `Export failed: ${message(err)}` }; } finally { diff --git a/src/features/sync/persist/fit.test.ts b/src/features/sync/persist/fit.test.ts new file mode 100644 index 0000000..c8ce5cf --- /dev/null +++ b/src/features/sync/persist/fit.test.ts @@ -0,0 +1,214 @@ +// Run with: pnpm test:dsp (node --test). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fitBackup, LibraryTooLargeError } from './fit.ts'; +import { BACKUP_FORMAT, encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../../../core/model/defaults.ts'; +import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; +import type { + ChordChart, + FavoriteEntry, + HistoryEntry, + TrackData, + TrackIdentity, +} from '../../../core/model/types.ts'; + +const measure = (b: Backup) => JSON.stringify(encodeBackup(b)).length; + +const T0 = 1_757_000_000_000; + +function song(n: number): TrackIdentity { + return makeTrackIdentity( + `https://www.youtube.com/watch?v=vid${n.toString().padStart(8, '0')}`, + `Song ${n}`, + 200 + n, + ); +} + +function chart(segments: number, computedAt: number): ChordChart { + return { + segments: Array.from({ length: segments }, (_, i) => ({ + startT: i * 2, + endT: i * 2 + 2, + label: ['C', 'G', 'Am', 'F'][i % 4], + confidence: 1, + })), + key: null, + coverage: 1, + analyzedFrom: 0, + analyzedTo: segments * 2, + computedAt, + }; +} + +function row(identity: TrackIdentity, updatedAt: number): HistoryEntry { + return { + identity, + params: { ...DEFAULT_PARAMS, transpose: 1 }, + pageUrl: `https://www.youtube.com/watch?v=${identity.normalizedUrl.slice(-11)}`, + createdAt: updatedAt, + updatedAt, + }; +} + +function record(identity: TrackIdentity, updatedAt: number, withChart: boolean): TrackData { + return { + identity, + markers: [{ id: 'm1', t: 10, label: 'Verse' }], + snippets: [], + sequenceLoop: false, + sequenceCountIn: false, + chordChart: withChart ? chart(40, updatedAt) : null, + updatedAt, + }; +} + +/** + * `recent` non-favorited songs (row + record + chart), `favorites` favorited + * ones (row + favorite + record + chart), `orphans` records with no row. + * Song n was touched at T0 + n minutes, so higher n = newer everywhere. + */ +function library(recent: number, favorites: number, orphans = 0): Backup { + const history: HistoryEntry[] = []; + const favs: FavoriteEntry[] = []; + const tracks: TrackData[] = []; + let n = 0; + for (let i = 0; i < recent; i++, n++) { + const id = song(n); + const at = T0 + n * 60_000; + history.push(row(id, at)); + tracks.push(record(id, at, true)); + } + for (let i = 0; i < favorites; i++, n++) { + const id = song(n); + const at = T0 + n * 60_000; + history.push(row(id, at)); + favs.push({ ...row(id, at), favoritedAt: at, lastAccessedAt: at }); + tracks.push(record(id, at, true)); + } + for (let i = 0; i < orphans; i++, n++) { + const id = song(n); + tracks.push(record(id, T0 + n * 60_000, false)); + } + // Storage enumeration order is arbitrary; make sure nothing relies on it. + tracks.reverse(); + return { + format: BACKUP_FORMAT, + version: 1, + exportedAt: T0, + appVersion: '', + settings: DEFAULT_SETTINGS, + uiPrefs: DEFAULT_UI_PREFS, + history, + favorites: favs, + eqPresets: [], + tracks, + }; +} + +const keys = (list: { identity: TrackIdentity }[]) => list.map((e) => e.identity.key).sort(); +const charts = (b: Backup) => b.tracks.filter((t) => t.chordChart?.segments.length).length; + +test('nothing is cut while the library fits', () => { + const b = library(5, 2); + const result = fitBackup(b, measure(b), measure); + assert.equal(result.trimmed, false); + assert.equal(result.size, measure(b)); + assert.deepEqual(result.backup, b); +}); + +test('recent songs go first, oldest first, taking their records along', () => { + const b = library(6, 2); + const withoutTwoOldest = fitBackup(b, measure(b) - 1, measure).backup; + assert.ok(withoutTwoOldest.history.length < b.history.length); + assert.equal(withoutTwoOldest.favorites.length, 2, 'favorites untouched'); + assert.equal(charts(withoutTwoOldest), charts(b) - (b.history.length - withoutTwoOldest.history.length), 'only the cut songs lost their charts'); + // The rows that survived are the newest ones. + const survivors = withoutTwoOldest.history.map((h) => h.updatedAt); + const cut = b.history + .filter((h) => !survivors.includes(h.updatedAt)) + .map((h) => h.updatedAt); + assert.ok(Math.max(...cut) < Math.min(...survivors)); + // Records follow their rows. + assert.deepEqual(keys(withoutTwoOldest.tracks), keys([...withoutTwoOldest.history])); +}); + +test('charts go next, oldest first; favorites and their markers stay', () => { + const b = library(3, 3); + const noRecent = fitBackup(b, measure(b), measure); + // Find the budget at which every non-favorite is gone but charts remain. + const favoritesOnly = { ...b, history: b.history.slice(3), tracks: b.tracks.filter((t) => keys(b.favorites).includes(t.identity.key)) }; + const result = fitBackup(b, measure(favoritesOnly) - 1, measure); + assert.equal(result.trimmed, true); + assert.equal(result.backup.favorites.length, 3); + assert.equal(result.backup.history.length, 3, 'favorites keep their Recent rows'); + assert.equal(result.backup.tracks.length, 3, 'favorites keep their records'); + assert.ok(result.backup.tracks.every((t) => t.markers.length === 1), 'markers intact'); + const kept = charts(result.backup); + assert.ok(kept > 0 && kept < 3, `some charts cut, not all: ${kept}`); + const keptAt = result.backup.tracks + .filter((t) => t.chordChart) + .map((t) => t.chordChart!.computedAt); + const cutAt = result.backup.tracks + .filter((t) => !t.chordChart) + .map((t) => t.updatedAt); + assert.ok(Math.max(...cutAt) < Math.min(...keptAt), 'the oldest charts went'); + assert.equal(noRecent.trimmed, false); +}); + +test('favorites go last, least recently accessed first', () => { + const b = library(2, 4); + // Exactly the two newest favorites, with their rows and chart-less records. + const newest = keys(b.favorites.slice(2)); + const expected = { + ...b, + history: b.history.filter((h) => newest.includes(h.identity.key)), + favorites: b.favorites.slice(2), + tracks: b.tracks + .filter((t) => newest.includes(t.identity.key)) + .map((t) => ({ ...t, chordChart: null })), + }; + const result = fitBackup(b, measure(expected), measure); + assert.equal(charts(result.backup), 0, 'every chart went before a favorite'); + assert.equal(result.backup.favorites.length, 2); + const survivors = result.backup.favorites.map((f) => f.lastAccessedAt); + const cut = b.favorites + .filter((f) => !survivors.includes(f.lastAccessedAt)) + .map((f) => f.lastAccessedAt); + assert.ok(Math.max(...cut) < Math.min(...survivors)); + assert.deepEqual(keys(result.backup.tracks), keys(result.backup.favorites), 'records follow'); + assert.deepEqual(keys(result.backup.history), keys(result.backup.favorites), 'rows follow'); +}); + +test('orphan records sit in the recent tier by their own edit time', () => { + const b = library(2, 1, 2); + const oneOrphanLess = { ...b, tracks: b.tracks.filter((t) => t.identity.key !== song(3).key) }; + const result = fitBackup(b, measure(oneOrphanLess), measure); + assert.equal(result.trimmed, true); + const kept = keys(result.backup.tracks); + assert.ok(!kept.includes(song(0).key), 'the oldest recent song went first'); + assert.ok(kept.includes(song(4).key), 'the newest orphan is newer than the recents and stays'); +}); + +test('the result is the largest plan that fits', () => { + const b = library(12, 0); + for (const keep of [1, 5, 11]) { + const exact = { ...b, history: b.history.slice(12 - keep), tracks: b.tracks.filter((t) => keys(b.history.slice(12 - keep)).includes(t.identity.key)) }; + const result = fitBackup(b, measure(exact), measure); + assert.equal(result.backup.history.length, keep, `budget for ${keep}`); + assert.equal(result.size, measure(exact)); + } +}); + +test('deterministic for equal input', () => { + const a = fitBackup(library(8, 3), 900, measure); + const b = fitBackup(library(8, 3), 900, measure); + assert.deepEqual(a, b); +}); + +test('too large when settings plus favorites alone are over budget', () => { + const b = library(2, 2); + const floor = measure({ ...b, history: [], tracks: [], favorites: [] }); + assert.throws(() => fitBackup(b, floor - 1, measure), LibraryTooLargeError); + assert.doesNotThrow(() => fitBackup(b, floor, measure)); +}); diff --git a/src/features/sync/persist/fit.ts b/src/features/sync/persist/fit.ts new file mode 100644 index 0000000..739e15a --- /dev/null +++ b/src/features/sync/persist/fit.ts @@ -0,0 +1,171 @@ +import type { Backup } from '../../../core/persist/backup-codec.ts'; +import type { FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types'; + +/** + * Cuts a backup down to a byte budget — the browser's sync quota — by + * dropping old things rather than capping counts. Nothing is trimmed while it + * fits; when it doesn't, tiers go in this order, each from its oldest end, + * and each cut no deeper than needed: + * + * 1. songs that aren't favorited: their Recent row and track record, by + * how recently they were played or edited (orphan records — a song no + * longer in Recent — sit in this tier by their own edit time); + * 2. chord charts, by when they were computed (they can be re-analyzed); + * 3. favorites, with their Recent row and track record, by last access. + * + * Settings, UI prefs and EQ presets are never cut. Songs are matched the way + * the library does (`isSameTrack`), so a record saved under a drifted + * duration still follows its favorite. + * + * `measure` is injected: the caller decides what "size" means (encoded JSON + * length in tests, the gzip+base64 blob for sync), so this stays pure and + * runs under `node --test` — hence relative `.ts` imports. + */ + +export interface FitResult { + backup: Backup; + /** Something had to go. */ + trimmed: boolean; + /** `measure` of the returned backup. */ + size: number; +} + +/** Thrown when even settings plus every favorite is over the budget. */ +export class LibraryTooLargeError extends Error { + constructor() { + super("Your library is too large for the browser's sync storage."); + this.name = 'LibraryTooLargeError'; + } +} + +/** How many of each tier's items (newest first) the built backup keeps. */ +interface Plan { + songs: number; + charts: number; + favorites: number; +} + +const TIERS: readonly (keyof Plan)[] = ['songs', 'charts', 'favorites']; + +/** The library matches rows by song, not key — see `isSameTrack`. */ +const songId = (identity: TrackIdentity) => `${identity.normalizedUrl}\n${identity.title}`; + +const byRecency = (a: { recency: number; key: string }, b: { recency: number; key: string }) => + b.recency - a.recency || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0); + +const hasChart = (track: TrackData) => !!track.chordChart && track.chordChart.segments.length > 0; + +interface Tiers { + /** Non-favorited songs, newest first. */ + songs: { id: string; recency: number; key: string }[]; + /** Chart-bearing track records, newest chart first. */ + charts: { key: string; recency: number }[]; + /** Favorites' songs, most recently accessed first. */ + favorites: { id: string; recency: number; key: string }[]; +} + +function collectTiers(backup: Backup): Tiers { + const favoriteIds = new Set(backup.favorites.map((f) => songId(f.identity))); + const songs = new Map(); + const touch = (identity: TrackIdentity, recency: number) => { + const id = songId(identity); + if (favoriteIds.has(id)) return; + const current = songs.get(id); + if (!current) songs.set(id, { id, recency, key: identity.key }); + else current.recency = Math.max(current.recency, recency); + }; + for (const entry of backup.history) touch(entry.identity, entry.updatedAt ?? 0); + for (const track of backup.tracks) touch(track.identity, track.updatedAt ?? 0); + + const charts = backup.tracks + .filter(hasChart) + .map((t) => ({ key: t.identity.key, recency: t.chordChart!.computedAt ?? 0 })) + .sort(byRecency); + + const favorites = new Map(); + for (const f of backup.favorites) { + const id = songId(f.identity); + const recency = f.lastAccessedAt ?? f.updatedAt ?? 0; + const current = favorites.get(id); + if (!current) favorites.set(id, { id, recency, key: f.identity.key }); + else current.recency = Math.max(current.recency, recency); + } + + return { + songs: [...songs.values()].sort(byRecency), + charts, + favorites: [...favorites.values()].sort(byRecency), + }; +} + +function build(backup: Backup, tiers: Tiers, plan: Plan): Backup { + const kept = new Set(); + for (const s of tiers.songs.slice(0, plan.songs)) kept.add(s.id); + for (const f of tiers.favorites.slice(0, plan.favorites)) kept.add(f.id); + const keptCharts = new Set(tiers.charts.slice(0, plan.charts).map((c) => c.key)); + + const keepRow = (row: HistoryEntry | FavoriteEntry) => kept.has(songId(row.identity)); + const tracks: TrackData[] = []; + for (const track of backup.tracks) { + if (!kept.has(songId(track.identity))) continue; + if (hasChart(track) && !keptCharts.has(track.identity.key)) { + tracks.push({ ...track, chordChart: null }); + } else { + tracks.push(track); + } + } + return { + ...backup, + history: backup.history.filter(keepRow), + favorites: backup.favorites.filter(keepRow), + tracks, + }; +} + +/** + * The largest backup, in the tier order above, whose `measure` is at most + * `budget`. Deterministic for equal input. Throws `LibraryTooLargeError` + * when nothing cuttable is left and it still doesn't fit. + */ +export function fitBackup( + backup: Backup, + budget: number, + measure: (backup: Backup) => number, +): FitResult { + const tiers = collectTiers(backup); + const full: Plan = { + songs: tiers.songs.length, + charts: tiers.charts.length, + favorites: tiers.favorites.length, + }; + const size = measure(backup); + if (size <= budget) return { backup, trimmed: false, size }; + + const plan = { ...full }; + const sizeOf = (p: Plan) => measure(build(backup, tiers, p)); + for (const tier of TIERS) { + // Earlier tiers are already empty. Does emptying this one fit? + const empty = sizeOf({ ...plan, [tier]: 0 }); + if (empty > budget) { + plan[tier] = 0; + continue; + } + // Yes — keep as many of its newest items as still fit. + let lo = 0; // known to fit + let hi = plan[tier]; // known not to fit (full plan didn't) + let loSize = empty; + while (hi - lo > 1) { + const mid = (lo + hi) >> 1; + const s = sizeOf({ ...plan, [tier]: mid }); + if (s <= budget) { + lo = mid; + loSize = s; + } else { + hi = mid; + } + } + plan[tier] = lo; + return { backup: build(backup, tiers, plan), trimmed: true, size: loSize }; + } + throw new LibraryTooLargeError(); +} From fd2a8450021ddefe0892fb9c0e48cf2fadbc7d9f Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 19:20:13 +0200 Subject: [PATCH 02/26] Remove server sync backend and related configurations - Deleted the server sync backend implementation in index.ts, including all related logic for handling backup snapshots. - Removed pnpm workspace configuration file as it is no longer needed. - Eliminated TypeScript configuration file for the server as the project structure has changed. - Cleared out wrangler configuration file, which defined the sync worker settings and rate limits. - Removed sync endpoint definitions and related utility functions from the sync feature module. - Deleted API functions for pulling, pushing, and deleting snapshots from the sync server. - Removed hash generation and cookie management for sync ID, ensuring no remnants of the sync functionality remain. - Cleaned up sync host definitions to prevent any references to the now-removed sync backend. --- server/README.md | 57 -- server/package.json | 17 - server/pnpm-lock.yaml | 888 --------------------------- server/pnpm-workspace.yaml | 6 - server/src/index.ts | 136 ---- server/tsconfig.json | 13 - server/wrangler.jsonc | 30 - src/features/sync/endpoint.ts | 9 - src/features/sync/panel/api.ts | 48 -- src/features/sync/panel/hash.ts | 37 -- src/features/sync/panel/id-cookie.ts | 82 --- src/features/sync/sync-hosts.ts | 22 - 12 files changed, 1345 deletions(-) delete mode 100644 server/README.md delete mode 100644 server/package.json delete mode 100644 server/pnpm-lock.yaml delete mode 100644 server/pnpm-workspace.yaml delete mode 100644 server/src/index.ts delete mode 100644 server/tsconfig.json delete mode 100644 server/wrangler.jsonc delete mode 100644 src/features/sync/endpoint.ts delete mode 100644 src/features/sync/panel/api.ts delete mode 100644 src/features/sync/panel/hash.ts delete mode 100644 src/features/sync/panel/id-cookie.ts delete mode 100644 src/features/sync/sync-hosts.ts diff --git a/server/README.md b/server/README.md deleted file mode 100644 index d82f63b..0000000 --- a/server/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# note-by-note-sync - -Minimal Cloudflare Worker + KV backend for Note by Note device sync. Stores one -backup snapshot per secret sync ID. No accounts, no auth beyond the ID itself. - -## API - -One route, `/v1/backup`. The sync ID goes in an **`X-Sync-Id` header**, never in -the path or query — URLs are recorded verbatim by Workers Logs and every other -edge request log, and the ID is the whole credential. - -- `GET /v1/backup` → the stored backup JSON, or `404` if the ID has no data. -- `PUT /v1/backup` → stores the request body (must be a Note by Note backup - ≤ 1 MiB), `204`. -- `DELETE /v1/backup` → removes the snapshot, `204`. - -IDs must match `[A-Za-z0-9_-]{43,64}`, matching what the extension generates -(32 random bytes as base64url). Shorter IDs are rejected: a hand-picked one -would be guessable, and guessing it is the whole attack. - -KV is keyed by `SHA-256(id)`, so the raw token isn't stored either. `GET` falls -back to the raw-ID key once, for blobs written before that change; the next -`PUT` rewrites them under the hashed key. - -Snapshots carry a 180-day TTL, refreshed on every write, so abandoned blobs age -out. The extension re-seeds an expired one automatically. - -Writes (`PUT`/`DELETE`) are rate-limited per IP via the `SYNC_WRITE_LIMIT` -binding — 60/minute, far above a real client's 5-second push debounce, and -enough to make scripted abuse of the free KV write quota uninteresting. Reads -are not limited, so a shared NAT can't lock its users out of pulling. - -Failures return a CORS-bearing `502` rather than letting the exception escape: -Cloudflare's own error page carries no CORS headers, which the browser reports -as a network failure indistinguishable from being offline. - -> **Upgrading from the pre-header API:** the old `/v1/:id` route is gone, so -> deploy this Worker and the matching extension build together. An older -> extension against this Worker (or vice versa) gets a `404` on every call. - -## Deploy (once) - -```sh -pnpm install -pnpm wrangler kv namespace create SYNC_KV # paste the id into wrangler.jsonc -pnpm run deploy -``` - -Then set the deployed URL as `SYNC_ENDPOINT` in `../src/features/sync/panel/api.ts` and rebuild the extension. - -## Local development - -```sh -pnpm run dev # serves http://localhost:8787 with a local KV emulation -``` - -The extension's dev build (`pnpm dev` at the repo root) targets `http://localhost:8787` automatically. diff --git a/server/package.json b/server/package.json deleted file mode 100644 index b7d9b1b..0000000 --- a/server/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "note-by-note-sync", - "private": true, - "version": "0.1.0", - "license": "GPL-2.0-or-later", - "type": "module", - "scripts": { - "dev": "wrangler dev", - "deploy": "wrangler deploy", - "check": "tsc --noEmit" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20250705.0", - "typescript": "^5.9.3", - "wrangler": "^4.24.0" - } -} diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml deleted file mode 100644 index 7f78ecb..0000000 --- a/server/pnpm-lock.yaml +++ /dev/null @@ -1,888 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - devDependencies: - '@cloudflare/workers-types': - specifier: ^4.20250705.0 - version: 4.20260702.1 - typescript: - specifier: ^5.9.3 - version: 5.9.3 - wrangler: - specifier: ^4.24.0 - version: 4.111.0(@cloudflare/workers-types@4.20260702.1) - -packages: - - '@cloudflare/kv-asset-handler@0.5.0': - resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} - engines: {node: '>=22.0.0'} - - '@cloudflare/unenv-preset@2.16.1': - resolution: {integrity: sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==} - peerDependencies: - unenv: 2.0.0-rc.24 - workerd: '>1.20260305.0 <2.0.0-0' - peerDependenciesMeta: - workerd: - optional: true - - '@cloudflare/workerd-darwin-64@1.20260710.1': - resolution: {integrity: sha512-OqJl2eWF5+y9jarMm3YqqCTUe7Hd4ihogX5jyRU8iaAgOVyDr/Bk6aXpPCVUi1/MHzO93a18R/TmSTtzmB0sQw==} - engines: {node: '>=16'} - cpu: [x64] - os: [darwin] - - '@cloudflare/workerd-darwin-arm64@1.20260710.1': - resolution: {integrity: sha512-MYBqWgUblO+VlGvO73zYsH3hB9tdRj+yLyt5IHDFWryipb2l1efmNiWtAOkIhSRfypqLYGFrfpaDm2Hg00XVKw==} - engines: {node: '>=16'} - cpu: [arm64] - os: [darwin] - - '@cloudflare/workerd-linux-64@1.20260710.1': - resolution: {integrity: sha512-lVWUgqI8qrkqvaCBGElu1kdaUFdAvaS2RD8K4qkCFP9hI3f5TCXumEs5qWSeZkvKum0+X/uJZ5hBFWsYI5SmoQ==} - engines: {node: '>=16'} - cpu: [x64] - os: [linux] - - '@cloudflare/workerd-linux-arm64@1.20260710.1': - resolution: {integrity: sha512-kDwDPItBjAI4JL0df9Fma2N+Qggbm77IB/DnroAkEGQ79fpR80sYMyuB/ZQKyjEk9f48Ocq7HCCLq59qVSyNqA==} - engines: {node: '>=16'} - cpu: [arm64] - os: [linux] - - '@cloudflare/workerd-windows-64@1.20260710.1': - resolution: {integrity: sha512-GcLHy1oN1dfK6g1Z7UDV9f5xMGyTfPwcjWQ0sfWKH31IsoEVCRapnj3IC0PoIrDbnoo6irGPP0CwVs3WzdTajw==} - engines: {node: '>=16'} - cpu: [x64] - os: [win32] - - '@cloudflare/workers-types@4.20260702.1': - resolution: {integrity: sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==} - - '@cspotcode/source-map-support@0.8.1': - resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} - engines: {node: '>=12'} - - '@emnapi/runtime@1.11.2': - resolution: {integrity: sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==} - - '@esbuild/aix-ppc64@0.28.1': - resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.28.1': - resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm@0.28.1': - resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-x64@0.28.1': - resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/darwin-arm64@0.28.1': - resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-x64@0.28.1': - resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.28.1': - resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.28.1': - resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/linux-arm64@0.28.1': - resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm@0.28.1': - resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-ia32@0.28.1': - resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.28.1': - resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-mips64el@0.28.1': - resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-ppc64@0.28.1': - resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-riscv64@0.28.1': - resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-s390x@0.28.1': - resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.28.1': - resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/netbsd-arm64@0.28.1': - resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.28.1': - resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/openbsd-arm64@0.28.1': - resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.28.1': - resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.28.1': - resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/sunos-x64@0.28.1': - resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/win32-arm64@0.28.1': - resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-ia32@0.28.1': - resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-x64@0.28.1': - resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@img/colour@1.1.0': - resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} - engines: {node: '>=18'} - - '@img/sharp-darwin-arm64@0.34.5': - resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [darwin] - - '@img/sharp-darwin-x64@0.34.5': - resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-darwin-arm64@1.2.4': - resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} - cpu: [arm64] - os: [darwin] - - '@img/sharp-libvips-darwin-x64@1.2.4': - resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-linux-arm64@1.2.4': - resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-arm@1.2.4': - resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-ppc64@1.2.4': - resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-riscv64@1.2.4': - resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-s390x@1.2.4': - resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-x64@1.2.4': - resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-linux-arm64@0.34.5': - resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-arm@0.34.5': - resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-ppc64@0.34.5': - resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-riscv64@0.34.5': - resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-s390x@0.34.5': - resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-x64@0.34.5': - resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-linuxmusl-arm64@0.34.5': - resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-linuxmusl-x64@0.34.5': - resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-wasm32@0.34.5': - resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [wasm32] - - '@img/sharp-win32-arm64@0.34.5': - resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [win32] - - '@img/sharp-win32-ia32@0.34.5': - resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ia32] - os: [win32] - - '@img/sharp-win32-x64@0.34.5': - resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [win32] - - '@jridgewell/resolve-uri@3.1.2': - resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} - engines: {node: '>=6.0.0'} - - '@jridgewell/sourcemap-codec@1.5.5': - resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - - '@jridgewell/trace-mapping@0.3.9': - resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - - '@poppinss/colors@4.1.6': - resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} - - '@poppinss/dumper@0.6.5': - resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} - - '@poppinss/exception@1.2.3': - resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} - - '@sindresorhus/is@7.2.0': - resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} - engines: {node: '>=18'} - - '@speed-highlight/core@1.2.17': - resolution: {integrity: sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==} - - blake3-wasm@2.1.5: - resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} - - cookie@1.1.1: - resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} - engines: {node: '>=18'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - error-stack-parser-es@1.0.5: - resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} - - esbuild@0.28.1: - resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} - engines: {node: '>=18'} - hasBin: true - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - kleur@4.1.5: - resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} - engines: {node: '>=6'} - - miniflare@4.20260710.0: - resolution: {integrity: sha512-x1LLRkU6o1p7hiKrB0TRnL0MJn6xFOT+/vrlEQINz5cRDKLP8ru4hBqWTIvXAetzr1acKAnmAaG84pQ4W/K14g==} - engines: {node: '>=22.0.0'} - hasBin: true - - path-to-regexp@6.3.0: - resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} - - pathe@2.0.3: - resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - - semver@7.8.5: - resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} - engines: {node: '>=10'} - hasBin: true - - sharp@0.34.5: - resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - - supports-color@10.2.2: - resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} - engines: {node: '>=18'} - - tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - - typescript@5.9.3: - resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} - engines: {node: '>=14.17'} - hasBin: true - - undici@7.28.0: - resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} - engines: {node: '>=20.18.1'} - - unenv@2.0.0-rc.24: - resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} - - workerd@1.20260710.1: - resolution: {integrity: sha512-U2sBPPrb9U97sBKnnMN6Kv8p65903P35nwMkPE9vSH/bRuRqkZ3a1EjUw3jV28RhiyXpkLF77Evzw8XimFxyTw==} - engines: {node: '>=16'} - hasBin: true - - wrangler@4.111.0: - resolution: {integrity: sha512-bffpI9EyrnpKkF/1S+RaIv8oRD93GtbsA7TlfWwOsGJGB7VO3jVbdGzpC9TU7Bqom3z7jUxcte4Z9MPhaQ4HoQ==} - engines: {node: '>=22.0.0'} - hasBin: true - peerDependencies: - '@cloudflare/workers-types': ^5.20260710.1 - peerDependenciesMeta: - '@cloudflare/workers-types': - optional: true - - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - - youch-core@0.3.3: - resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} - - youch@4.1.0-beta.10: - resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} - -snapshots: - - '@cloudflare/kv-asset-handler@0.5.0': {} - - '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260710.1)': - dependencies: - unenv: 2.0.0-rc.24 - optionalDependencies: - workerd: 1.20260710.1 - - '@cloudflare/workerd-darwin-64@1.20260710.1': - optional: true - - '@cloudflare/workerd-darwin-arm64@1.20260710.1': - optional: true - - '@cloudflare/workerd-linux-64@1.20260710.1': - optional: true - - '@cloudflare/workerd-linux-arm64@1.20260710.1': - optional: true - - '@cloudflare/workerd-windows-64@1.20260710.1': - optional: true - - '@cloudflare/workers-types@4.20260702.1': {} - - '@cspotcode/source-map-support@0.8.1': - dependencies: - '@jridgewell/trace-mapping': 0.3.9 - - '@emnapi/runtime@1.11.2': - dependencies: - tslib: 2.8.1 - optional: true - - '@esbuild/aix-ppc64@0.28.1': - optional: true - - '@esbuild/android-arm64@0.28.1': - optional: true - - '@esbuild/android-arm@0.28.1': - optional: true - - '@esbuild/android-x64@0.28.1': - optional: true - - '@esbuild/darwin-arm64@0.28.1': - optional: true - - '@esbuild/darwin-x64@0.28.1': - optional: true - - '@esbuild/freebsd-arm64@0.28.1': - optional: true - - '@esbuild/freebsd-x64@0.28.1': - optional: true - - '@esbuild/linux-arm64@0.28.1': - optional: true - - '@esbuild/linux-arm@0.28.1': - optional: true - - '@esbuild/linux-ia32@0.28.1': - optional: true - - '@esbuild/linux-loong64@0.28.1': - optional: true - - '@esbuild/linux-mips64el@0.28.1': - optional: true - - '@esbuild/linux-ppc64@0.28.1': - optional: true - - '@esbuild/linux-riscv64@0.28.1': - optional: true - - '@esbuild/linux-s390x@0.28.1': - optional: true - - '@esbuild/linux-x64@0.28.1': - optional: true - - '@esbuild/netbsd-arm64@0.28.1': - optional: true - - '@esbuild/netbsd-x64@0.28.1': - optional: true - - '@esbuild/openbsd-arm64@0.28.1': - optional: true - - '@esbuild/openbsd-x64@0.28.1': - optional: true - - '@esbuild/openharmony-arm64@0.28.1': - optional: true - - '@esbuild/sunos-x64@0.28.1': - optional: true - - '@esbuild/win32-arm64@0.28.1': - optional: true - - '@esbuild/win32-ia32@0.28.1': - optional: true - - '@esbuild/win32-x64@0.28.1': - optional: true - - '@img/colour@1.1.0': {} - - '@img/sharp-darwin-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-arm64': 1.2.4 - optional: true - - '@img/sharp-darwin-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-x64': 1.2.4 - optional: true - - '@img/sharp-libvips-darwin-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-darwin-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm@1.2.4': - optional: true - - '@img/sharp-libvips-linux-ppc64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-riscv64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-s390x@1.2.4': - optional: true - - '@img/sharp-libvips-linux-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - optional: true - - '@img/sharp-linux-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm64': 1.2.4 - optional: true - - '@img/sharp-linux-arm@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm': 1.2.4 - optional: true - - '@img/sharp-linux-ppc64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-ppc64': 1.2.4 - optional: true - - '@img/sharp-linux-riscv64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-riscv64': 1.2.4 - optional: true - - '@img/sharp-linux-s390x@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-s390x': 1.2.4 - optional: true - - '@img/sharp-linux-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-x64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - optional: true - - '@img/sharp-wasm32@0.34.5': - dependencies: - '@emnapi/runtime': 1.11.2 - optional: true - - '@img/sharp-win32-arm64@0.34.5': - optional: true - - '@img/sharp-win32-ia32@0.34.5': - optional: true - - '@img/sharp-win32-x64@0.34.5': - optional: true - - '@jridgewell/resolve-uri@3.1.2': {} - - '@jridgewell/sourcemap-codec@1.5.5': {} - - '@jridgewell/trace-mapping@0.3.9': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - - '@poppinss/colors@4.1.6': - dependencies: - kleur: 4.1.5 - - '@poppinss/dumper@0.6.5': - dependencies: - '@poppinss/colors': 4.1.6 - '@sindresorhus/is': 7.2.0 - supports-color: 10.2.2 - - '@poppinss/exception@1.2.3': {} - - '@sindresorhus/is@7.2.0': {} - - '@speed-highlight/core@1.2.17': {} - - blake3-wasm@2.1.5: {} - - cookie@1.1.1: {} - - detect-libc@2.1.2: {} - - error-stack-parser-es@1.0.5: {} - - esbuild@0.28.1: - optionalDependencies: - '@esbuild/aix-ppc64': 0.28.1 - '@esbuild/android-arm': 0.28.1 - '@esbuild/android-arm64': 0.28.1 - '@esbuild/android-x64': 0.28.1 - '@esbuild/darwin-arm64': 0.28.1 - '@esbuild/darwin-x64': 0.28.1 - '@esbuild/freebsd-arm64': 0.28.1 - '@esbuild/freebsd-x64': 0.28.1 - '@esbuild/linux-arm': 0.28.1 - '@esbuild/linux-arm64': 0.28.1 - '@esbuild/linux-ia32': 0.28.1 - '@esbuild/linux-loong64': 0.28.1 - '@esbuild/linux-mips64el': 0.28.1 - '@esbuild/linux-ppc64': 0.28.1 - '@esbuild/linux-riscv64': 0.28.1 - '@esbuild/linux-s390x': 0.28.1 - '@esbuild/linux-x64': 0.28.1 - '@esbuild/netbsd-arm64': 0.28.1 - '@esbuild/netbsd-x64': 0.28.1 - '@esbuild/openbsd-arm64': 0.28.1 - '@esbuild/openbsd-x64': 0.28.1 - '@esbuild/openharmony-arm64': 0.28.1 - '@esbuild/sunos-x64': 0.28.1 - '@esbuild/win32-arm64': 0.28.1 - '@esbuild/win32-ia32': 0.28.1 - '@esbuild/win32-x64': 0.28.1 - - fsevents@2.3.3: - optional: true - - kleur@4.1.5: {} - - miniflare@4.20260710.0: - dependencies: - '@cspotcode/source-map-support': 0.8.1 - sharp: 0.34.5 - undici: 7.28.0 - workerd: 1.20260710.1 - ws: 8.21.0 - youch: 4.1.0-beta.10 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - path-to-regexp@6.3.0: {} - - pathe@2.0.3: {} - - semver@7.8.5: {} - - sharp@0.34.5: - dependencies: - '@img/colour': 1.1.0 - detect-libc: 2.1.2 - semver: 7.8.5 - optionalDependencies: - '@img/sharp-darwin-arm64': 0.34.5 - '@img/sharp-darwin-x64': 0.34.5 - '@img/sharp-libvips-darwin-arm64': 1.2.4 - '@img/sharp-libvips-darwin-x64': 1.2.4 - '@img/sharp-libvips-linux-arm': 1.2.4 - '@img/sharp-libvips-linux-arm64': 1.2.4 - '@img/sharp-libvips-linux-ppc64': 1.2.4 - '@img/sharp-libvips-linux-riscv64': 1.2.4 - '@img/sharp-libvips-linux-s390x': 1.2.4 - '@img/sharp-libvips-linux-x64': 1.2.4 - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - '@img/sharp-linux-arm': 0.34.5 - '@img/sharp-linux-arm64': 0.34.5 - '@img/sharp-linux-ppc64': 0.34.5 - '@img/sharp-linux-riscv64': 0.34.5 - '@img/sharp-linux-s390x': 0.34.5 - '@img/sharp-linux-x64': 0.34.5 - '@img/sharp-linuxmusl-arm64': 0.34.5 - '@img/sharp-linuxmusl-x64': 0.34.5 - '@img/sharp-wasm32': 0.34.5 - '@img/sharp-win32-arm64': 0.34.5 - '@img/sharp-win32-ia32': 0.34.5 - '@img/sharp-win32-x64': 0.34.5 - - supports-color@10.2.2: {} - - tslib@2.8.1: - optional: true - - typescript@5.9.3: {} - - undici@7.28.0: {} - - unenv@2.0.0-rc.24: - dependencies: - pathe: 2.0.3 - - workerd@1.20260710.1: - optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20260710.1 - '@cloudflare/workerd-darwin-arm64': 1.20260710.1 - '@cloudflare/workerd-linux-64': 1.20260710.1 - '@cloudflare/workerd-linux-arm64': 1.20260710.1 - '@cloudflare/workerd-windows-64': 1.20260710.1 - - wrangler@4.111.0(@cloudflare/workers-types@4.20260702.1): - dependencies: - '@cloudflare/kv-asset-handler': 0.5.0 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260710.1) - blake3-wasm: 2.1.5 - esbuild: 0.28.1 - miniflare: 4.20260710.0 - path-to-regexp: 6.3.0 - unenv: 2.0.0-rc.24 - workerd: 1.20260710.1 - optionalDependencies: - '@cloudflare/workers-types': 4.20260702.1 - fsevents: 2.3.3 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - ws@8.21.0: {} - - youch-core@0.3.3: - dependencies: - '@poppinss/exception': 1.2.3 - error-stack-parser-es: 1.0.5 - - youch@4.1.0-beta.10: - dependencies: - '@poppinss/colors': 4.1.6 - '@poppinss/dumper': 0.6.5 - '@speed-highlight/core': 1.2.17 - cookie: 1.1.1 - youch-core: 0.3.3 diff --git a/server/pnpm-workspace.yaml b/server/pnpm-workspace.yaml deleted file mode 100644 index 14e463d..0000000 --- a/server/pnpm-workspace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Makes server/ its own install root (the repo root is a pnpm config root), -# and lets wrangler's runtime (workerd) install its binary. -allowBuilds: - esbuild: true - workerd: true - sharp: true diff --git a/server/src/index.ts b/server/src/index.ts deleted file mode 100644 index 068310e..0000000 --- a/server/src/index.ts +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Note by Note sync backend: a key-value store for backup snapshots, keyed by a - * client-generated secret ID. The ID is the whole capability — anyone holding - * it can read and write the blob — so CORS is open (`*`) and no other auth - * exists. Deployed once by the developer; see ../README.md. - * - * Because the ID *is* the credential it is passed in the `X-Sync-Id` header, - * never in the path or query: URLs are recorded verbatim by Workers Logs and - * every other edge-side request log. For the same reason KV is keyed by - * SHA-256(id) rather than the raw value, so a KV key listing discloses nothing - * usable either. - * - * KV is eventually consistent (~60 s across edges); the extension self-heals - * via periodic pulls, so no stronger consistency is needed here. - */ - -interface Env { - SYNC_KV: KVNamespace; - /** Per-IP write limiter; see `ratelimits` in wrangler.jsonc. */ - SYNC_WRITE_LIMIT: RateLimit; -} - -/** Matches the extension's generated IDs: 32 random bytes → 43-char base64url. - * Anything shorter would be guessable, so it is rejected before touching KV. */ -const ID_RE = /^[A-Za-z0-9_-]{43,64}$/; - -const MAX_BODY_BYTES = 1024 * 1024; - -/** Snapshots outlive any realistic gap between a user's sessions, but not - * forever — an abandoned blob eventually ages out instead of costing storage - * indefinitely. Refreshed on every write. The extension re-seeds an expired - * blob automatically (see `#reconcile` in sync.svelte.ts). */ -const TTL_SECONDS = 180 * 24 * 60 * 60; - -const CORS_HEADERS = { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': 'GET, PUT, DELETE, OPTIONS', - 'Access-Control-Allow-Headers': 'Content-Type, X-Sync-Id', - 'Access-Control-Max-Age': '86400', -}; - -function respond(status: number, body: string | null, headers: Record = {}) { - return new Response(body, { status, headers: { ...CORS_HEADERS, ...headers } }); -} - -/** KV key for a sync ID. Hex SHA-256 so the secret itself is never stored. */ -async function kvKey(id: string): Promise { - const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(id)); - return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join(''); -} - -export default { - async fetch(request: Request, env: Env): Promise { - if (request.method === 'OPTIONS') return respond(204, null); - - if (new URL(request.url).pathname !== '/v1/backup') return respond(404, 'Not found'); - - const id = request.headers.get('X-Sync-Id') ?? ''; - if (!ID_RE.test(id)) return respond(400, 'Invalid or missing sync ID'); - const key = await kvKey(id); - - if (request.method === 'GET') { - let value: string | null; - try { - value = await env.SYNC_KV.get(key); - // Pre-hashing deployments stored the blob under the raw ID. Fall back - // once; the next PUT rewrites it under the hashed key. - if (value === null) value = await env.SYNC_KV.get(id); - } catch { - return respond(502, 'Storage temporarily unavailable'); - } - if (value === null) return respond(404, 'No data for this sync ID'); - return respond(200, value, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - 'X-Content-Type-Options': 'nosniff', - }); - } - - if (request.method === 'PUT' || request.method === 'DELETE') { - // Writes are the expensive, abusable direction. Reads are left alone so a - // shared NAT can't lock its users out of pulling. - const { success } = await env.SYNC_WRITE_LIMIT.limit({ - key: request.headers.get('cf-connecting-ip') ?? 'unknown', - }); - if (!success) return respond(429, 'Too many requests', { 'Retry-After': '60' }); - } - - if (request.method === 'PUT') { - // Reject early when the client declares an oversized body, before buffering. - if (Number(request.headers.get('content-length')) > MAX_BODY_BYTES) { - return respond(413, 'Snapshot too large'); - } - const body = await request.text(); - // `body.length` counts UTF-16 units; the cap is bytes, so measure UTF-8. - if (new TextEncoder().encode(body).byteLength > MAX_BODY_BYTES) { - return respond(413, 'Snapshot too large'); - } - // Sniff the payload so the namespace can't be used as a generic dump. - let parsed: unknown; - try { - parsed = JSON.parse(body); - } catch { - return respond(400, 'Body is not valid JSON'); - } - const record = parsed as Record | null; - if ( - record === null || - typeof record !== 'object' || - record.format !== 'note-by-note-backup' || - typeof record.exportedAt !== 'number' - ) { - return respond(400, 'Body is not a Note by Note backup'); - } - try { - await env.SYNC_KV.put(key, body, { expirationTtl: TTL_SECONDS }); - } catch { - return respond(502, 'Storage temporarily unavailable'); - } - return respond(204, null); - } - - if (request.method === 'DELETE') { - try { - // Both keys: a blob written before the hashing change is still the - // user's, and "delete my data" has to mean it. - await Promise.all([env.SYNC_KV.delete(key), env.SYNC_KV.delete(id)]); - } catch { - return respond(502, 'Storage temporarily unavailable'); - } - return respond(204, null); - } - - return respond(405, 'Method not allowed', { Allow: 'GET, PUT, DELETE, OPTIONS' }); - }, -}; diff --git a/server/tsconfig.json b/server/tsconfig.json deleted file mode 100644 index a1efe2e..0000000 --- a/server/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "ES2022", - "moduleResolution": "bundler", - "lib": ["ES2022"], - "types": ["@cloudflare/workers-types"], - "strict": true, - "noEmit": true, - "skipLibCheck": true - }, - "include": ["src"] -} diff --git a/server/wrangler.jsonc b/server/wrangler.jsonc deleted file mode 100644 index 6bdee90..0000000 --- a/server/wrangler.jsonc +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "note-by-note-sync", - "main": "src/index.ts", - "compatibility_date": "2026-07-01", - "observability": { - "enabled": true - }, - "kv_namespaces": [ - { - "binding": "SYNC_KV", - "id": "53d23d1799a74bbca74ec8b98fa98b51", - "remote": true - } - ], - // Per-IP cap on writes (PUT/DELETE). A real client pushes on a 5 s debounce - // and pulls every 5 min, so this is far above normal use and only bites - // scripted abuse. `namespace_id` is a per-Worker label, not an account - // resource — self-hosters can leave it as-is. - "ratelimits": [ - { - "name": "SYNC_WRITE_LIMIT", - "namespace_id": "1001", - "simple": { - "limit": 60, - "period": 60 - } - } - ] -} \ No newline at end of file diff --git a/src/features/sync/endpoint.ts b/src/features/sync/endpoint.ts deleted file mode 100644 index 0646150..0000000 --- a/src/features/sync/endpoint.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { SYNC_ENDPOINT_DEV, SYNC_ENDPOINT_PROD, syncHostPattern } from './sync-hosts'; - -/** The Worker this build talks to: localhost in dev, the deployed one in prod. - * CORS is open there, so the sync calls themselves need no host permission; - * only the ID cookie does (see panel/id-cookie.ts). */ -export const SYNC_ENDPOINT = import.meta.env.DEV ? SYNC_ENDPOINT_DEV : SYNC_ENDPOINT_PROD; - -/** Optional host permission that lets the `cookies` API touch the sync host. */ -export const SYNC_ORIGIN_PATTERN = syncHostPattern(SYNC_ENDPOINT); diff --git a/src/features/sync/panel/api.ts b/src/features/sync/panel/api.ts deleted file mode 100644 index cb44cfe..0000000 --- a/src/features/sync/panel/api.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { parseBackup, type Backup } from '../../../core/persist/backup'; -import { SYNC_ENDPOINT } from '../endpoint'; - -export class SyncHttpError extends Error { - constructor( - public readonly status: number, - message: string, - ) { - super(message); - this.name = 'SyncHttpError'; - } -} - -/** The ID is the credential, so it travels in a header rather than the path — - * URLs end up verbatim in the Worker's request logs. */ -const BACKUP_URL = `${SYNC_ENDPOINT}/v1/backup`; - -function errorFor(status: number): SyncHttpError { - if (status === 429) { - return new SyncHttpError(status, 'Too many sync requests — try again in a minute.'); - } - if (status === 502) return new SyncHttpError(status, 'Sync storage is temporarily unavailable.'); - return new SyncHttpError(status, `Sync server error (${status})`); -} - -/** The stored snapshot, or null if the ID has no data yet (404). */ -export async function pullSnapshot(id: string): Promise { - const res = await fetch(BACKUP_URL, { headers: { 'X-Sync-Id': id } }); - if (res.status === 404) return null; - if (!res.ok) throw errorFor(res.status); - return parseBackup(await res.text()); -} - -export async function pushSnapshot(id: string, backup: Backup): Promise { - const res = await fetch(BACKUP_URL, { - method: 'PUT', - headers: { 'Content-Type': 'application/json', 'X-Sync-Id': id }, - body: JSON.stringify(backup), - }); - if (!res.ok) throw errorFor(res.status); -} - -/** Removes the snapshot from the server. A 404 counts as success — the goal is - * "no data under this ID", and it is already true. */ -export async function deleteSnapshot(id: string): Promise { - const res = await fetch(BACKUP_URL, { method: 'DELETE', headers: { 'X-Sync-Id': id } }); - if (!res.ok && res.status !== 404) throw errorFor(res.status); -} diff --git a/src/features/sync/panel/hash.ts b/src/features/sync/panel/hash.ts deleted file mode 100644 index 8eef05f..0000000 --- a/src/features/sync/panel/hash.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { Backup } from '../../../core/persist/backup'; - -/** JSON with object keys sorted at every level, so the same data always - * yields the same string regardless of construction order. */ -export function stableStringify(value: unknown): string { - if (Array.isArray(value)) { - return `[${value.map(stableStringify).join(',')}]`; - } - if (typeof value === 'object' && value !== null) { - const entries = Object.entries(value as Record) - .filter(([, v]) => v !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); - return `{${entries.join(',')}}`; - } - return JSON.stringify(value) ?? 'null'; -} - -/** - * Content hash of a snapshot's data fields — excludes `exportedAt` and - * `appVersion` so re-exporting unchanged data hashes the same. Tracks are - * sorted by identity key because their storage-enumeration order is arbitrary. - */ -export async function snapshotHash(backup: Backup): Promise { - const canonical = stableStringify({ - settings: backup.settings, - uiPrefs: backup.uiPrefs, - history: backup.history, - favorites: backup.favorites, - eqPresets: backup.eqPresets, - tracks: [...backup.tracks].sort((a, b) => - a.identity.key < b.identity.key ? -1 : a.identity.key > b.identity.key ? 1 : 0, - ), - }); - const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(canonical)); - return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join(''); -} diff --git a/src/features/sync/panel/id-cookie.ts b/src/features/sync/panel/id-cookie.ts deleted file mode 100644 index a4094cd..0000000 --- a/src/features/sync/panel/id-cookie.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { SYNC_ID_RE } from '../persist/sync-config'; -import { SYNC_ENDPOINT, SYNC_ORIGIN_PATTERN } from '../endpoint'; - -/** - * Durable copy of the sync ID — the canonical explanation; README, PRIVACY - * and the Settings copy defer to this file. - * - * `storage.sync` carries the ID to the user's other devices, but the browser - * purges it — on every device on the account — the moment the extension is - * uninstalled, and a reinstall then mints a fresh ID with no way back to the - * old snapshot. Cookies belong to the profile, not the extension, so one set on - * the sync server's domain is the one store that outlives an uninstall. It is - * read and written only through the `cookies` API and never rides along with a - * request: the sync calls carry the ID in a header, and a cookie is meaningless - * to the Worker anyway. - * - * The `cookies` API needs host access to the sync origin. That is an - * *optional* host permission (a required one would disable the extension on - * update until re-approved on Chrome, and is opt-in on Firefox regardless), so - * everything here is best-effort until `requestIdCookieAccess` succeeded in a - * user gesture — `` from Connect covers it too. `sync.durable` - * mirrors that state so the panel never promises a copy it can't keep. - * - * Lifecycle: written whenever sync is on with an ID (the store's `#reflect` - * is the single call site), removed by "Delete synced data" so a later - * reinstall does not resurrect an identity the user abandoned. Clearing browser - * cookies loses it — the panel keeps showing the ID for copying. - */ -const NAME = 'syncId'; -/** Chromium caps cookie lifetime at 400 days; refreshed once per panel start. */ -const MAX_AGE_S = 400 * 24 * 60 * 60; - -export async function hasIdCookieAccess(): Promise { - try { - return await browser.permissions.contains({ origins: [SYNC_ORIGIN_PATTERN] }); - } catch { - return false; - } -} - -/** Must run in a user gesture. Resolves true when access is held afterwards. */ -export async function requestIdCookieAccess(): Promise { - try { - return await browser.permissions.request({ origins: [SYNC_ORIGIN_PATTERN] }); - } catch { - return false; - } -} - -export async function readIdCookie(): Promise { - try { - const cookie = await browser.cookies.get({ url: SYNC_ENDPOINT, name: NAME }); - return cookie && SYNC_ID_RE.test(cookie.value) ? cookie.value : null; - } catch { - return null; - } -} - -/** Best-effort: a missing permission must not break sync itself. No `secure` - * attribute — the cookie is never sent, and localhost in dev is plain http. */ -export async function writeIdCookie(id: string): Promise { - try { - await browser.cookies.set({ - url: SYNC_ENDPOINT, - name: NAME, - value: id, - httpOnly: true, - sameSite: 'strict', - expirationDate: Math.floor(Date.now() / 1000) + MAX_AGE_S, - }); - } catch { - // Unsupported or not permitted here — the sync area still has the ID. - } -} - -export async function removeIdCookie(): Promise { - try { - await browser.cookies.remove({ url: SYNC_ENDPOINT, name: NAME }); - } catch { - // Nothing to remove, or no access — either way there is no copy to keep. - } -} diff --git a/src/features/sync/sync-hosts.ts b/src/features/sync/sync-hosts.ts deleted file mode 100644 index f23c3f2..0000000 --- a/src/features/sync/sync-hosts.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * The sync Worker's addresses (see /server) — the single place they are - * written down. Kept free of `import.meta.env` so wxt.config.ts can import it - * at manifest-build time; `endpoint.ts` picks the one the bundle talks to. - * Self-hosters change the production URL here and rebuild. - */ -export const SYNC_ENDPOINT_PROD = 'https://note-by-note-sync.oapp.workers.dev'; -/** `cd server ; pnpm run dev` — the dev build targets this automatically. */ -export const SYNC_ENDPOINT_DEV = 'http://localhost:8787'; - -/** Host match pattern for a Worker origin, as the manifest and the - * `permissions` API want it. */ -export function syncHostPattern(endpoint: string): string { - return `${new URL(endpoint).origin}/*`; -} - -/** Every pattern a build might hold for the sync host. The background worker - * uses this to tell the sync host apart from sites the user granted. */ -export const SYNC_HOST_PATTERNS: readonly string[] = [ - syncHostPattern(SYNC_ENDPOINT_PROD), - syncHostPattern(SYNC_ENDPOINT_DEV), -]; From 4ee5c7c03c41a4d0235f333e32a15154a6f95d6a Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 19:21:24 +0200 Subject: [PATCH 03/26] Refactor sync storage and backup handling - Updated tsconfig.json to remove the exclusion of the server directory. - Modified wxt.config.ts to improve comments and remove unused imports. - Added deletions.ts to manage deletion records for syncing. - Introduced hash.ts for content hashing of backups. - Created merge.test.ts to validate backup merging logic. - Implemented merge.ts to handle merging of local and remote backups. - Developed sync-area.ts for managing sync storage interactions. - Added sync-blob.test.ts to test blob packing and unpacking functionality. - Created sync-blob.ts to define the structure and methods for handling backup blobs in sync storage. --- .github/workflows/ci.yml | 24 - CLAUDE.md | 5 +- CONTRIBUTING.md | 6 +- PRIVACY.md | 45 +- README.md | 58 +- SECURITY.md | 36 +- pnpm-workspace.yaml | 3 +- src/core/persist/backup-codec.test.ts | 12 + src/core/persist/backup-codec.ts | 17 +- src/core/persist/backup.ts | 12 +- src/core/persist/deletions.ts | 71 ++ src/core/persist/storage.ts | 17 + src/entrypoints/background.ts | 20 +- src/features/library/persist/favorites.ts | 6 +- src/features/library/persist/history.ts | 7 +- .../settings/panel/SettingsView.svelte | 206 +----- src/features/sync/panel/sync.svelte.ts | 610 ++++++------------ src/features/sync/persist/fit.test.ts | 39 +- src/features/sync/persist/fit.ts | 20 +- src/features/sync/persist/hash.ts | 20 + src/features/sync/persist/merge.test.ts | 208 ++++++ src/features/sync/persist/merge.ts | 130 ++++ src/features/sync/persist/sync-area.ts | 85 +++ src/features/sync/persist/sync-blob.test.ts | 155 +++++ src/features/sync/persist/sync-blob.ts | 193 ++++++ src/features/sync/persist/sync-config.ts | 105 +-- store/long-description-firefox.md | 2 +- store/long-description.txt | 9 +- store/privacy-policy-firefox.md | 18 +- tsconfig.json | 4 +- wxt.config.ts | 20 +- 31 files changed, 1326 insertions(+), 837 deletions(-) create mode 100644 src/core/persist/deletions.ts create mode 100644 src/features/sync/persist/hash.ts create mode 100644 src/features/sync/persist/merge.test.ts create mode 100644 src/features/sync/persist/merge.ts create mode 100644 src/features/sync/persist/sync-area.ts create mode 100644 src/features/sync/persist/sync-blob.test.ts create mode 100644 src/features/sync/persist/sync-blob.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7b43f2..a057b44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,30 +47,6 @@ jobs: - name: Build (Firefox) run: pnpm build:firefox - server: - name: sync server - runs-on: ubuntu-latest - # Separate pnpm workspace with its own lockfile and its own tsconfig - # (Cloudflare Workers types), excluded from the root tsconfig — so `pnpm - # check` above does not cover it. - steps: - - uses: actions/checkout@v4 - - # No `version:` input — it reads `packageManager` from the root - # package.json, and passing both is an error. - - uses: pnpm/action-setup@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - run: pnpm install --frozen-lockfile - working-directory: server - - - name: Type check - run: pnpm run check - working-directory: server - # The e2e suite (e2e/run.mjs) is deliberately not run here: it needs a real # Chrome download plus generated WAV fixtures, and it is not currently # all-green — see CONTRIBUTING.md, which tracks the count. Run it locally for diff --git a/CLAUDE.md b/CLAUDE.md index 7bfc632..0b060b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,9 +38,6 @@ node e2e/run.mjs # add --headful to watch ``` The harness plays a 440 Hz tone and asserts on the **processed output** (e.g. 880 Hz after +12 st) via `window.__noteByNoteDebug` in the content script and `window.__panelDebug` in the side panel. -### Sync server (`server/`, separate pnpm workspace) -`cd server ; pnpm install` — it has its own lockfile, `tsconfig.json` (Cloudflare Workers types), and is `exclude`d from the root tsconfig. See [server/README.md](server/README.md) for deploy. `pnpm run dev` there serves `http://localhost:8787`, which the extension's dev build targets automatically. - ## Architecture This is a **multi-context extension**. The single most important structural fact: **the audio engine lives in the page (content script), not in the side panel.** The side panel is a thin UI mirror that connects to the engine over a typed `chrome.runtime` Port. This is why practice flows (loops, sequences, playback) survive the side panel closing. @@ -100,7 +97,7 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. - **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped, timestamps in seconds. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. -- Optional **cross-device sync** (`src/features/sync/` + `server/`): last-write-wins backup snapshots to a Cloudflare Worker + KV. The secret sync ID **is the whole capability** (open CORS, no other auth). The Worker URLs live once in [sync-hosts.ts](src/features/sync/sync-hosts.ts) (env-free, so `wxt.config.ts` imports it for the manifest); [endpoint.ts](src/features/sync/endpoint.ts) picks localhost in dev, the deployed Worker in prod. The ID rides `storage.sync` between devices and is additionally kept as a **cookie on the sync host** so it survives an uninstall ([id-cookie.ts](src/features/sync/panel/id-cookie.ts) is the canonical explanation). That needs the `cookies` permission plus host access to the sync origin, which is an **optional** host permission requested from the Sync settings / on enable / on connect (a required one would disable the extension on update in Chrome and is opt-in on Firefox); `sync.durable` mirrors whether it is held. The background worker filters the sync host out of the site-grant machinery (`siteOrigins` in [background.ts](src/entrypoints/background.ts)) so it is neither registered for the engine nor removed by Revoke Permissions. +- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`, TTL 30 d, cap 200) so a merge can't resurrect them; track records need none (an emptied record still wins on `updatedAt`). Pushes are debounced 5 s and spaced ≥ 30 s (the browser meters writes at 120/min); the per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is migrated from the server-era record on first load. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87e7a85..d7ab665 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,9 +28,9 @@ pnpm test:dsp # node --test, the unit tests (DSP, chords, backup codec, sync f pnpm build # production build → .output/chrome-mv3 ``` -CI runs all three on every pull request, plus the Firefox build and `server/`'s -own `pnpm run check`. Run them locally first anyway; the -turnaround is much faster than waiting on a runner. (A first-time contributor's +CI runs all three on every pull request, plus the Firefox build. Run them +locally first anyway; the turnaround is much faster than waiting on a runner. +(A first-time contributor's workflow run needs a maintainer to click approve, so it may sit for a bit.) There is no ESLint or Prettier config, deliberately — match the style of the code around you. diff --git a/PRIVACY.md b/PRIVACY.md index 22a7f6d..67bd06a 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -32,11 +32,14 @@ it without uninstalling. ## What is transmitted, and when -The extension makes exactly one kind of network request: the optional -**cross-device sync** backup. Nothing else in the extension talks to the network. +The extension makes **no network requests of its own**. The one thing that +leaves your device is the optional **cross-device sync** copy, and it leaves +through your browser's own sync (Chrome sync, Firefox Sync) — the same channel +that carries your bookmarks — to the other devices signed into the same +browser profile. If browser sync is off, nothing leaves the device. -Sync is **on by default**, but it only starts transmitting once you have -something to sync. When it does, it uploads a single snapshot containing: +Sync is **on by default**. When there is something to sync, it writes one +compact snapshot into the browser's synced extension storage containing: - your settings and UI preferences - your EQ presets @@ -52,37 +55,31 @@ tracks you practised on, or anything identifying you personally. ### Where it goes -Snapshots are stored by a Cloudflare Worker with Cloudflare KV, at -`https://note-by-note-sync.oapp.workers.dev`, operated by the author. The server -source is in [`server/`](server/) and can be self-hosted — self-hosters change -one constant and rebuild. - -- There are no accounts. A random 43-character **sync ID** is the only - credential; it *is* the capability, so treat it like a password. -- The ID travels in an `X-Sync-Id` header, never in the URL, so it does not land - in request logs. KV is keyed by its SHA-256, so the raw ID is not stored either. -- Snapshots are stored **unencrypted**. Whoever operates the server can read - them. Run your own if that matters to you. -- Snapshots expire after **180 days**, refreshed on every write. -- IP addresses are visible to Cloudflare as part of serving and rate-limiting - requests, per Cloudflare's own data handling. They are not stored by the - Worker or linked to a snapshot. +Into your browser vendor's sync storage, under your browser account, subject to +that vendor's own data handling (Google for Chrome sync, Mozilla for Firefox +Sync — the latter end-to-end encrypted). The author operates no server and can +see none of it. There are no accounts with us and no sync ID. + +The browser caps this storage at 100 KB per extension. The snapshot is stored +compact and compressed so a typical library fits with room to spare; when one +doesn't, the oldest songs and chord charts stay on the device that has them and +the Settings page says so. ### Turning it off and deleting the data -- `Settings → Sync` turns sync off. No further data is transmitted. -- `Settings → Sync → Delete synced data` removes the server-side copy. -- Doing nothing also works: an unused snapshot expires after 180 days. +- `Settings → Sync` turns sync off on that device. No further data is written. +- `Settings → Sync → Delete synced data` empties the synced copy (other devices + with sync still on will write theirs again). +- Uninstalling the extension makes the browser remove its synced storage. ## Permissions and why | Permission | Why | | --- | --- | -| `storage` | Saves your markers, loops, snippets and settings on your device. | +| `storage` | Saves your markers, loops, snippets and settings on your device, and — through the browser's synced storage — carries them to your other devices. | | `activeTab`, `scripting` | Injects the audio engine into the tab when you press **Connect**. | | `tabs` | Reads the active tab's URL and title to look up the practice data you saved for that track. | | `tabCapture`, `offscreen` (Chrome only) | Fallback audio path for pages that block the audio worklet. | -| `cookies` + access to the sync server's domain (optional) | Keeps a copy of your sync ID as a cookie on the sync server's domain, the one place the browser does not wipe when the extension is uninstalled — so a reinstall gets your data back. Access to that domain is requested from `Settings → Sync` (or comes with **Connect**), never at install time. The cookie is read only through the extension API and never sent with a request (the sync ID travels in a header). No other site's cookies are touched; `Delete synced data` removes it. | | Access to all sites (optional) | Requested **only** when you first press **Connect**, never at install time, because you choose which sites to practise on. `Settings → Revoke Permissions` takes it back. | ## Children diff --git a/README.md b/README.md index f4fc93b..36c4080 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,8 @@ draws a chart under the timeline. **Keeping your place.** Settings are stored per track against a normalized URL, so reopening a video brings back its pitch, speed, markers, loops and snippets — however you open it, not just from the library. Favorites and -recents live in a library tab, and optional cross-device sync pushes a snapshot -to a small Cloudflare Worker. +recents live in a library tab, and optional cross-device sync carries it all +to your other browsers through the browser's own sync — no server, no account. ## Installing it @@ -167,36 +167,30 @@ the other way round. are in marker chips, loop/sequence bounds, the tab-capture CTA and the vocal-reducer control — known and pre-existing. -## Sync server - -`server/` is a separate pnpm workspace (its own lockfile and tsconfig): a -Cloudflare Worker plus KV storing one backup snapshot per sync ID, last write -wins. There are no accounts. The 43-character sync ID *is* the credential, so -treat it like a password. Deploy notes in [server/README.md](server/README.md). - -The ID travels in an `X-Sync-Id` header rather than the URL, because URLs are -recorded verbatim by request logs, and KV is keyed by its SHA-256 so the raw -token isn't stored either. Writes are rate-limited per IP and snapshots carry a -180-day TTL, refreshed on every write. - -A snapshot is your settings, UI preferences, EQ presets, Recent and Favorites -(page URL, title, duration, thumbnail URL) and per-track data (markers with your -labels, loop ranges, snippets, chord charts). **No audio, ever.** It is stored -unencrypted, so whoever operates the Worker can read it — run your own if that -matters to you. Sync is on by default but only mints an ID once you have -something to sync; `Settings → Sync → Delete synced data` removes the server -copy. Nothing else in the extension talks to the network: there is no telemetry -and no analytics. - -The ID lives in the browser's synced extension storage, so a second device on -the same profile picks it up by itself. The browser wipes that storage (on -every device) when the extension is uninstalled, so the ID can also be kept as -a cookie on the sync server's domain, which outlives the extension — that is -what the `cookies` permission and the optional access to that one domain are -for (`Settings → Sync → Keep the ID after a reinstall`; the **Connect** grant -covers it too). How and why, and what it does not survive, is documented in -[id-cookie.ts](src/features/sync/panel/id-cookie.ts). Self-hosters change the -URL in [sync-hosts.ts](src/features/sync/sync-hosts.ts) and rebuild. +## Sync + +There is no server. Cross-device sync writes a compact, gzipped copy of your +data into the browser's synced extension storage (`browser.storage.sync`), and +the browser vendor's sync — Chrome sync, Firefox Sync — carries it to the other +devices signed into the same profile. No account with us, no ID to paste, no +network request of the extension's own; nothing in the extension talks to the +network at all, and there is no telemetry. + +A copy is your settings, UI preferences, EQ presets, Recent and Favorites (page +URL, title, duration) and per-track data (markers with your labels, loop +ranges, snippets, chord charts). **No audio, ever.** It is stored in the +compact backup format ([backup-codec.ts](src/core/persist/backup-codec.ts) — +the same file `Settings → Export` writes), which is ~9× smaller than the raw +data, so even a few hundred songs with chord charts fit the browser's 100 KB +quota; if a library still doesn't, the oldest songs, then the oldest chord +charts, stay on the device that has them ([fit.ts](src/features/sync/persist/fit.ts)). + +Two devices' copies are merged rather than overwritten +([merge.ts](src/features/sync/persist/merge.ts)): the more recently edited +version of each song wins, and a song you removed on one device stays removed +(deletions are dated, [deletions.ts](src/core/persist/deletions.ts)). Sync is +on by default; `Settings → Sync` turns it off, and `Delete synced data` empties +the synced copy. ## License diff --git a/SECURITY.md b/SECURITY.md index 902325b..548625e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,14 +19,12 @@ credited in the advisory unless you would rather not be. ## What is in scope -The extension itself, and the sync server under [`server/`](server/): +The extension itself: - Escaping the extension's boundaries — anything that lets a web page reach privileged extension APIs through the content script or the message port - Anything that causes the extension to grant, keep, or widen host permissions beyond what the user approved -- Cross-user data access on the sync server, or anything that lets a snapshot - be read or written without its sync ID - Injection through data the extension stores and later renders — track titles, marker labels, thumbnail URLs, restored sync snapshots @@ -35,23 +33,15 @@ The extension itself, and the sync server under [`server/`](server/): Two properties look like bugs but are the documented design. Reports of these will be closed as working-as-intended: -- **The sync ID is the entire capability.** There are no accounts. Anyone - holding the 43-character random ID can read and overwrite that snapshot, and - the server has open CORS and no other authentication. This is deliberate — - the ID is the secret, and the trade-off is spelled out in - [PRIVACY.md](PRIVACY.md). Reports that it is guessable need to show it is - actually guessable. -- **Sync is last-write-wins.** Two devices editing at once means one loses. - That is a data-loss property, not a security boundary; file it as a bug if - you can trigger it in a way the design does not predict. - -Also out of scope: findings against `note-by-note-sync.oapp.workers.dev` that -require volumetric traffic, and anything that needs the user to already be -running attacker-controlled code locally. - -## A note on scanning - -Please do not run automated scanners or load tests against the hosted sync -Worker — it is a personal Cloudflare account. The server is small and -self-hostable ([`server/README.md`](server/README.md)); test against your own -deployment instead. +- **Sync trusts the browser profile.** There is no server and no credential of + ours: the synced copy lives in the browser's own synced extension storage, + so whoever can sign into the browser profile can read and change it — the + same boundary as the bookmarks. The trade-off is spelled out in + [PRIVACY.md](PRIVACY.md). +- **Sync merges by recency.** Two devices editing the same song at once means + the later edit wins that song. That is a data-loss property, not a security + boundary; file it as a bug if you can trigger it in a way the design does + not predict. + +Also out of scope: anything that needs the user to already be running +attacker-controlled code locally. diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ebf96d7..90010e6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,4 @@ -# Not a monorepo (server/ has its own workspace) — this file exists only to -# hold pnpm settings, which pnpm 11 reads here rather than from package.json. +# Not a monorepo — this file exists only to hold pnpm settings, which pnpm 11 reads here rather than from package.json. packages: [] # Dependency build scripts. esbuild installs its native binary (the worklet diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index 85e404c..1483796 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -135,6 +135,7 @@ function backup(patch: Partial = {}): Backup { favorites: [], eqPresets: [], tracks: [], + deletions: {}, ...patch, }; } @@ -506,6 +507,17 @@ test('encode is deterministic regardless of track enumeration order', () => { assert.deepEqual(encodeBackup(shuffled), encodeBackup(b)); }); +test('deletion records travel in seconds and are omitted when empty', () => { + assert.equal('del' in encodeBackup(backup()), false); + const b = backup({ deletions: { 'h:abc:230': 1_757_112_345_678, 'h:*': 1_757_000_000_000 } }); + const enc = encodeBackup(b); + assert.deepEqual(enc.del, { 'h:*': 1757000000, 'h:abc:230': 1757112346 }); + assert.deepEqual(roundTrip(b).deletions, { 'h:*': 1757000000000, 'h:abc:230': 1757112346000 }); + const v1 = JSON.parse(JSON.stringify(backup())); + delete v1.deletions; + assert.deepEqual(parseBackupJson(v1).deletions, {}); +}); + test('exportedAt is kept to the second; appVersion is dropped', () => { const back = roundTrip(backup()); assert.equal(back.exportedAt, 1_757_200_000_000); diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index c1b89f7..58c356c 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -6,6 +6,7 @@ import { } from '../model/defaults.ts'; import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; import { identityKey } from '../model/track-identity.ts'; +import { normalizeDeletions, type Deletions } from './deletions.ts'; import type { ChordChart, ChordSegment, @@ -71,6 +72,9 @@ export interface Backup { eqPresets: EqPreset[]; /** Per-track markers and snippets, one entry per saved track. */ tracks: TrackData[]; + /** What was deleted and when — see `deletions.ts`. Absent in files from + * before sync merged; `{}` then. */ + deletions: Deletions; } // --------------------------------------------------------------------------- @@ -194,6 +198,8 @@ export interface CompactBackup { h: CompactEntry[]; f: CompactFavorite[]; t: CompactTrack[]; + /** Deletion records, dates in seconds; omitted when there are none. */ + del?: Record; } // --------------------------------------------------------------------------- @@ -737,7 +743,7 @@ export function encodeBackup(backup: Backup): CompactBackup { const h = backup.history.map((e) => encodeEntry(e, songs)); const f = backup.favorites.map((e) => encodeFavorite(e, songs)); const t = [...backup.tracks].sort(byKey).map((track) => encodeTrack(track, songs)); - return { + const out: CompactBackup = { format: BACKUP_FORMAT, version: COMPACT_VERSION, at: secs(backup.exportedAt), @@ -749,6 +755,11 @@ export function encodeBackup(backup: Backup): CompactBackup { f, t, }; + const deletions = Object.entries(normalizeDeletions(backup.deletions)).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ); + if (deletions.length) out.del = Object.fromEntries(deletions.map(([k, when]) => [k, secs(when)])); + return out; } /** Reads a compact (v2) backup, or throws an `Error` whose message is safe to @@ -769,6 +780,9 @@ export function decodeBackup(raw: unknown): Backup { favorites: requireArray(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), eqPresets: requireArray(raw.eq, 'eqPresets').map(decodeEqPreset), tracks: requireArray(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), + deletions: Object.fromEntries( + Object.entries(normalizeDeletions(raw.del)).map(([k, when]) => [k, when * 1000]), + ), }; } @@ -793,6 +807,7 @@ function normalizeV1(raw: Record): Backup { favorites: requireKeyedArray(raw.favorites, 'favorites'), eqPresets: requireArray(raw.eqPresets, 'eqPresets') as EqPreset[], tracks: requireKeyedArray(raw.tracks, 'tracks'), + deletions: normalizeDeletions(raw.deletions), }; } diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index 0bbea2b..b9e266e 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -5,7 +5,9 @@ import { parseBackupJson, type Backup, } from './backup-codec'; +import { mergeDeletions, pruneDeletions } from './deletions'; import { + deletionsItem, eqPresetsItem, favoritesItem, historyItem, @@ -28,7 +30,7 @@ async function loadAllTrackData(): Promise { } export async function createBackup(): Promise { - const [settings, uiPrefs, history, favorites, eqPresets, tracks] = + const [settings, uiPrefs, history, favorites, eqPresets, tracks, deletions] = await Promise.all([ settingsItem.getValue(), uiPrefsItem.getValue(), @@ -36,6 +38,7 @@ export async function createBackup(): Promise { favoritesItem.getValue(), eqPresetsItem.getValue(), loadAllTrackData(), + deletionsItem.getValue(), ]); return { format: BACKUP_FORMAT, @@ -48,6 +51,7 @@ export async function createBackup(): Promise { favorites, eqPresets, tracks, + deletions, }; } @@ -78,8 +82,11 @@ export function parseBackup(text: string): Backup { * Replaces every stored value with the backup's, dropping data the file does * not carry — a restore reproduces the machine it came from rather than * merging into whatever is here. Host permissions are left untouched. + * Deletion records are the one thing merged, not replaced: forgetting this + * device's would let a sync merge resurrect what it had removed. */ export async function restoreBackup(backup: Backup): Promise { + const deletions = await deletionsItem.getValue(); await removeAllTrackData(); await Promise.all([ settingsItem.setValue(backup.settings), @@ -87,6 +94,9 @@ export async function restoreBackup(backup: Backup): Promise { historyItem.setValue(backup.history), favoritesItem.setValue(backup.favorites), eqPresetsItem.setValue(backup.eqPresets), + deletionsItem.setValue( + pruneDeletions(mergeDeletions(deletions, backup.deletions ?? {}), Date.now()), + ), ...backup.tracks.map(saveTrackData), ]); } diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts new file mode 100644 index 0000000..e8153ed --- /dev/null +++ b/src/core/persist/deletions.ts @@ -0,0 +1,71 @@ +/** + * Deletion records ("tombstones"): what the user removed, and when. Without + * them a Recent row or a favorite deleted on one device would come straight + * back from another device's copy the next time the two merge — a merge is a + * union, and it cannot tell "never had it" from "removed it". Track records + * need none: an emptied record still exists, with a newer `updatedAt`, and + * wins its merge on that. + * + * One flat map, `key → when` (ms). Keys: `h:` a Recent row, + * `f:` a favorite, `h:*` "Clear Recent". A record older than the + * item it names (the song was played again after the deletion) is ignored, so + * re-adding always works. Records expire after a month and are capped, newest + * kept — long enough for any device that will ever sync again to see them. + * + * Pure and DOM-free (relative `.ts` imports; runs under `node --test`). + */ + +export type Deletions = Record; + +export const historyDeletion = (key: string) => `h:${key}`; +export const favoriteDeletion = (key: string) => `f:${key}`; +export const HISTORY_CLEARED = 'h:*'; + +export const DELETION_TTL_MS = 30 * 24 * 60 * 60_000; +export const DELETION_CAP = 200; + +/** When `key` was deleted, or 0 if it wasn't. */ +export function deletedAt(deletions: Deletions, key: string): number { + return deletions[key] ?? 0; +} + +/** Whether a Recent row with `updatedAt` is covered by a deletion — its own + * or a "Clear Recent" — dated at or after it. */ +export function historyDeleted(deletions: Deletions, key: string, updatedAt: number): boolean { + const when = Math.max(deletedAt(deletions, historyDeletion(key)), deletedAt(deletions, HISTORY_CLEARED)); + return when > 0 && when >= updatedAt; +} + +export function favoriteDeleted(deletions: Deletions, key: string, since: number): boolean { + const when = deletedAt(deletions, favoriteDeletion(key)); + return when > 0 && when >= since; +} + +/** Newest date per key. */ +export function mergeDeletions(a: Deletions, b: Deletions): Deletions { + const out: Deletions = { ...a }; + for (const [key, when] of Object.entries(b)) { + if (typeof when !== 'number') continue; + out[key] = Math.max(out[key] ?? 0, when); + } + return out; +} + +/** Drops records older than the TTL and, past the cap, the oldest. */ +export function pruneDeletions(deletions: Deletions, now: number): Deletions { + const kept = Object.entries(deletions) + .filter(([, when]) => typeof when === 'number' && Number.isFinite(when) && now - when < DELETION_TTL_MS) + .sort(([, a], [, b]) => b - a) + .slice(0, DELETION_CAP); + return Object.fromEntries(kept); +} + +/** Anything a file or a remote copy claims to be deletions, made safe. */ +export function normalizeDeletions(raw: unknown): Deletions { + if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return {}; + const out: Deletions = {}; + for (const [key, when] of Object.entries(raw as Record)) { + if (typeof when === 'number' && Number.isFinite(when) && when > 0) out[key] = when; + } + return out; +} diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index 8557bfb..22180c5 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -1,5 +1,6 @@ import { storage, type StorageItemKey, type WxtStorageItem } from '#imports'; import { DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults'; +import { pruneDeletions, type Deletions } from './deletions'; import type { EqPreset, FavoriteEntry, @@ -76,6 +77,22 @@ export const grantedOriginsItem = defineItem('local:grantedOrigins', { fallback: [], }); +/** What the user deleted and when, so a sync merge doesn't bring it back — + * see `deletions.ts`. Part of the backup; never part of "Reset Settings". */ +export const deletionsItem = defineItem('local:deletions', { + fallback: {}, +}); + +/** Dates a deletion (`historyDeletion(key)`, `favoriteDeletion(key)`, + * `HISTORY_CLEARED`) at now, pruning expired records on the way. */ +export async function recordDeletion(...keys: string[]): Promise { + const now = Date.now(); + const current = await deletionsItem.getValue(); + const next = { ...current }; + for (const key of keys) next[key] = now; + await deletionsItem.setValue(pruneDeletions(next, now)); +} + /** Per-track markers/snippets, keyed by TrackIdentity.key. */ export function trackDataKey(key: string) { return `local:track:${key}` as const; diff --git a/src/entrypoints/background.ts b/src/entrypoints/background.ts index 7c15cb5..d7e71ed 100644 --- a/src/entrypoints/background.ts +++ b/src/entrypoints/background.ts @@ -3,7 +3,6 @@ import { onMessage } from '@/core/messaging/rpc'; import { grantedOriginsItem } from '@/core/persist/storage'; import { CAN_CAPTURE_TAB, HAS_SIDE_PANEL_API } from '@/core/platform'; import { disablePanelForTab, enablePanelForTab, isPanelShowing } from '@/core/side-panel'; -import { SYNC_HOST_PATTERNS } from '@/features/sync/sync-hosts'; /** Firefox's sidebar API. WXT's `browser` types are Chromium-shaped and don't * declare it, so reach it through a narrow cast — only ever on the Firefox @@ -28,13 +27,6 @@ function originPattern(url: string): string | null { } } -/** The sync host is held for the ID cookie, not for practising on — it is - * neither a site to register the engine on nor one "Revoke Permissions" should - * take back. Everything else `permissions.getAll()` reports is a site grant. */ -function siteOrigins(origins: string[]): string[] { - return origins.filter((o) => !SYNC_HOST_PATTERNS.includes(o)); -} - /** Content-script match patterns for the origins we hold. `` is a * valid permission origin but not a valid registration match, so the broad * grant collapses to the http(s) wildcard. */ @@ -72,9 +64,8 @@ async function syncRegistration(matches: string[]) { * extensions UI, or revoke button). */ async function syncFromPermissions() { const { origins = [] } = await browser.permissions.getAll(); - const sites = siteOrigins(origins); - await grantedOriginsItem.setValue(sites); - await syncRegistration(registrationMatches(sites)); + await grantedOriginsItem.setValue(origins); + await syncRegistration(registrationMatches(origins)); } export default defineBackground(() => { @@ -178,11 +169,8 @@ export default defineBackground(() => { onMessage('revokeAllPermissions', async () => { const { origins = [] } = await browser.permissions.getAll(); - // `` covers the sync host, so revoking it also ends cookie access; - // the panel re-requests it from the Sync settings when needed. - const sites = siteOrigins(origins); - if (sites.length) { - await browser.permissions.remove({ origins: sites }).catch(() => { + if (origins.length) { + await browser.permissions.remove({ origins }).catch(() => { // Some patterns may already be gone; onRemoved still reconciles below. }); } diff --git a/src/features/library/persist/favorites.ts b/src/features/library/persist/favorites.ts index 17f4cda..9d483ab 100644 --- a/src/features/library/persist/favorites.ts +++ b/src/features/library/persist/favorites.ts @@ -1,6 +1,7 @@ import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; import { isSameTrack } from '../../../core/model/track-identity'; -import { favoritesItem } from '../../../core/persist/storage'; +import { favoriteDeletion } from '../../../core/persist/deletions'; +import { favoritesItem, recordDeletion } from '../../../core/persist/storage'; /** Star a song: copy the history entry into the Favorites library (top of the * manual order). No-op if already favorited. */ @@ -16,9 +17,12 @@ export async function addFavorite(entry: HistoryEntry): Promise { ]); } +/** Dated (`deletions.ts`) so a sync merge with another device's older copy + * doesn't star the song again. */ export async function removeFavorite(key: string): Promise { const list = await favoritesItem.getValue(); await favoritesItem.setValue(list.filter((e) => e.identity.key !== key)); + await recordDeletion(favoriteDeletion(key)); } /** Persist a new manual order (list of identity keys, complete). */ diff --git a/src/features/library/persist/history.ts b/src/features/library/persist/history.ts index b3feb85..942b06a 100644 --- a/src/features/library/persist/history.ts +++ b/src/features/library/persist/history.ts @@ -1,7 +1,8 @@ import { HISTORY_LIMIT } from '../../../core/model/defaults'; import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; import { isSameTrack } from '../../../core/model/track-identity'; -import { historyItem } from '../../../core/persist/storage'; +import { HISTORY_CLEARED, historyDeletion } from '../../../core/persist/deletions'; +import { historyItem, recordDeletion } from '../../../core/persist/storage'; /** Insert or refresh a Recent entry (newest first, LRU-capped). * @@ -46,11 +47,15 @@ export async function dedupeHistory(): Promise { if (kept.length !== list.length) await historyItem.setValue(kept); } +/** Deletions are dated (`deletions.ts`) so a sync merge with another device's + * older copy doesn't bring the row back. */ export async function removeHistoryEntry(key: string): Promise { const list = await historyItem.getValue(); await historyItem.setValue(list.filter((e) => e.identity.key !== key)); + await recordDeletion(historyDeletion(key)); } export async function clearHistory(): Promise { await historyItem.setValue([]); + await recordDeletion(HISTORY_CLEARED); } diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index 5c8877e..acddab5 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -190,20 +190,8 @@ let syncBusy = $state(false); let syncNotice = $state<{ ok: boolean; text: string } | null>(null); - let connectId = $state(''); - let idCopied = $state(false); async function setSyncEnabled(on: boolean) { - if (on && sync.syncId && !sync.lastSyncedAt) { - // An ID this device never synced with — inherited from another device - // through browser sync. Joining is pull-first, same as Connect. - const ok = confirm( - 'A sync ID from your other device was found. Replace all data on this ' + - "device with the synced copy? If the ID has no data yet, this device's " + - 'data is uploaded instead.', - ); - if (!ok) return; - } syncBusy = true; syncNotice = null; try { @@ -214,30 +202,18 @@ } } - /** An ID arrived from another device while this one already held data. */ - async function resolveConsent(accept: boolean) { - syncBusy = true; - syncNotice = null; - try { - // Accepting reloads the panel, so nothing after it runs. - if (accept) await sync.acceptRemote(); - else await sync.keepLocal(); - } finally { - syncBusy = false; - } - } - async function deleteSyncedData() { const ok = confirm( - 'Delete the synced copy of your data from the server? Sync will be turned ' + - 'off. Data on this device is not affected.', + "Delete the copy of your data in the browser's sync storage? Sync will be " + + 'turned off on this device. Data on this device is not affected, and other ' + + 'devices with sync on will upload their copy again.', ); if (!ok) return; syncBusy = true; syncNotice = null; try { await sync.deleteRemote(); - syncNotice = { ok: true, text: 'Synced data deleted from the server.' }; + syncNotice = { ok: true, text: 'Synced data deleted.' }; } catch (err) { syncNotice = { ok: false, text: `Delete failed: ${message(err)}` }; } finally { @@ -245,47 +221,6 @@ } } - async function keepAfterReinstall() { - syncNotice = null; - if (!(await sync.keepAfterReinstall())) { - syncNotice = { ok: false, text: 'Permission not granted — the ID is not kept after a reinstall.' }; - } - } - - async function copySyncId() { - if (!sync.syncId) return; - await navigator.clipboard.writeText(sync.syncId); - idCopied = true; - setTimeout(() => (idCopied = false), 1500); - } - - async function connectSync() { - const id = connectId.trim(); - if (!id) return; - const ok = confirm( - 'Replace all data on this device with the synced copy? ' + - "If the ID has no data yet, this device's data is uploaded instead.", - ); - if (!ok) return; - syncBusy = true; - syncNotice = null; - try { - // The 'applied' path never returns here — it reloads the panel. - const result = await sync.connectWithId(id); - if (result === 'uploaded') { - connectId = ''; - syncNotice = { - ok: true, - text: "No synced data found for that ID — this device's data was uploaded.", - }; - } - } catch (err) { - syncNotice = { ok: false, text: `Connect failed: ${message(err)}` }; - } finally { - syncBusy = false; - } - } - function lastSynced(ts: number): string { if (!ts) return 'never'; const minutes = Math.round((Date.now() - ts) / 60000); @@ -562,84 +497,37 @@
{@render prefText( 'Sync between devices', - 'Keep your settings, songs, presets, markers and snippets the same everywhere. No account needed — devices are linked by a private ID.', + "Keep your settings, songs, presets, markers and snippets the same everywhere. Uses your browser's built-in sync: sign in to the browser with sync turned on and it reaches your other devices. No account with us, no server.", )} - sync.enabled, (on) => void setSyncEnabled(on)} label="Sync between devices" />
- {#if sync.needsConsent} -
- {@render prefText( - 'A sync ID from your other device was found', - 'This device already has data of its own, so nothing has been changed yet. Use the synced copy and replace what is here, or keep this device and upload it instead.', - )} -
- - -
-
- {/if} - {#if sync.enabled && sync.syncId} -
- {@render prefText( - 'Your sync ID', - 'Devices signed into the same browser profile pick this ID up automatically; elsewhere, enter it by hand. Keep it private — anyone who has it can read and change your data.', - )} -
- {sync.syncId} + {#if sync.enabled} + {#if sync.pendingApply} +
+ {@render prefText( + 'Changes from another device are waiting', + 'They are applied when no song is loaded, or now — applying reloads the panel.', + )}
-
-
- - {@render prefText( - 'Keep the ID after a reinstall', - sync.durable - ? 'On. A copy is kept in this browser profile, outside the extension — but not past clearing the browser’s cookies, so keep a copy for that.' - : 'Uninstalling wipes the ID from the extension. Allow access to the sync server’s domain to keep a copy in this browser profile instead — nothing else is accessed.', - )} - {#if !sync.durable} - - {/if} -
+ {/if} + {#if sync.trimmed} +
+ The browser's sync storage is full, so the oldest songs and chord charts stay + on this device only. Everything else syncs. +
+ {/if}
- {:else} - {#if sync.syncId} -
- {sync.lastSyncedAt - ? 'Sync is off. Turn it back on to keep using your existing sync ID.' - : 'A sync ID from your other device was found — turn on sync to use it.'} -
- {:else if sync.enabled} -
- Sync is on. Your private ID is created as soon as there is something to - sync — a song in Recent, a marker, an EQ preset. On a new device, the ID - from your other devices arrives through browser sync within a minute or so; - if it doesn't, paste it below. -
- {/if} -
- {@render prefText( - 'Connect with a sync ID', - 'Paste the ID from your other device. Its synced data replaces what is on this device.', - )} -
- - -
-
{/if} {#if syncNotice}
packedChars(encodeBackup(backup)); + +function isEmpty(backup: Backup): boolean { + return ( + backup.history.length === 0 && + backup.favorites.length === 0 && + backup.tracks.length === 0 && + backup.eqPresets.length === 0 && + Object.keys(backup.deletions ?? {}).length === 0 + ); } /** - * Device sync: mirrors the backup snapshot (see `persist/backup.ts`) to the - * sync server under a secret ID, last-write-wins. Local changes are detected - * via storage change events and pushed after a debounce; remote changes are - * pulled at startup and on an interval, and applied via `restoreBackup` plus - * a panel reload (stores read storage once at startup — same rationale as the - * import flow in SettingsView). + * Cross-device sync through the browser's own synced storage — no server, + * no account, no ID: whoever signs into the same browser profile gets the + * data, because the browser vendor's sync carries it (see `sync-blob.ts` for + * the layout, `merge.ts` for how two copies become one). + * + * One routine, `#reconcile`, does everything: read the area, compare with + * what this device last saw, merge, then write locally and/or remotely as + * needed. Local changes (storage events) and remote ones (sync-area events) + * both just ask for a reconcile. Applying a merge that changes local data + * ends in a panel reload — stores read storage once at start-up, same as the + * import flow — so that is deferred while a track is loaded and picked up on + * the next quiet moment, panel open, or "Sync now". * * Runs only in the sidepanel: it is the sole writer of synced data, so there * is nothing to observe while it's closed. A change the panel didn't manage @@ -48,212 +78,108 @@ function errorMessage(err: unknown): string { */ class SyncStore { enabled = $state(false); - syncId = $state(null); lastSyncedAt = $state(0); lastError = $state(null); - /** An ID arrived from another device while this one already held data of its - * own. Applying would overwrite it, so the panel asks first — see - * `acceptRemote` / `keepLocal`. */ - needsConsent = $state(false); - /** Whether the ID's durable copy can be kept — host access to the sync - * origin is held (see id-cookie.ts). Drives the Settings copy. */ - durable = $state(false); + /** The last push left old songs or charts out to fit the quota. */ + trimmed = $state(false); + /** Bytes the area holds, by the browser's accounting. */ + usedBytes = $state(0); + /** Another device's changes are in, waiting for a moment without a track + * loaded (applying reloads the panel). "Sync now" applies them at once. */ + pendingApply = $state(false); #syncing = $state(false); status = $derived<'off' | 'syncing' | 'error' | 'idle'>( !this.enabled ? 'off' : this.#syncing ? 'syncing' : this.lastError ? 'error' : 'idle', ); + usedPercent = $derived( + this.usedBytes ? Math.max(1, Math.round((this.usedBytes / SYNC_QUOTA_BYTES) * 100)) : 0, + ); #config: SyncConfig = { ...DEFAULT_SYNC_CONFIG }; - /** Mirrors `syncIdItem` — the ID lives in browser-synced storage (see - * `persist/sync-config.ts`), separate from the per-device bookkeeping. */ - #id: string | null = null; - /** The ID last handed to `writeIdCookie` by this document, so `#reflect` - * writes the cookie once per identity rather than on every state change. */ - #cookieId: string | null = null; - /** Suppresses the echo of our own `syncConfigItem` write. The ID watcher - * needs no such flag: its own echo carries `value === #id` and is skipped by - * value, so a real event from another device is never dropped. */ + /** Suppresses the echo of our own `syncConfigItem` write. */ #writing = false; - /** Suppresses change events while `restoreBackup` writes a remote snapshot, - * so applying can't schedule a push of what was just pulled. */ + /** Suppresses local change events while a merge is being written. */ #applying = false; #pushTimer: ReturnType | undefined; - /** Serializes pushes and reconciles so they can't interleave. */ + #remoteTimer: ReturnType | undefined; + #retryTimer: ReturnType | undefined; + #lastPushAt = 0; + #tornSince = 0; + /** Serializes reconciles so they can't interleave. */ #queue: Promise = Promise.resolve(); async init() { - const [{ config, syncId }, kept, durable] = await Promise.all([ - loadSyncState(), - readIdCookie(), - hasIdCookieAccess(), - ]); - this.durable = durable; - this.#config = config; - this.#id = syncId; + this.#config = await loadSyncConfig(); this.#reflect(); - // Watchers go up before any write below, so nothing arriving from another - // device in the meantime is missed. syncConfigItem.watch((value) => { if (this.#writing) return; this.#config = value ?? { ...DEFAULT_SYNC_CONFIG }; this.#reflect(); }); - // The ID is browser-synced: another device on this browser profile can - // hand this one an ID (or replace it) at any time. - syncIdItem.watch((value) => { - if (value === this.#id) return; - if (value === null && this.#id) { - // The key was deleted, not replaced — the sync area was purged (an - // uninstall on another device, a sync reset). Another device changing - // identity on purpose always arrives as a different string. Put our ID - // back rather than drift into minting a new one over the same data. - // Consent is not a factor: it gates applying remote data (see - // `#startReconcile`), not holding an identity. - void this.#writeId(this.#id); - return; - } - this.#id = value; - this.#reflect(); - if (!this.#config.enabled) return; - // Identity changed while enabled: the bookkeeping refers to the old - // blob — reset it and reconcile against the new one. - void this.#saveConfig({ lastSyncedAt: 0, lastSyncedHash: null, pendingPush: false }).then( - () => this.#startReconcile({ allowApply: session.media === null }), - ); - }); - // Connect's `` grant (and Revoke Permissions) changes cookie - // access without going through this store. - browser.permissions.onAdded.addListener(() => void this.#refreshDurable()); - browser.permissions.onRemoved.addListener(() => void this.#refreshDurable()); - - // Best-effort like every other sync-area write: a failure here must not - // keep the listeners below from being installed. - await this.#recoverId(kept).catch(() => {}); - browser.storage.local.onChanged.addListener((changes) => { - if (this.#applying) return; - if (!this.#config.enabled) return; + if (this.#applying || !this.#config.enabled) return; if (!Object.keys(changes).some((key) => SYNCED_KEY_RE.test(key))) return; this.#onDataChanged(); }); - - if (this.#config.enabled && this.#id) { - await this.#startReconcile({ allowApply: true }); + onSyncAreaChanged(() => { + if (!this.#config.enabled) return; + clearTimeout(this.#remoteTimer); + this.#remoteTimer = setTimeout(() => { + void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })); + }, REMOTE_DEBOUNCE_MS); + }); + if (this.#config.enabled) { + await this.#enqueue(() => this.#reconcile({ allowApply: true })); } - - // A track being loaded defers remote applies (the reload would interrupt - // practice) to the next panel open or a manual "Sync now". setInterval(() => { - if (!this.#config.enabled || !this.#id) return; - void this.#startReconcile({ allowApply: session.media === null }); - }, PULL_INTERVAL_MS); - } - - /** - * Reconcile, but never silently overwrite data this device already has. - * - * Sync ships on, and the ID rides browser-profile sync, so a device can be - * handed an identity it never asked for. When that device is empty (a fresh - * install — the case this default exists for) adopting is what the user - * wants and there is nothing to lose, so consent is implied. When it already - * holds a library, applying the remote would delete it: raise `needsConsent` - * and let the panel ask instead. - */ - async #startReconcile(opts: { allowApply: boolean }) { - const id = this.#id; - if (!this.#config.enabled || !id) return; - if (this.#config.consentedId !== id) { - if (!(await this.#isPristine())) { - this.needsConsent = true; - return; - } - await this.#saveConfig({ consentedId: id }); - } - await this.#enqueue(() => this.#reconcile(opts)); - } - - /** Nothing here a remote snapshot could destroy. Settings and UI prefs are - * excluded deliberately — they are a keystroke to redo, and weighing them - * would make the common "installed, opened it once" path prompt for nothing. */ - async #isPristine(): Promise { - const local = await createBackup(); - return ( - local.history.length === 0 && - local.favorites.length === 0 && - local.tracks.length === 0 && - local.eqPresets.length === 0 - ); - } - - /** User chose the synced copy: apply it over this device's data. */ - async acceptRemote(): Promise { - const id = this.#id; - if (!id) return; - this.needsConsent = false; - await this.#saveConfig({ consentedId: id }); - await this.#enqueue(() => this.#reconcile({ allowApply: true })); - } - - /** User chose this device's data: keep it and let it win the next push. */ - async keepLocal(): Promise { - const id = this.#id; - if (!id) return; - this.needsConsent = false; - await this.#saveConfig({ consentedId: id, lastChangedAt: Date.now(), pendingPush: true }); - await this.#enqueue(() => this.#push({ force: true })); + if (!this.#config.enabled) return; + void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })); + }, SAFETY_INTERVAL_MS); } - /** Turns sync on. With no ID anywhere — none kept from an earlier enable, - * none received from another device via browser sync — generates one and - * uploads this device's data; otherwise reuses the ID and reconciles. */ async enable(): Promise { - // In a gesture, so the durable copy can be authorised in the same breath. - await this.#ensureDurable(); - const fresh = this.#id === null; - const id = this.#id ?? generateSyncId(); - await this.#saveId(id); - // Turning it on by hand is the consent. - await this.#saveConfig({ enabled: true, consentedId: id, lastError: null }); - this.needsConsent = false; - await this.#enqueue(() => - fresh ? this.#push({ force: true }) : this.#reconcile({ allowApply: true }), - ); + await this.#saveConfig({ enabled: true, lastError: null }); + await this.#enqueue(() => this.#reconcile({ allowApply: true })); } - /** Keeps the ID so re-enabling picks the same remote blob back up. */ async disable(): Promise { clearTimeout(this.#pushTimer); - this.needsConsent = false; + clearTimeout(this.#retryTimer); + this.pendingApply = false; await this.#saveConfig({ enabled: false }); } + /** Back up now and pull in the other devices' changes, reload included. */ + async syncNow(): Promise { + await this.#enqueue(() => this.#reconcile({ allowApply: true })); + } + /** - * Removes this ID's snapshot from the server. Sync is switched off too — - * leaving it on would re-upload from the next change and quietly undo the - * deletion. The ID is kept, so turning sync back on starts a fresh blob; its - * durable copy is not, so a reinstall does not quietly bring it back. + * Empties the synced copy and turns sync off — left on, the next change + * would quietly re-upload. Other devices with sync on will re-seed it from + * their own data the next time they change something. */ async deleteRemote(): Promise { - const id = this.#id; - if (!id) return; clearTimeout(this.#pushTimer); - this.needsConsent = false; + clearTimeout(this.#retryTimer); + this.pendingApply = false; await this.#enqueue(async () => { this.#syncing = true; try { - await deleteSnapshot(id); + await clearSyncArea(); + this.usedBytes = 0; await this.#saveConfig({ enabled: false, lastSyncedAt: 0, - lastSyncedHash: null, + lastRemoteHash: null, + lastLocalHash: null, pendingPush: false, lastError: null, + trimmed: false, }); - this.#cookieId = null; - await removeIdCookie(); } catch (err) { - await this.#saveConfig({ lastError: errorMessage(err) }); + await this.#saveConfig({ lastError: syncErrorMessage(err) }); throw err; } finally { this.#syncing = false; @@ -261,119 +187,11 @@ class SyncStore { }); } - /** - * Links this device to an existing sync ID. Pull-first: existing remote data - * replaces this device's (the UI confirms beforehand; ends in a reload) and - * never the other way around — a fresh install must not clobber the remote. - * Returns 'uploaded' when the ID had no data yet and local data seeded it. - */ - async connectWithId(rawId: string): Promise<'applied' | 'uploaded'> { - const id = rawId.trim(); - if (!SYNC_ID_RE.test(id)) throw new Error("That doesn't look like a sync ID."); - await this.#ensureDurable(); - await this.#saveId(id); - // Typing in someone else's ID, past the UI's confirm, is the consent. - await this.#saveConfig({ - enabled: true, - consentedId: id, - lastSyncedAt: 0, - lastSyncedHash: null, - pendingPush: false, - lastError: null, - }); - this.needsConsent = false; - return this.#enqueue(async () => { - try { - const remote = await pullSnapshot(id); - if (remote) { - await this.#applyRemote(remote); - return 'applied' as const; - } - await this.#push({ force: true }); - return 'uploaded' as const; - } catch (err) { - await this.#saveConfig({ lastError: errorMessage(err) }); - throw err; - } - }); - } - - async syncNow(): Promise { - await this.#startReconcile({ allowApply: true }); - } - - /** Settings → "Keep after reinstall". Must run in a user gesture. */ - async keepAfterReinstall(): Promise { - return this.#ensureDurable(); - } - - /** Asks for cookie access if it isn't held yet; a refusal is not an error — - * sync works without the durable copy, the panel just says so. */ - async #ensureDurable(): Promise { - if (!this.durable) this.durable = await requestIdCookieAccess(); - if (this.durable) await this.#onDurable(); - return this.durable; - } - - async #refreshDurable() { - const durable = await hasIdCookieAccess(); - if (durable === this.durable) return; - this.durable = durable; - if (durable) await this.#onDurable(); - } - - /** Access just became available. On a reinstall it usually arrives after - * `init` (Connect is the first thing pressed), so the cookie could not be - * read then — look again before an ID is minted over it; otherwise write - * (or refresh) the copy now that it is allowed. */ - async #onDurable() { - if (this.#id === null) await this.#recoverId(await readIdCookie()); - this.#cookieId = null; - this.#reflect(); - } - - /** No ID in the sync area: a fresh install, or a reinstall — the browser - * purged the extension's synced storage on uninstall. The cookie is this - * profile's own earlier ID, so taking it back is what the user expects and - * needs no consent; the next reconcile pulls the data. */ - async #recoverId(kept: string | null) { - if (this.#id === null && kept) await this.#adoptId(kept); - } - #reflect() { this.enabled = this.#config.enabled; - this.syncId = this.#id; this.lastSyncedAt = this.#config.lastSyncedAt; this.lastError = this.#config.lastError; - if (!this.#config.enabled || this.#config.consentedId === this.#id) this.needsConsent = false; - // The one place the durable copy is written: whenever sync is on with an - // ID — received, minted, connected or restored — once per identity per - // document (which also keeps its expiry rolling). - const keep = this.#config.enabled ? this.#id : null; - if (keep && keep !== this.#cookieId) { - this.#cookieId = keep; - void writeIdCookie(keep); - } - } - - /** An ID this device originated or recovered is its own — consent implied. */ - async #adoptId(id: string) { - await this.#saveId(id); - await this.#saveConfig({ consentedId: id }); - } - - /** Sync-area writes are quota-limited (~120/min), so no-ops are skipped. */ - async #saveId(id: string | null) { - if (id === this.#id) return; - this.#id = id; - this.#reflect(); - await this.#writeId(id); - } - - /** Writes the sync area only — `#id` is already what it should be, and the - * cookie follows from `#reflect`. */ - async #writeId(id: string | null) { - await syncIdItem.setValue(id); + this.trimmed = this.#config.trimmed; } async #saveConfig(patch: Partial) { @@ -397,139 +215,141 @@ class SyncStore { // Persisted immediately (not on the debounce) so a panel closed mid-burst // still knows there is unpushed data next time it opens. void this.#saveConfig({ pendingPush: true, lastChangedAt: Date.now() }); + this.#schedulePush(PUSH_DEBOUNCE_MS); + } + + #schedulePush(delayMs: number) { clearTimeout(this.#pushTimer); - this.#pushTimer = setTimeout(() => { - void this.#enqueue(() => this.#push()); - }, PUSH_DEBOUNCE_MS); + const spacing = this.#lastPushAt + MIN_PUSH_SPACING_MS - Date.now(); + this.#pushTimer = setTimeout( + () => void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })), + Math.max(delayMs, spacing), + ); } - /** Uploads the current snapshot. The hash guard makes echoes (and no-op - * writes) free; `force` skips it for seeding an empty remote. */ - async #push(opts: { force?: boolean } = {}) { + #retryIn(delayMs: number) { + clearTimeout(this.#retryTimer); + this.#retryTimer = setTimeout( + () => void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })), + delayMs, + ); + } + + /** + * The whole algorithm. Reads the area and this device's data, then: + * - nothing there → seed it (unless this device has nothing either); + * - torn → wait for the rest to land, retry; + * - unchanged since last look → push if this device changed something; + * - otherwise merge the two copies, write the result wherever it differs. + */ + async #reconcile(opts: { allowApply: boolean }) { if (!this.#config.enabled) return; - // Sync ships on, but an ID is minted only once there is something to push. - // Deferring leaves room for a profile-synced ID from another device to - // arrive first, and avoids seeding a blob for an install nobody uses. - // Minting our own means this device started the data set — consent implied. - if (!this.#id) { - await this.#adoptId(generateSyncId()); - opts = { force: true }; - } - const id = this.#id; - if (!id) return; this.#syncing = true; + let applied = false; try { + const { result, bytes } = await readSyncArea(); + this.usedBytes = bytes; const local = await createBackup(); - const hash = await snapshotHash(local); - if (!opts.force && hash === this.#config.lastSyncedHash) { - if (this.#config.pendingPush || this.#config.lastError) { - await this.#saveConfig({ pendingPush: false, lastError: null }); + const localHash = await contentHash(local); + const localChanged = localHash !== this.#config.lastLocalHash; + + if (result.kind === 'torn') { + if (!this.#tornSince) this.#tornSince = Date.now(); + if (Date.now() - this.#tornSince < TORN_GIVE_UP_MS) { + this.#retryIn(TORN_RETRY_MS); + return; } + // Nobody finished that write; ours replaces it. Whatever it carried + // comes back merged when its writer reconciles against ours. + this.#tornSince = 0; + await this.#push(local, localHash); return; } - await pushSnapshot(id, local); - await this.#saveConfig({ - lastSyncedAt: local.exportedAt, - lastSyncedHash: hash, - pendingPush: false, - lastError: null, - }); - } catch (err) { - // pendingPush stays set — retried on the next change, interval tick, - // startup, or manual sync. No retry timer. - await this.#saveConfig({ lastError: errorMessage(err) }); - } finally { - this.#syncing = false; - } - } - - /** Pull-and-decide: push, apply remote, or nothing — last write wins. */ - async #reconcile(opts: { allowApply: boolean }) { - const id = this.#id; - if (!this.#config.enabled || !id) return; - this.#syncing = true; - try { - const remote = await pullSnapshot(id); - const local = await createBackup(); - const localHash = await snapshotHash(local); - const localChanged = localHash !== this.#config.lastSyncedHash; + this.#tornSince = 0; - if (remote === null) { - // First device on this ID, or the blob expired server-side: seed it. - await this.#uploadLocal(local, localHash); + if (result.kind === 'none') { + if (isEmpty(local) && !this.#config.pendingPush) return; + await this.#push(local, localHash); return; } - if (remote.exportedAt === this.#config.lastSyncedAt) { - // Remote is still what we last synced; push if we have news. - if (localChanged || this.#config.pendingPush) await this.#uploadLocal(local, localHash); + + const { meta, base64 } = result; + if (meta.h === this.#config.lastRemoteHash) { + // Remote is what we last saw (our own echo included). + if (localChanged || this.#config.pendingPush) await this.#push(local, localHash); else if (this.#config.lastError) await this.#saveConfig({ lastError: null }); return; } - // Another device pushed since our last sync. - if ((await snapshotHash(remote)) === localHash) { - // Same content, different timestamp — adopt its bookkeeping, skip the reload. + + // Another device wrote since we last looked. + const remote = await unpackBackup(base64); + const remoteWins = !localChanged || remote.exportedAt > this.#config.lastChangedAt; + const merged = mergeBackups(local, remote, remoteWins); + const [mergedHash, remoteHash] = await Promise.all([ + contentHash(merged), + contentHash(remote), + ]); + const needApply = mergedHash !== localHash; + const needPush = mergedHash !== remoteHash; + + if (needApply && !opts.allowApply) { + // A track is loaded; applying would reload the panel mid-practice. + // Nothing is pushed either: a push now would carry only our side. + this.pendingApply = true; + return; + } + this.pendingApply = false; + if (needApply) { + // #applying stays set until the reload: nothing in between may + // schedule a push of what was just written. + this.#applying = true; + clearTimeout(this.#pushTimer); + await restoreBackup(merged); + applied = true; + } + if (needPush) { + await this.#push(merged, mergedHash); + } else { await this.#saveConfig({ - lastSyncedAt: remote.exportedAt, - lastSyncedHash: localHash, + lastSyncedAt: meta.at, + lastRemoteHash: meta.h, + lastLocalHash: mergedHash, pendingPush: false, lastError: null, }); - return; - } - if (!localChanged || remote.exportedAt > this.#config.lastChangedAt) { - if (opts.allowApply) await this.#applyRemote(remote); - // else: deferred — don't push over a newer remote either. - } else { - await this.#uploadLocal(local, localHash); } } catch (err) { - await this.#saveConfig({ lastError: errorMessage(err) }); + await this.#saveConfig({ lastError: syncErrorMessage(err) }); + if (classifySyncError(err) === 'rate') this.#retryIn(RATE_LIMIT_RETRY_MS); } finally { this.#syncing = false; + // Local data changed under the stores (same situation as an import): + // the reload is owed whether or not the push after it went through. + if (applied) location.reload(); } } - async #uploadLocal(local: Backup, hash: string) { - const id = this.#id; - if (!id) return; - await pushSnapshot(id, local); - await this.#saveConfig({ - lastSyncedAt: local.exportedAt, - lastSyncedHash: hash, - pendingPush: false, - lastError: null, - }); - } - - /** Overwrites local data with the remote snapshot and reloads the panel - * (mirrors the import flow — stores read storage once at startup). The hash - * is taken from a re-read because `parseBackup` backfills defaults, so what - * landed in storage can differ from the remote bytes. */ - async #applyRemoteImpl(remote: Backup) { - await restoreBackup(remote); - const applied = await createBackup(); + /** Writes `backup` to the area, cut to the quota if it must be. `hash` is + * the content hash of this device's full data, so a trimmed push doesn't + * read as "local changed" on the next pass. */ + async #push(backup: Backup, hash: string) { + const fitted = await fitBackup(backup, BUDGET_CHARS, measure); + const exportedAt = Date.now(); + const packed = await packBackup( + encodeBackup({ ...fitted.backup, exportedAt }), + browser.runtime.getManifest().version, + ); + await writeSyncArea(packed.items); + this.#lastPushAt = Date.now(); + this.usedBytes = itemsBytes(packed.items); await this.#saveConfig({ - lastSyncedAt: remote.exportedAt, - lastSyncedHash: await snapshotHash(applied), + lastSyncedAt: exportedAt, + lastRemoteHash: packed.meta.h, + lastLocalHash: hash, pendingPush: false, lastError: null, + trimmed: fitted.trimmed, }); - // After the reload, reconcile sees remote.exportedAt === lastSyncedAt and - // an unchanged hash — no loop. - location.reload(); - } - - async #applyRemote(remote: Backup) { - // #applying stays set: the page is about to reload, and nothing that - // happens between restore and reload should schedule a push. - this.#applying = true; - clearTimeout(this.#pushTimer); - try { - await this.#applyRemoteImpl(remote); - } catch (err) { - this.#applying = false; - throw err; - } } } diff --git a/src/features/sync/persist/fit.test.ts b/src/features/sync/persist/fit.test.ts index c8ce5cf..c4a3f01 100644 --- a/src/features/sync/persist/fit.test.ts +++ b/src/features/sync/persist/fit.test.ts @@ -103,23 +103,24 @@ function library(recent: number, favorites: number, orphans = 0): Backup { favorites: favs, eqPresets: [], tracks, + deletions: {}, }; } const keys = (list: { identity: TrackIdentity }[]) => list.map((e) => e.identity.key).sort(); const charts = (b: Backup) => b.tracks.filter((t) => t.chordChart?.segments.length).length; -test('nothing is cut while the library fits', () => { +test('nothing is cut while the library fits', async () => { const b = library(5, 2); - const result = fitBackup(b, measure(b), measure); + const result = await fitBackup(b, measure(b), measure); assert.equal(result.trimmed, false); assert.equal(result.size, measure(b)); assert.deepEqual(result.backup, b); }); -test('recent songs go first, oldest first, taking their records along', () => { +test('recent songs go first, oldest first, taking their records along', async () => { const b = library(6, 2); - const withoutTwoOldest = fitBackup(b, measure(b) - 1, measure).backup; + const withoutTwoOldest = (await fitBackup(b, measure(b) - 1, measure)).backup; assert.ok(withoutTwoOldest.history.length < b.history.length); assert.equal(withoutTwoOldest.favorites.length, 2, 'favorites untouched'); assert.equal(charts(withoutTwoOldest), charts(b) - (b.history.length - withoutTwoOldest.history.length), 'only the cut songs lost their charts'); @@ -133,12 +134,12 @@ test('recent songs go first, oldest first, taking their records along', () => { assert.deepEqual(keys(withoutTwoOldest.tracks), keys([...withoutTwoOldest.history])); }); -test('charts go next, oldest first; favorites and their markers stay', () => { +test('charts go next, oldest first; favorites and their markers stay', async () => { const b = library(3, 3); - const noRecent = fitBackup(b, measure(b), measure); + const noRecent = await fitBackup(b, measure(b), measure); // Find the budget at which every non-favorite is gone but charts remain. const favoritesOnly = { ...b, history: b.history.slice(3), tracks: b.tracks.filter((t) => keys(b.favorites).includes(t.identity.key)) }; - const result = fitBackup(b, measure(favoritesOnly) - 1, measure); + const result = await fitBackup(b, measure(favoritesOnly) - 1, measure); assert.equal(result.trimmed, true); assert.equal(result.backup.favorites.length, 3); assert.equal(result.backup.history.length, 3, 'favorites keep their Recent rows'); @@ -156,7 +157,7 @@ test('charts go next, oldest first; favorites and their markers stay', () => { assert.equal(noRecent.trimmed, false); }); -test('favorites go last, least recently accessed first', () => { +test('favorites go last, least recently accessed first', async () => { const b = library(2, 4); // Exactly the two newest favorites, with their rows and chart-less records. const newest = keys(b.favorites.slice(2)); @@ -168,7 +169,7 @@ test('favorites go last, least recently accessed first', () => { .filter((t) => newest.includes(t.identity.key)) .map((t) => ({ ...t, chordChart: null })), }; - const result = fitBackup(b, measure(expected), measure); + const result = await fitBackup(b, measure(expected), measure); assert.equal(charts(result.backup), 0, 'every chart went before a favorite'); assert.equal(result.backup.favorites.length, 2); const survivors = result.backup.favorites.map((f) => f.lastAccessedAt); @@ -180,35 +181,35 @@ test('favorites go last, least recently accessed first', () => { assert.deepEqual(keys(result.backup.history), keys(result.backup.favorites), 'rows follow'); }); -test('orphan records sit in the recent tier by their own edit time', () => { +test('orphan records sit in the recent tier by their own edit time', async () => { const b = library(2, 1, 2); const oneOrphanLess = { ...b, tracks: b.tracks.filter((t) => t.identity.key !== song(3).key) }; - const result = fitBackup(b, measure(oneOrphanLess), measure); + const result = await fitBackup(b, measure(oneOrphanLess), measure); assert.equal(result.trimmed, true); const kept = keys(result.backup.tracks); assert.ok(!kept.includes(song(0).key), 'the oldest recent song went first'); assert.ok(kept.includes(song(4).key), 'the newest orphan is newer than the recents and stays'); }); -test('the result is the largest plan that fits', () => { +test('the result is the largest plan that fits', async () => { const b = library(12, 0); for (const keep of [1, 5, 11]) { const exact = { ...b, history: b.history.slice(12 - keep), tracks: b.tracks.filter((t) => keys(b.history.slice(12 - keep)).includes(t.identity.key)) }; - const result = fitBackup(b, measure(exact), measure); + const result = await fitBackup(b, measure(exact), measure); assert.equal(result.backup.history.length, keep, `budget for ${keep}`); assert.equal(result.size, measure(exact)); } }); -test('deterministic for equal input', () => { - const a = fitBackup(library(8, 3), 900, measure); - const b = fitBackup(library(8, 3), 900, measure); +test('deterministic for equal input', async () => { + const a = await fitBackup(library(8, 3), 900, measure); + const b = await fitBackup(library(8, 3), 900, measure); assert.deepEqual(a, b); }); -test('too large when settings plus favorites alone are over budget', () => { +test('too large when settings plus favorites alone are over budget', async () => { const b = library(2, 2); const floor = measure({ ...b, history: [], tracks: [], favorites: [] }); - assert.throws(() => fitBackup(b, floor - 1, measure), LibraryTooLargeError); - assert.doesNotThrow(() => fitBackup(b, floor, measure)); + await assert.rejects(fitBackup(b, floor - 1, measure), LibraryTooLargeError); + await assert.doesNotReject(fitBackup(b, floor, measure)); }); diff --git a/src/features/sync/persist/fit.ts b/src/features/sync/persist/fit.ts index 739e15a..23020e1 100644 --- a/src/features/sync/persist/fit.ts +++ b/src/features/sync/persist/fit.ts @@ -17,9 +17,9 @@ import type { FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../. * the library does (`isSameTrack`), so a record saved under a drifted * duration still follows its favorite. * - * `measure` is injected: the caller decides what "size" means (encoded JSON - * length in tests, the gzip+base64 blob for sync), so this stays pure and - * runs under `node --test` — hence relative `.ts` imports. + * `measure` is injected (and may be async): the caller decides what "size" + * means — encoded JSON length in tests, the gzip+base64 blob for sync — so + * this stays pure and runs under `node --test`; hence relative `.ts` imports. */ export interface FitResult { @@ -127,25 +127,25 @@ function build(backup: Backup, tiers: Tiers, plan: Plan): Backup { * `budget`. Deterministic for equal input. Throws `LibraryTooLargeError` * when nothing cuttable is left and it still doesn't fit. */ -export function fitBackup( +export async function fitBackup( backup: Backup, budget: number, - measure: (backup: Backup) => number, -): FitResult { + measure: (backup: Backup) => number | Promise, +): Promise { const tiers = collectTiers(backup); const full: Plan = { songs: tiers.songs.length, charts: tiers.charts.length, favorites: tiers.favorites.length, }; - const size = measure(backup); + const size = await measure(backup); if (size <= budget) return { backup, trimmed: false, size }; const plan = { ...full }; - const sizeOf = (p: Plan) => measure(build(backup, tiers, p)); + const sizeOf = (p: Plan) => Promise.resolve(measure(build(backup, tiers, p))); for (const tier of TIERS) { // Earlier tiers are already empty. Does emptying this one fit? - const empty = sizeOf({ ...plan, [tier]: 0 }); + const empty = await sizeOf({ ...plan, [tier]: 0 }); if (empty > budget) { plan[tier] = 0; continue; @@ -156,7 +156,7 @@ export function fitBackup( let loSize = empty; while (hi - lo > 1) { const mid = (lo + hi) >> 1; - const s = sizeOf({ ...plan, [tier]: mid }); + const s = await sizeOf({ ...plan, [tier]: mid }); if (s <= budget) { lo = mid; loSize = s; diff --git a/src/features/sync/persist/hash.ts b/src/features/sync/persist/hash.ts new file mode 100644 index 0000000..b9af923 --- /dev/null +++ b/src/features/sync/persist/hash.ts @@ -0,0 +1,20 @@ +import { encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; + +/** + * Content identity of a backup: the compact encoding with its clock zeroed. + * Two devices holding the same data produce the same text — the codec is + * deterministic and idempotent — so hashes compare across devices, and a + * re-export of unchanged data hashes the same. Pure; `node --test`. + */ +export function contentText(backup: Backup): string { + return JSON.stringify({ ...encodeBackup(backup), at: 0 }); +} + +export async function sha256Hex(text: string): Promise { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)); + return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join(''); +} + +export async function contentHash(backup: Backup): Promise { + return sha256Hex(contentText(backup)); +} diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts new file mode 100644 index 0000000..cdbe1fb --- /dev/null +++ b/src/features/sync/persist/merge.test.ts @@ -0,0 +1,208 @@ +// Run with: pnpm test:dsp (node --test). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mergeBackups } from './merge.ts'; +import { BACKUP_FORMAT, type Backup } from '../../../core/persist/backup-codec.ts'; +import { + favoriteDeletion, + HISTORY_CLEARED, + historyDeletion, +} from '../../../core/persist/deletions.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, HISTORY_LIMIT } from '../../../core/model/defaults.ts'; +import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; +import type { ChordChart, FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types.ts'; + +const T0 = 1_757_000_000_000; +const NOW = T0 + 10 * 60_000; + +const song = (n: number, duration = 200) => + makeTrackIdentity(`https://www.youtube.com/watch?v=vid${n.toString().padStart(8, '0')}`, `Song ${n}`, duration); + +function row(identity: TrackIdentity, updatedAt: number, transpose = 0): HistoryEntry { + return { + identity, + params: { ...DEFAULT_PARAMS, transpose }, + pageUrl: identity.normalizedUrl, + createdAt: updatedAt, + updatedAt, + }; +} + +function fav(identity: TrackIdentity, updatedAt: number, lastAccessedAt = updatedAt): FavoriteEntry { + return { ...row(identity, updatedAt), favoritedAt: updatedAt, lastAccessedAt }; +} + +const chart = (computedAt: number): ChordChart => ({ + segments: [{ startT: 0, endT: 2, label: 'C', confidence: 1 }], + key: null, + coverage: 1, + analyzedFrom: 0, + analyzedTo: 2, + computedAt, +}); + +function record(identity: TrackIdentity, updatedAt: number, markers: number, withChart = false): TrackData { + return { + identity, + markers: Array.from({ length: markers }, (_, i) => ({ id: `m${i}`, t: i, label: '' })), + snippets: [], + sequenceLoop: false, + sequenceCountIn: false, + chordChart: withChart ? chart(updatedAt) : null, + updatedAt, + }; +} + +function backup(patch: Partial = {}): Backup { + return { + format: BACKUP_FORMAT, + version: 1, + exportedAt: T0, + appVersion: '', + settings: { ...DEFAULT_SETTINGS }, + uiPrefs: JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)), + history: [], + favorites: [], + eqPresets: [], + tracks: [], + deletions: {}, + ...patch, + }; +} + +const keys = (list: { identity: TrackIdentity }[]) => list.map((e) => e.identity.key); + +test('a row the other side trimmed away survives; the newer copy of a shared row wins', () => { + const local = backup({ history: [row(song(1), T0 + 1000, 1), row(song(2), T0)] }); + const remote = backup({ history: [row(song(1), T0 + 5000, 3)] }); + const merged = mergeBackups(local, remote, true, NOW); + assert.deepEqual(keys(merged.history), [song(1).key, song(2).key]); + assert.equal(merged.history[0].params.transpose, 3, 'newer copy'); +}); + +test('history is matched by song, so a drifted duration does not make a twin', () => { + const local = backup({ history: [row(song(1, 200), T0)] }); + const remote = backup({ history: [row(song(1, 201), T0 + 1)] }); + const merged = mergeBackups(local, remote, false, NOW); + assert.equal(merged.history.length, 1); + assert.equal(merged.history[0].identity.durationSec, 201); +}); + +test('a deletion beats the copy it postdates, but not a later re-play', () => { + const gone = song(1); + const local = backup({ deletions: { [historyDeletion(gone.key)]: T0 + 2000 } }); + const remote = backup({ history: [row(gone, T0 + 1000), row(song(2), T0)] }); + assert.deepEqual(keys(mergeBackups(local, remote, true, NOW).history), [song(2).key]); + const replayed = backup({ history: [row(gone, T0 + 3000)] }); + assert.deepEqual(keys(mergeBackups(local, replayed, true, NOW).history), [gone.key]); +}); + +test('"Clear Recent" travels and wipes older rows on the other side', () => { + const local = backup({ deletions: { [HISTORY_CLEARED]: T0 + 5000 } }); + const remote = backup({ history: [row(song(1), T0 + 1000), row(song(2), T0 + 6000)] }); + const merged = mergeBackups(local, remote, true, NOW); + assert.deepEqual(keys(merged.history), [song(2).key]); + assert.equal(merged.deletions[HISTORY_CLEARED], T0 + 5000, 'the record travels on'); +}); + +test('history is newest-first and capped', () => { + const local = backup({ + history: Array.from({ length: HISTORY_LIMIT }, (_, i) => row(song(i), T0 + i)), + }); + const remote = backup({ history: [row(song(999), T0 + 100_000)] }); + const merged = mergeBackups(local, remote, false, NOW); + assert.equal(merged.history.length, HISTORY_LIMIT); + assert.equal(merged.history[0].identity.key, song(999).key); +}); + +test('favorites: union in the winner order, deletions honoured, last access kept', () => { + const a = song(1); + const b = song(2); + const c = song(3); + const local = backup({ + favorites: [fav(b, T0, T0 + 9000), fav(a, T0)], + deletions: { [favoriteDeletion(c.key)]: T0 + 100 }, + }); + const remote = backup({ favorites: [fav(a, T0 + 1), fav(c, T0)] }); + const merged = mergeBackups(local, remote, true, NOW); + assert.deepEqual(keys(merged.favorites), [a.key, b.key], 'remote order first, c deleted'); + assert.equal(merged.favorites[0].updatedAt, T0 + 1); + assert.equal(merged.favorites[1].lastAccessedAt, T0 + 9000); +}); + +test('favorites: a newer copy adopts the other side’s later access time', () => { + const a = song(1); + const local = backup({ favorites: [fav(a, T0, T0 + 9000)] }); + const remote = backup({ favorites: [fav(a, T0 + 5, T0 + 5)] }); + const merged = mergeBackups(local, remote, true, NOW); + assert.equal(merged.favorites[0].updatedAt, T0 + 5); + assert.equal(merged.favorites[0].lastAccessedAt, T0 + 9000); +}); + +test('tracks: the newer record wins whole, an emptied one included', () => { + const a = song(1); + const local = backup({ tracks: [record(a, T0 + 1000, 0)] }); + const remote = backup({ tracks: [record(a, T0, 5), record(song(2), T0, 2)] }); + const merged = mergeBackups(local, remote, true, NOW); + assert.equal(merged.tracks.length, 2); + assert.equal(merged.tracks.find((t) => t.identity.key === a.key)?.markers.length, 0); +}); + +test('tracks: a winner without a chart adopts the other side’s', () => { + const a = song(1); + const local = backup({ tracks: [record(a, T0 + 1000, 3)] }); + const remote = backup({ tracks: [record(a, T0, 1, true)] }); + const merged = mergeBackups(local, remote, false, NOW); + assert.equal(merged.tracks[0].markers.length, 3); + assert.ok(merged.tracks[0].chordChart?.segments.length); +}); + +test('ties go to the winning side', () => { + const a = song(1); + const local = backup({ history: [row(a, T0, 1)], tracks: [record(a, T0, 1)] }); + const remote = backup({ history: [row(a, T0, 2)], tracks: [record(a, T0, 2)] }); + assert.equal(mergeBackups(local, remote, true, NOW).history[0].params.transpose, 2); + assert.equal(mergeBackups(local, remote, true, NOW).tracks[0].markers.length, 2); + assert.equal(mergeBackups(local, remote, false, NOW).history[0].params.transpose, 1); + assert.equal(mergeBackups(local, remote, false, NOW).tracks[0].markers.length, 1); +}); + +test('settings and prefs come from the winner; EQ presets are a union', () => { + const local = backup({ + settings: { ...DEFAULT_SETTINGS, theme: 'dark' }, + eqPresets: [{ name: 'Mine', gains: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] }], + }); + const remote = backup({ + settings: { ...DEFAULT_SETTINGS, theme: 'light' }, + eqPresets: [{ name: 'Theirs', gains: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] }, { name: 'Mine', gains: [9, 9, 9, 9, 9, 9, 9, 9, 9, 9] }], + }); + const remoteWins = mergeBackups(local, remote, true, NOW); + assert.equal(remoteWins.settings.theme, 'light'); + assert.deepEqual(remoteWins.eqPresets.map((p) => p.name), ['Theirs', 'Mine']); + assert.equal(remoteWins.eqPresets[1].gains[0], 9, 'winner’s copy of a shared name'); + const localWins = mergeBackups(local, remote, false, NOW); + assert.equal(localWins.settings.theme, 'dark'); + assert.deepEqual(localWins.eqPresets.map((p) => p.name), ['Mine', 'Theirs']); + assert.equal(localWins.eqPresets[0].gains[0], 1); +}); + +test('deletions are merged newest-per-key and expired ones dropped', () => { + const old = NOW - 40 * 24 * 60 * 60_000; + const local = backup({ deletions: { 'h:a': T0, 'h:old': old } }); + const remote = backup({ deletions: { 'h:a': T0 + 1, 'f:b': T0 } }); + assert.deepEqual(mergeBackups(local, remote, true, NOW).deletions, { 'h:a': T0 + 1, 'f:b': T0 }); +}); + +test('merging a library with itself changes nothing', () => { + const b = backup({ + history: [row(song(1), T0), row(song(2), T0 + 1)], + favorites: [fav(song(2), T0)], + tracks: [record(song(1), T0, 2, true)], + deletions: { 'h:x': T0 }, + }); + const merged = mergeBackups(b, b, true, NOW); + assert.deepEqual(merged.history, [...b.history].sort((x, y) => y.updatedAt - x.updatedAt)); + assert.deepEqual(merged.favorites, b.favorites); + assert.deepEqual(merged.tracks, b.tracks); + assert.deepEqual(merged.deletions, b.deletions); +}); diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts new file mode 100644 index 0000000..f276feb --- /dev/null +++ b/src/features/sync/persist/merge.ts @@ -0,0 +1,130 @@ +import { HISTORY_LIMIT } from '../../../core/model/defaults.ts'; +import type { Backup } from '../../../core/persist/backup-codec.ts'; +import { + favoriteDeleted, + historyDeleted, + mergeDeletions, + pruneDeletions, +} from '../../../core/persist/deletions.ts'; +import type { + EqPreset, + FavoriteEntry, + HistoryEntry, + TrackData, + TrackIdentity, +} from '../../../core/model/types'; + +/** + * Two devices' libraries into one. A union, item by item, so a copy another + * device trimmed to fit the quota never deletes anything here, and edits made + * on both sides since they last met both survive: + * + * - Recent rows and favorites: matched by song (URL + title, like the + * library itself), the more recently updated one wins; a dated deletion + * (`deletions.ts`) beats any copy it postdates. + * - Track records: matched by key, the more recently edited wins outright — + * an emptied record is still a record, so clearing markers sticks. A + * winner without a chart adopts the other's: absent may mean "trimmed", + * and a chart is only ever replaced by re-analysis. + * - Settings and UI prefs: the newer device's, as a whole. EQ presets: union + * by name, the newer device's order first. + * + * `remoteWins` breaks ties and picks the wholesale sections: true when the + * remote copy was written after this device's last local change. Pure; + * `node --test`. + */ + +const songId = (identity: TrackIdentity) => `${identity.normalizedUrl}\n${identity.title}`; + +const byUpdatedDesc = (a: { updatedAt: number }, b: { updatedAt: number }) => + (b.updatedAt ?? 0) - (a.updatedAt ?? 0); + +function mergeHistory( + first: HistoryEntry[], + second: HistoryEntry[], + deletions: Backup['deletions'], +): HistoryEntry[] { + const bySong = new Map(); + for (const entry of [...first, ...second]) { + if (historyDeleted(deletions, entry.identity.key, entry.updatedAt ?? 0)) continue; + const id = songId(entry.identity); + const current = bySong.get(id); + if (!current || (entry.updatedAt ?? 0) > (current.updatedAt ?? 0)) bySong.set(id, entry); + } + return [...bySong.values()].sort(byUpdatedDesc).slice(0, HISTORY_LIMIT); +} + +function mergeFavorites( + first: FavoriteEntry[], + second: FavoriteEntry[], + deletions: Backup['deletions'], +): FavoriteEntry[] { + // Insertion order = the winner's manual order, then the other side's extras. + const bySong = new Map(); + for (const entry of [...first, ...second]) { + const since = Math.max(entry.favoritedAt ?? 0, entry.updatedAt ?? 0); + if (favoriteDeleted(deletions, entry.identity.key, since)) continue; + const id = songId(entry.identity); + const current = bySong.get(id); + if (!current) { + bySong.set(id, entry); + } else if ((entry.updatedAt ?? 0) > (current.updatedAt ?? 0)) { + bySong.set(id, { + ...entry, + lastAccessedAt: Math.max(entry.lastAccessedAt ?? 0, current.lastAccessedAt ?? 0), + }); + } else if ((entry.lastAccessedAt ?? 0) > (current.lastAccessedAt ?? 0)) { + bySong.set(id, { ...current, lastAccessedAt: entry.lastAccessedAt }); + } + } + return [...bySong.values()]; +} + +const hasChart = (track: TrackData) => !!track.chordChart && track.chordChart.segments.length > 0; + +function mergeTracks(first: TrackData[], second: TrackData[]): TrackData[] { + const byKey = new Map(); + for (const track of [...first, ...second]) { + const current = byKey.get(track.identity.key); + if (!current) { + byKey.set(track.identity.key, track); + continue; + } + const [winner, loser] = + (track.updatedAt ?? 0) > (current.updatedAt ?? 0) ? [track, current] : [current, track]; + byKey.set( + track.identity.key, + !hasChart(winner) && hasChart(loser) ? { ...winner, chordChart: loser.chordChart } : winner, + ); + } + return [...byKey.values()]; +} + +function mergeEqPresets(first: EqPreset[], second: EqPreset[]): EqPreset[] { + const byName = new Map(); + for (const preset of [...first, ...second]) { + if (!byName.has(preset.name)) byName.set(preset.name, preset); + } + return [...byName.values()]; +} + +export function mergeBackups( + local: Backup, + remote: Backup, + remoteWins: boolean, + now = Date.now(), +): Backup { + const [winner, loser] = remoteWins ? [remote, local] : [local, remote]; + const deletions = pruneDeletions(mergeDeletions(local.deletions ?? {}, remote.deletions ?? {}), now); + return { + ...local, + exportedAt: Math.max(local.exportedAt ?? 0, remote.exportedAt ?? 0), + settings: winner.settings, + uiPrefs: winner.uiPrefs, + eqPresets: mergeEqPresets(winner.eqPresets, loser.eqPresets), + history: mergeHistory(winner.history, loser.history, deletions), + favorites: mergeFavorites(winner.favorites, loser.favorites, deletions), + tracks: mergeTracks(winner.tracks, loser.tracks), + deletions, + }; +} diff --git a/src/features/sync/persist/sync-area.ts b/src/features/sync/persist/sync-area.ts new file mode 100644 index 0000000..454506e --- /dev/null +++ b/src/features/sync/persist/sync-area.ts @@ -0,0 +1,85 @@ +import { LibraryTooLargeError } from './fit'; +import { + chunkKey, + isBlobKey, + itemsBytes, + NewerVersionError, + readBlob, + type BlobMeta, + type ReadResult, +} from './sync-blob'; + +/** + * The `browser.storage.sync` side of the blob layout in `sync-blob.ts`: + * reading what the browser has synced in, writing a packed blob, clearing it. + * Reads are local and free; writes are what the browser meters (120/min, + * 1800/hour, 100 KB), so the store spaces them out. + */ + +/** The pre-storage.sync builds kept the server sync ID here. */ +const LEGACY_KEYS = ['syncId']; + +export interface AreaRead { + result: ReadResult; + /** Everything in the area, foreign keys included — what the quota sees. */ + bytes: number; +} + +export async function readSyncArea(): Promise { + const items = (await browser.storage.sync.get(null)) as Record; + return { result: await readBlob(items), bytes: itemsBytes(items) }; +} + +/** Writes the blob, then drops chunks a larger earlier blob left behind and + * any legacy key. Two calls — a `set` can't remove — so a reader in between + * sees extra chunks, which `readBlob` ignores. */ +export async function writeSyncArea(items: Record): Promise { + const before = (await browser.storage.sync.get(null)) as Record; + await browser.storage.sync.set(items); + const stale = Object.keys(before).filter( + (key) => (isBlobKey(key) && !(key in items)) || LEGACY_KEYS.includes(key), + ); + if (stale.length) await browser.storage.sync.remove(stale); +} + +export async function clearSyncArea(): Promise { + const items = (await browser.storage.sync.get(null)) as Record; + const keys = Object.keys(items).filter((key) => isBlobKey(key) || LEGACY_KEYS.includes(key)); + if (keys.length) await browser.storage.sync.remove(keys); +} + +/** Fires when the browser syncs in a change to the blob (or one of our own + * writes lands). `storage.onChanged` with the area filter rather than + * `storage.sync.onChanged`, which older Firefox lacks. */ +export function onSyncAreaChanged(listener: () => void): void { + browser.storage.onChanged.addListener((changes, area) => { + if (area !== 'sync') return; + if (Object.keys(changes).some((key) => isBlobKey(key) || key === chunkKey(0))) listener(); + }); +} + +export type SyncErrorKind = 'rate' | 'quota' | 'newer' | 'other'; + +/** The browser reports quota trouble as thrown strings/errors with these + * names in the message; there is no error code to switch on. */ +export function classifySyncError(err: unknown): SyncErrorKind { + if (err instanceof NewerVersionError) return 'newer'; + if (err instanceof LibraryTooLargeError) return 'quota'; + const text = err instanceof Error ? err.message : String(err); + if (/MAX_WRITE_OPERATIONS|MAX_SUSTAINED_WRITE|MAX_ITEMS/i.test(text)) return 'rate'; + if (/QUOTA_BYTES|QuotaExceeded|quota/i.test(text)) return 'quota'; + return 'other'; +} + +export function syncErrorMessage(err: unknown): string { + switch (classifySyncError(err)) { + case 'rate': + return 'The browser is rate-limiting sync writes — retrying in a minute.'; + case 'quota': + return "Your library is too large for the browser's sync storage."; + case 'newer': + return 'Synced data was written by a newer version of Note by Note.'; + default: + return err instanceof Error && err.message ? err.message : 'Sync failed.'; + } +} diff --git a/src/features/sync/persist/sync-blob.test.ts b/src/features/sync/persist/sync-blob.test.ts new file mode 100644 index 0000000..f4de3bf --- /dev/null +++ b/src/features/sync/persist/sync-blob.test.ts @@ -0,0 +1,155 @@ +// Run with: pnpm test:dsp (node --test). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + base64ToBytes, + BUDGET_CHARS, + bytesToBase64, + CHUNK_CHARS, + chunkKey, + chunkString, + gunzipToText, + gzipText, + isBlobKey, + itemsBytes, + MAX_CHUNKS, + META_KEY, + NewerVersionError, + packBackup, + packedChars, + readBlob, + SYNC_QUOTA_BYTES, + unpackBackup, +} from './sync-blob.ts'; +import { fitBackup } from './fit.ts'; +import { BACKUP_FORMAT, encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../../../core/model/defaults.ts'; +import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; +import type { ChordChart, HistoryEntry, TrackData } from '../../../core/model/types.ts'; + +const T0 = 1_757_000_000_000; + +function chart(n: number): ChordChart { + const labels = ['C', 'Am', 'F', 'G', 'Dm7', 'E7', 'Bm', 'D']; + const segments = []; + let t = 0.5; + for (let i = 0; i < n; i++) { + const d = 1.2 + ((i * 7) % 11) * 0.13; + segments.push({ startT: t, endT: t + d, label: labels[(i * 5) % 8], confidence: 1 }); + t += d; + } + return { segments, key: { tonic: 'C', mode: 'major', confidence: 0.8 }, coverage: 1, analyzedFrom: 0, analyzedTo: t, computedAt: T0 }; +} + +/** `n` songs, each with a Recent row, a record with markers and a 160-segment + * chart — random ids so gzip can't cheat on repetition. */ +function library(n: number): Backup { + const history: HistoryEntry[] = []; + const tracks: TrackData[] = []; + for (let i = 0; i < n; i++) { + const vid = Math.random().toString(36).slice(2, 13); + const id = makeTrackIdentity(`https://www.youtube.com/watch?v=${vid}`, `Artist ${i} - Song ${vid} (Official Video)`, 180 + i); + history.push({ + identity: id, + params: { ...DEFAULT_PARAMS, transpose: i % 3, speed: 0.75 }, + pageUrl: `https://www.youtube.com/watch?v=${vid}`, + createdAt: T0, + updatedAt: T0 + i * 60_000, + }); + tracks.push({ + identity: id, + markers: Array.from({ length: 6 }, (_, k) => ({ id: `m${k}`, t: k * 30 + Math.random(), label: `Part ${k}` })), + snippets: [], + sequenceLoop: false, + sequenceCountIn: false, + chordChart: chart(160), + updatedAt: T0 + i * 60_000, + }); + } + return { + format: BACKUP_FORMAT, + version: 1, + exportedAt: T0, + appVersion: '1.0.3', + settings: DEFAULT_SETTINGS, + uiPrefs: DEFAULT_UI_PREFS, + history, + favorites: [], + eqPresets: [], + tracks, + deletions: {}, + }; +} + +test('base64 round-trips every byte value and a large buffer', () => { + const all = new Uint8Array(256).map((_, i) => i); + assert.deepEqual(base64ToBytes(bytesToBase64(all)), all); + const big = new Uint8Array(100_000).map(() => Math.floor(Math.random() * 256)); + assert.deepEqual(base64ToBytes(bytesToBase64(big)), big); +}); + +test('gzip round-trips text', async () => { + const text = 'Note by Note '.repeat(1000) + '— ünïcödé'; + assert.equal(await gunzipToText(await gzipText(text)), text); +}); + +test('chunking and keys', () => { + assert.deepEqual(chunkString('abcdefg', 3), ['abc', 'def', 'g']); + assert.deepEqual(chunkString('', 3), []); + assert.equal(chunkKey(0), 'nbn.0'); + assert.ok(isBlobKey('nbn.meta') && isBlobKey('nbn.10')); + assert.ok(!isBlobKey('syncId') && !isBlobKey('nbn.x')); +}); + +test('pack → items → read → unpack round-trips within the per-item and total limits', async () => { + const b = library(40); + const { items, meta, chars } = await packBackup(encodeBackup(b), '1.0.3'); + assert.equal(meta.n, Math.ceil(chars / CHUNK_CHARS)); + assert.equal(meta.at, T0); + for (const [key, value] of Object.entries(items)) { + assert.ok(key.length + JSON.stringify(value).length <= 8192, `${key} too big`); + } + assert.ok(itemsBytes(items) <= SYNC_QUOTA_BYTES); + const read = await readBlob(items); + assert.equal(read.kind, 'ok'); + if (read.kind !== 'ok') return; + const back = await unpackBackup(read.base64); + assert.equal(back.history.length, 40); + assert.equal(back.tracks[0].chordChart?.segments.length, 160); + assert.deepEqual(encodeBackup(back), encodeBackup(b)); +}); + +test('a 40-song library with charts is well inside the budget', async () => { + const chars = await packedChars(encodeBackup(library(40))); + assert.ok(chars < BUDGET_CHARS / 2, `${chars} of ${BUDGET_CHARS}`); +}); + +test('fit with the packed size as measure cuts a huge library to the budget', async () => { + const b = library(1200); + const measure = (x: Backup) => packedChars(encodeBackup(x)); + const result = await fitBackup(b, BUDGET_CHARS, measure); + assert.equal(result.trimmed, true); + assert.ok(result.size <= BUDGET_CHARS); + const { meta } = await packBackup(encodeBackup(result.backup), ''); + assert.ok(meta.n <= MAX_CHUNKS); + assert.ok(result.backup.history.length > 100, `kept ${result.backup.history.length}`); +}); + +test('torn and empty areas are told apart from a good blob', async () => { + const { items } = await packBackup(encodeBackup(library(3)), ''); + assert.equal((await readBlob({})).kind, 'none'); + assert.equal((await readBlob({ syncId: 'legacy' })).kind, 'none'); + const missing = { ...items }; + delete missing[chunkKey(0)]; + assert.equal((await readBlob(missing)).kind, 'torn'); + const corrupt = { ...items, [chunkKey(0)]: 'AAAA' + (items[chunkKey(0)] as string).slice(4) }; + assert.equal((await readBlob(corrupt)).kind, 'torn'); + assert.equal((await readBlob({ [META_KEY]: 'junk' })).kind, 'torn'); + const extra = { ...items, [chunkKey(99)]: 'stale' }; + assert.equal((await readBlob(extra)).kind, 'ok'); +}); + +test('a blob from a newer build is refused, not misread', async () => { + const { items, meta } = await packBackup(encodeBackup(library(1)), ''); + await assert.rejects(readBlob({ ...items, [META_KEY]: { ...meta, v: meta.v + 1 } }), NewerVersionError); +}); diff --git a/src/features/sync/persist/sync-blob.ts b/src/features/sync/persist/sync-blob.ts new file mode 100644 index 0000000..85a5e39 --- /dev/null +++ b/src/features/sync/persist/sync-blob.ts @@ -0,0 +1,193 @@ +import { + parseBackupJson, + type Backup, + type CompactBackup, +} from '../../../core/persist/backup-codec.ts'; +import { sha256Hex } from './hash.ts'; + +/** + * How a backup is laid out in `browser.storage.sync`, whose limits shape + * everything here: 100 KB in total, 8 KB per item, and the browser syncs + * each item on its own. + * + * compact JSON → gzip → base64 → fixed-size string chunks + * + * under `nbn.0 … nbn.N-1`, plus `nbn.meta` describing them. Strings, not + * bytes: Firefox structured-clones sync writes and rejects typed arrays, and + * base64 is what fits the per-item cap predictably. + * + * Because items travel separately, a reader can see a mix of two writes — a + * "torn" blob. `meta.h` is the hash of the joined base64; when it doesn't + * match, the reader waits for the rest to arrive rather than applying junk. + * + * Pure (no `browser`); the area I/O is in `sync-area.ts`. `node --test`, so + * relative `.ts` imports — `CompressionStream`, `Blob`, `Response`, `btoa` and + * `crypto.subtle` are all globals in Node ≥ 20 as well as in the browser. + */ + +export const META_KEY = 'nbn.meta'; +const CHUNK_PREFIX = 'nbn.'; +const BLOB_KEY_RE = /^nbn\.(meta|\d+)$/; + +/** Chunk payload length: `nbn.10` (6) + two quotes + 8000 stays under the + * 8192-byte per-item cap, which Chrome charges as key + JSON of the value. */ +export const CHUNK_CHARS = 8000; +export const MAX_CHUNKS = 11; +/** What `fitBackup` gets as its budget: base64 characters. 11 chunks plus + * meta come to ~88.3 KB of Chrome's 102,400-byte quota. */ +export const BUDGET_CHARS = CHUNK_CHARS * MAX_CHUNKS; +export const SYNC_QUOTA_BYTES = 102_400; + +/** Bump when a reader of this version could misread the layout. */ +export const BLOB_VERSION = 1; + +export interface BlobMeta { + v: number; + /** Chunk count. */ + n: number; + /** SHA-256 (hex) of the joined base64 — tear detection and echo recognition. */ + h: string; + /** `exportedAt` of the backup inside, ms — the last-write-wins clock. */ + at: number; + /** Writer's app version, for diagnostics only. */ + app: string; +} + +export type ReadResult = + | { kind: 'none' } + | { kind: 'torn' } + | { kind: 'ok'; meta: BlobMeta; base64: string }; + +/** Thrown when a blob was written by a build newer than this one. */ +export class NewerVersionError extends Error { + constructor() { + super('Synced data was written by a newer version of Note by Note.'); + this.name = 'NewerVersionError'; + } +} + +export function isBlobKey(key: string): boolean { + return BLOB_KEY_RE.test(key); +} + +export function chunkKey(index: number): string { + return `${CHUNK_PREFIX}${index}`; +} + +// --------------------------------------------------------------------------- +// gzip / base64 + +async function pipe( + bytes: Uint8Array, + stream: CompressionStream | DecompressionStream, +): Promise { + const out = new Blob([bytes as BlobPart]).stream().pipeThrough(stream); + return new Uint8Array(await new Response(out).arrayBuffer()); +} + +export async function gzipText(text: string): Promise { + return pipe(new TextEncoder().encode(text), new CompressionStream('gzip')); +} + +export async function gunzipToText(bytes: Uint8Array): Promise { + return new TextDecoder().decode(await pipe(bytes, new DecompressionStream('gzip'))); +} + +/** `btoa` wants a binary string; built in slices to stay clear of argument + * limits on large buffers. */ +export function bytesToBase64(bytes: Uint8Array): string { + let binary = ''; + const STEP = 0x8000; + for (let i = 0; i < bytes.length; i += STEP) { + binary += String.fromCharCode(...bytes.subarray(i, i + STEP)); + } + return btoa(binary); +} + +export function base64ToBytes(base64: string): Uint8Array { + const binary = atob(base64); + const out = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) out[i] = binary.charCodeAt(i); + return out; +} + +export function chunkString(text: string, size: number): string[] { + const chunks: string[] = []; + for (let i = 0; i < text.length; i += size) chunks.push(text.slice(i, i + size)); + return chunks; +} + +// --------------------------------------------------------------------------- +// pack / unpack + +/** The base64 a compact backup becomes — `fitBackup`'s measure. */ +export async function packedChars(compact: CompactBackup): Promise { + return bytesToBase64(await gzipText(JSON.stringify(compact))).length; +} + +export interface PackedBlob { + /** The items to write, meta included. */ + items: Record; + meta: BlobMeta; + /** base64 length, for the usage readout. */ + chars: number; +} + +export async function packBackup(compact: CompactBackup, app: string): Promise { + const base64 = bytesToBase64(await gzipText(JSON.stringify(compact))); + const chunks = chunkString(base64, CHUNK_CHARS); + if (chunks.length > MAX_CHUNKS) { + throw new Error("Your library is too large for the browser's sync storage."); + } + const meta: BlobMeta = { + v: BLOB_VERSION, + n: chunks.length, + h: await sha256Hex(base64), + at: compact.at * 1000, + app, + }; + const items: Record = { [META_KEY]: meta }; + chunks.forEach((chunk, i) => (items[chunkKey(i)] = chunk)); + return { items, meta, chars: base64.length }; +} + +/** What the area holds, classified. Only `nbn.*` keys are looked at. */ +export async function readBlob(items: Record): Promise { + const meta = items[META_KEY]; + if (meta === undefined) return { kind: 'none' }; + if ( + typeof meta !== 'object' || + meta === null || + typeof (meta as BlobMeta).v !== 'number' || + typeof (meta as BlobMeta).n !== 'number' || + typeof (meta as BlobMeta).h !== 'string' || + typeof (meta as BlobMeta).at !== 'number' + ) { + return { kind: 'torn' }; + } + const m = meta as BlobMeta; + if (m.v > BLOB_VERSION) throw new NewerVersionError(); + const chunks: string[] = []; + for (let i = 0; i < m.n; i++) { + const chunk = items[chunkKey(i)]; + if (typeof chunk !== 'string') return { kind: 'torn' }; + chunks.push(chunk); + } + const base64 = chunks.join(''); + if ((await sha256Hex(base64)) !== m.h) return { kind: 'torn' }; + return { kind: 'ok', meta: m, base64 }; +} + +export async function unpackBackup(base64: string): Promise { + return parseBackupJson(JSON.parse(await gunzipToText(base64ToBytes(base64)))); +} + +/** Chrome's accounting for `getBytesInUse`: key length plus the JSON length + * of the value, per item. */ +export function itemsBytes(items: Record): number { + let total = 0; + for (const [key, value] of Object.entries(items)) { + total += key.length + JSON.stringify(value).length; + } + return total; +} diff --git a/src/features/sync/persist/sync-config.ts b/src/features/sync/persist/sync-config.ts index 86b2391..aff979c 100644 --- a/src/features/sync/persist/sync-config.ts +++ b/src/features/sync/persist/sync-config.ts @@ -5,97 +5,58 @@ import { storage } from '#imports'; * this device's bookkeeping. */ export interface SyncConfig { enabled: boolean; - /** `exportedAt` of the last snapshot pushed or applied; 0 = never synced. */ + /** `exportedAt` of the last blob pushed or merged in; 0 = never synced. */ lastSyncedAt: number; - /** Wall clock of the last local data change — the local side of - * last-write-wins against a remote snapshot's `exportedAt`. */ + /** Wall clock of the last local data change — this device's side of + * "whose settings win" against a remote blob's clock. */ lastChangedAt: number; - /** Hash of the data fields at last sync; a matching hash means there is - * nothing new to push (also swallows the echo of applying a remote copy). */ - lastSyncedHash: string | null; + /** `meta.h` of the blob this device last reconciled with. The same hash on + * the next read means nothing new arrived (our own write echo included). */ + lastRemoteHash: string | null; + /** Content hash of this device's data at the last reconcile; a different + * one now means there is something to push. */ + lastLocalHash: string | null; /** A change happened but the push hasn't landed yet — survives the panel * closing mid-debounce so the next open retries. */ pendingPush: boolean; lastError: string | null; - /** The sync ID this device agreed to join, if any. Sync is on by default and - * the ID arrives over browser-profile sync, so a device can be handed an - * identity it never asked for — and adopting one means a remote snapshot can - * overwrite everything here. Consent is granted implicitly when there is - * nothing to lose (a fresh install, or this device minted the ID itself) and - * explicitly via the panel otherwise. Stored per *identity*, not as a flag: - * a second ID arriving later is a second decision. Per-device like the rest - * of this record — consent doesn't travel in a backup. */ - consentedId: string | null; + /** The last push had to leave old songs or charts out to fit the quota. */ + trimmed: boolean; } export const DEFAULT_SYNC_CONFIG: SyncConfig = { enabled: true, lastSyncedAt: 0, lastChangedAt: 0, - lastSyncedHash: null, + lastRemoteHash: null, + lastLocalHash: null, pendingPush: false, lastError: null, - consentedId: null, + trimmed: false, }; export const syncConfigItem = storage.defineItem('local:syncConfig', { fallback: DEFAULT_SYNC_CONFIG, }); -/** Secret capability token, 43-char base64url (32 random bytes). Lives in the - * browser-synced `sync` area so Chrome/Firefox carry it to the user's other - * devices on the same browser profile — a second install only needs the - * toggle, not a pasted ID. Kept after disable so re-enabling reuses the same - * remote blob. */ -export const syncIdItem = storage.defineItem('sync:syncId', { - fallback: null, -}); - -/** `consentedId` postdates the first release. A device that had already synced - * with its ID consented back when the user switched sync on, so re-asking on - * upgrade would be noise. Returns whether it wrote anything. */ -function backfillConsent(config: SyncConfig, syncId: string | null): boolean { - if (config.consentedId !== undefined) return false; - config.consentedId = config.lastSyncedAt > 0 ? syncId : null; - return true; -} - -/** Reads both items, moving a pre-split `syncId` out of `local:syncConfig` - * into the sync area on the first run after the update. */ -export async function loadSyncState(): Promise<{ config: SyncConfig; syncId: string | null }> { - const [raw, storedId] = await Promise.all([syncConfigItem.getValue(), syncIdItem.getValue()]); - const { syncId: legacyId, ...config } = raw as SyncConfig & { syncId?: string | null }; - if (legacyId === undefined) { - if (backfillConsent(config, storedId)) await syncConfigItem.setValue(config); - return { config, syncId: storedId }; - } - let syncId = storedId; - if (legacyId !== null && storedId === null) { - syncId = legacyId; - await syncIdItem.setValue(legacyId); - } else if (legacyId !== null && storedId !== legacyId) { - // Another device in the profile already established a different identity; - // adopt it and drop bookkeeping that referred to the old blob. - config.lastSyncedAt = 0; - config.lastSyncedHash = null; - config.pendingPush = false; - } - backfillConsent(config, syncId); +/** Reads the record, migrating one written by the server-era builds (which + * carried `syncId`/`consentedId` and hashes over a different shape): the + * on/off choice is kept, the bookkeeping starts over so the first reconcile + * merges rather than trusting stale hashes. */ +export async function loadSyncConfig(): Promise { + const raw = (await syncConfigItem.getValue()) as Partial & { + syncId?: unknown; + consentedId?: unknown; + lastSyncedHash?: unknown; + }; + const legacy = 'syncId' in raw || 'consentedId' in raw || 'lastSyncedHash' in raw; + if (!legacy) return { ...DEFAULT_SYNC_CONFIG, ...raw }; + const config: SyncConfig = { + ...DEFAULT_SYNC_CONFIG, + enabled: raw.enabled ?? true, + lastChangedAt: raw.lastChangedAt ?? 0, + pendingPush: true, + }; await syncConfigItem.setValue(config); - return { config, syncId }; -} - -/** The server accepts `[A-Za-z0-9_-]{43,64}` — same check client-side so a - * mistyped ID fails before a network round-trip. The lower bound matches what - * `generateSyncId` produces: anything shorter is guessable, and since the ID is - * the only credential, a hand-picked short one would be squattable. */ -export const SYNC_ID_RE = /^[A-Za-z0-9_-]{43,64}$/; - -/** 32 random bytes as base64url (43 chars). The ID is the whole secret. */ -export function generateSyncId(): string { - const bytes = crypto.getRandomValues(new Uint8Array(32)); - return btoa(String.fromCharCode(...bytes)) - .replaceAll('+', '-') - .replaceAll('/', '_') - .replace(/=+$/, ''); + return config; } diff --git a/store/long-description-firefox.md b/store/long-description-firefox.md index 89c3bd3..68d66cf 100644 --- a/store/long-description-firefox.md +++ b/store/long-description-firefox.md @@ -41,7 +41,7 @@ Most pages attach directly: a Web Audio graph on the page's own audio or video e - No telemetry. No analytics. No ads. No accounts. Nothing is sold or shared. - Audio never leaves your device. Every effect — pitch, speed, vocal reduction, EQ, chord detection — runs locally in your browser. -- Cross-device sync is the only network request the extension ever makes. It is on by default and can be switched off in Settings, and it carries no audio: just your settings, markers, loops, snippets, and the page URLs and titles of the tracks in your library. +- The extension makes no network requests. Cross-device sync goes through Firefox Sync (no server, no account with us); it is on by default, can be switched off in Settings, and carries no audio: just your settings, markers, loops, snippets, and the page URLs and titles of the tracks in your library. - Full policy: [PRIVACY.md](https://github.com/patrickiel/note-by-note/blob/main/PRIVACY.md) **Open source** diff --git a/store/long-description.txt b/store/long-description.txt index 54d8c1a..5418c35 100644 --- a/store/long-description.txt +++ b/store/long-description.txt @@ -73,10 +73,11 @@ PRIVACY • No telemetry. No analytics. No ads. No accounts. Nothing is sold or shared. • Audio never leaves your device. Every effect — pitch, speed, vocal reduction, EQ, chord detection — runs locally in your browser. -• Cross-device sync is the only network request the extension ever makes. It is - on by default and can be switched off in Settings, and it carries no audio: - just your settings, markers, loops, snippets, and the page URLs and titles of - the tracks in your library. +• The extension makes no network requests. Cross-device sync goes through your + browser's own sync (no server, no account with us); it is on by default, can + be switched off in Settings, and carries no audio: just your settings, + markers, loops, snippets, and the page URLs and titles of the tracks in your + library. • Full policy: https://github.com/patrickiel/note-by-note/blob/main/PRIVACY.md diff --git a/store/privacy-policy-firefox.md b/store/privacy-policy-firefox.md index 64d2d45..9d21bed 100644 --- a/store/privacy-policy-firefox.md +++ b/store/privacy-policy-firefox.md @@ -17,9 +17,9 @@ Uninstalling the extension removes all of it. Settings → Reset Settings clears **What is transmitted, and when** -The extension makes exactly one kind of network request: the optional cross-device sync backup. Nothing else in the extension talks to the network. +The extension makes no network requests of its own. The one thing that leaves your device is the optional cross-device sync copy, and it leaves through Firefox Sync — the same channel that carries your bookmarks — to the other devices signed into the same Firefox account. If Firefox Sync is off, nothing leaves the device. -Sync is on by default, but it only starts transmitting once you have something to sync. When it does, it uploads a single snapshot containing your settings and UI preferences, your EQ presets, your Recent and Favorites lists — including the page URL and title of tracks you practised — and your per-track data (markers and labels, loop ranges, snippets, chord charts). +Sync is on by default. When there is something to sync, it writes one compact snapshot into the browser's synced extension storage containing your settings and UI preferences, your EQ presets, your Recent and Favorites lists — including the page URL and title of tracks you practised — and your per-track data (markers and labels, loop ranges, snippets, chord charts). Because that snapshot contains the addresses of pages you have visited, this listing declares the `browsingActivity` data-collection category. @@ -27,19 +27,15 @@ Not included: audio, page content, keystrokes, browsing history beyond the track **Where it goes** -Snapshots are stored by a Cloudflare Worker with Cloudflare KV, at `https://note-by-note-sync.oapp.workers.dev`, operated by the author. The server source is in the repository and can be self-hosted — self-hosters change one URL (`src/features/sync/sync-hosts.ts`) and rebuild. +Into Firefox Sync's storage under your Firefox account, end-to-end encrypted, subject to Mozilla's own data handling. The author operates no server and can see none of it. There are no accounts with us and no sync ID. -- There are no accounts. A random 43-character sync ID is the only credential; it *is* the capability, so treat it like a password. -- The ID travels in an `X-Sync-Id` header, never in the URL, so it does not land in request logs. KV is keyed by its SHA-256, so the raw ID is not stored either. -- Snapshots are stored **unencrypted**. Whoever operates the server can read them. Run your own if that matters to you. -- Snapshots expire after 180 days, refreshed on every write. -- IP addresses are visible to Cloudflare as part of serving and rate-limiting requests, per Cloudflare's own data handling. They are not stored by the Worker or linked to a snapshot. +Firefox caps this storage at 100 KB per extension. The snapshot is stored compact and compressed so a typical library fits with room to spare; when one doesn't, the oldest songs and chord charts stay on the device that has them and the Settings page says so. **Turning it off and deleting the data** -- Settings → Sync turns sync off. No further data is transmitted. -- Settings → Sync → Delete synced data removes the server-side copy. -- Doing nothing also works: an unused snapshot expires after 180 days. +- Settings → Sync turns sync off on that device. No further data is written. +- Settings → Sync → Delete synced data empties the synced copy (other devices with sync still on will write theirs again). +- Uninstalling the extension makes Firefox remove its synced storage. **Permissions and why** diff --git a/tsconfig.json b/tsconfig.json index 196d90a..c57681c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,5 @@ // Unit tests run via Node type-stripping, which needs explicit .ts // extensions on relative imports. "allowImportingTsExtensions": true - }, - // The sync worker has its own tsconfig with Cloudflare types. - "exclude": ["server"] + } } diff --git a/wxt.config.ts b/wxt.config.ts index ce199f1..6b70fbd 100644 --- a/wxt.config.ts +++ b/wxt.config.ts @@ -2,7 +2,6 @@ import { mkdirSync, readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import tailwindcss from '@tailwindcss/vite'; import { defineConfig } from 'wxt'; -import { SYNC_ENDPOINT_PROD, SYNC_HOST_PATTERNS, syncHostPattern } from './src/features/sync/sync-hosts'; // Persistent dev profile: extensions installed here (e.g. uBlock Origin // Lite for YouTube-breakage repros), logins, and site data survive between @@ -104,7 +103,8 @@ export default defineConfig({ // `data_collection_permissions` only in 140 — which is also the // current ESR line, so nothing supported is left behind. strict_min_version: '140.0', - // Sync ships Recent/Favorites off the device, and those carry the + // Sync ships Recent/Favorites off the device (through Firefox + // Sync's own storage — no server of ours), and those carry the // page URL, title and thumbnail of every track practised — AMO // counts that as browsing activity. `required` rather than // `optional` because sync is on out of the box @@ -125,23 +125,19 @@ export default defineConfig({ // gets `sidebar_action`, which needs no permission). `tabCapture` and // `offscreen` are Chromium-only APIs — see src/core/platform.ts, which gates // every caller on the same build target. + // `storage` covers `storage.sync`, which is all cross-device sync needs: + // no host permission, no `cookies`, no `identity` (Chromium's "is browser + // sync on" probe would want `identity.email` — an install warning — so + // there is no such probe; the Settings copy tells the user instead). permissions: [ 'storage', - 'cookies', 'activeTab', 'scripting', 'tabs', ...(browser === 'firefox' ? [] : ['tabCapture', 'offscreen']), ], - // `` is what Connect asks for. The sync host is what the ID - // cookie needs (see src/features/sync/panel/id-cookie.ts) — requested from - // the Sync settings, and covered by `` too once that is granted. - // Non-production builds also list the localhost Worker so `pnpm dev` can - // exercise the cookie path. - optional_host_permissions: [ - '', - ...(mode === 'production' ? [syncHostPattern(SYNC_ENDPOINT_PROD)] : SYNC_HOST_PATTERNS), - ], + // `` is what Connect asks for. + optional_host_permissions: [''], action: { default_title: 'Note by Note', }, From f9640243fcd532054de5886d86e00d102803cf15 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 19:35:08 +0200 Subject: [PATCH 04/26] feat: enhance EQ preset management with updated timestamps and deletion handling --- CLAUDE.md | 2 +- src/core/model/track-identity.ts | 7 ++++ src/core/model/types.ts | 3 ++ src/core/persist/backup-codec.test.ts | 18 +++++++-- src/core/persist/backup-codec.ts | 23 ++++++++--- src/core/persist/deletions.ts | 47 ++++++++++++++-------- src/core/persist/storage.ts | 10 +++++ src/core/state/track-sync.svelte.ts | 5 ++- src/features/eq/persist/eq-presets.ts | 21 ++++++---- src/features/library/persist/favorites.ts | 14 ++++--- src/features/library/persist/history.ts | 17 +++++--- src/features/sync/persist/merge.test.ts | 48 +++++++++++++++++++++-- src/features/sync/persist/merge.ts | 26 +++++++++--- 13 files changed, 187 insertions(+), 54 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0b060b9..9d04a20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,7 +97,7 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. - **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped, timestamps in seconds. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. -- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`, TTL 30 d, cap 200) so a merge can't resurrect them; track records need none (an emptied record still wins on `updatedAt`). Pushes are debounced 5 s and spaced ≥ 30 s (the browser meters writes at 120/min); the per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is migrated from the server-era record on first load. Legacy `syncId` keys in the area are removed on the next write. +- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, `e:`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, TTL 30 d, cap 200) so a merge can't resurrect them. Songs are named by `songKey` (URL + title, **no duration** — the same identity the merge matches on, so a copy saved under a drifted duration is covered too); presets carry an optional `updatedAt` so a later save beats the deletion; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). Pushes are debounced 5 s and spaced ≥ 30 s (the browser meters writes at 120/min); the per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is migrated from the server-era record on first load. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. diff --git a/src/core/model/track-identity.ts b/src/core/model/track-identity.ts index e790a3d..ceb0cb5 100644 --- a/src/core/model/track-identity.ts +++ b/src/core/model/track-identity.ts @@ -63,6 +63,13 @@ export function identityKey(normalizedUrl: string, durationSec: number): string return `${hash(normalizedUrl)}:${durationSec}`; } +/** A short handle for what `isSameTrack` compares — URL and title, no + * duration — so a record about "this song" (a sync deletion, say) reaches + * every copy of it however its duration drifted. */ +export function songKey(identity: Pick): string { + return hash(`${identity.normalizedUrl}\n${identity.title}`); +} + export function makeTrackIdentity( pageUrl: string, title: string, diff --git a/src/core/model/types.ts b/src/core/model/types.ts index 6b1b1b1..582e09d 100644 --- a/src/core/model/types.ts +++ b/src/core/model/types.ts @@ -37,6 +37,9 @@ export interface EffectParams { export interface EqPreset { name: string; gains: number[]; + /** Last save. Absent on presets from before sync merged; reads as 0, so a + * dated deletion (deletions.ts) beats them and a later save beats it. */ + updatedAt?: number; } export interface Marker { diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index 1483796..cccf94b 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -255,10 +255,20 @@ test('settings: a keymap saved before an action existed is backfilled', () => { assert.equal(roundTrip(b).settings.keymap.zoomFit, DEFAULT_SETTINGS.keymap.zoomFit); }); -test('eq presets round-trip as tuples', () => { - const b = backup({ eqPresets: [{ name: 'Mine', gains: [1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0] }] }); - assert.deepEqual(encodeBackup(b).eq, [['Mine', 1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0]]); - assert.deepEqual(roundTrip(b).eqPresets, b.eqPresets); +test('eq presets round-trip as tuples, with or without a save time', () => { + const b = backup({ + eqPresets: [ + { name: 'Mine', gains: [1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0] }, + { name: 'Stamped', gains: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], updatedAt: 1_757_112_345_678 }, + ], + }); + assert.deepEqual(encodeBackup(b).eq, [ + ['Mine', [1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0]], + ['Stamped', [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 1757112346], + ]); + const back = roundTrip(b).eqPresets; + assert.deepEqual(back[0], b.eqPresets[0]); + assert.deepEqual(back[1], { ...b.eqPresets[1], updatedAt: 1757112346000 }); }); // --------------------------------------------------------------------------- diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 58c356c..85f4f22 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -134,6 +134,9 @@ export interface CompactFavorite extends CompactEntry { la: number; } +/** `[name, gains, updatedAt_s?]`. */ +export type CompactEqPreset = [string, number[]] | [string, number[], number]; + /** `[t_ms, label?]` — label omitted when empty. */ export type CompactMarker = [number] | [number, string]; @@ -192,8 +195,8 @@ export interface CompactBackup { s: Record; /** UI prefs that differ from the defaults. */ u: Record; - /** `[name, ...gains]` per saved EQ preset. */ - eq: (string | number)[][]; + /** `[name, gains, updatedAt_s?]` per saved EQ preset. */ + eq: CompactEqPreset[]; songs: CompactSong[]; h: CompactEntry[]; f: CompactFavorite[]; @@ -722,14 +725,22 @@ function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { // --------------------------------------------------------------------------- // Whole backup -function encodeEqPreset(preset: EqPreset): (string | number)[] { - return [preset.name ?? '', ...(preset.gains ?? []).map(round2)]; +/** `[name, gains, updatedAt_s?]`. */ +function encodeEqPreset(preset: EqPreset): CompactEqPreset { + const name = preset.name ?? ''; + const gains = (preset.gains ?? []).map(round2); + return preset.updatedAt ? [name, gains, secs(preset.updatedAt)] : [name, gains]; } function decodeEqPreset(raw: unknown): EqPreset { const r = arr(raw, 'eqPresets'); - if (r.length < 1) throw damaged('eqPresets'); - return { name: str(r[0], 'eqPresets'), gains: r.slice(1).map((g) => num(g, 'eqPresets')) }; + if (r.length < 2 || r.length > 3) throw damaged('eqPresets'); + const preset: EqPreset = { + name: str(r[0], 'eqPresets'), + gains: arr(r[1], 'eqPresets').map((g) => num(g, 'eqPresets')), + }; + if (r.length === 3) preset.updatedAt = num(r[2], 'eqPresets') * 1000; + return preset; } const byKey = (a: { identity: TrackIdentity }, b: { identity: TrackIdentity }) => diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index e8153ed..6258262 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -1,24 +1,29 @@ /** * Deletion records ("tombstones"): what the user removed, and when. Without - * them a Recent row or a favorite deleted on one device would come straight - * back from another device's copy the next time the two merge — a merge is a - * union, and it cannot tell "never had it" from "removed it". Track records - * need none: an emptied record still exists, with a newer `updatedAt`, and - * wins its merge on that. + * them a Recent row, a favorite or an EQ preset deleted on one device would + * come straight back from another device's copy the next time the two merge — + * a merge is a union, and it cannot tell "never had it" from "removed it". + * Track records need none: an emptied record still exists, with a newer + * `updatedAt`, and wins its merge on that. * - * One flat map, `key → when` (ms). Keys: `h:` a Recent row, - * `f:` a favorite, `h:*` "Clear Recent". A record older than the - * item it names (the song was played again after the deletion) is ignored, so - * re-adding always works. Records expire after a month and are capped, newest - * kept — long enough for any device that will ever sync again to see them. + * One flat map, `key → when` (ms). Keys: `h:` a Recent row, + * `f:` a favorite, `h:*` "Clear Recent", `e:` an EQ preset. + * Songs are named by `songKey` (URL + title, no duration — see + * track-identity.ts) so the record reaches every copy of the song, however + * its duration drifted, exactly like the merge matches them. A record older + * than the item it names (the song was played again, the preset saved again) + * is ignored, so re-adding always works. Records expire after a month and are + * capped, newest kept — long enough for any device that will ever sync again + * to see them. * * Pure and DOM-free (relative `.ts` imports; runs under `node --test`). */ export type Deletions = Record; -export const historyDeletion = (key: string) => `h:${key}`; -export const favoriteDeletion = (key: string) => `f:${key}`; +export const historyDeletion = (songKey: string) => `h:${songKey}`; +export const favoriteDeletion = (songKey: string) => `f:${songKey}`; +export const presetDeletion = (name: string) => `e:${name}`; export const HISTORY_CLEARED = 'h:*'; export const DELETION_TTL_MS = 30 * 24 * 60 * 60_000; @@ -31,16 +36,26 @@ export function deletedAt(deletions: Deletions, key: string): number { /** Whether a Recent row with `updatedAt` is covered by a deletion — its own * or a "Clear Recent" — dated at or after it. */ -export function historyDeleted(deletions: Deletions, key: string, updatedAt: number): boolean { - const when = Math.max(deletedAt(deletions, historyDeletion(key)), deletedAt(deletions, HISTORY_CLEARED)); +export function historyDeleted(deletions: Deletions, songKey: string, updatedAt: number): boolean { + const when = Math.max( + deletedAt(deletions, historyDeletion(songKey)), + deletedAt(deletions, HISTORY_CLEARED), + ); return when > 0 && when >= updatedAt; } -export function favoriteDeleted(deletions: Deletions, key: string, since: number): boolean { - const when = deletedAt(deletions, favoriteDeletion(key)); +export function favoriteDeleted(deletions: Deletions, songKey: string, since: number): boolean { + const when = deletedAt(deletions, favoriteDeletion(songKey)); return when > 0 && when >= since; } +/** Presets saved before they carried `updatedAt` read as 0: a deletion + * always beats them, a later save always beats the deletion. */ +export function presetDeleted(deletions: Deletions, name: string, updatedAt: number): boolean { + const when = deletedAt(deletions, presetDeletion(name)); + return when > 0 && when >= updatedAt; +} + /** Newest date per key. */ export function mergeDeletions(a: Deletions, b: Deletions): Deletions { const out: Deletions = { ...a }; diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index 22180c5..a423b00 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -93,6 +93,16 @@ export async function recordDeletion(...keys: string[]): Promise { await deletionsItem.setValue(pruneDeletions(next, now)); } +/** Forgets a deletion — the item was re-created here, and a record dated + * after its re-creation on another device would kill it in the merge. */ +export async function clearDeletion(...keys: string[]): Promise { + const current = await deletionsItem.getValue(); + if (!keys.some((key) => key in current)) return; + const next = { ...current }; + for (const key of keys) delete next[key]; + await deletionsItem.setValue(next); +} + /** Per-track markers/snippets, keyed by TrackIdentity.key. */ export function trackDataKey(key: string) { return `local:track:${key}` as const; diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index 28938c9..21ce5b6 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -132,7 +132,10 @@ class TrackSync { // the stale identity gets another go now the duration has settled. if (!adjusted) this.#restoreSaved(identity); if (adjusted) { - await removeHistoryEntry(staleKey); + // Housekeeping, not a user deletion: the song stays, only its + // stale-keyed twin goes — so no deletion record (which is per song + // and would kill the fresh row on the other devices). + await removeHistoryEntry(staleKey, { record: false }); await this.#saveCurrent(); // Only carry the stale key's slice over when the real key has none, so // this can't overwrite a saved record with an emptied one. diff --git a/src/features/eq/persist/eq-presets.ts b/src/features/eq/persist/eq-presets.ts index d0740c3..8380094 100644 --- a/src/features/eq/persist/eq-presets.ts +++ b/src/features/eq/persist/eq-presets.ts @@ -1,21 +1,28 @@ -import { eqPresetsItem } from '../../../core/persist/storage'; +import { presetDeletion } from '../../../core/persist/deletions'; +import { clearDeletion, eqPresetsItem, recordDeletion } from '../../../core/persist/storage'; /** Save the current curve under `name`. An existing preset with that name is * replaced in place — that's the edit and rename path, so there's no separate - * UI for either. */ + * UI for either. Stamped so a sync merge can tell this save from a deletion + * of the same name on another device (see deletions.ts). */ export async function saveEqPreset(name: string, gains: number[]): Promise { const list = await eqPresetsItem.getValue(); const index = list.findIndex((p) => p.name === name); + const preset = { name, gains, updatedAt: Date.now() }; if (index === -1) { - await eqPresetsItem.setValue([...list, { name, gains }]); - return; + await eqPresetsItem.setValue([...list, preset]); + } else { + const next = [...list]; + next[index] = preset; + await eqPresetsItem.setValue(next); } - const next = [...list]; - next[index] = { name, gains }; - await eqPresetsItem.setValue(next); + await clearDeletion(presetDeletion(name)); } +/** Dated (`deletions.ts`) so a sync merge with another device's copy doesn't + * bring the preset back. */ export async function deleteEqPreset(name: string): Promise { const list = await eqPresetsItem.getValue(); await eqPresetsItem.setValue(list.filter((p) => p.name !== name)); + await recordDeletion(presetDeletion(name)); } diff --git a/src/features/library/persist/favorites.ts b/src/features/library/persist/favorites.ts index 9d483ab..b4bd0f8 100644 --- a/src/features/library/persist/favorites.ts +++ b/src/features/library/persist/favorites.ts @@ -1,7 +1,7 @@ import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; -import { isSameTrack } from '../../../core/model/track-identity'; +import { isSameTrack, songKey } from '../../../core/model/track-identity'; import { favoriteDeletion } from '../../../core/persist/deletions'; -import { favoritesItem, recordDeletion } from '../../../core/persist/storage'; +import { clearDeletion, favoritesItem, recordDeletion } from '../../../core/persist/storage'; /** Star a song: copy the history entry into the Favorites library (top of the * manual order). No-op if already favorited. */ @@ -15,14 +15,18 @@ export async function addFavorite(entry: HistoryEntry): Promise { { ...entry, favoritedAt: now, lastAccessedAt: now }, ...list, ]); + // Starred again after an unstar: the deletion record must not outlive it. + await clearDeletion(favoriteDeletion(songKey(entry.identity))); } -/** Dated (`deletions.ts`) so a sync merge with another device's older copy - * doesn't star the song again. */ +/** Dated (`deletions.ts`, by song — every copy of it, whatever duration it + * was saved under) so a sync merge with another device's older copy doesn't + * star the song again. */ export async function removeFavorite(key: string): Promise { const list = await favoritesItem.getValue(); + const entry = list.find((e) => e.identity.key === key); await favoritesItem.setValue(list.filter((e) => e.identity.key !== key)); - await recordDeletion(favoriteDeletion(key)); + if (entry) await recordDeletion(favoriteDeletion(songKey(entry.identity))); } /** Persist a new manual order (list of identity keys, complete). */ diff --git a/src/features/library/persist/history.ts b/src/features/library/persist/history.ts index 942b06a..358d8cc 100644 --- a/src/features/library/persist/history.ts +++ b/src/features/library/persist/history.ts @@ -1,6 +1,6 @@ import { HISTORY_LIMIT } from '../../../core/model/defaults'; import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; -import { isSameTrack } from '../../../core/model/track-identity'; +import { isSameTrack, songKey } from '../../../core/model/track-identity'; import { HISTORY_CLEARED, historyDeletion } from '../../../core/persist/deletions'; import { historyItem, recordDeletion } from '../../../core/persist/storage'; @@ -47,12 +47,19 @@ export async function dedupeHistory(): Promise { if (kept.length !== list.length) await historyItem.setValue(kept); } -/** Deletions are dated (`deletions.ts`) so a sync merge with another device's - * older copy doesn't bring the row back. */ -export async function removeHistoryEntry(key: string): Promise { +/** The user removed a row: dated (`deletions.ts`, by song — every copy of it + * on every device, whatever duration it was saved under) so a sync merge with + * another device's older copy doesn't bring it back. `record: false` is for + * housekeeping that drops a stale twin of a song that stays — recording that + * would kill the song's fresh row on the other devices. */ +export async function removeHistoryEntry( + key: string, + { record = true }: { record?: boolean } = {}, +): Promise { const list = await historyItem.getValue(); + const entry = list.find((e) => e.identity.key === key); await historyItem.setValue(list.filter((e) => e.identity.key !== key)); - await recordDeletion(historyDeletion(key)); + if (record && entry) await recordDeletion(historyDeletion(songKey(entry.identity))); } export async function clearHistory(): Promise { diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts index cdbe1fb..c01cb18 100644 --- a/src/features/sync/persist/merge.test.ts +++ b/src/features/sync/persist/merge.test.ts @@ -7,9 +7,10 @@ import { favoriteDeletion, HISTORY_CLEARED, historyDeletion, + presetDeletion, } from '../../../core/persist/deletions.ts'; import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, HISTORY_LIMIT } from '../../../core/model/defaults.ts'; -import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; +import { makeTrackIdentity, songKey } from '../../../core/model/track-identity.ts'; import type { ChordChart, FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types.ts'; const T0 = 1_757_000_000_000; @@ -90,7 +91,7 @@ test('history is matched by song, so a drifted duration does not make a twin', ( test('a deletion beats the copy it postdates, but not a later re-play', () => { const gone = song(1); - const local = backup({ deletions: { [historyDeletion(gone.key)]: T0 + 2000 } }); + const local = backup({ deletions: { [historyDeletion(songKey(gone))]: T0 + 2000 } }); const remote = backup({ history: [row(gone, T0 + 1000), row(song(2), T0)] }); assert.deepEqual(keys(mergeBackups(local, remote, true, NOW).history), [song(2).key]); const replayed = backup({ history: [row(gone, T0 + 3000)] }); @@ -121,7 +122,7 @@ test('favorites: union in the winner order, deletions honoured, last access kept const c = song(3); const local = backup({ favorites: [fav(b, T0, T0 + 9000), fav(a, T0)], - deletions: { [favoriteDeletion(c.key)]: T0 + 100 }, + deletions: { [favoriteDeletion(songKey(c))]: T0 + 100 }, }); const remote = backup({ favorites: [fav(a, T0 + 1), fav(c, T0)] }); const merged = mergeBackups(local, remote, true, NOW); @@ -167,6 +168,47 @@ test('ties go to the winning side', () => { assert.equal(mergeBackups(local, remote, false, NOW).tracks[0].markers.length, 1); }); +test('a deletion reaches every copy of the song, whatever duration it was saved under', () => { + const url = 'https://www.youtube.com/watch?v=drifted0001'; + const s201 = makeTrackIdentity(url, 'Song', 201); + const s200 = makeTrackIdentity(url, 'Song', 200); + assert.notEqual(s201.key, s200.key); + const local = backup({ + deletions: { + [historyDeletion(songKey(s201))]: T0 + 5000, + [favoriteDeletion(songKey(s201))]: T0 + 5000, + }, + }); + const remote = backup({ history: [row(s200, T0)], favorites: [fav(s200, T0)] }); + const merged = mergeBackups(local, remote, false, NOW); + assert.equal(merged.history.length, 0); + assert.equal(merged.favorites.length, 0); + // A different song at the same URL (local files share one) is untouched. + const other = makeTrackIdentity(url, 'Other song', 200); + const kept = mergeBackups(local, backup({ history: [row(other, T0)] }), false, NOW); + assert.equal(kept.history.length, 1); +}); + +test('a deleted EQ preset stays deleted; a later save of the name brings it back', () => { + const gains = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const deleter = backup({ deletions: { [presetDeletion('Mine')]: T0 + 2000 } }); + const keeper = backup({ eqPresets: [{ name: 'Mine', gains, updatedAt: T0 + 1000 }] }); + assert.deepEqual(mergeBackups(deleter, keeper, false, NOW).eqPresets, []); + assert.deepEqual(mergeBackups(deleter, keeper, true, NOW).eqPresets, [], 'even when the keeper wins'); + const unstamped = backup({ eqPresets: [{ name: 'Mine', gains }] }); + assert.deepEqual(mergeBackups(deleter, unstamped, true, NOW).eqPresets, [], 'an undated preset loses to any deletion'); + const resaved = backup({ eqPresets: [{ name: 'Mine', gains, updatedAt: T0 + 3000 }] }); + assert.equal(mergeBackups(deleter, resaved, false, NOW).eqPresets.length, 1); + assert.equal(mergeBackups(deleter, resaved, false, NOW).deletions[presetDeletion('Mine')], T0 + 2000, 'the record still travels'); +}); + +test('a shared preset name goes to the later save', () => { + const local = backup({ eqPresets: [{ name: 'Mine', gains: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], updatedAt: T0 + 5 }] }); + const remote = backup({ eqPresets: [{ name: 'Mine', gains: [2, 0, 0, 0, 0, 0, 0, 0, 0, 0], updatedAt: T0 + 1 }] }); + assert.equal(mergeBackups(local, remote, true, NOW).eqPresets[0].gains[0], 1, 'later save beats the winner side'); + assert.equal(mergeBackups(local, remote, false, NOW).eqPresets[0].gains[0], 1); +}); + test('settings and prefs come from the winner; EQ presets are a union', () => { const local = backup({ settings: { ...DEFAULT_SETTINGS, theme: 'dark' }, diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index f276feb..b54facc 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -4,8 +4,10 @@ import { favoriteDeleted, historyDeleted, mergeDeletions, + presetDeleted, pruneDeletions, } from '../../../core/persist/deletions.ts'; +import { songKey } from '../../../core/model/track-identity.ts'; import type { EqPreset, FavoriteEntry, @@ -27,7 +29,8 @@ import type { * winner without a chart adopts the other's: absent may mean "trimmed", * and a chart is only ever replaced by re-analysis. * - Settings and UI prefs: the newer device's, as a whole. EQ presets: union - * by name, the newer device's order first. + * by name, the later save of a shared name, deletions honoured, the newer + * device's order first. * * `remoteWins` breaks ties and picks the wholesale sections: true when the * remote copy was written after this device's last local change. Pure; @@ -46,7 +49,7 @@ function mergeHistory( ): HistoryEntry[] { const bySong = new Map(); for (const entry of [...first, ...second]) { - if (historyDeleted(deletions, entry.identity.key, entry.updatedAt ?? 0)) continue; + if (historyDeleted(deletions, songKey(entry.identity), entry.updatedAt ?? 0)) continue; const id = songId(entry.identity); const current = bySong.get(id); if (!current || (entry.updatedAt ?? 0) > (current.updatedAt ?? 0)) bySong.set(id, entry); @@ -63,7 +66,7 @@ function mergeFavorites( const bySong = new Map(); for (const entry of [...first, ...second]) { const since = Math.max(entry.favoritedAt ?? 0, entry.updatedAt ?? 0); - if (favoriteDeleted(deletions, entry.identity.key, since)) continue; + if (favoriteDeleted(deletions, songKey(entry.identity), since)) continue; const id = songId(entry.identity); const current = bySong.get(id); if (!current) { @@ -100,10 +103,21 @@ function mergeTracks(first: TrackData[], second: TrackData[]): TrackData[] { return [...byKey.values()]; } -function mergeEqPresets(first: EqPreset[], second: EqPreset[]): EqPreset[] { +/** Union by name; a dated deletion beats any copy it postdates, and a shared + * name goes to the later save (the winner side on a tie, or when neither + * carries a save time). */ +function mergeEqPresets( + first: EqPreset[], + second: EqPreset[], + deletions: Backup['deletions'], +): EqPreset[] { const byName = new Map(); for (const preset of [...first, ...second]) { - if (!byName.has(preset.name)) byName.set(preset.name, preset); + if (presetDeleted(deletions, preset.name, preset.updatedAt ?? 0)) continue; + const current = byName.get(preset.name); + if (!current || (preset.updatedAt ?? 0) > (current.updatedAt ?? 0)) { + byName.set(preset.name, preset); + } } return [...byName.values()]; } @@ -121,7 +135,7 @@ export function mergeBackups( exportedAt: Math.max(local.exportedAt ?? 0, remote.exportedAt ?? 0), settings: winner.settings, uiPrefs: winner.uiPrefs, - eqPresets: mergeEqPresets(winner.eqPresets, loser.eqPresets), + eqPresets: mergeEqPresets(winner.eqPresets, loser.eqPresets, deletions), history: mergeHistory(winner.history, loser.history, deletions), favorites: mergeFavorites(winner.favorites, loser.favorites, deletions), tracks: mergeTracks(winner.tracks, loser.tracks), From dfbf5a924edb008405514bd444276806d3180168 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 19:54:42 +0200 Subject: [PATCH 05/26] refactor: remove clearDeletion function and its usages - Deleted the clearDeletion function from storage.ts, which was responsible for forgetting deletions of items. - Updated saveEqPreset and addFavorite functions to remove calls to clearDeletion, ensuring that deletion records are no longer cleared when items are recreated. - Adjusted sync.svelte.ts to improve the handling of sync state and configuration, including the removal of unnecessary state variables and timers. - Refactored the fit.ts file to streamline the backup fitting process, replacing the previous tiered approach with a more efficient cut-based method. - Enhanced merge.ts to utilize a unionNewest function for merging history, favorites, and tracks, improving clarity and maintainability. - Simplified sync-area.ts by removing unnecessary checks and consolidating event listeners. - Updated sync-blob.ts to remove the chars property from the PackedBlob interface, as it was no longer needed. - Refined sync-config.ts to improve the loading of sync configuration, ensuring legacy fields are handled appropriately without retaining unnecessary data. --- CLAUDE.md | 2 +- src/core/persist/backup-codec.test.ts | 4 +- src/core/persist/backup-codec.ts | 59 +++---- src/core/persist/deletions.ts | 37 +--- src/core/persist/storage.ts | 10 -- src/features/eq/persist/eq-presets.ts | 3 +- src/features/library/persist/favorites.ts | 4 +- src/features/sync/panel/sync.svelte.ts | 180 ++++++++------------ src/features/sync/persist/fit.ts | 170 +++++++----------- src/features/sync/persist/merge.ts | 160 +++++++---------- src/features/sync/persist/sync-area.ts | 43 ++--- src/features/sync/persist/sync-blob.test.ts | 5 +- src/features/sync/persist/sync-blob.ts | 4 +- src/features/sync/persist/sync-config.ts | 31 ++-- 14 files changed, 263 insertions(+), 449 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9d04a20..f5ad210 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,7 +97,7 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. - **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped, timestamps in seconds. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. -- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, `e:`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, TTL 30 d, cap 200) so a merge can't resurrect them. Songs are named by `songKey` (URL + title, **no duration** — the same identity the merge matches on, so a copy saved under a drifted duration is covered too); presets carry an optional `updatedAt` so a later save beats the deletion; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). Pushes are debounced 5 s and spaced ≥ 30 s (the browser meters writes at 120/min); the per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is migrated from the server-era record on first load. Legacy `syncId` keys in the area are removed on the next write. +- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, `e:`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, TTL 30 d, cap 200) so a merge can't resurrect them. Songs are named by `songKey` (URL + title, **no duration** — the same identity the merge matches on, so a copy saved under a drifted duration is covered too); presets carry an optional `updatedAt` so a later save beats the deletion; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index cccf94b..f7d8951 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -555,7 +555,7 @@ test('a v2 file routes through the codec; anything newer or foreign is refused', assert.throws(() => parseBackupJson('nope'), /isn't a Note by Note backup/); assert.throws( () => parseBackupJson({ format: BACKUP_FORMAT, version: 1, settings: {} }), - /"history" list is missing or damaged/, + /"history" list is damaged/, ); }); @@ -566,7 +566,7 @@ test('damaged v2 input is refused with the section named', () => { fn(raw); return () => decodeBackup(raw); }; - assert.throws(mutate((r) => delete r.songs), /"songs" list is missing or damaged/); + assert.throws(mutate((r) => delete r.songs), /"songs" list is damaged/); assert.throws(mutate((r) => r.songs[0][0] = 'yt:bad/id'), /"songs" list is damaged/); assert.throws(mutate((r) => r.h[0].i = 99), /"history" list is damaged/); assert.throws(mutate((r) => r.h[0].at = 'now'), /"history" list is damaged/); diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 85f4f22..65f5397 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -208,7 +208,7 @@ export interface CompactBackup { // --------------------------------------------------------------------------- // Shared helpers -export function isRecord(value: unknown): value is Record { +function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -216,24 +216,6 @@ function damaged(section: string): Error { return new Error(`This backup's "${section}" list is damaged.`); } -export function requireArray(value: unknown, field: string): unknown[] { - if (!Array.isArray(value)) { - throw new Error(`This backup's "${field}" list is missing or damaged.`); - } - return value; -} - -/** Entries are keyed by `identity.key`; without one they can't be stored or - * matched back to a track, so a file carrying them is not usable. */ -export function requireKeyedArray(value: unknown, field: string): T[] { - const list = requireArray(value, field); - const keyed = list.every( - (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.key === 'string', - ); - if (!keyed) throw damaged(field); - return list as T[]; -} - const roundTo = (dp: number) => { const f = 10 ** dp; return (x: number) => Math.round(x * f) / f; @@ -264,6 +246,17 @@ function rec(value: unknown, section: string): Record { return value; } +/** Entries are keyed by `identity.key`; without one they can't be stored or + * matched back to a track, so a file carrying them is not usable. */ +function keyedArr(value: unknown, section: string): T[] { + const list = arr(value, section); + const keyed = list.every( + (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.key === 'string', + ); + if (!keyed) throw damaged(section); + return list as T[]; +} + function jsonEqual(a: unknown, b: unknown): boolean { return JSON.stringify(a) === JSON.stringify(b); } @@ -467,7 +460,7 @@ class SongTable { } function decodeSongs(raw: unknown): TrackIdentity[] { - return requireArray(raw, 'songs').map((row) => { + return arr(raw, 'songs').map((row) => { const r = arr(row, 'songs'); if (r.length < 3 || r.length > 4) throw damaged('songs'); const normalizedUrl = longUrl(str(r[0], 'songs')); @@ -584,13 +577,9 @@ function encodeSnippet(snippet: Snippet): CompactSnippet { snippet.enabled === false ? 0 : 1, overrides, ]; - if (Object.keys(overrides).length === 0) { - out.pop(); - if (out[4] === 1) { - out.pop(); - if (out[3] === 1) out.pop(); - } - } + // Trailing defaults are left out: `{}` overrides, enabled, one repeat. + const isDefault = (v: unknown) => v === 1 || (isRecord(v) && Object.keys(v).length === 0); + while (out.length > 3 && isDefault(out[out.length - 1])) out.pop(); return out; } @@ -787,10 +776,10 @@ export function decodeBackup(raw: unknown): Backup { appVersion: '', settings: decodeSettings(raw.s), uiPrefs: decodeUiPrefs(raw.u), - history: requireArray(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), - favorites: requireArray(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), - eqPresets: requireArray(raw.eq, 'eqPresets').map(decodeEqPreset), - tracks: requireArray(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), + history: arr(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), + favorites: arr(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), + eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), + tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), deletions: Object.fromEntries( Object.entries(normalizeDeletions(raw.del)).map(([k, when]) => [k, when * 1000]), ), @@ -814,10 +803,10 @@ function normalizeV1(raw: Record): Backup { ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), }, - history: requireKeyedArray(raw.history, 'history'), - favorites: requireKeyedArray(raw.favorites, 'favorites'), - eqPresets: requireArray(raw.eqPresets, 'eqPresets') as EqPreset[], - tracks: requireKeyedArray(raw.tracks, 'tracks'), + history: keyedArr(raw.history, 'history'), + favorites: keyedArr(raw.favorites, 'favorites'), + eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], + tracks: keyedArr(raw.tracks, 'tracks'), deletions: normalizeDeletions(raw.deletions), }; } diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index 6258262..73ee5bc 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -29,40 +29,19 @@ export const HISTORY_CLEARED = 'h:*'; export const DELETION_TTL_MS = 30 * 24 * 60 * 60_000; export const DELETION_CAP = 200; -/** When `key` was deleted, or 0 if it wasn't. */ -export function deletedAt(deletions: Deletions, key: string): number { - return deletions[key] ?? 0; -} - -/** Whether a Recent row with `updatedAt` is covered by a deletion — its own - * or a "Clear Recent" — dated at or after it. */ -export function historyDeleted(deletions: Deletions, songKey: string, updatedAt: number): boolean { - const when = Math.max( - deletedAt(deletions, historyDeletion(songKey)), - deletedAt(deletions, HISTORY_CLEARED), - ); - return when > 0 && when >= updatedAt; -} - -export function favoriteDeleted(deletions: Deletions, songKey: string, since: number): boolean { - const when = deletedAt(deletions, favoriteDeletion(songKey)); - return when > 0 && when >= since; -} - -/** Presets saved before they carried `updatedAt` read as 0: a deletion - * always beats them, a later save always beats the deletion. */ -export function presetDeleted(deletions: Deletions, name: string, updatedAt: number): boolean { - const when = deletedAt(deletions, presetDeletion(name)); - return when > 0 && when >= updatedAt; +/** Whether any of `keys` carries a deletion dated at or after `since` — the + * item's own save time, so a later re-add always beats the record. */ +export function deletedSince(deletions: Deletions, since: number, ...keys: string[]): boolean { + return keys.some((key) => { + const when = deletions[key] ?? 0; + return when > 0 && when >= since; + }); } /** Newest date per key. */ export function mergeDeletions(a: Deletions, b: Deletions): Deletions { const out: Deletions = { ...a }; - for (const [key, when] of Object.entries(b)) { - if (typeof when !== 'number') continue; - out[key] = Math.max(out[key] ?? 0, when); - } + for (const [key, when] of Object.entries(b)) out[key] = Math.max(out[key] ?? 0, when); return out; } diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index a423b00..22180c5 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -93,16 +93,6 @@ export async function recordDeletion(...keys: string[]): Promise { await deletionsItem.setValue(pruneDeletions(next, now)); } -/** Forgets a deletion — the item was re-created here, and a record dated - * after its re-creation on another device would kill it in the merge. */ -export async function clearDeletion(...keys: string[]): Promise { - const current = await deletionsItem.getValue(); - if (!keys.some((key) => key in current)) return; - const next = { ...current }; - for (const key of keys) delete next[key]; - await deletionsItem.setValue(next); -} - /** Per-track markers/snippets, keyed by TrackIdentity.key. */ export function trackDataKey(key: string) { return `local:track:${key}` as const; diff --git a/src/features/eq/persist/eq-presets.ts b/src/features/eq/persist/eq-presets.ts index 8380094..6093260 100644 --- a/src/features/eq/persist/eq-presets.ts +++ b/src/features/eq/persist/eq-presets.ts @@ -1,5 +1,5 @@ import { presetDeletion } from '../../../core/persist/deletions'; -import { clearDeletion, eqPresetsItem, recordDeletion } from '../../../core/persist/storage'; +import { eqPresetsItem, recordDeletion } from '../../../core/persist/storage'; /** Save the current curve under `name`. An existing preset with that name is * replaced in place — that's the edit and rename path, so there's no separate @@ -16,7 +16,6 @@ export async function saveEqPreset(name: string, gains: number[]): Promise next[index] = preset; await eqPresetsItem.setValue(next); } - await clearDeletion(presetDeletion(name)); } /** Dated (`deletions.ts`) so a sync merge with another device's copy doesn't diff --git a/src/features/library/persist/favorites.ts b/src/features/library/persist/favorites.ts index b4bd0f8..2b5d85b 100644 --- a/src/features/library/persist/favorites.ts +++ b/src/features/library/persist/favorites.ts @@ -1,7 +1,7 @@ import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; import { isSameTrack, songKey } from '../../../core/model/track-identity'; import { favoriteDeletion } from '../../../core/persist/deletions'; -import { clearDeletion, favoritesItem, recordDeletion } from '../../../core/persist/storage'; +import { favoritesItem, recordDeletion } from '../../../core/persist/storage'; /** Star a song: copy the history entry into the Favorites library (top of the * manual order). No-op if already favorited. */ @@ -15,8 +15,6 @@ export async function addFavorite(entry: HistoryEntry): Promise { { ...entry, favoritedAt: now, lastAccessedAt: now }, ...list, ]); - // Starred again after an unstar: the deletion record must not outlive it. - await clearDeletion(favoriteDeletion(songKey(entry.identity))); } /** Dated (`deletions.ts`, by song — every copy of it, whatever duration it diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 45c7ed4..e4df755 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -13,8 +13,8 @@ import { unpackBackup, } from '../persist/sync-blob'; import { - classifySyncError, clearSyncArea, + isRateLimited, onSyncAreaChanged, readSyncArea, syncErrorMessage, @@ -66,22 +66,27 @@ function isEmpty(backup: Backup): boolean { * * One routine, `#reconcile`, does everything: read the area, compare with * what this device last saw, merge, then write locally and/or remotely as - * needed. Local changes (storage events) and remote ones (sync-area events) - * both just ask for a reconcile. Applying a merge that changes local data - * ends in a panel reload — stores read storage once at start-up, same as the - * import flow — so that is deferred while a track is loaded and picked up on - * the next quiet moment, panel open, or "Sync now". + * needed. Local changes (storage events), remote ones (sync-area events), + * retries and the push debounce all just ask for a reconcile later — one + * timer, latest request wins — and the push spacing is enforced in `#push`. + * Applying a merge that changes local data ends in a panel reload — stores + * read storage once at start-up, same as the import flow — so that is + * deferred while a track is loaded and picked up on the next quiet moment, + * panel open, or "Sync now". * - * Runs only in the sidepanel: it is the sole writer of synced data, so there - * is nothing to observe while it's closed. A change the panel didn't manage - * to push before closing is remembered via `pendingPush`. + * Runs in every open panel document (there can be more than one: a Firefox + * window each, the local-player tab). They share `syncConfig` through its + * watch, so at worst two push the same content, which the spacing absorbs. + * A change the panel didn't manage to push before closing is remembered via + * `pendingPush`. */ class SyncStore { - enabled = $state(false); - lastSyncedAt = $state(0); - lastError = $state(null); + config = $state({ ...DEFAULT_SYNC_CONFIG }); + enabled = $derived(this.config.enabled); + lastSyncedAt = $derived(this.config.lastSyncedAt); + lastError = $derived(this.config.lastError); /** The last push left old songs or charts out to fit the quota. */ - trimmed = $state(false); + trimmed = $derived(this.config.trimmed); /** Bytes the area holds, by the browser's accounting. */ usedBytes = $state(0); /** Another device's changes are in, waiting for a moment without a track @@ -96,63 +101,52 @@ class SyncStore { this.usedBytes ? Math.max(1, Math.round((this.usedBytes / SYNC_QUOTA_BYTES) * 100)) : 0, ); - #config: SyncConfig = { ...DEFAULT_SYNC_CONFIG }; - /** Suppresses the echo of our own `syncConfigItem` write. */ - #writing = false; /** Suppresses local change events while a merge is being written. */ #applying = false; - #pushTimer: ReturnType | undefined; - #remoteTimer: ReturnType | undefined; - #retryTimer: ReturnType | undefined; + #timer: ReturnType | undefined; #lastPushAt = 0; #tornSince = 0; /** Serializes reconciles so they can't interleave. */ #queue: Promise = Promise.resolve(); async init() { - this.#config = await loadSyncConfig(); - this.#reflect(); + this.config = await loadSyncConfig(); + // Another panel document's bookkeeping, and the echo of our own writes + // (which is the value already held — harmless). syncConfigItem.watch((value) => { - if (this.#writing) return; - this.#config = value ?? { ...DEFAULT_SYNC_CONFIG }; - this.#reflect(); + this.config = value ?? { ...DEFAULT_SYNC_CONFIG }; }); browser.storage.local.onChanged.addListener((changes) => { - if (this.#applying || !this.#config.enabled) return; + if (this.#applying || !this.config.enabled) return; if (!Object.keys(changes).some((key) => SYNCED_KEY_RE.test(key))) return; - this.#onDataChanged(); + // Persisted immediately (not on the debounce) so a panel closed + // mid-burst still knows there is unpushed data next time it opens. + void this.#saveConfig({ pendingPush: true, lastChangedAt: Date.now() }); + this.#reconcileIn(PUSH_DEBOUNCE_MS); }); onSyncAreaChanged(() => { - if (!this.#config.enabled) return; - clearTimeout(this.#remoteTimer); - this.#remoteTimer = setTimeout(() => { - void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })); - }, REMOTE_DEBOUNCE_MS); + if (this.config.enabled) this.#reconcileIn(REMOTE_DEBOUNCE_MS); }); - if (this.#config.enabled) { - await this.#enqueue(() => this.#reconcile({ allowApply: true })); - } + if (this.config.enabled) await this.#enqueue(() => this.#reconcile(true)); setInterval(() => { - if (!this.#config.enabled) return; - void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })); + if (this.config.enabled) void this.#enqueue(() => this.#reconcile()); }, SAFETY_INTERVAL_MS); } async enable(): Promise { await this.#saveConfig({ enabled: true, lastError: null }); - await this.#enqueue(() => this.#reconcile({ allowApply: true })); + await this.#enqueue(() => this.#reconcile(true)); } async disable(): Promise { - clearTimeout(this.#pushTimer); - clearTimeout(this.#retryTimer); + clearTimeout(this.#timer); this.pendingApply = false; await this.#saveConfig({ enabled: false }); } /** Back up now and pull in the other devices' changes, reload included. */ async syncNow(): Promise { - await this.#enqueue(() => this.#reconcile({ allowApply: true })); + await this.#enqueue(() => this.#reconcile(true)); } /** @@ -161,23 +155,13 @@ class SyncStore { * their own data the next time they change something. */ async deleteRemote(): Promise { - clearTimeout(this.#pushTimer); - clearTimeout(this.#retryTimer); - this.pendingApply = false; + await this.disable(); await this.#enqueue(async () => { this.#syncing = true; try { await clearSyncArea(); this.usedBytes = 0; - await this.#saveConfig({ - enabled: false, - lastSyncedAt: 0, - lastRemoteHash: null, - lastLocalHash: null, - pendingPush: false, - lastError: null, - trimmed: false, - }); + await this.#saveConfig({ ...DEFAULT_SYNC_CONFIG, enabled: false }); } catch (err) { await this.#saveConfig({ lastError: syncErrorMessage(err) }); throw err; @@ -187,22 +171,9 @@ class SyncStore { }); } - #reflect() { - this.enabled = this.#config.enabled; - this.lastSyncedAt = this.#config.lastSyncedAt; - this.lastError = this.#config.lastError; - this.trimmed = this.#config.trimmed; - } - async #saveConfig(patch: Partial) { - this.#config = { ...this.#config, ...patch }; - this.#reflect(); - this.#writing = true; - try { - await syncConfigItem.setValue(this.#config); - } finally { - this.#writing = false; - } + this.config = { ...this.config, ...patch }; + await syncConfigItem.setValue($state.snapshot(this.config)); } #enqueue(op: () => Promise): Promise { @@ -211,28 +182,10 @@ class SyncStore { return result; } - #onDataChanged() { - // Persisted immediately (not on the debounce) so a panel closed mid-burst - // still knows there is unpushed data next time it opens. - void this.#saveConfig({ pendingPush: true, lastChangedAt: Date.now() }); - this.#schedulePush(PUSH_DEBOUNCE_MS); - } - - #schedulePush(delayMs: number) { - clearTimeout(this.#pushTimer); - const spacing = this.#lastPushAt + MIN_PUSH_SPACING_MS - Date.now(); - this.#pushTimer = setTimeout( - () => void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })), - Math.max(delayMs, spacing), - ); - } - - #retryIn(delayMs: number) { - clearTimeout(this.#retryTimer); - this.#retryTimer = setTimeout( - () => void this.#enqueue(() => this.#reconcile({ allowApply: session.media === null })), - delayMs, - ); + /** Reconcile after `delayMs`; a later request replaces an earlier one. */ + #reconcileIn(delayMs: number) { + clearTimeout(this.#timer); + this.#timer = setTimeout(() => void this.#enqueue(() => this.#reconcile()), delayMs); } /** @@ -241,9 +194,13 @@ class SyncStore { * - torn → wait for the rest to land, retry; * - unchanged since last look → push if this device changed something; * - otherwise merge the two copies, write the result wherever it differs. + * + * Applying (writing a merge locally) reloads the panel, so it waits for a + * moment without a track loaded unless `force` — "Sync now", enabling, + * opening the panel. */ - async #reconcile(opts: { allowApply: boolean }) { - if (!this.#config.enabled) return; + async #reconcile(force = false) { + if (!this.config.enabled) return; this.#syncing = true; let applied = false; try { @@ -251,12 +208,12 @@ class SyncStore { this.usedBytes = bytes; const local = await createBackup(); const localHash = await contentHash(local); - const localChanged = localHash !== this.#config.lastLocalHash; + const localChanged = localHash !== this.config.lastLocalHash; if (result.kind === 'torn') { if (!this.#tornSince) this.#tornSince = Date.now(); if (Date.now() - this.#tornSince < TORN_GIVE_UP_MS) { - this.#retryIn(TORN_RETRY_MS); + this.#reconcileIn(TORN_RETRY_MS); return; } // Nobody finished that write; ours replaces it. Whatever it carried @@ -268,22 +225,22 @@ class SyncStore { this.#tornSince = 0; if (result.kind === 'none') { - if (isEmpty(local) && !this.#config.pendingPush) return; + if (isEmpty(local) && !this.config.pendingPush) return; await this.#push(local, localHash); return; } const { meta, base64 } = result; - if (meta.h === this.#config.lastRemoteHash) { + if (meta.h === this.config.lastRemoteHash) { // Remote is what we last saw (our own echo included). - if (localChanged || this.#config.pendingPush) await this.#push(local, localHash); - else if (this.#config.lastError) await this.#saveConfig({ lastError: null }); + if (localChanged || this.config.pendingPush) await this.#push(local, localHash); + else if (this.config.lastError) await this.#saveConfig({ lastError: null }); return; } // Another device wrote since we last looked. const remote = await unpackBackup(base64); - const remoteWins = !localChanged || remote.exportedAt > this.#config.lastChangedAt; + const remoteWins = !localChanged || remote.exportedAt > this.config.lastChangedAt; const merged = mergeBackups(local, remote, remoteWins); const [mergedHash, remoteHash] = await Promise.all([ contentHash(merged), @@ -292,7 +249,7 @@ class SyncStore { const needApply = mergedHash !== localHash; const needPush = mergedHash !== remoteHash; - if (needApply && !opts.allowApply) { + if (needApply && !force && session.media !== null) { // A track is loaded; applying would reload the panel mid-practice. // Nothing is pushed either: a push now would carry only our side. this.pendingApply = true; @@ -301,11 +258,12 @@ class SyncStore { this.pendingApply = false; if (needApply) { // #applying stays set until the reload: nothing in between may - // schedule a push of what was just written. + // schedule a push of what was just written. The reload is owed from + // here on, even if the restore fails halfway. this.#applying = true; - clearTimeout(this.#pushTimer); - await restoreBackup(merged); + clearTimeout(this.#timer); applied = true; + await restoreBackup(merged); } if (needPush) { await this.#push(merged, mergedHash); @@ -320,19 +278,25 @@ class SyncStore { } } catch (err) { await this.#saveConfig({ lastError: syncErrorMessage(err) }); - if (classifySyncError(err) === 'rate') this.#retryIn(RATE_LIMIT_RETRY_MS); + if (isRateLimited(err)) this.#reconcileIn(RATE_LIMIT_RETRY_MS); } finally { this.#syncing = false; - // Local data changed under the stores (same situation as an import): - // the reload is owed whether or not the push after it went through. + // Local data changed under the stores (same situation as an import). if (applied) location.reload(); } } - /** Writes `backup` to the area, cut to the quota if it must be. `hash` is - * the content hash of this device's full data, so a trimmed push doesn't - * read as "local changed" on the next pass. */ + /** Writes `backup` to the area, cut to the quota if it must be — or, too + * soon after the last write, comes back for it later (the next reconcile + * reaches the same conclusion; `pendingPush` and the hashes are untouched + * until the write lands). `hash` is the content hash of this device's full + * data, so a trimmed push doesn't read as "local changed" next time. */ async #push(backup: Backup, hash: string) { + const wait = this.#lastPushAt + MIN_PUSH_SPACING_MS - Date.now(); + if (wait > 0) { + this.#reconcileIn(wait); + return; + } const fitted = await fitBackup(backup, BUDGET_CHARS, measure); const exportedAt = Date.now(); const packed = await packBackup( diff --git a/src/features/sync/persist/fit.ts b/src/features/sync/persist/fit.ts index 23020e1..1c56029 100644 --- a/src/features/sync/persist/fit.ts +++ b/src/features/sync/persist/fit.ts @@ -1,21 +1,22 @@ +import { songKey } from '../../../core/model/track-identity.ts'; import type { Backup } from '../../../core/persist/backup-codec.ts'; -import type { FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types'; +import type { HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types'; /** * Cuts a backup down to a byte budget — the browser's sync quota — by * dropping old things rather than capping counts. Nothing is trimmed while it - * fits; when it doesn't, tiers go in this order, each from its oldest end, - * and each cut no deeper than needed: + * fits; when it doesn't, the cuttable things form one ordered list and the + * shortest prefix of it that makes the rest fit is dropped: * * 1. songs that aren't favorited: their Recent row and track record, by * how recently they were played or edited (orphan records — a song no - * longer in Recent — sit in this tier by their own edit time); + * longer in Recent — sit here by their own edit time); * 2. chord charts, by when they were computed (they can be re-analyzed); * 3. favorites, with their Recent row and track record, by last access. * - * Settings, UI prefs and EQ presets are never cut. Songs are matched the way - * the library does (`isSameTrack`), so a record saved under a drifted - * duration still follows its favorite. + * Each group oldest first. Settings, UI prefs and EQ presets are never cut. + * Songs are matched the way the library does (`songKey`: URL + title), so a + * record saved under a drifted duration still follows its favorite. * * `measure` is injected (and may be async): the caller decides what "size" * means — encoded JSON length in tests, the gzip+base64 blob for sync — so @@ -38,92 +39,63 @@ export class LibraryTooLargeError extends Error { } } -/** How many of each tier's items (newest first) the built backup keeps. */ -interface Plan { - songs: number; - charts: number; - favorites: number; -} - -const TIERS: readonly (keyof Plan)[] = ['songs', 'charts', 'favorites']; - -/** The library matches rows by song, not key — see `isSameTrack`. */ -const songId = (identity: TrackIdentity) => `${identity.normalizedUrl}\n${identity.title}`; - -const byRecency = (a: { recency: number; key: string }, b: { recency: number; key: string }) => - b.recency - a.recency || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0); - -const hasChart = (track: TrackData) => !!track.chordChart && track.chordChart.segments.length > 0; +export const hasChart = (track: TrackData) => + !!track.chordChart && track.chordChart.segments.length > 0; -interface Tiers { - /** Non-favorited songs, newest first. */ - songs: { id: string; recency: number; key: string }[]; - /** Chart-bearing track records, newest chart first. */ - charts: { key: string; recency: number }[]; - /** Favorites' songs, most recently accessed first. */ - favorites: { id: string; recency: number; key: string }[]; +/** A song (by `songKey`, row + record) or a chart (by track key). */ +interface Cut { + kind: 'song' | 'chart'; + id: string; + recency: number; } -function collectTiers(backup: Backup): Tiers { - const favoriteIds = new Set(backup.favorites.map((f) => songId(f.identity))); - const songs = new Map(); - const touch = (identity: TrackIdentity, recency: number) => { - const id = songId(identity); - if (favoriteIds.has(id)) return; - const current = songs.get(id); - if (!current) songs.set(id, { id, recency, key: identity.key }); - else current.recency = Math.max(current.recency, recency); +/** Oldest first; equal dates by id, so equal input cuts the same way. */ +const oldestFirst = (a: Cut, b: Cut) => + a.recency - b.recency || (a.id < b.id ? 1 : a.id > b.id ? -1 : 0); + +/** Everything that may go, in the order it goes. */ +function collectCuts(backup: Backup): Cut[] { + const favorites = new Set(backup.favorites.map((f) => songKey(f.identity))); + const songs = new Map(); + const touch = (identity: TrackIdentity, at: number) => { + const id = songKey(identity); + if (!favorites.has(id)) songs.set(id, Math.max(songs.get(id) ?? 0, at)); }; for (const entry of backup.history) touch(entry.identity, entry.updatedAt ?? 0); for (const track of backup.tracks) touch(track.identity, track.updatedAt ?? 0); - const charts = backup.tracks - .filter(hasChart) - .map((t) => ({ key: t.identity.key, recency: t.chordChart!.computedAt ?? 0 })) - .sort(byRecency); - - const favorites = new Map(); + const accessed = new Map(); for (const f of backup.favorites) { - const id = songId(f.identity); - const recency = f.lastAccessedAt ?? f.updatedAt ?? 0; - const current = favorites.get(id); - if (!current) favorites.set(id, { id, recency, key: f.identity.key }); - else current.recency = Math.max(current.recency, recency); + const id = songKey(f.identity); + const at = f.lastAccessedAt ?? f.updatedAt ?? 0; + accessed.set(id, Math.max(accessed.get(id) ?? 0, at)); } - return { - songs: [...songs.values()].sort(byRecency), - charts, - favorites: [...favorites.values()].sort(byRecency), - }; + const asCuts = (kind: Cut['kind'], m: Map) => + [...m].map(([id, recency]) => ({ kind, id, recency })).sort(oldestFirst); + const charts = backup.tracks + .filter(hasChart) + .map((t) => ({ kind: 'chart' as const, id: t.identity.key, recency: t.chordChart!.computedAt ?? 0 })) + .sort(oldestFirst); + return [...asCuts('song', songs), ...charts, ...asCuts('song', accessed)]; } -function build(backup: Backup, tiers: Tiers, plan: Plan): Backup { - const kept = new Set(); - for (const s of tiers.songs.slice(0, plan.songs)) kept.add(s.id); - for (const f of tiers.favorites.slice(0, plan.favorites)) kept.add(f.id); - const keptCharts = new Set(tiers.charts.slice(0, plan.charts).map((c) => c.key)); - - const keepRow = (row: HistoryEntry | FavoriteEntry) => kept.has(songId(row.identity)); - const tracks: TrackData[] = []; - for (const track of backup.tracks) { - if (!kept.has(songId(track.identity))) continue; - if (hasChart(track) && !keptCharts.has(track.identity.key)) { - tracks.push({ ...track, chordChart: null }); - } else { - tracks.push(track); - } - } +function apply(backup: Backup, cuts: Cut[]): Backup { + const songs = new Set(cuts.filter((c) => c.kind === 'song').map((c) => c.id)); + const charts = new Set(cuts.filter((c) => c.kind === 'chart').map((c) => c.id)); + const keep = (row: HistoryEntry | TrackData) => !songs.has(songKey(row.identity)); return { ...backup, - history: backup.history.filter(keepRow), - favorites: backup.favorites.filter(keepRow), - tracks, + history: backup.history.filter(keep), + favorites: backup.favorites.filter(keep), + tracks: backup.tracks + .filter(keep) + .map((t) => (charts.has(t.identity.key) ? { ...t, chordChart: null } : t)), }; } /** - * The largest backup, in the tier order above, whose `measure` is at most + * The largest backup, in the order above, whose `measure` is at most * `budget`. Deterministic for equal input. Throws `LibraryTooLargeError` * when nothing cuttable is left and it still doesn't fit. */ @@ -132,40 +104,26 @@ export async function fitBackup( budget: number, measure: (backup: Backup) => number | Promise, ): Promise { - const tiers = collectTiers(backup); - const full: Plan = { - songs: tiers.songs.length, - charts: tiers.charts.length, - favorites: tiers.favorites.length, - }; const size = await measure(backup); if (size <= budget) return { backup, trimmed: false, size }; - const plan = { ...full }; - const sizeOf = (p: Plan) => Promise.resolve(measure(build(backup, tiers, p))); - for (const tier of TIERS) { - // Earlier tiers are already empty. Does emptying this one fit? - const empty = await sizeOf({ ...plan, [tier]: 0 }); - if (empty > budget) { - plan[tier] = 0; - continue; - } - // Yes — keep as many of its newest items as still fit. - let lo = 0; // known to fit - let hi = plan[tier]; // known not to fit (full plan didn't) - let loSize = empty; - while (hi - lo > 1) { - const mid = (lo + hi) >> 1; - const s = await sizeOf({ ...plan, [tier]: mid }); - if (s <= budget) { - lo = mid; - loSize = s; - } else { - hi = mid; - } + // Size only shrinks as more of the list goes, so binary-search the + // shortest prefix that fits: `lo` is known not to, `hi` is known to. + const cuts = collectCuts(backup); + const sizeOf = (n: number) => measure(apply(backup, cuts.slice(0, n))); + let lo = 0; + let hi = cuts.length; + let hiSize = await sizeOf(hi); + if (hiSize > budget) throw new LibraryTooLargeError(); + while (hi - lo > 1) { + const mid = (lo + hi) >> 1; + const s = await sizeOf(mid); + if (s <= budget) { + hi = mid; + hiSize = s; + } else { + lo = mid; } - plan[tier] = lo; - return { backup: build(backup, tiers, plan), trimmed: true, size: loSize }; } - throw new LibraryTooLargeError(); + return { backup: apply(backup, cuts.slice(0, hi)), trimmed: true, size: hiSize }; } diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index b54facc..f062d81 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -1,20 +1,17 @@ import { HISTORY_LIMIT } from '../../../core/model/defaults.ts'; +import { songKey } from '../../../core/model/track-identity.ts'; import type { Backup } from '../../../core/persist/backup-codec.ts'; import { - favoriteDeleted, - historyDeleted, + deletedSince, + favoriteDeletion, + HISTORY_CLEARED, + historyDeletion, mergeDeletions, - presetDeleted, + presetDeletion, pruneDeletions, } from '../../../core/persist/deletions.ts'; -import { songKey } from '../../../core/model/track-identity.ts'; -import type { - EqPreset, - FavoriteEntry, - HistoryEntry, - TrackData, - TrackIdentity, -} from '../../../core/model/types'; +import { hasChart } from './fit.ts'; +import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; /** * Two devices' libraries into one. A union, item by item, so a copy another @@ -23,104 +20,44 @@ import type { * * - Recent rows and favorites: matched by song (URL + title, like the * library itself), the more recently updated one wins; a dated deletion - * (`deletions.ts`) beats any copy it postdates. + * (`deletions.ts`) beats any copy it postdates. A favorite's last access + * is the later of the two. * - Track records: matched by key, the more recently edited wins outright — * an emptied record is still a record, so clearing markers sticks. A * winner without a chart adopts the other's: absent may mean "trimmed", * and a chart is only ever replaced by re-analysis. * - Settings and UI prefs: the newer device's, as a whole. EQ presets: union - * by name, the later save of a shared name, deletions honoured, the newer - * device's order first. + * by name, the later save of a shared name, deletions honoured. * * `remoteWins` breaks ties and picks the wholesale sections: true when the - * remote copy was written after this device's last local change. Pure; - * `node --test`. + * remote copy was written after this device's last local change. The winning + * side's order comes first in every list. Pure; `node --test`. */ -const songId = (identity: TrackIdentity) => `${identity.normalizedUrl}\n${identity.title}`; - -const byUpdatedDesc = (a: { updatedAt: number }, b: { updatedAt: number }) => - (b.updatedAt ?? 0) - (a.updatedAt ?? 0); - -function mergeHistory( - first: HistoryEntry[], - second: HistoryEntry[], - deletions: Backup['deletions'], -): HistoryEntry[] { - const bySong = new Map(); - for (const entry of [...first, ...second]) { - if (historyDeleted(deletions, songKey(entry.identity), entry.updatedAt ?? 0)) continue; - const id = songId(entry.identity); - const current = bySong.get(id); - if (!current || (entry.updatedAt ?? 0) > (current.updatedAt ?? 0)) bySong.set(id, entry); - } - return [...bySong.values()].sort(byUpdatedDesc).slice(0, HISTORY_LIMIT); -} - -function mergeFavorites( - first: FavoriteEntry[], - second: FavoriteEntry[], - deletions: Backup['deletions'], -): FavoriteEntry[] { - // Insertion order = the winner's manual order, then the other side's extras. - const bySong = new Map(); - for (const entry of [...first, ...second]) { - const since = Math.max(entry.favoritedAt ?? 0, entry.updatedAt ?? 0); - if (favoriteDeleted(deletions, songKey(entry.identity), since)) continue; - const id = songId(entry.identity); - const current = bySong.get(id); - if (!current) { - bySong.set(id, entry); - } else if ((entry.updatedAt ?? 0) > (current.updatedAt ?? 0)) { - bySong.set(id, { - ...entry, - lastAccessedAt: Math.max(entry.lastAccessedAt ?? 0, current.lastAccessedAt ?? 0), - }); - } else if ((entry.lastAccessedAt ?? 0) > (current.lastAccessedAt ?? 0)) { - bySong.set(id, { ...current, lastAccessedAt: entry.lastAccessedAt }); - } - } - return [...bySong.values()]; -} - -const hasChart = (track: TrackData) => !!track.chordChart && track.chordChart.segments.length > 0; +const at = (item: { updatedAt?: number }) => item.updatedAt ?? 0; -function mergeTracks(first: TrackData[], second: TrackData[]): TrackData[] { - const byKey = new Map(); - for (const track of [...first, ...second]) { - const current = byKey.get(track.identity.key); - if (!current) { - byKey.set(track.identity.key, track); - continue; - } - const [winner, loser] = - (track.updatedAt ?? 0) > (current.updatedAt ?? 0) ? [track, current] : [current, track]; - byKey.set( - track.identity.key, - !hasChart(winner) && hasChart(loser) ? { ...winner, chordChart: loser.chordChart } : winner, - ); +/** Union by `id`: of two copies the later `updatedAt` wins (the first list's + * on a tie), `dead` ones are skipped, and `resolve` can fold the loser's + * fields into the winner. First list's order, then the second's extras. */ +function unionNewest( + first: T[], + second: T[], + id: (item: T) => string, + dead: (item: T) => boolean = () => false, + resolve: (winner: T, loser: T) => T = (winner) => winner, +): T[] { + const byId = new Map(); + for (const item of [...first, ...second]) { + if (dead(item)) continue; + const key = id(item); + const current = byId.get(key); + if (!current) byId.set(key, item); + else byId.set(key, at(item) > at(current) ? resolve(item, current) : resolve(current, item)); } - return [...byKey.values()]; + return [...byId.values()]; } -/** Union by name; a dated deletion beats any copy it postdates, and a shared - * name goes to the later save (the winner side on a tie, or when neither - * carries a save time). */ -function mergeEqPresets( - first: EqPreset[], - second: EqPreset[], - deletions: Backup['deletions'], -): EqPreset[] { - const byName = new Map(); - for (const preset of [...first, ...second]) { - if (presetDeleted(deletions, preset.name, preset.updatedAt ?? 0)) continue; - const current = byName.get(preset.name); - if (!current || (preset.updatedAt ?? 0) > (current.updatedAt ?? 0)) { - byName.set(preset.name, preset); - } - } - return [...byName.values()]; -} +const song = (entry: HistoryEntry) => songKey(entry.identity); export function mergeBackups( local: Backup, @@ -130,15 +67,38 @@ export function mergeBackups( ): Backup { const [winner, loser] = remoteWins ? [remote, local] : [local, remote]; const deletions = pruneDeletions(mergeDeletions(local.deletions ?? {}, remote.deletions ?? {}), now); + const history = unionNewest(winner.history, loser.history, song, (e) => + deletedSince(deletions, at(e), historyDeletion(song(e)), HISTORY_CLEARED), + ); return { ...local, exportedAt: Math.max(local.exportedAt ?? 0, remote.exportedAt ?? 0), settings: winner.settings, uiPrefs: winner.uiPrefs, - eqPresets: mergeEqPresets(winner.eqPresets, loser.eqPresets, deletions), - history: mergeHistory(winner.history, loser.history, deletions), - favorites: mergeFavorites(winner.favorites, loser.favorites, deletions), - tracks: mergeTracks(winner.tracks, loser.tracks), + eqPresets: unionNewest( + winner.eqPresets, + loser.eqPresets, + (p) => p.name, + (p) => deletedSince(deletions, at(p), presetDeletion(p.name)), + ), + history: history.sort((a, b) => at(b) - at(a)).slice(0, HISTORY_LIMIT), + favorites: unionNewest( + winner.favorites, + loser.favorites, + song, + (f) => deletedSince(deletions, Math.max(f.favoritedAt ?? 0, at(f)), favoriteDeletion(song(f))), + (w, l): FavoriteEntry => ({ + ...w, + lastAccessedAt: Math.max(w.lastAccessedAt ?? 0, l.lastAccessedAt ?? 0), + }), + ), + tracks: unionNewest( + winner.tracks, + loser.tracks, + (t) => t.identity.key, + undefined, + (w, l) => (!hasChart(w) && hasChart(l) ? { ...w, chordChart: l.chordChart } : w), + ), deletions, }; } diff --git a/src/features/sync/persist/sync-area.ts b/src/features/sync/persist/sync-area.ts index 454506e..fbc3749 100644 --- a/src/features/sync/persist/sync-area.ts +++ b/src/features/sync/persist/sync-area.ts @@ -1,13 +1,4 @@ -import { LibraryTooLargeError } from './fit'; -import { - chunkKey, - isBlobKey, - itemsBytes, - NewerVersionError, - readBlob, - type BlobMeta, - type ReadResult, -} from './sync-blob'; +import { isBlobKey, itemsBytes, readBlob, type BlobMeta, type ReadResult } from './sync-blob'; /** * The `browser.storage.sync` side of the blob layout in `sync-blob.ts`: @@ -53,33 +44,25 @@ export async function clearSyncArea(): Promise { * `storage.sync.onChanged`, which older Firefox lacks. */ export function onSyncAreaChanged(listener: () => void): void { browser.storage.onChanged.addListener((changes, area) => { - if (area !== 'sync') return; - if (Object.keys(changes).some((key) => isBlobKey(key) || key === chunkKey(0))) listener(); + if (area === 'sync' && Object.keys(changes).some(isBlobKey)) listener(); }); } -export type SyncErrorKind = 'rate' | 'quota' | 'newer' | 'other'; +const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err)); -/** The browser reports quota trouble as thrown strings/errors with these +/** The browser reports write metering as thrown strings/errors with these * names in the message; there is no error code to switch on. */ -export function classifySyncError(err: unknown): SyncErrorKind { - if (err instanceof NewerVersionError) return 'newer'; - if (err instanceof LibraryTooLargeError) return 'quota'; - const text = err instanceof Error ? err.message : String(err); - if (/MAX_WRITE_OPERATIONS|MAX_SUSTAINED_WRITE|MAX_ITEMS/i.test(text)) return 'rate'; - if (/QUOTA_BYTES|QuotaExceeded|quota/i.test(text)) return 'quota'; - return 'other'; +export function isRateLimited(err: unknown): boolean { + return /MAX_WRITE_OPERATIONS|MAX_SUSTAINED_WRITE|MAX_ITEMS/i.test(errorText(err)); } +/** Our own errors (`NewerVersionError`, `LibraryTooLargeError`) already read + * well; the browser's quota and metering ones are translated. */ export function syncErrorMessage(err: unknown): string { - switch (classifySyncError(err)) { - case 'rate': - return 'The browser is rate-limiting sync writes — retrying in a minute.'; - case 'quota': - return "Your library is too large for the browser's sync storage."; - case 'newer': - return 'Synced data was written by a newer version of Note by Note.'; - default: - return err instanceof Error && err.message ? err.message : 'Sync failed.'; + const text = errorText(err); + if (isRateLimited(err)) return 'The browser is rate-limiting sync writes — retrying in a minute.'; + if (/QUOTA_BYTES|QuotaExceeded|quota/i.test(text)) { + return "Your library is too large for the browser's sync storage."; } + return text || 'Sync failed.'; } diff --git a/src/features/sync/persist/sync-blob.test.ts b/src/features/sync/persist/sync-blob.test.ts index f4de3bf..b883f68 100644 --- a/src/features/sync/persist/sync-blob.test.ts +++ b/src/features/sync/persist/sync-blob.test.ts @@ -103,7 +103,10 @@ test('chunking and keys', () => { test('pack → items → read → unpack round-trips within the per-item and total limits', async () => { const b = library(40); - const { items, meta, chars } = await packBackup(encodeBackup(b), '1.0.3'); + const { items, meta } = await packBackup(encodeBackup(b), '1.0.3'); + const chars = Object.entries(items) + .filter(([key]) => key !== META_KEY) + .reduce((n, [, chunk]) => n + (chunk as string).length, 0); assert.equal(meta.n, Math.ceil(chars / CHUNK_CHARS)); assert.equal(meta.at, T0); for (const [key, value] of Object.entries(items)) { diff --git a/src/features/sync/persist/sync-blob.ts b/src/features/sync/persist/sync-blob.ts index 85a5e39..fb83209 100644 --- a/src/features/sync/persist/sync-blob.ts +++ b/src/features/sync/persist/sync-blob.ts @@ -129,8 +129,6 @@ export interface PackedBlob { /** The items to write, meta included. */ items: Record; meta: BlobMeta; - /** base64 length, for the usage readout. */ - chars: number; } export async function packBackup(compact: CompactBackup, app: string): Promise { @@ -148,7 +146,7 @@ export async function packBackup(compact: CompactBackup, app: string): Promise

= { [META_KEY]: meta }; chunks.forEach((chunk, i) => (items[chunkKey(i)] = chunk)); - return { items, meta, chars: base64.length }; + return { items, meta }; } /** What the area holds, classified. Only `nbn.*` keys are looked at. */ diff --git a/src/features/sync/persist/sync-config.ts b/src/features/sync/persist/sync-config.ts index aff979c..231e2fc 100644 --- a/src/features/sync/persist/sync-config.ts +++ b/src/features/sync/persist/sync-config.ts @@ -39,24 +39,17 @@ export const syncConfigItem = storage.defineItem('local:syncConfig', fallback: DEFAULT_SYNC_CONFIG, }); -/** Reads the record, migrating one written by the server-era builds (which - * carried `syncId`/`consentedId` and hashes over a different shape): the - * on/off choice is kept, the bookkeeping starts over so the first reconcile - * merges rather than trusting stale hashes. */ +/** The stored record over the defaults, known fields only. A record from the + * server-era builds (`syncId`, hashes over another shape) needs no more than + * that: its leftovers are dropped here and gone on the next write, and the + * hashes it lacks read as null, which sends the first reconcile down the + * merge path rather than trusting anything stale. */ export async function loadSyncConfig(): Promise { - const raw = (await syncConfigItem.getValue()) as Partial & { - syncId?: unknown; - consentedId?: unknown; - lastSyncedHash?: unknown; - }; - const legacy = 'syncId' in raw || 'consentedId' in raw || 'lastSyncedHash' in raw; - if (!legacy) return { ...DEFAULT_SYNC_CONFIG, ...raw }; - const config: SyncConfig = { - ...DEFAULT_SYNC_CONFIG, - enabled: raw.enabled ?? true, - lastChangedAt: raw.lastChangedAt ?? 0, - pendingPush: true, - }; - await syncConfigItem.setValue(config); - return config; + const raw = (await syncConfigItem.getValue()) as Partial; + const config: Record = { ...DEFAULT_SYNC_CONFIG }; + for (const key of Object.keys(DEFAULT_SYNC_CONFIG)) { + const value = raw[key as keyof SyncConfig]; + if (value !== undefined) config[key] = value; + } + return config as unknown as SyncConfig; } From d048410d216157e2ccac573e281c319d6e33e593 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 20:05:00 +0200 Subject: [PATCH 06/26] refactor: update timestamp handling to use milliseconds instead of seconds --- CLAUDE.md | 2 +- src/core/persist/backup-codec.test.ts | 20 ++++----- src/core/persist/backup-codec.ts | 57 ++++++++++++++------------ src/features/sync/panel/sync.svelte.ts | 35 ++++++++++------ src/features/sync/persist/sync-blob.ts | 2 +- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f5ad210..6fb6fe9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. -- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped, timestamps in seconds. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. +- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped. Timestamps stay in ms — they decide merges, and a delete and a re-add within one second must not collide. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. - Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, `e:`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, TTL 30 d, cap 200) so a merge can't resurrect them. Songs are named by `songKey` (URL + title, **no duration** — the same identity the merge matches on, so a copy saved under a drifted duration is covered too); presets carry an optional `updatedAt` so a later save beats the deletion; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index f7d8951..88e3438 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -264,11 +264,11 @@ test('eq presets round-trip as tuples, with or without a save time', () => { }); assert.deepEqual(encodeBackup(b).eq, [ ['Mine', [1, 2.5, 3, 4, 5, 6, 7, 8, 9, 0]], - ['Stamped', [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 1757112346], + ['Stamped', [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 1_757_112_345_678], ]); const back = roundTrip(b).eqPresets; assert.deepEqual(back[0], b.eqPresets[0]); - assert.deepEqual(back[1], { ...b.eqPresets[1], updatedAt: 1757112346000 }); + assert.deepEqual(back[1], { ...b.eqPresets[1], updatedAt: 1_757_112_345_678 }); }); // --------------------------------------------------------------------------- @@ -278,12 +278,12 @@ test('recent: a plain YouTube row is an index and a timestamp', () => { const b = backup({ history: [entry(ytSong)] }); const enc = encodeBackup(b); assert.deepEqual(enc.songs, [['yt:741FSo7Xb40', 'Symphony of Destruction', 230]]); - assert.deepEqual(enc.h, [{ i: 0, at: 1757112346 }]); + assert.deepEqual(enc.h, [{ i: 0, at: 1_757_112_345_678 }]); const back = roundTrip(b).history[0]; assert.deepEqual(back.identity, ytSong); assert.equal(back.pageUrl, YT_HREF); assert.equal(back.thumbnailUrl, YT_THUMB); - assert.equal(back.updatedAt, 1757112346000); + assert.equal(back.updatedAt, 1_757_112_345_678); assert.equal(back.createdAt, back.updatedAt); assert.deepEqual(back.params, params()); }); @@ -431,7 +431,7 @@ test('chart: times land within half a centisecond, gaps and key survive', () => assert.deepEqual(back.key, { tonic: 'A', mode: 'minor', confidence: 0.812 }); assert.equal(back.coverage, 0.988); assert.ok(Math.abs(back.analyzedTo - c.analyzedTo) <= 0.005); - assert.equal(back.computedAt, 1757112346000); + assert.equal(back.computedAt, 1_757_112_345_678); }); test('chart: contiguous segments need no gap array; unsorted input is sorted', () => { @@ -517,20 +517,20 @@ test('encode is deterministic regardless of track enumeration order', () => { assert.deepEqual(encodeBackup(shuffled), encodeBackup(b)); }); -test('deletion records travel in seconds and are omitted when empty', () => { +test('deletion records travel in ms and are omitted when empty', () => { assert.equal('del' in encodeBackup(backup()), false); const b = backup({ deletions: { 'h:abc:230': 1_757_112_345_678, 'h:*': 1_757_000_000_000 } }); const enc = encodeBackup(b); - assert.deepEqual(enc.del, { 'h:*': 1757000000, 'h:abc:230': 1757112346 }); - assert.deepEqual(roundTrip(b).deletions, { 'h:*': 1757000000000, 'h:abc:230': 1757112346000 }); + assert.deepEqual(enc.del, { 'h:*': 1_757_000_000_000, 'h:abc:230': 1_757_112_345_678 }); + assert.deepEqual(roundTrip(b).deletions, b.deletions); const v1 = JSON.parse(JSON.stringify(backup())); delete v1.deletions; assert.deepEqual(parseBackupJson(v1).deletions, {}); }); -test('exportedAt is kept to the second; appVersion is dropped', () => { +test('exportedAt is kept; appVersion is dropped', () => { const back = roundTrip(backup()); - assert.equal(back.exportedAt, 1_757_200_000_000); + assert.equal(back.exportedAt, 1_757_200_000_123); assert.equal(back.appVersion, ''); assert.equal(back.version, COMPACT_VERSION); }); diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 65f5397..112e9bb 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -118,7 +118,7 @@ export type CompactSong = [string, string, number] | [string, string, number, st export interface CompactEntry { /** Index into `songs`. */ i: number; - /** updatedAt, seconds. */ + /** updatedAt, ms. */ at: number; p?: CompactParams; /** pageUrl, when not the song's plain page. */ @@ -128,13 +128,13 @@ export interface CompactEntry { } export interface CompactFavorite extends CompactEntry { - /** favoritedAt, seconds. */ + /** favoritedAt, ms. */ fa: number; - /** lastAccessedAt, seconds. */ + /** lastAccessedAt, ms. */ la: number; } -/** `[name, gains, updatedAt_s?]`. */ +/** `[name, gains, updatedAt?]`. */ export type CompactEqPreset = [string, number[]] | [string, number[], number]; /** `[t_ms, label?]` — label omitted when empty. */ @@ -167,13 +167,13 @@ export interface CompactChart { cov: number; a0: number; a1: number; - /** computedAt, seconds. */ + /** computedAt, ms. */ c: number; } export interface CompactTrack { i: number; - /** updatedAt, seconds. */ + /** updatedAt, ms. */ at: number; m?: CompactMarker[]; s?: CompactSnippet[]; @@ -189,19 +189,19 @@ export interface CompactTrack { export interface CompactBackup { format: typeof BACKUP_FORMAT; version: typeof COMPACT_VERSION; - /** exportedAt, seconds. */ + /** exportedAt, ms. */ at: number; /** Settings that differ from the defaults (`lastUsedParams` as `lp`). */ s: Record; /** UI prefs that differ from the defaults. */ u: Record; - /** `[name, gains, updatedAt_s?]` per saved EQ preset. */ + /** `[name, gains, updatedAt?]` per saved EQ preset. */ eq: CompactEqPreset[]; songs: CompactSong[]; h: CompactEntry[]; f: CompactFavorite[]; t: CompactTrack[]; - /** Deletion records, dates in seconds; omitted when there are none. */ + /** Deletion records (ms); omitted when there are none. */ del?: Record; } @@ -224,7 +224,10 @@ const round2 = roundTo(2); const round3 = roundTo(3); const millis = (seconds: number) => Math.round(seconds * 1000); const centis = (seconds: number) => Math.round(seconds * 100); -const secs = (ms: number) => Math.round((Number.isFinite(ms) ? ms : 0) / 1000); +/** Timestamps stay in milliseconds: they decide merges (a deletion against a + * re-add, the newer of two edits), and rounding would let two actions within + * the same second read as one. */ +const stamp = (ms: number) => (Number.isFinite(ms) ? Math.round(ms) : 0); function num(value: unknown, section: string): number { if (typeof value !== 'number' || !Number.isFinite(value)) throw damaged(section); @@ -482,7 +485,7 @@ function songAt(songs: TrackIdentity[], index: unknown, section: string): TrackI // Recent / Favorites function encodeEntry(entry: HistoryEntry, songs: SongTable): CompactEntry { - const out: CompactEntry = { i: songs.add(entry.identity), at: secs(entry.updatedAt) }; + const out: CompactEntry = { i: songs.add(entry.identity), at: stamp(entry.updatedAt) }; const params = encodeParams(entry.params ?? DEFAULT_PARAMS); if (params) out.p = params; const pageUrl = entry.pageUrl ?? ''; @@ -496,7 +499,7 @@ function encodeEntry(entry: HistoryEntry, songs: SongTable): CompactEntry { function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): HistoryEntry { const c = rec(raw, section); const identity = songAt(songs, c.i, section); - const updatedAt = num(c.at, section) * 1000; + const updatedAt = num(c.at, section); const pageUrl = c.url === undefined ? defaultPageUrl(identity.normalizedUrl) : str(c.url, section); const thumbnailUrl = c.th === undefined ? youtubeThumbnailUrl(pageUrl) : str(c.th, section); const entry: HistoryEntry = { @@ -513,8 +516,8 @@ function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): His function encodeFavorite(entry: FavoriteEntry, songs: SongTable): CompactFavorite { return { ...encodeEntry(entry, songs), - fa: secs(entry.favoritedAt), - la: secs(entry.lastAccessedAt), + fa: stamp(entry.favoritedAt), + la: stamp(entry.lastAccessedAt), }; } @@ -522,8 +525,8 @@ function decodeFavorite(raw: unknown, songs: TrackIdentity[]): FavoriteEntry { const c = rec(raw, 'favorites'); return { ...decodeEntry(c, songs, 'favorites'), - favoritedAt: num(c.fa, 'favorites') * 1000, - lastAccessedAt: num(c.la, 'favorites') * 1000, + favoritedAt: num(c.fa, 'favorites'), + lastAccessedAt: num(c.la, 'favorites'), }; } @@ -637,7 +640,7 @@ export function encodeChart(chart: ChordChart): CompactChart { cov: round3(chart.coverage ?? 0), a0: centis(chart.analyzedFrom ?? 0), a1: centis(chart.analyzedTo ?? 0), - c: secs(chart.computedAt), + c: stamp(chart.computedAt), }; if (anyGap) out.g = g; if (chart.key) { @@ -679,12 +682,12 @@ export function decodeChart(raw: unknown): ChordChart { coverage: num(c.cov, 'tracks'), analyzedFrom: num(c.a0, 'tracks') / 100, analyzedTo: num(c.a1, 'tracks') / 100, - computedAt: num(c.c, 'tracks') * 1000, + computedAt: num(c.c, 'tracks'), }; } function encodeTrack(track: TrackData, songs: SongTable): CompactTrack { - const out: CompactTrack = { i: songs.add(track.identity), at: secs(track.updatedAt) }; + const out: CompactTrack = { i: songs.add(track.identity), at: stamp(track.updatedAt) }; if (track.markers?.length) out.m = track.markers.map(encodeMarker); if (track.snippets?.length) out.s = track.snippets.map(encodeSnippet); if (track.sequenceLoop) out.L = 1; @@ -705,7 +708,7 @@ function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { sequenceLoop: c.L !== undefined, sequenceCountIn: c.C !== undefined, chordChart: c.ch === undefined ? null : decodeChart(c.ch), - updatedAt: num(c.at, 'tracks') * 1000, + updatedAt: num(c.at, 'tracks'), }; if (c.ce !== undefined) track.chordsEnabled = num(c.ce, 'tracks') === 1; return track; @@ -714,11 +717,11 @@ function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { // --------------------------------------------------------------------------- // Whole backup -/** `[name, gains, updatedAt_s?]`. */ +/** `[name, gains, updatedAt?]`. */ function encodeEqPreset(preset: EqPreset): CompactEqPreset { const name = preset.name ?? ''; const gains = (preset.gains ?? []).map(round2); - return preset.updatedAt ? [name, gains, secs(preset.updatedAt)] : [name, gains]; + return preset.updatedAt ? [name, gains, stamp(preset.updatedAt)] : [name, gains]; } function decodeEqPreset(raw: unknown): EqPreset { @@ -728,7 +731,7 @@ function decodeEqPreset(raw: unknown): EqPreset { name: str(r[0], 'eqPresets'), gains: arr(r[1], 'eqPresets').map((g) => num(g, 'eqPresets')), }; - if (r.length === 3) preset.updatedAt = num(r[2], 'eqPresets') * 1000; + if (r.length === 3) preset.updatedAt = num(r[2], 'eqPresets'); return preset; } @@ -746,7 +749,7 @@ export function encodeBackup(backup: Backup): CompactBackup { const out: CompactBackup = { format: BACKUP_FORMAT, version: COMPACT_VERSION, - at: secs(backup.exportedAt), + at: stamp(backup.exportedAt), s: encodeSettings(backup.settings), u: encodeUiPrefs(backup.uiPrefs), eq: backup.eqPresets.map(encodeEqPreset), @@ -758,7 +761,7 @@ export function encodeBackup(backup: Backup): CompactBackup { const deletions = Object.entries(normalizeDeletions(backup.deletions)).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0, ); - if (deletions.length) out.del = Object.fromEntries(deletions.map(([k, when]) => [k, secs(when)])); + if (deletions.length) out.del = Object.fromEntries(deletions.map(([k, when]) => [k, stamp(when)])); return out; } @@ -772,7 +775,7 @@ export function decodeBackup(raw: unknown): Backup { return { format: BACKUP_FORMAT, version: COMPACT_VERSION, - exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at * 1000 : 0, + exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at : 0, appVersion: '', settings: decodeSettings(raw.s), uiPrefs: decodeUiPrefs(raw.u), @@ -781,7 +784,7 @@ export function decodeBackup(raw: unknown): Backup { eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), deletions: Object.fromEntries( - Object.entries(normalizeDeletions(raw.del)).map(([k, when]) => [k, when * 1000]), + Object.entries(normalizeDeletions(raw.del)).map(([k, when]) => [k, when]), ), }; } diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index e4df755..2f1e9b9 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -1,7 +1,7 @@ import { encodeBackup } from '../../../core/persist/backup-codec'; import { createBackup, restoreBackup, type Backup } from '../../../core/persist/backup'; import { session } from '../../../core/state/session.svelte'; -import { fitBackup } from '../persist/fit'; +import { fitBackup, type FitResult } from '../persist/fit'; import { contentHash } from '../persist/hash'; import { mergeBackups } from '../persist/merge'; import { @@ -47,6 +47,7 @@ const SAFETY_INTERVAL_MS = 5 * 60_000; const SYNCED_KEY_RE = /^(settings|uiPrefs|history|favorites|eqPresets|deletions|track:)/; const measure = (backup: Backup) => packedChars(encodeBackup(backup)); +const fit = (backup: Backup) => fitBackup(backup, BUDGET_CHARS, measure); function isEmpty(backup: Backup): boolean { return ( @@ -219,21 +220,21 @@ class SyncStore { // Nobody finished that write; ours replaces it. Whatever it carried // comes back merged when its writer reconciles against ours. this.#tornSince = 0; - await this.#push(local, localHash); + await this.#push(await fit(local), localHash); return; } this.#tornSince = 0; if (result.kind === 'none') { if (isEmpty(local) && !this.config.pendingPush) return; - await this.#push(local, localHash); + await this.#push(await fit(local), localHash); return; } const { meta, base64 } = result; if (meta.h === this.config.lastRemoteHash) { // Remote is what we last saw (our own echo included). - if (localChanged || this.config.pendingPush) await this.#push(local, localHash); + if (localChanged || this.config.pendingPush) await this.#push(await fit(local), localHash); else if (this.config.lastError) await this.#saveConfig({ lastError: null }); return; } @@ -242,12 +243,20 @@ class SyncStore { const remote = await unpackBackup(base64); const remoteWins = !localChanged || remote.exportedAt > this.config.lastChangedAt; const merged = mergeBackups(local, remote, remoteWins); + const fitted = await fit(merged); const [mergedHash, remoteHash] = await Promise.all([ contentHash(merged), contentHash(remote), ]); const needApply = mergedHash !== localHash; - const needPush = mergedHash !== remoteHash; + // Push our own edits, or a full copy the remote lacks. Once the merge + // is over the quota only our edits count: two devices holding + // different old songs would otherwise cut the copy differently and + // re-upload each other's cut forever. + const needPush = + localChanged || + this.config.pendingPush || + (!fitted.trimmed && mergedHash !== remoteHash); if (needApply && !force && session.media !== null) { // A track is loaded; applying would reload the panel mid-practice. @@ -266,7 +275,7 @@ class SyncStore { await restoreBackup(merged); } if (needPush) { - await this.#push(merged, mergedHash); + await this.#push(fitted, mergedHash); } else { await this.#saveConfig({ lastSyncedAt: meta.at, @@ -274,6 +283,7 @@ class SyncStore { lastLocalHash: mergedHash, pendingPush: false, lastError: null, + trimmed: fitted.trimmed, }); } } catch (err) { @@ -286,18 +296,17 @@ class SyncStore { } } - /** Writes `backup` to the area, cut to the quota if it must be — or, too - * soon after the last write, comes back for it later (the next reconcile - * reaches the same conclusion; `pendingPush` and the hashes are untouched - * until the write lands). `hash` is the content hash of this device's full - * data, so a trimmed push doesn't read as "local changed" next time. */ - async #push(backup: Backup, hash: string) { + /** Writes a fitted backup to the area — or, too soon after the last write, + * comes back for it later (the next reconcile reaches the same conclusion; + * `pendingPush` and the hashes are untouched until the write lands). + * `hash` is the content hash of this device's full data, so a trimmed push + * doesn't read as "local changed" next time. */ + async #push(fitted: FitResult, hash: string) { const wait = this.#lastPushAt + MIN_PUSH_SPACING_MS - Date.now(); if (wait > 0) { this.#reconcileIn(wait); return; } - const fitted = await fitBackup(backup, BUDGET_CHARS, measure); const exportedAt = Date.now(); const packed = await packBackup( encodeBackup({ ...fitted.backup, exportedAt }), diff --git a/src/features/sync/persist/sync-blob.ts b/src/features/sync/persist/sync-blob.ts index fb83209..97afd4e 100644 --- a/src/features/sync/persist/sync-blob.ts +++ b/src/features/sync/persist/sync-blob.ts @@ -141,7 +141,7 @@ export async function packBackup(compact: CompactBackup, app: string): Promise

= { [META_KEY]: meta }; From e463ee2270957c70bd7d52add47f18d93331a974 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 22:16:50 +0200 Subject: [PATCH 07/26] feat: implement backup fixture and enhance backup handling with improved deletion management --- src/core/model/types.ts | 3 +- src/core/persist/backup-codec.test.ts | 43 ++++++---------- src/core/persist/backup-codec.ts | 41 +++++----------- src/core/persist/backup.fixture.ts | 19 +++++++ src/core/persist/backup.ts | 16 +++--- src/core/persist/deletions.ts | 29 +++++++++-- src/features/chords/panel/chords.svelte.ts | 9 ++-- .../settings/panel/SettingsView.svelte | 2 +- src/features/sync/panel/sync.svelte.ts | 14 +----- src/features/sync/persist/fit.test.ts | 21 ++------ src/features/sync/persist/merge.test.ts | 49 ++++++++++++------- src/features/sync/persist/merge.ts | 12 +++-- src/features/sync/persist/sync-blob.test.ts | 19 ++----- 13 files changed, 138 insertions(+), 139 deletions(-) create mode 100644 src/core/persist/backup.fixture.ts diff --git a/src/core/model/types.ts b/src/core/model/types.ts index 582e09d..95ff57b 100644 --- a/src/core/model/types.ts +++ b/src/core/model/types.ts @@ -118,7 +118,8 @@ export interface TrackData { sequenceLoop: boolean; /** Count in on play and before each snippet repeat lap (not on section loop). */ sequenceCountIn: boolean; - /** Cached chord/key chart from the last analysis run. null = never analyzed. + /** Cached chord/key chart from the last analysis run. Empty segments with a + * computedAt date mean deleted; null = never analyzed or trimmed from sync. * (null, not undefined — patches serialize over the port, dropping undefined.) */ chordChart?: ChordChart | null; /** Chords panel switch. Kept apart from the chart so switching off hides the diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index 88e3438..e0cbacc 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -9,9 +9,11 @@ import { encodeBackup, encodeChart, encodeParams, + isEmptyBackup, parseBackupJson, type Backup, } from './backup-codec.ts'; +import { backupFixture } from './backup.fixture.ts'; import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; import { makeTrackIdentity } from '../model/track-identity.ts'; import type { @@ -42,12 +44,7 @@ const localSong = makeTrackIdentity( ); function params(patch: Partial = {}): EffectParams { - return { - ...DEFAULT_PARAMS, - eq: { enabled: false, gains: [...DEFAULT_PARAMS.eq.gains] }, - tuning: { ...DEFAULT_PARAMS.tuning }, - ...patch, - }; + return { ...structuredClone(DEFAULT_PARAMS), ...patch }; } function entry(identity: TrackIdentity, patch: Partial = {}): HistoryEntry { @@ -124,20 +121,7 @@ function track(identity: TrackIdentity, patch: Partial = {}): TrackDa } function backup(patch: Partial = {}): Backup { - return { - format: BACKUP_FORMAT, - version: 1, - exportedAt: 1_757_200_000_123, - appVersion: '1.0.3', - settings: { ...DEFAULT_SETTINGS, keymap: { ...DEFAULT_SETTINGS.keymap } }, - uiPrefs: JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)), - history: [], - favorites: [], - eqPresets: [], - tracks: [], - deletions: {}, - ...patch, - }; + return backupFixture({ exportedAt: 1_757_200_000_123, appVersion: '1.0.3', ...patch }); } /** A whole library the way storage would hold it: `n` Recent rows, a third of @@ -248,6 +232,13 @@ test('settings: only what differs from the defaults is written', () => { assert.deepEqual(back.uiPrefs, b.uiPrefs); }); +test('sync: customized settings or prefs count even without a library', () => { + assert.equal(isEmptyBackup(backup()), true); + assert.equal(isEmptyBackup(backup({ settings: { ...DEFAULT_SETTINGS, theme: 'dark' } })), false); + assert.equal(isEmptyBackup(backup({ uiPrefs: { ...DEFAULT_UI_PREFS, boundaryLabels: { start: 'Intro', end: '' } } })), false); + assert.equal(isEmptyBackup(backup({ deletions: { 'h:*': Date.now() } })), false); +}); + test('settings: a keymap saved before an action existed is backfilled', () => { const b = backup(); const { zoomFit: _dropped, ...keymap } = b.settings.keymap; @@ -395,20 +386,21 @@ test('tracks: markers, snippets and flags round-trip; ids are regenerated', () = assert.equal(back.chordChart, null); }); -test('tracks: an empty chart is not written; chordsEnabled keeps false/true', () => { +test('tracks: dated empty charts survive the codec; absent charts stay absent', () => { const b = backup({ tracks: [ track(ytSong, { chordChart: chart(0), chordsEnabled: false }), track(siteSong, { chordChart: undefined, chordsEnabled: true }), ], }); - const enc = encodeBackup(b).t; - assert.equal(enc.every((t) => t.ch === undefined), true); const back = roundTrip(b).tracks; const byKey = new Map(back.map((t) => [t.identity.key, t])); assert.equal(byKey.get(ytSong.key)?.chordsEnabled, false); assert.equal(byKey.get(siteSong.key)?.chordsEnabled, true); - assert.equal(byKey.get(ytSong.key)?.chordChart, null); + assert.deepEqual(byKey.get(ytSong.key)?.chordChart?.segments, []); + assert.equal(byKey.get(ytSong.key)?.chordChart?.computedAt, chart(0).computedAt); + assert.equal(byKey.get(siteSong.key)?.chordChart, null); + assert.deepEqual(encodeBackup(roundTrip(b)), encodeBackup(b)); }); test('chart: times land within half a centisecond, gaps and key survive', () => { @@ -506,9 +498,6 @@ test('encode is idempotent past the first pass and JSON-clean', () => { const again = encodeBackup(decodeBackup(JSON.parse(JSON.stringify(first)))); assert.deepEqual(again, first); assert.deepEqual(JSON.parse(JSON.stringify(first)), first); - const text = JSON.stringify(decodeBackup(first)); - assert.ok(!text.includes('null,') || true); // nulls are legitimate (chordChart, baseBpm) - assert.ok(!/NaN/.test(text)); }); test('encode is deterministic regardless of track enumeration order', () => { diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 112e9bb..4dca2bc 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -1,5 +1,4 @@ import { - DEFAULT_KEYMAP, DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, @@ -260,10 +259,6 @@ function keyedArr(value: unknown, section: string): T[] { return list as T[]; } -function jsonEqual(a: unknown, b: unknown): boolean { - return JSON.stringify(a) === JSON.stringify(b); -} - /** Keys of `value` whose (JSON) value differs from `defaults`, recursing into * plain objects. Keys unknown to `defaults` are kept verbatim. */ function diffPlain( @@ -277,7 +272,7 @@ function diffPlain( if (isRecord(v) && isRecord(d)) { const nested = diffPlain(v, d); if (Object.keys(nested).length) out[key] = nested; - } else if (!(key in defaults) || !jsonEqual(v, d)) { + } else if (!(key in defaults) || JSON.stringify(v) !== JSON.stringify(d)) { out[key] = v; } } @@ -331,16 +326,8 @@ export function encodeParams(p: EffectParams): CompactParams | undefined { return Object.keys(out).length ? out : undefined; } -function defaultParams(): EffectParams { - return { - ...DEFAULT_PARAMS, - eq: { enabled: false, gains: [...DEFAULT_PARAMS.eq.gains] }, - tuning: { ...DEFAULT_PARAMS.tuning }, - }; -} - export function decodeParams(raw: unknown, section: string): EffectParams { - const p = defaultParams(); + const p = structuredClone(DEFAULT_PARAMS); if (raw === undefined) return p; const c = rec(raw, section); if (c.t !== undefined) p.transpose = num(c.t, section); @@ -370,14 +357,9 @@ export function decodeParams(raw: unknown, section: string): EffectParams { // --------------------------------------------------------------------------- // Settings / UI prefs -const SETTINGS_DEFAULTS: Record = { - ...DEFAULT_SETTINGS, - keymap: { ...DEFAULT_KEYMAP }, -}; - export function encodeSettings(settings: Settings): Record { const { lastUsedParams, ...rest } = settings; - const out = diffPlain(rest, SETTINGS_DEFAULTS); + const out = diffPlain(rest, { ...DEFAULT_SETTINGS }); if (lastUsedParams) out.lp = encodeParams(lastUsedParams) ?? {}; return out; } @@ -385,7 +367,7 @@ export function encodeSettings(settings: Settings): Record { export function decodeSettings(raw: unknown): Settings { const diff = raw === undefined ? {} : rec(raw, 'settings'); const { lp, ...rest } = diff; - const settings = mergePlain(SETTINGS_DEFAULTS, rest) as unknown as Settings; + const settings = mergePlain({ ...DEFAULT_SETTINGS }, rest) as unknown as Settings; if (lp !== undefined) settings.lastUsedParams = decodeParams(lp, 'settings'); return settings; } @@ -693,9 +675,7 @@ function encodeTrack(track: TrackData, songs: SongTable): CompactTrack { if (track.sequenceLoop) out.L = 1; if (track.sequenceCountIn) out.C = 1; if (track.chordsEnabled !== undefined) out.ce = track.chordsEnabled ? 1 : 0; - if (track.chordChart && track.chordChart.segments?.length) { - out.ch = encodeChart(track.chordChart); - } + if (track.chordChart) out.ch = encodeChart(track.chordChart); return out; } @@ -765,6 +745,13 @@ export function encodeBackup(backup: Backup): CompactBackup { return out; } +/** Defaults alone need no sync blob; customized settings do, even without songs. */ +export function isEmptyBackup(backup: Backup): boolean { + const { songs, eq, s, u, del = {} } = encodeBackup(backup); + return songs.length === 0 && eq.length === 0 && + Object.keys(s).length === 0 && Object.keys(u).length === 0 && Object.keys(del).length === 0; +} + /** Reads a compact (v2) backup, or throws an `Error` whose message is safe to * show the user. Unknown keys are ignored so the format can grow. */ export function decodeBackup(raw: unknown): Backup { @@ -783,9 +770,7 @@ export function decodeBackup(raw: unknown): Backup { favorites: arr(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), - deletions: Object.fromEntries( - Object.entries(normalizeDeletions(raw.del)).map(([k, when]) => [k, when]), - ), + deletions: normalizeDeletions(raw.del), }; } diff --git a/src/core/persist/backup.fixture.ts b/src/core/persist/backup.fixture.ts new file mode 100644 index 0000000..88a708c --- /dev/null +++ b/src/core/persist/backup.fixture.ts @@ -0,0 +1,19 @@ +import { DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; +import { BACKUP_FORMAT, type Backup } from './backup-codec.ts'; + +export function backupFixture(patch: Partial = {}): Backup { + return { + format: BACKUP_FORMAT, + version: 1, + exportedAt: 1_757_000_000_000, + appVersion: '', + settings: structuredClone(DEFAULT_SETTINGS), + uiPrefs: structuredClone(DEFAULT_UI_PREFS), + history: [], + favorites: [], + eqPresets: [], + tracks: [], + deletions: {}, + ...patch, + }; +} diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index b9e266e..ac4b554 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -5,7 +5,7 @@ import { parseBackupJson, type Backup, } from './backup-codec'; -import { mergeDeletions, pruneDeletions } from './deletions'; +import { mergeDeletions, pruneDeletions, reviveBackup } from './deletions'; import { deletionsItem, eqPresetsItem, @@ -83,10 +83,14 @@ export function parseBackup(text: string): Backup { * not carry — a restore reproduces the machine it came from rather than * merging into whatever is here. Host permissions are left untouched. * Deletion records are the one thing merged, not replaced: forgetting this - * device's would let a sync merge resurrect what it had removed. + * device's would let a sync merge resurrect what it had removed. Manual file + * imports use `asNew` to re-add their contents; sync restores keep their dates. */ -export async function restoreBackup(backup: Backup): Promise { - const deletions = await deletionsItem.getValue(); +export async function restoreBackup(backup: Backup, { asNew = false } = {}): Promise { + const deletions = pruneDeletions( + mergeDeletions(await deletionsItem.getValue(), backup.deletions ?? {}), Date.now(), + ); + if (asNew) backup = reviveBackup(backup, deletions); await removeAllTrackData(); await Promise.all([ settingsItem.setValue(backup.settings), @@ -94,9 +98,7 @@ export async function restoreBackup(backup: Backup): Promise { historyItem.setValue(backup.history), favoritesItem.setValue(backup.favorites), eqPresetsItem.setValue(backup.eqPresets), - deletionsItem.setValue( - pruneDeletions(mergeDeletions(deletions, backup.deletions ?? {}), Date.now()), - ), + deletionsItem.setValue(deletions), ...backup.tracks.map(saveTrackData), ]); } diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index 73ee5bc..ee4c2ba 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -19,6 +19,8 @@ * Pure and DOM-free (relative `.ts` imports; runs under `node --test`). */ +import type { Backup } from './backup-codec.ts'; + export type Deletions = Record; export const historyDeletion = (songKey: string) => `h:${songKey}`; @@ -57,9 +59,26 @@ export function pruneDeletions(deletions: Deletions, now: number): Deletions { /** Anything a file or a remote copy claims to be deletions, made safe. */ export function normalizeDeletions(raw: unknown): Deletions { if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return {}; - const out: Deletions = {}; - for (const [key, when] of Object.entries(raw as Record)) { - if (typeof when === 'number' && Number.isFinite(when) && when > 0) out[key] = when; - } - return out; + return Object.fromEntries(Object.entries(raw).filter( + ([, when]) => typeof when === 'number' && Number.isFinite(when) && when > 0, + )); +} + +/** A manual import is a new edit. Keep deletion records for absent items, + * but date restored items after them so the next sync cannot delete them. */ +export function reviveBackup(backup: Backup, deletions: Deletions, now = Date.now()): Backup { + const updatedAt = Math.max(now, backup.exportedAt, ...Object.values(deletions)) + 1; + return { + ...backup, + exportedAt: updatedAt, + history: backup.history.map((e) => ({ ...e, updatedAt })), + favorites: backup.favorites.map((e) => ({ ...e, updatedAt })), + eqPresets: backup.eqPresets.map((e) => ({ ...e, updatedAt })), + tracks: backup.tracks.map((t) => ({ + ...t, + updatedAt, + chordChart: t.chordChart ? { ...t.chordChart, computedAt: updatedAt } : t.chordChart, + })), + deletions, + }; } diff --git a/src/features/chords/panel/chords.svelte.ts b/src/features/chords/panel/chords.svelte.ts index 59a6de7..bf8b417 100644 --- a/src/features/chords/panel/chords.svelte.ts +++ b/src/features/chords/panel/chords.svelte.ts @@ -94,8 +94,7 @@ class ChordsStore { this.#log(`analyze confirmed (${fromStart ? 'from start' : 'from here'}) — loading model`); this.phase = 'loading'; this.loadError = false; - this.chart = null; - this.#persistNow(); + this.clear(); try { await this.#ensureEngine().ready(); } catch (err) { @@ -178,7 +177,11 @@ class ChordsStore { /** Delete the analyzed chords for this track (and the persisted copy). */ clear() { - this.chart = null; + // An empty, dated chart is a deletion; null can also mean sync trimmed it. + this.chart = { + segments: [], key: null, coverage: 0, analyzedFrom: 0, analyzedTo: 0, + computedAt: Date.now(), + }; this.#persistNow(); } diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index acddab5..9de7f12 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -170,7 +170,7 @@ 'with the contents of this file? Your current data is lost.', ); if (!ok) return; - await restoreBackup(backup); + await restoreBackup(backup, { asNew: true }); // Every store reads storage once at start-up; a reload is the honest way // to get the whole panel — theme, open track, engine — onto new data. location.reload(); diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 2f1e9b9..84cb0b9 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -1,4 +1,4 @@ -import { encodeBackup } from '../../../core/persist/backup-codec'; +import { encodeBackup, isEmptyBackup } from '../../../core/persist/backup-codec'; import { createBackup, restoreBackup, type Backup } from '../../../core/persist/backup'; import { session } from '../../../core/state/session.svelte'; import { fitBackup, type FitResult } from '../persist/fit'; @@ -49,16 +49,6 @@ const SYNCED_KEY_RE = /^(settings|uiPrefs|history|favorites|eqPresets|deletions| const measure = (backup: Backup) => packedChars(encodeBackup(backup)); const fit = (backup: Backup) => fitBackup(backup, BUDGET_CHARS, measure); -function isEmpty(backup: Backup): boolean { - return ( - backup.history.length === 0 && - backup.favorites.length === 0 && - backup.tracks.length === 0 && - backup.eqPresets.length === 0 && - Object.keys(backup.deletions ?? {}).length === 0 - ); -} - /** * Cross-device sync through the browser's own synced storage — no server, * no account, no ID: whoever signs into the same browser profile gets the @@ -226,7 +216,7 @@ class SyncStore { this.#tornSince = 0; if (result.kind === 'none') { - if (isEmpty(local) && !this.config.pendingPush) return; + if (isEmptyBackup(local) && !this.config.pendingPush) return; await this.#push(await fit(local), localHash); return; } diff --git a/src/features/sync/persist/fit.test.ts b/src/features/sync/persist/fit.test.ts index c4a3f01..68ab5a1 100644 --- a/src/features/sync/persist/fit.test.ts +++ b/src/features/sync/persist/fit.test.ts @@ -2,8 +2,9 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { fitBackup, LibraryTooLargeError } from './fit.ts'; -import { BACKUP_FORMAT, encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; -import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../../../core/model/defaults.ts'; +import { encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; +import { backupFixture } from '../../../core/persist/backup.fixture.ts'; +import { DEFAULT_PARAMS } from '../../../core/model/defaults.ts'; import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; import type { ChordChart, @@ -92,19 +93,7 @@ function library(recent: number, favorites: number, orphans = 0): Backup { } // Storage enumeration order is arbitrary; make sure nothing relies on it. tracks.reverse(); - return { - format: BACKUP_FORMAT, - version: 1, - exportedAt: T0, - appVersion: '', - settings: DEFAULT_SETTINGS, - uiPrefs: DEFAULT_UI_PREFS, - history, - favorites: favs, - eqPresets: [], - tracks, - deletions: {}, - }; + return backupFixture({ history, favorites: favs, tracks }); } const keys = (list: { identity: TrackIdentity }[]) => list.map((e) => e.identity.key).sort(); @@ -136,7 +125,6 @@ test('recent songs go first, oldest first, taking their records along', async () test('charts go next, oldest first; favorites and their markers stay', async () => { const b = library(3, 3); - const noRecent = await fitBackup(b, measure(b), measure); // Find the budget at which every non-favorite is gone but charts remain. const favoritesOnly = { ...b, history: b.history.slice(3), tracks: b.tracks.filter((t) => keys(b.favorites).includes(t.identity.key)) }; const result = await fitBackup(b, measure(favoritesOnly) - 1, measure); @@ -154,7 +142,6 @@ test('charts go next, oldest first; favorites and their markers stay', async () .filter((t) => !t.chordChart) .map((t) => t.updatedAt); assert.ok(Math.max(...cutAt) < Math.min(...keptAt), 'the oldest charts went'); - assert.equal(noRecent.trimmed, false); }); test('favorites go last, least recently accessed first', async () => { diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts index c01cb18..0ce4842 100644 --- a/src/features/sync/persist/merge.test.ts +++ b/src/features/sync/persist/merge.test.ts @@ -2,14 +2,15 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { mergeBackups } from './merge.ts'; -import { BACKUP_FORMAT, type Backup } from '../../../core/persist/backup-codec.ts'; +import { backupFixture as backup } from '../../../core/persist/backup.fixture.ts'; import { favoriteDeletion, HISTORY_CLEARED, historyDeletion, presetDeletion, + reviveBackup, } from '../../../core/persist/deletions.ts'; -import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, HISTORY_LIMIT } from '../../../core/model/defaults.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, HISTORY_LIMIT } from '../../../core/model/defaults.ts'; import { makeTrackIdentity, songKey } from '../../../core/model/track-identity.ts'; import type { ChordChart, FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types.ts'; @@ -54,23 +55,6 @@ function record(identity: TrackIdentity, updatedAt: number, markers: number, wit }; } -function backup(patch: Partial = {}): Backup { - return { - format: BACKUP_FORMAT, - version: 1, - exportedAt: T0, - appVersion: '', - settings: { ...DEFAULT_SETTINGS }, - uiPrefs: JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)), - history: [], - favorites: [], - eqPresets: [], - tracks: [], - deletions: {}, - ...patch, - }; -} - const keys = (list: { identity: TrackIdentity }[]) => list.map((e) => e.identity.key); test('a row the other side trimmed away survives; the newer copy of a shared row wins', () => { @@ -158,6 +142,33 @@ test('tracks: a winner without a chart adopts the other side’s', () => { assert.ok(merged.tracks[0].chordChart?.segments.length); }); +test('a deleted chart beats stale analysis and later marker edits, but allows re-analysis', () => { + const deleted = backup({ tracks: [{ ...record(song(1), T0 + 1, 0), chordChart: { ...chart(T0 + 1), segments: [] } }] }); + const stale = backup({ tracks: [{ ...record(song(1), T0 + 2, 3), chordChart: chart(T0) }] }); + for (const remoteWins of [false, true]) { + const merged = mergeBackups(deleted, stale, remoteWins, NOW); + assert.equal(merged.tracks[0].markers.length, 3); + assert.deepEqual(merged.tracks[0].chordChart?.segments, []); + stale.tracks[0].chordChart = chart(T0 + 3); + assert.equal(mergeBackups(merged, stale, remoteWins, NOW).tracks[0].chordChart?.computedAt, T0 + 3); + stale.tracks[0].chordChart = chart(T0); + } +}); + +test('a manual import re-adds deleted rows and presets without reviving absent items', () => { + const del = { [HISTORY_CLEARED]: NOW, [favoriteDeletion(songKey(song(1)))]: NOW, [presetDeletion('Mine')]: NOW }; + const file = backup({ history: [row(song(1), T0)], favorites: [fav(song(1), T0)], eqPresets: [{ name: 'Mine', gains: [1] }] }); + const restored = reviveBackup(file, del, NOW); + const remote = backup({ deletions: del, history: [row(song(2), T0)] }); + for (const remoteWins of [false, true]) { + const merged = mergeBackups(restored, remote, remoteWins, NOW); + assert.deepEqual(keys(merged.history), [song(1).key]); + assert.deepEqual(keys(merged.favorites), [song(1).key]); + assert.equal(merged.eqPresets[0].name, 'Mine'); + } + assert.equal(file.history[0].updatedAt, T0, 'the original backup is unchanged'); +}); + test('ties go to the winning side', () => { const a = song(1); const local = backup({ history: [row(a, T0, 1)], tracks: [record(a, T0, 1)] }); diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index f062d81..ddd1f1c 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -10,7 +10,6 @@ import { presetDeletion, pruneDeletions, } from '../../../core/persist/deletions.ts'; -import { hasChart } from './fit.ts'; import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; /** @@ -24,8 +23,8 @@ import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; * is the later of the two. * - Track records: matched by key, the more recently edited wins outright — * an emptied record is still a record, so clearing markers sticks. A - * winner without a chart adopts the other's: absent may mean "trimmed", - * and a chart is only ever replaced by re-analysis. + * chart is chosen separately by computedAt: null may mean "trimmed", but + * an empty, dated chart is an explicit deletion and beats older analysis. * - Settings and UI prefs: the newer device's, as a whole. EQ presets: union * by name, the later save of a shared name, deletions honoured. * @@ -97,7 +96,12 @@ export function mergeBackups( loser.tracks, (t) => t.identity.key, undefined, - (w, l) => (!hasChart(w) && hasChart(l) ? { ...w, chordChart: l.chordChart } : w), + (w, l) => ({ + ...w, + chordChart: !w.chordChart || (l.chordChart?.computedAt ?? 0) > w.chordChart.computedAt + ? l.chordChart ?? w.chordChart + : w.chordChart, + }), ), deletions, }; diff --git a/src/features/sync/persist/sync-blob.test.ts b/src/features/sync/persist/sync-blob.test.ts index b883f68..807e352 100644 --- a/src/features/sync/persist/sync-blob.test.ts +++ b/src/features/sync/persist/sync-blob.test.ts @@ -22,8 +22,9 @@ import { unpackBackup, } from './sync-blob.ts'; import { fitBackup } from './fit.ts'; -import { BACKUP_FORMAT, encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; -import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../../../core/model/defaults.ts'; +import { encodeBackup, type Backup } from '../../../core/persist/backup-codec.ts'; +import { backupFixture } from '../../../core/persist/backup.fixture.ts'; +import { DEFAULT_PARAMS } from '../../../core/model/defaults.ts'; import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; import type { ChordChart, HistoryEntry, TrackData } from '../../../core/model/types.ts'; @@ -66,19 +67,7 @@ function library(n: number): Backup { updatedAt: T0 + i * 60_000, }); } - return { - format: BACKUP_FORMAT, - version: 1, - exportedAt: T0, - appVersion: '1.0.3', - settings: DEFAULT_SETTINGS, - uiPrefs: DEFAULT_UI_PREFS, - history, - favorites: [], - eqPresets: [], - tracks, - deletions: {}, - }; + return backupFixture({ history, tracks, appVersion: '1.0.3' }); } test('base64 round-trips every byte value and a large buffer', () => { From dba2c203f6c2bdcc6d90d88e6694d8ba1a843290 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 22:48:25 +0200 Subject: [PATCH 08/26] feat: enhance backup and sync handling with improved deletion management and media state updates --- src/core/persist/backup.ts | 11 ++- src/core/persist/deletions.ts | 4 +- src/core/persist/storage.ts | 13 ++- src/core/state/connect.svelte.ts | 4 +- src/core/state/session.svelte.ts | 15 +++- src/dev/mock.ts | 4 +- src/entrypoints/sidepanel/App.svelte | 3 + src/features/chords/panel/chords.svelte.ts | 8 +- src/features/sync/panel/sync.svelte.ts | 94 +++++++++++++++------- src/features/sync/persist/merge.test.ts | 27 +++++++ src/features/sync/persist/merge.ts | 42 +++++++--- src/features/sync/persist/sync-area.ts | 8 +- src/features/sync/persist/sync-blob.ts | 9 ++- 13 files changed, 186 insertions(+), 56 deletions(-) diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index ac4b554..4770043 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -11,7 +11,7 @@ import { eqPresetsItem, favoritesItem, historyItem, - removeAllTrackData, + removeTrackDataExcept, saveTrackData, settingsItem, uiPrefsItem, @@ -85,13 +85,19 @@ export function parseBackup(text: string): Backup { * Deletion records are the one thing merged, not replaced: forgetting this * device's would let a sync merge resurrect what it had removed. Manual file * imports use `asNew` to re-add their contents; sync restores keep their dates. + * + * Track records are written first and the leftovers removed afterwards, never + * the other way round. A sync merge calls this on every remote change, and + * the panel document can go away mid-restore (the user closes it, the tab + * changes) — wiping first would make that window cost every marker, snippet + * and chart in the library. This way the worst case is a stale record the + * next restore removes. */ export async function restoreBackup(backup: Backup, { asNew = false } = {}): Promise { const deletions = pruneDeletions( mergeDeletions(await deletionsItem.getValue(), backup.deletions ?? {}), Date.now(), ); if (asNew) backup = reviveBackup(backup, deletions); - await removeAllTrackData(); await Promise.all([ settingsItem.setValue(backup.settings), uiPrefsItem.setValue(backup.uiPrefs), @@ -101,4 +107,5 @@ export async function restoreBackup(backup: Backup, { asNew = false } = {}): Pro deletionsItem.setValue(deletions), ...backup.tracks.map(saveTrackData), ]); + await removeTrackDataExcept(new Set(backup.tracks.map((t) => t.identity.key))); } diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index ee4c2ba..2e0ca00 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -72,7 +72,9 @@ export function reviveBackup(backup: Backup, deletions: Deletions, now = Date.no ...backup, exportedAt: updatedAt, history: backup.history.map((e) => ({ ...e, updatedAt })), - favorites: backup.favorites.map((e) => ({ ...e, updatedAt })), + // `favoritedAt` too: it is what the merge dates a favorite by, so leaving + // it behind would let a deletion record outrank the star we just re-added. + favorites: backup.favorites.map((e) => ({ ...e, updatedAt, favoritedAt: updatedAt })), eqPresets: backup.eqPresets.map((e) => ({ ...e, updatedAt })), tracks: backup.tracks.map((t) => ({ ...t, diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index 22180c5..cb74b29 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -106,8 +106,15 @@ export async function saveTrackData(data: TrackData): Promise { await storage.setItem(trackDataKey(data.identity.key), toPlain(data)); } -export async function removeAllTrackData(): Promise { +/** Drops every stored track record whose identity key isn't in `keep` — a + * restore replaces the set of records rather than merging into it. Written + * as "remove what's left over" (rather than wiping first) so the records are + * only ever gone once their replacements are in: a restore interrupted + * halfway leaves stale records behind, never an empty library. */ +export async function removeTrackDataExcept(keep: Set): Promise { const snapshot = await browser.storage.local.get(null); - const keys = Object.keys(snapshot).filter((k) => k.startsWith('track:')); - if (keys.length) await browser.storage.local.remove(keys); + const stale = Object.keys(snapshot).filter( + (k) => k.startsWith('track:') && !keep.has(k.slice('track:'.length)), + ); + if (stale.length) await browser.storage.local.remove(stale); } diff --git a/src/core/state/connect.svelte.ts b/src/core/state/connect.svelte.ts index 0f5d3a6..44ca6f3 100644 --- a/src/core/state/connect.svelte.ts +++ b/src/core/state/connect.svelte.ts @@ -102,7 +102,7 @@ class ConnectionManager { if (isRestricted(tab.url) && !isLocalPlayer(tab.url)) { session.connection = 'restricted'; - session.media = null; + session.setMedia(null); return; } @@ -120,7 +120,7 @@ class ConnectionManager { if (!granted) { this.needsPermission = pattern; session.connection = 'idle'; - session.media = null; + session.setMedia(null); return; } this.needsPermission = null; diff --git a/src/core/state/session.svelte.ts b/src/core/state/session.svelte.ts index 932d612..b20edee 100644 --- a/src/core/state/session.svelte.ts +++ b/src/core/state/session.svelte.ts @@ -75,6 +75,10 @@ class SessionStore { /** Persistence hooks (set by track-sync). */ onMediaEvent: ((media: MediaInfo | null) => void) | null = null; + /** Every change to `media`, the connection layer's outright clears + * included — where `onMediaEvent` is only the engine's own media messages. + * Set by the panel root for whoever needs to know a track went away. */ + onMediaChanged: ((media: MediaInfo | null) => void) | null = null; onUserParamsChange: (() => void) | null = null; /** The engine link dropped. Whatever reconnects starts on the default preset, * so the track's saved settings have to go back on even if it never changed. */ @@ -85,6 +89,13 @@ class SessionStore { volume(volume: number): void; } | null = null; + /** The one place `media` is written, so `onMediaChanged` cannot miss a + * change: the connection layer clears it on paths that send no event. */ + setMedia(media: MediaInfo | null) { + this.media = media; + this.onMediaChanged?.(media); + } + attachTransport(send: (cmd: EngineCommand) => void) { this.#send = send; } @@ -122,7 +133,7 @@ class SessionStore { case 'snapshot': this.connection = event.state; this.#dspBlocked = !event.dspAvailable; - this.media = event.media; + this.setMedia(event.media); this.params = event.params; this.volume = event.volume; this.loop = event.loop; @@ -148,7 +159,7 @@ class SessionStore { this.#dspBlocked = !event.available; break; case 'media': - this.media = event.media; + this.setMedia(event.media); // Zero duration = metadata still loading: keep seeks gated a moment // longer (mirrors track-sync's zero-duration grace period). if (event.media?.duration) this.#setSourceChanging(false); diff --git a/src/dev/mock.ts b/src/dev/mock.ts index fbd22b0..db2ad20 100644 --- a/src/dev/mock.ts +++ b/src/dev/mock.ts @@ -13,12 +13,12 @@ const GUITAR_EQ = BUILTIN_EQ_PRESETS.find((p) => p.name === 'Guitar')!; * The store screenshots are taken from this state. */ export function installMockState() { session.connection = 'connected-direct'; - session.media = { + session.setMedia({ title: 'Megadeth - Symphony of Destruction - Guitar Tab | Lesson', pageUrl: 'https://youtube.com/watch?v=741FSo7Xb40', duration: 230, hasVideo: true, - }; + }); session.t = 141; session.playing = false; diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index 30a4d0f..87983da 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -32,6 +32,9 @@ console.error('[note-by-note] track sync failed', err); }); }; + // Applying another device's changes reloads this document, so a merge + // that arrived mid-practice waits for the track to go away. + session.onMediaChanged = (media) => sync.onMedia(media); session.onUserParamsChange = () => trackSync.onParamsChanged(); session.onEngineDetached = () => trackSync.onEngineLost(); // Diagnostics for the E2E harness; kept out of release builds. diff --git a/src/features/chords/panel/chords.svelte.ts b/src/features/chords/panel/chords.svelte.ts index bf8b417..223dc74 100644 --- a/src/features/chords/panel/chords.svelte.ts +++ b/src/features/chords/panel/chords.svelte.ts @@ -94,7 +94,13 @@ class ChordsStore { this.#log(`analyze confirmed (${fromStart ? 'from start' : 'from here'}) — loading model`); this.phase = 'loading'; this.loadError = false; - this.clear(); + // Null, not `clear()`: the old chart is being replaced, not deleted, and + // the run can still fail (the model load is a download). An empty dated + // chart is a deletion that outranks another device's real one, so a + // failed run here would wipe the chords everywhere; null just means + // "nothing on this device", which the merge fills back in. + this.chart = null; + this.#persistNow(); try { await this.#ensureEngine().ready(); } catch (err) { diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 84cb0b9..7e98d7b 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -1,6 +1,7 @@ import { encodeBackup, isEmptyBackup } from '../../../core/persist/backup-codec'; import { createBackup, restoreBackup, type Backup } from '../../../core/persist/backup'; import { session } from '../../../core/state/session.svelte'; +import type { MediaInfo } from '../../../core/model/types'; import { fitBackup, type FitResult } from '../persist/fit'; import { contentHash } from '../persist/hash'; import { mergeBackups } from '../persist/merge'; @@ -140,6 +141,17 @@ class SyncStore { await this.#enqueue(() => this.#reconcile(true)); } + /** The loaded track changed (wired to `session.onMediaChanged` by the panel + * root). A merge held back because applying it would reload the panel + * mid-practice has no other cue that its moment has come: without this it + * waits for the safety interval, up to five minutes after the track went + * away, while this device carries on pushing its own state. */ + onMedia(media: MediaInfo | null): void { + if (media === null && this.pendingApply && this.config.enabled) { + this.#reconcileIn(REMOTE_DEBOUNCE_MS); + } + } + /** * Empties the synced copy and turns sync off — left on, the next change * would quietly re-upload. Other devices with sync on will re-seed it from @@ -179,6 +191,18 @@ class SyncStore { this.#timer = setTimeout(() => void this.#enqueue(() => this.#reconcile()), delayMs); } + /** Whether a write would come too soon after the last one, scheduling the + * retry if so. Asked before the encoding work wherever a push is the only + * thing left to do: the debounce is 5 s and the spacing 30 s, so most of a + * burst's reconciles have nothing to do but come back later, and gzipping + * the whole library to find that out is pure waste on the panel's thread. */ + #pushTooSoon(): boolean { + const wait = this.#lastPushAt + MIN_PUSH_SPACING_MS - Date.now(); + if (wait <= 0) return false; + this.#reconcileIn(wait); + return true; + } + /** * The whole algorithm. Reads the area and this device's data, then: * - nothing there → seed it (unless this device has nothing either); @@ -197,9 +221,6 @@ class SyncStore { try { const { result, bytes } = await readSyncArea(); this.usedBytes = bytes; - const local = await createBackup(); - const localHash = await contentHash(local); - const localChanged = localHash !== this.config.lastLocalHash; if (result.kind === 'torn') { if (!this.#tornSince) this.#tornSince = Date.now(); @@ -209,23 +230,43 @@ class SyncStore { } // Nobody finished that write; ours replaces it. Whatever it carried // comes back merged when its writer reconciles against ours. - this.#tornSince = 0; - await this.#push(await fit(local), localHash); - return; } this.#tornSince = 0; + // When there is nothing new from another device, a push is the only + // thing this reconcile could do — and if the spacing says not yet, it + // need not read and encode the whole library to find that out. The + // debounce is 5 s against a 30 s spacing, so during a burst of edits + // that is most reconciles. + const pushIsAllThatIsLeft = + result.kind === 'torn' || + (this.config.pendingPush && + (result.kind === 'none' || result.meta.h === this.config.lastRemoteHash)); + if (pushIsAllThatIsLeft && this.#pushTooSoon()) return; + + const local = await createBackup(); + const localHash = await contentHash(local); + const localChanged = localHash !== this.config.lastLocalHash; + + if (result.kind === 'torn') { + if (!this.#pushTooSoon()) await this.#push(await fit(local), localHash); + return; + } + if (result.kind === 'none') { if (isEmptyBackup(local) && !this.config.pendingPush) return; - await this.#push(await fit(local), localHash); + if (!this.#pushTooSoon()) await this.#push(await fit(local), localHash); return; } const { meta, base64 } = result; if (meta.h === this.config.lastRemoteHash) { // Remote is what we last saw (our own echo included). - if (localChanged || this.config.pendingPush) await this.#push(await fit(local), localHash); - else if (this.config.lastError) await this.#saveConfig({ lastError: null }); + if (localChanged || this.config.pendingPush) { + if (!this.#pushTooSoon()) await this.#push(await fit(local), localHash); + } else if (this.config.lastError) { + await this.#saveConfig({ lastError: null }); + } return; } @@ -233,28 +274,31 @@ class SyncStore { const remote = await unpackBackup(base64); const remoteWins = !localChanged || remote.exportedAt > this.config.lastChangedAt; const merged = mergeBackups(local, remote, remoteWins); - const fitted = await fit(merged); - const [mergedHash, remoteHash] = await Promise.all([ - contentHash(merged), - contentHash(remote), - ]); + const mergedHash = await contentHash(merged); const needApply = mergedHash !== localHash; - // Push our own edits, or a full copy the remote lacks. Once the merge - // is over the quota only our edits count: two devices holding - // different old songs would otherwise cut the copy differently and - // re-upload each other's cut forever. - const needPush = - localChanged || - this.config.pendingPush || - (!fitted.trimmed && mergedHash !== remoteHash); if (needApply && !force && session.media !== null) { // A track is loaded; applying would reload the panel mid-practice. // Nothing is pushed either: a push now would carry only our side. + // Before the fit, so a track left loaded for an hour doesn't gzip the + // library on every tick to throw the result away. this.pendingApply = true; return; } this.pendingApply = false; + + const fitted = await fit(merged); + // Push our own edits, or a full copy the remote lacks. Once the merge + // is over the quota only our edits count: two devices holding + // different old songs would otherwise cut the copy differently and + // re-upload each other's cut forever. The remote's own hash is the + // last thing asked for — another full encode, and only the last term + // needs it. + const needPush = + localChanged || + this.config.pendingPush || + (!fitted.trimmed && mergedHash !== (await contentHash(remote))); + if (needApply) { // #applying stays set until the reload: nothing in between may // schedule a push of what was just written. The reload is owed from @@ -292,11 +336,7 @@ class SyncStore { * `hash` is the content hash of this device's full data, so a trimmed push * doesn't read as "local changed" next time. */ async #push(fitted: FitResult, hash: string) { - const wait = this.#lastPushAt + MIN_PUSH_SPACING_MS - Date.now(); - if (wait > 0) { - this.#reconcileIn(wait); - return; - } + if (this.#pushTooSoon()) return; const exportedAt = Date.now(); const packed = await packBackup( encodeBackup({ ...fitted.backup, exportedAt }), diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts index 0ce4842..b140919 100644 --- a/src/features/sync/persist/merge.test.ts +++ b/src/features/sync/persist/merge.test.ts @@ -100,6 +100,20 @@ test('history is newest-first and capped', () => { assert.equal(merged.history[0].identity.key, song(999).key); }); +test('the history cap never drops a favorited song’s row', () => { + const old = song(1); + const local = backup({ + favorites: [fav(old, T0)], + history: [ + row(old, T0), + ...Array.from({ length: HISTORY_LIMIT }, (_, i) => row(song(i + 2), T0 + 1000 + i)), + ], + }); + const merged = mergeBackups(local, backup({}), false, NOW); + assert.equal(merged.history.length, HISTORY_LIMIT, 'a plain row went instead'); + assert.ok(keys(merged.history).includes(old.key)); +}); + test('favorites: union in the winner order, deletions honoured, last access kept', () => { const a = song(1); const b = song(2); @@ -115,6 +129,19 @@ test('favorites: union in the winner order, deletions honoured, last access kept assert.equal(merged.favorites[1].lastAccessedAt, T0 + 9000); }); +test('favorites: practice on the other device does not undo an unfavorite', () => { + const a = song(1); + // Starred long ago, unfavorited here; the other device then played it and + // moved a slider, which bumps `updatedAt` but not `favoritedAt`. + const local = backup({ deletions: { [favoriteDeletion(songKey(a))]: T0 + 5000 } }); + const practised: FavoriteEntry = { ...fav(a, T0), updatedAt: T0 + 9000 }; + const remote = backup({ favorites: [practised] }); + assert.deepEqual(keys(mergeBackups(local, remote, true, NOW).favorites), []); + // Genuinely starring it again does beat the record. + const restarred = backup({ favorites: [fav(a, T0 + 6000)] }); + assert.deepEqual(keys(mergeBackups(local, restarred, true, NOW).favorites), [a.key]); +}); + test('favorites: a newer copy adopts the other side’s later access time', () => { const a = song(1); const local = backup({ favorites: [fav(a, T0, T0 + 9000)] }); diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index ddd1f1c..b98f502 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -58,6 +58,21 @@ function unionNewest( const song = (entry: HistoryEntry) => songKey(entry.identity); +/** Newest first, down to `HISTORY_LIMIT` — but only non-favorited rows are + * ever dropped. A favorited song whose Recent row went would leave the two + * library copies of it disagreeing, which is the drift `upsertHistory` and + * `fit.ts` both go out of their way to prevent; a union of two full libraries + * is exactly where the cap would otherwise reach one. Over the limit in + * favorites alone, the list simply stays long. */ +function capHistory(history: HistoryEntry[], favorites: FavoriteEntry[]): HistoryEntry[] { + const ordered = history.sort((a, b) => at(b) - at(a)); + const excess = ordered.length - HISTORY_LIMIT; + if (excess <= 0) return ordered; + const favorited = new Set(favorites.map((f) => song(f))); + const cut = new Set(ordered.filter((e) => !favorited.has(song(e))).slice(-excess)); + return ordered.filter((e) => !cut.has(e)); +} + export function mergeBackups( local: Backup, remote: Backup, @@ -69,6 +84,20 @@ export function mergeBackups( const history = unionNewest(winner.history, loser.history, song, (e) => deletedSince(deletions, at(e), historyDeletion(song(e)), HISTORY_CLEARED), ); + const favorites = unionNewest( + winner.favorites, + loser.favorites, + song, + // Anchored on `favoritedAt` — when the star was put there — not on + // `updatedAt`, which ordinary practice bumps (`touchFavorite` with new + // params). Taking the later of the two would let a slider nudge on one + // device outdate the other's unfavorite and re-star the song. + (f) => deletedSince(deletions, f.favoritedAt || at(f), favoriteDeletion(song(f))), + (w, l): FavoriteEntry => ({ + ...w, + lastAccessedAt: Math.max(w.lastAccessedAt ?? 0, l.lastAccessedAt ?? 0), + }), + ); return { ...local, exportedAt: Math.max(local.exportedAt ?? 0, remote.exportedAt ?? 0), @@ -80,17 +109,8 @@ export function mergeBackups( (p) => p.name, (p) => deletedSince(deletions, at(p), presetDeletion(p.name)), ), - history: history.sort((a, b) => at(b) - at(a)).slice(0, HISTORY_LIMIT), - favorites: unionNewest( - winner.favorites, - loser.favorites, - song, - (f) => deletedSince(deletions, Math.max(f.favoritedAt ?? 0, at(f)), favoriteDeletion(song(f))), - (w, l): FavoriteEntry => ({ - ...w, - lastAccessedAt: Math.max(w.lastAccessedAt ?? 0, l.lastAccessedAt ?? 0), - }), - ), + history: capHistory(history, favorites), + favorites, tracks: unionNewest( winner.tracks, loser.tracks, diff --git a/src/features/sync/persist/sync-area.ts b/src/features/sync/persist/sync-area.ts index fbc3749..926ed24 100644 --- a/src/features/sync/persist/sync-area.ts +++ b/src/features/sync/persist/sync-area.ts @@ -51,9 +51,11 @@ export function onSyncAreaChanged(listener: () => void): void { const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err)); /** The browser reports write metering as thrown strings/errors with these - * names in the message; there is no error code to switch on. */ + * names in the message; there is no error code to switch on. `MAX_ITEMS` is + * deliberately not here: it is a hard cap on the area, not metering, so the + * caller must not schedule a retry that can only fail the same way. */ export function isRateLimited(err: unknown): boolean { - return /MAX_WRITE_OPERATIONS|MAX_SUSTAINED_WRITE|MAX_ITEMS/i.test(errorText(err)); + return /MAX_WRITE_OPERATIONS|MAX_SUSTAINED_WRITE/i.test(errorText(err)); } /** Our own errors (`NewerVersionError`, `LibraryTooLargeError`) already read @@ -61,7 +63,7 @@ export function isRateLimited(err: unknown): boolean { export function syncErrorMessage(err: unknown): string { const text = errorText(err); if (isRateLimited(err)) return 'The browser is rate-limiting sync writes — retrying in a minute.'; - if (/QUOTA_BYTES|QuotaExceeded|quota/i.test(text)) { + if (/QUOTA_BYTES|QuotaExceeded|quota|MAX_ITEMS/i.test(text)) { return "Your library is too large for the browser's sync storage."; } return text || 'Sync failed.'; diff --git a/src/features/sync/persist/sync-blob.ts b/src/features/sync/persist/sync-blob.ts index 97afd4e..154fbc6 100644 --- a/src/features/sync/persist/sync-blob.ts +++ b/src/features/sync/persist/sync-blob.ts @@ -34,8 +34,13 @@ const BLOB_KEY_RE = /^nbn\.(meta|\d+)$/; export const CHUNK_CHARS = 8000; export const MAX_CHUNKS = 11; /** What `fitBackup` gets as its budget: base64 characters. 11 chunks plus - * meta come to ~88.3 KB of Chrome's 102,400-byte quota. */ -export const BUDGET_CHARS = CHUNK_CHARS * MAX_CHUNKS; + * meta come to ~88.3 KB of Chrome's 102,400-byte quota. The margin is what + * keeps `fit` and `packBackup` from disagreeing: the store fits a backup and + * packs it later with a fresh `exportedAt`, which gzips to a slightly + * different length. Filling the chunks exactly would let that difference + * spill into a twelfth chunk and throw where `fit` had just said it fits — + * and "too large" is not a retryable error, so sync would stay wedged. */ +export const BUDGET_CHARS = CHUNK_CHARS * MAX_CHUNKS - 512; export const SYNC_QUOTA_BYTES = 102_400; /** Bump when a reader of this version could misread the layout. */ From e5d4e3afaf66f11667c80cb9a3c02d4ed5c773c6 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Sun, 6 Sep 2026 23:07:10 +0200 Subject: [PATCH 09/26] feat: enhance sync handling with improved visibility and status management --- src/features/sync/panel/sync.svelte.ts | 43 ++++++++++++++++++------ src/features/sync/persist/sync-config.ts | 3 +- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 7e98d7b..3fd8b50 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -39,6 +39,9 @@ const TORN_RETRY_MS = 5000; /** … but not forever — past this, the writer died mid-write; our copy wins. */ const TORN_GIVE_UP_MS = 90_000; const RATE_LIMIT_RETRY_MS = 65_000; +/** A reconcile that finds everything in order takes milliseconds; without a + * floor "Syncing…" comes and goes inside a frame and the button reads dead. */ +const MIN_VISIBLE_SYNC_MS = 600; /** Belt and braces: the change event should carry everything, but a missed * one must not mean a device stays stale until the next panel open. */ const SAFETY_INTERVAL_MS = 5 * 60_000; @@ -84,10 +87,12 @@ class SyncStore { /** Another device's changes are in, waiting for a moment without a track * loaded (applying reloads the panel). "Sync now" applies them at once. */ pendingApply = $state(false); - #syncing = $state(false); + /** A depth, not a flag: "Sync now" holds it up for the minimum visible time + * around a reconcile that drops it as soon as it is done. */ + #busy = $state(0); status = $derived<'off' | 'syncing' | 'error' | 'idle'>( - !this.enabled ? 'off' : this.#syncing ? 'syncing' : this.lastError ? 'error' : 'idle', + !this.enabled ? 'off' : this.#busy > 0 ? 'syncing' : this.lastError ? 'error' : 'idle', ); usedPercent = $derived( this.usedBytes ? Math.max(1, Math.round((this.usedBytes / SYNC_QUOTA_BYTES) * 100)) : 0, @@ -136,9 +141,21 @@ class SyncStore { await this.#saveConfig({ enabled: false }); } - /** Back up now and pull in the other devices' changes, reload included. */ + /** Back up now and pull in the other devices' changes, reload included. + * Usually there is nothing to carry — the panel reconciled when it opened + * and after every change since — so what the click has to show for itself + * is the status line: "Syncing…" for long enough to read, then a refreshed + * "Last synced just now". */ async syncNow(): Promise { - await this.#enqueue(() => this.#reconcile(true)); + this.#busy++; + try { + await Promise.all([ + this.#enqueue(() => this.#reconcile(true)), + new Promise((resolve) => setTimeout(resolve, MIN_VISIBLE_SYNC_MS)), + ]); + } finally { + this.#busy--; + } } /** The loaded track changed (wired to `session.onMediaChanged` by the panel @@ -160,7 +177,7 @@ class SyncStore { async deleteRemote(): Promise { await this.disable(); await this.#enqueue(async () => { - this.#syncing = true; + this.#busy++; try { await clearSyncArea(); this.usedBytes = 0; @@ -169,7 +186,7 @@ class SyncStore { await this.#saveConfig({ lastError: syncErrorMessage(err) }); throw err; } finally { - this.#syncing = false; + this.#busy--; } }); } @@ -216,7 +233,7 @@ class SyncStore { */ async #reconcile(force = false) { if (!this.config.enabled) return; - this.#syncing = true; + this.#busy++; let applied = false; try { const { result, bytes } = await readSyncArea(); @@ -264,8 +281,12 @@ class SyncStore { // Remote is what we last saw (our own echo included). if (localChanged || this.config.pendingPush) { if (!this.#pushTooSoon()) await this.#push(await fit(local), localHash); - } else if (this.config.lastError) { - await this.#saveConfig({ lastError: null }); + } else { + // Nothing to send and nothing to fetch: this device is in agreement + // with the synced copy as of now, which is what the status line + // reports. Recording it is the only visible outcome a "Sync now" + // that finds everything already in order can have. + await this.#saveConfig({ lastSyncedAt: Date.now(), lastError: null }); } return; } @@ -312,7 +333,7 @@ class SyncStore { await this.#push(fitted, mergedHash); } else { await this.#saveConfig({ - lastSyncedAt: meta.at, + lastSyncedAt: Date.now(), lastRemoteHash: meta.h, lastLocalHash: mergedHash, pendingPush: false, @@ -324,7 +345,7 @@ class SyncStore { await this.#saveConfig({ lastError: syncErrorMessage(err) }); if (isRateLimited(err)) this.#reconcileIn(RATE_LIMIT_RETRY_MS); } finally { - this.#syncing = false; + this.#busy--; // Local data changed under the stores (same situation as an import). if (applied) location.reload(); } diff --git a/src/features/sync/persist/sync-config.ts b/src/features/sync/persist/sync-config.ts index 231e2fc..ca61b94 100644 --- a/src/features/sync/persist/sync-config.ts +++ b/src/features/sync/persist/sync-config.ts @@ -5,7 +5,8 @@ import { storage } from '#imports'; * this device's bookkeeping. */ export interface SyncConfig { enabled: boolean; - /** `exportedAt` of the last blob pushed or merged in; 0 = never synced. */ + /** When this device was last in agreement with the synced copy — a push, a + * merge, or a reconcile that found nothing to do; 0 = never synced. */ lastSyncedAt: number; /** Wall clock of the last local data change — this device's side of * "whose settings win" against a remote blob's clock. */ From e29f5bb0267f47def1744906ae7ec0c98b9d9051 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 15:50:43 +0200 Subject: [PATCH 10/26] feat: enhance backup handling with replacement imports and parameter alignment for merged songs --- src/core/persist/backup.ts | 16 ++++++--- src/core/persist/deletions.ts | 9 +++++- src/features/sync/panel/sync.svelte.ts | 19 ++++++++--- src/features/sync/persist/merge.test.ts | 43 +++++++++++++++++++++++++ src/features/sync/persist/merge.ts | 37 +++++++++++++++++---- 5 files changed, 108 insertions(+), 16 deletions(-) diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index 4770043..9e74733 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -5,7 +5,7 @@ import { parseBackupJson, type Backup, } from './backup-codec'; -import { mergeDeletions, pruneDeletions, reviveBackup } from './deletions'; +import { mergeDeletions, pruneDeletions, REPLACED_ALL, reviveBackup } from './deletions'; import { deletionsItem, eqPresetsItem, @@ -84,7 +84,9 @@ export function parseBackup(text: string): Backup { * merging into whatever is here. Host permissions are left untouched. * Deletion records are the one thing merged, not replaced: forgetting this * device's would let a sync merge resurrect what it had removed. Manual file - * imports use `asNew` to re-add their contents; sync restores keep their dates. + * imports use `asNew` to re-add their contents (and to date the removal of + * everything the file leaves out, which the other devices would otherwise + * union straight back); sync restores keep their dates. * * Track records are written first and the leftovers removed afterwards, never * the other way round. A sync merge calls this on every remote change, and @@ -94,10 +96,14 @@ export function parseBackup(text: string): Backup { * next restore removes. */ export async function restoreBackup(backup: Backup, { asNew = false } = {}): Promise { - const deletions = pruneDeletions( - mergeDeletions(await deletionsItem.getValue(), backup.deletions ?? {}), Date.now(), + const now = Date.now(); + let deletions = pruneDeletions( + mergeDeletions(await deletionsItem.getValue(), backup.deletions ?? {}), now, ); - if (asNew) backup = reviveBackup(backup, deletions); + if (asNew) { + deletions = { ...deletions, [REPLACED_ALL]: now }; + backup = reviveBackup(backup, deletions); + } await Promise.all([ settingsItem.setValue(backup.settings), uiPrefsItem.setValue(backup.uiPrefs), diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index 2e0ca00..6073aa9 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -7,7 +7,8 @@ * `updatedAt`, and wins its merge on that. * * One flat map, `key → when` (ms). Keys: `h:` a Recent row, - * `f:` a favorite, `h:*` "Clear Recent", `e:` an EQ preset. + * `f:` a favorite, `h:*` "Clear Recent", `e:` an EQ preset, + * `*` a replacement import (everything older, in every list). * Songs are named by `songKey` (URL + title, no duration — see * track-identity.ts) so the record reaches every copy of the song, however * its duration drifted, exactly like the merge matches them. A record older @@ -27,6 +28,12 @@ export const historyDeletion = (songKey: string) => `h:${songKey}`; export const favoriteDeletion = (songKey: string) => `f:${songKey}`; export const presetDeletion = (name: string) => `e:${name}`; export const HISTORY_CLEARED = 'h:*'; +/** A replacement import: everything older than this date is gone, in every + * list — Recent, favorites, presets and track records alike, which is what the + * import prompt promises. One record rather than one per dropped item: the + * whole library goes at once, and per-item records would hit `DELETION_CAP`. + * `reviveBackup` dates the file's own contents after it, so they survive. */ +export const REPLACED_ALL = '*'; export const DELETION_TTL_MS = 30 * 24 * 60 * 60_000; export const DELETION_CAP = 200; diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 3fd8b50..7ac81c5 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -293,7 +293,14 @@ class SyncStore { // Another device wrote since we last looked. const remote = await unpackBackup(base64); - const remoteWins = !localChanged || remote.exportedAt > this.config.lastChangedAt; + // Purely the clock: the remote copy wins if it was written after this + // device's last local change. Not "unless we changed something" — our + // own push records `lastLocalHash` as soon as the write resolves, and a + // write the browser later replaces (its own conflict resolution, or + // another device landing on top) would then read as "we changed + // nothing" and let an older copy overwrite the settings we just made. + // Losing that race now leaves local ahead, and `needPush` re-uploads. + const remoteWins = remote.exportedAt > this.config.lastChangedAt; const merged = mergeBackups(local, remote, remoteWins); const mergedHash = await contentHash(merged); const needApply = mergedHash !== localHash; @@ -322,12 +329,13 @@ class SyncStore { if (needApply) { // #applying stays set until the reload: nothing in between may - // schedule a push of what was just written. The reload is owed from - // here on, even if the restore fails halfway. + // schedule a push of what was just written. Owed only once the data + // is actually in — a restore that keeps failing (quota) would + // otherwise reload into the same reconcile, for ever. this.#applying = true; clearTimeout(this.#timer); - applied = true; await restoreBackup(merged); + applied = true; } if (needPush) { await this.#push(fitted, mergedHash); @@ -347,7 +355,10 @@ class SyncStore { } finally { this.#busy--; // Local data changed under the stores (same situation as an import). + // Nothing written, nothing to reload for: the error is on screen and + // this document goes on listening for local changes. if (applied) location.reload(); + else this.#applying = false; } } diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts index b140919..ebf6bc5 100644 --- a/src/features/sync/persist/merge.test.ts +++ b/src/features/sync/persist/merge.test.ts @@ -8,6 +8,7 @@ import { HISTORY_CLEARED, historyDeletion, presetDeletion, + REPLACED_ALL, reviveBackup, } from '../../../core/persist/deletions.ts'; import { DEFAULT_PARAMS, DEFAULT_SETTINGS, HISTORY_LIMIT } from '../../../core/model/defaults.ts'; @@ -182,6 +183,25 @@ test('a deleted chart beats stale analysis and later marker edits, but allows re } }); +test('a song’s two library copies come out of a merge with the same settings', () => { + const a = song(1); + // Starred here at +3; the other device, which hasn’t got the star yet, + // practised the song since and left it at 0. + const starred: FavoriteEntry = { ...fav(a, T0), params: row(a, T0, 3).params }; + const local = backup({ favorites: [starred], history: [row(a, T0, 3)] }); + const remote = backup({ history: [row(a, T0 + 1000, 0)] }); + for (const remoteWins of [false, true]) { + const merged = mergeBackups(local, remote, remoteWins, NOW); + assert.equal(merged.history[0].params.transpose, 0); + assert.equal(merged.favorites[0].params.transpose, 0, 'the favorite follows the newer row'); + } + // And the other way round, when the favorite is the fresher copy. + const practised = backup({ favorites: [{ ...fav(a, T0 + 1000), params: row(a, T0, 3).params }] }); + const merged = mergeBackups(practised, backup({ history: [row(a, T0, 0)] }), true, NOW); + assert.equal(merged.history[0].params.transpose, 3); + assert.equal(merged.favorites[0].params.transpose, 3); +}); + test('a manual import re-adds deleted rows and presets without reviving absent items', () => { const del = { [HISTORY_CLEARED]: NOW, [favoriteDeletion(songKey(song(1)))]: NOW, [presetDeletion('Mine')]: NOW }; const file = backup({ history: [row(song(1), T0)], favorites: [fav(song(1), T0)], eqPresets: [{ name: 'Mine', gains: [1] }] }); @@ -196,6 +216,29 @@ test('a manual import re-adds deleted rows and presets without reviving absent i assert.equal(file.history[0].updatedAt, T0, 'the original backup is unchanged'); }); +test('a replacement import removes what it leaves out, on the other devices too', () => { + const kept = song(1); + const dropped = song(2); + const file = backup({ history: [row(kept, T0)], tracks: [record(kept, T0, 1)] }); + const restored = reviveBackup(file, { [REPLACED_ALL]: NOW }, NOW); + const remote = backup({ + history: [row(dropped, T0)], + favorites: [fav(dropped, T0)], + eqPresets: [{ name: 'Gone', gains: [1] }], + tracks: [record(dropped, T0, 3)], + }); + for (const remoteWins of [false, true]) { + const merged = mergeBackups(restored, remote, remoteWins, NOW); + assert.deepEqual(keys(merged.history), [kept.key], 'only what the file carried'); + assert.deepEqual(keys(merged.favorites), []); + assert.deepEqual(merged.eqPresets, []); + assert.deepEqual(keys(merged.tracks), [kept.key]); + } + // What the other device did *after* the import is not the import's to drop. + const later = backup({ history: [row(dropped, NOW + 1)] }); + assert.equal(mergeBackups(restored, later, true, NOW).history.length, 2); +}); + test('ties go to the winning side', () => { const a = song(1); const local = backup({ history: [row(a, T0, 1)], tracks: [record(a, T0, 1)] }); diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index b98f502..6839f6a 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -9,6 +9,7 @@ import { mergeDeletions, presetDeletion, pruneDeletions, + REPLACED_ALL, } from '../../../core/persist/deletions.ts'; import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; @@ -27,6 +28,8 @@ import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; * an empty, dated chart is an explicit deletion and beats older analysis. * - Settings and UI prefs: the newer device's, as a whole. EQ presets: union * by name, the later save of a shared name, deletions honoured. + * - Last, the two library copies of a song (Recent and Favorites) are put + * back in step: they are written together and read as one. * * `remoteWins` breaks ties and picks the wholesale sections: true when the * remote copy was written after this device's last local change. The winning @@ -73,6 +76,25 @@ function capHistory(history: HistoryEntry[], favorites: FavoriteEntry[]): Histor return ordered.filter((e) => !cut.has(e)); } +/** Recent and Favorites each hold a copy of the same song's settings and are + * written together (`track-sync.#saveCurrent`); `findSavedEntry` and the chips + * in the list both take the two to agree. Merged apart they drift — star a song + * on one device, practise it unstarred on another, and only Recent hears about + * the new settings — so both copies take the newer one's params. Dates are left + * alone: this is a function of the merged lists, so every device works out the + * same answer from the same pair of copies. */ +function alignParams(history: HistoryEntry[], favorites: FavoriteEntry[]) { + const newest = new Map(); + for (const entry of [...history, ...favorites]) { + const best = newest.get(song(entry)); + if (!best || at(entry) > at(best)) newest.set(song(entry), entry); + } + return (entry: T): T => { + const best = newest.get(song(entry)); + return best && best !== entry ? { ...entry, params: best.params } : entry; + }; +} + export function mergeBackups( local: Backup, remote: Backup, @@ -82,7 +104,7 @@ export function mergeBackups( const [winner, loser] = remoteWins ? [remote, local] : [local, remote]; const deletions = pruneDeletions(mergeDeletions(local.deletions ?? {}, remote.deletions ?? {}), now); const history = unionNewest(winner.history, loser.history, song, (e) => - deletedSince(deletions, at(e), historyDeletion(song(e)), HISTORY_CLEARED), + deletedSince(deletions, at(e), historyDeletion(song(e)), HISTORY_CLEARED, REPLACED_ALL), ); const favorites = unionNewest( winner.favorites, @@ -92,12 +114,13 @@ export function mergeBackups( // `updatedAt`, which ordinary practice bumps (`touchFavorite` with new // params). Taking the later of the two would let a slider nudge on one // device outdate the other's unfavorite and re-star the song. - (f) => deletedSince(deletions, f.favoritedAt || at(f), favoriteDeletion(song(f))), + (f) => deletedSince(deletions, f.favoritedAt || at(f), favoriteDeletion(song(f)), REPLACED_ALL), (w, l): FavoriteEntry => ({ ...w, lastAccessedAt: Math.max(w.lastAccessedAt ?? 0, l.lastAccessedAt ?? 0), }), ); + const align = alignParams(history, favorites); return { ...local, exportedAt: Math.max(local.exportedAt ?? 0, remote.exportedAt ?? 0), @@ -107,15 +130,17 @@ export function mergeBackups( winner.eqPresets, loser.eqPresets, (p) => p.name, - (p) => deletedSince(deletions, at(p), presetDeletion(p.name)), + (p) => deletedSince(deletions, at(p), presetDeletion(p.name), REPLACED_ALL), ), - history: capHistory(history, favorites), - favorites, + history: capHistory(history.map(align), favorites), + favorites: favorites.map(align), tracks: unionNewest( winner.tracks, loser.tracks, (t) => t.identity.key, - undefined, + // No tombstone of their own (an emptied record is one), but a + // replacement import drops markers and snippets too. + (t) => deletedSince(deletions, at(t), REPLACED_ALL), (w, l) => ({ ...w, chordChart: !w.chordChart || (l.chordChart?.computedAt ?? 0) > w.chordChart.computedAt From 05c42357c7269dae12de24d22127c2f2a82e13ca Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 16:41:13 +0200 Subject: [PATCH 11/26] refactor: streamline merge logic and enhance data handling in sync process - Updated the merge function to improve the handling of history and favorites, ensuring that the most recent entries are prioritized and tombstones are properly pruned. - Simplified the union logic for merging backups, focusing on the latest updates and resolving conflicts based on timestamps and content. - Removed the lastChangedAt field from SyncConfig as it was deemed unnecessary for the current sync logic. - Enhanced the alignment of parameters between history and favorites to ensure consistency across devices. --- CLAUDE.md | 2 +- README.md | 10 +- src/core/model/types.ts | 28 +- src/core/persist/backup-codec.test.ts | 70 +++- src/core/persist/backup-codec.ts | 95 +++-- src/core/persist/backup.fixture.ts | 1 - src/core/persist/backup.ts | 41 +-- src/core/persist/deletions.ts | 174 +++++---- src/core/persist/storage.ts | 17 - src/features/eq/panel/eq-presets.svelte.ts | 7 +- src/features/eq/persist/eq-presets.ts | 20 +- .../library/panel/favorites.svelte.ts | 7 +- src/features/library/panel/history.svelte.ts | 8 +- src/features/library/persist/favorites.ts | 65 ++-- src/features/library/persist/history.ts | 46 ++- .../settings/panel/SettingsView.svelte | 6 +- .../settings/panel/settings.svelte.ts | 10 +- src/features/sync/panel/sync.svelte.ts | 19 +- src/features/sync/persist/fit.ts | 14 +- src/features/sync/persist/merge.test.ts | 331 ++++++++++-------- src/features/sync/persist/merge.ts | 217 ++++++------ src/features/sync/persist/sync-config.ts | 4 - 22 files changed, 712 insertions(+), 480 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6fb6fe9..cfbaa89 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,7 +97,7 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. - **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped. Timestamps stay in ms — they decide merges, and a delete and a re-add within one second must not collide. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. -- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs from the newer device, EQ presets by name) and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as dated **deletion records** ([core/persist/deletions.ts](src/core/persist/deletions.ts): `h:`, `f:`, `h:*`, `e:`, written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, TTL 30 d, cap 200) so a merge can't resurrect them. Songs are named by `songKey` (URL + title, **no duration** — the same identity the merge matches on, so a copy saved under a drifted duration is covered too); presets carry an optional `updatedAt` so a later save beats the deletion; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. +- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs each one dated item, EQ presets by name). `mergeBackups` is a **pure function of its two inputs** — no device clock, no "which side am I on" — so both devices compute the same result and the second has nothing left to push; that is also why every merged list is ordered by something the rows carry (`updatedAt`, the favorites' `orderedAt` rank, the preset name) rather than by whose array it was and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as **tombstones** ([core/persist/deletions.ts](src/core/persist/deletions.ts)): the removed row stays in its own list, marked `deleted` and dated, rather than being dropped — so the merge needs no deletion rules at all (last write wins, and a tombstone is a write), a re-add beats it by being newer, and a deletion can never reach an item it does not name. Written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, filtered out by the three panel stores so nothing downstream sees one, TTL 30 d. Songs are matched by `songKey` (URL + title, **no duration**), so a copy saved under a drifted duration is covered; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). A **replacement import** ([`replaceAll`](src/core/persist/deletions.ts)) is deliberately a *local* operation: it re-dates the file's rows and tombstones what **this device** held and the file omits — there is no "everything before now is gone" record, because a date range cannot be made safe across two devices' clocks. Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. diff --git a/README.md b/README.md index 36c4080..c3a82c4 100644 --- a/README.md +++ b/README.md @@ -187,10 +187,12 @@ charts, stay on the device that has them ([fit.ts](src/features/sync/persist/fit Two devices' copies are merged rather than overwritten ([merge.ts](src/features/sync/persist/merge.ts)): the more recently edited -version of each song wins, and a song you removed on one device stays removed -(deletions are dated, [deletions.ts](src/core/persist/deletions.ts)). Sync is -on by default; `Settings → Sync` turns it off, and `Delete synced data` empties -the synced copy. +version of each song wins, and a song you removed on one device stays removed — +a removal is kept as a dated, empty row rather than a gap, so a merge can tell +it from "never had it" ([deletions.ts](src/core/persist/deletions.ts)). The +merge reads nothing but the two copies, so both devices work out the same +answer. Sync is on by default; `Settings → Sync` turns it off, and `Delete +synced data` empties the synced copy. ## License diff --git a/src/core/model/types.ts b/src/core/model/types.ts index 95ff57b..6d77891 100644 --- a/src/core/model/types.ts +++ b/src/core/model/types.ts @@ -37,9 +37,13 @@ export interface EffectParams { export interface EqPreset { name: string; gains: number[]; - /** Last save. Absent on presets from before sync merged; reads as 0, so a - * dated deletion (deletions.ts) beats them and a later save beats it. */ + /** Last save — or, with `deleted`, the removal. Absent on presets from + * before sync merged; reads as 0, so any dated copy beats them. */ updatedAt?: number; + /** A tombstone: the preset was deleted at `updatedAt`, and the row is kept + * so a sync merge can tell "removed" from "never had it" (deletions.ts). + * `gains` is emptied — nothing reads them again. */ + deleted?: true; } export interface Marker { @@ -135,15 +139,28 @@ export interface HistoryEntry { thumbnailUrl?: string; pageUrl: string; createdAt: number; + /** Last save — or, with `deleted`, the removal. The one date a merge reads + * for this row (deletions.ts). */ updatedAt: number; + /** A tombstone: the row was removed at `updatedAt`, and is kept so a sync + * merge can tell "removed" from "never had it" (deletions.ts). The panel + * stores filter these out, so nothing downstream ever sees one. */ + deleted?: true; } /** A song the user starred (History → Favorites). Persists independently of * the LRU-capped Recent list. Stored array order = manual sort order. */ export interface FavoriteEntry extends HistoryEntry { + /** When the star was put there. Display only — the star and the unstar are + * the only writers of `updatedAt`, which is what the merge reads, so + * ordinary practice can no longer outdate another device's unfavorite. */ favoritedAt: number; /** Last time the track was opened or played, for "Last Accessed" sorting. */ lastAccessedAt: number; + /** When the manual order this row sits in was last set — by a drag + * (`setFavoritesOrder`) or by the star that put it on top. Decides whose + * order a merge keeps; absent on rows written before manual order synced. */ + orderedAt?: number; } export type FavoritesSort = 'lastAccessed' | 'title' | 'manual'; @@ -273,6 +290,11 @@ export interface Settings { /** Play an audible click on each count-in beat (accented downbeat). */ countInBeep: boolean; lastUsedParams?: EffectParams; + /** Last change, set by the settings store on every write. Settings travel + * between devices as one item with one date (see `merge.ts`); absent on + * settings written before that, which reads as 0. Never part of the file's + * settings diff — the codec carries it separately. */ + updatedAt?: number; } export type PanelId = @@ -300,4 +322,6 @@ export interface UiPrefs { accentHue: number; /** User overrides for the virtual Start/End marker labels (empty = default). */ boundaryLabels: { start: string; end: string }; + /** Last change — see `Settings.updatedAt`. */ + updatedAt?: number; } diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index e0cbacc..b548d40 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -236,7 +236,7 @@ test('sync: customized settings or prefs count even without a library', () => { assert.equal(isEmptyBackup(backup()), true); assert.equal(isEmptyBackup(backup({ settings: { ...DEFAULT_SETTINGS, theme: 'dark' } })), false); assert.equal(isEmptyBackup(backup({ uiPrefs: { ...DEFAULT_UI_PREFS, boundaryLabels: { start: 'Intro', end: '' } } })), false); - assert.equal(isEmptyBackup(backup({ deletions: { 'h:*': Date.now() } })), false); + assert.equal(isEmptyBackup(backup({ history: [entry(ytSong, { deleted: true })] })), false); }); test('settings: a keymap saved before an action existed is backfilled', () => { @@ -506,15 +506,56 @@ test('encode is deterministic regardless of track enumeration order', () => { assert.deepEqual(encodeBackup(shuffled), encodeBackup(b)); }); -test('deletion records travel in ms and are omitted when empty', () => { - assert.equal('del' in encodeBackup(backup()), false); - const b = backup({ deletions: { 'h:abc:230': 1_757_112_345_678, 'h:*': 1_757_000_000_000 } }); +test('a tombstone travels as the song and the date, and nothing else', () => { + const b = backup({ + history: [entry(ytSong, { deleted: true, updatedAt: 1_757_112_345_678 })], + favorites: [favorite(siteSong, { deleted: true, updatedAt: 1_757_112_345_679 })], + eqPresets: [{ name: 'Mine', gains: [3, -2], updatedAt: 1_757_000_000_000, deleted: true }], + }); const enc = encodeBackup(b); - assert.deepEqual(enc.del, { 'h:*': 1_757_000_000_000, 'h:abc:230': 1_757_112_345_678 }); - assert.deepEqual(roundTrip(b).deletions, b.deletions); - const v1 = JSON.parse(JSON.stringify(backup())); - delete v1.deletions; - assert.deepEqual(parseBackupJson(v1).deletions, {}); + assert.deepEqual(enc.h, [{ i: 0, at: 1_757_112_345_678, x: 1 }], 'params and URLs dropped'); + assert.equal('p' in enc.f[0], false); + assert.deepEqual(enc.eq, [['Mine', [], 1_757_000_000_000, 1]], 'gains dropped'); + + const back = roundTrip(b); + assert.equal(back.history[0].deleted, true); + assert.equal(back.history[0].updatedAt, 1_757_112_345_678); + assert.equal(back.favorites[0].deleted, true); + assert.deepEqual(back.eqPresets[0], { + name: 'Mine', + gains: [], + updatedAt: 1_757_000_000_000, + deleted: true, + }); + // Idempotent like every other rounding: what was stripped stays stripped, so + // two devices holding the same tombstone hash the same. + assert.deepEqual(encodeBackup(back), enc); +}); + +test('settings and prefs carry their own date, outside the diff', () => { + const b = backup({ + settings: { ...DEFAULT_SETTINGS, updatedAt: 1_757_000_000_000 }, + uiPrefs: { ...DEFAULT_UI_PREFS, updatedAt: 1_757_000_000_001 }, + }); + const enc = encodeBackup(b); + assert.deepEqual(enc.s, {}, 'the date is not a setting'); + assert.equal(enc.sat, 1_757_000_000_000); + assert.equal(enc.uat, 1_757_000_000_001); + assert.equal(isEmptyBackup(b), true, 'a date on the defaults is not data'); + assert.equal(roundTrip(b).settings.updatedAt, 1_757_000_000_000); + assert.equal(roundTrip(b).uiPrefs.updatedAt, 1_757_000_000_001); +}); + +test('a version 2 file still reads; its del map is dropped', () => { + const v2 = { + ...JSON.parse(JSON.stringify(encodeBackup(library(3)))), + version: 2, + del: { 'h:abc:230': 1_757_112_345_678 }, + }; + const back = parseBackupJson(v2); + assert.equal(back.history.length, 3); + assert.equal(back.history.some((e) => e.deleted), false); + assert.equal('deletions' in back, false); }); test('exportedAt is kept; appVersion is dropped', () => { @@ -536,11 +577,12 @@ test('a verbose v1 file still parses and is backfilled', () => { assert.equal(back.appVersion, '1.0.3'); }); -test('a v2 file routes through the codec; anything newer or foreign is refused', () => { - const v2 = JSON.parse(JSON.stringify(encodeBackup(library(2)))); - assert.equal(parseBackupJson(v2).history.length, 2); - assert.throws(() => parseBackupJson({ ...v2, version: 3 }), /newer version/); - assert.throws(() => parseBackupJson({ ...v2, format: 'other' }), /isn't a Note by Note backup/); +test('a compact file routes through the codec; anything newer or foreign is refused', () => { + const compact = JSON.parse(JSON.stringify(encodeBackup(library(2)))); + assert.equal(parseBackupJson(compact).history.length, 2); + assert.equal(parseBackupJson({ ...compact, version: 2 }).history.length, 2, 'v2 too'); + assert.throws(() => parseBackupJson({ ...compact, version: COMPACT_VERSION + 1 }), /newer version/); + assert.throws(() => parseBackupJson({ ...compact, format: 'other' }), /isn't a Note by Note backup/); assert.throws(() => parseBackupJson('nope'), /isn't a Note by Note backup/); assert.throws( () => parseBackupJson({ format: BACKUP_FORMAT, version: 1, settings: {} }), diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 4dca2bc..dd9f92a 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -5,7 +5,6 @@ import { } from '../model/defaults.ts'; import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; import { identityKey } from '../model/track-identity.ts'; -import { normalizeDeletions, type Deletions } from './deletions.ts'; import type { ChordChart, ChordSegment, @@ -53,8 +52,17 @@ export const BACKUP_FORMAT = 'note-by-note-backup'; * before the compact format. Still accepted on import. */ export const BACKUP_VERSION = 1; -/** The compact shape below. `parseBackupJson` rejects anything newer. */ -export const COMPACT_VERSION = 2; +/** The compact shape below. `parseBackupJson` rejects anything newer. + * + * 3 added tombstones (`x`) to Recent, Favorites and the EQ presets, the + * favorites' `orderedAt` (`oa`) and the settings/prefs dates (`sat`/`uat`), + * and dropped the `del` map that 2 carried. A 2 file still reads — it simply + * has no tombstones, and its `del` records are not translatable (a deletion + * key names a song by hash, and a tombstone has to *be* the row). */ +export const COMPACT_VERSION = 3; + +/** The oldest compact shape still readable. */ +export const COMPACT_MIN_VERSION = 2; /** Everything a user owns, in one file. Host permissions are deliberately out: * they live in the browser's permission store, and only a prompt can grant @@ -71,9 +79,6 @@ export interface Backup { eqPresets: EqPreset[]; /** Per-track markers and snippets, one entry per saved track. */ tracks: TrackData[]; - /** What was deleted and when — see `deletions.ts`. Absent in files from - * before sync merged; `{}` then. */ - deletions: Deletions; } // --------------------------------------------------------------------------- @@ -124,6 +129,9 @@ export interface CompactEntry { url?: string; /** thumbnailUrl, when not derivable from the page URL. */ th?: string; + /** A tombstone (`deletions.ts`): the row was removed at `at`. Nothing else + * is carried — a deletion is a song and a date. */ + x?: 1; } export interface CompactFavorite extends CompactEntry { @@ -131,10 +139,16 @@ export interface CompactFavorite extends CompactEntry { fa: number; /** lastAccessedAt, ms. */ la: number; + /** orderedAt, ms; omitted when the row predates manual order syncing. */ + oa?: number; } -/** `[name, gains, updatedAt?]`. */ -export type CompactEqPreset = [string, number[]] | [string, number[], number]; +/** `[name, gains, updatedAt?]`, or `[name, [], updatedAt, 1]` for a tombstone + * (`deletions.ts`). */ +export type CompactEqPreset = + | [string, number[]] + | [string, number[], number] + | [string, number[], number, 1]; /** `[t_ms, label?]` — label omitted when empty. */ export type CompactMarker = [number] | [number, string]; @@ -194,14 +208,16 @@ export interface CompactBackup { s: Record; /** UI prefs that differ from the defaults. */ u: Record; + /** settings.updatedAt, ms; omitted when never dated. */ + sat?: number; + /** uiPrefs.updatedAt, ms; omitted when never dated. */ + uat?: number; /** `[name, gains, updatedAt?]` per saved EQ preset. */ eq: CompactEqPreset[]; songs: CompactSong[]; h: CompactEntry[]; f: CompactFavorite[]; t: CompactTrack[]; - /** Deletion records (ms); omitted when there are none. */ - del?: Record; } // --------------------------------------------------------------------------- @@ -359,6 +375,9 @@ export function decodeParams(raw: unknown, section: string): EffectParams { export function encodeSettings(settings: Settings): Record { const { lastUsedParams, ...rest } = settings; + // Not a setting: the date rides beside the diff (`sat`), so a device that + // changed a setting and changed it back still encodes as empty. + delete (rest as { updatedAt?: number }).updatedAt; const out = diffPlain(rest, { ...DEFAULT_SETTINGS }); if (lastUsedParams) out.lp = encodeParams(lastUsedParams) ?? {}; return out; @@ -373,8 +392,10 @@ export function decodeSettings(raw: unknown): Settings { } export function encodeUiPrefs(uiPrefs: UiPrefs): Record { + const rest = { ...uiPrefs }; + delete rest.updatedAt; return diffPlain( - uiPrefs as unknown as Record, + rest as unknown as Record, DEFAULT_UI_PREFS as unknown as Record, ); } @@ -468,6 +489,9 @@ function songAt(songs: TrackIdentity[], index: unknown, section: string): TrackI function encodeEntry(entry: HistoryEntry, songs: SongTable): CompactEntry { const out: CompactEntry = { i: songs.add(entry.identity), at: stamp(entry.updatedAt) }; + // A tombstone is the song and the date it went; the rest was only ever there + // to be shown, and nothing shows a removed row. + if (entry.deleted) return { ...out, x: 1 }; const params = encodeParams(entry.params ?? DEFAULT_PARAMS); if (params) out.p = params; const pageUrl = entry.pageUrl ?? ''; @@ -492,24 +516,29 @@ function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): His updatedAt, }; if (thumbnailUrl !== undefined) entry.thumbnailUrl = thumbnailUrl; + if (c.x === 1) entry.deleted = true; return entry; } function encodeFavorite(entry: FavoriteEntry, songs: SongTable): CompactFavorite { - return { + const out: CompactFavorite = { ...encodeEntry(entry, songs), fa: stamp(entry.favoritedAt), la: stamp(entry.lastAccessedAt), }; + if (entry.orderedAt) out.oa = stamp(entry.orderedAt); + return out; } function decodeFavorite(raw: unknown, songs: TrackIdentity[]): FavoriteEntry { const c = rec(raw, 'favorites'); - return { + const favorite: FavoriteEntry = { ...decodeEntry(c, songs, 'favorites'), favoritedAt: num(c.fa, 'favorites'), lastAccessedAt: num(c.la, 'favorites'), }; + if (c.oa !== undefined) favorite.orderedAt = num(c.oa, 'favorites'); + return favorite; } // --------------------------------------------------------------------------- @@ -697,21 +726,23 @@ function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { // --------------------------------------------------------------------------- // Whole backup -/** `[name, gains, updatedAt?]`. */ +/** `[name, gains, updatedAt?]`, or `[name, [], updatedAt, 1]` tombstoned. */ function encodeEqPreset(preset: EqPreset): CompactEqPreset { const name = preset.name ?? ''; + if (preset.deleted) return [name, [], stamp(preset.updatedAt ?? 0), 1]; const gains = (preset.gains ?? []).map(round2); return preset.updatedAt ? [name, gains, stamp(preset.updatedAt)] : [name, gains]; } function decodeEqPreset(raw: unknown): EqPreset { const r = arr(raw, 'eqPresets'); - if (r.length < 2 || r.length > 3) throw damaged('eqPresets'); + if (r.length < 2 || r.length > 4) throw damaged('eqPresets'); const preset: EqPreset = { name: str(r[0], 'eqPresets'), gains: arr(r[1], 'eqPresets').map((g) => num(g, 'eqPresets')), }; - if (r.length === 3) preset.updatedAt = num(r[2], 'eqPresets'); + if (r.length >= 3) preset.updatedAt = num(r[2], 'eqPresets'); + if (r.length === 4) preset.deleted = true; return preset; } @@ -738,39 +769,46 @@ export function encodeBackup(backup: Backup): CompactBackup { f, t, }; - const deletions = Object.entries(normalizeDeletions(backup.deletions)).sort(([a], [b]) => - a < b ? -1 : a > b ? 1 : 0, - ); - if (deletions.length) out.del = Object.fromEntries(deletions.map(([k, when]) => [k, stamp(when)])); + if (backup.settings.updatedAt) out.sat = stamp(backup.settings.updatedAt); + if (backup.uiPrefs.updatedAt) out.uat = stamp(backup.uiPrefs.updatedAt); return out; } -/** Defaults alone need no sync blob; customized settings do, even without songs. */ +/** Defaults alone need no sync blob; customized settings do, even without + * songs. A tombstone counts as a song — it is the only record that the row + * was removed, and seeding without it would resurrect the row elsewhere. */ export function isEmptyBackup(backup: Backup): boolean { - const { songs, eq, s, u, del = {} } = encodeBackup(backup); + const { songs, eq, s, u } = encodeBackup(backup); return songs.length === 0 && eq.length === 0 && - Object.keys(s).length === 0 && Object.keys(u).length === 0 && Object.keys(del).length === 0; + Object.keys(s).length === 0 && Object.keys(u).length === 0; } /** Reads a compact (v2) backup, or throws an `Error` whose message is safe to * show the user. Unknown keys are ignored so the format can grow. */ export function decodeBackup(raw: unknown): Backup { - if (!isRecord(raw) || raw.format !== BACKUP_FORMAT || raw.version !== COMPACT_VERSION) { + const version = isRecord(raw) && typeof raw.version === 'number' ? raw.version : 0; + if ( + !isRecord(raw) || raw.format !== BACKUP_FORMAT || + version < COMPACT_MIN_VERSION || version > COMPACT_VERSION + ) { throw new Error("That file isn't a Note by Note backup."); } const songs = decodeSongs(raw.songs); + const settings = decodeSettings(raw.s); + const uiPrefs = decodeUiPrefs(raw.u); + if (typeof raw.sat === 'number' && Number.isFinite(raw.sat)) settings.updatedAt = raw.sat; + if (typeof raw.uat === 'number' && Number.isFinite(raw.uat)) uiPrefs.updatedAt = raw.uat; return { format: BACKUP_FORMAT, version: COMPACT_VERSION, exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at : 0, appVersion: '', - settings: decodeSettings(raw.s), - uiPrefs: decodeUiPrefs(raw.u), + settings, + uiPrefs, history: arr(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), favorites: arr(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), - deletions: normalizeDeletions(raw.del), }; } @@ -795,7 +833,6 @@ function normalizeV1(raw: Record): Backup { favorites: keyedArr(raw.favorites, 'favorites'), eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], tracks: keyedArr(raw.tracks, 'tracks'), - deletions: normalizeDeletions(raw.deletions), }; } @@ -810,5 +847,5 @@ export function parseBackupJson(raw: unknown): Backup { if (typeof raw.version !== 'number' || raw.version > COMPACT_VERSION) { throw new Error('That backup was made by a newer version of Note by Note.'); } - return raw.version === COMPACT_VERSION ? decodeBackup(raw) : normalizeV1(raw); + return raw.version >= COMPACT_MIN_VERSION ? decodeBackup(raw) : normalizeV1(raw); } diff --git a/src/core/persist/backup.fixture.ts b/src/core/persist/backup.fixture.ts index 88a708c..8b8420e 100644 --- a/src/core/persist/backup.fixture.ts +++ b/src/core/persist/backup.fixture.ts @@ -13,7 +13,6 @@ export function backupFixture(patch: Partial = {}): Backup { favorites: [], eqPresets: [], tracks: [], - deletions: {}, ...patch, }; } diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index 9e74733..dd26eab 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -5,9 +5,8 @@ import { parseBackupJson, type Backup, } from './backup-codec'; -import { mergeDeletions, pruneDeletions, REPLACED_ALL, reviveBackup } from './deletions'; +import { pruneTombstones, replaceAll } from './deletions'; import { - deletionsItem, eqPresetsItem, favoritesItem, historyItem, @@ -30,7 +29,7 @@ async function loadAllTrackData(): Promise { } export async function createBackup(): Promise { - const [settings, uiPrefs, history, favorites, eqPresets, tracks, deletions] = + const [settings, uiPrefs, history, favorites, eqPresets, tracks] = await Promise.all([ settingsItem.getValue(), uiPrefsItem.getValue(), @@ -38,7 +37,6 @@ export async function createBackup(): Promise { favoritesItem.getValue(), eqPresetsItem.getValue(), loadAllTrackData(), - deletionsItem.getValue(), ]); return { format: BACKUP_FORMAT, @@ -51,7 +49,6 @@ export async function createBackup(): Promise { favorites, eqPresets, tracks, - deletions, }; } @@ -82,11 +79,12 @@ export function parseBackup(text: string): Backup { * Replaces every stored value with the backup's, dropping data the file does * not carry — a restore reproduces the machine it came from rather than * merging into whatever is here. Host permissions are left untouched. - * Deletion records are the one thing merged, not replaced: forgetting this - * device's would let a sync merge resurrect what it had removed. Manual file - * imports use `asNew` to re-add their contents (and to date the removal of - * everything the file leaves out, which the other devices would otherwise - * union straight back); sync restores keep their dates. + * + * A manual file import passes `asNew`, which turns the file into "this is the + * library now": its rows are re-dated and everything this device held that the + * file leaves out becomes a tombstone, so a sync merge can't union it straight + * back (`replaceAll`, deletions.ts). A sync restore is already a merged result + * and keeps its dates. Expired tombstones are dropped on the way past. * * Track records are written first and the leftovers removed afterwards, never * the other way round. A sync merge calls this on every remote change, and @@ -97,21 +95,14 @@ export function parseBackup(text: string): Backup { */ export async function restoreBackup(backup: Backup, { asNew = false } = {}): Promise { const now = Date.now(); - let deletions = pruneDeletions( - mergeDeletions(await deletionsItem.getValue(), backup.deletions ?? {}), now, - ); - if (asNew) { - deletions = { ...deletions, [REPLACED_ALL]: now }; - backup = reviveBackup(backup, deletions); - } + const next = asNew ? replaceAll(backup, await createBackup(), now) : backup; await Promise.all([ - settingsItem.setValue(backup.settings), - uiPrefsItem.setValue(backup.uiPrefs), - historyItem.setValue(backup.history), - favoritesItem.setValue(backup.favorites), - eqPresetsItem.setValue(backup.eqPresets), - deletionsItem.setValue(deletions), - ...backup.tracks.map(saveTrackData), + settingsItem.setValue(next.settings), + uiPrefsItem.setValue(next.uiPrefs), + historyItem.setValue(pruneTombstones(next.history, now)), + favoritesItem.setValue(pruneTombstones(next.favorites, now)), + eqPresetsItem.setValue(pruneTombstones(next.eqPresets, now)), + ...next.tracks.map(saveTrackData), ]); - await removeTrackDataExcept(new Set(backup.tracks.map((t) => t.identity.key))); + await removeTrackDataExcept(new Set(next.tracks.map((t) => t.identity.key))); } diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index 6073aa9..02b3554 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -1,93 +1,119 @@ /** - * Deletion records ("tombstones"): what the user removed, and when. Without - * them a Recent row, a favorite or an EQ preset deleted on one device would - * come straight back from another device's copy the next time the two merge — - * a merge is a union, and it cannot tell "never had it" from "removed it". - * Track records need none: an emptied record still exists, with a newer - * `updatedAt`, and wins its merge on that. + * How a deletion travels between devices. * - * One flat map, `key → when` (ms). Keys: `h:` a Recent row, - * `f:` a favorite, `h:*` "Clear Recent", `e:` an EQ preset, - * `*` a replacement import (everything older, in every list). - * Songs are named by `songKey` (URL + title, no duration — see - * track-identity.ts) so the record reaches every copy of the song, however - * its duration drifted, exactly like the merge matches them. A record older - * than the item it names (the song was played again, the preset saved again) - * is ignored, so re-adding always works. Records expire after a month and are - * capped, newest kept — long enough for any device that will ever sync again - * to see them. + * A merge is a union, and a union cannot tell "never had it" from "removed + * it". So a removed row is not dropped: it stays in its own list, marked + * `deleted` and dated when the user removed it. That is a tombstone, and it + * is an ordinary row — which is the whole point: + * + * - the merge needs no deletion rules of its own. Last write wins, and a + * tombstone is a write (`merge.ts`); + * - re-adding works by itself: the new row simply out-dates the tombstone; + * - a deletion can only ever reach the item it *is*, so it cannot misfire + * on a song, preset or record it was never about; + * - the codec, the fit and the backup file carry it with everything else. + * + * Track records need no tombstone: an emptied record still exists, with a + * newer date, and wins its merge on that. + * + * The panel stores filter tombstones out, so nothing downstream sees one. + * They expire after a month — long enough for any device that will ever sync + * again to have seen them — and `pruneTombstones` drops them at merge and + * restore time. * * Pure and DOM-free (relative `.ts` imports; runs under `node --test`). */ import type { Backup } from './backup-codec.ts'; - -export type Deletions = Record; - -export const historyDeletion = (songKey: string) => `h:${songKey}`; -export const favoriteDeletion = (songKey: string) => `f:${songKey}`; -export const presetDeletion = (name: string) => `e:${name}`; -export const HISTORY_CLEARED = 'h:*'; -/** A replacement import: everything older than this date is gone, in every - * list — Recent, favorites, presets and track records alike, which is what the - * import prompt promises. One record rather than one per dropped item: the - * whole library goes at once, and per-item records would hit `DELETION_CAP`. - * `reviveBackup` dates the file's own contents after it, so they survive. */ -export const REPLACED_ALL = '*'; +import { songKey } from '../model/track-identity.ts'; +import type { HistoryEntry } from '../model/types'; export const DELETION_TTL_MS = 30 * 24 * 60 * 60_000; -export const DELETION_CAP = 200; -/** Whether any of `keys` carries a deletion dated at or after `since` — the - * item's own save time, so a later re-add always beats the record. */ -export function deletedSince(deletions: Deletions, since: number, ...keys: string[]): boolean { - return keys.some((key) => { - const when = deletions[key] ?? 0; - return when > 0 && when >= since; - }); +/** A row a merge dates and can tombstone. */ +export interface Deletable { + updatedAt?: number; + deleted?: true; } -/** Newest date per key. */ -export function mergeDeletions(a: Deletions, b: Deletions): Deletions { - const out: Deletions = { ...a }; - for (const [key, when] of Object.entries(b)) out[key] = Math.max(out[key] ?? 0, when); - return out; -} +/** The one date every merge decision reads: when this row last changed — was + * written, or was removed. */ +export const at = (item: Deletable) => item.updatedAt ?? 0; -/** Drops records older than the TTL and, past the cap, the oldest. */ -export function pruneDeletions(deletions: Deletions, now: number): Deletions { - const kept = Object.entries(deletions) - .filter(([, when]) => typeof when === 'number' && Number.isFinite(when) && now - when < DELETION_TTL_MS) - .sort(([, a], [, b]) => b - a) - .slice(0, DELETION_CAP); - return Object.fromEntries(kept); -} +export const isLive = (item: Deletable) => item.deleted !== true; -/** Anything a file or a remote copy claims to be deletions, made safe. */ -export function normalizeDeletions(raw: unknown): Deletions { - if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return {}; - return Object.fromEntries(Object.entries(raw).filter( - ([, when]) => typeof when === 'number' && Number.isFinite(when) && when > 0, - )); -} +/** The row, kept but marked removed as of `now`. */ +export const tombstone = (item: T, now: number): T => ({ + ...item, + updatedAt: now, + deleted: true, +}); + +/** Drops tombstones past the TTL. Live rows are never touched. */ +export const pruneTombstones = (list: T[], now: number): T[] => + list.filter((item) => isLive(item) || now - at(item) < DELETION_TTL_MS); + +const song = (entry: HistoryEntry) => songKey(entry.identity); + +/** + * A replacement import, as items: the file's contents dated now, plus a + * tombstone for everything this device held that the file leaves out. + * + * Deliberately a *local* operation — it removes what this device had, which + * is what the import prompt promises. A song only another device knows about + * is not this import's to delete, and there is no "everything before now is + * gone" record that could reach one: a date range cannot be made safe across + * two clocks, and every list dates its rows for its own reasons. + * + * Track records are not tombstoned either. The songs that named them are, so + * the records go quiet; a record another device still holds costs a few bytes + * (the first thing `fit.ts` cuts) and erring towards keeping markers is the + * right way round. + * + * `current` is this device's data as `createBackup` reads it. + */ +export function replaceAll(file: Backup, current: Backup, now = Date.now()): Backup { + const held = [...current.history, ...current.favorites, ...current.eqPresets]; + // Revived rows have to out-date every tombstone in play, this device's and + // any another device's clock dated ahead of ours. + const revivedAt = + Math.max(now, file.exportedAt ?? 0, ...held.filter((i) => !isLive(i)).map(at)) + 1; + + /** Tombstones for the live rows of `before` that `kept` does not carry, + * plus the tombstones `before` already had for anything else. */ + const removed = (before: T[], kept: T[], id: (item: T) => string): T[] => { + const keys = new Set(kept.map(id)); + return before + .filter((item) => !keys.has(id(item))) + .map((item) => (isLive(item) ? tombstone(item, now) : item)); + }; + const revived = (list: T[]): T[] => + list.map((item) => ({ ...item, updatedAt: revivedAt })); + const byName = (preset: { name: string }) => preset.name; -/** A manual import is a new edit. Keep deletion records for absent items, - * but date restored items after them so the next sync cannot delete them. */ -export function reviveBackup(backup: Backup, deletions: Deletions, now = Date.now()): Backup { - const updatedAt = Math.max(now, backup.exportedAt, ...Object.values(deletions)) + 1; return { - ...backup, - exportedAt: updatedAt, - history: backup.history.map((e) => ({ ...e, updatedAt })), - // `favoritedAt` too: it is what the merge dates a favorite by, so leaving - // it behind would let a deletion record outrank the star we just re-added. - favorites: backup.favorites.map((e) => ({ ...e, updatedAt, favoritedAt: updatedAt })), - eqPresets: backup.eqPresets.map((e) => ({ ...e, updatedAt })), - tracks: backup.tracks.map((t) => ({ - ...t, - updatedAt, - chordChart: t.chordChart ? { ...t.chordChart, computedAt: updatedAt } : t.chordChart, + ...file, + exportedAt: revivedAt, + settings: { ...file.settings, updatedAt: revivedAt }, + uiPrefs: { ...file.uiPrefs, updatedAt: revivedAt }, + history: [ + ...revived(file.history), + ...removed(current.history, file.history, song), + ], + favorites: [ + // The imported order is the newest statement about it, too. + ...revived(file.favorites).map((f) => ({ ...f, orderedAt: revivedAt })), + ...removed(current.favorites, file.favorites, song), + ], + eqPresets: [ + ...revived(file.eqPresets), + ...removed(current.eqPresets, file.eqPresets, byName), + ], + tracks: revived(file.tracks).map((track) => ({ + ...track, + chordChart: track.chordChart + ? { ...track.chordChart, computedAt: revivedAt } + : track.chordChart, })), - deletions, }; } diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index cb74b29..4a8d2cd 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -1,6 +1,5 @@ import { storage, type StorageItemKey, type WxtStorageItem } from '#imports'; import { DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults'; -import { pruneDeletions, type Deletions } from './deletions'; import type { EqPreset, FavoriteEntry, @@ -77,22 +76,6 @@ export const grantedOriginsItem = defineItem('local:grantedOrigins', { fallback: [], }); -/** What the user deleted and when, so a sync merge doesn't bring it back — - * see `deletions.ts`. Part of the backup; never part of "Reset Settings". */ -export const deletionsItem = defineItem('local:deletions', { - fallback: {}, -}); - -/** Dates a deletion (`historyDeletion(key)`, `favoriteDeletion(key)`, - * `HISTORY_CLEARED`) at now, pruning expired records on the way. */ -export async function recordDeletion(...keys: string[]): Promise { - const now = Date.now(); - const current = await deletionsItem.getValue(); - const next = { ...current }; - for (const key of keys) next[key] = now; - await deletionsItem.setValue(pruneDeletions(next, now)); -} - /** Per-track markers/snippets, keyed by TrackIdentity.key. */ export function trackDataKey(key: string) { return `local:track:${key}` as const; diff --git a/src/features/eq/panel/eq-presets.svelte.ts b/src/features/eq/panel/eq-presets.svelte.ts index d50078a..e864408 100644 --- a/src/features/eq/panel/eq-presets.svelte.ts +++ b/src/features/eq/panel/eq-presets.svelte.ts @@ -1,6 +1,7 @@ import { BUILTIN_EQ_PRESETS, EQ_BANDS } from '../../../core/model/defaults'; import type { EqPreset } from '../../../core/model/types'; import { deleteEqPreset, saveEqPreset } from '../persist/eq-presets'; +import { isLive } from '../../../core/persist/deletions'; import { eqPresetsItem } from '../../../core/persist/storage'; /** Slider gains are multiples of 0.5 dB, so anything closer than this is the @@ -8,12 +9,14 @@ import { eqPresetsItem } from '../../../core/persist/storage'; const GAIN_EPSILON = 0.01; class EqPresetsStore { + /** Live presets only — the stored list also carries deletion tombstones + * (`deletions.ts`). */ saved = $state([]); async init() { - this.saved = await eqPresetsItem.getValue(); + this.saved = (await eqPresetsItem.getValue()).filter(isLive); eqPresetsItem.watch((value) => { - this.saved = value ?? []; + this.saved = (value ?? []).filter(isLive); }); } diff --git a/src/features/eq/persist/eq-presets.ts b/src/features/eq/persist/eq-presets.ts index 6093260..45d8acf 100644 --- a/src/features/eq/persist/eq-presets.ts +++ b/src/features/eq/persist/eq-presets.ts @@ -1,10 +1,12 @@ -import { presetDeletion } from '../../../core/persist/deletions'; -import { eqPresetsItem, recordDeletion } from '../../../core/persist/storage'; +import type { EqPreset } from '../../../core/model/types'; +import { tombstone } from '../../../core/persist/deletions'; +import { eqPresetsItem } from '../../../core/persist/storage'; /** Save the current curve under `name`. An existing preset with that name is * replaced in place — that's the edit and rename path, so there's no separate - * UI for either. Stamped so a sync merge can tell this save from a deletion - * of the same name on another device (see deletions.ts). */ + * UI for either, and it is also what un-deletes a name this device or another + * one had tombstoned. Stamped so a sync merge can tell this save from a + * deletion of the same name on another device (see deletions.ts). */ export async function saveEqPreset(name: string, gains: number[]): Promise { const list = await eqPresetsItem.getValue(); const index = list.findIndex((p) => p.name === name); @@ -18,10 +20,12 @@ export async function saveEqPreset(name: string, gains: number[]): Promise } } -/** Dated (`deletions.ts`) so a sync merge with another device's copy doesn't - * bring the preset back. */ +/** The preset stays as a tombstone (`deletions.ts`), emptied of its gains, so + * a sync merge with another device's copy doesn't bring it back. */ export async function deleteEqPreset(name: string): Promise { const list = await eqPresetsItem.getValue(); - await eqPresetsItem.setValue(list.filter((p) => p.name !== name)); - await recordDeletion(presetDeletion(name)); + const now = Date.now(); + await eqPresetsItem.setValue( + list.map((p) => (p.name === name ? tombstone({ name, gains: [] }, now) : p)), + ); } diff --git a/src/features/library/panel/favorites.svelte.ts b/src/features/library/panel/favorites.svelte.ts index eeef566..f94cfb8 100644 --- a/src/features/library/panel/favorites.svelte.ts +++ b/src/features/library/panel/favorites.svelte.ts @@ -5,6 +5,7 @@ import { setFavoritesOrder, } from '../persist/favorites'; import { isSameTrack } from '../../../core/model/track-identity'; +import { isLive } from '../../../core/persist/deletions'; import { favoritesItem } from '../../../core/persist/storage'; /** Every write below is fired from a click handler as a floating promise, and @@ -19,12 +20,14 @@ async function write(what: string, run: () => Promise): Promise { } class FavoritesStore { + /** Live rows only — the stored list also carries unstar tombstones + * (`deletions.ts`). */ entries = $state([]); async init() { - this.entries = await favoritesItem.getValue(); + this.entries = (await favoritesItem.getValue()).filter(isLive); favoritesItem.watch((value) => { - this.entries = value ?? []; + this.entries = (value ?? []).filter(isLive); }); } diff --git a/src/features/library/panel/history.svelte.ts b/src/features/library/panel/history.svelte.ts index 93d06ae..960a122 100644 --- a/src/features/library/panel/history.svelte.ts +++ b/src/features/library/panel/history.svelte.ts @@ -1,17 +1,21 @@ import type { HistoryEntry } from '../../../core/model/types'; import { clearHistory, dedupeHistory, removeHistoryEntry } from '../persist/history'; +import { isLive } from '../../../core/persist/deletions'; import { historyItem } from '../../../core/persist/storage'; class HistoryStore { + /** Live rows only: the stored list also carries tombstones for what the + * user removed, which exist purely so a sync merge can't resurrect it + * (`deletions.ts`). Filtering here is what keeps them out of every screen. */ entries = $state([]); async init() { // Rows saved before dedupe-on-write can already be duplicated; collapse // them once, on the way in, so the list the user sees is the stored one. await dedupeHistory(); - this.entries = await historyItem.getValue(); + this.entries = (await historyItem.getValue()).filter(isLive); historyItem.watch((value) => { - this.entries = value ?? []; + this.entries = (value ?? []).filter(isLive); }); } diff --git a/src/features/library/persist/favorites.ts b/src/features/library/persist/favorites.ts index 2b5d85b..f3ca5c9 100644 --- a/src/features/library/persist/favorites.ts +++ b/src/features/library/persist/favorites.ts @@ -1,40 +1,56 @@ -import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; -import { isSameTrack, songKey } from '../../../core/model/track-identity'; -import { favoriteDeletion } from '../../../core/persist/deletions'; -import { favoritesItem, recordDeletion } from '../../../core/persist/storage'; +import type { EffectParams, FavoriteEntry, HistoryEntry, TrackIdentity } from '../../../core/model/types'; +import { isSameTrack } from '../../../core/model/track-identity'; +import { isLive, tombstone } from '../../../core/persist/deletions'; +import { favoritesItem } from '../../../core/persist/storage'; /** Star a song: copy the history entry into the Favorites library (top of the - * manual order). No-op if already favorited. */ + * manual order). No-op if already favorited. + * + * `updatedAt` is the star itself — the star and the unstar are its only + * writers, so it is the date the merge can trust for "is this song favorited" + * (`deletions.ts`, `merge.ts`). `orderedAt` says the manual order changed: + * this row went to the top. */ export async function addFavorite(entry: HistoryEntry): Promise { const list = await favoritesItem.getValue(); // By song, not by key — starring the same track after its duration settled // differently must not add a second row. - if (list.some((e) => isSameTrack(e.identity, entry.identity))) return; + if (list.some((e) => isLive(e) && isSameTrack(e.identity, entry.identity))) return; const now = Date.now(); + // Any tombstone for the song goes: this star is the newer statement. + const rest = list.filter((e) => !isSameTrack(e.identity, entry.identity)); await favoritesItem.setValue([ - { ...entry, favoritedAt: now, lastAccessedAt: now }, - ...list, + { ...entry, favoritedAt: now, lastAccessedAt: now, updatedAt: now, orderedAt: now }, + ...rest, ]); } -/** Dated (`deletions.ts`, by song — every copy of it, whatever duration it - * was saved under) so a sync merge with another device's older copy doesn't - * star the song again. */ +/** The star comes off: the row stays as a tombstone (`deletions.ts`) so a sync + * merge with another device's older copy doesn't star the song again. */ export async function removeFavorite(key: string): Promise { const list = await favoritesItem.getValue(); const entry = list.find((e) => e.identity.key === key); - await favoritesItem.setValue(list.filter((e) => e.identity.key !== key)); - if (entry) await recordDeletion(favoriteDeletion(songKey(entry.identity))); + const rest = list.filter((e) => e.identity.key !== key); + await favoritesItem.setValue(entry ? [tombstone(entry, Date.now()), ...rest] : rest); } -/** Persist a new manual order (list of identity keys, complete). */ +/** Persist a new manual order (list of identity keys, complete). + * + * The rank goes *into* each row, as `orderedAt` counting down from now, so the + * order is something the rows say rather than something the list's shape says. + * That is what lets a merge reproduce it — it sorts on the same field — instead + * of having to pick one device's array over the other's (`merge.ts`). */ export async function setFavoritesOrder(keys: string[]): Promise { const list = await favoritesItem.getValue(); - const byKey = new Map(list.map((e) => [e.identity.key, e])); + const live = list.filter(isLive); + const byKey = new Map(live.map((e) => [e.identity.key, e])); const next = keys.map((k) => byKey.get(k)).filter((e) => e !== undefined); // Keep entries missing from `keys` (e.g. added concurrently) at the top. - const missing = list.filter((e) => !keys.includes(e.identity.key)); - await favoritesItem.setValue([...missing, ...next]); + const missing = live.filter((e) => !keys.includes(e.identity.key)); + const now = Date.now(); + await favoritesItem.setValue([ + ...[...missing, ...next].map((e, i) => ({ ...e, orderedAt: now - i })), + ...list.filter((e) => !isLive(e)), + ]); } /** Refresh a favorite when its track is opened/played: bump Last Accessed and @@ -43,15 +59,19 @@ export async function setFavoritesOrder(keys: string[]): Promise { * Matched by song, like every other favorites operation. Matching on * `identity.key` would stop finding the row as soon as the duration drifted * (pre-roll ad, late metadata) — the favorite would then freeze while Recent, - * which matches by song, kept updating, and the two copies would disagree. */ + * which matches by song, kept updating, and the two copies would disagree. + * + * Leaves `updatedAt` alone: the fields it writes are a cache of the Recent row + * (the merge re-takes them from whichever copy is newer, `merge.ts`), and + * dating the row for them would let a slider nudge here outrank an unfavorite + * on another device. */ export async function touchFavorite( identity: TrackIdentity, patch?: { params?: EffectParams; pageUrl?: string; thumbnailUrl?: string }, ): Promise { const list = await favoritesItem.getValue(); - const index = list.findIndex((e) => isSameTrack(e.identity, identity)); + const index = list.findIndex((e) => isLive(e) && isSameTrack(e.identity, identity)); if (index === -1) return; - const now = Date.now(); const entry = list[index]; const next = [...list]; next[index] = { @@ -62,8 +82,7 @@ export async function touchFavorite( params: patch?.params ?? entry.params, pageUrl: patch?.pageUrl ?? entry.pageUrl, thumbnailUrl: patch?.thumbnailUrl ?? entry.thumbnailUrl, - lastAccessedAt: now, - updatedAt: patch?.params ? now : entry.updatedAt, - }; + lastAccessedAt: Date.now(), + } satisfies FavoriteEntry; await favoritesItem.setValue(next); } diff --git a/src/features/library/persist/history.ts b/src/features/library/persist/history.ts index 358d8cc..8b6b5d3 100644 --- a/src/features/library/persist/history.ts +++ b/src/features/library/persist/history.ts @@ -1,8 +1,17 @@ import { HISTORY_LIMIT } from '../../../core/model/defaults'; -import type { EffectParams, HistoryEntry, TrackIdentity } from '../../../core/model/types'; -import { isSameTrack, songKey } from '../../../core/model/track-identity'; -import { HISTORY_CLEARED, historyDeletion } from '../../../core/persist/deletions'; -import { historyItem, recordDeletion } from '../../../core/persist/storage'; +import type { HistoryEntry, EffectParams, TrackIdentity } from '../../../core/model/types'; +import { isSameTrack } from '../../../core/model/track-identity'; +import { isLive, tombstone } from '../../../core/persist/deletions'; +import { historyItem } from '../../../core/persist/storage'; + +/** Newest first, with the live rows capped at `HISTORY_LIMIT`. Tombstones + * (`deletions.ts`) are kept whatever the count — they are pruned by age, cost + * a few bytes each, and dropping one on a full list would let another device's + * older copy bring the row back. */ +function capped(list: HistoryEntry[]): HistoryEntry[] { + let live = 0; + return list.filter((entry) => !isLive(entry) || ++live <= HISTORY_LIMIT); +} /** Insert or refresh a Recent entry (newest first, LRU-capped). * @@ -19,7 +28,7 @@ export async function upsertHistory( ): Promise { const list = await historyItem.getValue(); const now = Date.now(); - const existing = list.find((e) => isSameTrack(e.identity, identity)); + const existing = list.find((e) => isLive(e) && isSameTrack(e.identity, identity)); if (!existing && onlyExisting) return; const entry = { identity, @@ -30,9 +39,11 @@ export async function upsertHistory( updatedAt: now, }; // Matched by song, not by key: this row supersedes every older one for the - // same song, so a duration that settled differently can't leave a twin behind. + // same song, so a duration that settled differently can't leave a twin + // behind — and a tombstone for the song goes with them, this play being the + // newer statement about it. const next = [entry, ...list.filter((e) => !isSameTrack(e.identity, identity))]; - await historyItem.setValue(next.slice(0, HISTORY_LIMIT)); + await historyItem.setValue(capped(next)); } /** Collapse rows written before saves were matched by song (one song split @@ -47,22 +58,25 @@ export async function dedupeHistory(): Promise { if (kept.length !== list.length) await historyItem.setValue(kept); } -/** The user removed a row: dated (`deletions.ts`, by song — every copy of it - * on every device, whatever duration it was saved under) so a sync merge with - * another device's older copy doesn't bring it back. `record: false` is for - * housekeeping that drops a stale twin of a song that stays — recording that - * would kill the song's fresh row on the other devices. */ +/** The user removed a row: it stays as a tombstone (`deletions.ts`) so a sync + * merge with another device's older copy doesn't bring it back, and moves to + * the front to keep the list newest-first. `record: false` is for housekeeping + * that drops a stale twin of a song that stays — a tombstone there would name + * the song, and kill its fresh row on the other devices. */ export async function removeHistoryEntry( key: string, { record = true }: { record?: boolean } = {}, ): Promise { const list = await historyItem.getValue(); const entry = list.find((e) => e.identity.key === key); - await historyItem.setValue(list.filter((e) => e.identity.key !== key)); - if (record && entry) await recordDeletion(historyDeletion(songKey(entry.identity))); + const rest = list.filter((e) => e.identity.key !== key); + await historyItem.setValue( + record && entry ? [tombstone(entry, Date.now()), ...rest] : rest, + ); } export async function clearHistory(): Promise { - await historyItem.setValue([]); - await recordDeletion(HISTORY_CLEARED); + const list = await historyItem.getValue(); + const now = Date.now(); + await historyItem.setValue(list.map((e) => (isLive(e) ? tombstone(e, now) : e))); } diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index 9de7f12..9200eb9 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -167,7 +167,11 @@ const backup = parseBackup(await file.text()); const ok = confirm( 'Replace all settings, history, favorites, presets, markers and snippets ' + - 'with the contents of this file? Your current data is lost.', + 'with the contents of this file? Your current data is lost.' + + // The import records what it drops (`deletions.ts`), and those + // records travel — so with sync on this is not only about this + // device, and the prompt has to say so. + (sync.enabled ? ' Your other synced devices lose the same songs.' : ''), ); if (!ok) return; await restoreBackup(backup, { asNew: true }); diff --git a/src/features/settings/panel/settings.svelte.ts b/src/features/settings/panel/settings.svelte.ts index 4d63ad4..88e107b 100644 --- a/src/features/settings/panel/settings.svelte.ts +++ b/src/features/settings/panel/settings.svelte.ts @@ -39,7 +39,10 @@ class SettingsStore { } async update(patch: Partial) { - const next = { ...this.current, ...patch }; + // Dated on every write: settings cross devices as one item with one date + // (`merge.ts`), so the later change wins without either device having to + // consult its own clock about the other's. + const next = { ...this.current, ...patch, updatedAt: Date.now() }; // Auto Reset and Remember settings are alternatives — enabling one // switches the other off. if (patch.rememberSettings) next.autoReset = false; @@ -59,7 +62,7 @@ class SettingsStore { } async reset() { - this.current = structuredClone(DEFAULT_SETTINGS); + this.current = { ...structuredClone(DEFAULT_SETTINGS), updatedAt: Date.now() }; this.onChange?.(this.current); await settingsItem.setValue($state.snapshot(this.current) as Settings); } @@ -85,7 +88,8 @@ class UiPrefsStore { async #save() { this.#writing = true; try { - await uiPrefsItem.setValue($state.snapshot(this.current)); + // Dated like Settings above — see `merge.ts`. + await uiPrefsItem.setValue({ ...$state.snapshot(this.current), updatedAt: Date.now() }); } finally { this.#writing = false; } diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 7ac81c5..51fc5b0 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -48,7 +48,7 @@ const SAFETY_INTERVAL_MS = 5 * 60_000; /** Raw storage keys (no `local:` prefix in change events) that belong to the * backup. `syncConfig` itself is deliberately absent. */ -const SYNCED_KEY_RE = /^(settings|uiPrefs|history|favorites|eqPresets|deletions|track:)/; +const SYNCED_KEY_RE = /^(settings|uiPrefs|history|favorites|eqPresets|track:)/; const measure = (backup: Backup) => packedChars(encodeBackup(backup)); const fit = (backup: Backup) => fitBackup(backup, BUDGET_CHARS, measure); @@ -118,7 +118,7 @@ class SyncStore { if (!Object.keys(changes).some((key) => SYNCED_KEY_RE.test(key))) return; // Persisted immediately (not on the debounce) so a panel closed // mid-burst still knows there is unpushed data next time it opens. - void this.#saveConfig({ pendingPush: true, lastChangedAt: Date.now() }); + void this.#saveConfig({ pendingPush: true }); this.#reconcileIn(PUSH_DEBOUNCE_MS); }); onSyncAreaChanged(() => { @@ -291,17 +291,12 @@ class SyncStore { return; } - // Another device wrote since we last looked. + // Another device wrote since we last looked. The merge asks this + // device nothing about itself — it is a function of the two copies + // alone (`merge.ts`), so the other device computes the same result and + // the two cannot end up pushing rival answers at each other. const remote = await unpackBackup(base64); - // Purely the clock: the remote copy wins if it was written after this - // device's last local change. Not "unless we changed something" — our - // own push records `lastLocalHash` as soon as the write resolves, and a - // write the browser later replaces (its own conflict resolution, or - // another device landing on top) would then read as "we changed - // nothing" and let an older copy overwrite the settings we just made. - // Losing that race now leaves local ahead, and `needPush` re-uploads. - const remoteWins = remote.exportedAt > this.config.lastChangedAt; - const merged = mergeBackups(local, remote, remoteWins); + const merged = mergeBackups(local, remote); const mergedHash = await contentHash(merged); const needApply = mergedHash !== localHash; diff --git a/src/features/sync/persist/fit.ts b/src/features/sync/persist/fit.ts index 1c56029..dc29d53 100644 --- a/src/features/sync/persist/fit.ts +++ b/src/features/sync/persist/fit.ts @@ -1,5 +1,6 @@ import { songKey } from '../../../core/model/track-identity.ts'; import type { Backup } from '../../../core/persist/backup-codec.ts'; +import { isLive } from '../../../core/persist/deletions.ts'; import type { HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types'; /** @@ -14,7 +15,9 @@ import type { HistoryEntry, TrackData, TrackIdentity } from '../../../core/model * 2. chord charts, by when they were computed (they can be re-analyzed); * 3. favorites, with their Recent row and track record, by last access. * - * Each group oldest first. Settings, UI prefs and EQ presets are never cut. + * Each group oldest first. Settings, UI prefs, EQ presets and tombstones + * (`deletions.ts`) are never cut — a dropped tombstone is a row another + * device brings straight back, and they are a handful of bytes each. * Songs are matched the way the library does (`songKey`: URL + title), so a * record saved under a drifted duration still follows its favorite. * @@ -55,17 +58,17 @@ const oldestFirst = (a: Cut, b: Cut) => /** Everything that may go, in the order it goes. */ function collectCuts(backup: Backup): Cut[] { - const favorites = new Set(backup.favorites.map((f) => songKey(f.identity))); + const favorites = new Set(backup.favorites.filter(isLive).map((f) => songKey(f.identity))); const songs = new Map(); const touch = (identity: TrackIdentity, at: number) => { const id = songKey(identity); if (!favorites.has(id)) songs.set(id, Math.max(songs.get(id) ?? 0, at)); }; - for (const entry of backup.history) touch(entry.identity, entry.updatedAt ?? 0); + for (const entry of backup.history.filter(isLive)) touch(entry.identity, entry.updatedAt ?? 0); for (const track of backup.tracks) touch(track.identity, track.updatedAt ?? 0); const accessed = new Map(); - for (const f of backup.favorites) { + for (const f of backup.favorites.filter(isLive)) { const id = songKey(f.identity); const at = f.lastAccessedAt ?? f.updatedAt ?? 0; accessed.set(id, Math.max(accessed.get(id) ?? 0, at)); @@ -83,7 +86,8 @@ function collectCuts(backup: Backup): Cut[] { function apply(backup: Backup, cuts: Cut[]): Backup { const songs = new Set(cuts.filter((c) => c.kind === 'song').map((c) => c.id)); const charts = new Set(cuts.filter((c) => c.kind === 'chart').map((c) => c.id)); - const keep = (row: HistoryEntry | TrackData) => !songs.has(songKey(row.identity)); + const keep = (row: HistoryEntry | TrackData) => + !isLive(row) || !songs.has(songKey(row.identity)); return { ...backup, history: backup.history.filter(keep), diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts index ebf6bc5..938e8e8 100644 --- a/src/features/sync/persist/merge.test.ts +++ b/src/features/sync/persist/merge.test.ts @@ -3,16 +3,10 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { mergeBackups } from './merge.ts'; import { backupFixture as backup } from '../../../core/persist/backup.fixture.ts'; -import { - favoriteDeletion, - HISTORY_CLEARED, - historyDeletion, - presetDeletion, - REPLACED_ALL, - reviveBackup, -} from '../../../core/persist/deletions.ts'; +import { encodeBackup } from '../../../core/persist/backup-codec.ts'; +import { isLive, replaceAll, tombstone } from '../../../core/persist/deletions.ts'; import { DEFAULT_PARAMS, DEFAULT_SETTINGS, HISTORY_LIMIT } from '../../../core/model/defaults.ts'; -import { makeTrackIdentity, songKey } from '../../../core/model/track-identity.ts'; +import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; import type { ChordChart, FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types.ts'; const T0 = 1_757_000_000_000; @@ -32,7 +26,7 @@ function row(identity: TrackIdentity, updatedAt: number, transpose = 0): History } function fav(identity: TrackIdentity, updatedAt: number, lastAccessedAt = updatedAt): FavoriteEntry { - return { ...row(identity, updatedAt), favoritedAt: updatedAt, lastAccessedAt }; + return { ...row(identity, updatedAt), favoritedAt: updatedAt, lastAccessedAt, orderedAt: updatedAt }; } const chart = (computedAt: number): ChordChart => ({ @@ -56,39 +50,45 @@ function record(identity: TrackIdentity, updatedAt: number, markers: number, wit }; } +/** What a screen would show: the panel stores filter tombstones out. */ +const live = (list: T[]) => list.filter(isLive); const keys = (list: { identity: TrackIdentity }[]) => list.map((e) => e.identity.key); +const shown = (list: { identity: TrackIdentity; deleted?: true }[]) => keys(live(list)); test('a row the other side trimmed away survives; the newer copy of a shared row wins', () => { const local = backup({ history: [row(song(1), T0 + 1000, 1), row(song(2), T0)] }); const remote = backup({ history: [row(song(1), T0 + 5000, 3)] }); - const merged = mergeBackups(local, remote, true, NOW); - assert.deepEqual(keys(merged.history), [song(1).key, song(2).key]); + const merged = mergeBackups(local, remote, NOW); + assert.deepEqual(shown(merged.history), [song(1).key, song(2).key]); assert.equal(merged.history[0].params.transpose, 3, 'newer copy'); }); test('history is matched by song, so a drifted duration does not make a twin', () => { const local = backup({ history: [row(song(1, 200), T0)] }); const remote = backup({ history: [row(song(1, 201), T0 + 1)] }); - const merged = mergeBackups(local, remote, false, NOW); + const merged = mergeBackups(local, remote, NOW); assert.equal(merged.history.length, 1); assert.equal(merged.history[0].identity.durationSec, 201); }); -test('a deletion beats the copy it postdates, but not a later re-play', () => { +test('a tombstone beats the copy it postdates, but not a later re-play', () => { const gone = song(1); - const local = backup({ deletions: { [historyDeletion(songKey(gone))]: T0 + 2000 } }); + const local = backup({ history: [tombstone(row(gone, T0), T0 + 2000)] }); const remote = backup({ history: [row(gone, T0 + 1000), row(song(2), T0)] }); - assert.deepEqual(keys(mergeBackups(local, remote, true, NOW).history), [song(2).key]); + assert.deepEqual(shown(mergeBackups(local, remote, NOW).history), [song(2).key]); const replayed = backup({ history: [row(gone, T0 + 3000)] }); - assert.deepEqual(keys(mergeBackups(local, replayed, true, NOW).history), [gone.key]); + assert.deepEqual(shown(mergeBackups(local, replayed, NOW).history), [gone.key]); }); -test('"Clear Recent" travels and wipes older rows on the other side', () => { - const local = backup({ deletions: { [HISTORY_CLEARED]: T0 + 5000 } }); +test('"Clear Recent" travels row by row and wipes older copies on the other side', () => { + // What `clearHistory` writes: every row it held, marked and dated. + const local = backup({ + history: [tombstone(row(song(1), T0), T0 + 5000), tombstone(row(song(2), T0), T0 + 5000)], + }); const remote = backup({ history: [row(song(1), T0 + 1000), row(song(2), T0 + 6000)] }); - const merged = mergeBackups(local, remote, true, NOW); - assert.deepEqual(keys(merged.history), [song(2).key]); - assert.equal(merged.deletions[HISTORY_CLEARED], T0 + 5000, 'the record travels on'); + const merged = mergeBackups(local, remote, NOW); + assert.deepEqual(shown(merged.history), [song(2).key], 'the row played since survives'); + assert.equal(merged.history.length, 2, 'the tombstone travels on'); }); test('history is newest-first and capped', () => { @@ -96,67 +96,80 @@ test('history is newest-first and capped', () => { history: Array.from({ length: HISTORY_LIMIT }, (_, i) => row(song(i), T0 + i)), }); const remote = backup({ history: [row(song(999), T0 + 100_000)] }); - const merged = mergeBackups(local, remote, false, NOW); + const merged = mergeBackups(local, remote, NOW); assert.equal(merged.history.length, HISTORY_LIMIT); assert.equal(merged.history[0].identity.key, song(999).key); }); -test('the history cap never drops a favorited song’s row', () => { +test('the cap counts live rows only, and never drops a favorited song’s row', () => { const old = song(1); const local = backup({ favorites: [fav(old, T0)], history: [ row(old, T0), + tombstone(row(song(500), T0), T0 + 1), ...Array.from({ length: HISTORY_LIMIT }, (_, i) => row(song(i + 2), T0 + 1000 + i)), ], }); - const merged = mergeBackups(local, backup({}), false, NOW); - assert.equal(merged.history.length, HISTORY_LIMIT, 'a plain row went instead'); - assert.ok(keys(merged.history).includes(old.key)); + const merged = mergeBackups(local, backup({}), NOW); + assert.equal(live(merged.history).length, HISTORY_LIMIT, 'a plain row went instead'); + assert.ok(shown(merged.history).includes(old.key)); + assert.equal(merged.history.length, HISTORY_LIMIT + 1, 'the tombstone is not what got cut'); }); -test('favorites: union in the winner order, deletions honoured, last access kept', () => { +test('favorites: union, tombstones honoured, last access kept', () => { const a = song(1); const b = song(2); const c = song(3); const local = backup({ - favorites: [fav(b, T0, T0 + 9000), fav(a, T0)], - deletions: { [favoriteDeletion(songKey(c))]: T0 + 100 }, + favorites: [fav(b, T0 + 20, T0 + 9000), fav(a, T0 + 10), tombstone(fav(c, T0), T0 + 100)], }); - const remote = backup({ favorites: [fav(a, T0 + 1), fav(c, T0)] }); - const merged = mergeBackups(local, remote, true, NOW); - assert.deepEqual(keys(merged.favorites), [a.key, b.key], 'remote order first, c deleted'); - assert.equal(merged.favorites[0].updatedAt, T0 + 1); - assert.equal(merged.favorites[1].lastAccessedAt, T0 + 9000); + const remote = backup({ favorites: [fav(a, T0 + 10), fav(c, T0)] }); + const merged = mergeBackups(local, remote, NOW); + assert.deepEqual(shown(merged.favorites), [b.key, a.key], 'by rank, c unstarred'); + assert.equal(live(merged.favorites)[1].lastAccessedAt, T0 + 10); + assert.equal(live(merged.favorites)[0].lastAccessedAt, T0 + 9000); }); test('favorites: practice on the other device does not undo an unfavorite', () => { const a = song(1); - // Starred long ago, unfavorited here; the other device then played it and - // moved a slider, which bumps `updatedAt` but not `favoritedAt`. - const local = backup({ deletions: { [favoriteDeletion(songKey(a))]: T0 + 5000 } }); - const practised: FavoriteEntry = { ...fav(a, T0), updatedAt: T0 + 9000 }; + // Starred long ago, unfavorited here; the other device then played it, which + // refreshes the row's cached fields but no longer dates it. + const local = backup({ favorites: [tombstone(fav(a, T0), T0 + 5000)] }); + const practised: FavoriteEntry = { ...fav(a, T0), lastAccessedAt: T0 + 9000, params: row(a, T0, 5).params }; const remote = backup({ favorites: [practised] }); - assert.deepEqual(keys(mergeBackups(local, remote, true, NOW).favorites), []); - // Genuinely starring it again does beat the record. + assert.deepEqual(shown(mergeBackups(local, remote, NOW).favorites), []); + // Genuinely starring it again does beat the tombstone. const restarred = backup({ favorites: [fav(a, T0 + 6000)] }); - assert.deepEqual(keys(mergeBackups(local, restarred, true, NOW).favorites), [a.key]); + assert.deepEqual(shown(mergeBackups(local, restarred, NOW).favorites), [a.key]); }); test('favorites: a newer copy adopts the other side’s later access time', () => { const a = song(1); const local = backup({ favorites: [fav(a, T0, T0 + 9000)] }); const remote = backup({ favorites: [fav(a, T0 + 5, T0 + 5)] }); - const merged = mergeBackups(local, remote, true, NOW); + const merged = mergeBackups(local, remote, NOW); assert.equal(merged.favorites[0].updatedAt, T0 + 5); assert.equal(merged.favorites[0].lastAccessedAt, T0 + 9000); }); +test('a manual reorder carries to the other device', () => { + const [a, b, c] = [song(1), song(2), song(3)]; + const before = [fav(a, T0), fav(b, T0), fav(c, T0)]; + // What `setFavoritesOrder([c, a, b])` writes: the rank, in the rows. + const reordered = [c, a, b].map((s, i) => ({ + ...before.find((f) => f.identity.key === s.key)!, + orderedAt: T0 + 5000 - i, + })); + const merged = mergeBackups(backup({ favorites: before }), backup({ favorites: reordered }), NOW); + assert.deepEqual(shown(merged.favorites), [c.key, a.key, b.key]); +}); + test('tracks: the newer record wins whole, an emptied one included', () => { const a = song(1); const local = backup({ tracks: [record(a, T0 + 1000, 0)] }); const remote = backup({ tracks: [record(a, T0, 5), record(song(2), T0, 2)] }); - const merged = mergeBackups(local, remote, true, NOW); + const merged = mergeBackups(local, remote, NOW); assert.equal(merged.tracks.length, 2); assert.equal(merged.tracks.find((t) => t.identity.key === a.key)?.markers.length, 0); }); @@ -165,7 +178,7 @@ test('tracks: a winner without a chart adopts the other side’s', () => { const a = song(1); const local = backup({ tracks: [record(a, T0 + 1000, 3)] }); const remote = backup({ tracks: [record(a, T0, 1, true)] }); - const merged = mergeBackups(local, remote, false, NOW); + const merged = mergeBackups(local, remote, NOW); assert.equal(merged.tracks[0].markers.length, 3); assert.ok(merged.tracks[0].chordChart?.segments.length); }); @@ -173,13 +186,12 @@ test('tracks: a winner without a chart adopts the other side’s', () => { test('a deleted chart beats stale analysis and later marker edits, but allows re-analysis', () => { const deleted = backup({ tracks: [{ ...record(song(1), T0 + 1, 0), chordChart: { ...chart(T0 + 1), segments: [] } }] }); const stale = backup({ tracks: [{ ...record(song(1), T0 + 2, 3), chordChart: chart(T0) }] }); - for (const remoteWins of [false, true]) { - const merged = mergeBackups(deleted, stale, remoteWins, NOW); + for (const [x, y] of [[deleted, stale], [stale, deleted]] as const) { + const merged = mergeBackups(x, y, NOW); assert.equal(merged.tracks[0].markers.length, 3); assert.deepEqual(merged.tracks[0].chordChart?.segments, []); - stale.tracks[0].chordChart = chart(T0 + 3); - assert.equal(mergeBackups(merged, stale, remoteWins, NOW).tracks[0].chordChart?.computedAt, T0 + 3); - stale.tracks[0].chordChart = chart(T0); + const reanalyzed = backup({ tracks: [{ ...stale.tracks[0], chordChart: chart(T0 + 3) }] }); + assert.equal(mergeBackups(merged, reanalyzed, NOW).tracks[0].chordChart?.computedAt, T0 + 3); } }); @@ -190,63 +202,86 @@ test('a song’s two library copies come out of a merge with the same settings', const starred: FavoriteEntry = { ...fav(a, T0), params: row(a, T0, 3).params }; const local = backup({ favorites: [starred], history: [row(a, T0, 3)] }); const remote = backup({ history: [row(a, T0 + 1000, 0)] }); - for (const remoteWins of [false, true]) { - const merged = mergeBackups(local, remote, remoteWins, NOW); + for (const [x, y] of [[local, remote], [remote, local]] as const) { + const merged = mergeBackups(x, y, NOW); assert.equal(merged.history[0].params.transpose, 0); assert.equal(merged.favorites[0].params.transpose, 0, 'the favorite follows the newer row'); } - // And the other way round, when the favorite is the fresher copy. - const practised = backup({ favorites: [{ ...fav(a, T0 + 1000), params: row(a, T0, 3).params }] }); - const merged = mergeBackups(practised, backup({ history: [row(a, T0, 0)] }), true, NOW); - assert.equal(merged.history[0].params.transpose, 3); - assert.equal(merged.favorites[0].params.transpose, 3); + assert.notEqual( + mergeBackups(local, remote, NOW).history[0].params, + mergeBackups(local, remote, NOW).favorites[0].params, + 'a copy each, not one object shared by both lists', + ); }); -test('a manual import re-adds deleted rows and presets without reviving absent items', () => { - const del = { [HISTORY_CLEARED]: NOW, [favoriteDeletion(songKey(song(1)))]: NOW, [presetDeletion('Mine')]: NOW }; - const file = backup({ history: [row(song(1), T0)], favorites: [fav(song(1), T0)], eqPresets: [{ name: 'Mine', gains: [1] }] }); - const restored = reviveBackup(file, del, NOW); - const remote = backup({ deletions: del, history: [row(song(2), T0)] }); - for (const remoteWins of [false, true]) { - const merged = mergeBackups(restored, remote, remoteWins, NOW); - assert.deepEqual(keys(merged.history), [song(1).key]); - assert.deepEqual(keys(merged.favorites), [song(1).key]); - assert.equal(merged.eqPresets[0].name, 'Mine'); - } +test('a manual import re-adds rows the devices had deleted', () => { + const a = song(1); + const file = backup({ + history: [row(a, T0)], + favorites: [fav(a, T0)], + eqPresets: [{ name: 'Mine', gains: [1] }], + }); + const current = backup({ + history: [tombstone(row(a, T0), NOW)], + favorites: [tombstone(fav(a, T0), NOW)], + eqPresets: [{ name: 'Mine', gains: [], updatedAt: NOW, deleted: true }], + }); + const imported = replaceAll(file, current, NOW); + // The other device still holds the tombstones the import is undoing. + const merged = mergeBackups(imported, current, NOW); + assert.deepEqual(shown(merged.history), [a.key]); + assert.deepEqual(shown(merged.favorites), [a.key]); + assert.deepEqual(live(merged.eqPresets).map((p) => p.name), ['Mine']); assert.equal(file.history[0].updatedAt, T0, 'the original backup is unchanged'); }); -test('a replacement import removes what it leaves out, on the other devices too', () => { +test('a replacement import removes what this device held, and only that', () => { const kept = song(1); const dropped = song(2); + const theirs = song(3); const file = backup({ history: [row(kept, T0)], tracks: [record(kept, T0, 1)] }); - const restored = reviveBackup(file, { [REPLACED_ALL]: NOW }, NOW); + const current = backup({ + history: [row(kept, T0), row(dropped, T0)], + favorites: [fav(dropped, T0)], + eqPresets: [{ name: 'Gone', gains: [1], updatedAt: T0 }], + }); + const imported = replaceAll(file, current, NOW); const remote = backup({ - history: [row(dropped, T0)], + history: [row(dropped, T0), row(theirs, T0)], favorites: [fav(dropped, T0)], - eqPresets: [{ name: 'Gone', gains: [1] }], - tracks: [record(dropped, T0, 3)], + eqPresets: [{ name: 'Gone', gains: [1], updatedAt: T0 }], }); - for (const remoteWins of [false, true]) { - const merged = mergeBackups(restored, remote, remoteWins, NOW); - assert.deepEqual(keys(merged.history), [kept.key], 'only what the file carried'); - assert.deepEqual(keys(merged.favorites), []); - assert.deepEqual(merged.eqPresets, []); - assert.deepEqual(keys(merged.tracks), [kept.key]); - } - // What the other device did *after* the import is not the import's to drop. - const later = backup({ history: [row(dropped, NOW + 1)] }); - assert.equal(mergeBackups(restored, later, true, NOW).history.length, 2); + const merged = mergeBackups(imported, remote, NOW); + assert.deepEqual(shown(merged.history).sort(), [kept.key, theirs.key].sort(), 'their own song stays'); + assert.deepEqual(shown(merged.favorites), []); + assert.deepEqual(live(merged.eqPresets), []); + // What the other device does after the import is not the import's to undo. + const later = backup({ history: [row(dropped, NOW + 5000)] }); + assert.equal(live(mergeBackups(imported, later, NOW).history).length, 2); +}); + +test('an item with no date of its own is not swept away by an unrelated deletion', () => { + // Legacy EQ presets and pre-`updatedAt` track records read as 0. Nothing but + // a tombstone of their own name may remove them. + const legacy = backup({ eqPresets: [{ name: 'Old', gains: [1] }] }); + const deleter = backup({ + history: [tombstone(row(song(1), T0), NOW)], + eqPresets: [{ name: 'Other', gains: [], updatedAt: NOW, deleted: true }], + }); + assert.deepEqual(live(mergeBackups(legacy, deleter, NOW).eqPresets).map((p) => p.name), ['Old']); }); -test('ties go to the winning side', () => { +test('a tie goes to the tombstone, and both devices break it the same way', () => { const a = song(1); - const local = backup({ history: [row(a, T0, 1)], tracks: [record(a, T0, 1)] }); - const remote = backup({ history: [row(a, T0, 2)], tracks: [record(a, T0, 2)] }); - assert.equal(mergeBackups(local, remote, true, NOW).history[0].params.transpose, 2); - assert.equal(mergeBackups(local, remote, true, NOW).tracks[0].markers.length, 2); - assert.equal(mergeBackups(local, remote, false, NOW).history[0].params.transpose, 1); - assert.equal(mergeBackups(local, remote, false, NOW).tracks[0].markers.length, 1); + const local = backup({ history: [row(a, T0, 1)] }); + const gone = backup({ history: [tombstone(row(a, T0, 2), T0)] }); + assert.deepEqual(shown(mergeBackups(local, gone, NOW).history), []); + assert.deepEqual(shown(mergeBackups(gone, local, NOW).history), []); + const other = backup({ history: [row(a, T0, 2)] }); + assert.equal( + mergeBackups(local, other, NOW).history[0].params.transpose, + mergeBackups(other, local, NOW).history[0].params.transpose, + ); }); test('a deletion reaches every copy of the song, whatever duration it was saved under', () => { @@ -255,77 +290,99 @@ test('a deletion reaches every copy of the song, whatever duration it was saved const s200 = makeTrackIdentity(url, 'Song', 200); assert.notEqual(s201.key, s200.key); const local = backup({ - deletions: { - [historyDeletion(songKey(s201))]: T0 + 5000, - [favoriteDeletion(songKey(s201))]: T0 + 5000, - }, + history: [tombstone(row(s201, T0), T0 + 5000)], + favorites: [tombstone(fav(s201, T0), T0 + 5000)], }); const remote = backup({ history: [row(s200, T0)], favorites: [fav(s200, T0)] }); - const merged = mergeBackups(local, remote, false, NOW); - assert.equal(merged.history.length, 0); - assert.equal(merged.favorites.length, 0); + const merged = mergeBackups(local, remote, NOW); + assert.equal(live(merged.history).length, 0); + assert.equal(live(merged.favorites).length, 0); // A different song at the same URL (local files share one) is untouched. const other = makeTrackIdentity(url, 'Other song', 200); - const kept = mergeBackups(local, backup({ history: [row(other, T0)] }), false, NOW); - assert.equal(kept.history.length, 1); + const kept = mergeBackups(local, backup({ history: [row(other, T0)] }), NOW); + assert.equal(live(kept.history).length, 1); }); test('a deleted EQ preset stays deleted; a later save of the name brings it back', () => { const gains = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]; - const deleter = backup({ deletions: { [presetDeletion('Mine')]: T0 + 2000 } }); + const deleter = backup({ eqPresets: [{ name: 'Mine', gains: [], updatedAt: T0 + 2000, deleted: true }] }); const keeper = backup({ eqPresets: [{ name: 'Mine', gains, updatedAt: T0 + 1000 }] }); - assert.deepEqual(mergeBackups(deleter, keeper, false, NOW).eqPresets, []); - assert.deepEqual(mergeBackups(deleter, keeper, true, NOW).eqPresets, [], 'even when the keeper wins'); + assert.deepEqual(live(mergeBackups(deleter, keeper, NOW).eqPresets), []); + assert.deepEqual(live(mergeBackups(keeper, deleter, NOW).eqPresets), [], 'either way round'); const unstamped = backup({ eqPresets: [{ name: 'Mine', gains }] }); - assert.deepEqual(mergeBackups(deleter, unstamped, true, NOW).eqPresets, [], 'an undated preset loses to any deletion'); + assert.deepEqual(live(mergeBackups(deleter, unstamped, NOW).eqPresets), [], 'an undated preset loses'); const resaved = backup({ eqPresets: [{ name: 'Mine', gains, updatedAt: T0 + 3000 }] }); - assert.equal(mergeBackups(deleter, resaved, false, NOW).eqPresets.length, 1); - assert.equal(mergeBackups(deleter, resaved, false, NOW).deletions[presetDeletion('Mine')], T0 + 2000, 'the record still travels'); + assert.equal(live(mergeBackups(deleter, resaved, NOW).eqPresets).length, 1); + assert.equal(mergeBackups(deleter, resaved, NOW).eqPresets.length, 1, 'and the tombstone is spent'); }); -test('a shared preset name goes to the later save', () => { - const local = backup({ eqPresets: [{ name: 'Mine', gains: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], updatedAt: T0 + 5 }] }); - const remote = backup({ eqPresets: [{ name: 'Mine', gains: [2, 0, 0, 0, 0, 0, 0, 0, 0, 0], updatedAt: T0 + 1 }] }); - assert.equal(mergeBackups(local, remote, true, NOW).eqPresets[0].gains[0], 1, 'later save beats the winner side'); - assert.equal(mergeBackups(local, remote, false, NOW).eqPresets[0].gains[0], 1); -}); - -test('settings and prefs come from the winner; EQ presets are a union', () => { - const local = backup({ - settings: { ...DEFAULT_SETTINGS, theme: 'dark' }, - eqPresets: [{ name: 'Mine', gains: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] }], - }); +test('a shared preset name goes to the later save; the list reads the same on both', () => { + const local = backup({ eqPresets: [{ name: 'Mine', gains: [1, 0], updatedAt: T0 + 5 }] }); const remote = backup({ - settings: { ...DEFAULT_SETTINGS, theme: 'light' }, - eqPresets: [{ name: 'Theirs', gains: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] }, { name: 'Mine', gains: [9, 9, 9, 9, 9, 9, 9, 9, 9, 9] }], + eqPresets: [{ name: 'Theirs', gains: [0, 1], updatedAt: T0 + 1 }, { name: 'Mine', gains: [9, 9], updatedAt: T0 + 1 }], }); - const remoteWins = mergeBackups(local, remote, true, NOW); - assert.equal(remoteWins.settings.theme, 'light'); - assert.deepEqual(remoteWins.eqPresets.map((p) => p.name), ['Theirs', 'Mine']); - assert.equal(remoteWins.eqPresets[1].gains[0], 9, 'winner’s copy of a shared name'); - const localWins = mergeBackups(local, remote, false, NOW); - assert.equal(localWins.settings.theme, 'dark'); - assert.deepEqual(localWins.eqPresets.map((p) => p.name), ['Mine', 'Theirs']); - assert.equal(localWins.eqPresets[0].gains[0], 1); + for (const [x, y] of [[local, remote], [remote, local]] as const) { + const merged = mergeBackups(x, y, NOW); + assert.deepEqual(merged.eqPresets.map((p) => p.name), ['Mine', 'Theirs']); + assert.equal(merged.eqPresets[0].gains[0], 1, 'later save wins'); + } +}); + +test('settings and prefs go by their own date, not by whose merge it is', () => { + const local = backup({ settings: { ...DEFAULT_SETTINGS, theme: 'dark', updatedAt: T0 + 5 } }); + const remote = backup({ settings: { ...DEFAULT_SETTINGS, theme: 'light', updatedAt: T0 + 1 } }); + assert.equal(mergeBackups(local, remote, NOW).settings.theme, 'dark'); + assert.equal(mergeBackups(remote, local, NOW).settings.theme, 'dark', 'the same on both devices'); }); -test('deletions are merged newest-per-key and expired ones dropped', () => { +test('undated settings on both sides still resolve the same way on both devices', () => { + const local = backup({ settings: { ...DEFAULT_SETTINGS, theme: 'dark' } }); + const remote = backup({ settings: { ...DEFAULT_SETTINGS, theme: 'light' } }); + assert.equal( + mergeBackups(local, remote, NOW).settings.theme, + mergeBackups(remote, local, NOW).settings.theme, + ); +}); + +test('tombstones expire', () => { const old = NOW - 40 * 24 * 60 * 60_000; - const local = backup({ deletions: { 'h:a': T0, 'h:old': old } }); - const remote = backup({ deletions: { 'h:a': T0 + 1, 'f:b': T0 } }); - assert.deepEqual(mergeBackups(local, remote, true, NOW).deletions, { 'h:a': T0 + 1, 'f:b': T0 }); + const local = backup({ history: [tombstone(row(song(1), old), old), tombstone(row(song(2), T0), T0)] }); + const merged = mergeBackups(local, backup({}), NOW); + assert.deepEqual(keys(merged.history), [song(2).key]); }); test('merging a library with itself changes nothing', () => { const b = backup({ - history: [row(song(1), T0), row(song(2), T0 + 1)], + history: [row(song(1), T0), row(song(2), T0 + 1), tombstone(row(song(9), T0), T0 + 2)], favorites: [fav(song(2), T0)], tracks: [record(song(1), T0, 2, true)], - deletions: { 'h:x': T0 }, }); - const merged = mergeBackups(b, b, true, NOW); - assert.deepEqual(merged.history, [...b.history].sort((x, y) => y.updatedAt - x.updatedAt)); - assert.deepEqual(merged.favorites, b.favorites); + const merged = mergeBackups(b, b, NOW); + assert.deepEqual(encodeBackup(merged), encodeBackup(mergeBackups(merged, b, NOW))); + assert.deepEqual(live(merged.favorites), b.favorites); assert.deepEqual(merged.tracks, b.tracks); - assert.deepEqual(merged.deletions, b.deletions); +}); + +test('both devices compute the same merge, and it absorbs a third pass', () => { + const a = backup({ + history: [row(song(1), T0 + 2), row(song(2), T0)], + favorites: [fav(song(2), T0, T0 + 9), { ...fav(song(4), T0), orderedAt: T0 + 50 }], + eqPresets: [{ name: 'Mine', gains: [1], updatedAt: T0 + 5 }], + tracks: [record(song(1), T0 + 3, 2, true)], + settings: { ...DEFAULT_SETTINGS, theme: 'dark', updatedAt: T0 + 7 }, + }); + const b = backup({ + history: [row(song(3), T0 + 1), tombstone(row(song(2), T0), T0 + 4)], + favorites: [fav(song(4), T0 + 6), tombstone(fav(song(2), T0), T0 + 4)], + eqPresets: [{ name: 'Theirs', gains: [2], updatedAt: T0 + 2 }], + tracks: [record(song(1), T0, 5), record(song(3), T0, 1)], + }); + const ab = mergeBackups(a, b, NOW); + const ba = mergeBackups(b, a, NOW); + assert.deepEqual(encodeBackup(ba), encodeBackup(ab), 'the same blob from either side'); + // Whoever pushed second has nothing to send back: re-merging your own copy + // against the published result reproduces it exactly. This is what stops + // two devices trading pushes for ever. + assert.deepEqual(encodeBackup(mergeBackups(a, ab, NOW)), encodeBackup(ab)); + assert.deepEqual(encodeBackup(mergeBackups(b, ab, NOW)), encodeBackup(ab)); }); diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index 6839f6a..635dd59 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -1,153 +1,174 @@ import { HISTORY_LIMIT } from '../../../core/model/defaults.ts'; import { songKey } from '../../../core/model/track-identity.ts'; import type { Backup } from '../../../core/persist/backup-codec.ts'; -import { - deletedSince, - favoriteDeletion, - HISTORY_CLEARED, - historyDeletion, - mergeDeletions, - presetDeletion, - pruneDeletions, - REPLACED_ALL, -} from '../../../core/persist/deletions.ts'; +import { at, isLive, pruneTombstones, type Deletable } from '../../../core/persist/deletions.ts'; import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; /** - * Two devices' libraries into one. A union, item by item, so a copy another - * device trimmed to fit the quota never deletes anything here, and edits made - * on both sides since they last met both survive: + * Two devices' libraries into one. + * + * One rule, applied to every list: union by id, and of two copies keep the one + * with the later date. A deletion is a row like any other (`deletions.ts`), so + * "removed over there" needs no case of its own, and neither does a re-add — + * it is simply newer. Ties go to the tombstone. * * - Recent rows and favorites: matched by song (URL + title, like the - * library itself), the more recently updated one wins; a dated deletion - * (`deletions.ts`) beats any copy it postdates. A favorite's last access - * is the later of the two. - * - Track records: matched by key, the more recently edited wins outright — - * an emptied record is still a record, so clearing markers sticks. A - * chart is chosen separately by computedAt: null may mean "trimmed", but - * an empty, dated chart is an explicit deletion and beats older analysis. - * - Settings and UI prefs: the newer device's, as a whole. EQ presets: union - * by name, the later save of a shared name, deletions honoured. + * library itself), so a copy saved under a drifted duration is the same + * song. A favorite's last access is the later of the two. + * - Track records: matched by key. An emptied record is still a record, so + * clearing markers sticks. A chart is chosen separately by `computedAt`: + * null may mean "trimmed", but an empty, dated chart is an explicit + * deletion and beats older analysis. + * - Settings and UI prefs: one item each, with one date, taken whole. + * EQ presets: union by name. * - Last, the two library copies of a song (Recent and Favorites) are put * back in step: they are written together and read as one. * - * `remoteWins` breaks ties and picks the wholesale sections: true when the - * remote copy was written after this device's last local change. The winning - * side's order comes first in every list. Pure; `node --test`. + * **A pure function of its two inputs.** It asks this device nothing about + * itself — no clock, no "which side am I on" — so both devices compute the + * same answer and the second one has nothing left to push. That is what stops + * two devices trading rival merges for ever, and it is why each list's *order* + * comes out of the rows as well (`orderOf`): a rule like "keep my order, then + * add theirs" reads the same on both devices and means something different on + * each. `node --test`. */ -const at = (item: { updatedAt?: number }) => item.updatedAt ?? 0; +const song = (entry: HistoryEntry) => songKey(entry.identity); + +const compare = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + +/** Key-order-independent text of a value. Only ever used to break a tie, and + * only so that both devices break it the same way. */ +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') { + return `{${Object.entries(value) + .sort(([a], [b]) => compare(a, b)) + .map(([key, v]) => `${JSON.stringify(key)}:${canonical(v)}`) + .join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +} + +/** Which of two copies of one item survives: the later date; on a tie the + * tombstone, because a removal and a re-add stamped in the same millisecond + * have to resolve the same way on both devices; failing that the one whose + * content sorts first, for the same reason. */ +function wins(item: T, current: T): boolean { + if (at(item) !== at(current)) return at(item) > at(current); + if (isLive(item) !== isLive(current)) return !isLive(item); + return compare(canonical(item), canonical(current)) < 0; +} -/** Union by `id`: of two copies the later `updatedAt` wins (the first list's - * on a tie), `dead` ones are skipped, and `resolve` can fold the loser's - * fields into the winner. First list's order, then the second's extras. */ -function unionNewest( +/** Union by `id`, keeping the winner of each pair; `resolve` folds the loser's + * fields into it. Unordered — every caller sorts the result. */ +function unionNewest( first: T[], second: T[], id: (item: T) => string, - dead: (item: T) => boolean = () => false, resolve: (winner: T, loser: T) => T = (winner) => winner, ): T[] { const byId = new Map(); for (const item of [...first, ...second]) { - if (dead(item)) continue; const key = id(item); const current = byId.get(key); if (!current) byId.set(key, item); - else byId.set(key, at(item) > at(current) ? resolve(item, current) : resolve(current, item)); + else byId.set(key, wins(item, current) ? resolve(item, current) : resolve(current, item)); } return [...byId.values()]; } -const song = (entry: HistoryEntry) => songKey(entry.identity); +/** Newest-first by `date`, ties by id. Every merged list is ordered by + * something each row carries, never by which copy it came from — that is what + * makes the order the same on both devices, and stable when a third merge + * runs over the result. */ +const orderOf = (date: (item: T) => number, id: (item: T) => string) => (a: T, b: T) => + date(b) - date(a) || compare(id(a), id(b)); -/** Newest first, down to `HISTORY_LIMIT` — but only non-favorited rows are - * ever dropped. A favorited song whose Recent row went would leave the two - * library copies of it disagreeing, which is the drift `upsertHistory` and - * `fit.ts` both go out of their way to prevent; a union of two full libraries - * is exactly where the cap would otherwise reach one. Over the limit in - * favorites alone, the list simply stays long. */ +/** Where a favorite sits in the manual order. `setFavoritesOrder` stamps the + * row's rank into it (`now - index`, so the list reads back highest first) and + * a new star takes `now`, which puts it on top. Rows from before manual order + * synced fall back to the star date — near enough to the order they were + * added in, which is the order they had. */ +const rankOf = (f: FavoriteEntry) => f.orderedAt ?? f.favoritedAt ?? at(f); + +/** Newest first, live rows down to `HISTORY_LIMIT` — but only non-favorited + * rows are ever dropped. A favorited song whose Recent row went would leave + * the two library copies of it disagreeing, which is the drift `upsertHistory` + * and `fit.ts` both go out of their way to prevent; a union of two full + * libraries is exactly where the cap would otherwise reach one. Over the limit + * in favorites alone, the list simply stays long. Tombstones are kept whatever + * the count — they go by age, not by rank. */ function capHistory(history: HistoryEntry[], favorites: FavoriteEntry[]): HistoryEntry[] { - const ordered = history.sort((a, b) => at(b) - at(a)); - const excess = ordered.length - HISTORY_LIMIT; + const ordered = [...history].sort(orderOf(at, song)); + const live = ordered.filter(isLive); + const excess = live.length - HISTORY_LIMIT; if (excess <= 0) return ordered; - const favorited = new Set(favorites.map((f) => song(f))); - const cut = new Set(ordered.filter((e) => !favorited.has(song(e))).slice(-excess)); + const favorited = new Set(favorites.filter(isLive).map(song)); + const cut = new Set(live.filter((e) => !favorited.has(song(e))).slice(-excess)); return ordered.filter((e) => !cut.has(e)); } /** Recent and Favorites each hold a copy of the same song's settings and are * written together (`track-sync.#saveCurrent`); `findSavedEntry` and the chips - * in the list both take the two to agree. Merged apart they drift — star a song - * on one device, practise it unstarred on another, and only Recent hears about - * the new settings — so both copies take the newer one's params. Dates are left - * alone: this is a function of the merged lists, so every device works out the - * same answer from the same pair of copies. */ -function alignParams(history: HistoryEntry[], favorites: FavoriteEntry[]) { + * in the list both take the two to agree. The favorite's copy is a cache — + * `touchFavorite` refreshes it without dating the row, so ordinary practice + * can't outrank another device's unfavorite — and this is where the cache is + * refilled: both copies take the fields of whichever row was written last. A + * function of the merged lists, so every device works out the same answer from + * the same pair of copies. */ +function alignCopies(history: HistoryEntry[], favorites: FavoriteEntry[]) { const newest = new Map(); for (const entry of [...history, ...favorites]) { + if (!isLive(entry)) continue; const best = newest.get(song(entry)); if (!best || at(entry) > at(best)) newest.set(song(entry), entry); } return (entry: T): T => { const best = newest.get(song(entry)); - return best && best !== entry ? { ...entry, params: best.params } : entry; + if (!best || best === entry || !isLive(entry)) return entry; + // A copy, not the same object: the two lists are written and encoded + // separately, and nothing here should be able to edit both at once. + const aligned: T = { ...entry, params: { ...best.params }, pageUrl: best.pageUrl }; + if (best.thumbnailUrl === undefined) delete aligned.thumbnailUrl; + else aligned.thumbnailUrl = best.thumbnailUrl; + return aligned; }; } -export function mergeBackups( - local: Backup, - remote: Backup, - remoteWins: boolean, - now = Date.now(), -): Backup { - const [winner, loser] = remoteWins ? [remote, local] : [local, remote]; - const deletions = pruneDeletions(mergeDeletions(local.deletions ?? {}, remote.deletions ?? {}), now); - const history = unionNewest(winner.history, loser.history, song, (e) => - deletedSince(deletions, at(e), historyDeletion(song(e)), HISTORY_CLEARED, REPLACED_ALL), - ); - const favorites = unionNewest( - winner.favorites, - loser.favorites, - song, - // Anchored on `favoritedAt` — when the star was put there — not on - // `updatedAt`, which ordinary practice bumps (`touchFavorite` with new - // params). Taking the later of the two would let a slider nudge on one - // device outdate the other's unfavorite and re-star the song. - (f) => deletedSince(deletions, f.favoritedAt || at(f), favoriteDeletion(song(f)), REPLACED_ALL), - (w, l): FavoriteEntry => ({ - ...w, - lastAccessedAt: Math.max(w.lastAccessedAt ?? 0, l.lastAccessedAt ?? 0), - }), - ); - const align = alignParams(history, favorites); +/** Settings and UI prefs travel whole, as one item with one date. */ +const newer = (a: T, b: T): T => (wins(b, a) ? b : a); + +export function mergeBackups(local: Backup, remote: Backup, now = Date.now()): Backup { + const history = pruneTombstones(unionNewest(local.history, remote.history, song), now); + const favorites = pruneTombstones( + unionNewest(local.favorites, remote.favorites, song, (winner, loser) => ({ + ...winner, + lastAccessedAt: Math.max(winner.lastAccessedAt ?? 0, loser.lastAccessedAt ?? 0), + orderedAt: Math.max(winner.orderedAt ?? 0, loser.orderedAt ?? 0), + })), + now, + ).sort(orderOf(rankOf, song)); + + const byName = (preset: { name: string }) => preset.name; + const align = alignCopies(history, favorites); return { ...local, exportedAt: Math.max(local.exportedAt ?? 0, remote.exportedAt ?? 0), - settings: winner.settings, - uiPrefs: winner.uiPrefs, - eqPresets: unionNewest( - winner.eqPresets, - loser.eqPresets, - (p) => p.name, - (p) => deletedSince(deletions, at(p), presetDeletion(p.name), REPLACED_ALL), - ), + settings: newer(local.settings, remote.settings), + uiPrefs: newer(local.uiPrefs, remote.uiPrefs), + // By name: the dropdown's order has to be the same on both devices, and + // "the order they were saved in" is not something the rows can say. + eqPresets: pruneTombstones(unionNewest(local.eqPresets, remote.eqPresets, byName), now) + .sort((a, b) => compare(a.name, b.name)), history: capHistory(history.map(align), favorites), favorites: favorites.map(align), - tracks: unionNewest( - winner.tracks, - loser.tracks, - (t) => t.identity.key, - // No tombstone of their own (an emptied record is one), but a - // replacement import drops markers and snippets too. - (t) => deletedSince(deletions, at(t), REPLACED_ALL), - (w, l) => ({ - ...w, - chordChart: !w.chordChart || (l.chordChart?.computedAt ?? 0) > w.chordChart.computedAt + tracks: unionNewest(local.tracks, remote.tracks, (t) => t.identity.key, (w, l) => ({ + ...w, + chordChart: + !w.chordChart || (l.chordChart?.computedAt ?? 0) > w.chordChart.computedAt ? l.chordChart ?? w.chordChart : w.chordChart, - }), - ), - deletions, + })).sort((a, b) => compare(a.identity.key, b.identity.key)), }; } diff --git a/src/features/sync/persist/sync-config.ts b/src/features/sync/persist/sync-config.ts index ca61b94..4041099 100644 --- a/src/features/sync/persist/sync-config.ts +++ b/src/features/sync/persist/sync-config.ts @@ -8,9 +8,6 @@ export interface SyncConfig { /** When this device was last in agreement with the synced copy — a push, a * merge, or a reconcile that found nothing to do; 0 = never synced. */ lastSyncedAt: number; - /** Wall clock of the last local data change — this device's side of - * "whose settings win" against a remote blob's clock. */ - lastChangedAt: number; /** `meta.h` of the blob this device last reconciled with. The same hash on * the next read means nothing new arrived (our own write echo included). */ lastRemoteHash: string | null; @@ -28,7 +25,6 @@ export interface SyncConfig { export const DEFAULT_SYNC_CONFIG: SyncConfig = { enabled: true, lastSyncedAt: 0, - lastChangedAt: 0, lastRemoteHash: null, lastLocalHash: null, pendingPush: false, From ffd4bdf7a75a2ff1ee8c9bb12338c6fec9c95fa6 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 17:06:15 +0200 Subject: [PATCH 12/26] refactor: unify track identity handling by using songKey for key derivation - Updated track identity logic to use songKey, which combines normalized URL and title, ensuring consistent key generation across the application. - Removed legacy key handling that included duration, as it is now treated as metadata. - Implemented storage migration to rekey existing track data to the new format. - Adjusted backup codec and sync logic to align with the new identity structure. - Updated tests to reflect changes in identity handling and ensure correct functionality with the new keying system. --- CLAUDE.md | 8 +-- src/core/model/track-identity.ts | 46 +++++++------- src/core/persist/backup-codec.test.ts | 31 ++++------ src/core/persist/backup-codec.ts | 24 ++++---- src/core/persist/deletions.ts | 3 +- src/core/persist/migrate.ts | 57 +++++++++++++++++ src/core/persist/rekey.test.ts | 65 ++++++++++++++++++++ src/core/persist/rekey.ts | 39 ++++++++++++ src/core/state/track-sync.svelte.ts | 25 ++++---- src/entrypoints/sidepanel/App.svelte | 8 ++- src/features/library/panel/history.svelte.ts | 5 +- src/features/library/persist/history.ts | 18 +----- src/features/sync/panel/sync.svelte.ts | 10 +-- src/features/sync/persist/fit.ts | 15 +++-- src/features/sync/persist/merge.test.ts | 12 +++- src/features/sync/persist/merge.ts | 25 ++++---- 16 files changed, 269 insertions(+), 122 deletions(-) create mode 100644 src/core/persist/migrate.ts create mode 100644 src/core/persist/rekey.test.ts create mode 100644 src/core/persist/rekey.ts diff --git a/CLAUDE.md b/CLAUDE.md index cfbaa89..2746e8a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,13 +95,13 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel - Features contribute boot init + event routing via `panel/panel.ts` (registered in `core/features.ts`) and per-track persistence via `persist.svelte.ts` (registered in `core/persist/track-data.ts`). ### Persistence & sync -- [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by a normalized track identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) + rounded duration, hashed to `local:track:`. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. -- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v2) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (key rebuilt from URL + duration via `identityKey`; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped. Timestamps stay in ms — they decide merges, and a delete and a re-add within one second must not collide. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. -- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs each one dated item, EQ presets by name). `mergeBackups` is a **pure function of its two inputs** — no device clock, no "which side am I on" — so both devices compute the same result and the second has nothing left to push; that is also why every merged list is ordered by something the rows carry (`updatedAt`, the favorites' `orderedAt` rank, the preset name) rather than by whose array it was and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as **tombstones** ([core/persist/deletions.ts](src/core/persist/deletions.ts)): the removed row stays in its own list, marked `deleted` and dated, rather than being dropped — so the merge needs no deletion rules at all (last write wins, and a tombstone is a write), a re-add beats it by being newer, and a deletion can never reach an item it does not name. Written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, filtered out by the three panel stores so nothing downstream sees one, TTL 30 d. Songs are matched by `songKey` (URL + title, **no duration**), so a copy saved under a drifted duration is covered; the stale-twin removal in `track-sync` passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). A **replacement import** ([`replaceAll`](src/core/persist/deletions.ts)) is deliberately a *local* operation: it re-dates the file's rows and tombstones what **this device** held and the file omits — there is no "everything before now is gone" record, because a date range cannot be made safe across two devices' clocks. Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. +- [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by the song's identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) plus the cleaned title, hashed by `songKey` to `local:track:`. **Duration is metadata, not identity** — it drifts (pre-roll ad, late metadata), and a key that moved with it split one song across several records, which every list then worked around. That is one key: track records, Recent rows, favorites and tombstones all answer to it. The title is in it because a URL alone is not enough — every local file reports the local-player page URL and is told apart only by its title. Storage written before this is re-keyed once at panel boot ([migrate.ts](src/core/persist/migrate.ts), collapsing logic in the node-tested [rekey.ts](src/core/persist/rekey.ts)); the wire format needs no migration, since a compact backup stores `[url, title, duration]` and each build derives its own key from those. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. +- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v3, reading v2 too) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (`[url, title, duration]`; the key is always derived, never stored; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped. Timestamps stay in ms — they decide merges, and a delete and a re-add within one second must not collide. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. +- Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs each one dated item, EQ presets by name). `mergeBackups` is a **pure function of its two inputs** — no device clock, no "which side am I on" — so both devices compute the same result and the second has nothing left to push; that is also why every merged list is ordered by something the rows carry (`updatedAt`, the favorites' `orderedAt` rank, the preset name) rather than by whose array it was and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as **tombstones** ([core/persist/deletions.ts](src/core/persist/deletions.ts)): the removed row stays in its own list, marked `deleted` and dated, rather than being dropped — so the merge needs no deletion rules at all (last write wins, and a tombstone is a write), a re-add beats it by being newer, and a deletion can never reach an item it does not name. Written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, filtered out by the three panel stores so nothing downstream sees one, TTL 30 d. Songs are matched on the one key above; the re-key path in `track-sync` (a title the site rewrote after the media event) passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). A **replacement import** ([`replaceAll`](src/core/persist/deletions.ts)) is deliberately a *local* operation: it re-dates the file's rows and tombstones what **this device** held and the file omits — there is no "everything before now is gone" record, because a date range cannot be made safe across two devices' clocks. Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. -- **`@/` does not work in two contexts** (they don't share the WXT/Vite resolver): the `node --test` files (`src/**/*.test.ts` and the modules they import as *values* — `fft.ts`, `center-cut-dsp.ts`, `detect-bpm.ts`, `backup-codec.ts`, `sync/persist/fit.ts`, and what those pull in: `defaults.ts`, `thumbnail.ts`, `track-identity.ts`) must use **relative imports with explicit `.ts` extensions**; the esbuild worklet bundles (`src/features/*/engine/*.worklet.ts`) must use **relative imports**. (`import type` is erased, so type-only imports may omit the extension.) +- **`@/` does not work in two contexts** (they don't share the WXT/Vite resolver): the `node --test` files (`src/**/*.test.ts` and the modules they import as *values* — `fft.ts`, `center-cut-dsp.ts`, `detect-bpm.ts`, `backup-codec.ts`, `sync/persist/fit.ts`, `core/persist/rekey.ts`, and what those pull in: `defaults.ts`, `thumbnail.ts`, `track-identity.ts`) must use **relative imports with explicit `.ts` extensions**; the esbuild worklet bundles (`src/features/*/engine/*.worklet.ts`) must use **relative imports**. (`import type` is erased, so type-only imports may omit the extension.) - **Both browsers build MV3** (`manifestVersion: 3` is pinned in [wxt.config.ts](wxt.config.ts) — Firefox would otherwise default to MV2 and drop `optional_host_permissions`). Chromium-only APIs are gated on the build-time flags in [core/platform.ts](src/core/platform.ts) (`CAN_CAPTURE_TAB`, `HAS_SIDE_PANEL_API`), never on runtime `browser.*` probes: Firefox has no `tabCapture`/`offscreen` (so no capture fallback — the offscreen entrypoint is excluded from that build) and no `sidePanel` (the same page is registered as `sidebar_action`). Panel-side the capability travels as a **prop**: an absent `oncapture`/`ontabaudio` is what makes the shared UI drop the affordance. - Debug globals are named `__noteByNote*` / `__panelDebug`. The processor name literal is `note-by-note-center-cut` and **must match on both sides** (`vocal-reducer.worklet.ts` registers it, `vocal-reducer.ts` constructs it) — mismatches throw `InvalidStateError` at runtime and `tsc` won't catch them. - A `MediaElementSource` can be created only once per element per document lifetime, so **extension reloads require a page reload** to reattach. diff --git a/src/core/model/track-identity.ts b/src/core/model/track-identity.ts index ceb0cb5..5bc6df1 100644 --- a/src/core/model/track-identity.ts +++ b/src/core/model/track-identity.ts @@ -48,39 +48,41 @@ function hash(text: string): string { return (h >>> 0).toString(16); } -/** Whether two library rows describe the same song. Deliberately not a `key` - * comparison: the duration baked into `key` drifts (pre-roll ads, metadata that - * settles late), which would split one song across several Recent rows. The - * title is what keeps local files apart — they all share the local-player URL. */ -export function isSameTrack(a: TrackIdentity, b: TrackIdentity): boolean { - return a.normalizedUrl === b.normalizedUrl && a.title === b.title; -} - -/** The storage key of a track: `${hash(normalizedUrl)}:${durationSec}`, with - * `durationSec` already rounded. Exported so a compact backup can leave the key - * out and rebuild it from the two strings it stores anyway (backup-codec.ts). */ -export function identityKey(normalizedUrl: string, durationSec: number): string { - return `${hash(normalizedUrl)}:${durationSec}`; -} - -/** A short handle for what `isSameTrack` compares — URL and title, no - * duration — so a record about "this song" (a sync deletion, say) reaches - * every copy of it however its duration drifted. */ +/** + * What makes a song itself: its normalized URL and its title, hashed. This is + * `TrackIdentity.key` — the storage key of its track record, the id every + * library list is matched on, and what a tombstone names. One key, so no two + * parts of the app can disagree about what counts as the same song. + * + * **Duration is not in it.** It drifts — a pre-roll ad, metadata that settles + * late — and a key that moved with it split one song across several records, + * which every list then had to work around. Duration is metadata now: stored, + * shown, and updated in place. + * + * The title is in it because the URL alone is not enough: every local file + * reports the local-player page URL and is told apart only by its title, and + * a page can hold more than one song. + */ export function songKey(identity: Pick): string { return hash(`${identity.normalizedUrl}\n${identity.title}`); } +/** Whether two library rows describe the same song. */ +export function isSameTrack(a: TrackIdentity, b: TrackIdentity): boolean { + return a.key === b.key; +} + export function makeTrackIdentity( pageUrl: string, title: string, durationSec: number, ): TrackIdentity { const normalizedUrl = normalizeUrl(pageUrl); - const duration = Number.isFinite(durationSec) ? Math.round(durationSec) : 0; + const cleaned = cleanTitle(title); return { - key: identityKey(normalizedUrl, duration), + key: songKey({ normalizedUrl, title: cleaned }), normalizedUrl, - title: cleanTitle(title), - durationSec: duration, + title: cleaned, + durationSec: Number.isFinite(durationSec) ? Math.round(durationSec) : 0, }; } diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index b548d40..7e6810c 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -461,29 +461,24 @@ test('identity: the key is rebuilt from the URL and duration, never stored', () ); }); -test('identity: a key that cannot be rebuilt travels explicitly', () => { - const odd = { ...ytSong, key: 'legacy:230' }; - const b = backup({ history: [entry(odd)] }); - const enc = encodeBackup(b); - assert.equal(enc.songs[0][3], 'legacy:230'); - assert.equal(roundTrip(b).history[0].identity.key, 'legacy:230'); +test('identity: a key a file carried is read past; the key is derived', () => { + // Older files stored a fourth element when the key didn't match the formula + // that build used. Every build derives its own key from the same two + // strings, which is what lets devices on either side of a key change read + // each other's blobs. + const enc = JSON.parse(JSON.stringify(encodeBackup(backup({ history: [entry(ytSong)] })))); + enc.songs[0][3] = 'legacy:230'; + assert.equal(decodeBackup(enc).history[0].identity.key, ytSong.key); }); -test('identity: one song is one row; a duration that drifted is another', () => { +test('identity: a duration that drifted is the same song', () => { const drifted = makeTrackIdentity(YT_HREF, ytSong.title, 231); - const b = backup({ - history: [entry(ytSong)], - favorites: [favorite(ytSong)], - tracks: [track(ytSong), track(drifted)], - }); + assert.equal(drifted.key, ytSong.key, 'duration is metadata, not identity'); + const b = backup({ history: [entry(ytSong)], favorites: [favorite(ytSong)], tracks: [track(ytSong)] }); const enc = encodeBackup(b); - assert.equal(enc.songs.length, 2); + assert.equal(enc.songs.length, 1, 'one table row for the song'); assert.equal(enc.h[0].i, enc.f[0].i); - const back = roundTrip(b); - assert.deepEqual( - back.tracks.map((t) => t.identity.key).sort(), - [ytSong.key, drifted.key].sort(), - ); + assert.deepEqual(roundTrip(b).tracks.map((t) => t.identity.key), [ytSong.key]); }); // --------------------------------------------------------------------------- diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index dd9f92a..26cb869 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -4,7 +4,7 @@ import { DEFAULT_UI_PREFS, } from '../model/defaults.ts'; import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; -import { identityKey } from '../model/track-identity.ts'; +import { songKey } from '../model/track-identity.ts'; import type { ChordChart, ChordSegment, @@ -114,10 +114,11 @@ export interface CompactParams { b?: number; } -/** `[normalizedUrl, title, durationSec, key?]`. YouTube watch URLs are - * shortened to `yt:`. `key` appears only when it can't be rebuilt from the - * other two — a safety net, never the case for keys this build made. */ -export type CompactSong = [string, string, number] | [string, string, number, string]; +/** `[normalizedUrl, title, durationSec]`. YouTube watch URLs are shortened to + * `yt:`. The key is never stored: it is `songKey` of the first two, which + * is why a file written by a build that keyed songs differently still reads — + * each build derives the key it uses from the same two strings. */ +export type CompactSong = [string, string, number]; export interface CompactEntry { /** Index into `songs`. */ @@ -451,14 +452,10 @@ class SongTable { const url = identity.normalizedUrl ?? ''; const title = identity.title ?? ''; const duration = Number.isFinite(identity.durationSec) ? identity.durationSec : 0; - const key = identity.key ?? identityKey(url, duration); - const tableKey = `${url}\n${title}\n${duration}\n${key}`; + const tableKey = `${url}\n${title}\n${duration}`; const existing = this.#index.get(tableKey); if (existing !== undefined) return existing; - const row: CompactSong = - key === identityKey(url, duration) - ? [shortUrl(url), title, duration] - : [shortUrl(url), title, duration, key]; + const row: CompactSong = [shortUrl(url), title, duration]; this.rows.push(row); this.#index.set(tableKey, this.rows.length - 1); return this.rows.length - 1; @@ -468,12 +465,13 @@ class SongTable { function decodeSongs(raw: unknown): TrackIdentity[] { return arr(raw, 'songs').map((row) => { const r = arr(row, 'songs'); + // A fourth element is a key from a build that stored one; the key is + // derived here either way, so it is read past rather than trusted. if (r.length < 3 || r.length > 4) throw damaged('songs'); const normalizedUrl = longUrl(str(r[0], 'songs')); const title = str(r[1], 'songs'); const durationSec = num(r[2], 'songs'); - const key = r.length === 4 ? str(r[3], 'songs') : identityKey(normalizedUrl, durationSec); - return { key, normalizedUrl, title, durationSec }; + return { key: songKey({ normalizedUrl, title }), normalizedUrl, title, durationSec }; }); } diff --git a/src/core/persist/deletions.ts b/src/core/persist/deletions.ts index 02b3554..6f9512f 100644 --- a/src/core/persist/deletions.ts +++ b/src/core/persist/deletions.ts @@ -25,7 +25,6 @@ */ import type { Backup } from './backup-codec.ts'; -import { songKey } from '../model/track-identity.ts'; import type { HistoryEntry } from '../model/types'; export const DELETION_TTL_MS = 30 * 24 * 60 * 60_000; @@ -53,7 +52,7 @@ export const tombstone = (item: T, now: number): T => ({ export const pruneTombstones = (list: T[], now: number): T[] => list.filter((item) => isLive(item) || now - at(item) < DELETION_TTL_MS); -const song = (entry: HistoryEntry) => songKey(entry.identity); +const song = (entry: HistoryEntry) => entry.identity.key; /** * A replacement import, as items: the file's contents dated now, plus a diff --git a/src/core/persist/migrate.ts b/src/core/persist/migrate.ts new file mode 100644 index 0000000..b913624 --- /dev/null +++ b/src/core/persist/migrate.ts @@ -0,0 +1,57 @@ +import { storage } from '#imports'; +import type { TrackData } from '../model/types'; +import { rekeyByIdentity } from './rekey'; +import { favoritesItem, historyItem, removeTrackDataExcept, saveTrackData } from './storage'; + +/** + * One-shot storage migrations, run once at panel boot before any store reads. + * + * Local only. The wire format needs no migration of its own: a compact backup + * stores a song as `[url, title, duration]` and every build derives the key it + * uses from those, so a device on either side of a key change reads the other's + * blob and re-encodes it identically (`backup-codec.ts`). + */ + +/** 2: `TrackIdentity.key` stopped being `hash(url):duration` and became + * `songKey` (url + title) — one key for the track record, the library lists + * and the tombstones alike. See `track-identity.ts`. */ +const SCHEMA_VERSION = 2; + +const schemaVersionItem = storage.defineItem('local:schemaVersion', { fallback: 0 }); + +/** Track records live one per storage key, so they are rewritten under the new + * key and the old ones removed afterwards — written first, never wiped first, + * the same order `restoreBackup` uses and for the same reason: interrupted + * halfway this leaves a stale record behind, never a missing one. */ +async function rekeyTrackData(): Promise { + const snapshot = await browser.storage.local.get(null); + const records = Object.entries(snapshot) + .filter(([key]) => key.startsWith('track:')) + .map(([, value]) => value as TrackData); + const kept = rekeyByIdentity(records); + await Promise.all(kept.map(saveTrackData)); + await removeTrackDataExcept(new Set(kept.map((t) => t.identity.key))); +} + +/** Idempotent, and cheap on the common path — one storage read when there is + * nothing to do. Awaited before the stores load so nothing reads half-migrated + * data; a failure is left to throw, since booting the panel onto keys that + * don't match its storage would be worse than not booting. */ +export async function migrateStorage(): Promise { + const from = await schemaVersionItem.getValue(); + if (from >= SCHEMA_VERSION) return; + + if (from < 2) { + const [history, favorites] = await Promise.all([ + historyItem.getValue(), + favoritesItem.getValue(), + ]); + await rekeyTrackData(); + await Promise.all([ + historyItem.setValue(rekeyByIdentity(history)), + favoritesItem.setValue(rekeyByIdentity(favorites)), + ]); + } + + await schemaVersionItem.setValue(SCHEMA_VERSION); +} diff --git a/src/core/persist/rekey.test.ts b/src/core/persist/rekey.test.ts new file mode 100644 index 0000000..b744e56 --- /dev/null +++ b/src/core/persist/rekey.test.ts @@ -0,0 +1,65 @@ +// Run with: pnpm test:dsp (node --test). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { rekeyByIdentity, type Keyed } from './rekey.ts'; +import { makeTrackIdentity, songKey } from '../model/track-identity.ts'; +import type { TrackIdentity } from '../model/types.ts'; + +const URL_A = 'https://www.youtube.com/watch?v=aaaaaaaaaaa'; +const URL_B = 'https://www.youtube.com/watch?v=bbbbbbbbbbb'; + +/** A row as an older build stored it: the key it used baked in the duration. */ +function stored(url: string, title: string, durationSec: number, updatedAt: number) { + const identity: TrackIdentity = { + ...makeTrackIdentity(url, title, durationSec), + key: `legacy:${durationSec}`, + }; + return { identity, updatedAt, mark: `${title}@${durationSec}` }; +} + +test('the key is re-derived from the URL and title', () => { + const [row] = rekeyByIdentity([stored(URL_A, 'Song', 200, 1)]); + assert.equal(row.identity.key, songKey(row.identity)); + assert.equal(row.identity.durationSec, 200, 'duration is kept, as metadata'); +}); + +test('copies of one song under drifted durations collapse to the newest', () => { + const rows = rekeyByIdentity([ + stored(URL_A, 'Song', 200, 10), + stored(URL_A, 'Song', 201, 30), + stored(URL_A, 'Song', 199, 20), + ]); + assert.equal(rows.length, 1); + assert.equal(rows[0].mark, 'Song@201', 'the most recently written copy'); +}); + +test('a different title at the same URL stays its own song', () => { + const rows = rekeyByIdentity([ + stored(URL_A, 'Song', 200, 10), + stored(URL_A, 'Other', 200, 10), + ]); + assert.equal(rows.length, 2); +}); + +test('list order is kept, at the position of the first copy seen', () => { + const rows = rekeyByIdentity([ + stored(URL_A, 'First', 200, 10), + stored(URL_B, 'Second', 200, 10), + stored(URL_A, 'First', 201, 99), + ]); + assert.deepEqual(rows.map((r) => r.mark), ['First@201', 'Second@200']); +}); + +test('rows without a usable identity are dropped, never merged into one', () => { + const junk = [ + { identity: undefined, updatedAt: 1 }, + { identity: {}, updatedAt: 2 }, + null, + ] as unknown as Keyed[]; + assert.deepEqual(rekeyByIdentity([...junk, stored(URL_A, 'Song', 200, 1)]).length, 1); +}); + +test('re-running it changes nothing', () => { + const once = rekeyByIdentity([stored(URL_A, 'Song', 200, 10), stored(URL_B, 'Two', 90, 20)]); + assert.deepEqual(rekeyByIdentity(once), once); +}); diff --git a/src/core/persist/rekey.ts b/src/core/persist/rekey.ts new file mode 100644 index 0000000..e0de1ca --- /dev/null +++ b/src/core/persist/rekey.ts @@ -0,0 +1,39 @@ +import { songKey } from '../model/track-identity.ts'; +import type { TrackIdentity } from '../model/types'; + +/** + * Re-deriving stored keys, for the migration in `migrate.ts`. + * + * Pure and DOM-free (relative `.ts` imports; runs under `node --test`) because + * it is the one step of that migration that can lose data: rows a key change + * brings together have to be collapsed, and the right copy has to survive. + */ + +export interface Keyed { + identity: TrackIdentity; + updatedAt?: number; +} + +const at = (row: Keyed) => row.updatedAt ?? 0; + +/** + * Rows under the key `songKey` derives from them now, with copies that land on + * the same key collapsed to the most recently written one — a song saved under + * two durations was always one song, and this is where its copies finally meet. + * + * List order is kept (Recent is newest-first, Favorites is the manual order): a + * survivor takes the position of the first copy seen, not of the copy that won. + * Rows without a usable identity are dropped rather than given a key derived + * from `undefined`, which would collide every one of them into a single row. + */ +export function rekeyByIdentity(rows: T[]): T[] { + const byKey = new Map(); + for (const row of rows) { + if (typeof row?.identity?.normalizedUrl !== 'string') continue; + const key = songKey(row.identity); + const next = { ...row, identity: { ...row.identity, key } }; + const current = byKey.get(key); + if (!current || at(next) >= at(current)) byKey.set(key, next); + } + return [...byKey.values()]; +} diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index 21ce5b6..1bd5afb 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -105,20 +105,19 @@ class TrackSync { if (identity.key === this.#applyingKey) return; if (identity.key === this.#identity?.key) { - // Same track — but the title may have settled late (SPA navigation). - if (identity.title !== this.#identity.title) { - this.#identity = identity; - if (this.#userAdjusted) await this.#saveCurrent(); - } - // A no-op unless something changed what this song is or what is applied to - // it: the title just settled, or the engine restarted on the defaults. + // Same song. Its duration may have settled since (a pre-roll ad, slow + // metadata) — duration is metadata, not identity, so it is updated in + // place and nothing is re-keyed (`track-identity.ts`). + if (identity.durationSec !== this.#identity.durationSec) this.#identity = identity; + // A no-op unless the engine restarted on the defaults. if (!this.#userAdjusted) this.#restoreSaved(identity); return; } - // Same page, new duration — the metadata settled late (ad, slow load) and - // the track got keyed with a stale duration. Re-key in place instead of - // treating it as a track switch, so Recents doesn't get a duplicate row. + // Same page, new title — the site rewrote document.title after the element + // fired, so the song was keyed under the placeholder. Re-key in place + // instead of treating it as a track switch, so Recent doesn't get a + // duplicate row. if (this.#identity && identity.normalizedUrl === this.#identity.normalizedUrl) { const staleKey = this.#identity.key; const adjusted = this.#userAdjusted; @@ -132,9 +131,9 @@ class TrackSync { // the stale identity gets another go now the duration has settled. if (!adjusted) this.#restoreSaved(identity); if (adjusted) { - // Housekeeping, not a user deletion: the song stays, only its - // stale-keyed twin goes — so no deletion record (which is per song - // and would kill the fresh row on the other devices). + // Housekeeping, not a user deletion: this is the same song under its + // real title, so no tombstone — that names the song, and would kill + // its fresh row on the other devices. await removeHistoryEntry(staleKey, { record: false }); await this.#saveCurrent(); // Only carry the stale key's slice over when the real key has none, so diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index 87983da..de8dee8 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -8,6 +8,7 @@ import { sendMessage } from '@/core/messaging/rpc'; import { openTabWithPanel } from '@/core/side-panel'; import { installMockState, installMockTicker } from '@/dev/mock'; + import { migrateStorage } from '@/core/persist/migrate'; import { connection } from '@/core/state/connect.svelte'; import { CAN_CAPTURE_TAB } from '@/core/platform'; import { features } from '@/core/features'; @@ -22,8 +23,11 @@ const mock = params.has('mock'); // ?mock=1&play=1 also runs the playhead, for previewing time-driven UI. const mockPlay = mock && params.has('play'); - // Each panel feature loads its own storage concurrently (see core/features.ts). - const ready = Promise.all(features.map((f) => f.init?.())).then( + // Storage first, then the features: a migration rewrites what they are about + // to read (see core/persist/migrate.ts). Each panel feature then loads its + // own storage concurrently (see core/features.ts). + const loadFeatures = () => Promise.all(features.map((f) => f.init?.())); + const ready = migrateStorage().then(loadFeatures).then( async () => { applyTheme(settings.current.theme); trackSync.init(); diff --git a/src/features/library/panel/history.svelte.ts b/src/features/library/panel/history.svelte.ts index 960a122..63fdd94 100644 --- a/src/features/library/panel/history.svelte.ts +++ b/src/features/library/panel/history.svelte.ts @@ -1,5 +1,5 @@ import type { HistoryEntry } from '../../../core/model/types'; -import { clearHistory, dedupeHistory, removeHistoryEntry } from '../persist/history'; +import { clearHistory, removeHistoryEntry } from '../persist/history'; import { isLive } from '../../../core/persist/deletions'; import { historyItem } from '../../../core/persist/storage'; @@ -10,9 +10,6 @@ class HistoryStore { entries = $state([]); async init() { - // Rows saved before dedupe-on-write can already be duplicated; collapse - // them once, on the way in, so the list the user sees is the stored one. - await dedupeHistory(); this.entries = (await historyItem.getValue()).filter(isLive); historyItem.watch((value) => { this.entries = (value ?? []).filter(isLive); diff --git a/src/features/library/persist/history.ts b/src/features/library/persist/history.ts index 8b6b5d3..4b694f2 100644 --- a/src/features/library/persist/history.ts +++ b/src/features/library/persist/history.ts @@ -38,26 +38,12 @@ export async function upsertHistory( createdAt: existing?.createdAt ?? now, updatedAt: now, }; - // Matched by song, not by key: this row supersedes every older one for the - // same song, so a duration that settled differently can't leave a twin - // behind — and a tombstone for the song goes with them, this play being the - // newer statement about it. + // This row supersedes any older one for the same song — a tombstone for it + // included, this play being the newer statement about it. const next = [entry, ...list.filter((e) => !isSameTrack(e.identity, identity))]; await historyItem.setValue(capped(next)); } -/** Collapse rows written before saves were matched by song (one song split - * across several durations). The list is newest-first, so the first row for a - * song wins and the older twins are dropped. */ -export async function dedupeHistory(): Promise { - const list = await historyItem.getValue(); - const kept: HistoryEntry[] = []; - for (const entry of list) { - if (!kept.some((e) => isSameTrack(e.identity, entry.identity))) kept.push(entry); - } - if (kept.length !== list.length) await historyItem.setValue(kept); -} - /** The user removed a row: it stays as a tombstone (`deletions.ts`) so a sync * merge with another device's older copy doesn't bring it back, and moves to * the front to keep the list newest-first. `record: false` is for housekeeping diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 51fc5b0..d4a13aa 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -69,11 +69,11 @@ const fit = (backup: Backup) => fitBackup(backup, BUDGET_CHARS, measure); * deferred while a track is loaded and picked up on the next quiet moment, * panel open, or "Sync now". * - * Runs in every open panel document (there can be more than one: a Firefox - * window each, the local-player tab). They share `syncConfig` through its - * watch, so at worst two push the same content, which the spacing absorbs. - * A change the panel didn't manage to push before closing is remembered via - * `pendingPush`. + * Runs in every open panel document, and there is more than one: Chromium + * gives each tab its own, Firefox each window. They share `syncConfig` through + * its watch but not their queues or timers, so at worst two push the same + * content, which the spacing absorbs. A change the panel didn't manage to push + * before closing is remembered via `pendingPush`. */ class SyncStore { config = $state({ ...DEFAULT_SYNC_CONFIG }); diff --git a/src/features/sync/persist/fit.ts b/src/features/sync/persist/fit.ts index dc29d53..b7c7b0d 100644 --- a/src/features/sync/persist/fit.ts +++ b/src/features/sync/persist/fit.ts @@ -1,4 +1,3 @@ -import { songKey } from '../../../core/model/track-identity.ts'; import type { Backup } from '../../../core/persist/backup-codec.ts'; import { isLive } from '../../../core/persist/deletions.ts'; import type { HistoryEntry, TrackData, TrackIdentity } from '../../../core/model/types'; @@ -18,8 +17,8 @@ import type { HistoryEntry, TrackData, TrackIdentity } from '../../../core/model * Each group oldest first. Settings, UI prefs, EQ presets and tombstones * (`deletions.ts`) are never cut — a dropped tombstone is a row another * device brings straight back, and they are a handful of bytes each. - * Songs are matched the way the library does (`songKey`: URL + title), so a - * record saved under a drifted duration still follows its favorite. + * A song's row and its track record share one key, so a record always goes + * with the row it belongs to. * * `measure` is injected (and may be async): the caller decides what "size" * means — encoded JSON length in tests, the gzip+base64 blob for sync — so @@ -45,7 +44,7 @@ export class LibraryTooLargeError extends Error { export const hasChart = (track: TrackData) => !!track.chordChart && track.chordChart.segments.length > 0; -/** A song (by `songKey`, row + record) or a chart (by track key). */ +/** A song (row + record) or a chart, both by the song's key. */ interface Cut { kind: 'song' | 'chart'; id: string; @@ -58,10 +57,10 @@ const oldestFirst = (a: Cut, b: Cut) => /** Everything that may go, in the order it goes. */ function collectCuts(backup: Backup): Cut[] { - const favorites = new Set(backup.favorites.filter(isLive).map((f) => songKey(f.identity))); + const favorites = new Set(backup.favorites.filter(isLive).map((f) => f.identity.key)); const songs = new Map(); const touch = (identity: TrackIdentity, at: number) => { - const id = songKey(identity); + const id = identity.key; if (!favorites.has(id)) songs.set(id, Math.max(songs.get(id) ?? 0, at)); }; for (const entry of backup.history.filter(isLive)) touch(entry.identity, entry.updatedAt ?? 0); @@ -69,7 +68,7 @@ function collectCuts(backup: Backup): Cut[] { const accessed = new Map(); for (const f of backup.favorites.filter(isLive)) { - const id = songKey(f.identity); + const id = f.identity.key; const at = f.lastAccessedAt ?? f.updatedAt ?? 0; accessed.set(id, Math.max(accessed.get(id) ?? 0, at)); } @@ -87,7 +86,7 @@ function apply(backup: Backup, cuts: Cut[]): Backup { const songs = new Set(cuts.filter((c) => c.kind === 'song').map((c) => c.id)); const charts = new Set(cuts.filter((c) => c.kind === 'chart').map((c) => c.id)); const keep = (row: HistoryEntry | TrackData) => - !isLive(row) || !songs.has(songKey(row.identity)); + !isLive(row) || !songs.has(row.identity.key); return { ...backup, history: backup.history.filter(keep), diff --git a/src/features/sync/persist/merge.test.ts b/src/features/sync/persist/merge.test.ts index 938e8e8..87785b2 100644 --- a/src/features/sync/persist/merge.test.ts +++ b/src/features/sync/persist/merge.test.ts @@ -284,19 +284,25 @@ test('a tie goes to the tombstone, and both devices break it the same way', () = ); }); -test('a deletion reaches every copy of the song, whatever duration it was saved under', () => { +test('a deletion reaches the song however its duration drifted', () => { const url = 'https://www.youtube.com/watch?v=drifted0001'; const s201 = makeTrackIdentity(url, 'Song', 201); const s200 = makeTrackIdentity(url, 'Song', 200); - assert.notEqual(s201.key, s200.key); + assert.equal(s201.key, s200.key, 'one song, one key — duration is metadata'); const local = backup({ history: [tombstone(row(s201, T0), T0 + 5000)], favorites: [tombstone(fav(s201, T0), T0 + 5000)], + tracks: [{ ...record(s201, T0 + 5000, 0), markers: [] }], + }); + const remote = backup({ + history: [row(s200, T0)], + favorites: [fav(s200, T0)], + tracks: [record(s200, T0, 3)], }); - const remote = backup({ history: [row(s200, T0)], favorites: [fav(s200, T0)] }); const merged = mergeBackups(local, remote, NOW); assert.equal(live(merged.history).length, 0); assert.equal(live(merged.favorites).length, 0); + assert.equal(merged.tracks.length, 1, 'and its record is one record, not two'); // A different song at the same URL (local files share one) is untouched. const other = makeTrackIdentity(url, 'Other song', 200); const kept = mergeBackups(local, backup({ history: [row(other, T0)] }), NOW); diff --git a/src/features/sync/persist/merge.ts b/src/features/sync/persist/merge.ts index 635dd59..544dbde 100644 --- a/src/features/sync/persist/merge.ts +++ b/src/features/sync/persist/merge.ts @@ -1,5 +1,4 @@ import { HISTORY_LIMIT } from '../../../core/model/defaults.ts'; -import { songKey } from '../../../core/model/track-identity.ts'; import type { Backup } from '../../../core/persist/backup-codec.ts'; import { at, isLive, pruneTombstones, type Deletable } from '../../../core/persist/deletions.ts'; import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; @@ -12,13 +11,12 @@ import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; * "removed over there" needs no case of its own, and neither does a re-add — * it is simply newer. Ties go to the tombstone. * - * - Recent rows and favorites: matched by song (URL + title, like the - * library itself), so a copy saved under a drifted duration is the same - * song. A favorite's last access is the later of the two. - * - Track records: matched by key. An emptied record is still a record, so - * clearing markers sticks. A chart is chosen separately by `computedAt`: - * null may mean "trimmed", but an empty, dated chart is an explicit - * deletion and beats older analysis. + * - Recent rows, favorites and track records: all matched on the one key a + * song has (`track-identity.ts`). A favorite's last access is the later of + * the two. An emptied record is still a record, so clearing markers + * sticks; a chart is chosen separately by `computedAt`, since null may + * mean "trimmed" but an empty, dated chart is an explicit deletion and + * beats older analysis. * - Settings and UI prefs: one item each, with one date, taken whole. * EQ presets: union by name. * - Last, the two library copies of a song (Recent and Favorites) are put @@ -33,7 +31,10 @@ import type { FavoriteEntry, HistoryEntry } from '../../../core/model/types'; * each. `node --test`. */ -const song = (entry: HistoryEntry) => songKey(entry.identity); +/** Every list is keyed the same way, on the one identity a song has + * (`track-identity.ts`) — Recent rows, favorites, tombstones and the track + * record all answer to it. */ +const song = (entry: { identity: { key: string } }) => entry.identity.key; const compare = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); @@ -100,7 +101,7 @@ const rankOf = (f: FavoriteEntry) => f.orderedAt ?? f.favoritedAt ?? at(f); * in favorites alone, the list simply stays long. Tombstones are kept whatever * the count — they go by age, not by rank. */ function capHistory(history: HistoryEntry[], favorites: FavoriteEntry[]): HistoryEntry[] { - const ordered = [...history].sort(orderOf(at, song)); + const ordered = [...history].sort(orderOf(at, song)); const live = ordered.filter(isLive); const excess = live.length - HISTORY_LIMIT; if (excess <= 0) return ordered; @@ -163,12 +164,12 @@ export function mergeBackups(local: Backup, remote: Backup, now = Date.now()): B .sort((a, b) => compare(a.name, b.name)), history: capHistory(history.map(align), favorites), favorites: favorites.map(align), - tracks: unionNewest(local.tracks, remote.tracks, (t) => t.identity.key, (w, l) => ({ + tracks: unionNewest(local.tracks, remote.tracks, song, (w, l) => ({ ...w, chordChart: !w.chordChart || (l.chordChart?.computedAt ?? 0) > w.chordChart.computedAt ? l.chordChart ?? w.chordChart : w.chordChart, - })).sort((a, b) => compare(a.identity.key, b.identity.key)), + })).sort((a, b) => compare(song(a), song(b))), }; } From cd77e304c90aaf26b4edf92a787f39d5ac0549c2 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 17:11:00 +0200 Subject: [PATCH 13/26] refactor: improve backup handling by enforcing key re-derivation and rejecting unsupported formats --- CLAUDE.md | 2 +- src/core/persist/backup-codec.test.ts | 44 ++++++++--------- src/core/persist/backup-codec.ts | 70 +++++++++++++-------------- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2746e8a..bb6caf3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel ### Persistence & sync - [storage.ts](src/core/persist/storage.ts) — WXT `storage.defineItem` wrappers (the full storage schema stays central here). **Per-track data is keyed by the song's identity** ([track-identity.ts](src/core/model/track-identity.ts)): site-aware URL normalization (strips `t`/`si`/`utm_*` etc.; collapses YouTube to `watch?v=`) plus the cleaned title, hashed by `songKey` to `local:track:`. **Duration is metadata, not identity** — it drifts (pre-roll ad, late metadata), and a key that moved with it split one song across several records, which every list then worked around. That is one key: track records, Recent rows, favorites and tombstones all answer to it. The title is in it because a URL alone is not enough — every local file reports the local-player page URL and is told apart only by its title. Storage written before this is re-keyed once at panel boot ([migrate.ts](src/core/persist/migrate.ts), collapsing logic in the node-tested [rekey.ts](src/core/persist/rekey.ts)); the wire format needs no migration, since a compact backup stores `[url, title, duration]` and each build derives its own key from those. The per-track `TrackData` record is assembled/scattered by feature descriptors ([core/persist/track-data.ts](src/core/persist/track-data.ts)). EQ presets and granted origins live in their own items so "Reset Settings" can't wipe them. -- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v3, reading v2 too) that the export writes and import reads alongside v1. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (`[url, title, duration]`; the key is always derived, never stored; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped. Timestamps stay in ms — they decide merges, and a delete and a re-add within one second must not collide. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. +- **Backup format** ([backup-codec.ts](src/core/persist/backup-codec.ts), pure): the in-memory `Backup` (v1, what `createBackup` builds) has a compact serialization (v3) that the export writes. Import reads exactly two shapes: v3, and the verbose v1 — the only format any released build wrote, so the only one an existing user's file is in (its stored keys are re-derived on the way in, see below). v2 never shipped outside this branch and is refused. Storage shapes are untouched — the codec only shrinks the wire: one `songs` identity table referenced by index (`[url, title, duration]`; the key is always derived, never stored; YouTube URLs as `yt:`), params/settings/UI prefs as deltas against the defaults, chord charts as parallel centisecond arrays with a label table, marker/snippet ids and derivable thumbnails/page URLs dropped. Timestamps stay in ms — they decide merges, and a delete and a re-add within one second must not collide. ~9× smaller before gzip; `encode(decode(encode(x)))` equals `encode(x)` so devices can compare content hashes. [sync/persist/fit.ts](src/features/sync/persist/fit.ts) cuts a `Backup` to a byte budget with an injected `measure` — no count caps: non-favorited songs oldest-first, then chord charts oldest-first, then favorites by last access; settings, prefs, presets and favorites' markers/snippets are never cut. - Optional **cross-device sync** (`src/features/sync/`) rides `browser.storage.sync` — no server, no ID, no cookies, no host permission; the browser vendor's sync carries the bytes. The compact backup (`encodeBackup`) is gzipped + base64 + chunked into `nbn.meta` / `nbn.0…` items of ≤ 8 KB ([sync-blob.ts](src/features/sync/persist/sync-blob.ts); `meta.h` over the joined base64 detects a **torn** read — items sync one by one — and recognises our own echo), cut to the 88 KB budget by [fit.ts](src/features/sync/persist/fit.ts) when it must be. The store ([sync.svelte.ts](src/features/sync/panel/sync.svelte.ts)) has one routine, `#reconcile`: read the area; `none` → seed; `torn` → retry for 90 s, then overwrite; same `meta.h` as last time → push if local changed; else **merge** ([merge.ts](src/features/sync/persist/merge.ts): union by song/key, newer `updatedAt` wins, settings/prefs each one dated item, EQ presets by name). `mergeBackups` is a **pure function of its two inputs** — no device clock, no "which side am I on" — so both devices compute the same result and the second has nothing left to push; that is also why every merged list is ordered by something the rows carry (`updatedAt`, the favorites' `orderedAt` rank, the preset name) rather than by whose array it was and write the result locally (`restoreBackup` + panel reload, deferred to a moment with no track loaded — `pendingApply`) and/or remotely. Removals travel as **tombstones** ([core/persist/deletions.ts](src/core/persist/deletions.ts)): the removed row stays in its own list, marked `deleted` and dated, rather than being dropped — so the merge needs no deletion rules at all (last write wins, and a tombstone is a write), a re-add beats it by being newer, and a deletion can never reach an item it does not name. Written by `removeHistoryEntry`/`clearHistory`/`removeFavorite`/`deleteEqPreset`, filtered out by the three panel stores so nothing downstream sees one, TTL 30 d. Songs are matched on the one key above; the re-key path in `track-sync` (a title the site rewrote after the media event) passes `record: false` because the song itself stays. Track records need none (an emptied record still wins on `updatedAt`). A **replacement import** ([`replaceAll`](src/core/persist/deletions.ts)) is deliberately a *local* operation: it re-dates the file's rows and tombstones what **this device** held and the file omits — there is no "everything before now is gone" record, because a date range cannot be made safe across two devices' clocks. Every trigger (local change, remote change, retry, safety interval) just asks for a reconcile on one timer; pushes are debounced 5 s and `#push` itself enforces the ≥ 30 s spacing (the browser meters writes at 120/min) by re-scheduling. The per-device bookkeeping (`local:syncConfig`: `lastRemoteHash`, `lastLocalHash`, `pendingPush`, `trimmed`) is read over its defaults, so a server-era record simply falls through to the merge path. Legacy `syncId` keys in the area are removed on the next write. ## Conventions & gotchas diff --git a/src/core/persist/backup-codec.test.ts b/src/core/persist/backup-codec.test.ts index 7e6810c..82f59a8 100644 --- a/src/core/persist/backup-codec.test.ts +++ b/src/core/persist/backup-codec.test.ts @@ -461,14 +461,24 @@ test('identity: the key is rebuilt from the URL and duration, never stored', () ); }); -test('identity: a key a file carried is read past; the key is derived', () => { - // Older files stored a fourth element when the key didn't match the formula - // that build used. Every build derives its own key from the same two - // strings, which is what lets devices on either side of a key change read - // each other's blobs. - const enc = JSON.parse(JSON.stringify(encodeBackup(backup({ history: [entry(ytSong)] })))); - enc.songs[0][3] = 'legacy:230'; - assert.equal(decodeBackup(enc).history[0].identity.key, ytSong.key); +test('identity: a v1 file’s stored keys are re-derived, not trusted', () => { + // v1 is the only format a released build wrote, and it wrote keys that baked + // in the duration. They are derived again on the way in, so a song a v1 file + // holds twice — two durations, one song — comes back as one row with its + // markers intact. + const drifted = { ...makeTrackIdentity(YT_HREF, ytSong.title, 231), key: 'a1b2:231' }; + const v1 = JSON.parse(JSON.stringify(backup({ + history: [ + { ...entry(ytSong), identity: { ...ytSong, key: 'a1b2:230' }, updatedAt: 100 }, + { ...entry(ytSong), identity: drifted, updatedAt: 200 }, + ], + tracks: [{ ...track(ytSong), identity: { ...ytSong, key: 'a1b2:230' } }], + }))); + const back = parseBackupJson(v1); + assert.equal(back.history.length, 1, 'two durations were always one song'); + assert.equal(back.history[0].identity.key, ytSong.key); + assert.equal(back.history[0].updatedAt, 200, 'the more recently written copy'); + assert.equal(back.tracks[0].identity.key, ytSong.key); }); test('identity: a duration that drifted is the same song', () => { @@ -541,18 +551,6 @@ test('settings and prefs carry their own date, outside the diff', () => { assert.equal(roundTrip(b).uiPrefs.updatedAt, 1_757_000_000_001); }); -test('a version 2 file still reads; its del map is dropped', () => { - const v2 = { - ...JSON.parse(JSON.stringify(encodeBackup(library(3)))), - version: 2, - del: { 'h:abc:230': 1_757_112_345_678 }, - }; - const back = parseBackupJson(v2); - assert.equal(back.history.length, 3); - assert.equal(back.history.some((e) => e.deleted), false); - assert.equal('deletions' in back, false); -}); - test('exportedAt is kept; appVersion is dropped', () => { const back = roundTrip(backup()); assert.equal(back.exportedAt, 1_757_200_000_123); @@ -572,10 +570,12 @@ test('a verbose v1 file still parses and is backfilled', () => { assert.equal(back.appVersion, '1.0.3'); }); -test('a compact file routes through the codec; anything newer or foreign is refused', () => { +test('only v1 and the current compact shape are read; the rest are refused', () => { const compact = JSON.parse(JSON.stringify(encodeBackup(library(2)))); assert.equal(parseBackupJson(compact).history.length, 2); - assert.equal(parseBackupJson({ ...compact, version: 2 }).history.length, 2, 'v2 too'); + // 2 never shipped: no file and no synced blob is in it, so it is refused + // rather than carried — and the message says which way it is wrong. + assert.throws(() => parseBackupJson({ ...compact, version: 2 }), /no longer reads/); assert.throws(() => parseBackupJson({ ...compact, version: COMPACT_VERSION + 1 }), /newer version/); assert.throws(() => parseBackupJson({ ...compact, format: 'other' }), /isn't a Note by Note backup/); assert.throws(() => parseBackupJson('nope'), /isn't a Note by Note backup/); diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 26cb869..3863f4a 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -5,6 +5,7 @@ import { } from '../model/defaults.ts'; import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; import { songKey } from '../model/track-identity.ts'; +import { rekeyByIdentity } from './rekey.ts'; import type { ChordChart, ChordSegment, @@ -48,22 +49,18 @@ import type { /** Marks a file as ours, so a stray JSON can be rejected on sight. */ export const BACKUP_FORMAT = 'note-by-note-backup'; -/** The verbose shape: what `createBackup` builds and what exports were - * before the compact format. Still accepted on import. */ +/** The verbose shape: what `createBackup` builds, and the only format any + * released build ever wrote — so it is the one an existing user's exported + * file is in, and import still accepts it. */ export const BACKUP_VERSION = 1; -/** The compact shape below. `parseBackupJson` rejects anything newer. - * - * 3 added tombstones (`x`) to Recent, Favorites and the EQ presets, the - * favorites' `orderedAt` (`oa`) and the settings/prefs dates (`sat`/`uat`), - * and dropped the `del` map that 2 carried. A 2 file still reads — it simply - * has no tombstones, and its `del` records are not translatable (a deletion - * key names a song by hash, and a tombstone has to *be* the row). */ +/** The compact shape below, and what the export writes. `parseBackupJson` + * takes this or the verbose v1, and nothing else: version 2 never shipped — + * it existed only on the branch this format grew on — so no file and no synced + * blob is in it, and carrying a compatibility path for it would be carrying + * one for nobody. */ export const COMPACT_VERSION = 3; -/** The oldest compact shape still readable. */ -export const COMPACT_MIN_VERSION = 2; - /** Everything a user owns, in one file. Host permissions are deliberately out: * they live in the browser's permission store, and only a prompt can grant * them — a backup that listed origins would restore access it can't give. */ @@ -116,8 +113,8 @@ export interface CompactParams { /** `[normalizedUrl, title, durationSec]`. YouTube watch URLs are shortened to * `yt:`. The key is never stored: it is `songKey` of the first two, which - * is why a file written by a build that keyed songs differently still reads — - * each build derives the key it uses from the same two strings. */ + * is why a device on either side of a key change reads the other's blob — + * each derives the key it uses from the same two strings. */ export type CompactSong = [string, string, number]; export interface CompactEntry { @@ -265,14 +262,15 @@ function rec(value: unknown, section: string): Record { return value; } -/** Entries are keyed by `identity.key`; without one they can't be stored or - * matched back to a track, so a file carrying them is not usable. */ -function keyedArr(value: unknown, section: string): T[] { +/** Rows are identified by their URL and title (`songKey`); without those they + * can't be stored or matched back to a track, so a file carrying them is not + * usable. The key a file may also carry is not read — it is derived. */ +function identifiedArr(value: unknown, section: string): T[] { const list = arr(value, section); - const keyed = list.every( - (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.key === 'string', + const identified = list.every( + (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.normalizedUrl === 'string', ); - if (!keyed) throw damaged(section); + if (!identified) throw damaged(section); return list as T[]; } @@ -465,9 +463,7 @@ class SongTable { function decodeSongs(raw: unknown): TrackIdentity[] { return arr(raw, 'songs').map((row) => { const r = arr(row, 'songs'); - // A fourth element is a key from a build that stored one; the key is - // derived here either way, so it is read past rather than trusted. - if (r.length < 3 || r.length > 4) throw damaged('songs'); + if (r.length !== 3) throw damaged('songs'); const normalizedUrl = longUrl(str(r[0], 'songs')); const title = str(r[1], 'songs'); const durationSec = num(r[2], 'songs'); @@ -784,11 +780,7 @@ export function isEmptyBackup(backup: Backup): boolean { /** Reads a compact (v2) backup, or throws an `Error` whose message is safe to * show the user. Unknown keys are ignored so the format can grow. */ export function decodeBackup(raw: unknown): Backup { - const version = isRecord(raw) && typeof raw.version === 'number' ? raw.version : 0; - if ( - !isRecord(raw) || raw.format !== BACKUP_FORMAT || - version < COMPACT_MIN_VERSION || version > COMPACT_VERSION - ) { + if (!isRecord(raw) || raw.format !== BACKUP_FORMAT || raw.version !== COMPACT_VERSION) { throw new Error("That file isn't a Note by Note backup."); } const songs = decodeSongs(raw.songs); @@ -810,9 +802,11 @@ export function decodeBackup(raw: unknown): Backup { }; } -/** The verbose v1 file: today's in-memory shape, written out as is. Objects - * are backfilled from the defaults so a file from an older build gains any - * setting added since. */ +/** The verbose v1 file: the in-memory shape, written out as is. Objects are + * backfilled from the defaults so a file from an older build gains any setting + * added since, and every row's key is re-derived — v1 files were written when + * a key baked in the duration, and rows that split across two of those are one + * song again here (`rekey.ts`, `track-identity.ts`). */ function normalizeV1(raw: Record): Backup { return { format: BACKUP_FORMAT, @@ -827,10 +821,10 @@ function normalizeV1(raw: Record): Backup { ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), }, - history: keyedArr(raw.history, 'history'), - favorites: keyedArr(raw.favorites, 'favorites'), + history: rekeyByIdentity(identifiedArr(raw.history, 'history')), + favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], - tracks: keyedArr(raw.tracks, 'tracks'), + tracks: rekeyByIdentity(identifiedArr(raw.tracks, 'tracks')), }; } @@ -842,8 +836,12 @@ export function parseBackupJson(raw: unknown): Backup { if (!isRecord(raw) || raw.format !== BACKUP_FORMAT) { throw new Error("That file isn't a Note by Note backup."); } - if (typeof raw.version !== 'number' || raw.version > COMPACT_VERSION) { + const version = typeof raw.version === 'number' ? raw.version : 0; + if (version > COMPACT_VERSION) { throw new Error('That backup was made by a newer version of Note by Note.'); } - return raw.version >= COMPACT_MIN_VERSION ? decodeBackup(raw) : normalizeV1(raw); + if (version === COMPACT_VERSION) return decodeBackup(raw); + if (version === BACKUP_VERSION) return normalizeV1(raw); + // Only 2 lands here, and only from the branch this format grew on. + throw new Error('That backup is in a format this version of Note by Note no longer reads.'); } From deaa88c83ce147d21605ae562ba882daaf05b8bf Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 17:22:50 +0200 Subject: [PATCH 14/26] Refactor sync configuration and library persistence - Simplified SyncConfig interface by removing unused properties and adding lastPushAt, usedBytes, and syncing. - Updated DEFAULT_SYNC_CONFIG to reflect changes in SyncConfig. - Improved loadSyncConfig function for better readability and performance. - Added legacy backup support with a new legacy-backup.ts file for handling previous backup formats. - Implemented library background synchronization with library-client and library-migration modules. - Introduced library persistence with a new library.ts file to manage shared and local library data. - Enhanced sync records management with new records.ts file for handling sync data and quota management. - Added alarms for library synchronization and safety checks. - Updated wxt.config.ts to include 'alarms' permission for background tasks. --- src/core/messaging/protocol.ts | 4 + src/core/model/track-identity.ts | 9 +- src/core/persist/backup-codec.ts | 929 ++---------------- src/core/persist/backup.ts | 106 +- src/core/persist/legacy-backup.ts | 365 +++++++ src/core/persist/library-background.ts | 108 ++ src/core/persist/library-client.ts | 13 + src/core/persist/library-migration.ts | 52 + src/core/persist/library.ts | 181 ++++ src/core/persist/storage.ts | 118 +-- src/core/state/track-sync.svelte.ts | 297 ++---- src/entrypoints/background.ts | 2 + src/entrypoints/sidepanel/App.svelte | 6 +- src/features/chords/panel/chords.svelte.ts | 11 +- src/features/eq/panel/eq-presets.svelte.ts | 18 +- .../library/panel/favorites.svelte.ts | 83 +- src/features/library/panel/history.svelte.ts | 37 +- src/features/library/panel/library.svelte.ts | 13 + src/features/library/panel/panel.ts | 11 +- src/features/library/panel/saved-settings.ts | 30 +- .../settings/panel/SettingsView.svelte | 42 +- .../settings/panel/settings.svelte.ts | 5 +- src/features/sync/panel/sync.svelte.ts | 387 +------- src/features/sync/persist/records.ts | 73 ++ src/features/sync/persist/sync-config.ts | 50 +- wxt.config.ts | 1 + 26 files changed, 1075 insertions(+), 1876 deletions(-) create mode 100644 src/core/persist/legacy-backup.ts create mode 100644 src/core/persist/library-background.ts create mode 100644 src/core/persist/library-client.ts create mode 100644 src/core/persist/library-migration.ts create mode 100644 src/core/persist/library.ts create mode 100644 src/features/library/panel/library.svelte.ts create mode 100644 src/features/sync/persist/records.ts diff --git a/src/core/messaging/protocol.ts b/src/core/messaging/protocol.ts index 735c601..3d6a876 100644 --- a/src/core/messaging/protocol.ts +++ b/src/core/messaging/protocol.ts @@ -1,3 +1,4 @@ +import type { Library, LibraryCommand } from '../persist/library'; import type { ConnectionState, EffectParams, @@ -121,6 +122,9 @@ export type OffscreenCommand = /** RPC handled by the background service worker (via @webext-core/messaging). */ export interface ProtocolMap { + libraryRead(): Promise; + libraryEdit(command: LibraryCommand): Promise; + librarySync(action: 'now' | 'enable' | 'disable' | 'delete'): Promise; /** Request per-origin host permission, inject + persist the content script. * Must run after the side panel already obtained the permission grant. */ ensureInjected(data: { tabId: number }): Promise<{ ok: boolean; error?: string }>; diff --git a/src/core/model/track-identity.ts b/src/core/model/track-identity.ts index 5bc6df1..fbbe412 100644 --- a/src/core/model/track-identity.ts +++ b/src/core/model/track-identity.ts @@ -14,7 +14,7 @@ function normalizeUrl(rawUrl: string): string { const host = url.hostname.replace(/^www\./, ''); // Site-aware rules: keep only the media id where we know it. - if (host.endsWith('youtube.com')) { + if ((host === 'youtube.com' || host.endsWith('.youtube.com'))) { const v = url.searchParams.get('v'); if (v) return `https://youtube.com/watch?v=${v}`; // Shorts / embeds carry the id in the path. @@ -64,7 +64,12 @@ function hash(text: string): string { * a page can hold more than one song. */ export function songKey(identity: Pick): string { - return hash(`${identity.normalizedUrl}\n${identity.title}`); + const url = identity.normalizedUrl; + if (url.startsWith('https://youtube.com/watch?v=')) return 'yt:' + url.slice('https://youtube.com/watch?v='.length); + // Local-player URLs include a browser-specific extension ID. The file name + // is the existing local-file discriminator; it must work across installations. + if (/^(chrome|moz)-extension:/.test(url)) return 'file:' + hash(cleanTitle(identity.title)); + return 'web:' + hash(url); } /** Whether two library rows describe the same song. */ diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 3863f4a..02f8270 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -1,847 +1,108 @@ -import { - DEFAULT_PARAMS, - DEFAULT_SETTINGS, - DEFAULT_UI_PREFS, -} from '../model/defaults.ts'; -import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; import { songKey } from '../model/track-identity.ts'; -import { rekeyByIdentity } from './rekey.ts'; -import type { - ChordChart, - ChordSegment, - EffectParams, - EqPreset, - FavoriteEntry, - HistoryEntry, - Marker, - Settings, - Snippet, - SnippetOverrides, - TrackData, - TrackIdentity, - UiPrefs, -} from '../model/types'; +import { emptyLibrary, type Library, type SharedLibrary } from './library.ts'; +import { parseBackupJson as parseLegacy } from './legacy-backup.ts'; +import { migrateBackup } from './library-migration.ts'; -/** - * The backup file format — the verbose in-memory `Backup` (v1) and its - * compact serialization (v2), which is what the export writes and what will - * ride the browser's sync storage. Nothing in `storage.local` changes: the - * codec shrinks the wire shape only, and `decodeBackup` hands back today's - * types. - * - * Pure and DOM-free so it runs under `node --test`; hence relative `.ts` - * imports and no `#imports` (see CLAUDE.md). - * - * Where the bytes go, and what v2 does about it: identities were repeated in - * Recent, Favorites and the track record (one `songs` table, referenced by - * index); every Recent row carried the full 13-field params object (a delta - * against the defaults, usually empty); chord charts spelled out four keys and - * 17-digit floats per segment (parallel arrays on a centisecond grid, labels - * through a per-chart table); settings/UI prefs carried every default (deep - * diff). Things derivable from what is kept are dropped: the identity key, - * YouTube thumbnails, the plain watch-page URL, marker/snippet ids. - * - * Every rounding is idempotent — `encode(decode(encode(x)))` deep-equals - * `encode(x)` — which is what will let two devices compare content hashes of - * data that both went through this. - */ - -/** Marks a file as ours, so a stray JSON can be rejected on sight. */ export const BACKUP_FORMAT = 'note-by-note-backup'; +export const BACKUP_VERSION = 4; +export interface Backup extends Library { format: typeof BACKUP_FORMAT; version: 4; exportedAt: number } -/** The verbose shape: what `createBackup` builds, and the only format any - * released build ever wrote — so it is the one an existing user's exported - * file is in, and import still accepts it. */ -export const BACKUP_VERSION = 1; - -/** The compact shape below, and what the export writes. `parseBackupJson` - * takes this or the verbose v1, and nothing else: version 2 never shipped — - * it existed only on the branch this format grew on — so no file and no synced - * blob is in it, and carrying a compatibility path for it would be carrying - * one for nobody. */ -export const COMPACT_VERSION = 3; - -/** Everything a user owns, in one file. Host permissions are deliberately out: - * they live in the browser's permission store, and only a prompt can grant - * them — a backup that listed origins would restore access it can't give. */ -export interface Backup { - format: typeof BACKUP_FORMAT; - version: number; - exportedAt: number; - appVersion: string; - settings: Settings; - uiPrefs: UiPrefs; - history: HistoryEntry[]; - favorites: FavoriteEntry[]; - eqPresets: EqPreset[]; - /** Per-track markers and snippets, one entry per saved track. */ - tracks: TrackData[]; -} - -// --------------------------------------------------------------------------- -// Compact shape - -/** Effect params as a delta against `DEFAULT_PARAMS`; absent = default. */ -export interface CompactParams { - /** transpose (semitones) */ - t?: number; - /** transposeEnabled off */ - te?: 0; - /** pitchCents */ - c?: number; - /** pitchEnabled off */ - ce?: 0; - /** speed */ - s?: number; - /** speedEnabled off */ - se?: 0; - /** vocalReduce */ - v?: number; - /** vocalReduceEnabled off */ - ve?: 0; - /** vocalMode 'isolate' */ - vm?: 1; - /** eq: [enabled, ...gains] — present when enabled or any gain is non-zero */ - e?: number[]; - /** tuning: [trackHz, instrumentHz] — present when not 440/440 */ - tu?: [number, number]; - /** power off */ - pw?: 0; - /** baseBpm */ - b?: number; +function object(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Damaged library data.'); + return value as Record; } - -/** `[normalizedUrl, title, durationSec]`. YouTube watch URLs are shortened to - * `yt:`. The key is never stored: it is `songKey` of the first two, which - * is why a device on either side of a key change reads the other's blob — - * each derives the key it uses from the same two strings. */ -export type CompactSong = [string, string, number]; - -export interface CompactEntry { - /** Index into `songs`. */ - i: number; - /** updatedAt, ms. */ - at: number; - p?: CompactParams; - /** pageUrl, when not the song's plain page. */ - url?: string; - /** thumbnailUrl, when not derivable from the page URL. */ - th?: string; - /** A tombstone (`deletions.ts`): the row was removed at `at`. Nothing else - * is carried — a deletion is a song and a date. */ - x?: 1; +function number(value: unknown) { + if (typeof value !== 'number' || !Number.isFinite(value)) throw new Error('Damaged library number.'); } - -export interface CompactFavorite extends CompactEntry { - /** favoritedAt, ms. */ - fa: number; - /** lastAccessedAt, ms. */ - la: number; - /** orderedAt, ms; omitted when the row predates manual order syncing. */ - oa?: number; +function string(value: unknown) { + if (typeof value !== 'string') throw new Error('Damaged library text.'); } - -/** `[name, gains, updatedAt?]`, or `[name, [], updatedAt, 1]` for a tombstone - * (`deletions.ts`). */ -export type CompactEqPreset = - | [string, number[]] - | [string, number[], number] - | [string, number[], number, 1]; - -/** `[t_ms, label?]` — label omitted when empty. */ -export type CompactMarker = [number] | [number, string]; - -/** `[name, start_ms, end_ms, repeats?, enabled?, overrides?]`, trailing - * defaults omitted (`1`, `1`, `{}`). `repeats` 0 stands for Infinity. */ -export type CompactSnippet = [string, number, number, number?, number?, CompactOverrides?]; - -export interface CompactOverrides { - s?: number; - t?: number; - v?: number; -} - -/** Segments as parallel arrays on a centisecond grid. */ -export interface CompactChart { - /** First segment start. */ - t0: number; - /** Durations. */ - d: number[]; - /** Gap before each segment (index 0 is always 0); omitted when all zero. */ - g?: number[]; - /** Label table, first-appearance order. */ - l: string[]; - /** Label index per segment. */ - i: number[]; - /** Key signature: [tonic, minor, confidence]; omitted when none. */ - k?: [string, 0 | 1, number]; - cov: number; - a0: number; - a1: number; - /** computedAt, ms. */ - c: number; -} - -export interface CompactTrack { - i: number; - /** updatedAt, ms. */ - at: number; - m?: CompactMarker[]; - s?: CompactSnippet[]; - /** sequenceLoop */ - L?: 1; - /** sequenceCountIn */ - C?: 1; - /** chordsEnabled, only when the record has the switch at all. */ - ce?: 0 | 1; - ch?: CompactChart; -} - -export interface CompactBackup { - format: typeof BACKUP_FORMAT; - version: typeof COMPACT_VERSION; - /** exportedAt, ms. */ - at: number; - /** Settings that differ from the defaults (`lastUsedParams` as `lp`). */ - s: Record; - /** UI prefs that differ from the defaults. */ - u: Record; - /** settings.updatedAt, ms; omitted when never dated. */ - sat?: number; - /** uiPrefs.updatedAt, ms; omitted when never dated. */ - uat?: number; - /** `[name, gains, updatedAt?]` per saved EQ preset. */ - eq: CompactEqPreset[]; - songs: CompactSong[]; - h: CompactEntry[]; - f: CompactFavorite[]; - t: CompactTrack[]; -} - -// --------------------------------------------------------------------------- -// Shared helpers - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function damaged(section: string): Error { - return new Error(`This backup's "${section}" list is damaged.`); -} - -const roundTo = (dp: number) => { - const f = 10 ** dp; - return (x: number) => Math.round(x * f) / f; -}; -const round2 = roundTo(2); -const round3 = roundTo(3); -const millis = (seconds: number) => Math.round(seconds * 1000); -const centis = (seconds: number) => Math.round(seconds * 100); -/** Timestamps stay in milliseconds: they decide merges (a deletion against a - * re-add, the newer of two edits), and rounding would let two actions within - * the same second read as one. */ -const stamp = (ms: number) => (Number.isFinite(ms) ? Math.round(ms) : 0); - -function num(value: unknown, section: string): number { - if (typeof value !== 'number' || !Number.isFinite(value)) throw damaged(section); - return value; -} - -function str(value: unknown, section: string): string { - if (typeof value !== 'string') throw damaged(section); - return value; -} - -function arr(value: unknown, section: string): unknown[] { - if (!Array.isArray(value)) throw damaged(section); +function array(value: unknown): any[] { + if (!Array.isArray(value)) throw new Error('Damaged library list.'); return value; } - -function rec(value: unknown, section: string): Record { - if (!isRecord(value)) throw damaged(section); - return value; -} - -/** Rows are identified by their URL and title (`songKey`); without those they - * can't be stored or matched back to a track, so a file carrying them is not - * usable. The key a file may also carry is not read — it is derived. */ -function identifiedArr(value: unknown, section: string): T[] { - const list = arr(value, section); - const identified = list.every( - (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.normalizedUrl === 'string', - ); - if (!identified) throw damaged(section); - return list as T[]; -} - -/** Keys of `value` whose (JSON) value differs from `defaults`, recursing into - * plain objects. Keys unknown to `defaults` are kept verbatim. */ -function diffPlain( - value: Record, - defaults: Record, -): Record { - const out: Record = {}; - for (const [key, v] of Object.entries(value)) { - if (v === undefined) continue; - const d = defaults[key]; - if (isRecord(v) && isRecord(d)) { - const nested = diffPlain(v, d); - if (Object.keys(nested).length) out[key] = nested; - } else if (!(key in defaults) || JSON.stringify(v) !== JSON.stringify(d)) { - out[key] = v; - } - } - return out; -} - -/** The inverse of `diffPlain`: a deep clone of `defaults` with `diff` laid - * over it. */ -function mergePlain( - defaults: Record, - diff: Record, -): Record { - const out: Record = JSON.parse(JSON.stringify(defaults)); - for (const [key, v] of Object.entries(diff)) { - if (v === undefined) continue; - const d = out[key]; - out[key] = isRecord(v) && isRecord(d) ? mergePlain(d, v) : v; - } - return out; -} - -// --------------------------------------------------------------------------- -// Effect params - -/** Old rows may predate a field (the switches, tuning, vocalMode, baseBpm); - * a missing one reads as its default, which is how the UI treats it too. */ -export function encodeParams(p: EffectParams): CompactParams | undefined { - const out: CompactParams = {}; - const transpose = round3(p.transpose ?? 0); - if (transpose !== 0) out.t = transpose; - if (p.transposeEnabled === false) out.te = 0; - const cents = round3(p.pitchCents ?? 0); - if (cents !== 0) out.c = cents; - if (p.pitchEnabled === false) out.ce = 0; - const speed = round3(p.speed ?? 1); - if (speed !== 1) out.s = speed; - if (p.speedEnabled === false) out.se = 0; - const vocal = round3(p.vocalReduce ?? 0); - if (vocal !== 0) out.v = vocal; - if (p.vocalReduceEnabled === false) out.ve = 0; - if (p.vocalMode === 'isolate') out.vm = 1; - const eq = p.eq ?? DEFAULT_PARAMS.eq; - const gains = (eq.gains ?? DEFAULT_PARAMS.eq.gains).map(round2); - if (eq.enabled || gains.some((g) => g !== 0)) out.e = [eq.enabled ? 1 : 0, ...gains]; - const tuning = p.tuning ?? DEFAULT_PARAMS.tuning; - const trackHz = round2(tuning.trackHz ?? 440); - const instrumentHz = round2(tuning.instrumentHz ?? 440); - if (trackHz !== 440 || instrumentHz !== 440) out.tu = [trackHz, instrumentHz]; - if (p.power === false) out.pw = 0; - if (typeof p.baseBpm === 'number' && Number.isFinite(p.baseBpm)) out.b = round2(p.baseBpm); - return Object.keys(out).length ? out : undefined; -} - -export function decodeParams(raw: unknown, section: string): EffectParams { - const p = structuredClone(DEFAULT_PARAMS); - if (raw === undefined) return p; - const c = rec(raw, section); - if (c.t !== undefined) p.transpose = num(c.t, section); - if (c.te !== undefined) p.transposeEnabled = false; - if (c.c !== undefined) p.pitchCents = num(c.c, section); - if (c.ce !== undefined) p.pitchEnabled = false; - if (c.s !== undefined) p.speed = num(c.s, section); - if (c.se !== undefined) p.speedEnabled = false; - if (c.v !== undefined) p.vocalReduce = num(c.v, section); - if (c.ve !== undefined) p.vocalReduceEnabled = false; - if (c.vm !== undefined) p.vocalMode = 'isolate'; - if (c.e !== undefined) { - const e = arr(c.e, section); - if (e.length !== 1 + p.eq.gains.length) throw damaged(section); - p.eq = { enabled: num(e[0], section) === 1, gains: e.slice(1).map((g) => num(g, section)) }; - } - if (c.tu !== undefined) { - const tu = arr(c.tu, section); - if (tu.length !== 2) throw damaged(section); - p.tuning = { trackHz: num(tu[0], section), instrumentHz: num(tu[1], section) }; - } - if (c.pw !== undefined) p.power = false; - if (c.b !== undefined) p.baseBpm = num(c.b, section); - return p; -} - -// --------------------------------------------------------------------------- -// Settings / UI prefs - -export function encodeSettings(settings: Settings): Record { - const { lastUsedParams, ...rest } = settings; - // Not a setting: the date rides beside the diff (`sat`), so a device that - // changed a setting and changed it back still encodes as empty. - delete (rest as { updatedAt?: number }).updatedAt; - const out = diffPlain(rest, { ...DEFAULT_SETTINGS }); - if (lastUsedParams) out.lp = encodeParams(lastUsedParams) ?? {}; - return out; -} - -export function decodeSettings(raw: unknown): Settings { - const diff = raw === undefined ? {} : rec(raw, 'settings'); - const { lp, ...rest } = diff; - const settings = mergePlain({ ...DEFAULT_SETTINGS }, rest) as unknown as Settings; - if (lp !== undefined) settings.lastUsedParams = decodeParams(lp, 'settings'); - return settings; -} - -export function encodeUiPrefs(uiPrefs: UiPrefs): Record { - const rest = { ...uiPrefs }; - delete rest.updatedAt; - return diffPlain( - rest as unknown as Record, - DEFAULT_UI_PREFS as unknown as Record, - ); -} - -export function decodeUiPrefs(raw: unknown): UiPrefs { - const diff = raw === undefined ? {} : rec(raw, 'uiPrefs'); - return mergePlain( - DEFAULT_UI_PREFS as unknown as Record, - diff, - ) as unknown as UiPrefs; -} - -// --------------------------------------------------------------------------- -// Songs (identities) - -const YT_WATCH = 'https://youtube.com/watch?v='; -const YT_ID_RE = /^[\w-]+$/; - -function shortUrl(normalizedUrl: string): string { - if (normalizedUrl.startsWith(YT_WATCH)) { - const id = normalizedUrl.slice(YT_WATCH.length); - if (YT_ID_RE.test(id)) return `yt:${id}`; - } - return normalizedUrl; -} - -function longUrl(short: string): string { - if (short.startsWith('yt:')) { - const id = short.slice(3); - if (!YT_ID_RE.test(id)) throw damaged('songs'); - return YT_WATCH + id; - } - return short; -} - -/** The page a song is opened at when the entry carries no `url` of its own: - * the engine records `location.href`, which on YouTube is the `www.` form of - * the watch page — so that, not the normalized URL, is the default there. */ -function defaultPageUrl(normalizedUrl: string): string { - if (normalizedUrl.startsWith(YT_WATCH)) { - return `https://www.youtube.com/watch?v=${normalizedUrl.slice(YT_WATCH.length)}`; - } - return normalizedUrl; -} - -/** One row per distinct (url, title, duration) — not per URL: the duration is - * part of the key, and a song whose duration drifted legitimately has two. */ -class SongTable { - rows: CompactSong[] = []; - #index = new Map(); - - add(identity: TrackIdentity): number { - const url = identity.normalizedUrl ?? ''; - const title = identity.title ?? ''; - const duration = Number.isFinite(identity.durationSec) ? identity.durationSec : 0; - const tableKey = `${url}\n${title}\n${duration}`; - const existing = this.#index.get(tableKey); - if (existing !== undefined) return existing; - const row: CompactSong = [shortUrl(url), title, duration]; - this.rows.push(row); - this.#index.set(tableKey, this.rows.length - 1); - return this.rows.length - 1; - } -} - -function decodeSongs(raw: unknown): TrackIdentity[] { - return arr(raw, 'songs').map((row) => { - const r = arr(row, 'songs'); - if (r.length !== 3) throw damaged('songs'); - const normalizedUrl = longUrl(str(r[0], 'songs')); - const title = str(r[1], 'songs'); - const durationSec = num(r[2], 'songs'); - return { key: songKey({ normalizedUrl, title }), normalizedUrl, title, durationSec }; - }); -} - -function songAt(songs: TrackIdentity[], index: unknown, section: string): TrackIdentity { - if (typeof index !== 'number' || !Number.isInteger(index) || index < 0 || index >= songs.length) { - throw damaged(section); - } - return songs[index]; -} - -// --------------------------------------------------------------------------- -// Recent / Favorites - -function encodeEntry(entry: HistoryEntry, songs: SongTable): CompactEntry { - const out: CompactEntry = { i: songs.add(entry.identity), at: stamp(entry.updatedAt) }; - // A tombstone is the song and the date it went; the rest was only ever there - // to be shown, and nothing shows a removed row. - if (entry.deleted) return { ...out, x: 1 }; - const params = encodeParams(entry.params ?? DEFAULT_PARAMS); - if (params) out.p = params; - const pageUrl = entry.pageUrl ?? ''; - if (pageUrl !== defaultPageUrl(entry.identity.normalizedUrl)) out.url = pageUrl; - if (entry.thumbnailUrl && entry.thumbnailUrl !== youtubeThumbnailUrl(pageUrl)) { - out.th = entry.thumbnailUrl; - } - return out; -} - -function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): HistoryEntry { - const c = rec(raw, section); - const identity = songAt(songs, c.i, section); - const updatedAt = num(c.at, section); - const pageUrl = c.url === undefined ? defaultPageUrl(identity.normalizedUrl) : str(c.url, section); - const thumbnailUrl = c.th === undefined ? youtubeThumbnailUrl(pageUrl) : str(c.th, section); - const entry: HistoryEntry = { - identity: { ...identity }, - params: decodeParams(c.p, section), - pageUrl, - createdAt: updatedAt, - updatedAt, - }; - if (thumbnailUrl !== undefined) entry.thumbnailUrl = thumbnailUrl; - if (c.x === 1) entry.deleted = true; - return entry; -} - -function encodeFavorite(entry: FavoriteEntry, songs: SongTable): CompactFavorite { - const out: CompactFavorite = { - ...encodeEntry(entry, songs), - fa: stamp(entry.favoritedAt), - la: stamp(entry.lastAccessedAt), - }; - if (entry.orderedAt) out.oa = stamp(entry.orderedAt); - return out; -} - -function decodeFavorite(raw: unknown, songs: TrackIdentity[]): FavoriteEntry { - const c = rec(raw, 'favorites'); - const favorite: FavoriteEntry = { - ...decodeEntry(c, songs, 'favorites'), - favoritedAt: num(c.fa, 'favorites'), - lastAccessedAt: num(c.la, 'favorites'), - }; - if (c.oa !== undefined) favorite.orderedAt = num(c.oa, 'favorites'); - return favorite; -} - -// --------------------------------------------------------------------------- -// Tracks - -function encodeMarker(marker: Marker): CompactMarker { - const t = millis(marker.t); - return marker.label ? [t, marker.label] : [t]; -} - -function decodeMarker(raw: unknown, index: number): Marker { - const r = arr(raw, 'tracks'); - if (r.length < 1 || r.length > 2) throw damaged('tracks'); - return { - id: `m${index + 1}`, - t: num(r[0], 'tracks') / 1000, - label: r.length === 2 ? str(r[1], 'tracks') : '', - }; -} - -function encodeOverrides(overrides: SnippetOverrides | undefined): CompactOverrides { - const out: CompactOverrides = {}; - if (!overrides) return out; - if (typeof overrides.speed === 'number') out.s = round3(overrides.speed); - if (typeof overrides.transpose === 'number') out.t = round3(overrides.transpose); - if (typeof overrides.vocalReduce === 'number') out.v = round3(overrides.vocalReduce); - return out; -} - -function decodeOverrides(raw: unknown): SnippetOverrides { - const c = rec(raw, 'tracks'); - const out: SnippetOverrides = {}; - if (c.s !== undefined) out.speed = num(c.s, 'tracks'); - if (c.t !== undefined) out.transpose = num(c.t, 'tracks'); - if (c.v !== undefined) out.vocalReduce = num(c.v, 'tracks'); - return out; -} - -function encodeSnippet(snippet: Snippet): CompactSnippet { - // `repeats: Infinity` is `null` once it has been through JSON (storage); - // both mean "forever", written as 0 — real counts start at 1. - const repeats = - typeof snippet.repeats === 'number' && Number.isFinite(snippet.repeats) ? snippet.repeats : 0; - const overrides = encodeOverrides(snippet.overrides); - const out: CompactSnippet = [ - snippet.name ?? '', - millis(snippet.startT), - millis(snippet.endT), - repeats, - snippet.enabled === false ? 0 : 1, - overrides, - ]; - // Trailing defaults are left out: `{}` overrides, enabled, one repeat. - const isDefault = (v: unknown) => v === 1 || (isRecord(v) && Object.keys(v).length === 0); - while (out.length > 3 && isDefault(out[out.length - 1])) out.pop(); - return out; -} - -function decodeSnippet(raw: unknown, index: number): Snippet { - const r = arr(raw, 'tracks'); - if (r.length < 3 || r.length > 6) throw damaged('tracks'); - const repeats = r.length > 3 ? num(r[3], 'tracks') : 1; +function versioned(value: unknown) { + const item = object(value); + number(item.at); + if (item.at < 0 || !('value' in item)) throw new Error('Damaged library revision.'); + return item; +} +/** Validate/backfill a JSON object against its version's defaults. */ +function defaults(value: unknown, fallback: T): T { + const source = object(value); + const result = structuredClone(fallback) as Record; + for (const [key, expected] of Object.entries(result)) { + if (!(key in source)) continue; + const next = source[key]; + if (expected === null) { if (next !== null) number(next); } + else if (Array.isArray(expected)) array(next).forEach(number); + else if (typeof expected === 'object') { result[key] = defaults(next, expected); continue; } + else if (typeof next !== typeof expected) throw new Error('Damaged library setting.'); + if (typeof next === 'number') number(next); + result[key] = next; + } + return result as T; +} +export function parseShared(value: unknown): SharedLibrary { + const shared = structuredClone(object(value)); + shared.settings = versioned(shared.settings); + shared.settings.value = defaults(shared.settings.value, DEFAULT_SETTINGS); + shared.favoriteOrder = versioned(shared.favoriteOrder); + array(shared.favoriteOrder.value).forEach(string); + for (const [key, raw] of Object.entries(object(shared.songs))) { + const song = object(raw); + song.practice = versioned(song.practice); + song.favorite = versioned(song.favorite); + if (typeof song.favorite.value !== 'boolean') throw new Error('Damaged favorite.'); + const practice = song.practice.value; + if (practice === null) continue; + object(practice); + const identity = object(practice.identity); + string(identity.normalizedUrl); string(identity.title); number(identity.durationSec); + if (key !== songKey(identity as any)) throw new Error('Song identity does not match its key.'); + identity.key = key; + string(practice.pageUrl); + if (practice.thumbnailUrl !== undefined) string(practice.thumbnailUrl); + if (practice.params !== undefined) practice.params = defaults(practice.params, DEFAULT_PARAMS); + array(practice.markers).forEach((m) => { object(m); string(m.id); string(m.label); number(m.t); }); + array(practice.snippets).forEach((s) => { + object(s); string(s.id); string(s.name); number(s.startT); number(s.endT); + if (s.repeats !== null && s.repeats !== Infinity) number(s.repeats); + if (typeof s.enabled !== 'boolean') throw new Error('Damaged snippet.'); + for (const amount of Object.values(object(s.overrides))) number(amount); + }); + if (typeof practice.sequenceLoop !== 'boolean' || typeof practice.sequenceCountIn !== 'boolean') throw new Error('Damaged sequence.'); + if (practice.chordsEnabled !== undefined && typeof practice.chordsEnabled !== 'boolean') throw new Error('Damaged chord setting.'); + } + for (const raw of Object.values(object(shared.presets))) { + const preset = versioned(raw); + if (preset.value !== null) array(preset.value).forEach(number); + } + return { settings: shared.settings, songs: shared.songs, presets: shared.presets, favoriteOrder: shared.favoriteOrder }; +} +export function parseLibrary(value: unknown): Library { + const source = object(value); + const local = object(source.local); + const charts = object(local.charts); + for (const chart of Object.values(charts)) { + if (chart === null) continue; + object(chart); number(chart.computedAt); number(chart.coverage); number(chart.analyzedFrom); number(chart.analyzedTo); + array(chart.segments).forEach((s) => { object(s); number(s.startT); number(s.endT); string(s.label); number(s.confidence); }); + if (chart.key !== null) { object(chart.key); string(chart.key.tonic); string(chart.key.mode); number(chart.key.confidence); } + } + const recent = object(local.recent); + for (const row of Object.values(recent)) { object(row); number(row.updatedAt); number(row.lastAccessedAt); } return { - id: `c${index + 1}`, - name: str(r[0], 'tracks'), - startT: num(r[1], 'tracks') / 1000, - endT: num(r[2], 'tracks') / 1000, - enabled: r.length > 4 ? num(r[4], 'tracks') === 1 : true, - repeats: repeats === 0 ? Infinity : repeats, - overrides: r.length > 5 ? decodeOverrides(r[5]) : {}, + shared: parseShared(source.shared), + local: { uiPrefs: defaults(local.uiPrefs, DEFAULT_UI_PREFS), recent, charts, + ...(local.lastUsedParams ? { lastUsedParams: defaults(local.lastUsedParams, DEFAULT_PARAMS) } : {}) }, }; } - -export function encodeChart(chart: ChordChart): CompactChart { - const segments = [...chart.segments].sort((a, b) => a.startT - b.startT); - const d: number[] = []; - const g: number[] = []; - const l: string[] = []; - const i: number[] = []; - const labelIndex = new Map(); - let t0 = 0; - let prevEnd = 0; - let anyGap = false; - segments.forEach((seg, n) => { - const start = centis(seg.startT); - const end = Math.max(start, centis(seg.endT)); - if (n === 0) { - t0 = start; - g.push(0); - } else { - const gap = start - prevEnd; - if (gap !== 0) anyGap = true; - g.push(gap); - } - d.push(end - start); - let li = labelIndex.get(seg.label); - if (li === undefined) { - li = l.length; - l.push(seg.label); - labelIndex.set(seg.label, li); - } - i.push(li); - prevEnd = end; - }); - const out: CompactChart = { - t0, - d, - l, - i, - cov: round3(chart.coverage ?? 0), - a0: centis(chart.analyzedFrom ?? 0), - a1: centis(chart.analyzedTo ?? 0), - c: stamp(chart.computedAt), - }; - if (anyGap) out.g = g; - if (chart.key) { - out.k = [chart.key.tonic, chart.key.mode === 'minor' ? 1 : 0, round3(chart.key.confidence)]; - } - return out; -} - -export function decodeChart(raw: unknown): ChordChart { - const c = rec(raw, 'tracks'); - const d = arr(c.d, 'tracks'); - const l = arr(c.l, 'tracks').map((label) => str(label, 'tracks')); - const i = arr(c.i, 'tracks'); - const g = c.g === undefined ? undefined : arr(c.g, 'tracks'); - if (i.length !== d.length || (g !== undefined && g.length !== d.length)) throw damaged('tracks'); - const segments: ChordSegment[] = []; - let acc = num(c.t0, 'tracks'); - for (let n = 0; n < d.length; n++) { - const li = num(i[n], 'tracks'); - if (!Number.isInteger(li) || li < 0 || li >= l.length) throw damaged('tracks'); - const start = acc + (g === undefined ? 0 : num(g[n], 'tracks')); - const end = start + num(d[n], 'tracks'); - segments.push({ startT: start / 100, endT: end / 100, label: l[li], confidence: 1 }); - acc = end; - } - let key: ChordChart['key'] = null; - if (c.k !== undefined) { - const k = arr(c.k, 'tracks'); - if (k.length !== 3) throw damaged('tracks'); - key = { - tonic: str(k[0], 'tracks'), - mode: num(k[1], 'tracks') === 1 ? 'minor' : 'major', - confidence: num(k[2], 'tracks'), - }; - } - return { - segments, - key, - coverage: num(c.cov, 'tracks'), - analyzedFrom: num(c.a0, 'tracks') / 100, - analyzedTo: num(c.a1, 'tracks') / 100, - computedAt: num(c.c, 'tracks'), - }; -} - -function encodeTrack(track: TrackData, songs: SongTable): CompactTrack { - const out: CompactTrack = { i: songs.add(track.identity), at: stamp(track.updatedAt) }; - if (track.markers?.length) out.m = track.markers.map(encodeMarker); - if (track.snippets?.length) out.s = track.snippets.map(encodeSnippet); - if (track.sequenceLoop) out.L = 1; - if (track.sequenceCountIn) out.C = 1; - if (track.chordsEnabled !== undefined) out.ce = track.chordsEnabled ? 1 : 0; - if (track.chordChart) out.ch = encodeChart(track.chordChart); - return out; -} - -function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { - const c = rec(raw, 'tracks'); - const track: TrackData = { - identity: { ...songAt(songs, c.i, 'tracks') }, - markers: c.m === undefined ? [] : arr(c.m, 'tracks').map(decodeMarker), - snippets: c.s === undefined ? [] : arr(c.s, 'tracks').map(decodeSnippet), - sequenceLoop: c.L !== undefined, - sequenceCountIn: c.C !== undefined, - chordChart: c.ch === undefined ? null : decodeChart(c.ch), - updatedAt: num(c.at, 'tracks'), - }; - if (c.ce !== undefined) track.chordsEnabled = num(c.ce, 'tracks') === 1; - return track; -} - -// --------------------------------------------------------------------------- -// Whole backup - -/** `[name, gains, updatedAt?]`, or `[name, [], updatedAt, 1]` tombstoned. */ -function encodeEqPreset(preset: EqPreset): CompactEqPreset { - const name = preset.name ?? ''; - if (preset.deleted) return [name, [], stamp(preset.updatedAt ?? 0), 1]; - const gains = (preset.gains ?? []).map(round2); - return preset.updatedAt ? [name, gains, stamp(preset.updatedAt)] : [name, gains]; -} - -function decodeEqPreset(raw: unknown): EqPreset { - const r = arr(raw, 'eqPresets'); - if (r.length < 2 || r.length > 4) throw damaged('eqPresets'); - const preset: EqPreset = { - name: str(r[0], 'eqPresets'), - gains: arr(r[1], 'eqPresets').map((g) => num(g, 'eqPresets')), - }; - if (r.length >= 3) preset.updatedAt = num(r[2], 'eqPresets'); - if (r.length === 4) preset.deleted = true; - return preset; -} - -const byKey = (a: { identity: TrackIdentity }, b: { identity: TrackIdentity }) => - a.identity.key < b.identity.key ? -1 : a.identity.key > b.identity.key ? 1 : 0; - -/** Deterministic for equal input: tracks are sorted by key (their storage - * enumeration order is arbitrary) and the song table is filled in Recent, - * Favorites, tracks order. */ -export function encodeBackup(backup: Backup): CompactBackup { - const songs = new SongTable(); - const h = backup.history.map((e) => encodeEntry(e, songs)); - const f = backup.favorites.map((e) => encodeFavorite(e, songs)); - const t = [...backup.tracks].sort(byKey).map((track) => encodeTrack(track, songs)); - const out: CompactBackup = { - format: BACKUP_FORMAT, - version: COMPACT_VERSION, - at: stamp(backup.exportedAt), - s: encodeSettings(backup.settings), - u: encodeUiPrefs(backup.uiPrefs), - eq: backup.eqPresets.map(encodeEqPreset), - songs: songs.rows, - h, - f, - t, - }; - if (backup.settings.updatedAt) out.sat = stamp(backup.settings.updatedAt); - if (backup.uiPrefs.updatedAt) out.uat = stamp(backup.uiPrefs.updatedAt); - return out; -} - -/** Defaults alone need no sync blob; customized settings do, even without - * songs. A tombstone counts as a song — it is the only record that the row - * was removed, and seeding without it would resurrect the row elsewhere. */ -export function isEmptyBackup(backup: Backup): boolean { - const { songs, eq, s, u } = encodeBackup(backup); - return songs.length === 0 && eq.length === 0 && - Object.keys(s).length === 0 && Object.keys(u).length === 0; -} - -/** Reads a compact (v2) backup, or throws an `Error` whose message is safe to - * show the user. Unknown keys are ignored so the format can grow. */ -export function decodeBackup(raw: unknown): Backup { - if (!isRecord(raw) || raw.format !== BACKUP_FORMAT || raw.version !== COMPACT_VERSION) { - throw new Error("That file isn't a Note by Note backup."); - } - const songs = decodeSongs(raw.songs); - const settings = decodeSettings(raw.s); - const uiPrefs = decodeUiPrefs(raw.u); - if (typeof raw.sat === 'number' && Number.isFinite(raw.sat)) settings.updatedAt = raw.sat; - if (typeof raw.uat === 'number' && Number.isFinite(raw.uat)) uiPrefs.updatedAt = raw.uat; - return { - format: BACKUP_FORMAT, - version: COMPACT_VERSION, - exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at : 0, - appVersion: '', - settings, - uiPrefs, - history: arr(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), - favorites: arr(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), - eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), - tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), - }; -} - -/** The verbose v1 file: the in-memory shape, written out as is. Objects are - * backfilled from the defaults so a file from an older build gains any setting - * added since, and every row's key is re-derived — v1 files were written when - * a key baked in the duration, and rows that split across two of those are one - * song again here (`rekey.ts`, `track-identity.ts`). */ -function normalizeV1(raw: Record): Backup { - return { - format: BACKUP_FORMAT, - version: BACKUP_VERSION, - exportedAt: typeof raw.exportedAt === 'number' ? raw.exportedAt : 0, - appVersion: typeof raw.appVersion === 'string' ? raw.appVersion : '', - settings: { - ...DEFAULT_SETTINGS, - ...(isRecord(raw.settings) ? raw.settings : {}), - } as Settings, - uiPrefs: { - ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), - ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), - }, - history: rekeyByIdentity(identifiedArr(raw.history, 'history')), - favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), - eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], - tracks: rekeyByIdentity(identifiedArr(raw.tracks, 'tracks')), - }; -} - -/** - * Reads a parsed backup file (any version this build knows) into a `Backup`, - * or throws an `Error` whose message is safe to show the user. - */ -export function parseBackupJson(raw: unknown): Backup { - if (!isRecord(raw) || raw.format !== BACKUP_FORMAT) { - throw new Error("That file isn't a Note by Note backup."); - } - const version = typeof raw.version === 'number' ? raw.version : 0; - if (version > COMPACT_VERSION) { - throw new Error('That backup was made by a newer version of Note by Note.'); - } - if (version === COMPACT_VERSION) return decodeBackup(raw); - if (version === BACKUP_VERSION) return normalizeV1(raw); - // Only 2 lands here, and only from the branch this format grew on. - throw new Error('That backup is in a format this version of Note by Note no longer reads.'); +export function parseBackupJson(value: unknown): Backup { + const raw = object(value); + if (raw.format !== BACKUP_FORMAT) throw new Error("That file isn't a Note by Note backup."); + if (raw.version > BACKUP_VERSION) throw new Error('That backup was made by a newer version of Note by Note.'); + const library = raw.version === BACKUP_VERSION ? parseLibrary(raw) : migrateBackup(parseLegacy(raw)); + return { format: BACKUP_FORMAT, version: BACKUP_VERSION, exportedAt: raw.exportedAt ?? raw.at ?? 0, ...library }; } diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index dd26eab..a19f751 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -1,108 +1,18 @@ -import type { TrackData } from '../model/types'; -import { - BACKUP_FORMAT, - BACKUP_VERSION, - parseBackupJson, - type Backup, -} from './backup-codec'; -import { pruneTombstones, replaceAll } from './deletions'; -import { - eqPresetsItem, - favoritesItem, - historyItem, - removeTrackDataExcept, - saveTrackData, - settingsItem, - uiPrefsItem, -} from './storage'; - -/** The file shape and its compact codec live in `backup-codec.ts` (pure, so - * they run under `node --test`); this module is the storage side. */ +import { BACKUP_FORMAT, BACKUP_VERSION, parseBackupJson, type Backup } from './backup-codec'; +import { editLibrary, readLibrary } from './library-client'; export type { Backup }; -/** Raw storage keys have no `local:` prefix — see `trackDataKey`. */ -async function loadAllTrackData(): Promise { - const snapshot = await browser.storage.local.get(null); - return Object.entries(snapshot) - .filter(([key]) => key.startsWith('track:')) - .map(([, value]) => value as TrackData); -} - export async function createBackup(): Promise { - const [settings, uiPrefs, history, favorites, eqPresets, tracks] = - await Promise.all([ - settingsItem.getValue(), - uiPrefsItem.getValue(), - historyItem.getValue(), - favoritesItem.getValue(), - eqPresetsItem.getValue(), - loadAllTrackData(), - ]); - return { - format: BACKUP_FORMAT, - version: BACKUP_VERSION, - exportedAt: Date.now(), - appVersion: browser.runtime.getManifest().version, - settings, - uiPrefs, - history, - favorites, - eqPresets, - tracks, - }; + return { format: BACKUP_FORMAT, version: BACKUP_VERSION, exportedAt: Date.now(), ...await readLibrary() }; } - -/** Suggested download name, e.g. `note-by-note-backup-2026-07-17.json`. */ -export function backupFilename(exportedAt: number): string { - const day = new Date(exportedAt).toISOString().slice(0, 10); - return `note-by-note-backup-${day}.json`; +export function backupFilename(at: number): string { + return 'note-by-note-backup-' + new Date(at).toISOString().slice(0, 10) + '.json'; } - -/** - * Reads a backup file's text into a `Backup`, or throws an `Error` whose - * message is safe to show the user. Accepts the compact format the export - * writes and the verbose one older builds wrote; either way objects are - * backfilled from the defaults so a file from an older build gains any setting - * added since. - */ export function parseBackup(text: string): Backup { let raw: unknown; - try { - raw = JSON.parse(text); - } catch { - throw new Error("That file isn't valid JSON."); - } + try { raw = JSON.parse(text); } catch { throw new Error("That file isn't valid JSON."); } return parseBackupJson(raw); } - -/** - * Replaces every stored value with the backup's, dropping data the file does - * not carry — a restore reproduces the machine it came from rather than - * merging into whatever is here. Host permissions are left untouched. - * - * A manual file import passes `asNew`, which turns the file into "this is the - * library now": its rows are re-dated and everything this device held that the - * file leaves out becomes a tombstone, so a sync merge can't union it straight - * back (`replaceAll`, deletions.ts). A sync restore is already a merged result - * and keeps its dates. Expired tombstones are dropped on the way past. - * - * Track records are written first and the leftovers removed afterwards, never - * the other way round. A sync merge calls this on every remote change, and - * the panel document can go away mid-restore (the user closes it, the tab - * changes) — wiping first would make that window cost every marker, snippet - * and chart in the library. This way the worst case is a stale record the - * next restore removes. - */ -export async function restoreBackup(backup: Backup, { asNew = false } = {}): Promise { - const now = Date.now(); - const next = asNew ? replaceAll(backup, await createBackup(), now) : backup; - await Promise.all([ - settingsItem.setValue(next.settings), - uiPrefsItem.setValue(next.uiPrefs), - historyItem.setValue(pruneTombstones(next.history, now)), - favoritesItem.setValue(pruneTombstones(next.favorites, now)), - eqPresetsItem.setValue(pruneTombstones(next.eqPresets, now)), - ...next.tracks.map(saveTrackData), - ]); - await removeTrackDataExcept(new Set(next.tracks.map((t) => t.identity.key))); +export function restoreBackup(backup: Backup): Promise { + return editLibrary({ type: 'import', library: backup }); } diff --git a/src/core/persist/legacy-backup.ts b/src/core/persist/legacy-backup.ts new file mode 100644 index 0000000..c651b63 --- /dev/null +++ b/src/core/persist/legacy-backup.ts @@ -0,0 +1,365 @@ +// Read-only adapter for previously supported backup formats. New writes use the library schema. +import { + DEFAULT_PARAMS, + DEFAULT_SETTINGS, + DEFAULT_UI_PREFS, +} from '../model/defaults.ts'; + +import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; + +import { songKey } from '../model/track-identity.ts'; + +import { rekeyByIdentity } from './rekey.ts'; + +import type { + ChordChart, + ChordSegment, + EffectParams, + EqPreset, + FavoriteEntry, + HistoryEntry, + Marker, + Settings, + Snippet, + SnippetOverrides, + TrackData, + TrackIdentity, + UiPrefs, +} from '../model/types'; + +export const BACKUP_FORMAT = 'note-by-note-backup'; + +export const BACKUP_VERSION = 1; + +export const COMPACT_VERSION = 3; + +export interface Backup { + format: typeof BACKUP_FORMAT; + version: number; + exportedAt: number; + appVersion: string; + settings: Settings; + uiPrefs: UiPrefs; + history: HistoryEntry[]; + favorites: FavoriteEntry[]; + eqPresets: EqPreset[]; + /** Per-track markers and snippets, one entry per saved track. */ + tracks: TrackData[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function damaged(section: string): Error { + return new Error(`This backup's "${section}" list is damaged.`); +} + +function num(value: unknown, section: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) throw damaged(section); + return value; +} + +function str(value: unknown, section: string): string { + if (typeof value !== 'string') throw damaged(section); + return value; +} + +function arr(value: unknown, section: string): unknown[] { + if (!Array.isArray(value)) throw damaged(section); + return value; +} + +function rec(value: unknown, section: string): Record { + if (!isRecord(value)) throw damaged(section); + return value; +} + +function identifiedArr(value: unknown, section: string): T[] { + const list = arr(value, section); + const identified = list.every( + (e) => isRecord(e) && isRecord(e.identity) && typeof e.identity.normalizedUrl === 'string', + ); + if (!identified) throw damaged(section); + return list as T[]; +} + +function mergePlain( + defaults: Record, + diff: Record, +): Record { + const out: Record = JSON.parse(JSON.stringify(defaults)); + for (const [key, v] of Object.entries(diff)) { + if (v === undefined) continue; + const d = out[key]; + out[key] = isRecord(v) && isRecord(d) ? mergePlain(d, v) : v; + } + return out; +} + +export function decodeParams(raw: unknown, section: string): EffectParams { + const p = structuredClone(DEFAULT_PARAMS); + if (raw === undefined) return p; + const c = rec(raw, section); + if (c.t !== undefined) p.transpose = num(c.t, section); + if (c.te !== undefined) p.transposeEnabled = false; + if (c.c !== undefined) p.pitchCents = num(c.c, section); + if (c.ce !== undefined) p.pitchEnabled = false; + if (c.s !== undefined) p.speed = num(c.s, section); + if (c.se !== undefined) p.speedEnabled = false; + if (c.v !== undefined) p.vocalReduce = num(c.v, section); + if (c.ve !== undefined) p.vocalReduceEnabled = false; + if (c.vm !== undefined) p.vocalMode = 'isolate'; + if (c.e !== undefined) { + const e = arr(c.e, section); + if (e.length !== 1 + p.eq.gains.length) throw damaged(section); + p.eq = { enabled: num(e[0], section) === 1, gains: e.slice(1).map((g) => num(g, section)) }; + } + if (c.tu !== undefined) { + const tu = arr(c.tu, section); + if (tu.length !== 2) throw damaged(section); + p.tuning = { trackHz: num(tu[0], section), instrumentHz: num(tu[1], section) }; + } + if (c.pw !== undefined) p.power = false; + if (c.b !== undefined) p.baseBpm = num(c.b, section); + return p; +} + +export function decodeSettings(raw: unknown): Settings { + const diff = raw === undefined ? {} : rec(raw, 'settings'); + const { lp, ...rest } = diff; + const settings = mergePlain({ ...DEFAULT_SETTINGS }, rest) as unknown as Settings; + if (lp !== undefined) settings.lastUsedParams = decodeParams(lp, 'settings'); + return settings; +} + +export function decodeUiPrefs(raw: unknown): UiPrefs { + const diff = raw === undefined ? {} : rec(raw, 'uiPrefs'); + return mergePlain( + DEFAULT_UI_PREFS as unknown as Record, + diff, + ) as unknown as UiPrefs; +} + +const YT_WATCH = 'https://youtube.com/watch?v='; + +const YT_ID_RE = /^[\w-]+$/; + +function longUrl(short: string): string { + if (short.startsWith('yt:')) { + const id = short.slice(3); + if (!YT_ID_RE.test(id)) throw damaged('songs'); + return YT_WATCH + id; + } + return short; +} + +function defaultPageUrl(normalizedUrl: string): string { + if (normalizedUrl.startsWith(YT_WATCH)) { + return `https://www.youtube.com/watch?v=${normalizedUrl.slice(YT_WATCH.length)}`; + } + return normalizedUrl; +} + +function decodeSongs(raw: unknown): TrackIdentity[] { + return arr(raw, 'songs').map((row) => { + const r = arr(row, 'songs'); + if (r.length !== 3) throw damaged('songs'); + const normalizedUrl = longUrl(str(r[0], 'songs')); + const title = str(r[1], 'songs'); + const durationSec = num(r[2], 'songs'); + return { key: songKey({ normalizedUrl, title }), normalizedUrl, title, durationSec }; + }); +} + +function songAt(songs: TrackIdentity[], index: unknown, section: string): TrackIdentity { + if (typeof index !== 'number' || !Number.isInteger(index) || index < 0 || index >= songs.length) { + throw damaged(section); + } + return songs[index]; +} + +function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): HistoryEntry { + const c = rec(raw, section); + const identity = songAt(songs, c.i, section); + const updatedAt = num(c.at, section); + const pageUrl = c.url === undefined ? defaultPageUrl(identity.normalizedUrl) : str(c.url, section); + const thumbnailUrl = c.th === undefined ? youtubeThumbnailUrl(pageUrl) : str(c.th, section); + const entry: HistoryEntry = { + identity: { ...identity }, + params: decodeParams(c.p, section), + pageUrl, + createdAt: updatedAt, + updatedAt, + }; + if (thumbnailUrl !== undefined) entry.thumbnailUrl = thumbnailUrl; + if (c.x === 1) entry.deleted = true; + return entry; +} + +function decodeFavorite(raw: unknown, songs: TrackIdentity[]): FavoriteEntry { + const c = rec(raw, 'favorites'); + const favorite: FavoriteEntry = { + ...decodeEntry(c, songs, 'favorites'), + favoritedAt: num(c.fa, 'favorites'), + lastAccessedAt: num(c.la, 'favorites'), + }; + if (c.oa !== undefined) favorite.orderedAt = num(c.oa, 'favorites'); + return favorite; +} + +function decodeMarker(raw: unknown, index: number): Marker { + const r = arr(raw, 'tracks'); + if (r.length < 1 || r.length > 2) throw damaged('tracks'); + return { + id: `m${index + 1}`, + t: num(r[0], 'tracks') / 1000, + label: r.length === 2 ? str(r[1], 'tracks') : '', + }; +} + +function decodeOverrides(raw: unknown): SnippetOverrides { + const c = rec(raw, 'tracks'); + const out: SnippetOverrides = {}; + if (c.s !== undefined) out.speed = num(c.s, 'tracks'); + if (c.t !== undefined) out.transpose = num(c.t, 'tracks'); + if (c.v !== undefined) out.vocalReduce = num(c.v, 'tracks'); + return out; +} + +function decodeSnippet(raw: unknown, index: number): Snippet { + const r = arr(raw, 'tracks'); + if (r.length < 3 || r.length > 6) throw damaged('tracks'); + const repeats = r.length > 3 ? num(r[3], 'tracks') : 1; + return { + id: `c${index + 1}`, + name: str(r[0], 'tracks'), + startT: num(r[1], 'tracks') / 1000, + endT: num(r[2], 'tracks') / 1000, + enabled: r.length > 4 ? num(r[4], 'tracks') === 1 : true, + repeats: repeats === 0 ? Infinity : repeats, + overrides: r.length > 5 ? decodeOverrides(r[5]) : {}, + }; +} + +export function decodeChart(raw: unknown): ChordChart { + const c = rec(raw, 'tracks'); + const d = arr(c.d, 'tracks'); + const l = arr(c.l, 'tracks').map((label) => str(label, 'tracks')); + const i = arr(c.i, 'tracks'); + const g = c.g === undefined ? undefined : arr(c.g, 'tracks'); + if (i.length !== d.length || (g !== undefined && g.length !== d.length)) throw damaged('tracks'); + const segments: ChordSegment[] = []; + let acc = num(c.t0, 'tracks'); + for (let n = 0; n < d.length; n++) { + const li = num(i[n], 'tracks'); + if (!Number.isInteger(li) || li < 0 || li >= l.length) throw damaged('tracks'); + const start = acc + (g === undefined ? 0 : num(g[n], 'tracks')); + const end = start + num(d[n], 'tracks'); + segments.push({ startT: start / 100, endT: end / 100, label: l[li], confidence: 1 }); + acc = end; + } + let key: ChordChart['key'] = null; + if (c.k !== undefined) { + const k = arr(c.k, 'tracks'); + if (k.length !== 3) throw damaged('tracks'); + key = { + tonic: str(k[0], 'tracks'), + mode: num(k[1], 'tracks') === 1 ? 'minor' : 'major', + confidence: num(k[2], 'tracks'), + }; + } + return { + segments, + key, + coverage: num(c.cov, 'tracks'), + analyzedFrom: num(c.a0, 'tracks') / 100, + analyzedTo: num(c.a1, 'tracks') / 100, + computedAt: num(c.c, 'tracks'), + }; +} + +function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { + const c = rec(raw, 'tracks'); + const track: TrackData = { + identity: { ...songAt(songs, c.i, 'tracks') }, + markers: c.m === undefined ? [] : arr(c.m, 'tracks').map(decodeMarker), + snippets: c.s === undefined ? [] : arr(c.s, 'tracks').map(decodeSnippet), + sequenceLoop: c.L !== undefined, + sequenceCountIn: c.C !== undefined, + chordChart: c.ch === undefined ? null : decodeChart(c.ch), + updatedAt: num(c.at, 'tracks'), + }; + if (c.ce !== undefined) track.chordsEnabled = num(c.ce, 'tracks') === 1; + return track; +} + +function decodeEqPreset(raw: unknown): EqPreset { + const r = arr(raw, 'eqPresets'); + if (r.length < 2 || r.length > 4) throw damaged('eqPresets'); + const preset: EqPreset = { + name: str(r[0], 'eqPresets'), + gains: arr(r[1], 'eqPresets').map((g) => num(g, 'eqPresets')), + }; + if (r.length >= 3) preset.updatedAt = num(r[2], 'eqPresets'); + if (r.length === 4) preset.deleted = true; + return preset; +} + +export function decodeBackup(raw: unknown): Backup { + if (!isRecord(raw) || raw.format !== BACKUP_FORMAT || raw.version !== COMPACT_VERSION) { + throw new Error("That file isn't a Note by Note backup."); + } + const songs = decodeSongs(raw.songs); + const settings = decodeSettings(raw.s); + const uiPrefs = decodeUiPrefs(raw.u); + if (typeof raw.sat === 'number' && Number.isFinite(raw.sat)) settings.updatedAt = raw.sat; + if (typeof raw.uat === 'number' && Number.isFinite(raw.uat)) uiPrefs.updatedAt = raw.uat; + return { + format: BACKUP_FORMAT, + version: COMPACT_VERSION, + exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at : 0, + appVersion: '', + settings, + uiPrefs, + history: arr(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), + favorites: arr(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), + eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), + tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), + }; +} + +function normalizeV1(raw: Record): Backup { + return { + format: BACKUP_FORMAT, + version: BACKUP_VERSION, + exportedAt: typeof raw.exportedAt === 'number' ? raw.exportedAt : 0, + appVersion: typeof raw.appVersion === 'string' ? raw.appVersion : '', + settings: { + ...DEFAULT_SETTINGS, + ...(isRecord(raw.settings) ? raw.settings : {}), + } as Settings, + uiPrefs: { + ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), + ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), + }, + history: rekeyByIdentity(identifiedArr(raw.history, 'history')), + favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), + eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], + tracks: rekeyByIdentity(identifiedArr(raw.tracks, 'tracks')), + }; +} + +export function parseBackupJson(raw: unknown): Backup { + if (!isRecord(raw) || raw.format !== BACKUP_FORMAT) { + throw new Error("That file isn't a Note by Note backup."); + } + const version = typeof raw.version === 'number' ? raw.version : 0; + if (version > COMPACT_VERSION) { + throw new Error('That backup was made by a newer version of Note by Note.'); + } + if (version === COMPACT_VERSION) return decodeBackup(raw); + if (version === BACKUP_VERSION) return normalizeV1(raw); + // Only 2 lands here, and only from the branch this format grew on. + throw new Error('That backup is in a format this version of Note by Note no longer reads.'); +} diff --git a/src/core/persist/library-background.ts b/src/core/persist/library-background.ts new file mode 100644 index 0000000..fe335ac --- /dev/null +++ b/src/core/persist/library-background.ts @@ -0,0 +1,108 @@ +import { onMessage } from '../messaging/rpc'; +import { applyCommand, canonical, emptyLibrary, mergeShared, type Library } from './library'; +import { libraryItem } from './library-client'; +import { parseBackupJson as parseLegacy } from './legacy-backup'; +import { parseLibrary } from './backup-codec'; +import { migrateBackup } from './library-migration'; +import { bytesUsed, changedRecords, legacyKeys, PREFIX, readRecords } from '../../features/sync/persist/records'; +import { loadSyncConfig, syncConfigItem, type SyncConfig } from '../../features/sync/persist/sync-config'; + +const WAKE = 'library-sync'; +const SAFETY = 'library-sync-safety'; + +/** One writer for edits, imports and merges. Persisted records survive worker restarts. */ +export function startLibraryBackground() { + let queue: Promise = Promise.resolve(); + const enqueue = (work: () => Promise): Promise => { + const result = queue.then(work, work); + queue = result.catch(() => {}); + return result; + }; + let ready: Promise | undefined; + const init = () => ready ??= (async () => { + const raw = await browser.storage.local.get(null); + if (!raw.library) { + const defaults = emptyLibrary(); + const legacy = parseLegacy({ format: 'note-by-note-backup', version: 1, + settings: raw.settings ?? defaults.shared.settings.value, uiPrefs: raw.uiPrefs ?? defaults.local.uiPrefs, + history: raw.history ?? [], favorites: raw.favorites ?? [], eqPresets: raw.eqPresets ?? [], + tracks: Object.entries(raw).filter(([key]) => key.startsWith('track:')).map(([, value]) => value), + }); + await libraryItem.setValue(migrateBackup(legacy)); + // The old records remain recoverable; they are never read or written after migration. + } + })().catch((error) => { ready = undefined; throw error; }); + const saveConfig = (config: SyncConfig) => syncConfigItem.setValue(config); + const schedule = async () => { + const config = await loadSyncConfig(); + if (config.enabled) await browser.alarms.create(WAKE, { when: Math.max(Date.now() + 5000, config.lastPushAt + 30000) }); + }; + const reconcile = async () => { + await init(); + const config = await loadSyncConfig(); + if (!config.enabled) return; + await saveConfig({ ...config, syncing: true }); + try { + const existing = await browser.storage.sync.get(null); + config.usedBytes = bytesUsed(existing); + const remote = await readRecords(existing); + const local = await libraryItem.getValue(); + const shared = mergeShared(local.shared, remote); + if (canonical(shared) !== canonical(local.shared)) await libraryItem.setValue({ ...local, shared }); + const changes = await changedRecords(shared, remote, existing); + if (Object.keys(changes).length) { + if (Date.now() < config.lastPushAt + 30000) { await schedule(); return; } + // Legacy bytes may occupy the quota. Their contents are durable locally before removal. + const oldKeys = legacyKeys(existing); + if (oldKeys.length) await browser.storage.sync.remove(oldKeys); + await browser.storage.sync.set(changes); + config.lastPushAt = Date.now(); + const final = { ...existing, ...changes }; + for (const key of oldKeys) delete final[key]; + config.usedBytes = bytesUsed(final); + } + config.lastSyncedAt = Date.now(); + config.lastError = null; + } catch (error) { + config.lastError = error instanceof Error ? error.message : String(error); + } finally { + await saveConfig({ ...config, syncing: false }); + } + }; + onMessage('libraryRead', () => enqueue(async () => { await init(); return libraryItem.getValue(); })); + onMessage('libraryEdit', ({ data }) => enqueue(async () => { + await init(); + if (data.type === 'import') data.library = parseLibrary(data.library); + const current = await libraryItem.getValue(); + const next = applyCommand(current, data); + await libraryItem.setValue(next); + if (canonical(current.shared) !== canonical(next.shared)) await schedule(); + })); + onMessage('librarySync', ({ data }) => enqueue(async () => { + const config = await loadSyncConfig(); + if (data === 'disable' || data === 'delete') { + await saveConfig({ ...config, enabled: false, syncing: false }); + await browser.alarms.clear(WAKE); + if (data === 'delete') { + const items = await browser.storage.sync.get(null); + const keys = Object.keys(items).filter((key) => key.startsWith(PREFIX) || legacyKeys(items).includes(key)); + if (keys.length) await browser.storage.sync.remove(keys); + await saveConfig({ ...config, enabled: false, syncing: false, lastSyncedAt: 0, usedBytes: 0, lastError: null }); + } + return; + } + if (data === 'enable') await saveConfig({ ...config, enabled: true }); + await reconcile(); + })); + browser.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === WAKE || alarm.name === SAFETY) void enqueue(reconcile); + }); + browser.storage.onChanged.addListener((changes, area) => { + if (area === 'sync' && Object.keys(changes).some((key) => key.startsWith(PREFIX) || key.startsWith('nbn.'))) { + void enqueue(reconcile); + } + }); + // Recreate the safety alarm on each worker start. No panel has to stay open. + void browser.alarms.create(SAFETY, { periodInMinutes: 1 }); + void enqueue(reconcile); +} diff --git a/src/core/persist/library-client.ts b/src/core/persist/library-client.ts new file mode 100644 index 0000000..639b8d5 --- /dev/null +++ b/src/core/persist/library-client.ts @@ -0,0 +1,13 @@ +import { storage } from '#imports'; +import { sendMessage } from '../messaging/rpc'; +import { emptyLibrary, type Library, type LibraryCommand } from './library'; + +/** Panels observe the library; the background is its only writer. */ +export const libraryItem = storage.defineItem('local:library', { fallback: emptyLibrary() }); +export const readLibrary = () => sendMessage('libraryRead', undefined); +export const editLibrary = (command: LibraryCommand) => + sendMessage('libraryEdit', JSON.parse(JSON.stringify(command)) as LibraryCommand); + +export function watchLibrary(select: (library: Library) => T, listener: (value: T) => void) { + return libraryItem.watch((value) => listener(select(value ?? emptyLibrary()))); +} diff --git a/src/core/persist/library-migration.ts b/src/core/persist/library-migration.ts new file mode 100644 index 0000000..f39107d --- /dev/null +++ b/src/core/persist/library-migration.ts @@ -0,0 +1,52 @@ +import { DEFAULT_PARAMS } from '../model/defaults.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; +import { cell, emptyLibrary, newest, type Library, type Practice, type SavedSong } from './library.ts'; +import type { Backup } from './legacy-backup.ts'; + +/** Collapse old Recent/Favorites/track copies once, at the boundary. */ +export function migrateBackup(backup: Backup): Library { + const library = emptyLibrary(); + const { shared, local } = library; + const { lastUsedParams, updatedAt, ...settings } = backup.settings; + shared.settings = cell({ ...shared.settings.value, ...settings }, updatedAt ?? 0); + local.lastUsedParams = lastUsedParams; + local.uiPrefs = { ...local.uiPrefs, ...backup.uiPrefs }; + const ensure = (identity: Practice['identity']): SavedSong => { + const normalized = makeTrackIdentity(identity.normalizedUrl, identity.title, identity.durationSec); + return shared.songs[normalized.key] ??= { + practice: cell({ identity: normalized, pageUrl: normalized.normalizedUrl, markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false }), + favorite: cell(false), + }; + }; + // History dates saved parameters; a favorite's star date must not outrank an edit. + const entries = [...backup.favorites, ...backup.history].filter((entry) => !entry.deleted) + .sort((a, b) => (a.updatedAt ?? 0) - (b.updatedAt ?? 0)); + for (const entry of entries) { + const song = ensure(entry.identity); + song.practice = cell({ ...song.practice.value!, params: { ...DEFAULT_PARAMS, ...entry.params }, + pageUrl: entry.pageUrl, thumbnailUrl: entry.thumbnailUrl }, entry.updatedAt ?? 0); + } + const tracks = [...backup.tracks].sort((a, b) => a.updatedAt - b.updatedAt); + for (const track of tracks) { + const song = ensure(track.identity); + const { identity, updatedAt, chordChart, ...data } = track; + song.practice = cell({ ...song.practice.value!, ...data }, Math.max(updatedAt, song.practice.at)); + local.charts[song.practice.value!.identity.key] = chordChart ?? null; + } + for (const entry of backup.history) { + if (entry.deleted) continue; + const key = ensure(entry.identity).practice.value!.identity.key; + local.recent[key] = { updatedAt: entry.updatedAt, lastAccessedAt: entry.updatedAt }; + } + for (const entry of backup.favorites) { + const song = ensure(entry.identity); + song.favorite = newest(song.favorite, cell(!entry.deleted, entry.deleted ? entry.updatedAt : entry.favoritedAt)); + const key = song.practice.value!.identity.key; + if (local.recent[key]) local.recent[key].lastAccessedAt = Math.max(local.recent[key].lastAccessedAt, entry.lastAccessedAt); + } + shared.favoriteOrder = cell(backup.favorites.filter((f) => !f.deleted) + .map((f) => makeTrackIdentity(f.identity.normalizedUrl, f.identity.title, f.identity.durationSec).key), + Math.max(0, ...backup.favorites.map((f) => f.orderedAt ?? f.favoritedAt ?? 0))); + for (const preset of backup.eqPresets) shared.presets[preset.name] = cell(preset.deleted ? null : preset.gains, preset.updatedAt ?? 0); + return library; +} diff --git a/src/core/persist/library.ts b/src/core/persist/library.ts new file mode 100644 index 0000000..d173d5d --- /dev/null +++ b/src/core/persist/library.ts @@ -0,0 +1,181 @@ +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, HISTORY_LIMIT } from '../model/defaults.ts'; +import type { ChordChart, EffectParams, FavoriteEntry, HistoryEntry, Settings, TrackData, TrackIdentity, UiPrefs } from '../model/types'; + +/** One revision per independently editable value. Null/false are durable deletions. */ +export interface Versioned { at: number; value: T } +export interface Practice extends Omit { + params?: EffectParams; + pageUrl: string; + thumbnailUrl?: string; +} +export interface SavedSong { + practice: Versioned; + favorite: Versioned; +} +export interface SharedLibrary { + settings: Versioned; + songs: Record; + presets: Record>; + favoriteOrder: Versioned; +} +export interface Library { + shared: SharedLibrary; + local: { + uiPrefs: UiPrefs; + recent: Record; + charts: Record; + lastUsedParams?: EffectParams; + }; +} + +export const cell = (value: T, at = 0): Versioned => ({ at, value }); +export function emptyLibrary(): Library { + return { + shared: { settings: cell(structuredClone(DEFAULT_SETTINGS)), songs: {}, presets: {}, favoriteOrder: cell([]) }, + local: { uiPrefs: structuredClone(DEFAULT_UI_PREFS), recent: {}, charts: {} }, + }; +} + +/** Stable comparison for equal revisions; also the JSON form used on the wire. */ +export function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') { + return `{${Object.entries(value).filter(([, v]) => v !== undefined) + .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([key, v]) => `${JSON.stringify(key)}:${canonical(v)}`).join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +} +export function newest(a: Versioned, b: Versioned): Versioned { + return a.at > b.at || (a.at === b.at && canonical(a.value) >= canonical(b.value)) ? a : b; +} +function mergeMap(a: Record, b: Record, merge: (a: T, b: T) => T) { + return Object.fromEntries([...new Set([...Object.keys(a), ...Object.keys(b)])].sort() + .map((key) => [key, a[key] && b[key] ? merge(a[key], b[key]) : a[key] ?? b[key]])); +} +export function mergeShared(a: SharedLibrary, b: SharedLibrary): SharedLibrary { + return { + settings: newest(a.settings, b.settings), + songs: mergeMap(a.songs, b.songs, (x, y) => ({ + practice: newest(x.practice, y.practice), favorite: newest(x.favorite, y.favorite), + })), + presets: mergeMap(a.presets, b.presets, newest), + favoriteOrder: newest(a.favoriteOrder, b.favoriteOrder), + }; +} +/** Every local edit follows all revisions this installation has observed. */ +export function nextRevision(shared: SharedLibrary, now = Date.now()): number { + let at = Math.max(now, shared.settings.at, shared.favoriteOrder.at); + for (const song of Object.values(shared.songs)) at = Math.max(at, song.practice.at, song.favorite.at); + for (const preset of Object.values(shared.presets)) at = Math.max(at, preset.at); + return at + 1; +} + +export type LibraryCommand = + | { type: 'practice'; identity: TrackIdentity; patch: Partial; recent: boolean } + | { type: 'favorite'; key: string; value: boolean } + | { type: 'order'; keys: string[] } + | { type: 'visit'; key: string } + | { type: 'recent.remove'; key?: string } + | { type: 'chart'; key: string; chart: ChordChart | null } + | { type: 'settings'; patch: Partial; reset?: boolean } + | { type: 'uiPrefs'; value: UiPrefs } + | { type: 'preset'; name: string; gains: number[] | null } + | { type: 'import'; library: Library }; + +/** Called only by the background writer. Incoming edits patch current saved data. */ +export function applyCommand(library: Library, command: LibraryCommand, now = Date.now()): Library { + const next = structuredClone(library); + const { shared, local } = next; + const at = nextRevision(shared, now); + switch (command.type) { + case 'practice': { + const key = command.identity.key; + const song = shared.songs[key] ?? { practice: cell(null), favorite: cell(false) }; + const base: Practice = song.practice.value ?? { + identity: command.identity, pageUrl: command.identity.normalizedUrl, + markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false, + }; + song.practice = cell({ ...base, ...command.patch, identity: command.identity }, at); + shared.songs[key] = song; + if (command.recent || local.recent[key]) local.recent[key] = { updatedAt: now, lastAccessedAt: now }; + const keep = Object.entries(local.recent).sort((a, b) => b[1].updatedAt - a[1].updatedAt).slice(0, HISTORY_LIMIT); + local.recent = Object.fromEntries(keep); + break; + } + case 'favorite': { + const song = shared.songs[command.key]; + if (!song?.practice.value) break; + song.favorite = cell(command.value, at); + if (command.value) shared.favoriteOrder = cell([command.key, ...shared.favoriteOrder.value.filter((k) => k !== command.key)], at); + break; + } + case 'order': shared.favoriteOrder = cell([...new Set(command.keys)], at); break; + case 'visit': { + const recent = local.recent[command.key]; + if (recent) recent.lastAccessedAt = now; + break; + } + case 'recent.remove': + if (command.key === undefined) local.recent = {}; + else delete local.recent[command.key]; + break; + case 'chart': local.charts[command.key] = command.chart; break; + case 'settings': { + const { lastUsedParams, updatedAt: ignored, ...patch } = command.patch; + if (lastUsedParams) local.lastUsedParams = lastUsedParams; + if (command.reset || Object.keys(patch).length) { + const value = { ...(command.reset ? DEFAULT_SETTINGS : shared.settings.value), ...patch }; + if (patch.rememberSettings) value.autoReset = false; + if (patch.autoReset) value.rememberSettings = false; + shared.settings = cell(value, at); + } + if (command.reset) delete local.lastUsedParams; + break; + } + case 'uiPrefs': local.uiPrefs = command.value; break; + case 'preset': shared.presets[command.name] = cell(command.gains, at); break; + case 'import': { + const file = structuredClone(command.library); + const revision = Math.max(at, nextRevision(file.shared, now)); + // Replacement names the records this device knows; absence is never a remote delete. + for (const key of new Set([...Object.keys(shared.songs), ...Object.keys(file.shared.songs)])) { + const song = file.shared.songs[key]; + shared.songs[key] = { practice: cell(song?.practice.value ?? null, revision), favorite: cell(song?.favorite.value ?? false, revision) }; + } + for (const name of new Set([...Object.keys(shared.presets), ...Object.keys(file.shared.presets)])) { + shared.presets[name] = cell(file.shared.presets[name]?.value ?? null, revision); + } + shared.settings = cell(file.shared.settings.value, revision); + shared.favoriteOrder = cell(file.shared.favoriteOrder.value, revision); + next.local = file.local; + break; + } + } + return next; +} + +/** UI rows are projections. They are never written back as library copies. */ +export function songEntry(key: string, library: Library): HistoryEntry | null { + const song = library.shared.songs[key]; + const practice = song?.practice.value; + if (!practice) return null; + return { + identity: practice.identity, pageUrl: practice.pageUrl, thumbnailUrl: practice.thumbnailUrl, + params: practice.params ?? structuredClone(DEFAULT_PARAMS), + createdAt: song.practice.at, updatedAt: library.local.recent[key]?.updatedAt ?? song.practice.at, + }; +} +export function recentEntries(library: Library): HistoryEntry[] { + return Object.keys(library.local.recent).map((key) => songEntry(key, library)) + .filter((entry) => entry !== null).sort((a, b) => b.updatedAt - a.updatedAt); +} +export function favoriteEntries(library: Library): FavoriteEntry[] { + const keys = [...new Set([...library.shared.favoriteOrder.value, ...Object.keys(library.shared.songs).sort()])]; + return keys.flatMap((key) => { + const song = library.shared.songs[key]; + const entry = songEntry(key, library); + return song?.favorite.value && entry ? [{ ...entry, favoritedAt: song.favorite.at, + lastAccessedAt: library.local.recent[key]?.lastAccessedAt ?? song.favorite.at }] : []; + }); +} diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index 4a8d2cd..2cda38d 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -1,103 +1,19 @@ -import { storage, type StorageItemKey, type WxtStorageItem } from '#imports'; -import { DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults'; -import type { - EqPreset, - FavoriteEntry, - HistoryEntry, - Settings, - TrackData, - UiPrefs, -} from '../model/types'; +import { storage } from '#imports'; +import type { Settings, UiPrefs } from '../model/types'; +import { editLibrary, readLibrary, watchLibrary } from './library-client'; -/** Rebuild a value as plain arrays/objects, stripping any Svelte `$state` - * proxies on the way. - * - * Chrome serializes storage writes to JSON and reads straight through a proxy; - * Firefox structured-clones them and throws DataCloneError, rejecting the write - * with nothing persisted. Panel stores are expected to `$state.snapshot` before - * writing, but one missed call is an invisible, browser-specific data-loss bug — - * so every write goes through here as well. - * - * A rebuild, not `structuredClone`: that throws on a proxy, which is the very - * case being defended against. Safe because this schema is JSON-shaped - * throughout (numbers, strings, booleans, arrays, plain objects); a Date, Map or - * typed array added later would need handling here first. - * - * Plain function, not the `$state.snapshot` rune: background.ts imports this - * module and runes only compile inside Svelte files. */ -function toPlain(value: T): T { - if (Array.isArray(value)) return value.map(toPlain) as T; - if (value === null || typeof value !== 'object') return value; - const out: Record = {}; - for (const [k, v] of Object.entries(value)) out[k] = toPlain(v); - return out as T; -} - -/** `storage.defineItem` with proxy-stripping on every write. Annotated rather - * than inferred: `storage.defineItem` is overloaded five ways, so deriving this - * signature from it makes the inference circular. */ -function defineItem( - key: StorageItemKey, - options: { fallback: T }, -): WxtStorageItem> { - const item = storage.defineItem(key, options); - const setValue = item.setValue.bind(item); - item.setValue = (value: T) => setValue(toPlain(value)); - return item; -} - -export const settingsItem = defineItem('local:settings', { - fallback: DEFAULT_SETTINGS, -}); - -export const uiPrefsItem = defineItem('local:uiPrefs', { - fallback: DEFAULT_UI_PREFS, -}); - -/** Recent history (Auto Save), newest first. */ -export const historyItem = defineItem('local:history', { - fallback: [], -}); - -/** Starred songs (History → Favorites). Array order = manual sort order. */ -export const favoritesItem = defineItem('local:favorites', { - fallback: [], +/** Feature stores read projections and send changes to the single background writer. */ +const settingsOf = (library: Awaited>): Settings => ({ + ...library.shared.settings.value, lastUsedParams: library.local.lastUsedParams, }); - -/** EQ curves the user saved (Equalizer → preset row). Array order = save order. - * Kept out of `settings` so Reset Settings can't wipe them. */ -export const eqPresetsItem = defineItem('local:eqPresets', { - fallback: [], -}); - -/** Origins the user has granted host permission for (mirrors permissions API, - * used to show/revoke the list without a permissions query round-trip). */ -export const grantedOriginsItem = defineItem('local:grantedOrigins', { - fallback: [], -}); - -/** Per-track markers/snippets, keyed by TrackIdentity.key. */ -export function trackDataKey(key: string) { - return `local:track:${key}` as const; -} - -export async function loadTrackData(key: string): Promise { - return (await storage.getItem(trackDataKey(key))) ?? null; -} - -export async function saveTrackData(data: TrackData): Promise { - await storage.setItem(trackDataKey(data.identity.key), toPlain(data)); -} - -/** Drops every stored track record whose identity key isn't in `keep` — a - * restore replaces the set of records rather than merging into it. Written - * as "remove what's left over" (rather than wiping first) so the records are - * only ever gone once their replacements are in: a restore interrupted - * halfway leaves stale records behind, never an empty library. */ -export async function removeTrackDataExcept(keep: Set): Promise { - const snapshot = await browser.storage.local.get(null); - const stale = Object.keys(snapshot).filter( - (k) => k.startsWith('track:') && !keep.has(k.slice('track:'.length)), - ); - if (stale.length) await browser.storage.local.remove(stale); -} +export const settingsItem = { + getValue: async () => settingsOf(await readLibrary()), + setValue: (value: Settings) => editLibrary({ type: 'settings', patch: value }), + watch: (listener: (value: Settings) => void) => watchLibrary(settingsOf, listener), +}; +export const uiPrefsItem = { + getValue: async () => (await readLibrary()).local.uiPrefs, + setValue: (value: UiPrefs) => editLibrary({ type: 'uiPrefs', value }), + watch: (listener: (value: UiPrefs) => void) => watchLibrary((library) => library.local.uiPrefs, listener), +}; +export const grantedOriginsItem = storage.defineItem('local:grantedOrigins', { fallback: [] }); diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index 1bd5afb..ca48392 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -1,266 +1,97 @@ import { DEFAULT_PARAMS } from '../model/defaults'; import { makeTrackIdentity } from '../model/track-identity'; -import type { EffectParams, HistoryEntry, MediaInfo, TrackData, TrackIdentity } from '../model/types'; -import { touchFavorite } from '../../features/library/persist/favorites'; -import { removeHistoryEntry, upsertHistory } from '../../features/library/persist/history'; -import { findSavedEntry } from '../../features/library/panel/saved-settings'; -import { loadTrackData, saveTrackData } from '../persist/storage'; +import type { EffectParams, HistoryEntry, MediaInfo, TrackIdentity } from '../model/types'; +import type { Practice } from '../persist/library'; +import { editLibrary, readLibrary } from '../persist/library-client'; import { openTabWithPanel } from '../side-panel'; -import { trackDataDescriptors } from '../persist/track-data'; import { session } from './session.svelte'; import { settings } from '../../features/settings/panel/settings.svelte'; +import { markers } from '../../features/markers/panel/markers.svelte'; +import { snippets } from '../../features/snippets/panel/snippets.svelte'; +import { chords } from '../../features/chords/panel/chords.svelte'; -/** Reacts to track changes — auto-saves the previous track to - * Recent, then resets / remembers / carries over params, and swaps the - * per-track markers & snippets in and out of storage. */ +/** An active session loads saved data once. Library updates never interrupt playback. */ class TrackSync { #identity: TrackIdentity | null = null; - #pageUrl = ''; - #thumbnailUrl: string | undefined; - #saveTimer: ReturnType | undefined; - /** The params last set for #identity. `session.params` is overwritten by the - * incoming engine's snapshot before that snapshot's media event reaches us, - * so the outgoing track has to be saved from this, not from the mirror. */ - #params: EffectParams | null = null; - /** True once the user touched a control on this track — gates the Recent save. */ - #userAdjusted = false; - /** Key of an in-flight #apply. Every connect delivers more than one snapshot - * and #identity is only assigned after an awaited save, so without this two - * runs both take the track-switch branch and the loser applies auto-reset - * over the params the winner just restored. */ - #applyingKey: string | null = null; - /** The song whose saved settings are currently applied, as `url\ntitle`. Set - * only on a successful restore, so a lookup that missed — the title had not - * settled yet — is retried on the next media event. Deliberately excludes the - * duration: drifting duration is what it has to survive. */ - #restoredFor: string | null = null; - #zeroDurationTimer: ReturnType | undefined; + #media: MediaInfo | null = null; + #generation = 0; + #restoring = false; + #edited = false; + #chordsEnabled = false; init() { - for (const d of trackDataDescriptors) { - d.bind(() => { - // Marker/snippet/chord edits count as adjusting a control. - this.#userAdjusted = true; - this.#persistTrackData(); - void this.#saveCurrent(); - }); - } + markers.onPersist = (list) => this.#save({ markers: list }); + snippets.onPersist = () => this.#save({ snippets: $state.snapshot(snippets.list), + sequenceLoop: snippets.sequenceLoop, sequenceCountIn: snippets.sequenceCountIn }); + chords.onPersist = () => { + if (!this.#identity) return; + // Generated analysis is local and does not date the saved practice record. + void editLibrary({ type: 'chart', key: this.#identity.key, chart: $state.snapshot(chords.chart) }); + if (this.#chordsEnabled !== chords.enabled) { + this.#chordsEnabled = chords.enabled; + this.#save({ chordsEnabled: chords.enabled }); + } + }; } - /** The engine link dropped — reload, reopened tab, tab switch. Whatever - * reconnects starts on the default preset, so this song has to be restored - * again; flush any live edits first so the restore reads them back. */ onEngineLost() { - // #saveCurrent reads #userAdjusted and #params before its first await, so - // clearing the flag on the next line can't race it. - void this.#saveCurrent(); - this.#userAdjusted = false; - this.#restoredFor = null; + this.#generation++; + this.#identity = null; + this.#media = null; } - /** Called for every media info event from the engine. */ async onMedia(media: MediaInfo | null) { - clearTimeout(this.#zeroDurationTimer); - if (!media) return; - if (!media.duration) { - // Duration 0 usually means metadata is still loading — but streams and - // unseekable sources never report one. Give durationchange a moment to - // supersede before keying the track without a duration. - this.#zeroDurationTimer = setTimeout(() => { - void this.#apply(media); - }, 3000); - return; - } - await this.#apply(media); - } - - /** Puts a song's saved settings back on, however it was opened — a clicked - * row, a typed URL, a link, an SPA navigation, a reload. Recent and Favorites - * hand a song back the way it was left; the "new song" preference governs only - * songs with nothing saved. */ - #restoreSaved(identity: TrackIdentity): boolean { - const token = `${identity.normalizedUrl}\n${identity.title}`; - if (this.#restoredFor === token) return false; - const entry = findSavedEntry(identity); - if (!entry) return false; - this.#restoredFor = token; - // $state.snapshot, not structuredClone: the entry belongs to a $state store, - // so its params are a proxy (structuredClone throws DataCloneError on one) - // and patchParams would otherwise assign its EQ band array by reference. - this.#applyParams($state.snapshot(entry.params) as EffectParams); - return true; - } - - /** Sets params on the track's behalf rather than the user's. patchParams fires - * onParamsChanged synchronously, so without this every restore and every auto - * reset would count as an edit and re-save the entry it just read. */ - #applyParams(params: EffectParams) { - session.patchParams(params); - this.#userAdjusted = false; - clearTimeout(this.#saveTimer); - } - - async #apply(media: MediaInfo) { + if (!media) { this.onEngineLost(); return; } const identity = makeTrackIdentity(media.pageUrl, media.title, media.duration); - if (identity.key === this.#applyingKey) return; - - if (identity.key === this.#identity?.key) { - // Same song. Its duration may have settled since (a pre-roll ad, slow - // metadata) — duration is metadata, not identity, so it is updated in - // place and nothing is re-keyed (`track-identity.ts`). - if (identity.durationSec !== this.#identity.durationSec) this.#identity = identity; - // A no-op unless the engine restarted on the defaults. - if (!this.#userAdjusted) this.#restoreSaved(identity); - return; - } - - // Same page, new title — the site rewrote document.title after the element - // fired, so the song was keyed under the placeholder. Re-key in place - // instead of treating it as a track switch, so Recent doesn't get a - // duplicate row. - if (this.#identity && identity.normalizedUrl === this.#identity.normalizedUrl) { - const staleKey = this.#identity.key; - const adjusted = this.#userAdjusted; - this.#identity = identity; - this.#pageUrl = media.pageUrl; - this.#thumbnailUrl = media.thumbnailUrl ?? this.#thumbnailUrl; - // The key was wrong until now, so the real key's slice was never loaded. - const data = await loadTrackData(identity.key); - if (data) for (const d of trackDataDescriptors) d.load(data); - // Their edits outrank anything stored; otherwise a lookup that missed on - // the stale identity gets another go now the duration has settled. - if (!adjusted) this.#restoreSaved(identity); - if (adjusted) { - // Housekeeping, not a user deletion: this is the same song under its - // real title, so no tombstone — that names the song, and would kill - // its fresh row on the other devices. - await removeHistoryEntry(staleKey, { record: false }); - await this.#saveCurrent(); - // Only carry the stale key's slice over when the real key has none, so - // this can't overwrite a saved record with an emptied one. - if (!data) this.#persistTrackData(); - } - return; - } - - this.#applyingKey = identity.key; + this.#media = media; + if (this.#identity?.key === identity.key) { this.#identity = identity; return; } + this.#identity = identity; + this.#edited = false; + const generation = ++this.#generation; + const saved = await readLibrary(); + if (generation !== this.#generation || this.#edited) return; + const practice = saved.shared.songs[identity.key]?.practice.value; + this.#restoring = true; try { - // Leaving the previous track: auto-save it with its final settings. - await this.#saveCurrent(); - - this.#identity = identity; - this.#pageUrl = media.pageUrl; - this.#thumbnailUrl = media.thumbnailUrl; - - // Restore this track's per-feature data (markers, snippets, chords) if - // we've seen it before — each feature scatters its own slice. - const data = await loadTrackData(identity.key); - for (const d of trackDataDescriptors) d.load(data); - - // A different song, so nothing is applied for it yet — even if we happen - // to be coming back to one restored earlier. Reset after both awaits, so - // this branch is the last writer and an event that slipped through during - // them can't restore only to be auto-reset over. - this.#restoredFor = null; - this.#userAdjusted = false; - // Starting params: the song's own saved settings > auto reset > remember - // > carry over. The last three are for songs with nothing saved. - if (!this.#restoreSaved(identity)) { - if (settings.current.autoReset) { - this.#applyParams(structuredClone(DEFAULT_PARAMS)); - } else if (settings.current.rememberSettings && settings.current.lastUsedParams) { - // $state.snapshot, not structuredClone: settings.current is a rune, so - // lastUsedParams is a proxy and structuredClone throws on it. - this.#applyParams($state.snapshot(settings.current.lastUsedParams) as EffectParams); - } - } - this.#params = $state.snapshot(session.params) as EffectParams; - - // If the track is favorited, bump its Last Accessed timestamp. - await touchFavorite(identity, { - pageUrl: media.pageUrl, - thumbnailUrl: media.thumbnailUrl, - }); + markers.load(practice?.markers ?? []); + snippets.load(practice?.snippets ?? [], practice?.sequenceLoop ?? false, practice?.sequenceCountIn ?? false); + chords.load(saved.local.charts[identity.key] ?? null, practice?.chordsEnabled); + this.#chordsEnabled = chords.enabled; + const params = practice?.params ?? (settings.current.autoReset ? DEFAULT_PARAMS : + settings.current.rememberSettings ? settings.current.lastUsedParams : undefined); + if (params) session.patchParams(structuredClone(params)); } finally { - this.#applyingKey = null; + this.#restoring = false; } + await editLibrary({ type: 'visit', key: identity.key }); } - /** Called on (debounced) param changes to keep history and - * "Remember settings" fresh without waiting for a track switch. */ onParamsChanged() { - this.#userAdjusted = true; - this.#params = $state.snapshot(session.params) as EffectParams; - clearTimeout(this.#saveTimer); - this.#saveTimer = setTimeout(() => { - void this.#saveCurrent(); - if (settings.current.rememberSettings) { - void settings.update({ - lastUsedParams: $state.snapshot(session.params) as EffectParams, - }); - } - }, 1500); - } - - async #saveCurrent() { - if (!this.#identity) return; - // Nothing was edited on this track — mirroring now would write auto-reset - // or carried-over state over what the user actually saved. - if (!this.#userAdjusted) return; - const params = this.#params ?? ($state.snapshot(session.params) as EffectParams); - // Both copies, from one value, in one call: a song that is favorited *and* - // in Recent must never end up with the two disagreeing. Auto Save off stops - // new rows being added, not an existing one being kept current. - await touchFavorite(this.#identity, { params }); - await upsertHistory( - this.#identity, - params, - this.#pageUrl, - this.#thumbnailUrl, - !settings.current.autoSave, - ); + if (this.#restoring) return; + const params = $state.snapshot(session.params) as EffectParams; + this.#save({ params }); + if (settings.current.rememberSettings) void editLibrary({ type: 'settings', patch: { lastUsedParams: params } }); } - #persistTrackData() { - if (!this.#identity) return; - // Build the single record; each feature descriptor fills in its own fields. - const data: TrackData = { - identity: this.#identity, - markers: [], - snippets: [], - sequenceLoop: false, - sequenceCountIn: false, - chordChart: null, - chordsEnabled: false, - updatedAt: Date.now(), - }; - for (const d of trackDataDescriptors) d.collect(data); - void saveTrackData(data); + #save(patch: Partial) { + if (!this.#identity || this.#restoring) return; + this.#edited = true; + void editLibrary({ type: 'practice', identity: this.#identity, + patch: { ...patch, pageUrl: this.#media?.pageUrl ?? this.#identity.normalizedUrl, + thumbnailUrl: this.#media?.thumbnailUrl }, recent: settings.current.autoSave, + }).catch((error) => console.error('[note-by-note] saving practice failed', error)); } - /** A row in the Songs list was clicked: go there. Its settings need no staging - * — arriving at a song is what puts them back on, whoever asked for it. */ async openHistoryEntry(tabId: number | null, entry: HistoryEntry) { - const target = makeTrackIdentity(entry.pageUrl, '', 0).normalizedUrl; - const playing = session.media?.pageUrl; - // Already on that page: apply in place. Re-navigating to the URL it is - // already on would reload for nothing (losing the playhead), and may not - // fire a media event at all. - if (playing && makeTrackIdentity(playing, '', 0).normalizedUrl === target) { - // Snapshot: `entry` belongs to a $state store, so patchParams would - // otherwise assign its EQ band array straight off the entry. - this.#applyParams($state.snapshot(entry.params) as EffectParams); + const playing = this.#identity?.key; + if (playing === entry.identity.key) { + // Explicitly opening the saved song adopts its current library revision. + this.#identity = null; + await this.onMedia(this.#media); return; } - - if (tabId != null) { - await browser.tabs.update(tabId, { url: entry.pageUrl }); - } else { - // No engine tab to reuse — a fresh one the panel follows (a plain create - // would activate a tab the panel isn't enabled on and hide it). - await openTabWithPanel(entry.pageUrl); - } + if (tabId != null) await browser.tabs.update(tabId, { url: entry.pageUrl }); + else await openTabWithPanel(entry.pageUrl); } } - export const trackSync = new TrackSync(); diff --git a/src/entrypoints/background.ts b/src/entrypoints/background.ts index d7e71ed..a98107f 100644 --- a/src/entrypoints/background.ts +++ b/src/entrypoints/background.ts @@ -1,3 +1,4 @@ +import { startLibraryBackground } from '@/core/persist/library-background'; import type { OffscreenCommand } from '@/core/messaging/protocol'; import { onMessage } from '@/core/messaging/rpc'; import { grantedOriginsItem } from '@/core/persist/storage'; @@ -69,6 +70,7 @@ async function syncFromPermissions() { } export default defineBackground(() => { + startLibraryBackground(); // Scope the panel to the tabs it was opened on: the manifest's // `side_panel.default_path` enables it everywhere, so once opened it would // follow the user to every tab. With the default disabled and the click diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index de8dee8..7791f67 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -8,7 +8,6 @@ import { sendMessage } from '@/core/messaging/rpc'; import { openTabWithPanel } from '@/core/side-panel'; import { installMockState, installMockTicker } from '@/dev/mock'; - import { migrateStorage } from '@/core/persist/migrate'; import { connection } from '@/core/state/connect.svelte'; import { CAN_CAPTURE_TAB } from '@/core/platform'; import { features } from '@/core/features'; @@ -27,7 +26,7 @@ // to read (see core/persist/migrate.ts). Each panel feature then loads its // own storage concurrently (see core/features.ts). const loadFeatures = () => Promise.all(features.map((f) => f.init?.())); - const ready = migrateStorage().then(loadFeatures).then( + const ready = loadFeatures().then( async () => { applyTheme(settings.current.theme); trackSync.init(); @@ -36,9 +35,6 @@ console.error('[note-by-note] track sync failed', err); }); }; - // Applying another device's changes reloads this document, so a merge - // that arrived mid-practice waits for the track to go away. - session.onMediaChanged = (media) => sync.onMedia(media); session.onUserParamsChange = () => trackSync.onParamsChanged(); session.onEngineDetached = () => trackSync.onEngineLost(); // Diagnostics for the E2E harness; kept out of release builds. diff --git a/src/features/chords/panel/chords.svelte.ts b/src/features/chords/panel/chords.svelte.ts index 223dc74..59a6de7 100644 --- a/src/features/chords/panel/chords.svelte.ts +++ b/src/features/chords/panel/chords.svelte.ts @@ -94,11 +94,6 @@ class ChordsStore { this.#log(`analyze confirmed (${fromStart ? 'from start' : 'from here'}) — loading model`); this.phase = 'loading'; this.loadError = false; - // Null, not `clear()`: the old chart is being replaced, not deleted, and - // the run can still fail (the model load is a download). An empty dated - // chart is a deletion that outranks another device's real one, so a - // failed run here would wipe the chords everywhere; null just means - // "nothing on this device", which the merge fills back in. this.chart = null; this.#persistNow(); try { @@ -183,11 +178,7 @@ class ChordsStore { /** Delete the analyzed chords for this track (and the persisted copy). */ clear() { - // An empty, dated chart is a deletion; null can also mean sync trimmed it. - this.chart = { - segments: [], key: null, coverage: 0, analyzedFrom: 0, analyzedTo: 0, - computedAt: Date.now(), - }; + this.chart = null; this.#persistNow(); } diff --git a/src/features/eq/panel/eq-presets.svelte.ts b/src/features/eq/panel/eq-presets.svelte.ts index e864408..e48bfe4 100644 --- a/src/features/eq/panel/eq-presets.svelte.ts +++ b/src/features/eq/panel/eq-presets.svelte.ts @@ -1,8 +1,7 @@ import { BUILTIN_EQ_PRESETS, EQ_BANDS } from '../../../core/model/defaults'; import type { EqPreset } from '../../../core/model/types'; -import { deleteEqPreset, saveEqPreset } from '../persist/eq-presets'; -import { isLive } from '../../../core/persist/deletions'; -import { eqPresetsItem } from '../../../core/persist/storage'; +import { editLibrary, readLibrary, watchLibrary } from '../../../core/persist/library-client'; +import type { Library } from '../../../core/persist/library'; /** Slider gains are multiples of 0.5 dB, so anything closer than this is the * same curve; the tolerance only guards against float drift. */ @@ -14,10 +13,11 @@ class EqPresetsStore { saved = $state([]); async init() { - this.saved = (await eqPresetsItem.getValue()).filter(isLive); - eqPresetsItem.watch((value) => { - this.saved = (value ?? []).filter(isLive); - }); + const select = (library: Library): EqPreset[] => Object.entries(library.shared.presets) + .filter(([, preset]) => preset.value !== null) + .map(([name, preset]) => ({ name, gains: preset.value!, updatedAt: preset.at })); + this.saved = select(await readLibrary()); + watchLibrary(select, (value) => { this.saved = value; }); } /** Built-ins first, then the user's, as listed in the dropdown. */ @@ -42,11 +42,11 @@ class EqPresetsStore { } async save(name: string, gains: number[]) { - await saveEqPreset(name, gains); + await editLibrary({ type: 'preset', name, gains }); } async remove(name: string) { - await deleteEqPreset(name); + await editLibrary({ type: 'preset', name, gains: null }); } } diff --git a/src/features/library/panel/favorites.svelte.ts b/src/features/library/panel/favorites.svelte.ts index f94cfb8..12a1ccf 100644 --- a/src/features/library/panel/favorites.svelte.ts +++ b/src/features/library/panel/favorites.svelte.ts @@ -1,70 +1,13 @@ -import type { FavoriteEntry, HistoryEntry, TrackIdentity } from '../../../core/model/types'; -import { - addFavorite, - removeFavorite, - setFavoritesOrder, -} from '../persist/favorites'; -import { isSameTrack } from '../../../core/model/track-identity'; -import { isLive } from '../../../core/persist/deletions'; -import { favoritesItem } from '../../../core/persist/storage'; - -/** Every write below is fired from a click handler as a floating promise, and - * the store only repaints from the `favoritesItem.watch` callback — so a failed - * write repaints nothing and reads as a dead button. Log instead of vanishing. */ -async function write(what: string, run: () => Promise): Promise { - try { - await run(); - } catch (err) { - console.error(`[note-by-note] favorites: ${what} failed:`, err); - } -} - -class FavoritesStore { - /** Live rows only — the stored list also carries unstar tombstones - * (`deletions.ts`). */ - entries = $state([]); - - async init() { - this.entries = (await favoritesItem.getValue()).filter(isLive); - favoritesItem.watch((value) => { - this.entries = (value ?? []).filter(isLive); - }); - } - - /** By song, not by key: a favorite stored under a duration that has since - * drifted is still this track, and its star has to read as lit. */ - has(identity: TrackIdentity): boolean { - return this.entries.some((e) => isSameTrack(e.identity, identity)); - } - - async toggle(entry: HistoryEntry) { - // Unstar the row as it was stored — its key may differ from this one's. - const existing = this.entries.find((e) => isSameTrack(e.identity, entry.identity)); - // $state.snapshot: `entry` belongs to the history store, so it and its - // nested identity/params are proxies. Firefox structured-clones storage - // writes and throws DataCloneError on a proxy (Chrome, which serializes to - // JSON, does not) — without this the star silently never lights. - await write('toggle', () => - existing - ? removeFavorite(existing.identity.key) - : addFavorite($state.snapshot(entry) as HistoryEntry), - ); - } - - async remove(key: string) { - await write('remove', () => removeFavorite(key)); - } - - /** Commit a new manual order (complete list of identity keys). Applied - * optimistically so the list doesn't snap back while storage round-trips. */ - async reorder(keys: string[]) { - const byKey = new Map(this.entries.map((e) => [e.identity.key, e])); - const next = keys - .map((k) => byKey.get(k)) - .filter((e): e is (typeof this.entries)[number] => e !== undefined); - if (next.length === this.entries.length) this.entries = next; - await write('reorder', () => setFavoritesOrder(keys)); - } -} - -export const favorites = new FavoritesStore(); +import type { HistoryEntry, TrackIdentity } from '../../../core/model/types'; +import { favoriteEntries } from '../../../core/persist/library'; +import { editLibrary } from '../../../core/persist/library-client'; +import { library } from './library.svelte'; + +export const favorites = { + get entries() { return favoriteEntries(library.current); }, + has: (identity: TrackIdentity) => library.current.shared.songs[identity.key]?.favorite.value === true, + toggle: (entry: HistoryEntry) => editLibrary({ type: 'favorite', key: entry.identity.key, + value: !library.current.shared.songs[entry.identity.key]?.favorite.value }), + remove: (key: string) => editLibrary({ type: 'favorite', key, value: false }), + reorder: (keys: string[]) => editLibrary({ type: 'order', keys }), +}; diff --git a/src/features/library/panel/history.svelte.ts b/src/features/library/panel/history.svelte.ts index 63fdd94..db01383 100644 --- a/src/features/library/panel/history.svelte.ts +++ b/src/features/library/panel/history.svelte.ts @@ -1,28 +1,9 @@ -import type { HistoryEntry } from '../../../core/model/types'; -import { clearHistory, removeHistoryEntry } from '../persist/history'; -import { isLive } from '../../../core/persist/deletions'; -import { historyItem } from '../../../core/persist/storage'; - -class HistoryStore { - /** Live rows only: the stored list also carries tombstones for what the - * user removed, which exist purely so a sync merge can't resurrect it - * (`deletions.ts`). Filtering here is what keeps them out of every screen. */ - entries = $state([]); - - async init() { - this.entries = (await historyItem.getValue()).filter(isLive); - historyItem.watch((value) => { - this.entries = (value ?? []).filter(isLive); - }); - } - - async remove(key: string) { - await removeHistoryEntry(key); - } - - async clear() { - await clearHistory(); - } -} - -export const history = new HistoryStore(); +import { recentEntries } from '../../../core/persist/library'; +import { editLibrary } from '../../../core/persist/library-client'; +import { library } from './library.svelte'; + +export const history = { + get entries() { return recentEntries(library.current); }, + remove: (key: string) => editLibrary({ type: 'recent.remove', key }), + clear: () => editLibrary({ type: 'recent.remove' }), +}; diff --git a/src/features/library/panel/library.svelte.ts b/src/features/library/panel/library.svelte.ts new file mode 100644 index 0000000..c7e1344 --- /dev/null +++ b/src/features/library/panel/library.svelte.ts @@ -0,0 +1,13 @@ +import { emptyLibrary, type Library } from '../../../core/persist/library'; +import { readLibrary, watchLibrary } from '../../../core/persist/library-client'; + +class LibraryStore { + current = $state.raw(emptyLibrary()); + async init() { + let changed = false; + watchLibrary((library) => library, (library) => { changed = true; this.current = library; }); + const initial = await readLibrary(); + if (!changed) this.current = initial; + } +} +export const library = new LibraryStore(); diff --git a/src/features/library/panel/panel.ts b/src/features/library/panel/panel.ts index ef3f890..28097aa 100644 --- a/src/features/library/panel/panel.ts +++ b/src/features/library/panel/panel.ts @@ -1,10 +1,3 @@ import type { PanelFeature } from '../../../core/features'; -import { favorites } from './favorites.svelte'; -import { history } from './history.svelte'; - -/** Recent history + favorites load from storage at panel boot. */ -export const libraryFeature: PanelFeature = { - async init() { - await Promise.all([history.init(), favorites.init()]); - }, -}; +import { library } from './library.svelte'; +export const libraryFeature: PanelFeature = { init: () => library.init() }; diff --git a/src/features/library/panel/saved-settings.ts b/src/features/library/panel/saved-settings.ts index 03b02d4..4a53690 100644 --- a/src/features/library/panel/saved-settings.ts +++ b/src/features/library/panel/saved-settings.ts @@ -1,25 +1,9 @@ -import type { HistoryEntry, TrackIdentity } from '../../../core/model/types'; -import { favorites } from './favorites.svelte'; -import { history } from './history.svelte'; +import type { TrackIdentity } from '../../../core/model/types'; +import { songEntry } from '../../../core/persist/library'; +import { library } from './library.svelte'; -/** The settings saved for a song that is being *visited* — a typed URL, a link, - * an SPA navigation — or null when it has none. - * - * Deliberately looser than `isSameTrack`: at a page's first media event the - * title has usually not settled (sites rewrite document.title after the element - * fires), so demanding a title match would miss the very case this exists for. - * The URL alone is not enough either — every local file reports the local-player - * page URL and is told apart only by its title. So the title is required exactly - * when the URL is ambiguous, i.e. when it holds more than one song. */ -export function findSavedEntry(identity: TrackIdentity): HistoryEntry | null { - // Favorites first: the two lists are written together and cannot disagree, - // but with Auto Save off only the favorite is guaranteed to exist. - const pool = [...favorites.entries, ...history.entries].filter( - (e) => e.identity.normalizedUrl === identity.normalizedUrl, - ); - const titled = pool.filter((e) => e.identity.title === identity.title); - if (titled.length > 0) return titled[0]; - // Count songs, not rows — a favorited song is a row in both lists. - const oneSong = new Set(pool.map((e) => e.identity.title)).size === 1; - return oneSong ? pool[0] : null; +/** Saved settings belong to the song even when it is absent from Recent/Favorites. */ +export function findSavedEntry(identity: TrackIdentity) { + const song = library.current.shared.songs[identity.key]; + return song?.practice.value?.params ? songEntry(identity.key, library.current) : null; } diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index 9200eb9..6d4a8df 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -18,7 +18,6 @@ parseBackup, restoreBackup, } from '@/core/persist/backup'; - import { encodeBackup } from '@/core/persist/backup-codec'; import { history } from '@/features/library/panel/history.svelte'; import { applyTheme, settings } from '@/features/settings/panel/settings.svelte'; import { session } from '@/core/state/session.svelte'; @@ -139,7 +138,7 @@ const backup = await createBackup(); // The compact form — a fraction of the verbose one and the shape that // will ride the browser's sync storage; import reads both. - const text = JSON.stringify(encodeBackup(backup)); + const text = JSON.stringify(backup, null, 2); const blob = new Blob([text], { type: 'application/json' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); @@ -149,7 +148,7 @@ // The download reads the blob after click() returns, so the URL has to // outlive this task. setTimeout(() => URL.revokeObjectURL(url), 0); - const songs = backup.history.length + backup.favorites.length; + const songs = Object.values(backup.shared.songs).filter((song) => song.practice.value !== null).length; const kb = Math.max(1, Math.round(new TextEncoder().encode(text).length / 1024)); notice = { ok: true, text: `Saved ${link.download} (${songs} songs, ${kb} KB).` }; } catch (err) { @@ -171,13 +170,11 @@ // The import records what it drops (`deletions.ts`), and those // records travel — so with sync on this is not only about this // device, and the prompt has to say so. - (sync.enabled ? ' Your other synced devices lose the same songs.' : ''), + (sync.enabled ? ' The replacement of saved practice data, favorites, presets and settings also syncs.' : ''), ); if (!ok) return; - await restoreBackup(backup, { asNew: true }); - // Every store reads storage once at start-up; a reload is the honest way - // to get the whole panel — theme, open track, engine — onto new data. - location.reload(); + await restoreBackup(backup); + notice = { ok: true, text: 'Backup imported. Reopen the current song to use its imported practice settings.' }; } catch (err) { notice = { ok: false, text: `Import failed: ${message(err)}` }; } finally { @@ -501,7 +498,7 @@

{@render prefText( 'Sync between devices', - "Keep your settings, songs, presets, markers and snippets the same everywhere. Uses your browser's built-in sync: sign in to the browser with sync turned on and it reaches your other devices. No account with us, no server.", + "Sync saved practice settings, favorites, presets, markers and snippets. Recent, layout and chord analysis stay on this device. Uses your browser's built-in sync: sign in to the browser with sync turned on and it reaches your other devices. No account with us, no server.", )} sync.enabled, (on) => void setSyncEnabled(on)} @@ -509,29 +506,6 @@ />
{#if sync.enabled} - {#if sync.pendingApply} -
- {@render prefText( - 'Changes from another device are waiting', - 'They are applied when no song is loaded, or now — applying reloads the panel.', - )} - -
- {/if} - {#if sync.trimmed} -
- The browser's sync storage is full, so the oldest songs and chord charts stay - on this device only. Everything else syncs. -
- {/if}
void sync.syncNow()} - {@attach tooltip('Back up now and pull in changes from your other devices')} + {@attach tooltip('Sync saved data; your current practice session keeps playing')} > Sync now @@ -594,7 +568,7 @@ - {@render prefText('Backup file', 'Export or import all your data. Rarely needed with sync on.')} + {@render prefText('Backup file', 'Export or import everything, including local history and chord analysis.')}
diff --git a/src/features/settings/panel/settings.svelte.ts b/src/features/settings/panel/settings.svelte.ts index e48b26c..371004f 100644 --- a/src/features/settings/panel/settings.svelte.ts +++ b/src/features/settings/panel/settings.svelte.ts @@ -34,7 +34,12 @@ class SettingsStore { this.loaded = true; settingsItem.watch((value) => { if (this.#writing) return; + const before = this.current.theme; this.current = this.#withDefaults(value); + // Settings can land here without any local control having been touched — + // a backup import, or a merge from another device — and `applyTheme` is + // what actually paints . + if (this.current.theme !== before) applyTheme(this.current.theme); this.onChange?.(this.current); }); } diff --git a/src/features/sync/persist/records.test.ts b/src/features/sync/persist/records.test.ts index 0b555ec..af8da0a 100644 --- a/src/features/sync/persist/records.test.ts +++ b/src/features/sync/persist/records.test.ts @@ -7,12 +7,12 @@ import { changedRecords, readRecords, bytesUsed } from './records.ts'; const song = (id: number) => makeTrackIdentity('https://youtube.com/watch?v=song' + id, 'Song', 200); test('records round-trip independently, and an unchanged copy has nothing to upload', async () => { const library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); - const items = await changedRecords(library.shared, emptyLibrary().shared, {}); + const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); const restored = await readRecords(items); assert.equal(canonical(restored), canonical(library.shared)); - assert.deepEqual(await changedRecords(library.shared, restored, items), {}); + assert.deepEqual((await changedRecords(library.shared, restored, items)).changes, {}); const edited = applyCommand(library, { type: 'favorite', key: song(1).key, value: true }); - const changes = await changedRecords(edited.shared, restored, items); + const { changes } = await changedRecords(edited.shared, restored, items); assert.deepEqual(Object.keys(changes), ['nbn4:order', 'nbn4:song:' + song(1).key]); assert.ok(bytesUsed(items) < 8192); }); @@ -20,7 +20,7 @@ test('records round-trip independently, and an unchanged copy has nothing to upl test('partial arrival yields complete individual songs, with no global blob to assemble', async () => { let library = emptyLibrary(); for (const n of [1, 2]) library = applyCommand(library, { type: 'practice', identity: song(n), patch: {}, recent: true }); - const items = await changedRecords(library.shared, emptyLibrary().shared, {}); + const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); const key = 'nbn4:song:' + song(2).key; const partial = await readRecords({ [key]: items[key] }); assert.deepEqual(Object.keys(partial.songs), [song(2).key]); @@ -33,6 +33,16 @@ test('capacity failure leaves the library intact and never silently trims record assert.equal(canonical(library), before); }); +test('a record too large to sync is reported and left behind, never blocking the rest', async () => { + let library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); + const markers = Array.from({ length: 2000 }, (_, n) => ({ id: 'm' + n, t: n, label: 'Marker ' + n })); + library = applyCommand(library, { type: 'practice', identity: song(2), patch: { markers }, recent: true }); + const { changes, skipped } = await changedRecords(library.shared, emptyLibrary().shared, {}); + assert.deepEqual(skipped, ['nbn4:song:' + song(2).key]); + assert.ok(Object.keys(changes).includes('nbn4:song:' + song(1).key)); + assert.equal(library.shared.songs[song(2).key].practice.value!.markers.length, 2000); +}); + test('unsupported records are rejected before any application or upload', async () => { await assert.rejects(readRecords({ 'nbn4:settings': { version: 5, data: '' } }), /Unsupported/); }); diff --git a/src/features/sync/persist/records.ts b/src/features/sync/persist/records.ts index 6986504..432c5d3 100644 --- a/src/features/sync/persist/records.ts +++ b/src/features/sync/persist/records.ts @@ -54,14 +54,21 @@ export async function readRecords(items: Record): Promise; skipped: string[] } +export const skippedMessage = (skipped: string[]) => `${skipped.length} saved ${skipped.length === 1 + ? 'record is' : 'records are'} too large to sync. Everything else synced; all data is kept on this device.`; + /** Write complete independent records. There is no chunk assembly or truncation. */ -export async function changedRecords(local: SharedLibrary, remote: SharedLibrary, existing: Record) { +export async function changedRecords(local: SharedLibrary, remote: SharedLibrary, existing: Record): Promise { const previous = records(remote); const changes: Record = {}; + const skipped: string[] = []; for (const [key, value] of Object.entries(records(local))) { if (existing[key] !== undefined && canonical(value) === canonical(previous[key])) continue; const item = { version: 4, data: await compress(canonical(value)) }; - if (bytesUsed({ [key]: item }) > 8192) throw new Error('A saved record is too large to sync. All data is kept on this device; export a backup to transfer it.'); + // One outsized song must not hold back every other record for good. + if (bytesUsed({ [key]: item }) > 8192) { skipped.push(key); continue; } changes[key] = item; } const proposed = { ...existing, ...changes }; @@ -69,5 +76,5 @@ export async function changedRecords(local: SharedLibrary, remote: SharedLibrary if (bytesUsed(proposed) > QUOTA_BYTES || Object.keys(proposed).length > 512) { throw new Error('Browser sync storage is full. All data is kept on this device; export a backup to transfer it.'); } - return changes; + return { changes, skipped }; } diff --git a/store/privacy-policy-firefox.md b/store/privacy-policy-firefox.md index bef544d..9db5e4b 100644 --- a/store/privacy-policy-firefox.md +++ b/store/privacy-policy-firefox.md @@ -40,7 +40,7 @@ Firefox caps synced storage at 100 KB per extension and 8 KB per item. If a comp **Permissions and why** - `storage` — saves your markers, loops, snippets and settings on your device. -- `alarms` ? retries background sync while the panel is closed. +- `alarms` — retries background sync while the panel is closed. - `activeTab`, `scripting` — injects the audio engine into the tab when you press Connect. - `tabs` — reads the active tab's URL and title to look up the practice data you saved for that track. - Access to all sites (optional) — requested **only** when you first press Connect, never at install time, because you choose which sites to practise on. Settings → Revoke Permissions takes it back. From 9bc0d6b91280f0dc6920d122e1d4ec62cf0162b1 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 21:16:18 +0200 Subject: [PATCH 20/26] Refactor library persistence and synchronization logic - Introduced a PUSH_INTERVAL constant to manage synchronization timing. - Simplified migration logic by removing unnecessary settings defaults. - Enhanced sync configuration handling with defaults for missing fields. - Improved handling of shared library records, ensuring proper entry definitions. - Updated sync logic to track used bytes and handle quota limits more effectively. - Refactored session management to streamline media handling. - Modularized favorites and history management into separate files for better organization. - Adjusted tests to reflect changes in migration and backup handling. - Updated documentation to clarify sync behavior and data handling. --- .gitignore | 5 - CLAUDE.md | 17 +- CONTRIBUTING.md | 7 +- PUBLISHING.md | 3 +- README.md | 26 +- SECURITY.md | 2 +- e2e/library.mjs | 11 +- package.json | 1 + src/core/model/track-identity.ts | 7 +- src/core/model/types.ts | 36 +- src/core/persist/backup-codec.ts | 4 +- src/core/persist/legacy-backup.ts | 327 ++---------------- src/core/persist/library-background.ts | 36 +- src/core/persist/library-migration.ts | 10 +- src/core/persist/library.test.ts | 29 +- src/core/persist/library.ts | 22 +- src/core/persist/rekey.ts | 2 +- src/core/persist/storage.ts | 1 - src/core/state/connect.svelte.ts | 4 +- src/core/state/session.svelte.ts | 8 +- src/dev/mock.ts | 4 +- src/entrypoints/sidepanel/App.svelte | 5 +- src/features/eq/panel/eq-presets.svelte.ts | 2 +- src/features/library/panel/LibraryView.svelte | 4 +- .../{favorites.svelte.ts => favorites.ts} | 0 .../panel/{history.svelte.ts => history.ts} | 0 .../settings/panel/SettingsView.svelte | 2 +- .../settings/panel/settings.svelte.ts | 15 +- src/features/sync/panel/sync.svelte.ts | 7 +- src/features/sync/persist/records.ts | 38 +- src/features/sync/persist/sync-config.ts | 8 +- wxt.config.ts | 8 +- 32 files changed, 161 insertions(+), 490 deletions(-) rename src/features/library/panel/{favorites.svelte.ts => favorites.ts} (100%) rename src/features/library/panel/{history.svelte.ts => history.ts} (100%) diff --git a/.gitignore b/.gitignore index 88bd626..afc8fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,11 +14,6 @@ stats-*.json .wxt web-ext.config.ts -# Cloudflare Workers -.wrangler -.dev.vars -.dev.vars.* - # Editor directories and files .vscode/* !.vscode/extensions.json diff --git a/CLAUDE.md b/CLAUDE.md index de059e0..3d10c13 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,6 +35,7 @@ pnpm dlx @puppeteer/browsers install chrome@stable --path ./.browsers # once node e2e/make-tone.mjs ; node e2e/make-stereo-mix.mjs # once, generates WAV fixtures pnpm wxt build --mode testing # `testing` mode grants host perms so no native prompts block the run node e2e/run.mjs # add --headful to watch +node e2e/library.mjs # background library + sync integration (pnpm test:e2e:library) ``` The harness plays a 440 Hz tone and asserts on the **processed output** (e.g. 880 Hz after +12 st) via `window.__noteByNoteDebug` in the content script and `window.__panelDebug` in the side panel. @@ -106,17 +107,21 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel Feature persistence is wired directly in track-sync; there is no descriptor registry. - Sync stores independent gzip-compressed records in browser.storage.sync (records.ts). Versioned practice and favorite values merge independently. Explicit null/false - deletions are retained. No quota trimming, global blob or deletion expiry. - Quota failures leave local data intact. Background alarms retry independently of panels. -- Backups use the readable v4 library schema. legacy-backup.ts only reads v1/v3; - library-migration.ts collapses old copies once. v2 remains unsupported. Old local - storage is retained for recovery, but only local:library is used after migration. + deletions are retained. There is no global blob. Because the browser caps sync at + 512 items, library.ts prune() keeps a song while it is favorited, in Recent, or + among the SONG_LIMIT most recently opened, and retains only the newest + DELETION_LIMIT deletions (defaults.ts). Quota failures leave local data intact. + Background alarms retry independently of panels. +- Backups use the readable v4 library schema. legacy-backup.ts only reads v1, the + format every released build wrote; library-migration.ts collapses old copies once. + Old local storage is retained for recovery, but only local:library is used after + migration. - Web identity uses provider ID/normalized URL. Title and duration are metadata. Local files retain a filename discriminator independent of the extension URL. ## Conventions & gotchas - Path alias `@/` → `src/` (so `@/core/*`, `@/features/*`, `@/ui/*`, `@/dev/*` all resolve). WXT provides the `#imports` virtual module (`storage`, `defineBackground`, `defineContentScript`, the `browser` global) — no explicit import of `browser`. -- **`@/` does not work in two contexts** (they don't share the WXT/Vite resolver): the `node --test` files (`src/**/*.test.ts` and the modules they import as *values* — `fft.ts`, `center-cut-dsp.ts`, `detect-bpm.ts`, `backup-codec.ts`, `sync/persist/records.ts`, `core/persist/rekey.ts`, and what those pull in: `defaults.ts`, `thumbnail.ts`, `track-identity.ts`) must use **relative imports with explicit `.ts` extensions**; the esbuild worklet bundles (`src/features/*/engine/*.worklet.ts`) must use **relative imports**. (`import type` is erased, so type-only imports may omit the extension.) +- **`@/` does not work in two contexts** (they don't share the WXT/Vite resolver): the `node --test` files (`src/**/*.test.ts` and the modules they import as *values* — `fft.ts`, `center-cut-dsp.ts`, `detect-bpm.ts`, `backup-codec.ts`, `sync/persist/records.ts`, `core/persist/{library,library-migration,legacy-backup,rekey}.ts`, and what those pull in: `defaults.ts`, `thumbnail.ts`, `track-identity.ts`) must use **relative imports with explicit `.ts` extensions**; the esbuild worklet bundles (`src/features/*/engine/*.worklet.ts`) must use **relative imports**. (`import type` is erased, so type-only imports may omit the extension.) - **Both browsers build MV3** (`manifestVersion: 3` is pinned in [wxt.config.ts](wxt.config.ts) — Firefox would otherwise default to MV2 and drop `optional_host_permissions`). Chromium-only APIs are gated on the build-time flags in [core/platform.ts](src/core/platform.ts) (`CAN_CAPTURE_TAB`, `HAS_SIDE_PANEL_API`), never on runtime `browser.*` probes: Firefox has no `tabCapture`/`offscreen` (so no capture fallback — the offscreen entrypoint is excluded from that build) and no `sidePanel` (the same page is registered as `sidebar_action`). Panel-side the capability travels as a **prop**: an absent `oncapture`/`ontabaudio` is what makes the shared UI drop the affordance. - Debug globals are named `__noteByNote*` / `__panelDebug`. The processor name literal is `note-by-note-center-cut` and **must match on both sides** (`vocal-reducer.worklet.ts` registers it, `vocal-reducer.ts` constructs it) — mismatches throw `InvalidStateError` at runtime and `tsc` won't catch them. - A `MediaElementSource` can be created only once per element per document lifetime, so **extension reloads require a page reload** to reattach. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0d0a30f..8a4f175 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,10 +54,11 @@ For the browser-level e2e suite (it plays a 440 Hz tone and asserts on the - **A media element can host exactly one `MediaElementSourceNode`** for the lifetime of the document. Reloading the extension therefore means reloading the page too. -- **The e2e suite is not currently all-green** — 22 of 30. The audio path passes - end to end; the failures are in marker chips, loop and sequence bounds, the +- **The e2e suite is not currently all-green.** The audio path passes end to + end; the failures are in marker chips, loop and sequence bounds, the tab-capture CTA, and the vocal-reducer control. They pre-date any change you - are about to make; compare against a clean checkout before assuming otherwise. + are about to make; compare the tally against a clean checkout before assuming + otherwise. ## Architecture in one paragraph diff --git a/PUBLISHING.md b/PUBLISHING.md index 8f7df87..c4a1855 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -44,7 +44,8 @@ review (a few days; `` + tab capture can stretch it). Only revisit the **Privacy practices** tab if permissions or data use changed — keep it consistent with the manifest's `data_collection_permissions` -(*browsing activity*: sync uploads Recent/Favorites, which carry URLs/titles). +(*browsing activity*: sync ships saved songs and Favorites, which carry +URLs/titles; Recent stays on the device). ## 3 — Firefox Add-ons (AMO) diff --git a/README.md b/README.md index 6c89fdf..951c802 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,9 @@ the other way round. extension means reloading the page too. - `note-by-note-center-cut` is a string literal on both sides of the worklet boundary — `tsc` won't catch a mismatch, you'll get an `InvalidStateError`. -- The playback E2E suite has 47 checks. `node e2e/library.mjs` additionally - checks concurrent library edits, remote updates, restart recovery and sync capacity. +- `node e2e/run.mjs` prints a pass/fail tally for the playback suite; + `node e2e/library.mjs` (`pnpm test:e2e:library`) additionally checks concurrent + library edits, remote updates, restart recovery and sync capacity. ## Sync @@ -180,17 +181,20 @@ configuration until the song is reopened. Sync never reloads the panel. Each song is a complete, independently compressed sync item. Practice edits and favorite membership have separate revisions; preset deletion is an explicit null. Revisions advance past everything a device has observed, with deterministic ties. -There is no whole-library blob, chunk assembly, automatic trimming or timed -expiry of deletion records. If a record or library exceeds the browser's capacity, -local data remains saved and Settings reports the error. Export a backup to transfer -all data, including local history and analysis. +There is no whole-library blob and no chunk assembly. Because the browser caps +sync at 512 items, a song is kept while it is favorited, in Recent, or among the +300 most recently opened; past that it becomes a dated deletion so the removal +crosses devices, and only the newest 100 deletions are kept. If a record or +library exceeds the browser's capacity, local data remains saved and Settings +reports the error. Export a backup to transfer all data, including local history +and analysis. Backups are readable version-4 JSON containing shared and local sections. Imports -accept the previously supported v1/v3 formats and convert them once. Replacing a -backup replaces this device's library and dates the named changes for sync; it does -not delete songs known only to another device. Old local storage is retained as a -recovery copy after the first migration. Upgrade all devices before using the new -sync format; older builds cannot read it. +also accept the version-1 format every released build wrote, and convert it once. +Replacing a backup replaces this device's library and dates the named changes for +sync; it does not delete songs known only to another device. Old local storage is +retained as a recovery copy after the first migration. Upgrade all devices before +using the new sync format; older builds cannot read it. ## License diff --git a/SECURITY.md b/SECURITY.md index 548625e..1154d58 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -26,7 +26,7 @@ The extension itself: - Anything that causes the extension to grant, keep, or widen host permissions beyond what the user approved - Injection through data the extension stores and later renders — track titles, - marker labels, thumbnail URLs, restored sync snapshots + marker labels, thumbnail URLs, synced records and imported backups ## What is not a vulnerability diff --git a/e2e/library.mjs b/e2e/library.mjs index ff200fd..185afa1 100644 --- a/e2e/library.mjs +++ b/e2e/library.mjs @@ -3,17 +3,20 @@ import assert from 'node:assert/strict'; import { globSync, mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { resolve, join } from 'node:path'; +import { dirname, resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import puppeteer from 'puppeteer-core'; import { emptyLibrary, applyCommand, canonical } from '../src/core/persist/library.ts'; import { makeTrackIdentity } from '../src/core/model/track-identity.ts'; import { DEFAULT_PARAMS } from '../src/core/model/defaults.ts'; import { changedRecords } from '../src/features/sync/persist/records.ts'; -const extension = resolve('.output/chrome-mv3-testing'); -const executablePath = globSync(resolve('.browsers/chrome/*/chrome-win64/chrome.exe'))[0]; +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const extension = resolve(root, '.output', 'chrome-mv3-testing'); +const executablePath = globSync(resolve(root, '.browsers', 'chrome', '*', 'chrome-win64', 'chrome.exe'))[0]; +if (!executablePath) throw new Error('Chrome for Testing not found under .browsers/'); const profile = mkdtempSync(join(tmpdir(), 'note-by-note-library-')); -const launch = () => puppeteer.launch({ executablePath, headless: true, userDataDir: profile, +const launch = () => puppeteer.launch({ executablePath, headless: !process.argv.includes('--headful'), userDataDir: profile, args: [`--disable-extensions-except=${extension}`, `--load-extension=${extension}`, '--mute-audio'] }); let browser; async function panel() { diff --git a/package.json b/package.json index 053a6be..62c6c7c 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "check": "svelte-check --tsconfig ./tsconfig.json", "test:dsp": "node --test \"src/**/*.test.ts\"", "test:e2e": "node e2e/run.mjs", + "test:e2e:library": "node e2e/library.mjs", "release": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/release.ps1", "release:dry": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/release.ps1 -DryRun", "postinstall": "node scripts/copy-rubberband-wasm.mjs && node scripts/build-rubberband-worklet.mjs && node scripts/build-vocal-worklet.mjs && node scripts/build-pcm-tap-worklet.mjs && wxt prepare" diff --git a/src/core/model/track-identity.ts b/src/core/model/track-identity.ts index 8b4eb0f..7d75cd1 100644 --- a/src/core/model/track-identity.ts +++ b/src/core/model/track-identity.ts @@ -20,7 +20,7 @@ function normalizeUrl(rawUrl: string): string { if (host === 'youtube.com' || host.endsWith('.youtube.com')) { const v = url.searchParams.get('v') ?? /^\/(?:shorts|embed)\/([^/]+)/.exec(url.pathname)?.[1]; if (v) return `https://youtube.com/watch?v=${v}`; - // Shorts / embeds carry the id in the path. + // Any other youtube path is its own page (a channel, a playlist view). return `https://youtube.com${url.pathname}`; } if (host === 'youtu.be') { @@ -62,11 +62,6 @@ export function songKey(identity: Pick return 'web:' + hash(url); } -/** Whether two library rows describe the same song. */ -export function isSameTrack(a: TrackIdentity, b: TrackIdentity): boolean { - return a.key === b.key; -} - export function makeTrackIdentity( pageUrl: string, title: string, diff --git a/src/core/model/types.ts b/src/core/model/types.ts index 1f742d8..72aeb6c 100644 --- a/src/core/model/types.ts +++ b/src/core/model/types.ts @@ -37,13 +37,6 @@ export interface EffectParams { export interface EqPreset { name: string; gains: number[]; - /** Last save — or, with `deleted`, the removal. Absent on presets from - * before sync merged; reads as 0, so any dated copy beats them. */ - updatedAt?: number; - /** A tombstone: the preset was deleted at `updatedAt`, and the row is kept - * so a sync merge can tell "removed" from "never had it" (legacy backups). - * `gains` is emptied — nothing reads them again. */ - deleted?: true; } export interface Marker { @@ -106,7 +99,9 @@ export interface ChordChart { /** Stable identity of a piece of media, so settings/markers/snippets survive * reloads and URL noise. */ export interface TrackIdentity { - /** `${hash(normalizedUrl)}:${round(duration)}` */ + /** The saved song's key — see `songKey` in core/model/track-identity.ts. + * Derived from the provider id or normalized URL, never from the duration, + * which drifts (pre-roll ads, metadata that settles late). */ key: string; normalizedUrl: string; title: string; @@ -136,29 +131,19 @@ export interface HistoryEntry { params: EffectParams; thumbnailUrl?: string; pageUrl: string; - createdAt: number; - /** Last save — or, with `deleted`, the removal. The one date a merge reads - * for this row (legacy backups). */ + /** Last save — the date Recent sorts and displays by. */ updatedAt: number; - /** A tombstone: the row was removed at `updatedAt`, and is kept so a sync - * merge can tell "removed" from "never had it" (legacy backups). The panel - * stores filter these out, so nothing downstream ever sees one. */ - deleted?: true; } /** A song the user starred (History → Favorites). Persists independently of * the LRU-capped Recent list. Stored array order = manual sort order. */ export interface FavoriteEntry extends HistoryEntry { - /** When the star was put there. Display only — the star and the unstar are - * the only writers of `updatedAt`, which is what the merge reads, so - * ordinary practice can no longer outdate another device's unfavorite. */ + /** When the star was put there — the favorite's own revision date, kept + * apart from `updatedAt` so ordinary practice cannot outdate an unfavorite + * made on another device. */ favoritedAt: number; /** Last time the track was opened or played, for "Last Accessed" sorting. */ lastAccessedAt: number; - /** When the manual order this row sits in was last set — by a drag - * (`setFavoritesOrder`) or by the star that put it on top. Decides whose - * order a merge keeps; absent on rows written before manual order synced. */ - orderedAt?: number; } export type FavoritesSort = 'lastAccessed' | 'title' | 'manual'; @@ -288,11 +273,6 @@ export interface Settings { /** Play an audible click on each count-in beat (accented downbeat). */ countInBeep: boolean; lastUsedParams?: EffectParams; - /** Last change, set by the settings store on every write. Settings travel - * between devices as one item with one date (see `merge.ts`); absent on - * settings written before that, which reads as 0. Never part of the file's - * settings diff — the codec carries it separately. */ - updatedAt?: number; } export type PanelId = @@ -320,6 +300,4 @@ export interface UiPrefs { accentHue: number; /** User overrides for the virtual Start/End marker labels (empty = default). */ boundaryLabels: { start: string; end: string }; - /** Last change — see `Settings.updatedAt`. */ - updatedAt?: number; } diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index d074144..c7a80ad 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -39,7 +39,7 @@ function defaults(value: unknown, fallback: T): T { else if (Array.isArray(expected)) array(next).forEach(number); else if (typeof expected === 'object') { result[key] = defaults(next, expected); continue; } else if (typeof next !== typeof expected) throw new Error('Damaged library setting.'); - if (typeof next === 'number') number(next); + else if (typeof next === 'number') number(next); result[key] = next; } return result as T; @@ -106,5 +106,5 @@ export function parseBackupJson(value: unknown): Backup { if (raw.format !== BACKUP_FORMAT) throw new Error("That file isn't a Note by Note backup."); if (raw.version > BACKUP_VERSION) throw new Error('That backup was made by a newer version of Note by Note.'); const library = raw.version === BACKUP_VERSION ? parseLibrary(raw) : migrateBackup(parseLegacy(raw)); - return { format: BACKUP_FORMAT, version: BACKUP_VERSION, exportedAt: raw.exportedAt ?? raw.at ?? 0, ...library }; + return { format: BACKUP_FORMAT, version: BACKUP_VERSION, exportedAt: raw.exportedAt ?? 0, ...library }; } diff --git a/src/core/persist/legacy-backup.ts b/src/core/persist/legacy-backup.ts index c651b63..fdd37eb 100644 --- a/src/core/persist/legacy-backup.ts +++ b/src/core/persist/legacy-backup.ts @@ -1,48 +1,45 @@ -// Read-only adapter for previously supported backup formats. New writes use the library schema. -import { - DEFAULT_PARAMS, - DEFAULT_SETTINGS, - DEFAULT_UI_PREFS, -} from '../model/defaults.ts'; - -import { youtubeThumbnailUrl } from '../model/thumbnail.ts'; - -import { songKey } from '../model/track-identity.ts'; +// Read-only adapter for the version-1 backup format, the only one any released +// build ever wrote. New writes use the library schema (backup-codec.ts). +import { DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; import { rekeyByIdentity } from './rekey.ts'; import type { - ChordChart, - ChordSegment, - EffectParams, EqPreset, FavoriteEntry, HistoryEntry, - Marker, Settings, - Snippet, - SnippetOverrides, TrackData, - TrackIdentity, UiPrefs, } from '../model/types'; -export const BACKUP_FORMAT = 'note-by-note-backup'; +const BACKUP_FORMAT = 'note-by-note-backup'; -export const BACKUP_VERSION = 1; +const BACKUP_VERSION = 1; -export const COMPACT_VERSION = 3; +/** Rows in a v1 file carry sync bookkeeping the live model no longer has: a + * tombstone told an old merge "removed" from "never had it", and every row and + * section carried its own date. `library-migration.ts` reads them once, here. */ +export interface LegacyHistoryEntry extends HistoryEntry { + deleted?: true; +} +export interface LegacyFavoriteEntry extends FavoriteEntry, LegacyHistoryEntry { + /** When the manual order this row sat in was last set. */ + orderedAt?: number; +} +export interface LegacyEqPreset extends EqPreset { + updatedAt?: number; + deleted?: true; +} export interface Backup { format: typeof BACKUP_FORMAT; version: number; - exportedAt: number; - appVersion: string; - settings: Settings; + settings: Settings & { updatedAt?: number }; uiPrefs: UiPrefs; - history: HistoryEntry[]; - favorites: FavoriteEntry[]; - eqPresets: EqPreset[]; + history: LegacyHistoryEntry[]; + favorites: LegacyFavoriteEntry[]; + eqPresets: LegacyEqPreset[]; /** Per-track markers and snippets, one entry per saved track. */ tracks: TrackData[]; } @@ -55,26 +52,11 @@ function damaged(section: string): Error { return new Error(`This backup's "${section}" list is damaged.`); } -function num(value: unknown, section: string): number { - if (typeof value !== 'number' || !Number.isFinite(value)) throw damaged(section); - return value; -} - -function str(value: unknown, section: string): string { - if (typeof value !== 'string') throw damaged(section); - return value; -} - function arr(value: unknown, section: string): unknown[] { if (!Array.isArray(value)) throw damaged(section); return value; } -function rec(value: unknown, section: string): Record { - if (!isRecord(value)) throw damaged(section); - return value; -} - function identifiedArr(value: unknown, section: string): T[] { const list = arr(value, section); const identified = list.every( @@ -84,268 +66,21 @@ function identifiedArr(value: unknown, section: string): T[] { return list as T[]; } -function mergePlain( - defaults: Record, - diff: Record, -): Record { - const out: Record = JSON.parse(JSON.stringify(defaults)); - for (const [key, v] of Object.entries(diff)) { - if (v === undefined) continue; - const d = out[key]; - out[key] = isRecord(v) && isRecord(d) ? mergePlain(d, v) : v; - } - return out; -} - -export function decodeParams(raw: unknown, section: string): EffectParams { - const p = structuredClone(DEFAULT_PARAMS); - if (raw === undefined) return p; - const c = rec(raw, section); - if (c.t !== undefined) p.transpose = num(c.t, section); - if (c.te !== undefined) p.transposeEnabled = false; - if (c.c !== undefined) p.pitchCents = num(c.c, section); - if (c.ce !== undefined) p.pitchEnabled = false; - if (c.s !== undefined) p.speed = num(c.s, section); - if (c.se !== undefined) p.speedEnabled = false; - if (c.v !== undefined) p.vocalReduce = num(c.v, section); - if (c.ve !== undefined) p.vocalReduceEnabled = false; - if (c.vm !== undefined) p.vocalMode = 'isolate'; - if (c.e !== undefined) { - const e = arr(c.e, section); - if (e.length !== 1 + p.eq.gains.length) throw damaged(section); - p.eq = { enabled: num(e[0], section) === 1, gains: e.slice(1).map((g) => num(g, section)) }; - } - if (c.tu !== undefined) { - const tu = arr(c.tu, section); - if (tu.length !== 2) throw damaged(section); - p.tuning = { trackHz: num(tu[0], section), instrumentHz: num(tu[1], section) }; - } - if (c.pw !== undefined) p.power = false; - if (c.b !== undefined) p.baseBpm = num(c.b, section); - return p; -} - -export function decodeSettings(raw: unknown): Settings { - const diff = raw === undefined ? {} : rec(raw, 'settings'); - const { lp, ...rest } = diff; - const settings = mergePlain({ ...DEFAULT_SETTINGS }, rest) as unknown as Settings; - if (lp !== undefined) settings.lastUsedParams = decodeParams(lp, 'settings'); - return settings; -} - -export function decodeUiPrefs(raw: unknown): UiPrefs { - const diff = raw === undefined ? {} : rec(raw, 'uiPrefs'); - return mergePlain( - DEFAULT_UI_PREFS as unknown as Record, - diff, - ) as unknown as UiPrefs; -} - -const YT_WATCH = 'https://youtube.com/watch?v='; - -const YT_ID_RE = /^[\w-]+$/; - -function longUrl(short: string): string { - if (short.startsWith('yt:')) { - const id = short.slice(3); - if (!YT_ID_RE.test(id)) throw damaged('songs'); - return YT_WATCH + id; - } - return short; -} - -function defaultPageUrl(normalizedUrl: string): string { - if (normalizedUrl.startsWith(YT_WATCH)) { - return `https://www.youtube.com/watch?v=${normalizedUrl.slice(YT_WATCH.length)}`; - } - return normalizedUrl; -} - -function decodeSongs(raw: unknown): TrackIdentity[] { - return arr(raw, 'songs').map((row) => { - const r = arr(row, 'songs'); - if (r.length !== 3) throw damaged('songs'); - const normalizedUrl = longUrl(str(r[0], 'songs')); - const title = str(r[1], 'songs'); - const durationSec = num(r[2], 'songs'); - return { key: songKey({ normalizedUrl, title }), normalizedUrl, title, durationSec }; - }); -} - -function songAt(songs: TrackIdentity[], index: unknown, section: string): TrackIdentity { - if (typeof index !== 'number' || !Number.isInteger(index) || index < 0 || index >= songs.length) { - throw damaged(section); - } - return songs[index]; -} - -function decodeEntry(raw: unknown, songs: TrackIdentity[], section: string): HistoryEntry { - const c = rec(raw, section); - const identity = songAt(songs, c.i, section); - const updatedAt = num(c.at, section); - const pageUrl = c.url === undefined ? defaultPageUrl(identity.normalizedUrl) : str(c.url, section); - const thumbnailUrl = c.th === undefined ? youtubeThumbnailUrl(pageUrl) : str(c.th, section); - const entry: HistoryEntry = { - identity: { ...identity }, - params: decodeParams(c.p, section), - pageUrl, - createdAt: updatedAt, - updatedAt, - }; - if (thumbnailUrl !== undefined) entry.thumbnailUrl = thumbnailUrl; - if (c.x === 1) entry.deleted = true; - return entry; -} - -function decodeFavorite(raw: unknown, songs: TrackIdentity[]): FavoriteEntry { - const c = rec(raw, 'favorites'); - const favorite: FavoriteEntry = { - ...decodeEntry(c, songs, 'favorites'), - favoritedAt: num(c.fa, 'favorites'), - lastAccessedAt: num(c.la, 'favorites'), - }; - if (c.oa !== undefined) favorite.orderedAt = num(c.oa, 'favorites'); - return favorite; -} - -function decodeMarker(raw: unknown, index: number): Marker { - const r = arr(raw, 'tracks'); - if (r.length < 1 || r.length > 2) throw damaged('tracks'); - return { - id: `m${index + 1}`, - t: num(r[0], 'tracks') / 1000, - label: r.length === 2 ? str(r[1], 'tracks') : '', - }; -} - -function decodeOverrides(raw: unknown): SnippetOverrides { - const c = rec(raw, 'tracks'); - const out: SnippetOverrides = {}; - if (c.s !== undefined) out.speed = num(c.s, 'tracks'); - if (c.t !== undefined) out.transpose = num(c.t, 'tracks'); - if (c.v !== undefined) out.vocalReduce = num(c.v, 'tracks'); - return out; -} - -function decodeSnippet(raw: unknown, index: number): Snippet { - const r = arr(raw, 'tracks'); - if (r.length < 3 || r.length > 6) throw damaged('tracks'); - const repeats = r.length > 3 ? num(r[3], 'tracks') : 1; - return { - id: `c${index + 1}`, - name: str(r[0], 'tracks'), - startT: num(r[1], 'tracks') / 1000, - endT: num(r[2], 'tracks') / 1000, - enabled: r.length > 4 ? num(r[4], 'tracks') === 1 : true, - repeats: repeats === 0 ? Infinity : repeats, - overrides: r.length > 5 ? decodeOverrides(r[5]) : {}, - }; -} - -export function decodeChart(raw: unknown): ChordChart { - const c = rec(raw, 'tracks'); - const d = arr(c.d, 'tracks'); - const l = arr(c.l, 'tracks').map((label) => str(label, 'tracks')); - const i = arr(c.i, 'tracks'); - const g = c.g === undefined ? undefined : arr(c.g, 'tracks'); - if (i.length !== d.length || (g !== undefined && g.length !== d.length)) throw damaged('tracks'); - const segments: ChordSegment[] = []; - let acc = num(c.t0, 'tracks'); - for (let n = 0; n < d.length; n++) { - const li = num(i[n], 'tracks'); - if (!Number.isInteger(li) || li < 0 || li >= l.length) throw damaged('tracks'); - const start = acc + (g === undefined ? 0 : num(g[n], 'tracks')); - const end = start + num(d[n], 'tracks'); - segments.push({ startT: start / 100, endT: end / 100, label: l[li], confidence: 1 }); - acc = end; - } - let key: ChordChart['key'] = null; - if (c.k !== undefined) { - const k = arr(c.k, 'tracks'); - if (k.length !== 3) throw damaged('tracks'); - key = { - tonic: str(k[0], 'tracks'), - mode: num(k[1], 'tracks') === 1 ? 'minor' : 'major', - confidence: num(k[2], 'tracks'), - }; - } - return { - segments, - key, - coverage: num(c.cov, 'tracks'), - analyzedFrom: num(c.a0, 'tracks') / 100, - analyzedTo: num(c.a1, 'tracks') / 100, - computedAt: num(c.c, 'tracks'), - }; -} - -function decodeTrack(raw: unknown, songs: TrackIdentity[]): TrackData { - const c = rec(raw, 'tracks'); - const track: TrackData = { - identity: { ...songAt(songs, c.i, 'tracks') }, - markers: c.m === undefined ? [] : arr(c.m, 'tracks').map(decodeMarker), - snippets: c.s === undefined ? [] : arr(c.s, 'tracks').map(decodeSnippet), - sequenceLoop: c.L !== undefined, - sequenceCountIn: c.C !== undefined, - chordChart: c.ch === undefined ? null : decodeChart(c.ch), - updatedAt: num(c.at, 'tracks'), - }; - if (c.ce !== undefined) track.chordsEnabled = num(c.ce, 'tracks') === 1; - return track; -} - -function decodeEqPreset(raw: unknown): EqPreset { - const r = arr(raw, 'eqPresets'); - if (r.length < 2 || r.length > 4) throw damaged('eqPresets'); - const preset: EqPreset = { - name: str(r[0], 'eqPresets'), - gains: arr(r[1], 'eqPresets').map((g) => num(g, 'eqPresets')), - }; - if (r.length >= 3) preset.updatedAt = num(r[2], 'eqPresets'); - if (r.length === 4) preset.deleted = true; - return preset; -} - -export function decodeBackup(raw: unknown): Backup { - if (!isRecord(raw) || raw.format !== BACKUP_FORMAT || raw.version !== COMPACT_VERSION) { - throw new Error("That file isn't a Note by Note backup."); - } - const songs = decodeSongs(raw.songs); - const settings = decodeSettings(raw.s); - const uiPrefs = decodeUiPrefs(raw.u); - if (typeof raw.sat === 'number' && Number.isFinite(raw.sat)) settings.updatedAt = raw.sat; - if (typeof raw.uat === 'number' && Number.isFinite(raw.uat)) uiPrefs.updatedAt = raw.uat; - return { - format: BACKUP_FORMAT, - version: COMPACT_VERSION, - exportedAt: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at : 0, - appVersion: '', - settings, - uiPrefs, - history: arr(raw.h, 'history').map((e) => decodeEntry(e, songs, 'history')), - favorites: arr(raw.f, 'favorites').map((e) => decodeFavorite(e, songs)), - eqPresets: arr(raw.eq, 'eqPresets').map(decodeEqPreset), - tracks: arr(raw.t, 'tracks').map((t) => decodeTrack(t, songs)), - }; -} - function normalizeV1(raw: Record): Backup { return { format: BACKUP_FORMAT, version: BACKUP_VERSION, - exportedAt: typeof raw.exportedAt === 'number' ? raw.exportedAt : 0, - appVersion: typeof raw.appVersion === 'string' ? raw.appVersion : '', settings: { ...DEFAULT_SETTINGS, ...(isRecord(raw.settings) ? raw.settings : {}), - } as Settings, + } as Settings & { updatedAt?: number }, uiPrefs: { ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), }, - history: rekeyByIdentity(identifiedArr(raw.history, 'history')), - favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), - eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], + history: rekeyByIdentity(identifiedArr(raw.history, 'history')), + favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), + eqPresets: arr(raw.eqPresets, 'eqPresets') as LegacyEqPreset[], tracks: rekeyByIdentity(identifiedArr(raw.tracks, 'tracks')), }; } @@ -354,12 +89,6 @@ export function parseBackupJson(raw: unknown): Backup { if (!isRecord(raw) || raw.format !== BACKUP_FORMAT) { throw new Error("That file isn't a Note by Note backup."); } - const version = typeof raw.version === 'number' ? raw.version : 0; - if (version > COMPACT_VERSION) { - throw new Error('That backup was made by a newer version of Note by Note.'); - } - if (version === COMPACT_VERSION) return decodeBackup(raw); - if (version === BACKUP_VERSION) return normalizeV1(raw); - // Only 2 lands here, and only from the branch this format grew on. + if (raw.version === BACKUP_VERSION) return normalizeV1(raw); throw new Error('That backup is in a format this version of Note by Note no longer reads.'); } diff --git a/src/core/persist/library-background.ts b/src/core/persist/library-background.ts index 630f353..a837463 100644 --- a/src/core/persist/library-background.ts +++ b/src/core/persist/library-background.ts @@ -4,11 +4,14 @@ import { libraryItem } from './library-client'; import { parseBackupJson as parseLegacy } from './legacy-backup'; import { parseLibrary } from './backup-codec'; import { migrateBackup } from './library-migration'; -import { bytesUsed, changedRecords, legacyKeys, PREFIX, readRecords, skippedMessage } from '../../features/sync/persist/records'; -import { loadSyncConfig, syncConfigItem, type SyncConfig } from '../../features/sync/persist/sync-config'; +import { bytesUsed, changedRecords, PREFIX, readRecords, skippedMessage } from '../../features/sync/persist/records'; +import { loadSyncConfig, syncConfigItem } from '../../features/sync/persist/sync-config'; const WAKE = 'library-sync'; const SAFETY = 'library-sync-safety'; +/** Chromium allows ~2 writes/second sustained; one push per 30 s is well under + * it and still lets a burst of edits ride out together. */ +const PUSH_INTERVAL = 30000; /** One writer for edits, imports and merges. Persisted records survive worker restarts. */ export function startLibraryBackground() { @@ -29,7 +32,7 @@ export function startLibraryBackground() { typeof (row as { identity?: { normalizedUrl?: unknown } })?.identity?.normalizedUrl === 'string'); try { return migrateBackup(parseLegacy({ format: 'note-by-note-backup', version: 1, - settings: raw.settings ?? defaults.shared.settings.value, uiPrefs: raw.uiPrefs ?? defaults.local.uiPrefs, + settings: raw.settings, uiPrefs: raw.uiPrefs, history: identified(raw.history), favorites: identified(raw.favorites), eqPresets: Array.isArray(raw.eqPresets) ? raw.eqPresets : [], tracks: identified(Object.entries(raw).filter(([key]) => key.startsWith('track:')).map(([, value]) => value)), @@ -45,16 +48,15 @@ export function startLibraryBackground() { if ((await browser.storage.local.get('library')).library) return; await libraryItem.setValue(migrate(await browser.storage.local.get(null))); })().catch((error) => { ready = undefined; throw error; }); - const saveConfig = (config: SyncConfig) => syncConfigItem.setValue(config); const schedule = async () => { const config = await loadSyncConfig(); - if (config.enabled) await browser.alarms.create(WAKE, { when: Math.max(Date.now() + 5000, config.lastPushAt + 30000) }); + if (config.enabled) await browser.alarms.create(WAKE, { when: Math.max(Date.now() + 5000, config.lastPushAt + PUSH_INTERVAL) }); }; const reconcile = async () => { await init(); const config = await loadSyncConfig(); if (!config.enabled) return; - await saveConfig({ ...config, syncing: true }); + await syncConfigItem.setValue({ ...config, syncing: true }); try { const existing = await browser.storage.sync.get(null); config.usedBytes = bytesUsed(existing); @@ -62,24 +64,19 @@ export function startLibraryBackground() { const local = await libraryItem.getValue(); const shared = mergeShared(local.shared, remote); if (canonical(shared) !== canonical(local.shared)) await libraryItem.setValue({ ...local, shared }); - const { changes, skipped } = await changedRecords(shared, remote, existing); + const { changes, skipped, usedBytes } = await changedRecords(shared, remote, existing); if (Object.keys(changes).length) { - if (Date.now() < config.lastPushAt + 30000) { await schedule(); return; } - // Legacy bytes may occupy the quota. Their contents are durable locally before removal. - const oldKeys = legacyKeys(existing); - if (oldKeys.length) await browser.storage.sync.remove(oldKeys); + if (Date.now() < config.lastPushAt + PUSH_INTERVAL) { await schedule(); return; } await browser.storage.sync.set(changes); config.lastPushAt = Date.now(); - const final = { ...existing, ...changes }; - for (const key of oldKeys) delete final[key]; - config.usedBytes = bytesUsed(final); + config.usedBytes = usedBytes; } config.lastSyncedAt = Date.now(); config.lastError = skipped.length ? skippedMessage(skipped) : null; } catch (error) { config.lastError = error instanceof Error ? error.message : String(error); } finally { - await saveConfig({ ...config, syncing: false }); + await syncConfigItem.setValue({ ...config, syncing: false }); } }; onMessage('libraryRead', () => enqueue(async () => { await init(); return libraryItem.getValue(); })); @@ -96,26 +93,25 @@ export function startLibraryBackground() { if (data === 'disable' || data === 'delete') { // Disabled before anything is removed, so a worker restart part-way // through the delete cannot wake up and upload the library again. - await saveConfig(data === 'delete' + await syncConfigItem.setValue(data === 'delete' ? { ...config, enabled: false, syncing: false, lastSyncedAt: 0, usedBytes: 0, lastError: null } : { ...config, enabled: false, syncing: false }); await browser.alarms.clear(WAKE); if (data === 'delete') { const items = await browser.storage.sync.get(null); - const old = new Set(legacyKeys(items)); - const keys = Object.keys(items).filter((key) => key.startsWith(PREFIX) || old.has(key)); + const keys = Object.keys(items).filter((key) => key.startsWith(PREFIX)); if (keys.length) await browser.storage.sync.remove(keys); } return; } - if (data === 'enable') await saveConfig({ ...config, enabled: true }); + if (data === 'enable') await syncConfigItem.setValue({ ...config, enabled: true }); await reconcile(); })); browser.alarms.onAlarm.addListener((alarm) => { if (alarm.name === WAKE || alarm.name === SAFETY) void enqueue(reconcile); }); browser.storage.onChanged.addListener((changes, area) => { - if (area === 'sync' && Object.keys(changes).some((key) => key.startsWith(PREFIX) || key.startsWith('nbn.'))) { + if (area === 'sync' && Object.keys(changes).some((key) => key.startsWith(PREFIX))) { void enqueue(reconcile); } }); diff --git a/src/core/persist/library-migration.ts b/src/core/persist/library-migration.ts index a1a7c4a..29fa0ac 100644 --- a/src/core/persist/library-migration.ts +++ b/src/core/persist/library-migration.ts @@ -1,6 +1,6 @@ import { DEFAULT_PARAMS } from '../model/defaults.ts'; import { makeTrackIdentity } from '../model/track-identity.ts'; -import { cell, emptyLibrary, newest, type Library, type Practice, type SavedSong } from './library.ts'; +import { cell, defineEntry, emptyLibrary, newest, type Library, type Practice, type SavedSong } from './library.ts'; import type { Backup } from './legacy-backup.ts'; /** Collapse old Recent/Favorites/track copies once, at the boundary. */ @@ -46,10 +46,10 @@ export function migrateBackup(backup: Backup): Library { local.lastAccessed[key] = Math.max(local.lastAccessed[key] ?? 0, entry.lastAccessedAt); } shared.favoriteOrder = cell(backup.favorites.filter((f) => !f.deleted) - .map((f) => makeTrackIdentity(f.identity.normalizedUrl, f.identity.title, f.identity.durationSec).key), + .map((f) => ensure(f.identity).practice.value!.identity.key), Math.max(0, ...backup.favorites.map((f) => f.orderedAt ?? f.favoritedAt ?? 0))); - for (const preset of backup.eqPresets) Object.defineProperty(shared.presets, preset.name, { - value: cell(preset.deleted ? null : preset.gains, preset.updatedAt ?? 0), enumerable: true, writable: true, configurable: true, - }); + for (const preset of backup.eqPresets) { + defineEntry(shared.presets, preset.name, cell(preset.deleted ? null : preset.gains, preset.updatedAt ?? 0)); + } return library; } diff --git a/src/core/persist/library.test.ts b/src/core/persist/library.test.ts index 15e44c3..b82edf1 100644 --- a/src/core/persist/library.test.ts +++ b/src/core/persist/library.test.ts @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { applyCommand, canonical, cell, emptyLibrary, favoriteEntries, mergeShared, nextRevision, recentEntries } from './library.ts'; +import { applyCommand, canonical, emptyLibrary, favoriteEntries, mergeShared, nextRevision, recentEntries } from './library.ts'; import { migrateBackup } from './library-migration.ts'; import { parseBackupJson } from './backup-codec.ts'; import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, DELETION_LIMIT, SONG_LIMIT } from '../model/defaults.ts'; @@ -102,8 +102,8 @@ test('replacement import dates both present and absent records after observed fu }); test('migration collapses parameters, favorites and markers without syncing local data', () => { - const entry = { identity, pageUrl: identity.normalizedUrl, params: { ...DEFAULT_PARAMS, speed: 0.7 }, createdAt: 1, updatedAt: 10 }; - const migrated = migrateBackup({ format: 'note-by-note-backup', version: 1, exportedAt: 20, appVersion: '', + const entry = { identity, pageUrl: identity.normalizedUrl, params: { ...DEFAULT_PARAMS, speed: 0.7 }, updatedAt: 10 }; + const migrated = migrateBackup({ format: 'note-by-note-backup', version: 1, settings: { ...DEFAULT_SETTINGS, lastUsedParams: DEFAULT_PARAMS }, uiPrefs: DEFAULT_UI_PREFS, eqPresets: [], history: [entry], favorites: [{ ...entry, params: DEFAULT_PARAMS, updatedAt: 5, favoritedAt: 15, lastAccessedAt: 16 }], tracks: [{ identity, updatedAt: 11, markers: [{ id: 'm', t: 42, label: '' }], snippets: [], sequenceLoop: false, sequenceCountIn: false, chordChart: null }], @@ -120,7 +120,7 @@ test('new backups round-trip complete data and reject malformed or unsupported f const backup = { format: 'note-by-note-backup', version: 4, exportedAt: 100, ...save() }; assert.deepEqual(parseBackupJson(JSON.parse(JSON.stringify(backup))), backup); assert.throws(() => parseBackupJson({ ...backup, version: 5 }), /newer version/); - assert.throws(() => parseBackupJson({ ...backup, version: 2 }), /no longer reads/); + assert.throws(() => parseBackupJson({ ...backup, version: 3 }), /no longer reads/); const damaged = structuredClone(backup); damaged.shared.songs[identity.key].practice.value!.identity.key = 'old-key'; assert.equal(parseBackupJson(damaged).shared.songs[identity.key].practice.value!.identity.key, identity.key); @@ -128,27 +128,6 @@ test('new backups round-trip complete data and reject malformed or unsupported f assert.throws(() => parseBackupJson(damaged), /identity/); }); -test('compact v3 imports preserve tuning, markers, snippets, local chords and favorite order', () => { - const backup = parseBackupJson({ format: 'note-by-note-backup', version: 3, at: 1000, - s: { lp: { s: 0.75 } }, u: { markerView: 'list' }, eq: [['Bass', [1, 2], 5]], - songs: [['yt:example', 'Song', 200]], - h: [{ i: 0, at: 10, p: { s: 0.86, c: -2, tu: [442, 440] } }], - f: [{ i: 0, at: 10, fa: 12, la: 15, p: { s: 0.86, c: -2, tu: [442, 440] } }], - t: [{ i: 0, at: 11, m: [[3753, 'Verse']], s: [['Loop', 1000, 4000, 0]], ce: 1, - ch: { t0: 100, d: [200], l: ['C'], i: [0], cov: 1, a0: 100, a1: 300, c: 9 } }], - }); - const practice = backup.shared.songs['yt:example'].practice.value!; - assert.equal(practice.params!.speed, 0.86); - assert.deepEqual(practice.params!.tuning, { trackHz: 442, instrumentHz: 440 }); - assert.equal(practice.markers[0].t, 3.753); - assert.equal(practice.snippets[0].repeats, Infinity); - assert.deepEqual(backup.shared.favoriteOrder.value, ['yt:example']); - assert.equal(backup.local.charts['yt:example']!.segments[0].label, 'C'); - assert.equal(backup.local.lastUsedParams!.speed, 0.75); - assert.equal(canonical(backup.shared).includes('chordChart'), false); - assert.equal(canonical(parseBackupJson(JSON.parse(JSON.stringify(backup)))), canonical(backup)); -}); - test('preset names are data, including names matching object properties', () => { let library = emptyLibrary(); for (const name of ['constructor', '__proto__']) library = applyCommand(library, { type: 'preset', name, gains: [1] }); diff --git a/src/core/persist/library.ts b/src/core/persist/library.ts index 74d3195..dddf55a 100644 --- a/src/core/persist/library.ts +++ b/src/core/persist/library.ts @@ -2,7 +2,7 @@ import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, DELETION_LIMIT, HIS import type { ChordChart, EffectParams, FavoriteEntry, HistoryEntry, Settings, TrackData, TrackIdentity, UiPrefs } from '../model/types'; /** One revision per independently editable value. Null/false are durable deletions. */ -export interface Versioned { at: number; value: T } +interface Versioned { at: number; value: T } export interface Practice extends Omit { params?: EffectParams; pageUrl: string; @@ -30,6 +30,12 @@ export interface Library { } export const cell = (value: T, at = 0): Versioned => ({ at, value }); +/** Song keys and preset names are user data, so they may spell an object + * property (`__proto__`, `constructor`). Defining the entry writes the map the + * plain assignment would only appear to. */ +export function defineEntry(target: Record, key: string, value: T): void { + Object.defineProperty(target, key, { value, enumerable: true, writable: true, configurable: true }); +} export function emptyLibrary(): Library { return { shared: { settings: cell(structuredClone(DEFAULT_SETTINGS)), songs: {}, presets: {}, favoriteOrder: cell([]) }, @@ -164,7 +170,7 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da } case 'chart': local.charts[command.key] = command.chart; break; case 'settings': { - const { lastUsedParams, updatedAt: ignored, ...patch } = command.patch; + const { lastUsedParams, ...patch } = command.patch; if (lastUsedParams) local.lastUsedParams = lastUsedParams; if (command.reset || Object.keys(patch).length) { const value = { ...(command.reset ? structuredClone(DEFAULT_SETTINGS) : shared.settings.value), ...patch }; @@ -176,9 +182,7 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da break; } case 'uiPrefs': local.uiPrefs = command.value; break; - case 'preset': Object.defineProperty(shared.presets, command.name, { - value: cell(command.gains, at), enumerable: true, writable: true, configurable: true, - }); break; + case 'preset': defineEntry(shared.presets, command.name, cell(command.gains, at)); break; case 'import': { const file = structuredClone(command.library); const revision = Math.max(at, nextRevision(file.shared, now)); @@ -188,8 +192,8 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da shared.songs[key] = { practice: cell(song?.practice.value ?? null, revision), favorite: cell(song?.favorite.value ?? false, revision) }; } for (const name of new Set([...Object.keys(shared.presets), ...Object.keys(file.shared.presets)])) { - Object.defineProperty(shared.presets, name, { value: cell(Object.hasOwn(file.shared.presets, name) ? file.shared.presets[name].value : null, revision), - enumerable: true, writable: true, configurable: true }); + defineEntry(shared.presets, name, + cell(Object.hasOwn(file.shared.presets, name) ? file.shared.presets[name].value : null, revision)); } shared.settings = cell(file.shared.settings.value, revision); shared.favoriteOrder = cell(file.shared.favoriteOrder.value, revision); @@ -202,14 +206,14 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da } /** UI rows are projections. They are never written back as library copies. */ -export function songEntry(key: string, library: Library): HistoryEntry | null { +function songEntry(key: string, library: Library): HistoryEntry | null { const song = library.shared.songs[key]; const practice = song?.practice.value; if (!practice) return null; return { identity: practice.identity, pageUrl: practice.pageUrl, thumbnailUrl: practice.thumbnailUrl, params: practice.params ?? structuredClone(DEFAULT_PARAMS), - createdAt: song.practice.at, updatedAt: library.local.recent[key] ?? song.practice.at, + updatedAt: library.local.recent[key] ?? song.practice.at, }; } export function recentEntries(library: Library): HistoryEntry[] { diff --git a/src/core/persist/rekey.ts b/src/core/persist/rekey.ts index e0de1ca..b273aac 100644 --- a/src/core/persist/rekey.ts +++ b/src/core/persist/rekey.ts @@ -2,7 +2,7 @@ import { songKey } from '../model/track-identity.ts'; import type { TrackIdentity } from '../model/types'; /** - * Re-deriving stored keys, for the migration in `migrate.ts`. + * Re-deriving stored keys, for the one-time migration in `library-migration.ts`. * * Pure and DOM-free (relative `.ts` imports; runs under `node --test`) because * it is the one step of that migration that can lose data: rows a key change diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index 2cda38d..d000bf4 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -8,7 +8,6 @@ const settingsOf = (library: Awaited>): Settings }); export const settingsItem = { getValue: async () => settingsOf(await readLibrary()), - setValue: (value: Settings) => editLibrary({ type: 'settings', patch: value }), watch: (listener: (value: Settings) => void) => watchLibrary(settingsOf, listener), }; export const uiPrefsItem = { diff --git a/src/core/state/connect.svelte.ts b/src/core/state/connect.svelte.ts index 44ca6f3..0f5d3a6 100644 --- a/src/core/state/connect.svelte.ts +++ b/src/core/state/connect.svelte.ts @@ -102,7 +102,7 @@ class ConnectionManager { if (isRestricted(tab.url) && !isLocalPlayer(tab.url)) { session.connection = 'restricted'; - session.setMedia(null); + session.media = null; return; } @@ -120,7 +120,7 @@ class ConnectionManager { if (!granted) { this.needsPermission = pattern; session.connection = 'idle'; - session.setMedia(null); + session.media = null; return; } this.needsPermission = null; diff --git a/src/core/state/session.svelte.ts b/src/core/state/session.svelte.ts index aaf2768..932d612 100644 --- a/src/core/state/session.svelte.ts +++ b/src/core/state/session.svelte.ts @@ -85,10 +85,6 @@ class SessionStore { volume(volume: number): void; } | null = null; - setMedia(media: MediaInfo | null) { - this.media = media; - } - attachTransport(send: (cmd: EngineCommand) => void) { this.#send = send; } @@ -126,7 +122,7 @@ class SessionStore { case 'snapshot': this.connection = event.state; this.#dspBlocked = !event.dspAvailable; - this.setMedia(event.media); + this.media = event.media; this.params = event.params; this.volume = event.volume; this.loop = event.loop; @@ -152,7 +148,7 @@ class SessionStore { this.#dspBlocked = !event.available; break; case 'media': - this.setMedia(event.media); + this.media = event.media; // Zero duration = metadata still loading: keep seeks gated a moment // longer (mirrors track-sync's zero-duration grace period). if (event.media?.duration) this.#setSourceChanging(false); diff --git a/src/dev/mock.ts b/src/dev/mock.ts index db2ad20..fbd22b0 100644 --- a/src/dev/mock.ts +++ b/src/dev/mock.ts @@ -13,12 +13,12 @@ const GUITAR_EQ = BUILTIN_EQ_PRESETS.find((p) => p.name === 'Guitar')!; * The store screenshots are taken from this state. */ export function installMockState() { session.connection = 'connected-direct'; - session.setMedia({ + session.media = { title: 'Megadeth - Symphony of Destruction - Guitar Tab | Lesson', pageUrl: 'https://youtube.com/watch?v=741FSo7Xb40', duration: 230, hasVideo: true, - }); + }; session.t = 141; session.playing = false; diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index a05eba8..e67c11a 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -23,8 +23,7 @@ // ?mock=1&play=1 also runs the playhead, for previewing time-driven UI. const mockPlay = mock && params.has('play'); // Feature stores load projections from the background-owned library. - const loadFeatures = () => Promise.all(features.map((f) => f.init?.())); - const ready = loadFeatures().then( + const ready = Promise.all(features.map((f) => f.init?.())).then( async () => { applyTheme(settings.current.theme); trackSync.init(); @@ -45,7 +44,7 @@ }; } installShortcuts(); - // Fire-and-forget: opening the panel must not wait on the network. + // Fire-and-forget: opening the panel must not wait on storage. void sync.init(); if (mock) { installMockState(); diff --git a/src/features/eq/panel/eq-presets.svelte.ts b/src/features/eq/panel/eq-presets.svelte.ts index 685421d..cfc0200 100644 --- a/src/features/eq/panel/eq-presets.svelte.ts +++ b/src/features/eq/panel/eq-presets.svelte.ts @@ -14,7 +14,7 @@ class EqPresetsStore { async init() { const select = (library: Library): EqPreset[] => Object.entries(library.shared.presets) .filter(([, preset]) => preset.value !== null) - .map(([name, preset]) => ({ name, gains: preset.value!, updatedAt: preset.at })); + .map(([name, preset]) => ({ name, gains: preset.value! })); this.saved = select(await readLibrary()); watchLibrary(select, (value) => { this.saved = value; }); } diff --git a/src/features/library/panel/LibraryView.svelte b/src/features/library/panel/LibraryView.svelte index 4992181..ac352c5 100644 --- a/src/features/library/panel/LibraryView.svelte +++ b/src/features/library/panel/LibraryView.svelte @@ -7,8 +7,8 @@ import { cleanTitle } from '@/core/model/track-identity'; import { youtubeThumbnailUrl } from '@/core/model/thumbnail'; import type { FavoriteEntry, FavoritesSort, HistoryEntry } from '@/core/model/types'; - import { favorites } from '@/features/library/panel/favorites.svelte'; - import { history } from '@/features/library/panel/history.svelte'; + import { favorites } from '@/features/library/panel/favorites'; + import { history } from '@/features/library/panel/history'; import { uiPrefs } from '@/features/settings/panel/settings.svelte'; import { view } from '@/core/state/view.svelte'; diff --git a/src/features/library/panel/favorites.svelte.ts b/src/features/library/panel/favorites.ts similarity index 100% rename from src/features/library/panel/favorites.svelte.ts rename to src/features/library/panel/favorites.ts diff --git a/src/features/library/panel/history.svelte.ts b/src/features/library/panel/history.ts similarity index 100% rename from src/features/library/panel/history.svelte.ts rename to src/features/library/panel/history.ts diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index 3dc7c76..59c3d45 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -18,7 +18,7 @@ parseBackup, restoreBackup, } from '@/core/persist/backup'; - import { history } from '@/features/library/panel/history.svelte'; + import { history } from '@/features/library/panel/history'; import { applyTheme, settings } from '@/features/settings/panel/settings.svelte'; import { session } from '@/core/state/session.svelte'; import { view } from '@/core/state/view.svelte'; diff --git a/src/features/settings/panel/settings.svelte.ts b/src/features/settings/panel/settings.svelte.ts index 371004f..76ba31d 100644 --- a/src/features/settings/panel/settings.svelte.ts +++ b/src/features/settings/panel/settings.svelte.ts @@ -3,7 +3,8 @@ import type { PanelId, SectionId, Settings, UiPrefs } from '../../../core/model/ import { editLibrary } from '../../../core/persist/library-client'; import { settingsItem, uiPrefsItem } from '../../../core/persist/storage'; -/** Settings synced two-way with storage.local. Components mutate via `update`. */ +/** Mirror of the library's settings. Components mutate via `update`, which + * sends the patch to the background writer. */ class SettingsStore { current = $state(structuredClone(DEFAULT_SETTINGS)); loaded = $state(false); @@ -45,10 +46,9 @@ class SettingsStore { } async update(patch: Partial) { - // Dated on every write: settings cross devices as one item with one date - // (`merge.ts`), so the later change wins without either device having to - // consult its own clock about the other's. - const next = { ...this.current, ...patch, updatedAt: Date.now() }; + // Settings cross devices as one revisioned item (see `core/persist/library.ts`), + // so the write carries no date of its own. + const next = { ...this.current, ...patch }; // Auto Reset and Remember settings are alternatives — enabling one // switches the other off. if (patch.rememberSettings) next.autoReset = false; @@ -68,7 +68,7 @@ class SettingsStore { } async reset() { - this.current = { ...structuredClone(DEFAULT_SETTINGS), updatedAt: Date.now() }; + this.current = structuredClone(DEFAULT_SETTINGS); this.onChange?.(this.current); await editLibrary({ type: 'settings', patch: {}, reset: true }); } @@ -94,8 +94,7 @@ class UiPrefsStore { async #save() { this.#writing = true; try { - // Dated like Settings above — see `merge.ts`. - await uiPrefsItem.setValue({ ...$state.snapshot(this.current), updatedAt: Date.now() }); + await uiPrefsItem.setValue($state.snapshot(this.current)); } finally { this.#writing = false; } diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 833769f..620935a 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -1,5 +1,6 @@ import { sendMessage } from '../../../core/messaging/rpc'; -import { DEFAULT_SYNC_CONFIG, loadSyncConfig, syncConfigItem } from '../persist/sync-config'; +import { QUOTA_BYTES } from '../persist/records'; +import { DEFAULT_SYNC_CONFIG, loadSyncConfig, syncConfigItem, withSyncDefaults } from '../persist/sync-config'; /** Status projection only. Sync runs in the background, independently of panel lifetime. */ class SyncStore { @@ -8,10 +9,10 @@ class SyncStore { lastSyncedAt = $derived(this.config.lastSyncedAt); lastError = $derived(this.config.lastError); status = $derived(!this.enabled ? 'off' : this.config.syncing ? 'syncing' : this.lastError ? 'error' : 'idle'); - usedPercent = $derived(Math.round(this.config.usedBytes / 102400 * 100)); + usedPercent = $derived(Math.round(this.config.usedBytes / QUOTA_BYTES * 100)); async init() { this.config = await loadSyncConfig(); - syncConfigItem.watch((value) => { this.config = { ...DEFAULT_SYNC_CONFIG, ...value }; }); + syncConfigItem.watch((value) => { this.config = withSyncDefaults(value); }); } enable = () => sendMessage('librarySync', 'enable'); disable = () => sendMessage('librarySync', 'disable'); diff --git a/src/features/sync/persist/records.ts b/src/features/sync/persist/records.ts index 432c5d3..487f885 100644 --- a/src/features/sync/persist/records.ts +++ b/src/features/sync/persist/records.ts @@ -1,14 +1,14 @@ -import { canonical, emptyLibrary, type SharedLibrary } from '../../../core/persist/library.ts'; +import { canonical, defineEntry, emptyLibrary, type SharedLibrary } from '../../../core/persist/library.ts'; import { parseShared } from '../../../core/persist/backup-codec.ts'; -import { migrateBackup } from '../../../core/persist/library-migration.ts'; -import { parseBackupJson as parseLegacy } from '../../../core/persist/legacy-backup.ts'; export const PREFIX = 'nbn4:'; +/** `browser.storage.sync` caps the whole area, one item, and the item count. */ export const QUOTA_BYTES = 102400; +const ITEM_BYTES = 8192; +const ITEM_COUNT = 512; const encoder = new TextEncoder(); export const bytesUsed = (items: Record) => Object.entries(items) .reduce((total, [key, value]) => total + encoder.encode(key + JSON.stringify(value)).length, 0); -export const legacyKeys = (items: Record) => Object.keys(items).filter((key) => /^nbn\.(meta|\d+)$/.test(key) || key === 'syncId'); async function compress(text: string): Promise { const buffer = await new Response(new Blob([text]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer(); @@ -20,7 +20,7 @@ async function decompress(data: string): Promise { const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0)); return new Response(new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'))).text(); } -export function records(shared: SharedLibrary): Record { +function records(shared: SharedLibrary): Record { return Object.fromEntries([ [PREFIX + 'settings', shared.settings], [PREFIX + 'order', shared.favoriteOrder], ...Object.entries(shared.songs).map(([key, value]) => [PREFIX + 'song:' + key, value]), @@ -29,33 +29,21 @@ export function records(shared: SharedLibrary): Record { } export async function readRecords(items: Record): Promise { const shared = emptyLibrary().shared; - const keys = Object.keys(items).filter((key) => key.startsWith(PREFIX)); - for (const key of keys) { + for (const key of Object.keys(items).filter((key) => key.startsWith(PREFIX))) { const item = items[key] as { version: number; data: string }; if (item?.version !== 4 || typeof item.data !== 'string') throw new Error('Unsupported synced data. Update Note by Note on all devices.'); const value = JSON.parse(await decompress(item.data)); const name = key.slice(PREFIX.length); if (name === 'settings') shared.settings = value; else if (name === 'order') shared.favoriteOrder = value; - else if (name.startsWith('song:')) Object.defineProperty(shared.songs, name.slice(5), { value, enumerable: true, writable: true, configurable: true }); - else if (name.startsWith('preset:')) Object.defineProperty(shared.presets, name.slice(7), { value, enumerable: true, writable: true, configurable: true }); + else if (name.startsWith('song:')) defineEntry(shared.songs, name.slice(5), value); + else if (name.startsWith('preset:')) defineEntry(shared.presets, name.slice(7), value); else throw new Error('Unsupported synced record. Update Note by Note on all devices.'); } - // One-time read of the previous shared blob. Never write that format again. - if (!keys.length && items['nbn.meta']) { - const meta = items['nbn.meta'] as { v: number; n: number; h: string }; - if (meta.v !== 1 || !Number.isInteger(meta.n) || meta.n < 1 || meta.n > 12) throw new Error('Unsupported legacy sync data.'); - const chunks = Array.from({ length: meta.n }, (_, i) => items['nbn.' + i]); - if (chunks.some((chunk) => typeof chunk !== 'string')) throw new Error('Previous sync data is still arriving. Try again shortly.'); - const base64 = chunks.join(''); - const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', encoder.encode(base64))); - if ([...digest].map((n) => n.toString(16).padStart(2, '0')).join('') !== meta.h) throw new Error('Previous sync data is still arriving. Try again shortly.'); - return migrateBackup(parseLegacy(JSON.parse(await decompress(base64)))).shared; - } return parseShared(shared); } /** One record that cannot fit is reported and left behind, never truncated. */ -export interface RecordChanges { changes: Record; skipped: string[] } +export interface RecordChanges { changes: Record; skipped: string[]; usedBytes: number } export const skippedMessage = (skipped: string[]) => `${skipped.length} saved ${skipped.length === 1 ? 'record is' : 'records are'} too large to sync. Everything else synced; all data is kept on this device.`; @@ -68,13 +56,13 @@ export async function changedRecords(local: SharedLibrary, remote: SharedLibrary if (existing[key] !== undefined && canonical(value) === canonical(previous[key])) continue; const item = { version: 4, data: await compress(canonical(value)) }; // One outsized song must not hold back every other record for good. - if (bytesUsed({ [key]: item }) > 8192) { skipped.push(key); continue; } + if (encoder.encode(key + JSON.stringify(item)).length > ITEM_BYTES) { skipped.push(key); continue; } changes[key] = item; } const proposed = { ...existing, ...changes }; - for (const key of legacyKeys(proposed)) delete proposed[key]; - if (bytesUsed(proposed) > QUOTA_BYTES || Object.keys(proposed).length > 512) { + const usedBytes = bytesUsed(proposed); + if (usedBytes > QUOTA_BYTES || Object.keys(proposed).length > ITEM_COUNT) { throw new Error('Browser sync storage is full. All data is kept on this device; export a backup to transfer it.'); } - return { changes, skipped }; + return { changes, skipped, usedBytes }; } diff --git a/src/features/sync/persist/sync-config.ts b/src/features/sync/persist/sync-config.ts index dc20ff5..11b8f9c 100644 --- a/src/features/sync/persist/sync-config.ts +++ b/src/features/sync/persist/sync-config.ts @@ -11,8 +11,6 @@ export const DEFAULT_SYNC_CONFIG: SyncConfig = { enabled: true, lastSyncedAt: 0, lastPushAt: 0, lastError: null, usedBytes: 0, syncing: false, }; export const syncConfigItem = storage.defineItem('local:syncConfig', { fallback: DEFAULT_SYNC_CONFIG }); -export async function loadSyncConfig(): Promise { - const raw = await syncConfigItem.getValue(); - return Object.fromEntries(Object.entries(DEFAULT_SYNC_CONFIG).map(([key, fallback]) => - [key, raw[key as keyof SyncConfig] ?? fallback])) as unknown as SyncConfig; -} +/** A stored config written before a field existed still lacks it. */ +export const withSyncDefaults = (value: Partial | null): SyncConfig => ({ ...DEFAULT_SYNC_CONFIG, ...value }); +export const loadSyncConfig = async (): Promise => withSyncDefaults(await syncConfigItem.getValue()); diff --git a/wxt.config.ts b/wxt.config.ts index 3bb5c6f..58601f1 100644 --- a/wxt.config.ts +++ b/wxt.config.ts @@ -103,10 +103,10 @@ export default defineConfig({ // `data_collection_permissions` only in 140 — which is also the // current ESR line, so nothing supported is left behind. strict_min_version: '140.0', - // Sync ships Recent/Favorites off the device (through Firefox - // Sync's own storage — no server of ours), and those carry the - // page URL, title and thumbnail of every track practised — AMO - // counts that as browsing activity. `required` rather than + // Sync ships saved songs and Favorites off the device (through + // Firefox Sync's own storage — no server of ours), and those carry + // the page URL, title and thumbnail of every track practised — AMO + // counts that as browsing activity. (Recent stays on the device.) `required` rather than // `optional` because sync is on out of the box // (DEFAULT_SYNC_CONFIG.enabled), and a manifest claim that // contradicts the submission form is a rejection. Nothing else is From 47090b9c47168a0bd0e7aa03dc94d4b354df36d0 Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 22:07:35 +0200 Subject: [PATCH 21/26] refactor: enhance library synchronization and pruning logic, improve error handling for records --- src/core/persist/backup-codec.ts | 4 ++ src/core/persist/library-background.ts | 30 +++++--- src/core/persist/library.test.ts | 29 +++++++- src/core/persist/library.ts | 16 ++++- src/core/state/track-sync.svelte.ts | 16 +++-- src/features/library/panel/favorites.ts | 13 ++-- src/features/sync/persist/records.test.ts | 36 ++++++++++ src/features/sync/persist/records.ts | 84 +++++++++++++++++++---- 8 files changed, 191 insertions(+), 37 deletions(-) diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index c7a80ad..c4576ac 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -56,6 +56,10 @@ export function parseShared(value: unknown): SharedLibrary { song.favorite = versioned(song.favorite); if (typeof song.favorite.value !== 'boolean') throw new Error('Damaged favorite.'); const practice = song.practice.value; + // Checked ahead of the tombstone skip: a deleted song carries no identity to + // match the key against, so the shape is all that stands between an imported + // file and an arbitrary property name in the songs map. + if (!/^(yt|file|web):/.test(key)) throw new Error('Damaged song key.'); if (practice === null) continue; object(practice); const identity = object(practice.identity); diff --git a/src/core/persist/library-background.ts b/src/core/persist/library-background.ts index a837463..f2be912 100644 --- a/src/core/persist/library-background.ts +++ b/src/core/persist/library-background.ts @@ -1,5 +1,5 @@ import { onMessage } from '../messaging/rpc'; -import { applyCommand, canonical, emptyLibrary, mergeShared, type Library } from './library'; +import { applyCommand, canonical, emptyLibrary, mergeShared, pruned, type Library } from './library'; import { libraryItem } from './library-client'; import { parseBackupJson as parseLegacy } from './legacy-backup'; import { parseLibrary } from './backup-codec'; @@ -46,7 +46,9 @@ export function startLibraryBackground() { const init = () => ready ??= (async () => { // Only the migration needs every old key. Ordinary wakes read one item. if ((await browser.storage.local.get('library')).library) return; - await libraryItem.setValue(migrate(await browser.storage.local.get(null))); + // Pruned on the way in: an old library can hold far more songs than sync + // allows, and without this the first reconcile would fail on every retry. + await libraryItem.setValue(pruned(migrate(await browser.storage.local.get(null)))); })().catch((error) => { ready = undefined; throw error; }); const schedule = async () => { const config = await loadSyncConfig(); @@ -62,17 +64,22 @@ export function startLibraryBackground() { config.usedBytes = bytesUsed(existing); const remote = await readRecords(existing); const local = await libraryItem.getValue(); - const shared = mergeShared(local.shared, remote); - if (canonical(shared) !== canonical(local.shared)) await libraryItem.setValue({ ...local, shared }); - const { changes, skipped, usedBytes } = await changedRecords(shared, remote, existing); - if (Object.keys(changes).length) { + // Pruned here too: a merge can carry in more songs than the limits allow, + // and only `applyCommand` would otherwise ever bring it back under them. + const merged = pruned({ ...local, shared: mergeShared(local.shared, remote) }); + if (canonical(merged) !== canonical(local)) await libraryItem.setValue(merged); + const { changes, removals, skipped, usedBytes } = await changedRecords(merged.shared, remote, existing); + // Recorded before the throttle returns, or a merge that succeeded would + // leave the previous run's error on screen until a push happens to be due. + config.lastSyncedAt = Date.now(); + config.lastError = skipped.length ? skippedMessage(skipped) : null; + if (Object.keys(changes).length || removals.length) { if (Date.now() < config.lastPushAt + PUSH_INTERVAL) { await schedule(); return; } - await browser.storage.sync.set(changes); + if (removals.length) await browser.storage.sync.remove(removals); + if (Object.keys(changes).length) await browser.storage.sync.set(changes); config.lastPushAt = Date.now(); config.usedBytes = usedBytes; } - config.lastSyncedAt = Date.now(); - config.lastError = skipped.length ? skippedMessage(skipped) : null; } catch (error) { config.lastError = error instanceof Error ? error.message : String(error); } finally { @@ -116,6 +123,9 @@ export function startLibraryBackground() { } }); // Recreate the safety alarm on each worker start. No panel has to stay open. - void browser.alarms.create(SAFETY, { periodInMinutes: 1 }); + // Only a net: `libraryEdit` schedules WAKE and `storage.onChanged` catches + // remote writes, so this never needs to be the thing that notices a change — + // and each run wakes the worker to decompress every record. + void browser.alarms.create(SAFETY, { periodInMinutes: 30 }); void enqueue(reconcile); } diff --git a/src/core/persist/library.test.ts b/src/core/persist/library.test.ts index b82edf1..39609ca 100644 --- a/src/core/persist/library.test.ts +++ b/src/core/persist/library.test.ts @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { applyCommand, canonical, emptyLibrary, favoriteEntries, mergeShared, nextRevision, recentEntries } from './library.ts'; +import { applyCommand, canonical, cell, emptyLibrary, favoriteEntries, mergeShared, nextRevision, pruned, recentEntries } from './library.ts'; import { migrateBackup } from './library-migration.ts'; import { parseBackupJson } from './backup-codec.ts'; import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, DELETION_LIMIT, SONG_LIMIT } from '../model/defaults.ts'; @@ -134,3 +134,30 @@ test('preset names are data, including names matching object properties', () => assert.deepEqual(Object.keys(library.shared.presets), ['constructor', '__proto__']); assert.deepEqual(mergeShared(emptyLibrary().shared, library.shared).presets, library.shared.presets); }); + +test('every write path prunes, so a merge or migration cannot leave the library oversized', () => { + const track = (n: number) => makeTrackIdentity('https://www.youtube.com/watch?v=p' + n, 'Song ' + n, 200); + // Built directly: a merge or a migration lands songs without going through + // applyCommand, which is the only place that used to prune. + const oversized = emptyLibrary(); + for (let n = 0; n < SONG_LIMIT + DELETION_LIMIT + 50; n++) { + const key = track(n).key; + oversized.shared.songs[key] = { practice: cell({ identity: track(n), pageUrl: track(n).normalizedUrl, + markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false }, 100 + n), favorite: cell(false, 100) }; + oversized.local.lastAccessed[key] = 100 + n; + } + const trimmed = pruned(oversized, 1000); + assert.ok(Object.keys(trimmed.shared.songs).length <= SONG_LIMIT + DELETION_LIMIT); + assert.equal(canonical(pruned(trimmed, 2000)), canonical(trimmed), 'idempotent, so it never manufactures a write'); +}); + +test('an imported tombstone cannot name an arbitrary object property', () => { + const backup = { format: 'note-by-note-backup', version: 4, exportedAt: 100, ...structuredClone(save()) }; + // A deleted song carries no identity to check the key against, so the key + // shape is the only thing standing between a file and the songs map. + Object.defineProperty(backup.shared.songs, '__proto__', { + value: { practice: { at: 0, value: null }, favorite: { at: 0, value: false } }, + enumerable: true, writable: true, configurable: true, + }); + assert.throws(() => parseBackupJson(backup), /song key/); +}); diff --git a/src/core/persist/library.ts b/src/core/persist/library.ts index dddf55a..3e71981 100644 --- a/src/core/persist/library.ts +++ b/src/core/persist/library.ts @@ -108,7 +108,7 @@ function prune(library: Library, at: number): void { .filter((key) => !kept.has(key) && shared.songs[key].practice.value !== null) .sort((a, b) => (local.lastAccessed[b] ?? 0) - (local.lastAccessed[a] ?? 0)); for (const key of rest.slice(Math.max(0, SONG_LIMIT - kept.size))) { - shared.songs[key] = { practice: cell(null, at), favorite: cell(false, at) }; + defineEntry(shared.songs, key, { practice: cell(null, at), favorite: cell(false, at) }); } const deleted = Object.keys(shared.songs).filter((key) => shared.songs[key].practice.value === null) .sort((a, b) => shared.songs[b].practice.at - shared.songs[a].practice.at); @@ -123,6 +123,16 @@ function prune(library: Library, at: number): void { if (order.length !== shared.favoriteOrder.value.length) shared.favoriteOrder = cell(order, at); } +/** Every path that writes the library prunes, not only edits: a merge or a + * migration can carry in more songs than the sync limits allow, and nothing else + * would ever bring it back under them. Idempotent — with nothing to drop the + * result is identical, so it never manufactures a write of its own. */ +export function pruned(library: Library, now = Date.now()): Library { + const next = structuredClone(library); + prune(next, nextRevision(next.shared, now)); + return next; +} + /** Called only by the background writer. Incoming edits patch current saved data. */ export function applyCommand(library: Library, command: LibraryCommand, now = Date.now()): Library { const next = structuredClone(library); @@ -137,7 +147,7 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false, }; song.practice = cell({ ...base, ...command.patch, identity: command.identity }, at); - shared.songs[key] = song; + defineEntry(shared.songs, key, song); if (command.recent || key in local.recent) local.recent[key] = now; local.lastAccessed[key] = now; const keep = Object.entries(local.recent).sort((a, b) => b[1] - a[1]).slice(0, HISTORY_LIMIT); @@ -189,7 +199,7 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da // Replacement names the records this device knows; absence is never a remote delete. for (const key of new Set([...Object.keys(shared.songs), ...Object.keys(file.shared.songs)])) { const song = file.shared.songs[key]; - shared.songs[key] = { practice: cell(song?.practice.value ?? null, revision), favorite: cell(song?.favorite.value ?? false, revision) }; + defineEntry(shared.songs, key, { practice: cell(song?.practice.value ?? null, revision), favorite: cell(song?.favorite.value ?? false, revision) }); } for (const name of new Set([...Object.keys(shared.presets), ...Object.keys(file.shared.presets)])) { defineEntry(shared.presets, name, diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index 5da96b5..cf0c03a 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -45,11 +45,17 @@ class TrackSync { } async onMedia(media: MediaInfo | null) { - if (!media) { this.onEngineLost(); return; } + // Null media is a transient engine state (detecting, no player, mid source + // change), not the end of the session — dropping the track here would throw + // away edits made before the next event. Real loss arrives via `onEngineLost`. + if (!media) return; const identity = makeTrackIdentity(media.pageUrl, media.title, media.duration); - this.#media = media; - if (this.#identity?.key === identity.key) { this.#identity = identity; return; } + if (this.#identity?.key === identity.key) { this.#media = media; this.#identity = identity; return; } + // Flushed against the outgoing track's media: `#save` reads `#media` for the + // saved pageUrl and thumbnail, so replacing it first files this song's URL + // under the previous song's identity. this.#flushParams(); + this.#media = media; this.#identity = identity; this.#hasSavedParams = false; const generation = ++this.#generation; @@ -69,7 +75,9 @@ class TrackSync { this.#chordsEnabled = chords.enabled; const params = practice?.params ?? (settings.current.autoReset ? DEFAULT_PARAMS : settings.current.rememberSettings ? settings.current.lastUsedParams : undefined); - if (params) session.patchParams(structuredClone(params)); + // $state.snapshot, not structuredClone: `settings.current` is a rune, so + // `lastUsedParams` is a proxy and structuredClone throws on it. + if (params) session.patchParams($state.snapshot(params) as EffectParams); } finally { // A newer track already owns the flag; only its own load may clear it. if (generation === this.#generation) this.#restoring = false; diff --git a/src/features/library/panel/favorites.ts b/src/features/library/panel/favorites.ts index 12a1ccf..3aacd97 100644 --- a/src/features/library/panel/favorites.ts +++ b/src/features/library/panel/favorites.ts @@ -3,11 +3,16 @@ import { favoriteEntries } from '../../../core/persist/library'; import { editLibrary } from '../../../core/persist/library-client'; import { library } from './library.svelte'; +/** The list only repaints from the storage watch, so a rejected write leaves the + * control looking dead. Log instead of vanishing. */ +const write = (run: Promise) => + run.catch((error: unknown) => console.error('[note-by-note] library write failed', error)); + export const favorites = { get entries() { return favoriteEntries(library.current); }, has: (identity: TrackIdentity) => library.current.shared.songs[identity.key]?.favorite.value === true, - toggle: (entry: HistoryEntry) => editLibrary({ type: 'favorite', key: entry.identity.key, - value: !library.current.shared.songs[entry.identity.key]?.favorite.value }), - remove: (key: string) => editLibrary({ type: 'favorite', key, value: false }), - reorder: (keys: string[]) => editLibrary({ type: 'order', keys }), + toggle: (entry: HistoryEntry) => write(editLibrary({ type: 'favorite', key: entry.identity.key, + value: !library.current.shared.songs[entry.identity.key]?.favorite.value })), + remove: (key: string) => write(editLibrary({ type: 'favorite', key, value: false })), + reorder: (keys: string[]) => write(editLibrary({ type: 'order', keys })), }; diff --git a/src/features/sync/persist/records.test.ts b/src/features/sync/persist/records.test.ts index af8da0a..4aeb418 100644 --- a/src/features/sync/persist/records.test.ts +++ b/src/features/sync/persist/records.test.ts @@ -46,3 +46,39 @@ test('a record too large to sync is reported and left behind, never blocking the test('unsupported records are rejected before any application or upload', async () => { await assert.rejects(readRecords({ 'nbn4:settings': { version: 5, data: '' } }), /Unsupported/); }); + +test('a record the library no longer holds is removed remotely, not merged back', async () => { + const library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); + const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); + const remote = await readRecords(items); + // The song is gone locally: sync must drop it rather than read it back forever. + const { removals } = await changedRecords(emptyLibrary().shared, remote, items); + assert.deepEqual(removals, ['nbn4:song:' + song(1).key]); + // A record this build does not own belongs to a newer one and is left alone. + const foreign = { ...items, 'nbn4:future:1': { version: 4, data: '' } }; + assert.ok(!(await changedRecords(library.shared, remote, foreign)).removals.includes('nbn4:future:1')); +}); + +test('one damaged record costs only itself, never the rest or the upload', async () => { + let library = emptyLibrary(); + for (const n of [1, 2]) library = applyCommand(library, { type: 'practice', identity: song(n), patch: {}, recent: true }); + const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); + const damaged = { ...items, ['nbn4:song:' + song(1).key]: { version: 4, data: 'not-gzip-at-all' } }; + const remote = await readRecords(damaged); + assert.deepEqual(Object.keys(remote.songs), [song(2).key]); + // The undamaged song still round-trips, so this device is not locked out. + assert.equal(canonical(remote.songs[song(2).key]), canonical(library.shared.songs[song(2).key])); +}); + +test('over budget, songs are held back and reported; the small records still sync', async () => { + let library = emptyLibrary(); + const markers = Array.from({ length: 300 }, (_, n) => ({ id: 'm' + n, t: n, label: 'Marker ' + n })); + for (const n of [1, 2, 3]) library = applyCommand(library, { type: 'practice', identity: song(n), patch: { markers }, recent: true }); + const before = canonical(library); + const padded = { unrelated: 'x'.repeat(102400 - 3000) }; + const { changes, skipped, usedBytes } = await changedRecords(library.shared, emptyLibrary().shared, padded); + assert.ok(skipped.length > 0, 'the songs that do not fit are reported'); + assert.ok(Object.keys(changes).includes('nbn4:settings'), 'settings are small and always get through'); + assert.ok(usedBytes <= 102400, 'what is written fits the quota'); + assert.equal(canonical(library), before, 'nothing is trimmed from the library itself'); +}); diff --git a/src/features/sync/persist/records.ts b/src/features/sync/persist/records.ts index 487f885..545e51e 100644 --- a/src/features/sync/persist/records.ts +++ b/src/features/sync/persist/records.ts @@ -27,42 +27,96 @@ function records(shared: SharedLibrary): Record { ...Object.entries(shared.presets).map(([key, value]) => [PREFIX + 'preset:' + key, value]), ]); } +/** Record kinds this build owns. Anything else is read past and left in place — + * it may belong to a newer build, and removing it would destroy that data. */ +const OWNED = /^(settings|order|song:|preset:)/; + +/** Each record is validated on its own, so one damaged record costs only itself. + * Validating the whole set at once would let a single bad song block every other + * song *and* this device's uploads, with "Delete synced data" the only way out. + * A record written by a newer build is the one hard failure: this build cannot + * read it, and uploading its own view over it would lose data. */ export async function readRecords(items: Record): Promise { const shared = emptyLibrary().shared; + const blank = emptyLibrary().shared; + // The casts assert nothing: `parseShared` validates the value at runtime and + // throws for this record alone if it does not hold up. + const validate = (patch: Partial) => parseShared({ ...blank, ...patch }); + const map = (key: string, value: unknown) => { + const one: Record = {}; + defineEntry(one, key, value); + return one; + }; for (const key of Object.keys(items).filter((key) => key.startsWith(PREFIX))) { - const item = items[key] as { version: number; data: string }; - if (item?.version !== 4 || typeof item.data !== 'string') throw new Error('Unsupported synced data. Update Note by Note on all devices.'); - const value = JSON.parse(await decompress(item.data)); + const item = items[key] as { version?: unknown; data?: unknown }; + if (typeof item?.version === 'number' && item.version > 4) { + throw new Error('Unsupported synced data. Update Note by Note on all devices.'); + } const name = key.slice(PREFIX.length); - if (name === 'settings') shared.settings = value; - else if (name === 'order') shared.favoriteOrder = value; - else if (name.startsWith('song:')) defineEntry(shared.songs, name.slice(5), value); - else if (name.startsWith('preset:')) defineEntry(shared.presets, name.slice(7), value); - else throw new Error('Unsupported synced record. Update Note by Note on all devices.'); + try { + if (item?.version !== 4 || typeof item.data !== 'string') throw new Error('Damaged record.'); + const value: unknown = JSON.parse(await decompress(item.data)); + // Any other name falls through untouched: `OWNED` keeps it off the removal + // list too, so a record this build has never heard of is left alone. + if (name === 'settings') shared.settings = validate({ settings: value as SharedLibrary['settings'] }).settings; + else if (name === 'order') shared.favoriteOrder = validate({ favoriteOrder: value as SharedLibrary['favoriteOrder'] }).favoriteOrder; + else if (name.startsWith('song:')) { + const id = name.slice(5); + defineEntry(shared.songs, id, validate({ songs: map(id, value) as SharedLibrary['songs'] }).songs[id]); + } else if (name.startsWith('preset:')) { + const id = name.slice(7); + defineEntry(shared.presets, id, validate({ presets: map(id, value) as SharedLibrary['presets'] }).presets[id]); + } + } catch (error) { + console.warn('[note-by-note] a synced record could not be read and was skipped', key, error); + } } - return parseShared(shared); + return shared; } /** One record that cannot fit is reported and left behind, never truncated. */ -export interface RecordChanges { changes: Record; skipped: string[]; usedBytes: number } +export interface RecordChanges { changes: Record; removals: string[]; skipped: string[]; usedBytes: number } export const skippedMessage = (skipped: string[]) => `${skipped.length} saved ${skipped.length === 1 ? 'record is' : 'records are'} too large to sync. Everything else synced; all data is kept on this device.`; /** Write complete independent records. There is no chunk assembly or truncation. */ export async function changedRecords(local: SharedLibrary, remote: SharedLibrary, existing: Record): Promise { const previous = records(remote); + const next = records(local); const changes: Record = {}; const skipped: string[] = []; - for (const [key, value] of Object.entries(records(local))) { + for (const [key, value] of Object.entries(next)) { if (existing[key] !== undefined && canonical(value) === canonical(previous[key])) continue; const item = { version: 4, data: await compress(canonical(value)) }; // One outsized song must not hold back every other record for good. if (encoder.encode(key + JSON.stringify(item)).length > ITEM_BYTES) { skipped.push(key); continue; } changes[key] = item; } - const proposed = { ...existing, ...changes }; - const usedBytes = bytesUsed(proposed); - if (usedBytes > QUOTA_BYTES || Object.keys(proposed).length > ITEM_COUNT) { + // A record the library no longer holds is dropped remotely. Without this a + // pruned song is merged straight back on the next read, and the item count + // climbs until the cap is hit and nothing can be written at all. + const removals = Object.keys(existing).filter((key) => key.startsWith(PREFIX) + && OWNED.test(key.slice(PREFIX.length)) && !(key in next)); + const build = () => { + const proposed: Record = { ...existing, ...changes }; + for (const key of removals) delete proposed[key]; + return proposed; + }; + const overBudget = (proposed: Record) => + bytesUsed(proposed) > QUOTA_BYTES || Object.keys(proposed).length > ITEM_COUNT; + // Over budget, songs are held back largest-first instead of the whole batch + // failing: settings, order and presets are small and must always get through, + // and a song left behind stays on this device and is retried next reconcile. + const droppable = Object.keys(changes).filter((key) => key.slice(PREFIX.length).startsWith('song:')) + .sort((a, b) => encoder.encode(JSON.stringify(changes[b])).length - encoder.encode(JSON.stringify(changes[a])).length); + let proposed = build(); + for (const key of droppable) { + if (!overBudget(proposed)) break; + delete changes[key]; + skipped.push(key); + proposed = build(); + } + if (overBudget(proposed)) { throw new Error('Browser sync storage is full. All data is kept on this device; export a backup to transfer it.'); } - return { changes, skipped, usedBytes }; + return { changes, removals, skipped, usedBytes: bytesUsed(proposed) }; } From 8b07d01bf5c945bfd288bd3723529caad852a77d Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Mon, 7 Sep 2026 23:02:58 +0200 Subject: [PATCH 22/26] feat: implement library state management and sync improvements - Introduced LibraryStore to manage library state and initialization. - Updated track-sync to handle practice parameters correctly. - Refactored local-player to apply theme settings from the library. - Enhanced sidepanel to initialize library state and settings. - Removed deprecated chords panel and related code. - Simplified EQ presets management by leveraging library state. - Updated favorites and history features to use new library state. - Removed old library panel and related feature code. - Refactored settings management to utilize library state. - Improved sync records handling and added snapshot validation. - Updated privacy policy to reflect changes in sync data handling. --- CLAUDE.md | 32 +-- PRIVACY.md | 8 +- README.md | 42 ++-- e2e/library.mjs | 119 +++++++-- src/core/features.ts | 33 --- src/core/model/defaults.ts | 9 - src/core/model/types.ts | 4 +- src/core/persist/backup-codec.ts | 53 ++-- src/core/persist/legacy-backup.ts | 31 +-- src/core/persist/library-background.ts | 50 ++-- src/core/persist/library-client.ts | 11 +- src/core/persist/library-migration.ts | 43 ++-- src/core/persist/library.test.ts | 234 ++++++++++-------- src/core/persist/library.ts | 203 +++++---------- src/core/persist/storage.ts | 16 -- src/core/state/connect.svelte.ts | 14 +- src/core/state/library.svelte.ts | 14 ++ src/core/state/track-sync.svelte.ts | 2 +- src/entrypoints/local-player/main.ts | 7 +- src/entrypoints/sidepanel/App.svelte | 11 +- src/features/chords/panel/panel.ts | 17 -- src/features/eq/panel/eq-presets.svelte.ts | 15 +- src/features/eq/panel/panel.ts | 7 - src/features/library/panel/favorites.ts | 6 +- src/features/library/panel/history.ts | 2 +- src/features/library/panel/library.svelte.ts | 13 - src/features/library/panel/panel.ts | 3 - .../settings/panel/SettingsView.svelte | 16 +- src/features/settings/panel/panel.ts | 9 - .../settings/panel/settings.svelte.ts | 135 ++-------- src/features/sync/persist/records.test.ts | 122 ++++----- src/features/sync/persist/records.ts | 139 ++++------- store/privacy-policy-firefox.md | 4 +- 33 files changed, 577 insertions(+), 847 deletions(-) delete mode 100644 src/core/features.ts create mode 100644 src/core/state/library.svelte.ts delete mode 100644 src/features/chords/panel/panel.ts delete mode 100644 src/features/eq/panel/panel.ts delete mode 100644 src/features/library/panel/library.svelte.ts delete mode 100644 src/features/library/panel/panel.ts delete mode 100644 src/features/settings/panel/panel.ts diff --git a/CLAUDE.md b/CLAUDE.md index 3d10c13..c9a2f62 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,13 +45,13 @@ This is a **multi-context extension**. The single most important structural fact ### Source layout (vertical feature slices) The tree is organized by **feature**, not by layer: -- **`src/core/`** — shared platform: `engine/` (controller, media-engine/-detect, attach-audio), `audio/` (pipeline, fft, silence-detector), `messaging/` (protocol shell, ports, rpc), `model/` (shared types + defaults + format + track-identity + thumbnail), `persist/` (library, backup, migration), `state/` (session, track-sync, connect, view), and `features.ts` (the panel-feature registry). -- **`src/features//`** — one folder per product feature (chords, pitch, speed, vocal-reducer, eq, loops, markers, snippets, count-in, library, sync, settings, shortcuts), each with an `engine/` subfolder (content-script code: worklets, schedulers, DSP factories) and/or a `panel/` subfolder (side-panel stores + components), plus optional `protocol.ts` (its wire-message fragment), `panel/panel.ts` (registration object). **`engine/` and `panel/` never cross-import**, so the content and panel bundles stay separate. +- **`src/core/`** — shared platform: `engine/` (controller, media-engine/-detect, attach-audio), `audio/` (pipeline, fft, silence-detector), `messaging/` (protocol shell, ports, rpc), `model/` (shared types + defaults + format + track-identity + thumbnail), `persist/` (library, backup, migration), and `state/` (session, library, track-sync, connect, view). +- **`src/features//`** — one folder per product feature (chords, pitch, speed, vocal-reducer, eq, loops, markers, snippets, count-in, library, sync, settings, shortcuts), each with an `engine/` subfolder (content-script code: worklets, schedulers, DSP factories) and/or a `panel/` subfolder (side-panel stores + components), plus optional `protocol.ts` (its wire-message fragment). **`engine/` and `panel/` never cross-import**, so the content and panel bundles stay separate. - **`src/ui/`** — shared/presentational UI (Workspace, Panel, PanelStack, Timeline, chrome bars, `shared/` primitives, icons, dismiss). - **`src/dev/`** — preview-only helpers (`browser-shim`, `mock`). - **`src/entrypoints/`** — thin WXT composition roots (unchanged location). -**Dependency direction:** `entrypoints → core composition roots (pipeline, controller, protocol, App, features.ts, track-sync) → features → core primitives (model, messaging, audio/fft, ui)`. Composition roots **import feature contributions** (the "light registration"); **features never import the orchestrators**. Domain types stay central in `core/model/types.ts` (they are the shared engine↔panel wire + persistence contract). +**Dependency direction:** `entrypoints → core composition roots (pipeline, controller, protocol, App, track-sync) → features → core primitives (model, messaging, audio/fft, ui)`. Composition roots wire feature behavior directly; features never import the orchestrators. Domain types stay central in `core/model/types.ts` (the shared engine↔panel contract). ### Execution contexts (`src/entrypoints/`) - **`sidepanel/`** — the Svelte UI. Holds no engine state of its own; mirrors the active tab's engine. @@ -91,29 +91,31 @@ Both worklet processors are shipped as **static files under `public/worklets/`** ### State layer (Svelte 5 runes stores, `*.svelte.ts`) Runes stores (classes with `$state`), one singleton exported per file. All panel-side. Split by ownership: -- **Core (`src/core/state/`):** `session` — mirror of the active tab's engine + the command surface panels call (while no engine is attached, commands fall back to **optimistic local state**, staged and pushed on connect); `connection` (`connect.svelte.ts`) — owns the port lifecycle (one `` prompt from the banner's Connect button in a user gesture, injection, reconnect, capture start/stop; a `#generation` counter drops stale async work) and iterates the **panel-feature registry** ([core/features.ts](src/core/features.ts)) to route engine events into feature stores; `track-sync` — reacts to track changes (auto-save to Recent, reset/remember/carry-over params) loads saved practice data and wires feature edits directly; `view`. +- **Core (`src/core/state/`):** `session` mirrors the active tab's engine and provides panel commands; `library` holds the panel's single saved-data snapshot; `connection` owns permissions, injection, port lifecycle and capture, routing chord events directly; `track-sync` loads saved practice data and wires feature edits; `view` selects the open panel. - **Feature-owned (`src/features//panel/`):** `markers`, `snippets`, `chords`, `settings`, `favorites`/`history` (library), `eq-presets`, `shortcuts`. Preview data (`mock`) lives in `src/dev/`. -- Features contribute boot init + event routing via `panel/panel.ts` (registered in `core/features.ts`) and submit per-track edits through track-sync. +- App loads the library once. Settings, UI preferences, presets and song lists derive from it; App effects apply the theme and send engine settings. Features submit per-track edits through track-sync. ### Persistence & sync - One local library owns shared songs/settings/presets/order and device-local Recent, UI preferences, last-used parameters and analysis. See core/persist/library.ts. - The background service is the only writer (library-background.ts). Panels use - library-client.ts commands and storage watches. Commands patch the latest saved + library-client.ts commands and one storage watch. Commands patch the latest saved data; Recent and Favorites are projections, not persistent song copies. - Track-sync loads a practice session once and submits edits to the saved library. Receiving remote changes never reloads or silently replaces the active session. Feature persistence is wired directly in track-sync; there is no descriptor registry. -- Sync stores independent gzip-compressed records in browser.storage.sync (records.ts). - Versioned practice and favorite values merge independently. Explicit null/false - deletions are retained. There is no global blob. Because the browser caps sync at - 512 items, library.ts prune() keeps a song while it is favorited, in Recent, or - among the SONG_LIMIT most recently opened, and retains only the newest - DELETION_LIMIT deletions (defaults.ts). Quota failures leave local data intact. - Background alarms retry independently of panels. -- Backups use the readable v4 library schema. legacy-backup.ts only reads v1, the - format every released build wrote; library-migration.ts collapses old copies once. +- Sync copies the same SharedLibrary snapshot used locally (records.ts). One + updatedAt timestamp chooses the whole winner; equal dates adopt the remote copy. + There are no field merges, deletion markers, or automatic song pruning. Song + dates only support display and sorting. Gzip data spans fixed size-limited slots; + a hash prevents partial or mixed snapshots from being applied. Capacity failures + preserve local data and the last successful upload. Background alarms retry + independently of panels. +- Backups use the readable v2 library schema. legacy-backup.ts reads the + released v1 format, and + library-migration.ts collapses its old copies once. + Only released formats need compatibility adapters; intermediate PR formats do not. Old local storage is retained for recovery, but only local:library is used after migration. - Web identity uses provider ID/normalized URL. Title and duration are metadata. diff --git a/PRIVACY.md b/PRIVACY.md index e88b73a..83b13cc 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -38,7 +38,7 @@ through your browser's own sync (Chrome sync, Firefox Sync) — the same channel that carries your bookmarks — to the other devices signed into the same browser profile. If browser sync is off, nothing leaves the device. -Sync is **on by default**. It writes independent compressed records into the +Sync is **on by default**. It writes a compressed library snapshot into the browser's synced extension storage containing: - settings and EQ presets @@ -63,8 +63,10 @@ Sync — the latter end-to-end encrypted). The author operates no server and can see none of it. There are no accounts with us and no sync ID. The browser caps synced storage at 100 KB per extension and 8 KB per item. -If a compressed record or the library exceeds those limits, Settings reports an -error. All data remains saved locally; no songs are automatically trimmed. +The snapshot is split into size-limited pieces. If it exceeds the total capacity, +Settings reports an error. All data remains saved locally; no songs are +automatically trimmed. The newest complete library replaces older copies, so +edits made on two devices at once can overwrite each other. ### Turning it off and deleting the data diff --git a/README.md b/README.md index 951c802..441efaf 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ the engine. ## Tests `pnpm test:dsp` runs the unit tests under `node --test`: the center-cut -math, the CQT, chord decoding, library migration, merge rules, backups and independent sync records. Fast, no browser. +math, the CQT, chord decoding, library migration, snapshot selection, backups and sync transport. Fast, no browser. The e2e harness is the interesting one. It launches Chrome for Testing with the extension installed, plays a 440 Hz tone, and asserts on the *processed output* — @@ -175,26 +175,30 @@ chord analysis stay on the device. Audio is never transferred. One saved song owns its parameters, markers and snippets. Recent and Favorites are views of that song, so there are no saved-settings copies to keep aligned. The background is the only library writer; it handles edits, imports and remote -updates even when the panel is closed. An active practice session keeps its loaded +updates even when the panel is closed. Each panel loads and watches one library; +settings, UI preferences, presets and song lists read that same copy. +An active practice session keeps its loaded configuration until the song is reopened. Sync never reloads the panel. -Each song is a complete, independently compressed sync item. Practice edits and -favorite membership have separate revisions; preset deletion is an explicit null. -Revisions advance past everything a device has observed, with deterministic ties. -There is no whole-library blob and no chunk assembly. Because the browser caps -sync at 512 items, a song is kept while it is favorited, in Recent, or among the -300 most recently opened; past that it becomes a dated deletion so the removal -crosses devices, and only the newest 100 deletions are kept. If a record or -library exceeds the browser's capacity, local data remains saved and Settings -reports the error. Export a backup to transfer all data, including local history -and analysis. - -Backups are readable version-4 JSON containing shared and local sections. Imports -also accept the version-1 format every released build wrote, and convert it once. -Replacing a backup replaces this device's library and dates the named changes for -sync; it does not delete songs known only to another device. Old local storage is -retained as a recovery copy after the first migration. Upgrade all devices before -using the new sync format; older builds cannot read it. +Local storage and sync use the same shared library snapshot, with one timestamp. +The most recently edited snapshot replaces the older copy in full. On equal +timestamps the synced copy wins. Edits made on two devices at once can overwrite +each other, even when they affect different songs; there are no field merges or +deletion markers. + +The snapshot is gzip-compressed and split across fixed storage slots to fit the +browser's 8 KB item limit. A content hash ensures all parts belong to the same +complete snapshot before it is used. If the library exceeds sync capacity, +Settings reports an error and retains the complete local library and the last +successful synced copy. Songs are not automatically discarded to make it fit. +Export a backup to transfer all data, including local history and analysis. + +Backups are readable version-2 JSON containing shared and local sections. Imports +also accept the released version-1 format, converting it once. +Importing a backup replaces the entire library and dates it +as a new edit for sync, including removal of songs absent from the file. Old local +storage is retained as a recovery copy after the first migration. Upgrade all +devices before using the new sync format; older builds cannot read it. ## License diff --git a/e2e/library.mjs b/e2e/library.mjs index 185afa1..2686035 100644 --- a/e2e/library.mjs +++ b/e2e/library.mjs @@ -6,23 +6,25 @@ import { tmpdir } from 'node:os'; import { dirname, resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import puppeteer from 'puppeteer-core'; -import { emptyLibrary, applyCommand, canonical } from '../src/core/persist/library.ts'; +import { emptyLibrary, applyCommand } from '../src/core/persist/library.ts'; import { makeTrackIdentity } from '../src/core/model/track-identity.ts'; -import { DEFAULT_PARAMS } from '../src/core/model/defaults.ts'; -import { changedRecords } from '../src/features/sync/persist/records.ts'; +import { DEFAULT_PARAMS, DEFAULT_UI_PREFS } from '../src/core/model/defaults.ts'; +import { encodeSnapshot, readSnapshot, SNAPSHOT_KEY, PREFIX } from '../src/features/sync/persist/records.ts'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const extension = resolve(root, '.output', 'chrome-mv3-testing'); const executablePath = globSync(resolve(root, '.browsers', 'chrome', '*', 'chrome-win64', 'chrome.exe'))[0]; if (!executablePath) throw new Error('Chrome for Testing not found under .browsers/'); const profile = mkdtempSync(join(tmpdir(), 'note-by-note-library-')); -const launch = () => puppeteer.launch({ executablePath, headless: !process.argv.includes('--headful'), userDataDir: profile, +const launch = () => puppeteer.launch({ executablePath, headless: !process.argv.includes('--headful'), userDataDir: profile, protocolTimeout: 30000, args: [`--disable-extensions-except=${extension}`, `--load-extension=${extension}`, '--mute-audio'] }); let browser; +const panelErrors = []; async function panel() { const target = await browser.waitForTarget((target) => target.type() === 'service_worker'); const id = new URL(target.url()).host; const page = await browser.newPage(); + page.on('pageerror', (error) => { panelErrors.push(error.message); console.error('Panel error:', error.message); }); await page.goto(`chrome-extension://${id}/sidepanel.html?mock=1`); return page; } @@ -44,6 +46,34 @@ try { const second = await panel(); await sync(first, 'disable'); await edit(first, { type: 'import', library: emptyLibrary() }); + await first.bringToFront(); + await first.waitForSelector('button[aria-label="Settings"]'); + await first.click('button[aria-label="Settings"]'); + await first.waitForSelector('button[aria-label="Light"]'); + await first.click('button[aria-label="Light"]'); + await Promise.all([first, second].map((page) => page.waitForFunction(() => document.documentElement.dataset.theme === 'light', { polling: 50 }))); + assert.equal((await read(first)).shared.settings.theme, 'light'); + await edit(second, { type: 'settings', patch: { theme: 'dark' } }); + await first.waitForFunction(() => document.querySelector('button[aria-label="Dark"]')?.getAttribute('aria-checked') === 'true'); + await first.click('button[aria-label="Close settings"]'); + console.log('PASS settings controls and both panel themes follow the saved library'); + + const collapse = 'section[aria-label="Looper"] button[aria-expanded]'; + await first.waitForSelector(collapse); + await first.click(collapse); + await second.waitForFunction((selector) => document.querySelector(selector)?.getAttribute('aria-expanded') === 'false', { polling: 50 }, collapse); + assert.equal((await read(first)).local.uiPrefs.collapsedSections.looper, true); + await second.bringToFront(); + await second.click(collapse); + await first.waitForFunction((selector) => document.querySelector(selector)?.getAttribute('aria-expanded') === 'true', { polling: 50 }, collapse); + console.log('PASS preference controls update both panels through one library watch'); + + await edit(first, { type: 'preset', name: 'Study EQ', gains: [1, 2, 3] }); + await Promise.all([first, second].map((page) => page.waitForSelector('select[aria-label="EQ preset"] option[value="Study EQ"]'))); + await edit(second, { type: 'preset', name: 'Study EQ', gains: null }); + await first.waitForFunction(() => !document.querySelector('select[aria-label="EQ preset"] option[value="Study EQ"]'), { polling: 50 }); + console.log('PASS preset options follow library additions and deletions'); + await Promise.all(Array.from({ length: 12 }, (_, n) => edit(n % 2 ? first : second, { type: 'practice', identity: identity(n), patch: { params: { ...DEFAULT_PARAMS, speed: 0.8 } }, recent: true, }))); @@ -55,50 +85,105 @@ try { edit(second, { type: 'practice', identity: identity(0), patch: { params: { ...DEFAULT_PARAMS, speed: 0.5 } }, recent: true }), ]); const saved = await read(first); - assert.equal(saved.shared.songs[identity(0).key].practice.value.markers[0].t, 5); - assert.equal(saved.shared.songs[identity(0).key].practice.value.params.speed, 0.5); + assert.equal(saved.shared.songs[identity(0).key].practice.markers[0].t, 5); + assert.equal(saved.shared.songs[identity(0).key].practice.params.speed, 0.5); console.log('PASS concurrent commands patch the latest record'); await first.evaluate(() => { window.libraryReloadSentinel = 42; }); const remote = applyCommand(saved, { type: 'favorite', key: identity(0).key, value: true }, Date.now() + 1000); - const { changes: remoteItems } = await changedRecords(remote.shared, emptyLibrary().shared, {}); + const { items: remoteItems } = await encodeSnapshot(remote.shared); await first.evaluate((items) => chrome.storage.sync.set(items), remoteItems); await sync(first, 'enable'); - const merged = await read(first); - assert.equal(merged.shared.songs[identity(0).key].favorite.value, true); - assert.deepEqual(merged.local, saved.local); + const received = await read(first); + assert.deepEqual(received.shared, remote.shared); + assert.deepEqual(received.local, saved.local); assert.equal(await first.evaluate(() => window.libraryReloadSentinel), 42); console.log('PASS remote updates preserve local data and do not reload the panel'); + await sync(first, 'disable'); + const replacement = applyCommand(emptyLibrary(), { type: 'preset', name: 'Remote only', gains: [1] }, remote.shared.updatedAt + 1000); + const { items: replacementItems } = await encodeSnapshot(replacement.shared); + // Header arrives before the content. Neither the old data nor a partial + // replacement may be uploaded over this newer snapshot. + await first.evaluate(({ key, header }) => chrome.storage.sync.set({ [key]: header }), + { key: SNAPSHOT_KEY, header: replacementItems[SNAPSHOT_KEY] }); + await sync(first, 'enable'); + assert.deepEqual((await read(first)).shared, remote.shared); + assert.match(await first.evaluate(async () => (await chrome.storage.local.get('syncConfig')).syncConfig.lastError), /complete synced library/); + await first.evaluate((items) => chrome.storage.sync.set(items), replacementItems); + await sync(first, 'now'); + assert.deepEqual((await read(first)).shared, replacement.shared); + console.log('PASS incomplete arrivals wait; the newer snapshot replaces the whole library'); + + const edited = { type: 'practice', identity: identity(98), patch: {}, recent: true }; + await edit(first, edited); + const newest = await read(first); + await first.evaluate(async ({ items }) => { + const { syncConfig } = await chrome.storage.local.get('syncConfig'); + await chrome.storage.local.set({ syncConfig: { ...syncConfig, lastPushAt: 0 } }); + await chrome.storage.sync.set(items); + }, { items: remoteItems }); + await sync(first, 'now'); + assert.deepEqual(await readSnapshot(await first.evaluate(() => chrome.storage.sync.get(null))), newest.shared); + console.log('PASS an older remote snapshot is replaced by the newer local copy'); + await sync(first, 'disable'); await first.evaluate(() => chrome.storage.sync.clear()); await edit(first, { type: 'practice', identity: identity(99), patch: {}, recent: true }); const persisted = await read(first); + // Missing UI preferences must restore defaults without losing saved work. + // Close the panels before seeding raw storage, so their watches only see + // normalized data from the background when they reopen. + const worker = await (await browser.waitForTarget((target) => target.type() === 'service_worker')).worker(); + await first.close(); + await second.close(); + await worker.evaluate(async () => { + const { library } = await chrome.storage.local.get('library'); + delete library.local.uiPrefs; + await chrome.storage.local.set({ library }); + }); await browser.close(); browser = await launch(); const reopened = await panel(); - assert.equal(canonical((await read(reopened)).shared), canonical(persisted.shared)); + await reopened.waitForSelector('button[aria-label="Settings"]'); + const restored = await read(reopened); + assert.deepEqual(restored.local, { ...persisted.local, uiPrefs: DEFAULT_UI_PREFS }); + assert.deepEqual(await reopened.evaluate(async () => (await chrome.storage.local.get('library')).library), restored); + console.log('PASS missing preferences recover on restart without losing saved work'); + assert.deepEqual((await read(reopened)).shared, persisted.shared); await reopened.evaluate(async () => { const { syncConfig } = await chrome.storage.local.get('syncConfig'); await chrome.storage.local.set({ syncConfig: { ...syncConfig, lastPushAt: 0 } }); }); await sync(reopened, 'enable'); - const remoteKeys = await reopened.evaluate(async () => Object.keys(await chrome.storage.sync.get(null))); - assert.ok(remoteKeys.includes('nbn4:song:' + identity(99).key)); + const uploaded = await readSnapshot(await reopened.evaluate(() => chrome.storage.sync.get(null))); + assert.deepEqual(uploaded, persisted.shared); console.log('PASS restart preserves saved work and uploads it without the original panels'); - const large = Array.from({ length: 1000 }, (_, n) => ({ id: `m${n}`, t: n, label: crypto.randomUUID() })); + // A failed/interrupted upload is repaired from the local complete snapshot. + await reopened.evaluate(async (key) => { + const { syncConfig } = await chrome.storage.local.get('syncConfig'); + await chrome.storage.local.set({ syncConfig: { ...syncConfig, lastPushAt: 0 } }); + await chrome.storage.sync.set({ [key]: 'interrupted' }); + }, PREFIX + 'chunk:0'); + await sync(reopened, 'now'); + assert.deepEqual(await readSnapshot(await reopened.evaluate(() => chrome.storage.sync.get(null))), persisted.shared); + console.log('PASS an interrupted local upload is repaired'); + + const large = Array.from({ length: 5000 }, (_, n) => ({ id: `m${n}`, t: n, label: crypto.randomUUID() })); await edit(reopened, { type: 'practice', identity: identity(100), patch: { markers: large }, recent: true }); await sync(reopened, 'now'); const config = await reopened.evaluate(async () => (await chrome.storage.local.get('syncConfig')).syncConfig); - assert.match(config.lastError, /too large/); - assert.equal((await read(reopened)).shared.songs[identity(100).key].practice.value.markers.length, 1000); + assert.match(config.lastError, /storage is full/); + assert.equal((await read(reopened)).shared.songs[identity(100).key].practice.markers.length, 5000); + assert.deepEqual(await readSnapshot(await reopened.evaluate(() => chrome.storage.sync.get(null))), persisted.shared); console.log('PASS sync capacity errors retain every local marker'); await sync(reopened, 'delete'); assert.equal(Object.keys(await reopened.evaluate(() => chrome.storage.sync.get(null))).length, 0); - assert.equal((await read(reopened)).shared.songs[identity(100).key].practice.value.markers.length, 1000); + assert.equal((await read(reopened)).shared.songs[identity(100).key].practice.markers.length, 5000); console.log('PASS deleting the remote copy disables sync and preserves the local library'); + assert.deepEqual(panelErrors, [], 'panels must not report unhandled errors'); } finally { await browser?.close(); } diff --git a/src/core/features.ts b/src/core/features.ts deleted file mode 100644 index 91f555c..0000000 --- a/src/core/features.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { EngineEvent } from './messaging/protocol'; -import { chordsFeature } from '../features/chords/panel/panel'; -import { eqFeature } from '../features/eq/panel/panel'; -import { libraryFeature } from '../features/library/panel/panel'; -import { settingsFeature } from '../features/settings/panel/panel'; - -/** The snapshot variant of EngineEvent (the engine's full "current state"). */ -export type SnapshotEvent = Extract; - -/** A side-panel feature's registration surface. The composition roots — App - * (boot init) and the connection manager (port event routing) — iterate these - * instead of hard-coding each feature. This is the "light registration" seam: - * core imports the feature contributions; features never import the roots. - * A feature only implements the hooks it needs. */ -export interface PanelFeature { - /** Async storage load at panel boot. Run concurrently across features. */ - init?(): Promise | void; - /** Route an engine→panel event into feature-owned state that lives outside - * the session mirror (e.g. chords' PCM stream). */ - routeEvent?(event: EngineEvent): void; - /** React to a fresh engine snapshot (the full current-state message). */ - onSnapshot?(snapshot: SnapshotEvent): void; - /** The engine port disconnected (navigation/reload/teardown). */ - onDisconnect?(): void; -} - -/** Panel boot and engine-event contributions. */ -export const features: PanelFeature[] = [ - settingsFeature, - libraryFeature, - eqFeature, - chordsFeature, -]; diff --git a/src/core/model/defaults.ts b/src/core/model/defaults.ts index dc56ec6..909d818 100644 --- a/src/core/model/defaults.ts +++ b/src/core/model/defaults.ts @@ -44,15 +44,6 @@ export function countInDurationMs(beats: number, bpm: number): number { export const HISTORY_LIMIT = 200; -/** A saved song lives while it is favorited, in Recent, or among the most - * recently opened. Past that the record is dropped, because every song is one - * `browser.storage.sync` item and the browser caps those at 512. */ -export const SONG_LIMIT = 300; -/** Dropped songs stay as dated deletions so the removal crosses devices. Only - * the newest are kept; a device offline for longer than that many deletions - * re-adds its own copy, which is the cost of never growing without bound. */ -export const DELETION_LIMIT = 100; - export const DEFAULT_KEYMAP: Record = { playPause: 'Space', seekBack: 'ArrowLeft', diff --git a/src/core/model/types.ts b/src/core/model/types.ts index 72aeb6c..2398e89 100644 --- a/src/core/model/types.ts +++ b/src/core/model/types.ts @@ -138,9 +138,7 @@ export interface HistoryEntry { /** A song the user starred (History → Favorites). Persists independently of * the LRU-capped Recent list. Stored array order = manual sort order. */ export interface FavoriteEntry extends HistoryEntry { - /** When the star was put there — the favorite's own revision date, kept - * apart from `updatedAt` so ordinary practice cannot outdate an unfavorite - * made on another device. */ + /** When the song was starred, for display and sorting. */ favoritedAt: number; /** Last time the track was opened or played, for "Last Accessed" sorting. */ lastAccessedAt: number; diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index c4576ac..6a553ef 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -1,12 +1,12 @@ import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; import { songKey } from '../model/track-identity.ts'; -import { type Library, type SharedLibrary } from './library.ts'; +import type { Library, SharedLibrary } from './library.ts'; import { parseBackupJson as parseLegacy } from './legacy-backup.ts'; import { migrateBackup } from './library-migration.ts'; export const BACKUP_FORMAT = 'note-by-note-backup'; -export const BACKUP_VERSION = 4; -export interface Backup extends Library { format: typeof BACKUP_FORMAT; version: 4; exportedAt: number } +export const BACKUP_VERSION = 2; +export interface Backup extends Library { format: typeof BACKUP_FORMAT; version: typeof BACKUP_VERSION; exportedAt: number } function object(value: unknown): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Damaged library data.'); @@ -22,15 +22,9 @@ function array(value: unknown): any[] { if (!Array.isArray(value)) throw new Error('Damaged library list.'); return value; } -function versioned(value: unknown) { - const item = object(value); - number(item.at); - if (item.at < 0 || !('value' in item)) throw new Error('Damaged library revision.'); - return item; -} -/** Validate/backfill a JSON object against its version's defaults. */ +/** Missing preference groups use defaults, just like missing individual fields. */ function defaults(value: unknown, fallback: T): T { - const source = object(value); + const source = object(value ?? {}); const result = structuredClone(fallback) as Record; for (const [key, expected] of Object.entries(result)) { if (!(key in source)) continue; @@ -46,22 +40,17 @@ function defaults(value: unknown, fallback: T): T { } export function parseShared(value: unknown): SharedLibrary { const shared = structuredClone(object(value)); - shared.settings = versioned(shared.settings); - shared.settings.value = defaults(shared.settings.value, DEFAULT_SETTINGS); - shared.favoriteOrder = versioned(shared.favoriteOrder); - array(shared.favoriteOrder.value).forEach(string); + number(shared.updatedAt); + if (shared.updatedAt < 0) throw new Error('Damaged library revision.'); + shared.settings = defaults(shared.settings, DEFAULT_SETTINGS); + array(shared.favoriteOrder).forEach(string); for (const [key, raw] of Object.entries(object(shared.songs))) { const song = object(raw); - song.practice = versioned(song.practice); - song.favorite = versioned(song.favorite); - if (typeof song.favorite.value !== 'boolean') throw new Error('Damaged favorite.'); - const practice = song.practice.value; - // Checked ahead of the tombstone skip: a deleted song carries no identity to - // match the key against, so the shape is all that stands between an imported - // file and an arbitrary property name in the songs map. + if (song.favoritedAt !== null) number(song.favoritedAt); + const practice = song.practice; if (!/^(yt|file|web):/.test(key)) throw new Error('Damaged song key.'); - if (practice === null) continue; object(practice); + number(practice.updatedAt); const identity = object(practice.identity); string(identity.normalizedUrl); string(identity.title); number(identity.durationSec); if (key !== songKey(identity as any)) throw new Error('Song identity does not match its key.'); @@ -79,25 +68,23 @@ export function parseShared(value: unknown): SharedLibrary { if (typeof practice.sequenceLoop !== 'boolean' || typeof practice.sequenceCountIn !== 'boolean') throw new Error('Damaged sequence.'); if (practice.chordsEnabled !== undefined && typeof practice.chordsEnabled !== 'boolean') throw new Error('Damaged chord setting.'); } - for (const raw of Object.values(object(shared.presets))) { - const preset = versioned(raw); - if (preset.value !== null) array(preset.value).forEach(number); - } - return { settings: shared.settings, songs: shared.songs, presets: shared.presets, favoriteOrder: shared.favoriteOrder }; + for (const preset of Object.values(object(shared.presets))) array(preset).forEach(number); + return { updatedAt: shared.updatedAt, settings: shared.settings, songs: shared.songs, + presets: shared.presets, favoriteOrder: shared.favoriteOrder }; } export function parseLibrary(value: unknown): Library { const source = object(value); - const local = object(source.local); - const charts = object(local.charts); + const local = object(source.local ?? {}); + const charts = object(local.charts ?? {}); for (const chart of Object.values(charts)) { if (chart === null) continue; object(chart); number(chart.computedAt); number(chart.coverage); number(chart.analyzedFrom); number(chart.analyzedTo); array(chart.segments).forEach((s) => { object(s); number(s.startT); number(s.endT); string(s.label); number(s.confidence); }); if (chart.key !== null) { object(chart.key); string(chart.key.tonic); string(chart.key.mode); number(chart.key.confidence); } } - const recent = object(local.recent); + const lastAccessed = object(local.lastAccessed ?? {}); + const recent = object(local.recent ?? {}); Object.values(recent).forEach(number); - const lastAccessed = object(local.lastAccessed); Object.values(lastAccessed).forEach(number); return { shared: parseShared(source.shared), @@ -109,6 +96,6 @@ export function parseBackupJson(value: unknown): Backup { const raw = object(value); if (raw.format !== BACKUP_FORMAT) throw new Error("That file isn't a Note by Note backup."); if (raw.version > BACKUP_VERSION) throw new Error('That backup was made by a newer version of Note by Note.'); - const library = raw.version === BACKUP_VERSION ? parseLibrary(raw) : migrateBackup(parseLegacy(raw)); + const library = parseLibrary(raw.version === BACKUP_VERSION ? raw : migrateBackup(parseLegacy(raw))); return { format: BACKUP_FORMAT, version: BACKUP_VERSION, exportedAt: raw.exportedAt ?? 0, ...library }; } diff --git a/src/core/persist/legacy-backup.ts b/src/core/persist/legacy-backup.ts index fdd37eb..3d6e962 100644 --- a/src/core/persist/legacy-backup.ts +++ b/src/core/persist/legacy-backup.ts @@ -17,29 +17,14 @@ const BACKUP_FORMAT = 'note-by-note-backup'; const BACKUP_VERSION = 1; -/** Rows in a v1 file carry sync bookkeeping the live model no longer has: a - * tombstone told an old merge "removed" from "never had it", and every row and - * section carried its own date. `library-migration.ts` reads them once, here. */ -export interface LegacyHistoryEntry extends HistoryEntry { - deleted?: true; -} -export interface LegacyFavoriteEntry extends FavoriteEntry, LegacyHistoryEntry { - /** When the manual order this row sat in was last set. */ - orderedAt?: number; -} -export interface LegacyEqPreset extends EqPreset { - updatedAt?: number; - deleted?: true; -} - export interface Backup { format: typeof BACKUP_FORMAT; version: number; - settings: Settings & { updatedAt?: number }; + settings: Settings; uiPrefs: UiPrefs; - history: LegacyHistoryEntry[]; - favorites: LegacyFavoriteEntry[]; - eqPresets: LegacyEqPreset[]; + history: HistoryEntry[]; + favorites: FavoriteEntry[]; + eqPresets: EqPreset[]; /** Per-track markers and snippets, one entry per saved track. */ tracks: TrackData[]; } @@ -73,14 +58,14 @@ function normalizeV1(raw: Record): Backup { settings: { ...DEFAULT_SETTINGS, ...(isRecord(raw.settings) ? raw.settings : {}), - } as Settings & { updatedAt?: number }, + } as Settings, uiPrefs: { ...(JSON.parse(JSON.stringify(DEFAULT_UI_PREFS)) as UiPrefs), ...(isRecord(raw.uiPrefs) ? raw.uiPrefs : {}), }, - history: rekeyByIdentity(identifiedArr(raw.history, 'history')), - favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), - eqPresets: arr(raw.eqPresets, 'eqPresets') as LegacyEqPreset[], + history: rekeyByIdentity(identifiedArr(raw.history, 'history')), + favorites: rekeyByIdentity(identifiedArr(raw.favorites, 'favorites')), + eqPresets: arr(raw.eqPresets, 'eqPresets') as EqPreset[], tracks: rekeyByIdentity(identifiedArr(raw.tracks, 'tracks')), }; } diff --git a/src/core/persist/library-background.ts b/src/core/persist/library-background.ts index f2be912..658b5c7 100644 --- a/src/core/persist/library-background.ts +++ b/src/core/persist/library-background.ts @@ -1,10 +1,10 @@ import { onMessage } from '../messaging/rpc'; -import { applyCommand, canonical, emptyLibrary, mergeShared, pruned, type Library } from './library'; +import { applyCommand, emptyLibrary, newestSnapshot, type Library } from './library'; import { libraryItem } from './library-client'; import { parseBackupJson as parseLegacy } from './legacy-backup'; import { parseLibrary } from './backup-codec'; import { migrateBackup } from './library-migration'; -import { bytesUsed, changedRecords, PREFIX, readRecords, skippedMessage } from '../../features/sync/persist/records'; +import { bytesUsed, encodeSnapshot, IncompleteSnapshot, PREFIX, readSnapshot, SNAPSHOT_KEY } from '../../features/sync/persist/records'; import { loadSyncConfig, syncConfigItem } from '../../features/sync/persist/sync-config'; const WAKE = 'library-sync'; @@ -13,7 +13,7 @@ const SAFETY = 'library-sync-safety'; * it and still lets a burst of edits ride out together. */ const PUSH_INTERVAL = 30000; -/** One writer for edits, imports and merges. Persisted records survive worker restarts. */ +/** One writer for edits, imports and sync. Saved snapshots survive worker restarts. */ export function startLibraryBackground() { let queue: Promise = Promise.resolve(); const enqueue = (work: () => Promise): Promise => { @@ -31,12 +31,12 @@ export function startLibraryBackground() { const identified = (value: unknown) => (Array.isArray(value) ? value : []).filter((row) => typeof (row as { identity?: { normalizedUrl?: unknown } })?.identity?.normalizedUrl === 'string'); try { - return migrateBackup(parseLegacy({ format: 'note-by-note-backup', version: 1, + return parseLibrary(migrateBackup(parseLegacy({ format: 'note-by-note-backup', version: 1, settings: raw.settings, uiPrefs: raw.uiPrefs, history: identified(raw.history), favorites: identified(raw.favorites), eqPresets: Array.isArray(raw.eqPresets) ? raw.eqPresets : [], tracks: identified(Object.entries(raw).filter(([key]) => key.startsWith('track:')).map(([, value]) => value)), - })); + }))); } catch (error) { console.error('[note-by-note] the previous library could not be migrated; its records are kept', error); return defaults; @@ -45,10 +45,12 @@ export function startLibraryBackground() { let ready: Promise | undefined; const init = () => ready ??= (async () => { // Only the migration needs every old key. Ordinary wakes read one item. - if ((await browser.storage.local.get('library')).library) return; - // Pruned on the way in: an old library can hold far more songs than sync - // allows, and without this the first reconcile would fail on every retry. - await libraryItem.setValue(pruned(migrate(await browser.storage.local.get(null)))); + const { library } = await browser.storage.local.get('library'); + if (library) { + await libraryItem.setValue(parseLibrary(library)); + } else { + await libraryItem.setValue(migrate(await browser.storage.local.get(null))); + } })().catch((error) => { ready = undefined; throw error; }); const schedule = async () => { const config = await loadSyncConfig(); @@ -62,24 +64,24 @@ export function startLibraryBackground() { try { const existing = await browser.storage.sync.get(null); config.usedBytes = bytesUsed(existing); - const remote = await readRecords(existing); const local = await libraryItem.getValue(); - // Pruned here too: a merge can carry in more songs than the limits allow, - // and only `applyCommand` would otherwise ever bring it back under them. - const merged = pruned({ ...local, shared: mergeShared(local.shared, remote) }); - if (canonical(merged) !== canonical(local)) await libraryItem.setValue(merged); - const { changes, removals, skipped, usedBytes } = await changedRecords(merged.shared, remote, existing); - // Recorded before the throttle returns, or a merge that succeeded would - // leave the previous run's error on screen until a push happens to be due. - config.lastSyncedAt = Date.now(); - config.lastError = skipped.length ? skippedMessage(skipped) : null; - if (Object.keys(changes).length || removals.length) { + const remote = await readSnapshot(existing).catch((error) => { + // Repair an interrupted upload from our complete local copy. A newer + // incomplete snapshot must finish arriving before we can choose a winner. + if (error instanceof IncompleteSnapshot && local.shared.updatedAt >= error.updatedAt) return null; + throw error; + }); + const shared = remote ? newestSnapshot(local.shared, remote) : local.shared; + if (shared !== local.shared) await libraryItem.setValue({ ...local, shared }); + config.lastError = null; + if (shared !== remote || !existing[SNAPSHOT_KEY]) { + const { items, usedBytes } = await encodeSnapshot(shared, existing); if (Date.now() < config.lastPushAt + PUSH_INTERVAL) { await schedule(); return; } - if (removals.length) await browser.storage.sync.remove(removals); - if (Object.keys(changes).length) await browser.storage.sync.set(changes); + await browser.storage.sync.set(items); config.lastPushAt = Date.now(); config.usedBytes = usedBytes; } + config.lastSyncedAt = Date.now(); } catch (error) { config.lastError = error instanceof Error ? error.message : String(error); } finally { @@ -93,7 +95,7 @@ export function startLibraryBackground() { const current = await libraryItem.getValue(); const next = applyCommand(current, data); await libraryItem.setValue(next); - if (canonical(current.shared) !== canonical(next.shared)) await schedule(); + if (current.shared.updatedAt !== next.shared.updatedAt) await schedule(); })); onMessage('librarySync', ({ data }) => enqueue(async () => { const config = await loadSyncConfig(); @@ -125,7 +127,7 @@ export function startLibraryBackground() { // Recreate the safety alarm on each worker start. No panel has to stay open. // Only a net: `libraryEdit` schedules WAKE and `storage.onChanged` catches // remote writes, so this never needs to be the thing that notices a change — - // and each run wakes the worker to decompress every record. + // and each run wakes the worker to read the shared snapshot. void browser.alarms.create(SAFETY, { periodInMinutes: 30 }); void enqueue(reconcile); } diff --git a/src/core/persist/library-client.ts b/src/core/persist/library-client.ts index ff026a5..36b2938 100644 --- a/src/core/persist/library-client.ts +++ b/src/core/persist/library-client.ts @@ -1,17 +1,10 @@ import { storage } from '#imports'; import { sendMessage } from '../messaging/rpc'; -import { canonical, emptyLibrary, type Library, type LibraryCommand } from './library'; +import { emptyLibrary, type Library, type LibraryCommand } from './library'; /** Panels observe the library; the background is its only writer. */ export const libraryItem = storage.defineItem('local:library', { fallback: emptyLibrary() }); -export const readLibrary = async () => await sendMessage('libraryRead', undefined); +export const readLibrary = async () => sendMessage('libraryRead', undefined); export const editLibrary = async (command: LibraryCommand): Promise => { await sendMessage('libraryEdit', JSON.parse(JSON.stringify(command)) as LibraryCommand); }; - -export function watchLibrary(select: (library: Library) => T, listener: (value: T) => void) { - return libraryItem.watch((value, previous) => { - const selected = select(value ?? emptyLibrary()); - if (canonical(selected) !== canonical(select(previous ?? emptyLibrary()))) listener(selected); - }); -} diff --git a/src/core/persist/library-migration.ts b/src/core/persist/library-migration.ts index 29fa0ac..8a4316a 100644 --- a/src/core/persist/library-migration.ts +++ b/src/core/persist/library-migration.ts @@ -1,55 +1,54 @@ import { DEFAULT_PARAMS } from '../model/defaults.ts'; import { makeTrackIdentity } from '../model/track-identity.ts'; -import { cell, defineEntry, emptyLibrary, newest, type Library, type Practice, type SavedSong } from './library.ts'; +import { defineEntry, emptyLibrary, type Library, type Practice, type SavedSong } from './library.ts'; import type { Backup } from './legacy-backup.ts'; /** Collapse old Recent/Favorites/track copies once, at the boundary. */ export function migrateBackup(backup: Backup): Library { const library = emptyLibrary(); const { shared, local } = library; - const { lastUsedParams, updatedAt, ...settings } = backup.settings; - shared.settings = cell({ ...shared.settings.value, ...settings }, updatedAt ?? 0); + const { lastUsedParams, ...settings } = backup.settings; + shared.settings = { ...shared.settings, ...settings }; local.lastUsedParams = lastUsedParams; local.uiPrefs = { ...local.uiPrefs, ...backup.uiPrefs }; const ensure = (identity: Practice['identity']): SavedSong => { const normalized = makeTrackIdentity(identity.normalizedUrl, identity.title, identity.durationSec); return shared.songs[normalized.key] ??= { - practice: cell({ identity: normalized, pageUrl: normalized.normalizedUrl, markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false }), - favorite: cell(false), + practice: { identity: normalized, pageUrl: normalized.normalizedUrl, updatedAt: 0, + markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false }, + favoritedAt: null, }; }; + const observe = (at = 0) => { shared.updatedAt = Math.max(shared.updatedAt, at); }; // History dates saved parameters; a favorite's star date must not outrank an edit. - const entries = [...backup.favorites, ...backup.history].filter((entry) => !entry.deleted) + const entries = [...backup.favorites, ...backup.history] .sort((a, b) => (a.updatedAt ?? 0) - (b.updatedAt ?? 0)); for (const entry of entries) { const song = ensure(entry.identity); - song.practice = cell({ ...song.practice.value!, params: { ...DEFAULT_PARAMS, ...entry.params }, - pageUrl: entry.pageUrl, thumbnailUrl: entry.thumbnailUrl }, entry.updatedAt ?? 0); + song.practice = { ...song.practice, params: { ...DEFAULT_PARAMS, ...entry.params }, + pageUrl: entry.pageUrl, thumbnailUrl: entry.thumbnailUrl, updatedAt: entry.updatedAt ?? 0 }; + observe(entry.updatedAt); } - const tracks = [...backup.tracks].sort((a, b) => a.updatedAt - b.updatedAt); - for (const track of tracks) { + for (const track of [...backup.tracks].sort((a, b) => a.updatedAt - b.updatedAt)) { const song = ensure(track.identity); const { identity, updatedAt, chordChart, ...data } = track; - song.practice = cell({ ...song.practice.value!, ...data }, Math.max(updatedAt, song.practice.at)); - local.charts[song.practice.value!.identity.key] = chordChart ?? null; + song.practice = { ...song.practice, ...data, updatedAt: Math.max(updatedAt, song.practice.updatedAt) }; + local.charts[song.practice.identity.key] = chordChart ?? null; + observe(updatedAt); } for (const entry of backup.history) { - if (entry.deleted) continue; - const key = ensure(entry.identity).practice.value!.identity.key; + const key = ensure(entry.identity).practice.identity.key; local.recent[key] = entry.updatedAt; local.lastAccessed[key] = entry.updatedAt; } for (const entry of backup.favorites) { const song = ensure(entry.identity); - song.favorite = newest(song.favorite, cell(!entry.deleted, entry.deleted ? entry.updatedAt : entry.favoritedAt)); - const key = song.practice.value!.identity.key; + song.favoritedAt = entry.favoritedAt; + const key = song.practice.identity.key; local.lastAccessed[key] = Math.max(local.lastAccessed[key] ?? 0, entry.lastAccessedAt); + observe(entry.favoritedAt); } - shared.favoriteOrder = cell(backup.favorites.filter((f) => !f.deleted) - .map((f) => ensure(f.identity).practice.value!.identity.key), - Math.max(0, ...backup.favorites.map((f) => f.orderedAt ?? f.favoritedAt ?? 0))); - for (const preset of backup.eqPresets) { - defineEntry(shared.presets, preset.name, cell(preset.deleted ? null : preset.gains, preset.updatedAt ?? 0)); - } + shared.favoriteOrder = backup.favorites.map((f) => ensure(f.identity).practice.identity.key); + for (const preset of backup.eqPresets) defineEntry(shared.presets, preset.name, preset.gains); return library; } diff --git a/src/core/persist/library.test.ts b/src/core/persist/library.test.ts index 39609ca..4a5da5e 100644 --- a/src/core/persist/library.test.ts +++ b/src/core/persist/library.test.ts @@ -1,17 +1,16 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { applyCommand, canonical, cell, emptyLibrary, favoriteEntries, mergeShared, nextRevision, pruned, recentEntries } from './library.ts'; -import { migrateBackup } from './library-migration.ts'; -import { parseBackupJson } from './backup-codec.ts'; -import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, DELETION_LIMIT, SONG_LIMIT } from '../model/defaults.ts'; +import { applyCommand, emptyLibrary, favoriteEntries, newestSnapshot, recentEntries } from './library.ts'; +import { parseBackupJson, parseLibrary } from './backup-codec.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; import { makeTrackIdentity } from '../model/track-identity.ts'; const identity = makeTrackIdentity('https://www.youtube.com/watch?v=example', 'Song', 200); -const save = (library = emptyLibrary(), speed = 0.8) => applyCommand(library, { +const save = (library = emptyLibrary(), speed = 0.8, now = 100) => applyCommand(library, { type: 'practice', identity, patch: { params: { ...DEFAULT_PARAMS, speed } }, recent: true, -}, 100); +}, now); -test('Recent and Favorites project the same saved parameters; removing Recent preserves the song', () => { +test('Recent and Favorites project the same saved parameters; removing Recent preserves a favorite', () => { let library = applyCommand(save(), { type: 'favorite', key: identity.key, value: true }, 200); library = save(library, 0.5); assert.equal(recentEntries(library)[0].params.speed, 0.5); @@ -19,145 +18,170 @@ test('Recent and Favorites project the same saved parameters; removing Recent pr library = applyCommand(library, { type: 'recent.remove', key: identity.key }); assert.equal(recentEntries(library).length, 0); assert.equal(favoriteEntries(library)[0].params.speed, 0.5); - assert.equal(Object.keys(library.shared.songs).length, 1); library = applyCommand(library, { type: 'visit', key: identity.key }, 1000); assert.equal(recentEntries(library).length, 0); assert.equal(favoriteEntries(library)[0].lastAccessedAt, 1000); }); -test('an independent practice edit cannot undo an unfavorite', () => { +test('the newer complete snapshot wins, including conflicting edits and unrelated changes', () => { const base = applyCommand(save(), { type: 'favorite', key: identity.key, value: true }, 200); const removed = applyCommand(base, { type: 'favorite', key: identity.key, value: false }, 300); - const edited = save(base, 0.4); - const merged = mergeShared(edited.shared, removed.shared); - assert.equal(merged.songs[identity.key].favorite.value, false); - assert.equal(merged.songs[identity.key].practice.value!.params!.speed, 0.4); + const edited = save(base, 0.4, 400); + const winner = newestSnapshot(removed.shared, edited.shared); + assert.equal(winner, edited.shared); + assert.equal(newestSnapshot(edited.shared, removed.shared), edited.shared); + // The later edit wins as a whole, even though it carries the older star. + assert.equal(winner.songs[identity.key].favoritedAt, 200); + assert.equal(winner.songs[identity.key].practice.params!.speed, 0.4); + const otherDevice = applyCommand(emptyLibrary(), { type: 'preset', name: 'Only remote', gains: [2] }, 500); + assert.equal(newestSnapshot(edited.shared, otherDevice.shared), otherDevice.shared); + assert.deepEqual(otherDevice.shared.songs, {}); }); -test('merge is commutative, associative and idempotent, including deletions and order', () => { - const a = save().shared; - const b = applyCommand(save(), { type: 'favorite', key: identity.key, value: true }, 200).shared; - const c = applyCommand(save(), { type: 'import', library: emptyLibrary() }, 300).shared; - for (const x of [a, b, c]) for (const y of [a, b, c]) for (const z of [a, b, c]) { - assert.equal(canonical(mergeShared(x, y)), canonical(mergeShared(y, x))); - assert.equal(canonical(mergeShared(x, x)), canonical(x)); - assert.equal(canonical(mergeShared(mergeShared(x, y), z)), canonical(mergeShared(x, mergeShared(y, z)))); - } +test('equal timestamps adopt the synced snapshot and stay settled', () => { + const local = save(emptyLibrary(), 0.8, 100).shared; + const remote = save(emptyLibrary(), 0.5, 100).shared; + assert.equal(newestSnapshot(local, remote), remote); + assert.equal(newestSnapshot(remote, remote), remote); }); -test('local activity, layout, last-used parameters and chord analysis never change shared data', () => { +test('local activity, layout, last-used parameters and chord analysis never date shared data', () => { const original = save(); let next = applyCommand(original, { type: 'visit', key: identity.key }); next = applyCommand(next, { type: 'chart', key: identity.key, chart: null }); - next = applyCommand(next, { type: 'uiPrefs', value: { ...DEFAULT_UI_PREFS, markerView: 'list' } }); + next = applyCommand(next, { type: 'uiPrefs', patch: { markerView: 'list' } }); next = applyCommand(next, { type: 'settings', patch: { lastUsedParams: { ...DEFAULT_PARAMS, speed: 0.2 } } }); assert.deepEqual(next.shared, original.shared); }); -test('removing from history deletes the saved song, and the deletion crosses devices', () => { - const library = save(); - const removed = applyCommand(library, { type: 'recent.remove', key: identity.key }, 300); - assert.equal(removed.shared.songs[identity.key].practice.value, null); - assert.equal(recentEntries(removed).length, 0); - assert.equal(removed.local.lastAccessed[identity.key], undefined); - // An older copy from another device must not resurrect it. - assert.equal(mergeShared(library.shared, removed.shared).songs[identity.key].practice.value, null); - const cleared = applyCommand(library, { type: 'recent.remove' }, 300); - assert.equal(cleared.shared.songs[identity.key].practice.value, null); -}); - -test('saved songs stay inside the sync record limit, keeping favorites and the newest', () => { - const track = (n: number) => makeTrackIdentity('https://www.youtube.com/watch?v=s' + n, 'Song ' + n, 200); - let library = emptyLibrary(); - for (let n = 0; n < SONG_LIMIT + DELETION_LIMIT + 20; n++) { - library = applyCommand(library, { type: 'practice', identity: track(n), patch: {}, recent: false }, 100 + n); - if (n === 0) library = applyCommand(library, { type: 'favorite', key: track(0).key, value: true }, 100 + n); - } - const live = Object.values(library.shared.songs).filter((song) => song.practice.value !== null); - assert.equal(live.length, SONG_LIMIT); - assert.ok(library.shared.songs[track(0).key].practice.value, 'a favorite outlives the limit'); - assert.equal(library.shared.songs[track(1).key], undefined, 'the oldest deletions are finally forgotten'); - // Deletions are bounded too, so the synced record count cannot grow forever. - assert.ok(Object.keys(library.shared.songs).length <= SONG_LIMIT + DELETION_LIMIT); - assert.deepEqual(library.shared.favoriteOrder.value, [track(0).key]); +test('deleting songs and presets removes them from the winning snapshot without tombstones', () => { + const library = applyCommand(save(), { type: 'preset', name: 'Old', gains: [1] }, 200); + let removed = applyCommand(library, { type: 'recent.remove', key: identity.key }, 300); + removed = applyCommand(removed, { type: 'preset', name: 'Old', gains: null }, 400); + assert.deepEqual(removed.shared.songs, {}); + assert.deepEqual(removed.shared.presets, {}); + assert.deepEqual(removed.local.recent, {}); + assert.deepEqual(removed.local.lastAccessed, {}); + assert.equal(newestSnapshot(library.shared, removed.shared), removed.shared); + assert.deepEqual(applyCommand(library, { type: 'recent.remove' }).shared.songs, {}); }); -test('field patches preserve other session and remote edits', () => { +test('field patches from panels preserve other changes on the same device', () => { let library = save(); library = applyCommand(library, { type: 'practice', identity, patch: { markers: [{ id: 'm', t: 4, label: 'Verse' }] }, recent: true }); library = save(library, 0.3); - assert.deepEqual(library.shared.songs[identity.key].practice.value!.markers, [{ id: 'm', t: 4, label: 'Verse' }]); + assert.deepEqual(library.shared.songs[identity.key].practice.markers, [{ id: 'm', t: 4, label: 'Verse' }]); +}); + +test('preference patches preserve other controls and do not create a sync revision', () => { + const original = save(); + let library = applyCommand(original, { type: 'uiPrefs', patch: { collapsedSections: { looper: true } } }); + library = applyCommand(library, { type: 'uiPrefs', patch: { collapsedSections: { tools: true } } }); + library = applyCommand(library, { type: 'uiPrefs', patch: { boundaryLabels: { start: 'Intro' } } }); + library = applyCommand(library, { type: 'uiPrefs', patch: { boundaryLabels: { end: 'Outro' }, markerView: 'list' } }); + assert.deepEqual(library.local.uiPrefs.collapsedSections, { ...DEFAULT_UI_PREFS.collapsedSections, looper: true, tools: true }); + assert.deepEqual(library.local.uiPrefs.boundaryLabels, { start: 'Intro', end: 'Outro' }); + assert.equal(library.local.uiPrefs.markerView, 'list'); + assert.deepEqual(library.shared, original.shared); +}); + +test('the storage boundary fills defaults for older settings and nested UI preferences', () => { + const older = JSON.parse(JSON.stringify(save())); + delete older.shared.settings.keymap.playPause; + delete older.shared.settings.countInBeep; + delete older.local.uiPrefs.collapsedSections.looper; + delete older.local.uiPrefs.boundaryLabels; + const restored = parseLibrary(older); + assert.equal(restored.shared.settings.keymap.playPause, DEFAULT_SETTINGS.keymap.playPause); + assert.equal(restored.shared.settings.countInBeep, DEFAULT_SETTINGS.countInBeep); + assert.equal(restored.local.uiPrefs.collapsedSections.looper, DEFAULT_UI_PREFS.collapsedSections.looper); + assert.deepEqual(restored.local.uiPrefs.boundaryLabels, DEFAULT_UI_PREFS.boundaryLabels); }); -test('replacement import dates both present and absent records after observed future revisions', () => { - let current = save(); - current = applyCommand(current, { type: 'preset', name: 'Old', gains: [1] }, 100000); +test('missing or null preference groups use defaults without changing saved songs or dates', () => { + const saved = applyCommand(save(), { type: 'favorite', key: identity.key, value: true }, 200); + for (const absent of [undefined, null]) { + const older = { ...saved, local: { ...saved.local, uiPrefs: absent } }; + const before = structuredClone(older); + assert.deepEqual(parseLibrary(older), saved); + assert.deepEqual(older, before); + const partial = { ...saved, shared: { ...saved.shared, settings: { theme: 'dark', keymap: absent } }, + local: { ...saved.local, uiPrefs: { markerView: 'list', collapsed: absent, collapsedSections: absent, boundaryLabels: absent } } }; + const restored = parseLibrary(partial); + assert.deepEqual(restored.shared.settings, { ...DEFAULT_SETTINGS, theme: 'dark' }); + assert.deepEqual(restored.local.uiPrefs, { ...DEFAULT_UI_PREFS, markerView: 'list' }); + assert.deepEqual(restored.shared.songs, saved.shared.songs); + assert.deepEqual(parseLibrary(restored), restored); + assert.deepEqual(parseLibrary({ ...saved, shared: { ...saved.shared, settings: absent } }), saved); + } +}); + +test('absent local metadata defaults independently while malformed present data is rejected', () => { + const saved = save(); + for (const absent of [undefined, null]) { + for (const key of ['recent', 'lastAccessed', 'charts']) { + const older = { ...saved, local: { ...saved.local, [key]: absent } }; + assert.deepEqual(parseLibrary(older), { ...saved, local: { ...saved.local, [key]: {} } }); + } + assert.deepEqual(parseLibrary({ ...saved, local: absent }), { ...saved, local: emptyLibrary().local }); + } + for (const invalid of [false, 1, 'invalid', []]) { + assert.throws(() => parseLibrary({ ...saved, local: { ...saved.local, uiPrefs: invalid } }), /data/); + } + assert.throws(() => parseLibrary({ ...saved, shared: null }), /data/); + assert.throws(() => parseLibrary({ ...saved, local: { ...saved.local, recent: { [identity.key]: NaN } } }), /number/); +}); + +test('replacement imports and subsequent edits advance beyond observed or imported future dates', () => { + const current = save(emptyLibrary(), 0.8, 100000); const replaced = applyCommand(current, { type: 'import', library: emptyLibrary() }, 1); - assert.equal(replaced.shared.songs[identity.key].practice.value, null); - assert.equal(replaced.shared.presets.Old.value, null); - assert.deepEqual(mergeShared(current.shared, replaced.shared), replaced.shared); - const revived = save(replaced); - assert.ok(revived.shared.songs[identity.key].practice.at > replaced.shared.songs[identity.key].practice.at); - assert.ok(nextRevision(revived.shared, 0) > 100000); + assert.deepEqual(replaced.shared.songs, {}); + assert.equal(replaced.shared.updatedAt, 100001); + assert.equal(newestSnapshot(current.shared, replaced.shared), replaced.shared); + assert.ok(save(replaced).shared.updatedAt > replaced.shared.updatedAt); + const future = save(emptyLibrary(), 0.5, 200000); + assert.equal(applyCommand(current, { type: 'import', library: future }, 1).shared.updatedAt, 200001); }); -test('migration collapses parameters, favorites and markers without syncing local data', () => { +test('released version-1 backups import parameters, favorites, markers and presets', () => { const entry = { identity, pageUrl: identity.normalizedUrl, params: { ...DEFAULT_PARAMS, speed: 0.7 }, updatedAt: 10 }; - const migrated = migrateBackup({ format: 'note-by-note-backup', version: 1, - settings: { ...DEFAULT_SETTINGS, lastUsedParams: DEFAULT_PARAMS }, uiPrefs: DEFAULT_UI_PREFS, eqPresets: [], + const migrated = parseBackupJson({ format: 'note-by-note-backup', version: 1, + settings: { ...DEFAULT_SETTINGS, lastUsedParams: DEFAULT_PARAMS }, uiPrefs: DEFAULT_UI_PREFS, + eqPresets: [{ name: 'Practice', gains: [1, 2, 3] }], history: [entry], favorites: [{ ...entry, params: DEFAULT_PARAMS, updatedAt: 5, favoritedAt: 15, lastAccessedAt: 16 }], tracks: [{ identity, updatedAt: 11, markers: [{ id: 'm', t: 42, label: '' }], snippets: [], sequenceLoop: false, sequenceCountIn: false, chordChart: null }], }); const song = migrated.shared.songs[identity.key]; - assert.equal(song.practice.value!.params!.speed, 0.7); - assert.equal(song.practice.value!.markers[0].t, 42); - assert.equal(song.favorite.value, true); + assert.equal(song.practice.params!.speed, 0.7); + assert.equal(song.practice.markers[0].t, 42); + assert.equal(song.favoritedAt, 15); + assert.equal(migrated.shared.updatedAt, 15); assert.equal(migrated.local.lastAccessed[identity.key], 16); - assert.equal(migrated.shared.settings.value.lastUsedParams, undefined); + assert.equal(migrated.shared.settings.lastUsedParams, undefined); + assert.deepEqual(migrated.shared.presets, { Practice: [1, 2, 3] }); + assert.deepEqual(migrated.local.lastUsedParams, DEFAULT_PARAMS); + assert.equal(migrated.version, 2); + const exported = JSON.parse(JSON.stringify(migrated)); + assert.deepEqual(parseBackupJson(exported), exported); }); -test('new backups round-trip complete data and reject malformed or unsupported formats', () => { - const backup = { format: 'note-by-note-backup', version: 4, exportedAt: 100, ...save() }; +test('new backups round-trip and reject malformed or unsupported formats', () => { + const backup = { format: 'note-by-note-backup', version: 2, exportedAt: 100, ...save() }; assert.deepEqual(parseBackupJson(JSON.parse(JSON.stringify(backup))), backup); - assert.throws(() => parseBackupJson({ ...backup, version: 5 }), /newer version/); - assert.throws(() => parseBackupJson({ ...backup, version: 3 }), /no longer reads/); + for (const version of [3, 4, 5]) assert.throws(() => parseBackupJson({ ...backup, version }), /newer version/); + assert.throws(() => parseBackupJson({ ...backup, version: 0 }), /no longer reads/); const damaged = structuredClone(backup); - damaged.shared.songs[identity.key].practice.value!.identity.key = 'old-key'; - assert.equal(parseBackupJson(damaged).shared.songs[identity.key].practice.value!.identity.key, identity.key); - damaged.shared.songs[identity.key].practice.value!.identity.normalizedUrl = 'https://different.example'; + damaged.shared.songs[identity.key].practice.identity.key = 'old-key'; + assert.equal(parseBackupJson(damaged).shared.songs[identity.key].practice.identity.key, identity.key); + damaged.shared.songs[identity.key].practice.identity.normalizedUrl = 'https://different.example'; assert.throws(() => parseBackupJson(damaged), /identity/); + assert.throws(() => parseBackupJson({ ...backup, shared: { ...backup.shared, updatedAt: -1 } }), /revision/); }); -test('preset names are data, including names matching object properties', () => { +test('preset names matching object properties can be saved, backed up and deleted', () => { let library = emptyLibrary(); for (const name of ['constructor', '__proto__']) library = applyCommand(library, { type: 'preset', name, gains: [1] }); - assert.deepEqual(Object.keys(library.shared.presets), ['constructor', '__proto__']); - assert.deepEqual(mergeShared(emptyLibrary().shared, library.shared).presets, library.shared.presets); -}); - -test('every write path prunes, so a merge or migration cannot leave the library oversized', () => { - const track = (n: number) => makeTrackIdentity('https://www.youtube.com/watch?v=p' + n, 'Song ' + n, 200); - // Built directly: a merge or a migration lands songs without going through - // applyCommand, which is the only place that used to prune. - const oversized = emptyLibrary(); - for (let n = 0; n < SONG_LIMIT + DELETION_LIMIT + 50; n++) { - const key = track(n).key; - oversized.shared.songs[key] = { practice: cell({ identity: track(n), pageUrl: track(n).normalizedUrl, - markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false }, 100 + n), favorite: cell(false, 100) }; - oversized.local.lastAccessed[key] = 100 + n; - } - const trimmed = pruned(oversized, 1000); - assert.ok(Object.keys(trimmed.shared.songs).length <= SONG_LIMIT + DELETION_LIMIT); - assert.equal(canonical(pruned(trimmed, 2000)), canonical(trimmed), 'idempotent, so it never manufactures a write'); -}); - -test('an imported tombstone cannot name an arbitrary object property', () => { - const backup = { format: 'note-by-note-backup', version: 4, exportedAt: 100, ...structuredClone(save()) }; - // A deleted song carries no identity to check the key against, so the key - // shape is the only thing standing between a file and the songs map. - Object.defineProperty(backup.shared.songs, '__proto__', { - value: { practice: { at: 0, value: null }, favorite: { at: 0, value: false } }, - enumerable: true, writable: true, configurable: true, - }); - assert.throws(() => parseBackupJson(backup), /song key/); + assert.deepEqual(Object.keys(parseLibrary(library).shared.presets), ['constructor', '__proto__']); + library = applyCommand(library, { type: 'preset', name: '__proto__', gains: null }); + assert.deepEqual(Object.keys(library.shared.presets), ['constructor']); }); diff --git a/src/core/persist/library.ts b/src/core/persist/library.ts index 3e71981..66d3a22 100644 --- a/src/core/persist/library.ts +++ b/src/core/persist/library.ts @@ -1,22 +1,23 @@ -import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, DELETION_LIMIT, HISTORY_LIMIT, SONG_LIMIT } from '../model/defaults.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS, HISTORY_LIMIT } from '../model/defaults.ts'; import type { ChordChart, EffectParams, FavoriteEntry, HistoryEntry, Settings, TrackData, TrackIdentity, UiPrefs } from '../model/types'; -/** One revision per independently editable value. Null/false are durable deletions. */ -interface Versioned { at: number; value: T } -export interface Practice extends Omit { +export interface Practice extends Omit { params?: EffectParams; pageUrl: string; thumbnailUrl?: string; } export interface SavedSong { - practice: Versioned; - favorite: Versioned; + practice: Practice; + /** Display metadata, not a sync revision. Null means not favorited. */ + favoritedAt: number | null; } +/** The same snapshot is saved locally and sent to browser sync. */ export interface SharedLibrary { - settings: Versioned; + updatedAt: number; + settings: Settings; songs: Record; - presets: Record>; - favoriteOrder: Versioned; + presets: Record; + favoriteOrder: string[]; } export interface Library { shared: SharedLibrary; @@ -29,59 +30,26 @@ export interface Library { }; } -export const cell = (value: T, at = 0): Versioned => ({ at, value }); -/** Song keys and preset names are user data, so they may spell an object - * property (`__proto__`, `constructor`). Defining the entry writes the map the - * plain assignment would only appear to. */ +/** Preset names may spell object properties such as __proto__. */ export function defineEntry(target: Record, key: string, value: T): void { Object.defineProperty(target, key, { value, enumerable: true, writable: true, configurable: true }); } export function emptyLibrary(): Library { return { - shared: { settings: cell(structuredClone(DEFAULT_SETTINGS)), songs: {}, presets: {}, favoriteOrder: cell([]) }, + shared: { updatedAt: 0, settings: structuredClone(DEFAULT_SETTINGS), songs: {}, presets: {}, favoriteOrder: [] }, local: { uiPrefs: structuredClone(DEFAULT_UI_PREFS), recent: {}, lastAccessed: {}, charts: {} }, }; } -/** Stable comparison for equal revisions; also the JSON form used on the wire. */ -export function canonical(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; - if (value && typeof value === 'object') { - return `{${Object.entries(value).filter(([, v]) => v !== undefined) - .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) - .map(([key, v]) => `${JSON.stringify(key)}:${canonical(v)}`).join(',')}}`; - } - return JSON.stringify(value) ?? 'null'; -} -export function newest(a: Versioned, b: Versioned): Versioned { - if (a.at === b.at) { - const deleted = (value: T) => value === null || value === false; - if (deleted(a.value) !== deleted(b.value)) return deleted(a.value) ? a : b; - } - return a.at > b.at || (a.at === b.at && canonical(a.value) >= canonical(b.value)) ? a : b; -} -function mergeMap(a: Record, b: Record, merge: (a: T, b: T) => T) { - return Object.fromEntries([...new Set([...Object.keys(a), ...Object.keys(b)])].sort() - .map((key) => [key, Object.hasOwn(a, key) && Object.hasOwn(b, key) ? merge(a[key], b[key]) : Object.hasOwn(a, key) ? a[key] : b[key]])); -} -export function mergeShared(a: SharedLibrary, b: SharedLibrary): SharedLibrary { - return { - settings: newest(a.settings, b.settings), - songs: mergeMap(a.songs, b.songs, (x, y) => ({ - practice: newest(x.practice, y.practice), favorite: newest(x.favorite, y.favorite), - })), - presets: mergeMap(a.presets, b.presets, newest), - favoriteOrder: newest(a.favoriteOrder, b.favoriteOrder), - }; -} -/** Every local edit follows all revisions this installation has observed. */ -export function nextRevision(shared: SharedLibrary, now = Date.now()): number { - let at = Math.max(now, shared.settings.at, shared.favoriteOrder.at); - for (const song of Object.values(shared.songs)) at = Math.max(at, song.practice.at, song.favorite.at); - for (const preset of Object.values(shared.presets)) at = Math.max(at, preset.at); - return at + 1; +/** One winner for the entire library. On equal timestamps the synced copy wins. */ +export function newestSnapshot(local: SharedLibrary, remote: SharedLibrary): SharedLibrary { + return local.updatedAt > remote.updatedAt ? local : remote; } +export type UiPrefsPatch = { + [K in keyof UiPrefs]?: UiPrefs[K] extends object ? Partial : UiPrefs[K]; +}; + export type LibraryCommand = | { type: 'practice'; identity: TrackIdentity; patch: Partial; recent: boolean } | { type: 'favorite'; key: string; value: boolean } @@ -90,91 +58,49 @@ export type LibraryCommand = | { type: 'recent.remove'; key?: string } | { type: 'chart'; key: string; chart: ChordChart | null } | { type: 'settings'; patch: Partial; reset?: boolean } - | { type: 'uiPrefs'; value: UiPrefs } + | { type: 'uiPrefs'; patch: UiPrefsPatch } | { type: 'preset'; name: string; gains: number[] | null } | { type: 'import'; library: Library }; -/** A song is kept while it is favorited, listed in Recent, or among the most - * recently opened. Anything else becomes a dated deletion so that dropping it - * crosses devices instead of being merged straight back; the oldest deletions - * are finally forgotten. Without this every song ever played would stay in - * `shared.songs` forever and eventually fill the sync quota, after which - * nothing at all syncs. */ -function prune(library: Library, at: number): void { - const { shared, local } = library; - const kept = new Set(Object.keys(shared.songs) - .filter((key) => shared.songs[key].favorite.value || key in local.recent)); - const rest = Object.keys(shared.songs) - .filter((key) => !kept.has(key) && shared.songs[key].practice.value !== null) - .sort((a, b) => (local.lastAccessed[b] ?? 0) - (local.lastAccessed[a] ?? 0)); - for (const key of rest.slice(Math.max(0, SONG_LIMIT - kept.size))) { - defineEntry(shared.songs, key, { practice: cell(null, at), favorite: cell(false, at) }); - } - const deleted = Object.keys(shared.songs).filter((key) => shared.songs[key].practice.value === null) - .sort((a, b) => shared.songs[b].practice.at - shared.songs[a].practice.at); - for (const key of deleted.slice(DELETION_LIMIT)) delete shared.songs[key]; - for (const key of deleted) { - delete local.recent[key]; - delete local.lastAccessed[key]; - delete local.charts[key]; - } - // A star that no longer names a saved song only costs sync bytes. - const order = shared.favoriteOrder.value.filter((key) => shared.songs[key]?.favorite.value); - if (order.length !== shared.favoriteOrder.value.length) shared.favoriteOrder = cell(order, at); -} - -/** Every path that writes the library prunes, not only edits: a merge or a - * migration can carry in more songs than the sync limits allow, and nothing else - * would ever bring it back under them. Idempotent — with nothing to drop the - * result is identical, so it never manufactures a write of its own. */ -export function pruned(library: Library, now = Date.now()): Library { - const next = structuredClone(library); - prune(next, nextRevision(next.shared, now)); - return next; -} - -/** Called only by the background writer. Incoming edits patch current saved data. */ +/** Only the background writes. Commands from panels patch the current snapshot. */ export function applyCommand(library: Library, command: LibraryCommand, now = Date.now()): Library { - const next = structuredClone(library); + const next = structuredClone(command.type === 'import' ? command.library : library); const { shared, local } = next; - const at = nextRevision(shared, now); switch (command.type) { case 'practice': { const key = command.identity.key; - const song = shared.songs[key] ?? { practice: cell(null), favorite: cell(false) }; - const base: Practice = song.practice.value ?? { - identity: command.identity, pageUrl: command.identity.normalizedUrl, + const song = shared.songs[key] ?? { practice: { + identity: command.identity, pageUrl: command.identity.normalizedUrl, updatedAt: now, markers: [], snippets: [], sequenceLoop: false, sequenceCountIn: false, - }; - song.practice = cell({ ...base, ...command.patch, identity: command.identity }, at); + }, favoritedAt: null }; + song.practice = { ...song.practice, ...command.patch, identity: command.identity, updatedAt: now }; defineEntry(shared.songs, key, song); if (command.recent || key in local.recent) local.recent[key] = now; local.lastAccessed[key] = now; - const keep = Object.entries(local.recent).sort((a, b) => b[1] - a[1]).slice(0, HISTORY_LIMIT); - local.recent = Object.fromEntries(keep); + local.recent = Object.fromEntries(Object.entries(local.recent).sort((a, b) => b[1] - a[1]).slice(0, HISTORY_LIMIT)); break; } case 'favorite': { const song = shared.songs[command.key]; - if (!song?.practice.value) break; - song.favorite = cell(command.value, at); - if (command.value) shared.favoriteOrder = cell([command.key, ...shared.favoriteOrder.value.filter((k) => k !== command.key)], at); + if (!song) break; + song.favoritedAt = command.value ? now : null; + shared.favoriteOrder = shared.favoriteOrder.filter((key) => key !== command.key); + if (command.value) shared.favoriteOrder.unshift(command.key); break; } - case 'order': shared.favoriteOrder = cell([...new Set(command.keys)], at); break; + case 'order': shared.favoriteOrder = [...new Set(command.keys)].filter((key) => shared.songs[key]?.favoritedAt != null); break; case 'visit': { - if (shared.songs[command.key]?.practice.value) local.lastAccessed[command.key] = now; + if (shared.songs[command.key]) local.lastAccessed[command.key] = now; break; } case 'recent.remove': { - // "Remove from history" is the only delete the UI offers, so it removes - // the saved song itself. A favorite is kept — its own row only unstars — - // and falls back to the limit above once it is neither. const keys = command.key === undefined ? Object.keys(local.recent) : [command.key]; for (const key of keys) { delete local.recent[key]; - const song = shared.songs[key]; - if (song && !song.favorite.value) song.practice = cell(null, at); + if (shared.songs[key]?.favoritedAt != null) continue; + delete shared.songs[key]; + delete local.lastAccessed[key]; + delete local.charts[key]; } break; } @@ -182,48 +108,43 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da case 'settings': { const { lastUsedParams, ...patch } = command.patch; if (lastUsedParams) local.lastUsedParams = lastUsedParams; - if (command.reset || Object.keys(patch).length) { - const value = { ...(command.reset ? structuredClone(DEFAULT_SETTINGS) : shared.settings.value), ...patch }; - if (patch.rememberSettings) value.autoReset = false; - if (patch.autoReset) value.rememberSettings = false; - shared.settings = cell(value, at); - } + shared.settings = { ...(command.reset ? structuredClone(DEFAULT_SETTINGS) : shared.settings), ...patch }; + if (patch.rememberSettings) shared.settings.autoReset = false; + if (patch.autoReset) shared.settings.rememberSettings = false; if (command.reset) delete local.lastUsedParams; break; } - case 'uiPrefs': local.uiPrefs = command.value; break; - case 'preset': defineEntry(shared.presets, command.name, cell(command.gains, at)); break; - case 'import': { - const file = structuredClone(command.library); - const revision = Math.max(at, nextRevision(file.shared, now)); - // Replacement names the records this device knows; absence is never a remote delete. - for (const key of new Set([...Object.keys(shared.songs), ...Object.keys(file.shared.songs)])) { - const song = file.shared.songs[key]; - defineEntry(shared.songs, key, { practice: cell(song?.practice.value ?? null, revision), favorite: cell(song?.favorite.value ?? false, revision) }); - } - for (const name of new Set([...Object.keys(shared.presets), ...Object.keys(file.shared.presets)])) { - defineEntry(shared.presets, name, - cell(Object.hasOwn(file.shared.presets, name) ? file.shared.presets[name].value : null, revision)); - } - shared.settings = cell(file.shared.settings.value, revision); - shared.favoriteOrder = cell(file.shared.favoriteOrder.value, revision); - next.local = file.local; + case 'uiPrefs': { + local.uiPrefs = { + ...local.uiPrefs, ...command.patch, + collapsed: { ...local.uiPrefs.collapsed, ...command.patch.collapsed }, + collapsedSections: { ...local.uiPrefs.collapsedSections, ...command.patch.collapsedSections }, + boundaryLabels: { ...local.uiPrefs.boundaryLabels, ...command.patch.boundaryLabels }, + }; break; } + case 'preset': { + if (command.gains === null) delete shared.presets[command.name]; + else defineEntry(shared.presets, command.name, command.gains); + break; + } + } + // Local activity never makes an older shared snapshot win. Imports are an + // explicit replacement, even when the file happens to contain the same data. + if (command.type === 'import' || JSON.stringify(shared) !== JSON.stringify(library.shared)) { + shared.updatedAt = Math.max(now, library.shared.updatedAt + 1, shared.updatedAt + 1); } - prune(next, at); return next; } /** UI rows are projections. They are never written back as library copies. */ function songEntry(key: string, library: Library): HistoryEntry | null { - const song = library.shared.songs[key]; - const practice = song?.practice.value; + const practice = library.shared.songs[key]?.practice; if (!practice) return null; return { identity: practice.identity, pageUrl: practice.pageUrl, thumbnailUrl: practice.thumbnailUrl, params: practice.params ?? structuredClone(DEFAULT_PARAMS), - updatedAt: library.local.recent[key] ?? song.practice.at, + updatedAt: library.local.recent[key] ?? practice.updatedAt, }; } export function recentEntries(library: Library): HistoryEntry[] { @@ -231,11 +152,11 @@ export function recentEntries(library: Library): HistoryEntry[] { .filter((entry) => entry !== null).sort((a, b) => b.updatedAt - a.updatedAt); } export function favoriteEntries(library: Library): FavoriteEntry[] { - const keys = [...new Set([...library.shared.favoriteOrder.value, ...Object.keys(library.shared.songs).sort()])]; + const keys = [...new Set([...library.shared.favoriteOrder, ...Object.keys(library.shared.songs).sort()])]; return keys.flatMap((key) => { const song = library.shared.songs[key]; const entry = songEntry(key, library); - return song?.favorite.value && entry ? [{ ...entry, favoritedAt: song.favorite.at, - lastAccessedAt: library.local.lastAccessed[key] ?? song.favorite.at }] : []; + return song?.favoritedAt != null && entry ? [{ ...entry, favoritedAt: song.favoritedAt, + lastAccessedAt: library.local.lastAccessed[key] ?? song.favoritedAt }] : []; }); } diff --git a/src/core/persist/storage.ts b/src/core/persist/storage.ts index d000bf4..f79ad1a 100644 --- a/src/core/persist/storage.ts +++ b/src/core/persist/storage.ts @@ -1,18 +1,2 @@ import { storage } from '#imports'; -import type { Settings, UiPrefs } from '../model/types'; -import { editLibrary, readLibrary, watchLibrary } from './library-client'; - -/** Feature stores read projections and send changes to the single background writer. */ -const settingsOf = (library: Awaited>): Settings => ({ - ...library.shared.settings.value, lastUsedParams: library.local.lastUsedParams, -}); -export const settingsItem = { - getValue: async () => settingsOf(await readLibrary()), - watch: (listener: (value: Settings) => void) => watchLibrary(settingsOf, listener), -}; -export const uiPrefsItem = { - getValue: async () => (await readLibrary()).local.uiPrefs, - setValue: (value: UiPrefs) => editLibrary({ type: 'uiPrefs', value }), - watch: (listener: (value: UiPrefs) => void) => watchLibrary((library) => library.local.uiPrefs, listener), -}; export const grantedOriginsItem = storage.defineItem('local:grantedOrigins', { fallback: [] }); diff --git a/src/core/state/connect.svelte.ts b/src/core/state/connect.svelte.ts index 0f5d3a6..6a7a094 100644 --- a/src/core/state/connect.svelte.ts +++ b/src/core/state/connect.svelte.ts @@ -1,7 +1,7 @@ import { UI_PORT, type EngineCommand, type EngineEvent } from '../messaging/protocol'; import { connectToTab, type TypedPort } from '../messaging/ports'; import { sendMessage } from '../messaging/rpc'; -import { features } from '../features'; +import { chords } from '../../features/chords/panel/chords.svelte'; import { session } from './session.svelte'; import { settings } from '../../features/settings/panel/settings.svelte'; @@ -16,7 +16,7 @@ function isLocalPlayer(url: string | undefined): boolean { /** A fresh engine starts on the default preset, so this runs on every attach as * well as on change — a no-op while nothing is attached. */ -function pushSettings() { +export function pushSettings() { session.send({ type: 'settings', seekInterval: settings.current.seekInterval, @@ -39,8 +39,6 @@ class ConnectionManager { #generation = 0; async init() { - settings.onChange = pushSettings; - // Chromium opens one panel document per tab at `sidepanel.html?tabId=N` // (see core/side-panel.ts): pin to that tab. Hidden behind another tab // this document stays alive, and following activation there would leave @@ -137,10 +135,8 @@ class ConnectionManager { this.#port = port; port.onMessage((event) => { session.apply(event); - // Feature-owned traffic that lives outside the session mirror (e.g. the - // chords store) is routed through the panel feature registry. - for (const f of features) f.routeEvent?.(event); - if (event.type === 'snapshot') for (const f of features) f.onSnapshot?.(event); + if (event.type === 'pcm') chords.pushPcm(event.samples, event.sampleRate, event.t, event.speed); + if (event.type === 'snapshot') chords.syncActive(event.chordActive); if (event.type === 'state' || event.type === 'snapshot') { void this.#refineNoPlayer(); } @@ -149,7 +145,7 @@ class ConnectionManager { if (this.#port !== port) return; this.#port = null; session.detachTransport(); - for (const f of features) f.onDisconnect?.(); + chords.onDisconnect(); if (session.connection !== 'restricted' && session.connection !== 'idle') { session.connection = 'stale'; } diff --git a/src/core/state/library.svelte.ts b/src/core/state/library.svelte.ts new file mode 100644 index 0000000..090353b --- /dev/null +++ b/src/core/state/library.svelte.ts @@ -0,0 +1,14 @@ +import { emptyLibrary, type Library } from '../persist/library'; +import { libraryItem, readLibrary } from '../persist/library-client'; + +/** One panel-side copy. Settings, preferences, presets and song lists read it. */ +class LibraryStore { + current = $state.raw(emptyLibrary()); + async init() { + let changed = false; + libraryItem.watch((value) => { changed = true; this.current = value ?? emptyLibrary(); }); + const initial = await readLibrary(); + if (!changed) this.current = initial; + } +} +export const library = new LibraryStore(); diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index cf0c03a..6e2038e 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -67,7 +67,7 @@ class TrackSync { try { const saved = await readLibrary(); if (generation !== this.#generation) return; - const practice = saved.shared.songs[identity.key]?.practice.value; + const practice = saved.shared.songs[identity.key]?.practice; this.#hasSavedParams = !!practice?.params; markers.load(practice?.markers ?? []); snippets.load(practice?.snippets ?? [], practice?.sequenceLoop ?? false, practice?.sequenceCountIn ?? false); diff --git a/src/entrypoints/local-player/main.ts b/src/entrypoints/local-player/main.ts index 73fb579..bb43deb 100644 --- a/src/entrypoints/local-player/main.ts +++ b/src/entrypoints/local-player/main.ts @@ -2,14 +2,13 @@ import { mount } from 'svelte'; import LocalPlayer from './LocalPlayer.svelte'; import '@/assets/theme.css'; import { applyTheme } from '@/features/settings/panel/settings.svelte'; -import { settingsItem } from '@/core/persist/storage'; +import { readLibrary } from '@/core/persist/library-client'; // Match the side panel's chosen theme. Apply 'auto' synchronously (follows the // OS, live) to avoid a flash, then refine from the stored choice once loaded. applyTheme('auto'); -void settingsItem - .getValue() - .then((s) => applyTheme(s?.theme ?? 'auto')) +void readLibrary() + .then((library) => applyTheme(library.shared.settings.theme)) .catch(() => {}); const app = mount(LocalPlayer, { diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index e67c11a..2bfa3bc 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -8,9 +8,9 @@ import { sendMessage } from '@/core/messaging/rpc'; import { openTabWithPanel } from '@/core/side-panel'; import { installMockState, installMockTicker } from '@/dev/mock'; - import { connection } from '@/core/state/connect.svelte'; + import { connection, pushSettings } from '@/core/state/connect.svelte'; import { CAN_CAPTURE_TAB } from '@/core/platform'; - import { features } from '@/core/features'; + import { library } from '@/core/state/library.svelte'; import { session } from '@/core/state/session.svelte'; import { applyTheme, settings } from '@/features/settings/panel/settings.svelte'; import { installShortcuts } from '@/features/shortcuts/panel/shortcuts'; @@ -22,10 +22,11 @@ const mock = params.has('mock'); // ?mock=1&play=1 also runs the playhead, for previewing time-driven UI. const mockPlay = mock && params.has('play'); - // Feature stores load projections from the background-owned library. - const ready = Promise.all(features.map((f) => f.init?.())).then( + $effect(() => applyTheme(settings.current.theme)); + $effect(pushSettings); + + const ready = library.init().then( async () => { - applyTheme(settings.current.theme); trackSync.init(); session.onMediaEvent = (media) => { trackSync.onMedia(media).catch((err: unknown) => { diff --git a/src/features/chords/panel/panel.ts b/src/features/chords/panel/panel.ts deleted file mode 100644 index 954bfbd..0000000 --- a/src/features/chords/panel/panel.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { EngineEvent } from '../../../core/messaging/protocol'; -import type { PanelFeature, SnapshotEvent } from '../../../core/features'; -import { chords } from './chords.svelte'; - -/** Chord/key detection lives in its own track-scoped store (like markers), so - * its engine traffic is routed here rather than through the session mirror. */ -export const chordsFeature: PanelFeature = { - routeEvent(event: EngineEvent) { - if (event.type === 'pcm') chords.pushPcm(event.samples, event.sampleRate, event.t, event.speed); - }, - onSnapshot(snapshot: SnapshotEvent) { - chords.syncActive(snapshot.chordActive); - }, - onDisconnect() { - chords.onDisconnect(); - }, -}; diff --git a/src/features/eq/panel/eq-presets.svelte.ts b/src/features/eq/panel/eq-presets.svelte.ts index cfc0200..e0efa0b 100644 --- a/src/features/eq/panel/eq-presets.svelte.ts +++ b/src/features/eq/panel/eq-presets.svelte.ts @@ -1,23 +1,14 @@ import { BUILTIN_EQ_PRESETS, EQ_BANDS } from '../../../core/model/defaults'; import type { EqPreset } from '../../../core/model/types'; -import { editLibrary, readLibrary, watchLibrary } from '../../../core/persist/library-client'; -import type { Library } from '../../../core/persist/library'; +import { editLibrary } from '../../../core/persist/library-client'; +import { library } from '../../../core/state/library.svelte'; /** Slider gains are multiples of 0.5 dB, so anything closer than this is the * same curve; the tolerance only guards against float drift. */ const GAIN_EPSILON = 0.01; class EqPresetsStore { - /** Projection of non-deleted presets. */ - saved = $state([]); - - async init() { - const select = (library: Library): EqPreset[] => Object.entries(library.shared.presets) - .filter(([, preset]) => preset.value !== null) - .map(([name, preset]) => ({ name, gains: preset.value! })); - this.saved = select(await readLibrary()); - watchLibrary(select, (value) => { this.saved = value; }); - } + saved = $derived(Object.entries(library.current.shared.presets).map(([name, gains]) => ({ name, gains }))); /** Built-ins first, then the user's, as listed in the dropdown. */ get all(): EqPreset[] { diff --git a/src/features/eq/panel/panel.ts b/src/features/eq/panel/panel.ts deleted file mode 100644 index 19b05e1..0000000 --- a/src/features/eq/panel/panel.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { PanelFeature } from '../../../core/features'; -import { eqPresets } from './eq-presets.svelte'; - -/** User-saved EQ presets load from storage at panel boot. */ -export const eqFeature: PanelFeature = { - init: () => eqPresets.init(), -}; diff --git a/src/features/library/panel/favorites.ts b/src/features/library/panel/favorites.ts index 3aacd97..cb416b0 100644 --- a/src/features/library/panel/favorites.ts +++ b/src/features/library/panel/favorites.ts @@ -1,7 +1,7 @@ import type { HistoryEntry, TrackIdentity } from '../../../core/model/types'; import { favoriteEntries } from '../../../core/persist/library'; import { editLibrary } from '../../../core/persist/library-client'; -import { library } from './library.svelte'; +import { library } from '../../../core/state/library.svelte'; /** The list only repaints from the storage watch, so a rejected write leaves the * control looking dead. Log instead of vanishing. */ @@ -10,9 +10,9 @@ const write = (run: Promise) => export const favorites = { get entries() { return favoriteEntries(library.current); }, - has: (identity: TrackIdentity) => library.current.shared.songs[identity.key]?.favorite.value === true, + has: (identity: TrackIdentity) => library.current.shared.songs[identity.key]?.favoritedAt != null, toggle: (entry: HistoryEntry) => write(editLibrary({ type: 'favorite', key: entry.identity.key, - value: !library.current.shared.songs[entry.identity.key]?.favorite.value })), + value: library.current.shared.songs[entry.identity.key]?.favoritedAt == null })), remove: (key: string) => write(editLibrary({ type: 'favorite', key, value: false })), reorder: (keys: string[]) => write(editLibrary({ type: 'order', keys })), }; diff --git a/src/features/library/panel/history.ts b/src/features/library/panel/history.ts index db01383..cbe3c01 100644 --- a/src/features/library/panel/history.ts +++ b/src/features/library/panel/history.ts @@ -1,6 +1,6 @@ import { recentEntries } from '../../../core/persist/library'; import { editLibrary } from '../../../core/persist/library-client'; -import { library } from './library.svelte'; +import { library } from '../../../core/state/library.svelte'; export const history = { get entries() { return recentEntries(library.current); }, diff --git a/src/features/library/panel/library.svelte.ts b/src/features/library/panel/library.svelte.ts deleted file mode 100644 index c7e1344..0000000 --- a/src/features/library/panel/library.svelte.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { emptyLibrary, type Library } from '../../../core/persist/library'; -import { readLibrary, watchLibrary } from '../../../core/persist/library-client'; - -class LibraryStore { - current = $state.raw(emptyLibrary()); - async init() { - let changed = false; - watchLibrary((library) => library, (library) => { changed = true; this.current = library; }); - const initial = await readLibrary(); - if (!changed) this.current = initial; - } -} -export const library = new LibraryStore(); diff --git a/src/features/library/panel/panel.ts b/src/features/library/panel/panel.ts deleted file mode 100644 index 28097aa..0000000 --- a/src/features/library/panel/panel.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { PanelFeature } from '../../../core/features'; -import { library } from './library.svelte'; -export const libraryFeature: PanelFeature = { init: () => library.init() }; diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index 59c3d45..a4c798a 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -19,7 +19,7 @@ restoreBackup, } from '@/core/persist/backup'; import { history } from '@/features/library/panel/history'; - import { applyTheme, settings } from '@/features/settings/panel/settings.svelte'; + import { settings } from '@/features/settings/panel/settings.svelte'; import { session } from '@/core/state/session.svelte'; import { view } from '@/core/state/view.svelte'; import { sync } from '@/features/sync/panel/sync.svelte'; @@ -47,7 +47,7 @@ let notice = $state<{ ok: boolean; text: string } | null>(null); /** UI-level view of the `autoReset` / `rememberSettings` pair, which the - * settings store keeps mutually exclusive. Both off = carry over. */ + * background keeps mutually exclusive. Both off = carry over. */ type NewSongBehavior = 'defaults' | 'keep' | 'lastUsed'; const themeOptions: { value: Theme; label: string; icon: IconName }[] = [ @@ -89,11 +89,6 @@ }); } - function setTheme(value: Theme) { - void settings.update({ theme: value }); - applyTheme(value); - } - function setTabAudio(on: boolean) { void settings.update({ tabAudio: on }); ontabaudio?.(on); @@ -118,7 +113,6 @@ function resetSettingsConfirmed() { if (!confirm('Restore all extension settings to their defaults?')) return; void settings.reset(); - applyTheme('auto'); } function revokeConfirmed() { @@ -146,7 +140,7 @@ // The download reads the blob after click() returns, so the URL has to // outlive this task. setTimeout(() => URL.revokeObjectURL(url), 0); - const songs = Object.values(backup.shared.songs).filter((song) => song.practice.value !== null).length; + const songs = Object.keys(backup.shared.songs).length; const kb = Math.max(1, Math.round(new TextEncoder().encode(text).length / 1024)); notice = { ok: true, text: `Saved ${link.download} (${songs} songs, ${kb} KB).` }; } catch (err) { @@ -307,7 +301,7 @@ void settings.update({ theme })} />
@@ -493,7 +487,7 @@
{@render prefText( 'Sync between devices', - "Sync saved practice settings, favorites, presets, markers and snippets. Recent, layout and chord analysis stay on this device. Uses your browser's built-in sync: sign in to the browser with sync turned on and it reaches your other devices. No account with us, no server.", + "Sync saved practice settings, favorites, presets, markers and snippets. The most recently edited library replaces the older copy, so changes made on two devices at once can overwrite each other. Recent, layout and chord analysis stay on this device. Uses your browser's built-in sync: sign in to the browser with sync turned on and it reaches your other devices. No account with us, no server.", )} sync.enabled, (on) => void setSyncEnabled(on)} diff --git a/src/features/settings/panel/panel.ts b/src/features/settings/panel/panel.ts deleted file mode 100644 index b6c008e..0000000 --- a/src/features/settings/panel/panel.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { PanelFeature } from '../../../core/features'; -import { settings, uiPrefs } from './settings.svelte'; - -/** Settings + cosmetic UI prefs load from storage at panel boot. */ -export const settingsFeature: PanelFeature = { - async init() { - await Promise.all([settings.init(), uiPrefs.init()]); - }, -}; diff --git a/src/features/settings/panel/settings.svelte.ts b/src/features/settings/panel/settings.svelte.ts index 76ba31d..b61a0b0 100644 --- a/src/features/settings/panel/settings.svelte.ts +++ b/src/features/settings/panel/settings.svelte.ts @@ -1,140 +1,45 @@ -import { DEFAULT_KEYMAP, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../../../core/model/defaults'; import type { PanelId, SectionId, Settings, UiPrefs } from '../../../core/model/types'; +import type { UiPrefsPatch } from '../../../core/persist/library'; import { editLibrary } from '../../../core/persist/library-client'; -import { settingsItem, uiPrefsItem } from '../../../core/persist/storage'; +import { library } from '../../../core/state/library.svelte'; -/** Mirror of the library's settings. Components mutate via `update`, which - * sends the patch to the background writer. */ +/** Views of the single library copy. All changes go through the background. */ class SettingsStore { - current = $state(structuredClone(DEFAULT_SETTINGS)); - loaded = $state(false); - #writing = false; + current = $derived({ + ...library.current.shared.settings, lastUsedParams: library.current.local.lastUsedParams, + }); - /** Wired by the connection layer, which pushes the engine-relevant settings - * to the tab. Fires on every path that lands a new value in - * `current` — the engine can't observe this store itself. */ - onChange: ((next: Settings) => void) | null = null; - - /** Stored settings may predate newly added fields — backfill from defaults so - * a missing key never reaches the engine as `undefined` (which the port drops, - * e.g. a NaN count-in duration that never elapses). */ - #withDefaults(value: Settings | null): Settings { - return { - ...structuredClone(DEFAULT_SETTINGS), - ...value, - // Merged one level deeper: a keymap stored before an action existed would - // otherwise leave that action `undefined`, which the dispatcher can never - // match (the hotkey silently does nothing) and the Help sheet — which - // reads the keymap unconditionally — renders as a blank row. - keymap: { ...DEFAULT_KEYMAP, ...value?.keymap }, - }; - } - - async init() { - this.current = this.#withDefaults(await settingsItem.getValue()); - this.loaded = true; - settingsItem.watch((value) => { - if (this.#writing) return; - const before = this.current.theme; - this.current = this.#withDefaults(value); - // Settings can land here without any local control having been touched — - // a backup import, or a merge from another device — and `applyTheme` is - // what actually paints . - if (this.current.theme !== before) applyTheme(this.current.theme); - this.onChange?.(this.current); - }); - } - - async update(patch: Partial) { - // Settings cross devices as one revisioned item (see `core/persist/library.ts`), - // so the write carries no date of its own. - const next = { ...this.current, ...patch }; - // Auto Reset and Remember settings are alternatives — enabling one - // switches the other off. - if (patch.rememberSettings) next.autoReset = false; - if (patch.autoReset) next.rememberSettings = false; - this.current = next; - this.onChange?.(next); - this.#writing = true; - try { - // $state.snapshot, like UiPrefsStore below: `next` is spread off the - // `current` rune, so nested `keymap`/`lastUsedParams` are still proxies. - // Firefox structured-clones storage writes and throws DataCloneError on a - // proxy — settings would apply for the session but never persist. - await editLibrary({ type: 'settings', patch: $state.snapshot(patch) as Partial }); - } finally { - this.#writing = false; - } + update(patch: Partial) { + return editLibrary({ type: 'settings', patch: $state.snapshot(patch) }); } - async reset() { - this.current = structuredClone(DEFAULT_SETTINGS); - this.onChange?.(this.current); - await editLibrary({ type: 'settings', patch: {}, reset: true }); + reset() { + return editLibrary({ type: 'settings', patch: {}, reset: true }); } } class UiPrefsStore { - current = $state(structuredClone(DEFAULT_UI_PREFS)); - #writing = false; - - /** Stored values may predate newly added prefs — backfill from defaults. */ - #withDefaults(value: UiPrefs | null): UiPrefs { - return { ...structuredClone(DEFAULT_UI_PREFS), ...value }; - } - - async init() { - this.current = this.#withDefaults(await uiPrefsItem.getValue()); - uiPrefsItem.watch((value) => { - if (this.#writing) return; - this.current = this.#withDefaults(value); - }); - } + current = $derived(library.current.local.uiPrefs); - async #save() { - this.#writing = true; - try { - await uiPrefsItem.setValue($state.snapshot(this.current)); - } finally { - this.#writing = false; - } + update(patch: UiPrefsPatch) { + return editLibrary({ type: 'uiPrefs', patch }); } toggleCollapsed(panel: PanelId) { - this.current.collapsed[panel] = !this.current.collapsed[panel]; - void this.#save(); + return this.update({ collapsed: { [panel]: !this.current.collapsed[panel] } }); } toggleSectionCollapsed(section: SectionId) { - this.current.collapsedSections[section] = !this.current.collapsedSections[section]; - void this.#save(); - } - - setMarkerView(view: UiPrefs['markerView']) { - this.current.markerView = view; - void this.#save(); - } - - setTimelineFollow(on: boolean) { - this.current.timelineFollow = on; - void this.#save(); - } - - setFavoritesSort(sort: UiPrefs['favoritesSort']) { - this.current.favoritesSort = sort; - void this.#save(); + return this.update({ collapsedSections: { [section]: !this.current.collapsedSections[section] } }); } - setLibraryTab(tab: UiPrefs['libraryTab']) { - this.current.libraryTab = tab; - void this.#save(); - } + setMarkerView(markerView: UiPrefs['markerView']) { return this.update({ markerView }); } + setTimelineFollow(timelineFollow: boolean) { return this.update({ timelineFollow }); } + setFavoritesSort(favoritesSort: UiPrefs['favoritesSort']) { return this.update({ favoritesSort }); } + setLibraryTab(libraryTab: UiPrefs['libraryTab']) { return this.update({ libraryTab }); } - /** Override a virtual boundary marker's label; empty text restores the - * default ("Start"/"End"). */ setBoundaryLabel(which: 'start' | 'end', label: string) { - this.current.boundaryLabels[which] = label.trim(); - void this.#save(); + void this.update({ boundaryLabels: { [which]: label.trim() } }); } } diff --git a/src/features/sync/persist/records.test.ts b/src/features/sync/persist/records.test.ts index 4aeb418..6bcceff 100644 --- a/src/features/sync/persist/records.test.ts +++ b/src/features/sync/persist/records.test.ts @@ -1,84 +1,70 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { applyCommand, emptyLibrary, canonical } from '../../../core/persist/library.ts'; +import { randomBytes } from 'node:crypto'; +import { applyCommand, emptyLibrary } from '../../../core/persist/library.ts'; import { makeTrackIdentity } from '../../../core/model/track-identity.ts'; -import { changedRecords, readRecords, bytesUsed } from './records.ts'; +import { encodeSnapshot, readSnapshot, bytesUsed, IncompleteSnapshot, PREFIX, SNAPSHOT_KEY } from './records.ts'; -const song = (id: number) => makeTrackIdentity('https://youtube.com/watch?v=song' + id, 'Song', 200); -test('records round-trip independently, and an unchanged copy has nothing to upload', async () => { - const library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); - const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); - const restored = await readRecords(items); - assert.equal(canonical(restored), canonical(library.shared)); - assert.deepEqual((await changedRecords(library.shared, restored, items)).changes, {}); - const edited = applyCommand(library, { type: 'favorite', key: song(1).key, value: true }); - const { changes } = await changedRecords(edited.shared, restored, items); - assert.deepEqual(Object.keys(changes), ['nbn4:order', 'nbn4:song:' + song(1).key]); - assert.ok(bytesUsed(items) < 8192); -}); - -test('partial arrival yields complete individual songs, with no global blob to assemble', async () => { - let library = emptyLibrary(); - for (const n of [1, 2]) library = applyCommand(library, { type: 'practice', identity: song(n), patch: {}, recent: true }); - const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); - const key = 'nbn4:song:' + song(2).key; - const partial = await readRecords({ [key]: items[key] }); - assert.deepEqual(Object.keys(partial.songs), [song(2).key]); -}); +const song = makeTrackIdentity('https://youtube.com/watch?v=song', 'Song', 200); +const save = (label = '', now = 100) => applyCommand(emptyLibrary(), { + type: 'practice', identity: song, patch: { markers: [{ id: 'm', t: 1, label }] }, recent: true, +}, now).shared; -test('capacity failure leaves the library intact and never silently trims records', async () => { - const library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); - const before = canonical(library); - await assert.rejects(changedRecords(library.shared, emptyLibrary().shared, { unrelated: 'x'.repeat(102400) }), /storage is full/); - assert.equal(canonical(library), before); +test('sync round-trips exactly the shared snapshot, including large songs across chunks', async () => { + const shared = save(randomBytes(18000).toString('base64')); + const { items, usedBytes } = await encodeSnapshot(shared); + assert.deepEqual(await readSnapshot(items), shared); + assert.ok((items[SNAPSHOT_KEY] as { chunks: number }).chunks > 1); + assert.equal(usedBytes, bytesUsed(items)); + for (const [key, value] of Object.entries(items)) assert.ok(bytesUsed({ [key]: value }) <= 8192); + assert.deepEqual((await encodeSnapshot(shared, items)).items, items); }); -test('a record too large to sync is reported and left behind, never blocking the rest', async () => { - let library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); - const markers = Array.from({ length: 2000 }, (_, n) => ({ id: 'm' + n, t: n, label: 'Marker ' + n })); - library = applyCommand(library, { type: 'practice', identity: song(2), patch: { markers }, recent: true }); - const { changes, skipped } = await changedRecords(library.shared, emptyLibrary().shared, {}); - assert.deepEqual(skipped, ['nbn4:song:' + song(2).key]); - assert.ok(Object.keys(changes).includes('nbn4:song:' + song(1).key)); - assert.equal(library.shared.songs[song(2).key].practice.value!.markers.length, 2000); +test('missing, reordered and mixed chunks never produce a partial library', async () => { + const { items } = await encodeSnapshot(save(randomBytes(18000).toString('base64'))); + const missing = { ...items }; + delete missing[PREFIX + 'chunk:1']; + await assert.rejects(readSnapshot(missing), IncompleteSnapshot); + const noHeader = { ...items }; + delete noHeader[SNAPSHOT_KEY]; + await assert.rejects(readSnapshot(noHeader), IncompleteSnapshot); + const { items: other } = await encodeSnapshot(save('other device at the same timestamp')); + await assert.rejects(readSnapshot({ ...items, [PREFIX + 'chunk:0']: other[PREFIX + 'chunk:0'] }), IncompleteSnapshot); + const reversed = Object.fromEntries(Object.entries(items).reverse()); + assert.deepEqual(await readSnapshot(reversed), await readSnapshot(items)); }); -test('unsupported records are rejected before any application or upload', async () => { - await assert.rejects(readRecords({ 'nbn4:settings': { version: 5, data: '' } }), /Unsupported/); +test('a smaller replacement clears all old chunks in the same write', async () => { + const { items: large } = await encodeSnapshot(save(randomBytes(18000).toString('base64'))); + const small = applyCommand({ shared: save(), local: emptyLibrary().local }, { type: 'import', library: emptyLibrary() }, 200).shared; + const { items } = await encodeSnapshot(small, large); + assert.equal(items[PREFIX + 'chunk:1'], ''); + assert.deepEqual(await readSnapshot({ ...large, ...items }), small); }); -test('a record the library no longer holds is removed remotely, not merged back', async () => { - const library = applyCommand(emptyLibrary(), { type: 'practice', identity: song(1), patch: {}, recent: true }); - const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); - const remote = await readRecords(items); - // The song is gone locally: sync must drop it rather than read it back forever. - const { removals } = await changedRecords(emptyLibrary().shared, remote, items); - assert.deepEqual(removals, ['nbn4:song:' + song(1).key]); - // A record this build does not own belongs to a newer one and is left alone. - const foreign = { ...items, 'nbn4:future:1': { version: 4, data: '' } }; - assert.ok(!(await changedRecords(library.shared, remote, foreign)).removals.includes('nbn4:future:1')); +test('capacity failures leave both the library and existing sync storage intact', async () => { + const shared = save(randomBytes(100000).toString('base64')); + const before = structuredClone(shared); + const { items: existing } = await encodeSnapshot(save('last complete upload')); + const existingBefore = structuredClone(existing); + await assert.rejects(encodeSnapshot(shared, existing), /storage is full/); + assert.deepEqual(shared, before); + assert.deepEqual(existing, existingBefore); + await assert.rejects(encodeSnapshot(save(), { unrelated: 'x'.repeat(102400) }), /storage is full/); }); -test('one damaged record costs only itself, never the rest or the upload', async () => { - let library = emptyLibrary(); - for (const n of [1, 2]) library = applyCommand(library, { type: 'practice', identity: song(n), patch: {}, recent: true }); - const { changes: items } = await changedRecords(library.shared, emptyLibrary().shared, {}); - const damaged = { ...items, ['nbn4:song:' + song(1).key]: { version: 4, data: 'not-gzip-at-all' } }; - const remote = await readRecords(damaged); - assert.deepEqual(Object.keys(remote.songs), [song(2).key]); - // The undamaged song still round-trips, so this device is not locked out. - assert.equal(canonical(remote.songs[song(2).key]), canonical(library.shared.songs[song(2).key])); +test('empty sync storage is distinct from a valid empty library or an incomplete transfer', async () => { + assert.equal(await readSnapshot({ unrelated: 'kept' }), null); + const { items } = await encodeSnapshot(emptyLibrary().shared); + assert.deepEqual(await readSnapshot(items), emptyLibrary().shared); + const broken = { ...items, [PREFIX + 'chunk:0']: '' }; + await assert.rejects(readSnapshot(broken), (error: unknown) => error instanceof IncompleteSnapshot && error.updatedAt === 0); }); -test('over budget, songs are held back and reported; the small records still sync', async () => { - let library = emptyLibrary(); - const markers = Array.from({ length: 300 }, (_, n) => ({ id: 'm' + n, t: n, label: 'Marker ' + n })); - for (const n of [1, 2, 3]) library = applyCommand(library, { type: 'practice', identity: song(n), patch: { markers }, recent: true }); - const before = canonical(library); - const padded = { unrelated: 'x'.repeat(102400 - 3000) }; - const { changes, skipped, usedBytes } = await changedRecords(library.shared, emptyLibrary().shared, padded); - assert.ok(skipped.length > 0, 'the songs that do not fit are reported'); - assert.ok(Object.keys(changes).includes('nbn4:settings'), 'settings are small and always get through'); - assert.ok(usedBytes <= 102400, 'what is written fits the quota'); - assert.equal(canonical(library), before, 'nothing is trimmed from the library itself'); +test('unsupported and damaged snapshots fail before adoption or upload', async () => { + const { items } = await encodeSnapshot(save()); + const header = items[SNAPSHOT_KEY] as Record; + await assert.rejects(readSnapshot({ ...items, [SNAPSHOT_KEY]: { ...header, version: 2 } }), /Unsupported/); + await assert.rejects(readSnapshot({ ...items, [SNAPSHOT_KEY]: { ...header, chunks: 10000 } }), /Damaged/); + await assert.rejects(readSnapshot({ ...items, [SNAPSHOT_KEY]: { ...header, updatedAt: 999 } }), /revision/); }); diff --git a/src/features/sync/persist/records.ts b/src/features/sync/persist/records.ts index 545e51e..dbc0346 100644 --- a/src/features/sync/persist/records.ts +++ b/src/features/sync/persist/records.ts @@ -1,11 +1,11 @@ -import { canonical, defineEntry, emptyLibrary, type SharedLibrary } from '../../../core/persist/library.ts'; +import type { SharedLibrary } from '../../../core/persist/library'; import { parseShared } from '../../../core/persist/backup-codec.ts'; -export const PREFIX = 'nbn4:'; -/** `browser.storage.sync` caps the whole area, one item, and the item count. */ +export const PREFIX = 'nbn:'; +export const SNAPSHOT_KEY = PREFIX + 'library'; export const QUOTA_BYTES = 102400; -const ITEM_BYTES = 8192; -const ITEM_COUNT = 512; +const CHUNK_SIZE = 7800; // Leaves room for the key and JSON below the 8192-byte item limit. +const MAX_CHUNKS = Math.ceil(QUOTA_BYTES / CHUNK_SIZE); const encoder = new TextEncoder(); export const bytesUsed = (items: Record) => Object.entries(items) .reduce((total, [key, value]) => total + encoder.encode(key + JSON.stringify(value)).length, 0); @@ -20,103 +20,52 @@ async function decompress(data: string): Promise { const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0)); return new Response(new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'))).text(); } -function records(shared: SharedLibrary): Record { - return Object.fromEntries([ - [PREFIX + 'settings', shared.settings], [PREFIX + 'order', shared.favoriteOrder], - ...Object.entries(shared.songs).map(([key, value]) => [PREFIX + 'song:' + key, value]), - ...Object.entries(shared.presets).map(([key, value]) => [PREFIX + 'preset:' + key, value]), - ]); +async function hash(data: string): Promise { + const digest = await crypto.subtle.digest('SHA-256', encoder.encode(data)); + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join(''); } -/** Record kinds this build owns. Anything else is read past and left in place — - * it may belong to a newer build, and removing it would destroy that data. */ -const OWNED = /^(settings|order|song:|preset:)/; -/** Each record is validated on its own, so one damaged record costs only itself. - * Validating the whole set at once would let a single bad song block every other - * song *and* this device's uploads, with "Delete synced data" the only way out. - * A record written by a newer build is the one hard failure: this build cannot - * read it, and uploading its own view over it would lose data. */ -export async function readRecords(items: Record): Promise { - const shared = emptyLibrary().shared; - const blank = emptyLibrary().shared; - // The casts assert nothing: `parseShared` validates the value at runtime and - // throws for this record alone if it does not hold up. - const validate = (patch: Partial) => parseShared({ ...blank, ...patch }); - const map = (key: string, value: unknown) => { - const one: Record = {}; - defineEntry(one, key, value); - return one; - }; - for (const key of Object.keys(items).filter((key) => key.startsWith(PREFIX))) { - const item = items[key] as { version?: unknown; data?: unknown }; - if (typeof item?.version === 'number' && item.version > 4) { - throw new Error('Unsupported synced data. Update Note by Note on all devices.'); - } - const name = key.slice(PREFIX.length); - try { - if (item?.version !== 4 || typeof item.data !== 'string') throw new Error('Damaged record.'); - const value: unknown = JSON.parse(await decompress(item.data)); - // Any other name falls through untouched: `OWNED` keeps it off the removal - // list too, so a record this build has never heard of is left alone. - if (name === 'settings') shared.settings = validate({ settings: value as SharedLibrary['settings'] }).settings; - else if (name === 'order') shared.favoriteOrder = validate({ favoriteOrder: value as SharedLibrary['favoriteOrder'] }).favoriteOrder; - else if (name.startsWith('song:')) { - const id = name.slice(5); - defineEntry(shared.songs, id, validate({ songs: map(id, value) as SharedLibrary['songs'] }).songs[id]); - } else if (name.startsWith('preset:')) { - const id = name.slice(7); - defineEntry(shared.presets, id, validate({ presets: map(id, value) as SharedLibrary['presets'] }).presets[id]); - } - } catch (error) { - console.warn('[note-by-note] a synced record could not be read and was skipped', key, error); - } +export class IncompleteSnapshot extends Error { + readonly updatedAt: number; + constructor(updatedAt: number) { + super('Waiting for the complete synced library. All data is kept on this device.'); + this.updatedAt = updatedAt; } - return shared; } -/** One record that cannot fit is reported and left behind, never truncated. */ -export interface RecordChanges { changes: Record; removals: string[]; skipped: string[]; usedBytes: number } -export const skippedMessage = (skipped: string[]) => `${skipped.length} saved ${skipped.length === 1 - ? 'record is' : 'records are'} too large to sync. Everything else synced; all data is kept on this device.`; -/** Write complete independent records. There is no chunk assembly or truncation. */ -export async function changedRecords(local: SharedLibrary, remote: SharedLibrary, existing: Record): Promise { - const previous = records(remote); - const next = records(local); - const changes: Record = {}; - const skipped: string[] = []; - for (const [key, value] of Object.entries(next)) { - if (existing[key] !== undefined && canonical(value) === canonical(previous[key])) continue; - const item = { version: 4, data: await compress(canonical(value)) }; - // One outsized song must not hold back every other record for good. - if (encoder.encode(key + JSON.stringify(item)).length > ITEM_BYTES) { skipped.push(key); continue; } - changes[key] = item; +/** No partial library is ever applied, even if browser sync delivers keys separately. */ +export async function readSnapshot(items: Record): Promise { + const header = items[SNAPSHOT_KEY]; + if (!header) { + if (Object.keys(items).some((key) => key.startsWith(PREFIX + 'chunk:'))) throw new IncompleteSnapshot(Infinity); + return null; } - // A record the library no longer holds is dropped remotely. Without this a - // pruned song is merged straight back on the next read, and the item count - // climbs until the cap is hit and nothing can be written at all. - const removals = Object.keys(existing).filter((key) => key.startsWith(PREFIX) - && OWNED.test(key.slice(PREFIX.length)) && !(key in next)); - const build = () => { - const proposed: Record = { ...existing, ...changes }; - for (const key of removals) delete proposed[key]; - return proposed; - }; - const overBudget = (proposed: Record) => - bytesUsed(proposed) > QUOTA_BYTES || Object.keys(proposed).length > ITEM_COUNT; - // Over budget, songs are held back largest-first instead of the whole batch - // failing: settings, order and presets are small and must always get through, - // and a song left behind stays on this device and is retried next reconcile. - const droppable = Object.keys(changes).filter((key) => key.slice(PREFIX.length).startsWith('song:')) - .sort((a, b) => encoder.encode(JSON.stringify(changes[b])).length - encoder.encode(JSON.stringify(changes[a])).length); - let proposed = build(); - for (const key of droppable) { - if (!overBudget(proposed)) break; - delete changes[key]; - skipped.push(key); - proposed = build(); + if (header.version !== 1) throw new Error('Unsupported synced data. Update Note by Note on all devices.'); + if (!Number.isFinite(header.updatedAt) || header.updatedAt < 0 || !Number.isInteger(header.chunks) + || header.chunks < 1 || header.chunks > MAX_CHUNKS || typeof header.hash !== 'string') { + throw new Error('Damaged synced library.'); + } + const chunks = Array.from({ length: header.chunks }, (_, n) => items[PREFIX + 'chunk:' + n]); + if (chunks.some((chunk) => typeof chunk !== 'string') || await hash(chunks.join('')) !== header.hash) { + throw new IncompleteSnapshot(header.updatedAt); } - if (overBudget(proposed)) { + const snapshot = parseShared(JSON.parse(await decompress(chunks.join('')))); + if (snapshot.updatedAt !== header.updatedAt) throw new Error('Damaged synced library revision.'); + return snapshot; +} + +/** Upload the entire snapshot or report capacity failure before changing storage. */ +export async function encodeSnapshot(shared: SharedLibrary, existing: Record = {}) { + const data = await compress(JSON.stringify(shared)); + const items: Record = { + [SNAPSHOT_KEY]: { version: 1, updatedAt: shared.updatedAt, chunks: Math.ceil(data.length / CHUNK_SIZE), hash: await hash(data) }, + }; + // Fixed slots let a smaller snapshot clear its old tail in the same set() call. + for (let n = 0; n < MAX_CHUNKS; n++) items[PREFIX + 'chunk:' + n] = data.slice(n * CHUNK_SIZE, (n + 1) * CHUNK_SIZE); + const usedBytes = bytesUsed({ ...existing, ...items }); + if (data.length > CHUNK_SIZE * MAX_CHUNKS || usedBytes > QUOTA_BYTES + || Object.keys({ ...existing, ...items }).length > 512) { throw new Error('Browser sync storage is full. All data is kept on this device; export a backup to transfer it.'); } - return { changes, removals, skipped, usedBytes: bytesUsed(proposed) }; + return { items, usedBytes }; } diff --git a/store/privacy-policy-firefox.md b/store/privacy-policy-firefox.md index 9db5e4b..9c81bd6 100644 --- a/store/privacy-policy-firefox.md +++ b/store/privacy-policy-firefox.md @@ -19,7 +19,7 @@ Uninstalling the extension removes all of it. Settings → Reset Settings clears The extension makes no network requests of its own. The one thing that leaves your device is the optional cross-device sync copy, and it leaves through Firefox Sync — the same channel that carries your bookmarks — to the other devices signed into the same Firefox account. If Firefox Sync is off, nothing leaves the device. -Sync is on by default. It writes independent compressed records containing settings, EQ presets, saved songs (including URLs, titles, durations and thumbnail URLs), favorites and manual order, practice parameters, markers, labels and snippets. Recent activity, UI layout, last-used parameters and generated chord analysis stay on the device and are included in manual backup exports. +Sync is on by default. It writes a compressed library snapshot containing settings, EQ presets, saved songs (including URLs, titles, durations and thumbnail URLs), favorites and manual order, practice parameters, markers, labels and snippets. Recent activity, UI layout, last-used parameters and generated chord analysis stay on the device and are included in manual backup exports. Because that data contains the addresses of pages you have visited, this listing declares the `browsingActivity` data-collection category. @@ -29,7 +29,7 @@ Not included: audio, page content, keystrokes, browsing history beyond the track Into Firefox Sync's storage under your Firefox account, end-to-end encrypted, subject to Mozilla's own data handling. The author operates no server and can see none of it. There are no accounts with us and no sync ID. -Firefox caps synced storage at 100 KB per extension and 8 KB per item. If a compressed record or the library exceeds those limits, Settings reports an error. All data remains saved locally; no songs are automatically trimmed. +Firefox caps synced storage at 100 KB per extension and 8 KB per item. The snapshot is split into size-limited pieces. If it exceeds the total capacity, Settings reports an error. All data remains saved locally; no songs are automatically trimmed. The newest complete library replaces older copies, so edits made on two devices at once can overwrite each other. **Turning it off and deleting the data** From c985cf2f123dedd84da23d3e840203c49223404d Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Tue, 8 Sep 2026 07:22:54 +0200 Subject: [PATCH 23/26] Implement library recovery and backup features - Add `recoverLegacyStorage` function to salvage legacy data while retaining original records. - Create tests for recovery scenarios, ensuring unrelated saved work is preserved. - Introduce `LibraryRecovery` component for user interaction during recovery failures. - Enhance `applyCommand` to reject edits from sessions opened before a replacement import. - Update settings and sync logic to handle import revisions and prevent stale edits. - Modify sync configuration to support incomplete chunks and recover lost headers. - Implement export and import functionality for recovery data in `LibraryRecovery` component. --- CLAUDE.md | 8 + README.md | 8 +- e2e/library.mjs | 32 +++- src/core/persist/backup-codec.ts | 31 ++-- src/core/persist/library-background.test.ts | 150 ++++++++++++++++++ src/core/persist/library-background.ts | 110 ++++++++----- src/core/persist/library-recovery.test.ts | 56 +++++++ src/core/persist/library-recovery.ts | 64 ++++++++ src/core/persist/library.test.ts | 20 +++ src/core/persist/library.ts | 20 ++- src/core/state/connect.svelte.ts | 13 +- src/core/state/persistence.test.ts | 134 ++++++++++++++++ src/core/state/track-sync.svelte.ts | 40 ++++- src/entrypoints/sidepanel/App.svelte | 3 + .../settings/panel/SettingsView.svelte | 2 +- src/features/sync/panel/sync.svelte.ts | 6 +- src/features/sync/persist/records.test.ts | 11 +- src/features/sync/persist/records.ts | 16 +- src/features/sync/persist/sync-config.ts | 3 + src/ui/LibraryRecovery.svelte | 50 ++++++ 20 files changed, 696 insertions(+), 81 deletions(-) create mode 100644 src/core/persist/library-background.test.ts create mode 100644 src/core/persist/library-recovery.test.ts create mode 100644 src/core/persist/library-recovery.ts create mode 100644 src/core/state/persistence.test.ts create mode 100644 src/ui/LibraryRecovery.svelte diff --git a/CLAUDE.md b/CLAUDE.md index c9a2f62..e4301d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,6 +104,8 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel data; Recent and Favorites are projections, not persistent song copies. - Track-sync loads a practice session once and submits edits to the saved library. Receiving remote changes never reloads or silently replaces the active session. + Explicit imports reload active sessions and advance a device-local import + revision; the writer rejects practice edits carrying an older revision. Feature persistence is wired directly in track-sync; there is no descriptor registry. - Sync copies the same SharedLibrary snapshot used locally (records.ts). One updatedAt timestamp chooses the whole winner; equal dates adopt the remote copy. @@ -112,12 +114,18 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel a hash prevents partial or mixed snapshots from being applied. Capacity failures preserve local data and the last successful upload. Background alarms retry independently of panels. + Identical snapshots do not rewrite storage or toggle sync status. Missing + headers are recovered from complete gzip chunks; partial headerless uploads + get a persisted grace period before repair from the complete local copy. - Backups use the readable v2 library schema. legacy-backup.ts reads the released v1 format, and library-migration.ts collapses its old copies once. Only released formats need compatibility adapters; intermediate PR formats do not. Old local storage is retained for recovery, but only local:library is used after migration. + Automatic legacy migration salvages fields independently (library-recovery.ts). + Invalid current libraries remain untouched and open a recovery screen; a valid + recovery import retains the damaged value under local:libraryRecovery. - Web identity uses provider ID/normalized URL. Title and duration are metadata. Local files retain a filename discriminator independent of the extension URL. diff --git a/README.md b/README.md index 441efaf..174867e 100644 --- a/README.md +++ b/README.md @@ -177,8 +177,9 @@ are views of that song, so there are no saved-settings copies to keep aligned. The background is the only library writer; it handles edits, imports and remote updates even when the panel is closed. Each panel loads and watches one library; settings, UI preferences, presets and song lists read that same copy. -An active practice session keeps its loaded -configuration until the song is reopened. Sync never reloads the panel. +An active practice session keeps its loaded configuration when sync arrives. +An explicit backup import reloads open songs with the imported practice settings +and cancels pending edits from before the import. Sync never reloads the panel. Local storage and sync use the same shared library snapshot, with one timestamp. The most recently edited snapshot replaces the older copy in full. On equal @@ -199,6 +200,9 @@ Importing a backup replaces the entire library and dates it as a new edit for sync, including removal of songs absent from the file. Old local storage is retained as a recovery copy after the first migration. Upgrade all devices before using the new sync format; older builds cannot read it. +Automatic migration recovers valid songs and fields independently of damaged +records. If a saved library cannot be opened, the panel offers retry, recovery +export and backup import; importing retains a copy of the damaged library locally. ## License diff --git a/e2e/library.mjs b/e2e/library.mjs index 2686035..bf95873 100644 --- a/e2e/library.mjs +++ b/e2e/library.mjs @@ -1,7 +1,7 @@ /** Background-library integration checks in an isolated Chrome profile. * Run after `wxt build --mode testing`: node e2e/library.mjs */ import assert from 'node:assert/strict'; -import { globSync, mkdtempSync } from 'node:fs'; +import { globSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -36,7 +36,9 @@ async function rpc(page, type, data) { }, { type, data }); } const read = (page) => rpc(page, 'libraryRead'); -const edit = (page, command) => rpc(page, 'libraryEdit', command); +const edit = async (page, command) => rpc(page, 'libraryEdit', + command.type === 'practice' || command.type === 'chart' + ? { ...command, importRevision: (await read(page)).local.importRevision ?? 0 } : command); const sync = (page, action) => rpc(page, 'librarySync', action); const identity = (n) => makeTrackIdentity(`https://youtube.com/watch?v=integration${n}`, `Song ${n}`, 200); @@ -172,6 +174,11 @@ try { const large = Array.from({ length: 5000 }, (_, n) => ({ id: `m${n}`, t: n, label: crypto.randomUUID() })); await edit(reopened, { type: 'practice', identity: identity(100), patch: { markers: large }, recent: true }); + // Capacity is checked only when an upload is due, before any remote write. + await reopened.evaluate(async () => { + const { syncConfig } = await chrome.storage.local.get('syncConfig'); + await chrome.storage.local.set({ syncConfig: { ...syncConfig, lastPushAt: 0 } }); + }); await sync(reopened, 'now'); const config = await reopened.evaluate(async () => (await chrome.storage.local.get('syncConfig')).syncConfig); assert.match(config.lastError, /storage is full/); @@ -183,6 +190,27 @@ try { assert.equal(Object.keys(await reopened.evaluate(() => chrome.storage.sync.get(null))).length, 0); assert.equal((await read(reopened)).shared.songs[identity(100).key].practice.markers.length, 5000); console.log('PASS deleting the remote copy disables sync and preserves the local library'); + + // A corrupt saved record must expose a working recovery UI on the next wake. + const recoveryBackup = await read(reopened); + const damaged = structuredClone(recoveryBackup); + damaged.local.recent.broken = 'not a date'; + await reopened.evaluate((library) => chrome.storage.local.set({ library }), damaged); + await browser.close(); + browser = await launch(); + const recovery = await panel(); + await recovery.setViewport({ width: 400, height: 700 }); + await recovery.waitForSelector('main[aria-label="Library recovery"]'); + assert.match(await recovery.$eval('main', (node) => node.textContent), /saved data is still on this device/); + await recovery.screenshot({ path: resolve(root, '.output', 'pr12-recovery.png') }); + const file = join(profile, 'restore.json'); + writeFileSync(file, JSON.stringify({ format: 'note-by-note-backup', version: 2, exportedAt: Date.now(), ...recoveryBackup })); + recovery.once('dialog', (dialog) => dialog.accept()); + await (await recovery.$('input[aria-label="Import backup"]')).uploadFile(file); + await recovery.waitForSelector('button[aria-label="Settings"]'); + assert.deepEqual((await read(recovery)).shared.songs, recoveryBackup.shared.songs); + assert.deepEqual(await recovery.evaluate(async () => (await chrome.storage.local.get('libraryRecovery')).libraryRecovery), damaged); + console.log('PASS corrupt data shows recovery controls and backup import restores the panel'); assert.deepEqual(panelErrors, [], 'panels must not report unhandled errors'); } finally { await browser?.close(); diff --git a/src/core/persist/backup-codec.ts b/src/core/persist/backup-codec.ts index 6a553ef..7e97f2e 100644 --- a/src/core/persist/backup-codec.ts +++ b/src/core/persist/backup-codec.ts @@ -23,18 +23,24 @@ function array(value: unknown): any[] { return value; } /** Missing preference groups use defaults, just like missing individual fields. */ -function defaults(value: unknown, fallback: T): T { - const source = object(value ?? {}); +export function defaults(value: unknown, fallback: T, recover = false): T { + let source: Record; + try { source = object(value ?? {}); } catch (error) { + if (!recover) throw error; + return structuredClone(fallback); + } const result = structuredClone(fallback) as Record; for (const [key, expected] of Object.entries(result)) { if (!(key in source)) continue; - const next = source[key]; - if (expected === null) { if (next !== null) number(next); } - else if (Array.isArray(expected)) array(next).forEach(number); - else if (typeof expected === 'object') { result[key] = defaults(next, expected); continue; } - else if (typeof next !== typeof expected) throw new Error('Damaged library setting.'); - else if (typeof next === 'number') number(next); - result[key] = next; + try { + const next = source[key]; + if (expected === null) { if (next !== null) number(next); } + else if (Array.isArray(expected)) array(next).forEach(number); + else if (typeof expected === 'object') { result[key] = defaults(next, expected, recover); continue; } + else if (typeof next !== typeof expected) throw new Error('Damaged library setting.'); + else if (typeof next === 'number') number(next); + result[key] = next; + } catch (error) { if (!recover) throw error; } } return result as T; } @@ -86,10 +92,15 @@ export function parseLibrary(value: unknown): Library { const recent = object(local.recent ?? {}); Object.values(recent).forEach(number); Object.values(lastAccessed).forEach(number); + if (local.importRevision !== undefined) { + number(local.importRevision); + if (!Number.isSafeInteger(local.importRevision) || local.importRevision < 0) throw new Error('Damaged import revision.'); + } return { shared: parseShared(source.shared), local: { uiPrefs: defaults(local.uiPrefs, DEFAULT_UI_PREFS), recent, lastAccessed, charts, - ...(local.lastUsedParams ? { lastUsedParams: defaults(local.lastUsedParams, DEFAULT_PARAMS) } : {}) }, + ...(local.lastUsedParams ? { lastUsedParams: defaults(local.lastUsedParams, DEFAULT_PARAMS) } : {}), + ...(local.importRevision !== undefined ? { importRevision: local.importRevision } : {}) }, }; } export function parseBackupJson(value: unknown): Backup { diff --git a/src/core/persist/library-background.test.ts b/src/core/persist/library-background.test.ts new file mode 100644 index 0000000..8b85ec1 --- /dev/null +++ b/src/core/persist/library-background.test.ts @@ -0,0 +1,150 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { build } from 'esbuild'; +import { fileURLToPath } from 'node:url'; +import { applyCommand, emptyLibrary } from './library.ts'; +import { bytesUsed, encodeSnapshot, readSnapshot, PREFIX, SNAPSHOT_KEY } from '../../features/sync/persist/records.ts'; + +// Bundle the real worker with only its browser/RPC boundaries replaced. Each +// start gets fresh queues and listeners while its saved storage/alarms survive. +const bundled = await build({ entryPoints: [fileURLToPath(new URL('./library-background.ts', import.meta.url))], + bundle: true, write: false, platform: 'node', format: 'esm', plugins: [{ name: 'browser-test', setup(builder) { + builder.onResolve({ filter: /^(#imports)$|\/messaging\/rpc$/ }, ({ path }) => ({ path, namespace: 'test' })); + builder.onLoad({ filter: /.*/, namespace: 'test' }, ({ path }) => ({ contents: path === '#imports' + ? 'export const storage = { defineItem: (...args) => ({ getValue: () => globalThis.libraryTest.item(...args).getValue(), setValue: (v) => globalThis.libraryTest.item(...args).setValue(v) }) };' + : 'export const onMessage = (...args) => globalThis.libraryTest.onMessage(...args); export const sendMessage = () => {};', loader: 'js' })); + } }] }); +const { startLibraryBackground } = await import('data:text/javascript;base64,' + Buffer.from(bundled.outputFiles[0].text).toString('base64')); + +function harness(local: Record, sync: Record = {}) { + const areas = { local: structuredClone(local), sync: structuredClone(sync) }; + const writes: { area: string; items: Record }[] = []; + const alarms = new Map>(); + const handlers = new Map Promise>(); + let changed: ((changes: Record, area: string) => void)[] = []; + let alarmListeners: ((alarm: { name: string }) => void)[] = []; + const area = (name: keyof typeof areas) => ({ + async get(key: string | null) { return structuredClone(key === null ? areas[name] : { [key]: areas[name][key] }); }, + async set(items: Record) { + writes.push({ area: name, items: structuredClone(items) }); + Object.assign(areas[name], structuredClone(items)); + for (const listener of changed) listener(items, name); + }, + async remove(keys: string[]) { for (const key of keys) delete areas[name][key]; }, + }); + const browserMock = { + storage: { local: area('local'), sync: area('sync'), onChanged: { addListener: (fn: typeof changed[number]) => changed.push(fn) } }, + alarms: { + async get(name: string) { return alarms.get(name); }, + async create(name: string, info: { when?: number; periodInMinutes?: number }) { + alarms.set(name, { scheduledTime: info.when ?? Date.now() + info.periodInMinutes! * 60000 }); + }, + async clear(name: string) { return alarms.delete(name); }, + onAlarm: { addListener: (fn: typeof alarmListeners[number]) => alarmListeners.push(fn) }, + }, + }; + const global = globalThis as any; + global.browser = browserMock; + global.libraryTest = { + onMessage: (name: string, fn: typeof handlers extends Map ? V : never) => handlers.set(name, fn), + item: (key: string, { fallback }: { fallback: unknown }) => ({ + async getValue() { return structuredClone(areas.local[key.split(':')[1]] ?? fallback); }, + async setValue(value: unknown) { await browserMock.storage.local.set({ [key.split(':')[1]]: value }); }, + }), + }; + const rpc = (name: string, data?: unknown) => handlers.get(name)!({ data }); + return { areas, writes, alarms, rpc, + async start() { changed = []; alarmListeners = []; handlers.clear(); startLibraryBackground(); await rpc('librarySync', 'now'); }, + async fire(name: string) { for (const listener of alarmListeners) listener({ name }); await rpc('librarySync', 'now'); }, + }; +} +const config = { enabled: true, syncing: false, lastPushAt: 0, lastSyncedAt: 10, usedBytes: 0, lastError: null }; +const saved = () => applyCommand(emptyLibrary(), { type: 'preset', name: 'Saved', gains: [1] }, 100); + +test('settled sync and worker restarts do not rewrite data, flicker status or postpone safety alarms', async (t) => { + let now = 100000; + t.mock.method(Date, 'now', () => now); + const library = saved(); + const { items } = await encodeSnapshot(library.shared); + const h = harness({ library, syncConfig: { ...config, usedBytes: bytesUsed(items) } }, items); + await h.start(); + const deadline = h.alarms.get('library-sync-safety')!.scheduledTime; + assert.deepEqual(h.writes, []); + now += 10000; + await h.start(); + await h.rpc('libraryRead'); + await h.fire('library-sync-safety'); + assert.deepEqual(h.writes, []); + assert.equal(h.alarms.get('library-sync-safety')!.scheduledTime, deadline); +}); + +test('a different equal-date remote snapshot is adopted once', async () => { + const library = saved(); + const remote = { ...library.shared, presets: { Remote: [2] } }; + const { items } = await encodeSnapshot(remote); + const h = harness({ library, syncConfig: config }, items); + await h.start(); + assert.deepEqual(h.areas.local.library.shared, remote); + assert.equal(h.writes.filter((w) => w.items.library).length, 1); + h.writes.length = 0; + await h.rpc('librarySync', 'now'); + assert.deepEqual(h.writes, []); +}); + +test('rate-limited uploads defer compression and status changes', async (t) => { + t.mock.method(Date, 'now', () => 100000); + const library = saved(); + const { items } = await encodeSnapshot(emptyLibrary().shared); + const h = harness({ library, syncConfig: { ...config, lastPushAt: 99999, usedBytes: bytesUsed(items) } }, items); + let compressed = 0; + const compress = globalThis.CompressionStream; + t.mock.method(globalThis, 'CompressionStream', class extends compress { constructor(format: CompressionFormat) { super(format); compressed++; } }); + await h.start(); + assert.equal(compressed, 0); + assert.deepEqual(h.writes, []); + assert.equal(h.alarms.get('library-sync')!.scheduledTime, 129999); +}); + +test('complete headerless data keeps the newer remote revision and repairs the header', async () => { + const library = saved(); + const remote = { ...library.shared, updatedAt: 200, presets: { Newer: [2] } }; + const { items } = await encodeSnapshot(remote); + delete items[SNAPSHOT_KEY]; + const h = harness({ library, syncConfig: config }, items); + await h.start(); + assert.deepEqual(h.areas.local.library.shared, remote); + assert.deepEqual(await readSnapshot(h.areas.sync), remote); + assert.ok(h.areas.sync[SNAPSHOT_KEY]); +}); + +test('orphaned partial chunks repair after a bounded wait that survives worker restarts', async (t) => { + let now = 100000; + t.mock.method(Date, 'now', () => now); + const library = saved(); + const h = harness({ library, syncConfig: config }, { [PREFIX + 'chunk:0']: 'partial' }); + await h.start(); + assert.equal(h.areas.local.syncConfig.incompleteSince, now); + assert.equal(h.writes.filter((w) => w.area === 'sync').length, 0); + now += 60000; + await h.start(); + assert.equal(h.areas.local.syncConfig.incompleteSince, 100000); + now += 60001; + await h.fire('library-sync'); + assert.deepEqual(await readSnapshot(h.areas.sync), library.shared); + assert.equal(h.areas.local.syncConfig.lastError, null); +}); + +test('a damaged saved library stays intact and can be replaced through the recovery import', async () => { + const damaged = saved() as any; + damaged.local.recent = { broken: 'bad date' }; + const h = harness({ library: damaged, syncConfig: config }); + await h.start(); + await assert.rejects(h.rpc('libraryRead'), /number/); + assert.deepEqual(h.areas.local.library, damaged); + const restored = saved(); + await h.rpc('libraryEdit', { type: 'import', library: restored }); + const library = await h.rpc('libraryRead'); + assert.deepEqual(library.shared.presets, restored.shared.presets); + assert.deepEqual(h.areas.local.libraryRecovery, damaged); + assert.equal(library.local.importRevision, 1); +}); diff --git a/src/core/persist/library-background.ts b/src/core/persist/library-background.ts index 658b5c7..35e4d4f 100644 --- a/src/core/persist/library-background.ts +++ b/src/core/persist/library-background.ts @@ -1,10 +1,9 @@ import { onMessage } from '../messaging/rpc'; import { applyCommand, emptyLibrary, newestSnapshot, type Library } from './library'; import { libraryItem } from './library-client'; -import { parseBackupJson as parseLegacy } from './legacy-backup'; import { parseLibrary } from './backup-codec'; -import { migrateBackup } from './library-migration'; -import { bytesUsed, encodeSnapshot, IncompleteSnapshot, PREFIX, readSnapshot, SNAPSHOT_KEY } from '../../features/sync/persist/records'; +import { recoverLegacyStorage } from './library-recovery'; +import { bytesUsed, encodeSnapshot, hash, IncompleteSnapshot, PREFIX, readSnapshot, SNAPSHOT_KEY } from '../../features/sync/persist/records'; import { loadSyncConfig, syncConfigItem } from '../../features/sync/persist/sync-config'; const WAKE = 'library-sync'; @@ -12,6 +11,7 @@ const SAFETY = 'library-sync-safety'; /** Chromium allows ~2 writes/second sustained; one push per 30 s is well under * it and still lets a burst of edits ride out together. */ const PUSH_INTERVAL = 30000; +const INCOMPLETE_GRACE = 120000; /** One writer for edits, imports and sync. Saved snapshots survive worker restarts. */ export function startLibraryBackground() { @@ -21,35 +21,15 @@ export function startLibraryBackground() { queue = result.catch(() => {}); return result; }; - /** `parseLegacy` rejects a damaged file, which is right for a backup the user - * picked but fatal here: a single bad row would leave the panel with no - * library at all, on every start, forever. Unusable rows are dropped instead, - * and a parse that still fails yields an empty library. Either way the old - * records are left in place, so nothing is beyond recovery. */ - const migrate = (raw: Record): Library => { - const defaults = emptyLibrary(); - const identified = (value: unknown) => (Array.isArray(value) ? value : []).filter((row) => - typeof (row as { identity?: { normalizedUrl?: unknown } })?.identity?.normalizedUrl === 'string'); - try { - return parseLibrary(migrateBackup(parseLegacy({ format: 'note-by-note-backup', version: 1, - settings: raw.settings, uiPrefs: raw.uiPrefs, - history: identified(raw.history), favorites: identified(raw.favorites), - eqPresets: Array.isArray(raw.eqPresets) ? raw.eqPresets : [], - tracks: identified(Object.entries(raw).filter(([key]) => key.startsWith('track:')).map(([, value]) => value)), - }))); - } catch (error) { - console.error('[note-by-note] the previous library could not be migrated; its records are kept', error); - return defaults; - } - }; let ready: Promise | undefined; const init = () => ready ??= (async () => { // Only the migration needs every old key. Ordinary wakes read one item. const { library } = await browser.storage.local.get('library'); - if (library) { - await libraryItem.setValue(parseLibrary(library)); + if (library !== undefined) { + const normalized = parseLibrary(library); + if (JSON.stringify(normalized) !== JSON.stringify(library)) await libraryItem.setValue(normalized); } else { - await libraryItem.setValue(migrate(await browser.storage.local.get(null))); + await libraryItem.setValue(recoverLegacyStorage(await browser.storage.local.get(null))); } })().catch((error) => { ready = undefined; throw error; }); const schedule = async () => { @@ -57,44 +37,86 @@ export function startLibraryBackground() { if (config.enabled) await browser.alarms.create(WAKE, { when: Math.max(Date.now() + 5000, config.lastPushAt + PUSH_INTERVAL) }); }; const reconcile = async () => { - await init(); const config = await loadSyncConfig(); if (!config.enabled) return; - await syncConfigItem.setValue({ ...config, syncing: true }); + const before = JSON.stringify(config); + let active = false; + const begin = async () => { + if (!active) { active = true; await syncConfigItem.setValue({ ...config, syncing: true }); } + }; try { + await init(); const existing = await browser.storage.sync.get(null); config.usedBytes = bytesUsed(existing); const local = await libraryItem.getValue(); - const remote = await readSnapshot(existing).catch((error) => { + const remote = await readSnapshot(existing).catch(async (error) => { // Repair an interrupted upload from our complete local copy. A newer // incomplete snapshot must finish arriving before we can choose a winner. - if (error instanceof IncompleteSnapshot && local.shared.updatedAt >= error.updatedAt) return null; + if (error instanceof IncompleteSnapshot) { + if (error.updatedAt !== null && local.shared.updatedAt >= error.updatedAt) return null; + if (error.updatedAt === null) { + const fingerprint = await hash(JSON.stringify(Object.entries(existing) + .filter(([key]) => key.startsWith(PREFIX)).sort(([a], [b]) => a.localeCompare(b)))); + if (config.incompleteHash !== fingerprint || config.incompleteSince === undefined) { + config.incompleteHash = fingerprint; + config.incompleteSince = Date.now(); + } + // Let separate key arrivals settle. A permanently orphaned upload + // is repaired from our complete copy after a bounded, durable wait. + if (Date.now() >= config.incompleteSince + INCOMPLETE_GRACE) return null; + await browser.alarms.create(WAKE, { when: config.incompleteSince + INCOMPLETE_GRACE }); + } + } throw error; }); const shared = remote ? newestSnapshot(local.shared, remote) : local.shared; - if (shared !== local.shared) await libraryItem.setValue({ ...local, shared }); + const adopting = shared !== local.shared; + if (adopting) { await begin(); await libraryItem.setValue({ ...local, shared }); } config.lastError = null; - if (shared !== remote || !existing[SNAPSHOT_KEY]) { - const { items, usedBytes } = await encodeSnapshot(shared, existing); + delete config.incompleteSince; + delete config.incompleteHash; + const uploading = !remote || shared.updatedAt > remote.updatedAt || !existing[SNAPSHOT_KEY]; + if (uploading) { if (Date.now() < config.lastPushAt + PUSH_INTERVAL) { await schedule(); return; } + await begin(); + const { items, usedBytes } = await encodeSnapshot(shared, existing); await browser.storage.sync.set(items); config.lastPushAt = Date.now(); config.usedBytes = usedBytes; } - config.lastSyncedAt = Date.now(); + if (adopting || uploading || !config.lastSyncedAt) config.lastSyncedAt = Date.now(); } catch (error) { config.lastError = error instanceof Error ? error.message : String(error); } finally { - await syncConfigItem.setValue({ ...config, syncing: false }); + config.syncing = false; + if (active || before !== JSON.stringify(config)) await syncConfigItem.setValue(config); } }; onMessage('libraryRead', () => enqueue(async () => { await init(); return libraryItem.getValue(); })); onMessage('libraryEdit', ({ data }) => enqueue(async () => { - await init(); - if (data.type === 'import') data.library = parseLibrary(data.library); - const current = await libraryItem.getValue(); + let current: Library; + if (data.type === 'import') { + data.library = parseLibrary(data.library); + // Recovery imports must work even when initialization cannot parse the + // saved library. Keep that original available before replacing it. + const raw = (await browser.storage.local.get('library')).library; + try { current = raw === undefined ? emptyLibrary() : parseLibrary(raw); } + catch { + await browser.storage.local.set({ libraryRecovery: raw }); + current = emptyLibrary(); + const damaged = raw as Partial | null; + const updatedAt = damaged?.shared?.updatedAt; + const importRevision = damaged?.local?.importRevision; + if (typeof updatedAt === 'number' && Number.isFinite(updatedAt) && updatedAt >= 0) current.shared.updatedAt = updatedAt; + if (typeof importRevision === 'number' && Number.isSafeInteger(importRevision) && importRevision >= 0) current.local.importRevision = importRevision; + } + } else { + await init(); + current = await libraryItem.getValue(); + } const next = applyCommand(current, data); await libraryItem.setValue(next); + if (data.type === 'import') ready = Promise.resolve(); if (current.shared.updatedAt !== next.shared.updatedAt) await schedule(); })); onMessage('librarySync', ({ data }) => enqueue(async () => { @@ -124,10 +146,10 @@ export function startLibraryBackground() { void enqueue(reconcile); } }); - // Recreate the safety alarm on each worker start. No panel has to stay open. - // Only a net: `libraryEdit` schedules WAKE and `storage.onChanged` catches - // remote writes, so this never needs to be the thing that notices a change — - // and each run wakes the worker to read the shared snapshot. - void browser.alarms.create(SAFETY, { periodInMinutes: 30 }); + // Creating an existing alarm postpones it. Keep its deadline across wakes; + // recreate only when the browser has dropped it (for example after restart). + void browser.alarms.get(SAFETY).then((alarm) => { + if (!alarm) return browser.alarms.create(SAFETY, { periodInMinutes: 30 }); + }); void enqueue(reconcile); } diff --git a/src/core/persist/library-recovery.test.ts b/src/core/persist/library-recovery.test.ts new file mode 100644 index 0000000..f444e32 --- /dev/null +++ b/src/core/persist/library-recovery.test.ts @@ -0,0 +1,56 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { recoverLegacyStorage } from './library-recovery.ts'; +import { DEFAULT_PARAMS, DEFAULT_SETTINGS } from '../model/defaults.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; +import { parseBackupJson } from './backup-codec.ts'; + +const a = makeTrackIdentity('https://youtube.com/watch?v=a', 'A', 100); +const b = makeTrackIdentity('https://youtube.com/watch?v=b', 'B', 200); +const marker = { id: 'good', t: 12, label: 'Verse' }; +const snippet = { id: 's', name: 'Solo', startT: 2, endT: 8, repeats: null, enabled: true, overrides: {} }; + +test('one damaged legacy row or field cannot discard unrelated saved work', () => { + const raw = { + history: [{ identity: a, pageUrl: a.normalizedUrl, params: { ...DEFAULT_PARAMS, speed: 0.7 } }, + { identity: b, updatedAt: 50, pageUrl: b.normalizedUrl, params: DEFAULT_PARAMS }, null], + favorites: [{ identity: b, updatedAt: 50, favoritedAt: 60, lastAccessedAt: 70, params: DEFAULT_PARAMS }], + 'track:a': { identity: a, updatedAt: 10, markers: [marker, { ...marker, id: 'bad', t: NaN }], + snippets: [snippet, { ...snippet, id: 'bad', startT: 'broken' }], sequenceLoop: true, chordChart: { coverage: 'bad' } }, + 'track:b': { identity: b, updatedAt: 20, markers: [{ ...marker, id: 'b' }], snippets: [snippet] }, + eqPresets: [{ name: 'Keep me', gains: [1, 2] }, { name: 'Bad', gains: ['bad'] }], + settings: { theme: 'dark', seekInterval: 'broken', keymap: { playPause: 'KeyP', seekBack: 123 } }, + uiPrefs: { collapsedSections: { looper: true, tools: 'broken' } }, + }; + const before = structuredClone(raw); + const recovered = recoverLegacyStorage(raw); + assert.deepEqual(raw, before, 'source records stay available for recovery'); + assert.deepEqual(Object.keys(recovered.shared.songs), [a.key, b.key]); + assert.deepEqual(recovered.shared.songs[a.key].practice.markers, [marker]); + assert.deepEqual(recovered.shared.songs[a.key].practice.snippets, [snippet]); + assert.equal(recovered.shared.songs[a.key].practice.params!.speed, 0.7); + assert.equal(recovered.local.recent[a.key], 0, 'missing dates use a safe default'); + assert.equal(recovered.shared.songs[b.key].favoritedAt, 60); + assert.equal(recovered.shared.songs[b.key].practice.markers[0].id, 'b'); + assert.deepEqual(recovered.shared.presets, { 'Keep me': [1, 2] }); + assert.equal(recovered.shared.settings.theme, 'dark'); + assert.equal(recovered.shared.settings.seekInterval, DEFAULT_SETTINGS.seekInterval); + assert.equal(recovered.shared.settings.keymap.playPause, 'KeyP'); + assert.equal(recovered.shared.settings.keymap.seekBack, DEFAULT_SETTINGS.keymap.seekBack); + assert.equal(recovered.local.uiPrefs.collapsedSections.looper, true); + assert.equal(recovered.local.charts[a.key], null); +}); + +test('a malformed legacy group does not discard other groups', () => { + const recovered = recoverLegacyStorage({ history: false, favorites: [null], settings: 'bad', + 'track:good': { identity: a, markers: [marker] }, eqPresets: [{ name: 'Still here', gains: [3] }] }); + assert.equal(recovered.shared.songs[a.key].practice.markers[0].t, 12); + assert.deepEqual(recovered.shared.presets, { 'Still here': [3] }); +}); + +test('automatic recovery does not make backup imports accept damaged records', () => { + assert.throws(() => parseBackupJson({ format: 'note-by-note-backup', version: 1, + history: [], favorites: [], eqPresets: [], + tracks: [{ identity: a, updatedAt: 1, markers: [{ ...marker, t: 'broken' }], snippets: [], sequenceLoop: false, sequenceCountIn: false }], + }), /number/); +}); diff --git a/src/core/persist/library-recovery.ts b/src/core/persist/library-recovery.ts new file mode 100644 index 0000000..1abcfc4 --- /dev/null +++ b/src/core/persist/library-recovery.ts @@ -0,0 +1,64 @@ +import { DEFAULT_PARAMS, DEFAULT_SETTINGS, DEFAULT_UI_PREFS } from '../model/defaults.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; +import type { ChordChart, FavoriteEntry, HistoryEntry, TrackData, TrackIdentity } from '../model/types'; +import { defaults, parseLibrary } from './backup-codec.ts'; +import { emptyLibrary } from './library.ts'; +import { migrateBackup } from './library-migration.ts'; +import { rekeyByIdentity } from './rekey.ts'; + +const record = (value: unknown): Record => + value && typeof value === 'object' && !Array.isArray(value) ? value : {}; +const list = (value: unknown): any[] => Array.isArray(value) ? value : []; +const finite = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value); +const date = (value: unknown) => finite(value) && value >= 0 ? value : 0; + +/** The automatic upgrade salvages each field independently. Backup-file imports + * remain strict, and the original storage keys are retained for recovery. */ +export function recoverLegacyStorage(raw: Record) { + const identified = (rows: unknown): (Record & { identity: TrackIdentity; updatedAt: number })[] => list(rows).flatMap((value) => { + const row = record(value), identity = record(row.identity); + if (typeof identity.normalizedUrl !== 'string') return []; + return [{ ...row, identity: makeTrackIdentity(identity.normalizedUrl, + typeof identity.title === 'string' ? identity.title : identity.normalizedUrl, + finite(identity.durationSec) ? identity.durationSec : 0), updatedAt: date(row.updatedAt) }]; + }); + const entry = (row: ReturnType[number]): HistoryEntry => ({ + identity: row.identity, updatedAt: row.updatedAt, + pageUrl: typeof row.pageUrl === 'string' ? row.pageUrl : row.identity.normalizedUrl, + ...(typeof row.thumbnailUrl === 'string' ? { thumbnailUrl: row.thumbnailUrl } : {}), + params: defaults(row.params, DEFAULT_PARAMS, true), + }); + const chart = (value: unknown): ChordChart | null => { + try { + const candidate = emptyLibrary(); + candidate.local.charts.recovery = value as ChordChart; + return parseLibrary(candidate).local.charts.recovery; + } catch { return null; } + }; + const tracks: TrackData[] = identified(Object.entries(raw) + .filter(([key]) => key.startsWith('track:')).map(([, value]) => value)).map((row) => ({ + identity: row.identity, updatedAt: row.updatedAt, + markers: list(row.markers).filter((m) => m && typeof m.id === 'string' + && typeof m.label === 'string' && finite(m.t)), + snippets: list(row.snippets).filter((s) => s && typeof s.id === 'string' && typeof s.name === 'string' + && finite(s.startT) && finite(s.endT) && (s.repeats === null || s.repeats === Infinity || finite(s.repeats)) + && typeof s.enabled === 'boolean' && s.overrides && typeof s.overrides === 'object' + && !Array.isArray(s.overrides) && Object.values(s.overrides).every(finite)), + sequenceLoop: row.sequenceLoop === true, sequenceCountIn: row.sequenceCountIn === true, + ...(typeof row.chordsEnabled === 'boolean' ? { chordsEnabled: row.chordsEnabled } : {}), + chordChart: chart(row.chordChart ?? null), + })); + const history = identified(raw.history).map(entry); + const favorites: FavoriteEntry[] = identified(raw.favorites).map((row) => ({ + ...entry(row), favoritedAt: date(row.favoritedAt), lastAccessedAt: date(row.lastAccessedAt), + })); + const settings = defaults(raw.settings, DEFAULT_SETTINGS, true); + const lastUsed = record(raw.settings).lastUsedParams; + if (lastUsed) settings.lastUsedParams = defaults(lastUsed, DEFAULT_PARAMS, true); + return parseLibrary(migrateBackup({ format: 'note-by-note-backup', version: 1, + settings, uiPrefs: defaults(raw.uiPrefs, DEFAULT_UI_PREFS, true), + history: rekeyByIdentity(history), favorites: rekeyByIdentity(favorites), tracks: rekeyByIdentity(tracks), + eqPresets: list(raw.eqPresets).filter((p) => p && typeof p.name === 'string' + && Array.isArray(p.gains) && p.gains.every(finite)), + })); +} diff --git a/src/core/persist/library.test.ts b/src/core/persist/library.test.ts index 4a5da5e..5c0ede6 100644 --- a/src/core/persist/library.test.ts +++ b/src/core/persist/library.test.ts @@ -8,6 +8,7 @@ import { makeTrackIdentity } from '../model/track-identity.ts'; const identity = makeTrackIdentity('https://www.youtube.com/watch?v=example', 'Song', 200); const save = (library = emptyLibrary(), speed = 0.8, now = 100) => applyCommand(library, { type: 'practice', identity, patch: { params: { ...DEFAULT_PARAMS, speed } }, recent: true, + importRevision: library.local.importRevision, }, now); test('Recent and Favorites project the same saved parameters; removing Recent preserves a favorite', () => { @@ -43,6 +44,25 @@ test('equal timestamps adopt the synced snapshot and stay settled', () => { const remote = save(emptyLibrary(), 0.5, 100).shared; assert.equal(newestSnapshot(local, remote), remote); assert.equal(newestSnapshot(remote, remote), remote); + assert.equal(newestSnapshot(local, structuredClone(local)), local, 'identical remote data retains the local object'); +}); + +test('imports reject edits from old sessions, including songs absent from the backup', () => { + const original = save(); + const imported = applyCommand(original, { type: 'import', library: emptyLibrary() }); + const command = { type: 'practice' as const, identity, patch: { markers: [{ id: 'old', t: 4, label: 'Old' }] }, recent: true }; + assert.throws(() => applyCommand(imported, command), /backup was imported/); + assert.throws(() => applyCommand(imported, { type: 'chart', key: identity.key, chart: null }), /backup was imported/); + assert.throws(() => applyCommand(imported, { type: 'settings', patch: { lastUsedParams: DEFAULT_PARAMS }, importRevision: 0 }), /backup was imported/); + const edited = applyCommand(imported, { ...command, importRevision: imported.local.importRevision }); + assert.equal(edited.shared.songs[identity.key].practice.markers[0].id, 'old'); + const importedAgain = applyCommand(imported, { type: 'import', library: original }); + assert.throws(() => applyCommand(importedAgain, { ...command, importRevision: imported.local.importRevision }), /backup was imported/); + assert.equal(parseLibrary(importedAgain).local.importRevision, 2); +}); + +test('favoriting a missing song reports the failure', () => { + assert.throws(() => applyCommand(emptyLibrary(), { type: 'favorite', key: identity.key, value: true }), /no longer in the library/); }); test('local activity, layout, last-used parameters and chord analysis never date shared data', () => { diff --git a/src/core/persist/library.ts b/src/core/persist/library.ts index 66d3a22..9cca1b0 100644 --- a/src/core/persist/library.ts +++ b/src/core/persist/library.ts @@ -27,6 +27,8 @@ export interface Library { lastAccessed: Record; charts: Record; lastUsedParams?: EffectParams; + /** Invalidates edits from sessions opened before a replacement import. */ + importRevision?: number; }; } @@ -43,6 +45,7 @@ export function emptyLibrary(): Library { /** One winner for the entire library. On equal timestamps the synced copy wins. */ export function newestSnapshot(local: SharedLibrary, remote: SharedLibrary): SharedLibrary { + if (local.updatedAt === remote.updatedAt && JSON.stringify(local) === JSON.stringify(remote)) return local; return local.updatedAt > remote.updatedAt ? local : remote; } @@ -51,22 +54,28 @@ export type UiPrefsPatch = { }; export type LibraryCommand = - | { type: 'practice'; identity: TrackIdentity; patch: Partial; recent: boolean } + | { type: 'practice'; identity: TrackIdentity; patch: Partial; recent: boolean; importRevision?: number } | { type: 'favorite'; key: string; value: boolean } | { type: 'order'; keys: string[] } | { type: 'visit'; key: string } | { type: 'recent.remove'; key?: string } - | { type: 'chart'; key: string; chart: ChordChart | null } - | { type: 'settings'; patch: Partial; reset?: boolean } + | { type: 'chart'; key: string; chart: ChordChart | null; importRevision?: number } + | { type: 'settings'; patch: Partial; reset?: boolean; importRevision?: number } | { type: 'uiPrefs'; patch: UiPrefsPatch } | { type: 'preset'; name: string; gains: number[] | null } | { type: 'import'; library: Library }; /** Only the background writes. Commands from panels patch the current snapshot. */ export function applyCommand(library: Library, command: LibraryCommand, now = Date.now()): Library { + if ((command.type === 'practice' || command.type === 'chart' + || (command.type === 'settings' && command.importRevision !== undefined)) + && (command.importRevision ?? 0) !== (library.local.importRevision ?? 0)) { + throw new Error('A backup was imported. Reload the song before saving more practice edits.'); + } const next = structuredClone(command.type === 'import' ? command.library : library); const { shared, local } = next; switch (command.type) { + case 'import': local.importRevision = (library.local.importRevision ?? 0) + 1; break; case 'practice': { const key = command.identity.key; const song = shared.songs[key] ?? { practice: { @@ -82,7 +91,10 @@ export function applyCommand(library: Library, command: LibraryCommand, now = Da } case 'favorite': { const song = shared.songs[command.key]; - if (!song) break; + if (!song) { + if (command.value) throw new Error('This song is no longer in the library. Save it before adding it to Favorites.'); + break; + } song.favoritedAt = command.value ? now : null; shared.favoriteOrder = shared.favoriteOrder.filter((key) => key !== command.key); if (command.value) shared.favoriteOrder.unshift(command.key); diff --git a/src/core/state/connect.svelte.ts b/src/core/state/connect.svelte.ts index 6a7a094..b5e9c7c 100644 --- a/src/core/state/connect.svelte.ts +++ b/src/core/state/connect.svelte.ts @@ -16,16 +16,19 @@ function isLocalPlayer(url: string | undefined): boolean { /** A fresh engine starts on the default preset, so this runs on every attach as * well as on change — a no-op while nothing is attached. */ -export function pushSettings() { - session.send({ - type: 'settings', +let lastSettings = ''; +export function pushSettings(force = false) { + const command = { + type: 'settings' as const, seekInterval: settings.current.seekInterval, lowLatency: settings.current.lowLatency, formantPreserved: settings.current.formantPreserved, countInBeats: settings.current.countInBeats, countInBpm: settings.current.countInBpm, countInBeep: settings.current.countInBeep, - }); + }; + const serialized = JSON.stringify(command); + if ((force || serialized !== lastSettings) && session.send(command)) lastSettings = serialized; } /** Side-panel side of the connection: binds the session store to the engine in @@ -152,7 +155,7 @@ class ConnectionManager { }); session.attachTransport((cmd) => port.send(cmd)); port.send({ type: 'hello' }); - pushSettings(); + pushSettings(true); } /** "No compatible player": engine finds nothing but the tab is audible. */ diff --git a/src/core/state/persistence.test.ts b/src/core/state/persistence.test.ts new file mode 100644 index 0000000..7dee617 --- /dev/null +++ b/src/core/state/persistence.test.ts @@ -0,0 +1,134 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { build, transformSync } from 'esbuild'; +import { compileModule } from 'svelte/compiler'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { applyCommand, emptyLibrary, type Library, type LibraryCommand } from '../persist/library.ts'; +import { DEFAULT_PARAMS } from '../model/defaults.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; + +const stubs: Record = { + '/library-client': 'export const readLibrary = () => h.read(); export const editLibrary = (command) => h.edit(command); export const libraryItem = { watch: (fn) => h.watch = fn };', + '/library.svelte': 'export const library = h.library;', + '/session.svelte': 'export const session = h.session;', + '/settings.svelte': 'export const settings = h.settings;', + '/markers.svelte': 'export const markers = h.markers;', + '/snippets.svelte': 'export const snippets = h.snippets;', + '/chords.svelte': 'export const chords = h.chords;', + '/side-panel': 'export const openTabWithPanel = (url) => h.navigations.push(url);', + '/messaging/ports': 'export const connectToTab = () => {};', + '/messaging/rpc': 'export const sendMessage = () => {};', + '/persist/sync-config': 'export const DEFAULT_SYNC_CONFIG = {}; export const withSyncDefaults = (v) => v; export const loadSyncConfig = () => h.loadConfig(); export const syncConfigItem = { watch: (fn) => h.watch = fn };', +}; +async function bundle(file: string) { + const built = await build({ entryPoints: [fileURLToPath(new URL(file, import.meta.url))], bundle: true, + write: false, platform: 'node', format: 'esm', plugins: [{ name: 'panel-test', setup(builder) { + builder.onResolve({ filter: /.*/ }, ({ path }) => { + const stub = Object.keys(stubs).find((suffix) => path.endsWith(suffix)); + return stub ? { path: stub, namespace: 'test' } : undefined; + }); + builder.onLoad({ filter: /.*/, namespace: 'test' }, ({ path }) => ({ contents: 'const h = globalThis.panelTest; ' + stubs[path] })); + builder.onLoad({ filter: /\.svelte\.ts$/, namespace: 'file' }, ({ path }) => ({ + contents: compileModule(transformSync(readFileSync(path, 'utf8'), { loader: 'ts', target: 'esnext' }).code, + { filename: path, generate: 'client' }).js.code, + })); + } }] }); + return built.outputFiles[0].text; +} +const [trackCode, syncCode, connectionCode] = await Promise.all([ + bundle('./track-sync.svelte.ts'), bundle('../../features/sync/panel/sync.svelte.ts'), bundle('./connect.svelte.ts'), +]); +let instance = 0; +const load = (code: string) => import('data:text/javascript;base64,' + Buffer.from(code).toString('base64') + '#' + instance++); +function harness(initial: Library) { + let saved = structuredClone(initial); + const h = { + library: { current: saved }, + settings: { get current() { return saved.shared.settings; } }, + session: { params: structuredClone(DEFAULT_PARAMS), commands: [] as unknown[], + patchParams(params: object) { Object.assign(this.params, params); }, + send(command: unknown) { this.commands.push(command); return true; }, + stopSequence() {}, clearLoop() {}, + }, + markers: { list: [] as any[], onPersist: null as any, load(list: any[]) { this.list = list; } }, + snippets: { list: [] as any[], sequenceLoop: false, sequenceCountIn: false, onPersist: null as any, + load(list: any[], loop: boolean, countIn: boolean) { this.list = list; this.sequenceLoop = loop; this.sequenceCountIn = countIn; } }, + chords: { chart: null, enabled: false, onPersist: null as any, load() {} }, + navigations: [] as string[], edits: [] as LibraryCommand[], watch: null as any, + async read() { return structuredClone(saved); }, + async edit(command: LibraryCommand) { h.edits.push(command); saved = applyCommand(saved, command); }, + replace(library: Library) { saved = applyCommand(saved, { type: 'import', library }); h.library.current = saved; h.watch(saved); }, + }; + (globalThis as any).panelTest = h; + (globalThis as any).browser = { tabs: { update: (_id: number, { url }: { url: string }) => h.navigations.push(url) } }; + return h; +} +const a = makeTrackIdentity('chrome-extension://test/local-player.html', 'A.mp3', 100); +const b = makeTrackIdentity('chrome-extension://test/local-player.html', 'B.mp3', 100); +const media = { title: a.title, pageUrl: a.normalizedUrl, duration: 100, hasVideo: false }; +const withMarker = (id: string) => applyCommand(emptyLibrary(), { type: 'practice', identity: a, + patch: { params: DEFAULT_PARAMS, markers: [{ id, t: 3, label: id }] }, recent: true }); + +test('import replaces active markers and cancels pending parameter saves before another edit', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = harness(withMarker('old')); + const { trackSync } = await load(trackCode); + trackSync.init(); + await trackSync.onMedia(media); + h.session.params.speed = 0.5; + trackSync.onParamsChanged(); + h.replace(withMarker('imported')); + // Even an input event in the async reload gap cannot save the old list. + h.markers.onPersist([{ id: 'stale', t: 1, label: '' }]); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(h.markers.list[0].id, 'imported'); + t.mock.timers.tick(2000); + assert.equal(h.edits.filter((command) => command.type === 'practice').length, 0); + h.markers.list[0].t = 20; + h.markers.onPersist(h.markers.list); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual((await h.read()).shared.songs[a.key].practice.markers, [{ id: 'imported', t: 20, label: 'imported' }]); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 1); +}); + +test('opening another local preset keeps the loaded file and applies its parameters in place', async () => { + const h = harness(withMarker('A marker')); + const { trackSync } = await load(trackCode); + trackSync.init(); + await trackSync.onMedia(media); + await trackSync.openHistoryEntry(4, { identity: b, pageUrl: b.normalizedUrl, updatedAt: 1, params: { ...DEFAULT_PARAMS, speed: 0.6 } }); + assert.deepEqual(h.navigations, []); + assert.equal(h.session.params.speed, 0.6); + assert.equal(h.markers.list[0].id, 'A marker'); + h.markers.onPersist(h.markers.list); + assert.ok((await h.read()).shared.songs[a.key]); + assert.equal((await h.read()).shared.songs[b.key], undefined); +}); + +test('sync status changes during the initial read cannot be missed or overwritten', async () => { + let finish!: (value: unknown) => void; + const h = { watch: null as any, loadConfig: () => new Promise((resolve) => { finish = resolve; }) }; + (globalThis as any).panelTest = h; + const { sync } = await load(syncCode); + const ready = sync.init(); + assert.equal(typeof h.watch, 'function'); + h.watch({ enabled: true, syncing: false, lastError: null }); + finish({ enabled: true, syncing: true }); + await ready; + assert.equal(sync.status, 'idle'); +}); + +test('engine settings are sent only on meaningful changes or a forced engine attach', async () => { + const h = harness(emptyLibrary()); + const { pushSettings } = await load(connectionCode); + pushSettings(); + await h.edit({ type: 'uiPrefs', patch: { markerView: 'list' } }); + pushSettings(); + assert.equal(h.session.commands.length, 1); + await h.edit({ type: 'settings', patch: { countInBeats: 8 } }); + pushSettings(); + assert.equal(h.session.commands.length, 2); + pushSettings(true); + assert.equal(h.session.commands.length, 3); +}); diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index 6e2038e..ed42845 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -2,13 +2,14 @@ import { DEFAULT_PARAMS } from '../model/defaults'; import { makeTrackIdentity } from '../model/track-identity'; import type { EffectParams, HistoryEntry, MediaInfo, TrackIdentity } from '../model/types'; import type { Practice } from '../persist/library'; -import { editLibrary, readLibrary } from '../persist/library-client'; +import { editLibrary, libraryItem, readLibrary } from '../persist/library-client'; import { openTabWithPanel } from '../side-panel'; import { session } from './session.svelte'; import { settings } from '../../features/settings/panel/settings.svelte'; import { markers } from '../../features/markers/panel/markers.svelte'; import { snippets } from '../../features/snippets/panel/snippets.svelte'; import { chords } from '../../features/chords/panel/chords.svelte'; +import { library } from './library.svelte'; /** An active session loads saved data once. Library updates never interrupt playback. */ class TrackSync { @@ -18,18 +19,35 @@ class TrackSync { #restoring = false; #hasSavedParams = false; #chordsEnabled = false; + #importRevision = 0; /** Parameter changes arrive per input event while a slider is dragged, and * every save rewrites the whole library. Coalesce them into one write. */ #paramsTimer: ReturnType | undefined; init() { + this.#importRevision = library.current.local.importRevision ?? 0; + libraryItem.watch((value) => { + const revision = value?.local.importRevision ?? 0; + if (revision === this.#importRevision) return; + this.#importRevision = revision; + // An explicit import replaces the open session too. Never flush pending + // pre-import edits; the background also rejects already queued stale edits. + clearTimeout(this.#paramsTimer); + this.#paramsTimer = undefined; + this.#generation++; + this.#identity = null; + session.stopSequence(); + session.clearLoop(); + void this.onMedia(this.#media).catch((error) => console.error('[note-by-note] loading imported practice failed', error)); + }); markers.onPersist = (list) => this.#save({ markers: list }); snippets.onPersist = () => this.#save({ snippets: $state.snapshot(snippets.list), sequenceLoop: snippets.sequenceLoop, sequenceCountIn: snippets.sequenceCountIn }); chords.onPersist = () => { if (!this.#identity || this.#restoring) return; // Generated analysis is local and does not date the saved practice record. - void editLibrary({ type: 'chart', key: this.#identity.key, chart: $state.snapshot(chords.chart) }); + void editLibrary({ type: 'chart', key: this.#identity.key, chart: $state.snapshot(chords.chart), + importRevision: this.#importRevision }).catch((error) => console.error('[note-by-note] saving chart failed', error)); if (this.#chordsEnabled !== chords.enabled) { this.#chordsEnabled = chords.enabled; this.#save({ chordsEnabled: chords.enabled }); @@ -67,6 +85,7 @@ class TrackSync { try { const saved = await readLibrary(); if (generation !== this.#generation) return; + this.#importRevision = saved.local.importRevision ?? 0; const practice = saved.shared.songs[identity.key]?.practice; this.#hasSavedParams = !!practice?.params; markers.load(practice?.markers ?? []); @@ -101,14 +120,15 @@ class TrackSync { if (this.#restoring) return; const params = $state.snapshot(session.params) as EffectParams; this.#save({ params }); - if (settings.current.rememberSettings) void editLibrary({ type: 'settings', patch: { lastUsedParams: params } }); + if (settings.current.rememberSettings) void editLibrary({ type: 'settings', patch: { lastUsedParams: params }, + importRevision: this.#importRevision }).catch((error) => console.error('[note-by-note] saving last-used settings failed', error)); } #save(patch: Partial) { if (!this.#identity || this.#restoring) return; if (!this.#hasSavedParams) patch = { params: $state.snapshot(session.params), ...patch }; this.#hasSavedParams = true; - void editLibrary({ type: 'practice', identity: this.#identity, + void editLibrary({ type: 'practice', identity: this.#identity, importRevision: this.#importRevision, patch: { ...patch, pageUrl: this.#media?.pageUrl ?? this.#identity.normalizedUrl, thumbnailUrl: this.#media?.thumbnailUrl }, recent: settings.current.autoSave, }).catch((error) => console.error('[note-by-note] saving practice failed', error)); @@ -116,6 +136,13 @@ class TrackSync { async openHistoryEntry(tabId: number | null, entry: HistoryEntry) { const playing = this.#identity?.key; + if (playing?.startsWith('file:') && entry.identity.key.startsWith('file:') && playing !== entry.identity.key) { + // Local files share a player page. Its File object cannot survive a reload + // or be restored from a saved URL: apply the chosen preset to the loaded + // file, retaining that file's identity, playhead, markers and snippets. + session.patchParams($state.snapshot(entry.params) as EffectParams); + return; + } if (playing === entry.identity.key) { // Explicitly opening the saved song adopts its current library revision. this.#flushParams(); @@ -123,8 +150,9 @@ class TrackSync { await this.onMedia(this.#media); return; } - if (tabId != null) await browser.tabs.update(tabId, { url: entry.pageUrl }); - else await openTabWithPanel(entry.pageUrl); + const url = entry.identity.key.startsWith('file:') ? browser.runtime.getURL('/local-player.html') : entry.pageUrl; + if (tabId != null) await browser.tabs.update(tabId, { url }); + else await openTabWithPanel(url); } } export const trackSync = new TrackSync(); diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index 2bfa3bc..43edb8f 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -5,6 +5,7 @@ import LibraryView from '@/features/library/panel/LibraryView.svelte'; import SettingsView from '@/features/settings/panel/SettingsView.svelte'; import TooltipLayer from '@/ui/shared/TooltipLayer.svelte'; + import LibraryRecovery from '@/ui/LibraryRecovery.svelte'; import { sendMessage } from '@/core/messaging/rpc'; import { openTabWithPanel } from '@/core/side-panel'; import { installMockState, installMockTicker } from '@/dev/mock'; @@ -101,4 +102,6 @@
+{:catch error} + {/await} diff --git a/src/features/settings/panel/SettingsView.svelte b/src/features/settings/panel/SettingsView.svelte index a4c798a..d070c86 100644 --- a/src/features/settings/panel/SettingsView.svelte +++ b/src/features/settings/panel/SettingsView.svelte @@ -163,7 +163,7 @@ ); if (!ok) return; await restoreBackup(backup); - notice = { ok: true, text: 'Backup imported. Reopen the current song to use its imported practice settings.' }; + notice = { ok: true, text: 'Backup imported. Open songs now use the imported practice settings.' }; } catch (err) { notice = { ok: false, text: `Import failed: ${message(err)}` }; } finally { diff --git a/src/features/sync/panel/sync.svelte.ts b/src/features/sync/panel/sync.svelte.ts index 620935a..86f5264 100644 --- a/src/features/sync/panel/sync.svelte.ts +++ b/src/features/sync/panel/sync.svelte.ts @@ -11,8 +11,10 @@ class SyncStore { status = $derived(!this.enabled ? 'off' : this.config.syncing ? 'syncing' : this.lastError ? 'error' : 'idle'); usedPercent = $derived(Math.round(this.config.usedBytes / QUOTA_BYTES * 100)); async init() { - this.config = await loadSyncConfig(); - syncConfigItem.watch((value) => { this.config = withSyncDefaults(value); }); + let changed = false; + syncConfigItem.watch((value) => { changed = true; this.config = withSyncDefaults(value); }); + const initial = await loadSyncConfig(); + if (!changed) this.config = initial; } enable = () => sendMessage('librarySync', 'enable'); disable = () => sendMessage('librarySync', 'disable'); diff --git a/src/features/sync/persist/records.test.ts b/src/features/sync/persist/records.test.ts index 6bcceff..552feb1 100644 --- a/src/features/sync/persist/records.test.ts +++ b/src/features/sync/persist/records.test.ts @@ -27,7 +27,7 @@ test('missing, reordered and mixed chunks never produce a partial library', asyn await assert.rejects(readSnapshot(missing), IncompleteSnapshot); const noHeader = { ...items }; delete noHeader[SNAPSHOT_KEY]; - await assert.rejects(readSnapshot(noHeader), IncompleteSnapshot); + assert.deepEqual(await readSnapshot(noHeader), await readSnapshot(items)); const { items: other } = await encodeSnapshot(save('other device at the same timestamp')); await assert.rejects(readSnapshot({ ...items, [PREFIX + 'chunk:0']: other[PREFIX + 'chunk:0'] }), IncompleteSnapshot); const reversed = Object.fromEntries(Object.entries(items).reverse()); @@ -61,6 +61,15 @@ test('empty sync storage is distinct from a valid empty library or an incomplete await assert.rejects(readSnapshot(broken), (error: unknown) => error instanceof IncompleteSnapshot && error.updatedAt === 0); }); +test('a lost header is recovered from complete chunks with the original revision', async () => { + const shared = save(); + const { items } = await encodeSnapshot(shared); + delete items[SNAPSHOT_KEY]; + assert.deepEqual(await readSnapshot(items), shared); + items[PREFIX + 'chunk:0'] = 'interrupted'; + await assert.rejects(readSnapshot(items), (error: unknown) => error instanceof IncompleteSnapshot && error.updatedAt === null); +}); + test('unsupported and damaged snapshots fail before adoption or upload', async () => { const { items } = await encodeSnapshot(save()); const header = items[SNAPSHOT_KEY] as Record; diff --git a/src/features/sync/persist/records.ts b/src/features/sync/persist/records.ts index dbc0346..b399010 100644 --- a/src/features/sync/persist/records.ts +++ b/src/features/sync/persist/records.ts @@ -20,14 +20,14 @@ async function decompress(data: string): Promise { const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0)); return new Response(new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'))).text(); } -async function hash(data: string): Promise { +export async function hash(data: string): Promise { const digest = await crypto.subtle.digest('SHA-256', encoder.encode(data)); return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join(''); } export class IncompleteSnapshot extends Error { - readonly updatedAt: number; - constructor(updatedAt: number) { + readonly updatedAt: number | null; + constructor(updatedAt: number | null) { super('Waiting for the complete synced library. All data is kept on this device.'); this.updatedAt = updatedAt; } @@ -37,7 +37,15 @@ export class IncompleteSnapshot extends Error { export async function readSnapshot(items: Record): Promise { const header = items[SNAPSHOT_KEY]; if (!header) { - if (Object.keys(items).some((key) => key.startsWith(PREFIX + 'chunk:'))) throw new IncompleteSnapshot(Infinity); + if (Object.keys(items).some((key) => key.startsWith(PREFIX + 'chunk:'))) { + // A lost header need not lose the snapshot: gzip verifies its own checksum. + // Recover its revision from complete slots before choosing which copy wins. + const chunks = Array.from({ length: MAX_CHUNKS }, (_, n) => items[PREFIX + 'chunk:' + n] ?? ''); + try { + if (chunks.some((chunk) => typeof chunk !== 'string')) throw new Error('Damaged chunk.'); + return parseShared(JSON.parse(await decompress(chunks.join('')))); + } catch { throw new IncompleteSnapshot(null); } + } return null; } if (header.version !== 1) throw new Error('Unsupported synced data. Update Note by Note on all devices.'); diff --git a/src/features/sync/persist/sync-config.ts b/src/features/sync/persist/sync-config.ts index 11b8f9c..00dc799 100644 --- a/src/features/sync/persist/sync-config.ts +++ b/src/features/sync/persist/sync-config.ts @@ -6,6 +6,9 @@ export interface SyncConfig { lastError: string | null; usedBytes: number; syncing: boolean; + /** Persisted across worker wakes while headerless chunks are still arriving. */ + incompleteSince?: number; + incompleteHash?: string; } export const DEFAULT_SYNC_CONFIG: SyncConfig = { enabled: true, lastSyncedAt: 0, lastPushAt: 0, lastError: null, usedBytes: 0, syncing: false, diff --git a/src/ui/LibraryRecovery.svelte b/src/ui/LibraryRecovery.svelte new file mode 100644 index 0000000..af36ae2 --- /dev/null +++ b/src/ui/LibraryRecovery.svelte @@ -0,0 +1,50 @@ + + +
+

Your saved library could not be opened

+

Your saved data is still on this device. Retry, export it for recovery, or restore a Note by Note backup.

+

{message(error)}

+
+ + + +
+ {#if notice}

{notice}

{/if} +
From 7909ec34671926efa3658e45e6cad3a4ef1a58bd Mon Sep 17 00:00:00 2001 From: Patrick Demichiel Date: Tue, 8 Sep 2026 12:01:09 +0200 Subject: [PATCH 24/26] feat: enhance library synchronization and session management - Update `startLibraryBackground` to set `lastSyncedAt` on adopting new shared state. - Modify `editLibrary` to return the updated timestamp after editing. - Add tests for migrating URL aliases and ensuring correct handling of library edits. - Refactor `rekey` logic to improve identity handling for YouTube URLs. - Improve connection management in `connect.svelte` to handle connection failures gracefully. - Enhance session management in `session.svelte` to restore playback state without triggering edits. - Update `track-sync` to better manage practice edits and parameter changes. - Improve recovery UI messaging for clarity on data export and recovery options. --- CLAUDE.md | 6 + e2e/library.mjs | 41 ++- src/core/messaging/protocol.ts | 3 +- src/core/persist/backup.ts | 4 +- src/core/persist/library-background.test.ts | 50 ++++ src/core/persist/library-background.ts | 10 +- src/core/persist/library-client.ts | 4 +- src/core/persist/library.test.ts | 15 + src/core/persist/rekey.test.ts | 13 + src/core/persist/rekey.ts | 9 +- src/core/state/connect.svelte.ts | 10 + src/core/state/persistence.test.ts | 295 ++++++++++++++++++-- src/core/state/session.svelte.ts | 11 +- src/core/state/track-sync.svelte.ts | 205 ++++++++------ src/entrypoints/sidepanel/App.svelte | 11 +- src/ui/LibraryRecovery.svelte | 6 +- 16 files changed, 564 insertions(+), 129 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e4301d3..48075c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,10 +103,16 @@ Runes stores (classes with `$state`), one singleton exported per file. All panel library-client.ts commands and one storage watch. Commands patch the latest saved data; Recent and Favorites are projections, not persistent song copies. - Track-sync loads a practice session once and submits edits to the saved library. + Restoration uses the initialized panel mirror and does not emit user edits. + Parameter, marker and snippet edits capture their track and values before being + coalesced; pending patches remain readable until the library watch acknowledges + their committed revision. Switching tracks or hiding the panel flushes the batch. Receiving remote changes never reloads or silently replaces the active session. Explicit imports reload active sessions and advance a device-local import revision; the writer rejects practice edits carrying an older revision. Feature persistence is wired directly in track-sync; there is no descriptor registry. +- The writer validates the resulting library before every command commit. Connection + failures use connection state; only library initialization can open recovery. - Sync copies the same SharedLibrary snapshot used locally (records.ts). One updatedAt timestamp chooses the whole winner; equal dates adopt the remote copy. There are no field merges, deletion markers, or automatic song pruning. Song diff --git a/e2e/library.mjs b/e2e/library.mjs index bf95873..49aaf4f 100644 --- a/e2e/library.mjs +++ b/e2e/library.mjs @@ -48,6 +48,39 @@ try { const second = await panel(); await sync(first, 'disable'); await edit(first, { type: 'import', library: emptyLibrary() }); + + // Exercise App's real startup without an actual website or player. A failed + // injection RPC must leave the library usable and show connection state. + const disconnected = await browser.newPage(); + disconnected.on('pageerror', (error) => panelErrors.push(error.message)); + await disconnected.evaluateOnNewDocument(() => { + chrome.tabs.get = (_id, callback) => { + const tab = { id: 123, url: 'https://example.test/song' }; + if (callback) { callback(tab); return; } + return Promise.resolve(tab); + }; + chrome.permissions.contains = (_permissions, callback) => { + if (callback) { callback(true); return; } + return Promise.resolve(true); + }; + const send = chrome.runtime.sendMessage.bind(chrome.runtime); + chrome.runtime.sendMessage = (...args) => { + if (args[0]?.type === 'ensureInjected') { + globalThis.__injectionFailed = true; + throw new Error('Simulated injection RPC failure'); + } + return send(...args); + }; + }); + const beforeConnection = await read(first); + await disconnected.goto(first.url().replace('?mock=1', '?tabId=123')); + await disconnected.waitForSelector('button[aria-label="Settings"]'); + await disconnected.waitForFunction(() => globalThis.__injectionFailed && globalThis.__panelDebug?.connection() === 'stale'); + assert.equal(await disconnected.$('main[aria-label="Library recovery"]'), null); + assert.deepEqual(await read(disconnected), beforeConnection); + await disconnected.close(); + console.log('PASS injection failure keeps the library open and uses normal connection state'); + await first.bringToFront(); await first.waitForSelector('button[aria-label="Settings"]'); await first.click('button[aria-label="Settings"]'); @@ -82,6 +115,12 @@ try { assert.equal(Object.keys((await read(first)).shared.songs).length, 12); console.log('PASS concurrent panels preserve all 12 songs'); + const beforeInvalid = await read(first); + await assert.rejects(edit(first, { type: 'practice', identity: identity(0), + patch: { markers: [{ id: 'invalid', t: null, label: 'Invalid time' }] }, recent: true }), /Damaged library number/); + assert.deepEqual(await read(first), beforeInvalid); + console.log('PASS invalid edits leave the persisted library unchanged'); + await Promise.all([ edit(first, { type: 'practice', identity: identity(0), patch: { markers: [{ id: 'm', t: 5, label: 'Verse' }] }, recent: true }), edit(second, { type: 'practice', identity: identity(0), patch: { params: { ...DEFAULT_PARAMS, speed: 0.5 } }, recent: true }), @@ -202,7 +241,7 @@ try { await recovery.setViewport({ width: 400, height: 700 }); await recovery.waitForSelector('main[aria-label="Library recovery"]'); assert.match(await recovery.$eval('main', (node) => node.textContent), /saved data is still on this device/); - await recovery.screenshot({ path: resolve(root, '.output', 'pr12-recovery.png') }); + await recovery.screenshot({ path: resolve(root, '.output', 'library-recovery.png') }); const file = join(profile, 'restore.json'); writeFileSync(file, JSON.stringify({ format: 'note-by-note-backup', version: 2, exportedAt: Date.now(), ...recoveryBackup })); recovery.once('dialog', (dialog) => dialog.accept()); diff --git a/src/core/messaging/protocol.ts b/src/core/messaging/protocol.ts index 3d6a876..d47fcd0 100644 --- a/src/core/messaging/protocol.ts +++ b/src/core/messaging/protocol.ts @@ -123,7 +123,8 @@ export type OffscreenCommand = /** RPC handled by the background service worker (via @webext-core/messaging). */ export interface ProtocolMap { libraryRead(): Promise; - libraryEdit(command: LibraryCommand): Promise; + /** The committed shared revision lets panels retire edits once their watch catches up. */ + libraryEdit(command: LibraryCommand): Promise; librarySync(action: 'now' | 'enable' | 'disable' | 'delete'): Promise; /** Request per-origin host permission, inject + persist the content script. * Must run after the side panel already obtained the permission grant. */ diff --git a/src/core/persist/backup.ts b/src/core/persist/backup.ts index a19f751..c91aaf2 100644 --- a/src/core/persist/backup.ts +++ b/src/core/persist/backup.ts @@ -13,6 +13,6 @@ export function parseBackup(text: string): Backup { try { raw = JSON.parse(text); } catch { throw new Error("That file isn't valid JSON."); } return parseBackupJson(raw); } -export function restoreBackup(backup: Backup): Promise { - return editLibrary({ type: 'import', library: backup }); +export async function restoreBackup(backup: Backup): Promise { + await editLibrary({ type: 'import', library: backup }); } diff --git a/src/core/persist/library-background.test.ts b/src/core/persist/library-background.test.ts index 8b85ec1..0a5315e 100644 --- a/src/core/persist/library-background.test.ts +++ b/src/core/persist/library-background.test.ts @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import { build } from 'esbuild'; import { fileURLToPath } from 'node:url'; import { applyCommand, emptyLibrary } from './library.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; import { bytesUsed, encodeSnapshot, readSnapshot, PREFIX, SNAPSHOT_KEY } from '../../features/sync/persist/records.ts'; // Bundle the real worker with only its browser/RPC boundaries replaced. Each @@ -148,3 +149,52 @@ test('a damaged saved library stays intact and can be replaced through the recov assert.deepEqual(h.areas.local.libraryRecovery, damaged); assert.equal(library.local.importRevision, 1); }); + +test('invalid edits are rejected without writing or poisoning the next worker startup', async () => { + const original = saved(); + const h = harness({ library: original, syncConfig: { ...config, enabled: false } }); + await h.start(); + await h.rpc('libraryRead'); + h.writes.length = 0; + const identity = makeTrackIdentity('https://youtube.com/watch?v=validation', 'Song', 100); + const invalid = [ + { type: 'practice', identity, patch: { markers: [{ id: 'm', t: NaN, label: 'Bad time' }] }, recent: true }, + { type: 'practice', identity, patch: { snippets: [{ id: 's', name: 'Bad range', startT: NaN, + endT: 20, repeats: 1, enabled: true, overrides: {} }] }, recent: true }, + { type: 'settings', patch: { seekInterval: NaN } }, + { type: 'preset', name: 'Bad gains', gains: [NaN] }, + ]; + for (const command of invalid) { + await assert.rejects(h.rpc('libraryEdit', JSON.parse(JSON.stringify(command))), /Damaged/); + assert.deepEqual(h.areas.local.library, original); + } + assert.deepEqual(h.writes, []); + await h.start(); + assert.deepEqual(await h.rpc('libraryRead'), original); + const revision = await h.rpc('libraryEdit', { type: 'preset', name: 'Valid after rejection', gains: [2] }); + assert.equal(revision, h.areas.local.library.shared.updatedAt); + assert.deepEqual(h.areas.local.library.shared.presets['Valid after rejection'], [2]); +}); + +test('validation preserves infinite snippet repeats encoded as null', async () => { + const h = harness({ library: saved(), syncConfig: { ...config, enabled: false } }); + await h.start(); + const identity = makeTrackIdentity('https://youtube.com/watch?v=infinite', 'Song', 100); + await h.rpc('libraryEdit', JSON.parse(JSON.stringify({ type: 'practice', identity, recent: true, + patch: { snippets: [{ id: 's', name: 'Repeat', startT: 2, endT: 8, repeats: Infinity, enabled: true, overrides: {} }] } }))); + await h.start(); + assert.equal((await h.rpc('libraryRead')).shared.songs[identity.key].practice.snippets[0].repeats, null); +}); + +test('adopting headerless remote data dates the pull even while header repair is rate limited', async (t) => { + t.mock.method(Date, 'now', () => 100000); + const library = saved(); + const remote = { ...library.shared, updatedAt: 200, presets: { Remote: [2] } }; + const { items } = await encodeSnapshot(remote); + delete items[SNAPSHOT_KEY]; + const h = harness({ library, syncConfig: { ...config, lastPushAt: 99999 } }, items); + await h.start(); + assert.deepEqual(h.areas.local.library.shared, remote); + assert.equal(h.areas.local.syncConfig.lastSyncedAt, 100000); + assert.equal(h.writes.filter((w) => w.area === 'sync').length, 0); +}); diff --git a/src/core/persist/library-background.ts b/src/core/persist/library-background.ts index 35e4d4f..272e62d 100644 --- a/src/core/persist/library-background.ts +++ b/src/core/persist/library-background.ts @@ -71,7 +71,11 @@ export function startLibraryBackground() { }); const shared = remote ? newestSnapshot(local.shared, remote) : local.shared; const adopting = shared !== local.shared; - if (adopting) { await begin(); await libraryItem.setValue({ ...local, shared }); } + if (adopting) { + await begin(); + await libraryItem.setValue({ ...local, shared }); + config.lastSyncedAt = Date.now(); + } config.lastError = null; delete config.incompleteSince; delete config.incompleteHash; @@ -114,10 +118,12 @@ export function startLibraryBackground() { await init(); current = await libraryItem.getValue(); } - const next = applyCommand(current, data); + // Reject malformed edits before they can poison the next worker startup. + const next = parseLibrary(applyCommand(current, data)); await libraryItem.setValue(next); if (data.type === 'import') ready = Promise.resolve(); if (current.shared.updatedAt !== next.shared.updatedAt) await schedule(); + return next.shared.updatedAt; })); onMessage('librarySync', ({ data }) => enqueue(async () => { const config = await loadSyncConfig(); diff --git a/src/core/persist/library-client.ts b/src/core/persist/library-client.ts index 36b2938..31f0e3a 100644 --- a/src/core/persist/library-client.ts +++ b/src/core/persist/library-client.ts @@ -5,6 +5,6 @@ import { emptyLibrary, type Library, type LibraryCommand } from './library'; /** Panels observe the library; the background is its only writer. */ export const libraryItem = storage.defineItem('local:library', { fallback: emptyLibrary() }); export const readLibrary = async () => sendMessage('libraryRead', undefined); -export const editLibrary = async (command: LibraryCommand): Promise => { - await sendMessage('libraryEdit', JSON.parse(JSON.stringify(command)) as LibraryCommand); +export const editLibrary = async (command: LibraryCommand): Promise => { + return sendMessage('libraryEdit', JSON.parse(JSON.stringify(command)) as LibraryCommand); }; diff --git a/src/core/persist/library.test.ts b/src/core/persist/library.test.ts index 5c0ede6..b2fb53e 100644 --- a/src/core/persist/library.test.ts +++ b/src/core/persist/library.test.ts @@ -185,6 +185,21 @@ test('released version-1 backups import parameters, favorites, markers and prese assert.deepEqual(parseBackupJson(exported), exported); }); +test('version-1 URL aliases migrate to one favorite with the newest practice data', () => { + const entries = ['https://youtube.com/shorts/example', 'https://youtube.com/embed/example', + 'https://youtube.com/watch?v=example'].map((url, i) => ({ + identity: { key: 'old:' + i, normalizedUrl: url, title: 'Song', durationSec: 100 }, + pageUrl: url, params: { ...DEFAULT_PARAMS, speed: 0.5 + i * 0.1 }, + updatedAt: i + 1, favoritedAt: i + 1, lastAccessedAt: i + 1, + })); + const migrated = parseBackupJson({ format: 'note-by-note-backup', version: 1, + history: entries, favorites: entries, tracks: [], eqPresets: [] }); + assert.deepEqual(Object.keys(migrated.shared.songs), ['yt:example']); + assert.deepEqual(migrated.shared.favoriteOrder, ['yt:example']); + assert.equal(migrated.shared.songs['yt:example'].practice.params!.speed, 0.7); + assert.equal(migrated.local.recent['yt:example'], 3); +}); + test('new backups round-trip and reject malformed or unsupported formats', () => { const backup = { format: 'note-by-note-backup', version: 2, exportedAt: 100, ...save() }; assert.deepEqual(parseBackupJson(JSON.parse(JSON.stringify(backup))), backup); diff --git a/src/core/persist/rekey.test.ts b/src/core/persist/rekey.test.ts index 6acc90a..f21a79d 100644 --- a/src/core/persist/rekey.test.ts +++ b/src/core/persist/rekey.test.ts @@ -71,3 +71,16 @@ test('re-running it changes nothing', () => { const once = rekeyByIdentity([stored(URL_A, 'Song', 200, 10), stored(URL_B, 'Two', 90, 20)]); assert.deepEqual(rekeyByIdentity(once), once); }); + +test('previously distinct shorts, embed and watch URLs collapse before migration', () => { + const rows = ['https://youtube.com/shorts/aaaaaaaaaaa', + 'https://youtube.com/embed/aaaaaaaaaaa', 'https://youtube.com/watch?v=aaaaaaaaaaa'].map((url, i) => ({ + ...stored(url, 'Song', 100, i + 1), + identity: { key: 'old:' + i, normalizedUrl: url, title: 'Song', durationSec: 100 }, + })); + const result = rekeyByIdentity(rows); + assert.equal(result.length, 1); + assert.equal(result[0].identity.key, 'yt:aaaaaaaaaaa'); + assert.equal(result[0].updatedAt, 3); + assert.deepEqual(rekeyByIdentity(result), result); +}); diff --git a/src/core/persist/rekey.ts b/src/core/persist/rekey.ts index b273aac..1958471 100644 --- a/src/core/persist/rekey.ts +++ b/src/core/persist/rekey.ts @@ -1,4 +1,4 @@ -import { songKey } from '../model/track-identity.ts'; +import { makeTrackIdentity } from '../model/track-identity.ts'; import type { TrackIdentity } from '../model/types'; /** @@ -17,7 +17,7 @@ export interface Keyed { const at = (row: Keyed) => row.updatedAt ?? 0; /** - * Rows under the key `songKey` derives from them now, with copies that land on + * Rows under the identity derived from their URL now, with copies that land on * the same key collapsed to the most recently written one — a song saved under * two durations was always one song, and this is where its copies finally meet. * @@ -30,8 +30,9 @@ export function rekeyByIdentity(rows: T[]): T[] { const byKey = new Map(); for (const row of rows) { if (typeof row?.identity?.normalizedUrl !== 'string') continue; - const key = songKey(row.identity); - const next = { ...row, identity: { ...row.identity, key } }; + const identity = makeTrackIdentity(row.identity.normalizedUrl, row.identity.title, row.identity.durationSec); + const key = identity.key; + const next = { ...row, identity }; const current = byKey.get(key); if (!current || at(next) >= at(current)) byKey.set(key, next); } diff --git a/src/core/state/connect.svelte.ts b/src/core/state/connect.svelte.ts index b5e9c7c..57f4f94 100644 --- a/src/core/state/connect.svelte.ts +++ b/src/core/state/connect.svelte.ts @@ -87,6 +87,16 @@ class ConnectionManager { async #connect() { const generation = ++this.#generation; + try { + await this.#attach(generation); + } catch (error) { + if (generation !== this.#generation) return; + session.connection = 'stale'; + console.error('[note-by-note] connecting to the player failed', error); + } + } + + async #attach(generation: number) { this.#port?.disconnect(); this.#port = null; session.detachTransport(); diff --git a/src/core/state/persistence.test.ts b/src/core/state/persistence.test.ts index 7dee617..5ea59bd 100644 --- a/src/core/state/persistence.test.ts +++ b/src/core/state/persistence.test.ts @@ -17,8 +17,8 @@ const stubs: Record = { '/snippets.svelte': 'export const snippets = h.snippets;', '/chords.svelte': 'export const chords = h.chords;', '/side-panel': 'export const openTabWithPanel = (url) => h.navigations.push(url);', - '/messaging/ports': 'export const connectToTab = () => {};', - '/messaging/rpc': 'export const sendMessage = () => {};', + '/messaging/ports': 'export const connectToTab = () => h.port;', + '/messaging/rpc': 'export const sendMessage = (...args) => h.rpc(...args);', '/persist/sync-config': 'export const DEFAULT_SYNC_CONFIG = {}; export const withSyncDefaults = (v) => v; export const loadSyncConfig = () => h.loadConfig(); export const syncConfigItem = { watch: (fn) => h.watch = fn };', }; async function bundle(file: string) { @@ -36,73 +36,280 @@ async function bundle(file: string) { } }] }); return built.outputFiles[0].text; } -const [trackCode, syncCode, connectionCode] = await Promise.all([ +const [trackCode, syncCode, connectionCode, sessionCode] = await Promise.all([ bundle('./track-sync.svelte.ts'), bundle('../../features/sync/panel/sync.svelte.ts'), bundle('./connect.svelte.ts'), + bundle('./session.svelte.ts'), ]); let instance = 0; const load = (code: string) => import('data:text/javascript;base64,' + Buffer.from(code).toString('base64') + '#' + instance++); -function harness(initial: Library) { +async function harness(initial: Library) { let saved = structuredClone(initial); + const { session } = await load(sessionCode); const h = { - library: { current: saved }, + library: { current: structuredClone(saved) }, settings: { get current() { return saved.shared.settings; } }, - session: { params: structuredClone(DEFAULT_PARAMS), commands: [] as unknown[], - patchParams(params: object) { Object.assign(this.params, params); }, - send(command: unknown) { this.commands.push(command); return true; }, - stopSequence() {}, clearLoop() {}, - }, + session, commands: [] as unknown[], autoPublish: true, markers: { list: [] as any[], onPersist: null as any, load(list: any[]) { this.list = list; } }, snippets: { list: [] as any[], sequenceLoop: false, sequenceCountIn: false, onPersist: null as any, load(list: any[], loop: boolean, countIn: boolean) { this.list = list; this.sequenceLoop = loop; this.sequenceCountIn = countIn; } }, - chords: { chart: null, enabled: false, onPersist: null as any, load() {} }, + chords: { chart: null, enabled: false, onPersist: null as any, load() {}, onDisconnect() {} }, + rpc: async (..._args: unknown[]): Promise => ({ ok: true }), + port: { disconnected: false, disconnect() {}, onMessage() {}, onDisconnect() {}, send(_command: unknown) {} }, navigations: [] as string[], edits: [] as LibraryCommand[], watch: null as any, async read() { return structuredClone(saved); }, - async edit(command: LibraryCommand) { h.edits.push(command); saved = applyCommand(saved, command); }, - replace(library: Library) { saved = applyCommand(saved, { type: 'import', library }); h.library.current = saved; h.watch(saved); }, + async edit(command: LibraryCommand) { + h.edits.push(structuredClone(command)); + saved = applyCommand(saved, JSON.parse(JSON.stringify(command))); + if (h.autoPublish) h.publish(); + return saved.shared.updatedAt; + }, + publish() { h.library.current = structuredClone(saved); h.watch?.(h.library.current); }, + replace(library: Library) { saved = applyCommand(saved, { type: 'import', library }); h.publish(); }, }; (globalThis as any).panelTest = h; (globalThis as any).browser = { tabs: { update: (_id: number, { url }: { url: string }) => h.navigations.push(url) } }; + session.attachTransport((command: unknown) => h.commands.push(command)); return h; } +async function connectTrack(h: Awaited>) { + const { trackSync } = await load(trackCode); + trackSync.init(); + h.session.onUserParamsChange = () => trackSync.onParamsChanged(); + h.session.onEngineDetached = () => trackSync.onEngineLost(); + return trackSync; +} const a = makeTrackIdentity('chrome-extension://test/local-player.html', 'A.mp3', 100); const b = makeTrackIdentity('chrome-extension://test/local-player.html', 'B.mp3', 100); const media = { title: a.title, pageUrl: a.normalizedUrl, duration: 100, hasVideo: false }; const withMarker = (id: string) => applyCommand(emptyLibrary(), { type: 'practice', identity: a, patch: { params: DEFAULT_PARAMS, markers: [{ id, t: 3, label: id }] }, recent: true }); +test('real session restoration reaches the engine without triggering a user edit', async () => { + const h = await harness(emptyLibrary()); + let edits = 0; + h.session.onUserParamsChange = () => edits++; + h.session.restoreParams({ ...DEFAULT_PARAMS, speed: 0.6 }); + assert.equal(h.session.params.speed, 0.6); + assert.equal(edits, 0); + assert.equal((h.commands[0] as any).patch.speed, 0.6); + h.session.patchParams({ speed: 0.8 }); + assert.equal(edits, 1); +}); + +test('remembered edits work before connecting and on immediate track switches before storage echoes', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const initial = applyCommand(emptyLibrary(), { type: 'settings', patch: { + rememberSettings: true, lastUsedParams: { ...DEFAULT_PARAMS, speed: 0.9 }, + } }); + const h = await harness(initial); + const trackSync = await connectTrack(h); + h.autoPublish = false; + h.session.patchParams({ speed: 0.7 }); + t.mock.timers.tick(2000); + assert.equal((await h.read()).local.lastUsedParams!.speed, 0.7); + assert.deepEqual((await h.read()).shared.songs, {}); + await trackSync.onMedia(media); + assert.equal(h.session.params.speed, 0.7); + h.session.patchParams({ speed: 0.6 }); + await trackSync.onMedia({ ...media, title: b.title }); + assert.equal(h.session.params.speed, 0.6); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.6); + h.publish(); + // Imported preferences and last-used values replace all pending local ones. + const imported = applyCommand(emptyLibrary(), { type: 'settings', patch: { + rememberSettings: true, lastUsedParams: { ...DEFAULT_PARAMS, speed: 0.8 }, + } }); + h.session.patchParams({ speed: 0.5 }); + h.replace(imported); + assert.equal(h.session.params.speed, 0.8); + t.mock.timers.tick(2000); + assert.equal((await h.read()).local.lastUsedParams!.speed, 0.8); +}); + +test('track changes use the ready mirror and a visit failure cannot mix song data', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const initial = applyCommand(withMarker('A marker'), { type: 'practice', identity: b, + patch: { params: { ...DEFAULT_PARAMS, speed: 0.6 }, markers: [{ id: 'B', t: 7, label: 'B marker' }] }, recent: true }); + const h = await harness(initial); + const trackSync = await connectTrack(h); + t.mock.method(h, 'read', async () => { throw new Error('The worker read is unavailable'); }); + await trackSync.onMedia(media); + t.mock.timers.tick(2000); + assert.equal(h.edits.filter((c) => c.type === 'practice').length, 0); + const edit = h.edit.bind(h); + t.mock.method(h, 'edit', (command: LibraryCommand) => command.type === 'visit' + ? Promise.reject(new Error('Visit failed')) : edit(command)); + await assert.rejects(trackSync.onMedia({ ...media, title: b.title }), /Visit failed/); + assert.equal(h.markers.list[0].id, 'B'); + h.markers.list[0].t = 9; + h.markers.onPersist(h.markers.list); + t.mock.timers.tick(2000); + assert.equal(h.library.current.shared.songs[a.key].practice.markers[0].t, 3); + assert.equal(h.library.current.shared.songs[b.key].practice.markers[0].t, 9); +}); + +test('pending edits capture their track and values before an engine snapshot replaces parameters', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); + await trackSync.onMedia(media); + h.session.patchParams({ speed: 0.75 }); + // An incoming engine snapshot changes params before it reports the new media. + h.session.params = structuredClone(DEFAULT_PARAMS); + await trackSync.onMedia({ ...media, title: b.title }); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.75); + assert.equal((await h.read()).shared.songs[a.key].practice.pageUrl, media.pageUrl); + t.mock.timers.tick(2000); + assert.equal((await h.read()).shared.songs[b.key], undefined); +}); + +test('rapid return and explicit reopen retain edits until the library watch catches up', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); + await trackSync.onMedia(media); + h.autoPublish = false; + h.session.patchParams({ speed: 0.75 }); + h.markers.list[0].t = 12; + h.markers.onPersist(h.markers.list); + await trackSync.onMedia({ ...media, title: b.title }); + assert.equal(h.library.current.shared.songs[a.key].practice.params!.speed, 1); + await trackSync.onMedia(media); + assert.equal(h.session.params.speed, 0.75); + assert.equal(h.markers.list[0].t, 12); + await trackSync.openHistoryEntry(4, { identity: a, pageUrl: a.normalizedUrl, updatedAt: 1, params: DEFAULT_PARAMS }); + assert.equal(h.session.params.speed, 0.75); + h.publish(); + await h.edit({ type: 'practice', identity: a, patch: { params: { ...DEFAULT_PARAMS, speed: 0.9 } }, recent: true }); + h.publish(); + // Acknowledged patches must not mask subsequent library changes. + await trackSync.openHistoryEntry(4, { identity: a, pageUrl: a.normalizedUrl, updatedAt: 1, params: DEFAULT_PARAMS }); + assert.equal(h.session.params.speed, 0.9); +}); + +test('marker and snippet bursts share one immutable save and engine detach flushes it', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); + await trackSync.onMedia(media); + for (let i = 0; i < 50; i++) { + h.markers.list[0].t = i; + h.markers.onPersist(h.markers.list); + } + h.snippets.list = [{ id: 's', name: 'Verse', startT: 2, endT: 8, repeats: Infinity, enabled: true, overrides: {} }]; + h.snippets.onPersist(); + assert.equal(h.edits.filter((c) => c.type === 'practice').length, 0); + // Mutating the live store later cannot alter the already queued edit. + h.markers.list[0].t = 99; + h.session.detachTransport(); + const practice = (await h.read()).shared.songs[a.key].practice; + assert.equal(practice.markers[0].t, 49); + assert.equal(practice.snippets[0].endT, 8); + assert.equal(practice.snippets[0].repeats, null); + assert.equal(h.edits.filter((c) => c.type === 'practice').length, 1); + await trackSync.onMedia(media); + h.session.patchParams({ speed: 0.7 }); + t.mock.timers.tick(2000); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.7); +}); + +test('failed hydration leaves no writable track and a later load can recover', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); + await trackSync.onMedia(media); + const loadMarkers = h.markers.load.bind(h.markers); + let failed = false; + t.mock.method(h.markers, 'load', (list: any[]) => { + if (!failed) { failed = true; throw new Error('Cannot hydrate'); } + loadMarkers(list); + }); + await assert.rejects(trackSync.onMedia({ ...media, title: b.title }), /Cannot hydrate/); + assert.deepEqual(h.markers.list, []); + h.session.patchParams({ speed: 0.5 }); + t.mock.timers.tick(2000); + assert.equal((await h.read()).shared.songs[b.key], undefined); + await trackSync.onMedia({ ...media, title: b.title }); + h.session.patchParams({ speed: 0.6 }); + t.mock.timers.tick(2000); + assert.equal((await h.read()).shared.songs[b.key].practice.params!.speed, 0.6); +}); + +test('a later edit survives acknowledgement of an earlier save and panel flush commits it', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); + await trackSync.onMedia(media); + h.autoPublish = false; + h.session.patchParams({ speed: 0.8 }); + trackSync.flush(); + h.session.patchParams({ speed: 0.6 }); + await new Promise((resolve) => setImmediate(resolve)); + h.publish(); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.8); + await trackSync.onMedia({ ...media, title: b.title }); + await trackSync.onMedia(media); + assert.equal(h.session.params.speed, 0.6); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.6); +}); + +test('engine loss while a visit is in flight cannot disable later saves', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); + const edit = h.edit.bind(h); + let finish!: () => void; + let delay = true; + t.mock.method(h, 'edit', async (command: LibraryCommand) => { + if (command.type === 'visit' && delay) { + delay = false; + await new Promise((resolve) => { finish = resolve; }); + } + return edit(command); + }); + const loading = trackSync.onMedia(media); + h.session.detachTransport(); + finish(); + await loading; + await trackSync.onMedia(media); + h.session.patchParams({ speed: 0.7 }); + t.mock.timers.tick(2000); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.7); +}); + test('import replaces active markers and cancels pending parameter saves before another edit', async (t) => { t.mock.timers.enable({ apis: ['setTimeout'] }); - const h = harness(withMarker('old')); - const { trackSync } = await load(trackCode); - trackSync.init(); + const h = await harness(withMarker('old')); + const trackSync = await connectTrack(h); await trackSync.onMedia(media); - h.session.params.speed = 0.5; - trackSync.onParamsChanged(); + h.session.patchParams({ speed: 0.5 }); h.replace(withMarker('imported')); - // Even an input event in the async reload gap cannot save the old list. - h.markers.onPersist([{ id: 'stale', t: 1, label: '' }]); - await new Promise((resolve) => setImmediate(resolve)); + // Replacement is synchronous, so there is no gap with the old stores. assert.equal(h.markers.list[0].id, 'imported'); t.mock.timers.tick(2000); assert.equal(h.edits.filter((command) => command.type === 'practice').length, 0); h.markers.list[0].t = 20; h.markers.onPersist(h.markers.list); + t.mock.timers.tick(2000); await new Promise((resolve) => setImmediate(resolve)); assert.deepEqual((await h.read()).shared.songs[a.key].practice.markers, [{ id: 'imported', t: 20, label: 'imported' }]); assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 1); }); -test('opening another local preset keeps the loaded file and applies its parameters in place', async () => { - const h = harness(withMarker('A marker')); - const { trackSync } = await load(trackCode); - trackSync.init(); +test('opening another local preset preserves pending edits and never saves the preview onto the loaded file', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const h = await harness(withMarker('A marker')); + const trackSync = await connectTrack(h); await trackSync.onMedia(media); + h.session.patchParams({ speed: 0.8 }); await trackSync.openHistoryEntry(4, { identity: b, pageUrl: b.normalizedUrl, updatedAt: 1, params: { ...DEFAULT_PARAMS, speed: 0.6 } }); assert.deepEqual(h.navigations, []); assert.equal(h.session.params.speed, 0.6); assert.equal(h.markers.list[0].id, 'A marker'); h.markers.onPersist(h.markers.list); + t.mock.timers.tick(2000); assert.ok((await h.read()).shared.songs[a.key]); + assert.equal((await h.read()).shared.songs[a.key].practice.params!.speed, 0.8); assert.equal((await h.read()).shared.songs[b.key], undefined); }); @@ -120,15 +327,45 @@ test('sync status changes during the initial read cannot be missed or overwritte }); test('engine settings are sent only on meaningful changes or a forced engine attach', async () => { - const h = harness(emptyLibrary()); + const h = await harness(emptyLibrary()); const { pushSettings } = await load(connectionCode); pushSettings(); await h.edit({ type: 'uiPrefs', patch: { markerView: 'list' } }); pushSettings(); - assert.equal(h.session.commands.length, 1); + assert.equal(h.commands.length, 1); await h.edit({ type: 'settings', patch: { countInBeats: 8 } }); pushSettings(); - assert.equal(h.session.commands.length, 2); + assert.equal(h.commands.length, 2); pushSettings(true); - assert.equal(h.session.commands.length, 3); + assert.equal(h.commands.length, 3); +}); + +test('an injection RPC rejection uses connection state and keeps reconnect listeners working', async (t) => { + const h = await harness(withMarker('A marker')); + const global = globalThis as any; + const previousLocation = global.location; + global.location = { search: '?tabId=4' }; + t.after(() => { global.location = previousLocation; }); + let onUpdated: (tabId: number, info: object) => void = () => {}; + global.browser = { + runtime: { getURL: (path: string) => 'chrome-extension://test' + path }, + permissions: { contains: async () => true }, + tabs: { get: async () => ({ id: 4, url: 'https://example.com/song' }), + onUpdated: { addListener: (fn: typeof onUpdated) => { onUpdated = fn; } } }, + }; + let attempts = 0; + h.rpc = async () => { + if (++attempts === 1) throw new Error('Worker restarted'); + return { ok: true }; + }; + t.mock.method(console, 'error', () => {}); + const send = t.mock.method(h.port, 'send', (_command: unknown) => {}); + const { connection } = await load(connectionCode); + await connection.init(); + assert.equal(h.session.connection, 'stale'); + assert.equal((await h.read()).shared.songs[a.key].practice.markers[0].id, 'A marker'); + onUpdated(4, { status: 'complete' }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(attempts, 2); + assert.ok(send.mock.calls.some((call) => (call.arguments[0] as any)?.type === 'hello')); }); diff --git a/src/core/state/session.svelte.ts b/src/core/state/session.svelte.ts index 932d612..02a6056 100644 --- a/src/core/state/session.svelte.ts +++ b/src/core/state/session.svelte.ts @@ -299,6 +299,16 @@ class SessionStore { // ─── Effect params ─────────────────────────────────────────── patchParams(patch: Partial) { + this.#applyParams(patch); + this.onUserParamsChange?.(); + } + + /** Restore playback state without treating it as an edit to the saved song. */ + restoreParams(params: EffectParams) { + this.#applyParams(params); + } + + #applyParams(patch: Partial) { if (patch.speed !== undefined) { patch.speed = clampSpeed(patch.speed); } @@ -306,7 +316,6 @@ class SessionStore { const snapshot = $state.snapshot(patch) as Partial; this.send({ type: 'params', patch: snapshot }); if (this.capturing) this.captureRelay?.params(snapshot); - this.onUserParamsChange?.(); } /** Ask the engine to measure the playing tempo and set `baseBpm`. The engine diff --git a/src/core/state/track-sync.svelte.ts b/src/core/state/track-sync.svelte.ts index ed42845..0257a06 100644 --- a/src/core/state/track-sync.svelte.ts +++ b/src/core/state/track-sync.svelte.ts @@ -1,8 +1,8 @@ import { DEFAULT_PARAMS } from '../model/defaults'; import { makeTrackIdentity } from '../model/track-identity'; import type { EffectParams, HistoryEntry, MediaInfo, TrackIdentity } from '../model/types'; -import type { Practice } from '../persist/library'; -import { editLibrary, libraryItem, readLibrary } from '../persist/library-client'; +import type { Library, LibraryCommand, Practice } from '../persist/library'; +import { editLibrary, libraryItem } from '../persist/library-client'; import { openTabWithPanel } from '../side-panel'; import { session } from './session.svelte'; import { settings } from '../../features/settings/panel/settings.svelte'; @@ -11,141 +11,182 @@ import { snippets } from '../../features/snippets/panel/snippets.svelte'; import { chords } from '../../features/chords/panel/chords.svelte'; import { library } from './library.svelte'; +type PracticeEdit = Extract; +type RememberedParams = { params: EffectParams; importRevision: number }; + /** An active session loads saved data once. Library updates never interrupt playback. */ class TrackSync { #identity: TrackIdentity | null = null; #media: MediaInfo | null = null; - #generation = 0; - #restoring = false; - #hasSavedParams = false; + #baselineParams: EffectParams | null = null; #chordsEnabled = false; #importRevision = 0; - /** Parameter changes arrive per input event while a slider is dragged, and - * every save rewrites the whole library. Coalesce them into one write. */ - #paramsTimer: ReturnType | undefined; + #timer: ReturnType | undefined; + #pending: PracticeEdit | undefined; + #rememberParams: RememberedParams | undefined; + #lastUsed: RememberedParams | undefined; + /** Edits stay available for A -> B -> A until the storage watch acknowledges + * their commit. Only unacknowledged patches are overlaid on the library. */ + #drafts = new Map; committedAt?: number }>(); init() { this.#importRevision = library.current.local.importRevision ?? 0; libraryItem.watch((value) => { - const revision = value?.local.importRevision ?? 0; + if (!value) return; + this.#acknowledge(value); + const revision = value.local.importRevision ?? 0; if (revision === this.#importRevision) return; this.#importRevision = revision; - // An explicit import replaces the open session too. Never flush pending - // pre-import edits; the background also rejects already queued stale edits. - clearTimeout(this.#paramsTimer); - this.#paramsTimer = undefined; - this.#generation++; + // Imports replace the session too. Discard pending pre-import edits; + // the writer rejects old revisions already in flight. + this.#cancelPending(); + this.#drafts.clear(); + this.#lastUsed = undefined; this.#identity = null; session.stopSequence(); session.clearLoop(); - void this.onMedia(this.#media).catch((error) => console.error('[note-by-note] loading imported practice failed', error)); + // Use the event's snapshot, independently of storage-listener ordering. + void this.onMedia(this.#media, value).catch((error) => console.error('[note-by-note] loading imported practice failed', error)); }); - markers.onPersist = (list) => this.#save({ markers: list }); - snippets.onPersist = () => this.#save({ snippets: $state.snapshot(snippets.list), + markers.onPersist = (list) => this.#queue({ markers: list }); + snippets.onPersist = () => this.#queue({ snippets: $state.snapshot(snippets.list), sequenceLoop: snippets.sequenceLoop, sequenceCountIn: snippets.sequenceCountIn }); chords.onPersist = () => { - if (!this.#identity || this.#restoring) return; + if (!this.#identity) return; // Generated analysis is local and does not date the saved practice record. void editLibrary({ type: 'chart', key: this.#identity.key, chart: $state.snapshot(chords.chart), importRevision: this.#importRevision }).catch((error) => console.error('[note-by-note] saving chart failed', error)); if (this.#chordsEnabled !== chords.enabled) { this.#chordsEnabled = chords.enabled; - this.#save({ chordsEnabled: chords.enabled }); + this.#queue({ chordsEnabled: chords.enabled }); } }; } onEngineLost() { - this.#flushParams(); - this.#generation++; + this.#flush(); this.#identity = null; this.#media = null; + this.#baselineParams = null; } - async onMedia(media: MediaInfo | null) { - // Null media is a transient engine state (detecting, no player, mid source - // change), not the end of the session — dropping the track here would throw - // away edits made before the next event. Real loss arrives via `onEngineLost`. + /** Commit before the panel is hidden or its document is closed. */ + flush() { this.#flush(); } + + async onMedia(media: MediaInfo | null, saved = library.current) { + // Null is transient (detecting / source change). Real loss has its own hook. if (!media) return; const identity = makeTrackIdentity(media.pageUrl, media.title, media.duration); if (this.#identity?.key === identity.key) { this.#media = media; this.#identity = identity; return; } - // Flushed against the outgoing track's media: `#save` reads `#media` for the - // saved pageUrl and thumbnail, so replacing it first files this song's URL - // under the previous song's identity. - this.#flushParams(); + this.#flush(); + this.#identity = null; this.#media = media; - this.#identity = identity; - this.#hasSavedParams = false; - const generation = ++this.#generation; - // Saving is closed for the whole load, not just the apply: the stores still - // hold the previous song, and `#identity` already names this one, so any - // edit landing inside the await would write that song's markers, snippets - // and parameters onto this one. - this.#restoring = true; + this.#baselineParams = null; + this.#importRevision = saved.local.importRevision ?? 0; + // Hydration is synchronous: no worker read can leave the previous song's + // stores attached to a new identity. Restoration never emits user edits. try { - const saved = await readLibrary(); - if (generation !== this.#generation) return; - this.#importRevision = saved.local.importRevision ?? 0; - const practice = saved.shared.songs[identity.key]?.practice; - this.#hasSavedParams = !!practice?.params; - markers.load(practice?.markers ?? []); - snippets.load(practice?.snippets ?? [], practice?.sequenceLoop ?? false, practice?.sequenceCountIn ?? false); - chords.load(saved.local.charts[identity.key] ?? null, practice?.chordsEnabled); + const practice = { ...saved.shared.songs[identity.key]?.practice, ...this.#drafts.get(identity.key)?.patch }; + markers.load($state.snapshot(practice.markers ?? [])); + snippets.load($state.snapshot(practice.snippets ?? []), practice.sequenceLoop ?? false, practice.sequenceCountIn ?? false); + chords.load($state.snapshot(saved.local.charts[identity.key] ?? null), practice.chordsEnabled); this.#chordsEnabled = chords.enabled; - const params = practice?.params ?? (settings.current.autoReset ? DEFAULT_PARAMS : - settings.current.rememberSettings ? settings.current.lastUsedParams : undefined); - // $state.snapshot, not structuredClone: `settings.current` is a rune, so - // `lastUsedParams` is a proxy and structuredClone throws on it. - if (params) session.patchParams($state.snapshot(params) as EffectParams); - } finally { - // A newer track already owns the flag; only its own load may clear it. - if (generation === this.#generation) this.#restoring = false; + const params = practice.params ?? (saved.shared.settings.autoReset ? DEFAULT_PARAMS : + saved.shared.settings.rememberSettings ? this.#lastUsed?.params ?? saved.local.lastUsedParams : undefined); + if (params) session.restoreParams($state.snapshot(params) as EffectParams); + this.#baselineParams = $state.snapshot(session.params) as EffectParams; + this.#identity = identity; + } catch (error) { + markers.load([]); + snippets.load([], false, false); + chords.load(null, false); + throw error; } - if (generation !== this.#generation) return; await editLibrary({ type: 'visit', key: identity.key }); } onParamsChanged() { - if (this.#restoring) return; - clearTimeout(this.#paramsTimer); - this.#paramsTimer = setTimeout(() => this.#flushParams(), 1500); + const params = $state.snapshot(session.params) as EffectParams; + this.#rememberParams = settings.current.rememberSettings ? { params, importRevision: this.#importRevision } : undefined; + if (this.#rememberParams) this.#lastUsed = this.#rememberParams; + if (this.#identity) { + this.#baselineParams = params; + this.#queue({ params }); + } else if (this.#rememberParams) { + // Remember parameter edits made before a player is connected too. + this.#schedule(); + } } - /** Writes the parameters a drag settled on. Also called before the track - * changes, so the last edit is never lost to the pending timer. */ - #flushParams() { - if (this.#paramsTimer === undefined) return; - clearTimeout(this.#paramsTimer); - this.#paramsTimer = undefined; - if (this.#restoring) return; - const params = $state.snapshot(session.params) as EffectParams; - this.#save({ params }); - if (settings.current.rememberSettings) void editLibrary({ type: 'settings', patch: { lastUsedParams: params }, - importRevision: this.#importRevision }).catch((error) => console.error('[note-by-note] saving last-used settings failed', error)); + #queue(patch: Partial) { + if (!this.#identity || !this.#baselineParams) return; + const key = this.#identity.key; + const draft = this.#drafts.get(key); + if (!library.current.shared.songs[key]?.practice.params && !draft?.patch.params) { + patch = { params: this.#baselineParams, ...patch }; + } + // Capture values now, including the URL and revision. Engine echoes and + // navigation cannot change what a delayed save writes. + this.#pending = $state.snapshot({ type: 'practice', identity: this.#identity, + importRevision: this.#importRevision, recent: settings.current.autoSave, + patch: { ...this.#pending?.patch, ...patch, + pageUrl: this.#media?.pageUrl ?? this.#identity.normalizedUrl, thumbnailUrl: this.#media?.thumbnailUrl }, + }) as PracticeEdit; + this.#drafts.set(key, { patch: { ...draft?.patch, ...this.#pending.patch } }); + this.#schedule(); } - #save(patch: Partial) { - if (!this.#identity || this.#restoring) return; - if (!this.#hasSavedParams) patch = { params: $state.snapshot(session.params), ...patch }; - this.#hasSavedParams = true; - void editLibrary({ type: 'practice', identity: this.#identity, importRevision: this.#importRevision, - patch: { ...patch, pageUrl: this.#media?.pageUrl ?? this.#identity.normalizedUrl, - thumbnailUrl: this.#media?.thumbnailUrl }, recent: settings.current.autoSave, - }).catch((error) => console.error('[note-by-note] saving practice failed', error)); + #schedule() { + clearTimeout(this.#timer); + this.#timer = setTimeout(() => this.#flush(), 1500); + } + + #cancelPending() { + clearTimeout(this.#timer); + this.#timer = undefined; + this.#pending = undefined; + this.#rememberParams = undefined; + } + + #flush() { + const command = this.#pending; + const params = this.#rememberParams; + this.#cancelPending(); + if (command) { + const draft = this.#drafts.get(command.identity.key)!; + void editLibrary(command).then((committedAt) => { + draft.committedAt = committedAt; + this.#acknowledge(library.current); + }).catch((error) => { + if (this.#drafts.get(command.identity.key) === draft) this.#drafts.delete(command.identity.key); + console.error('[note-by-note] saving practice failed', error); + }); + } + if (params) void editLibrary({ type: 'settings', patch: { lastUsedParams: params.params }, + importRevision: params.importRevision }).then(() => this.#acknowledge(library.current)).catch((error) => { + if (this.#lastUsed === params) this.#lastUsed = undefined; + console.error('[note-by-note] saving last-used settings failed', error); + }); + } + + #acknowledge(saved: Library) { + if (this.#lastUsed && JSON.stringify(saved.local.lastUsedParams) === JSON.stringify(this.#lastUsed.params)) this.#lastUsed = undefined; + for (const [key, draft] of this.#drafts) { + if (draft.committedAt !== undefined && saved.shared.updatedAt >= draft.committedAt) this.#drafts.delete(key); + } } async openHistoryEntry(tabId: number | null, entry: HistoryEntry) { const playing = this.#identity?.key; if (playing?.startsWith('file:') && entry.identity.key.startsWith('file:') && playing !== entry.identity.key) { - // Local files share a player page. Its File object cannot survive a reload - // or be restored from a saved URL: apply the chosen preset to the loaded - // file, retaining that file's identity, playhead, markers and snippets. - session.patchParams($state.snapshot(entry.params) as EffectParams); + // Preserve actual edits to A before previewing B's preset on its player. + // File objects cannot be restored from a saved URL. + this.#flush(); + session.restoreParams($state.snapshot(entry.params) as EffectParams); return; } if (playing === entry.identity.key) { - // Explicitly opening the saved song adopts its current library revision. - this.#flushParams(); + this.#flush(); this.#identity = null; await this.onMedia(this.#media); return; diff --git a/src/entrypoints/sidepanel/App.svelte b/src/entrypoints/sidepanel/App.svelte index 43edb8f..e24d061 100644 --- a/src/entrypoints/sidepanel/App.svelte +++ b/src/entrypoints/sidepanel/App.svelte @@ -26,7 +26,9 @@ $effect(() => applyTheme(settings.current.theme)); $effect(pushSettings); - const ready = library.init().then( + // Only a saved-library failure belongs in the recovery screen. + const ready = library.init(); + void ready.then( async () => { trackSync.init(); session.onMediaEvent = (media) => { @@ -53,7 +55,7 @@ if (mockPlay) installMockTicker(); } else await connection.init(); }, - ); + ).catch((error) => console.error('[note-by-note] initializing panel failed', error)); // Opened from here rather than via the background: the side panel has to // follow the user to the player tab, and only this document holds the @@ -73,6 +75,11 @@ } + trackSync.flush()} /> + { + if (document.visibilityState === 'hidden') trackSync.flush(); +}} /> + {#await ready then}