diff --git a/docs/architecture.md b/docs/architecture.md index f84f445cd..c522edaae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -147,7 +147,9 @@ tick that writes more than the room left, such as one value larger than its byte budget, can still cross that mark. The idle truncation empties the WAL with no further write, on the Worker or on the main thread without one. -Space goes back to the disk once. When every one-time pass has finished and +A new journal is created in incremental auto-vacuum, so it gives free pages +back from its first compaction ticks. A journal created before that gets its +space back once. When every one-time pass has finished and the trace was cut once, the next core start checks the journal before opening it (`journal/vacuum.ts`): with free pages over a quarter of the file and over 256 MiB, and room on the volume for the live pages plus 512 MiB, it writes diff --git a/packages/core/src/journal.ts b/packages/core/src/journal.ts index 69a72c3db..fe52cc2ba 100644 --- a/packages/core/src/journal.ts +++ b/packages/core/src/journal.ts @@ -93,6 +93,7 @@ export class Journal { this.stream = new StreamBuffer(this.db, this, this.onError); this.processes = new ProcessRows(this.db); try { + if ((this.db.query('PRAGMA page_count').get() as { page_count: number }).page_count === 0) this.db.exec('PRAGMA auto_vacuum = INCREMENTAL'); // Only a new file takes it: no `journal/vacuum.ts` ever. this.db.exec('PRAGMA journal_mode = WAL'); this.db.exec('PRAGMA synchronous = NORMAL'); this.db.exec('PRAGMA busy_timeout = 5000'); diff --git a/packages/core/src/journal/retention.ts b/packages/core/src/journal/retention.ts index fa1097827..7240ffb6b 100644 --- a/packages/core/src/journal/retention.ts +++ b/packages/core/src/journal/retention.ts @@ -167,7 +167,7 @@ function dailyPass(journal: Journal, now: number, atKey: string, throughKey: str /** * Gives free pages back to the disk on a journal in incremental auto-vacuum - * (`journal/vacuum.ts` turns it on), `pages` per tick, until the free list is + * (set when the journal is created, or by `journal/vacuum.ts` for an older one), `pages` per tick, until the free list is * back to `FREE_KEPT_BYTES`. */ function freePages(journal: Journal, pages: number): Step { diff --git a/packages/core/test/journal-vacuum.test.ts b/packages/core/test/journal-vacuum.test.ts index f4d258b8d..a1f06178f 100644 --- a/packages/core/test/journal-vacuum.test.ts +++ b/packages/core/test/journal-vacuum.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, expect, spyOn, test } from 'bun:test'; +import { Database } from 'bun:sqlite'; import * as fs from 'node:fs'; import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -16,9 +17,18 @@ const when = { share: 0.25, bytes: 4 * MiB }; const file = () => join(dir, 'journal.db'); const pragma = (journal: Journal, name: string) => Object.values(journal.db.query(`PRAGMA ${name}`).get() as Record)[0]!; -/** A journal whose background passes finished, holding `freed` bytes of free pages and a kept message. */ +/** A journal file made before new journals started in incremental auto-vacuum: its first page is written without it. */ +function legacyJournalFile(): void { + const db = new Database(file(), { create: true }); + db.exec('CREATE TABLE legacy (x); DROP TABLE legacy;'); + db.close(); +} + +/** An older journal whose background passes finished, holding `freed` bytes of free pages and a kept message. */ function journalWithFreePages(freed: number, finished = true): void { + legacyJournalFile(); const journal = new Journal(file()); + expect(pragma(journal, 'auto_vacuum')).toBe(0); journal.putMessage({ id: 'msg_kept', threadId: 'thr_a', turnId: 'trn_a', role: 'assistant', parts: [{ type: 'text', text: 'kept' }], state: 'complete', createdAt: 1 }); const insert = journal.db.query("INSERT INTO events (thread_id, ts, type, version, payload) VALUES ('thr_b', 1, 'turn.finished', 1, ?)"); journal.db.transaction(() => { for (let n = 0; n < freed / 4096; n++) insert.run('x'.repeat(4000)); })(); @@ -69,6 +79,19 @@ test('once the passes are done, a start gives the free pages back and turns on i } }); +test('a new journal starts in incremental auto-vacuum and never needs the VACUUM', () => { + const journal = new Journal(file()); + try { + expect(pragma(journal, 'auto_vacuum')).toBe(2); + expect(journal.db.query('PRAGMA journal_mode').get()).toEqual({ journal_mode: 'wal' }); + for (const key of ONE_TIME_PASSES) journal.setSetting(key, Number.MAX_SAFE_INTEGER); + journal.setSetting(PRUNED_AT, 1); + } finally { + journal.close(); + } + expect(vacuumPlan(file(), { share: 0, bytes: 0 })).toBeNull(); +}); + test('no VACUUM before the passes finished or under the thresholds, and the journal is left as it was', () => { journalWithFreePages(12 * MiB, false); expect(vacuumPlan(file(), when)).toBeNull();