diff --git a/cli/commands/db/mark-as-migrated.mjs b/cli/commands/db/mark-as-migrated.mjs index a9e2d5a5e..f26d32c5a 100644 --- a/cli/commands/db/mark-as-migrated.mjs +++ b/cli/commands/db/mark-as-migrated.mjs @@ -6,7 +6,6 @@ import { join } from 'pathe' import { createDrizzleClient, getDatabaseMigrationFiles, AppliedDatabaseMigrationsQuery } from '@nuxthub/core/db' import { sql } from 'drizzle-orm' import { loadDotenv, dotenvArg } from '../../utils/dotenv.mjs' -import { quoteIdentifier } from '../../utils/db.mjs' export default defineCommand({ meta: { @@ -76,8 +75,7 @@ export default defineCommand({ } // If a specific migration is provided, mark it as applied if (args.name) { - const migrationsTable = quoteIdentifier('_hub_migrations', dialect) - await db[execute](sql.raw(`INSERT INTO ${migrationsTable} (name) VALUES ('${args.name}');`)) + await db[execute](sql`INSERT INTO ${sql.identifier('_hub_migrations')} (name) VALUES (${args.name})`) consola.success(`Local migration \`${args.name}\` marked as applied.`) return closeDb() } @@ -88,7 +86,6 @@ export default defineCommand({ return closeDb() } consola.info(`Found \`${pendingMigrations.length}\` pending migration${pendingMigrations.length === 1 ? '' : 's'}`) - const migrationsTable = quoteIdentifier('_hub_migrations', dialect) let migrationsMarkedAsApplied = 0 for (const migration of pendingMigrations) { const confirmed = await consola.prompt(`Mark migration \`${migration.name}\` as applied?`, { @@ -100,7 +97,7 @@ export default defineCommand({ consola.info(`Migration \`${migration.name}\` skipped.`) continue } - await db[execute](sql.raw(`INSERT INTO ${migrationsTable} (name) VALUES ('${migration.name}');`)) + await db[execute](sql`INSERT INTO ${sql.identifier('_hub_migrations')} (name) VALUES (${migration.name})`) consola.success(`Migration \`${migration.name}\` marked as applied.`) migrationsMarkedAsApplied++ } diff --git a/cli/commands/db/squash.mjs b/cli/commands/db/squash.mjs index 92fd9010a..22a9124f8 100644 --- a/cli/commands/db/squash.mjs +++ b/cli/commands/db/squash.mjs @@ -5,7 +5,7 @@ import { readFile, writeFile, rm } from 'node:fs/promises' import { join, resolve } from 'pathe' import { buildDatabaseSchema, createDrizzleClient } from '@nuxthub/core/db' import { sql } from 'drizzle-orm' -import { getTsconfigAliases, quoteIdentifier } from '../../utils/db.mjs' +import { getTsconfigAliases } from '../../utils/db.mjs' export default defineCommand({ meta: { @@ -204,7 +204,7 @@ export default defineCommand({ const db = await createDrizzleClient(hubConfig.db, hubDir) const dialect = hubConfig.db.dialect const execute = dialect === 'sqlite' ? 'run' : 'execute' - await db[execute](sql.raw(`INSERT INTO ${quoteIdentifier('_hub_migrations', dialect)} (name) VALUES ('${newMigration.tag}');`)) + await db[execute](sql`INSERT INTO ${sql.identifier('_hub_migrations')} (name) VALUES (${newMigration.tag})`) await db.$client?.end?.() consola.success(`Migration \`${newMigration.tag}\` marked as applied.`) } else { diff --git a/docs/content/docs/2.database/4.migrations.md b/docs/content/docs/2.database/4.migrations.md index 77cbb502b..cf97ab17a 100644 --- a/docs/content/docs/2.database/4.migrations.md +++ b/docs/content/docs/2.database/4.migrations.md @@ -116,6 +116,25 @@ When running the development server, NuxtHub will automatically apply the migrat See the CI/CD guide for production migration workflows, including D1-specific handling. :: +### PostgreSQL transactions + +On PostgreSQL, each migration file runs in its own transaction together with its `_hub_migrations` row, so a failing statement leaves neither partial changes nor a history row. An advisory lock serializes migrations, so instances starting at the same time apply each file once. + +Some statements cannot run in a transaction: + +- `CREATE INDEX CONCURRENTLY`, `REINDEX CONCURRENTLY`, `VACUUM`, and other statements PostgreSQL rejects inside a transaction block +- `BEGIN`, `COMMIT` and `ROLLBACK` written in the migration file +- using an enum value in the same file that adds it with `ALTER TYPE ... ADD VALUE` + +For enum values, add the value in one migration and use it in the next. For the other statements, put them in their own migration file that starts with `-- nuxthub:no-transaction`: + +```sql [server/db/migrations/postgresql/0005_posts_slug_index.sql] +-- nuxthub:no-transaction +CREATE INDEX CONCURRENTLY IF NOT EXISTS posts_slug_idx ON posts (slug); +``` + +NuxtHub runs this file statement by statement, outside a transaction, and records it once the last statement succeeds. If a statement fails, earlier statements stay applied, so write them with `IF NOT EXISTS` so the file can run again. These files keep a session-level lock on one connection, so they need the `postgres-js`, `node-postgres` or `pglite` driver and a session connection, not a transaction-mode pooler such as PgBouncer. + ## Post-migration queries ::important diff --git a/package.json b/package.json index a5349020f..dd2506a60 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ } }, "devDependencies": { + "@electric-sql/pglite": "^0.3.15", "@nuxt/devtools": "^3.1.1", "@nuxt/eslint-config": "^1.15.1", "@nuxt/module-builder": "^1.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95e9f15be..44a2413c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -90,6 +90,9 @@ importers: specifier: ^4.2.1 version: 4.6.7 devDependencies: + '@electric-sql/pglite': + specifier: ^0.3.15 + version: 0.3.15 '@nuxt/devtools': specifier: ^3.1.1 version: 3.2.4(vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.33.0)(terser@5.48.0)(yaml@2.8.2))(vue@3.5.44(typescript@5.9.3)) diff --git a/src/db/lib/migrations.ts b/src/db/lib/migrations.ts index 1f1cf2d61..b466d8334 100644 --- a/src/db/lib/migrations.ts +++ b/src/db/lib/migrations.ts @@ -1,13 +1,57 @@ import { consola } from 'consola' import { join, relative } from 'pathe' import type { ResolvedHubConfig } from '@nuxthub/core' -import { AppliedDatabaseMigrationsQuery, getCreateMigrationsTableQuery, splitSqlQueries } from './utils' +import { AppliedDatabaseMigrationsQuery, NoTransactionMigrationDirective, PostgresqlMigrationsLockedMessage, dollarQuote, getAtomicPostgresqlMigrationQuery, getCreateMigrationsTableQuery, getRecordPostgresqlMigrationQuery, isNoTransactionMigration, splitSqlQueries } from './utils' import { useDatabaseMigrationsStorage, getDatabaseMigrationFiles, useDatabaseQueriesStorage, getDatabaseQueryFiles } from './storage' function getRelativePath(fullPath: string) { return relative(process.cwd(), fullPath) } +type PostgresqlSession = { + query: (text: string) => Promise + release: () => Promise +} + +async function reservePostgresqlSession(client: any): Promise { + // postgres-js + if (typeof client?.reserve === 'function') { + const reserved = await client.reserve() + return { query: async text => [...await reserved.unsafe(text)], release: async () => reserved.release() } + } + // node-postgres Pool + if (typeof client?.connect === 'function' && 'totalCount' in client) { + const connection = await client.connect() + return { query: async text => (await connection.query(text)).rows, release: async () => connection.release() } + } + // node-postgres Client and PGlite hold a single connection + if (typeof client?.query === 'function') { + return { query: async text => (await client.query(text)).rows, release: async () => {} } + } +} + +// Lock attempts never wait inside a statement: a waiting statement holds a snapshot that `CREATE INDEX CONCURRENTLY` waits for +async function retryWhileLocked(attempt: () => Promise, log: ReturnType): Promise { + let waiting = false + while (true) { + try { + return await attempt() + } catch (error: any) { + if (!(error.cause?.message || error.message)?.includes(PostgresqlMigrationsLockedMessage)) throw error + } + if (!waiting) log.info('Waiting for another instance to finish database migrations...') + waiting = true + await new Promise(resolve => setTimeout(resolve, 500)) + } +} + +async function acquirePostgresqlMigrationsLock(session: PostgresqlSession, log: ReturnType) { + await retryWhileLocked(async () => { + const [row] = await session.query(`SELECT pg_try_advisory_lock(hashtext('nuxthub'), hashtext('migrations')) AS locked`) + if (!row?.locked) throw new Error(PostgresqlMigrationsLockedMessage) + }, log) +} + export async function applyDatabaseMigrations(hub: ResolvedHubConfig, db: any) { if (!hub.db) return // Create a logger for this function (at runtime so we can have the debug level when run by the CLI) @@ -23,7 +67,7 @@ export async function applyDatabaseMigrations(hub: ResolvedHubConfig, db: any) { const drizzleOrmPkg = 'drizzle-orm' const sql = await import(drizzleOrmPkg).then(m => m.sql) try { - await db[execute](sql.raw(createMigrationsTableQuery)) + await retryWhileLocked(() => db[execute](sql.raw(createMigrationsTableQuery)), log) } catch (error: any) { const message = error.cause?.message || error.message log.error(`Failed to create migrations table\n${message}`) @@ -53,28 +97,64 @@ export async function applyDatabaseMigrations(hub: ResolvedHubConfig, db: any) { return } + const migrations = [] for (const migration of pendingMigrations) { - let query = await migrationsStorage.getItem(migration.filename) - if (!query) continue - query += `\nINSERT INTO _hub_migrations (name) values ('${migration.name}');` - const queries = splitSqlQueries(query) + const query = await migrationsStorage.getItem(migration.filename) + if (query) migrations.push({ ...migration, query }) + } - try { - log.debug(`Applying database migration \`${getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))}\`...`) - for (const query of queries) { - await db[execute](sql.raw(query)) - } - } catch (error: any) { - const message = error.cause?.message || error.message - log.error(`Failed to apply migration \`${getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))}\`\n${message}`) - if (message?.includes('already exists')) { - log.info(`To mark this migration as applied, run \`npx nuxt db mark-as-migrated ${migration.name}\``) - log.info('To drop a table, run `npx nuxt db drop `') - } + let session: PostgresqlSession | undefined + if (dialect === 'postgresql' && migrations.some(migration => isNoTransactionMigration(migration.query))) { + session = hub.db.driver === 'neon-http' ? undefined : await reservePostgresqlSession(db.$client) + if (!session) { + log.error(`Migrations starting with \`${NoTransactionMigrationDirective}\` need a driver with session connections, such as \`postgres-js\`, \`node-postgres\` or \`pglite\`.`) return false } + await acquirePostgresqlMigrationsLock(session, log) + } - log.success(`Database migration \`${getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))}\` applied`) + try { + for (const migration of migrations) { + const migrationPath = getRelativePath(join(hub.dir!, 'db/migrations', migration.filename)) + try { + log.debug(`Applying database migration \`${migrationPath}\`...`) + if (dialect !== 'postgresql') { + for (const query of splitSqlQueries(migration.query)) { + await db[execute](sql.raw(query)) + } + await db[execute](sql`INSERT INTO _hub_migrations (name) VALUES (${migration.name})`) + } else if (!session) { + await retryWhileLocked(() => db[execute](sql.raw(getAtomicPostgresqlMigrationQuery(migration.name, migration.query))), log) + } else if (!isNoTransactionMigration(migration.query)) { + await session.query(getAtomicPostgresqlMigrationQuery(migration.name, migration.query)) + } else { + const applied = await session.query(`SELECT 1 FROM _hub_migrations WHERE name = ${dollarQuote(migration.name)}`) + if (applied.length) continue + for (const query of splitSqlQueries(migration.query)) { + await session.query(query) + } + await session.query(getRecordPostgresqlMigrationQuery(migration.name)) + } + } catch (error: any) { + const message = error.cause?.message || error.message + log.error(`Failed to apply migration \`${migrationPath}\`\n${message}`) + if (message?.includes('already exists')) { + log.info(`To mark this migration as applied, run \`npx nuxt db mark-as-migrated ${migration.name}\``) + log.info('To drop a table, run `npx nuxt db drop `') + } + if (dialect === 'postgresql' && /cannot be executed from a function|cannot run inside a transaction block|unsafe use of new value|transaction commands/.test(message || '')) { + log.info(`PostgreSQL migrations run in a transaction. Split the file so a new enum value is used in a later migration, or move the statement to a migration that starts with \`${NoTransactionMigrationDirective}\`.`) + } + return false + } + + log.success(`Database migration \`${migrationPath}\` applied`) + } + } finally { + if (session) { + await session.query(`SELECT pg_advisory_unlock(hashtext('nuxthub'), hashtext('migrations'))`).catch(() => {}) + await session.release() + } } !import.meta.dev && log.success('Database migrations applied successfully.') return true diff --git a/src/db/lib/utils.ts b/src/db/lib/utils.ts index 28109e49c..b5024f927 100644 --- a/src/db/lib/utils.ts +++ b/src/db/lib/utils.ts @@ -62,6 +62,52 @@ export function getMigrationMetadata(filename: string): { filename: string, name } } +export const PostgresqlMigrationsLockedMessage = 'nuxthub: migrations are locked by another instance' + +const PostgresqlMigrationsLock = `IF NOT pg_try_advisory_xact_lock(hashtext('nuxthub'), hashtext('migrations')) THEN + RAISE EXCEPTION '${PostgresqlMigrationsLockedMessage}' USING ERRCODE = 'lock_not_available'; + END IF;` + +export const NoTransactionMigrationDirective = '-- nuxthub:no-transaction' + +export function dollarQuote(value: string): string { + let tag = '$nuxthub$' + while (value.includes(tag)) tag = `${tag.slice(0, -1)}_$` + return `${tag}${value}${tag}` +} + +export function isNoTransactionMigration(sqlFileContent: string): boolean { + for (const line of sqlFileContent.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + if (!trimmed.startsWith('--')) return false + if (trimmed === NoTransactionMigrationDirective) return true + } + return false +} + +export function getAtomicPostgresqlMigrationQuery(name: string, sqlFileContent: string): string { + return `DO ${dollarQuote(` +BEGIN + ${PostgresqlMigrationsLock} + -- another instance may have applied it while this one waited for the lock + IF NOT EXISTS (SELECT 1 FROM _hub_migrations WHERE name = ${dollarQuote(name)}) THEN + EXECUTE ${dollarQuote(sqlFileContent)}; + INSERT INTO _hub_migrations (name) VALUES (${dollarQuote(name)}); + END IF; +END +`)};` +} + +export function getRecordPostgresqlMigrationQuery(name: string): string { + return `DO ${dollarQuote(` +BEGIN + ${PostgresqlMigrationsLock} + INSERT INTO _hub_migrations (name) VALUES (${dollarQuote(name)}) ON CONFLICT (name) DO NOTHING; +END +`)};` +} + /** * Get the appropriate create table query for the migrations table based on the database dialect */ @@ -70,7 +116,12 @@ export function getCreateMigrationsTableQuery(db: { dialect: string }): string { switch (dialect) { case 'postgresql': - return CreateDatabaseMigrationsTableQueryPostgresql + return `DO ${dollarQuote(` +BEGIN + ${PostgresqlMigrationsLock} + ${CreateDatabaseMigrationsTableQueryPostgresql} +END +`)};` case 'mysql': return CreateDatabaseMigrationsTableQueryMysql case 'sqlite': diff --git a/test/database.migrations.test.ts b/test/database.migrations.test.ts new file mode 100644 index 000000000..861afdcae --- /dev/null +++ b/test/database.migrations.test.ts @@ -0,0 +1,151 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'pathe' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { PGlite } from '@electric-sql/pglite' +import { drizzle } from 'drizzle-orm/pglite' +import { drizzle as drizzleLibsql } from 'drizzle-orm/libsql' +import { createClient } from '@libsql/client' +import type { ResolvedHubConfig } from '../src/types' +import { applyDatabaseMigrations } from '../src/db/lib/migrations' + +let rootDir: string + +beforeEach(async () => { + rootDir = await mkdtemp(join(tmpdir(), 'nuxthub-migrations-')) + await mkdir(join(rootDir, 'db/migrations'), { recursive: true }) +}) + +afterEach(async () => { + await rm(rootDir, { recursive: true, force: true }) +}) + +function hub(dialect: 'postgresql' | 'sqlite') { + return { dir: rootDir, db: { dialect } } as ResolvedHubConfig +} + +function writeMigration(filename: string, content: string) { + return writeFile(join(rootDir, 'db/migrations', filename), content) +} + +describe('PostgreSQL migrations', () => { + let client: PGlite + let db: ReturnType + let executedQueries: string[] + + beforeEach(() => { + client = new PGlite() + executedQueries = [] + db = drizzle(client, { logger: { logQuery: query => executedQueries.push(query) } }) + }) + + afterEach(async () => { + await client.close() + }) + + it('rolls back a failed migration and its history row', async () => { + await writeMigration('0001_failure.postgresql.sql', ` + CREATE TABLE partially_applied (id integer); + --> statement-breakpoint + ALTER TABLE missing_table ADD COLUMN value integer; + `) + + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBe(false) + + const result = await client.query(` + SELECT + to_regclass('partially_applied') IS NOT NULL AS ddl_applied, + (SELECT count(*)::int FROM _hub_migrations) AS history_rows + `) + expect(result.rows).toEqual([{ ddl_applied: false, history_rows: 0 }]) + }) + + it('applies each migration once when runners overlap', async () => { + await writeMigration('0001_once.postgresql.sql', ` + CREATE FUNCTION migration_value() RETURNS text AS $$ + BEGIN + RETURN '$nuxthub$'; + END; + $$ LANGUAGE plpgsql; + --> statement-breakpoint + CREATE TABLE applied_once (value text DEFAULT '$nuxthub$'); + `) + + expect(await Promise.all([ + applyDatabaseMigrations(hub('postgresql'), db), + applyDatabaseMigrations(hub('postgresql'), db) + ])).toEqual([true, true]) + + const lockedQueries = executedQueries.filter(query => query.startsWith('DO ')) + expect(lockedQueries).toHaveLength(4) + expect(lockedQueries.every(query => query.includes('pg_try_advisory_xact_lock'))).toBe(true) + + const result = await client.query(` + SELECT + migration_value() AS function_value, + (SELECT count(*)::int FROM _hub_migrations) AS history_rows + `) + expect(result.rows).toEqual([{ function_value: '$nuxthub$', history_rows: 1 }]) + }) + + it('records migration names that contain quotes', async () => { + await writeMigration('0001_it\'s.postgresql.sql', 'CREATE TABLE quoted (id integer);') + + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBe(true) + + const result = await client.query('SELECT name FROM _hub_migrations') + expect(result.rows).toEqual([{ name: '0001_it\'s' }]) + }) + + it('runs statements that cannot run in a transaction only from opted-out files', async () => { + const vacuum = ` + CREATE TABLE vacuumed (id integer); + --> statement-breakpoint + VACUUM vacuumed; + ` + await writeMigration('0001_vacuum.postgresql.sql', vacuum) + + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBe(false) + expect((await client.query('SELECT to_regclass(\'vacuumed\') IS NOT NULL AS ddl_applied')).rows).toEqual([{ ddl_applied: false }]) + + await writeMigration('0001_vacuum.postgresql.sql', `-- nuxthub:no-transaction${vacuum}`) + + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBe(true) + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBeUndefined() + const result = await client.query(` + SELECT + to_regclass('vacuumed') IS NOT NULL AS ddl_applied, + (SELECT count(*)::int FROM _hub_migrations) AS history_rows + `) + expect(result.rows).toEqual([{ ddl_applied: true, history_rows: 1 }]) + }) + + it('needs a separate migration to use a new enum value', async () => { + await writeMigration('0001_enum.postgresql.sql', 'CREATE TYPE status AS ENUM (\'draft\');') + await writeMigration('0002_enum.postgresql.sql', ` + ALTER TYPE status ADD VALUE 'published'; + --> statement-breakpoint + CREATE TABLE posts (status status DEFAULT 'published'); + `) + + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBe(false) + + await writeMigration('0002_enum.postgresql.sql', 'ALTER TYPE status ADD VALUE \'published\';') + await writeMigration('0003_enum.postgresql.sql', 'CREATE TABLE posts (status status DEFAULT \'published\');') + + expect(await applyDatabaseMigrations(hub('postgresql'), db)).toBe(true) + }) +}) + +describe('SQLite migrations', () => { + it('records migration names that contain quotes', async () => { + const client = createClient({ url: ':memory:' }) + const db = drizzleLibsql(client) + await writeMigration('0001_it\'s.sqlite.sql', 'CREATE TABLE quoted (id integer);') + + expect(await applyDatabaseMigrations(hub('sqlite'), db)).toBe(true) + const result = await client.execute('SELECT name FROM _hub_migrations') + expect(result.rows.map(row => row.name)).toEqual(['0001_it\'s']) + client.close() + }) +})