Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions cli/commands/db/mark-as-migrated.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import { join } from 'pathe'
import { createDrizzleClient, getDatabaseMigrationFiles, AppliedDatabaseMigrationsQuery } from '@nuxthub/core/db'
import { sql } from 'drizzle-orm'
import { loadDotenv, dotenvArg } from '../../utils/dotenv.mjs'
import { quoteIdentifier } from '../../utils/db.mjs'

export default defineCommand({
meta: {
Expand Down Expand Up @@ -76,8 +75,7 @@ export default defineCommand({
}
// If a specific migration is provided, mark it as applied
if (args.name) {
const migrationsTable = quoteIdentifier('_hub_migrations', dialect)
await db[execute](sql.raw(`INSERT INTO ${migrationsTable} (name) VALUES ('${args.name}');`))
await db[execute](sql`INSERT INTO ${sql.identifier('_hub_migrations')} (name) VALUES (${args.name})`)
consola.success(`Local migration \`${args.name}\` marked as applied.`)
return closeDb()
}
Expand All @@ -88,7 +86,6 @@ export default defineCommand({
return closeDb()
}
consola.info(`Found \`${pendingMigrations.length}\` pending migration${pendingMigrations.length === 1 ? '' : 's'}`)
const migrationsTable = quoteIdentifier('_hub_migrations', dialect)
let migrationsMarkedAsApplied = 0
for (const migration of pendingMigrations) {
const confirmed = await consola.prompt(`Mark migration \`${migration.name}\` as applied?`, {
Expand All @@ -100,7 +97,7 @@ export default defineCommand({
consola.info(`Migration \`${migration.name}\` skipped.`)
continue
}
await db[execute](sql.raw(`INSERT INTO ${migrationsTable} (name) VALUES ('${migration.name}');`))
await db[execute](sql`INSERT INTO ${sql.identifier('_hub_migrations')} (name) VALUES (${migration.name})`)
consola.success(`Migration \`${migration.name}\` marked as applied.`)
migrationsMarkedAsApplied++
}
Expand Down
4 changes: 2 additions & 2 deletions cli/commands/db/squash.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { readFile, writeFile, rm } from 'node:fs/promises'
import { join, resolve } from 'pathe'
import { buildDatabaseSchema, createDrizzleClient } from '@nuxthub/core/db'
import { sql } from 'drizzle-orm'
import { getTsconfigAliases, quoteIdentifier } from '../../utils/db.mjs'
import { getTsconfigAliases } from '../../utils/db.mjs'

export default defineCommand({
meta: {
Expand Down Expand Up @@ -204,7 +204,7 @@ export default defineCommand({
const db = await createDrizzleClient(hubConfig.db, hubDir)
const dialect = hubConfig.db.dialect
const execute = dialect === 'sqlite' ? 'run' : 'execute'
await db[execute](sql.raw(`INSERT INTO ${quoteIdentifier('_hub_migrations', dialect)} (name) VALUES ('${newMigration.tag}');`))
await db[execute](sql`INSERT INTO ${sql.identifier('_hub_migrations')} (name) VALUES (${newMigration.tag})`)
await db.$client?.end?.()
consola.success(`Migration \`${newMigration.tag}\` marked as applied.`)
} else {
Expand Down
19 changes: 19 additions & 0 deletions docs/content/docs/2.database/4.migrations.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,25 @@ When running the development server, NuxtHub will automatically apply the migrat
See the CI/CD guide for production migration workflows, including D1-specific handling.
::

### PostgreSQL transactions

On PostgreSQL, each migration file runs in its own transaction together with its `_hub_migrations` row, so a failing statement leaves neither partial changes nor a history row. An advisory lock serializes migrations, so instances starting at the same time apply each file once.

Some statements cannot run in a transaction:

- `CREATE INDEX CONCURRENTLY`, `REINDEX CONCURRENTLY`, `VACUUM`, and other statements PostgreSQL rejects inside a transaction block
- `BEGIN`, `COMMIT` and `ROLLBACK` written in the migration file
- using an enum value in the same file that adds it with `ALTER TYPE ... ADD VALUE`

For enum values, add the value in one migration and use it in the next. For the other statements, put them in their own migration file that starts with `-- nuxthub:no-transaction`:

```sql [server/db/migrations/postgresql/0005_posts_slug_index.sql]
-- nuxthub:no-transaction
CREATE INDEX CONCURRENTLY IF NOT EXISTS posts_slug_idx ON posts (slug);
```

NuxtHub runs this file statement by statement, outside a transaction, and records it once the last statement succeeds. If a statement fails, earlier statements stay applied, so write them with `IF NOT EXISTS` so the file can run again. These files keep a session-level lock on one connection, so they need the `postgres-js`, `node-postgres` or `pglite` driver and a session connection, not a transaction-mode pooler such as PgBouncer.

## Post-migration queries

::important
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@
}
},
"devDependencies": {
"@electric-sql/pglite": "^0.3.15",
"@nuxt/devtools": "^3.1.1",
"@nuxt/eslint-config": "^1.15.1",
"@nuxt/module-builder": "^1.0.2",
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

118 changes: 99 additions & 19 deletions src/db/lib/migrations.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,57 @@
import { consola } from 'consola'
import { join, relative } from 'pathe'
import type { ResolvedHubConfig } from '@nuxthub/core'
import { AppliedDatabaseMigrationsQuery, getCreateMigrationsTableQuery, splitSqlQueries } from './utils'
import { AppliedDatabaseMigrationsQuery, NoTransactionMigrationDirective, PostgresqlMigrationsLockedMessage, dollarQuote, getAtomicPostgresqlMigrationQuery, getCreateMigrationsTableQuery, getRecordPostgresqlMigrationQuery, isNoTransactionMigration, splitSqlQueries } from './utils'
import { useDatabaseMigrationsStorage, getDatabaseMigrationFiles, useDatabaseQueriesStorage, getDatabaseQueryFiles } from './storage'

function getRelativePath(fullPath: string) {
return relative(process.cwd(), fullPath)
}

type PostgresqlSession = {
query: (text: string) => Promise<any[]>
release: () => Promise<void>
}

async function reservePostgresqlSession(client: any): Promise<PostgresqlSession | undefined> {
// postgres-js
if (typeof client?.reserve === 'function') {
const reserved = await client.reserve()
return { query: async text => [...await reserved.unsafe(text)], release: async () => reserved.release() }
}
// node-postgres Pool
if (typeof client?.connect === 'function' && 'totalCount' in client) {
const connection = await client.connect()
return { query: async text => (await connection.query(text)).rows, release: async () => connection.release() }
}
// node-postgres Client and PGlite hold a single connection
if (typeof client?.query === 'function') {
return { query: async text => (await client.query(text)).rows, release: async () => {} }
}
}

// Lock attempts never wait inside a statement: a waiting statement holds a snapshot that `CREATE INDEX CONCURRENTLY` waits for
async function retryWhileLocked<T>(attempt: () => Promise<T>, log: ReturnType<typeof consola.withTag>): Promise<T> {
let waiting = false
while (true) {
try {
return await attempt()
} catch (error: any) {
if (!(error.cause?.message || error.message)?.includes(PostgresqlMigrationsLockedMessage)) throw error
}
if (!waiting) log.info('Waiting for another instance to finish database migrations...')
waiting = true
await new Promise(resolve => setTimeout(resolve, 500))
}
}

async function acquirePostgresqlMigrationsLock(session: PostgresqlSession, log: ReturnType<typeof consola.withTag>) {
await retryWhileLocked(async () => {
const [row] = await session.query(`SELECT pg_try_advisory_lock(hashtext('nuxthub'), hashtext('migrations')) AS locked`)
if (!row?.locked) throw new Error(PostgresqlMigrationsLockedMessage)
}, log)
}

export async function applyDatabaseMigrations(hub: ResolvedHubConfig, db: any) {
if (!hub.db) return
// Create a logger for this function (at runtime so we can have the debug level when run by the CLI)
Expand All @@ -23,7 +67,7 @@ export async function applyDatabaseMigrations(hub: ResolvedHubConfig, db: any) {
const drizzleOrmPkg = 'drizzle-orm'
const sql = await import(drizzleOrmPkg).then(m => m.sql)
try {
await db[execute](sql.raw(createMigrationsTableQuery))
await retryWhileLocked(() => db[execute](sql.raw(createMigrationsTableQuery)), log)
} catch (error: any) {
const message = error.cause?.message || error.message
log.error(`Failed to create migrations table\n${message}`)
Expand Down Expand Up @@ -53,28 +97,64 @@ export async function applyDatabaseMigrations(hub: ResolvedHubConfig, db: any) {
return
}

const migrations = []
for (const migration of pendingMigrations) {
let query = await migrationsStorage.getItem<string>(migration.filename)
if (!query) continue
query += `\nINSERT INTO _hub_migrations (name) values ('${migration.name}');`
const queries = splitSqlQueries(query)
const query = await migrationsStorage.getItem<string>(migration.filename)
if (query) migrations.push({ ...migration, query })
}

try {
log.debug(`Applying database migration \`${getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))}\`...`)
for (const query of queries) {
await db[execute](sql.raw(query))
}
} catch (error: any) {
const message = error.cause?.message || error.message
log.error(`Failed to apply migration \`${getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))}\`\n${message}`)
if (message?.includes('already exists')) {
log.info(`To mark this migration as applied, run \`npx nuxt db mark-as-migrated ${migration.name}\``)
log.info('To drop a table, run `npx nuxt db drop <table-name>`')
}
let session: PostgresqlSession | undefined
if (dialect === 'postgresql' && migrations.some(migration => isNoTransactionMigration(migration.query))) {
session = hub.db.driver === 'neon-http' ? undefined : await reservePostgresqlSession(db.$client)
if (!session) {
log.error(`Migrations starting with \`${NoTransactionMigrationDirective}\` need a driver with session connections, such as \`postgres-js\`, \`node-postgres\` or \`pglite\`.`)
return false
}
await acquirePostgresqlMigrationsLock(session, log)
}

log.success(`Database migration \`${getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))}\` applied`)
try {
for (const migration of migrations) {
const migrationPath = getRelativePath(join(hub.dir!, 'db/migrations', migration.filename))
try {
log.debug(`Applying database migration \`${migrationPath}\`...`)
if (dialect !== 'postgresql') {
for (const query of splitSqlQueries(migration.query)) {
await db[execute](sql.raw(query))
}
await db[execute](sql`INSERT INTO _hub_migrations (name) VALUES (${migration.name})`)
} else if (!session) {
await retryWhileLocked(() => db[execute](sql.raw(getAtomicPostgresqlMigrationQuery(migration.name, migration.query))), log)
} else if (!isNoTransactionMigration(migration.query)) {
await session.query(getAtomicPostgresqlMigrationQuery(migration.name, migration.query))
} else {
const applied = await session.query(`SELECT 1 FROM _hub_migrations WHERE name = ${dollarQuote(migration.name)}`)
if (applied.length) continue
for (const query of splitSqlQueries(migration.query)) {
await session.query(query)
}
await session.query(getRecordPostgresqlMigrationQuery(migration.name))
}
} catch (error: any) {
const message = error.cause?.message || error.message
log.error(`Failed to apply migration \`${migrationPath}\`\n${message}`)
if (message?.includes('already exists')) {
log.info(`To mark this migration as applied, run \`npx nuxt db mark-as-migrated ${migration.name}\``)
log.info('To drop a table, run `npx nuxt db drop <table-name>`')
}
if (dialect === 'postgresql' && /cannot be executed from a function|cannot run inside a transaction block|unsafe use of new value|transaction commands/.test(message || '')) {
log.info(`PostgreSQL migrations run in a transaction. Split the file so a new enum value is used in a later migration, or move the statement to a migration that starts with \`${NoTransactionMigrationDirective}\`.`)
}
return false
}

log.success(`Database migration \`${migrationPath}\` applied`)
}
} finally {
if (session) {
await session.query(`SELECT pg_advisory_unlock(hashtext('nuxthub'), hashtext('migrations'))`).catch(() => {})
await session.release()
}
}
!import.meta.dev && log.success('Database migrations applied successfully.')
return true
Expand Down
53 changes: 52 additions & 1 deletion src/db/lib/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,52 @@ export function getMigrationMetadata(filename: string): { filename: string, name
}
}

export const PostgresqlMigrationsLockedMessage = 'nuxthub: migrations are locked by another instance'

const PostgresqlMigrationsLock = `IF NOT pg_try_advisory_xact_lock(hashtext('nuxthub'), hashtext('migrations')) THEN
RAISE EXCEPTION '${PostgresqlMigrationsLockedMessage}' USING ERRCODE = 'lock_not_available';
END IF;`

export const NoTransactionMigrationDirective = '-- nuxthub:no-transaction'

export function dollarQuote(value: string): string {
let tag = '$nuxthub$'
while (value.includes(tag)) tag = `${tag.slice(0, -1)}_$`
return `${tag}${value}${tag}`
}

export function isNoTransactionMigration(sqlFileContent: string): boolean {
for (const line of sqlFileContent.split('\n')) {
const trimmed = line.trim()
if (!trimmed) continue
if (!trimmed.startsWith('--')) return false
if (trimmed === NoTransactionMigrationDirective) return true
}
return false
}

export function getAtomicPostgresqlMigrationQuery(name: string, sqlFileContent: string): string {
return `DO ${dollarQuote(`
BEGIN
${PostgresqlMigrationsLock}
-- another instance may have applied it while this one waited for the lock
IF NOT EXISTS (SELECT 1 FROM _hub_migrations WHERE name = ${dollarQuote(name)}) THEN
EXECUTE ${dollarQuote(sqlFileContent)};
INSERT INTO _hub_migrations (name) VALUES (${dollarQuote(name)});
END IF;
END
`)};`
}

export function getRecordPostgresqlMigrationQuery(name: string): string {
return `DO ${dollarQuote(`
BEGIN
${PostgresqlMigrationsLock}
INSERT INTO _hub_migrations (name) VALUES (${dollarQuote(name)}) ON CONFLICT (name) DO NOTHING;
END
`)};`
}

/**
* Get the appropriate create table query for the migrations table based on the database dialect
*/
Expand All @@ -70,7 +116,12 @@ export function getCreateMigrationsTableQuery(db: { dialect: string }): string {

switch (dialect) {
case 'postgresql':
return CreateDatabaseMigrationsTableQueryPostgresql
return `DO ${dollarQuote(`
BEGIN
${PostgresqlMigrationsLock}
${CreateDatabaseMigrationsTableQueryPostgresql}
END
`)};`
case 'mysql':
return CreateDatabaseMigrationsTableQueryMysql
case 'sqlite':
Expand Down
Loading
Loading