diff --git a/pgpm/cli/__tests__/import-e2e.test.ts b/pgpm/cli/__tests__/import-e2e.test.ts index b632b3dd4..50391419d 100644 --- a/pgpm/cli/__tests__/import-e2e.test.ts +++ b/pgpm/cli/__tests__/import-e2e.test.ts @@ -14,6 +14,7 @@ * * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars. */ +import { readBundleArchiveFile } from '@pgpmjs/core'; import { copyBlockToInsert, preprocessDumpText } from '@pgpmjs/import'; import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/transform'; import * as fs from 'fs'; @@ -165,6 +166,25 @@ describe('pgpm import e2e', () => { expect(diffCatalogSnapshots(snapOriginal, snapPartitioned)).toEqual([]); }); + it('composes module + linear SQL + bundle projections from one import run', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import dump.sql --pkg imp-emit --emit-sql imp-emit.sql --emit-bundle imp-emit.bundle.tar.gz + `, + {} + ); + + expect(fs.existsSync(path.join(wsDir, 'imp-emit', 'pgpm.plan'))).toBe(true); + + const sql = fs.readFileSync(path.join(wsDir, 'imp-emit.sql'), 'utf-8'); + expect(sql).toMatch(/CREATE SCHEMA imp_app/); + expect(sql).toMatch(/CREATE TABLE imp_app\.users/); + + const bundle = readBundleArchiveFile(path.join(wsDir, 'imp-emit.bundle.tar.gz')); + expect(bundle.changes.length).toBeGreaterThan(0); + }); + it('--with-data deploys COPY/INSERT data as seed fixtures', async () => { await fixture.runTerminalCommands( ` diff --git a/pgpm/cli/__tests__/transform-e2e.test.ts b/pgpm/cli/__tests__/transform-e2e.test.ts index 9170a46ae..cfeab8d76 100644 --- a/pgpm/cli/__tests__/transform-e2e.test.ts +++ b/pgpm/cli/__tests__/transform-e2e.test.ts @@ -9,6 +9,7 @@ * * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars. */ +import { readBundleArchiveFile } from '@pgpmjs/core'; import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/transform'; import * as fs from 'fs'; import * as path from 'path'; @@ -276,6 +277,28 @@ describe('pgpm transform e2e', () => { expect(diffCatalogSnapshots(snapOriginal, snapPartitioned)).toEqual([]); }); + it('composes module + linear SQL + bundle projections from one transform run', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS}/${MODULE_NAME} + pgpm transform --granularity consolidated --out ../emit-out --emit-sql ../emit-out/transform.sql --emit-bundle ../emit-out/transform.bundle.tar.gz + `, + {} + ); + + const outDir = path.join(wsDir, 'emit-out', `${MODULE_NAME}-consolidated`); + expect(fs.existsSync(path.join(outDir, 'pgpm.plan'))).toBe(true); + + // linear SQL projection: real, deployable SQL in plan order + const sql = fs.readFileSync(path.join(wsDir, 'emit-out', 'transform.sql'), 'utf-8'); + expect(sql).toMatch(/CREATE SCHEMA tfx_app/); + expect(sql).toMatch(/CREATE TABLE tfx_app\.users/); + + // bundle projection: a readable content-addressed archive of the changes + const bundle = readBundleArchiveFile(path.join(wsDir, 'emit-out', 'transform.bundle.tar.gz')); + expect(bundle.changes.length).toBeGreaterThan(0); + }); + it('round-trips consolidated -> atomic -> consolidated to the same object set', async () => { // consolidated already exists at -consolidated (from --check test) const c1 = path.join(wsDir, `${MODULE_NAME}-consolidated`); diff --git a/pgpm/cli/src/commands/import.ts b/pgpm/cli/src/commands/import.ts index cd52405d3..0f4a2b644 100644 --- a/pgpm/cli/src/commands/import.ts +++ b/pgpm/cli/src/commands/import.ts @@ -18,6 +18,12 @@ import { cliExitWithError, CLIOptions, Inquirerer, ParsedArgs } from 'inquirerer import * as path from 'path'; import { checkOverwrite, writePackage } from '../utils/emit-package'; +import { + hasEmitProjection, + parseEmitProjectionTargets, + projectModule, + STDOUT_TARGET +} from '../utils/module-projections'; const log = new Logger('import'); @@ -59,14 +65,24 @@ Options: splitting the import into multiple pgpm packages with derived cross-package requires --write Allow overwriting an existing module directory + --emit-sql Also project the imported module into a single linear + SQL script (- for stdout). Requires a single output + package (not supported with --partition). + --emit-bundle Also project the imported module into a + content-addressed .bundle.tar.gz. Same single-package + requirement as --emit-sql. --cwd Working directory (default: current directory) --dry-run Print the resulting plan/paths without writing +The imported module is the canonical artifact; --emit-sql and --emit-bundle are +pure projections of it (the same machinery pgpm package and pgpm diff use). + Examples: pgpm import dump.sql --pkg my-app pgpm import dump.sql --pkg my-app --granularity consolidated --naming flat pgpm import ./sql-files --pkg my-app --out ./packages --with-data pgpm import dump.sql --pkg my-app --partition partition.json + pgpm import dump.sql --pkg my-app --emit-sql my-app.sql `; const NAMING_STYLES = ['directory', 'flat'] as const; @@ -119,6 +135,13 @@ export default async ( const withData = Boolean(argv['with-data'] ?? argv.withData); const write = Boolean(argv.write); const dryRun = Boolean(argv['dry-run'] ?? argv.dryRun); + const emit = parseEmitProjectionTargets(argv, cwd); + const emitRequested = hasEmitProjection(emit); + const sqlToStdout = emit.emitSql === STDOUT_TARGET; + + if (emitRequested && dryRun) { + await cliExitWithError('--emit-sql/--emit-bundle cannot be combined with --dry-run.'); + } let source; try { @@ -128,7 +151,7 @@ export default async ( return; } - if (source.files.length > 1) { + if (source.files.length > 1 && !sqlToStdout) { log.info(`concatenated ${source.files.length} .sql files in sorted order`); } @@ -175,24 +198,38 @@ export default async ( } } } else { + if (emitRequested && packages.length !== 1) { + await cliExitWithError( + '--emit-sql/--emit-bundle require a single output package; a --partition import emits multiple packages.' + ); + } for (const pkg of packages) { const guard = checkOverwrite(path.join(outBase, pkg.name), source.files[0], write); if (guard) { await cliExitWithError(guard); } } + let firstDir: string | undefined; for (const pkg of packages) { const dir = writePackage(outBase, pkg); - log.success(`wrote ${pkg.rows.length} changes to ${dir}`); + firstDir ??= dir; + if (!sqlToStdout) { + log.success(`wrote ${pkg.rows.length} changes to ${dir}`); + } + } + if (emitRequested && firstDir) { + await projectModule(firstDir, emit, sqlToStdout ? undefined : msg => log.success(msg)); } } const { summary } = result; - log.success( - `import: ${summary.statements} statement(s) -> ${summary.changes} change(s), ` + - `${summary.skippedPreamble} preamble skipped, ${summary.skippedData} data skipped, ` + - `${summary.misc} in misc, ${result.warnings.length} warning(s)` - ); + if (!sqlToStdout) { + log.success( + `import: ${summary.statements} statement(s) -> ${summary.changes} change(s), ` + + `${summary.skippedPreamble} preamble skipped, ${summary.skippedData} data skipped, ` + + `${summary.misc} in misc, ${result.warnings.length} warning(s)` + ); + } prompter.close(); return argv; diff --git a/pgpm/cli/src/commands/transform.ts b/pgpm/cli/src/commands/transform.ts index 88190f212..d9f336a22 100644 --- a/pgpm/cli/src/commands/transform.ts +++ b/pgpm/cli/src/commands/transform.ts @@ -19,6 +19,12 @@ import type { PgConfig } from 'pg-env'; import { getPgEnvOptions } from 'pg-env'; import { checkOverwrite, writePackage } from '../utils/emit-package'; +import { + hasEmitProjection, + parseEmitProjectionTargets, + projectModule, + STDOUT_TARGET +} from '../utils/module-projections'; import { catalogDifferences, withScratchDatabases } from '../utils/scratch-db'; export { checkOverwrite } from '../utils/emit-package'; @@ -45,14 +51,24 @@ Options: --cwd Module or workspace directory (default: current directory) --out Output directory (default: sibling -) --write Allow writing over an existing/module directory + --emit-sql Also project the transformed module into a single + linear SQL script (- for stdout). Requires a single + output package (no workspace/partition fan-out). + --emit-bundle Also project the transformed module into a + content-addressed .bundle.tar.gz. Same single-package + requirement as --emit-sql. --check Deploy original and transformed output into scratch databases and assert the catalogs are equivalent --dry-run Print the resulting plan/paths without writing +The transformed module is the canonical artifact; --emit-sql and --emit-bundle +are pure projections of it (the same machinery pgpm package and pgpm diff use). + Examples: pgpm transform --granularity object pgpm transform --granularity atomic --naming flat --out ./out pgpm transform --granularity object --partition partition.json --check + pgpm transform --granularity consolidated --emit-sql migration.sql `; const NAMING_STYLES = ['directory', 'flat'] as const; @@ -223,6 +239,9 @@ export default async ( const write = Boolean(argv.write); const check = Boolean(argv.check); const dryRun = Boolean(argv['dry-run'] ?? argv.dryRun); + const emit = parseEmitProjectionTargets(argv, cwd); + const emitRequested = hasEmitProjection(emit); + const sqlToStdout = emit.emitSql === STDOUT_TARGET; const pkg = new PgpmPackage(path.resolve(cwd)); @@ -245,6 +264,15 @@ export default async ( return; } + if (emitRequested && modulePaths.length > 1) { + await cliExitWithError( + '--emit-sql/--emit-bundle target a single module; they are not supported when transforming a multi-module workspace.' + ); + } + if (emitRequested && dryRun) { + await cliExitWithError('--emit-sql/--emit-bundle cannot be combined with --dry-run.'); + } + for (const modulePath of modulePaths) { let transformed: TransformedModule; try { @@ -274,10 +302,24 @@ export default async ( } } + if (emitRequested && transformed.packages.length !== 1) { + await cliExitWithError( + '--emit-sql/--emit-bundle require a single output package; a --partition transform emits multiple packages.' + ); + } + const sourceRequires = readControlRequires(modulePath, transformed.name); + let firstDir: string | undefined; for (const pkgRows of transformed.packages) { const dir = writePackage(transformed.outBase, pkgRows, sourceRequires); - log.success(`wrote ${pkgRows.rows.length} changes to ${dir}`); + firstDir ??= dir; + if (!sqlToStdout) { + log.success(`wrote ${pkgRows.rows.length} changes to ${dir}`); + } + } + + if (emitRequested && firstDir) { + await projectModule(firstDir, emit, sqlToStdout ? undefined : msg => log.success(msg)); } if (check) { diff --git a/pgpm/cli/src/utils/module-projections.ts b/pgpm/cli/src/utils/module-projections.ts index 17621b961..b2e389ea9 100644 --- a/pgpm/cli/src/utils/module-projections.ts +++ b/pgpm/cli/src/utils/module-projections.ts @@ -48,3 +48,59 @@ export const emitModuleBundle = async (moduleDir: string, target: string): Promi fs.mkdirSync(path.dirname(resolved), { recursive: true }); writeBundleArchiveFile(bundle, resolved); }; + +/** Resolved `--emit-sql` / `--emit-bundle` projection targets for a command. */ +export interface EmitProjectionTargets { + /** Linear SQL target (absolute path, or `-` for stdout). */ + emitSql?: string; + /** Bundle archive target (absolute path). */ + emitBundle?: string; +} + +/** + * Parse the shared `--emit-sql` / `--emit-bundle` projection flags off a parsed + * argv, resolving file targets against `cwd` (the `-` stdout sentinel for SQL is + * preserved). Keeps every command's projection flags identical. + */ +export const parseEmitProjectionTargets = ( + argv: Record, + cwd: string +): EmitProjectionTargets => { + const sqlRaw = argv['emit-sql'] ?? argv.emitSql; + const emitSql = + typeof sqlRaw === 'string' && sqlRaw + ? sqlRaw === STDOUT_TARGET + ? STDOUT_TARGET + : path.resolve(cwd, sqlRaw) + : undefined; + const bundleRaw = argv['emit-bundle'] ?? argv.emitBundle; + const emitBundle = + typeof bundleRaw === 'string' && bundleRaw ? path.resolve(cwd, bundleRaw) : undefined; + return { emitSql, emitBundle }; +}; + +/** Whether any projection target was requested. */ +export const hasEmitProjection = (targets: EmitProjectionTargets): boolean => + Boolean(targets.emitSql || targets.emitBundle); + +/** + * Run the requested SQL/bundle projections against a written module directory. + * `onSuccess` (when provided) is invoked with a human-readable line per emitted + * artifact; it is skipped for stdout SQL so the stream stays valid SQL. + */ +export const projectModule = async ( + moduleDir: string, + targets: EmitProjectionTargets, + onSuccess?: (message: string) => void +): Promise => { + if (targets.emitSql) { + await emitModuleSql(moduleDir, targets.emitSql); + if (targets.emitSql !== STDOUT_TARGET) { + onSuccess?.(`wrote linear SQL to ${targets.emitSql}`); + } + } + if (targets.emitBundle) { + await emitModuleBundle(moduleDir, targets.emitBundle); + onSuccess?.(`wrote bundle to ${targets.emitBundle}`); + } +};