Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions pgpm/cli/__tests__/import-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*
* PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
*/
import { readBundleArchiveFile } from '@pgpmjs/core';
import { copyBlockToInsert, preprocessDumpText } from '@pgpmjs/import';
import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/transform';
import * as fs from 'fs';
Expand Down Expand Up @@ -165,6 +166,25 @@ describe('pgpm import e2e', () => {
expect(diffCatalogSnapshots(snapOriginal, snapPartitioned)).toEqual([]);
});

it('composes module + linear SQL + bundle projections from one import run', async () => {
await fixture.runTerminalCommands(
`
cd ${WS}
pgpm import dump.sql --pkg imp-emit --emit-sql imp-emit.sql --emit-bundle imp-emit.bundle.tar.gz
`,
{}
);

expect(fs.existsSync(path.join(wsDir, 'imp-emit', 'pgpm.plan'))).toBe(true);

const sql = fs.readFileSync(path.join(wsDir, 'imp-emit.sql'), 'utf-8');
expect(sql).toMatch(/CREATE SCHEMA imp_app/);
expect(sql).toMatch(/CREATE TABLE imp_app\.users/);

const bundle = readBundleArchiveFile(path.join(wsDir, 'imp-emit.bundle.tar.gz'));
expect(bundle.changes.length).toBeGreaterThan(0);
});

it('--with-data deploys COPY/INSERT data as seed fixtures', async () => {
await fixture.runTerminalCommands(
`
Expand Down
23 changes: 23 additions & 0 deletions pgpm/cli/__tests__/transform-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*
* PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
*/
import { readBundleArchiveFile } from '@pgpmjs/core';
import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/transform';
import * as fs from 'fs';
import * as path from 'path';
Expand Down Expand Up @@ -276,6 +277,28 @@ describe('pgpm transform e2e', () => {
expect(diffCatalogSnapshots(snapOriginal, snapPartitioned)).toEqual([]);
});

it('composes module + linear SQL + bundle projections from one transform run', async () => {
await fixture.runTerminalCommands(
`
cd ${WS}/${MODULE_NAME}
pgpm transform --granularity consolidated --out ../emit-out --emit-sql ../emit-out/transform.sql --emit-bundle ../emit-out/transform.bundle.tar.gz
`,
{}
);

const outDir = path.join(wsDir, 'emit-out', `${MODULE_NAME}-consolidated`);
expect(fs.existsSync(path.join(outDir, 'pgpm.plan'))).toBe(true);

// linear SQL projection: real, deployable SQL in plan order
const sql = fs.readFileSync(path.join(wsDir, 'emit-out', 'transform.sql'), 'utf-8');
expect(sql).toMatch(/CREATE SCHEMA tfx_app/);
expect(sql).toMatch(/CREATE TABLE tfx_app\.users/);

// bundle projection: a readable content-addressed archive of the changes
const bundle = readBundleArchiveFile(path.join(wsDir, 'emit-out', 'transform.bundle.tar.gz'));
expect(bundle.changes.length).toBeGreaterThan(0);
});

it('round-trips consolidated -> atomic -> consolidated to the same object set', async () => {
// consolidated already exists at <module>-consolidated (from --check test)
const c1 = path.join(wsDir, `${MODULE_NAME}-consolidated`);
Expand Down
51 changes: 44 additions & 7 deletions pgpm/cli/src/commands/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ import { cliExitWithError, CLIOptions, Inquirerer, ParsedArgs } from 'inquirerer
import * as path from 'path';

import { checkOverwrite, writePackage } from '../utils/emit-package';
import {
hasEmitProjection,
parseEmitProjectionTargets,
projectModule,
STDOUT_TARGET
} from '../utils/module-projections';

const log = new Logger('import');

Expand Down Expand Up @@ -59,14 +65,24 @@ Options:
splitting the import into multiple pgpm packages with
derived cross-package requires
--write Allow overwriting an existing module directory
--emit-sql <file|-> Also project the imported module into a single linear
SQL script (- for stdout). Requires a single output
package (not supported with --partition).
--emit-bundle <file> Also project the imported module into a
content-addressed .bundle.tar.gz. Same single-package
requirement as --emit-sql.
--cwd <directory> Working directory (default: current directory)
--dry-run Print the resulting plan/paths without writing

The imported module is the canonical artifact; --emit-sql and --emit-bundle are
pure projections of it (the same machinery pgpm package and pgpm diff use).

Examples:
pgpm import dump.sql --pkg my-app
pgpm import dump.sql --pkg my-app --granularity consolidated --naming flat
pgpm import ./sql-files --pkg my-app --out ./packages --with-data
pgpm import dump.sql --pkg my-app --partition partition.json
pgpm import dump.sql --pkg my-app --emit-sql my-app.sql
`;

const NAMING_STYLES = ['directory', 'flat'] as const;
Expand Down Expand Up @@ -119,6 +135,13 @@ export default async (
const withData = Boolean(argv['with-data'] ?? argv.withData);
const write = Boolean(argv.write);
const dryRun = Boolean(argv['dry-run'] ?? argv.dryRun);
const emit = parseEmitProjectionTargets(argv, cwd);
const emitRequested = hasEmitProjection(emit);
const sqlToStdout = emit.emitSql === STDOUT_TARGET;

if (emitRequested && dryRun) {
await cliExitWithError('--emit-sql/--emit-bundle cannot be combined with --dry-run.');
}

let source;
try {
Expand All @@ -128,7 +151,7 @@ export default async (
return;
}

if (source.files.length > 1) {
if (source.files.length > 1 && !sqlToStdout) {
log.info(`concatenated ${source.files.length} .sql files in sorted order`);
}

Expand Down Expand Up @@ -175,24 +198,38 @@ export default async (
}
}
} else {
if (emitRequested && packages.length !== 1) {
await cliExitWithError(
'--emit-sql/--emit-bundle require a single output package; a --partition import emits multiple packages.'
);
}
for (const pkg of packages) {
const guard = checkOverwrite(path.join(outBase, pkg.name), source.files[0], write);
if (guard) {
await cliExitWithError(guard);
}
}
let firstDir: string | undefined;
for (const pkg of packages) {
const dir = writePackage(outBase, pkg);
log.success(`wrote ${pkg.rows.length} changes to ${dir}`);
firstDir ??= dir;
if (!sqlToStdout) {
log.success(`wrote ${pkg.rows.length} changes to ${dir}`);
}
}
if (emitRequested && firstDir) {
await projectModule(firstDir, emit, sqlToStdout ? undefined : msg => log.success(msg));
}
}

const { summary } = result;
log.success(
`import: ${summary.statements} statement(s) -> ${summary.changes} change(s), ` +
`${summary.skippedPreamble} preamble skipped, ${summary.skippedData} data skipped, ` +
`${summary.misc} in misc, ${result.warnings.length} warning(s)`
);
if (!sqlToStdout) {
log.success(
`import: ${summary.statements} statement(s) -> ${summary.changes} change(s), ` +
`${summary.skippedPreamble} preamble skipped, ${summary.skippedData} data skipped, ` +
`${summary.misc} in misc, ${result.warnings.length} warning(s)`
);
}

prompter.close();
return argv;
Expand Down
44 changes: 43 additions & 1 deletion pgpm/cli/src/commands/transform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ import type { PgConfig } from 'pg-env';
import { getPgEnvOptions } from 'pg-env';

import { checkOverwrite, writePackage } from '../utils/emit-package';
import {
hasEmitProjection,
parseEmitProjectionTargets,
projectModule,
STDOUT_TARGET
} from '../utils/module-projections';
import { catalogDifferences, withScratchDatabases } from '../utils/scratch-db';

export { checkOverwrite } from '../utils/emit-package';
Expand All @@ -45,14 +51,24 @@ Options:
--cwd <directory> Module or workspace directory (default: current directory)
--out <dir> Output directory (default: sibling <module>-<granularity>)
--write Allow writing over an existing/module directory
--emit-sql <file|-> Also project the transformed module into a single
linear SQL script (- for stdout). Requires a single
output package (no workspace/partition fan-out).
--emit-bundle <file> Also project the transformed module into a
content-addressed .bundle.tar.gz. Same single-package
requirement as --emit-sql.
--check Deploy original and transformed output into scratch
databases and assert the catalogs are equivalent
--dry-run Print the resulting plan/paths without writing

The transformed module is the canonical artifact; --emit-sql and --emit-bundle
are pure projections of it (the same machinery pgpm package and pgpm diff use).

Examples:
pgpm transform --granularity object
pgpm transform --granularity atomic --naming flat --out ./out
pgpm transform --granularity object --partition partition.json --check
pgpm transform --granularity consolidated --emit-sql migration.sql
`;

const NAMING_STYLES = ['directory', 'flat'] as const;
Expand Down Expand Up @@ -223,6 +239,9 @@ export default async (
const write = Boolean(argv.write);
const check = Boolean(argv.check);
const dryRun = Boolean(argv['dry-run'] ?? argv.dryRun);
const emit = parseEmitProjectionTargets(argv, cwd);
const emitRequested = hasEmitProjection(emit);
const sqlToStdout = emit.emitSql === STDOUT_TARGET;

const pkg = new PgpmPackage(path.resolve(cwd));

Expand All @@ -245,6 +264,15 @@ export default async (
return;
}

if (emitRequested && modulePaths.length > 1) {
await cliExitWithError(
'--emit-sql/--emit-bundle target a single module; they are not supported when transforming a multi-module workspace.'
);
}
if (emitRequested && dryRun) {
await cliExitWithError('--emit-sql/--emit-bundle cannot be combined with --dry-run.');
}

for (const modulePath of modulePaths) {
let transformed: TransformedModule;
try {
Expand Down Expand Up @@ -274,10 +302,24 @@ export default async (
}
}

if (emitRequested && transformed.packages.length !== 1) {
await cliExitWithError(
'--emit-sql/--emit-bundle require a single output package; a --partition transform emits multiple packages.'
);
}

const sourceRequires = readControlRequires(modulePath, transformed.name);
let firstDir: string | undefined;
for (const pkgRows of transformed.packages) {
const dir = writePackage(transformed.outBase, pkgRows, sourceRequires);
log.success(`wrote ${pkgRows.rows.length} changes to ${dir}`);
firstDir ??= dir;
if (!sqlToStdout) {
log.success(`wrote ${pkgRows.rows.length} changes to ${dir}`);
}
}

if (emitRequested && firstDir) {
await projectModule(firstDir, emit, sqlToStdout ? undefined : msg => log.success(msg));
}

if (check) {
Expand Down
56 changes: 56 additions & 0 deletions pgpm/cli/src/utils/module-projections.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,59 @@ export const emitModuleBundle = async (moduleDir: string, target: string): Promi
fs.mkdirSync(path.dirname(resolved), { recursive: true });
writeBundleArchiveFile(bundle, resolved);
};

/** Resolved `--emit-sql` / `--emit-bundle` projection targets for a command. */
export interface EmitProjectionTargets {
/** Linear SQL target (absolute path, or `-` for stdout). */
emitSql?: string;
/** Bundle archive target (absolute path). */
emitBundle?: string;
}

/**
* Parse the shared `--emit-sql` / `--emit-bundle` projection flags off a parsed
* argv, resolving file targets against `cwd` (the `-` stdout sentinel for SQL is
* preserved). Keeps every command's projection flags identical.
*/
export const parseEmitProjectionTargets = (
argv: Record<string, unknown>,
cwd: string
): EmitProjectionTargets => {
const sqlRaw = argv['emit-sql'] ?? argv.emitSql;
const emitSql =
typeof sqlRaw === 'string' && sqlRaw
? sqlRaw === STDOUT_TARGET
? STDOUT_TARGET
: path.resolve(cwd, sqlRaw)
: undefined;
const bundleRaw = argv['emit-bundle'] ?? argv.emitBundle;
const emitBundle =
typeof bundleRaw === 'string' && bundleRaw ? path.resolve(cwd, bundleRaw) : undefined;
return { emitSql, emitBundle };
};

/** Whether any projection target was requested. */
export const hasEmitProjection = (targets: EmitProjectionTargets): boolean =>
Boolean(targets.emitSql || targets.emitBundle);

/**
* Run the requested SQL/bundle projections against a written module directory.
* `onSuccess` (when provided) is invoked with a human-readable line per emitted
* artifact; it is skipped for stdout SQL so the stream stays valid SQL.
*/
export const projectModule = async (
moduleDir: string,
targets: EmitProjectionTargets,
onSuccess?: (message: string) => void
): Promise<void> => {
if (targets.emitSql) {
await emitModuleSql(moduleDir, targets.emitSql);
if (targets.emitSql !== STDOUT_TARGET) {
onSuccess?.(`wrote linear SQL to ${targets.emitSql}`);
}
}
if (targets.emitBundle) {
await emitModuleBundle(moduleDir, targets.emitBundle);
onSuccess?.(`wrote bundle to ${targets.emitBundle}`);
}
};
Loading