Merge pull request #1581 from constructive-io/feat/pgpm-diff-append

feat(pgpm): pgpm diff --append-module — append the delta into an existing module

Author: pyramation

pgpm/cli/__tests__/diff-e2e.test.ts

@@ -251,6 +251,47 @@ describe('pgpm diff e2e', () => {
     expect(bundle.changes.length).toBeGreaterThan(0);
   });
 
+  it('appends the delta into an existing module and fresh-deploys to the v2 catalog', async () => {
+    // A standalone module with v1's schema (named so pgpm can deploy it), so
+    // the shared diff-v1 module is untouched.
+    const appendDir = path.join(wsDir, 'diff-append');
+    writeModule(appendDir, 'diff-append', V1);
+    const originalChangeCount = V1.length;
+    // existing change scripts must be left byte-for-byte untouched
+    const sentinel = 'schemas/dfx/tables/products/table';
+    const deployBefore = fs.readFileSync(path.join(appendDir, 'deploy', `${sentinel}.sql`), 'utf-8');
+
+    await fixture.runTerminalCommands(
+      `
+      cd ${WS}
+      pgpm diff diff-append diff-v2 --append-module diff-append
+      `,
+      {}
+    );
+
+    // existing changes are preserved (names + untouched scripts) and new ones appended
+    const planAfter = fs.readFileSync(path.join(appendDir, 'pgpm.plan'), 'utf-8');
+    for (const c of V1) expect(planAfter).toContain(c.name);
+    const changeLines = planAfter
+      .split('\n')
+      .filter(l => l.trim() && !l.startsWith('%') && !l.startsWith('@'));
+    expect(changeLines.length).toBeGreaterThan(originalChangeCount);
+    expect(fs.readFileSync(path.join(appendDir, 'deploy', `${sentinel}.sql`), 'utf-8')).toBe(deployBefore);
+
+    // the appended module (v1 + delta) fresh-deploys to a v2-equivalent catalog
+    const testDb = await fixture.setupTestDatabase();
+    await fixture.runTerminalCommands(
+      `
+      cd ${WS}/diff-append
+      pgpm deploy --database ${testDb.name} --package diff-append --yes
+      `,
+      { database: testDb.name }
+    );
+    const snapAppended = await snapshotCatalog(testDb);
+    const snapV2 = await snapshotCatalog(v2Db);
+    expect(diffCatalogSnapshots(withoutColumnOrder(snapAppended), withoutColumnOrder(snapV2))).toEqual([]);
+  });
+
   it.each(['atomic', 'object', 'consolidated'])(
     'emits a %s-granularity migration that reaches the same v2 catalog (dial parity)',
     async granularity => {

pgpm/cli/src/commands/diff.ts

@@ -7,6 +7,7 @@ import {
 } from '@pgpmjs/diff';
 import { Logger } from '@pgpmjs/logger';
 import {
+  appendModule,
   diffChangeSets,
   EXPORT_GRANULARITIES,
   ExportGranularity,
@@ -58,6 +59,10 @@ Options:
                            verify per change, spec-derived paths, graph-derived
                            requires) into <dir>/<pkg>
   --emit-module <dir>      Alias of --emit-migration
+  --append-module <dir>    Append the delta into an EXISTING pgpm module at
+                           <dir> (new changes only; existing changes, scripts,
+                           and .control are left untouched). Standalone: not
+                           combinable with --emit-*/--verify.
   --emit-sql <file|->      Also project the delta to a single consolidated SQL
                            file (deparsed in plan order); - writes to stdout
   --emit-bundle <file>     Also project the delta to a content-addressed
@@ -238,6 +243,10 @@ export default async (
   const emitModuleDir = typeof emitModuleRaw === 'string' && emitModuleRaw
     ? path.resolve(cwd, emitModuleRaw)
     : undefined;
+  const appendModuleRaw = argv['append-module'] ?? argv.appendModule;
+  const appendModuleDir = typeof appendModuleRaw === 'string' && appendModuleRaw
+    ? path.resolve(cwd, appendModuleRaw)
+    : undefined;
   const emitSqlRaw = argv['emit-sql'] ?? argv.emitSql;
   const emitSql = typeof emitSqlRaw === 'string' && emitSqlRaw
     ? (emitSqlRaw === STDOUT_TARGET ? STDOUT_TARGET : path.resolve(cwd, emitSqlRaw))
@@ -249,6 +258,12 @@ export default async (
   const sqlToStdout = emitSql === STDOUT_TARGET;
   const pkgName = (argv.pkg as string) || 'diff-migration';
 
+  if (appendModuleDir && (emitModuleDir || emitSql || emitBundle || verify)) {
+    await cliExitWithError(
+      '--append-module is standalone; it cannot be combined with --emit-migration/--emit-module/--emit-sql/--emit-bundle/--verify.'
+    );
+  }
+
   await loadModule();
 
   let sideA: DiffSide;
@@ -287,6 +302,22 @@ export default async (
     if (!sqlToStdout) printSummary(result, sideA.label, sideB.label);
   }
 
+  if (appendModuleDir) {
+    const rows = deltaChangesToRows(result.changes);
+    if (rows.length === 0) {
+      log.info('no migration changes to append (sides are identical).');
+    } else {
+      const appended = appendModule(appendModuleDir, rows);
+      for (const w of appended.warnings) console.warn(`diff: ${w}`);
+      log.success(
+        `appended ${appended.added.length} change(s) to ${appended.dir}` +
+        (appended.skipped.length ? ` (${appended.skipped.length} skipped)` : '')
+      );
+    }
+    prompter.close();
+    return argv;
+  }
+
   let migrationDir: string | undefined;
   const needModule = Boolean(emitModuleDir || emitSql || emitBundle) || (verify && !result.identical);
   if (needModule) {

pgpm/transform/src/index.ts

@@ -41,8 +41,8 @@ export {
   defaultChangeName,
   restructureChanges,
 } from './granularity-driver';
-export type { PgpmModuleModel } from './module-emit';
-export { checkOverwrite, writeControlFile, writeModule } from './module-emit';
+export type { AppendModuleResult, PgpmModuleModel } from './module-emit';
+export { appendModule, checkOverwrite, writeControlFile, writeModule } from './module-emit';
 export type { ModuleSource, ModuleSourceChange } from './module-source';
 export { loadModuleSource, stripTransactionWrapper } from './module-source';
 export type { PartitionedPackageRows, PartitionExportRowsResult } from './partition';

pgpm/transform/src/module-emit.ts

@@ -9,7 +9,15 @@
  * `pgpm import` all emit byte-identical module layouts instead of each
  * carrying its own copy of the writer.
  */
-import { PgpmRow, SqlWriteOptions, writePgpmFiles, writePgpmPlan } from '@pgpmjs/ast';
+import {
+  parseAuthor,
+  parsePlanFile,
+  PgpmRow,
+  SqlWriteOptions,
+  writePgpmFiles,
+  writePgpmPlan,
+  writePlanFile
+} from '@pgpmjs/ast';
 import * as fs from 'fs';
 import * as path from 'path';
 
@@ -71,6 +79,101 @@ export const writeModule = (
   return dir;
 };
 
+/** Result of appending changes into an existing module (see {@link appendModule}). */
+export interface AppendModuleResult {
+  /** The module directory that was appended to. */
+  dir: string;
+  /** Change names that were added to the plan. */
+  added: string[];
+  /** Change names skipped because they already exist in the plan. */
+  skipped: string[];
+  /** Non-fatal notices (skips, dropped dangling dependencies). */
+  warnings: string[];
+}
+
+/**
+ * Append plan-ordered {@link PgpmRow}s into an *existing* module rather than
+ * writing a fresh package. Existing changes, their scripts, and the `.control`
+ * file are left untouched; only the new changes are written (deploy/revert/
+ * verify) and appended to `pgpm.plan` after the current changes.
+ *
+ * Rows whose change name already exists are skipped (never overwritten).
+ * A new change's dependency bracket is filtered to names that resolve within
+ * the plan (existing changes, other appended changes, or `pkg:`-external
+ * references); a dangling internal dependency is dropped with a warning
+ * (plan order still sequences it after the current changes).
+ */
+export const appendModule = (
+  moduleDir: string,
+  rows: PgpmRow[],
+  options: { author?: string } = {}
+): AppendModuleResult => {
+  const planPath = path.join(moduleDir, 'pgpm.plan');
+  if (!fs.existsSync(planPath)) {
+    throw new Error(
+      `No pgpm.plan found at ${planPath}; append mode expects an existing pgpm module directory.`
+    );
+  }
+
+  const parsed = parsePlanFile(planPath);
+  if (!parsed.data) {
+    throw new Error(
+      `Failed to parse ${planPath}: ${parsed.errors
+        .map(e => `line ${e.line}: ${e.message}`)
+        .join('; ')}`
+    );
+  }
+  const plan = parsed.data;
+
+  const existingNames = new Set(plan.changes.map(c => c.name));
+  const incomingNames = new Set(rows.map(r => r.deploy));
+  const appended = new Set<string>();
+  const added: string[] = [];
+  const skipped: string[] = [];
+  const warnings: string[] = [];
+  const newRows: PgpmRow[] = [];
+
+  const { fullName, email } = parseAuthor(options.author || 'constructive');
+  const timestamp = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
+
+  for (const row of rows) {
+    if (existingNames.has(row.deploy) || appended.has(row.deploy)) {
+      skipped.push(row.deploy);
+      warnings.push(`change ${row.deploy} already exists in ${plan.package}; left untouched`);
+      continue;
+    }
+    const deps = (row.deps ?? []).filter(dep => {
+      if (dep.includes(':')) return true; // cross-package (pkg:change) external
+      if (existingNames.has(dep) || incomingNames.has(dep)) return true;
+      warnings.push(`change ${row.deploy}: dropped dependency ${dep} (not present in ${plan.package})`);
+      return false;
+    });
+    plan.changes.push({
+      name: row.deploy,
+      dependencies: deps,
+      timestamp,
+      planner: fullName,
+      email: email || `${fullName}@constructive.io`,
+      comment: `add ${row.name ?? row.deploy}`
+    });
+    newRows.push({ ...row, deps });
+    appended.add(row.deploy);
+    added.push(row.deploy);
+  }
+
+  if (newRows.length) {
+    const opts: SqlWriteOptions = {
+      outdir: path.dirname(moduleDir),
+      name: path.basename(moduleDir),
+      replacer: (str: string) => str
+    };
+    writePgpmFiles(newRows, opts);
+    writePlanFile(planPath, plan);
+  }
+
+  return { dir: moduleDir, added, skipped, warnings };
+};
+
 /**
  * Guard against clobbering: writing into the source directory requires
  * `--write`, as does overwriting any existing package directory.