From 4bfb5aa1fbaaf1e9cf53f7f7d27584f503a0cd8a Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 1 Aug 2026 02:41:19 +0000 Subject: [PATCH] =?UTF-8?q?feat(pgpm):=20pgpm=20diff=20--append-module=20?= =?UTF-8?q?=E2=80=94=20append=20the=20delta=20into=20an=20existing=20modul?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an append mode to pgpm diff: instead of writing a fresh package, emit the semantic delta into an existing pgpm module's plan. Existing changes, their deploy/revert/verify scripts, and the .control file are left byte-for-byte untouched; only the new changes are written and appended to pgpm.plan after the current changes. - appendModule() in @pgpmjs/transform: parses the existing plan, appends new Change entries + writes only the new change scripts (no rmSync of the tree), skips name collisions (never overwrites), and filters dangling internal deps out of the bracket with a warning (plan order still sequences them last) - pgpm diff --append-module : standalone (rejected with --emit-*/--verify, whose semantics assume the fresh-delta module) - e2e: append v1+delta into an existing module, assert existing scripts are untouched and the appended module fresh-deploys to the v2 catalog --- pgpm/cli/__tests__/diff-e2e.test.ts | 41 +++++++++++ pgpm/cli/src/commands/diff.ts | 31 ++++++++ pgpm/transform/src/index.ts | 4 +- pgpm/transform/src/module-emit.ts | 105 +++++++++++++++++++++++++++- 4 files changed, 178 insertions(+), 3 deletions(-) diff --git a/pgpm/cli/__tests__/diff-e2e.test.ts b/pgpm/cli/__tests__/diff-e2e.test.ts index 7bde82faf..ea1ab7028 100644 --- a/pgpm/cli/__tests__/diff-e2e.test.ts +++ b/pgpm/cli/__tests__/diff-e2e.test.ts @@ -251,6 +251,47 @@ describe('pgpm diff e2e', () => { expect(bundle.changes.length).toBeGreaterThan(0); }); + it('appends the delta into an existing module and fresh-deploys to the v2 catalog', async () => { + // A standalone module with v1's schema (named so pgpm can deploy it), so + // the shared diff-v1 module is untouched. + const appendDir = path.join(wsDir, 'diff-append'); + writeModule(appendDir, 'diff-append', V1); + const originalChangeCount = V1.length; + // existing change scripts must be left byte-for-byte untouched + const sentinel = 'schemas/dfx/tables/products/table'; + const deployBefore = fs.readFileSync(path.join(appendDir, 'deploy', `${sentinel}.sql`), 'utf-8'); + + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm diff diff-append diff-v2 --append-module diff-append + `, + {} + ); + + // existing changes are preserved (names + untouched scripts) and new ones appended + const planAfter = fs.readFileSync(path.join(appendDir, 'pgpm.plan'), 'utf-8'); + for (const c of V1) expect(planAfter).toContain(c.name); + const changeLines = planAfter + .split('\n') + .filter(l => l.trim() && !l.startsWith('%') && !l.startsWith('@')); + expect(changeLines.length).toBeGreaterThan(originalChangeCount); + expect(fs.readFileSync(path.join(appendDir, 'deploy', `${sentinel}.sql`), 'utf-8')).toBe(deployBefore); + + // the appended module (v1 + delta) fresh-deploys to a v2-equivalent catalog + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands( + ` + cd ${WS}/diff-append + pgpm deploy --database ${testDb.name} --package diff-append --yes + `, + { database: testDb.name } + ); + const snapAppended = await snapshotCatalog(testDb); + const snapV2 = await snapshotCatalog(v2Db); + expect(diffCatalogSnapshots(withoutColumnOrder(snapAppended), withoutColumnOrder(snapV2))).toEqual([]); + }); + it.each(['atomic', 'object', 'consolidated'])( 'emits a %s-granularity migration that reaches the same v2 catalog (dial parity)', async granularity => { diff --git a/pgpm/cli/src/commands/diff.ts b/pgpm/cli/src/commands/diff.ts index 21a51e42e..af8c777b0 100644 --- a/pgpm/cli/src/commands/diff.ts +++ b/pgpm/cli/src/commands/diff.ts @@ -7,6 +7,7 @@ import { } from '@pgpmjs/diff'; import { Logger } from '@pgpmjs/logger'; import { + appendModule, diffChangeSets, EXPORT_GRANULARITIES, ExportGranularity, @@ -58,6 +59,10 @@ Options: verify per change, spec-derived paths, graph-derived requires) into / --emit-module Alias of --emit-migration + --append-module Append the delta into an EXISTING pgpm module at + (new changes only; existing changes, scripts, + and .control are left untouched). Standalone: not + combinable with --emit-*/--verify. --emit-sql Also project the delta to a single consolidated SQL file (deparsed in plan order); - writes to stdout --emit-bundle Also project the delta to a content-addressed @@ -238,6 +243,10 @@ export default async ( const emitModuleDir = typeof emitModuleRaw === 'string' && emitModuleRaw ? path.resolve(cwd, emitModuleRaw) : undefined; + const appendModuleRaw = argv['append-module'] ?? argv.appendModule; + const appendModuleDir = typeof appendModuleRaw === 'string' && appendModuleRaw + ? path.resolve(cwd, appendModuleRaw) + : undefined; const emitSqlRaw = argv['emit-sql'] ?? argv.emitSql; const emitSql = typeof emitSqlRaw === 'string' && emitSqlRaw ? (emitSqlRaw === STDOUT_TARGET ? STDOUT_TARGET : path.resolve(cwd, emitSqlRaw)) @@ -249,6 +258,12 @@ export default async ( const sqlToStdout = emitSql === STDOUT_TARGET; const pkgName = (argv.pkg as string) || 'diff-migration'; + if (appendModuleDir && (emitModuleDir || emitSql || emitBundle || verify)) { + await cliExitWithError( + '--append-module is standalone; it cannot be combined with --emit-migration/--emit-module/--emit-sql/--emit-bundle/--verify.' + ); + } + await loadModule(); let sideA: DiffSide; @@ -287,6 +302,22 @@ export default async ( if (!sqlToStdout) printSummary(result, sideA.label, sideB.label); } + if (appendModuleDir) { + const rows = deltaChangesToRows(result.changes); + if (rows.length === 0) { + log.info('no migration changes to append (sides are identical).'); + } else { + const appended = appendModule(appendModuleDir, rows); + for (const w of appended.warnings) console.warn(`diff: ${w}`); + log.success( + `appended ${appended.added.length} change(s) to ${appended.dir}` + + (appended.skipped.length ? ` (${appended.skipped.length} skipped)` : '') + ); + } + prompter.close(); + return argv; + } + let migrationDir: string | undefined; const needModule = Boolean(emitModuleDir || emitSql || emitBundle) || (verify && !result.identical); if (needModule) { diff --git a/pgpm/transform/src/index.ts b/pgpm/transform/src/index.ts index a5549f918..ebd7c58e6 100644 --- a/pgpm/transform/src/index.ts +++ b/pgpm/transform/src/index.ts @@ -41,8 +41,8 @@ export { defaultChangeName, restructureChanges, } from './granularity-driver'; -export type { PgpmModuleModel } from './module-emit'; -export { checkOverwrite, writeControlFile, writeModule } from './module-emit'; +export type { AppendModuleResult, PgpmModuleModel } from './module-emit'; +export { appendModule, checkOverwrite, writeControlFile, writeModule } from './module-emit'; export type { ModuleSource, ModuleSourceChange } from './module-source'; export { loadModuleSource, stripTransactionWrapper } from './module-source'; export type { PartitionedPackageRows, PartitionExportRowsResult } from './partition'; diff --git a/pgpm/transform/src/module-emit.ts b/pgpm/transform/src/module-emit.ts index 60cf67910..85798bc21 100644 --- a/pgpm/transform/src/module-emit.ts +++ b/pgpm/transform/src/module-emit.ts @@ -9,7 +9,15 @@ * `pgpm import` all emit byte-identical module layouts instead of each * carrying its own copy of the writer. */ -import { PgpmRow, SqlWriteOptions, writePgpmFiles, writePgpmPlan } from '@pgpmjs/ast'; +import { + parseAuthor, + parsePlanFile, + PgpmRow, + SqlWriteOptions, + writePgpmFiles, + writePgpmPlan, + writePlanFile +} from '@pgpmjs/ast'; import * as fs from 'fs'; import * as path from 'path'; @@ -71,6 +79,101 @@ export const writeModule = ( return dir; }; +/** Result of appending changes into an existing module (see {@link appendModule}). */ +export interface AppendModuleResult { + /** The module directory that was appended to. */ + dir: string; + /** Change names that were added to the plan. */ + added: string[]; + /** Change names skipped because they already exist in the plan. */ + skipped: string[]; + /** Non-fatal notices (skips, dropped dangling dependencies). */ + warnings: string[]; +} + +/** + * Append plan-ordered {@link PgpmRow}s into an *existing* module rather than + * writing a fresh package. Existing changes, their scripts, and the `.control` + * file are left untouched; only the new changes are written (deploy/revert/ + * verify) and appended to `pgpm.plan` after the current changes. + * + * Rows whose change name already exists are skipped (never overwritten). + * A new change's dependency bracket is filtered to names that resolve within + * the plan (existing changes, other appended changes, or `pkg:`-external + * references); a dangling internal dependency is dropped with a warning + * (plan order still sequences it after the current changes). + */ +export const appendModule = ( + moduleDir: string, + rows: PgpmRow[], + options: { author?: string } = {} +): AppendModuleResult => { + const planPath = path.join(moduleDir, 'pgpm.plan'); + if (!fs.existsSync(planPath)) { + throw new Error( + `No pgpm.plan found at ${planPath}; append mode expects an existing pgpm module directory.` + ); + } + + const parsed = parsePlanFile(planPath); + if (!parsed.data) { + throw new Error( + `Failed to parse ${planPath}: ${parsed.errors + .map(e => `line ${e.line}: ${e.message}`) + .join('; ')}` + ); + } + const plan = parsed.data; + + const existingNames = new Set(plan.changes.map(c => c.name)); + const incomingNames = new Set(rows.map(r => r.deploy)); + const appended = new Set(); + const added: string[] = []; + const skipped: string[] = []; + const warnings: string[] = []; + const newRows: PgpmRow[] = []; + + const { fullName, email } = parseAuthor(options.author || 'constructive'); + const timestamp = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + + for (const row of rows) { + if (existingNames.has(row.deploy) || appended.has(row.deploy)) { + skipped.push(row.deploy); + warnings.push(`change ${row.deploy} already exists in ${plan.package}; left untouched`); + continue; + } + const deps = (row.deps ?? []).filter(dep => { + if (dep.includes(':')) return true; // cross-package (pkg:change) external + if (existingNames.has(dep) || incomingNames.has(dep)) return true; + warnings.push(`change ${row.deploy}: dropped dependency ${dep} (not present in ${plan.package})`); + return false; + }); + plan.changes.push({ + name: row.deploy, + dependencies: deps, + timestamp, + planner: fullName, + email: email || `${fullName}@constructive.io`, + comment: `add ${row.name ?? row.deploy}` + }); + newRows.push({ ...row, deps }); + appended.add(row.deploy); + added.push(row.deploy); + } + + if (newRows.length) { + const opts: SqlWriteOptions = { + outdir: path.dirname(moduleDir), + name: path.basename(moduleDir), + replacer: (str: string) => str + }; + writePgpmFiles(newRows, opts); + writePlanFile(planPath, plan); + } + + return { dir: moduleDir, added, skipped, warnings }; +}; + /** * Guard against clobbering: writing into the source directory requires * `--write`, as does overwriting any existing package directory.