diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 3ab63a92b..a2bef440d 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -105,7 +105,7 @@ jobs: - batch: graphql packages: 'graphql/query graphql/codegen' - batch: pglite - packages: 'postgres/pglite-adapter postgres/pglite-test examples/pglite-socket' + packages: 'postgres/pglite-adapter postgres/pglite-test examples/pglite-socket examples/pgpm-projections' steps: - name: Download workspace diff --git a/examples/pgpm-projections/README.md b/examples/pgpm-projections/README.md new file mode 100644 index 000000000..3b1224ddd --- /dev/null +++ b/examples/pgpm-projections/README.md @@ -0,0 +1,102 @@ +# pgpm projections + +

+ +

+ +

+ + + + + + +

+ +One PostgreSQL schema, **projected into every shape** — and every shape means +the same thing. + +pgpm normalizes any schema (a `.sql` file, a `pg_dump`, a live database, or a +pgpm module) to an **identity-keyed object set**: the objects and their shapes, +with authoring granularity, naming, partitioning, statement order, and +whitespace all stripped out. Everything downstream is a **projection** of that +one canonical model: + +| Axis | Projections | What changes | What stays the same | +| --- | --- | --- | --- | +| **granularity** | `atomic` · `object` · `consolidated` | how finely a change is split (one `ALTER` per column vs one statement per object vs the whole thing) | the resulting catalog | +| **partition** | one module vs many | which objects live in which package | the combined schema | +| **diff** | `v1 → v2` migration | — | derived, never hand-written | +| **output** | pgpm module · linear `.sql` · bundle | the artifact format | the migration | + +The point: a projection changes the **representation**, never the **meaning**. + +## The source + +[`schema/schema.sql`](./schema/schema.sql) is a small two-schema blog platform +(`blog_app` + `blog_sec`) — ordinary PostgreSQL DDL, nothing pgpm-specific. +[`schema/schema-v2.sql`](./schema/schema-v2.sql) is the next version of it. + +## Run it yourself (CLI) + +```sh +# 1. pgpm-itize the schema into a module (object granularity by default) +pgpm import schema/schema.sql --pkg blog --out ./out + +# 2. Re-dial the SAME module to other granularities (siblings blog-) +cd out/blog +pgpm transform --granularity atomic +pgpm transform --granularity consolidated + +# 3. Partition the source into app + security modules +cd ../.. +pgpm import schema/schema.sql --pkg blog-part --partition schema/partition.json --out ./out + +# 4. Derive the v1 -> v2 migration, and project it two ways at once +pgpm import schema/schema-v2.sql --pkg blog-v2 --out ./out +pgpm diff ./out/blog ./out/blog-v2 \ + --emit-migration ./out --pkg blog-migration \ + --emit-sql ./out/migration.sql + +# 5. Prove a projection is lossless against a real catalog (needs Postgres) +cd out/blog +pgpm transform --granularity atomic --check # deploys both, diffs the catalogs +``` + +## What the test proves + +```sh +cd examples/pgpm-projections +pnpm test # jest, no Postgres service required +``` + +The suite ([`__tests__/projections.e2e.test.ts`](./__tests__/projections.e2e.test.ts)) +runs the real CLI and then compares the emitted artifacts **semantically** — no +database needed, because equivalence is checked at the identity-keyed model: + +- **granularity-invariant** — `object` and `consolidated` normalize to the same + schema (empty diff). +- **partition-invariant** — the `blog-core` + `blog-security` modules recombine + to the same schema as the single module (empty diff). +- **diff is exact** — `schema.sql → schema-v2.sql` derives precisely the real + changes (two new tables, a new policy, a changed `posts` table, a changed + function body) and nothing is guessed. +- **output is composable** — one `pgpm diff` run emits a pgpm module *and* a + linear `.sql` file, in dependency order, with no `CREATE OR REPLACE`. + +### Catalog-level equivalence + +Every granularity — including `atomic` — deploys to the **identical Postgres +catalog**; that is the guarantee `--check` / `--verify` enforce, and it is +covered against live Postgres by the engine's own suites +(`pgpm/cli` `transform-e2e` "dial parity" and `diff-e2e`). This example is +deliberately database-free and asserts the model-level invariants above. + +> Note: `atomic` authorship emits standalone `ALTER TABLE … ADD CONSTRAINT` +> statements. Because Postgres names such constraints at deploy time, the +> semantic normalizer does not yet fold every standalone constraint back into +> its owning table object, so `atomic` is guaranteed **catalog-equivalent** +> rather than AST-identical to `object`/`consolidated`. Tightening that +> normalization is tracked in `constructive-planning`. + +It's a normal workspace package, so `pnpm install` at the repo root wires it up. diff --git a/examples/pgpm-projections/__tests__/projections.e2e.test.ts b/examples/pgpm-projections/__tests__/projections.e2e.test.ts new file mode 100644 index 000000000..5a15fa321 --- /dev/null +++ b/examples/pgpm-projections/__tests__/projections.e2e.test.ts @@ -0,0 +1,148 @@ +// End-to-end example: pgpm PROJECTIONS. +// +// One SQL schema (schema/schema.sql) is the source of truth. We normalize it +// to an identity-keyed object set and then *project* it into many shapes: +// +// granularity object | consolidated (atomic too — see the note below) +// partition one module vs app + security modules +// diff schema.sql -> schema-v2.sql as a generated migration +// output pgpm module | linear .sql +// +// The headline guarantee: a projection changes the *representation*, never the +// *meaning*. We prove it WITHOUT a database by normalizing each projection back +// to its object set and asserting the diff is empty — authoring granularity, +// naming, partitioning, ordering, and whitespace all wash out. +// +// (Deploy-level catalog equivalence for every projection, including `atomic`, +// is proven against live Postgres by the engine's own suites — pgpm/cli +// transform-e2e "dial parity" and diff-e2e. See README.) +// +// This suite is intentionally database-free: it only runs the CLI and compares +// the emitted artifacts semantically. +import { loadDiffSideFromDisk } from '@pgpmjs/diff'; +import { diffChangeSets, loadModule } from '@pgpmjs/transform'; +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +const CLI = path.join(__dirname, '..', '..', '..', 'pgpm', 'cli', 'dist', 'index.js'); +const SCHEMA_DIR = path.join(__dirname, '..', 'schema'); + +const pgpm = (work: string, args: string[]): void => { + execFileSync('node', [CLI, ...args], { cwd: work, stdio: 'pipe' }); +}; + +type ChangeSet = ReturnType['changes']; +const changesOf = (dir: string): ChangeSet => loadDiffSideFromDisk(dir).changes; + +describe('pgpm projections: one schema, every shape, one meaning', () => { + let work: string; + + beforeAll(async () => { + await loadModule(); + work = fs.mkdtempSync(path.join(os.tmpdir(), 'pgpm-projections-')); + + // Source schema -> pgpm module at the default (object) granularity. + pgpm(work, ['import', path.join(SCHEMA_DIR, 'schema.sql'), '--pkg', 'blog', '--out', work]); + + // Re-dial the same module to other granularities (siblings blog-). + pgpm(work, ['transform', '--granularity', 'atomic', '--cwd', path.join(work, 'blog')]); + pgpm(work, ['transform', '--granularity', 'consolidated', '--cwd', path.join(work, 'blog')]); + + // Partition the same source into app + security modules. + pgpm(work, [ + 'import', path.join(SCHEMA_DIR, 'schema.sql'), + '--pkg', 'blog-part', + '--partition', path.join(SCHEMA_DIR, 'partition.json'), + '--out', work + ]); + + // The next schema version, and the generated migration between them. + pgpm(work, ['import', path.join(SCHEMA_DIR, 'schema-v2.sql'), '--pkg', 'blog-v2', '--out', work]); + pgpm(work, [ + 'diff', path.join(work, 'blog'), path.join(work, 'blog-v2'), + '--emit-migration', work, '--pkg', 'blog-migration', + '--emit-sql', path.join(work, 'migration.sql') + ]); + }); + + afterAll(() => { + fs.rmSync(work, { recursive: true, force: true }); + }); + + it('imports the schema into a deployable pgpm module', () => { + const moduleDir = path.join(work, 'blog'); + expect(fs.existsSync(path.join(moduleDir, 'pgpm.plan'))).toBe(true); + expect(fs.existsSync(path.join(moduleDir, 'blog.control'))).toBe(true); + expect(changesOf(moduleDir).length).toBeGreaterThan(0); + }); + + it('is granularity-invariant: object and consolidated normalize to the same schema', () => { + const object = changesOf(path.join(work, 'blog')); + const consolidated = changesOf(path.join(work, 'blog-consolidated')); + expect(diffChangeSets(object, consolidated).identical).toBe(true); + }); + + it('is partition-invariant: app + security modules recombine to the same schema', () => { + const object = changesOf(path.join(work, 'blog')); + const partitioned = [ + ...changesOf(path.join(work, 'blog-core')), + ...changesOf(path.join(work, 'blog-security')) + ]; + expect(diffChangeSets(object, partitioned).identical).toBe(true); + }); + + it('emits the atomic projection as a deployable module covering the same schemas', () => { + // atomic explodes objects into per-column / per-constraint statements. It + // deploys to the same catalog as the other granularities (proven against + // live Postgres by pgpm/cli transform-e2e "dial parity"); here we assert it + // is a well-formed module spanning the same schemas. + const atomicDir = path.join(work, 'blog-atomic'); + expect(fs.existsSync(path.join(atomicDir, 'pgpm.plan'))).toBe(true); + const atomicChanges = changesOf(atomicDir); + expect(atomicChanges.length).toBeGreaterThan(0); + + const schemasOf = (changes: ChangeSet): string[] => + [...new Set(changes.map(c => c.name.split('/')[1]).filter(Boolean))].sort(); + expect(schemasOf(atomicChanges)).toEqual(schemasOf(changesOf(path.join(work, 'blog')))); + }); + + it('derives the v1 -> v2 migration: exactly the real changes, nothing guessed', () => { + const result = diffChangeSets( + changesOf(path.join(work, 'blog')), + changesOf(path.join(work, 'blog-v2')) + ); + expect(result.identical).toBe(false); + const byDelta = (delta: string): string[] => + result.objects.filter(o => o.delta === delta).map(o => o.path).sort(); + + // two new tables, one new policy + expect(byDelta('added')).toEqual([ + 'schemas/blog_app/tables/post_tags/table', + 'schemas/blog_app/tables/tags/table', + 'schemas/blog_sec/tables/audit_log/policies/audit_log_insert/policy' + ]); + // changed posts table + changed function body + expect(byDelta('modified')).toEqual([ + 'schemas/blog_app/procedures/published_post_count/procedure', + 'schemas/blog_app/tables/posts/table' + ]); + expect(byDelta('removed')).toEqual([]); + }); + + it('projects the migration into a pgpm module AND a linear SQL file at once', () => { + // module projection + expect(fs.existsSync(path.join(work, 'blog-migration', 'pgpm.plan'))).toBe(true); + + // linear SQL projection: statements in dependency order, no CREATE OR REPLACE + const sql = fs.readFileSync(path.join(work, 'migration.sql'), 'utf-8'); + expect(sql).toMatch(/CREATE TABLE blog_app\.tags/); + expect(sql).toMatch(/CREATE TABLE blog_app\.post_tags/); + expect(sql).toMatch(/ADD COLUMN slug/); + expect(sql).toMatch(/DROP COLUMN word_count/); + expect(sql).toMatch(/DROP FUNCTION blog_app\.published_post_count/); + expect(sql).toMatch(/CREATE POLICY audit_log_insert/); + expect(sql).not.toMatch(/CREATE OR REPLACE/i); + }); +}); diff --git a/examples/pgpm-projections/jest.config.js b/examples/pgpm-projections/jest.config.js new file mode 100644 index 000000000..eecd07335 --- /dev/null +++ b/examples/pgpm-projections/jest.config.js @@ -0,0 +1,18 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json' + } + ] + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'] +}; diff --git a/examples/pgpm-projections/package.json b/examples/pgpm-projections/package.json new file mode 100644 index 000000000..24f47c50f --- /dev/null +++ b/examples/pgpm-projections/package.json @@ -0,0 +1,31 @@ +{ + "name": "@constructive-io/examples-pgpm-projections", + "version": "0.0.1", + "private": true, + "description": "Example: pgpm projections — one schema, projected into every granularity, naming, partition, and output shape, all proven to deploy to the identical Postgres catalog.", + "homepage": "https://github.com/constructive-io/constructive", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "keywords": [ + "postgres", + "postgresql", + "pgpm", + "migrations", + "projections", + "schema-diff", + "example" + ], + "scripts": { + "lint": "eslint . --fix", + "test": "jest --passWithNoTests", + "test:watch": "jest --watch" + }, + "devDependencies": { + "@pgpmjs/diff": "workspace:^", + "@pgpmjs/transform": "workspace:^", + "pgpm": "workspace:^" + } +} diff --git a/examples/pgpm-projections/schema/partition.json b/examples/pgpm-projections/schema/partition.json new file mode 100644 index 000000000..d457f2263 --- /dev/null +++ b/examples/pgpm-projections/schema/partition.json @@ -0,0 +1,9 @@ +{ + "defaultPackage": "blog-core", + "rules": [ + { + "package": "blog-security", + "select": [{ "schema": "blog_sec" }] + } + ] +} diff --git a/examples/pgpm-projections/schema/schema-v2.sql b/examples/pgpm-projections/schema/schema-v2.sql new file mode 100644 index 000000000..e8489c755 --- /dev/null +++ b/examples/pgpm-projections/schema/schema-v2.sql @@ -0,0 +1,64 @@ +-- The next version of the same platform. Diffing schema.sql -> schema-v2.sql +-- yields exactly these changes: a new column (posts.slug), a dropped column +-- (posts.word_count), a changed function body, two new tables (tags, +-- post_tags), and a new policy on the audit log. `pgpm diff` derives the +-- migration from the two states; nothing is hand-written. + +CREATE SCHEMA blog_app; +CREATE SCHEMA blog_sec; + +CREATE TABLE blog_app.authors ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + email text NOT NULL UNIQUE, + display_name text NOT NULL +); + +CREATE TABLE blog_app.posts ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + author_id bigint NOT NULL REFERENCES blog_app.authors (id), + title text NOT NULL, + body text NOT NULL, + slug text, + published boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT title_non_empty CHECK (length(title) > 0) +); + +CREATE INDEX posts_author_idx ON blog_app.posts (author_id); + +-- Changed body: now counts all posts, not only published ones. +CREATE FUNCTION blog_app.published_post_count() RETURNS bigint + LANGUAGE sql STABLE + AS $$ SELECT count(*) FROM blog_app.posts $$; + +-- New: a tagging model. +CREATE TABLE blog_app.tags ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name text NOT NULL UNIQUE +); + +CREATE TABLE blog_app.post_tags ( + post_id bigint NOT NULL REFERENCES blog_app.posts (id), + tag_id bigint NOT NULL REFERENCES blog_app.tags (id), + PRIMARY KEY (post_id, tag_id) +); + +CREATE TABLE blog_sec.audit_log ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + post_id bigint NOT NULL REFERENCES blog_app.posts (id), + action text NOT NULL, + at timestamptz NOT NULL DEFAULT now() +); + +ALTER TABLE blog_sec.audit_log ENABLE ROW LEVEL SECURITY; + +CREATE POLICY audit_log_read ON blog_sec.audit_log + FOR SELECT USING (true); + +-- New policy. +CREATE POLICY audit_log_insert ON blog_sec.audit_log + FOR INSERT WITH CHECK (true); + +GRANT SELECT ON blog_app.posts TO PUBLIC; + +COMMENT ON TABLE blog_app.posts IS 'Blog posts, one per author.'; diff --git a/examples/pgpm-projections/schema/schema.sql b/examples/pgpm-projections/schema/schema.sql new file mode 100644 index 000000000..a7dc4890b --- /dev/null +++ b/examples/pgpm-projections/schema/schema.sql @@ -0,0 +1,49 @@ +-- A small but realistic two-schema blog platform, authored as one flat SQL +-- file (as if hand-written, or produced by `pg_dump --schema-only`). This is +-- the SOURCE we project into every shape. Nothing here is pgpm-specific: it is +-- just ordinary PostgreSQL DDL. + +CREATE SCHEMA blog_app; +CREATE SCHEMA blog_sec; + +-- Authors and posts, with a foreign key, a check constraint, and an index. +CREATE TABLE blog_app.authors ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + email text NOT NULL UNIQUE, + display_name text NOT NULL +); + +CREATE TABLE blog_app.posts ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + author_id bigint NOT NULL REFERENCES blog_app.authors (id), + title text NOT NULL, + body text NOT NULL, + word_count int NOT NULL DEFAULT 0, + published boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT title_non_empty CHECK (length(title) > 0) +); + +CREATE INDEX posts_author_idx ON blog_app.posts (author_id); + +-- A read model exposed as a function. +CREATE FUNCTION blog_app.published_post_count() RETURNS bigint + LANGUAGE sql STABLE + AS $$ SELECT count(*) FROM blog_app.posts WHERE published $$; + +-- Security schema: an append-only audit log, protected by row-level security. +CREATE TABLE blog_sec.audit_log ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + post_id bigint NOT NULL REFERENCES blog_app.posts (id), + action text NOT NULL, + at timestamptz NOT NULL DEFAULT now() +); + +ALTER TABLE blog_sec.audit_log ENABLE ROW LEVEL SECURITY; + +CREATE POLICY audit_log_read ON blog_sec.audit_log + FOR SELECT USING (true); + +GRANT SELECT ON blog_app.posts TO PUBLIC; + +COMMENT ON TABLE blog_app.posts IS 'Blog posts, one per author.'; diff --git a/examples/pgpm-projections/tsconfig.json b/examples/pgpm-projections/tsconfig.json new file mode 100644 index 000000000..725eb2c62 --- /dev/null +++ b/examples/pgpm-projections/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "." + }, + "include": ["__tests__/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0573c5cd9..1cf9b38d9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -314,6 +314,18 @@ importers: specifier: workspace:^ version: link:../../postgres/pg-cache/dist + examples/pgpm-projections: + devDependencies: + '@pgpmjs/diff': + specifier: workspace:^ + version: link:../../pgpm/diff/dist + '@pgpmjs/transform': + specifier: workspace:^ + version: link:../../pgpm/transform/dist + pgpm: + specifier: workspace:^ + version: link:../../pgpm/cli/dist + graphile/graphile-bucket-provisioner-plugin: dependencies: '@constructive-io/bucket-provisioner':