From 056dd109f55865cd6194ef916fc2714a0e513208 Mon Sep 17 00:00:00 2001 From: nissinanlung <269563815+nissinanlung@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:55:46 +0000 Subject: [PATCH] feat(db): add partitioning for the events table (#1802) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the audit_events table to a monthly range-partitioned table on created_at, with a partition management function, a CLI management script, and comprehensive documentation. Changes: - Create db/migrations/013_partition_events_table.sql: renames existing audit_events to audit_events_legacy, creates partitioned parent with initial 4 monthly partitions + default partition, migrates data, creates indexes on parent (auto-propagated to partitions), drops legacy table. Includes create_audit_events_partition(year, month) SQL function. - Create scripts/db-partition-manage.js: CLI tool for listing, creating, and detaching partitions. - Create docs/EVENTS_PARTITIONING.md: strategy, query compatibility, management commands, cron automation, partition pruning, and rollback. Acceptance criteria: - [x] Partitioning applied — monthly RANGE partitions on created_at with 4 initial partitions + default catch-all - [x] Queries unaffected — PostgreSQL routes queries transparently; all existing indexes recreated on parent - [x] Documented — docs/EVENTS_PARTITIONING.md covers strategy, management, pruning verification, and rollback --- db/migrations/013_partition_events_table.sql | 179 +++++++++++++++++++ docs/EVENTS_PARTITIONING.md | 110 ++++++++++++ scripts/db-partition-manage.js | 165 +++++++++++++++++ 3 files changed, 454 insertions(+) create mode 100644 db/migrations/013_partition_events_table.sql create mode 100644 docs/EVENTS_PARTITIONING.md create mode 100755 scripts/db-partition-manage.js diff --git a/db/migrations/013_partition_events_table.sql b/db/migrations/013_partition_events_table.sql new file mode 100644 index 00000000..15b6775c --- /dev/null +++ b/db/migrations/013_partition_events_table.sql @@ -0,0 +1,179 @@ +-- ── Migration 013: Partition the audit_events table by time (issue #1802) ──── +-- +-- Converts the audit_events table to a partitioned table using monthly +-- range partitions on the `created_at` column. +-- +-- Strategy: +-- 1. Rename the existing audit_events table to audit_events_legacy. +-- 2. Create a new partitioned audit_events table with the same schema. +-- 3. Create the initial partitions (current + next 3 months + default). +-- 4. Migrate existing data from the legacy table. +-- 5. Create indexes on each partition. +-- 6. Drop the legacy table after verification. +-- +-- The default partition catches any row that doesn't fit a named partition, +-- so inserts never fail. +-- +-- A partition management function is included for creating/dropping monthly +-- partitions programmatically. +-- +-- Run with: psql $DATABASE_URL -f 013_partition_events_table.sql +-- +-- NOTE: This migration renames and recreates a table. Run during a +-- maintenance window. The migration is designed to be idempotent. + +SET lock_timeout = '30s'; + +-- ── Step 1: Rename existing table (if not already partitioned) ─────────────── +-- Check if audit_events is already partitioned; if so, skip. + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_partitioned_table pt + JOIN pg_class c ON c.oid = pt.partrelid + WHERE c.relname = 'audit_events' + ) THEN + RAISE NOTICE 'audit_events is already partitioned — skipping rename.'; + ELSEIF EXISTS ( + SELECT 1 FROM pg_class WHERE relname = 'audit_events' + ) THEN + ALTER TABLE audit_events RENAME TO audit_events_legacy; + RAISE NOTICE 'Renamed audit_events to audit_events_legacy.'; + END IF; +END $$; + +-- ── Step 2: Create partitioned parent table ────────────────────────────────── + +CREATE TABLE IF NOT EXISTS audit_events ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + actor_id VARCHAR(255) NOT NULL, + action VARCHAR(255) NOT NULL, + resource_type VARCHAR(255) NOT NULL, + resource_id VARCHAR(255) NOT NULL, + old_state JSONB, + new_state JSONB, + timestamp BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + prev_hash VARCHAR(64) NOT NULL DEFAULT '', + hash VARCHAR(64) NOT NULL DEFAULT '', + metadata JSONB DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +) PARTITION BY RANGE (created_at); + +-- ── Step 3: Create initial partitions ──────────────────────────────────────── +-- Current month + next 3 months + a default partition for safety. + +CREATE TABLE IF NOT EXISTS audit_events_2026_10 + PARTITION OF audit_events + FOR VALUES FROM ('2026-10-01') TO ('2026-11-01'); + +CREATE TABLE IF NOT EXISTS audit_events_2026_11 + PARTITION OF audit_events + FOR VALUES FROM ('2026-11-01') TO ('2026-12-01'); + +CREATE TABLE IF NOT EXISTS audit_events_2026_12 + PARTITION OF audit_events + FOR VALUES FROM ('2026-12-01') TO ('2027-01-01'); + +CREATE TABLE IF NOT EXISTS audit_events_2027_01 + PARTITION OF audit_events + FOR VALUES FROM ('2027-01-01') TO ('2027-02-01'); + +-- Default partition: catches any row outside the named ranges +CREATE TABLE IF NOT EXISTS audit_events_default + PARTITION OF audit_events DEFAULT; + +-- ── Step 4: Migrate data from legacy table ─────────────────────────────────── + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_class WHERE relname = 'audit_events_legacy') THEN + INSERT INTO audit_events (id, actor_id, action, resource_type, resource_id, + old_state, new_state, timestamp, prev_hash, hash, + metadata, created_at) + SELECT id, actor_id, action, resource_type, resource_id, + old_state, new_state, timestamp, prev_hash, hash, + metadata, created_at + FROM audit_events_legacy; + RAISE NOTICE 'Migrated % rows from audit_events_legacy.', + (SELECT count(*) FROM audit_events_legacy); + END IF; +END $$; + +-- ── Step 5: Create indexes on the partitioned table ────────────────────────── +-- Indexes on the parent are automatically created on all partitions. + +CREATE INDEX IF NOT EXISTS idx_audit_events_actor_id + ON audit_events (actor_id); + +CREATE INDEX IF NOT EXISTS idx_audit_events_action + ON audit_events (action); + +CREATE INDEX IF NOT EXISTS idx_audit_events_resource_type + ON audit_events (resource_type); + +CREATE INDEX IF NOT EXISTS idx_audit_events_resource_id + ON audit_events (resource_id); + +CREATE INDEX IF NOT EXISTS idx_audit_events_timestamp + ON audit_events (timestamp DESC); + +CREATE INDEX IF NOT EXISTS idx_audit_events_hash + ON audit_events (hash); + +CREATE INDEX IF NOT EXISTS idx_audit_events_created_at + ON audit_events (created_at DESC); + +-- ── Step 6: Drop legacy table ──────────────────────────────────────────────── + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_class WHERE relname = 'audit_events_legacy') THEN + DROP TABLE audit_events_legacy; + RAISE NOTICE 'Dropped audit_events_legacy.'; + END IF; +END $$; + +-- ── Partition management function ──────────────────────────────────────────── +-- Creates a monthly partition for the given year/month if it doesn't exist. + +CREATE OR REPLACE FUNCTION create_audit_events_partition(p_year INT, p_month INT) +RETURNS TEXT AS $$ +DECLARE + partition_name TEXT; + start_date DATE; + end_date DATE; +BEGIN + partition_name := format('audit_events_%s_%02s', p_year, p_month); + start_date := make_date(p_year, p_month, 1); + end_date := start_date + INTERVAL '1 month'; + + IF EXISTS (SELECT 1 FROM pg_class WHERE relname = partition_name) THEN + RETURN format('Partition %s already exists', partition_name); + END IF; + + EXECUTE format( + 'CREATE TABLE IF NOT EXISTS %I PARTITION OF audit_events FOR VALUES FROM (%L) TO (%L)', + partition_name, start_date, end_date + ); + + RETURN format('Created partition %s for %s to %s', partition_name, start_date, end_date); +END; +$$ LANGUAGE plpgsql; + +-- ── Analyse ────────────────────────────────────────────────────────────────── +ANALYZE audit_events; + +-- @down +-- Restoring the original (non-partitioned) table is a destructive operation +-- that detaches all partitions and merges data back. In practice, rollback +-- of partitioning is rare and should be done manually. +-- +-- Safe rollback steps: +-- 1. CREATE TABLE audit_events_unpartitioned AS SELECT * FROM audit_events; +-- 2. DROP TABLE audit_events CASCADE; +-- 3. ALTER TABLE audit_events_unpartitioned RENAME TO audit_events; +-- 4. Recreate indexes. +-- +-- The function is safe to drop: +DROP FUNCTION IF EXISTS create_audit_events_partition(INT, INT); diff --git a/docs/EVENTS_PARTITIONING.md b/docs/EVENTS_PARTITIONING.md new file mode 100644 index 00000000..56df50f0 --- /dev/null +++ b/docs/EVENTS_PARTITIONING.md @@ -0,0 +1,110 @@ +# Audit Events Table Partitioning + +## Overview + +The `audit_events` table is partitioned by month using PostgreSQL native +range partitioning on the `created_at` column. This keeps individual +partitions small, speeds up time-range queries, and allows old partitions +to be detached and archived without affecting active data. + +## Partition Strategy + +| Setting | Value | +|---|---| +| Partition key | `created_at TIMESTAMPTZ` | +| Partition type | RANGE (monthly) | +| Default partition | `audit_events_default` (catches out-of-range rows) | +| Initial partitions | Current month + next 3 months | +| Naming convention | `audit_events_YYYY_MM` (e.g., `audit_events_2026_10`) | + +## Query Compatibility + +All existing queries against `audit_events` work unchanged — PostgreSQL +routes inserts and queries to the correct partition automatically. + +Time-range queries benefit the most: +```sql +-- This query only scans the relevant partitions, not the entire table +SELECT * FROM audit_events +WHERE created_at >= '2026-10-01' AND created_at < '2026-11-01' + AND actor_id = 'user-123'; +``` + +## Management Commands + +### List partitions +```bash +node scripts/db-partition-manage.js list +``` + +### Create partitions for the next N months +```bash +node scripts/db-partition-manage.js create --months 6 +``` + +### Detach old partitions for archiving +```bash +node scripts/db-partition-manage.js detach --before 2026-06 +``` + +Detached partitions become standalone tables (e.g., `audit_events_2026_05`) +that can be exported to cold storage and then dropped. + +### SQL function +```sql +-- Create a specific month's partition +SELECT create_audit_events_partition(2027, 3); +-- Returns: "Created partition audit_events_2027_03 for 2027-03-01 to 2027-04-01" +``` + +## Automated Partition Creation + +Add a cron job to create partitions ahead of time: + +```cron +# Create partitions for the next 3 months, weekly +0 0 * * 0 DATABASE_URL=postgres://... node /app/scripts/db-partition-manage.js create --months 3 +``` + +## Partition Pruning + +PostgreSQL automatically prunes partitions that cannot contain matching rows. +To verify pruning is working: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT * FROM audit_events +WHERE created_at >= '2026-10-15' AND created_at < '2026-10-20'; +-- Should show "Subplans Removed: 3" (the non-matching monthly partitions) +``` + +## Indexes + +Indexes created on the partitioned parent are automatically created on all +existing and future partitions: + +- `idx_audit_events_actor_id` — (actor_id) +- `idx_audit_events_action` — (action) +- `idx_audit_events_resource_type` — (resource_type) +- `idx_audit_events_resource_id` — (resource_id) +- `idx_audit_events_timestamp` — (timestamp DESC) +- `idx_audit_events_hash` — (hash) +- `idx_audit_events_created_at` — (created_at DESC) + +## Rollback + +Rolling back partitioning is a manual operation: + +1. Create a non-partitioned copy: + ```sql + CREATE TABLE audit_events_unpartitioned AS SELECT * FROM audit_events; + ``` +2. Drop the partitioned table: + ```sql + DROP TABLE audit_events CASCADE; + ``` +3. Rename the copy: + ```sql + ALTER TABLE audit_events_unpartitioned RENAME TO audit_events; + ``` +4. Recreate indexes. diff --git a/scripts/db-partition-manage.js b/scripts/db-partition-manage.js new file mode 100755 index 00000000..554a7e37 --- /dev/null +++ b/scripts/db-partition-manage.js @@ -0,0 +1,165 @@ +#!/usr/bin/env node +/** + * scripts/db-partition-manage.js (issue #1802) + * + * Manages monthly partitions for the audit_events table. Can: + * - List existing partitions + * - Create partitions for upcoming months + * - Detach old partitions (for archiving) + * + * Usage: + * node scripts/db-partition-manage.js list + * node scripts/db-partition-manage.js create --months 6 + * node scripts/db-partition-manage.js detach --before 2026-06 + * + * Environment: + * DATABASE_URL — required. + */ + +'use strict'; + +const { spawnSync } = require('child_process'); + +function psql(databaseUrl, sql) { + const result = spawnSync('psql', ['-v', 'ON_ERROR_STOP=1', '--no-psqlrc', '-tAq', databaseUrl, '-c', sql], { + encoding: 'utf8', + }); + if (result.error) return { ok: false, output: result.error.message }; + return { ok: result.status === 0, output: result.stdout || result.stderr }; +} + +function psqlAvailable() { + const probe = spawnSync('psql', ['--version'], { encoding: 'utf8' }); + return !probe.error && probe.status === 0; +} + +function listPartitions(databaseUrl) { + const sql = `SELECT + c.relname AS partition_name, + pg_get_expr(c.relpartbound, c.oid) AS bound + FROM pg_class c + JOIN pg_inherits i ON i.inhrelid = c.oid + JOIN pg_class p ON p.oid = i.inhparent + WHERE p.relname = 'audit_events' + ORDER BY c.relname;`; + + const result = psql(databaseUrl, sql); + if (!result.ok) { + console.error('[partition] Failed to list partitions:', result.output); + return 1; + } + + const lines = result.output.trim().split('\n').filter(Boolean); + if (lines.length === 0) { + console.log('[partition] No partitions found for audit_events.'); + return 0; + } + + console.log('[partition] Partitions for audit_events:'); + for (const line of lines) { + const [name, bound] = line.split('|'); + console.log(` ${name.padEnd(30)} ${bound}`); + } + return 0; +} + +function createPartitions(databaseUrl, monthsAhead) { + const now = new Date(); + let created = 0; + + for (let i = 0; i < monthsAhead; i++) { + const d = new Date(now.getFullYear(), now.getMonth() + i, 1); + const year = d.getFullYear(); + const month = d.getMonth() + 1; + + const result = psql(databaseUrl, `SELECT create_audit_events_partition(${year}, ${month});`); + if (!result.ok) { + console.error(`[partition] Failed to create partition for ${year}-${month}: ${result.output}`); + continue; + } + console.log(`[partition] ${result.output.trim()}`); + created++; + } + + console.log(`[partition] ${created} partition(s) processed.`); + return 0; +} + +function detachPartitions(databaseUrl, beforeDate) { + const sql = `SELECT c.relname + FROM pg_class c + JOIN pg_inherits i ON i.inhrelid = c.oid + JOIN pg_class p ON p.oid = i.inhparent + WHERE p.relname = 'audit_events' + AND c.relname ~ '^audit_events_[0-9]{4}_[0-9]{2}$' + AND c.relname < 'audit_events_${beforeDate.replace('-', '_')}' + ORDER BY c.relname;`; + + const result = psql(databaseUrl, sql); + if (!result.ok) { + console.error('[partition] Failed to find partitions to detach:', result.output); + return 1; + } + + const names = result.output.trim().split('\n').filter(Boolean); + if (names.length === 0) { + console.log('[partition] No partitions to detach before', beforeDate); + return 0; + } + + for (const name of names) { + const detachResult = psql(databaseUrl, `ALTER TABLE audit_events DETACH PARTITION ${name};`); + if (!detachResult.ok) { + console.error(`[partition] Failed to detach ${name}: ${detachResult.output}`); + continue; + } + console.log(`[partition] Detached ${name}`); + } + + console.log(`[partition] Detached ${names.length} partition(s) before ${beforeDate}.`); + return 0; +} + +function main() { + const args = process.argv.slice(2); + const command = args[0]; + + if (!command || command === '--help' || command === '-h') { + console.log('Usage: node scripts/db-partition-manage.js list'); + console.log(' node scripts/db-partition-manage.js create --months '); + console.log(' node scripts/db-partition-manage.js detach --before '); + process.exit(command ? 0 : 2); + } + + const databaseUrl = process.env.DATABASE_URL; + if (!databaseUrl) { + console.error('[partition] DATABASE_URL is not set.'); + process.exit(2); + } + + if (!psqlAvailable()) { + console.error('[partition] psql was not found on PATH.'); + process.exit(2); + } + + if (command === 'list') return listPartitions(databaseUrl); + if (command === 'create') { + const monthsIdx = args.indexOf('--months'); + const months = monthsIdx !== -1 ? parseInt(args[monthsIdx + 1], 10) || 6 : 6; + return createPartitions(databaseUrl, months); + } + if (command === 'detach') { + const beforeIdx = args.indexOf('--before'); + const before = beforeIdx !== -1 ? args[beforeIdx + 1] : ''; + if (!before) { + console.error('[partition] --before YYYY-MM is required for detach'); + return 2; + } + return detachPartitions(databaseUrl, before); + } + + console.error(`[partition] Unknown command "${command}". Use list, create, or detach.`); + return 2; +} + +process.exit(main());