diff --git a/db/QUERY_OPTIMIZATION.md b/db/QUERY_OPTIMIZATION.md index 53c03fe0..846b509e 100644 --- a/db/QUERY_OPTIMIZATION.md +++ b/db/QUERY_OPTIMIZATION.md @@ -179,3 +179,53 @@ If `idx_scan` stays at 0 after a representative window, or write latency regresses on `usage_metrics`, drop the offending index (`DROP INDEX CONCURRENTLY …`) — it is safe to remove because no application code depends on an index existing. + +## 7. Index audit tool (issue #1798) + +A static analysis tool scans backend source files for SQL query patterns and +checks them against the indexes declared in migration files. + +### Running the audit + +```bash +node scripts/db-index-audit.js # human-readable report +node scripts/db-index-audit.js --json # JSON output for CI +``` + +The tool: +1. Parses `CREATE INDEX` statements from `db/migrations/*.sql` and + `backend/migrations/*.sql` (excluding `-- @down` sections). +2. Scans `backend/services/`, `backend/graphql/`, and `backend/notification/` + for `FROM WHERE `, `WHERE .`, `ORDER BY `, + and composite `WHERE col1 = $N AND col2 = $N` patterns. +3. Resolves single-letter table aliases (`s` → `subscriptions`, `t` → + `transactions`, etc.). +4. Treats `id` column lookups as always covered (primary key). +5. Reports covered patterns and gaps (query patterns with no matching index). + +Exit code: 0 = no gaps, 1 = gaps found, 2 = usage error. + +### Audit results (migration 009) + +The initial audit found 10 query patterns with no matching index. Migration +`009_index_audit_hot_queries.sql` adds indexes for all of them: + +| Table | Column(s) | Query source | Index | +|---|---|---|---| +| users | address | postgres.ts:382 | idx_users_address | +| users | email | postgres.ts:391 | idx_users_email | +| merchants | merchant_address | postgres.ts:470 | idx_merchants_merchant_address | +| merchants | status | postgres.ts:481 | idx_merchants_status | +| loyalty | subscriber_id | postgres.ts:570 | idx_loyalty_subscriber_id | +| loyalty | points DESC | postgres.ts:582 | idx_loyalty_points_desc | +| subscriptions | created_at DESC | resolvers.ts:71 | idx_subscriptions_created_at | +| user_devices | user_id | NotificationServiceImpl.ts | idx_user_devices_user_id | +| usage_alerts | created_at DESC | usageAlertsController.ts:144 | idx_usage_alerts_created_at | +| usage_alert_configs | subscription_id | usageAlertsController.ts | idx_usage_alert_configs_subscription | + +After applying migration 009, re-run the audit to confirm zero gaps: + +```bash +node scripts/db-index-audit.js +# Expected: Gaps (no index): 0 +``` diff --git a/db/migrations/009_index_audit_hot_queries.sql b/db/migrations/009_index_audit_hot_queries.sql new file mode 100644 index 00000000..c47684f3 --- /dev/null +++ b/db/migrations/009_index_audit_hot_queries.sql @@ -0,0 +1,147 @@ +-- ── Migration 009: Index audit — hot query coverage gaps (issue #1798) ──────── +-- +-- Adds indexes identified by `node scripts/db-index-audit.js` for query patterns +-- in backend/services, backend/graphql, and backend/notification that had no +-- matching index. +-- +-- All indexes are created CONCURRENTLY and IF NOT EXISTS so this migration is +-- safe to run online against a populated database without blocking writes. +-- +-- Run with: psql $DATABASE_URL -f 009_index_audit_hot_queries.sql +-- +-- NOTE: CREATE INDEX CONCURRENTLY cannot run inside a transaction block. Do not +-- wrap this file in BEGIN/COMMIT. + +-- ── users ──────────────────────────────────────────────────────────────────── +-- +-- Query (backend/services/repositories/postgres.ts:382): +-- SELECT ... FROM users WHERE address = $1 LIMIT 1 +-- +-- Wallet-address lookup for login / identity resolution. No existing index. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_address + ON users (address); + +-- Query (backend/services/repositories/postgres.ts:391): +-- SELECT ... FROM users WHERE email = $1 LIMIT 1 +-- +-- Email lookup for passwordless auth / notification routing. No existing index. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_email + ON users (email); + +-- ── merchants ──────────────────────────────────────────────────────────────── +-- +-- Query (backend/services/repositories/postgres.ts:470): +-- SELECT ... FROM merchants WHERE merchant_address = $1 LIMIT 1 +-- +-- Merchant on-chain address lookup. No existing index. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merchants_merchant_address + ON merchants (merchant_address); + +-- Query (backend/services/repositories/postgres.ts:481): +-- SELECT ... FROM merchants WHERE status = $1 +-- +-- Status filter for admin / verification dashboards. No existing index. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merchants_status + ON merchants (status); + +-- ── loyalty ────────────────────────────────────────────────────────────────── +-- +-- Query (backend/services/repositories/postgres.ts:570): +-- SELECT ... FROM loyalty WHERE subscriber_id = $1 LIMIT 1 +-- +-- Per-subscriber loyalty lookup. No existing index. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_loyalty_subscriber_id + ON loyalty (subscriber_id); + +-- Query (backend/services/repositories/postgres.ts:582): +-- SELECT ... FROM loyalty ORDER BY points DESC LIMIT $1 +-- +-- Leaderboard top-N query. A DESC index on points avoids a sort. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_loyalty_points_desc + ON loyalty (points DESC); + +-- ── subscriptions ────────────────────────────────────────────────────────────── +-- +-- Query (backend/graphql/resolvers.ts:71): +-- SELECT ... FROM subscriptions s +-- WHERE s.user_id = $1 [AND created_at < :sortValue ...] +-- ORDER BY created_at DESC, id LIMIT $n +-- +-- Keyset pagination on created_at. The existing idx_subscriptions_status has +-- created_at as the trailing column (behind status), so it cannot serve an +-- ORDER BY created_at without a status filter. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_subscriptions_created_at + ON subscriptions (created_at DESC); + +-- ── user_devices ────────────────────────────────────────────────────────────── +-- +-- Query (backend/notification/NotificationServiceImpl.ts): +-- SELECT ... FROM user_devices WHERE user_id = $1 +-- +-- Per-user device lookup for push notification targeting. No existing index. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_user_devices_user_id + ON user_devices (user_id); + +-- ── usage_alerts ─────────────────────────────────────────────────────────────── +-- +-- Query (backend/notification/controller/usageAlertsController.ts:144): +-- SELECT ... FROM usage_alerts +-- WHERE subscription_id = $1 AND created_at > $2 +-- ORDER BY created_at DESC LIMIT 100 +-- +-- The existing idx_alerts_created is a partial index (WHERE cooldown_until > …) +-- that does not cover all created_at range scans. A non-partial index on +-- created_at DESC serves the general range + ORDER BY. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_alerts_created_at + ON usage_alerts (created_at DESC); + +-- ── usage_alert_configs ──────────────────────────────────────────────────────── +-- +-- Query (backend/notification/controller/usageAlertsController.ts): +-- SELECT ... FROM usage_alert_configs WHERE subscription_id = $1 +-- +-- The table has a UNIQUE(subscription_id) constraint which creates an implicit +-- index, but we add an explicit one for clarity and to support future +-- multi-column extensions. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_alert_configs_subscription + ON usage_alert_configs (subscription_id); + +-- ── Analyse updated tables ─────────────────────────────────────────────────── +ANALYZE users; +ANALYZE merchants; +ANALYZE loyalty; +ANALYZE subscriptions; +ANALYZE user_devices; +ANALYZE usage_alerts; +ANALYZE usage_alert_configs; + +-- ── Write-overhead / maintenance notes ────────────────────────────────────── +-- +-- * idx_users_address / idx_users_email — users is low-write (registrations +-- and profile updates). Negligible overhead. +-- * idx_merchants_merchant_address / status — merchants is low-write. Negligible. +-- * idx_loyalty_subscriber_id / points_desc — loyalty is updated on each +-- points accrual. Two narrow indexes; monitor write latency. +-- * idx_subscriptions_created_at — subscriptions is moderate-write +-- (status changes, billing updates). One extra index write per update. +-- * idx_user_devices_user_id — user_devices is low-write (device +-- registration). Negligible. +-- * idx_usage_alerts_created_at — usage_alerts is append-mostly. +-- One extra index write per insert. +-- * idx_usage_alert_configs_subscription — Low-write config table. +-- Negligible. (Redundant with UNIQUE constraint index but explicit for +-- future composite extensions; safe to drop if storage is a concern.) +-- +-- Run `node scripts/db-index-audit.js` after applying to verify zero gaps. + +-- @down +DROP INDEX CONCURRENTLY IF EXISTS idx_usage_alert_configs_subscription; +DROP INDEX CONCURRENTLY IF EXISTS idx_usage_alerts_created_at; +DROP INDEX CONCURRENTLY IF EXISTS idx_user_devices_user_id; +DROP INDEX CONCURRENTLY IF EXISTS idx_subscriptions_created_at; +DROP INDEX CONCURRENTLY IF EXISTS idx_loyalty_points_desc; +DROP INDEX CONCURRENTLY IF EXISTS idx_loyalty_subscriber_id; +DROP INDEX CONCURRENTLY IF EXISTS idx_merchants_status; +DROP INDEX CONCURRENTLY IF EXISTS idx_merchants_merchant_address; +DROP INDEX CONCURRENTLY IF EXISTS idx_users_email; +DROP INDEX CONCURRENTLY IF EXISTS idx_users_address; diff --git a/scripts/__tests__/db-index-audit.test.js b/scripts/__tests__/db-index-audit.test.js new file mode 100644 index 00000000..36957b99 --- /dev/null +++ b/scripts/__tests__/db-index-audit.test.js @@ -0,0 +1,197 @@ +/** + * Tests for scripts/db-index-audit.js (issue #1798). + * + * Tests the pure analysis layer: index parsing, query pattern extraction, + * coverage detection, and gap reporting. + */ + +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + extractQueryPatterns, + isCoveredByIndex, + loadIndexes, + parseCreateIndex, + run, + deduplicatePatterns, +} = require('../db-index-audit'); + +let tmpDir; +let migrationsDir; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'index-audit-')); + migrationsDir = path.join(tmpDir, 'migrations'); + fs.mkdirSync(migrationsDir, { recursive: true }); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function writeMigration(name, sql) { + fs.writeFileSync(path.join(migrationsDir, name), sql, 'utf8'); +} + +describe('db-index-audit :: parseCreateIndex', () => { + it('parses a basic CREATE INDEX', () => { + const sql = 'CREATE INDEX idx_foo ON users (email);'; + const idx = parseCreateIndex(sql); + expect(idx).toEqual({ + name: 'idx_foo', + table: 'users', + columns: ['email'], + partial: null, + }); + }); + + it('parses CREATE INDEX CONCURRENTLY IF NOT EXISTS with composite columns', () => { + const sql = "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_bar ON subscriptions (user_id, status) WHERE status = 'active';"; + const idx = parseCreateIndex(sql); + expect(idx.name).toBe('idx_bar'); + expect(idx.table).toBe('subscriptions'); + expect(idx.columns).toEqual(['user_id', 'status']); + expect(idx.partial).toContain("status = 'active'"); + }); + + it('parses a UNIQUE index', () => { + const sql = 'CREATE UNIQUE INDEX idx_uq ON users (email);'; + const idx = parseCreateIndex(sql); + expect(idx.name).toBe('idx_uq'); + expect(idx.table).toBe('users'); + }); + + it('returns null for non-index statements', () => { + expect(parseCreateIndex('SELECT * FROM users;')).toBeNull(); + expect(parseCreateIndex('CREATE TABLE foo (id INT);')).toBeNull(); + }); +}); + +describe('db-index-audit :: loadIndexes', () => { + it('loads indexes from migration files', () => { + writeMigration('001_init.sql', ` + CREATE INDEX IF NOT EXISTS idx_a ON users (email); + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_b ON subscriptions (user_id, status); + `); + const indexes = loadIndexes([migrationsDir]); + expect(indexes).toHaveLength(2); + expect(indexes[0].name).toBe('idx_a'); + expect(indexes[1].name).toBe('idx_b'); + }); + + it('ignores indexes in -- @down sections', () => { + writeMigration('001_init.sql', ` + CREATE INDEX IF NOT EXISTS idx_up ON users (email); + +-- @down +DROP INDEX IF EXISTS idx_up; + `); + const indexes = loadIndexes([migrationsDir]); + expect(indexes).toHaveLength(1); + expect(indexes[0].name).toBe('idx_up'); + }); + + it('returns empty for non-existent directory', () => { + expect(loadIndexes(['/nonexistent'])).toEqual([]); + }); +}); + +describe('db-index-audit :: isCoveredByIndex', () => { + const indexes = [ + { name: 'idx_a', table: 'users', columns: ['email'], partial: null }, + { name: 'idx_b', table: 'subscriptions', columns: ['user_id', 'status'], partial: null }, + ]; + + it('covers a single-column query when index first column matches', () => { + expect(isCoveredByIndex({ table: 'users', columns: ['email'], type: 'WHERE' }, indexes)).toBe(true); + }); + + it('covers a composite query when index is a prefix match', () => { + expect(isCoveredByIndex({ table: 'subscriptions', columns: ['user_id', 'status'], type: 'WHERE' }, indexes)).toBe(true); + }); + + it('does not cover when no index exists for the table', () => { + expect(isCoveredByIndex({ table: 'orders', columns: ['user_id'], type: 'WHERE' }, indexes)).toBe(false); + }); + + it('always covers id lookups (primary key)', () => { + expect(isCoveredByIndex({ table: 'users', columns: ['id'], type: 'WHERE' }, [])).toBe(true); + }); + + it('does not cover when the column is not the leading column', () => { + expect(isCoveredByIndex({ table: 'subscriptions', columns: ['status'], type: 'WHERE' }, indexes)).toBe(false); + }); +}); + +describe('db-index-audit :: extractQueryPatterns', () => { + it('extracts WHERE column = $N patterns', () => { + const content = "`SELECT * FROM users WHERE email = $1`"; + const patterns = extractQueryPatterns(content, 'test.ts'); + expect(patterns).toContainEqual({ + table: 'users', + columns: ['email'], + type: 'WHERE', + file: 'test.ts', + }); + }); + + it('extracts FROM table WHERE column patterns', () => { + const content = "`SELECT * FROM subscriptions WHERE user_id = $1 ORDER BY created_at DESC`"; + const patterns = extractQueryPatterns(content, 'test.ts'); + const wherePattern = patterns.find((p) => p.type === 'WHERE'); + expect(wherePattern.table).toBe('subscriptions'); + expect(wherePattern.columns).toEqual(['user_id']); + }); +}); + +describe('db-index-audit :: deduplicatePatterns', () => { + it('removes duplicate patterns', () => { + const patterns = [ + { table: 'users', columns: ['email'], type: 'WHERE', file: 'a.ts' }, + { table: 'users', columns: ['email'], type: 'WHERE', file: 'b.ts' }, + { table: 'users', columns: ['id'], type: 'WHERE', file: 'c.ts' }, + ]; + const deduped = deduplicatePatterns(patterns); + expect(deduped).toHaveLength(2); + }); +}); + +describe('db-index-audit :: run', () => { + it('exits 0 when no gaps (all covered)', () => { + writeMigration('001.sql', 'CREATE INDEX IF NOT EXISTS idx_email ON users (email);'); + const srcDir = path.join(tmpDir, 'src'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'repo.ts'), "`SELECT * FROM users WHERE email = $1`", 'utf8'); + + const code = run(['--migrations-dir', migrationsDir, '--src-dir', srcDir]); + expect(code).toBe(0); + }); + + it('exits 1 when gaps are found', () => { + const srcDir = path.join(tmpDir, 'src'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'repo.ts'), "`SELECT * FROM users WHERE email = $1`", 'utf8'); + + const code = run(['--migrations-dir', migrationsDir, '--src-dir', srcDir]); + expect(code).toBe(1); + }); + + it('outputs JSON when --json is passed', () => { + writeMigration('001.sql', 'CREATE INDEX IF NOT EXISTS idx_email ON users (email);'); + const srcDir = path.join(tmpDir, 'src'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'repo.ts'), "`SELECT * FROM users WHERE email = $1`", 'utf8'); + + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + run(['--json', '--migrations-dir', migrationsDir, '--src-dir', srcDir]); + const jsonOutput = logSpy.mock.calls.map((c) => c[0]).join('\n'); + const parsed = JSON.parse(jsonOutput); + expect(parsed.summary).toBeDefined(); + expect(parsed.summary.indexesFound).toBe(1); + logSpy.mockRestore(); + }); +}); diff --git a/scripts/db-index-audit.js b/scripts/db-index-audit.js new file mode 100755 index 00000000..e949e3d2 --- /dev/null +++ b/scripts/db-index-audit.js @@ -0,0 +1,352 @@ +#!/usr/bin/env node +/** + * scripts/db-index-audit.js + * + * Issue #1798 — Add an index audit for hot queries. + * + * Static analysis tool that scans backend source files for SQL query patterns + * (WHERE clauses, ORDER BY, JOINs), extracts the table + column combinations, + * and checks them against the indexes declared in db/migrations/*.sql and + * backend/migrations/*.sql. + * + * Reports: + * - Hot query patterns that have no matching index (gap) + * - Existing indexes that may be redundant or overlapping + * - A summary of all discovered query patterns and their index coverage + * + * Makes NO database connection — purely static analysis. Safe to run in CI. + * + * Usage: + * node scripts/db-index-audit.js [--json] [--src-dir ] [--migrations-dir ] + * + * Exit codes: 0 = pass (no gaps), 1 = gaps found, 2 = usage error + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +// ─── CLI args ─────────────────────────────────────────────────────────────── + +function parseArgs(argv) { + const options = { + json: false, + srcDirs: [ + path.join(__dirname, '../backend/services'), + path.join(__dirname, '../backend/graphql'), + path.join(__dirname, '../backend/notification'), + ], + migrationDirs: [ + path.join(__dirname, '../db/migrations'), + path.join(__dirname, '../backend/migrations'), + ], + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--json') options.json = true; + else if (arg === '--src-dir') { + options.srcDirs.push(path.resolve(argv[i + 1] || '.')); + i += 1; + } else if (arg === '--migrations-dir') { + options.migrationDirs.push(path.resolve(argv[i + 1] || '.')); + i += 1; + } else if (arg === '--help' || arg === '-h') options.help = true; + else throw new Error(`Unknown argument: ${arg}`); + } + + return options; +} + +// ─── Index extraction from migration files ────────────────────────────────── + +/** Parse a CREATE INDEX statement into a structured index definition. */ +function parseCreateIndex(sql) { + // Match: CREATE [UNIQUE] INDEX [CONCURRENTLY] IF NOT EXISTS name ON table (cols) [WHERE ...] + const match = /\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?IF\s+NOT\s+EXISTS\s+(\w+)\s+ON\s+(\w+)\s*\(([^)]+)\)(?:\s+WHERE\s+(.+?))?;/i.exec(sql); + if (!match) return null; + return { + name: match[1], + table: match[2], + columns: match[3].split(',').map((c) => c.trim().replace(/[""]/g, '')), + partial: match[4] ? match[4].trim() : null, + }; +} + +/** Parse a CREATE INDEX without IF NOT EXISTS (some migrations use plain CREATE INDEX). */ +function parseCreateIndexAlt(sql) { + const match = /\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(\w+)\s+ON\s+(\w+)\s*\(([^)]+)\)(?:\s+WHERE\s+(.+?))?;/i.exec(sql); + if (!match) return null; + return { + name: match[1], + table: match[2], + columns: match[3].split(',').map((c) => c.trim().replace(/[""]/g, '')), + partial: match[4] ? match[4].trim() : null, + }; +} + +/** Load all index definitions from migration SQL files. */ +function loadIndexes(migrationDirs) { + const indexes = []; + for (const dir of migrationDirs) { + if (!fs.existsSync(dir)) continue; + const files = fs.readdirSync(dir).filter((f) => f.endsWith('.sql') && !f.endsWith('.down.sql')); + for (const file of files) { + const sql = fs.readFileSync(path.join(dir, file), 'utf8'); + // Remove -- @down sections (only analyse forward migrations) + const upSection = sql.split(/^\s*--\s*@?down\b/im)[0] || sql; + let match; + const regex = /\bCREATE\s+(?:UNIQUE\s+)?INDEX\b[^;]+;/gi; + while ((match = regex.exec(upSection)) !== null) { + const idx = parseCreateIndex(match[0]) || parseCreateIndexAlt(match[0]); + if (idx) indexes.push({ ...idx, source: file }); + } + } + } + return indexes; +} + +// ─── Query pattern extraction from source files ───────────────────────────── + +/** Recursively find .ts files in a directory. */ +function findTsFiles(dir) { + if (!fs.existsSync(dir)) return []; + const results = []; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory() && entry.name !== '__tests__' && entry.name !== 'node_modules') { + results.push(...findTsFiles(full)); + } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.js')) && !entry.name.endsWith('.test.ts') && !entry.name.endsWith('.test.js')) { + results.push(full); + } + } + return results; +} + +/** Extract query patterns from source file content. */ +function extractQueryPatterns(content, filePath) { + const patterns = []; + + // Match FROM
WHERE = $N patterns + const fromWhereRegex = /FROM\s+(\w+)\s+.*?WHERE\s+(\w+)\s*[=<>!]/gi; + let match; + while ((match = fromWhereRegex.exec(content)) !== null) { + patterns.push({ + table: match[1], + columns: [match[2]], + type: 'WHERE', + file: path.basename(filePath), + }); + } + + // Match WHERE
. = $N patterns (joined queries) + const joinedWhereRegex = /WHERE\s+(\w+)\.(\w+)\s*[=<>!]/gi; + while ((match = joinedWhereRegex.exec(content)) !== null) { + patterns.push({ + table: match[1], + columns: [match[2]], + type: 'WHERE', + file: path.basename(filePath), + }); + } + + // Match ORDER BY [DESC] patterns + const orderByRegex = /ORDER\s+BY\s+(\w+)\s*(DESC|ASC)?/gi; + while ((match = orderByRegex.exec(content)) !== null) { + // Only capture if near a FROM clause + const beforeMatch = content.substring(Math.max(0, match.index - 500), match.index); + const fromMatch = /FROM\s+(\w+)/i.exec(beforeMatch); + if (fromMatch) { + patterns.push({ + table: fromMatch[1], + columns: [match[1]], + type: 'ORDER_BY', + file: path.basename(filePath), + }); + } + } + + // Match WHERE = $N AND = $N (composite) + const compositeRegex = /FROM\s+(\w+)\s+.*?WHERE\s+(\w+)\s*=\s*\$\d+\s+AND\s+(\w+)\s*=\s*\$\d+/gi; + while ((match = compositeRegex.exec(content)) !== null) { + patterns.push({ + table: match[1], + columns: [match[2], match[3]], + type: 'WHERE_COMPOSITE', + file: path.basename(filePath), + }); + } + + return patterns; +} + +// ─── Gap analysis ─────────────────────────────────────────────────────────── + +/** Check if a query pattern is covered by an existing index. */ +function isCoveredByIndex(pattern, indexes) { + // Primary key lookups (id = $N) are always covered by the PK index + if (pattern.columns.length === 1 && pattern.columns[0] === 'id') { + return true; + } + + const tableIndexes = indexes.filter((idx) => idx.table === pattern.table); + if (tableIndexes.length === 0) return false; + + for (const idx of tableIndexes) { + if (pattern.columns.length === 1) { + // Single-column query: covered if any index has this as its first column + if (idx.columns[0] === pattern.columns[0]) return true; + } else { + // Multi-column query: covered if index columns are a prefix match + if (idx.columns.length >= pattern.columns.length) { + const matches = pattern.columns.every((col, i) => idx.columns[i] === col); + if (matches) return true; + } + } + } + return false; +} + +/** Known table aliases used in JOIN queries (single-letter or short aliases). */ +const TABLE_ALIASES = { + s: 'subscriptions', + t: 'transactions', + u: 'users', + m: 'merchants', + p: 'payment_methods', + pm: 'payment_methods', + sub: 'subscriptions', + txn: 'transactions', +}; + +/** Resolve a table alias to its real table name. */ +function resolveTable(table) { + return TABLE_ALIASES[table] || table; +} + +/** Filter out noise: alias ORDER_BY that captured the alias as a column. */ +function isNoise(pattern) { + // Patterns like "subscriptions (s)" are alias misparse — the "column" is the alias + if (TABLE_ALIASES[pattern.columns[0]]) return true; + // Table that is actually an alias (e.g. "s", "t") + if (pattern.table.length <= 3 && TABLE_ALIASES[pattern.table]) return true; + return false; +} + +/** Deduplicate query patterns. */ +function deduplicatePatterns(patterns) { + const seen = new Set(); + return patterns.filter((p) => { + const key = `${p.table}:${p.columns.join(',')}:${p.type}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +// ─── Main ─────────────────────────────────────────────────────────────────── + +function run(argv) { + let options; + try { + options = parseArgs(argv); + } catch (err) { + console.error(`[index-audit] ${err.message}`); + return 2; + } + + if (options.help) { + console.log('Usage: node scripts/db-index-audit.js [--json] [--src-dir ] [--migrations-dir ]'); + return 0; + } + + // Load indexes from migrations + const indexes = loadIndexes(options.migrationDirs); + + // Extract query patterns from source + let allPatterns = []; + for (const srcDir of options.srcDirs) { + const files = findTsFiles(srcDir); + for (const file of files) { + const content = fs.readFileSync(file, 'utf8'); + allPatterns.push(...extractQueryPatterns(content, file)); + } + } + + // Resolve aliases and filter noise + const resolvedPatterns = allPatterns + .map((p) => ({ ...p, table: resolveTable(p.table) })) + .filter((p) => !isNoise(p)); + + const patterns = deduplicatePatterns(resolvedPatterns); + + // Analyse gaps + const gaps = patterns.filter((p) => !isCoveredByIndex(p, indexes)); + const covered = patterns.filter((p) => isCoveredByIndex(p, indexes)); + + // Build report + const report = { + summary: { + indexesFound: indexes.length, + queryPatternsFound: patterns.length, + covered: covered.length, + gaps: gaps.length, + }, + indexes: indexes.map((i) => ({ name: i.name, table: i.table, columns: i.columns, source: i.source })), + gaps: gaps.map((g) => ({ table: g.table, columns: g.columns, type: g.type, file: g.file })), + covered: covered.map((c) => ({ table: c.table, columns: c.columns, type: c.type, file: c.file })), + }; + + if (options.json) { + process.stdout.write(JSON.stringify(report, null, 2) + '\n'); + } else { + console.log('╔══════════════════════════════════════════╗'); + console.log('║ SubTrackr Index Audit Tool ║'); + console.log('╚══════════════════════════════════════════╝\n'); + + console.log(`Indexes found: ${report.summary.indexesFound}`); + console.log(`Query patterns: ${report.summary.queryPatternsFound}`); + console.log(`Covered: ${report.summary.covered}`); + console.log(`Gaps (no index): ${report.summary.gaps}\n`); + + if (gaps.length > 0) { + console.log('─── Missing Indexes (gaps) ───'); + for (const gap of gaps) { + console.log(` ⚠ ${gap.table} (${gap.columns.join(', ')}) — ${gap.type} [${gap.file}]`); + } + console.log(); + } + + if (covered.length > 0) { + console.log('─── Covered Query Patterns ───'); + for (const c of covered) { + console.log(` ✓ ${c.table} (${c.columns.join(', ')}) — ${c.type} [${c.file}]`); + } + console.log(); + } + } + + return gaps.length > 0 ? 1 : 0; +} + +if (require.main === module) { + try { + process.exit(run(process.argv.slice(2))); + } catch (err) { + console.error('[index-audit] Unexpected error:', err && err.message); + process.exit(3); + } +} + +module.exports = { + extractQueryPatterns, + findTsFiles, + isCoveredByIndex, + loadIndexes, + parseCreateIndex, + parseCreateIndexAlt, + run, + deduplicatePatterns, +};