diff --git a/.env.example b/.env.example index 7f343b4b..8e7af699 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,21 @@ DB_USER=postgres # Required: set a strong password DB_PASSWORD= +# ── Connection Pool Tuning (issue #1797) ────────────────────────────────────── +# All values have sane defaults; override only when needed. +DB_POOL_MAX=20 # Max connections in the primary pool +DB_POOL_MIN=0 # Min connections to keep open +DB_IDLE_TIMEOUT_MS=10000 # Idle connection timeout (ms) +DB_CONNECTION_TIMEOUT_MS=30000 # Connection acquisition timeout (ms) +DB_STATEMENT_TIMEOUT_MS=30000 # Per-statement timeout (ms) +DB_POOL_MAX_USES=0 # Max client reuses before recycle (0 = unlimited) +DB_QUERY_WAIT_WARNING_MS=5000 # Warn when a query waits longer than this (ms) + +# Pool metrics collection +DB_POOL_METRICS_ENABLED=true # Set to "false" to disable +DB_POOL_METRICS_INTERVAL_MS=10000 # Sampling interval (ms) +DB_POOL_METRICS_ALERT_THRESHOLD=0.8 # Alert when utilization exceeds this (0–1) + # Redis REDIS_HOST=localhost REDIS_PORT=6379 diff --git a/backend/config/__tests__/database.test.ts b/backend/config/__tests__/database.test.ts index 10418edc..236acf1d 100644 --- a/backend/config/__tests__/database.test.ts +++ b/backend/config/__tests__/database.test.ts @@ -94,3 +94,53 @@ describe('database config', () => { ]); }); }); + +// ── Pool tuning options (issue #1797) ─────────────────────────────────────── + +describe('pool tuning configuration (issue #1797)', () => { + it('loads default pool tuning when env vars are unset', () => { + const config = loadDatabaseConfig({}); + expect(config.poolTuning.minConnections).toBe(0); + expect(config.poolTuning.maxClientUses).toBe(0); + expect(config.poolTuning.queryWaitWarningMs).toBe(5_000); + expect(config.poolTuning.metricsEnabled).toBe(true); + expect(config.poolTuning.metricsIntervalMs).toBe(10_000); + expect(config.poolTuning.metricsAlertThreshold).toBe(0.8); + }); + + it('reads custom pool tuning from env', () => { + const config = loadDatabaseConfig({ + DB_POOL_MIN: '5', + DB_POOL_MAX_USES: '100', + DB_QUERY_WAIT_WARNING_MS: '2000', + DB_POOL_METRICS_ENABLED: 'false', + DB_POOL_METRICS_INTERVAL_MS: '5000', + DB_POOL_METRICS_ALERT_THRESHOLD: '0.9', + }); + expect(config.poolTuning.minConnections).toBe(5); + expect(config.poolTuning.maxClientUses).toBe(100); + expect(config.poolTuning.queryWaitWarningMs).toBe(2_000); + expect(config.poolTuning.metricsEnabled).toBe(false); + expect(config.poolTuning.metricsIntervalMs).toBe(5_000); + expect(config.poolTuning.metricsAlertThreshold).toBe(0.9); + }); + + it('falls back for invalid pool tuning values', () => { + const config = loadDatabaseConfig({ + DB_POOL_MIN: '-1', + DB_POOL_MAX_USES: 'not-a-number', + DB_POOL_METRICS_ALERT_THRESHOLD: '2.0', + DB_POOL_METRICS_INTERVAL_MS: '-5', + }); + expect(config.poolTuning.minConnections).toBe(0); + expect(config.poolTuning.maxClientUses).toBe(0); + expect(config.poolTuning.metricsAlertThreshold).toBe(0.8); + expect(config.poolTuning.metricsIntervalMs).toBe(10_000); + }); + + it('DB_POOL_METRICS_ENABLED only disables on "false"', () => { + expect(loadDatabaseConfig({ DB_POOL_METRICS_ENABLED: 'false' }).poolTuning.metricsEnabled).toBe(false); + expect(loadDatabaseConfig({ DB_POOL_METRICS_ENABLED: 'true' }).poolTuning.metricsEnabled).toBe(true); + expect(loadDatabaseConfig({ DB_POOL_METRICS_ENABLED: undefined }).poolTuning.metricsEnabled).toBe(true); + }); +}); diff --git a/backend/config/database.ts b/backend/config/database.ts index bbd82379..65edc795 100644 --- a/backend/config/database.ts +++ b/backend/config/database.ts @@ -44,6 +44,8 @@ export interface DatabaseConfig { staleReadDefaultSeconds: number; /** How often to poll replication lag (ms). Default: 5000 */ lagPollIntervalMs: number; + /** Extended pool tuning options (issue #1797) */ + poolTuning: PoolTuningOptions; } export const DEFAULT_DATABASE_CONFIG: Readonly<{ @@ -95,6 +97,60 @@ function poolDefaults(env: NodeJS.ProcessEnv): Pick< }; } +/** + * Extended pool tuning options loaded from environment. + * All have sensible defaults so the pool works out-of-the-box. + */ +export interface PoolTuningOptions { + /** Minimum connections to keep open. Default: 0 (pg-pool default) */ + minConnections: number; + /** Max times a client is reused before being recycled. Default: 0 (unlimited) */ + maxClientUses: number; + /** Log a warning when a query waits longer than this (ms). Default: 5000 */ + queryWaitWarningMs: number; + /** Whether to enable pool metrics collection. Default: true */ + metricsEnabled: boolean; + /** Pool metrics sampling interval in ms. Default: 10000 */ + metricsIntervalMs: number; + /** Alert threshold for pool utilization (0–1). Default: 0.8 */ + metricsAlertThreshold: number; +} + +export const DEFAULT_POOL_TUNING: Readonly = { + minConnections: 0, + maxClientUses: 0, + queryWaitWarningMs: 5_000, + metricsEnabled: true, + metricsIntervalMs: 10_000, + metricsAlertThreshold: 0.8, +}; + +function loadPoolTuning(env: NodeJS.ProcessEnv): PoolTuningOptions { + return { + minConnections: parsePositiveInt(env.DB_POOL_MIN, DEFAULT_POOL_TUNING.minConnections), + maxClientUses: parsePositiveInt(env.DB_POOL_MAX_USES, DEFAULT_POOL_TUNING.maxClientUses), + queryWaitWarningMs: parsePositiveInt( + env.DB_QUERY_WAIT_WARNING_MS, + DEFAULT_POOL_TUNING.queryWaitWarningMs, + ), + metricsEnabled: env.DB_POOL_METRICS_ENABLED !== 'false', + metricsIntervalMs: parsePositiveInt( + env.DB_POOL_METRICS_INTERVAL_MS, + DEFAULT_POOL_TUNING.metricsIntervalMs, + ), + metricsAlertThreshold: parsePositiveFloat( + env.DB_POOL_METRICS_ALERT_THRESHOLD, + DEFAULT_POOL_TUNING.metricsAlertThreshold, + ), + }; +} + +function parsePositiveFloat(value: string | undefined, fallback: number): number { + if (value === undefined || value === '') return fallback; + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : fallback; +} + function primaryFromParsed( parsed: ParsedConnectionString, env: NodeJS.ProcessEnv, @@ -160,13 +216,14 @@ export function loadDatabaseConfig(env: NodeJS.ProcessEnv = process.env): Databa DEFAULT_DATABASE_CONFIG.replicationLagFailoverMs, ), staleReadDefaultSeconds: parsePositiveInt( - env.DB_STALE_READ_DEFAULT_SECONDS, + env.DB_STALE_READ_SECONDS, DEFAULT_DATABASE_CONFIG.staleReadDefaultSeconds, ), lagPollIntervalMs: parsePositiveInt( env.DB_LAG_POLL_INTERVAL_MS, DEFAULT_DATABASE_CONFIG.lagPollIntervalMs, ), + poolTuning: loadPoolTuning(env), }; } diff --git a/backend/shared/db/__tests__/poolMetrics.test.ts b/backend/shared/db/__tests__/poolMetrics.test.ts new file mode 100644 index 00000000..0624a3ae --- /dev/null +++ b/backend/shared/db/__tests__/poolMetrics.test.ts @@ -0,0 +1,188 @@ +import { PoolMetricsCollector, type PoolMetricsSnapshot } from '../poolMetrics'; +import type { Pool } from '../connectionPool'; + +/** Mock pool that simulates pg-pool counters. */ +function createMockPool(overrides: Partial = {}): Pool & { + max: number; + setCounts: (total: number, idle: number, waiting: number) => void; +} { + let totalCount = 0; + let idleCount = 0; + let waitingCount = 0; + const max = overrides.max ?? 20; + + const pool: Pool & { max: number; setCounts: (t: number, i: number, w: number) => void } = { + max, + get totalCount() { return totalCount; }, + get idleCount() { return idleCount; }, + get waitingCount() { return waitingCount; }, + query: jest.fn(), + connect: jest.fn(), + end: jest.fn(), + on: jest.fn(), + setCounts: (t: number, i: number, w: number) => { + totalCount = t; + idleCount = i; + waitingCount = w; + }, + }; + return pool; +} + +describe('PoolMetricsCollector', () => { + it('takes a snapshot with correct derived fields', () => { + const pool = createMockPool({ max: 20 }); + pool.setCounts(15, 8, 0); + + const collector = new PoolMetricsCollector(pool); + const snapshot = collector.sample(); + + expect(snapshot.totalCount).toBe(15); + expect(snapshot.idleCount).toBe(8); + expect(snapshot.waitingCount).toBe(0); + expect(snapshot.activeCount).toBe(7); + expect(snapshot.maxConnections).toBe(20); + expect(snapshot.utilization).toBeCloseTo(0.35, 2); + expect(snapshot.saturated).toBe(false); + }); + + it('detects saturation when waitingCount > 0', () => { + const pool = createMockPool({ max: 10 }); + pool.setCounts(10, 0, 3); + + const collector = new PoolMetricsCollector(pool); + const snapshot = collector.sample(); + + expect(snapshot.saturated).toBe(true); + expect(snapshot.activeCount).toBe(10); + expect(snapshot.utilization).toBe(1); + }); + + it('calls onSnapshot for each sample', () => { + const pool = createMockPool({ max: 20 }); + pool.setCounts(5, 3, 0); + + const snapshots: PoolMetricsSnapshot[] = []; + const collector = new PoolMetricsCollector(pool, { + onSnapshot: (s) => snapshots.push(s), + }); + + collector.sample(); + pool.setCounts(8, 4, 0); + collector.sample(); + + expect(snapshots).toHaveLength(2); + expect(snapshots[0].activeCount).toBe(2); + expect(snapshots[1].activeCount).toBe(4); + }); + + it('fires onAlert when utilization crosses the threshold (edge-triggered)', () => { + const pool = createMockPool({ max: 10 }); + pool.setCounts(7, 3, 0); // 40% utilization + + const alerts: PoolMetricsSnapshot[] = []; + const collector = new PoolMetricsCollector(pool, { + alertThreshold: 0.8, + onAlert: (s) => alerts.push(s), + }); + + collector.sample(); // 40% — no alert + expect(alerts).toHaveLength(0); + + pool.setCounts(9, 1, 0); // 80% — crosses threshold + collector.sample(); + expect(alerts).toHaveLength(1); + expect(alerts[0].utilization).toBeGreaterThanOrEqual(0.8); + + pool.setCounts(10, 0, 0); // 100% — still alerting, no new alert + collector.sample(); + expect(alerts).toHaveLength(1); // edge-triggered + + pool.setCounts(5, 5, 0); // 0% — clears alerting + collector.sample(); + expect(alerts).toHaveLength(1); + + pool.setCounts(9, 1, 0); // 80% — re-crosses threshold + collector.sample(); + expect(alerts).toHaveLength(2); // new alert after recovery + }); + + it('getHealthSummary reports healthy under normal load', () => { + const pool = createMockPool({ max: 20 }); + pool.setCounts(10, 5, 0); + + const collector = new PoolMetricsCollector(pool, { alertThreshold: 0.8 }); + collector.sample(); + + const health = collector.getHealthSummary(); + expect(health.healthy).toBe(true); + expect(health.saturated).toBe(false); + expect(health.snapshot).not.toBeNull(); + }); + + it('getHealthSummary reports unhealthy when saturated', () => { + const pool = createMockPool({ max: 10 }); + pool.setCounts(10, 0, 5); + + const collector = new PoolMetricsCollector(pool); + collector.sample(); + + const health = collector.getHealthSummary(); + expect(health.healthy).toBe(false); + expect(health.saturated).toBe(true); + }); + + it('getHealthSummary returns healthy before any sample is taken', () => { + const pool = createMockPool(); + const collector = new PoolMetricsCollector(pool); + const health = collector.getHealthSummary(); + expect(health.healthy).toBe(true); + expect(health.snapshot).toBeNull(); + }); + + it('start and stop control the sampling interval', () => { + const pool = createMockPool({ max: 20 }); + pool.setCounts(5, 3, 0); + + const collector = new PoolMetricsCollector(pool, { intervalMs: 100 }); + expect(collector.isRunning()).toBe(false); + + collector.start(); + expect(collector.isRunning()).toBe(true); + + collector.stop(); + expect(collector.isRunning()).toBe(false); + }); + + it('start is idempotent', () => { + const pool = createMockPool(); + const collector = new PoolMetricsCollector(pool, { intervalMs: 100 }); + collector.start(); + collector.start(); + expect(collector.isRunning()).toBe(true); + collector.stop(); + }); + + it('getLastSnapshot returns the most recent sample', () => { + const pool = createMockPool({ max: 20 }); + pool.setCounts(5, 3, 0); + + const collector = new PoolMetricsCollector(pool); + expect(collector.getLastSnapshot()).toBeNull(); + + collector.sample(); + expect(collector.getLastSnapshot()).not.toBeNull(); + expect(collector.getLastSnapshot()?.activeCount).toBe(2); + }); + + it('handles zero maxConnections gracefully', () => { + const pool = createMockPool({ max: 0 }); + pool.setCounts(0, 0, 0); + + const collector = new PoolMetricsCollector(pool); + const snapshot = collector.sample(); + + expect(snapshot.utilization).toBe(0); + expect(snapshot.maxConnections).toBe(0); + }); +}); diff --git a/backend/shared/db/connectionPool.ts b/backend/shared/db/connectionPool.ts index 7af2eccb..1a1883da 100644 --- a/backend/shared/db/connectionPool.ts +++ b/backend/shared/db/connectionPool.ts @@ -56,6 +56,8 @@ export interface Pool { totalCount: number; idleCount: number; waitingCount: number; + /** Configured max connections (pg-pool exposes this) */ + max?: number; } function defaultPoolConfig(): Required { @@ -124,3 +126,5 @@ export async function closePool(): Promise { } export type { ReadWritePool }; +export { PoolMetricsCollector } from './poolMetrics'; +export type { PoolMetricsSnapshot, PoolMetricsOptions, ReplicaLagState } from './poolMetrics'; diff --git a/backend/shared/db/poolMetrics.ts b/backend/shared/db/poolMetrics.ts new file mode 100644 index 00000000..358cdaf3 --- /dev/null +++ b/backend/shared/db/poolMetrics.ts @@ -0,0 +1,161 @@ +/** + * Connection pool metrics collector (issue #1797). + * + * Periodically samples pool health counters (total, idle, waiting) and exposes + * them as a structured snapshot for monitoring, dashboards and alerts. + * + * The collector is transport-agnostic: it produces a `PoolMetricsSnapshot` + * that can be serialized to JSON for Prometheus export, log shipping, or an + * internal `/health/db` endpoint. + */ + +import type { Pool, ReadWritePool } from './connectionPool'; + +export interface PoolMetricsSnapshot { + /** Epoch ms when the sample was taken */ + timestamp: number; + /** Total connections open (active + idle) */ + totalCount: number; + /** Idle connections available for reuse */ + idleCount: number; + /** Requests blocked waiting for a connection */ + waitingCount: number; + /** Active (checked-out) connections: total - idle */ + activeCount: number; + /** Configured max connections */ + maxConnections: number; + /** Utilisation ratio: active / max (0–1) */ + utilization: number; + /** True when waitingCount > 0 (saturation signal) */ + saturated: boolean; + /** Read-replica lag states when replica routing is enabled */ + replicas?: ReplicaLagState[]; +} + +export interface ReplicaLagState { + name: string; + lagMs: number | null; + rerouted: boolean; +} + +export interface PoolMetricsOptions { + /** Sampling interval in ms. Default: 10 000 */ + intervalMs?: number; + /** Called on every snapshot for log/metrics shipping */ + onSnapshot?: (snapshot: PoolMetricsSnapshot) => void; + /** Alert when utilization exceeds this threshold (0–1). Default: 0.8 */ + alertThreshold?: number; + /** Called when utilization crosses the alert threshold */ + onAlert?: (snapshot: PoolMetricsSnapshot) => void; +} + +export class PoolMetricsCollector { + private timer: ReturnType | null = null; + private readonly intervalMs: number; + private readonly alertThreshold: number; + private onSnapshot?: (snapshot: PoolMetricsSnapshot) => void; + private onAlert?: (snapshot: PoolMetricsSnapshot) => void; + private lastSnapshot: PoolMetricsSnapshot | null = null; + private wasAlerting = false; + private readonly maxConnections: number; + + constructor( + private pool: Pool | ReadWritePool, + opts: PoolMetricsOptions = {}, + ) { + this.intervalMs = opts.intervalMs ?? 10_000; + this.alertThreshold = opts.alertThreshold ?? 0.8; + this.onSnapshot = opts.onSnapshot; + this.onAlert = opts.onAlert; + this.maxConnections = (pool as Pool & { max?: number }).max ?? 20; + } + + /** Start periodic sampling. Returns this for chaining. */ + start(): this { + if (this.timer) return this; + // Sample immediately, then on interval + void this.sample(); + this.timer = setInterval(() => void this.sample(), this.intervalMs); + return this; + } + + /** Stop periodic sampling. */ + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** True when the collector is actively sampling. */ + isRunning(): boolean { + return this.timer !== null; + } + + /** Take a single snapshot without starting the interval. */ + sample(): PoolMetricsSnapshot { + const pool = this.pool as Pool & { + getLagStates?: () => { name: string; lagMs: number | null; rerouted: boolean }[]; + }; + + const totalCount = pool.totalCount ?? 0; + const idleCount = pool.idleCount ?? 0; + const waitingCount = pool.waitingCount ?? 0; + const activeCount = totalCount - idleCount; + const utilization = this.maxConnections > 0 ? activeCount / this.maxConnections : 0; + const saturated = waitingCount > 0; + + const snapshot: PoolMetricsSnapshot = { + timestamp: Date.now(), + totalCount, + idleCount, + waitingCount, + activeCount, + maxConnections: this.maxConnections, + utilization: Math.round(utilization * 1000) / 1000, + saturated, + }; + + // Attach replica lag states if available + if (typeof pool.getLagStates === 'function') { + snapshot.replicas = pool.getLagStates(); + } + + this.lastSnapshot = snapshot; + this.onSnapshot?.(snapshot); + + // Alert on threshold crossing (edge-triggered) + const isAlerting = utilization >= this.alertThreshold; + if (isAlerting && !this.wasAlerting) { + this.onAlert?.(snapshot); + } + this.wasAlerting = isAlerting; + + return snapshot; + } + + /** Return the most recent snapshot, or null if none has been taken. */ + getLastSnapshot(): PoolMetricsSnapshot | null { + return this.lastSnapshot; + } + + /** Return a JSON-safe summary suitable for `/health/db` responses. */ + getHealthSummary(): { + healthy: boolean; + utilization: number; + saturated: boolean; + snapshot: PoolMetricsSnapshot | null; + } { + const snapshot = this.lastSnapshot; + if (!snapshot) { + return { healthy: true, utilization: 0, saturated: false, snapshot: null }; + } + const healthy = !snapshot.saturated && snapshot.utilization < this.alertThreshold; + return { + healthy, + utilization: snapshot.utilization, + saturated: snapshot.saturated, + snapshot, + }; + } +}