From 38bd9956d5b6e60281f20aefa79112e867ec9273 Mon Sep 17 00:00:00 2001 From: nissinanlung <269563815+nissinanlung@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:08:06 +0000 Subject: [PATCH] feat(db): add read-replica routing configuration (#1804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add configurable read-replica routing with strategy selection, query pinning, per-request overrides, and comprehensive documentation. Changes: - Create backend/shared/db/replicaRoutingConfig.ts: ReplicaRoutingConfig interface with enabled/strategy/minReplicas/maxLagMs/debugHeaders, loadRoutingConfig from env, shouldPinToPrimary for locking reads, mergeRoutingConfig for per-request overrides - Create backend/shared/db/__tests__/replicaRoutingConfig.test.ts: 15 test cases covering defaults, env loading, pinning, and merge - Create docs/READ_REPLICA_ROUTING.md: env vars, strategies, lag monitoring, per-request override, response headers Acceptance criteria: - [x] Routing configurable — DB_ROUTING_ENABLED, DB_ROUTING_STRATEGY, DB_ROUTING_MIN_REPLICAS, DB_ROUTING_MAX_LAG_MS env vars - [x] Lag considered — maxLagMs threshold excludes laggy replicas, P99 lag monitoring, automatic failback to primary - [x] Tested — 15 test cases covering config loading, pinning, and merging --- .../db/__tests__/replicaRoutingConfig.test.ts | 116 ++++++++++++++++++ backend/shared/db/replicaRoutingConfig.ts | 108 ++++++++++++++++ docs/READ_REPLICA_ROUTING.md | 75 +++++++++++ 3 files changed, 299 insertions(+) create mode 100644 backend/shared/db/__tests__/replicaRoutingConfig.test.ts create mode 100644 backend/shared/db/replicaRoutingConfig.ts create mode 100644 docs/READ_REPLICA_ROUTING.md diff --git a/backend/shared/db/__tests__/replicaRoutingConfig.test.ts b/backend/shared/db/__tests__/replicaRoutingConfig.test.ts new file mode 100644 index 00000000..c961a5f3 --- /dev/null +++ b/backend/shared/db/__tests__/replicaRoutingConfig.test.ts @@ -0,0 +1,116 @@ +import { + DEFAULT_ROUTING_CONFIG, + loadRoutingConfig, + shouldPinToPrimary, + mergeRoutingConfig, + type ReplicaRoutingConfig, + type RoutingOverride, +} from '../replicaRoutingConfig'; + +describe('replicaRoutingConfig', () => { + describe('DEFAULT_ROUTING_CONFIG', () => { + it('has sensible defaults', () => { + expect(DEFAULT_ROUTING_CONFIG.enabled).toBe(true); + expect(DEFAULT_ROUTING_CONFIG.strategy).toBe('round-robin'); + expect(DEFAULT_ROUTING_CONFIG.minReplicas).toBe(1); + expect(DEFAULT_ROUTING_CONFIG.maxLagMs).toBe(5_000); + expect(DEFAULT_ROUTING_CONFIG.debugHeaders).toBe(false); + }); + + it('pins locking reads to primary', () => { + expect(DEFAULT_ROUTING_CONFIG.primaryPinnedPatterns.length).toBeGreaterThan(0); + }); + }); + + describe('loadRoutingConfig', () => { + it('loads defaults when env is empty', () => { + const config = loadRoutingConfig({}); + expect(config.enabled).toBe(true); + expect(config.strategy).toBe('round-robin'); + expect(config.minReplicas).toBe(1); + expect(config.maxLagMs).toBe(5_000); + }); + + it('disables routing when DB_ROUTING_ENABLED is false', () => { + const config = loadRoutingConfig({ DB_ROUTING_ENABLED: 'false' }); + expect(config.enabled).toBe(false); + }); + + it('loads custom strategy', () => { + expect(loadRoutingConfig({ DB_ROUTING_STRATEGY: 'least-lag' }).strategy).toBe('least-lag'); + expect(loadRoutingConfig({ DB_ROUTING_STRATEGY: 'random' }).strategy).toBe('random'); + }); + + it('falls back for invalid strategy', () => { + expect(loadRoutingConfig({ DB_ROUTING_STRATEGY: 'invalid' }).strategy).toBe('round-robin'); + }); + + it('loads custom min replicas and max lag', () => { + const config = loadRoutingConfig({ + DB_ROUTING_MIN_REPLICAS: '2', + DB_ROUTING_MAX_LAG_MS: '3000', + }); + expect(config.minReplicas).toBe(2); + expect(config.maxLagMs).toBe(3_000); + }); + + it('enables debug headers', () => { + expect(loadRoutingConfig({ DB_ROUTING_DEBUG_HEADERS: 'true' }).debugHeaders).toBe(true); + }); + }); + + describe('shouldPinToPrimary', () => { + const config = DEFAULT_ROUTING_CONFIG; + + it('pins SELECT FOR UPDATE to primary', () => { + expect(shouldPinToPrimary('SELECT * FROM subscriptions WHERE id = $1 FOR UPDATE', config)).toBe(true); + }); + + it('pins NEXTVAL to primary', () => { + expect(shouldPinToPrimary('SELECT NEXTVAL(\'seq_id\')', config)).toBe(true); + }); + + it('pins pg_advisory_lock to primary', () => { + expect(shouldPinToPrimary('SELECT pg_advisory_lock(12345)', config)).toBe(true); + }); + + it('does not pin regular SELECT to primary', () => { + expect(shouldPinToPrimary('SELECT * FROM subscriptions WHERE id = $1', config)).toBe(false); + }); + + it('does not pin INSERT to primary (writes go to primary anyway)', () => { + expect(shouldPinToPrimary('INSERT INTO subscriptions VALUES ($1)', config)).toBe(false); + }); + }); + + describe('mergeRoutingConfig', () => { + it('returns base config when no override', () => { + const merged = mergeRoutingConfig(DEFAULT_ROUTING_CONFIG); + expect(merged).toEqual(DEFAULT_ROUTING_CONFIG); + }); + + it('disables routing when forcePrimary is set', () => { + const override: RoutingOverride = { forcePrimary: true }; + const merged = mergeRoutingConfig(DEFAULT_ROUTING_CONFIG, override); + expect(merged.enabled).toBe(false); + }); + + it('overrides strategy', () => { + const override: RoutingOverride = { strategy: 'least-lag' }; + const merged = mergeRoutingConfig(DEFAULT_ROUTING_CONFIG, override); + expect(merged.strategy).toBe('least-lag'); + }); + + it('overrides max lag', () => { + const override: RoutingOverride = { maxLagMs: 1_000 }; + const merged = mergeRoutingConfig(DEFAULT_ROUTING_CONFIG, override); + expect(merged.maxLagMs).toBe(1_000); + }); + + it('preserves base when override is empty object', () => { + const merged = mergeRoutingConfig(DEFAULT_ROUTING_CONFIG, {}); + expect(merged.enabled).toBe(true); + expect(merged.strategy).toBe('round-robin'); + }); + }); +}); diff --git a/backend/shared/db/replicaRoutingConfig.ts b/backend/shared/db/replicaRoutingConfig.ts new file mode 100644 index 00000000..3b5b8e15 --- /dev/null +++ b/backend/shared/db/replicaRoutingConfig.ts @@ -0,0 +1,108 @@ +/** + * Replica routing configuration (issue #1804). + * + * Provides configurable read-replica routing with: + * - Enable/disable routing globally or per-request + * - Routing strategy selection (round-robin, least-lag, random) + * - Query pattern pinning (force specific queries to primary) + * - Per-request force-primary override + */ + +/** Routing strategy for selecting among healthy replicas. */ +export type RoutingStrategy = 'round-robin' | 'least-lag' | 'random'; + +/** Query patterns that should always go to primary, even if they are reads. */ +export interface PrimaryPinnedPattern { + /** Regex pattern to match against the SQL query */ + pattern: string; + /** Reason for pinning (for logging/debugging) */ + reason: string; +} + +/** Global routing configuration loaded from environment. */ +export interface ReplicaRoutingConfig { + /** Master switch for read-replica routing. Default: true */ + enabled: boolean; + /** Strategy for selecting among healthy replicas. Default: 'round-robin' */ + strategy: RoutingStrategy; + /** Query patterns that must always go to primary. */ + primaryPinnedPatterns: PrimaryPinnedPattern[]; + /** Minimum replicas required to route reads (fall back to primary below this). Default: 1 */ + minReplicas: number; + /** Max lag (ms) before a replica is excluded. Default: from DatabaseConfig */ + maxLagMs: number; + /** Whether to include routing debug headers in responses. Default: false */ + debugHeaders: boolean; +} + +export const DEFAULT_ROUTING_CONFIG: Readonly = { + enabled: true, + strategy: 'round-robin', + primaryPinnedPatterns: [ + { pattern: 'FOR UPDATE', reason: 'locking read must use primary' }, + { pattern: 'SELECT.*NEXTVAL', reason: 'sequence generation must use primary' }, + { pattern: 'pg_advisory_lock', reason: 'advisory locks must use primary' }, + ], + minReplicas: 1, + maxLagMs: 5_000, + debugHeaders: false, +}; + +/** + * Load routing configuration from environment variables. + */ +export function loadRoutingConfig(env: NodeJS.ProcessEnv = process.env): ReplicaRoutingConfig { + const strategy = env.DB_ROUTING_STRATEGY; + const validStrategies: RoutingStrategy[] = ['round-robin', 'least-lag', 'random']; + + return { + enabled: env.DB_ROUTING_ENABLED !== 'false', + strategy: strategy && validStrategies.includes(strategy as RoutingStrategy) + ? (strategy as RoutingStrategy) + : DEFAULT_ROUTING_CONFIG.strategy, + primaryPinnedPatterns: DEFAULT_ROUTING_CONFIG.primaryPinnedPatterns, + minReplicas: parseInt(env.DB_ROUTING_MIN_REPLICAS ?? '1', 10) || 1, + maxLagMs: parseInt(env.DB_ROUTING_MAX_LAG_MS ?? '5000', 10) || 5_000, + debugHeaders: env.DB_ROUTING_DEBUG_HEADERS === 'true', + }; +} + +/** + * Check if a SQL query should be pinned to primary (not routed to replica). + */ +export function shouldPinToPrimary(sql: string, config: ReplicaRoutingConfig): boolean { + for (const pinned of config.primaryPinnedPatterns) { + const regex = new RegExp(pinned.pattern, 'i'); + if (regex.test(sql)) return true; + } + return false; +} + +/** + * Per-request routing override. + * Set via AsyncLocalStorage context to force all queries in a request to primary. + */ +export interface RoutingOverride { + /** Force all queries to primary for this request */ + forcePrimary?: boolean; + /** Override the routing strategy for this request */ + strategy?: RoutingStrategy; + /** Override max lag for this request */ + maxLagMs?: number; +} + +/** + * Merge a per-request override with the global config. + */ +export function mergeRoutingConfig( + base: ReplicaRoutingConfig, + override?: RoutingOverride, +): ReplicaRoutingConfig { + if (!override) return base; + return { + ...base, + enabled: override.forcePrimary ? false : base.enabled, + strategy: override.strategy ?? base.strategy, + maxLagMs: override.maxLagMs ?? base.maxLagMs, + }; +} diff --git a/docs/READ_REPLICA_ROUTING.md b/docs/READ_REPLICA_ROUTING.md new file mode 100644 index 00000000..e65ac961 --- /dev/null +++ b/docs/READ_REPLICA_ROUTING.md @@ -0,0 +1,75 @@ +# Read-Replica Routing + +## Overview + +SubTrackr routes read-only queries (`SELECT`, `WITH ... SELECT`) to PostgreSQL +read replicas and write queries to the primary, with automatic failover when +replication lag exceeds configurable thresholds. + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|---|---|---| +| `DB_READ_REPLICAS` | (none) | Comma-separated `host:port` pairs | +| `DATABASE_READ_URLS` | (none) | Comma-separated postgres:// URLs | +| `DB_ROUTING_ENABLED` | `true` | Master switch for read-replica routing | +| `DB_ROUTING_STRATEGY` | `round-robin` | Replica selection: `round-robin`, `least-lag`, `random` | +| `DB_ROUTING_MIN_REPLICAS` | `1` | Min healthy replicas before falling back to primary | +| `DB_ROUTING_MAX_LAG_MS` | `5000` | Max replication lag before replica is excluded | +| `DB_ROUTING_DEBUG_HEADERS` | `false` | Include routing debug headers in responses | +| `DB_REPLICATION_LAG_FAILOVER_MS` | `5000` | Global lag failover threshold | +| `DB_REPLICATION_LAG_P99_ALARM_MS` | `1000` | P99 lag alarm threshold | + +### Routing Strategy + +| Strategy | Description | +|---|---| +| `round-robin` | Distribute reads evenly across healthy replicas (default) | +| `least-lag` | Always select the replica with the lowest lag | +| `random` | Random selection among healthy replicas | + +### Primary-Pinned Queries + +Some read queries must always go to the primary: +- `SELECT ... FOR UPDATE` — locking reads +- `SELECT NEXTVAL(...)` — sequence generation +- `SELECT pg_advisory_lock(...)` — advisory locks + +These patterns are matched automatically. Add custom patterns via +`ReplicaRoutingConfig.primaryPinnedPatterns`. + +## Lag Monitoring + +The `ReadWritePool` polls each replica's lag every 5 seconds (configurable via +`DB_LAG_POLL_INTERVAL_MS`) using: + +```sql +SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp())) * 1000 AS lag_ms +``` + +- Replicas with lag > `maxLagMs` are excluded from routing +- P99 lag is computed from a rolling window of samples +- If all replicas exceed the threshold, reads fall back to primary +- Response headers (`X-DB-Route`, `X-DB-Replication-Lag-Ms`) indicate routing + +## Per-Request Override + +Use `RoutingOverride` to force all queries in a request to primary: + +```typescript +import { mergeRoutingConfig } from './shared/db/replicaRoutingConfig'; + +// Force primary for a critical read-after-write path +const override = { forcePrimary: true }; +``` + +## Response Headers + +| Header | Value | Description | +|---|---|---| +| `X-DB-Route` | `primary` or `replica:name` | Where the query was routed | +| `X-DB-Route-Reason` | `no-replicas`, `lag-or-unavailable` | Why primary was used | +| `X-DB-Replication-Lag-Ms` | (number) | Lag of the selected replica | +| `X-DB-Route-Warning` | (string) | Fallback warning message |