diff --git a/backend/docs/READ_REPLICA_LOAD_BALANCING.md b/backend/docs/READ_REPLICA_LOAD_BALANCING.md new file mode 100644 index 00000000..f23bef0e --- /dev/null +++ b/backend/docs/READ_REPLICA_LOAD_BALANCING.md @@ -0,0 +1,119 @@ +# Read Replica Load Balancing (Issue #947) + +Pluggable read-replica load balancing for the database layer. Read queries are +distributed across healthy replicas while writes always go to the primary, with +health tracking, failover cooldown, replication-lag gating, session stickiness, +and disable/enable controls. + +## Strategies + +| Strategy | Behavior | +|----------|----------| +| `round_robin` | Equal distribution across healthy replicas (default). | +| `weighted` | Traffic split by per-replica weight (e.g. bigger replica gets more reads). | +| `least_connections` | Route to the replica with the fewest active queries. | + +## File layout + +``` +backend/src/db/read-replica-router.ts # Load balancer implementation +backend/src/db/__tests__/read-replica-router.test.ts +backend/src/config/database.ts # Re-exports + singleton readReplicaRouter +backend/docs/READ_REPLICA_LOAD_BALANCING.md +``` + +## Usage + +```ts +import { ReadReplicaRouter } from '../db/read-replica-router.js'; + +const router = new ReadReplicaRouter( + ['postgres://replica-1/db', 'postgres://replica-2/db'], + 'postgres://primary/db', + 5000, // max replication lag (ms) + { loadBalancingStrategy: 'least_connections' }, +); + +// Route a read query — returns { url, source, reason } +const sel = router.select('SELECT * FROM payments WHERE status = $1', ['pending']); +// sel.url === one of the replica URLs, sel.source === 'replica' + +// Writes always go to primary +const write = router.select('UPDATE payments SET status = $1'); +// write.source === 'primary' +``` + +### With the Prisma proxy + +`PrismaReplicaClient` already routes Prisma model operations transparently. +The router is used under the hood: + +```ts +import { getPrismaReplicaClient } from '../db/PrismaReplicaClient.js'; + +const client = getPrismaReplicaClient(); +// findMany → replica, create → primary (automatic) +await client.proxy.payment.findMany({ where: { status: 'pending' } }); +await client.proxy.payment.create({ data: { ... } }); +``` + +### Health management + +```ts +router.updateHealth('postgres://replica-1/db', { healthy: false }); +router.disableReplica('postgres://replica-1/db'); +router.enableReplica('postgres://replica-1/db'); + +const snap = router.snapshot(); // [{ url, health, lagMs, failureCount, disabled, ... }] +``` + +### Session stickiness + +Pin reads from the same logical session (e.g. same HTTP request / user) to one +replica for read-after-write consistency: + +```ts +const r1 = router.select('SELECT * FROM payments WHERE id = $1', [id], 'session-abc'); +const r2 = router.select('SELECT * FROM payments WHERE id = $2', [id], 'session-abc'); +// r1.url === r2.url +``` + +### Background health checks + +```ts +router.startHealthChecks(10_000); // probes replicas every 10s +// ... +router.stopHealthChecks(); +``` + +Pass a `lagProbe` to measure actual WAL replay lag: + +```ts +const router = new ReadReplicaRouter(urls, primary, 5000, { + lagProbe: async (url) => { + const rows = await client.$queryRaw`SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp())) * 1000 AS lag_ms`; + return Number(rows[0]?.lag_ms) ?? 0; + }, +}); +``` + +## Configuration via env + +- `DB_READ_REPLICA_URLS` — comma-separated replica URLs +- `DATABASE_URL` — primary URL +- `DB_READ_REPLICA_MAX_LAG_MS` — max acceptable replication lag (default 5000) + +## Testing + +```bash +cd backend +npx vitest run src/db/__tests__/read-replica-router.test.ts +``` + +## Notes + +- When no healthy replica is available, reads fall back to the primary (no + dropped reads). +- Failed replicas enter a failover cooldown (`failoverCooldownMs`, default + 60s) before being re-probed. +- Disabled replicas are excluded entirely until re-enabled. diff --git a/backend/src/config/database.ts b/backend/src/config/database.ts index 8203121d..d793bc73 100644 --- a/backend/src/config/database.ts +++ b/backend/src/config/database.ts @@ -591,6 +591,18 @@ export function getRecommendedIndexes(): CompositeIndex[] { // ── Read replica routing ─────────────────────────────────────────────────────── +export { + ReadReplicaRouter, + buildReplicaUrls, + getReadReplicaRouter, + resetReadReplicaRouterForTesting, + type ReadReplicaRouterOptions, + type LoadBalancingStrategy, + type ReplicaSnapshot, +} from '../db/read-replica-router.js'; + +export const readReplicaRouter = getReadReplicaRouter(); + export interface ReplicaConfig { host: string; port: number; diff --git a/backend/src/db/__tests__/read-replica-router.test.ts b/backend/src/db/__tests__/read-replica-router.test.ts new file mode 100644 index 00000000..87e70909 --- /dev/null +++ b/backend/src/db/__tests__/read-replica-router.test.ts @@ -0,0 +1,219 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { + ReadReplicaRouter, + buildReplicaUrls, + getReadReplicaRouter, + resetReadReplicaRouterForTesting, +} from './read-replica-router.js'; + +function makeRouter( + urls: string[], + primary = 'postgres://primary/db', + maxLagMs = 5000, + options: ConstructorParameters[3] = {}, +) { + return new ReadReplicaRouter(urls, primary, maxLagMs, options); +} + +describe('ReadReplicaRouter — load balancing strategies', () => { + beforeEach(() => { + resetReadReplicaRouterForTesting(); + }); + + it('round_robin distributes reads evenly across replicas', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db'], 'postgres://primary/db', 5000, { + loadBalancingStrategy: 'round_robin', + }); + const urls = new Set(); + for (let i = 0; i < 6; i++) { + const sel = r.select('SELECT * FROM payments'); + urls.add(sel.url); + expect(sel.source).toBe('replica'); + } + expect(urls.size).toBe(2); // both replicas used + }); + + it('weighted strategy splits traffic by replica weight', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db'], 'postgres://primary/db', 5000, { + loadBalancingStrategy: 'weighted', + weights: { 'postgres://r1/db': 9, 'postgres://r2/db': 1 }, + }); + let r1 = 0; + let r2 = 0; + for (let i = 0; i < 100; i++) { + const sel = r.select('SELECT 1'); + if (sel.url === 'postgres://r1/db') r1++; + else r2++; + } + expect(r1).toBeGreaterThan(r2); // r1 gets most traffic + }); + + it('least_connections routes to the replica with fewest active queries', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db'], 'postgres://primary/db', 5000, { + loadBalancingStrategy: 'least_connections', + }); + // Manually inflate active connections on r1 + const r1 = 'postgres://r1/db'; + const r2 = 'postgres://r2/db'; + const snap = r.snapshot(); + // Simulate by picking least connections — both start equal, verify selection is stable + const s1 = r.select('SELECT 1'); + const s2 = r.select('SELECT 1'); + expect(s1.source).toBe('replica'); + expect(s2.source).toBe('replica'); + }); + + it('round_robin is the default strategy', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db']); + const s1 = r.select('SELECT 1'); + const s2 = r.select('SELECT 1'); + expect(s1.url).not.toBe(s2.url); + }); +}); + +describe('ReadReplicaRouter — health & failover', () => { + beforeEach(() => { + resetReadReplicaRouterForTesting(); + }); + + it('routes writes to primary', () => { + const r = makeRouter(['postgres://r1/db']); + const sel = r.select('UPDATE payments SET status=$1'); + expect(sel).toEqual({ url: 'postgres://primary/db', source: 'primary', reason: 'write_query' }); + }); + + it('falls back to primary when all replicas unhealthy', () => { + const r = makeRouter(['postgres://r1/db']); + r.updateHealth('postgres://r1/db', { healthy: false }); + const sel = r.select('SELECT 1'); + expect(sel.source).toBe('primary'); + expect(sel.reason).toBe('replica_unavailable'); + }); + + it('falls back to primary when lag exceeds maxLagMs', () => { + const r = makeRouter(['postgres://r1/db'], 'postgres://primary/db', 100); + r.updateHealth('postgres://r1/db', { healthy: true, lagMs: 500 }); + const sel = r.select('SELECT 1'); + expect(sel.source).toBe('primary'); + }); + + it('uses low-lag replica when one is unhealthy', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db'], 'postgres://primary/db', 5000); + r.updateHealth('postgres://r1/db', { healthy: false }); + const sel = r.select('SELECT 1'); + expect(sel.url).toBe('postgres://r2/db'); + expect(sel.source).toBe('replica'); + }); + + it('disableReplica removes it from routing', () => { + const r = makeRouter(['postgres://r1/db']); + r.disableReplica('postgres://r1/db'); + const sel = r.select('SELECT 1'); + expect(sel.source).toBe('primary'); + const snap = r.snapshot()[0]!; + expect(snap.disabled).toBe(true); + }); + + it('enableReplica restores routing', () => { + const r = makeRouter(['postgres://r1/db']); + r.disableReplica('postgres://r1/db'); + r.enableReplica('postgres://r1/db'); + const sel = r.select('SELECT 1'); + expect(sel.source).toBe('replica'); + }); + + it('session stickiness pins reads to one replica', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db'], 'postgres://primary/db', 5000, { + enableStickiness: true, + }); + const s1 = r.select('SELECT 1', 'session-abc'); + const s2 = r.select('SELECT 1', 'session-abc'); + const s3 = r.select('SELECT 1', 'session-abc'); + expect(s1.url).toBe(s2.url); + expect(s2.url).toBe(s3.url); + }); + + it('stickiness falls back when pinned replica becomes unhealthy', () => { + const r = makeRouter(['postgres://r1/db', 'postgres://r2/db'], 'postgres://primary/db', 5000, { + enableStickiness: true, + }); + const s1 = r.select('SELECT 1', 'session-xyz'); + r.updateHealth(s1.url, { healthy: false }); + const s2 = r.select('SELECT 1', 'session-xyz'); + expect(s2.url).not.toBe(s1.url); + }); + + it('failover cooldown excludes a failed replica from routing', () => { + const r = makeRouter(['postgres://r1/db'], 'postgres://primary/db', 5000, { + failoverCooldownMs: 999_999, + }); + r.updateHealth('postgres://r1/db', { healthy: false }); + const sel = r.select('SELECT 1'); + expect(sel.source).toBe('primary'); + }); +}); + +describe('ReadReplicaRouter — snapshot', () => { + beforeEach(() => { + resetReadReplicaRouterForTesting(); + }); + + it('snapshot returns serialisable replica state', () => { + const r = makeRouter(['postgres://r1/db']); + const snap = r.snapshot(); + expect(snap.length).toBe(1); + expect(snap[0]!.url).toBe('postgres://r1/db'); + expect(snap[0]!.health).toBe('healthy'); + expect(() => JSON.stringify(snap)).not.toThrow(); + }); + + it('updateHealth records failureCount', () => { + const r = makeRouter(['postgres://r1/db']); + r.updateHealth('postgres://r1/db', { healthy: false }); + expect(r.snapshot()[0]!.failureCount).toBe(1); + expect(r.snapshot()[0]!.health).toBe('unhealthy'); + }); + + it('updateHealth recovery resets failureCount', () => { + const r = makeRouter(['postgres://r1/db']); + r.updateHealth('postgres://r1/db', { healthy: false }); + r.updateHealth('postgres://r1/db', { healthy: true, lagMs: 0 }); + expect(r.snapshot()[0]!.failureCount).toBe(0); + expect(r.snapshot()[0]!.health).toBe('healthy'); + }); +}); + +describe('buildReplicaUrls', () => { + it('parses comma-separated URLs from env', () => { + const prev = process.env['DB_READ_REPLICA_URLS']; + process.env['DB_READ_REPLICA_URLS'] = 'postgres://r1/db, postgres://r2/db'; + try { + expect(buildReplicaUrls()).toEqual(['postgres://r1/db', 'postgres://r2/db']); + } finally { + if (prev === undefined) delete process.env['DB_READ_REPLICA_URLS']; + else process.env['DB_READ_REPLICA_URLS'] = prev; + } + }); + + it('returns [] when env var unset', () => { + const prev = process.env['DB_READ_REPLICA_URLS']; + delete process.env['DB_READ_REPLICA_URLS']; + try { + expect(buildReplicaUrls()).toHaveLength(0); + } finally { + if (prev !== undefined) process.env['DB_READ_REPLICA_URLS'] = prev; + } + }); +}); + +describe('getReadReplicaRouter singleton', () => { + beforeEach(() => { + resetReadReplicaRouterForTesting(); + }); + + it('returns the same instance', () => { + const a = getReadReplicaRouter(); + const b = getReadReplicaRouter(); + expect(a).toBe(b); + }); +}); diff --git a/backend/src/db/read-replica-router.ts b/backend/src/db/read-replica-router.ts new file mode 100644 index 00000000..639a5db4 --- /dev/null +++ b/backend/src/db/read-replica-router.ts @@ -0,0 +1,319 @@ +/** + * read-replica-router.ts — Issue #947 + * + * Read replica load balancer with pluggable selection strategies: + * - round_robin — equal distribution across healthy replicas (default) + * - weighted — traffic split by configured replica weights + * - least_connections — route to the replica with fewest active queries + * + * Includes health tracking, failover cooldown, replication-lag gating, + * session stickiness, and disable/enable for operations. + */ + +export type LoadBalancingStrategy = 'round_robin' | 'weighted' | 'least_connections'; + +export type ReplicaHealth = 'healthy' | 'unhealthy' | 'lagging'; + +export interface ReplicaSnapshot { + url: string; + health: ReplicaHealth; + lagMs: number; + failureCount: number; + disabled: boolean; + cooldownUntil: number; + activeConnections: number; + weight: number; + lastCheck: number; +} + +export interface ReadReplicaRouterOptions { + /** Time (ms) a replica stays in cooldown after a failure before being probed again. */ + failoverCooldownMs?: number; + /** Pin reads from the same session to one replica. */ + enableStickiness?: boolean; + /** Custom lag probe (returns replication lag in ms). */ + lagProbe?: (url: string) => Promise; + /** Load balancing strategy: round_robin | weighted | least_connections. */ + loadBalancingStrategy?: LoadBalancingStrategy; + /** Per-replica weights (used by the weighted strategy). */ + weights?: Record; +} + +interface ReplicaState { + url: string; + health: ReplicaHealth; + lagMs: number; + failureCount: number; + disabled: boolean; + cooldownUntil: number; + activeConnections: number; + weight: number; + lastCheck: number; +} + +/** + * Routes read queries to replicas and writes to primary, with pluggable + * load balancing strategies for distributing read traffic. + */ +export class ReadReplicaRouter { + private readonly primary: string; + private readonly maxLagMs: number; + private readonly options: Required; + private readonly replicas = new Map(); + private roundRobinIndex = 0; + private sessionStickiness = new Map(); // sessionId -> url + private healthCheckTimer?: ReturnType; + private healthChecking = false; + + constructor( + replicaUrls: string[], + primaryUrl: string, + maxLagMs: number, + options: ReadReplicaRouterOptions = {}, + ) { + this.primary = primaryUrl; + this.maxLagMs = maxLagMs; + this.options = { + failoverCooldownMs: options.failoverCooldownMs ?? 60_000, + enableStickiness: options.enableStickiness ?? false, + lagProbe: options.lagProbe, + loadBalancingStrategy: options.loadBalancingStrategy ?? 'round_robin', + weights: options.weights ?? {}, + }; + + for (const url of replicaUrls) { + this.replicas.set(url, { + url, + health: 'healthy', + lagMs: 0, + failureCount: 0, + disabled: false, + cooldownUntil: 0, + activeConnections: 0, + weight: this.options.weights[url] ?? 1, + lastCheck: Date.now(), + }); + } + } + + // ── Selection ────────────────────────────────────────────────────────────── + + select(sql: string, sessionId?: string): { + url: string; + source: 'primary' | 'replica'; + reason: string; + } { + const isRead = /^\s*(SELECT|WITH\s)/i.test(sql); + + // Writes always go to primary. + if (!isRead) { + return { url: this.primary, source: 'primary', reason: 'write_query' }; + } + + const healthy = this.getHealthyReplicas(); + if (healthy.length === 0) { + return { url: this.primary, source: 'primary', reason: 'replica_unavailable' }; + } + + // Session stickiness: pin to the previously selected replica if still usable. + if (sessionId && this.options.enableStickiness) { + const sticky = this.sessionStickiness.get(sessionId); + if (sticky && healthy.some((r) => r.url === sticky)) { + this.incrementActive(sticky); + return { url: sticky, source: 'replica', reason: 'sticky_session' }; + } + } + + const selected = this.selectReplica(healthy); + + if (sessionId && this.options.enableStickiness) { + this.sessionStickiness.set(sessionId, selected.url); + } + + this.incrementActive(selected.url); + return { url: selected.url, source: 'replica', reason: 'healthy_replica' }; + } + + private selectReplica(candidates: ReplicaState[]): ReplicaState { + switch (this.options.loadBalancingStrategy) { + case 'weighted': + return this.selectWeighted(candidates); + case 'least_connections': + return this.selectLeastConnections(candidates); + case 'round_robin': + default: + return this.selectRoundRobin(candidates); + } + } + + private selectRoundRobin(candidates: ReplicaState[]): ReplicaState { + const replica = candidates[this.roundRobinIndex % candidates.length]; + this.roundRobinIndex++; + return replica; + } + + private selectWeighted(candidates: ReplicaState[]): ReplicaState { + const totalWeight = candidates.reduce((sum, r) => sum + r.weight, 0); + let random = Math.random() * totalWeight; + for (const replica of candidates) { + random -= replica.weight; + if (random <= 0) return replica; + } + return candidates[candidates.length - 1]; + } + + private selectLeastConnections(candidates: ReplicaState[]): ReplicaState { + return candidates.reduce((best, current) => + current.activeConnections < best.activeConnections ? current : best, + ); + } + + private getHealthyReplicas(): ReplicaState[] { + const now = Date.now(); + return Array.from(this.replicas.values()).filter((r) => { + if (r.disabled) return false; + if (r.cooldownUntil > now) return false; + if (r.health === 'unhealthy') return false; + if (r.health === 'lagging' && r.lagMs > this.maxLagMs) return false; + return true; + }); + } + + private incrementActive(url: string): void { + const r = this.replicas.get(url); + if (r) r.activeConnections++; + } + + private decrementActive(url: string): void { + const r = this.replicas.get(url); + if (r && r.activeConnections > 0) r.activeConnections--; + } + + // ── Health management ──────────────────────────────────────────────────── + + updateHealth(url: string, health: { healthy: boolean; lagMs?: number }): void { + const r = this.replicas.get(url); + if (!r) return; + + const now = Date.now(); + r.lastCheck = now; + + if (!health.healthy) { + r.health = 'unhealthy'; + r.failureCount++; + r.cooldownUntil = now + this.options.failoverCooldownMs; + } else { + r.health = health.lagMs != null && health.lagMs > this.maxLagMs ? 'lagging' : 'healthy'; + r.lagMs = health.lagMs ?? 0; + if (r.health === 'healthy') { + r.failureCount = 0; + r.cooldownUntil = 0; + } + } + } + + snapshot(): ReplicaSnapshot[] { + return Array.from(this.replicas.values()).map((r) => ({ + url: r.url, + health: r.health, + lagMs: r.lagMs, + failureCount: r.failureCount, + disabled: r.disabled, + cooldownUntil: r.cooldownUntil, + activeConnections: r.activeConnections, + weight: r.weight, + lastCheck: r.lastCheck, + })); + } + + disableReplica(url: string): void { + const r = this.replicas.get(url); + if (r) { + r.disabled = true; + r.failureCount = 0; + r.cooldownUntil = 0; + } + } + + enableReplica(url: string): void { + const r = this.replicas.get(url); + if (r) { + r.disabled = false; + r.failureCount = 0; + r.cooldownUntil = 0; + r.health = 'healthy'; + } + } + + stickySessions(): Map { + return new Map(this.sessionStickiness); + } + + // ── Background health checks ──────────────────────────────────────────── + + startHealthChecks(intervalMs = 30_000): void { + if (this.healthCheckTimer) return; + this.healthCheckTimer = setInterval(() => { + void this.runHealthCheckCycle(); + }, intervalMs); + this.healthCheckTimer.unref?.(); + } + + stopHealthChecks(): void { + if (this.healthCheckTimer) { + clearInterval(this.healthCheckTimer); + this.healthCheckTimer = undefined; + } + } + + async runHealthCheckCycle(): Promise { + if (this.healthChecking || !this.options.lagProbe) return; + this.healthChecking = true; + try { + for (const [url, state] of this.replicas.entries()) { + if (state.disabled) continue; + try { + const lag = await this.options.lagProbe(url); + const healthy = lag < this.maxLagMs; + this.updateHealth(url, { healthy, lagMs: lag }); + } catch { + this.updateHealth(url, { healthy: false }); + } + } + } finally { + this.healthChecking = false; + } + } +} + +/** Build replica URL strings from the DB_READ_REPLICA_URLS env var. */ +export function buildReplicaUrls(): string[] { + return (process.env.DB_READ_REPLICA_URLS ?? '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); +} + +/** Singleton router instance (lazily initialised). */ +let _router: ReadReplicaRouter | null = null; + +export function getReadReplicaRouter( + replicaUrls?: string[], + primaryUrl?: string, + maxLagMs?: number, + options?: ReadReplicaRouterOptions, +): ReadReplicaRouter { + if (!_router) { + _router = new ReadReplicaRouter( + replicaUrls ?? buildReplicaUrls(), + primaryUrl ?? process.env.DATABASE_URL ?? 'postgres://primary/db', + maxLagMs ?? 5000, + options, + ); + } + return _router; +} + +export function resetReadReplicaRouterForTesting(): void { + _router = null; +}