diff --git a/backend/__tests__/gracefulShutdown.test.ts b/backend/__tests__/gracefulShutdown.test.ts new file mode 100644 index 00000000..d46862ef --- /dev/null +++ b/backend/__tests__/gracefulShutdown.test.ts @@ -0,0 +1,146 @@ +import http from 'node:http'; +import { startServer } from '../server'; +import type { Pool } from '../shared/db/connectionPool'; +import { PlanCacheService } from '../subscription/domain/PlanCacheService'; +import { InMemoryPlanRepository } from '../subscription/domain/PlanRepository'; +import type { RedisClient } from '../shared/cache/types'; +import { setPlanCacheService } from '../subscription/planCacheRegistry'; + +class FakeRedis implements RedisClient { + private store = new Map(); + async get(key: string): Promise { return this.store.get(key) ?? null; } + async set(key: string, value: string): Promise<'OK'> { this.store.set(key, value); return 'OK'; } + async del(): Promise { return 1; } + async keys(): Promise { return []; } + async ping(): Promise { return 'PONG'; } + async quit(): Promise<'OK'> { return 'OK'; } +} + +function makeMockPool(): Pool { + return { + query: jest.fn(async () => ({ rows: [], rowCount: 0 })), + connect: jest.fn(), + end: jest.fn(), + on: jest.fn(), + totalCount: 0, + idleCount: 0, + waitingCount: 0, + } as unknown as Pool; +} + +function makeBootstrap() { + const redis = new FakeRedis(); + const planCache = new PlanCacheService(redis, new InMemoryPlanRepository()); + return { planCache, redis, repository: new InMemoryPlanRepository() }; +} + +async function listenEphemeral(server: http.Server): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + return typeof address === 'object' && address ? address.port : 0; +} + +describe('Graceful Shutdown & Connection Draining (Issue #1792)', () => { + afterEach(() => { + setPlanCacheService(null); + }); + + it('drains in-flight requests and waits for completion before shutdown resolves', async () => { + const pool = makeMockPool(); + const planBootstrap = makeBootstrap(); + const running = await startServer({ + pool, + planBootstrap, + listen: false, + drainTimeoutMs: 5_000, + }); + const port = await listenEphemeral(running.server); + + let requestCompleted = false; + + // Send a request to an endpoint + const reqPromise = new Promise<{ status: number }>((resolve, reject) => { + const req = http.request( + { host: '127.0.0.1', port, path: '/health/live', method: 'GET' }, + (res) => { + res.on('data', () => {}); + res.on('end', () => { + requestCompleted = true; + resolve({ status: res.statusCode ?? 0 }); + }); + }, + ); + req.on('error', reject); + req.end(); + }); + + const response = await reqPromise; + expect(response.status).toBe(200); + expect(requestCompleted).toBe(true); + expect(running.getInFlightRequests?.()).toBe(0); + + running.removeSignalListeners?.(); + await running.shutdown(); + }); + + it('forces socket closure when in-flight request exceeds drain timeout', async () => { + const pool = makeMockPool(); + const planBootstrap = makeBootstrap(); + + // Configure a short drain timeout (200ms) + const running = await startServer({ + pool, + planBootstrap, + listen: false, + drainTimeoutMs: 200, + }); + const port = await listenEphemeral(running.server); + + // Open an uncompleted connection that stays active + const client = http.request({ + host: '127.0.0.1', + port, + path: '/plans', + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + // Swallow the expected ECONNRESET/socket hang up when the server forcefully closes the socket + client.on('error', () => {}); + // Send partial body and do not end the stream yet to keep request in-flight + client.write('{"name": "Hang'); + + + // Give the server 50ms to register the connection + await new Promise((r) => setTimeout(r, 50)); + expect(running.getInFlightRequests?.()).toBeGreaterThanOrEqual(1); + + const start = Date.now(); + running.removeSignalListeners?.(); + // Shutdown should wait up to drainTimeoutMs (200ms) then force socket destruction + await running.shutdown(); + const duration = Date.now() - start; + + expect(duration).toBeGreaterThanOrEqual(150); + expect(running.isDraining?.()).toBe(true); + + client.destroy(); + await new Promise((r) => setTimeout(r, 50)); + }); + + + it('registers and removes signal listeners cleanly', async () => { + const pool = makeMockPool(); + const planBootstrap = makeBootstrap(); + const running = await startServer({ + pool, + planBootstrap, + listen: false, + }); + + expect(typeof running.removeSignalListeners).toBe('function'); + // Remove signal listeners so process signal table is not polluted + running.removeSignalListeners?.(); + + await running.shutdown(); + }); +}); diff --git a/backend/__tests__/probes.test.ts b/backend/__tests__/probes.test.ts new file mode 100644 index 00000000..32419c86 --- /dev/null +++ b/backend/__tests__/probes.test.ts @@ -0,0 +1,195 @@ +import http from 'node:http'; +import { startServer } from '../server'; +import type { Pool } from '../shared/db/connectionPool'; +import { PlanCacheService } from '../subscription/domain/PlanCacheService'; +import { InMemoryPlanRepository } from '../subscription/domain/PlanRepository'; +import type { RedisClient } from '../shared/cache/types'; +import { setPlanCacheService } from '../subscription/planCacheRegistry'; + +class FakeRedis implements RedisClient { + public healthy = true; + private store = new Map(); + + async get(key: string): Promise { + return this.store.get(key) ?? null; + } + async set(key: string, value: string): Promise<'OK'> { + this.store.set(key, value); + return 'OK'; + } + async del(...keys: string[]): Promise { + let n = 0; + for (const k of keys) { + if (this.store.delete(k)) n++; + } + return n; + } + async keys(): Promise { + return []; + } + async ping(): Promise { + if (!this.healthy) throw new Error('Redis connection refused'); + return 'PONG'; + } + async quit(): Promise<'OK'> { + return 'OK'; + } +} + +function makeMockPool(healthy = true): Pool { + return { + query: jest.fn(async (sql: string) => { + if (!healthy) throw new Error('Postgres connection timeout'); + return { rows: [{ '?column?': 1 }], rowCount: 1 }; + }), + connect: jest.fn(), + end: jest.fn(), + on: jest.fn(), + totalCount: 1, + idleCount: 1, + waitingCount: 0, + } as unknown as Pool; +} + +function makeBootstrap(redis = new FakeRedis()) { + const planCache = new PlanCacheService(redis, new InMemoryPlanRepository()); + return { planCache, redis, repository: new InMemoryPlanRepository() }; +} + +function request(port: number, path: string): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const req = http.request( + { + host: '127.0.0.1', + port, + path, + method: 'GET', + }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => { + resolve({ + status: res.statusCode ?? 0, + body: Buffer.concat(chunks).toString('utf8'), + }); + }); + }, + ); + req.on('error', reject); + req.end(); + }); +} + +async function listenEphemeral(server: http.Server): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + return typeof address === 'object' && address ? address.port : 0; +} + +describe('Health and Readiness Probes (Issue #1795)', () => { + afterEach(() => { + setPlanCacheService(null); + }); + + it('responds with 200 and uptime on liveness probe endpoints (/health/live, /livez)', async () => { + const pool = makeMockPool(); + const planBootstrap = makeBootstrap(); + const running = await startServer({ pool, planBootstrap, listen: false }); + const port = await listenEphemeral(running.server); + + const liveRes = await request(port, '/health/live'); + expect(liveRes.status).toBe(200); + const liveJson = JSON.parse(liveRes.body); + expect(liveJson.status).toBe('ok'); + expect(typeof liveJson.uptime).toBe('number'); + + const livezRes = await request(port, '/livez'); + expect(livezRes.status).toBe(200); + const livezJson = JSON.parse(livezRes.body); + expect(livezJson.status).toBe('ok'); + + running.removeSignalListeners?.(); + await running.shutdown(); + }); + + it('responds with 200 when all dependencies (DB, Redis cache) are ready', async () => { + const pool = makeMockPool(true); + const planBootstrap = makeBootstrap(); + const running = await startServer({ pool, planBootstrap, listen: false }); + const port = await listenEphemeral(running.server); + + const readyRes = await request(port, '/health/ready'); + expect(readyRes.status).toBe(200); + const readyJson = JSON.parse(readyRes.body); + expect(readyJson.status).toBe('ok'); + expect(readyJson.ready).toBe(true); + expect(readyJson.checks.database).toBe('ok'); + expect(readyJson.checks.cache).toBe('ok'); + + const readyzRes = await request(port, '/readyz'); + expect(readyzRes.status).toBe(200); + expect(JSON.parse(readyzRes.body).ready).toBe(true); + + running.removeSignalListeners?.(); + await running.shutdown(); + }); + + it('responds with 503 degraded when database dependency check fails', async () => { + const pool = makeMockPool(false); // failing DB query + const planBootstrap = makeBootstrap(); + const running = await startServer({ pool, planBootstrap, listen: false }); + const port = await listenEphemeral(running.server); + + const readyRes = await request(port, '/health/ready'); + expect(readyRes.status).toBe(503); + const readyJson = JSON.parse(readyRes.body); + expect(readyJson.status).toBe('degraded'); + expect(readyJson.ready).toBe(false); + expect(readyJson.checks.database).toBe('failed'); + expect(readyJson.checks.dbError).toContain('Postgres connection timeout'); + + running.removeSignalListeners?.(); + await running.shutdown(); + }); + + it('responds with 503 degraded when cache dependency check fails', async () => { + const pool = makeMockPool(true); + const fakeRedis = new FakeRedis(); + fakeRedis.healthy = false; // failing cache ping + const planBootstrap = makeBootstrap(fakeRedis); + const running = await startServer({ pool, planBootstrap, listen: false }); + const port = await listenEphemeral(running.server); + + const readyRes = await request(port, '/health/ready'); + expect(readyRes.status).toBe(503); + const readyJson = JSON.parse(readyRes.body); + expect(readyJson.status).toBe('degraded'); + expect(readyJson.ready).toBe(false); + expect(readyJson.checks.cache).toBe('failed'); + + running.removeSignalListeners?.(); + await running.shutdown(); + }); + + it('transitions to draining status during shutdown and marks readiness as terminating', async () => { + const pool = makeMockPool(true); + const planBootstrap = makeBootstrap(); + const running = await startServer({ pool, planBootstrap, listen: false }); + const port = await listenEphemeral(running.server); + + // Initial readiness is ok + const initialRes = await request(port, '/health/ready'); + expect(initialRes.status).toBe(200); + expect(running.isDraining?.()).toBe(false); + + // Trigger shutdown + const shutdownPromise = running.shutdown(); + expect(running.isDraining?.()).toBe(true); + + running.removeSignalListeners?.(); + await shutdownPromise; + }); + + +}); diff --git a/backend/server.ts b/backend/server.ts index e7c3e9ca..a126defc 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -69,6 +69,8 @@ export interface StartServerOptions { listen?: boolean; /** Tracer for request spans. Defaults to one configured from OTEL_* env vars. */ tracer?: Tracer; + /** Maximum time in milliseconds to wait for in-flight requests to drain during shutdown. Defaults to 10,000ms. */ + drainTimeoutMs?: number; } export interface RunningServer { @@ -78,8 +80,12 @@ export interface RunningServer { monitoredPool: MonitoredPool; port: number; shutdown: () => Promise; + getInFlightRequests?: () => number; + isDraining?: () => boolean; + removeSignalListeners?: () => void; } + async function ensurePlanCache(pool: Pool): Promise { const bootstrapped = await bootstrapPlanCache({ pool, warmOnStart: true }); if (bootstrapped) { @@ -180,11 +186,16 @@ function buildRateLimitMiddleware() { allowMissingKey: true, skipPaths: [ '/health', + '/health/live', + '/health/ready', + '/livez', + '/readyz', '/metrics/plan-cache', '/metrics/compression', '/metrics/pool', '/metrics/build', ], + // Per-key tier: resolved from the server-side record, not from a // client-supplied header. See resolveTierForApiKey. getTier: (apiKey, req) => resolveTierForApiKey(apiKey, req), @@ -417,11 +428,16 @@ export async function startServer(options: StartServerOptions = {}): Promise(); + const drainTimeoutMs = options.drainTimeoutMs ?? Number(process.env.DRAIN_TIMEOUT_MS ?? 10_000); + const handleRequest = async ( req: http.IncomingMessage, res: http.ServerResponse, @@ -472,8 +493,26 @@ export async function startServer(options: StartServerOptions = {}): Promise { + if (!inFlightDecremented) { + inFlightDecremented = true; + inFlightRequests = Math.max(0, inFlightRequests - 1); + } + }; + res.once('finish', onReqFinished); + res.once('close', onReqFinished); + applySecurityHeadersToResponse(res); + const originalWriteHead = res.writeHead.bind(res); res.writeHead = ((statusCode: number, statusMessage?: string | Record) => { if (!res.headersSent) { @@ -543,8 +582,64 @@ export async function startServer(options: StartServerOptions = {}): Promise handleRequest(req, res)), ); + server.on('connection', (socket) => { + activeSockets.add(socket); + socket.once('close', () => { + activeSockets.delete(socket); + }); + }); + const port = options.port ?? Number(process.env.PORT ?? 3001); const host = options.host ?? process.env.HOST ?? '0.0.0.0'; const shutdown = async (): Promise => { + if (isShuttingDown) return; + isShuttingDown = true; clearInterval(sessionSweepTimer); - await new Promise((resolve, reject) => { + + // Stop accepting new connections + const closeServerPromise = new Promise((resolve, reject) => { server.close((err) => (err ? reject(err) : resolve())); }); + + // Close idle keep-alive connections immediately + if (typeof (server as any).closeIdleConnections === 'function') { + (server as any).closeIdleConnections(); + } + + // Drain in-flight connections with bounded drain timeout + await new Promise((resolve) => { + if (inFlightRequests === 0) { + resolve(); + return; + } + + const checkInterval = setInterval(() => { + if (inFlightRequests === 0) { + clearInterval(checkInterval); + clearTimeout(timeoutTimer); + resolve(); + } + }, 50); + + const timeoutTimer = setTimeout(() => { + clearInterval(checkInterval); + console.warn( + `[Server] Drain timeout of ${drainTimeoutMs}ms exceeded with ${inFlightRequests} in-flight requests remaining; forcing connection termination.`, + ); + for (const socket of activeSockets) { + socket.destroy(); + } + resolve(); + }, drainTimeoutMs); + + checkInterval.unref(); + timeoutTimer.unref(); + }); + + await closeServerPromise.catch(() => {}); + await shutdownPlanCache(planBootstrap); if (!options.pool) { await closePool(); @@ -1869,7 +2018,7 @@ export async function startServer(options: StartServerOptions = {}): Promise { - console.info(`[Server] Received ${signal}, shutting down…`); + console.info(`[Server] Received ${signal}, draining connections and shutting down…`); shutdown() .then(() => process.exit(0)) .catch((err) => { @@ -1878,10 +2027,28 @@ export async function startServer(options: StartServerOptions = {}): Promise handleSignal('SIGTERM')); - process.once('SIGINT', () => handleSignal('SIGINT')); + const sigtermListener = () => handleSignal('SIGTERM'); + const sigintListener = () => handleSignal('SIGINT'); - return { server, pool, planBootstrap, monitoredPool, port, shutdown }; + process.once('SIGTERM', sigtermListener); + process.once('SIGINT', sigintListener); + + const removeSignalListeners = () => { + process.removeListener('SIGTERM', sigtermListener); + process.removeListener('SIGINT', sigintListener); + }; + + return { + server, + pool, + planBootstrap, + monitoredPool, + port, + shutdown, + getInFlightRequests: () => inFlightRequests, + isDraining: () => isShuttingDown, + removeSignalListeners, + }; } if (require.main === module) { @@ -1890,3 +2057,4 @@ if (require.main === module) { process.exit(1); }); } + diff --git a/backend/services/shared/__tests__/outboundRequest.test.ts b/backend/services/shared/__tests__/outboundRequest.test.ts new file mode 100644 index 00000000..0cd760d1 --- /dev/null +++ b/backend/services/shared/__tests__/outboundRequest.test.ts @@ -0,0 +1,149 @@ +import { + outboundRequest, + isIdempotentMethod, + isIdempotentRequest, + OutboundTimeoutError, +} from '../outboundRequest'; + +describe('outboundRequest', () => { + const fakeSleep = jest.fn(async () => {}); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('idempotency checks', () => { + it('identifies standard HTTP idempotent methods', () => { + expect(isIdempotentMethod('GET')).toBe(true); + expect(isIdempotentMethod('get')).toBe(true); + expect(isIdempotentMethod('PUT')).toBe(true); + expect(isIdempotentMethod('DELETE')).toBe(true); + expect(isIdempotentMethod('HEAD')).toBe(true); + expect(isIdempotentMethod('OPTIONS')).toBe(true); + expect(isIdempotentMethod('POST')).toBe(false); + expect(isIdempotentMethod('PATCH')).toBe(false); + }); + + it('considers requests with Idempotency-Key headers safe to retry', () => { + expect(isIdempotentRequest({ method: 'POST' })).toBe(false); + expect( + isIdempotentRequest({ + method: 'POST', + headers: { 'idempotency-key': 'key-123' }, + }), + ).toBe(true); + expect( + isIdempotentRequest({ + method: 'PATCH', + headers: { 'X-Idempotency-Key': 'key-456' }, + }), + ).toBe(true); + }); + }); + + describe('timeouts', () => { + it('aborts and throws OutboundTimeoutError when request hangs beyond timeout', async () => { + const slowFetch = jest.fn((_url: string, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + const err = new Error('The operation was aborted'); + err.name = 'AbortError'; + reject(err); + }); + }); + }); + + await expect( + outboundRequest('https://api.external.com/hang', { + timeoutMs: 50, + maxRetries: 1, + fetchImpl: slowFetch as unknown as typeof fetch, + }), + ).rejects.toThrow(OutboundTimeoutError); + }); + }); + + describe('bounded retries', () => { + it('retries idempotent GET requests on transient 5xx errors up to maxRetries', async () => { + let callCount = 0; + const mockFetch = jest.fn(async () => { + callCount++; + if (callCount < 3) { + return new Response('Internal Server Error', { status: 503 }); + } + return new Response('{"ok":true}', { status: 200 }); + }); + + const response = await outboundRequest('https://api.example.com/data', { + method: 'GET', + maxRetries: 3, + sleep: fakeSleep, + fetchImpl: mockFetch as unknown as typeof fetch, + }); + + expect(response.status).toBe(200); + expect(callCount).toBe(3); + expect(fakeSleep).toHaveBeenCalledTimes(2); + }); + + it('retries POST requests when an Idempotency-Key is provided', async () => { + let callCount = 0; + const mockFetch = jest.fn(async () => { + callCount++; + if (callCount === 1) { + return new Response('Gateway Timeout', { status: 504 }); + } + return new Response('{"charged":true}', { status: 201 }); + }); + + const response = await outboundRequest('https://api.example.com/charge', { + method: 'POST', + headers: { 'Idempotency-Key': 'charge-abc-123' }, + maxRetries: 3, + sleep: fakeSleep, + fetchImpl: mockFetch as unknown as typeof fetch, + }); + + expect(response.status).toBe(201); + expect(callCount).toBe(2); + }); + + it('does NOT retry non-idempotent POST requests without idempotency keys', async () => { + let callCount = 0; + const mockFetch = jest.fn(async () => { + callCount++; + return new Response('Service Unavailable', { status: 503 }); + }); + + const response = await outboundRequest('https://api.example.com/charge', { + method: 'POST', + maxRetries: 3, + sleep: fakeSleep, + fetchImpl: mockFetch as unknown as typeof fetch, + }); + + expect(response.status).toBe(503); + expect(callCount).toBe(1); // Exactly 1 attempt, no retry! + expect(fakeSleep).not.toHaveBeenCalled(); + }); + + it('does NOT retry client errors (400, 401, 404)', async () => { + let callCount = 0; + const mockFetch = jest.fn(async () => { + callCount++; + return new Response('Bad Request', { status: 400 }); + }); + + const response = await outboundRequest('https://api.example.com/data', { + method: 'GET', + maxRetries: 3, + sleep: fakeSleep, + fetchImpl: mockFetch as unknown as typeof fetch, + }); + + expect(response.status).toBe(400); + expect(callCount).toBe(1); + expect(fakeSleep).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/backend/services/shared/__tests__/providerCircuitBreaker.test.ts b/backend/services/shared/__tests__/providerCircuitBreaker.test.ts new file mode 100644 index 00000000..4dfe6b0c --- /dev/null +++ b/backend/services/shared/__tests__/providerCircuitBreaker.test.ts @@ -0,0 +1,139 @@ +import { + ProviderCircuitBreakerRegistry, + executeWithProviderFallback, + CircuitOpenError, +} from '../providerCircuitBreaker'; + +describe('ProviderCircuitBreakerRegistry', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('maintains independent circuit breaker states per provider', async () => { + const registry = new ProviderCircuitBreakerRegistry({ + defaultOptions: { failureThreshold: 2, recoveryTimeoutMs: 10_000 }, + }); + + const failingStripe = async () => { + throw new Error('Stripe timeout'); + }; + const healthyPaddle = async () => 'paddle_ok'; + + // Trip Stripe breaker + await expect(registry.execute('stripe', failingStripe)).rejects.toThrow('Stripe timeout'); + await expect(registry.execute('stripe', failingStripe)).rejects.toThrow('Stripe timeout'); + + expect(registry.getState('stripe')).toBe('open'); + expect(registry.getState('paddle')).toBe('closed'); + + // Paddle remains completely unaffected + const paddleResult = await registry.execute('paddle', healthyPaddle); + expect(paddleResult).toBe('paddle_ok'); + expect(registry.getState('paddle')).toBe('closed'); + }); + + it('triggers fallback function when provider call fails', async () => { + const registry = new ProviderCircuitBreakerRegistry({ + defaultOptions: { failureThreshold: 3 }, + }); + + const fallback = jest.fn(async (err: Error) => ({ + status: 'fallback_applied', + cause: err.message, + })); + + const result = await registry.execute( + 'stripe', + async () => { + throw new Error('503 Service Unavailable'); + }, + fallback, + ); + + expect(result).toEqual({ + status: 'fallback_applied', + cause: '503 Service Unavailable', + }); + expect(fallback).toHaveBeenCalledTimes(1); + }); + + it('triggers registered default fallback when circuit is open', async () => { + const registry = new ProviderCircuitBreakerRegistry({ + defaultOptions: { failureThreshold: 1, recoveryTimeoutMs: 5_000 }, + }); + + registry.registerFallback('quickbooks', (err) => ({ + degraded: true, + errorName: err.name, + })); + + // First call fails and trips circuit + const res1 = await registry.execute('quickbooks', async () => { + throw new Error('Network error'); + }); + expect(res1).toEqual({ degraded: true, errorName: 'Error' }); + expect(registry.getState('quickbooks')).toBe('open'); + + // Second call is short-circuited and directly uses registered fallback + const res2 = await registry.execute('quickbooks', async () => 'never_called'); + expect(res2).toEqual({ degraded: true, errorName: 'CircuitOpenError' }); + }); + + it('recovers provider circuit breaker after recovery timeout', async () => { + const registry = new ProviderCircuitBreakerRegistry({ + defaultOptions: { failureThreshold: 1, recoveryTimeoutMs: 5_000, successThreshold: 1 }, + }); + + // Trip breaker + await expect(registry.execute('circle', async () => { + throw new Error('failed'); + })).rejects.toThrow('failed'); + + expect(registry.getState('circle')).toBe('open'); + + // Advance time past recovery timeout + jest.advanceTimersByTime(5_000); + + // Call in half-open succeeds and transitions back to closed + const res = await registry.execute('circle', async () => 'circle_restored'); + expect(res).toBe('circle_restored'); + expect(registry.getState('circle')).toBe('closed'); + }); + + it('supports executeWithProviderFallback convenience function', async () => { + const fallback = jest.fn(() => 'cached_backup_data'); + + const result = await executeWithProviderFallback( + 'freshbooks', + async () => { + throw new Error('Gateway timeout'); + }, + fallback, + ); + + expect(result).toBe('cached_backup_data'); + expect(fallback).toHaveBeenCalled(); + }); + + it('allows manual reset of individual and all provider breakers', async () => { + const registry = new ProviderCircuitBreakerRegistry({ + defaultOptions: { failureThreshold: 1 }, + }); + + await expect(registry.execute('p1', async () => { throw new Error('e1'); })).rejects.toThrow(); + await expect(registry.execute('p2', async () => { throw new Error('e2'); })).rejects.toThrow(); + + expect(registry.getAllStates()).toEqual({ p1: 'open', p2: 'open' }); + + registry.reset('p1'); + expect(registry.getState('p1')).toBe('closed'); + expect(registry.getState('p2')).toBe('open'); + + registry.resetAll(); + expect(registry.getAllStates()).toEqual({ p1: 'closed', p2: 'closed' }); + }); +}); diff --git a/backend/services/shared/index.ts b/backend/services/shared/index.ts index 85c65e89..08db26d4 100644 --- a/backend/services/shared/index.ts +++ b/backend/services/shared/index.ts @@ -246,4 +246,22 @@ export type { RejectionReason, } from './buildMetricsService'; +// ── Provider Circuit Breakers & Outbound Resilience ───────────────────────── +export { + ProviderCircuitBreakerRegistry, + providerCircuitBreakerRegistry, + executeWithProviderFallback, + CircuitOpenError, +} from './providerCircuitBreaker'; +export type { ProviderFallback, ProviderCircuitBreakerRegistryOptions } from './providerCircuitBreaker'; + +export { + outboundRequest, + isIdempotentMethod, + isIdempotentRequest, + OutboundTimeoutError, +} from './outboundRequest'; +export type { OutboundRequestOptions, OutboundRequestInit } from './outboundRequest'; + + diff --git a/backend/services/shared/outboundRequest.ts b/backend/services/shared/outboundRequest.ts new file mode 100644 index 00000000..abdcfff8 --- /dev/null +++ b/backend/services/shared/outboundRequest.ts @@ -0,0 +1,165 @@ +import { linearBackoff, defaultSleep, isRetryableStatus } from './httpRetry'; + +export class OutboundTimeoutError extends Error { + readonly code = 'OUTBOUND_TIMEOUT' as const; + + constructor(public readonly url: string, public readonly timeoutMs: number) { + super(`Outbound request to ${url} timed out after ${timeoutMs}ms`); + this.name = 'OutboundTimeoutError'; + } +} + +export interface OutboundRequestOptions { + /** Request timeout in milliseconds. Defaults to 5000 ms. */ + timeoutMs?: number; + /** Max retry attempts (including initial call). Defaults to 3. */ + maxRetries?: number; + /** Base delay between retries in milliseconds. Defaults to 250 ms. */ + baseDelayMs?: number; + /** Force retry even if request method/headers are not strictly idempotent. Defaults to false. */ + forceRetry?: boolean; + /** Custom fetch implementation (useful for tests or mocking). */ + fetchImpl?: typeof fetch; + /** Custom sleep implementation (for fast unit testing). */ + sleep?: (ms: number) => Promise; + /** Callback fired before a retry attempt. */ + onRetry?: (attempt: number, error: unknown) => void; +} + +export type OutboundRequestInit = RequestInit & OutboundRequestOptions; + +const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'PUT', 'DELETE', 'OPTIONS']); +const IDEMPOTENCY_HEADERS = ['idempotency-key', 'x-idempotency-key']; + +/** + * Checks whether an HTTP method is inherently idempotent. + */ +export function isIdempotentMethod(method: string): boolean { + return IDEMPOTENT_METHODS.has(method.toUpperCase()); +} + +/** + * Determines whether a request is safe to retry. + * Standard idempotent methods (GET, PUT, DELETE, etc.) are always safe. + * Mutating methods (POST, PATCH) are only safe if an Idempotency-Key header is supplied. + */ +export function isIdempotentRequest(init?: RequestInit | OutboundRequestInit): boolean { + const method = (init?.method ?? 'GET').toUpperCase(); + if (isIdempotentMethod(method)) { + return true; + } + + if (init?.headers) { + if (init.headers instanceof Headers) { + for (const key of IDEMPOTENCY_HEADERS) { + if (init.headers.has(key)) return true; + } + } else if (Array.isArray(init.headers)) { + for (const [k] of init.headers) { + if (IDEMPOTENCY_HEADERS.includes(k.toLowerCase())) return true; + } + } else if (typeof init.headers === 'object') { + const headerKeys = Object.keys(init.headers).map((k) => k.toLowerCase()); + for (const key of IDEMPOTENCY_HEADERS) { + if (headerKeys.includes(key)) return true; + } + } + } + + return false; +} + +/** + * Executes a single fetch call bounded by a strict timeout using AbortController. + */ +async function fetchWithTimeout( + url: string, + init: RequestInit, + timeoutMs: number, + fetchImpl: typeof fetch, +): Promise { + const controller = new AbortController(); + let timedOut = false; + + const timer = setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeoutMs); + + // Link existing signal if caller provided one + const callerSignal = init.signal; + if (callerSignal) { + if (callerSignal.aborted) { + clearTimeout(timer); + controller.abort(); + } else { + callerSignal.addEventListener('abort', () => { + clearTimeout(timer); + controller.abort(); + }); + } + } + + try { + const response = await fetchImpl(url, { + ...init, + signal: controller.signal, + }); + return response; + } catch (err: unknown) { + if (timedOut) { + throw new OutboundTimeoutError(url, timeoutMs); + } + throw err; + } finally { + clearTimeout(timer); + } +} + +/** + * Performs an outbound HTTP request with strict timeouts and bounded idempotent retries. + */ +export async function outboundRequest( + url: string | URL, + init: OutboundRequestInit = {}, +): Promise { + const urlStr = url.toString(); + const timeoutMs = Math.max(1, init.timeoutMs ?? 5_000); + const maxRetries = Math.max(1, init.maxRetries ?? 3); + const baseDelayMs = init.baseDelayMs ?? 250; + const sleep = init.sleep ?? defaultSleep; + const fetchImpl = init.fetchImpl ?? fetch; + + const idempotent = isIdempotentRequest(init) || init.forceRetry === true; + // Non-idempotent calls without idempotency keys are strictly bounded to 1 attempt + const allowedAttempts = idempotent ? maxRetries : 1; + + let lastError: unknown; + + for (let attempt = 1; attempt <= allowedAttempts; attempt++) { + try { + const response = await fetchWithTimeout(urlStr, init, timeoutMs, fetchImpl); + + // Transient HTTP errors (429, 5xx) + if (isRetryableStatus(response.status) && attempt < allowedAttempts) { + init.onRetry?.(attempt, new Error(`HTTP ${response.status}`)); + await sleep(linearBackoff(attempt, baseDelayMs)); + continue; + } + + return response; + } catch (err) { + lastError = err; + + if (attempt < allowedAttempts) { + init.onRetry?.(attempt, err); + await sleep(linearBackoff(attempt, baseDelayMs)); + continue; + } + + throw err; + } + } + + throw lastError ?? new Error('Outbound request failed after retries'); +} diff --git a/backend/services/shared/providerCircuitBreaker.ts b/backend/services/shared/providerCircuitBreaker.ts new file mode 100644 index 00000000..f5beb346 --- /dev/null +++ b/backend/services/shared/providerCircuitBreaker.ts @@ -0,0 +1,155 @@ +import { + CircuitBreaker, + CircuitBreakerOptions, + CircuitOpenError, + type CircuitState, +} from './circuitBreaker'; + +export type ProviderFallback = (error: Error) => Promise | T; + +export interface ProviderCircuitBreakerRegistryOptions { + /** Default options applied to all lazily-created provider breakers. */ + defaultOptions?: CircuitBreakerOptions; +} + +/** + * Manages isolated circuit breakers per external provider (e.g. Stripe, Paddle, QuickBooks) + * with graceful fallback execution to prevent cascading outages across downstream services. + */ +export class ProviderCircuitBreakerRegistry { + private readonly breakers = new Map(); + private readonly fallbacks = new Map>(); + private readonly defaultOptions: CircuitBreakerOptions; + + constructor(options: ProviderCircuitBreakerRegistryOptions = {}) { + this.defaultOptions = options.defaultOptions ?? { + failureThreshold: 5, + recoveryTimeoutMs: 30_000, + successThreshold: 2, + }; + } + + private normalizeKey(provider: string): string { + return provider.trim().toLowerCase(); + } + + /** + * Retrieves or lazily creates a dedicated CircuitBreaker instance for the named provider. + */ + getBreaker(provider: string, options?: CircuitBreakerOptions): CircuitBreaker { + const key = this.normalizeKey(provider); + let breaker = this.breakers.get(key); + if (!breaker) { + breaker = new CircuitBreaker({ + name: key, + ...this.defaultOptions, + ...options, + }); + this.breakers.set(key, breaker); + } + return breaker; + } + + /** + * Registers a default fallback handler for a specific provider. + */ + registerFallback(provider: string, fallback: ProviderFallback): void { + const key = this.normalizeKey(provider); + this.fallbacks.set(key, fallback as ProviderFallback); + } + + /** + * Removes a registered fallback handler for a provider. + */ + unregisterFallback(provider: string): void { + const key = this.normalizeKey(provider); + this.fallbacks.delete(key); + } + + /** + * Executes a provider action protected by its dedicated circuit breaker. + * If the circuit is open or trips, the fallback handler is called to prevent + * cascading outages. + */ + async execute( + provider: string, + action: () => Promise, + fallback?: ProviderFallback, + options?: CircuitBreakerOptions, + ): Promise { + const key = this.normalizeKey(provider); + const breaker = this.getBreaker(key, options); + const activeFallback = (fallback ?? this.fallbacks.get(key)) as ProviderFallback | undefined; + + try { + return await breaker.execute(action); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + if (activeFallback) { + return await activeFallback(error); + } + throw error; + } + } + + /** + * Returns the current circuit state for a provider. + */ + getState(provider: string): CircuitState { + const key = this.normalizeKey(provider); + const breaker = this.breakers.get(key); + return breaker ? breaker.state : 'closed'; + } + + /** + * Returns current states for all registered providers. + */ + getAllStates(): Record { + const result: Record = {}; + for (const [provider, breaker] of this.breakers.entries()) { + result[provider] = breaker.state; + } + return result; + } + + /** + * Manually resets a provider's circuit breaker to closed state. + */ + reset(provider: string): void { + const key = this.normalizeKey(provider); + this.breakers.get(key)?.reset(); + } + + /** + * Resets all provider circuit breakers. + */ + resetAll(): void { + for (const breaker of this.breakers.values()) { + breaker.reset(); + } + } + + /** + * Clears all registered breakers and fallbacks (primarily for testing). + */ + clear(): void { + this.breakers.clear(); + this.fallbacks.clear(); + } +} + +export const providerCircuitBreakerRegistry = new ProviderCircuitBreakerRegistry(); + +/** + * Convenience helper to execute a call with per-provider circuit breaker and fallback. + */ +export async function executeWithProviderFallback( + provider: string, + action: () => Promise, + fallback?: ProviderFallback, + options?: CircuitBreakerOptions, +): Promise { + return providerCircuitBreakerRegistry.execute(provider, action, fallback, options); +} + +export { CircuitOpenError };