From dcd86f503af4631b52eb5f349b311de3740c8425 Mon Sep 17 00:00:00 2001 From: maikNeta Date: Fri, 16 Oct 2026 13:50:17 +0100 Subject: [PATCH 1/2] fix: close rate-lock reuse gap, guard overlapping webhook retries, document key rotation windows - fiat-crypto: bind a rate lock to the user who created it and consume it on first use, so the same lock can no longer be confirmed twice (duplicate conversions) or confirmed by a different user than the one who locked it. - webhooks: add an in-flight guard around the delivery worker's 1s ticker so a slow or backlogged pass can never overlap with the next tick and deliver the same webhook twice. - WEBHOOK_VERIFICATION.md: document the overlap/retention/replay-tolerance windows WebhookKeyRegistry actually enforces during key rotation, which were implemented but never written down. Closes #1278, Closes #1279, Closes #1280, Closes #1281. --- WEBHOOK_VERIFICATION.md | 31 ++++++++++ backend/src/routes/stripe.ts | 12 ++-- .../services/__tests__/fiat-crypto.test.ts | 30 ++++++++++ backend/src/services/fiat-crypto.ts | 18 +++++- backend/src/services/webhooks.test.ts | 58 +++++++++++++++++++ backend/src/services/webhooks.ts | 16 ++++- 6 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 backend/src/services/__tests__/fiat-crypto.test.ts create mode 100644 backend/src/services/webhooks.test.ts diff --git a/WEBHOOK_VERIFICATION.md b/WEBHOOK_VERIFICATION.md index 64523a7e..3def73fd 100644 --- a/WEBHOOK_VERIFICATION.md +++ b/WEBHOOK_VERIFICATION.md @@ -20,6 +20,37 @@ Implement webhook signature verification for all incoming webhooks to prevent sp ## Complexity Estimate 200 points - Verification, secure storage. +## Key Rotation: Overlap, Retention, and Replay Tolerance + +`WebhookKeyRegistry` (`backend/src/services/webhookKeys.ts`) governs how long a +rotated secret keeps working and how it is retired. Get these windows wrong +and either genuine events get rejected mid-rotation, or revoked secrets keep +verifying signatures long after they should not. + +- **`overlapSeconds`** (default `DEFAULT_OVERLAP_SECONDS` = 72h) — passed to + `registry.rotate({ overlapSeconds })`. For this long after a secret is + rotated, the *previous* secret still verifies signatures alongside the new + one. Set this to at least as long as the slowest provider takes to pick up + a newly configured secret (e.g. Stripe can take a while to fully cut over + after a dashboard change) — too short, and in-flight webhooks signed with + the old secret start failing verification before the provider has switched. +- **`retentionSeconds`** (default `DEFAULT_RETENTION_SECONDS` = 7 days) — how + long an expired or revoked key is kept around (rather than purged) so that + verification attempts against it can still be logged and classified by + reason, instead of just vanishing. This is operational/audit retention, not + a security control — a key past its `overlapSeconds` window no longer + verifies anything regardless of retention. +- **`toleranceSeconds`** (default `DEFAULT_TOLERANCE_SECONDS` = 300) — the + replay-protection clock-skew window: a webhook's timestamp must be within + this many seconds of `registry` time, in either direction, or verification + rejects it as expired/replayed. Widen this only to accommodate real clock + drift between you and the provider, since every second added is a second a + captured signature + body could be replayed. + +All three are constructor options on `WebhookKeyRegistry` (`new +WebhookKeyRegistry({ overlapSeconds, retentionSeconds, toleranceSeconds })`) +and default to the constants above when omitted. + ## Implementation Plan 1. Create webhook verification service with HMAC-SHA256 2. Implement per-provider secret management diff --git a/backend/src/routes/stripe.ts b/backend/src/routes/stripe.ts index 5b54d415..ad48f6ca 100644 --- a/backend/src/routes/stripe.ts +++ b/backend/src/routes/stripe.ts @@ -296,17 +296,21 @@ stripeRouter.get('/rates', asyncHandler(async (req, res) => { res.json({ from, to, rate, updatedAt: new Date().toISOString() }); })); -stripeRouter.post('/convert/lock-rate', asyncHandler(async (req, res) => { +stripeRouter.post('/convert/lock-rate', asyncHandler(async (req: any, res) => { const { fromCurrency = 'USD', toAsset = 'USDC', amount } = req.body; if (!amount) return res.status(400).json({ error: 'amount required' }); - const lock = await lockRate(fromCurrency, toAsset, amount); + const userId = req.user?.id ?? req.headers['x-actor-id']?.toString(); + if (!userId) return res.status(401).json({ error: 'authentication required' }); + const lock = await lockRate(userId, fromCurrency, toAsset, amount); res.json(lock); })); -stripeRouter.post('/convert/confirm', asyncHandler(async (req, res) => { +stripeRouter.post('/convert/confirm', asyncHandler(async (req: any, res) => { const { rateLockId, stripePaymentIntentId } = req.body; if (!rateLockId || !stripePaymentIntentId) return res.status(400).json({ error: 'rateLockId and stripePaymentIntentId required' }); - const conversion = await confirmConversion(rateLockId, stripePaymentIntentId); + const userId = req.user?.id ?? req.headers['x-actor-id']?.toString(); + if (!userId) return res.status(401).json({ error: 'authentication required' }); + const conversion = await confirmConversion(rateLockId, userId, stripePaymentIntentId); res.json(conversion); })); diff --git a/backend/src/services/__tests__/fiat-crypto.test.ts b/backend/src/services/__tests__/fiat-crypto.test.ts new file mode 100644 index 00000000..a1d73589 --- /dev/null +++ b/backend/src/services/__tests__/fiat-crypto.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import { lockRate, confirmConversion, getConversion } from '../fiat-crypto.js'; + +const USER_A = 'user_a'; +const USER_B = 'user_b'; + +describe('fiat-crypto conversions', () => { + it('ties a confirmed conversion to the user who locked the rate', async () => { + const lock = await lockRate(USER_A, 'USD', 'USDC', 100); + const conversion = await confirmConversion(lock.id, USER_A, 'pi_123'); + expect(conversion.userId).toBe(USER_A); + }); + + it('rejects confirming a rate lock that belongs to a different user', async () => { + const lock = await lockRate(USER_A, 'USD', 'USDC', 100); + await expect(confirmConversion(lock.id, USER_B, 'pi_123')).rejects.toThrow(); + }); + + it('rejects confirming the same rate lock twice, even for its own owner', async () => { + const lock = await lockRate(USER_A, 'USD', 'USDC', 100); + const first = await confirmConversion(lock.id, USER_A, 'pi_first'); + await expect(confirmConversion(lock.id, USER_A, 'pi_second')).rejects.toThrow( + 'Rate lock has already been used', + ); + + // Only the first confirmation produced a conversion. + const stored = await getConversion(first.id); + expect(stored?.stripePaymentIntentId).toBe('pi_first'); + }); +}); diff --git a/backend/src/services/fiat-crypto.ts b/backend/src/services/fiat-crypto.ts index c0b1d402..38346f34 100644 --- a/backend/src/services/fiat-crypto.ts +++ b/backend/src/services/fiat-crypto.ts @@ -6,12 +6,15 @@ export type ConversionStatus = 'pending' | 'rate_locked' | 'completed' | 'failed export interface RateLock { id: string; + userId: string; fromCurrency: string; toAsset: string; rate: number; amount: number; expiresAt: number; locked: boolean; + /** Set once this lock has been consumed by `confirmConversion`, so it cannot be reused. */ + consumedAt?: number; } export interface ConversionRecord { @@ -44,10 +47,11 @@ function getCurrentRate(fromCurrency: string, toAsset: string): number { return 1.0; } -export async function lockRate(fromCurrency: string, toAsset: string, amount: number): Promise { +export async function lockRate(userId: string, fromCurrency: string, toAsset: string, amount: number): Promise { const rate = getCurrentRate(fromCurrency, toAsset); const lock: RateLock = { id: randomUUID(), + userId, fromCurrency, toAsset, rate, @@ -59,15 +63,23 @@ export async function lockRate(fromCurrency: string, toAsset: string, amount: nu return lock; } -export async function confirmConversion(rateLockId: string, stripePaymentIntentId: string): Promise { +export async function confirmConversion(rateLockId: string, userId: string, stripePaymentIntentId: string): Promise { const lock = rateLocks.get(rateLockId); if (!lock) throw new Error('Rate lock not found'); + // A lock belongs to whoever created it, and can be spent exactly once -- otherwise + // one locked rate could be confirmed repeatedly (duplicate conversions from a single + // amount) or confirmed by a user who never created it. + if (lock.userId !== userId) throw new Error('Rate lock not found'); + if (lock.consumedAt !== undefined) throw new Error('Rate lock has already been used'); if (Date.now() > lock.expiresAt) throw new Error('Rate lock has expired'); + lock.consumedAt = Date.now(); + rateLocks.set(lock.id, lock); + const cryptoAmount = (lock.amount * lock.rate).toFixed(7); const conversion: ConversionRecord = { id: randomUUID(), - userId: 'user_default', + userId: lock.userId, fromCurrency: lock.fromCurrency, toAsset: lock.toAsset, fiatAmount: lock.amount, diff --git a/backend/src/services/webhooks.test.ts b/backend/src/services/webhooks.test.ts new file mode 100644 index 00000000..3edc8c5b --- /dev/null +++ b/backend/src/services/webhooks.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + upsertWebhookConfig, + enqueueWebhookEvent, + startWebhookWorker, + stopWebhookWorker, + getWebhookDelivery, +} from './webhooks.js'; + +describe('webhook worker', () => { + afterEach(() => { + stopWebhookWorker(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it('never runs two overlapping passes while a delivery is still in flight', async () => { + vi.useFakeTimers(); + + let resolveFetch: (() => void) | undefined; + const fetchMock = vi.fn( + () => + new Promise((resolve) => { + resolveFetch = () => resolve({ ok: true, status: 200, text: async () => '{}' } as Response); + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const config = upsertWebhookConfig({ + merchantId: 'merchant_1', + url: 'https://merchant.example/webhook', + secret: 'a_very_secret_value', + }); + const { delivery } = enqueueWebhookEvent({ + merchantId: config.merchantId, + type: 'payment.completed', + payload: { amount: 100 }, + }); + expect(delivery).toBeDefined(); + + startWebhookWorker(); + + // First tick: picks up the delivery and starts the (slow, unresolved) fetch. + await vi.advanceTimersByTimeAsync(1_000); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // Several more ticks elapse while the first attempt is still in flight. + // Without the in-flight guard, each tick would start another pass and + // call fetch again for the same still-"processing" delivery. + await vi.advanceTimersByTimeAsync(5_000); + expect(fetchMock).toHaveBeenCalledTimes(1); + + resolveFetch?.(); + await vi.advanceTimersByTimeAsync(0); + + expect(getWebhookDelivery(delivery!.id)?.status).toBe('delivered'); + }); +}); diff --git a/backend/src/services/webhooks.ts b/backend/src/services/webhooks.ts index b283996f..fd9b697c 100644 --- a/backend/src/services/webhooks.ts +++ b/backend/src/services/webhooks.ts @@ -48,6 +48,8 @@ export interface WebhookDeliveryLog { type WorkerState = { timer?: NodeJS.Timeout; running: boolean; + /** True while a pass over due deliveries is in flight, so the 1s ticker never overlaps itself. */ + processing: boolean; }; const MAX_ATTEMPTS = 5; @@ -60,7 +62,7 @@ const deliveries = new Map(); const idempotencyIndex = new Map(); const deadLetterQueue: WebhookDeliveryLog[] = []; -const worker: WorkerState = { running: false }; +const worker: WorkerState = { running: false, processing: false }; function nowIso(): string { return new Date().toISOString(); @@ -254,7 +256,16 @@ export function startWebhookWorker(): void { if (worker.running) return; worker.running = true; worker.timer = setInterval(() => { - void processDueDeliveries(); + // A pass that is still in flight (e.g. a slow endpoint, or a backlog bigger + // than one tick can clear) must finish before another one starts. Without + // this guard, the next 1s tick starts a second, overlapping pass that can + // pick up and deliver the same still-"pending"/"retrying" delivery a + // second time before the first pass has marked it "processing". + if (worker.processing) return; + worker.processing = true; + void processDueDeliveries().finally(() => { + worker.processing = false; + }); }, 1_000); } @@ -262,6 +273,7 @@ export function stopWebhookWorker(): void { if (worker.timer) clearInterval(worker.timer); worker.timer = undefined; worker.running = false; + worker.processing = false; } export function listWebhookDeliveries(): WebhookDeliveryLog[] { From d30b8f071771f2a9e0ba112c090a0fb8118b1283 Mon Sep 17 00:00:00 2001 From: maikNeta Date: Sat, 17 Oct 2026 17:32:52 +0100 Subject: [PATCH 2/2] fix: prevent provider quota exhaustion, improve gateway reliability, fix payment correctness, enhance payout reliability --- backend/src/services/gateway-integration.ts | 305 ++++++++++++++++++++ backend/src/services/payment-correctness.ts | 247 ++++++++++++++++ backend/src/services/payout-reliability.ts | 261 +++++++++++++++++ backend/src/services/quota.ts | 143 +++++++++ backend/src/services/retry/payment.ts | 45 ++- 5 files changed, 998 insertions(+), 3 deletions(-) create mode 100644 backend/src/services/gateway-integration.ts create mode 100644 backend/src/services/payment-correctness.ts create mode 100644 backend/src/services/payout-reliability.ts create mode 100644 backend/src/services/quota.ts diff --git a/backend/src/services/gateway-integration.ts b/backend/src/services/gateway-integration.ts new file mode 100644 index 00000000..53349774 --- /dev/null +++ b/backend/src/services/gateway-integration.ts @@ -0,0 +1,305 @@ +/** + * gateway-integration.ts — Issues #1162, #1160 + * + * Enhanced gateway integration with error classification, circuit breaking, + * request deduplication, and payout reliability improvements. + */ + +import { quotaService } from './quota.js'; + +export type GatewayErrorType = + | 'RATE_LIMIT' + | 'TIMEOUT' + | 'TRANSIENT' + | 'PERMANENT' + | 'INVALID_REQUEST' + | 'AUTHENTICATION' + | 'INSUFFICIENT_BALANCE'; + +export interface GatewayError { + type: GatewayErrorType; + message: string; + statusCode?: number; + retryable: boolean; + retryAfterMs?: number; +} + +export interface GatewayRequest { + id: string; + method: string; + payload: Record; + providerId: string; + createdAt: number; + expiresAt: number; +} + +export interface GatewayResponse { + success: boolean; + data?: T; + error?: GatewayError; + requestId: string; + statusCode: number; +} + +export class GatewayIntegrationService { + private requestCache: Map> = new Map(); + private circuitBreakers: Map = new Map(); + private readonly CIRCUIT_BREAKER_THRESHOLD = 5; + private readonly CIRCUIT_BREAKER_RESET_MS = 60000; + private readonly REQUEST_DEDUP_TIMEOUT_MS = 30000; + + constructor() { + this.initializeCircuitBreakers(); + } + + private initializeCircuitBreakers(): void { + const providers = ['stripe', 'stellar', 'solana']; + for (const provider of providers) { + this.circuitBreakers.set(provider, { + open: false, + failureCount: 0, + lastFailureTime: 0, + }); + } + } + + classifyError(error: Error, statusCode?: number): GatewayError { + const message = error.message.toLowerCase(); + + if (statusCode === 429 || message.includes('rate limit')) { + return { + type: 'RATE_LIMIT', + message: 'Rate limit exceeded. Please retry after delay.', + statusCode, + retryable: true, + retryAfterMs: 30000, + }; + } + + if (statusCode === 408 || message.includes('timeout')) { + return { + type: 'TIMEOUT', + message: 'Request timeout. The operation may have partially succeeded.', + statusCode, + retryable: true, + retryAfterMs: 5000, + }; + } + + if (statusCode === 401 || message.includes('unauthorized') || message.includes('authentication')) { + return { + type: 'AUTHENTICATION', + message: 'Authentication failed. Check credentials.', + statusCode, + retryable: false, + }; + } + + if (statusCode === 400 || message.includes('invalid')) { + return { + type: 'INVALID_REQUEST', + message: 'Invalid request parameters.', + statusCode, + retryable: false, + }; + } + + if (statusCode === 402 || message.includes('insufficient')) { + return { + type: 'INSUFFICIENT_BALANCE', + message: 'Insufficient balance or funds.', + statusCode, + retryable: true, + retryAfterMs: 3600000, + }; + } + + if (statusCode && statusCode >= 500) { + return { + type: 'TRANSIENT', + message: 'Server error. Will retry automatically.', + statusCode, + retryable: true, + retryAfterMs: 10000, + }; + } + + return { + type: 'PERMANENT', + message: error.message, + statusCode, + retryable: false, + }; + } + + isCircuitBreakerOpen(providerId: string): boolean { + const breaker = this.circuitBreakers.get(providerId); + if (!breaker) return false; + + if (!breaker.open) return false; + + const timeSinceLastFailure = Date.now() - breaker.lastFailureTime; + if (timeSinceLastFailure > this.CIRCUIT_BREAKER_RESET_MS) { + breaker.open = false; + breaker.failureCount = 0; + return false; + } + + return true; + } + + recordFailure(providerId: string): void { + const breaker = this.circuitBreakers.get(providerId); + if (!breaker) return; + + breaker.failureCount++; + breaker.lastFailureTime = Date.now(); + + if (breaker.failureCount >= this.CIRCUIT_BREAKER_THRESHOLD) { + breaker.open = true; + } + } + + recordSuccess(providerId: string): void { + const breaker = this.circuitBreakers.get(providerId); + if (breaker) { + breaker.failureCount = Math.max(0, breaker.failureCount - 1); + } + } + + getCacheKey(method: string, payload: Record, providerId: string): string { + const hash = JSON.stringify({ method, payload, providerId }); + return Buffer.from(hash).toString('base64'); + } + + getCachedRequest(method: string, payload: Record, providerId: string): GatewayResponse | undefined { + const key = this.getCacheKey(method, payload, providerId); + const cached = this.requestCache.get(key); + + if (cached && cached.success) { + const age = Date.now() - (cached as any).cachedAt; + if (age < this.REQUEST_DEDUP_TIMEOUT_MS) { + return cached; + } + } + + return undefined; + } + + cacheRequest(method: string, payload: Record, providerId: string, response: GatewayResponse): void { + const key = this.getCacheKey(method, payload, providerId); + (response as any).cachedAt = Date.now(); + this.requestCache.set(key, response); + + // Cleanup old cache entries after 5 minutes + if (this.requestCache.size > 1000) { + const now = Date.now(); + for (const [k, v] of this.requestCache.entries()) { + if (now - (v as any).cachedAt > 300000) { + this.requestCache.delete(k); + } + } + } + } + + async executeWithErrorHandling( + providerId: string, + method: string, + payload: Record, + operation: () => Promise + ): Promise> { + // Check circuit breaker + if (this.isCircuitBreakerOpen(providerId)) { + return { + success: false, + error: { + type: 'TRANSIENT', + message: `Circuit breaker open for ${providerId}. Provider temporarily unavailable.`, + retryable: true, + retryAfterMs: 30000, + }, + requestId: '', + statusCode: 503, + }; + } + + // Check quota + if (!quotaService.canMakeRequest(providerId)) { + return { + success: false, + error: { + type: 'RATE_LIMIT', + message: `Quota exceeded for ${providerId}`, + retryable: true, + retryAfterMs: 60000, + }, + requestId: '', + statusCode: 429, + }; + } + + // Check cache for idempotent operations + const cached = this.getCachedRequest(method, payload, providerId); + if (cached) { + return cached as GatewayResponse; + } + + try { + quotaService.recordRequest(providerId); + const data = await operation(); + const response: GatewayResponse = { + success: true, + data, + requestId: `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + statusCode: 200, + }; + + this.cacheRequest(method, payload, providerId, response); + this.recordSuccess(providerId); + return response; + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + const statusCode = (err as any)?.statusCode; + const gatewayError = this.classifyError(error, statusCode); + + if (!gatewayError.retryable) { + this.recordFailure(providerId); + } + + return { + success: false, + error: gatewayError, + requestId: `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + statusCode: statusCode || 500, + }; + } + } + + getCircuitBreakerStatus(providerId?: string): Record { + const result: Record = {}; + + const providers = providerId ? [providerId] : Array.from(this.circuitBreakers.keys()); + for (const provider of providers) { + const breaker = this.circuitBreakers.get(provider); + if (breaker) { + result[provider] = { open: breaker.open, failureCount: breaker.failureCount }; + } + } + + return result; + } + + resetCircuitBreaker(providerId: string): void { + const breaker = this.circuitBreakers.get(providerId); + if (breaker) { + breaker.open = false; + breaker.failureCount = 0; + } + } + + clearCache(): void { + this.requestCache.clear(); + } +} + +export const gatewayIntegrationService = new GatewayIntegrationService(); diff --git a/backend/src/services/payment-correctness.ts b/backend/src/services/payment-correctness.ts new file mode 100644 index 00000000..6841230d --- /dev/null +++ b/backend/src/services/payment-correctness.ts @@ -0,0 +1,247 @@ +/** + * payment-correctness.ts — Issue #1159 + * + * Payment state validation and consistency checks to prevent payment + * correctness bugs and state inconsistencies. + */ + +export type PaymentState = 'pending' | 'processing' | 'succeeded' | 'failed' | 'cancelled'; + +export interface PaymentRecord { + id: string; + userId: string; + amount: number; + currency: string; + state: PaymentState; + createdAt: number; + updatedAt: number; + metadata?: Record; +} + +export interface PaymentValidationResult { + valid: boolean; + errors: string[]; + warnings: string[]; +} + +export class PaymentCorrectnessService { + private payments: Map = new Map(); + private stateTransitions: Map = new Map([ + ['pending', ['processing', 'cancelled']], + ['processing', ['succeeded', 'failed', 'cancelled']], + ['succeeded', []], + ['failed', ['pending', 'cancelled']], + ['cancelled', []], + ]); + + validatePayment(payment: PaymentRecord): PaymentValidationResult { + const errors: string[] = []; + const warnings: string[] = []; + + if (!payment.id || typeof payment.id !== 'string') { + errors.push('Payment ID must be a non-empty string'); + } + + if (!payment.userId || typeof payment.userId !== 'string') { + errors.push('User ID must be a non-empty string'); + } + + if (typeof payment.amount !== 'number' || payment.amount <= 0) { + errors.push('Payment amount must be a positive number'); + } + + if (!payment.currency || typeof payment.currency !== 'string') { + errors.push('Currency must be specified'); + } + + const validStates: PaymentState[] = ['pending', 'processing', 'succeeded', 'failed', 'cancelled']; + if (!validStates.includes(payment.state)) { + errors.push(`Payment state '${payment.state}' is invalid`); + } + + if (payment.createdAt > payment.updatedAt) { + errors.push('Created timestamp cannot be after updated timestamp'); + } + + if (payment.updatedAt > Date.now()) { + warnings.push('Updated timestamp is in the future'); + } + + if (payment.state === 'succeeded' && !payment.metadata?.successAt) { + warnings.push('Successful payment should have successAt timestamp in metadata'); + } + + if (payment.state === 'failed' && !payment.metadata?.failureReason) { + warnings.push('Failed payment should have failureReason in metadata'); + } + + return { + valid: errors.length === 0, + errors, + warnings, + }; + } + + validateStateTransition(fromState: PaymentState, toState: PaymentState): { valid: boolean; reason?: string } { + const allowedTransitions = this.stateTransitions.get(fromState); + + if (!allowedTransitions) { + return { valid: false, reason: `Unknown state: ${fromState}` }; + } + + if (!allowedTransitions.includes(toState)) { + return { + valid: false, + reason: `Cannot transition from ${fromState} to ${toState}. Allowed: ${allowedTransitions.join(', ')}`, + }; + } + + return { valid: true }; + } + + createPayment( + id: string, + userId: string, + amount: number, + currency: string, + metadata?: Record + ): { success: boolean; payment?: PaymentRecord; error?: string } { + const now = Date.now(); + const payment: PaymentRecord = { + id, + userId, + amount, + currency, + state: 'pending', + createdAt: now, + updatedAt: now, + metadata, + }; + + const validation = this.validatePayment(payment); + if (!validation.valid) { + return { + success: false, + error: `Payment validation failed: ${validation.errors.join('; ')}`, + }; + } + + this.payments.set(id, payment); + return { success: true, payment }; + } + + updatePaymentState( + paymentId: string, + newState: PaymentState, + metadata?: Record + ): { success: boolean; payment?: PaymentRecord; error?: string } { + const payment = this.payments.get(paymentId); + + if (!payment) { + return { success: false, error: `Payment ${paymentId} not found` }; + } + + const transition = this.validateStateTransition(payment.state, newState); + if (!transition.valid) { + return { success: false, error: transition.reason }; + } + + payment.state = newState; + payment.updatedAt = Date.now(); + + if (metadata) { + payment.metadata = { ...payment.metadata, ...metadata }; + } + + if (newState === 'succeeded') { + payment.metadata = { ...payment.metadata, successAt: Date.now() }; + } else if (newState === 'failed') { + payment.metadata = { ...payment.metadata, failureAt: Date.now() }; + } + + this.payments.set(paymentId, payment); + return { success: true, payment }; + } + + getPayment(paymentId: string): PaymentRecord | undefined { + return this.payments.get(paymentId); + } + + getPaymentsByUser(userId: string): PaymentRecord[] { + return Array.from(this.payments.values()).filter((p) => p.userId === userId); + } + + getPaymentsByState(state: PaymentState): PaymentRecord[] { + return Array.from(this.payments.values()).filter((p) => p.state === state); + } + + validatePaymentChain(paymentIds: string[]): { valid: boolean; errors: string[] } { + const errors: string[] = []; + + for (const id of paymentIds) { + const payment = this.payments.get(id); + if (!payment) { + errors.push(`Payment ${id} not found`); + continue; + } + + const validation = this.validatePayment(payment); + if (!validation.valid) { + errors.push(`Payment ${id}: ${validation.errors.join('; ')}`); + } + } + + return { + valid: errors.length === 0, + errors, + }; + } + + getConsistencyReport(): { + totalPayments: number; + byState: Record; + inconsistencies: string[]; + } { + const payments = Array.from(this.payments.values()); + const byState: Record = { + pending: 0, + processing: 0, + succeeded: 0, + failed: 0, + cancelled: 0, + }; + + const inconsistencies: string[] = []; + + for (const payment of payments) { + byState[payment.state]++; + + const validation = this.validatePayment(payment); + if (!validation.valid) { + inconsistencies.push(`${payment.id}: ${validation.errors.join('; ')}`); + } + } + + return { + totalPayments: payments.length, + byState, + inconsistencies, + }; + } + + cleanup(): void { + const now = Date.now(); + const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + + for (const [id, payment] of this.payments.entries()) { + const age = now - payment.updatedAt; + if (payment.state === 'succeeded' || payment.state === 'failed' || payment.state === 'cancelled') { + if (age > thirtyDaysMs) { + this.payments.delete(id); + } + } + } + } +} + +export const paymentCorrectnessService = new PaymentCorrectnessService(); diff --git a/backend/src/services/payout-reliability.ts b/backend/src/services/payout-reliability.ts new file mode 100644 index 00000000..45344f63 --- /dev/null +++ b/backend/src/services/payout-reliability.ts @@ -0,0 +1,261 @@ +/** + * payout-reliability.ts — Issue #1160 + * + * Payout reliability improvements with idempotency, deduplication, + * state tracking, and recovery mechanisms. + */ + +export type PayoutStatus = 'pending' | 'in_progress' | 'confirmed' | 'failed' | 'recovered'; + +export interface PayoutAttempt { + id: string; + timestamp: number; + status: 'pending' | 'success' | 'failed'; + error?: string; + txHash?: string; +} + +export interface PayoutRecord { + id: string; + userId: string; + recipientAddress: string; + amount: number; + currency: string; + chainId: string; + status: PayoutStatus; + idempotencyKey: string; + attempts: PayoutAttempt[]; + createdAt: number; + updatedAt: number; + confirmedAt?: number; + failedAt?: number; +} + +export class PayoutReliabilityService { + private payouts: Map = new Map(); + private idempotencyMap: Map = new Map(); // idempotencyKey → payoutId + private recipientIndex: Map> = new Map(); // recipientAddress → Set + private readonly MAX_ATTEMPTS = 5; + private readonly CONFIRMATION_TIMEOUT_MS = 3600000; // 1 hour + + createPayout( + id: string, + userId: string, + recipientAddress: string, + amount: number, + currency: string, + chainId: string, + idempotencyKey: string + ): { success: boolean; payout?: PayoutRecord; error?: string } { + // Check idempotency - return existing payout if already created + const existingId = this.idempotencyMap.get(idempotencyKey); + if (existingId) { + const existingPayout = this.payouts.get(existingId); + if (existingPayout) { + return { success: true, payout: existingPayout }; + } + } + + // Validate inputs + if (!recipientAddress || recipientAddress.length < 10) { + return { success: false, error: 'Invalid recipient address' }; + } + + if (amount <= 0) { + return { success: false, error: 'Payout amount must be positive' }; + } + + const now = Date.now(); + const payout: PayoutRecord = { + id, + userId, + recipientAddress, + amount, + currency, + chainId, + status: 'pending', + idempotencyKey, + attempts: [], + createdAt: now, + updatedAt: now, + }; + + this.payouts.set(id, payout); + this.idempotencyMap.set(idempotencyKey, id); + + const recipients = this.recipientIndex.get(recipientAddress) || new Set(); + recipients.add(id); + this.recipientIndex.set(recipientAddress, recipients); + + return { success: true, payout }; + } + + recordAttempt( + payoutId: string, + success: boolean, + options?: { error?: string; txHash?: string } + ): { success: boolean; payout?: PayoutRecord; error?: string } { + const payout = this.payouts.get(payoutId); + if (!payout) { + return { success: false, error: `Payout ${payoutId} not found` }; + } + + if (payout.attempts.length >= this.MAX_ATTEMPTS) { + return { success: false, error: 'Maximum attempt limit reached' }; + } + + const attempt: PayoutAttempt = { + id: `attempt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + timestamp: Date.now(), + status: success ? 'success' : 'failed', + error: options?.error, + txHash: options?.txHash, + }; + + payout.attempts.push(attempt); + payout.updatedAt = Date.now(); + + if (success) { + payout.status = 'confirmed'; + payout.confirmedAt = Date.now(); + } else if (payout.attempts.length >= this.MAX_ATTEMPTS) { + payout.status = 'failed'; + payout.failedAt = Date.now(); + } else { + payout.status = 'in_progress'; + } + + this.payouts.set(payoutId, payout); + return { success: true, payout }; + } + + getPayout(payoutId: string): PayoutRecord | undefined { + return this.payouts.get(payoutId); + } + + getPayoutByIdempotencyKey(idempotencyKey: string): PayoutRecord | undefined { + const id = this.idempotencyMap.get(idempotencyKey); + return id ? this.payouts.get(id) : undefined; + } + + getPayoutsByRecipient(recipientAddress: string): PayoutRecord[] { + const ids = this.recipientIndex.get(recipientAddress) || new Set(); + return Array.from(ids) + .map((id) => this.payouts.get(id)) + .filter(Boolean) as PayoutRecord[]; + } + + getPayoutsByUser(userId: string): PayoutRecord[] { + return Array.from(this.payouts.values()).filter((p) => p.userId === userId); + } + + getPendingPayouts(): PayoutRecord[] { + return Array.from(this.payouts.values()).filter( + (p) => p.status === 'pending' || p.status === 'in_progress' + ); + } + + getFailedPayouts(): PayoutRecord[] { + return Array.from(this.payouts.values()).filter((p) => p.status === 'failed'); + } + + getConfirmedPayouts(): PayoutRecord[] { + return Array.from(this.payouts.values()).filter((p) => p.status === 'confirmed'); + } + + canRetry(payoutId: string): boolean { + const payout = this.payouts.get(payoutId); + if (!payout) return false; + + if (payout.status === 'confirmed') return false; + if (payout.attempts.length >= this.MAX_ATTEMPTS) return false; + + return true; + } + + detectStuckPayouts(): PayoutRecord[] { + const now = Date.now(); + return Array.from(this.payouts.values()).filter((p) => { + if (p.status === 'confirmed' || p.status === 'failed') { + return false; + } + + const age = now - p.updatedAt; + return age > this.CONFIRMATION_TIMEOUT_MS; + }); + } + + recoverStuckPayout(payoutId: string): { success: boolean; payout?: PayoutRecord; error?: string } { + const payout = this.payouts.get(payoutId); + if (!payout) { + return { success: false, error: `Payout ${payoutId} not found` }; + } + + if (payout.status === 'confirmed' || payout.status === 'failed') { + return { success: false, error: 'Cannot recover completed payout' }; + } + + payout.status = 'recovered'; + payout.updatedAt = Date.now(); + this.payouts.set(payoutId, payout); + + return { success: true, payout }; + } + + getReliabilityMetrics(): { + totalPayouts: number; + confirmed: number; + failed: number; + pending: number; + inProgress: number; + recovered: number; + successRate: number; + avgAttemptsPerPayout: number; + stuckPayouts: number; + } { + const payouts = Array.from(this.payouts.values()); + const confirmed = payouts.filter((p) => p.status === 'confirmed').length; + const failed = payouts.filter((p) => p.status === 'failed').length; + const pending = payouts.filter((p) => p.status === 'pending').length; + const inProgress = payouts.filter((p) => p.status === 'in_progress').length; + const recovered = payouts.filter((p) => p.status === 'recovered').length; + + const totalAttempts = payouts.reduce((sum, p) => sum + p.attempts.length, 0); + const stuckPayouts = this.detectStuckPayouts().length; + + return { + totalPayouts: payouts.length, + confirmed, + failed, + pending, + inProgress, + recovered, + successRate: payouts.length > 0 ? (confirmed / payouts.length) * 100 : 0, + avgAttemptsPerPayout: payouts.length > 0 ? totalAttempts / payouts.length : 0, + stuckPayouts, + }; + } + + cleanup(): void { + const now = Date.now(); + const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; + + for (const [id, payout] of this.payouts.entries()) { + const age = now - payout.updatedAt; + if ((payout.status === 'confirmed' || payout.status === 'failed') && age > thirtyDaysMs) { + this.payouts.delete(id); + this.idempotencyMap.delete(payout.idempotencyKey); + + const recipients = this.recipientIndex.get(payout.recipientAddress); + if (recipients) { + recipients.delete(id); + if (recipients.size === 0) { + this.recipientIndex.delete(payout.recipientAddress); + } + } + } + } + } +} + +export const payoutReliabilityService = new PayoutReliabilityService(); diff --git a/backend/src/services/quota.ts b/backend/src/services/quota.ts new file mode 100644 index 00000000..bd911bbe --- /dev/null +++ b/backend/src/services/quota.ts @@ -0,0 +1,143 @@ +/** + * quota.ts — Issue #1024 + * + * Provider quota tracking and rate limiting to prevent exhausting + * provider quotas during failed payout retries. + */ + +export interface QuotaConfig { + maxRequestsPerHour: number; + maxRequestsPerDay: number; + burstLimit: number; +} + +export interface QuotaUsage { + providerId: string; + hourlyCount: number; + dailyCount: number; + lastResetHour: number; + lastResetDay: number; + burstRemaining: number; +} + +const DEFAULT_QUOTAS: Record = { + stripe: { + maxRequestsPerHour: 1000, + maxRequestsPerDay: 20000, + burstLimit: 50, + }, + stellar: { + maxRequestsPerHour: 500, + maxRequestsPerDay: 10000, + burstLimit: 25, + }, + solana: { + maxRequestsPerHour: 300, + maxRequestsPerDay: 7200, + burstLimit: 10, + }, +}; + +export class QuotaService { + private quotaUsage: Map = new Map(); + private quotaConfigs: Map = new Map(); + + constructor() { + this.initializeQuotas(); + } + + private initializeQuotas(): void { + for (const [provider, config] of Object.entries(DEFAULT_QUOTAS)) { + this.quotaConfigs.set(provider, config); + this.quotaUsage.set(provider, { + providerId: provider, + hourlyCount: 0, + dailyCount: 0, + lastResetHour: Date.now(), + lastResetDay: Date.now(), + burstRemaining: config.burstLimit, + }); + } + } + + private resetHourlyIfNeeded(usage: QuotaUsage, config: QuotaConfig): void { + const now = Date.now(); + const hourMs = 60 * 60 * 1000; + if (now - usage.lastResetHour > hourMs) { + usage.hourlyCount = 0; + usage.lastResetHour = now; + usage.burstRemaining = config.burstLimit; + } + } + + private resetDailyIfNeeded(usage: QuotaUsage): void { + const now = Date.now(); + const dayMs = 24 * 60 * 60 * 1000; + if (now - usage.lastResetDay > dayMs) { + usage.dailyCount = 0; + usage.lastResetDay = now; + } + } + + canMakeRequest(providerId: string, isBurst: boolean = false): boolean { + const config = this.quotaConfigs.get(providerId); + const usage = this.quotaUsage.get(providerId); + + if (!config || !usage) { + return true; + } + + this.resetHourlyIfNeeded(usage, config); + this.resetDailyIfNeeded(usage); + + if (isBurst && usage.burstRemaining > 0) { + return true; + } + + return ( + usage.hourlyCount < config.maxRequestsPerHour && + usage.dailyCount < config.maxRequestsPerDay + ); + } + + recordRequest(providerId: string, isBurst: boolean = false): void { + const usage = this.quotaUsage.get(providerId); + const config = this.quotaConfigs.get(providerId); + + if (!usage || !config) { + return; + } + + this.resetHourlyIfNeeded(usage, config); + this.resetDailyIfNeeded(usage); + + usage.hourlyCount++; + usage.dailyCount++; + + if (isBurst && usage.burstRemaining > 0) { + usage.burstRemaining--; + } + } + + getUsage(providerId: string): QuotaUsage | undefined { + return this.quotaUsage.get(providerId); + } + + setQuotaConfig(providerId: string, config: Partial): void { + const existing = this.quotaConfigs.get(providerId); + if (existing) { + this.quotaConfigs.set(providerId, { ...existing, ...config }); + } + } + + getQuotaConfig(providerId: string): QuotaConfig | undefined { + return this.quotaConfigs.get(providerId); + } + + resetAll(): void { + this.quotaUsage.clear(); + this.initializeQuotas(); + } +} + +export const quotaService = new QuotaService(); diff --git a/backend/src/services/retry/payment.ts b/backend/src/services/retry/payment.ts index af7d8291..bce90867 100644 --- a/backend/src/services/retry/payment.ts +++ b/backend/src/services/retry/payment.ts @@ -1,9 +1,12 @@ import { retryService, RetryResult, RetryConfig, createRetryConfig, RetryMetrics } from './index.js'; +import { quotaService } from '../quota.js'; export interface PaymentRetryConfig extends RetryConfig { paymentType: string; + providerId?: string; enableCircuitBreaker: boolean; notifyOnFailure: boolean; + respectQuota?: boolean; } export interface PaymentAttempt { @@ -47,30 +50,37 @@ const DEFAULT_PAYMENT_CONFIGS: Record> = { backoffMultiplier: 2, enableCircuitBreaker: true, notifyOnFailure: true, + respectQuota: true, }, usdc: { maxRetries: 3, initialDelayMs: 10000, maxDelayMs: 180000, backoffMultiplier: 2, + providerId: 'stripe', enableCircuitBreaker: true, notifyOnFailure: true, + respectQuota: true, }, xlm: { maxRetries: 2, initialDelayMs: 5000, maxDelayMs: 60000, backoffMultiplier: 1.5, + providerId: 'stellar', enableCircuitBreaker: true, notifyOnFailure: true, + respectQuota: true, }, stripe: { maxRetries: 3, initialDelayMs: 2000, maxDelayMs: 30000, backoffMultiplier: 2, + providerId: 'stripe', enableCircuitBreaker: false, notifyOnFailure: true, + respectQuota: true, }, }; @@ -141,6 +151,30 @@ class PaymentRetryService { return { success: true, attempts: 0, totalTimeMs: 0 }; } + // Check provider quota before attempting retry + if (config.respectQuota && config.providerId && !quotaService.canMakeRequest(config.providerId)) { + state.status = 'exhausted'; + state.error = `Provider quota exhausted for ${config.providerId}`; + + if (config.notifyOnFailure) { + this.sendFailureNotification(paymentId, state); + } + + this.emit({ + type: 'retry_exhausted', + paymentId, + timestamp: new Date(), + data: { error: `Provider quota exhausted for ${config.providerId}` }, + }); + + return { + success: false, + attempts: state.attempts.length, + totalTimeMs: 0, + error: new Error(state.error), + }; + } + if (!manualOverride && this.manualRetryQueue.has(paymentId)) { this.manualRetryQueue.delete(paymentId); } @@ -160,6 +194,11 @@ class PaymentRetryService { backoffMultiplier: config.backoffMultiplier, }); + // Record quota usage on attempt + if (config.respectQuota && config.providerId) { + quotaService.recordRequest(config.providerId, result.attempts > 1); + } + const attempt: PaymentAttempt = { paymentId, attemptNumber: result.attempts, @@ -184,11 +223,11 @@ class PaymentRetryService { if (result.attempts >= maxAttempts) { state.status = 'exhausted'; state.error = result.error?.message; - + if (config.notifyOnFailure) { this.sendFailureNotification(paymentId, state); } - + this.emit({ type: 'retry_exhausted', paymentId, @@ -199,7 +238,7 @@ class PaymentRetryService { state.status = 'pending'; const delay = config.initialDelayMs * Math.pow(config.backoffMultiplier, result.attempts - 1); state.nextRetryTime = new Date(Date.now() + delay); - + this.emit({ type: 'retry_attempt', paymentId,