From 79e85ecff951aaf86ddf312eb656dda34dfd3c75 Mon Sep 17 00:00:00 2001 From: Georgechisom Date: Sat, 17 Oct 2026 15:58:30 +0100 Subject: [PATCH] feat(agenticpay): documentation, payout reliability, gateway integrations, payment security - Add webhook retry strategy documentation with best practices (issue #1158) - Add payout reliability service with dead letter queue and health monitoring (issue #1155) - Add unified gateway adapter interface with failover support (issue #1157) - Add payment fraud detection with risk scoring and velocity checks (issue #1156) Resolves #1155, #1156, #1157, #1158 --- backend/src/integrations/gateway-adapter.ts | 407 +++++++++++++ .../src/security/payment-fraud-detection.ts | 565 ++++++++++++++++++ backend/src/services/payout-reliability.ts | 427 +++++++++++++ docs/WEBHOOK_RETRY_STRATEGY.md | 294 +++++++++ 4 files changed, 1693 insertions(+) create mode 100644 backend/src/integrations/gateway-adapter.ts create mode 100644 backend/src/security/payment-fraud-detection.ts create mode 100644 backend/src/services/payout-reliability.ts create mode 100644 docs/WEBHOOK_RETRY_STRATEGY.md diff --git a/backend/src/integrations/gateway-adapter.ts b/backend/src/integrations/gateway-adapter.ts new file mode 100644 index 00000000..ac76cdb2 --- /dev/null +++ b/backend/src/integrations/gateway-adapter.ts @@ -0,0 +1,407 @@ +// Issue #1157: Improve gateway integrations (iteration 5) +// Unified gateway adapter interface with health monitoring and failover + +/** + * Payment gateway types + */ +export type GatewayType = 'stellar' | 'stripe' | 'paypal' | 'coinbase'; + +/** + * Payment request + */ +export interface PaymentRequest { + amount: string; + currency: string; + recipient: string; + metadata?: Record; + idempotencyKey: string; +} + +/** + * Payment response + */ +export interface PaymentResponse { + success: boolean; + transactionId: string; + transactionHash?: string; + status: 'pending' | 'completed' | 'failed'; + errorCode?: string; + errorMessage?: string; + gatewayResponse?: unknown; +} + +/** + * Gateway health status + */ +export interface GatewayHealth { + gatewayType: GatewayType; + isHealthy: boolean; + lastCheckAt: string; + responseTimeMs: number; + errorRate: number; + successRate: number; + consecutiveFailures: number; +} + +/** + * Gateway configuration + */ +export interface GatewayConfig { + gatewayType: GatewayType; + enabled: boolean; + priority: number; + maxRetries: number; + timeoutMs: number; + healthCheckIntervalMs: number; + circuitBreakerThreshold: number; + credentials: Record; +} + +/** + * Circuit breaker state + */ +enum CircuitState { + Closed = 'closed', + Open = 'open', + HalfOpen = 'half_open', +} + +/** + * Base gateway adapter interface + */ +export interface IGatewayAdapter { + getType(): GatewayType; + processPayment(request: PaymentRequest): Promise; + checkHealth(): Promise; + getHealthStatus(): GatewayHealth; +} + +/** + * Abstract base gateway adapter with common functionality + */ +export abstract class BaseGatewayAdapter implements IGatewayAdapter { + protected config: GatewayConfig; + protected health: GatewayHealth; + protected circuitState: CircuitState = CircuitState.Closed; + protected circuitOpenedAt?: Date; + protected requestTimes: number[] = []; + protected requestResults: boolean[] = []; + + constructor(config: GatewayConfig) { + this.config = config; + this.health = { + gatewayType: config.gatewayType, + isHealthy: true, + lastCheckAt: new Date().toISOString(), + responseTimeMs: 0, + errorRate: 0, + successRate: 1, + consecutiveFailures: 0, + }; + } + + abstract getType(): GatewayType; + protected abstract executePayment(request: PaymentRequest): Promise; + + /** + * Process payment with circuit breaker and monitoring + */ + async processPayment(request: PaymentRequest): Promise { + // Check circuit breaker + if (this.circuitState === CircuitState.Open) { + if (this.shouldAttemptReset()) { + this.circuitState = CircuitState.HalfOpen; + } else { + throw new Error( + `Gateway ${this.config.gatewayType} circuit is open. Try again later.`, + ); + } + } + + const startTime = Date.now(); + + try { + const response = await this.executePayment(request); + const processingTime = Date.now() - startTime; + + this.recordSuccess(processingTime); + + // Close circuit if half-open + if (this.circuitState === CircuitState.HalfOpen) { + this.circuitState = CircuitState.Closed; + this.health.consecutiveFailures = 0; + } + + return response; + } catch (error) { + const processingTime = Date.now() - startTime; + this.recordFailure(processingTime); + + // Check if should open circuit + if (this.health.consecutiveFailures >= this.config.circuitBreakerThreshold) { + this.openCircuit(); + } + + throw error; + } + } + + /** + * Check health of the gateway + */ + async checkHealth(): Promise { + const startTime = Date.now(); + + try { + await this.performHealthCheck(); + const responseTime = Date.now() - startTime; + + this.health.isHealthy = true; + this.health.responseTimeMs = responseTime; + this.health.lastCheckAt = new Date().toISOString(); + + return true; + } catch { + this.health.isHealthy = false; + this.health.lastCheckAt = new Date().toISOString(); + + return false; + } + } + + /** + * Perform gateway-specific health check + */ + protected abstract performHealthCheck(): Promise; + + /** + * Get current health status + */ + getHealthStatus(): GatewayHealth { + return { ...this.health }; + } + + /** + * Record successful request + */ + private recordSuccess(processingTimeMs: number): void { + this.requestTimes.push(processingTimeMs); + this.requestResults.push(true); + this.trimMetrics(); + this.updateMetrics(); + this.health.consecutiveFailures = 0; + } + + /** + * Record failed request + */ + private recordFailure(processingTimeMs: number): void { + this.requestTimes.push(processingTimeMs); + this.requestResults.push(false); + this.trimMetrics(); + this.updateMetrics(); + this.health.consecutiveFailures++; + } + + /** + * Trim metrics to last 100 requests + */ + private trimMetrics(): void { + if (this.requestTimes.length > 100) { + this.requestTimes.shift(); + this.requestResults.shift(); + } + } + + /** + * Update health metrics + */ + private updateMetrics(): void { + // Average response time + if (this.requestTimes.length > 0) { + this.health.responseTimeMs = + this.requestTimes.reduce((a, b) => a + b, 0) / this.requestTimes.length; + } + + // Success and error rates + if (this.requestResults.length > 0) { + const successCount = this.requestResults.filter((r) => r).length; + this.health.successRate = successCount / this.requestResults.length; + this.health.errorRate = 1 - this.health.successRate; + } + } + + /** + * Open circuit breaker + */ + private openCircuit(): void { + this.circuitState = CircuitState.Open; + this.circuitOpenedAt = new Date(); + this.health.isHealthy = false; + } + + /** + * Check if should attempt circuit reset + */ + private shouldAttemptReset(): boolean { + if (!this.circuitOpenedAt) return false; + + const elapsed = Date.now() - this.circuitOpenedAt.getTime(); + return elapsed > 60000; // 1 minute cooldown + } +} + +/** + * Gateway manager with failover support + */ +export class GatewayManager { + private adapters: Map = new Map(); + private healthCheckInterval?: NodeJS.Timeout; + + /** + * Register a gateway adapter + */ + registerAdapter(adapter: IGatewayAdapter): void { + this.adapters.set(adapter.getType(), adapter); + } + + /** + * Unregister a gateway adapter + */ + unregisterAdapter(gatewayType: GatewayType): void { + this.adapters.delete(gatewayType); + } + + /** + * Process payment with automatic failover + */ + async processPayment( + request: PaymentRequest, + preferredGateway?: GatewayType, + ): Promise { + // Get available gateways sorted by priority and health + const gateways = this.getAvailableGateways(preferredGateway); + + if (gateways.length === 0) { + throw new Error('No available payment gateways'); + } + + let lastError: Error | null = null; + + // Try each gateway in order + for (const adapter of gateways) { + try { + const response = await adapter.processPayment(request); + return { + ...response, + gatewayUsed: adapter.getType(), + }; + } catch (error) { + lastError = error instanceof Error ? error : new Error('Unknown error'); + // Continue to next gateway + } + } + + // All gateways failed + throw new Error( + `Payment failed on all available gateways. Last error: ${lastError?.message}`, + ); + } + + /** + * Get available gateways sorted by priority and health + */ + private getAvailableGateways(preferredGateway?: GatewayType): IGatewayAdapter[] { + const adapters = Array.from(this.adapters.values()); + + // Filter to healthy gateways + const healthyAdapters = adapters.filter((adapter) => { + const health = adapter.getHealthStatus(); + return health.isHealthy; + }); + + // If preferred gateway is healthy, put it first + if (preferredGateway) { + const preferred = healthyAdapters.find((a) => a.getType() === preferredGateway); + if (preferred) { + return [ + preferred, + ...healthyAdapters.filter((a) => a.getType() !== preferredGateway), + ]; + } + } + + // Sort by success rate + return healthyAdapters.sort((a, b) => { + const healthA = a.getHealthStatus(); + const healthB = b.getHealthStatus(); + return healthB.successRate - healthA.successRate; + }); + } + + /** + * Get health status of all gateways + */ + getAllHealthStatus(): GatewayHealth[] { + return Array.from(this.adapters.values()).map((adapter) => + adapter.getHealthStatus(), + ); + } + + /** + * Get health status of specific gateway + */ + getHealthStatus(gatewayType: GatewayType): GatewayHealth | null { + const adapter = this.adapters.get(gatewayType); + return adapter ? adapter.getHealthStatus() : null; + } + + /** + * Start periodic health checks + */ + startHealthChecks(intervalMs: number = 60000): void { + this.stopHealthChecks(); + + this.healthCheckInterval = setInterval(async () => { + for (const adapter of this.adapters.values()) { + await adapter.checkHealth(); + } + }, intervalMs); + } + + /** + * Stop health checks + */ + stopHealthChecks(): void { + if (this.healthCheckInterval) { + clearInterval(this.healthCheckInterval); + this.healthCheckInterval = undefined; + } + } + + /** + * Get aggregate metrics + */ + getAggregateMetrics(): { + totalGateways: number; + healthyGateways: number; + averageSuccessRate: number; + averageResponseTime: number; + } { + const allHealth = this.getAllHealthStatus(); + const healthyCount = allHealth.filter((h) => h.isHealthy).length; + + const avgSuccessRate = + allHealth.reduce((sum, h) => sum + h.successRate, 0) / allHealth.length || 0; + + const avgResponseTime = + allHealth.reduce((sum, h) => sum + h.responseTimeMs, 0) / allHealth.length || 0; + + return { + totalGateways: allHealth.length, + healthyGateways: healthyCount, + averageSuccessRate: avgSuccessRate, + averageResponseTime: avgResponseTime, + }; + } +} diff --git a/backend/src/security/payment-fraud-detection.ts b/backend/src/security/payment-fraud-detection.ts new file mode 100644 index 00000000..7ed1916a --- /dev/null +++ b/backend/src/security/payment-fraud-detection.ts @@ -0,0 +1,565 @@ +// Issue #1156: Strengthen payment security (iteration 5) +// Implements fraud detection and risk scoring for payments + +import { createHash } from 'crypto'; + +/** + * Payment transaction for fraud analysis + */ +export interface PaymentTransaction { + id: string; + amount: string; + currency: string; + merchantId: string; + customerId?: string; + customerEmail?: string; + customerIp?: string; + deviceFingerprint?: string; + billingAddress?: Address; + shippingAddress?: Address; + paymentMethod: PaymentMethod; + timestamp: string; + metadata?: Record; +} + +/** + * Address structure + */ +export interface Address { + line1: string; + line2?: string; + city: string; + state?: string; + postalCode: string; + country: string; +} + +/** + * Payment method details + */ +export interface PaymentMethod { + type: 'card' | 'bank_transfer' | 'crypto' | 'wallet'; + last4?: string; + brand?: string; + walletAddress?: string; +} + +/** + * Risk assessment result + */ +export interface RiskAssessment { + transactionId: string; + riskScore: number; // 0-100, higher = more risky + riskLevel: 'low' | 'medium' | 'high' | 'critical'; + decision: 'approve' | 'review' | 'decline'; + factors: RiskFactor[]; + recommendation: string; + assessedAt: string; +} + +/** + * Risk factor + */ +export interface RiskFactor { + type: string; + severity: 'low' | 'medium' | 'high'; + description: string; + score: number; +} + +/** + * Fraud detection configuration + */ +export interface FraudDetectionConfig { + /** Enable fraud detection */ + enabled: boolean; + /** Automatic decline threshold (risk score) */ + autoDeclineThreshold: number; + /** Manual review threshold (risk score) */ + manualReviewThreshold: number; + /** Enable velocity checks */ + enableVelocityChecks: boolean; + /** Enable geolocation checks */ + enableGeolocationChecks: boolean; + /** Enable device fingerprinting */ + enableDeviceFingerprinting: boolean; + /** Enable behavioral analysis */ + enableBehavioralAnalysis: boolean; +} + +/** + * Default configuration + */ +export const DEFAULT_FRAUD_CONFIG: FraudDetectionConfig = { + enabled: true, + autoDeclineThreshold: 80, + manualReviewThreshold: 60, + enableVelocityChecks: true, + enableGeolocationChecks: true, + enableDeviceFingerprinting: true, + enableBehavioralAnalysis: true, +}; + +/** + * Transaction velocity tracker + */ +interface VelocityRecord { + count: number; + totalAmount: number; + timestamps: string[]; +} + +/** + * Fraud detection service + */ +export class PaymentFraudDetectionService { + private config: FraudDetectionConfig; + private velocityTracker: Map = new Map(); + private blocklist: Set = new Set(); + private trustedCustomers: Set = new Set(); + + constructor(config: Partial = {}) { + this.config = { ...DEFAULT_FRAUD_CONFIG, ...config }; + } + + /** + * Assess risk for a payment transaction + */ + async assessRisk(transaction: PaymentTransaction): Promise { + if (!this.config.enabled) { + return this.createLowRiskAssessment(transaction); + } + + const factors: RiskFactor[] = []; + + // Check blocklist + const blocklistFactor = this.checkBlocklist(transaction); + if (blocklistFactor) { + factors.push(blocklistFactor); + } + + // Check trusted customers + if (this.isTrustedCustomer(transaction)) { + return this.createTrustedCustomerAssessment(transaction); + } + + // Amount anomaly check + const amountFactor = this.checkAmountAnomaly(transaction); + if (amountFactor) { + factors.push(amountFactor); + } + + // Velocity checks + if (this.config.enableVelocityChecks) { + const velocityFactor = this.checkVelocity(transaction); + if (velocityFactor) { + factors.push(velocityFactor); + } + } + + // Geolocation checks + if (this.config.enableGeolocationChecks && transaction.customerIp) { + const geoFactor = this.checkGeolocation(transaction); + if (geoFactor) { + factors.push(geoFactor); + } + } + + // Device fingerprint checks + if (this.config.enableDeviceFingerprinting && transaction.deviceFingerprint) { + const deviceFactor = this.checkDeviceFingerprint(transaction); + if (deviceFactor) { + factors.push(deviceFactor); + } + } + + // Address verification + const addressFactor = this.checkAddressMismatch(transaction); + if (addressFactor) { + factors.push(addressFactor); + } + + // Payment method risk + const paymentMethodFactor = this.checkPaymentMethod(transaction); + if (paymentMethodFactor) { + factors.push(paymentMethodFactor); + } + + // Calculate overall risk score + const riskScore = this.calculateRiskScore(factors); + const riskLevel = this.determineRiskLevel(riskScore); + const decision = this.makeDecision(riskScore); + + return { + transactionId: transaction.id, + riskScore, + riskLevel, + decision, + factors, + recommendation: this.generateRecommendation(riskScore, factors), + assessedAt: new Date().toISOString(), + }; + } + + /** + * Check if entity is on blocklist + */ + private checkBlocklist(transaction: PaymentTransaction): RiskFactor | null { + const identifiers = [ + transaction.customerEmail, + transaction.customerIp, + transaction.deviceFingerprint, + transaction.paymentMethod.walletAddress, + ].filter(Boolean) as string[]; + + for (const identifier of identifiers) { + if (this.blocklist.has(identifier)) { + return { + type: 'blocklist', + severity: 'high', + description: 'Customer or payment method is on blocklist', + score: 90, + }; + } + } + + return null; + } + + /** + * Check if customer is trusted + */ + private isTrustedCustomer(transaction: PaymentTransaction): boolean { + if (!transaction.customerId) return false; + return this.trustedCustomers.has(transaction.customerId); + } + + /** + * Check for amount anomalies + */ + private checkAmountAnomaly(transaction: PaymentTransaction): RiskFactor | null { + const amount = parseFloat(transaction.amount); + + // Check for suspiciously large amounts + if (amount > 10000) { + return { + type: 'high_amount', + severity: 'high', + description: `Transaction amount $${amount} is unusually high`, + score: 40, + }; + } + + // Check for suspiciously round amounts (potential test transactions) + if (amount % 100 === 0 && amount >= 1000) { + return { + type: 'round_amount', + severity: 'low', + description: 'Transaction amount is suspiciously round', + score: 10, + }; + } + + return null; + } + + /** + * Check transaction velocity + */ + private checkVelocity(transaction: PaymentTransaction): RiskFactor | null { + const key = this.getVelocityKey(transaction); + const record = this.velocityTracker.get(key) || { + count: 0, + totalAmount: 0, + timestamps: [], + }; + + const now = new Date(); + const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000); + + // Filter to last hour + record.timestamps = record.timestamps.filter( + (ts) => new Date(ts) > oneHourAgo, + ); + + record.count = record.timestamps.length + 1; + record.totalAmount += parseFloat(transaction.amount); + record.timestamps.push(transaction.timestamp); + + this.velocityTracker.set(key, record); + + // Check for velocity violations + if (record.count > 10) { + return { + type: 'high_velocity', + severity: 'high', + description: `${record.count} transactions in the last hour`, + score: 60, + }; + } + + if (record.totalAmount > 50000) { + return { + type: 'high_volume', + severity: 'medium', + description: `$${record.totalAmount} in transaction volume in the last hour`, + score: 40, + }; + } + + return null; + } + + /** + * Check geolocation for anomalies + */ + private checkGeolocation(transaction: PaymentTransaction): RiskFactor | null { + // Simplified geolocation check + // In production, use actual IP geolocation service + + if (!transaction.customerIp) return null; + + // Check for known risky countries (simplified) + const riskyPatterns = ['tor-exit', 'proxy', 'vpn']; + + if (riskyPatterns.some((pattern) => transaction.customerIp!.includes(pattern))) { + return { + type: 'suspicious_ip', + severity: 'medium', + description: 'Transaction from suspicious IP address', + score: 30, + }; + } + + return null; + } + + /** + * Check device fingerprint + */ + private checkDeviceFingerprint( + transaction: PaymentTransaction, + ): RiskFactor | null { + if (!transaction.deviceFingerprint) return null; + + // Check if device has been used for multiple failed transactions + // This would require historical data in production + + return null; + } + + /** + * Check for address mismatches + */ + private checkAddressMismatch(transaction: PaymentTransaction): RiskFactor | null { + if (!transaction.billingAddress || !transaction.shippingAddress) { + return null; + } + + const billing = transaction.billingAddress; + const shipping = transaction.shippingAddress; + + // Check country mismatch + if (billing.country !== shipping.country) { + return { + type: 'address_mismatch', + severity: 'medium', + description: 'Billing and shipping addresses in different countries', + score: 25, + }; + } + + // Check significant distance (simplified) + if (billing.city !== shipping.city && billing.state !== shipping.state) { + return { + type: 'address_distance', + severity: 'low', + description: 'Billing and shipping addresses are far apart', + score: 15, + }; + } + + return null; + } + + /** + * Check payment method risk + */ + private checkPaymentMethod(transaction: PaymentTransaction): RiskFactor | null { + const method = transaction.paymentMethod; + + // Crypto payments have higher baseline risk + if (method.type === 'crypto') { + return { + type: 'crypto_payment', + severity: 'low', + description: 'Cryptocurrency payments have elevated risk', + score: 10, + }; + } + + return null; + } + + /** + * Calculate overall risk score from factors + */ + private calculateRiskScore(factors: RiskFactor[]): number { + if (factors.length === 0) return 0; + + // Sum individual scores with severity weighting + const weightedSum = factors.reduce((sum, factor) => { + const weight = + factor.severity === 'high' ? 1.5 : factor.severity === 'medium' ? 1.2 : 1.0; + return sum + factor.score * weight; + }, 0); + + // Cap at 100 + return Math.min(Math.round(weightedSum), 100); + } + + /** + * Determine risk level from score + */ + private determineRiskLevel( + riskScore: number, + ): 'low' | 'medium' | 'high' | 'critical' { + if (riskScore >= 80) return 'critical'; + if (riskScore >= 60) return 'high'; + if (riskScore >= 30) return 'medium'; + return 'low'; + } + + /** + * Make decision based on risk score + */ + private makeDecision(riskScore: number): 'approve' | 'review' | 'decline' { + if (riskScore >= this.config.autoDeclineThreshold) { + return 'decline'; + } + if (riskScore >= this.config.manualReviewThreshold) { + return 'review'; + } + return 'approve'; + } + + /** + * Generate recommendation message + */ + private generateRecommendation( + riskScore: number, + factors: RiskFactor[], + ): string { + if (riskScore >= this.config.autoDeclineThreshold) { + return 'Automatically decline this transaction due to high fraud risk'; + } + + if (riskScore >= this.config.manualReviewThreshold) { + const highRiskFactors = factors + .filter((f) => f.severity === 'high') + .map((f) => f.type); + return `Manual review required. Primary concerns: ${highRiskFactors.join(', ') || 'multiple medium-risk factors'}`; + } + + return 'Approve transaction - risk level acceptable'; + } + + /** + * Create low risk assessment for disabled fraud detection + */ + private createLowRiskAssessment( + transaction: PaymentTransaction, + ): RiskAssessment { + return { + transactionId: transaction.id, + riskScore: 0, + riskLevel: 'low', + decision: 'approve', + factors: [], + recommendation: 'Fraud detection disabled - auto-approved', + assessedAt: new Date().toISOString(), + }; + } + + /** + * Create assessment for trusted customers + */ + private createTrustedCustomerAssessment( + transaction: PaymentTransaction, + ): RiskAssessment { + return { + transactionId: transaction.id, + riskScore: 5, + riskLevel: 'low', + decision: 'approve', + factors: [ + { + type: 'trusted_customer', + severity: 'low', + description: 'Customer has trusted status', + score: -50, // Negative score to reduce risk + }, + ], + recommendation: 'Approve - trusted customer', + assessedAt: new Date().toISOString(), + }; + } + + /** + * Get velocity tracking key for customer + */ + private getVelocityKey(transaction: PaymentTransaction): string { + const identifier = transaction.customerId || transaction.customerIp || 'anonymous'; + return createHash('sha256').update(identifier).digest('hex'); + } + + /** + * Add entity to blocklist + */ + addToBlocklist(identifier: string): void { + this.blocklist.add(identifier); + } + + /** + * Remove entity from blocklist + */ + removeFromBlocklist(identifier: string): void { + this.blocklist.delete(identifier); + } + + /** + * Add customer to trusted list + */ + addTrustedCustomer(customerId: string): void { + this.trustedCustomers.add(customerId); + } + + /** + * Remove customer from trusted list + */ + removeTrustedCustomer(customerId: string): void { + this.trustedCustomers.delete(customerId); + } + + /** + * Clear velocity tracking (for testing) + */ + clearVelocityTracking(): void { + this.velocityTracker.clear(); + } + + /** + * Get statistics + */ + getStatistics(): { + blocklistSize: number; + trustedCustomersCount: number; + velocityRecordsCount: number; + } { + return { + blocklistSize: this.blocklist.size, + trustedCustomersCount: this.trustedCustomers.size, + velocityRecordsCount: this.velocityTracker.size, + }; + } +} diff --git a/backend/src/services/payout-reliability.ts b/backend/src/services/payout-reliability.ts new file mode 100644 index 00000000..eeeb5f9c --- /dev/null +++ b/backend/src/services/payout-reliability.ts @@ -0,0 +1,427 @@ +// Issue #1155: Improve payout reliability (iteration 5) +// Implements payout queue management with dead letter queue and health monitoring + +import { EventEmitter } from 'events'; + +/** + * Payout request structure + */ +export interface PayoutRequest { + id: string; + recipientAddress: string; + amount: string; + currency: string; + merchantId: string; + createdAt: string; + attempts: number; + lastAttemptAt?: string; + lastError?: string; + status: 'pending' | 'processing' | 'completed' | 'failed' | 'dead_letter'; + metadata?: Record; +} + +/** + * Payout result + */ +export interface PayoutResult { + success: boolean; + transactionHash?: string; + error?: string; + retriable: boolean; +} + +/** + * Payout reliability configuration + */ +export interface PayoutReliabilityConfig { + /** Maximum retry attempts before moving to dead letter queue */ + maxRetries: number; + /** Initial retry delay in milliseconds */ + initialRetryDelayMs: number; + /** Maximum retry delay in milliseconds */ + maxRetryDelayMs: number; + /** Retry backoff multiplier */ + backoffMultiplier: number; + /** Enable dead letter queue */ + enableDeadLetterQueue: boolean; + /** Health check interval in milliseconds */ + healthCheckIntervalMs: number; + /** Concurrent payout processing limit */ + concurrencyLimit: number; +} + +/** + * Default configuration + */ +export const DEFAULT_PAYOUT_CONFIG: PayoutReliabilityConfig = { + maxRetries: 5, + initialRetryDelayMs: 1000, + maxRetryDelayMs: 300000, + backoffMultiplier: 2, + enableDeadLetterQueue: true, + healthCheckIntervalMs: 60000, + concurrencyLimit: 10, +}; + +/** + * Payout health metrics + */ +export interface PayoutHealthMetrics { + totalProcessed: number; + successCount: number; + failureCount: number; + deadLetterCount: number; + averageProcessingTimeMs: number; + queueDepth: number; + isHealthy: boolean; + lastHealthCheckAt: string; +} + +/** + * Payout processor interface + */ +export interface IPayoutProcessor { + process(request: PayoutRequest): Promise; +} + +/** + * Payout reliability service with queue management and monitoring + */ +export class PayoutReliabilityService extends EventEmitter { + private queue: PayoutRequest[] = []; + private deadLetterQueue: PayoutRequest[] = []; + private processing: Set = new Set(); + private metrics: PayoutHealthMetrics; + private config: PayoutReliabilityConfig; + private processor: IPayoutProcessor; + private healthCheckInterval?: NodeJS.Timeout; + private processingTimes: number[] = []; + + constructor( + processor: IPayoutProcessor, + config: Partial = {}, + ) { + super(); + this.processor = processor; + this.config = { ...DEFAULT_PAYOUT_CONFIG, ...config }; + this.metrics = this.initializeMetrics(); + this.startHealthMonitoring(); + } + + /** + * Initialize metrics + */ + private initializeMetrics(): PayoutHealthMetrics { + return { + totalProcessed: 0, + successCount: 0, + failureCount: 0, + deadLetterCount: 0, + averageProcessingTimeMs: 0, + queueDepth: 0, + isHealthy: true, + lastHealthCheckAt: new Date().toISOString(), + }; + } + + /** + * Enqueue a payout request + */ + async enqueue(request: PayoutRequest): Promise { + this.queue.push(request); + this.emit('payout:enqueued', request); + + // Process queue if under concurrency limit + if (this.processing.size < this.config.concurrencyLimit) { + this.processQueue(); + } + } + + /** + * Process the payout queue + */ + private async processQueue(): Promise { + while ( + this.queue.length > 0 && + this.processing.size < this.config.concurrencyLimit + ) { + const request = this.queue.shift(); + if (!request) continue; + + this.processPayoutWithRetry(request); + } + } + + /** + * Process a single payout with retry logic + */ + private async processPayoutWithRetry(request: PayoutRequest): Promise { + this.processing.add(request.id); + this.emit('payout:processing', request); + + const startTime = Date.now(); + + try { + const result = await this.processor.process(request); + const processingTime = Date.now() - startTime; + + this.recordProcessingTime(processingTime); + + if (result.success) { + await this.handleSuccess(request, result); + } else { + await this.handleFailure(request, result); + } + } catch (error) { + const processingTime = Date.now() - startTime; + this.recordProcessingTime(processingTime); + + await this.handleFailure(request, { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + retriable: true, + }); + } finally { + this.processing.delete(request.id); + + // Continue processing queue + this.processQueue(); + } + } + + /** + * Handle successful payout + */ + private async handleSuccess( + request: PayoutRequest, + result: PayoutResult, + ): Promise { + request.status = 'completed'; + this.metrics.successCount++; + this.metrics.totalProcessed++; + + this.emit('payout:completed', { + request, + transactionHash: result.transactionHash, + }); + } + + /** + * Handle failed payout + */ + private async handleFailure( + request: PayoutRequest, + result: PayoutResult, + ): Promise { + request.attempts++; + request.lastAttemptAt = new Date().toISOString(); + request.lastError = result.error; + + // Check if should retry + if (result.retriable && request.attempts < this.config.maxRetries) { + // Schedule retry with exponential backoff + const delay = this.calculateRetryDelay(request.attempts); + + this.emit('payout:retry_scheduled', { + request, + attempt: request.attempts, + delayMs: delay, + }); + + setTimeout(() => { + this.enqueue(request); + }, delay); + } else { + // Move to dead letter queue or mark as failed + if (this.config.enableDeadLetterQueue) { + await this.moveToDeadLetterQueue(request); + } else { + request.status = 'failed'; + this.metrics.failureCount++; + this.metrics.totalProcessed++; + + this.emit('payout:failed', request); + } + } + } + + /** + * Move payout to dead letter queue + */ + private async moveToDeadLetterQueue(request: PayoutRequest): Promise { + request.status = 'dead_letter'; + this.deadLetterQueue.push(request); + this.metrics.deadLetterCount++; + this.metrics.totalProcessed++; + + this.emit('payout:dead_letter', request); + + // Alert if dead letter queue is growing + if (this.deadLetterQueue.length > 100) { + this.emit('alert:dead_letter_threshold', { + count: this.deadLetterQueue.length, + threshold: 100, + }); + } + } + + /** + * Calculate retry delay with exponential backoff + */ + private calculateRetryDelay(attempt: number): number { + const delay = Math.min( + this.config.initialRetryDelayMs * Math.pow(this.config.backoffMultiplier, attempt - 1), + this.config.maxRetryDelayMs, + ); + + // Add jitter to prevent thundering herd + const jitter = Math.random() * 0.1 * delay; + return delay + jitter; + } + + /** + * Record processing time for metrics + */ + private recordProcessingTime(timeMs: number): void { + this.processingTimes.push(timeMs); + + // Keep only last 100 measurements + if (this.processingTimes.length > 100) { + this.processingTimes.shift(); + } + + // Update average + this.metrics.averageProcessingTimeMs = + this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length; + } + + /** + * Start health monitoring + */ + private startHealthMonitoring(): void { + this.healthCheckInterval = setInterval(() => { + this.performHealthCheck(); + }, this.config.healthCheckIntervalMs); + } + + /** + * Perform health check + */ + private performHealthCheck(): void { + this.metrics.queueDepth = this.queue.length; + this.metrics.lastHealthCheckAt = new Date().toISOString(); + + // Health criteria + const criteria = { + queueDepthOk: this.queue.length < 1000, + deadLetterOk: this.deadLetterQueue.length < 100, + successRateOk: + this.metrics.totalProcessed === 0 || + this.metrics.successCount / this.metrics.totalProcessed > 0.95, + processingTimeOk: this.metrics.averageProcessingTimeMs < 5000, + }; + + this.metrics.isHealthy = Object.values(criteria).every((v) => v); + + this.emit('health:check', { + metrics: this.metrics, + criteria, + }); + + if (!this.metrics.isHealthy) { + this.emit('alert:unhealthy', { + metrics: this.metrics, + criteria, + }); + } + } + + /** + * Get current health metrics + */ + getHealthMetrics(): PayoutHealthMetrics { + return { ...this.metrics }; + } + + /** + * Get dead letter queue + */ + getDeadLetterQueue(): PayoutRequest[] { + return [...this.deadLetterQueue]; + } + + /** + * Retry payout from dead letter queue + */ + async retryDeadLetter(requestId: string): Promise { + const index = this.deadLetterQueue.findIndex((r) => r.id === requestId); + if (index === -1) return false; + + const request = this.deadLetterQueue.splice(index, 1)[0]; + this.metrics.deadLetterCount--; + + // Reset attempts and enqueue + request.attempts = 0; + request.status = 'pending'; + await this.enqueue(request); + + this.emit('payout:retry_from_dead_letter', request); + return true; + } + + /** + * Bulk retry dead letter queue + */ + async retryAllDeadLetters(): Promise { + const count = this.deadLetterQueue.length; + const requests = this.deadLetterQueue.splice(0); + + for (const request of requests) { + request.attempts = 0; + request.status = 'pending'; + await this.enqueue(request); + } + + this.metrics.deadLetterCount = 0; + this.emit('payout:bulk_retry_dead_letter', { count }); + + return count; + } + + /** + * Clear dead letter queue + */ + clearDeadLetterQueue(): number { + const count = this.deadLetterQueue.length; + this.deadLetterQueue = []; + this.metrics.deadLetterCount = 0; + + this.emit('dead_letter:cleared', { count }); + return count; + } + + /** + * Stop the service + */ + stop(): void { + if (this.healthCheckInterval) { + clearInterval(this.healthCheckInterval); + } + this.removeAllListeners(); + } + + /** + * Get queue statistics + */ + getQueueStats(): { + pending: number; + processing: number; + deadLetter: number; + } { + return { + pending: this.queue.length, + processing: this.processing.size, + deadLetter: this.deadLetterQueue.length, + }; + } +} diff --git a/docs/WEBHOOK_RETRY_STRATEGY.md b/docs/WEBHOOK_RETRY_STRATEGY.md new file mode 100644 index 00000000..c11ecfe1 --- /dev/null +++ b/docs/WEBHOOK_RETRY_STRATEGY.md @@ -0,0 +1,294 @@ +# Webhook Retry Strategy + +## Overview + +This document describes the webhook delivery and retry strategy for AgenticPay. It ensures reliable event notification delivery to merchant endpoints with exponential backoff and circuit breaker patterns. + +## Issue + +**#1158 - Improve documentation (iteration 5)** + +This documentation increment covers the webhook retry mechanism, failure handling, and best practices for webhook consumers. + +## Webhook Delivery Flow + +``` +Event Triggered → Webhook Queued → Delivery Attempt + ↓ + Success? → Mark Delivered + ↓ + Failed + ↓ + Retry with Backoff + ↓ + Max Retries? → Mark Failed & Alert +``` + +## Retry Configuration + +### Default Settings + +```typescript +const DEFAULT_WEBHOOK_CONFIG = { + maxRetries: 5, + initialBackoffMs: 1000, // 1 second + maxBackoffMs: 300000, // 5 minutes + backoffMultiplier: 2, // Exponential backoff + timeoutMs: 10000, // 10 second request timeout + circuitBreakerThreshold: 10, // Failed attempts before opening circuit + circuitBreakerResetMs: 60000, // 1 minute cooldown +}; +``` + +### Retry Schedule + +| Attempt | Delay | Cumulative Time | +|---------|-------|-----------------| +| 1 | Immediate | 0s | +| 2 | 1s | 1s | +| 3 | 2s | 3s | +| 4 | 4s | 7s | +| 5 | 8s | 15s | +| 6 | 16s | 31s | + +After 6 attempts over ~30 seconds, the webhook is marked as failed and requires manual intervention. + +## HTTP Response Handling + +### Success Conditions + +Webhook delivery is considered successful when: +- HTTP status code is 2xx (200-299) +- Response received within timeout window +- No network errors + +### Retry Conditions + +Webhooks are retried on: +- HTTP 5xx errors (server errors) +- HTTP 429 (rate limit - respects Retry-After header) +- Network timeouts +- Connection refused/reset +- DNS resolution failures + +### No Retry Conditions + +Webhooks are NOT retried on: +- HTTP 4xx errors (except 429) - indicates client error +- Invalid SSL certificates (security policy) +- Endpoint returns malformed response + +## Circuit Breaker + +The circuit breaker protects both systems from cascading failures: + +### States + +1. **Closed (Normal)**: All webhooks delivered normally +2. **Open (Failing)**: No delivery attempts, webhooks queued +3. **Half-Open (Testing)**: Single probe request to test recovery + +### Behavior + +- Circuit opens after 10 consecutive failures to same endpoint +- While open, webhooks are queued but not delivered +- After 1 minute cooldown, circuit moves to half-open +- Single successful delivery closes circuit +- Single failed delivery reopens circuit + +## Event Types and Priority + +### High Priority (Immediate delivery, aggressive retry) + +- `payment.completed` +- `payment.failed` +- `escrow.released` +- `subscription.canceled` + +### Standard Priority (Normal retry schedule) + +- `invoice.created` +- `customer.updated` +- `dispute.opened` + +### Low Priority (Delayed delivery, reduced retries) + +- `analytics.generated` +- `report.ready` + +## Webhook Security + +### Signature Verification + +All webhooks include a signature header for verification: + +``` +X-AgenticPay-Signature: t=1234567890,v1=abc123def456... +``` + +Verification algorithm: +```typescript +const payload = `${timestamp}.${JSON.stringify(body)}`; +const signature = crypto + .createHmac('sha256', webhookSecret) + .update(payload) + .digest('hex'); +``` + +### Replay Protection + +- Timestamps older than 5 minutes are rejected +- Duplicate event IDs within 24 hours are rejected + +## Monitoring and Debugging + +### Webhook Logs + +Each webhook attempt is logged with: +- Event ID +- Attempt number +- HTTP status code +- Response time +- Error details (if failed) + +### Dashboard Metrics + +Available in merchant dashboard: +- Delivery success rate (last 24h, 7d, 30d) +- Average delivery time +- Failed webhooks requiring attention +- Circuit breaker status per endpoint + +### Testing Webhooks + +Use the webhook testing endpoint: + +```bash +POST /api/v1/webhooks/test +{ + "url": "https://your-endpoint.com/webhooks", + "eventType": "payment.completed" +} +``` + +## Best Practices for Consumers + +### 1. Idempotency + +Always handle webhooks idempotently: + +```typescript +app.post('/webhooks', async (req, res) => { + const eventId = req.body.id; + + // Check if already processed + if (await isEventProcessed(eventId)) { + return res.status(200).send('Already processed'); + } + + // Process event + await processEvent(req.body); + + // Mark as processed + await markEventProcessed(eventId); + + res.status(200).send('OK'); +}); +``` + +### 2. Quick Response + +Respond quickly (< 5 seconds) to avoid timeouts: + +```typescript +app.post('/webhooks', async (req, res) => { + // Acknowledge receipt immediately + res.status(200).send('OK'); + + // Process asynchronously + processEventAsync(req.body).catch(handleError); +}); +``` + +### 3. Verify Signatures + +Always verify webhook signatures: + +```typescript +import { verifyWebhookSignature } from '@agenticpay/sdk'; + +app.post('/webhooks', (req, res) => { + const signature = req.headers['x-agenticpay-signature']; + + if (!verifyWebhookSignature(req.body, signature, webhookSecret)) { + return res.status(401).send('Invalid signature'); + } + + // Process webhook... +}); +``` + +### 4. Error Handling + +Return appropriate status codes: + +```typescript +app.post('/webhooks', async (req, res) => { + try { + await processEvent(req.body); + res.status(200).send('OK'); + } catch (error) { + if (error.isRetryable) { + // Temporary error - AgenticPay should retry + res.status(500).send('Temporary error'); + } else { + // Permanent error - don't retry + res.status(400).send('Invalid event'); + } + } +}); +``` + +## Troubleshooting + +### Webhooks Not Received + +1. Check endpoint is publicly accessible +2. Verify SSL certificate is valid +3. Check firewall rules allow AgenticPay IPs +4. Review webhook logs in dashboard + +### High Failure Rate + +1. Check endpoint response time (should be < 5s) +2. Verify signature verification is correct +3. Review error logs for specific failure reasons +4. Check if circuit breaker is open + +### Duplicate Events + +1. Implement idempotency checks using event ID +2. Store processed event IDs for 24 hours minimum +3. Return 200 OK for already-processed events + +## Configuration + +Webhook settings can be configured per merchant: + +```typescript +await sdk.webhooks.updateConfig({ + maxRetries: 3, // Reduce retry attempts + timeoutMs: 5000, // Shorter timeout + enableCircuitBreaker: true, // Enable protection +}); +``` + +## Changelog + +- **2024-10-17**: Initial webhook retry strategy documentation (Issue #1158) + +## Related Documentation + +- [Webhook Verification](./WEBHOOK_VERIFICATION.md) +- [API Documentation](./api.md) +- [Troubleshooting Guide](./troubleshooting.md)