feat(backend): SEP-0001 retry with backoff; Payment Processor Telemetry sanitization, health and retries

backend/.env.example

@@ -97,3 +97,18 @@ RESEND_API_KEY=your_resend_api_key
 # Sender address for receipt emails (must be a verified domain in Resend)
 # Defaults to: receipts@notifications.dripsnetwork.com
 RECEIPT_FROM_EMAIL=receipts@yourdomain.com
+
+# SEP-0001 stellar.toml merchant lookup retry (#1459; clamped 1-5 attempts, <=5000ms)
+# SEP0001_RETRY_MAX_ATTEMPTS=3
+# SEP0001_RETRY_BASE_DELAY_MS=100
+# SEP0001_RETRY_MAX_DELAY_MS=1000
+
+# Payment Processor Telemetry (#1477-#1479, docs/PAYMENT_PROCESSOR.md section 6)
+# PAYMENT_PROCESSOR_TELEMETRY_MAX_ASSET_LABELS=50
+# PAYMENT_PROCESSOR_HEALTH_WINDOW_MS=300000
+# PAYMENT_PROCESSOR_HEALTH_MIN_SAMPLES=20
+# PAYMENT_PROCESSOR_DEGRADED_ERROR_RATIO=0.05
+# PAYMENT_PROCESSOR_UNHEALTHY_ERROR_RATIO=0.2
+# PAYMENT_PROCESSOR_RETRY_MAX_ATTEMPTS=3
+# PAYMENT_PROCESSOR_RETRY_BASE_DELAY_MS=50
+# PAYMENT_PROCESSOR_RETRY_MAX_DELAY_MS=500

backend/SEP0001_GENERATOR_GUIDE.md

@@ -262,6 +262,33 @@ console.log('Accounts:', toml.ACCOUNTS);
 - Wait 1 hour for changes to propagate
 - Clear CDN cache if using a CDN
 
+## Retry with Exponential Backoff (#1459)
+
+The only I/O behind `GET /.well-known/stellar.toml` is the merchant lookup.
+It is retried when it fails transiently (`src/lib/sep0001-retry.js`):
+
+- Retried: network errors, 408/429/5xx, connection-class Postgres errors.
+- Not retried: unknown or invalid merchant ids, PostgREST client errors.
+- Delay: full jitter, `random(0, min(max, base * 2^n))`.
+
+| Variable | Default | Clamp |
+|---|---|---|
+| `SEP0001_RETRY_MAX_ATTEMPTS` | 3 | 1-5 |
+| `SEP0001_RETRY_BASE_DELAY_MS` | 100 | 0-5000 |
+| `SEP0001_RETRY_MAX_DELAY_MS` | 1000 | base-5000 |
+
+If the lookup is still failing transiently after the last attempt, the
+endpoint answers **503** with `Retry-After: 5` instead of 500, so wallets
+retry later instead of treating the domain as having no stellar.toml.
+Deterministic lookup errors still return 500. Every error response carries
+`Cache-Control: no-store`, so a CDN never caches a failure for the hour that
+successful responses are cached.
+
+Retry outcomes are exported as
+`sep0001_merchant_lookup_retries_total{result="scheduled|recovered|exhausted"}`
+on `GET /metrics`. Logs carry the error code, attempt count and message, never
+the merchant record.
+
 ## Security Considerations
 
 1. **Public Information**: stellar.toml is publicly accessible

backend/docs/PAYMENT_PROCESSOR.md

@@ -4,7 +4,9 @@ Architecture, granular metrics, and testing guide for the Payment Processor
 module (`src/routes/payments.js` + `src/services/paymentService.js`).
 
 Covers issues **#1086** (end-to-end testing), **#1087** (legacy refactor),
-**#1088** (granular metrics) and **#1089** (load testing).
+**#1088** (granular metrics), **#1089** (load testing), and Payment Processor
+Telemetry: **#1477** (payload sanitization and strict validation), **#1478**
+(alert metrics and health) and **#1479** (retry with exponential backoff).
 
 ---
 
@@ -118,3 +120,105 @@ Include the printed percentile tables in PRs touching the processor.
 - Verification uses a DB-level conditional update so a `tx_hash` can only be
   claimed once even under concurrent confirmation attempts
 - Signature verification failure keeps payments pending (fail-closed)
+
+---
+
+## 6. Payment Processor Telemetry (#1477, #1478, #1479)
+
+| File | Role |
+|---|---|
+| `src/lib/payment-processor-telemetry.js` | Sanitising, validating recorders; rolling-window health; new series |
+| `src/lib/payment-processor-retry.js` | Retried idempotent reads (status / verification lookup) |
+| `src/lib/backoff-retry.js` | Shared full-jitter backoff used by the processor and SEP-0001 |
+| `docs/alerts/payment-processor.rules.yml` | Prometheus alert rules |
+
+### 6.1 Payload sanitization and strict validation (#1477)
+
+`routes/payments.js` and `services/paymentService.js` no longer call the
+`payment_processor_*` metric objects directly. They call the recorders
+(`recordSessionTelemetry`, `recordVerificationTelemetry`,
+`recordRefundTelemetry`, `recordListRequestTelemetry`,
+`recordStatusCacheTelemetry`, `recordProcessorRetryTelemetry`), which:
+
+| Input | Rule | Otherwise |
+|---|---|---|
+| payload | plain object | rejected (`invalid_payload`) |
+| `outcome` / `stage` / `operation` | fixed allow-list per series | rejected (`invalid_outcome` / `invalid_stage` / `invalid_operation`) |
+| `asset` | trimmed, upper-cased, `^[A-Z0-9]{1,12}$`; `native` becomes `XLM` | label `invalid`; missing becomes `unknown` |
+| distinct assets | at most `PAYMENT_PROCESSOR_TELEMETRY_MAX_ASSET_LABELS` (default 50) | label `other` |
+| duration | finite, `0` to `3600` seconds | count still recorded, observation dropped, `invalid_duration` |
+
+Before this, `req.body.asset` went straight into a label, so a client could
+create a new Prometheus series on every request. Rejected payloads are never
+written to the business series; they increment
+`payment_processor_telemetry_rejected_total{metric, reason}` and log one
+warning with the metric and reason only. Recorders never throw.
+
+Allow-lists:
+
+- sessions: created, validation_failed, persistence_failed, sandbox_skipped
+- verifications: confirmed, already_confirmed, already_confirmed_concurrent,
+  pending_no_match, signature_invalid, underpayment, overpayment,
+  tx_claim_conflict, error
+- refunds: stage generate or confirm; outcome success, rejected, not_found,
+  failed_on_chain, error
+- list requests: success, pool_fallback, error
+
+### 6.2 Alert metrics and health (#1478)
+
+| Series | Type | Notes |
+|---|---|---|
+| `payment_processor_health_state` | gauge | 0 healthy, 1 degraded, 2 unhealthy; refreshed at scrape |
+| `payment_processor_session_failure_ratio` | gauge | persistence_failed / (created + persistence_failed) |
+| `payment_processor_verification_error_ratio` | gauge | error / all verifications |
+| `payment_processor_refund_error_ratio` | gauge | (error + failed_on_chain) / all refund operations |
+| `payment_processor_telemetry_rejection_ratio` | gauge | rejected / all telemetry events |
+| `payment_processor_telemetry_rejected_total` | counter | `metric`, `reason` |
+| `payment_processor_retries_total` | counter | `operation`, `result` (see 6.3) |
+
+Health uses a rolling window (`PAYMENT_PROCESSOR_HEALTH_WINDOW_MS`, default
+5 min) and needs `PAYMENT_PROCESSOR_HEALTH_MIN_SAMPLES` (default 20) before a
+ratio counts:
+
+- `unhealthy`: any error ratio >= `PAYMENT_PROCESSOR_UNHEALTHY_ERROR_RATIO` (0.2)
+- `degraded`: any error ratio >= `PAYMENT_PROCESSOR_DEGRADED_ERROR_RATIO`
+  (0.05), telemetry rejections at that ratio, or any exhausted read retry
+- `healthy`: otherwise
+
+Validation failures (bad client input) do not count as processor errors.
+
+`GET /health/payment-processor` returns the snapshot (503 only when
+unhealthy). `GET /health` lists `services.payment_processor` for information
+only. The body holds counts and ratios, never payment ids, amounts, addresses
+or asset codes.
+
+### 6.3 Retry with exponential backoff (#1479)
+
+The first Supabase read of the status poll and of verification
+(`status_lookup`, `verify_lookup`, in both the route and the service) is
+retried with full jitter, `random(0, min(max, base * 2^n))`:
+
+| Variable | Default | Clamp |
+|---|---|---|
+| `PAYMENT_PROCESSOR_RETRY_MAX_ATTEMPTS` | 3 | 1-5 |
+| `PAYMENT_PROCESSOR_RETRY_BASE_DELAY_MS` | 50 | 0-3000 |
+| `PAYMENT_PROCESSOR_RETRY_MAX_DELAY_MS` | 500 | base-3000 |
+
+- Retried: network errors, 408/429/5xx, connection-class Postgres errors
+  (shared `isRetryableSessionError`).
+- Not retried: not-found, invalid ids, missing columns (the status route still
+  falls through its schema-tolerant query shapes), PostgREST client errors.
+- Writes (status transitions, tx claims, refunds) are never wrapped.
+- Each query is rebuilt per attempt (PostgREST builders are single-use).
+- After the last attempt the final `{ data, error }` is returned unchanged, so
+  response codes are as before.
+
+`payment_processor_retries_total{operation, result}` records `scheduled`,
+`recovered` and `exhausted`; exhausted retries also degrade health.
+
+### 6.4 Tests
+
+```bash
+npx vitest run src/lib/payment-processor-telemetry.test.js src/lib/payment-processor-retry.test.js src/lib/backoff-retry.test.js
+npx vitest run tests/integration/payment-processor-telemetry.test.js tests/e2e/payment-processor.e2e.test.js
+```

backend/docs/alerts/payment-processor.rules.yml

@@ -0,0 +1,82 @@
+# Prometheus alerting rules for Payment Processor Telemetry (issues #1477-#1479).
+#
+# Load with:   rule_files: ["docs/alerts/payment-processor.rules.yml"]
+# Validate:    promtool check rules docs/alerts/payment-processor.rules.yml
+#
+# Metric reference: docs/PAYMENT_PROCESSOR.md (section 6)
+
+groups:
+  - name: payment-processor
+    rules:
+      - alert: PaymentProcessorUnhealthy
+        expr: max(payment_processor_health_state) >= 2
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Payment processor reports unhealthy
+          description: >
+            Session persistence, verification or refund errors are above the
+            unhealthy ratio over the rolling window. See
+            GET /health/payment-processor for the counts and reasons.
+
+      - alert: PaymentProcessorDegraded
+        expr: max(payment_processor_health_state) == 1
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: Payment processor degraded for 15 minutes
+          description: >
+            An error ratio is above the degraded threshold, telemetry payloads
+            are being rejected, or read retries are being exhausted.
+
+      - alert: PaymentProcessorSessionPersistenceFailures
+        expr: |
+          sum(rate(payment_processor_sessions_total{outcome="persistence_failed"}[10m]))
+            /
+          clamp_min(sum(rate(payment_processor_sessions_total{outcome=~"created|persistence_failed"}[10m])), 1e-9)
+            > 0.05
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: More than 5% of payment sessions fail to persist
+          description: >
+            Session inserts are failing after their own retries. Check
+            database health and the "Payment session operation failed" logs.
+
+      - alert: PaymentProcessorReadRetriesExhausted
+        expr: sum(increase(payment_processor_retries_total{result="exhausted"}[10m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: Payment processor reads failed after all retries
+          description: >
+            Status or verification lookups kept failing with transient errors
+            until PAYMENT_PROCESSOR_RETRY_MAX_ATTEMPTS ran out. Customers saw
+            5xx on payment-status or verify-payment.
+
+      - alert: PaymentProcessorTelemetryRejections
+        expr: sum(increase(payment_processor_telemetry_rejected_total[15m])) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: Payment processor telemetry payloads are being rejected
+          description: >
+            A code path is recording an outcome, stage or duration outside the
+            allow-list. The business series are not affected (rejected
+            payloads are dropped) but the gap hides real traffic. Check the
+            "Payment processor telemetry payload rejected" log for the metric.
+
+      - alert: PaymentProcessorAssetLabelOverflow
+        expr: sum(increase(payment_processor_sessions_total{asset=~"other|invalid"}[1h])) > 100
+        labels:
+          severity: info
+        annotations:
+          summary: Many sessions recorded under asset "other" or "invalid"
+          description: >
+            Either the distinct asset label cap
+            (PAYMENT_PROCESSOR_TELEMETRY_MAX_ASSET_LABELS) is too low for real
+            traffic, or clients are sending malformed asset codes.

backend/src/app.js

@@ -49,6 +49,7 @@ import { versionDeprecationMiddleware } from "./lib/version-deprecation.js";
 import oracleRouter from "./routes/oracle.js";
 import { getPaymentSessionValidatorHealth } from "./lib/payment-session-validator.js";
 import { getExchangeRateOracleHealth } from "./lib/exchange-rate-oracle-telemetry.js";
+import { getPaymentProcessorHealth } from "./lib/payment-processor-telemetry.js";
 import { configureExchangeRateCoordination } from "./services/exchangeRateService.js";
 
 export async function createApp({ redisClient }) {
@@ -262,6 +263,7 @@ export async function createApp({ redisClient }) {
         // Informational only — does not affect `ok` / the status code.
         payment_session_validator: getPaymentSessionValidatorHealth().status,
         exchange_rate_oracle_cache: getExchangeRateOracleHealth().status,
+        payment_processor: getPaymentProcessorHealth().status,
       },
     });
   });
@@ -314,6 +316,31 @@ export async function createApp({ redisClient }) {
     res.status(health.status === "unhealthy" ? 503 : 200).json(health);
   });
 
+  /**
+   * @swagger
+   * /health/payment-processor:
+   *   get:
+   *     summary: Payment Processor telemetry health
+   *     description: >
+   *       Rolling-window session, verification and refund outcome counts,
+   *       error ratios, telemetry rejections and exhausted read retries for
+   *       the payment processor (issue #1478). Returns 503 only when
+   *       unhealthy (an error ratio above the unhealthy threshold); degraded
+   *       still returns 200. The body contains no payment ids, amounts,
+   *       addresses or asset codes.
+   *     tags: [Health]
+   *     security: []
+   *     responses:
+   *       200:
+   *         description: Processor healthy or degraded
+   *       503:
+   *         description: Processor unhealthy
+   */
+  app.get("/health/payment-processor", (_req, res) => {
+    const health = getPaymentProcessorHealth();
+    res.status(health.status === "unhealthy" ? 503 : 200).json(health);
+  });
+
   const verifyPaymentRateLimit = createVerifyPaymentRateLimit({
     store: redisAvailable ? createRedisRateLimitStore({ client: redisClient }) : undefined,
   });