Fix backend monitoring, transfers, and yield fees

.github/workflows/backend-governance.yml

@@ -62,6 +62,9 @@ jobs:
       - name: Type-check backend
         run: npm run build
 
+      - name: Validate Prometheus alert rules
+        run: docker run --rm --entrypoint=/bin/promtool -v "${{ github.workspace }}:/workspace" prom/prometheus:v2.55.0 check rules /workspace/backend/monitoring/prometheus-alerts.yml
+
       - name: Validate Prisma schema and migration consistency
         run: npm run prisma:schema-check
 

backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md

@@ -0,0 +1,40 @@
+# Backend Prometheus Alert Runbook
+
+These alerts enforce the backend operational SLOs below. Prometheus rule annotations link to the relevant response section.
+
+| Signal | SLO / threshold |
+| --- | --- |
+| Background jobs | Every registered job remains healthy; no sustained dead-letter backlog |
+| Event outbox | Oldest pending or retrying event remains under 60 seconds; backlog stays at or below 100 |
+| Email queue | Oldest queued or retrying email remains under 5 minutes; backlog stays at or below 100 |
+| Webhook delivery | No failed deliveries remain unresolved; pending backlog stays at or below 100 |
+| Database queries | P95 query latency remains below 1 second; no sustained query-budget breaches |
+| Database backups | Scheduled backup job remains healthy |
+
+## Background Jobs
+
+1. Check `/admin/jobs/monitor` and `/admin/jobs/metrics` for the unhealthy job, recurring failures, and dead letters.
+2. Inspect the job's logs and dependency health. Resolve the underlying cause before retrying or discarding dead-letter work.
+3. For `databaseBackup`, verify `BACKUP_ENABLED`, the S3 bucket/endpoint/credentials, and the most recent object in the configured backup destination. Do not mark the job healthy until a usable backup is confirmed.
+
+## Webhooks And Event Outbox
+
+1. Check `/admin/jobs/monitor` for webhook delivery metrics and inspect the admin webhook delivery/dead-letter endpoints.
+2. Check event outbox entries for pending, failed, or dead-lettered records. Verify downstream webhook endpoint health and delivery secrets.
+3. Restore downstream availability, then retry only the affected dead-letter records. Confirm that backlog age and count return below the SLO.
+
+## Email Queue
+
+1. Inspect `/admin/emails/queue` and worker logs for retry counts and the latest delivery errors.
+2. Verify the mail provider configuration and connectivity. Replay dead-lettered messages only after the provider issue is resolved.
+3. Confirm queue age and pending count return below the SLO.
+
+## Database And Dependencies
+
+1. Compare P95 query latency and budget-breach metrics by Prisma model/action; correlate with database CPU, connection saturation, and slow-query logs.
+2. Check database connectivity, pool configuration, and recent migrations. Reduce the offending query load or address the slow query before scaling the pool.
+3. For external dependency latency, check the affected provider's status and dependency-specific operations before changing retry settings.
+
+## Alert Validation
+
+The backend governance workflow runs `promtool check rules` against `backend/monitoring/prometheus-alerts.yml` on backend changes.

backend/docs/TRANSFER_ORCHESTRATION.md

@@ -114,6 +114,21 @@ await resolveInDoubtTransfer(storeKey, { discard: true });
 `listInDoubtTransfers()` and `getInDoubtTransfer(storeKey)` expose the queue.
 Neither leaks the internal fingerprint.
 
+The admin API exposes the same controls:
+
+```http
+GET /admin/transfers/in-doubt
+POST /admin/transfers/{storeKey}/resolve
+```
+
+Both routes require an admin API key. The resolve body must be exactly one of
+`{"transactionHash":"<on-chain-hash>"}` or `{"discard":true}`. Use the hash
+only after confirming the transfer landed; discard only after checking that no
+matching transaction landed, because it releases the idempotency key for a new
+submission. Each resolution attempt is recorded as the
+`transfers.in_doubt.confirm` or `transfers.in_doubt.discard` admin audit action.
+Unknown or already-resolved keys return `404`.
+
 ## Errors
 
 Every failure is a `TransferOrchestrationError` subclass carrying `code`,

backend/monitoring/prometheus-alerts.yml

@@ -64,4 +64,93 @@ groups:
           service: yieldvault-backend
         annotations:
           summary: "YieldVault external dependency is slow"
-          description: "P95 {{ $labels.dependency }} {{ $labels.operation }} latency is above 2 seconds."
+          description: "P95 {{ $labels.dependency }} {{ $labels.operation }} latency is above 2 seconds."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#database-and-dependencies"
+
+      - alert: YieldVaultBackgroundJobUnhealthy
+        expr: job_health_status == 0
+        for: 5m
+        labels:
+          severity: critical
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault background job is unhealthy"
+          description: "Job {{ $labels.job_name }} is reporting degraded health."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#background-jobs"
+
+      - alert: YieldVaultJobDeadLettersPresent
+        expr: job_dead_letter_count > 0
+        for: 5m
+        labels:
+          severity: warning
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault background job has dead letters"
+          description: "Job {{ $labels.job_name }} has {{ $value }} dead-letter records."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#background-jobs"
+
+      - alert: YieldVaultDatabaseBackupUnhealthy
+        expr: job_health_status{job_name="databaseBackup"} == 0 or job_dead_letter_count{job_name="databaseBackup"} > 0
+        for: 5m
+        labels:
+          severity: critical
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault database backup job is unhealthy"
+          description: "The scheduled database backup job is reporting degraded health."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#database-backups"
+
+      - alert: YieldVaultEventOutboxLagHigh
+        expr: event_outbox_oldest_pending_age_seconds > 60 or event_outbox_pending_count > 100
+        for: 5m
+        labels:
+          severity: warning
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault event outbox is lagging"
+          description: "Outbox events are older than 60 seconds or more than 100 events are awaiting relay."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#webhooks-and-event-outbox"
+
+      - alert: YieldVaultEmailQueueLagHigh
+        expr: email_queue_oldest_pending_age_seconds > 300 or email_queue_pending_count > 100
+        for: 5m
+        labels:
+          severity: warning
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault email queue is lagging"
+          description: "Queued email is older than 5 minutes or more than 100 emails are awaiting delivery."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#email-queue"
+
+      - alert: YieldVaultWebhookDeliveryBacklog
+        expr: webhook_delivery_failed_count > 0 or webhook_delivery_pending_count > 100
+        for: 5m
+        labels:
+          severity: warning
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault webhook deliveries are failing or backlogged"
+          description: "Webhook deliveries have failures or more than 100 pending deliveries."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#webhooks-and-event-outbox"
+
+      - alert: YieldVaultDatabaseQueryLatencyHigh
+        expr: histogram_quantile(0.95, sum by (model, action, le) (rate(db_query_duration_seconds_bucket[5m]))) > 1
+        for: 5m
+        labels:
+          severity: warning
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault database query latency is high"
+          description: "P95 {{ $labels.model }}.{{ $labels.action }} query latency is above the 1-second SLO."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#database-and-dependencies"
+
+      - alert: YieldVaultDatabaseQueryBudgetBreaches
+        expr: sum(rate(db_query_budget_breach_total[5m])) > 0
+        for: 10m
+        labels:
+          severity: warning
+          service: yieldvault-backend
+        annotations:
+          summary: "YieldVault database query budgets are being exceeded"
+          description: "Database query budget breaches continue to occur."
+          runbook_url: "https://github.com/Junirezz/YieldVault-RWA/blob/main/backend/docs/PROMETHEUS_ALERTS_RUNBOOK.md#database-and-dependencies"

backend/src/emailQueue.ts

@@ -158,6 +158,25 @@ export class EmailQueueService {
     });
   }
 
+  async getQueueMetrics(): Promise<{ pending: number; oldestPendingAgeSeconds: number }> {
+    const where = { status: { in: ['pending', 'failed'] } };
+    const [pending, oldestPending] = await Promise.all([
+      this.queueDelegate.count({ where }),
+      this.queueDelegate.findFirst({
+        where,
+        orderBy: { createdAt: 'asc' },
+        select: { createdAt: true },
+      }),
+    ]);
+
+    return {
+      pending,
+      oldestPendingAgeSeconds: oldestPending?.createdAt
+        ? Math.max(0, (Date.now() - new Date(oldestPending.createdAt).getTime()) / 1000)
+        : 0,
+    };
+  }
+
   async replayEmail(emailId: string): Promise<EmailQueueItem> {
     await this.queueDelegate.findUniqueOrThrow({
       where: { id: emailId },

backend/src/eventOutbox.ts

@@ -127,6 +127,7 @@ export interface OutboxMetrics {
   deadLettered: number;
   locked: number;
   total: number;
+  oldestPendingAgeSeconds: number;
 }
 
 // ─── Configuration ───────────────────────────────────────────────────────────
@@ -775,7 +776,7 @@ class EventOutboxService {
 
   /** Returns a snapshot of outbox metrics for monitoring/administration. */
   async getMetrics(): Promise<OutboxMetrics> {
-    const [pending, relayed, failed, deadLettered, locked, total] = await Promise.all([
+    const [pending, relayed, failed, deadLettered, locked, total, oldestPending] = await Promise.all([
       getEventOutboxDelegate().count({ where: { status: 'pending' } }),
       getEventOutboxDelegate().count({ where: { status: 'relayed' } }),
       getEventOutboxDelegate().count({ where: { status: 'failed' } }),
@@ -787,9 +788,24 @@ class EventOutboxService {
         },
       }),
       getEventOutboxDelegate().count(),
+      getEventOutboxDelegate().findFirst({
+        where: { status: { in: ['pending', 'failed'] } },
+        orderBy: { createdAt: 'asc' },
+        select: { createdAt: true },
+      }),
     ]);
 
-    return { pending, relayed, failed, deadLettered, locked, total };
+    return {
+      pending,
+      relayed,
+      failed,
+      deadLettered,
+      locked,
+      total,
+      oldestPendingAgeSeconds: oldestPending
+        ? Math.max(0, (Date.now() - new Date(oldestPending.createdAt).getTime()) / 1000)
+        : 0,
+    };
   }
 
   /** Lists outbox entries with optional filters. */

backend/src/index.ts

@@ -64,6 +64,8 @@ import { corsMiddleware } from './middleware/cors';
 import { geofencingMiddleware } from './middleware/geofencing';
 import { registerInvalidationHook } from './middleware/cache';
 import { invalidateVaultCaches } from './vaultDataCache';
+import { getRedisCacheHealth, redisCacheClient } from './redisCache';
+import { validate, LoginSchema, NonceRequestSchema, RefreshSchema, WebhookRegisterSchema, WebhookUpdateSchema, ApyBackfillBodySchema, MaintenanceToggleSchema, MaintenanceWindowBodySchema, FeatureFlagOverrideSchema, CacheInvalidateSchema, EventReplayBodySchema, WithdrawalLimitOverrideSchema, AllowlistWalletBodySchema, ImpersonationSessionBodySchema, ApiKeyRegisterSchema, ApiKeyRotateSchema, ApiKeyRevokeSchema, WebhookVerifyBodySchema, BulkExportBodySchema, TransactionBackfillBodySchema, GovernanceSnapshotExportSchema, ReportExportBodySchema, ChecksumVerifyBodySchema, DeadLetterResolveSchema, DeadLetterIdsSchema, DeadLetterProcessSchema, ScopedTokenCreateSchema, PaginationQuerySchema, WebhookListQuerySchema, IdParamSchema, WindowIdParamSchema, CreateVaultBodySchema, TransferResolveBodySchema, TransferResolveParamsSchema } from './middleware/validate';
 import { redisCacheClient } from './redisCache';
 import { validate, LoginSchema, NonceRequestSchema, RefreshSchema } from './middleware/validate';
 import { tieredJsonBodyParser } from './middleware/payloadLimit';
@@ -97,6 +99,12 @@ import {
   rateLimitEvents,
   httpErrorCount,
   httpClientErrorCount,
+  eventOutboxPendingCount,
+  eventOutboxOldestPendingAgeSeconds,
+  emailQueuePendingCount,
+  emailQueueOldestPendingAgeSeconds,
+  webhookDeliveryPendingCount,
+  webhookDeliveryFailedCount,
 } from './metrics';
 import { latencyMonitoringService } from './latencyMonitoring';
 import { listEndpointSlaRegistry } from './endpointSlaRegistry';
@@ -116,6 +124,20 @@ import {
 import { diagnosticsBundleHandler } from './diagnosticsBundle';
 import { errorBoundaryMiddleware } from './middleware/errorBoundary';
 import { apiErrorContractMiddleware, sendApiError } from './middleware/apiError';
+import {
+  exportGovernanceSnapshots,
+  listGovernanceSnapshots,
+} from './governanceSnapshotExport';
+import {
+  getIdempotencyRetentionMetrics,
+  pruneStaleIdempotencyRecords,
+  startIdempotencyRetentionScheduler,
+} from './idempotencyRetention';
+import { scopedAdminTokenStore } from './scopedAdminTokens';
+import {
+  listInDoubtTransfers,
+  resolveInDoubtTransfer,
+} from './transferOrchestrator';
 import { startIdempotencyRetentionScheduler } from './idempotencyRetention';
 import vaultRoutes from './routes/vault';
 import transactionRoutes from './routes/transactions';
@@ -300,6 +322,23 @@ app.get('/metrics', async (_req: Request, res: Response) => {
   try {
     syncJobGovernanceMetrics();
     latencyMonitoringService.syncSloMetrics();
+    try {
+      const [outbox, emails] = await Promise.all([
+        eventOutboxService.getMetrics(),
+        emailQueueService.getQueueMetrics(),
+      ]);
+      const webhooks = getWebhookDeliveryMetrics();
+      eventOutboxPendingCount.set(outbox.pending + outbox.failed);
+      eventOutboxOldestPendingAgeSeconds.set(outbox.oldestPendingAgeSeconds);
+      emailQueuePendingCount.set(emails.pending);
+      emailQueueOldestPendingAgeSeconds.set(emails.oldestPendingAgeSeconds);
+      webhookDeliveryPendingCount.set(webhooks.pending);
+      webhookDeliveryFailedCount.set(webhooks.failed);
+    } catch (error) {
+      logger.log('warn', 'Failed to refresh queue Prometheus gauges', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
     res.set('Content-Type', register.contentType);
     res.end(await register.metrics());
   } catch (err) {
@@ -334,6 +373,47 @@ app.get('/admin/sla/registry', validateApiKey, (_req: Request, res: Response) =>
   });
 });
 
+app.get('/admin/transfers/in-doubt', validateApiKey, (_req: Request, res: Response) => {
+  res.status(200).json({ transfers: listInDoubtTransfers() });
+});
+
+app.post(
+  '/admin/transfers/:key/resolve',
+  validateApiKey,
+  validate({ params: TransferResolveParamsSchema, body: TransferResolveBodySchema }),
+  async (req: Request, res: Response) => {
+    const { key } = req.params;
+    const resolution = req.body as
+      | { transactionHash: string }
+      | { discard: true };
+    const action = 'discard' in resolution ? 'discard' : 'confirm';
+
+    try {
+      const resolved = await resolveInDoubtTransfer(key, resolution);
+      const statusCode = resolved ? 200 : 404;
+      await recordAdminAuditLog(req, `transfers.in_doubt.${action}`, statusCode, {
+        storeKey: key,
+        resolution: action,
+      });
+      if (!resolved) {
+        res.status(404).json({ error: 'In-doubt transfer not found' });
+        return;
+      }
+      res.status(200).json({ resolved: true, storeKey: key, resolution: action });
+    } catch (error) {
+      await recordAdminAuditLog(req, `transfers.in_doubt.${action}`, 500, {
+        storeKey: key,
+        resolution: action,
+      });
+      logger.log('error', 'Failed to resolve in-doubt transfer', {
+        storeKey: key,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      res.status(500).json({ error: 'Failed to resolve in-doubt transfer' });
+    }
+  },
+);
+
 /**
  * GET /admin/rate-limits
  * Rate-limit monitoring snapshot: allowed vs limited counts by tier.