Feat/multi 681 675 676 677

.env.example

@@ -23,11 +23,22 @@ DATABASE_URL="postgresql://parashield:secret@localhost:5432/parashield_db?schema
 # Redis password used by docker-compose.yml (default: redispass)
 # REDIS_PASSWORD=redispass
 
+# Distributed locking for scheduled background jobs (#675) — optional.
+# Stable lock keys are used per job (lock:job:auth-cleanup, lock:job:oracle-worker,
+# lock:job:claims-worker). Lock TTL in milliseconds. Defaults: 300000 (5m) for auth cleanup,
+# 900000 (15m) for oracle worker and claims worker.
+# AUTH_CLEANUP_LOCK_TTL_MS=300000
+# ORACLE_WORKER_LOCK_TTL_MS=900000
+# CLAIMS_WORKER_LOCK_TTL_MS=900000
+
 # ─── Authentication ────────────────────────────────────────────────────────────
 # Secret used to sign JWT tokens — REQUIRED. In production, this MUST be a cryptographically secure random string of at least 32 bytes (256 bits).
 # For development/local only, a non-production default of sufficient length (>= 32 chars) is provided below:
 JWT_SECRET="non-production-dev-jwt-secret-min-32-chars-long-do-not-use-in-production"
 
+# Challenge nonce expiration time in seconds (#681). Default: 300 (5 minutes).
+# AUTH_NONCE_TTL_SECONDS=300
+
 # ─── Webhook secret encryption (#606) ─────────────────────────────────────────
 # When set, webhook signing secrets are encrypted at rest with AES-256-GCM
 # before being stored in the database. Strongly recommended in production.

README.md

@@ -154,32 +154,103 @@ Multiple origins are comma-separated. The server logs the active origin list at
 
 ---
 
-## Error handling
+## Database Connection Pool Sizing (#676)
 
-All error responses follow the same envelope shape regardless of endpoint:
+Prisma connection pool sizing and timeouts are configurable via environment variables or directly as query parameters on `DATABASE_URL`.
+
+| Environment Variable | Default | Description | Valid Range |
+|---|---|---|---|
+| `DATABASE_CONNECTION_LIMIT` | `10` | Maximum connections in the pool per application instance | 1 – 500 |
+| `DATABASE_POOL_TIMEOUT` | `10` | Seconds a query waits for an available pool connection before throwing a timeout error | 1 – 300 |
+| `DATABASE_CONNECT_TIMEOUT` | `5` | Seconds establishing a new TCP connection to PostgreSQL may take before timing out | 1 – 120 |
+
+### Precedence
+
+Explicit query parameters in `DATABASE_URL` (e.g. `postgresql://user:pass@host:5432/db?connection_limit=25&pool_timeout=15`) always take precedence over the environment variables. Environment variables are used only when the corresponding query parameter is not present in the URL.
+
+Startup validation verifies all values are positive integers within sane bounds and fails fast with a clear error on misconfiguration.
+
+### Sizing Guidance
+
+When running multiple backend instances (e.g., in Kubernetes, ECS, or behind a load balancer), ensure the aggregate connections from all instances do not exceed PostgreSQL's `max_connections`:
+
+$$\text{Total Connections} = (\text{Number of Instances} \times \text{DATABASE\_CONNECTION\_LIMIT}) + \text{Reserved Connections}$$
+
+**Rule**: $\text{Total Connections} < \text{PostgreSQL } \texttt{max\_connections}$
+
+- **Example 1 (Default)**: 5 instances $\times$ 10 connections = 50 connections. Fits comfortably within a default PostgreSQL `max_connections = 100` setting, leaving 50 connections for database migrations, administrative tasks, and replicas.
+- **Example 2 (High Throughput)**: 10 instances $\times$ 25 connections = 250 connections. Requires PostgreSQL `max_connections` configured to $\ge 300$.
+
+### Pool Monitoring & Health Checks
+
+Pool utilization is monitored continuously via `/api/v1/health`:
+- Exposes `active`, `idle`, `waiting`, `max`, `poolTimeout`, and `connectTimeout`.
+- Flags health as degraded when connection utilization reaches `DB_POOL_EXHAUSTION_WARN_PERCENT` (default 90%) to detect exhaustion before queries queue or fail.
+- Exposes genuine Prisma runtime engine metrics when supported.
+- The effective pool configuration is logged at server startup.
+
+### Load Testing
+
+A k6 stress test script is provided in `loadtest/k6/pool-stress.js` to evaluate pool behavior and latency under concurrent load:
+
+```bash
+# Run pool stress test with k6
+npm run test:load:pool
+
+# Or with custom target URL and thresholds
+BASE_URL=http://localhost:3001/api/v1 k6 run loadtest/k6/pool-stress.js
+```
+
+---
+
+## Error handling (RFC 7807 Problem Details)
+
+All error responses use `Content-Type: application/problem+json` conforming to [RFC 7807](https://datatracker.ietf.org/doc/html/rfc7807). Legacy fields remain present in the response body for backward compatibility, but are deprecated:
 
 ```json
 {
+  "type": "https://parashield.io/errors/not-found",
+  "title": "Not Found",
+  "status": 404,
+  "detail": "Policy not found",
+  "instance": "/api/v1/policies/abc123",
   "success": false,
   "errorCode": "NOT_FOUND",
   "error": "Policy not found",
+  "message": "Policy not found",
   "statusCode": 404,
   "path": "/api/v1/policies/abc123",
   "timestamp": "2024-01-15T10:30:00.000Z"
 }
 ```
 
+### RFC 7807 Members
+
+| Member | Type | Description |
+|---|---|---|
+| `type` | `string (URI)` | Stable URI reference identifying the error type. |
+| `title` | `string` | Short human-readable summary of problem type. |
+| `status` | `number` | HTTP status code. |
+| `detail` | `string` | Human-readable explanation specific to this occurrence. |
+| `instance` | `string (URI)` | Request path that generated the error. |
+| `errors` | `array` | (Extension) Validation error details on 400 validation failures. |
+| `correlationId` | `string` | (Extension) Request correlation ID when header is provided. |
+
+### Legacy Fields (Deprecated)
+
 | Field | Type | Description |
 |---|---|---|
-| `success` | `boolean` | Always `false` for errors. |
-| `errorCode` | `string` | Stable machine-readable code (see table below). Key off this, not `error`. |
-| `error` | `string` \| `object` | Human-readable message, or NestJS validation error details for 400s. May change between versions. |
-| `statusCode` | `number` | Mirrors the HTTP status code. |
-| `path` | `string` | The request path that produced the error. |
+| `success` | `boolean` | Always `false` for errors (Deprecated: use `status`). |
+| `errorCode` | `string` | Stable machine-readable code (Deprecated: use `type`). |
+| `error` | `string` \| `object` | Error message string (Deprecated: use `detail`). |
+| `message` | `string` | Human-readable message (Deprecated: use `detail`). |
+| `statusCode` | `number` | Mirrors the HTTP status code (Deprecated: use `status`). |
+| `path` | `string` | The request path (Deprecated: use `instance`). |
 | `timestamp` | `string` | ISO-8601 UTC timestamp. |
 
 ### Error codes
 
+
 | `errorCode` | HTTP status | When it occurs |
 |---|---|---|
 | `VALIDATION_ERROR` | 400 | Request body fails class-validator rules (missing/invalid fields, wrong types). The `error` field contains an array of violation objects. |

loadtest/k6/pool-stress.js

@@ -0,0 +1,55 @@
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+
+/**
+ * k6 load test script for database connection pool sizing and stress testing (#676).
+ *
+ * Simulates concurrent traffic against database-backed endpoints to evaluate
+ * pool exhaustion behavior under different DATABASE_CONNECTION_LIMIT and
+ * DATABASE_POOL_TIMEOUT settings.
+ *
+ * Usage:
+ *   k6 run loadtest/k6/pool-stress.js
+ * Or with custom URL and VUs:
+ *   BASE_URL=http://localhost:3001/api/v1 k6 run loadtest/k6/pool-stress.js
+ */
+export const options = {
+  stages: [
+    { duration: '30s', target: 20 },  // Ramp up to 20 virtual users (2x default pool)
+    { duration: '1m', target: 50 },   // Stress test at 50 virtual users (5x default pool)
+    { duration: '30s', target: 100 }, // Spike to 100 virtual users
+    { duration: '30s', target: 0 },   // Ramp down
+  ],
+  thresholds: {
+    // Assert that less than 1% of requests fail during steady state
+    http_req_failed: ['rate<0.05'],
+    // 95% of database requests should complete within 1.5 seconds
+    http_req_duration: ['p(95)<1500'],
+  },
+};
+
+const baseUrl = __ENV.BASE_URL || 'http://localhost:3001/api/v1';
+
+export default function () {
+  // Test health endpoint which monitors pool utilization and runs DB queries
+  const healthRes = http.get(`${baseUrl}/health`);
+  check(healthRes, {
+    'health responds 200 or 503': (res) => res.status === 200 || res.status === 503,
+    'pool info is present': (res) => {
+      try {
+        const body = JSON.parse(res.body);
+        return body.checks && body.checks.database && body.checks.database.pool !== undefined;
+      } catch {
+        return false;
+      }
+    },
+  });
+
+  // Query database-backed products catalog
+  const productsRes = http.get(`${baseUrl}/products`);
+  check(productsRes, {
+    'products endpoint responds 200': (res) => res.status === 200,
+  });
+
+  sleep(0.5);
+}

package.json

@@ -9,6 +9,7 @@
     "start:dev": "nest start --watch",
     "start:prod": "node dist/main",
     "test:load": "k6 run loadtest/k6/smoke.js",
+    "test:load:pool": "k6 run loadtest/k6/pool-stress.js",
     "lint": "eslint \"{src,test}/**/*.ts\" --fix",
     "test": "jest",
     "test:watch": "jest --watch",

src/app.module.ts

@@ -40,6 +40,15 @@ function validateConfig(config: Record<string, unknown>) {
     errors.push(jwtSecretValidation.error!);
   }
 
+  // #681 — Validate AUTH_NONCE_TTL_SECONDS if configured
+  const nonceTtl = config['AUTH_NONCE_TTL_SECONDS'] as string | undefined;
+  if (nonceTtl !== undefined && nonceTtl !== '') {
+    const parsed = Number(nonceTtl);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 86400) {
+      errors.push('AUTH_NONCE_TTL_SECONDS must be a positive integer between 1 and 86400');
+    }
+  }
+
   if (!config['DATABASE_URL']) {
     errors.push('DATABASE_URL is required');
   } else {
@@ -50,6 +59,32 @@ function validateConfig(config: Record<string, unknown>) {
     }
   }
 
+  // #676 — Validate Prisma connection pool tuning environment variables
+  const connLimit = config['DATABASE_CONNECTION_LIMIT'] as string | undefined;
+  if (connLimit !== undefined && connLimit !== '') {
+    const parsed = Number(connLimit);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 500) {
+      errors.push('DATABASE_CONNECTION_LIMIT must be a positive integer between 1 and 500');
+    }
+  }
+
+  const poolTimeout = config['DATABASE_POOL_TIMEOUT'] as string | undefined;
+  if (poolTimeout !== undefined && poolTimeout !== '') {
+    const parsed = Number(poolTimeout);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 300) {
+      errors.push('DATABASE_POOL_TIMEOUT must be a positive integer between 1 and 300');
+    }
+  }
+
+  const connectTimeout = config['DATABASE_CONNECT_TIMEOUT'] as string | undefined;
+  if (connectTimeout !== undefined && connectTimeout !== '') {
+    const parsed = Number(connectTimeout);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 120) {
+      errors.push('DATABASE_CONNECT_TIMEOUT must be a positive integer between 1 and 120');
+    }
+  }
+
+
   if (!config['STELLAR_RPC_URL']) {
     errors.push('STELLAR_RPC_URL is required');
   }

src/auth/auth-cleanup.worker.ts

@@ -1,40 +1,56 @@
-import { Injectable, Inject, Logger } from '@nestjs/common';
+import { Injectable, Inject, Logger, Optional } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
+import { ConfigService } from '@nestjs/config';
 import type Redis from 'ioredis';
 import { PrismaService } from '../prisma/prisma.service';
 import { recordWorkerHeartbeat } from '../common/worker-heartbeat';
+import { DistributedLockService } from '../redis/distributed-lock.service';
 
 /**
  * AuthCleanupWorker — periodically prunes expired AuthChallenge rows.
  *
- * #489 — this is the only place expired challenges are pruned. The cleanup
- * used to also run on every GET /auth/challenge, which put a table-wide
- * deleteMany on the hot path of every login. Stale rows are harmless in the
- * meantime: login rejects an expired nonce and the next challenge for the
- * same wallet overwrites its row via upsert. This runs independently of
- * request traffic, mirroring the pattern used by ClaimsWorker.
+ * Wrapped with DistributedLockService (#675) to prevent multiple instances from
+ * competing on database table locks during challenge cleanup.
  */
 @Injectable()
 export class AuthCleanupWorker {
   private readonly logger = new Logger(AuthCleanupWorker.name);
+  private readonly lockService: DistributedLockService;
 
   constructor(
     private readonly prisma: PrismaService,
     @Inject('REDIS_CLIENT') private readonly redis: Redis,
-  ) {}
+    @Optional() lockService?: DistributedLockService,
+    @Optional() private readonly configService?: ConfigService,
+  ) {
+    this.lockService = lockService ?? new DistributedLockService(this.redis);
+  }
 
   @Cron(CronExpression.EVERY_10_MINUTES)
   async cleanupExpiredChallenges(): Promise<void> {
-    const { count } = await this.prisma.authChallenge.deleteMany({
-      where: { expiresAt: { lt: new Date() } },
-    });
+    const ttlMs = parseInt(
+      this.configService?.get<string>('AUTH_CLEANUP_LOCK_TTL_MS') ?? '300000',
+      10,
+    );
 
-    if (count > 0) {
-      this.logger.log(`Cleaned up ${count} expired auth challenge(s)`);
-    }
+    await this.lockService.runWithLock(
+      'lock:job:auth-cleanup',
+      ttlMs,
+      async () => {
+        const { count } = await this.prisma.authChallenge.deleteMany({
+          where: { expiresAt: { lt: new Date() } },
+        });
 
-    await recordWorkerHeartbeat(this.redis, 'auth-cleanup').catch((err) =>
-      this.logger.warn(`Failed to record worker heartbeat: ${err instanceof Error ? err.message : String(err)}`),
+        if (count > 0) {
+          this.logger.log(`Cleaned up ${count} expired auth challenge(s)`);
+        }
+
+        await recordWorkerHeartbeat(this.redis, 'auth-cleanup').catch((err) =>
+          this.logger.warn(`Failed to record worker heartbeat: ${err instanceof Error ? err.message : String(err)}`),
+        );
+      },
+      { enableHeartbeat: false },
     );
   }
 }
+