Fix/serialization lb body logging key rotation

.env.example

@@ -95,6 +95,8 @@ WATCHED_ASSETS=XLM,USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN
 
 # API key auth
 ADMIN_API_KEY=
+# API_KEY_ROTATION_GRACE_SECONDS: how long the previous key remains valid after rotation.
+API_KEY_ROTATION_GRACE_SECONDS=0
 
 # LOG_LEVEL: string enum (debug, info, warn, error). Default: info.
 LOG_LEVEL=info

README.md

@@ -249,6 +249,7 @@ The application reads configurations from the `.env` file at the root.
 | `PRICE_STALE_THRESHOLD` | Stale threshold in minutes | 5 | No |
 | `PRICE_ANOMALY_THRESHOLD` | Anomaly detection threshold % | 10 | No |
 | `ADMIN_API_KEY` | Bootstrap admin bearer token for API key management | undefined | Yes, for protected endpoints |
+| `API_KEY_ROTATION_GRACE_SECONDS` | How long the previous key stays valid after rotation | 0 | No |
 | `LOG_LEVEL` | Logging level | info | No |
 | `API_DOCS_ENABLED` | Serve the OpenAPI specification and Swagger UI at `/api-docs` | true | No |
 | `STATIC_ASSETS_DIR` | Directory served at `/static` when no CDN is configured | public | No |
@@ -286,6 +287,7 @@ The application reads configurations from the `.env` file at the root.
 
 Each price source (`stellar_dex`, `coingecko`, `coinmarketcap`) gets one circuit breaker shared across every watched asset. `CIRCUIT_BREAKER_FAILURE_THRESHOLD` only counts genuine failures for an asset a source is supposed to support (network errors, unexpected empty responses) — a source being asked about an asset it doesn't support at all (e.g. CoinGecko has no mapping for anything but XLM) is filtered out before it ever reaches the breaker, so it can't trip that source offline for the other assets it does support.
 | `ADMIN_API_KEY` | Bootstrap admin bearer token for API key management | empty | Yes, for protected endpoints |
+| `API_KEY_ROTATION_GRACE_SECONDS` | How long the previous key stays valid after rotation | 0 | No |
 | `AIRDROP_CSV_MAX_BYTES` | Maximum recipient CSV upload size in bytes | 5242880 (5 MiB) | No |
 | `AIRDROP_JSON_MAX_BYTES` | Maximum JSON request body size; 2 MiB accommodates 10,000 inline recipients | 2097152 (2 MiB) | No |
 | `AIRDROP_RATELIMIT_WINDOW` | Per-IP airdrop mutation rate-limit window in seconds | 60 | No |

deploy/docker-compose.prod.yml

@@ -0,0 +1,59 @@
+services:
+  api:
+    build:
+      context: ..
+      target: production
+    environment:
+      - PORT=4000
+      - NODE_ENV=production
+      - REDIS_URL=redis://redis:6379
+      - DATABASE_URL=postgres://smartdrop:smartdrop@postgres:5432/smartdrop
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    deploy:
+      replicas: 3
+      restart_policy:
+        condition: on-failure
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:4000/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  loadbalancer:
+    image: nginx:1.27-alpine
+    ports:
+      - "80:80"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - api
+
+  redis:
+    image: redis:7-alpine
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: smartdrop
+      POSTGRES_PASSWORD: smartdrop
+      POSTGRES_DB: smartdrop
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "smartdrop"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+volumes:
+  postgres_data:

deploy/nginx.conf

@@ -0,0 +1,33 @@
+worker_processes auto;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    upstream smartdrop_api {
+        least_conn;
+        server api:4000;
+    }
+
+    server {
+        listen 80;
+
+        location / {
+            proxy_pass http://smartdrop_api;
+            proxy_http_version 1.1;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_read_timeout 60s;
+        }
+
+        location /health {
+            proxy_pass http://smartdrop_api;
+            access_log off;
+        }
+    }
+}

src/config.js

@@ -92,6 +92,7 @@ const env = cleanEnv(rawEnv, {
   LEASE_RENEW_INTERVAL_MS: positiveInteger({ default: 5000 }),
   ADMIN_API_KEY: str({ default: '' }),
   BCRYPT_ROUNDS: positiveInteger({ default: 12 }),
+  API_KEY_ROTATION_GRACE_SECONDS: num({ default: 0 }),
   // 32-byte AES-256 key for encrypting webhook secrets at rest (issue #145),
   // hex or base64 encoded. Empty in dev — webhookEncryption falls back to an
   // insecure fixed key and logs a warning; production must set this.
@@ -252,6 +253,7 @@ const config = module.exports = {
   auth: {
     adminApiKey: env.ADMIN_API_KEY,
     bcryptRounds: env.BCRYPT_ROUNDS,
+    apiKeyRotationGraceSeconds: env.API_KEY_ROTATION_GRACE_SECONDS,
   },
   apiKeyRateLimit: {
     windowSeconds: env.API_KEY_RATELIMIT_WINDOW_SECONDS,
@@ -437,6 +439,7 @@ function reload() {
     auth: {
       adminApiKey: reloaded.ADMIN_API_KEY,
       bcryptRounds: reloaded.BCRYPT_ROUNDS,
+      apiKeyRotationGraceSeconds: reloaded.API_KEY_ROTATION_GRACE_SECONDS,
     },
     // Issue #373: reload() already re-validated COINMARKETCAP_API_KEY above
     // (it's in the env schema passed to validateEnv) but never applied it —

src/index.js

@@ -43,6 +43,7 @@ const {
   auditApiKeyUsage,
 } = require("./middleware/auth");
 const { errorHandler, notFoundHandler } = require("./middleware/errorHandler");
+const { jsonReplacer } = require("./utils/serialization");
 const { checkDatabase } = require("./services/dbHealth");
 const pricesRouter = require("./routes/prices");
 const alertsRouter = require("./routes/alerts");

src/middleware/requestLogger.js

@@ -22,6 +22,27 @@
 
 const logger = require("../logger");
 const config = require("../config");
+const { redactInfo } = require("../services/logRedaction");
+
+const MAX_LOGGED_BODY_CHARS = 4096;
+
+function requestBodyForLog(req) {
+  if (!req.body || typeof req.body !== "object") return undefined;
+  if (String(req.headers["content-type"] || "").startsWith("multipart/")) {
+    return undefined;
+  }
+
+  try {
+    const sanitized = redactInfo({ requestPayload: req.body }).requestPayload;
+    const serialized = JSON.stringify(sanitized);
+    if (serialized.length > MAX_LOGGED_BODY_CHARS) {
+      return { truncated: true, preview: serialized.slice(0, MAX_LOGGED_BODY_CHARS) };
+    }
+    return sanitized;
+  } catch {
+    return undefined;
+  }
+}
 
 function requestLoggerMiddleware(req, res, next) {
   const startedAt = process.hrtime.bigint();
@@ -34,6 +55,7 @@ function requestLoggerMiddleware(req, res, next) {
       res.statusCode >= 500 ? "error" : res.statusCode >= 400 ? "warn" : "info";
 
     const correlationId = req.id || req.correlationId;
+    const requestPayload = requestBodyForLog(req);
 
     logger[level]("HTTP request", {
       method: req.method,
@@ -46,6 +68,7 @@ function requestLoggerMiddleware(req, res, next) {
       correlation_id: correlationId,
       requestId: correlationId,
       request_id: correlationId,
+      ...(requestPayload ? { request_payload: requestPayload } : {}),
     });
 
     // Distinct, greppable/alertable warning for slow requests (issue #244),

src/routes/keys.js

@@ -60,9 +60,12 @@ router.post(
   validate(keyRotateBodySchema),
   async (req, res, next) => {
     try {
-      const { tier } = req.validated.body;
+      const { tier, grace_period_seconds: gracePeriodSeconds } = req.validated.body;
 
-      const rotated = await apiKeys.rotateKey(req.params.id, { tier });
+      const rotated = await apiKeys.rotateKey(req.params.id, {
+        tier,
+        gracePeriodSeconds,
+      });
       if (!rotated) {
         return next(
           new AppError("API_KEY_NOT_FOUND", "API key not found", 404),

src/services/apiKeys.js

@@ -47,7 +47,10 @@ async function listKeys() {
   const ids = await redis.zrevrange(IDS_KEY, 0, -1);
   if (ids.length === 0) return [];
   const records = await cache.mget(ids.map(keyPath));
-  return records.filter(Boolean).map(sanitize);
+  const now = Date.now();
+  return records
+    .filter((record) => record && (!record.expires_at || Date.parse(record.expires_at) > now))
+    .map(sanitize);
 }
 
 function normalizeTier(tier) {
@@ -128,28 +131,47 @@ async function rotateKey(id, options = {}) {
     scopes: oldRecord.scopes,
     tier: options.tier ? normalizeTier(options.tier) : oldRecord.tier,
     created_at: now,
+    rotated_at: now,
+    rotated_from: oldRecord.id,
+    rotation_count: (oldRecord.rotation_count || 0) + 1,
     last_used_at: null,
   };
 
+  const configuredGrace = Number.isFinite(options.gracePeriodSeconds)
+    ? options.gracePeriodSeconds
+    : config.auth.apiKeyRotationGraceSeconds;
+  const graceSeconds = Math.max(0, configuredGrace);
   const redis = cache.getClient();
   const multi = redis.multi();
 
-  // Create new key
   multi.set(keyPath(newRecord.id), JSON.stringify(newRecord));
   multi.set(hashPath(hashed), newRecord.id);
   multi.zadd(IDS_KEY, Date.now(), newRecord.id);
 
-  // Revoke old key
-  multi.del(keyPath(id));
-  multi.del(hashPath(oldRecord.key_hash));
-  multi.zrem(IDS_KEY, id);
+  let previousKeyExpiresAt = null;
+  if (graceSeconds > 0) {
+    previousKeyExpiresAt = new Date(Date.now() + graceSeconds * 1000).toISOString();
+    const retiredRecord = {
+      ...oldRecord,
+      revoked_at: now,
+      expires_at: previousKeyExpiresAt,
+      rotated_to: newRecord.id,
+    };
+    multi.set(keyPath(id), JSON.stringify(retiredRecord));
+    multi.zadd(IDS_KEY, Date.now(), id);
+  } else {
+    multi.del(keyPath(id));
+    multi.del(hashPath(oldRecord.key_hash));
+    multi.zrem(IDS_KEY, id);
+  }
 
   await multi.exec();
 
   return {
     api_key: newApiKey,
     key: sanitize(newRecord),
     rotated_from: sanitize(oldRecord),
+    previous_key_expires_at: previousKeyExpiresAt,
   };
 }
 
@@ -178,6 +200,16 @@ async function validateApiKey(apiKey) {
   const record = await getKey(id);
   if (!record || record.key_hash !== hashed) return null;
 
+  if (record.expires_at && Date.parse(record.expires_at) <= Date.now()) {
+    const redis = cache.getClient();
+    const multi = redis.multi();
+    multi.del(keyPath(record.id));
+    multi.del(hashPath(record.key_hash));
+    multi.zrem(IDS_KEY, record.id);
+    await multi.exec();
+    return null;
+  }
+
   // Keys created before tiers existed have no `tier`; resolve them to the
   // default tier rather than leaving the rate limiter to guess.
   if (!record.tier) {

src/utils/serialization.js

@@ -0,0 +1,13 @@
+'use strict';
+
+function jsonReplacer(_key, value) {
+  if (typeof value === 'bigint') return value.toString();
+  if (value instanceof Date) return value.toISOString();
+  return value;
+}
+
+function serialize(value) {
+  return JSON.stringify(value, jsonReplacer);
+}
+
+module.exports = { jsonReplacer, serialize };

src/validation/schemas.js

@@ -125,6 +125,7 @@ const keyCreateBodySchema = z.object({
 const keyRotateBodySchema = z.object({
   // Preserves label and scopes from the old key, but allows overriding tier
   tier: z.enum(Object.keys(config.apiKeyRateLimit.tiers)).optional(),
+  grace_period_seconds: z.number().int().min(0).max(86400).optional(),
 });
 
 const alertCreateBodySchema = z.object({