feat(admin): zero-downtime key rotation with bounded overlap window (#501)

.env.example

@@ -157,6 +157,9 @@ ADMIN_TOKEN=super-secret-admin-token
 # Set to "false" to disable API-key auth entirely (e.g. local dev). Any other
 # value (or unset) leaves auth enabled, so every non-public route needs a key.
 REQUIRE_API_KEY=true
+# Default overlap duration in hours for key rotation via POST /admin/keys/:id/rotate (default: 24)
+# During this window, both old and replacement keys function simultaneously to prevent service disruption.
+KEY_ROTATION_OVERLAP_HOURS=24
 # Fallback rate limit (requests per minute per IP) for unauthenticated or public requests (default: 100)
 RATE_LIMIT_IP_MAX=100
 # Requests this single instance will accept per minute in total, across every

src/__tests__/keyRotation.test.ts

@@ -0,0 +1,244 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import Fastify from 'fastify'
+
+vi.mock('../db', () => ({
+  prisma: {
+    apiKey: {
+      findUnique: vi.fn(),
+      findMany: vi.fn(),
+      create: vi.fn(),
+      update: vi.fn(),
+    },
+  },
+}))
+
+vi.mock('../redis', () => ({
+  redis: {
+    get: vi.fn(),
+    multi: vi.fn().mockReturnValue({
+      set: vi.fn().mockReturnThis(),
+      exec: vi.fn().mockResolvedValue([]),
+    }),
+    on: vi.fn(),
+  },
+}))
+
+import { prisma } from '../db'
+import { redis } from '../redis'
+import { registerAdminRoutes } from '../api/admin'
+import { registerApiKeyAuth, hashApiKey } from '../api/auth'
+
+const mockFindUnique = prisma.apiKey.findUnique as any
+const mockCreate = prisma.apiKey.create as any
+const mockUpdate = prisma.apiKey.update as any
+const mockRedisGet = redis.get as any
+
+const ORIGINAL_ENV = { ...process.env }
+
+beforeEach(() => {
+  vi.clearAllMocks()
+  process.env.ADMIN_TOKEN = 'admin-secret-token'
+})
+
+afterEach(() => {
+  process.env = { ...ORIGINAL_ENV }
+})
+
+describe('Issue #501: Key Rotation (POST /admin/keys/:id/rotate)', () => {
+  it('rejects rotation without admin authorization', async () => {
+    const app = Fastify()
+    await app.register(registerAdminRoutes)
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/admin/keys/old-key-id/rotate',
+    })
+    expect(res.statusCode).toBe(401)
+  })
+
+  it('rejects rotation if key is not found or already revoked', async () => {
+    const app = Fastify()
+    await app.register(registerAdminRoutes)
+
+    mockFindUnique.mockResolvedValueOnce(null)
+    const resNotFound = await app.inject({
+      method: 'POST',
+      url: '/admin/keys/non-existent/rotate',
+      headers: { 'x-admin-token': 'admin-secret-token' },
+    })
+    expect(resNotFound.statusCode).toBe(404)
+
+    mockFindUnique.mockResolvedValueOnce({
+      id: 'revoked-key',
+      revokedAt: new Date(),
+    })
+    const resRevoked = await app.inject({
+      method: 'POST',
+      url: '/admin/keys/revoked-key/rotate',
+      headers: { 'x-admin-token': 'admin-secret-token' },
+    })
+    expect(resRevoked.statusCode).toBe(400)
+    expect(resRevoked.json().error).toContain('revoked')
+  })
+
+  it('successfully rotates a key with bounded overlap window, carries quotas and usage', async () => {
+    const oldKeyRecord = {
+      id: 'old-key-uuid',
+      label: 'Partner Billing Key',
+      ratePerMin: 120,
+      ratePerDay: 50000,
+      monthlyQuotaCents: 100000,
+      dailyQuotaCents: 5000,
+      overagePolicy: 'charge_402',
+      revokedAt: null,
+      expiresAt: null,
+    }
+    mockFindUnique.mockResolvedValue(oldKeyRecord)
+    mockUpdate.mockResolvedValue({})
+    mockCreate.mockImplementation(({ data }: any) => {
+      return Promise.resolve({
+        id: 'new-key-uuid',
+        ...data,
+        createdAt: new Date(),
+      })
+    })
+
+    // Mock existing usage on old key
+    mockRedisGet.mockImplementation((key: string) => {
+      if (key.includes('old-key-uuid') && key.endsWith(':calls')) return Promise.resolve('45')
+      if (key.includes('old-key-uuid') && key.endsWith(':cents')) return Promise.resolve('450')
+      return Promise.resolve(null)
+    })
+
+    const app = Fastify()
+    await app.register(registerAdminRoutes)
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/admin/keys/old-key-uuid/rotate',
+      headers: { 'x-admin-token': 'admin-secret-token' },
+      payload: { overlapHours: 48 },
+    })
+
+    expect(res.statusCode).toBe(201)
+    const body = res.json()
+    expect(body.id).toBe('new-key-uuid')
+    expect(body.oldKeyId).toBe('old-key-uuid')
+    expect(body.label).toBe('Partner Billing Key')
+    expect(body.ratePerMin).toBe(120)
+    expect(body.ratePerDay).toBe(50000)
+    expect(body.monthlyQuotaCents).toBe(100000)
+    expect(body.dailyQuotaCents).toBe(5000)
+    expect(body.overagePolicy).toBe('charge_402')
+    expect(body.key).toMatch(/^lens_[a-f0-9]{48}$/)
+    expect(body.overlapHours).toBe(48)
+    expect(body.overlapEndsAt).toBeDefined()
+
+    // Assert that the old key was updated with expiresAt matching overlap window
+    expect(mockUpdate).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: { id: 'old-key-uuid' },
+        data: expect.objectContaining({
+          expiresAt: expect.any(Date),
+        }),
+      })
+    )
+  })
+
+  it('both keys work during overlap, and the old one fails after overlap window ends', async () => {
+    const oldPlainKey = 'lens_old_key_during_overlap_1234567890123'
+    const newPlainKey = 'lens_new_key_during_overlap_1234567890123'
+    const oldHash = hashApiKey(oldPlainKey)
+    const newHash = hashApiKey(newPlainKey)
+
+    const futureExpiration = new Date(Date.now() + 24 * 3600 * 1000)
+    const pastExpiration = new Date(Date.now() - 1000) // 1 second in the past
+
+    const app = Fastify()
+    await app.register(registerApiKeyAuth)
+    app.get('/protected', async (req, reply) => reply.send({ success: true, key: req.apiKey?.id }))
+
+    // State 1: During overlap (both valid)
+    mockFindUnique.mockImplementation(({ where }: any) => {
+      if (where.hash === oldHash) {
+        return Promise.resolve({
+          id: 'old-key-id',
+          label: 'Customer Key',
+          ratePerMin: 60,
+          ratePerDay: 5000,
+          revokedAt: null,
+          expiresAt: futureExpiration, // still valid
+        })
+      }
+      if (where.hash === newHash) {
+        return Promise.resolve({
+          id: 'new-key-id',
+          label: 'Customer Key',
+          ratePerMin: 60,
+          ratePerDay: 5000,
+          revokedAt: null,
+          expiresAt: null,
+        })
+      }
+      return Promise.resolve(null)
+    })
+
+    const resOldDuringOverlap = await app.inject({
+      method: 'GET',
+      url: '/protected',
+      headers: { authorization: `Bearer ${oldPlainKey}` },
+    })
+    expect(resOldDuringOverlap.statusCode).toBe(200)
+    expect(resOldDuringOverlap.json().key).toBe('old-key-id')
+
+    const resNewDuringOverlap = await app.inject({
+      method: 'GET',
+      url: '/protected',
+      headers: { authorization: `Bearer ${newPlainKey}` },
+    })
+    expect(resNewDuringOverlap.statusCode).toBe(200)
+    expect(resNewDuringOverlap.json().key).toBe('new-key-id')
+
+    // State 2: After overlap ends (old key expired)
+    mockFindUnique.mockImplementation(({ where }: any) => {
+      if (where.hash === oldHash) {
+        return Promise.resolve({
+          id: 'old-key-id',
+          label: 'Customer Key',
+          ratePerMin: 60,
+          ratePerDay: 5000,
+          revokedAt: null,
+          expiresAt: pastExpiration, // EXPIRED
+        })
+      }
+      if (where.hash === newHash) {
+        return Promise.resolve({
+          id: 'new-key-id',
+          label: 'Customer Key',
+          ratePerMin: 60,
+          ratePerDay: 5000,
+          revokedAt: null,
+          expiresAt: null,
+        })
+      }
+      return Promise.resolve(null)
+    })
+
+    const resOldAfterOverlap = await app.inject({
+      method: 'GET',
+      url: '/protected',
+      headers: { authorization: `Bearer ${oldPlainKey}` },
+    })
+    expect(resOldAfterOverlap.statusCode).toBe(401)
+    expect(resOldAfterOverlap.json().code).toBe('expired')
+
+    // New key continues to work smoothly without outage
+    const resNewAfterOverlap = await app.inject({
+      method: 'GET',
+      url: '/protected',
+      headers: { authorization: `Bearer ${newPlainKey}` },
+    })
+    expect(resNewAfterOverlap.statusCode).toBe(200)
+    expect(resNewAfterOverlap.json().key).toBe('new-key-id')
+  })
+})

src/api/admin.ts

@@ -3,6 +3,7 @@ import { randomBytes } from 'crypto'
 import { prisma } from '../db'
 import { admin_auth_failures_total } from '../metrics'
 import { hashApiKey, isAdminAuthorized } from './auth'
+import { copyUsageHistory } from '../x402/metering'
 
 /** Generates a new opaque API key with a recognizable `lens_` prefix. */
 export function generateApiKey(): string {
@@ -19,6 +20,10 @@ interface CreateKeyBody {
   overagePolicy?: 'block' | 'charge_402' | 'allow_overage'
 }
 
+interface RotateKeyBody {
+  overlapHours?: number
+}
+
 /**
  * Registers admin-only endpoints for minting and revoking API keys.
  *
@@ -114,4 +119,76 @@ export async function registerAdminRoutes(app: FastifyInstance) {
       return reply.status(200).send({ id, revoked: true })
     },
   )
+
+  app.post<{ Params: { id: string }; Body: RotateKeyBody }>(
+    '/admin/keys/:id/rotate',
+    { config: { public: true } },
+    async (req: FastifyRequest<{ Params: { id: string }; Body: RotateKeyBody }>, reply: FastifyReply) => {
+      if (!isAdminAuthorized(req)) {
+        admin_auth_failures_total.inc()
+        return reply.status(401).send({
+          error: 'Unauthorized',
+          code: 'admin_unauthorized',
+          message: 'Provide a valid X-Admin-Token header.',
+        })
+      }
+
+      const { id } = req.params
+      const oldKey = await prisma.apiKey.findUnique({ where: { id } })
+      if (!oldKey) {
+        return reply.status(404).send({ error: 'API key not found' })
+      }
+      if (oldKey.revokedAt) {
+        return reply.status(400).send({ error: 'Cannot rotate a revoked API key' })
+      }
+
+      const defaultOverlap = parseInt(process.env.KEY_ROTATION_OVERLAP_HOURS ?? '24', 10) || 24
+      const overlapHours =
+        req.body?.overlapHours !== undefined &&
+        Number.isFinite(req.body.overlapHours) &&
+        req.body.overlapHours > 0
+          ? req.body.overlapHours
+          : defaultOverlap
+
+      const overlapEndsAt = new Date(Date.now() + overlapHours * 3600 * 1000)
+
+      // Set bounded expiration on old key so it automatically stops working when overlap ends
+      await prisma.apiKey.update({
+        where: { id: oldKey.id },
+        data: { expiresAt: overlapEndsAt },
+      })
+
+      // Generate replacement key inheriting quotas and metadata
+      const plaintext = generateApiKey()
+      const newKey = await prisma.apiKey.create({
+        data: {
+          hash: hashApiKey(plaintext),
+          label: oldKey.label,
+          ratePerMin: oldKey.ratePerMin,
+          ratePerDay: oldKey.ratePerDay,
+          monthlyQuotaCents: oldKey.monthlyQuotaCents,
+          dailyQuotaCents: oldKey.dailyQuotaCents,
+          overagePolicy: oldKey.overagePolicy,
+        },
+      })
+
+      // Carry across usage history from old key to replacement key
+      await copyUsageHistory(oldKey.id, newKey.id)
+
+      return reply.status(201).send({
+        id: newKey.id,
+        key: plaintext,
+        label: newKey.label,
+        oldKeyId: oldKey.id,
+        overlapEndsAt: overlapEndsAt.toISOString(),
+        overlapHours,
+        ratePerMin: newKey.ratePerMin,
+        ratePerDay: newKey.ratePerDay,
+        monthlyQuotaCents: newKey.monthlyQuotaCents,
+        dailyQuotaCents: newKey.dailyQuotaCents,
+        overagePolicy: newKey.overagePolicy,
+        createdAt: newKey.createdAt,
+      })
+    },
+  )
 }

src/openapi/coverage.ts

@@ -34,10 +34,11 @@ export const INTERNAL_ROUTES: readonly string[] = [
   // protocol; the equivalent GraphQL subscription is documented instead.
   '/ws',
 
-  // Operator-only API-key issuance and revocation. Authenticated with a shared
+  // Operator-only API-key issuance, rotation, and revocation. Authenticated with a shared
   // ADMIN_TOKEN rather than a client API key (see src/api/admin.ts).
   '/admin/keys',
   '/admin/keys/{id}',
+  '/admin/keys/{id}/rotate',
 
   // Operator-only usage/quota reporting for minted keys.
   '/admin/usage',