From 7243a6d00709b0d487753e4d782ba06b93939fec Mon Sep 17 00:00:00 2001 From: sparkysolutions10-pixelmy Date: Tue, 20 Oct 2026 00:19:25 +0530 Subject: [PATCH] feat(admin): zero-downtime key rotation with bounded overlap window (#501) --- .env.example | 3 + src/__tests__/keyRotation.test.ts | 244 ++++++++++++++++++++++++++++++ src/api/admin.ts | 77 ++++++++++ src/openapi/coverage.ts | 3 +- src/x402/metering.ts | 38 +++++ 5 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/keyRotation.test.ts diff --git a/.env.example b/.env.example index a23c6be6..968aacd5 100644 --- a/.env.example +++ b/.env.example @@ -157,6 +157,9 @@ ADMIN_TOKEN=super-secret-admin-token # Set to "false" to disable API-key auth entirely (e.g. local dev). Any other # value (or unset) leaves auth enabled, so every non-public route needs a key. REQUIRE_API_KEY=true +# Default overlap duration in hours for key rotation via POST /admin/keys/:id/rotate (default: 24) +# During this window, both old and replacement keys function simultaneously to prevent service disruption. +KEY_ROTATION_OVERLAP_HOURS=24 # Fallback rate limit (requests per minute per IP) for unauthenticated or public requests (default: 100) RATE_LIMIT_IP_MAX=100 # Requests this single instance will accept per minute in total, across every diff --git a/src/__tests__/keyRotation.test.ts b/src/__tests__/keyRotation.test.ts new file mode 100644 index 00000000..5fa21b3a --- /dev/null +++ b/src/__tests__/keyRotation.test.ts @@ -0,0 +1,244 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import Fastify from 'fastify' + +vi.mock('../db', () => ({ + prisma: { + apiKey: { + findUnique: vi.fn(), + findMany: vi.fn(), + create: vi.fn(), + update: vi.fn(), + }, + }, +})) + +vi.mock('../redis', () => ({ + redis: { + get: vi.fn(), + multi: vi.fn().mockReturnValue({ + set: vi.fn().mockReturnThis(), + exec: vi.fn().mockResolvedValue([]), + }), + on: vi.fn(), + }, +})) + +import { prisma } from '../db' +import { redis } from '../redis' +import { registerAdminRoutes } from '../api/admin' +import { registerApiKeyAuth, hashApiKey } from '../api/auth' + +const mockFindUnique = prisma.apiKey.findUnique as any +const mockCreate = prisma.apiKey.create as any +const mockUpdate = prisma.apiKey.update as any +const mockRedisGet = redis.get as any + +const ORIGINAL_ENV = { ...process.env } + +beforeEach(() => { + vi.clearAllMocks() + process.env.ADMIN_TOKEN = 'admin-secret-token' +}) + +afterEach(() => { + process.env = { ...ORIGINAL_ENV } +}) + +describe('Issue #501: Key Rotation (POST /admin/keys/:id/rotate)', () => { + it('rejects rotation without admin authorization', async () => { + const app = Fastify() + await app.register(registerAdminRoutes) + + const res = await app.inject({ + method: 'POST', + url: '/admin/keys/old-key-id/rotate', + }) + expect(res.statusCode).toBe(401) + }) + + it('rejects rotation if key is not found or already revoked', async () => { + const app = Fastify() + await app.register(registerAdminRoutes) + + mockFindUnique.mockResolvedValueOnce(null) + const resNotFound = await app.inject({ + method: 'POST', + url: '/admin/keys/non-existent/rotate', + headers: { 'x-admin-token': 'admin-secret-token' }, + }) + expect(resNotFound.statusCode).toBe(404) + + mockFindUnique.mockResolvedValueOnce({ + id: 'revoked-key', + revokedAt: new Date(), + }) + const resRevoked = await app.inject({ + method: 'POST', + url: '/admin/keys/revoked-key/rotate', + headers: { 'x-admin-token': 'admin-secret-token' }, + }) + expect(resRevoked.statusCode).toBe(400) + expect(resRevoked.json().error).toContain('revoked') + }) + + it('successfully rotates a key with bounded overlap window, carries quotas and usage', async () => { + const oldKeyRecord = { + id: 'old-key-uuid', + label: 'Partner Billing Key', + ratePerMin: 120, + ratePerDay: 50000, + monthlyQuotaCents: 100000, + dailyQuotaCents: 5000, + overagePolicy: 'charge_402', + revokedAt: null, + expiresAt: null, + } + mockFindUnique.mockResolvedValue(oldKeyRecord) + mockUpdate.mockResolvedValue({}) + mockCreate.mockImplementation(({ data }: any) => { + return Promise.resolve({ + id: 'new-key-uuid', + ...data, + createdAt: new Date(), + }) + }) + + // Mock existing usage on old key + mockRedisGet.mockImplementation((key: string) => { + if (key.includes('old-key-uuid') && key.endsWith(':calls')) return Promise.resolve('45') + if (key.includes('old-key-uuid') && key.endsWith(':cents')) return Promise.resolve('450') + return Promise.resolve(null) + }) + + const app = Fastify() + await app.register(registerAdminRoutes) + + const res = await app.inject({ + method: 'POST', + url: '/admin/keys/old-key-uuid/rotate', + headers: { 'x-admin-token': 'admin-secret-token' }, + payload: { overlapHours: 48 }, + }) + + expect(res.statusCode).toBe(201) + const body = res.json() + expect(body.id).toBe('new-key-uuid') + expect(body.oldKeyId).toBe('old-key-uuid') + expect(body.label).toBe('Partner Billing Key') + expect(body.ratePerMin).toBe(120) + expect(body.ratePerDay).toBe(50000) + expect(body.monthlyQuotaCents).toBe(100000) + expect(body.dailyQuotaCents).toBe(5000) + expect(body.overagePolicy).toBe('charge_402') + expect(body.key).toMatch(/^lens_[a-f0-9]{48}$/) + expect(body.overlapHours).toBe(48) + expect(body.overlapEndsAt).toBeDefined() + + // Assert that the old key was updated with expiresAt matching overlap window + expect(mockUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'old-key-uuid' }, + data: expect.objectContaining({ + expiresAt: expect.any(Date), + }), + }) + ) + }) + + it('both keys work during overlap, and the old one fails after overlap window ends', async () => { + const oldPlainKey = 'lens_old_key_during_overlap_1234567890123' + const newPlainKey = 'lens_new_key_during_overlap_1234567890123' + const oldHash = hashApiKey(oldPlainKey) + const newHash = hashApiKey(newPlainKey) + + const futureExpiration = new Date(Date.now() + 24 * 3600 * 1000) + const pastExpiration = new Date(Date.now() - 1000) // 1 second in the past + + const app = Fastify() + await app.register(registerApiKeyAuth) + app.get('/protected', async (req, reply) => reply.send({ success: true, key: req.apiKey?.id })) + + // State 1: During overlap (both valid) + mockFindUnique.mockImplementation(({ where }: any) => { + if (where.hash === oldHash) { + return Promise.resolve({ + id: 'old-key-id', + label: 'Customer Key', + ratePerMin: 60, + ratePerDay: 5000, + revokedAt: null, + expiresAt: futureExpiration, // still valid + }) + } + if (where.hash === newHash) { + return Promise.resolve({ + id: 'new-key-id', + label: 'Customer Key', + ratePerMin: 60, + ratePerDay: 5000, + revokedAt: null, + expiresAt: null, + }) + } + return Promise.resolve(null) + }) + + const resOldDuringOverlap = await app.inject({ + method: 'GET', + url: '/protected', + headers: { authorization: `Bearer ${oldPlainKey}` }, + }) + expect(resOldDuringOverlap.statusCode).toBe(200) + expect(resOldDuringOverlap.json().key).toBe('old-key-id') + + const resNewDuringOverlap = await app.inject({ + method: 'GET', + url: '/protected', + headers: { authorization: `Bearer ${newPlainKey}` }, + }) + expect(resNewDuringOverlap.statusCode).toBe(200) + expect(resNewDuringOverlap.json().key).toBe('new-key-id') + + // State 2: After overlap ends (old key expired) + mockFindUnique.mockImplementation(({ where }: any) => { + if (where.hash === oldHash) { + return Promise.resolve({ + id: 'old-key-id', + label: 'Customer Key', + ratePerMin: 60, + ratePerDay: 5000, + revokedAt: null, + expiresAt: pastExpiration, // EXPIRED + }) + } + if (where.hash === newHash) { + return Promise.resolve({ + id: 'new-key-id', + label: 'Customer Key', + ratePerMin: 60, + ratePerDay: 5000, + revokedAt: null, + expiresAt: null, + }) + } + return Promise.resolve(null) + }) + + const resOldAfterOverlap = await app.inject({ + method: 'GET', + url: '/protected', + headers: { authorization: `Bearer ${oldPlainKey}` }, + }) + expect(resOldAfterOverlap.statusCode).toBe(401) + expect(resOldAfterOverlap.json().code).toBe('expired') + + // New key continues to work smoothly without outage + const resNewAfterOverlap = await app.inject({ + method: 'GET', + url: '/protected', + headers: { authorization: `Bearer ${newPlainKey}` }, + }) + expect(resNewAfterOverlap.statusCode).toBe(200) + expect(resNewAfterOverlap.json().key).toBe('new-key-id') + }) +}) diff --git a/src/api/admin.ts b/src/api/admin.ts index 6cd01f68..a35318b7 100644 --- a/src/api/admin.ts +++ b/src/api/admin.ts @@ -3,6 +3,7 @@ import { randomBytes } from 'crypto' import { prisma } from '../db' import { admin_auth_failures_total } from '../metrics' import { hashApiKey, isAdminAuthorized } from './auth' +import { copyUsageHistory } from '../x402/metering' /** Generates a new opaque API key with a recognizable `lens_` prefix. */ export function generateApiKey(): string { @@ -19,6 +20,10 @@ interface CreateKeyBody { overagePolicy?: 'block' | 'charge_402' | 'allow_overage' } +interface RotateKeyBody { + overlapHours?: number +} + /** * Registers admin-only endpoints for minting and revoking API keys. * @@ -114,4 +119,76 @@ export async function registerAdminRoutes(app: FastifyInstance) { return reply.status(200).send({ id, revoked: true }) }, ) + + app.post<{ Params: { id: string }; Body: RotateKeyBody }>( + '/admin/keys/:id/rotate', + { config: { public: true } }, + async (req: FastifyRequest<{ Params: { id: string }; Body: RotateKeyBody }>, reply: FastifyReply) => { + if (!isAdminAuthorized(req)) { + admin_auth_failures_total.inc() + return reply.status(401).send({ + error: 'Unauthorized', + code: 'admin_unauthorized', + message: 'Provide a valid X-Admin-Token header.', + }) + } + + const { id } = req.params + const oldKey = await prisma.apiKey.findUnique({ where: { id } }) + if (!oldKey) { + return reply.status(404).send({ error: 'API key not found' }) + } + if (oldKey.revokedAt) { + return reply.status(400).send({ error: 'Cannot rotate a revoked API key' }) + } + + const defaultOverlap = parseInt(process.env.KEY_ROTATION_OVERLAP_HOURS ?? '24', 10) || 24 + const overlapHours = + req.body?.overlapHours !== undefined && + Number.isFinite(req.body.overlapHours) && + req.body.overlapHours > 0 + ? req.body.overlapHours + : defaultOverlap + + const overlapEndsAt = new Date(Date.now() + overlapHours * 3600 * 1000) + + // Set bounded expiration on old key so it automatically stops working when overlap ends + await prisma.apiKey.update({ + where: { id: oldKey.id }, + data: { expiresAt: overlapEndsAt }, + }) + + // Generate replacement key inheriting quotas and metadata + const plaintext = generateApiKey() + const newKey = await prisma.apiKey.create({ + data: { + hash: hashApiKey(plaintext), + label: oldKey.label, + ratePerMin: oldKey.ratePerMin, + ratePerDay: oldKey.ratePerDay, + monthlyQuotaCents: oldKey.monthlyQuotaCents, + dailyQuotaCents: oldKey.dailyQuotaCents, + overagePolicy: oldKey.overagePolicy, + }, + }) + + // Carry across usage history from old key to replacement key + await copyUsageHistory(oldKey.id, newKey.id) + + return reply.status(201).send({ + id: newKey.id, + key: plaintext, + label: newKey.label, + oldKeyId: oldKey.id, + overlapEndsAt: overlapEndsAt.toISOString(), + overlapHours, + ratePerMin: newKey.ratePerMin, + ratePerDay: newKey.ratePerDay, + monthlyQuotaCents: newKey.monthlyQuotaCents, + dailyQuotaCents: newKey.dailyQuotaCents, + overagePolicy: newKey.overagePolicy, + createdAt: newKey.createdAt, + }) + }, + ) } diff --git a/src/openapi/coverage.ts b/src/openapi/coverage.ts index 3e09587f..e0c4b7e6 100644 --- a/src/openapi/coverage.ts +++ b/src/openapi/coverage.ts @@ -34,10 +34,11 @@ export const INTERNAL_ROUTES: readonly string[] = [ // protocol; the equivalent GraphQL subscription is documented instead. '/ws', - // Operator-only API-key issuance and revocation. Authenticated with a shared + // Operator-only API-key issuance, rotation, and revocation. Authenticated with a shared // ADMIN_TOKEN rather than a client API key (see src/api/admin.ts). '/admin/keys', '/admin/keys/{id}', + '/admin/keys/{id}/rotate', // Operator-only usage/quota reporting for minted keys. '/admin/usage', diff --git a/src/x402/metering.ts b/src/x402/metering.ts index a0a8f5d7..eb40eeeb 100644 --- a/src/x402/metering.ts +++ b/src/x402/metering.ts @@ -137,3 +137,41 @@ export function parseCents(price: string): number { if (!match) return 0 return Math.round(parseFloat(match[1]) * 100) } + +export async function copyUsageHistory(sourceKeyId: string, targetKeyId: string): Promise { + const db = redis as unknown as Redis + if (!db || typeof db.get !== 'function') return + try { + const [ + dailyCalls, + dailyCents, + monthlyCalls, + monthlyCents, + ] = await Promise.all([ + db.get(getDailyBase(sourceKeyId) + ':calls'), + db.get(getDailyBase(sourceKeyId) + ':cents'), + db.get(getMonthlyBase(sourceKeyId) + ':calls'), + db.get(getMonthlyBase(sourceKeyId) + ':cents'), + ]) + + if (typeof db.multi !== 'function') return + const multi = db.multi() + if (dailyCalls) { + multi.set(getDailyBase(targetKeyId) + ':calls', dailyCalls, 'EX', secondsUntilEndOfDay()) + } + if (dailyCents) { + multi.set(getDailyBase(targetKeyId) + ':cents', dailyCents, 'EX', secondsUntilEndOfDay()) + } + if (monthlyCalls) { + multi.set(getMonthlyBase(targetKeyId) + ':calls', monthlyCalls, 'EX', secondsUntilEndOfMonth()) + } + if (monthlyCents) { + multi.set(getMonthlyBase(targetKeyId) + ':cents', monthlyCents, 'EX', secondsUntilEndOfMonth()) + } + if (typeof multi.exec === 'function') { + await multi.exec() + } + } catch { + // Graceful fallback if Redis mock or server fails + } +}