From bb542e687fc9ccb52943f696fd803129749af55b Mon Sep 17 00:00:00 2001 From: TaprootFreakAI <315477232+TaprootFreakAI@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:25:40 +0200 Subject: [PATCH 1/2] Store validated client metadata on request logs Persist client IP, country, Cloudflare ray, user agent, accept-language, and origin when those headers validate. The same fields are attached to events logged during the request, including passkey registration and login. --- SPEC.md | 25 +++- docs/handbook/endpoints.md | 4 +- docs/handbook/functions.md | 20 ++- docs/schema/api_log.sql | 11 +- e2e/functions.spec.ts | 73 +++++++++++ src/__tests__/lib/api-log.test.ts | 76 +++++++++++- src/__tests__/lib/debug-catalog.test.ts | 12 ++ src/__tests__/lib/log.test.ts | 70 +++++++++++ src/__tests__/lib/request-meta.test.ts | 110 ++++++++++++++++ src/__tests__/routes/debug-api-log.test.ts | 12 ++ src/__tests__/routes/debug-catalog.test.ts | 19 ++- src/__tests__/routes/diagnostics.test.ts | 23 ++++ src/lib/api-log.ts | 60 ++++++++- src/lib/log.ts | 94 ++++++++------ src/lib/request-meta.ts | 138 +++++++++++++++++++++ src/routes/diagnostics.ts | 25 +--- 16 files changed, 696 insertions(+), 76 deletions(-) create mode 100644 src/__tests__/lib/request-meta.test.ts create mode 100644 src/lib/request-meta.ts diff --git a/SPEC.md b/SPEC.md index b8588fda0..e74bacf4d 100644 --- a/SPEC.md +++ b/SPEC.md @@ -2276,15 +2276,29 @@ Success → **Response** `200`: "status": 200, "ms": 8, "accountId": "", - "authKind": "session" + "authKind": "session", + "clientIp": null, + "clientCountry": null, + "cfRay": null, + "userAgent": null, + "acceptLanguage": null, + "origin": null } ] } ``` `authKind` is `session`, `debug`, `spend`, or `none`. `accountId` is the -session account when `authKind` is `session`; otherwise JSON `null`. An -empty log returns `"logs": []`. When `DATABASE_URL` is unset the default +session account when `authKind` is `session`; otherwise JSON `null`. +`clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and +`origin` are always present and are JSON `null` when that header is missing +or fails validation. `clientIp` is `CF-Connecting-IP` only when it is an +IPv4 or IPv6 address. `clientCountry` is `CF-IPCountry`, uppercased, when it +is two letters or digits. `cfRay` is `CF-Ray` when it is 16 hex digits, a +hyphen, and three letters. `userAgent` and `acceptLanguage` are those +headers with controls removed and at most 200 characters. `origin` is an +`https` origin, or `http://localhost` or `http://127.0.0.1`, with an +optional port. An empty log returns `"logs": []`. When `DATABASE_URL` is unset the default in-memory store starts empty; when set, rows come from Postgres `api_log`. Environment: @@ -2321,7 +2335,10 @@ include PRF output, the recovery phrase, a session token, a view key, nsec, Authorization, Cookie, a WebAuthn challenge, attestation, or signatures. `User-Agent` is not a body field. The server may store it as `userAgent` after stripping controls and truncating to 200 characters. An empty result -is omitted. +is omitted. The server also stores `clientIp`, `clientCountry`, `cfRay`, +`acceptLanguage`, and `origin` when those request headers validate, using +the same rules as `api_log`. Those names are not body keys. Absent values +are omitted. Invalid JSON or a field outside the allowlist → **Response** `400`: diff --git a/docs/handbook/endpoints.md b/docs/handbook/endpoints.md index fe2532db8..1a7c154ca 100644 --- a/docs/handbook/endpoints.md +++ b/docs/handbook/endpoints.md @@ -142,14 +142,14 @@ ## Endpoint: GET /debug/api-log -- **Purpose:** Operator listing of HTTP audit rows newest-first (cap 200): method, redacted path, status, ms, nullable `accountId` (always present; JSON `null` unless `authKind` is `session`), and `authKind` (`session` | `debug` | `spend` | `none`). No query string, Authorization, bodies, or tokens. OPTIONS and `/healthz` are not stored. +- **Purpose:** Operator listing of HTTP audit rows newest-first (cap 200): method, redacted path, status, ms, nullable `accountId` (always present; JSON `null` unless `authKind` is `session`), `authKind` (`session` | `debug` | `spend` | `none`), and nullable `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and `origin` (JSON `null` when the header is missing or invalid). No query string, Authorization, bodies, or tokens. OPTIONS and `/healthz` are not stored. - **Errors:** 503 `{ error: 'Debug is not configured' }` when `DEBUG_TOKEN` is unset or blank; 401 `{ error: 'Unauthorized' }` when the Bearer token does not match; 503 `{ error: 'Log is unavailable' }` if the store throws (`api_log.list.failed`). - **Used by:** Operators attributing who called the API (`gifts-debug api-log`). - **Auth:** `Authorization: Bearer` with `DEBUG_TOKEN`. Not an end-user session. ## Endpoint: POST /diagnostics -- **Purpose:** Public client diagnostic ingest with no auth. The JSON object requires `event` (`client.` plus 1–60 of `a-z`, digits, and `.`). Optional `name` (1–40 letters), `message` (1–120 of letters, digits, `.`, `_`, `:`, space, `-`; no slash), `prfPresent` (boolean, never the bytes), `challengeId` (64 lowercase hex), `accountId` (UUID), `stage` (`register` / `authenticate` / `seed` / `login` / `unhandled`), `status` (integer 100–599), and `path` (string; `/view/` becomes `/view/:viewKey`; rejected if it contains `?` or 32 lowercase hex digits (`0-9`, `a-f`) in a row). Any other key is rejected. A valid body is stored as a `client` row and answered with 204 and an empty body. `User-Agent` is not a body field; the server may store it as `userAgent` after stripping controls and truncating to 200. Rows are kept forever (no TTL, no DELETE). Secrets are not stored: no PRF bytes, mnemonic, session token, view key, nsec, Authorization, Cookie, WebAuthn challenge, attestation, signatures, or raw request bodies. +- **Purpose:** Public client diagnostic ingest with no auth. The JSON object requires `event` (`client.` plus 1–60 of `a-z`, digits, and `.`). Optional `name` (1–40 letters), `message` (1–120 of letters, digits, `.`, `_`, `:`, space, `-`; no slash), `prfPresent` (boolean, never the bytes), `challengeId` (64 lowercase hex), `accountId` (UUID), `stage` (`register` / `authenticate` / `seed` / `login` / `unhandled`), `status` (integer 100–599), and `path` (string; `/view/` becomes `/view/:viewKey`; rejected if it contains `?` or 32 lowercase hex digits (`0-9`, `a-f`) in a row). Any other key is rejected. A valid body is stored as a `client` row and answered with 204 and an empty body. `User-Agent` is not a body field; the server may store it as `userAgent` after stripping controls and truncating to 200. The server also stores `clientIp`, `clientCountry`, `cfRay`, `acceptLanguage`, and `origin` when those headers validate, using the same rules as the audit log; those names are not body keys, and absent values are omitted. Rows are kept forever (no TTL, no DELETE). Secrets are not stored: no PRF bytes, mnemonic, session token, view key, nsec, Authorization, Cookie, WebAuthn challenge, attestation, signatures, or raw request bodies. - **Errors:** 400 `{ error: 'Invalid diagnostics' }` when JSON or any field fails the allowlist; 429 `{ error: 'Too many diagnostics' }` when the IP window (60) or the global window (600) in 60 seconds is full (the accept timestamp is not recorded on 429); 500 `{ error: 'Log is unavailable' }` when the insert throws (that failure does not consume a rate-limit slot). - **Auth:** None. No session and no debug bearer. diff --git a/docs/handbook/functions.md b/docs/handbook/functions.md index 79db26528..299b41edb 100644 --- a/docs/handbook/functions.md +++ b/docs/handbook/functions.md @@ -983,7 +983,7 @@ ## Function: logEvent -- **Purpose:** One JSON line on `console.warn` (`ts` + `event` + fields). Never log secrets. +- **Purpose:** One JSON line on `console.warn` (`ts` + `event` + fields). During a request, validated client fields from `readClientRequestMeta` are merged underneath the explicit fields, and explicit fields win. Absent client fields are omitted. Outside a request, only the explicit fields are written. Never log secrets. - **Inputs:** `event` string, optional `LogFields`. - **Returns / side effects:** void. - **Used by:** Auth, me, lightning-address, requestLog. @@ -1618,6 +1618,20 @@ - **Returns / side effects:** Lowercase hex. - **Used by:** `issueSession`, passkey begin, verification nonce. +## Function: presentClientFields + +- **Purpose:** Copies the non-null fields of a validated client-request record, in `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, `origin` order. Absent headers stay omitted so a log line does not print nulls. +- **Inputs:** `ClientRequestMeta`. +- **Returns / side effects:** A string map. No I/O. +- **Used by:** `requestLog` (the request-scoped log fields) and `diagnosticsRoutes`. + +## Function: readClientRequestMeta + +- **Purpose:** Reads `cf-connecting-ip`, `cf-ipcountry`, `cf-ray`, `user-agent`, `accept-language`, and `origin`. Keeps an IPv4 address with no leading zeros, or an IPv6 address including compressed and IPv4-mapped forms, unchanged. Keeps a country code of two letters or digits after trim and uppercase, so `T1` stays. Keeps a Cloudflare ray id of 16 hex digits, a hyphen, and three letters. Strips controls from the user agent and accept-language and cuts each at 200 characters. Keeps an `https` origin, or `http://localhost` or `http://127.0.0.1`, with an optional port. Drops a port or zone id on the IP, a hostname, `x-forwarded-for`, userinfo, a path, a query, or any other `http` host. Does not read the socket. +- **Inputs:** A headers object with `get(name)`. +- **Returns / side effects:** `ClientRequestMeta`. Each field is the original text or `null`. No I/O. +- **Used by:** `requestLog` and `diagnosticsRoutes`. + ## Function: readPublicBrandFile - **Purpose:** Reads `public/` relative to a root directory. @@ -1627,7 +1641,7 @@ ## Function: requestLog -- **Purpose:** Hono middleware: `http.request` JSON after the handler, then one `api_log` row. Skips `/healthz` and OPTIONS. Never logs the query string, body, or Authorization. Path is passed through `requestLogPath` so `/view/` is redacted. `ms` is handler duration (captured once after `next`). Auth-classification failure still stores `authKind: 'none'` with `accountId` null. Store write failure logs `api_log.write.failed` and does not replace the response. +- **Purpose:** Hono middleware: `http.request` JSON after the handler, then one `api_log` row. Skips `/healthz` and OPTIONS. Never logs the query string, body, or Authorization. Path is passed through `requestLogPath` so `/view/` is redacted. `ms` is handler duration (captured once after `next`). Auth-classification failure still stores `authKind: 'none'` with `accountId` null. The row always includes `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and `origin`, each null when that header is missing or invalid. `logEvent` during the request merges the present values underneath the explicit fields, and explicit fields win. Store write failure logs `api_log.write.failed` and does not replace the response. - **Inputs:** `{ apiLogStore, authStore, debugToken, spendApiToken, now? }`. - **Returns / side effects:** `MiddlewareHandler`. - **Used by:** `createApp`. @@ -1722,7 +1736,7 @@ ## Function: diagnosticsRoutes - **Purpose:** Public `POST /` ingest mounted at `/diagnostics`. No auth. Only allowlisted scalar keys are stored on a `client` row, then the response is 204 with an empty body. The per-IP cap (60) and the global cap (600) per 60_000 ms are reserved before the insert await, so two overlapping requests cannot share one slot, and released if that insert throws. Expired per-IP buckets are dropped on the first request of a new minute, so a one-off address does not stay for the process lifetime. Rows are kept forever (no TTL, no DELETE). Secrets and raw bodies are not stored. -- **Inputs:** `{ store: DiagnosticStore, now?: () => number }`. Optional `cf-connecting-ip` is the per-IP key only when it matches a short IP token; any other value is ignored. Optional `User-Agent` has controls stripped, is truncated to 200, and is omitted when nothing remains. +- **Inputs:** `{ store: DiagnosticStore, now?: () => number }`. Optional `cf-connecting-ip` is the per-IP key only when it matches a short IP token; any other value is ignored for the cap. Validated `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and `origin` from `readClientRequestMeta` are stored when present. `User-Agent` and `Accept-Language` have controls stripped, are truncated to 200, and are omitted when nothing remains. A body key named `clientIp` is rejected. - **Returns / side effects:** 204 empty on accept; 400 `{ error: 'Invalid diagnostics' }` when JSON or a field fails the allowlist; 429 `{ error: 'Too many diagnostics' }` over the cap, without recording an accept timestamp (at most one `diagnostics.rate_limited` server row per window, and only after that append resolves); 500 `{ error: 'Log is unavailable' }` when the client-row insert throws. No PRF bytes, mnemonic, session token, view key, nsec, Authorization, Cookie, WebAuthn challenge, attestation, signatures, or raw body are stored. ## Function: debugDiagnosticsRoutes diff --git a/docs/schema/api_log.sql b/docs/schema/api_log.sql index 86e594b82..d1a8e37f4 100644 --- a/docs/schema/api_log.sql +++ b/docs/schema/api_log.sql @@ -1,7 +1,8 @@ -- HTTP request audit log. One row per request after the handler (except -- OPTIONS and `/healthz`). Path is redacted (`/view/:viewKey`); no query --- string, Authorization, or bodies. Covered by db_change attach-all-public-tables --- when migrateApiLogSchema runs before migrateDbChangeSchema. +-- string, request body, Authorization, Cookie, or tokens. Validated client +-- fields are stored by the nullable columns added below. Covered by db_change +-- attach-all-public-tables when migrateApiLogSchema runs before migrateDbChangeSchema. CREATE TABLE IF NOT EXISTS api_log ( id uuid PRIMARY KEY, @@ -14,3 +15,9 @@ CREATE TABLE IF NOT EXISTS api_log ( auth_kind text NOT NULL CHECK (auth_kind IN ('session', 'debug', 'spend', 'none')) ); CREATE INDEX IF NOT EXISTS api_log_created_at_idx ON api_log (created_at DESC, id DESC); +ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_ip text; +ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_country text; +ALTER TABLE api_log ADD COLUMN IF NOT EXISTS cf_ray text; +ALTER TABLE api_log ADD COLUMN IF NOT EXISTS user_agent text; +ALTER TABLE api_log ADD COLUMN IF NOT EXISTS accept_language text; +ALTER TABLE api_log ADD COLUMN IF NOT EXISTS origin text; diff --git a/e2e/functions.spec.ts b/e2e/functions.spec.ts index 4246f62ad..4860a3a4e 100644 --- a/e2e/functions.spec.ts +++ b/e2e/functions.spec.ts @@ -854,6 +854,79 @@ test('Function: diagnosticsRoutes — POST /diagnostics accepts an allowlisted c expect(res.status()).toBe(204); }); +test('Function: readClientRequestMeta — GET /info stores validated client headers on the audit row', async ({ + request, +}) => { + const userAgent = 'e2e-request-meta-read'; + const res = await request.get('/info', { + headers: { + 'cf-connecting-ip': '192.0.2.1', + 'cf-ipcountry': 't1', + 'cf-ray': '0123456789abcdef-ZRH', + 'user-agent': userAgent, + 'accept-language': 'de-CH,de;q=0.9', + origin: 'https://21.gifts', + }, + }); + expect(res.status()).toBe(200); + const listed = await request.get('/debug/api-log', { + headers: { authorization: 'Bearer e2e-debug-token' }, + }); + expect(listed.status()).toBe(200); + const body = (await listed.json()) as { + logs: Array<{ + path: string; + userAgent: string | null; + clientIp: string | null; + clientCountry: string | null; + cfRay: string | null; + acceptLanguage: string | null; + origin: string | null; + }>; + }; + const row = body.logs.find((log) => log.userAgent === userAgent && log.path === '/info'); + expect(row).toMatchObject({ + clientIp: '192.0.2.1', + clientCountry: 'T1', + cfRay: '0123456789abcdef-ZRH', + acceptLanguage: 'de-CH,de;q=0.9', + origin: 'https://21.gifts', + }); +}); + +test('Function: presentClientFields — POST /diagnostics stores present client headers', async ({ + request, +}) => { + const res = await request.post('/diagnostics', { + headers: { + 'cf-connecting-ip': '198.51.100.10', + 'cf-ipcountry': 'ch', + 'cf-ray': 'fedcba9876543210-zrh', + 'user-agent': 'e2e-present-fields', + 'accept-language': 'en', + origin: 'http://127.0.0.1:3000', + }, + data: { event: 'client.e2e.present.fields' }, + }); + expect(res.status()).toBe(204); + const listed = await request.get('/debug/diagnostics', { + headers: { authorization: 'Bearer e2e-debug-token' }, + }); + expect(listed.status()).toBe(200); + const body = (await listed.json()) as { + logs: Array<{ event: string; fields: Record }>; + }; + const row = body.logs.find((log) => log.event === 'client.e2e.present.fields'); + expect(row?.fields).toMatchObject({ + clientIp: '198.51.100.10', + clientCountry: 'CH', + cfRay: 'fedcba9876543210-zrh', + userAgent: 'e2e-present-fields', + acceptLanguage: 'en', + origin: 'http://127.0.0.1:3000', + }); +}); + test('Function: debugDiagnosticsRoutes — GET /debug/diagnostics without bearer is 401', async ({ request, }) => { diff --git a/src/__tests__/lib/api-log.test.ts b/src/__tests__/lib/api-log.test.ts index 6f021add7..41d4fd8c5 100644 --- a/src/__tests__/lib/api-log.test.ts +++ b/src/__tests__/lib/api-log.test.ts @@ -41,6 +41,12 @@ const EARLY: ApiLogRow = { ms: 1, accountId: null, authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }; const LATE: ApiLogRow = { @@ -52,6 +58,12 @@ const LATE: ApiLogRow = { ms: 4, accountId: 'acc', authKind: 'session', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }; const TIE_LOW: ApiLogRow = { @@ -63,6 +75,12 @@ const TIE_LOW: ApiLogRow = { ms: 1, accountId: null, authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }; const TIE_HIGH: ApiLogRow = { @@ -74,6 +92,12 @@ const TIE_HIGH: ApiLogRow = { ms: 2, accountId: 'acc', authKind: 'session', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }; describe('serializeDebugApiLog', () => { @@ -87,6 +111,12 @@ describe('serializeDebugApiLog', () => { ms: 4, accountId: 'acc', authKind: 'session', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }); }); }); @@ -119,6 +149,14 @@ describe('migrateApiLogSchema', () => { const sql = new MockSql(); await migrateApiLogSchema(sql); expect(sql.executes.map((row) => row.text)).toEqual([...API_LOG_SCHEMA_SQL]); + expect(API_LOG_SCHEMA_SQL.slice(2)).toEqual([ + 'ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_ip text', + 'ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_country text', + 'ALTER TABLE api_log ADD COLUMN IF NOT EXISTS cf_ray text', + 'ALTER TABLE api_log ADD COLUMN IF NOT EXISTS user_agent text', + 'ALTER TABLE api_log ADD COLUMN IF NOT EXISTS accept_language text', + 'ALTER TABLE api_log ADD COLUMN IF NOT EXISTS origin text', + ]); }); }); @@ -127,7 +165,10 @@ describe('PostgresApiLogStore', () => { const sql = new MockSql(); await new PostgresApiLogStore(sql).append(LATE); expect(sql.executes[0]?.text).toMatch( - /INSERT INTO api_log \(id, created_at, method, path, status, ms, account_id, auth_kind\)/, + /INSERT INTO api_log \(id, created_at, method, path, status, ms, account_id, auth_kind, client_ip, client_country, cf_ray, user_agent, accept_language, origin\)/, + ); + expect(sql.executes[0]?.text).toMatch( + /VALUES \(\$1,\$2,\$3,\$4,\$5,\$6,\$7,\$8,\$9,\$10,\$11,\$12,\$13,\$14\)/, ); expect(sql.executes[0]?.text).not.toMatch(/ON CONFLICT/i); expect(sql.executes[0]?.params).toEqual([ @@ -139,6 +180,12 @@ describe('PostgresApiLogStore', () => { 4, 'acc', 'session', + null, + null, + null, + null, + null, + null, ]); }); @@ -154,6 +201,12 @@ describe('PostgresApiLogStore', () => { ms: '2', account_id: 'acc', auth_kind: 'session', + client_ip: '192.0.2.1', + client_country: 'CH', + cf_ray: '0123456789abcdef-ZRH', + user_agent: 'Agent', + accept_language: 'de-CH', + origin: 'https://21.gifts', }, { id: 'a', @@ -164,9 +217,18 @@ describe('PostgresApiLogStore', () => { ms: 1, account_id: null, auth_kind: 'mystery', + client_ip: null, + client_country: null, + cf_ray: null, + user_agent: null, + accept_language: null, + origin: null, }, ]; const listed = await new PostgresApiLogStore(sql).listLatest(50); + expect(sql.queries[0]?.text).toMatch( + /account_id, auth_kind, client_ip, client_country, cf_ray, user_agent, accept_language, origin/, + ); expect(sql.queries[0]?.text).toMatch(/ORDER BY created_at DESC, id DESC\s+LIMIT \$1/); expect(sql.queries[0]?.params).toEqual([50]); expect(listed).toEqual([ @@ -179,6 +241,12 @@ describe('PostgresApiLogStore', () => { ms: 2, accountId: 'acc', authKind: 'session', + clientIp: '192.0.2.1', + clientCountry: 'CH', + cfRay: '0123456789abcdef-ZRH', + userAgent: 'Agent', + acceptLanguage: 'de-CH', + origin: 'https://21.gifts', }, { id: 'a', @@ -189,6 +257,12 @@ describe('PostgresApiLogStore', () => { ms: 1, accountId: null, authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }, ]); }); diff --git a/src/__tests__/lib/debug-catalog.test.ts b/src/__tests__/lib/debug-catalog.test.ts index 8a336b027..be87f4a08 100644 --- a/src/__tests__/lib/debug-catalog.test.ts +++ b/src/__tests__/lib/debug-catalog.test.ts @@ -623,6 +623,12 @@ describe('loadDebugTables', () => { ms: 1, accountId: null, authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }, ]), }, @@ -635,6 +641,12 @@ describe('loadDebugTables', () => { path: '/healthz', status: 200, authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }), ]); const stripped = new InMemoryMessageStore() as MessageStore; diff --git a/src/__tests__/lib/log.test.ts b/src/__tests__/lib/log.test.ts index 2a0261772..124a7e77d 100644 --- a/src/__tests__/lib/log.test.ts +++ b/src/__tests__/lib/log.test.ts @@ -90,6 +90,11 @@ describe('logEvent', () => { logEvent('second'); expect(sink).not.toHaveBeenCalled(); }); + + it('does not add clientIp outside a request', () => { + logEvent('probe'); + expect(parsedEvents(warn)[0]).not.toHaveProperty('clientIp'); + }); }); describe('errorLogFields', () => { @@ -186,6 +191,14 @@ describe('requestLog', () => { ); app.get('/healthz', (c) => c.text('ok')); app.get('/info', (c) => c.text('info')); + app.get('/probe', (c) => { + logEvent('probe'); + return c.text('probe'); + }); + app.get('/probe-explicit', (c) => { + logEvent('probe', { clientIp: 'kept' }); + return c.text('probe'); + }); app.options('/info', (c) => c.body(null, 204)); app.get('/view/:viewKey', (c) => c.json({ error: 'Not found' }, 404)); return app; @@ -223,6 +236,63 @@ describe('requestLog', () => { expect(rows[0]?.accountId).toBeNull(); }); + it('logs and stores all validated client metadata', async () => { + const store = new InMemoryApiLogStore(); + await appWithRequestLog(store).request('/info', { + headers: { + 'cf-connecting-ip': '192.0.2.1', + 'cf-ipcountry': 't1', + 'cf-ray': '0123456789aBCDef-ZrH', + 'user-agent': 'Test Agent', + 'accept-language': 'de-CH', + origin: 'https://21.gifts', + }, + }); + const line = parsedEvents(warn).find((event) => event['event'] === 'http.request'); + expect(line).toMatchObject({ + clientIp: '192.0.2.1', + clientCountry: 'T1', + cfRay: '0123456789aBCDef-ZrH', + userAgent: 'Test Agent', + acceptLanguage: 'de-CH', + origin: 'https://21.gifts', + }); + expect((await store.listLatest(10))[0]).toMatchObject({ + clientIp: '192.0.2.1', + clientCountry: 'T1', + cfRay: '0123456789aBCDef-ZrH', + userAgent: 'Test Agent', + acceptLanguage: 'de-CH', + origin: 'https://21.gifts', + }); + }); + + it('stores invalid clientIp as null and omits it from the log line', async () => { + const store = new InMemoryApiLogStore(); + await appWithRequestLog(store).request('/info', { + headers: { 'cf-connecting-ip': 'not-an-ip' }, + }); + const line = parsedEvents(warn).find((event) => event['event'] === 'http.request'); + expect(line).not.toHaveProperty('clientIp'); + expect((await store.listLatest(10))[0]?.clientIp).toBeNull(); + }); + + it('adds request clientIp to logEvent inside the request', async () => { + await appWithRequestLog().request('/probe', { + headers: { 'cf-connecting-ip': '192.0.2.1' }, + }); + const probe = parsedEvents(warn).find((event) => event['event'] === 'probe'); + expect(probe?.['clientIp']).toBe('192.0.2.1'); + }); + + it('lets explicit logEvent fields override request client metadata', async () => { + await appWithRequestLog().request('/probe-explicit', { + headers: { 'cf-connecting-ip': '192.0.2.1' }, + }); + const probe = parsedEvents(warn).find((event) => event['event'] === 'probe'); + expect(probe?.['clientIp']).toBe('kept'); + }); + it('emits http.request for GET /view/<64-hex> with redacted path', async () => { const key = 'a'.repeat(64); await appWithRequestLog().request('/view/' + key); diff --git a/src/__tests__/lib/request-meta.test.ts b/src/__tests__/lib/request-meta.test.ts new file mode 100644 index 000000000..f01cbd026 --- /dev/null +++ b/src/__tests__/lib/request-meta.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest'; +import { + presentClientFields, + readClientRequestMeta, + type ClientRequestMeta, +} from '@/lib/request-meta'; + +function read(headers: Record): ClientRequestMeta { + return readClientRequestMeta({ get: (name) => headers[name] }); +} + +describe('readClientRequestMeta', () => { + it('keeps valid IPv4 addresses unchanged', () => { + for (const clientIp of ['0.0.0.0', '192.0.2.1', '255.255.255.255']) { + expect(read({ 'cf-connecting-ip': clientIp }).clientIp).toBe(clientIp); + } + }); + + it('rejects invalid IPv4 addresses', () => { + for (const clientIp of [ + '1.2.3', + '01.2.3.4', + '256.1.1.1', + '192.0.2.1:443', + ' 192.0.2.1', + '192.0.2.1 ', + ]) { + expect(read({ 'cf-connecting-ip': clientIp }).clientIp).toBeNull(); + } + }); + + it('keeps valid IPv6 addresses unchanged', () => { + for (const clientIp of ['::1', '::ffff:192.0.2.1', '2001:db8::1']) { + expect(read({ 'cf-connecting-ip': clientIp }).clientIp).toBe(clientIp); + } + }); + + it('rejects invalid IPv6 addresses', () => { + for (const clientIp of [ + '', + 'fe80::1%eth0', + '::ffff:192.0.2.01', + '::1/128', + '1::2::3', + '::12345', + '1:2:3:4:5:6:7:8::', + '2001:db8:1', + ]) { + expect(read({ 'cf-connecting-ip': clientIp }).clientIp).toBeNull(); + } + }); + + it('keeps an uncompressed IPv6 address', () => { + expect(read({ 'cf-connecting-ip': '2001:db8:0:0:0:0:0:1' }).clientIp).toBe( + '2001:db8:0:0:0:0:0:1', + ); + }); + + it('does not use x-forwarded-for as the client IP', () => { + expect(read({ 'x-forwarded-for': '192.0.2.1' }).clientIp).toBeNull(); + }); + + it('normalizes and validates the client country', () => { + expect(read({ 'cf-ipcountry': 't1' }).clientCountry).toBe('T1'); + expect(read({ 'cf-ipcountry': 'CHE' }).clientCountry).toBeNull(); + expect(read({ 'cf-ipcountry': ' ch ' }).clientCountry).toBe('CH'); + }); + + it('validates cf-ray and preserves its original case', () => { + expect(read({ 'cf-ray': '0123456789aBCDef-ZrH' }).cfRay).toBe('0123456789aBCDef-ZrH'); + expect(read({ 'cf-ray': '0123456789abcdef-ZH' }).cfRay).toBeNull(); + }); + + it('strips accept-language controls the same way as user-agent', () => { + expect(read({ 'accept-language': 'de-CH,de;q=0.9' }).acceptLanguage).toBe('de-CH,de;q=0.9'); + expect(read({ 'accept-language': '\n' }).acceptLanguage).toBeNull(); + }); + + it('strips user-agent controls and caps the result at 200 characters', () => { + expect(read({ 'user-agent': 'Mozilla\nX' }).userAgent).toBe('MozillaX'); + expect(read({ 'user-agent': '\u0000\n\u007f' }).userAgent).toBeNull(); + expect(read({ 'user-agent': 'x'.repeat(201) }).userAgent).toBe('x'.repeat(200)); + }); + + it('validates allowed origins', () => { + expect(read({ origin: 'https://21.gifts/path' }).origin).toBeNull(); + expect(read({ origin: 'http://localhost:3000' }).origin).toBe('http://localhost:3000'); + expect(read({ origin: 'http://example.com' }).origin).toBeNull(); + expect(read({ origin: 'https://21.gifts' }).origin).toBe('https://21.gifts'); + }); +}); + +describe('presentClientFields', () => { + it('omits null fields and returns present strings in interface order', () => { + expect( + presentClientFields({ + clientIp: '192.0.2.1', + clientCountry: null, + cfRay: '0123456789abcdef-ZRH', + userAgent: null, + acceptLanguage: 'de-CH', + origin: null, + }), + ).toEqual({ + clientIp: '192.0.2.1', + cfRay: '0123456789abcdef-ZRH', + acceptLanguage: 'de-CH', + }); + }); +}); diff --git a/src/__tests__/routes/debug-api-log.test.ts b/src/__tests__/routes/debug-api-log.test.ts index 99d9aa98f..d8c454ca4 100644 --- a/src/__tests__/routes/debug-api-log.test.ts +++ b/src/__tests__/routes/debug-api-log.test.ts @@ -19,6 +19,12 @@ const ROW: ApiLogRow = { ms: 8, accountId: 'staff', authKind: 'session', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }; describe('debugApiLogRoutes', () => { @@ -73,6 +79,12 @@ describe('debugApiLogRoutes', () => { ms: 8, accountId: 'staff', authKind: 'session', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }, ], }); diff --git a/src/__tests__/routes/debug-catalog.test.ts b/src/__tests__/routes/debug-catalog.test.ts index 5600a14ca..f82523d97 100644 --- a/src/__tests__/routes/debug-catalog.test.ts +++ b/src/__tests__/routes/debug-catalog.test.ts @@ -190,6 +190,12 @@ describe('debugCatalogRoutes', () => { ms: 1, accountId: null, authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, }, ]), debugToken: 'secret', @@ -201,7 +207,18 @@ describe('debugCatalogRoutes', () => { expect(res.status).toBe(200); const body = (await res.json()) as { table: string; rows: Array<{ path: string }> }; expect(body.table).toBe('api_log'); - expect(body.rows).toEqual([expect.objectContaining({ path: '/healthz', authKind: 'none' })]); + expect(body.rows).toEqual([ + expect.objectContaining({ + path: '/healthz', + authKind: 'none', + clientIp: null, + clientCountry: null, + cfRay: null, + userAgent: null, + acceptLanguage: null, + origin: null, + }), + ]); }); it('returns 503 when a store throw escapes the dump', async () => { diff --git a/src/__tests__/routes/diagnostics.test.ts b/src/__tests__/routes/diagnostics.test.ts index 61b15ec06..c4631c78d 100644 --- a/src/__tests__/routes/diagnostics.test.ts +++ b/src/__tests__/routes/diagnostics.test.ts @@ -100,6 +100,7 @@ describe('diagnosticsRoutes', () => { { name: 'path not a string', body: { event: EVENT, path: 1 } }, { name: 'path with hex run', body: { event: EVENT, path: `/x/${'a'.repeat(32)}` } }, { name: 'path with query', body: { event: EVENT, path: '/login?x=1' } }, + { name: 'clientIp in the body', body: { event: EVENT, clientIp: '192.0.2.1' } }, ]; it.each(invalidBodies)('returns 400 for $name', async (tc) => { const store = new InMemoryDiagnosticStore(); @@ -154,6 +155,28 @@ describe('diagnosticsRoutes', () => { }); }); + it('stores validated client headers and omits invalid ones', async () => { + const store = new InMemoryDiagnosticStore(); + const app = mount(store, now); + const res = await post(app, { + 'cf-connecting-ip': '192.0.2.1', + 'cf-ipcountry': 't1', + 'cf-ray': '0123456789abcdef-ZRH', + 'user-agent': 'Test Agent', + 'accept-language': 'de-CH', + origin: 'https://21.gifts', + }); + expect(res.status).toBe(204); + expect((await store.listLatest(1))[0]?.fields).toEqual({ + clientIp: '192.0.2.1', + clientCountry: 'T1', + cfRay: '0123456789abcdef-ZRH', + userAgent: 'Test Agent', + acceptLanguage: 'de-CH', + origin: 'https://21.gifts', + }); + }); + it('strips controls, truncates, and omits empty user agents', async () => { async function accept(ua: string): Promise { const store = new InMemoryDiagnosticStore(); diff --git a/src/lib/api-log.ts b/src/lib/api-log.ts index d26911205..fb9c83fab 100644 --- a/src/lib/api-log.ts +++ b/src/lib/api-log.ts @@ -31,6 +31,18 @@ export interface ApiLogRow { accountId: string | null; /** Bearer class. */ authKind: ApiLogAuthKind; + /** Validated client IP address, or null. */ + clientIp: string | null; + /** Validated client country code, or null. */ + clientCountry: string | null; + /** Validated Cloudflare ray id, or null. */ + cfRay: string | null; + /** Sanitized user agent, or null. */ + userAgent: string | null; + /** Sanitized accepted languages, or null. */ + acceptLanguage: string | null; + /** Validated request origin, or null. */ + origin: string | null; } /** Operator JSON for one audit row. */ @@ -51,6 +63,18 @@ export interface DebugApiLog { accountId: string | null; /** Bearer class. */ authKind: ApiLogAuthKind; + /** Validated client IP address, or null. */ + clientIp: string | null; + /** Validated client country code, or null. */ + clientCountry: string | null; + /** Validated Cloudflare ray id, or null. */ + cfRay: string | null; + /** Sanitized user agent, or null. */ + userAgent: string | null; + /** Sanitized accepted languages, or null. */ + acceptLanguage: string | null; + /** Validated request origin, or null. */ + origin: string | null; } /** @@ -69,6 +93,12 @@ export function serializeDebugApiLog(row: ApiLogRow): DebugApiLog { ms: row.ms, accountId: row.accountId, authKind: row.authKind, + clientIp: row.clientIp, + clientCountry: row.clientCountry, + cfRay: row.cfRay, + userAgent: row.userAgent, + acceptLanguage: row.acceptLanguage, + origin: row.origin, }; } @@ -105,6 +135,12 @@ export const API_LOG_SCHEMA_SQL: readonly string[] = [ auth_kind text NOT NULL CHECK (auth_kind IN ('session', 'debug', 'spend', 'none')) )`, `CREATE INDEX IF NOT EXISTS api_log_created_at_idx ON api_log (created_at DESC, id DESC)`, + `ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_ip text`, + `ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_country text`, + `ALTER TABLE api_log ADD COLUMN IF NOT EXISTS cf_ray text`, + `ALTER TABLE api_log ADD COLUMN IF NOT EXISTS user_agent text`, + `ALTER TABLE api_log ADD COLUMN IF NOT EXISTS accept_language text`, + `ALTER TABLE api_log ADD COLUMN IF NOT EXISTS origin text`, ]; /** @@ -171,6 +207,12 @@ interface ApiLogSqlRow { ms: number | string; account_id: string | null; auth_kind: string; + client_ip: string | null; + client_country: string | null; + cf_ray: string | null; + user_agent: string | null; + accept_language: string | null; + origin: string | null; } /** @@ -193,8 +235,8 @@ export class PostgresApiLogStore implements ApiLogStore { */ async append(row: ApiLogRow): Promise { await this.#sql.execute( - `INSERT INTO api_log (id, created_at, method, path, status, ms, account_id, auth_kind) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, + `INSERT INTO api_log (id, created_at, method, path, status, ms, account_id, auth_kind, client_ip, client_country, cf_ray, user_agent, accept_language, origin) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`, [ row.id, row.createdAt, @@ -204,6 +246,12 @@ export class PostgresApiLogStore implements ApiLogStore { row.ms, row.accountId, row.authKind, + row.clientIp, + row.clientCountry, + row.cfRay, + row.userAgent, + row.acceptLanguage, + row.origin, ], ); } @@ -216,7 +264,7 @@ export class PostgresApiLogStore implements ApiLogStore { */ async listLatest(limit: number): Promise { const rows = await this.#sql.query( - `SELECT id, created_at, method, path, status, ms, account_id, auth_kind + `SELECT id, created_at, method, path, status, ms, account_id, auth_kind, client_ip, client_country, cf_ray, user_agent, accept_language, origin FROM api_log ORDER BY created_at DESC, id DESC LIMIT $1`, @@ -250,5 +298,11 @@ function mapApiLogRow(row: ApiLogSqlRow): ApiLogRow { ms: Number(row.ms), accountId: row.account_id, authKind: parseAuthKind(row.auth_kind), + clientIp: row.client_ip, + clientCountry: row.client_country, + cfRay: row.cf_ray, + userAgent: row.user_agent, + acceptLanguage: row.accept_language, + origin: row.origin, }; } diff --git a/src/lib/log.ts b/src/lib/log.ts index a2ab56ffc..099512397 100644 --- a/src/lib/log.ts +++ b/src/lib/log.ts @@ -1,7 +1,9 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; import type { MiddlewareHandler } from 'hono'; import type { ApiLogStore } from '@/lib/api-log'; import type { AuthStore } from '@/lib/auth/store'; import { resolveRequestAuth } from '@/lib/request-auth'; +import { presentClientFields, readClientRequestMeta } from '@/lib/request-meta'; /** JSON-serialisable event fields. No nested objects. */ export type LogFields = { readonly [key: string]: string | number | boolean }; @@ -11,6 +13,7 @@ export type DiagnosticSink = (event: string, fields: LogFields | undefined) => v let diagnosticSink: DiagnosticSink | null = null; let diagnosticSinkDepth = 0; +const requestClientFields = new AsyncLocalStorage>(); /** * Install or clear the diagnostic persistence hook used by {@link logEvent}. @@ -36,13 +39,15 @@ export function setDiagnosticSink(next: DiagnosticSink | null): void { * @returns void */ export function logEvent(event: string, fields?: LogFields): void { - console.warn(JSON.stringify({ ts: new Date().toISOString(), event, ...fields })); + const storedFields = requestClientFields.getStore(); + const merged = storedFields === undefined ? fields : { ...storedFields, ...fields }; + console.warn(JSON.stringify({ ts: new Date().toISOString(), event, ...merged })); if (diagnosticSink === null || diagnosticSinkDepth !== 0) { return; } diagnosticSinkDepth += 1; try { - diagnosticSink(event, fields); + diagnosticSink(event, merged); } catch { // Sink failures must not escape logEvent. } finally { @@ -122,48 +127,57 @@ export interface RequestLogDeps { */ export function requestLog(deps: RequestLogDeps): MiddlewareHandler { return async (c, next) => { - const started = Date.now(); - await next(); - if (c.req.method === 'OPTIONS' || c.req.path === '/healthz') { - return; - } - const ms = Date.now() - started; - logEvent('http.request', { - method: c.req.method, - path: requestLogPath(c.req.path), - status: c.res.status, - ms, - }); - const clock = deps.now ?? Date.now; - let accountId: string | null = null; - let authKind: 'session' | 'debug' | 'spend' | 'none' = 'none'; - try { - const auth = await resolveRequestAuth({ - authorizationHeader: c.req.header('authorization'), - debugToken: deps.debugToken, - spendApiToken: deps.spendApiToken, - authStore: deps.authStore, - now: clock(), - }); - accountId = auth.accountId; - authKind = auth.authKind; - } catch { - accountId = null; - authKind = 'none'; - } - try { - await deps.apiLogStore.append({ - id: crypto.randomUUID(), - createdAt: new Date(clock()), + const meta = readClientRequestMeta(c.req.raw.headers); + return requestClientFields.run(presentClientFields(meta), async () => { + const started = Date.now(); + await next(); + if (c.req.method === 'OPTIONS' || c.req.path === '/healthz') { + return; + } + const ms = Date.now() - started; + logEvent('http.request', { method: c.req.method, path: requestLogPath(c.req.path), status: c.res.status, ms, - accountId, - authKind, }); - } catch { - logEvent('api_log.write.failed'); - } + const clock = deps.now ?? Date.now; + let accountId: string | null = null; + let authKind: 'session' | 'debug' | 'spend' | 'none' = 'none'; + try { + const auth = await resolveRequestAuth({ + authorizationHeader: c.req.header('authorization'), + debugToken: deps.debugToken, + spendApiToken: deps.spendApiToken, + authStore: deps.authStore, + now: clock(), + }); + accountId = auth.accountId; + authKind = auth.authKind; + } catch { + accountId = null; + authKind = 'none'; + } + try { + await deps.apiLogStore.append({ + id: crypto.randomUUID(), + createdAt: new Date(clock()), + method: c.req.method, + path: requestLogPath(c.req.path), + status: c.res.status, + ms, + accountId, + authKind, + clientIp: meta.clientIp, + clientCountry: meta.clientCountry, + cfRay: meta.cfRay, + userAgent: meta.userAgent, + acceptLanguage: meta.acceptLanguage, + origin: meta.origin, + }); + } catch { + logEvent('api_log.write.failed'); + } + }); }; } diff --git a/src/lib/request-meta.ts b/src/lib/request-meta.ts new file mode 100644 index 000000000..d405cff5f --- /dev/null +++ b/src/lib/request-meta.ts @@ -0,0 +1,138 @@ +/** + * Validated client metadata read from request headers. + */ +export interface ClientRequestMeta { + clientIp: string | null; + clientCountry: string | null; + cfRay: string | null; + userAgent: string | null; + acceptLanguage: string | null; + origin: string | null; +} + +function isValidIpv4(value: string): boolean { + const octets = value.split('.'); + return ( + octets.length === 4 && + octets.every((octet) => /^(?:0|[1-9]\d{0,2})$/.test(octet) && Number(octet) <= 255) + ); +} + +function isValidIpv6(value: string): boolean { + if (!value || value.includes('%') || !/^[0-9a-fA-F:.]+$/.test(value)) { + return false; + } + + let hexAddress = value; + if (hexAddress.includes('.')) { + const finalColon = hexAddress.lastIndexOf(':'); + const ipv4 = hexAddress.slice(finalColon + 1); + if (finalColon < 0 || !isValidIpv4(ipv4)) { + return false; + } + hexAddress = `${hexAddress.slice(0, finalColon + 1)}0:0`; + } + + const sides = hexAddress.split('::'); + if (sides.length > 2) { + return false; + } + + const groups = sides.flatMap((side) => (side ? side.split(':') : [])); + if (groups.some((group) => !/^[0-9a-fA-F]{1,4}$/.test(group))) { + return false; + } + + return sides.length === 1 ? groups.length === 8 : groups.length <= 7; +} + +function readTextHeader( + headers: { get(name: string): string | null | undefined }, + name: string, +): string | null { + const value = headers.get(name); + if (value === null || value === undefined) { + return null; + } + let cleaned = ''; + for (const char of value) { + const code = char.charCodeAt(0); + if (code >= 0x20 && code !== 0x7f) { + cleaned += char; + } + } + cleaned = cleaned.slice(0, 200); + return cleaned === '' ? null : cleaned; +} + +/** + * Reads and validates client metadata from request headers. + */ +export function readClientRequestMeta(headers: { + get(name: string): string | null | undefined; +}): ClientRequestMeta { + const rawClientIp = headers.get('cf-connecting-ip'); + const clientIp = + rawClientIp !== null && + rawClientIp !== undefined && + (isValidIpv4(rawClientIp) || isValidIpv6(rawClientIp)) + ? rawClientIp + : null; + + const rawClientCountry = headers.get('cf-ipcountry'); + const normalizedClientCountry = + rawClientCountry === null || rawClientCountry === undefined + ? null + : rawClientCountry.trim().toUpperCase(); + const clientCountry = + normalizedClientCountry !== null && /^[A-Z0-9]{2}$/.test(normalizedClientCountry) + ? normalizedClientCountry + : null; + + const rawCfRay = headers.get('cf-ray'); + const cfRay = + rawCfRay !== null && rawCfRay !== undefined && /^[0-9a-f]{16}-[A-Za-z]{3}$/i.test(rawCfRay) + ? rawCfRay + : null; + + const rawOrigin = headers.get('origin'); + const origin = + rawOrigin !== null && + rawOrigin !== undefined && + (/^https:\/\/[A-Za-z0-9.-]{1,253}(?::\d{1,5})?$/.test(rawOrigin) || + /^http:\/\/(?:localhost|127\.0\.0\.1)(?::\d{1,5})?$/.test(rawOrigin)) + ? rawOrigin + : null; + + return { + clientIp, + clientCountry, + cfRay, + userAgent: readTextHeader(headers, 'user-agent'), + acceptLanguage: readTextHeader(headers, 'accept-language'), + origin, + }; +} + +/** + * Returns the non-null client metadata fields. + */ +export function presentClientFields(meta: ClientRequestMeta): { + [key: string]: string; +} { + const fields: { [key: string]: string } = {}; + for (const key of [ + 'clientIp', + 'clientCountry', + 'cfRay', + 'userAgent', + 'acceptLanguage', + 'origin', + ] as const) { + const value = meta[key]; + if (value !== null) { + fields[key] = value; + } + } + return fields; +} diff --git a/src/routes/diagnostics.ts b/src/routes/diagnostics.ts index 9f91523c8..334df5c41 100644 --- a/src/routes/diagnostics.ts +++ b/src/routes/diagnostics.ts @@ -1,6 +1,7 @@ import { Hono } from 'hono'; import type { DiagnosticStore } from '@/lib/diagnostic-log'; import { requestLogPath } from '@/lib/log'; +import { presentClientFields, readClientRequestMeta } from '@/lib/request-meta'; const EVENT_RE = /^client\.[a-z0-9.]{1,60}$/; const NAME_RE = /^[A-Za-z]{1,40}$/; @@ -58,21 +59,6 @@ function releaseReserved(timestamps: number[], reserved: number): void { } } -function sanitizeUserAgent(raw: string | undefined): string | undefined { - if (raw === undefined) { - return undefined; - } - let cleaned = ''; - for (const char of raw) { - const code = char.charCodeAt(0); - if (code >= 0x20 && code !== 0x7f) { - cleaned += char; - } - } - cleaned = cleaned.slice(0, 200); - return cleaned === '' ? undefined : cleaned; -} - function parseClientBody( body: unknown, ): { ok: true; event: string; fields: ClientFields } | { ok: false } { @@ -229,11 +215,10 @@ export function diagnosticsRoutes(deps: { store: DiagnosticStore; now?: () => nu } ipBucket.push(now); } - const fields: ClientFields = { ...parsed.fields }; - const userAgent = sanitizeUserAgent(c.req.header('user-agent')); - if (userAgent !== undefined) { - fields['userAgent'] = userAgent; - } + const fields: ClientFields = { + ...parsed.fields, + ...presentClientFields(readClientRequestMeta(c.req.raw.headers)), + }; try { await deps.store.append({ id: crypto.randomUUID(), From 8be6f5157a46ac5ef6543f125597e7752340559f Mon Sep 17 00:00:00 2001 From: TaprootFreakAI <315477232+TaprootFreakAI@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:21:44 +0200 Subject: [PATCH 2/2] Document request metadata and cover an invalid ray Add parameter docs, list the new files, and assert an invalid Cloudflare ray is omitted from the stored diagnostic row. --- CONTRIBUTING.md | 2 ++ src/__tests__/routes/diagnostics.test.ts | 3 +-- src/lib/request-meta.ts | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31ee09549..02cba2968 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,6 +105,7 @@ api/ │ │ ├── diagnostic-log.ts # Diagnostic event store (`diagnostic_event`) │ │ ├── debug-db.ts # Operator read of every public table (`GET /debug/db`) │ │ ├── request-auth.ts # Classify bearer for api_log (session/debug/spend/none) +│ │ ├── request-meta.ts # Validated client IP, country, ray, user agent, language, origin │ │ ├── conversation-store.ts # ConversationStore port, memory + Postgres │ │ ├── conversation-push.ts # notifyConversationMessage (DM Web Push; no in-app rows) │ │ ├── notification.ts # Notification public JSON + bell fan-out (`notifyForumPost` / `notifyForumReply` / `notifyZap`) filtered by `notificationLevel` (`parseNotificationLevel` / `isStaffAccount` / `wantsNotification`); staff `notifyModeratorProposed`; targeted `notifyModeratorAppointed` and `notifyExternalForumReply` (not fan-out; the latter reaches only the parent note's author) @@ -227,6 +228,7 @@ api/ │ │ ├── diagnostic-log.test.ts │ │ ├── debug-db.test.ts │ │ ├── request-auth.test.ts +│ │ ├── request-meta.test.ts │ │ ├── funding.test.ts │ │ ├── funding-store.test.ts │ │ ├── postgres-text-array.test.ts diff --git a/src/__tests__/routes/diagnostics.test.ts b/src/__tests__/routes/diagnostics.test.ts index c4631c78d..0a9caf269 100644 --- a/src/__tests__/routes/diagnostics.test.ts +++ b/src/__tests__/routes/diagnostics.test.ts @@ -161,7 +161,7 @@ describe('diagnosticsRoutes', () => { const res = await post(app, { 'cf-connecting-ip': '192.0.2.1', 'cf-ipcountry': 't1', - 'cf-ray': '0123456789abcdef-ZRH', + 'cf-ray': 'not-a-ray', 'user-agent': 'Test Agent', 'accept-language': 'de-CH', origin: 'https://21.gifts', @@ -170,7 +170,6 @@ describe('diagnosticsRoutes', () => { expect((await store.listLatest(1))[0]?.fields).toEqual({ clientIp: '192.0.2.1', clientCountry: 'T1', - cfRay: '0123456789abcdef-ZRH', userAgent: 'Test Agent', acceptLanguage: 'de-CH', origin: 'https://21.gifts', diff --git a/src/lib/request-meta.ts b/src/lib/request-meta.ts index d405cff5f..2036a1758 100644 --- a/src/lib/request-meta.ts +++ b/src/lib/request-meta.ts @@ -67,6 +67,10 @@ function readTextHeader( /** * Reads and validates client metadata from request headers. + * + * @param headers - Header lookup. Only `cf-connecting-ip`, `cf-ipcountry`, + * `cf-ray`, `user-agent`, `accept-language`, and `origin` are read. + * @returns Validated fields. Each is the original text or `null`. */ export function readClientRequestMeta(headers: { get(name: string): string | null | undefined; @@ -116,6 +120,9 @@ export function readClientRequestMeta(headers: { /** * Returns the non-null client metadata fields. + * + * @param meta - Validated client metadata. + * @returns Present string fields, in interface order. Nulls are omitted. */ export function presentClientFields(meta: ClientRequestMeta): { [key: string]: string;