01a0ee50 - Store validated client metadata on request logs

CONTRIBUTING.md

@@ -105,6 +105,7 @@ api/
 │   │   ├── diagnostic-log.ts # Diagnostic event store (`diagnostic_event`)
 │   │   ├── debug-db.ts       # Operator read of every public table (`GET /debug/db`)
 │   │   ├── request-auth.ts   # Classify bearer for api_log (session/debug/spend/none)
+│   │   ├── request-meta.ts   # Validated client IP, country, ray, user agent, language, origin
 │   │   ├── conversation-store.ts  # ConversationStore port, memory + Postgres
 │   │   ├── conversation-push.ts  # notifyConversationMessage (DM Web Push; no in-app rows)
 │   │   ├── notification.ts   # Notification public JSON + bell fan-out (`notifyForumPost` / `notifyForumReply` / `notifyZap`) filtered by `notificationLevel` (`parseNotificationLevel` / `isStaffAccount` / `wantsNotification`); staff `notifyModeratorProposed`; targeted `notifyModeratorAppointed` and `notifyExternalForumReply` (not fan-out; the latter reaches only the parent note's author)
@@ -227,6 +228,7 @@ api/
 │       │   ├── diagnostic-log.test.ts
 │       │   ├── debug-db.test.ts
 │       │   ├── request-auth.test.ts
+│       │   ├── request-meta.test.ts
 │       │   ├── funding.test.ts
 │       │   ├── funding-store.test.ts
 │       │   ├── postgres-text-array.test.ts

SPEC.md

@@ -2276,15 +2276,29 @@ Success → **Response** `200`:
       "status": 200,
       "ms": 8,
       "accountId": "<uuid>",
-      "authKind": "session"
+      "authKind": "session",
+      "clientIp": null,
+      "clientCountry": null,
+      "cfRay": null,
+      "userAgent": null,
+      "acceptLanguage": null,
+      "origin": null
     }
   ]
 }
 ```
 
 `authKind` is `session`, `debug`, `spend`, or `none`. `accountId` is the
-session account when `authKind` is `session`; otherwise JSON `null`. An
-empty log returns `"logs": []`. When `DATABASE_URL` is unset the default
+session account when `authKind` is `session`; otherwise JSON `null`.
+`clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and
+`origin` are always present and are JSON `null` when that header is missing
+or fails validation. `clientIp` is `CF-Connecting-IP` only when it is an
+IPv4 or IPv6 address. `clientCountry` is `CF-IPCountry`, uppercased, when it
+is two letters or digits. `cfRay` is `CF-Ray` when it is 16 hex digits, a
+hyphen, and three letters. `userAgent` and `acceptLanguage` are those
+headers with controls removed and at most 200 characters. `origin` is an
+`https` origin, or `http://localhost` or `http://127.0.0.1`, with an
+optional port. An empty log returns `"logs": []`. When `DATABASE_URL` is unset the default
 in-memory store starts empty; when set, rows come from Postgres `api_log`.
 
 Environment:
@@ -2321,7 +2335,10 @@ include PRF output, the recovery phrase, a session token, a view key, nsec,
 Authorization, Cookie, a WebAuthn challenge, attestation, or signatures.
 `User-Agent` is not a body field. The server may store it as `userAgent`
 after stripping controls and truncating to 200 characters. An empty result
-is omitted.
+is omitted. The server also stores `clientIp`, `clientCountry`, `cfRay`,
+`acceptLanguage`, and `origin` when those request headers validate, using
+the same rules as `api_log`. Those names are not body keys. Absent values
+are omitted.
 
 Invalid JSON or a field outside the allowlist → **Response** `400`:
 

docs/handbook/endpoints.md

@@ -142,14 +142,14 @@
 
 ## Endpoint: GET /debug/api-log
 
-- **Purpose:** Operator listing of HTTP audit rows newest-first (cap 200): method, redacted path, status, ms, nullable `accountId` (always present; JSON `null` unless `authKind` is `session`), and `authKind` (`session` | `debug` | `spend` | `none`). No query string, Authorization, bodies, or tokens. OPTIONS and `/healthz` are not stored.
+- **Purpose:** Operator listing of HTTP audit rows newest-first (cap 200): method, redacted path, status, ms, nullable `accountId` (always present; JSON `null` unless `authKind` is `session`), `authKind` (`session` | `debug` | `spend` | `none`), and nullable `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and `origin` (JSON `null` when the header is missing or invalid). No query string, Authorization, bodies, or tokens. OPTIONS and `/healthz` are not stored.
 - **Errors:** 503 `{ error: 'Debug is not configured' }` when `DEBUG_TOKEN` is unset or blank; 401 `{ error: 'Unauthorized' }` when the Bearer token does not match; 503 `{ error: 'Log is unavailable' }` if the store throws (`api_log.list.failed`).
 - **Used by:** Operators attributing who called the API (`gifts-debug api-log`).
 - **Auth:** `Authorization: Bearer` with `DEBUG_TOKEN`. Not an end-user session.
 
 ## Endpoint: POST /diagnostics
 
-- **Purpose:** Public client diagnostic ingest with no auth. The JSON object requires `event` (`client.` plus 1–60 of `a-z`, digits, and `.`). Optional `name` (1–40 letters), `message` (1–120 of letters, digits, `.`, `_`, `:`, space, `-`; no slash), `prfPresent` (boolean, never the bytes), `challengeId` (64 lowercase hex), `accountId` (UUID), `stage` (`register` / `authenticate` / `seed` / `login` / `unhandled`), `status` (integer 100–599), and `path` (string; `/view/<segment>` becomes `/view/:viewKey`; rejected if it contains `?` or 32 lowercase hex digits (`0-9`, `a-f`) in a row). Any other key is rejected. A valid body is stored as a `client` row and answered with 204 and an empty body. `User-Agent` is not a body field; the server may store it as `userAgent` after stripping controls and truncating to 200. Rows are kept forever (no TTL, no DELETE). Secrets are not stored: no PRF bytes, mnemonic, session token, view key, nsec, Authorization, Cookie, WebAuthn challenge, attestation, signatures, or raw request bodies.
+- **Purpose:** Public client diagnostic ingest with no auth. The JSON object requires `event` (`client.` plus 1–60 of `a-z`, digits, and `.`). Optional `name` (1–40 letters), `message` (1–120 of letters, digits, `.`, `_`, `:`, space, `-`; no slash), `prfPresent` (boolean, never the bytes), `challengeId` (64 lowercase hex), `accountId` (UUID), `stage` (`register` / `authenticate` / `seed` / `login` / `unhandled`), `status` (integer 100–599), and `path` (string; `/view/<segment>` becomes `/view/:viewKey`; rejected if it contains `?` or 32 lowercase hex digits (`0-9`, `a-f`) in a row). Any other key is rejected. A valid body is stored as a `client` row and answered with 204 and an empty body. `User-Agent` is not a body field; the server may store it as `userAgent` after stripping controls and truncating to 200. The server also stores `clientIp`, `clientCountry`, `cfRay`, `acceptLanguage`, and `origin` when those headers validate, using the same rules as the audit log; those names are not body keys, and absent values are omitted. Rows are kept forever (no TTL, no DELETE). Secrets are not stored: no PRF bytes, mnemonic, session token, view key, nsec, Authorization, Cookie, WebAuthn challenge, attestation, signatures, or raw request bodies.
 - **Errors:** 400 `{ error: 'Invalid diagnostics' }` when JSON or any field fails the allowlist; 429 `{ error: 'Too many diagnostics' }` when the IP window (60) or the global window (600) in 60 seconds is full (the accept timestamp is not recorded on 429); 500 `{ error: 'Log is unavailable' }` when the insert throws (that failure does not consume a rate-limit slot).
 - **Auth:** None. No session and no debug bearer.
 

docs/handbook/functions.md

@@ -983,7 +983,7 @@
 
 ## Function: logEvent
 
-- **Purpose:** One JSON line on `console.warn` (`ts` + `event` + fields). Never log secrets.
+- **Purpose:** One JSON line on `console.warn` (`ts` + `event` + fields). During a request, validated client fields from `readClientRequestMeta` are merged underneath the explicit fields, and explicit fields win. Absent client fields are omitted. Outside a request, only the explicit fields are written. Never log secrets.
 - **Inputs:** `event` string, optional `LogFields`.
 - **Returns / side effects:** void.
 - **Used by:** Auth, me, lightning-address, requestLog.
@@ -1618,6 +1618,20 @@
 - **Returns / side effects:** Lowercase hex.
 - **Used by:** `issueSession`, passkey begin, verification nonce.
 
+## Function: presentClientFields
+
+- **Purpose:** Copies the non-null fields of a validated client-request record, in `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, `origin` order. Absent headers stay omitted so a log line does not print nulls.
+- **Inputs:** `ClientRequestMeta`.
+- **Returns / side effects:** A string map. No I/O.
+- **Used by:** `requestLog` (the request-scoped log fields) and `diagnosticsRoutes`.
+
+## Function: readClientRequestMeta
+
+- **Purpose:** Reads `cf-connecting-ip`, `cf-ipcountry`, `cf-ray`, `user-agent`, `accept-language`, and `origin`. Keeps an IPv4 address with no leading zeros, or an IPv6 address including compressed and IPv4-mapped forms, unchanged. Keeps a country code of two letters or digits after trim and uppercase, so `T1` stays. Keeps a Cloudflare ray id of 16 hex digits, a hyphen, and three letters. Strips controls from the user agent and accept-language and cuts each at 200 characters. Keeps an `https` origin, or `http://localhost` or `http://127.0.0.1`, with an optional port. Drops a port or zone id on the IP, a hostname, `x-forwarded-for`, userinfo, a path, a query, or any other `http` host. Does not read the socket.
+- **Inputs:** A headers object with `get(name)`.
+- **Returns / side effects:** `ClientRequestMeta`. Each field is the original text or `null`. No I/O.
+- **Used by:** `requestLog` and `diagnosticsRoutes`.
+
 ## Function: readPublicBrandFile
 
 - **Purpose:** Reads `public/<name>` relative to a root directory.
@@ -1627,7 +1641,7 @@
 
 ## Function: requestLog
 
-- **Purpose:** Hono middleware: `http.request` JSON after the handler, then one `api_log` row. Skips `/healthz` and OPTIONS. Never logs the query string, body, or Authorization. Path is passed through `requestLogPath` so `/view/<segment>` is redacted. `ms` is handler duration (captured once after `next`). Auth-classification failure still stores `authKind: 'none'` with `accountId` null. Store write failure logs `api_log.write.failed` and does not replace the response.
+- **Purpose:** Hono middleware: `http.request` JSON after the handler, then one `api_log` row. Skips `/healthz` and OPTIONS. Never logs the query string, body, or Authorization. Path is passed through `requestLogPath` so `/view/<segment>` is redacted. `ms` is handler duration (captured once after `next`). Auth-classification failure still stores `authKind: 'none'` with `accountId` null. The row always includes `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and `origin`, each null when that header is missing or invalid. `logEvent` during the request merges the present values underneath the explicit fields, and explicit fields win. Store write failure logs `api_log.write.failed` and does not replace the response.
 - **Inputs:** `{ apiLogStore, authStore, debugToken, spendApiToken, now? }`.
 - **Returns / side effects:** `MiddlewareHandler`.
 - **Used by:** `createApp`.
@@ -1722,7 +1736,7 @@
 ## Function: diagnosticsRoutes
 
 - **Purpose:** Public `POST /` ingest mounted at `/diagnostics`. No auth. Only allowlisted scalar keys are stored on a `client` row, then the response is 204 with an empty body. The per-IP cap (60) and the global cap (600) per 60_000 ms are reserved before the insert await, so two overlapping requests cannot share one slot, and released if that insert throws. Expired per-IP buckets are dropped on the first request of a new minute, so a one-off address does not stay for the process lifetime. Rows are kept forever (no TTL, no DELETE). Secrets and raw bodies are not stored.
-- **Inputs:** `{ store: DiagnosticStore, now?: () => number }`. Optional `cf-connecting-ip` is the per-IP key only when it matches a short IP token; any other value is ignored. Optional `User-Agent` has controls stripped, is truncated to 200, and is omitted when nothing remains.
+- **Inputs:** `{ store: DiagnosticStore, now?: () => number }`. Optional `cf-connecting-ip` is the per-IP key only when it matches a short IP token; any other value is ignored for the cap. Validated `clientIp`, `clientCountry`, `cfRay`, `userAgent`, `acceptLanguage`, and `origin` from `readClientRequestMeta` are stored when present. `User-Agent` and `Accept-Language` have controls stripped, are truncated to 200, and are omitted when nothing remains. A body key named `clientIp` is rejected.
 - **Returns / side effects:** 204 empty on accept; 400 `{ error: 'Invalid diagnostics' }` when JSON or a field fails the allowlist; 429 `{ error: 'Too many diagnostics' }` over the cap, without recording an accept timestamp (at most one `diagnostics.rate_limited` server row per window, and only after that append resolves); 500 `{ error: 'Log is unavailable' }` when the client-row insert throws. No PRF bytes, mnemonic, session token, view key, nsec, Authorization, Cookie, WebAuthn challenge, attestation, signatures, or raw body are stored.
 
 ## Function: debugDiagnosticsRoutes

docs/schema/api_log.sql

@@ -1,7 +1,8 @@
 -- HTTP request audit log. One row per request after the handler (except
 -- OPTIONS and `/healthz`). Path is redacted (`/view/:viewKey`); no query
--- string, Authorization, or bodies. Covered by db_change attach-all-public-tables
--- when migrateApiLogSchema runs before migrateDbChangeSchema.
+-- string, request body, Authorization, Cookie, or tokens. Validated client
+-- fields are stored by the nullable columns added below. Covered by db_change
+-- attach-all-public-tables when migrateApiLogSchema runs before migrateDbChangeSchema.
 
 CREATE TABLE IF NOT EXISTS api_log (
   id uuid PRIMARY KEY,
@@ -14,3 +15,9 @@ CREATE TABLE IF NOT EXISTS api_log (
   auth_kind text NOT NULL CHECK (auth_kind IN ('session', 'debug', 'spend', 'none'))
 );
 CREATE INDEX IF NOT EXISTS api_log_created_at_idx ON api_log (created_at DESC, id DESC);
+ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_ip text;
+ALTER TABLE api_log ADD COLUMN IF NOT EXISTS client_country text;
+ALTER TABLE api_log ADD COLUMN IF NOT EXISTS cf_ray text;
+ALTER TABLE api_log ADD COLUMN IF NOT EXISTS user_agent text;
+ALTER TABLE api_log ADD COLUMN IF NOT EXISTS accept_language text;
+ALTER TABLE api_log ADD COLUMN IF NOT EXISTS origin text;

e2e/functions.spec.ts

@@ -854,6 +854,79 @@ test('Function: diagnosticsRoutes — POST /diagnostics accepts an allowlisted c
   expect(res.status()).toBe(204);
 });
 
+test('Function: readClientRequestMeta — GET /info stores validated client headers on the audit row', async ({
+  request,
+}) => {
+  const userAgent = 'e2e-request-meta-read';
+  const res = await request.get('/info', {
+    headers: {
+      'cf-connecting-ip': '192.0.2.1',
+      'cf-ipcountry': 't1',
+      'cf-ray': '0123456789abcdef-ZRH',
+      'user-agent': userAgent,
+      'accept-language': 'de-CH,de;q=0.9',
+      origin: 'https://21.gifts',
+    },
+  });
+  expect(res.status()).toBe(200);
+  const listed = await request.get('/debug/api-log', {
+    headers: { authorization: 'Bearer e2e-debug-token' },
+  });
+  expect(listed.status()).toBe(200);
+  const body = (await listed.json()) as {
+    logs: Array<{
+      path: string;
+      userAgent: string | null;
+      clientIp: string | null;
+      clientCountry: string | null;
+      cfRay: string | null;
+      acceptLanguage: string | null;
+      origin: string | null;
+    }>;
+  };
+  const row = body.logs.find((log) => log.userAgent === userAgent && log.path === '/info');
+  expect(row).toMatchObject({
+    clientIp: '192.0.2.1',
+    clientCountry: 'T1',
+    cfRay: '0123456789abcdef-ZRH',
+    acceptLanguage: 'de-CH,de;q=0.9',
+    origin: 'https://21.gifts',
+  });
+});
+
+test('Function: presentClientFields — POST /diagnostics stores present client headers', async ({
+  request,
+}) => {
+  const res = await request.post('/diagnostics', {
+    headers: {
+      'cf-connecting-ip': '198.51.100.10',
+      'cf-ipcountry': 'ch',
+      'cf-ray': 'fedcba9876543210-zrh',
+      'user-agent': 'e2e-present-fields',
+      'accept-language': 'en',
+      origin: 'http://127.0.0.1:3000',
+    },
+    data: { event: 'client.e2e.present.fields' },
+  });
+  expect(res.status()).toBe(204);
+  const listed = await request.get('/debug/diagnostics', {
+    headers: { authorization: 'Bearer e2e-debug-token' },
+  });
+  expect(listed.status()).toBe(200);
+  const body = (await listed.json()) as {
+    logs: Array<{ event: string; fields: Record<string, unknown> }>;
+  };
+  const row = body.logs.find((log) => log.event === 'client.e2e.present.fields');
+  expect(row?.fields).toMatchObject({
+    clientIp: '198.51.100.10',
+    clientCountry: 'CH',
+    cfRay: 'fedcba9876543210-zrh',
+    userAgent: 'e2e-present-fields',
+    acceptLanguage: 'en',
+    origin: 'http://127.0.0.1:3000',
+  });
+});
+
 test('Function: debugDiagnosticsRoutes — GET /debug/diagnostics without bearer is 401', async ({
   request,
 }) => {