Merge pull request #575 from Podcastindex-org/aggrivator-bot-signing

add well-known signing directory for web bot auth support in aggrivator

Author: daveajones

package.json

@@ -85,6 +85,7 @@
         "production": "NODE_ENV='production' && webpack --env.file=production --mode production && node server",
         "start": "node server",
         "test": "react-scripts test --env=jsdom",
+        "test:server": "node server/test/well-known-signing-directory.test.js",
         "eject": "react-scripts eject",
         "flow": "flow",
         "format": "prettier --write 'src/**/*.js'",

server/data/http-message-signatures-directory.json

@@ -0,0 +1,11 @@
+{
+  "keys": [
+    {
+      "kty": "OKP",
+      "crv": "Ed25519",
+      "x": "REPLACE_ME",
+      "kid": "REPLACE_ME",
+      "use": "sig"
+    }
+  ]
+}

server/index.js

@@ -109,6 +109,50 @@ app.get('/.well-known/lnurlp/podcastindex', (req, res) => {
   res.redirect('https://getalby.com/.well-known/lnurlp/podcastindex')
 })
 
+// ------------------------------------------------
+// ------ Web Bot Auth key directory (JWKS) -------
+// ------------------------------------------------
+//
+// Publishes the PUBLIC Ed25519 key(s) the Aggrivator crawler uses to sign its
+// requests, so Cloudflare and other verifiers can confirm a signed request
+// really comes from us (Web Bot Auth / RFC 9421 HTTP Message Signatures).
+//
+// The body is served from server/data/http-message-signatures-directory.json so
+// that adding/rotating a key is a content edit, not a code change. The exact
+// media type below is mandatory — verifiers reject application/json.
+//
+// TODO(ops): replace the REPLACE_ME placeholder values in that JSON file with
+// the real JWKS produced on the signing server. Never put private key material
+// (a "d" member) in that file. The `kid` must match the crawler's advertised
+// keyid.
+//
+// CDN/WAF: this endpoint must be reachable by automated clients with no bot
+// challenge, redirect, or auth. The app applies no such gating to this path
+// (the POW middleware only guards /api/*), but if Cloudflare (or any WAF) is in
+// front of the site, ensure this exact path is allowed through unchallenged.
+app.get('/.well-known/http-message-signatures-directory', (req, res) => {
+  fs.readFile(
+    './server/data/http-message-signatures-directory.json',
+    'utf8',
+    (err, data) => {
+      if (err) {
+        res.status(500).json({ error: 'key directory unavailable' })
+        return
+      }
+      // Use setHeader (raw Node) rather than res.set/res.type so Express does
+      // not append "; charset=utf-8" — verifiers expect this media type exactly.
+      res.setHeader(
+        'Content-Type',
+        'application/http-message-signatures-directory+json'
+      )
+      res.setHeader('Cache-Control', 'public, max-age=3600')
+      // Send a Buffer, not a string: res.send() force-appends "; charset=utf-8"
+      // to the Content-Type for string bodies, which we must avoid here.
+      res.send(Buffer.from(data))
+    }
+  )
+})
+
 // ------------------------------------------------
 // ------------ Reverse proxy for API -------------
 // ------------------------------------------------
@@ -452,6 +496,11 @@ app.get('*', (req, res) => {
 const PORT = process.env.PORT || 333
 
 // start express server on port 5001 (default)
-app.listen(PORT, () => {
-  console.log(`server started on port ${PORT}`)
-})
+// Only listen when run directly, so the app can be imported by tests.
+if (require.main === module) {
+  app.listen(PORT, () => {
+    console.log(`server started on port ${PORT}`)
+  })
+}
+
+module.exports = app

server/test/well-known-signing-directory.test.js

@@ -0,0 +1,94 @@
+/*
+ * Self-contained test for the Web Bot Auth key directory endpoint:
+ *   GET /.well-known/http-message-signatures-directory
+ *
+ * There is no server-side test runner in this repo (`yarn test` is react-scripts
+ * for the UI), so this file is a plain Node script: boot the Express app on an
+ * ephemeral port, make real requests, assert, exit non-zero on failure.
+ *
+ * Run with:  node server/test/well-known-signing-directory.test.js
+ */
+const assert = require('assert')
+const fetch = require('node-fetch')
+const app = require('../index')
+
+const PATH = '/.well-known/http-message-signatures-directory'
+const EXPECTED_CONTENT_TYPE =
+  'application/http-message-signatures-directory+json'
+
+async function run() {
+  const server = app.listen(0)
+  await new Promise((resolve) => server.once('listening', resolve))
+  const { port } = server.address()
+  const url = `http://127.0.0.1:${port}${PATH}`
+
+  let failures = 0
+  const check = (name, fn) => {
+    try {
+      fn()
+      console.log(`  ok   - ${name}`)
+    } catch (err) {
+      failures++
+      console.error(`  FAIL - ${name}\n         ${err.message}`)
+    }
+  }
+
+  try {
+    // --- GET ---
+    const getRes = await fetch(url)
+    const body = await getRes.text()
+
+    check('GET returns 200', () => assert.strictEqual(getRes.status, 200))
+    check('GET sets exact Content-Type', () =>
+      assert.strictEqual(
+        getRes.headers.get('content-type'),
+        EXPECTED_CONTENT_TYPE
+      )
+    )
+    check('GET sets a public Cache-Control', () => {
+      const cc = getRes.headers.get('cache-control') || ''
+      assert.ok(/public/.test(cc) && /max-age=\d+/.test(cc), `got "${cc}"`)
+    })
+    check('GET body is valid JSON with a keys array', () => {
+      const json = JSON.parse(body)
+      assert.ok(Array.isArray(json.keys), 'keys is not an array')
+      assert.ok(json.keys.length >= 1, 'keys is empty')
+      const k = json.keys[0]
+      assert.strictEqual(k.kty, 'OKP')
+      assert.strictEqual(k.crv, 'Ed25519')
+      assert.ok('x' in k, 'missing x')
+      assert.ok('kid' in k, 'missing kid')
+    })
+    check('GET body contains no private key material', () => {
+      assert.ok(!/"d"\s*:/.test(body), 'response contains a private key "d"')
+    })
+
+    // --- HEAD ---
+    const headRes = await fetch(url, { method: 'HEAD' })
+    const headBody = await headRes.text()
+    check('HEAD returns 200', () => assert.strictEqual(headRes.status, 200))
+    check('HEAD sets exact Content-Type', () =>
+      assert.strictEqual(
+        headRes.headers.get('content-type'),
+        EXPECTED_CONTENT_TYPE
+      )
+    )
+    check('HEAD has an empty body', () =>
+      assert.strictEqual(headBody, '')
+    )
+  } finally {
+    server.close()
+  }
+
+  if (failures > 0) {
+    console.error(`\n${failures} check(s) failed`)
+    process.exit(1)
+  }
+  console.log('\nAll checks passed')
+  process.exit(0)
+}
+
+run().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})