Sell training crawlers a day pass on /crawl (x402-gateway 0.11)

package.json

@@ -58,12 +58,13 @@
     "test:only": "node --test \"test/*.test.ts\""
   },
   "dependencies": {
-    "@profullstack/footer": "latest",
     "@hono/node-server": "^1.14.0",
     "@profullstack/emailer": "^1.0.3",
+    "@profullstack/footer": "latest",
     "@profullstack/hqtui": "^0.5.0",
     "@profullstack/leaderboard": "^0.3.1",
     "@profullstack/synconfig": "^0.1.2",
+    "@profullstack/x402-gateway": "^0.11.0",
     "@simplewebauthn/server": "^14.0.1",
     "hono": "^4.6.14",
     "pg": "^8.13.1"

src/config.ts

@@ -44,6 +44,12 @@ export interface Config {
    * absent from every page rather than present and broken.
    */
   coinpay: CoinPayConfig | null;
+  /**
+   * The crawl door (x402-gateway): a scoped CoinPay key and the USDC payee. Both or
+   * nothing; a board without them never sells crawl passes. Optional so a
+   * hand-built config in a test can leave it out.
+   */
+  crawl?: { apiKey: string; payTo: string } | null;
   version: string;
 }
 
@@ -182,6 +188,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
     openaiApiKey: env['OPENAI_API_KEY']?.trim() || null,
     writerModel: env['WRITER_MODEL']?.trim() || null,
     coinpay: coinpayConfig(env),
+    crawl:
+      env['COINPAY_X402_KEY']?.trim() && env['CRAWL_PAY_TO']?.trim()
+        ? { apiKey: env['COINPAY_X402_KEY'].trim(), payTo: env['CRAWL_PAY_TO'].trim() }
+        : null,
     version: env['npm_package_version']?.trim() || VERSION,
   };
 }

src/server/app.tsx

@@ -8,6 +8,9 @@
  */
 
 import { Hono } from 'hono';
+import { SESSION_COOKIE } from '../core/auth.ts';
+import { createGateway } from '@profullstack/x402-gateway';
+import { x402Gateway } from '@profullstack/x402-gateway/hono';
 import type pg from 'pg';
 import type { Config } from '../config.ts';
 import { apiRoutes } from './routes/api.ts';
@@ -43,6 +46,26 @@ export function createApp(
   const deps: Deps = { pool, config, mailer, coinpay, rankings: createRankings(pool, config) };
   const app = new Hono<AppEnv>();
 
+  // The crawl door, first: a training crawler, headless browser or SEO scraper is
+  // sold a day pass ($1 over x402, or any coin / PayPal through CoinPay's hosted
+  // checkout on /crawl) before the board spends a database query on it. People,
+  // verified search crawlers and the AI chat/search fetchers pass through, as does
+  // anyone signed in. Ahead of the security headers, whose form-action would stop
+  // the /crawl/checkout redirect to CoinPay.
+  if (config.crawl) {
+    const crawlGateway = createGateway({
+      siteUrl: config.publicUrl,
+      siteName: config.boardName,
+      coinpay: { apiKey: config.crawl.apiKey },
+      payTo: config.crawl.payTo,
+      contact: 'mailto:anthony@profullstack.com',
+      openPaths: ['/api/health', '/llms.txt', '/.well-known/'],
+      exempt: (request) =>
+        new RegExp(`(?:^|;\\s*)${SESSION_COOKIE}=`).test(request.headers.get('cookie') ?? ''),
+    });
+    app.use('*', x402Gateway(crawlGateway));
+  }
+
   app.use('*', withViewer(deps));
   // A payer is sent from a form on this board to CoinPay's hosted pay page,
   // and the policy has to say so or the browser stops the redirect.

test/crawl-gateway.test.ts

@@ -0,0 +1,61 @@
+/**
+ * The crawl door: training crawlers are sold a day pass before the board does
+ * any work for them; people are not. No database and no network: the offer is
+ * built locally and the 402 comes before the first query.
+ */
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+
+const { createApp } = await import('../dist/server/app.js');
+const { loadConfig } = await import('../dist/config.js');
+
+let queries = 0;
+const pool = {
+  query: async () => {
+    queries += 1;
+    return { rows: [] };
+  },
+};
+const config = loadConfig({
+  PUBLIC_URL: 'https://agenticjobs.work',
+  COINPAY_X402_KEY: 'cp_live_test_crawl_key',
+  CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5',
+});
+const app = createApp(pool as never, config, null, null);
+
+const GPTBOT =
+  'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)';
+
+test('crawl config is read from COINPAY_X402_KEY and CRAWL_PAY_TO, both or nothing', () => {
+  assert.ok(config.crawl);
+  assert.equal(loadConfig({ COINPAY_X402_KEY: 'cp_live_x' }).crawl, null);
+});
+
+test('GPTBot gets 402 with PAYMENT-REQUIRED, before any query', async () => {
+  queries = 0;
+  const res = await app.request('/', { headers: { 'user-agent': GPTBOT, accept: 'application/json' } });
+  assert.equal(res.status, 402);
+  assert.ok(res.headers.get('payment-required'));
+  assert.equal(queries, 0);
+});
+
+test('/crawl sells the day pass', async () => {
+  const res = await app.request('/crawl', { headers: { 'user-agent': 'curl/8', accept: 'application/json' } });
+  assert.equal(res.status, 402);
+  const body = (await res.json()) as { accepts: Array<{ amount: string }> };
+  assert.equal(body.accepts[0]?.amount, '1000000');
+});
+
+test('a browser and the status check are not charged', async () => {
+  const chrome = {
+    'user-agent':
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+    'sec-fetch-mode': 'navigate',
+    'sec-fetch-site': 'none',
+    'sec-fetch-dest': 'document',
+  };
+  const health = await app.request('/api/health', { headers: chrome });
+  assert.equal(health.status, 200);
+  const monitor = await app.request('/api/health', { headers: { 'user-agent': 'GPTBot/1.2' } });
+  assert.equal(monitor.status, 200);
+});