diff --git a/package.json b/package.json index fb73520..347e53b 100644 --- a/package.json +++ b/package.json @@ -58,12 +58,13 @@ "test:only": "node --test \"test/*.test.ts\"" }, "dependencies": { - "@profullstack/footer": "latest", "@hono/node-server": "^1.14.0", "@profullstack/emailer": "^1.0.3", + "@profullstack/footer": "latest", "@profullstack/hqtui": "^0.5.0", "@profullstack/leaderboard": "^0.3.1", "@profullstack/synconfig": "^0.1.2", + "@profullstack/x402-gateway": "^0.11.0", "@simplewebauthn/server": "^14.0.1", "hono": "^4.6.14", "pg": "^8.13.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e880ff..78ced65 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,7 +16,7 @@ importers: version: 1.0.3 '@profullstack/footer': specifier: latest - version: 0.2.1 + version: 0.2.5 '@profullstack/hqtui': specifier: ^0.5.0 version: 0.5.0 @@ -26,6 +26,9 @@ importers: '@profullstack/synconfig': specifier: ^0.1.2 version: 0.1.2 + '@profullstack/x402-gateway': + specifier: ^0.11.0 + version: 0.11.0 '@simplewebauthn/server': specifier: ^14.0.1 version: 14.0.1 @@ -131,8 +134,8 @@ packages: '@profullstack/emailer@1.0.3': resolution: {integrity: sha512-VyAtCcfnSI6LOwJvWogCr3ZNmEsaZw+iuWrqnosLsdcKsbRyCrCmXigMNZJZEjlgYlR+lK//PwNPG+xtCFvWAw==} - '@profullstack/footer@0.2.1': - resolution: {integrity: sha512-KHmKVwrlzCBrSAAJ4zobNFCxTg8VF0gvNMeAut00dX3+gbWCtk6eOwIR2AaqEC7smj6Aio6cIexzt/dUnv1McA==} + '@profullstack/footer@0.2.5': + resolution: {integrity: sha512-/CKwFP3m+qM0ZFTC9Rj11fgClutJzeYliX44Qp4qwrv2JgA/1k4tVOB3mhkPhaeckUhnfg5UyESDI1uoWr9c2w==} engines: {node: '>=18'} hasBin: true peerDependencies: @@ -146,6 +149,10 @@ packages: engines: {bun: '>=1.1', node: '>=22.6'} hasBin: true + '@profullstack/keys@0.1.0': + resolution: {integrity: sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w==} + engines: {node: '>=20.11'} + '@profullstack/leaderboard@0.3.1': resolution: {integrity: sha512-LbxLwy+RvT/Qez0mfWCk4o9hvcq0kzwBZ0vF+CYp+oGvyqJWUaSTLhCHWGtKKAI3c0ZUXj6ll0iuRjbuZUE2Sg==} engines: {node: '>=20.11'} @@ -154,6 +161,10 @@ packages: resolution: {integrity: sha512-t1Pdms+k/+A7tgFCAcNx8Dry0kwboPEue9Pw2pbn/chXSMfp1dxrcFz672zWkDgvcB7YBmYpIjnvW4NkNkE4Rg==} engines: {node: '>=22'} + '@profullstack/x402-gateway@0.11.0': + resolution: {integrity: sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q==} + engines: {node: '>=20.11'} + '@simplewebauthn/server@14.0.1': resolution: {integrity: sha512-kjWgcm9NdSv+Tobo4Swxd1WnEawsBogpYFMjg9pKsVLPN3FdCmR6BipkL+DcPDgmQ7UKbbBZqRqn3E3YaA7NBw==} engines: {node: '>=20.0.0'} @@ -391,14 +402,20 @@ snapshots: '@profullstack/emailer@1.0.3': {} - '@profullstack/footer@0.2.1': {} + '@profullstack/footer@0.2.5': {} '@profullstack/hqtui@0.5.0': {} + '@profullstack/keys@0.1.0': {} + '@profullstack/leaderboard@0.3.1': {} '@profullstack/synconfig@0.1.2': {} + '@profullstack/x402-gateway@0.11.0': + dependencies: + '@profullstack/keys': 0.1.0 + '@simplewebauthn/server@14.0.1': dependencies: '@hexagon/base64': 1.1.28 diff --git a/src/config.ts b/src/config.ts index 68db124..3665dbb 100644 --- a/src/config.ts +++ b/src/config.ts @@ -44,6 +44,12 @@ export interface Config { * absent from every page rather than present and broken. */ coinpay: CoinPayConfig | null; + /** + * The crawl door (x402-gateway): a scoped CoinPay key and the USDC payee. Both or + * nothing; a board without them never sells crawl passes. Optional so a + * hand-built config in a test can leave it out. + */ + crawl?: { apiKey: string; payTo: string } | null; version: string; } @@ -182,6 +188,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { openaiApiKey: env['OPENAI_API_KEY']?.trim() || null, writerModel: env['WRITER_MODEL']?.trim() || null, coinpay: coinpayConfig(env), + crawl: + env['COINPAY_X402_KEY']?.trim() && env['CRAWL_PAY_TO']?.trim() + ? { apiKey: env['COINPAY_X402_KEY'].trim(), payTo: env['CRAWL_PAY_TO'].trim() } + : null, version: env['npm_package_version']?.trim() || VERSION, }; } diff --git a/src/server/app.tsx b/src/server/app.tsx index 844e72a..bddcbf5 100644 --- a/src/server/app.tsx +++ b/src/server/app.tsx @@ -8,6 +8,9 @@ */ import { Hono } from 'hono'; +import { SESSION_COOKIE } from '../core/auth.ts'; +import { createGateway } from '@profullstack/x402-gateway'; +import { x402Gateway } from '@profullstack/x402-gateway/hono'; import type pg from 'pg'; import type { Config } from '../config.ts'; import { apiRoutes } from './routes/api.ts'; @@ -43,6 +46,26 @@ export function createApp( const deps: Deps = { pool, config, mailer, coinpay, rankings: createRankings(pool, config) }; const app = new Hono(); + // The crawl door, first: a training crawler, headless browser or SEO scraper is + // sold a day pass ($1 over x402, or any coin / PayPal through CoinPay's hosted + // checkout on /crawl) before the board spends a database query on it. People, + // verified search crawlers and the AI chat/search fetchers pass through, as does + // anyone signed in. Ahead of the security headers, whose form-action would stop + // the /crawl/checkout redirect to CoinPay. + if (config.crawl) { + const crawlGateway = createGateway({ + siteUrl: config.publicUrl, + siteName: config.boardName, + coinpay: { apiKey: config.crawl.apiKey }, + payTo: config.crawl.payTo, + contact: 'mailto:anthony@profullstack.com', + openPaths: ['/api/health', '/llms.txt', '/.well-known/'], + exempt: (request) => + new RegExp(`(?:^|;\\s*)${SESSION_COOKIE}=`).test(request.headers.get('cookie') ?? ''), + }); + app.use('*', x402Gateway(crawlGateway)); + } + app.use('*', withViewer(deps)); // A payer is sent from a form on this board to CoinPay's hosted pay page, // and the policy has to say so or the browser stops the redirect. diff --git a/test/crawl-gateway.test.ts b/test/crawl-gateway.test.ts new file mode 100644 index 0000000..91fcb6d --- /dev/null +++ b/test/crawl-gateway.test.ts @@ -0,0 +1,61 @@ +/** + * The crawl door: training crawlers are sold a day pass before the board does + * any work for them; people are not. No database and no network: the offer is + * built locally and the 402 comes before the first query. + */ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +const { createApp } = await import('../dist/server/app.js'); +const { loadConfig } = await import('../dist/config.js'); + +let queries = 0; +const pool = { + query: async () => { + queries += 1; + return { rows: [] }; + }, +}; +const config = loadConfig({ + PUBLIC_URL: 'https://agenticjobs.work', + COINPAY_X402_KEY: 'cp_live_test_crawl_key', + CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5', +}); +const app = createApp(pool as never, config, null, null); + +const GPTBOT = + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; + +test('crawl config is read from COINPAY_X402_KEY and CRAWL_PAY_TO, both or nothing', () => { + assert.ok(config.crawl); + assert.equal(loadConfig({ COINPAY_X402_KEY: 'cp_live_x' }).crawl, null); +}); + +test('GPTBot gets 402 with PAYMENT-REQUIRED, before any query', async () => { + queries = 0; + const res = await app.request('/', { headers: { 'user-agent': GPTBOT, accept: 'application/json' } }); + assert.equal(res.status, 402); + assert.ok(res.headers.get('payment-required')); + assert.equal(queries, 0); +}); + +test('/crawl sells the day pass', async () => { + const res = await app.request('/crawl', { headers: { 'user-agent': 'curl/8', accept: 'application/json' } }); + assert.equal(res.status, 402); + const body = (await res.json()) as { accepts: Array<{ amount: string }> }; + assert.equal(body.accepts[0]?.amount, '1000000'); +}); + +test('a browser and the status check are not charged', async () => { + const chrome = { + 'user-agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-dest': 'document', + }; + const health = await app.request('/api/health', { headers: chrome }); + assert.equal(health.status, 200); + const monitor = await app.request('/api/health', { headers: { 'user-agent': 'GPTBot/1.2' } }); + assert.equal(monitor.status, 200); +});