diff --git a/CHANGELOG.md b/CHANGELOG.md index ed16de2e..1a21e3ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- Backend: `/readyz` now verifies the database, Redis and Soroban RPC (RPC is `degraded` unless + `READINESS_REQUIRE_RPC=true`), and `/healthz` reports uptime and RPC pool state (#1251). +- Backend: IPFS pin service for campaign images, rules, badge and metadata via Pinata, with + `POST /campaigns/:id/ipfs-pins` and background pinning on campaign creation (#1255). +- Backend: GitHub webhook listener (`POST /webhooks/github`, HMAC-verified) that records merged PRs + and completed issues for developer-bounty task verification (#1256). +- Backend: real-time campaign leaderboard over WebSocket (`leaderboard` channel) with rank-change + diffs (#1257). Fixes the WebSocket server constructor for `ws` 8. - Backend: asynchronous campaign export worker (`POST /campaigns/:id/exports`) on the durable job queue (#1260), Horizon deposit watcher for operator reward-token deposits (#1261), IP-cluster fraud detection with an admin review API (#1258), and a generated OpenAPI reference with Swagger diff --git a/backend/.env.example b/backend/.env.example index d8451908..96e235d3 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -152,6 +152,25 @@ VAPID_SUBJECT=mailto:notifications@trivela.app # POST /api/v1/social-tasks/twitter/verify responds 503 TWITTER_NOT_CONFIGURED. # TWITTER_BEARER_TOKEN= +# ── Readiness probes (#1251) ────────────────────────────────────────────── +# /readyz checks the database, Redis (when REDIS_URL is set) and the Soroban RPC. +# The RPC only fails readiness when this is true; otherwise it reports "degraded". +# READINESS_REQUIRE_RPC=false +# Per-check timeout in milliseconds. +# READINESS_TIMEOUT_MS=2000 + +# ── IPFS pinning (#1255) ────────────────────────────────────────────────── +# Pins campaign images, rules, badge and metadata JSON through Pinata. +# PINATA_JWT= +# PINATA_API_URL=https://api.pinata.cloud + +# ── GitHub webhook (#1256) ──────────────────────────────────────────────── +# Shared secret configured on the GitHub webhook (POST /api/v1/webhooks/github). +# GITHUB_WEBHOOK_SECRET= + +# ── Live leaderboard (#1257) ────────────────────────────────────────────── +# Coalesce bursts of awarded points into one leaderboard push (milliseconds). +# LEADERBOARD_DEBOUNCE_MS=200 # ── Fraud detection (#1258) ─────────────────────────────────────────────── # Distinct accounts registering from one IP subnet (/24 IPv4, /64 IPv6) within the # window before a fraud flag is raised for review. diff --git a/backend/src/db/migrations/044_github_task_events.js b/backend/src/db/migrations/044_github_task_events.js new file mode 100644 index 00000000..3685da0d --- /dev/null +++ b/backend/src/db/migrations/044_github_task_events.js @@ -0,0 +1,26 @@ +export const version = 44; +export const description = 'Add github_task_events for verified developer-bounty tasks'; + +/** + * Merged pull requests and closed issues reported by the GitHub webhook (#1256). + * `delivery_id` is unique so GitHub redeliveries are idempotent. + */ +export function up(db) { + db.exec(` + CREATE TABLE IF NOT EXISTS github_task_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + delivery_id TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL CHECK (kind IN ('pr_merged', 'issue_closed')), + repo TEXT NOT NULL, + number INTEGER NOT NULL, + github_login TEXT NOT NULL, + url TEXT, + title TEXT, + occurred_at TEXT, + recorded_at TEXT NOT NULL, + UNIQUE (kind, repo, number, github_login) + ); + CREATE INDEX IF NOT EXISTS idx_github_task_events_login + ON github_task_events(github_login COLLATE NOCASE, id DESC); + `); +} diff --git a/backend/src/db/migrations/045_ipfs_pins.js b/backend/src/db/migrations/045_ipfs_pins.js new file mode 100644 index 00000000..b6224a27 --- /dev/null +++ b/backend/src/db/migrations/045_ipfs_pins.js @@ -0,0 +1,21 @@ +export const version = 45; +export const description = 'Add ipfs_pins for campaign asset and metadata pinning'; + +/** Pins created by the IPFS pin service (#1255). `content_hash` de-duplicates identical content. */ +export function up(db) { + db.exec(` + CREATE TABLE IF NOT EXISTS ipfs_pins ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + campaign_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('metadata', 'image', 'rules', 'badge')), + content_hash TEXT NOT NULL, + cid TEXT NOT NULL, + name TEXT, + size_bytes INTEGER, + provider TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE (campaign_id, kind, content_hash) + ); + CREATE INDEX IF NOT EXISTS idx_ipfs_pins_campaign ON ipfs_pins(campaign_id, id DESC); + `); +} diff --git a/backend/src/index.js b/backend/src/index.js index d04cc5e0..4e708170 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -112,6 +112,11 @@ import { createNotificationPreferencesRoutes, } from './routes/notifications.js'; import { createOperatorBalanceJob } from './jobs/operatorBalanceJob.js'; +import { createGithubTaskStore } from './services/githubWebhook.js'; +import { createGithubTaskRoutes, createGithubWebhookRoutes } from './routes/githubWebhook.js'; +import { createIpfsPinService, createPinataProvider } from './services/ipfsPinService.js'; +import { IPFS_PIN_JOB_TYPE, createIpfsPinRoutes } from './routes/ipfsPins.js'; +import { createLeaderboardGateway, leaderboardRoom } from './websocket/leaderboardGateway.js'; import { createSqliteExportJobRepository } from './dal/sqliteExportJobRepository.js'; import { CAMPAIGN_EXPORT_JOB_TYPE, @@ -377,6 +382,25 @@ export async function createApp(options = {}) { const auditLogRepository = dal.auditLogs; const webhookRepository = dal.webhooks; const referralRepository = dal.referrals; + + // #1257 — live leaderboard pushes over the WebSocket gateway + const leaderboardGateway = createLeaderboardGateway({ + getLeaderboard: (campaignId) => + referralRepository + .getLeaderboard(campaignId, { limit: 10 }) + .rows.map((/** @type {any} */ r) => ({ + wallet: r.referrerAddress, + rank: r.rank, + points: r.referralCount, + })), + publish: (campaignId, message) => + getWebSocketServer()?.broadcast(leaderboardRoom(campaignId), message) ?? 0, + debounceMs: normalizePositiveInteger( + /** @type {any} */ (options.leaderboardDebounceMs) ?? process.env.LEADERBOARD_DEBOUNCE_MS, + 200, + ), + logger: log, + }); const variantRepository = dal.variants; const cohortRepository = dal.cohorts; const pushSubscriptionRepository = dal.pushSubscriptions; @@ -704,6 +728,19 @@ export async function createApp(options = {}) { app.use(createDeprecationMiddleware({ log })); app.use(traceparentMiddleware()); app.use(requestLogger); + // #1256 — GitHub webhook. Mounted before express.json(): the HMAC signature is + // computed over the raw request bytes. + const githubTaskStore = createGithubTaskStore({ db: dal.db, logger: log }); + app.use( + API_V1_PREFIX, + createGithubWebhookRoutes({ + store: githubTaskStore, + getSecret: () => + String(options.githubWebhookSecret ?? process.env.GITHUB_WEBHOOK_SECRET ?? ''), + logger: log, + }), + ); + app.use(express.json({ limit: jsonBodyLimit })); const uploadDir = process.env.UPLOAD_DIR ?? './uploads'; @@ -856,6 +893,21 @@ export async function createApp(options = {}) { setInterval(doExport, 24 * 60 * 60 * 1_000).unref?.(); } + // #1255 — IPFS pinning of campaign assets (enabled when PINATA_JWT is set) + const ipfsPinService = + /** @type {any} */ (options.ipfsPinService) ?? + (process.env.PINATA_JWT + ? createIpfsPinService({ + db: dal.db, + provider: createPinataProvider({ + jwt: process.env.PINATA_JWT, + baseUrl: process.env.PINATA_API_URL, + fetchImpl, + }), + fetchImpl, + logger: log, + }) + : null); // #1260 — asynchronous campaign exports, processed off the request path const campaignExportJobRepository = createSqliteExportJobRepository({ db: dal.db }); const campaignExportWorker = createCampaignExportWorker({ @@ -870,6 +922,11 @@ export async function createApp(options = {}) { const durableJobQueue = createDurableJobQueue({ store: jobQueueStore, handlers: { + // #1255 — background pinning, enqueued when a campaign is created + [IPFS_PIN_JOB_TYPE]: async (/** @type {any} */ payload) => { + const target = campaignRepository.getById(String(payload?.campaignId)); + if (target && ipfsPinService) await ipfsPinService.pinCampaign(target); + }, [CAMPAIGN_EXPORT_JOB_TYPE]: campaignExportWorker.handle, // #922 — end-of-campaign claimable balance creation, enqueued from // POST /campaigns/:id/claimable-balances instead of running inline. @@ -1028,12 +1085,53 @@ export async function createApp(options = {}) { res.json(payload); }); - const probeHandlers = createProbeHandlers({ getIsShuttingDown: () => isShuttingDown }); + // #1251 — readiness verifies the database, Redis (when configured) and the + // Soroban RPC. The RPC is reported but only fails readiness when + // READINESS_REQUIRE_RPC=true, so an RPC blip does not evict every pod. + const readinessRequireRpc = ['1', 'true'].includes( + String(options.readinessRequireRpc ?? process.env.READINESS_REQUIRE_RPC ?? '').toLowerCase(), + ); + const probeHandlers = createProbeHandlers({ + getIsShuttingDown: () => isShuttingDown, + timeoutMs: normalizePositiveInteger( + /** @type {any} */ (options.readinessTimeoutMs) ?? process.env.READINESS_TIMEOUT_MS, + 2000, + ), + checks: { + database: { + run: () => { + dal.db.prepare('SELECT 1 AS ok').get(); + }, + }, + redis: usageRedisClient + ? { + run: async () => { + const pong = await usageRedisClient.ping(); + if (pong !== 'PONG') throw new Error(`unexpected PING reply: ${pong}`); + }, + } + : null, + rpc: { + required: readinessRequireRpc, + run: async () => { + const result = await checkSorobanRpcHealth({ + rpcUrl: rpcPool.getHealthyRpcUrl(), + fetchImpl, + }); + if (/** @type {any} */ (result).status !== 'ok') { + throw new Error(`Soroban RPC is ${/** @type {any} */ (result).status}`); + } + return { pool: rpcPool.getStatus() }; + }, + }, + }, + healthDetails: () => ({ rpc: rpcPool.getStatus() }), + }); app.get('/health/live', probeHandlers.livenessHandler); app.get('/health/ready', probeHandlers.readinessHandler); app.get('/livez', probeHandlers.livenessHandler); app.get('/readyz', probeHandlers.readinessHandler); - app.get('/healthz', probeHandlers.livenessHandler); + app.get('/healthz', probeHandlers.healthHandler); app.get('/ready', (_req, res) => { if (isShuttingDown) { @@ -1547,6 +1645,10 @@ export async function createApp(options = {}) { }); } + if (ipfsPinService && !options.disableJobs) { + durableJobQueue.enqueue(IPFS_PIN_JOB_TYPE, { campaignId: campaign.id }, { maxAttempts: 3 }); + } + shortCache.clear(); return res.status(201).json(serializeCampaign(campaign)); } catch (error) { @@ -2956,6 +3058,8 @@ export async function createApp(options = {}) { }); } + // Points were awarded to the referrer: push new standings to WebSocket subscribers (#1257). + leaderboardGateway.trigger(campaign.id); // #1258 — flag clusters of signups from one IP subnet / proxy exit node. // Detection is advisory: a failure here must never fail the signup. try { @@ -3205,6 +3309,19 @@ export async function createApp(options = {}) { }); app.use(prefix, rateLimiter, auditRouter); + // #1256 — verified GitHub tasks (master key) + app.use( + `${prefix}/admin/github-tasks`, + rateLimiter, + requireMasterKey, + createGithubTaskRoutes({ store: githubTaskStore }), + ); + + // #1255 — IPFS pinning of campaign assets (API key; the router applies the guard itself) + app.use( + prefix, + rateLimiter, + createIpfsPinRoutes({ getService: () => ipfsPinService, campaignRepository, guard }), // #1258 — fraud flag review (master key) app.use( `${prefix}/admin/fraud`, @@ -3405,8 +3522,11 @@ export async function createApp(options = {}) { // Central error handler — must be registered after all routes app.use(errorHandler); + app._leaderboardGateway = leaderboardGateway; + app._close = () => { isShuttingDown = true; + leaderboardGateway.stop(); depositWatcher.stop(); try { dal.db.close(); @@ -3444,6 +3564,8 @@ export async function startServer(options = {}) { try { initializeWebSocket(server, { path: process.env.WEBSOCKET_PATH || '/ws', + leaderboardSnapshot: (/** @type {string} */ campaignId) => + app._leaderboardGateway.snapshot(campaignId), }); log.info('WebSocket server initialized on /ws'); } catch (error) { diff --git a/backend/src/middleware/probes.js b/backend/src/middleware/probes.js index ea406bfa..666ea419 100644 --- a/backend/src/middleware/probes.js +++ b/backend/src/middleware/probes.js @@ -1,28 +1,132 @@ // @ts-check +/** + * @typedef {{ + * run: () => unknown | Promise, + * required?: boolean, + * }} ProbeCheck + */ + +/** + * @template T + * @param {Promise} promise + * @param {number} ms + * @param {string} message + * @returns {Promise} + */ +function withTimeout(promise, ms, message) { + /** @type {ReturnType | undefined} */ + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), ms); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + /** * Creates health and probe handlers for Kubernetes liveness and readiness checks. + * + * - `/livez` liveness: the process is up. Never touches dependencies. + * - `/healthz` health: liveness plus uptime and any cheap, cached details + * (e.g. the RPC pool state) supplied via `healthDetails`. + * - `/readyz` readiness: runs the dependency `checks` (database, Redis, + * Soroban RPC, ...) in parallel, each under `timeoutMs`. It returns + * 503 only when a *required* check fails (or the process is + * shutting down); a failing optional check reports `degraded` but + * still returns 200, so an RPC blip does not pull every pod out of + * rotation. + * * @param {object} [options] * @param {() => boolean} [options.getIsShuttingDown] + * @param {Record} [options.checks] + * @param {number} [options.timeoutMs] per-check timeout (default 2000) + * @param {() => Record} [options.healthDetails] + * @param {() => number} [options.now] */ export function createProbeHandlers(options = {}) { const getIsShuttingDown = options.getIsShuttingDown ?? (() => false); + const checks = options.checks ?? {}; + const timeoutMs = options.timeoutMs ?? 2000; + const healthDetails = options.healthDetails; + const now = options.now ?? (() => Date.now()); + const startedAt = now(); + + const uptimeSeconds = () => Math.floor((now() - startedAt) / 1000); + + /** + * @param {string} name + * @param {ProbeCheck} check + */ + async function runCheck(name, check) { + const required = check.required !== false; + const started = now(); + try { + const detail = await withTimeout( + Promise.resolve().then(() => check.run()), + timeoutMs, + `${name} check timed out after ${timeoutMs}ms`, + ); + return { + status: 'ok', + required, + latencyMs: now() - started, + ...(detail && typeof detail === 'object' ? detail : {}), + }; + } catch (err) { + return { + status: 'fail', + required, + latencyMs: now() - started, + error: err instanceof Error ? err.message : String(err), + }; + } + } /** Liveness probe handler - checks if the application process is running */ function livenessHandler(_req, res) { res.status(200).json({ status: 'ok', live: true }); } + /** Health handler - liveness plus uptime and cached dependency details */ + function healthHandler(_req, res) { + res.status(200).json({ + status: 'ok', + live: true, + uptimeSeconds: uptimeSeconds(), + ...(healthDetails ? healthDetails() : {}), + }); + } + /** Readiness probe handler - checks if application is ready to receive traffic */ - function readinessHandler(_req, res) { + async function readinessHandler(_req, res) { if (getIsShuttingDown()) { return res.status(503).json({ status: 'shutting_down', ready: false }); } - res.status(200).json({ status: 'ok', ready: true }); + + const entries = Object.entries(checks).filter((entry) => entry[1]); + const results = await Promise.all( + entries.map(async ([name, check]) => [ + name, + await runCheck(name, /** @type {ProbeCheck} */ (check)), + ]), + ); + const byName = Object.fromEntries(results); + + const requiredFailed = results.some(([, r]) => r.status === 'fail' && r.required); + const optionalFailed = results.some(([, r]) => r.status === 'fail' && !r.required); + const status = requiredFailed ? 'unavailable' : optionalFailed ? 'degraded' : 'ok'; + + res.status(requiredFailed ? 503 : 200).json({ + status, + ready: !requiredFailed, + uptimeSeconds: uptimeSeconds(), + checks: byName, + }); } return { livenessHandler, + healthHandler, readinessHandler, }; } diff --git a/backend/src/middleware/probes.test.js b/backend/src/middleware/probes.test.js new file mode 100644 index 00000000..ed665121 --- /dev/null +++ b/backend/src/middleware/probes.test.js @@ -0,0 +1,198 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createProbeHandlers } from './probes.js'; + +function makeRes() { + return { + statusCode: 200, + body: null, + status(code) { + this.statusCode = code; + return this; + }, + json(payload) { + this.body = payload; + return this; + }, + }; +} + +const ok = () => ({ run: () => {} }); +const fail = (message, extra = {}) => ({ + run: () => { + throw new Error(message); + }, + ...extra, +}); + +test('liveness never runs dependency checks', () => { + let ran = false; + const { livenessHandler } = createProbeHandlers({ + checks: { database: { run: () => (ran = true) } }, + }); + const res = makeRes(); + livenessHandler({}, res); + assert.equal(res.statusCode, 200); + assert.deepEqual(res.body, { status: 'ok', live: true }); + assert.equal(ran, false); +}); + +test('readiness with no checks stays ready', async () => { + const { readinessHandler } = createProbeHandlers(); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 200); + assert.equal(res.body.ready, true); + assert.equal(res.body.status, 'ok'); +}); + +test('readiness reports every passing check with its latency', async () => { + const { readinessHandler } = createProbeHandlers({ + checks: { database: ok(), redis: { run: () => ({ mode: 'standalone' }) }, skipped: null }, + }); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 200); + assert.deepEqual(Object.keys(res.body.checks), ['database', 'redis']); + assert.equal(res.body.checks.database.status, 'ok'); + assert.equal(typeof res.body.checks.database.latencyMs, 'number'); + assert.equal(res.body.checks.redis.mode, 'standalone'); +}); + +test('a failing required check returns 503 and names the failure', async () => { + const { readinessHandler } = createProbeHandlers({ + checks: { database: fail('disk I/O error'), redis: ok() }, + }); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 503); + assert.equal(res.body.ready, false); + assert.equal(res.body.status, 'unavailable'); + assert.equal(res.body.checks.database.status, 'fail'); + assert.equal(res.body.checks.database.error, 'disk I/O error'); + assert.equal(res.body.checks.redis.status, 'ok'); +}); + +test('a failing optional check degrades but stays ready', async () => { + const { readinessHandler } = createProbeHandlers({ + checks: { database: ok(), rpc: fail('rpc down', { required: false }) }, + }); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 200); + assert.equal(res.body.ready, true); + assert.equal(res.body.status, 'degraded'); + assert.equal(res.body.checks.rpc.required, false); +}); + +test('a hanging check times out instead of hanging the probe', async () => { + const { readinessHandler } = createProbeHandlers({ + timeoutMs: 20, + checks: { rpc: { run: () => new Promise(() => {}) } }, + }); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 503); + assert.match(res.body.checks.rpc.error, /timed out after 20ms/); +}); + +test('async rejections are caught', async () => { + const { readinessHandler } = createProbeHandlers({ + checks: { redis: { run: async () => Promise.reject(new Error('ECONNREFUSED')) } }, + }); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 503); + assert.equal(res.body.checks.redis.error, 'ECONNREFUSED'); +}); + +test('shutting down short-circuits readiness without running checks', async () => { + let ran = false; + const { readinessHandler } = createProbeHandlers({ + getIsShuttingDown: () => true, + checks: { database: { run: () => (ran = true) } }, + }); + const res = makeRes(); + await readinessHandler({}, res); + assert.equal(res.statusCode, 503); + assert.deepEqual(res.body, { status: 'shutting_down', ready: false }); + assert.equal(ran, false); +}); + +test('healthz adds uptime and the supplied cached details', () => { + let t = 1_000; + const { healthHandler } = createProbeHandlers({ + now: () => t, + healthDetails: () => ({ rpc: { healthy: 2 } }), + }); + t += 5_500; + const res = makeRes(); + healthHandler({}, res); + assert.equal(res.statusCode, 200); + assert.equal(res.body.uptimeSeconds, 5); + assert.deepEqual(res.body.rpc, { healthy: 2 }); +}); + +// ── Wired into the app (#1251) ─────────────────────────────────────────────── +import request from 'supertest'; +import { createApp } from '../index.js'; + +async function makeApp(options = {}) { + return createApp({ + dbPath: ':memory:', + campaigns: [], + disableJobs: true, + skipEnvValidation: true, + disableRedis: true, + rateLimit: { windowMs: 60_000, maxRequests: 10_000 }, + ...options, + }); +} + +const rpcOk = async () => ({ + ok: true, + json: async () => ({ result: { status: 'healthy', latestLedger: 1 } }), +}); +const rpcDown = async () => { + throw new Error('connect ECONNREFUSED'); +}; + +test('/readyz: database ok, RPC healthy -> 200 ok', async () => { + const app = await makeApp({ fetchImpl: rpcOk }); + const res = await request(app).get('/readyz').expect(200); + assert.equal(res.body.status, 'ok'); + assert.equal(res.body.checks.database.status, 'ok'); + assert.equal(res.body.checks.rpc.status, 'ok'); + assert.equal(res.body.checks.redis, undefined, 'no Redis configured, so no Redis check'); +}); + +test('/readyz: RPC down is reported as degraded but stays ready by default', async () => { + const app = await makeApp({ fetchImpl: rpcDown }); + const res = await request(app).get('/readyz').expect(200); + assert.equal(res.body.status, 'degraded'); + assert.equal(res.body.checks.rpc.status, 'fail'); + assert.equal(res.body.checks.rpc.required, false); +}); + +test('/readyz: RPC down fails readiness when READINESS_REQUIRE_RPC is set', async () => { + const app = await makeApp({ fetchImpl: rpcDown, readinessRequireRpc: true }); + const res = await request(app).get('/readyz').expect(503); + assert.equal(res.body.ready, false); + assert.equal(res.body.checks.rpc.required, true); +}); + +test('/readyz: a broken database fails readiness', async () => { + const app = await makeApp({ fetchImpl: rpcOk }); + app._close(); // closes the SQLite handle and starts shutdown + const res = await request(app).get('/health/ready'); + assert.equal(res.statusCode, 503); +}); + +test('/healthz reports uptime and the RPC pool without touching the network', async () => { + const app = await makeApp({ fetchImpl: rpcDown }); + const res = await request(app).get('/healthz').expect(200); + assert.equal(res.body.live, true); + assert.equal(typeof res.body.uptimeSeconds, 'number'); + assert.ok(res.body.rpc); + await request(app).get('/livez').expect(200); +}); diff --git a/backend/src/routes/githubWebhook.js b/backend/src/routes/githubWebhook.js new file mode 100644 index 00000000..3b4887c3 --- /dev/null +++ b/backend/src/routes/githubWebhook.js @@ -0,0 +1,112 @@ +// @ts-check +import express, { Router } from 'express'; +import { extractVerifiedTask, verifyGithubSignature } from '../services/githubWebhook.js'; + +/** + * GitHub webhook listener (#1256). + * + * POST /webhooks/github (public; authenticated by the HMAC signature) + * + * Mount this BEFORE the global `express.json()` — the signature is computed + * over the exact bytes GitHub sent, so the body is parsed here from the raw + * buffer. + * + * @param {{ + * store: ReturnType, + * getSecret: () => string, + * logger?: { warn?: Function }, + * }} options + */ +export function createGithubWebhookRoutes({ store, getSecret, logger = console }) { + const router = Router(); + + router.post('/webhooks/github', express.raw({ type: '*/*', limit: '2mb' }), (req, res) => { + const secret = getSecret(); + if (!secret) { + return res + .status(503) + .json({ error: 'GitHub webhook is not configured', code: 'GITHUB_WEBHOOK_NOT_CONFIGURED' }); + } + + const raw = Buffer.isBuffer(req.body) ? req.body : Buffer.from(''); + const signature = req.get('x-hub-signature-256'); + if (!verifyGithubSignature(raw, signature, secret)) { + logger.warn?.('github:webhook invalid signature'); + return res.status(401).json({ error: 'Invalid signature', code: 'INVALID_SIGNATURE' }); + } + + const event = req.get('x-github-event') ?? ''; + const deliveryId = req.get('x-github-delivery') ?? ''; + if (event === 'ping') return res.status(200).json({ ok: true, event: 'ping' }); + if (!deliveryId) { + return res.status(400).json({ error: 'Missing X-GitHub-Delivery', code: 'VALIDATION_ERROR' }); + } + + /** @type {Record} */ + let payload; + try { + payload = JSON.parse(raw.toString('utf8')); + } catch { + return res.status(400).json({ error: 'Invalid JSON payload', code: 'VALIDATION_ERROR' }); + } + + const task = extractVerifiedTask(event, payload); + if (!task) return res.status(202).json({ ok: true, ignored: true, event }); + + const recorded = store.record(deliveryId, task); + return res + .status(recorded ? 201 : 200) + .json({ + ok: true, + verified: true, + duplicate: !recorded, + kind: task.kind, + repo: task.repo, + number: task.number, + }); + }); + + return router; +} + +/** + * Master-key API to look up verified tasks before crediting a reward. + * + * GET /?githubLogin=&repo=&limit= + * GET /verify?githubLogin=&repo=&number=&kind= + * (mounted at /admin/github-tasks) + * + * @param {{ store: ReturnType }} options + */ +export function createGithubTaskRoutes({ store }) { + const router = Router(); + + router.get('/verify', (req, res) => { + const githubLogin = typeof req.query.githubLogin === 'string' ? req.query.githubLogin : ''; + if (!githubLogin) { + return res.status(400).json({ error: 'githubLogin is required', code: 'VALIDATION_ERROR' }); + } + const number = req.query.number !== undefined ? Number(req.query.number) : undefined; + if (number !== undefined && !Number.isInteger(number)) { + return res.status(400).json({ error: 'number must be an integer', code: 'VALIDATION_ERROR' }); + } + const verified = store.isVerified({ + githubLogin, + number, + kind: typeof req.query.kind === 'string' ? req.query.kind : undefined, + repo: typeof req.query.repo === 'string' ? req.query.repo : undefined, + }); + return res.json({ githubLogin, verified }); + }); + + router.get('/', (req, res) => { + const data = store.list({ + githubLogin: typeof req.query.githubLogin === 'string' ? req.query.githubLogin : undefined, + repo: typeof req.query.repo === 'string' ? req.query.repo : undefined, + limit: Number.parseInt(String(req.query.limit ?? ''), 10) || 50, + }); + return res.json({ data, total: data.length }); + }); + + return router; +} diff --git a/backend/src/routes/ipfsPins.js b/backend/src/routes/ipfsPins.js new file mode 100644 index 00000000..201f896c --- /dev/null +++ b/backend/src/routes/ipfsPins.js @@ -0,0 +1,53 @@ +// @ts-check +import { Router } from 'express'; + +/** Durable-queue job type that pins a campaign in the background. */ +export const IPFS_PIN_JOB_TYPE = 'ipfs_pin_campaign'; + +/** + * IPFS pinning endpoints (#1255). + * + * POST /campaigns/:id/ipfs-pins pin the campaign's image, rules, badge and metadata + * GET /campaigns/:id/ipfs-pins list stored pins + * + * @param {{ + * getService: () => ReturnType | null, + * campaignRepository: { getById: (id: string) => any }, + * guard?: import('express').RequestHandler[], + * }} options + */ +export function createIpfsPinRoutes({ getService, campaignRepository, guard = [] }) { + const router = Router(); + + router.post('/campaigns/:id/ipfs-pins', ...guard, async (req, res) => { + const service = getService(); + if (!service) { + return res + .status(503) + .json({ error: 'IPFS pinning is not configured', code: 'IPFS_PIN_NOT_CONFIGURED' }); + } + const campaign = campaignRepository.getById(req.params.id); + if (!campaign) return res.status(404).json({ error: 'Campaign not found', code: 'NOT_FOUND' }); + + try { + const result = await service.pinCampaign(campaign); + return res.status(201).json(result); + } catch (err) { + return res.status(502).json({ + error: `Pinning failed: ${err instanceof Error ? err.message : String(err)}`, + code: 'IPFS_PIN_FAILED', + }); + } + }); + + router.get('/campaigns/:id/ipfs-pins', ...guard, (req, res) => { + const service = getService(); + if (!campaignRepository.getById(req.params.id)) { + return res.status(404).json({ error: 'Campaign not found', code: 'NOT_FOUND' }); + } + const data = service ? service.list(req.params.id) : []; + return res.json({ data, total: data.length, configured: Boolean(service) }); + }); + + return router; +} diff --git a/backend/src/services/githubWebhook.js b/backend/src/services/githubWebhook.js new file mode 100644 index 00000000..c7ac9730 --- /dev/null +++ b/backend/src/services/githubWebhook.js @@ -0,0 +1,146 @@ +// @ts-check +import { createHmac, timingSafeEqual } from 'node:crypto'; + +/** + * GitHub webhook verification for developer-bounty campaigns (#1256). + * + * A reward task is only considered verified when GitHub itself says so: a pull + * request `closed` with `merged: true`, or an issue `closed` as completed. The + * payload is authenticated with the shared secret (`X-Hub-Signature-256`) and + * every delivery is recorded once, so redeliveries are harmless. + */ + +/** + * Constant-time check of GitHub's `sha256=` signature over the raw body. + * + * @param {Buffer | string} rawBody + * @param {string | undefined} signatureHeader + * @param {string} secret + */ +export function verifyGithubSignature(rawBody, signatureHeader, secret) { + if (!secret || typeof signatureHeader !== 'string') return false; + const expected = `sha256=${createHmac('sha256', secret).update(rawBody).digest('hex')}`; + const a = Buffer.from(signatureHeader); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); +} + +/** + * Turn a GitHub webhook payload into a verified task, or null when the event is + * not one that proves a task was completed. + * + * @param {string} event the `X-GitHub-Event` header + * @param {Record} payload + * @returns {{ kind: 'pr_merged' | 'issue_closed', repo: string, number: number, + * githubLogin: string, url: string | null, title: string | null, + * occurredAt: string | null } | null} + */ +export function extractVerifiedTask(event, payload) { + const repo = payload?.repository?.full_name; + if (typeof repo !== 'string' || !repo) return null; + + if (event === 'pull_request') { + const pr = payload.pull_request; + // A PR that was closed without merging is not a completed task. + if (payload.action !== 'closed' || pr?.merged !== true) return null; + const login = pr.user?.login; + if (!login) return null; + return { + kind: 'pr_merged', + repo, + number: Number(pr.number), + githubLogin: String(login), + url: pr.html_url ?? null, + title: pr.title ?? null, + occurredAt: pr.merged_at ?? null, + }; + } + + if (event === 'issues') { + const issue = payload.issue; + // `not_planned` closures are not completed work. + if (payload.action !== 'closed' || issue?.state_reason === 'not_planned') return null; + const login = payload.sender?.login; + if (!login) return null; + return { + kind: 'issue_closed', + repo, + number: Number(issue.number), + githubLogin: String(login), + url: issue.html_url ?? null, + title: issue.title ?? null, + occurredAt: issue.closed_at ?? null, + }; + } + + return null; +} + +/** + * @param {{ db: InstanceType, logger?: { info?: Function } }} options + */ +export function createGithubTaskStore({ db, logger = console }) { + const insert = db.prepare(` + INSERT OR IGNORE INTO github_task_events + (delivery_id, kind, repo, number, github_login, url, title, occurred_at, recorded_at) + VALUES (@deliveryId, @kind, @repo, @number, @githubLogin, @url, @title, @occurredAt, @recordedAt) + `); + + return { + /** + * @param {string} deliveryId + * @param {NonNullable>} task + * @returns {boolean} true when newly recorded, false for a redelivery/duplicate + */ + record(deliveryId, task) { + const result = insert.run({ deliveryId, ...task, recordedAt: new Date().toISOString() }); + if (result.changes > 0) { + logger.info?.( + { repo: task.repo, kind: task.kind, number: task.number }, + 'github:task_verified', + ); + } + return result.changes > 0; + }, + + /** + * Has this GitHub user completed the given task? Login match is case-insensitive. + * @param {{ kind?: string, repo?: string, number?: number, githubLogin: string }} query + */ + isVerified({ kind, repo, number, githubLogin }) { + const where = ['github_login = ? COLLATE NOCASE']; + const params = [githubLogin]; + if (kind) (where.push('kind = ?'), params.push(kind)); + if (repo) (where.push('repo = ? COLLATE NOCASE'), params.push(repo)); + if (number !== undefined) (where.push('number = ?'), params.push(number)); + return Boolean( + db + .prepare(`SELECT 1 FROM github_task_events WHERE ${where.join(' AND ')} LIMIT 1`) + .get(...params), + ); + }, + + /** @param {{ githubLogin?: string, repo?: string, limit?: number }} [filter] */ + list({ githubLogin, repo, limit = 50 } = {}) { + const where = []; + const params = []; + if (githubLogin) (where.push('github_login = ? COLLATE NOCASE'), params.push(githubLogin)); + if (repo) (where.push('repo = ? COLLATE NOCASE'), params.push(repo)); + const clause = where.length ? `WHERE ${where.join(' AND ')}` : ''; + return db + .prepare(`SELECT * FROM github_task_events ${clause} ORDER BY id DESC LIMIT ?`) + .all(...params, Math.min(Math.max(limit, 1), 200)) + .map((r) => ({ + id: r.id, + kind: r.kind, + repo: r.repo, + number: r.number, + githubLogin: r.github_login, + url: r.url, + title: r.title, + occurredAt: r.occurred_at, + recordedAt: r.recorded_at, + })); + }, + }; +} diff --git a/backend/src/services/githubWebhook.test.js b/backend/src/services/githubWebhook.test.js new file mode 100644 index 00000000..5995ff03 --- /dev/null +++ b/backend/src/services/githubWebhook.test.js @@ -0,0 +1,197 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createHmac } from 'node:crypto'; +import Database from 'better-sqlite3'; +import request from 'supertest'; +import { runMigrations } from '../db/migrate.js'; +import { createApp } from '../index.js'; +import { + createGithubTaskStore, + extractVerifiedTask, + verifyGithubSignature, +} from './githubWebhook.js'; + +const SECRET = 'whsec_test'; +const sign = (body, secret = SECRET) => + `sha256=${createHmac('sha256', secret).update(body).digest('hex')}`; + +const mergedPr = (over = {}) => ({ + action: 'closed', + repository: { full_name: 'acme/widgets' }, + pull_request: { + number: 12, + merged: true, + merged_at: '2026-09-26T00:00:00Z', + title: 'Fix bug', + html_url: 'https://github.com/acme/widgets/pull/12', + user: { login: 'Octo-Dev' }, + }, + ...over, +}); + +const closedIssue = (over = {}) => ({ + action: 'closed', + repository: { full_name: 'acme/widgets' }, + issue: { + number: 7, + title: 'Bug', + state_reason: 'completed', + html_url: 'https://github.com/acme/widgets/issues/7', + closed_at: '2026-09-26T01:00:00Z', + }, + sender: { login: 'octo-dev' }, + ...over, +}); + +test('verifyGithubSignature accepts the right HMAC and rejects everything else', () => { + const body = Buffer.from('{"a":1}'); + assert.equal(verifyGithubSignature(body, sign(body), SECRET), true); + assert.equal(verifyGithubSignature(body, sign(body, 'other'), SECRET), false); + assert.equal(verifyGithubSignature(body, sign(Buffer.from('{"a":2}')), SECRET), false); + assert.equal(verifyGithubSignature(body, undefined, SECRET), false); + assert.equal(verifyGithubSignature(body, 'sha256=abc', SECRET), false, 'wrong length'); + assert.equal(verifyGithubSignature(body, sign(body), ''), false, 'no secret configured'); +}); + +test('extractVerifiedTask: only merged PRs and completed issue closures count', () => { + const pr = extractVerifiedTask('pull_request', mergedPr()); + assert.deepEqual( + { kind: pr.kind, repo: pr.repo, number: pr.number, githubLogin: pr.githubLogin }, + { kind: 'pr_merged', repo: 'acme/widgets', number: 12, githubLogin: 'Octo-Dev' }, + ); + + const issue = extractVerifiedTask('issues', closedIssue()); + assert.equal(issue.kind, 'issue_closed'); + assert.equal(issue.githubLogin, 'octo-dev'); + + const unmerged = mergedPr(); + unmerged.pull_request.merged = false; + assert.equal(extractVerifiedTask('pull_request', unmerged), null, 'closed without merging'); + assert.equal(extractVerifiedTask('pull_request', mergedPr({ action: 'opened' })), null); + assert.equal(extractVerifiedTask('issues', closedIssue({ action: 'opened' })), null); + const notPlanned = closedIssue(); + notPlanned.issue.state_reason = 'not_planned'; + assert.equal( + extractVerifiedTask('issues', notPlanned), + null, + 'not_planned is not completed work', + ); + assert.equal(extractVerifiedTask('push', { repository: { full_name: 'a/b' } }), null); + assert.equal(extractVerifiedTask('pull_request', {}), null); +}); + +test('store: records once per delivery/task and matches logins case-insensitively', async () => { + const db = new Database(':memory:'); + await runMigrations(db); + const store = createGithubTaskStore({ db, logger: {} }); + const task = extractVerifiedTask('pull_request', mergedPr()); + + assert.equal(store.record('d-1', task), true); + assert.equal(store.record('d-1', task), false, 'same delivery'); + assert.equal(store.record('d-2', task), false, 'same task, new delivery id'); + + assert.equal( + store.isVerified({ githubLogin: 'octo-dev', repo: 'ACME/widgets', number: 12 }), + true, + ); + assert.equal(store.isVerified({ githubLogin: 'octo-dev', number: 13 }), false); + assert.equal(store.isVerified({ githubLogin: 'someone-else' }), false); + assert.equal(store.list({ githubLogin: 'OCTO-DEV' }).length, 1); +}); + +async function makeApp(options = {}) { + return createApp({ + dbPath: ':memory:', + campaigns: [], + disableJobs: true, + skipEnvValidation: true, + rateLimit: { windowMs: 60_000, maxRequests: 10_000 }, + apiKeys: 'k', + masterKey: 'master', + githubWebhookSecret: SECRET, + ...options, + }); +} + +function deliver(app, event, payload, { id = `d-${Math.random()}`, secret = SECRET } = {}) { + const body = JSON.stringify(payload); + return request(app) + .post('/api/v1/webhooks/github') + .set('Content-Type', 'application/json') + .set('X-GitHub-Event', event) + .set('X-GitHub-Delivery', id) + .set('X-Hub-Signature-256', sign(body, secret)) + .send(body); +} + +test('webhook: verifies a merged PR, dedupes redeliveries and exposes the result to admins', async () => { + const app = await makeApp(); + + const first = await deliver(app, 'pull_request', mergedPr(), { id: 'delivery-1' }).expect(201); + assert.deepEqual( + { verified: first.body.verified, duplicate: first.body.duplicate, kind: first.body.kind }, + { verified: true, duplicate: false, kind: 'pr_merged' }, + ); + const again = await deliver(app, 'pull_request', mergedPr(), { id: 'delivery-1' }).expect(200); + assert.equal(again.body.duplicate, true); + + const verify = await request(app) + .get('/api/v1/admin/github-tasks/verify?githubLogin=octo-dev&repo=acme/widgets&number=12') + .set('X-API-Key', 'master') + .expect(200); + assert.equal(verify.body.verified, true); + + const list = await request(app) + .get('/api/v1/admin/github-tasks') + .set('X-API-Key', 'master') + .expect(200); + assert.equal(list.body.total, 1); +}); + +test('webhook: a closed issue is recorded; ignored events return 202', async () => { + const app = await makeApp(); + await deliver(app, 'issues', closedIssue()).expect(201); + const ignored = await deliver(app, 'push', { repository: { full_name: 'a/b' } }).expect(202); + assert.equal(ignored.body.ignored, true); + const unmerged = mergedPr(); + unmerged.pull_request.merged = false; + await deliver(app, 'pull_request', unmerged).expect(202); +}); + +test('webhook: rejects bad signatures, missing config, bad input; ping works', async () => { + const app = await makeApp(); + const bad = await deliver(app, 'pull_request', mergedPr(), { secret: 'wrong' }).expect(401); + assert.equal(bad.body.code, 'INVALID_SIGNATURE'); + + await request(app) + .post('/api/v1/webhooks/github') + .set('Content-Type', 'application/json') + .set('X-GitHub-Event', 'pull_request') + .send(JSON.stringify(mergedPr())) + .expect(401); + + const body = '{not json'; + await request(app) + .post('/api/v1/webhooks/github') + .set('Content-Type', 'application/json') + .set('X-GitHub-Event', 'issues') + .set('X-GitHub-Delivery', 'x') + .set('X-Hub-Signature-256', sign(body)) + .send(body) + .expect(400); + + await deliver(app, 'ping', { zen: 'hi' }).expect(200); + + const unconfigured = await makeApp({ githubWebhookSecret: '' }); + const res = await deliver(unconfigured, 'ping', {}).expect(503); + assert.equal(res.body.code, 'GITHUB_WEBHOOK_NOT_CONFIGURED'); +}); + +test('admin task lookup requires the master key and a githubLogin', async () => { + const app = await makeApp(); + await request(app).get('/api/v1/admin/github-tasks').set('X-API-Key', 'k').expect(401); + await request(app) + .get('/api/v1/admin/github-tasks/verify') + .set('X-API-Key', 'master') + .expect(400); +}); diff --git a/backend/src/services/ipfsPinService.js b/backend/src/services/ipfsPinService.js new file mode 100644 index 00000000..f04dc980 --- /dev/null +++ b/backend/src/services/ipfsPinService.js @@ -0,0 +1,306 @@ +// @ts-check +import { createHash } from 'node:crypto'; +import { isIP } from 'node:net'; + +/** + * Automated IPFS pinning for campaign assets (#1255). + * + * Pins a campaign's image, rules and badge JSON, then a metadata document that + * links them by `ipfs://` CID, through a pinning provider (Pinata). Pins are + * content-addressed and de-duplicated per campaign: re-pinning unchanged + * content returns the stored CID without a provider call. + */ + +const DEFAULT_MAX_IMAGE_BYTES = 5 * 1024 * 1024; + +/** Stable JSON: object keys sorted, so equal content always hashes equally. */ +export function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const keys = Object.keys(value).sort(); + return `{${keys + .filter((k) => value[k] !== undefined) + .map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +/** @param {string | Buffer} data */ +const sha256 = (data) => createHash('sha256').update(data).digest('hex'); + +/** + * Only public https image hosts may be fetched, so campaign data cannot point + * the server at internal services (SSRF). Redirects are refused separately. + * + * @param {string} rawUrl + * @returns {URL} + */ +export function assertPublicHttpsUrl(rawUrl) { + let url; + try { + url = new URL(rawUrl); + } catch { + throw new Error('imageUrl is not a valid URL'); + } + if (url.protocol !== 'https:') throw new Error('imageUrl must use https'); + const host = url.hostname.replace(/^\[|\]$/g, '').toLowerCase(); + const isPrivateV4 = /^(10\.|127\.|0\.|169\.254\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test( + host, + ); + const isPrivateV6 = isIP(host) === 6 && /^(::1?$|f[cd]|fe80)/.test(host); + if ( + host === 'localhost' || + host.endsWith('.localhost') || + host.endsWith('.internal') || + isPrivateV4 || + isPrivateV6 + ) { + throw new Error('imageUrl must point to a public host'); + } + return url; +} + +/** + * Pinata provider (https://docs.pinata.cloud). Authenticates with a JWT. + * + * @param {{ jwt: string, baseUrl?: string, fetchImpl?: typeof fetch }} options + */ +export function createPinataProvider({ + jwt, + baseUrl = 'https://api.pinata.cloud', + fetchImpl = fetch, +}) { + if (!jwt) throw new Error('PINATA_JWT is required'); + const base = baseUrl.replace(/\/$/, ''); + const headers = { Authorization: `Bearer ${jwt}` }; + + /** @param {Response} res */ + async function parse(res) { + if (!res.ok) { + const body = await res.text().catch(() => ''); + const err = /** @type {Error & { status?: number }} */ ( + new Error(`Pinata request failed (${res.status}): ${body.slice(0, 200)}`) + ); + err.status = res.status; + throw err; + } + const json = await res.json(); + if (!json?.IpfsHash) throw new Error('Pinata response missing IpfsHash'); + return { cid: String(json.IpfsHash), size: Number(json.PinSize ?? 0) }; + } + + return { + name: 'pinata', + /** @param {unknown} content @param {{ name: string }} meta */ + async pinJson(content, { name }) { + const res = await fetchImpl(`${base}/pinning/pinJSONToIPFS`, { + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + body: JSON.stringify({ pinataContent: content, pinataMetadata: { name } }), + }); + return parse(res); + }, + /** @param {Buffer} buffer @param {{ name: string, mimeType: string }} meta */ + async pinFile(buffer, { name, mimeType }) { + const form = new FormData(); + form.append('file', new Blob([buffer], { type: mimeType }), name); + form.append('pinataMetadata', JSON.stringify({ name })); + const res = await fetchImpl(`${base}/pinning/pinFileToIPFS`, { + method: 'POST', + headers, + body: form, + }); + return parse(res); + }, + }; +} + +/** + * @param {{ + * db: InstanceType, + * provider: ReturnType, + * fetchImpl?: typeof fetch, + * logger?: { info?: Function, warn?: Function }, + * maxImageBytes?: number, + * attempts?: number, + * retryBaseMs?: number, + * sleep?: (ms: number) => Promise, + * }} options + */ +export function createIpfsPinService({ + db, + provider, + fetchImpl = fetch, + logger = console, + maxImageBytes = DEFAULT_MAX_IMAGE_BYTES, + attempts = 3, + retryBaseMs = 250, + sleep = (ms) => new Promise((r) => setTimeout(r, ms)), +}) { + const findStmt = db.prepare( + 'SELECT * FROM ipfs_pins WHERE campaign_id = ? AND kind = ? AND content_hash = ?', + ); + const insertStmt = db.prepare(` + INSERT OR IGNORE INTO ipfs_pins + (campaign_id, kind, content_hash, cid, name, size_bytes, provider, created_at) + VALUES (@campaignId, @kind, @contentHash, @cid, @name, @size, @provider, @createdAt) + `); + + /** Retry transient provider failures (429, 5xx, network) with backoff. */ + async function withRetry(/** @type {() => Promise} */ fn) { + let lastError; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await fn(); + } catch (err) { + lastError = err; + const status = /** @type {any} */ (err)?.status; + const transient = status === undefined || status === 429 || status >= 500; + if (!transient || attempt === attempts) break; + await sleep(retryBaseMs * 2 ** (attempt - 1)); + } + } + throw lastError; + } + + /** @returns {ReturnType} */ + function store(campaignId, kind, contentHash, name, { cid, size }) { + insertStmt.run({ + campaignId: String(campaignId), + kind, + contentHash, + cid, + name, + size, + provider: provider.name, + createdAt: new Date().toISOString(), + }); + return rowToPin(findStmt.get(String(campaignId), kind, contentHash)); + } + + /** + * @param {string | number} campaignId + * @param {'metadata' | 'rules' | 'badge'} kind + * @param {unknown} content + */ + async function pinJsonDocument(campaignId, kind, content) { + const canonical = canonicalJson(content); + const contentHash = sha256(canonical); + const existing = findStmt.get(String(campaignId), kind, contentHash); + if (existing) return { ...rowToPin(existing), cached: true }; + + const name = `campaign-${campaignId}-${kind}.json`; + const result = await withRetry(() => provider.pinJson(JSON.parse(canonical), { name })); + logger.info?.({ campaignId, kind, cid: result.cid }, 'ipfs:pinned'); + return { ...store(campaignId, kind, contentHash, name, result), cached: false }; + } + + /** @param {string | number} campaignId @param {string} imageUrl */ + async function pinImage(campaignId, imageUrl) { + const url = assertPublicHttpsUrl(imageUrl); + const res = await fetchImpl(url, { redirect: 'error', headers: { Accept: 'image/*' } }); + if (!res.ok) throw new Error(`Image download failed (${res.status})`); + const mimeType = (res.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase(); + if (!mimeType.startsWith('image/')) + throw new Error(`Refusing to pin non-image content (${mimeType || 'unknown'})`); + const declared = Number(res.headers.get('content-length') ?? 0); + if (declared > maxImageBytes) throw new Error(`Image exceeds ${maxImageBytes} bytes`); + const buffer = Buffer.from(await res.arrayBuffer()); + if (buffer.length > maxImageBytes) throw new Error(`Image exceeds ${maxImageBytes} bytes`); + + const contentHash = sha256(buffer); + const existing = findStmt.get(String(campaignId), 'image', contentHash); + if (existing) return { ...rowToPin(existing), cached: true }; + + const ext = mimeType.split('/')[1]?.replace(/[^a-z0-9]/g, '') || 'img'; + const name = `campaign-${campaignId}-image.${ext}`; + const result = await withRetry(() => provider.pinFile(buffer, { name, mimeType })); + logger.info?.({ campaignId, kind: 'image', cid: result.cid }, 'ipfs:pinned'); + return { ...store(campaignId, 'image', contentHash, name, result), cached: false }; + } + + /** + * Pin everything a campaign publishes: image, rules, badge, then the metadata + * document linking them. An image failure is reported but does not stop the + * JSON documents from being pinned. + * + * @param {{ id: string | number, name: string, description?: string, imageUrl?: string | null, + * category?: string | null, tags?: string[], rewardPerAction?: number, startDate?: string | null, + * endDate?: string | null }} campaign + */ + async function pinCampaign(campaign) { + const id = campaign.id; + /** @type {Array>} */ + const pins = []; + /** @type {Array<{ kind: string, error: string }>} */ + const errors = []; + + let imageCid = null; + if (campaign.imageUrl) { + try { + const pin = await pinImage(id, campaign.imageUrl); + pins.push(pin); + imageCid = pin.cid; + } catch (err) { + errors.push({ kind: 'image', error: err instanceof Error ? err.message : String(err) }); + logger.warn?.({ campaignId: id, err }, 'ipfs:image_pin_failed'); + } + } + + const rules = await pinJsonDocument(id, 'rules', { + rewardPerAction: campaign.rewardPerAction ?? 0, + startDate: campaign.startDate ?? null, + endDate: campaign.endDate ?? null, + category: campaign.category ?? null, + }); + pins.push(rules); + + const badge = await pinJsonDocument(id, 'badge', { + name: `${campaign.name} participant`, + description: `Awarded for participating in ${campaign.name}`, + image: imageCid ? `ipfs://${imageCid}` : null, + attributes: [{ trait_type: 'campaign', value: String(id) }], + }); + pins.push(badge); + + const metadata = await pinJsonDocument(id, 'metadata', { + schema: 'trivela.campaign/1', + id: String(id), + name: campaign.name, + description: campaign.description ?? '', + category: campaign.category ?? null, + tags: campaign.tags ?? [], + image: imageCid ? `ipfs://${imageCid}` : null, + rules: `ipfs://${rules.cid}`, + badge: `ipfs://${badge.cid}`, + }); + pins.push(metadata); + + return { metadataCid: metadata.cid, metadataUri: `ipfs://${metadata.cid}`, pins, errors }; + } + + /** @param {string | number} campaignId */ + function list(campaignId) { + return db + .prepare('SELECT * FROM ipfs_pins WHERE campaign_id = ? ORDER BY id DESC') + .all(String(campaignId)) + .map(rowToPin); + } + + return { pinCampaign, pinImage, pinJsonDocument, list }; +} + +function rowToPin(row) { + return { + id: row.id, + campaignId: row.campaign_id, + kind: row.kind, + cid: row.cid, + uri: `ipfs://${row.cid}`, + name: row.name, + sizeBytes: row.size_bytes, + provider: row.provider, + createdAt: row.created_at, + }; +} diff --git a/backend/src/services/ipfsPinService.test.js b/backend/src/services/ipfsPinService.test.js new file mode 100644 index 00000000..a0394f51 --- /dev/null +++ b/backend/src/services/ipfsPinService.test.js @@ -0,0 +1,269 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import Database from 'better-sqlite3'; +import request from 'supertest'; +import { runMigrations } from '../db/migrate.js'; +import { createApp } from '../index.js'; +import { + assertPublicHttpsUrl, + canonicalJson, + createIpfsPinService, + createPinataProvider, +} from './ipfsPinService.js'; + +const silent = { info() {}, warn() {} }; + +function fakeProvider() { + const calls = []; + let n = 0; + return { + name: 'fake', + calls, + async pinJson(content, { name }) { + calls.push({ type: 'json', name, content }); + return { cid: `bafyjson${++n}`, size: JSON.stringify(content).length }; + }, + async pinFile(buffer, { name, mimeType }) { + calls.push({ type: 'file', name, mimeType, bytes: buffer.length }); + return { cid: `bafyfile${++n}`, size: buffer.length }; + }, + }; +} + +function imageFetch({ + status = 200, + type = 'image/png', + body = Buffer.from('PNGDATA'), + length, +} = {}) { + return async () => ({ + ok: status < 400, + status, + headers: { + get: (h) => ({ 'content-type': type, 'content-length': length })[h.toLowerCase()] ?? null, + }, + arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength), + }); +} + +async function makeService(over = {}) { + const db = new Database(':memory:'); + await runMigrations(db); + const provider = over.provider ?? fakeProvider(); + const service = createIpfsPinService({ + db, + provider, + fetchImpl: over.fetchImpl ?? imageFetch(), + logger: silent, + sleep: async () => {}, + ...over.options, + }); + return { db, provider, service }; +} + +const campaign = { + id: 7, + name: 'Spring Drop', + description: 'Win points', + imageUrl: 'https://cdn.example.com/a.png', + category: 'DeFi', + tags: ['x'], + rewardPerAction: 5, +}; + +test('canonicalJson is order-independent and skips undefined', () => { + assert.equal(canonicalJson({ b: 1, a: [2, { d: 1, c: undefined }] }), '{"a":[2,{"d":1}],"b":1}'); +}); + +test('assertPublicHttpsUrl blocks non-https and internal hosts', () => { + assert.equal(assertPublicHttpsUrl('https://cdn.example.com/a.png').hostname, 'cdn.example.com'); + for (const bad of [ + 'http://cdn.example.com/a.png', + 'https://localhost/a.png', + 'https://127.0.0.1/a.png', + 'https://10.0.0.5/a.png', + 'https://192.168.1.1/a.png', + 'https://172.20.0.1/a.png', + 'https://169.254.169.254/latest', + 'https://[::1]/a.png', + 'https://db.internal/a.png', + 'file:///etc/passwd', + 'not a url', + ]) { + assert.throws(() => assertPublicHttpsUrl(bad), undefined, bad); + } +}); + +test('pinCampaign pins image, rules, badge and a metadata document linking them', async () => { + const { service, provider } = await makeService(); + + const result = await service.pinCampaign(campaign); + + assert.deepEqual( + result.pins.map((p) => p.kind), + ['image', 'rules', 'badge', 'metadata'], + ); + assert.deepEqual(result.errors, []); + assert.equal(result.metadataUri, `ipfs://${result.metadataCid}`); + + const metadata = provider.calls.find((c) => c.name.endsWith('metadata.json')).content; + assert.equal(metadata.image, `ipfs://${result.pins[0].cid}`); + assert.equal(metadata.rules, `ipfs://${result.pins[1].cid}`); + assert.equal(metadata.badge, `ipfs://${result.pins[2].cid}`); + assert.equal(metadata.name, 'Spring Drop'); + + assert.equal(service.list(7).length, 4); +}); + +test('re-pinning unchanged content is served from the store, not the provider', async () => { + const { service, provider } = await makeService(); + await service.pinCampaign(campaign); + const callsAfterFirst = provider.calls.length; + + const again = await service.pinCampaign(campaign); + + assert.equal(provider.calls.length, callsAfterFirst); + assert.ok(again.pins.every((p) => p.cached)); + + // Changed content pins again. + await service.pinCampaign({ ...campaign, description: 'New copy' }); + assert.ok(provider.calls.length > callsAfterFirst); +}); + +test('an image failure is reported but the JSON documents still pin', async () => { + const { service } = await makeService({ fetchImpl: imageFetch({ type: 'text/html' }) }); + const result = await service.pinCampaign(campaign); + assert.equal(result.errors.length, 1); + assert.match(result.errors[0].error, /non-image/); + assert.deepEqual( + result.pins.map((p) => p.kind), + ['rules', 'badge', 'metadata'], + ); +}); + +test('oversized images are refused', async () => { + const { service } = await makeService({ + fetchImpl: imageFetch({ length: '999999999' }), + options: { maxImageBytes: 1024 }, + }); + await assert.rejects(() => service.pinImage(7, 'https://cdn.example.com/big.png'), /exceeds/); +}); + +test('transient provider errors are retried with backoff; permanent ones are not', async () => { + let calls = 0; + const flaky = { + name: 'flaky', + async pinJson() { + calls++; + if (calls < 3) throw Object.assign(new Error('rate limited'), { status: 429 }); + return { cid: 'bafyok', size: 1 }; + }, + async pinFile() { + return { cid: 'x', size: 1 }; + }, + }; + const { service } = await makeService({ provider: flaky }); + const pin = await service.pinJsonDocument(1, 'rules', { a: 1 }); + assert.equal(pin.cid, 'bafyok'); + assert.equal(calls, 3); + + let permanent = 0; + const rejecting = { + name: 'reject', + async pinJson() { + permanent++; + throw Object.assign(new Error('unauthorized'), { status: 401 }); + }, + async pinFile() {}, + }; + const { service: s2 } = await makeService({ provider: rejecting }); + await assert.rejects(() => s2.pinJsonDocument(1, 'rules', { a: 1 }), /unauthorized/); + assert.equal(permanent, 1); +}); + +test('createPinataProvider sends the JWT and maps the response', async () => { + const seen = []; + const provider = createPinataProvider({ + jwt: 'jwt-123', + baseUrl: 'https://pinata.test/', + fetchImpl: async (url, init) => { + seen.push({ url, init }); + return { ok: true, status: 200, json: async () => ({ IpfsHash: 'QmABC', PinSize: 42 }) }; + }, + }); + + const result = await provider.pinJson({ hello: 'world' }, { name: 'doc.json' }); + + assert.deepEqual(result, { cid: 'QmABC', size: 42 }); + assert.equal(seen[0].url, 'https://pinata.test/pinning/pinJSONToIPFS'); + assert.equal(seen[0].init.headers.Authorization, 'Bearer jwt-123'); + assert.deepEqual(JSON.parse(seen[0].init.body).pinataContent, { hello: 'world' }); + assert.throws(() => createPinataProvider({ jwt: '' }), /PINATA_JWT/); + + const failing = createPinataProvider({ + jwt: 'j', + fetchImpl: async () => ({ ok: false, status: 503, text: async () => 'down' }), + }); + await assert.rejects( + () => failing.pinJson({}, { name: 'x' }), + (e) => e.status === 503, + ); +}); + +test('routes: pin on demand, list pins, 503 when unconfigured, 404 for unknown campaigns', async () => { + const { db, provider } = await makeService(); + const app = await createApp({ + dbPath: ':memory:', + campaigns: [], + disableJobs: true, + skipEnvValidation: true, + rateLimit: { windowMs: 60_000, maxRequests: 10_000 }, + apiKeys: 'k', + ipfsPinService: createIpfsPinService({ + db, + provider, + fetchImpl: imageFetch(), + logger: silent, + sleep: async () => {}, + }), + }); + const created = await request(app) + .post('/api/v1/campaigns') + .set('X-API-Key', 'k') + .send({ name: 'Pin Me', rewardPerAction: 1 }) + .expect(201); + + const pinned = await request(app) + .post(`/api/v1/campaigns/${created.body.id}/ipfs-pins`) + .set('X-API-Key', 'k') + .expect(201); + assert.match(pinned.body.metadataUri, /^ipfs:\/\//); + const list = await request(app) + .get(`/api/v1/campaigns/${created.body.id}/ipfs-pins`) + .set('X-API-Key', 'k') + .expect(200); + assert.equal(list.body.configured, true); + assert.ok(list.body.total >= 3); + + await request(app).post('/api/v1/campaigns/nope/ipfs-pins').set('X-API-Key', 'k').expect(404); + await request(app).post(`/api/v1/campaigns/${created.body.id}/ipfs-pins`).expect(401); + + const bare = await createApp({ + dbPath: ':memory:', + campaigns: [], + disableJobs: true, + skipEnvValidation: true, + rateLimit: { windowMs: 60_000, maxRequests: 10_000 }, + apiKeys: 'k', + }); + const c2 = await request(bare) + .post('/api/v1/campaigns') + .set('X-API-Key', 'k') + .send({ name: 'No Pins', rewardPerAction: 1 }) + .expect(201); + const off = await request(bare) + .post(`/api/v1/campaigns/${c2.body.id}/ipfs-pins`) + .set('X-API-Key', 'k') + .expect(503); + assert.equal(off.body.code, 'IPFS_PIN_NOT_CONFIGURED'); +}); diff --git a/backend/src/websocket/index.js b/backend/src/websocket/index.js index 49199b9a..76fcd45c 100644 --- a/backend/src/websocket/index.js +++ b/backend/src/websocket/index.js @@ -16,6 +16,7 @@ function initializeWebSocket(httpServer, options = {}) { wsServerInstance = new WebSocketServer(httpServer, { path: options.path || '/ws', verifyClient: options.verifyClient, + leaderboardSnapshot: options.leaderboardSnapshot, }); return wsServerInstance; diff --git a/backend/src/websocket/leaderboardGateway.js b/backend/src/websocket/leaderboardGateway.js new file mode 100644 index 00000000..1c6f22c3 --- /dev/null +++ b/backend/src/websocket/leaderboardGateway.js @@ -0,0 +1,146 @@ +// @ts-check +import { maskWallet } from './server.js'; + +/** + * Real-time campaign leaderboard gateway (#1257). + * + * When points are awarded, the leaderboard is recomputed (bursts coalesced + * into one recompute), diffed against the last snapshot and — only if the + * ranking actually changed — pushed to every WebSocket client subscribed to + * that campaign's leaderboard: + * + * -> { "type": "subscribe", "channel": "leaderboard", "campaignId": "42" } + * <- { "type": "leaderboard_snapshot", "campaignId": "42", "entries": [...] } + * <- { "type": "leaderboard_update", "campaignId": "42", "entries": [...], "changes": [...] } + * + * Wallets are masked before they leave the server. + */ + +export const LEADERBOARD_CHANNEL = 'leaderboard'; + +/** @param {string | number} campaignId */ +export const leaderboardRoom = (campaignId) => `${LEADERBOARD_CHANNEL}:campaign:${campaignId}`; + +/** + * @typedef {{ wallet: string, rank: number, points: number }} Entry + * @typedef {{ wallet: string, rank: number, previousRank: number | null, + * points: number, previousPoints: number | null, + * change: 'new' | 'up' | 'down' | 'same' }} Change + */ + +/** + * Compare two ranked lists and describe what moved. Only entries whose rank or + * points differ from `previous` are returned. + * + * @param {Entry[]} previous + * @param {Entry[]} next + * @returns {Change[]} + */ +export function diffLeaderboards(previous, next) { + const before = new Map(previous.map((e) => [e.wallet, e])); + /** @type {Change[]} */ + const changes = []; + for (const entry of next) { + const old = before.get(entry.wallet); + if (!old) { + changes.push({ ...entry, previousRank: null, previousPoints: null, change: 'new' }); + } else if (old.rank !== entry.rank || old.points !== entry.points) { + changes.push({ + ...entry, + previousRank: old.rank, + previousPoints: old.points, + change: entry.rank < old.rank ? 'up' : entry.rank > old.rank ? 'down' : 'same', + }); + } + } + return changes; +} + +/** + * @param {{ + * getLeaderboard: (campaignId: string) => Entry[], + * publish: (campaignId: string, message: Record) => number, + * limit?: number, + * debounceMs?: number, + * logger?: { warn?: Function }, + * }} options + */ +export function createLeaderboardGateway({ + getLeaderboard, + publish, + limit = 10, + debounceMs = 200, + logger = console, +}) { + /** @type {Map} */ + const last = new Map(); + /** @type {Map>} */ + const timers = new Map(); + + const mask = (/** @type {Entry[]} */ entries) => + entries.map((e) => ({ rank: e.rank, wallet: maskWallet(e.wallet), points: e.points })); + + /** @param {string | number} campaignId */ + function top(campaignId) { + return getLeaderboard(String(campaignId)).slice(0, limit); + } + + /** Current leaderboard, e.g. to greet a new subscriber. */ + function snapshot(/** @type {string | number} */ campaignId) { + const entries = top(campaignId); + if (!last.has(String(campaignId))) last.set(String(campaignId), entries); + return { + type: 'leaderboard_snapshot', + campaignId: String(campaignId), + entries: mask(entries), + timestamp: new Date().toISOString(), + }; + } + + /** + * Recompute now and publish when the ranking changed. + * @returns {number} clients notified (0 when nothing changed) + */ + function flush(/** @type {string | number} */ campaignId) { + const id = String(campaignId); + timers.delete(id); + try { + const next = top(id); + const previous = last.get(id) ?? []; + const changes = diffLeaderboards(previous, next); + last.set(id, next); + if (changes.length === 0) return 0; + return publish(id, { + type: 'leaderboard_update', + campaignId: id, + entries: mask(next), + changes: changes.map((c) => ({ ...c, wallet: maskWallet(c.wallet) })), + timestamp: new Date().toISOString(), + }); + } catch (err) { + logger.warn?.({ err, campaignId: id }, 'leaderboard:publish_failed'); + return 0; + } + } + + /** + * Call after points were awarded in a campaign. Bursts within `debounceMs` + * produce a single recompute. + */ + function trigger(/** @type {string | number} */ campaignId) { + const id = String(campaignId); + if (debounceMs <= 0) return flush(id); + if (timers.has(id)) return 0; + const timer = setTimeout(() => flush(id), debounceMs); + timer.unref?.(); + timers.set(id, timer); + return 0; + } + + function stop() { + for (const timer of timers.values()) clearTimeout(timer); + timers.clear(); + } + + return { trigger, flush, snapshot, stop }; +} diff --git a/backend/src/websocket/leaderboardGateway.test.js b/backend/src/websocket/leaderboardGateway.test.js new file mode 100644 index 00000000..4db6154e --- /dev/null +++ b/backend/src/websocket/leaderboardGateway.test.js @@ -0,0 +1,165 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import WebSocket from 'ws'; +import WebSocketServer from './server.js'; +import { + createLeaderboardGateway, + diffLeaderboards, + leaderboardRoom, +} from './leaderboardGateway.js'; + +const W = (n) => + `G${String(n).padStart(3, '0')}AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`; +const entry = (wallet, rank, points) => ({ wallet, rank, points }); + +test('diffLeaderboards labels new, up, down and points-only changes; omits unchanged', () => { + const previous = [entry('a', 1, 5), entry('b', 2, 4), entry('c', 3, 3), entry('d', 4, 1)]; + const next = [entry('b', 1, 6), entry('a', 2, 5), entry('c', 3, 3), entry('e', 4, 1)]; + + const byWallet = Object.fromEntries(diffLeaderboards(previous, next).map((c) => [c.wallet, c])); + + assert.equal(byWallet.b.change, 'up'); + assert.deepEqual([byWallet.b.previousRank, byWallet.b.previousPoints], [2, 4]); + assert.equal(byWallet.a.change, 'down'); + assert.equal(byWallet.e.change, 'new'); + assert.equal(byWallet.e.previousRank, null); + assert.equal(byWallet.c, undefined, 'c is unchanged'); + assert.equal(byWallet.d, undefined, 'd dropped out; only current entries are reported'); +}); + +test('a points change at the same rank is reported as same', () => { + const [c] = diffLeaderboards([entry('a', 1, 5)], [entry('a', 1, 7)]); + assert.equal(c.change, 'same'); + assert.equal(c.previousPoints, 5); +}); + +function makeGateway(initial) { + let board = initial; + const published = []; + const gateway = createLeaderboardGateway({ + getLeaderboard: () => board, + publish: (campaignId, message) => { + published.push({ campaignId, message }); + return 1; + }, + debounceMs: 0, + logger: {}, + }); + return { gateway, published, set: (b) => (board = b) }; +} + +test('publishes only when the ranking changed, with masked wallets', () => { + const { gateway, published, set } = makeGateway([entry(W(1), 1, 3)]); + gateway.snapshot('9'); + + assert.equal(gateway.trigger('9'), 0, 'nothing changed since the snapshot'); + assert.equal(published.length, 0); + + set([entry(W(2), 1, 4), entry(W(1), 2, 3)]); + assert.equal(gateway.trigger('9'), 1); + + const { campaignId, message } = published[0]; + assert.equal(campaignId, '9'); + assert.equal(message.type, 'leaderboard_update'); + assert.ok( + message.entries.every((e) => e.wallet.includes('…')), + 'wallets are masked', + ); + assert.ok(!JSON.stringify(message).includes(W(2)), 'no full wallet leaves the server'); + assert.deepEqual(message.changes.map((c) => c.change).sort(), ['down', 'new']); + + assert.equal(gateway.trigger('9'), 0, 'a repeat trigger with no change is silent'); +}); + +test('bursts within the debounce window produce one publish', async () => { + let board = [entry('a', 1, 1)]; + const published = []; + const gateway = createLeaderboardGateway({ + getLeaderboard: () => board, + publish: (id, m) => (published.push(m), 1), + debounceMs: 20, + logger: {}, + }); + gateway.snapshot('1'); + board = [entry('a', 1, 2)]; + gateway.trigger('1'); + board = [entry('a', 1, 3)]; + gateway.trigger('1'); + gateway.trigger('1'); + + await new Promise((r) => setTimeout(r, 60)); + + assert.equal(published.length, 1); + assert.equal(published[0].entries[0].points, 3, 'the publish reflects the latest state'); + gateway.stop(); +}); + +test('honours the limit and survives a failing data source', () => { + const many = Array.from({ length: 30 }, (_, i) => entry(`w${i}`, i + 1, 100 - i)); + const { gateway } = makeGateway(many); + assert.equal(gateway.snapshot('1').entries.length, 10); + + const broken = createLeaderboardGateway({ + getLeaderboard: () => { + throw new Error('db gone'); + }, + publish: () => 1, + debounceMs: 0, + logger: {}, + }); + assert.equal(broken.trigger('1'), 0); +}); + +test('end to end: subscribers get a snapshot, then live updates when points are awarded', async () => { + let board = [entry(W(1), 1, 2)]; + /** @type {WebSocketServer} */ + let wsServer; + const gateway = createLeaderboardGateway({ + getLeaderboard: () => board, + publish: (id, message) => wsServer.broadcast(leaderboardRoom(id), message), + debounceMs: 0, + logger: {}, + }); + + const http = createServer(); + wsServer = new WebSocketServer(http, { + path: '/ws', + leaderboardSnapshot: (id) => gateway.snapshot(id), + }); + http.listen(0); + await once(http, 'listening'); + const { port } = http.address(); + + const client = new WebSocket(`ws://127.0.0.1:${port}/ws`); + const messages = []; + client.on('message', (m) => messages.push(JSON.parse(m.toString()))); + await once(client, 'open'); + const waitFor = async (type) => { + for (let i = 0; i < 100; i++) { + const found = messages.find((m) => m.type === type); + if (found) return found; + await new Promise((r) => setTimeout(r, 10)); + } + throw new Error(`no ${type} message; got ${messages.map((m) => m.type)}`); + }; + + client.send(JSON.stringify({ type: 'subscribe', channel: 'leaderboard', campaignId: '5' })); + const snapshot = await waitFor('leaderboard_snapshot'); + assert.equal(snapshot.campaignId, '5'); + assert.equal(snapshot.entries.length, 1); + + board = [entry(W(2), 1, 9), entry(W(1), 2, 2)]; + gateway.trigger('5'); + const update = await waitFor('leaderboard_update'); + assert.equal(update.entries[0].points, 9); + assert.equal(update.changes.find((c) => c.change === 'new').rank, 1); + + // A client subscribed to a different campaign hears nothing. + assert.equal(wsServer.broadcast(leaderboardRoom('other'), { type: 'x' }), 0); + + client.close(); + wsServer.close(); + http.close(); +}); diff --git a/backend/src/websocket/server.js b/backend/src/websocket/server.js index dbb7d21b..3e897418 100644 --- a/backend/src/websocket/server.js +++ b/backend/src/websocket/server.js @@ -1,4 +1,4 @@ -import WebSocket from 'ws'; +import WebSocket, { WebSocketServer as WsServer } from 'ws'; import { EventEmitter } from 'events'; import { log as logger } from '../middleware/logger.js'; @@ -18,12 +18,15 @@ class WebSocketServer extends EventEmitter { constructor(httpServer, options = {}) { super(); - this.wss = new WebSocket.Server({ + this.wss = new WsServer({ server: httpServer, path: options.path || '/ws', verifyClient: options.verifyClient, }); + /** @type {((campaignId: string) => object) | null} */ + this.leaderboardSnapshot = options.leaderboardSnapshot ?? null; + this.clients = new Map(); // clientId -> { ws, subscriptions, metadata } this.rooms = new Map(); // roomId -> Set of clientIds @@ -148,6 +151,15 @@ class WebSocketServer extends EventEmitter { channel: roomId, timestamp: new Date().toISOString(), }); + + // Greet leaderboard subscribers with the current standings (#1257). + if (channel === 'leaderboard' && campaignId && this.leaderboardSnapshot) { + try { + this.send(clientId, this.leaderboardSnapshot(String(campaignId))); + } catch (error) { + logger.error(`Failed to send leaderboard snapshot to ${clientId}:`, error); + } + } } handleUnsubscribe(clientId, message) { diff --git a/docs/BACKEND_PROBES_IPFS_GITHUB_LEADERBOARD.md b/docs/BACKEND_PROBES_IPFS_GITHUB_LEADERBOARD.md new file mode 100644 index 00000000..8843f0bc --- /dev/null +++ b/docs/BACKEND_PROBES_IPFS_GITHUB_LEADERBOARD.md @@ -0,0 +1,74 @@ +# Readiness probes, IPFS pinning, GitHub verification and live leaderboard + +## Health and readiness — `/healthz`, `/readyz` + +| Endpoint | Purpose | +| ---------- | ------------------------------------------------------------------------------ | +| `/livez` | Liveness. Never touches a dependency. | +| `/healthz` | Liveness plus `uptimeSeconds` and the cached RPC pool state. | +| `/readyz` | Readiness. Runs the dependency checks below, in parallel, each with a timeout. | + +`/readyz` checks the **database** (`SELECT 1`), **Redis** (`PING`, only when Redis is configured) +and the **Soroban RPC**. It returns `503` when the process is shutting down or a _required_ check +fails. The database and Redis are required. The RPC is optional by default: a failure reports +`"status": "degraded"` with HTTP 200, so an RPC blip does not evict every pod. Set +`READINESS_REQUIRE_RPC=true` to make it required. `READINESS_TIMEOUT_MS` (default 2000) bounds each +check. + +```json +{ + "status": "degraded", + "ready": true, + "uptimeSeconds": 812, + "checks": { + "database": { "status": "ok", "required": true, "latencyMs": 0 }, + "rpc": { + "status": "fail", + "required": false, + "latencyMs": 2001, + "error": "rpc check timed out after 2000ms" + } + } +} +``` + +## IPFS pinning + +With `PINATA_JWT` set, `POST /api/v1/campaigns/:id/ipfs-pins` (API key) pins the campaign's +**image**, **rules** JSON, **badge** JSON and a **metadata** document that links them by `ipfs://` +CID, and `GET` lists the stored pins. New campaigns are also pinned in the background through the +durable job queue. + +- Pins are content-addressed and de-duplicated per campaign; unchanged content is not re-pinned. +- Transient provider errors (network, 429, 5xx) are retried with backoff; auth errors are not. +- Images must be public `https` URLs (no private or internal hosts, no redirects), `image/*`, at + most 5 MiB. DNS-level rebinding is not covered; run the server with egress restrictions if that + matters. + +## GitHub webhook — developer bounty verification + +Point a GitHub webhook (content type `application/json`, events **Pull requests** and **Issues**) at +`POST /api/v1/webhooks/github` and set `GITHUB_WEBHOOK_SECRET`. The `X-Hub-Signature-256` header is +verified in constant time over the raw body. A task counts as verified only when GitHub reports a +pull request **merged**, or an issue **closed as completed** (`not_planned` is ignored). Each +delivery is recorded once, so GitHub retries are harmless. + +Before crediting a reward, look the task up with the master key: + +```bash +curl "$API/api/v1/admin/github-tasks/verify?githubLogin=octo-dev&repo=acme/widgets&number=12" \ + -H "X-API-Key: $MASTER_KEY" +``` + +## Live leaderboard over WebSocket + +Connect to `/ws` and subscribe: + +```json +{ "type": "subscribe", "channel": "leaderboard", "campaignId": "42" } +``` + +The server replies with `leaderboard_snapshot`, then sends `leaderboard_update` (top 10 with +`changes` marking each entry `new`, `up`, `down` or `same`) whenever awarded points change the +ranking. Bursts are coalesced (`LEADERBOARD_DEBOUNCE_MS`) and wallets are masked. Awards currently +trigger on referral credits, the campaign leaderboard the API already exposes.