diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 9bc00d0..99987ce 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -667,6 +667,32 @@ await liveCheck( }; }, ); +await liveCheck( + "24", + "Webhook", + "create({ webhook }) returns the crawl with that webhook: its url, events and metadata", + async () => { + const webhook = { + url: "https://example.com/zenrows-hook", + events: ["crawl.completed"], + metadata: { k: "v" }, + }; + const r = await errOf(create({ url: CRAWL_URL, depth: 1, maxPages: 1, webhook })); + if (r.error) + return { status: FAIL, actual: JSON.stringify(errSummary(r.error)), evidence: r.error.stack }; + await client.stop(r.value.crawl_id); + const got = r.value.webhook; + const ok = + got?.url === webhook.url && + JSON.stringify(got.events) === JSON.stringify(webhook.events) && + JSON.stringify(got.metadata) === JSON.stringify(webhook.metadata); + return { + status: passIf(ok), + actual: `${r.value.crawl_id}: webhook ${JSON.stringify(got ?? null)}`, + evidence: JSON.stringify(r.value), + }; + }, +); await check("15", "Bad key", "any call with the key `invalid` → 401", async () => { const bad = new ZenRowsCrawlClient("invalid", { baseURL: API_BASE, retries: 0 }); const r = await errOf(bad.get("c_does_not_exist")); @@ -761,6 +787,7 @@ H.naming({ exclude_patterns: "excludePatterns", output_format: "outputFormat", discovery: "discovery", + webhook: "webhook", idempotency_key: "idempotencyKey", cursor: "cursor", limit: "limit", diff --git a/acceptance/crawl/surface/typecheck.ts b/acceptance/crawl/surface/typecheck.ts index 3e26655..e24dd34 100644 --- a/acceptance/crawl/surface/typecheck.ts +++ b/acceptance/crawl/surface/typecheck.ts @@ -5,6 +5,7 @@ import { type CrawlList, type CrawlResult, type CrawlStop, + type CrawlWebhook, type CrawlWithResults, type CreateCrawlParams, ZenRows, @@ -27,11 +28,13 @@ export async function main(): Promise { maxPages: 10, includePatterns: ["/p/"], outputFormat: "html", + webhook: { url: "https://example.com/hook", events: ["crawl.completed"], metadata: { k: "v" } }, }; const c: Crawl = await client.crawl.create(params, { idempotencyKey: "k" }); // depth is optional: left out, the crawl has no depth limit and the response omits it. const unbounded: Crawl = await client.crawl.create({ url: "https://example.com/" }); const depth: number | undefined = unbounded.depth; + const hook: CrawlWebhook | null | undefined = c.webhook; const w: Crawl = await standalone.wait(c.crawl_id, { timeout: 1 }); const g: CrawlWithResults = await client.crawl.get(w.crawl_id, { limit: 2 }); const next: string | null = g.next_cursor; @@ -54,5 +57,5 @@ export async function main(): Promise { console.log(code, status); } } - console.log(next, listNext, s.status, depth); + console.log(next, listNext, s.status, depth, hook?.events); } diff --git a/src/crawl/client.ts b/src/crawl/client.ts index 1ebeaf0..ad5f2bd 100644 --- a/src/crawl/client.ts +++ b/src/crawl/client.ts @@ -9,9 +9,12 @@ import type { CrawlOutputFormat, CrawlResult, CrawlStop, + CrawlWebhookEventType, CrawlWithResults, } from "./types.js"; +export * from "./webhook.js"; + const DEFAULT_CRAWL_API_URL = "https://api.zenrows.com/v1"; const DEFAULT_TIMEOUT_SECONDS = 30; const DEFAULT_WAIT_TIMEOUT_SECONDS = 600; @@ -38,6 +41,19 @@ export interface CrawlClientConfig { timeout?: number; } +/** Where to send a crawl's events, and which ones. */ +export interface CrawlWebhookParams { + /** The absolute `https` URL each event is posted to, up to 2,048 characters. Its host must resolve. */ + url: string; + /** The events to send, each named once, in any order. Omit it for all four. */ + events?: CrawlWebhookEventType[]; + /** + * Sent back with every event, to match it to your own records. Up to 20 keys, each 1 to 64 + * characters; each value any JSON, at most 1,024 bytes as compact JSON. + */ + metadata?: Record; +} + export interface CreateCrawlParams { /** The page the crawl starts from: an absolute public `http`/`https` URL. */ url: string; @@ -64,6 +80,14 @@ export interface CreateCrawlParams { * links in the page's HTML, and is what a crawl runs with when you leave this out. */ discovery?: CrawlDiscoveryStrategy[]; + /** + * Get notified when the crawl starts, completes, stops or fails: each event is POSTed to + * `webhook.url` as a `CrawlWebhookEvent`, signed with the account's webhook signing key + * (create it in the dashboard). Check it with `verifyCrawlWebhookSignature()`. A failed + * delivery is retried for up to 3 days. The account needs a signing key, or the create throws a 422 + * `webhook_signing_key_required`; a bad `url`, `events` or `metadata` is a 422 `invalid_webhook`. + */ + webhook?: CrawlWebhookParams; } export interface WaitForCrawlOptions { @@ -145,6 +169,7 @@ export class ZenRowsCrawlClient { exclude_patterns: params.excludePatterns, output_format: params.outputFormat, discovery: params.discovery, + webhook: params.webhook, }; return this.transport.requestJson("POST", "/crawls", { body, diff --git a/src/crawl/types.ts b/src/crawl/types.ts index ebc5c0a..7868121 100644 --- a/src/crawl/types.ts +++ b/src/crawl/types.ts @@ -29,6 +29,16 @@ export type CrawlContentStatus = "pending" | "fetched" | "failed"; export type CrawlOutputFormat = "html" | "markdown"; /** How a crawl finds the URLs it follows on each page it walks. */ export type CrawlDiscoveryStrategy = "links"; +/** + * The events a crawl's webhook can be sent. `crawl.started`: the crawl was created and runs. + * `crawl.completed`, `crawl.stopped`, `crawl.failed`: the crawl reached that `status`; exactly + * one of these three is sent per crawl. + */ +export type CrawlWebhookEventType = + | "crawl.started" + | "crawl.completed" + | "crawl.stopped" + | "crawl.failed"; export interface CrawlCoverage { pages_fetched: number; @@ -46,6 +56,43 @@ export interface CrawlRunError { detail: string; } +/** + * The webhook a crawl sends its events to, as the crawl runs with it. Fixed at create: there is + * no update. + */ +export interface CrawlWebhook { + url: string; + /** The events the crawl sends: the ones the create asked for, or all four when it left them out. */ + events: Extensible[]; + /** Sent back with every event. Null or absent when the create set none. */ + metadata?: Record | null; +} + +/** + * The JSON body of one crawl event (`schema: "zenrows.webhook/v1"`). Verify the raw body with + * `verifyCrawlWebhookSignature()` before parsing it. Every delivery of one event carries the same + * `event_id` (also sent as the `X-ZenRows-Event-Id` header): drop one you have already handled. + * Events can arrive out of order, so a short crawl's end can come before its `crawl.started`: + * order them by `occurred_at`, or read the crawl with `get()`. + */ +export interface CrawlWebhookEvent { + schema: Extensible<"zenrows.webhook/v1">; + /** E.g. `ev_01M5AKXYZ_completed`: the crawl's id without its `c_`, and the event's last part. */ + event_id: string; + event_type: Extensible; + /** When the crawl was created, for `crawl.started`; its `finished_at`, for the others. */ + occurred_at: string; + resource: { + type: Extensible<"crawl">; + /** The crawl's `crawl_id`. */ + id: string; + /** The crawl on the API. */ + url?: string; + }; + /** The webhook's `metadata`. Absent when the crawl has none. */ + metadata?: Record; +} + export interface Crawl { crawl_id: string; status: Extensible; @@ -66,6 +113,8 @@ export interface Crawl { * Absent when the server does not send the field. */ discovery?: Extensible[]; + /** Where the crawl sends its events. Absent (or null) on a crawl created without a webhook. */ + webhook?: CrawlWebhook | null; duplicates_removed?: number; /** Absent while the crawl runs. */ finished_at?: string; diff --git a/src/crawl/webhook.ts b/src/crawl/webhook.ts new file mode 100644 index 0000000..0fa1bc5 --- /dev/null +++ b/src/crawl/webhook.ts @@ -0,0 +1,25 @@ +import { type VerifyWebhookSignatureOptions, verifyWebhookSignature } from "../batch/webhook.js"; + +/** + * Check a crawl event's `X-Signature` header, before you trust or parse its body. + * + * The header reads `t=,v1=,kid=`. The signature covers + * `t`, a literal `.`, then the raw request body, keyed with the account's webhook signing key + * (base64, from the dashboard); this helper decodes it to the key bytes. + * + * Pass the body exactly as received. `secrets` maps each key id to its base64 secret, so a + * receiver accepts both the old and the new key during a rotation; the header's `kid` picks the + * one to check. The comparison is constant-time. + * + * Returns `false` for a missing or malformed header, an unknown `kid`, a secret that isn't a + * 32-byte base64 key, a timestamp further than `toleranceSeconds` (default 300) from now, or a + * wrong signature. An event can arrive more than once: drop a repeat by its `event_id`. + */ +export function verifyCrawlWebhookSignature( + rawBody: string | Uint8Array, + signatureHeader: string | null | undefined, + secrets: Record, + options: VerifyWebhookSignatureOptions = {}, +): boolean { + return verifyWebhookSignature(rawBody, signatureHeader, secrets, options); +} diff --git a/tests/crawl-webhook.test.ts b/tests/crawl-webhook.test.ts new file mode 100644 index 0000000..22fe29f --- /dev/null +++ b/tests/crawl-webhook.test.ts @@ -0,0 +1,156 @@ +import { createHmac } from "node:crypto"; +import { http, HttpResponse } from "msw"; +import { describe, expect, expectTypeOf, test } from "vitest"; +import { + type CrawlWebhookEvent, + type CrawlWebhookParams, + ZenRowsCrawlClient, + verifyCrawlWebhookSignature, +} from "../src"; +import type { CrawlWithResults } from "../src/crawl/types"; +import { server } from "./_setup"; + +const BASE = "https://api.zenrows.com/v1"; + +function crawl(overrides: Partial = {}): CrawlWithResults { + return { + crawl_id: "c_01M5AKXYZ", + status: "running", + url: "https://example.com/", + max_items: 100, + max_pages: 100, + coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 }, + created_at: "2026-10-27T10:40:00Z", + results: [], + next_cursor: "n", + ...overrides, + }; +} + +describe("ZenRowsCrawlClient — webhook", () => { + test("create sends the webhook as given and returns the crawl's webhook", async () => { + let body: unknown; + const webhook = { + url: "https://hooks.example.com/zenrows", + events: ["crawl.completed", "crawl.failed"], + metadata: { site: "books", n: 1 }, + } satisfies CrawlWebhookParams; + server.use( + http.post(`${BASE}/crawls`, async ({ request }) => { + body = await request.json(); + return HttpResponse.json(crawl({ webhook }), { status: 202 }); + }), + ); + const created = await new ZenRowsCrawlClient("API_KEY").create({ + url: "https://example.com/", + webhook, + }); + expect(body).toEqual({ url: "https://example.com/", webhook }); + expect(created.webhook).toEqual(webhook); + }); + + test("a webhook with only a url sends only the url, and reads back all four events", async () => { + let body: unknown; + const all = ["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"]; + server.use( + http.post(`${BASE}/crawls`, async ({ request }) => { + body = await request.json(); + return HttpResponse.json( + crawl({ webhook: { url: "https://hooks.example.com/z", events: all, metadata: null } }), + { status: 202 }, + ); + }), + ); + const created = await new ZenRowsCrawlClient("API_KEY").create({ + url: "https://example.com/", + webhook: { url: "https://hooks.example.com/z" }, + }); + expect(body).toEqual({ + url: "https://example.com/", + webhook: { url: "https://hooks.example.com/z" }, + }); + expect(created.webhook?.events).toEqual(all); + expect(created.webhook?.metadata).toBeNull(); + }); + + test("a crawl without a webhook parses, on get and on list", async () => { + server.use( + http.get(`${BASE}/crawls/c_01M5AKXYZ`, () => HttpResponse.json(crawl())), + http.get(`${BASE}/crawls`, () => HttpResponse.json({ crawls: [crawl()] })), + ); + const client = new ZenRowsCrawlClient("API_KEY"); + expect((await client.get("c_01M5AKXYZ")).webhook).toBeUndefined(); + expect((await client.list()).crawls[0]?.webhook).toBeUndefined(); + }); + + test("a 422 webhook_signing_key_required surfaces as ZenRowsCrawlError with its code", async () => { + server.use( + http.post(`${BASE}/crawls`, () => + HttpResponse.json( + { + code: "webhook_signing_key_required", + title: "Unprocessable Content", + detail: "Create a webhook signing key in the dashboard, then retry.", + status: 422, + }, + { status: 422, headers: { "Content-Type": "application/problem+json" } }, + ), + ), + ); + await expect( + new ZenRowsCrawlClient("API_KEY", { retries: 0 }).create({ + url: "https://example.com/", + webhook: { url: "https://hooks.example.com/z" }, + }), + ).rejects.toMatchObject({ + name: "ZenRowsCrawlError", + status: 422, + code: "webhook_signing_key_required", + }); + }); + + test("the event type matches the contract's example", () => { + const event: CrawlWebhookEvent = { + schema: "zenrows.webhook/v1", + event_id: "ev_01M5AKXYZ_completed", + event_type: "crawl.completed", + occurred_at: "2026-10-27T10:42:10Z", + resource: { type: "crawl", id: "c_01M5AKXYZ", url: `${BASE}/crawls/c_01M5AKXYZ` }, + metadata: { site: "golfclub" }, + }; + expect(event.resource.id).toBe("c_01M5AKXYZ"); + expectTypeOf().toEqualTypeOf< + ("crawl.started" | "crawl.completed" | "crawl.stopped" | "crawl.failed")[] | undefined + >(); + }); +}); + +describe("verifyCrawlWebhookSignature", () => { + const raw = Buffer.alloc(32, 9); + const secrets = { k1: raw.toString("base64") }; + const T = 1793097730; + const BODY = + '{"schema":"zenrows.webhook/v1","event_id":"ev_01M5AKXYZ_completed","event_type":"crawl.completed","occurred_at":"2026-10-27T10:42:10Z","resource":{"type":"crawl","id":"c_01M5AKXYZ","url":"https://api.zenrows.com/v1/crawls/c_01M5AKXYZ"},"metadata":{"site":"golfclub"}}'; + const sig = createHmac("sha256", raw).update(`${T}.${BODY}`).digest("hex"); + + test("accepts a genuine crawl event and rejects a tampered one", () => { + const header = `t=${T},v1=${sig},kid=k1`; + expect(verifyCrawlWebhookSignature(BODY, header, secrets, { now: T })).toBe(true); + expect(verifyCrawlWebhookSignature(Buffer.from(BODY), header, secrets, { now: T })).toBe(true); + expect( + verifyCrawlWebhookSignature(BODY.replace("golfclub", "golfclub2"), header, secrets, { + now: T, + }), + ).toBe(false); + }); + + test("rejects an unknown kid, a stale timestamp and a missing header", () => { + expect(verifyCrawlWebhookSignature(BODY, `t=${T},v1=${sig},kid=k9`, secrets, { now: T })).toBe( + false, + ); + expect( + verifyCrawlWebhookSignature(BODY, `t=${T},v1=${sig},kid=k1`, secrets, { now: T + 301 }), + ).toBe(false); + expect(verifyCrawlWebhookSignature(BODY, undefined, secrets, { now: T })).toBe(false); + }); +});