feat(crawl): get notified by webhook when a crawl starts, completes, stops or fails

acceptance/crawl/surface/test.mjs

@@ -667,6 +667,32 @@ await liveCheck(
     };
   },
 );
+await liveCheck(
+  "24",
+  "Webhook",
+  "create({ webhook }) returns the crawl with that webhook: its url, events and metadata",
+  async () => {
+    const webhook = {
+      url: "https://example.com/zenrows-hook",
+      events: ["crawl.completed"],
+      metadata: { k: "v" },
+    };
+    const r = await errOf(create({ url: CRAWL_URL, depth: 1, maxPages: 1, webhook }));
+    if (r.error)
+      return { status: FAIL, actual: JSON.stringify(errSummary(r.error)), evidence: r.error.stack };
+    await client.stop(r.value.crawl_id);
+    const got = r.value.webhook;
+    const ok =
+      got?.url === webhook.url &&
+      JSON.stringify(got.events) === JSON.stringify(webhook.events) &&
+      JSON.stringify(got.metadata) === JSON.stringify(webhook.metadata);
+    return {
+      status: passIf(ok),
+      actual: `${r.value.crawl_id}: webhook ${JSON.stringify(got ?? null)}`,
+      evidence: JSON.stringify(r.value),
+    };
+  },
+);
 await check("15", "Bad key", "any call with the key `invalid` → 401", async () => {
   const bad = new ZenRowsCrawlClient("invalid", { baseURL: API_BASE, retries: 0 });
   const r = await errOf(bad.get("c_does_not_exist"));
@@ -761,6 +787,7 @@ H.naming({
     exclude_patterns: "excludePatterns",
     output_format: "outputFormat",
     discovery: "discovery",
+    webhook: "webhook",
     idempotency_key: "idempotencyKey",
     cursor: "cursor",
     limit: "limit",

acceptance/crawl/surface/typecheck.ts

@@ -5,6 +5,7 @@ import {
   type CrawlList,
   type CrawlResult,
   type CrawlStop,
+  type CrawlWebhook,
   type CrawlWithResults,
   type CreateCrawlParams,
   ZenRows,
@@ -27,11 +28,13 @@ export async function main(): Promise<void> {
     maxPages: 10,
     includePatterns: ["/p/"],
     outputFormat: "html",
+    webhook: { url: "https://example.com/hook", events: ["crawl.completed"], metadata: { k: "v" } },
   };
   const c: Crawl = await client.crawl.create(params, { idempotencyKey: "k" });
   // depth is optional: left out, the crawl has no depth limit and the response omits it.
   const unbounded: Crawl = await client.crawl.create({ url: "https://example.com/" });
   const depth: number | undefined = unbounded.depth;
+  const hook: CrawlWebhook | null | undefined = c.webhook;
   const w: Crawl = await standalone.wait(c.crawl_id, { timeout: 1 });
   const g: CrawlWithResults = await client.crawl.get(w.crawl_id, { limit: 2 });
   const next: string | null = g.next_cursor;
@@ -54,5 +57,5 @@ export async function main(): Promise<void> {
       console.log(code, status);
     }
   }
-  console.log(next, listNext, s.status, depth);
+  console.log(next, listNext, s.status, depth, hook?.events);
 }

src/crawl/client.ts

@@ -9,9 +9,12 @@ import type {
   CrawlOutputFormat,
   CrawlResult,
   CrawlStop,
+  CrawlWebhookEventType,
   CrawlWithResults,
 } from "./types.js";
 
+export * from "./webhook.js";
+
 const DEFAULT_CRAWL_API_URL = "https://api.zenrows.com/v1";
 const DEFAULT_TIMEOUT_SECONDS = 30;
 const DEFAULT_WAIT_TIMEOUT_SECONDS = 600;
@@ -38,6 +41,19 @@ export interface CrawlClientConfig {
   timeout?: number;
 }
 
+/** Where to send a crawl's events, and which ones. */
+export interface CrawlWebhookParams {
+  /** The absolute `https` URL each event is posted to, up to 2,048 characters. Its host must resolve. */
+  url: string;
+  /** The events to send, each named once, in any order. Omit it for all four. */
+  events?: CrawlWebhookEventType[];
+  /**
+   * Sent back with every event, to match it to your own records. Up to 20 keys, each 1 to 64
+   * characters; each value any JSON, at most 1,024 bytes as compact JSON.
+   */
+  metadata?: Record<string, unknown>;
+}
+
 export interface CreateCrawlParams {
   /** The page the crawl starts from: an absolute public `http`/`https` URL. */
   url: string;
@@ -64,6 +80,14 @@ export interface CreateCrawlParams {
    * links in the page's HTML, and is what a crawl runs with when you leave this out.
    */
   discovery?: CrawlDiscoveryStrategy[];
+  /**
+   * Get notified when the crawl starts, completes, stops or fails: each event is POSTed to
+   * `webhook.url` as a `CrawlWebhookEvent`, signed with the account's webhook signing key
+   * (create it in the dashboard). Check it with `verifyCrawlWebhookSignature()`. A failed
+   * delivery is retried for up to 3 days. The account needs a signing key, or the create throws a 422
+   * `webhook_signing_key_required`; a bad `url`, `events` or `metadata` is a 422 `invalid_webhook`.
+   */
+  webhook?: CrawlWebhookParams;
 }
 
 export interface WaitForCrawlOptions {
@@ -145,6 +169,7 @@ export class ZenRowsCrawlClient {
       exclude_patterns: params.excludePatterns,
       output_format: params.outputFormat,
       discovery: params.discovery,
+      webhook: params.webhook,
     };
     return this.transport.requestJson("POST", "/crawls", {
       body,

src/crawl/types.ts

@@ -29,6 +29,16 @@ export type CrawlContentStatus = "pending" | "fetched" | "failed";
 export type CrawlOutputFormat = "html" | "markdown";
 /** How a crawl finds the URLs it follows on each page it walks. */
 export type CrawlDiscoveryStrategy = "links";
+/**
+ * The events a crawl's webhook can be sent. `crawl.started`: the crawl was created and runs.
+ * `crawl.completed`, `crawl.stopped`, `crawl.failed`: the crawl reached that `status`; exactly
+ * one of these three is sent per crawl.
+ */
+export type CrawlWebhookEventType =
+  | "crawl.started"
+  | "crawl.completed"
+  | "crawl.stopped"
+  | "crawl.failed";
 
 export interface CrawlCoverage {
   pages_fetched: number;
@@ -46,6 +56,43 @@ export interface CrawlRunError {
   detail: string;
 }
 
+/**
+ * The webhook a crawl sends its events to, as the crawl runs with it. Fixed at create: there is
+ * no update.
+ */
+export interface CrawlWebhook {
+  url: string;
+  /** The events the crawl sends: the ones the create asked for, or all four when it left them out. */
+  events: Extensible<CrawlWebhookEventType>[];
+  /** Sent back with every event. Null or absent when the create set none. */
+  metadata?: Record<string, unknown> | null;
+}
+
+/**
+ * The JSON body of one crawl event (`schema: "zenrows.webhook/v1"`). Verify the raw body with
+ * `verifyCrawlWebhookSignature()` before parsing it. Every delivery of one event carries the same
+ * `event_id` (also sent as the `X-ZenRows-Event-Id` header): drop one you have already handled.
+ * Events can arrive out of order, so a short crawl's end can come before its `crawl.started`:
+ * order them by `occurred_at`, or read the crawl with `get()`.
+ */
+export interface CrawlWebhookEvent {
+  schema: Extensible<"zenrows.webhook/v1">;
+  /** E.g. `ev_01M5AKXYZ_completed`: the crawl's id without its `c_`, and the event's last part. */
+  event_id: string;
+  event_type: Extensible<CrawlWebhookEventType>;
+  /** When the crawl was created, for `crawl.started`; its `finished_at`, for the others. */
+  occurred_at: string;
+  resource: {
+    type: Extensible<"crawl">;
+    /** The crawl's `crawl_id`. */
+    id: string;
+    /** The crawl on the API. */
+    url?: string;
+  };
+  /** The webhook's `metadata`. Absent when the crawl has none. */
+  metadata?: Record<string, unknown>;
+}
+
 export interface Crawl {
   crawl_id: string;
   status: Extensible<CrawlStatus>;
@@ -66,6 +113,8 @@ export interface Crawl {
    * Absent when the server does not send the field.
    */
   discovery?: Extensible<CrawlDiscoveryStrategy>[];
+  /** Where the crawl sends its events. Absent (or null) on a crawl created without a webhook. */
+  webhook?: CrawlWebhook | null;
   duplicates_removed?: number;
   /** Absent while the crawl runs. */
   finished_at?: string;

src/crawl/webhook.ts

@@ -0,0 +1,25 @@
+import { type VerifyWebhookSignatureOptions, verifyWebhookSignature } from "../batch/webhook.js";
+
+/**
+ * Check a crawl event's `X-Signature` header, before you trust or parse its body.
+ *
+ * The header reads `t=<unix seconds>,v1=<hex HMAC-SHA256>,kid=<key id>`. The signature covers
+ * `t`, a literal `.`, then the raw request body, keyed with the account's webhook signing key
+ * (base64, from the dashboard); this helper decodes it to the key bytes.
+ *
+ * Pass the body exactly as received. `secrets` maps each key id to its base64 secret, so a
+ * receiver accepts both the old and the new key during a rotation; the header's `kid` picks the
+ * one to check. The comparison is constant-time.
+ *
+ * Returns `false` for a missing or malformed header, an unknown `kid`, a secret that isn't a
+ * 32-byte base64 key, a timestamp further than `toleranceSeconds` (default 300) from now, or a
+ * wrong signature. An event can arrive more than once: drop a repeat by its `event_id`.
+ */
+export function verifyCrawlWebhookSignature(
+  rawBody: string | Uint8Array,
+  signatureHeader: string | null | undefined,
+  secrets: Record<string, string>,
+  options: VerifyWebhookSignatureOptions = {},
+): boolean {
+  return verifyWebhookSignature(rawBody, signatureHeader, secrets, options);
+}

tests/crawl-webhook.test.ts

@@ -0,0 +1,156 @@
+import { createHmac } from "node:crypto";
+import { http, HttpResponse } from "msw";
+import { describe, expect, expectTypeOf, test } from "vitest";
+import {
+  type CrawlWebhookEvent,
+  type CrawlWebhookParams,
+  ZenRowsCrawlClient,
+  verifyCrawlWebhookSignature,
+} from "../src";
+import type { CrawlWithResults } from "../src/crawl/types";
+import { server } from "./_setup";
+
+const BASE = "https://api.zenrows.com/v1";
+
+function crawl(overrides: Partial<CrawlWithResults> = {}): CrawlWithResults {
+  return {
+    crawl_id: "c_01M5AKXYZ",
+    status: "running",
+    url: "https://example.com/",
+    max_items: 100,
+    max_pages: 100,
+    coverage: { pages_fetched: 0, pages_failed: 0, items_found: 0 },
+    created_at: "2026-10-27T10:40:00Z",
+    results: [],
+    next_cursor: "n",
+    ...overrides,
+  };
+}
+
+describe("ZenRowsCrawlClient — webhook", () => {
+  test("create sends the webhook as given and returns the crawl's webhook", async () => {
+    let body: unknown;
+    const webhook = {
+      url: "https://hooks.example.com/zenrows",
+      events: ["crawl.completed", "crawl.failed"],
+      metadata: { site: "books", n: 1 },
+    } satisfies CrawlWebhookParams;
+    server.use(
+      http.post(`${BASE}/crawls`, async ({ request }) => {
+        body = await request.json();
+        return HttpResponse.json(crawl({ webhook }), { status: 202 });
+      }),
+    );
+    const created = await new ZenRowsCrawlClient("API_KEY").create({
+      url: "https://example.com/",
+      webhook,
+    });
+    expect(body).toEqual({ url: "https://example.com/", webhook });
+    expect(created.webhook).toEqual(webhook);
+  });
+
+  test("a webhook with only a url sends only the url, and reads back all four events", async () => {
+    let body: unknown;
+    const all = ["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"];
+    server.use(
+      http.post(`${BASE}/crawls`, async ({ request }) => {
+        body = await request.json();
+        return HttpResponse.json(
+          crawl({ webhook: { url: "https://hooks.example.com/z", events: all, metadata: null } }),
+          { status: 202 },
+        );
+      }),
+    );
+    const created = await new ZenRowsCrawlClient("API_KEY").create({
+      url: "https://example.com/",
+      webhook: { url: "https://hooks.example.com/z" },
+    });
+    expect(body).toEqual({
+      url: "https://example.com/",
+      webhook: { url: "https://hooks.example.com/z" },
+    });
+    expect(created.webhook?.events).toEqual(all);
+    expect(created.webhook?.metadata).toBeNull();
+  });
+
+  test("a crawl without a webhook parses, on get and on list", async () => {
+    server.use(
+      http.get(`${BASE}/crawls/c_01M5AKXYZ`, () => HttpResponse.json(crawl())),
+      http.get(`${BASE}/crawls`, () => HttpResponse.json({ crawls: [crawl()] })),
+    );
+    const client = new ZenRowsCrawlClient("API_KEY");
+    expect((await client.get("c_01M5AKXYZ")).webhook).toBeUndefined();
+    expect((await client.list()).crawls[0]?.webhook).toBeUndefined();
+  });
+
+  test("a 422 webhook_signing_key_required surfaces as ZenRowsCrawlError with its code", async () => {
+    server.use(
+      http.post(`${BASE}/crawls`, () =>
+        HttpResponse.json(
+          {
+            code: "webhook_signing_key_required",
+            title: "Unprocessable Content",
+            detail: "Create a webhook signing key in the dashboard, then retry.",
+            status: 422,
+          },
+          { status: 422, headers: { "Content-Type": "application/problem+json" } },
+        ),
+      ),
+    );
+    await expect(
+      new ZenRowsCrawlClient("API_KEY", { retries: 0 }).create({
+        url: "https://example.com/",
+        webhook: { url: "https://hooks.example.com/z" },
+      }),
+    ).rejects.toMatchObject({
+      name: "ZenRowsCrawlError",
+      status: 422,
+      code: "webhook_signing_key_required",
+    });
+  });
+
+  test("the event type matches the contract's example", () => {
+    const event: CrawlWebhookEvent = {
+      schema: "zenrows.webhook/v1",
+      event_id: "ev_01M5AKXYZ_completed",
+      event_type: "crawl.completed",
+      occurred_at: "2026-10-27T10:42:10Z",
+      resource: { type: "crawl", id: "c_01M5AKXYZ", url: `${BASE}/crawls/c_01M5AKXYZ` },
+      metadata: { site: "golfclub" },
+    };
+    expect(event.resource.id).toBe("c_01M5AKXYZ");
+    expectTypeOf<CrawlWebhookParams["events"]>().toEqualTypeOf<
+      ("crawl.started" | "crawl.completed" | "crawl.stopped" | "crawl.failed")[] | undefined
+    >();
+  });
+});
+
+describe("verifyCrawlWebhookSignature", () => {
+  const raw = Buffer.alloc(32, 9);
+  const secrets = { k1: raw.toString("base64") };
+  const T = 1793097730;
+  const BODY =
+    '{"schema":"zenrows.webhook/v1","event_id":"ev_01M5AKXYZ_completed","event_type":"crawl.completed","occurred_at":"2026-10-27T10:42:10Z","resource":{"type":"crawl","id":"c_01M5AKXYZ","url":"https://api.zenrows.com/v1/crawls/c_01M5AKXYZ"},"metadata":{"site":"golfclub"}}';
+  const sig = createHmac("sha256", raw).update(`${T}.${BODY}`).digest("hex");
+
+  test("accepts a genuine crawl event and rejects a tampered one", () => {
+    const header = `t=${T},v1=${sig},kid=k1`;
+    expect(verifyCrawlWebhookSignature(BODY, header, secrets, { now: T })).toBe(true);
+    expect(verifyCrawlWebhookSignature(Buffer.from(BODY), header, secrets, { now: T })).toBe(true);
+    expect(
+      verifyCrawlWebhookSignature(BODY.replace("golfclub", "golfclub2"), header, secrets, {
+        now: T,
+      }),
+    ).toBe(false);
+  });
+
+  test("rejects an unknown kid, a stale timestamp and a missing header", () => {
+    expect(verifyCrawlWebhookSignature(BODY, `t=${T},v1=${sig},kid=k9`, secrets, { now: T })).toBe(
+      false,
+    );
+    expect(
+      verifyCrawlWebhookSignature(BODY, `t=${T},v1=${sig},kid=k1`, secrets, { now: T + 301 }),
+    ).toBe(false);
+    expect(verifyCrawlWebhookSignature(BODY, undefined, secrets, { now: T })).toBe(false);
+  });
+});