feat(crawl): get notified by webhook when a crawl starts, completes, stops or fails

acceptance/crawl/README.md

@@ -81,6 +81,7 @@ and signal checks.
 | 19 | No match | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0, with a `note` about the patterns) |
 | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish |
 | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) |
+| 25 | Webhook | a create with `--webhook`, one `--webhook-event` and one `--webhook-metadata` returns a crawl that carries that `webhook`; it is stopped at once |
 
 Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags.
 

acceptance/crawl/surface/test.mjs

@@ -892,6 +892,40 @@ await liveCheck(
   },
 );
 
+await liveCheck(
+  "25",
+  "Webhook",
+  "crawl create --webhook --webhook-event crawl.completed --webhook-metadata k=v: the crawl carries that webhook",
+  async () => {
+    const want = {
+      url: "https://example.com/zenrows-hook",
+      events: ["crawl.completed"],
+      metadata: { k: "v" },
+    };
+    const r = createWith([
+      CRAWL_URL,
+      "--depth",
+      "1",
+      "--max-pages",
+      "1",
+      "--webhook",
+      want.url,
+      "--webhook-event",
+      want.events[0],
+      "--webhook-metadata",
+      "k=v",
+    ]);
+    if (r.code !== 0 || !r.json?.crawl_id) return { status: FAIL, actual: H.trim(show(r), 400) };
+    z(["crawl", "stop", r.json.crawl_id, "--json"]);
+    const got = r.json.crawl?.webhook;
+    return {
+      status: passIf(JSON.stringify(got) === JSON.stringify(want)),
+      actual: `${r.json.crawl_id}: webhook ${JSON.stringify(got)}`,
+      evidence: show(r),
+    };
+  },
+);
+
 H.naming({
   operations: {
     create: "crawl create",

skills/crawl/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawl
-description: "Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs."
+description: "Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it or get notified by webhook, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs."
 version: 0.1.0
 requires_backend_capabilities: [crawl]
 ---
@@ -71,6 +71,36 @@ of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only
 `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download`
 reports the crawl's `status`, and `partial: true` while it runs.
 
+## Webhooks
+
+Instead of polling, have Crawl post its events to a URL you run:
+
+```
+zenrows crawl create <url> --depth 1 --webhook https://example.com/hooks/zenrows \
+  --webhook-event crawl.completed --webhook-event crawl.failed \
+  --webhook-metadata site=books --json
+```
+
+- `--webhook <url>` takes an absolute `https` URL. It is fixed at create.
+- `--webhook-event` repeats; each of `crawl.started`, `crawl.completed`,
+  `crawl.stopped`, `crawl.failed` at most once. Leave it out for all four.
+  Exactly one of the last three is sent per crawl.
+- `--webhook-metadata key=value` repeats (up to 20 keys of 1 to 64
+  characters, each value at most 1,024 bytes as JSON); every event carries it
+  back, to match the crawl to your own records.
+- The crawl shows `webhook` as `{url, events, metadata}`, `events` being the
+  effective list; a crawl without one has none.
+- Events are signed JSON (`schema: zenrows.webhook/v1`, `event_id`,
+  `event_type`, `occurred_at`, `resource {type: crawl, id, url}`, `metadata`).
+  Verify the `X-Signature: t=…,v1=…,kid=…` header: `v1` is the hex
+  HMAC-SHA256 of `"<t>." + body` with the account's webhook signing key
+  (Settings → API Keys). Events can repeat (drop one whose `event_id` you
+  handled) and arrive out of order (sort by `occurred_at`).
+- The account needs a webhook signing key: without one the create gets 422
+  `webhook_signing_key_required` (create the key, then retry). A bad URL,
+  event or metadata gets 422 `invalid_webhook`. Both are
+  `CRAWL_INVALID_REQUEST`, and nothing was created.
+
 ## While it runs, and after
 
 ```

src/cli/commands/crawl.ts

@@ -28,6 +28,10 @@ import {
   type CrawlResult,
   type CrawlStop,
   type CreateCrawlParams,
+  type CreateCrawlWebhook,
+  type CrawlWebhookEvent,
+  CRAWL_WEBHOOK_EVENTS,
+  WEBHOOK_METADATA_LIMITS,
 } from "../../core/crawl-api.ts";
 import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts";
 import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.ts";
@@ -60,6 +64,13 @@ export const crawl: Command = {
     "                                    fetch counted by --max-pages",
     "    --discovery links               how the crawl finds the URLs it follows; links (the",
     "                                    default) follows the links in each page's HTML",
+    "    --webhook <url>                 post the crawl's events to this https URL, signed with the",
+    "                                    account's webhook signing key (Settings → API Keys)",
+    "    --webhook-event <event>         with --webhook: send only this event (repeatable); one of",
+    "                                    crawl.started, crawl.completed, crawl.stopped, crawl.failed;",
+    "                                    leave it out for all four",
+    "    --webhook-metadata <key=value>  with --webhook: sent back with every event (repeatable,",
+    "                                    up to 20)",
     "    --follow                        wait until the crawl ends (Ctrl-C stops the wait, not the crawl)",
     "    --timeout <s>                   with --follow: stop waiting after s seconds (default 600)",
     "    --idempotency-key <key>         so a retried create starts no second crawl",
@@ -132,6 +143,9 @@ async function createCmd(rest: string[], ctx: RunContext): Promise<number> {
     "exclude-patterns": { type: "string", multiple: true },
     "output-format": { type: "string" },
     discovery: { type: "string", multiple: true },
+    webhook: { type: "string" },
+    "webhook-event": { type: "string", multiple: true },
+    "webhook-metadata": { type: "string", multiple: true },
     follow: { type: "boolean" },
     timeout: { type: "string" },
     "idempotency-key": { type: "string" },
@@ -186,6 +200,8 @@ async function createCmd(rest: string[], ctx: RunContext): Promise<number> {
     }
     params.discovery = ["links"];
   }
+  const webhook = webhookOf(values);
+  if (webhook) params.webhook = webhook;
   const timeoutMs = timeoutFlag(values.timeout);
   const idempotencyKey = asString(values["idempotency-key"]);
 
@@ -567,6 +583,11 @@ function printCrawl(
   if (cov) {
     log.info(`  ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`);
   }
+  const hook = (c as Crawl).webhook;
+  if (hook?.url) {
+    const meta = hook.metadata && Object.keys(hook.metadata).length ? ` · metadata: ${JSON.stringify(hook.metadata)}` : "";
+    log.info(`  webhook: ${hook.url} · events: ${(hook.events ?? []).join(", ") || "all"}${meta}`);
+  }
   if (extra.note) log.dim(`  next: zenrows crawl wait ${c.crawl_id}`);
   else if (note) log.warn(`  ${note}`);
   if (err) printError(err, false);
@@ -621,6 +642,77 @@ function pageLimit(v: unknown, max: number): number | undefined {
   return n;
 }
 
+/**
+ * `--webhook`, `--webhook-event` and `--webhook-metadata key=value` as the create's
+ * `webhook`, or undefined without `--webhook`. Checks the shapes and the API's limits
+ * before any request; whether the URL's host resolves is the API's to judge.
+ */
+function webhookOf(values: Record<string, unknown>): CreateCrawlWebhook | undefined {
+  const url = asString(values.webhook);
+  const events = patterns(values["webhook-event"]);
+  const pairs = patterns(values["webhook-metadata"]);
+  if (url === undefined) {
+    if (events.length || pairs.length) {
+      throw webhookUsage(
+        `${events.length ? "--webhook-event" : "--webhook-metadata"} needs --webhook.`,
+        "Pass --webhook <https-url> with it, or leave it out.",
+      );
+    }
+    return undefined;
+  }
+  let parsed: URL | undefined;
+  try {
+    parsed = new URL(url);
+  } catch {
+    parsed = undefined;
+  }
+  if (parsed?.protocol !== "https:") {
+    throw webhookUsage(`Invalid --webhook value '${url}': it must be an absolute https URL.`, "Pass e.g. --webhook https://example.com/hooks/zenrows.");
+  }
+  const webhook: CreateCrawlWebhook = { url };
+  if (events.length) {
+    const known = new Set<string>(CRAWL_WEBHOOK_EVENTS);
+    const unknown = events.find((e) => !known.has(e));
+    if (unknown !== undefined) {
+      throw webhookUsage(`Invalid --webhook-event value '${unknown}'.`, `Pass one of ${CRAWL_WEBHOOK_EVENTS.join(", ")}, or leave it out for all four.`);
+    }
+    const repeated = events.find((e, i) => events.indexOf(e) !== i);
+    if (repeated !== undefined) {
+      throw webhookUsage(`--webhook-event '${repeated}' is given twice.`, "Name each event once.");
+    }
+    webhook.events = events as CrawlWebhookEvent[];
+  }
+  if (pairs.length) {
+    const { keys, keyLength, valueBytes } = WEBHOOK_METADATA_LIMITS;
+    const metadata: Record<string, string> = {};
+    for (const p of pairs) {
+      const eq = p.indexOf("=");
+      if (eq <= 0) throw webhookUsage(`Invalid --webhook-metadata '${p}': use key=value.`, "Pass e.g. --webhook-metadata site=books.");
+      const key = p.slice(0, eq);
+      const value = p.slice(eq + 1);
+      if ([...key].length > keyLength) throw webhookUsage(`--webhook-metadata key '${key}' is longer than ${keyLength} characters.`, "Use a shorter key.");
+      if (key in metadata) throw webhookUsage(`--webhook-metadata key '${key}' is given twice.`, "Name each key once.");
+      if (Buffer.byteLength(JSON.stringify(value)) > valueBytes) {
+        throw webhookUsage(`--webhook-metadata '${key}' is longer than ${valueBytes} bytes as JSON.`, "Use a shorter value.");
+      }
+      metadata[key] = value;
+    }
+    if (Object.keys(metadata).length > keys) throw webhookUsage(`--webhook-metadata has more than ${keys} keys.`, `Pass at most ${keys}.`);
+    webhook.metadata = metadata;
+  }
+  return webhook;
+}
+
+function webhookUsage(message: string, next_action: string): ToolkitError {
+  return new ToolkitError({
+    code: "INVALID_USAGE",
+    message,
+    likely_cause: "The webhook flags do not match what Crawl accepts.",
+    next_action,
+    suggested_commands: ["zenrows crawl --help"],
+  });
+}
+
 function patterns(v: unknown): string[] {
   const list = Array.isArray(v) ? v : typeof v === "string" ? [v] : [];
   return list.filter((p): p is string => typeof p === "string" && p.length > 0);

src/core/crawl-api.ts

@@ -10,8 +10,9 @@
  * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable
  * for tests.
  *
- * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do
- * not model pass through untouched.
+ * `output_format` takes `html` or `markdown` (absent = URLs only). A `webhook` gets the
+ * crawl's events (`crawl.started` and how it ended) posted to a URL, signed with the
+ * account's webhook signing key. Response fields we do not model pass through untouched.
  */
 import { Readable } from "node:stream";
 import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.ts";
@@ -71,6 +72,16 @@ export interface Crawl {
   discovery?: string[];
   duplicates_removed?: number;
   finished_at?: string;
+  /** Where the crawl's events go. Absent or null on a crawl created without one. */
+  webhook?: CrawlWebhook | null;
+  [k: string]: unknown;
+}
+
+/** A crawl's webhook, as the API shows it: `events` is always the effective list. */
+export interface CrawlWebhook {
+  url: string;
+  events: CrawlWebhookEvent[];
+  metadata?: Record<string, unknown> | null;
   [k: string]: unknown;
 }
 
@@ -120,6 +131,18 @@ export interface CreateCrawlParams {
   output_format?: CrawlOutputFormat;
   /** How the crawl finds the URLs it follows on each page it walks. */
   discovery?: CrawlDiscoveryStrategy[];
+  /** Post the crawl's events to this URL. Fixed at create. */
+  webhook?: CreateCrawlWebhook;
+}
+
+/** The `webhook` of a create. Only the fields set are sent. */
+export interface CreateCrawlWebhook {
+  /** Absolute `https` URL each event is posted to. */
+  url: string;
+  /** The events to send, each once. Absent = all four. */
+  events?: CrawlWebhookEvent[];
+  /** Sent back with every event: up to 20 keys of 1 to 64 characters, each value at most 1,024 bytes as JSON. */
+  metadata?: Record<string, unknown>;
 }
 
 /** RFC 9457 problem body. */
@@ -138,6 +161,11 @@ export const CRAWL_NOT_ENABLED_CODE = "REQS008";
 export type CrawlOutputFormat = "html" | "markdown";
 /** How a crawl finds the URLs it follows on each page it walks. */
 export type CrawlDiscoveryStrategy = "links";
+/** The events a crawl's webhook can send. */
+export const CRAWL_WEBHOOK_EVENTS = ["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"] as const;
+export type CrawlWebhookEvent = (typeof CRAWL_WEBHOOK_EVENTS)[number];
+/** The API's limits on a webhook's `metadata`. */
+export const WEBHOOK_METADATA_LIMITS = { keys: 20, keyLength: 64, valueBytes: 1024 } as const;
 
 
 /** A crawl stops progressing once its status is anything but `running`. */
@@ -427,7 +455,9 @@ function classifyProblem(
       next_action:
         serverCode === "idempotency_key_reused"
           ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is."
-          : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.",
+          : serverCode === "webhook_signing_key_required"
+            ? "Nothing was created. Create a webhook signing key in the dashboard (Settings → API Keys), then retry."
+            : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.",
     });
   }
   return new ToolkitError({
@@ -449,6 +479,12 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo
   if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns;
   if (params.output_format !== undefined) body.output_format = params.output_format;
   if (params.discovery?.length) body.discovery = params.discovery;
+  if (params.webhook !== undefined) {
+    const webhook: Record<string, unknown> = { url: params.webhook.url };
+    if (params.webhook.events?.length) webhook.events = params.webhook.events;
+    if (params.webhook.metadata !== undefined) webhook.metadata = params.webhook.metadata;
+    body.webhook = webhook;
+  }
   return crawlJson<Crawl>("POST", "/crawls", {
     ...opts,
     body,

tests/crawl-api.test.ts

@@ -284,6 +284,36 @@ test("422 idempotency_key_reused says to send a new key, not to retry as is", as
   );
 });
 
+test("createCrawl sends the webhook with only the fields set", async () => {
+  const { impl, calls } = stubFetch(202, crawlBody);
+  await createCrawl({ url: "https://example.com/products/", webhook: { url: "https://example.com/hook" } }, { apiKey: "k", fetchImpl: impl });
+  assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)).webhook, { url: "https://example.com/hook" });
+  await createCrawl(
+    {
+      url: "https://example.com/products/",
+      webhook: { url: "https://example.com/hook", events: ["crawl.completed"], metadata: { k: "v" } },
+    },
+    { apiKey: "k", fetchImpl: impl },
+  );
+  assert.deepEqual(JSON.parse(String(calls[1]!.init?.body)).webhook, {
+    url: "https://example.com/hook",
+    events: ["crawl.completed"],
+    metadata: { k: "v" },
+  });
+});
+
+test("422 webhook_signing_key_required says to create the signing key", async () => {
+  const { impl } = problem(422, "webhook_signing_key_required");
+  await assert.rejects(
+    () => createCrawl({ url: "https://example.com/products/", webhook: { url: "https://example.com/hook" } }, { apiKey: "k", fetchImpl: impl }),
+    (e: unknown) =>
+      e instanceof ToolkitError &&
+      e.code === "CRAWL_INVALID_REQUEST" &&
+      e.server_code === "webhook_signing_key_required" &&
+      /signing key/.test(e.next_action),
+  );
+});
+
 test("409 maps to CRAWL_REQUEST_IN_FLIGHT", async () => {
   const { impl } = problem(409, "idempotency_request_in_flight");
   await assert.rejects(