From 4938ef2378ef9184801223df4ecb998b4d4e0ce7 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 30 Oct 2026 09:47:13 +0000 Subject: [PATCH] feat(crawl): get notified by webhook when a crawl starts, completes, stops or fails Crawls are long-running, and polling them until they end is the only way the CLI lets a caller learn how one finished. The Crawl API now posts signed events to a URL given at create, so the CLI can hand that URL, the events to send and the metadata to echo back to the API, and shows the webhook a crawl carries. --- acceptance/crawl/README.md | 1 + acceptance/crawl/surface/test.mjs | 34 ++++++++++++ skills/crawl/SKILL.md | 32 ++++++++++- src/cli/commands/crawl.ts | 92 +++++++++++++++++++++++++++++++ src/core/crawl-api.ts | 42 +++++++++++++- tests/crawl-api.test.ts | 30 ++++++++++ tests/crawl-command.test.ts | 48 ++++++++++++++++ 7 files changed, 275 insertions(+), 4 deletions(-) diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index c79f89f..2e8776c 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -81,6 +81,7 @@ and signal checks. | 19 | No match | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0, with a `note` about the patterns) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | +| 25 | Webhook | a create with `--webhook`, one `--webhook-event` and one `--webhook-metadata` returns a crawl that carries that `webhook`; it is stopped at once | Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags. diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index ba18a63..2cf5337 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -892,6 +892,40 @@ await liveCheck( }, ); +await liveCheck( + "25", + "Webhook", + "crawl create --webhook --webhook-event crawl.completed --webhook-metadata k=v: the crawl carries that webhook", + async () => { + const want = { + url: "https://example.com/zenrows-hook", + events: ["crawl.completed"], + metadata: { k: "v" }, + }; + const r = createWith([ + CRAWL_URL, + "--depth", + "1", + "--max-pages", + "1", + "--webhook", + want.url, + "--webhook-event", + want.events[0], + "--webhook-metadata", + "k=v", + ]); + if (r.code !== 0 || !r.json?.crawl_id) return { status: FAIL, actual: H.trim(show(r), 400) }; + z(["crawl", "stop", r.json.crawl_id, "--json"]); + const got = r.json.crawl?.webhook; + return { + status: passIf(JSON.stringify(got) === JSON.stringify(want)), + actual: `${r.json.crawl_id}: webhook ${JSON.stringify(got)}`, + evidence: show(r), + }; + }, +); + H.naming({ operations: { create: "crawl create", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index a2e741c..01b2572 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -1,6 +1,6 @@ --- name: crawl -description: "Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs." +description: "Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it or get notified by webhook, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs." version: 0.1.0 requires_backend_capabilities: [crawl] --- @@ -71,6 +71,36 @@ of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download` reports the crawl's `status`, and `partial: true` while it runs. +## Webhooks + +Instead of polling, have Crawl post its events to a URL you run: + +``` +zenrows crawl create --depth 1 --webhook https://example.com/hooks/zenrows \ + --webhook-event crawl.completed --webhook-event crawl.failed \ + --webhook-metadata site=books --json +``` + +- `--webhook ` takes an absolute `https` URL. It is fixed at create. +- `--webhook-event` repeats; each of `crawl.started`, `crawl.completed`, + `crawl.stopped`, `crawl.failed` at most once. Leave it out for all four. + Exactly one of the last three is sent per crawl. +- `--webhook-metadata key=value` repeats (up to 20 keys of 1 to 64 + characters, each value at most 1,024 bytes as JSON); every event carries it + back, to match the crawl to your own records. +- The crawl shows `webhook` as `{url, events, metadata}`, `events` being the + effective list; a crawl without one has none. +- Events are signed JSON (`schema: zenrows.webhook/v1`, `event_id`, + `event_type`, `occurred_at`, `resource {type: crawl, id, url}`, `metadata`). + Verify the `X-Signature: t=…,v1=…,kid=…` header: `v1` is the hex + HMAC-SHA256 of `"." + body` with the account's webhook signing key + (Settings → API Keys). Events can repeat (drop one whose `event_id` you + handled) and arrive out of order (sort by `occurred_at`). +- The account needs a webhook signing key: without one the create gets 422 + `webhook_signing_key_required` (create the key, then retry). A bad URL, + event or metadata gets 422 `invalid_webhook`. Both are + `CRAWL_INVALID_REQUEST`, and nothing was created. + ## While it runs, and after ``` diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 75a88c6..e5df484 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -28,6 +28,10 @@ import { type CrawlResult, type CrawlStop, type CreateCrawlParams, + type CreateCrawlWebhook, + type CrawlWebhookEvent, + CRAWL_WEBHOOK_EVENTS, + WEBHOOK_METADATA_LIMITS, } from "../../core/crawl-api.ts"; import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.ts"; @@ -60,6 +64,13 @@ export const crawl: Command = { " fetch counted by --max-pages", " --discovery links how the crawl finds the URLs it follows; links (the", " default) follows the links in each page's HTML", + " --webhook post the crawl's events to this https URL, signed with the", + " account's webhook signing key (Settings → API Keys)", + " --webhook-event with --webhook: send only this event (repeatable); one of", + " crawl.started, crawl.completed, crawl.stopped, crawl.failed;", + " leave it out for all four", + " --webhook-metadata with --webhook: sent back with every event (repeatable,", + " up to 20)", " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", " --timeout with --follow: stop waiting after s seconds (default 600)", " --idempotency-key so a retried create starts no second crawl", @@ -132,6 +143,9 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { "exclude-patterns": { type: "string", multiple: true }, "output-format": { type: "string" }, discovery: { type: "string", multiple: true }, + webhook: { type: "string" }, + "webhook-event": { type: "string", multiple: true }, + "webhook-metadata": { type: "string", multiple: true }, follow: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, @@ -186,6 +200,8 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { } params.discovery = ["links"]; } + const webhook = webhookOf(values); + if (webhook) params.webhook = webhook; const timeoutMs = timeoutFlag(values.timeout); const idempotencyKey = asString(values["idempotency-key"]); @@ -567,6 +583,11 @@ function printCrawl( if (cov) { log.info(` ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`); } + const hook = (c as Crawl).webhook; + if (hook?.url) { + const meta = hook.metadata && Object.keys(hook.metadata).length ? ` · metadata: ${JSON.stringify(hook.metadata)}` : ""; + log.info(` webhook: ${hook.url} · events: ${(hook.events ?? []).join(", ") || "all"}${meta}`); + } if (extra.note) log.dim(` next: zenrows crawl wait ${c.crawl_id}`); else if (note) log.warn(` ${note}`); if (err) printError(err, false); @@ -621,6 +642,77 @@ function pageLimit(v: unknown, max: number): number | undefined { return n; } +/** + * `--webhook`, `--webhook-event` and `--webhook-metadata key=value` as the create's + * `webhook`, or undefined without `--webhook`. Checks the shapes and the API's limits + * before any request; whether the URL's host resolves is the API's to judge. + */ +function webhookOf(values: Record): CreateCrawlWebhook | undefined { + const url = asString(values.webhook); + const events = patterns(values["webhook-event"]); + const pairs = patterns(values["webhook-metadata"]); + if (url === undefined) { + if (events.length || pairs.length) { + throw webhookUsage( + `${events.length ? "--webhook-event" : "--webhook-metadata"} needs --webhook.`, + "Pass --webhook with it, or leave it out.", + ); + } + return undefined; + } + let parsed: URL | undefined; + try { + parsed = new URL(url); + } catch { + parsed = undefined; + } + if (parsed?.protocol !== "https:") { + throw webhookUsage(`Invalid --webhook value '${url}': it must be an absolute https URL.`, "Pass e.g. --webhook https://example.com/hooks/zenrows."); + } + const webhook: CreateCrawlWebhook = { url }; + if (events.length) { + const known = new Set(CRAWL_WEBHOOK_EVENTS); + const unknown = events.find((e) => !known.has(e)); + if (unknown !== undefined) { + throw webhookUsage(`Invalid --webhook-event value '${unknown}'.`, `Pass one of ${CRAWL_WEBHOOK_EVENTS.join(", ")}, or leave it out for all four.`); + } + const repeated = events.find((e, i) => events.indexOf(e) !== i); + if (repeated !== undefined) { + throw webhookUsage(`--webhook-event '${repeated}' is given twice.`, "Name each event once."); + } + webhook.events = events as CrawlWebhookEvent[]; + } + if (pairs.length) { + const { keys, keyLength, valueBytes } = WEBHOOK_METADATA_LIMITS; + const metadata: Record = {}; + for (const p of pairs) { + const eq = p.indexOf("="); + if (eq <= 0) throw webhookUsage(`Invalid --webhook-metadata '${p}': use key=value.`, "Pass e.g. --webhook-metadata site=books."); + const key = p.slice(0, eq); + const value = p.slice(eq + 1); + if ([...key].length > keyLength) throw webhookUsage(`--webhook-metadata key '${key}' is longer than ${keyLength} characters.`, "Use a shorter key."); + if (key in metadata) throw webhookUsage(`--webhook-metadata key '${key}' is given twice.`, "Name each key once."); + if (Buffer.byteLength(JSON.stringify(value)) > valueBytes) { + throw webhookUsage(`--webhook-metadata '${key}' is longer than ${valueBytes} bytes as JSON.`, "Use a shorter value."); + } + metadata[key] = value; + } + if (Object.keys(metadata).length > keys) throw webhookUsage(`--webhook-metadata has more than ${keys} keys.`, `Pass at most ${keys}.`); + webhook.metadata = metadata; + } + return webhook; +} + +function webhookUsage(message: string, next_action: string): ToolkitError { + return new ToolkitError({ + code: "INVALID_USAGE", + message, + likely_cause: "The webhook flags do not match what Crawl accepts.", + next_action, + suggested_commands: ["zenrows crawl --help"], + }); +} + function patterns(v: unknown): string[] { const list = Array.isArray(v) ? v : typeof v === "string" ? [v] : []; return list.filter((p): p is string => typeof p === "string" && p.length > 0); diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 91d6e43..98e214c 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -10,8 +10,9 @@ * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable * for tests. * - * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do - * not model pass through untouched. + * `output_format` takes `html` or `markdown` (absent = URLs only). A `webhook` gets the + * crawl's events (`crawl.started` and how it ended) posted to a URL, signed with the + * account's webhook signing key. Response fields we do not model pass through untouched. */ import { Readable } from "node:stream"; import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.ts"; @@ -71,6 +72,16 @@ export interface Crawl { discovery?: string[]; duplicates_removed?: number; finished_at?: string; + /** Where the crawl's events go. Absent or null on a crawl created without one. */ + webhook?: CrawlWebhook | null; + [k: string]: unknown; +} + +/** A crawl's webhook, as the API shows it: `events` is always the effective list. */ +export interface CrawlWebhook { + url: string; + events: CrawlWebhookEvent[]; + metadata?: Record | null; [k: string]: unknown; } @@ -120,6 +131,18 @@ export interface CreateCrawlParams { output_format?: CrawlOutputFormat; /** How the crawl finds the URLs it follows on each page it walks. */ discovery?: CrawlDiscoveryStrategy[]; + /** Post the crawl's events to this URL. Fixed at create. */ + webhook?: CreateCrawlWebhook; +} + +/** The `webhook` of a create. Only the fields set are sent. */ +export interface CreateCrawlWebhook { + /** Absolute `https` URL each event is posted to. */ + url: string; + /** The events to send, each once. Absent = all four. */ + events?: CrawlWebhookEvent[]; + /** Sent back with every event: up to 20 keys of 1 to 64 characters, each value at most 1,024 bytes as JSON. */ + metadata?: Record; } /** RFC 9457 problem body. */ @@ -138,6 +161,11 @@ export const CRAWL_NOT_ENABLED_CODE = "REQS008"; export type CrawlOutputFormat = "html" | "markdown"; /** How a crawl finds the URLs it follows on each page it walks. */ export type CrawlDiscoveryStrategy = "links"; +/** The events a crawl's webhook can send. */ +export const CRAWL_WEBHOOK_EVENTS = ["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"] as const; +export type CrawlWebhookEvent = (typeof CRAWL_WEBHOOK_EVENTS)[number]; +/** The API's limits on a webhook's `metadata`. */ +export const WEBHOOK_METADATA_LIMITS = { keys: 20, keyLength: 64, valueBytes: 1024 } as const; /** A crawl stops progressing once its status is anything but `running`. */ @@ -427,7 +455,9 @@ function classifyProblem( next_action: serverCode === "idempotency_key_reused" ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is." - : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.", + : serverCode === "webhook_signing_key_required" + ? "Nothing was created. Create a webhook signing key in the dashboard (Settings → API Keys), then retry." + : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.", }); } return new ToolkitError({ @@ -449,6 +479,12 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns; if (params.output_format !== undefined) body.output_format = params.output_format; if (params.discovery?.length) body.discovery = params.discovery; + if (params.webhook !== undefined) { + const webhook: Record = { url: params.webhook.url }; + if (params.webhook.events?.length) webhook.events = params.webhook.events; + if (params.webhook.metadata !== undefined) webhook.metadata = params.webhook.metadata; + body.webhook = webhook; + } return crawlJson("POST", "/crawls", { ...opts, body, diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index a4a5add..96dae16 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -284,6 +284,36 @@ test("422 idempotency_key_reused says to send a new key, not to retry as is", as ); }); +test("createCrawl sends the webhook with only the fields set", async () => { + const { impl, calls } = stubFetch(202, crawlBody); + await createCrawl({ url: "https://example.com/products/", webhook: { url: "https://example.com/hook" } }, { apiKey: "k", fetchImpl: impl }); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)).webhook, { url: "https://example.com/hook" }); + await createCrawl( + { + url: "https://example.com/products/", + webhook: { url: "https://example.com/hook", events: ["crawl.completed"], metadata: { k: "v" } }, + }, + { apiKey: "k", fetchImpl: impl }, + ); + assert.deepEqual(JSON.parse(String(calls[1]!.init?.body)).webhook, { + url: "https://example.com/hook", + events: ["crawl.completed"], + metadata: { k: "v" }, + }); +}); + +test("422 webhook_signing_key_required says to create the signing key", async () => { + const { impl } = problem(422, "webhook_signing_key_required"); + await assert.rejects( + () => createCrawl({ url: "https://example.com/products/", webhook: { url: "https://example.com/hook" } }, { apiKey: "k", fetchImpl: impl }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "CRAWL_INVALID_REQUEST" && + e.server_code === "webhook_signing_key_required" && + /signing key/.test(e.next_action), + ); +}); + test("409 maps to CRAWL_REQUEST_IN_FLIGHT", async () => { const { impl } = problem(409, "idempotency_request_in_flight"); await assert.rejects( diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 2d3abf5..4f981a7 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -800,3 +800,51 @@ test("crawl create sends --discovery links, and refuses any other strategy befor }, ); }); + +test("crawl create sends --webhook, --webhook-event and --webhook-metadata as webhook, and prints it", async () => { + const webhook = { url: "https://example.com/hook", events: ["crawl.completed"], metadata: { k: "v" } }; + await withCrawlWorkspace( + () => json({ ...crawlBody, webhook }, 202), + async (calls) => { + const { code, out } = await run([ + "create", + "https://example.com/products/", + "--webhook", + "https://example.com/hook", + "--webhook-event", + "crawl.completed", + "--webhook-metadata", + "k=v", + ]); + assert.equal(code, 0); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)).webhook, webhook); + assert.deepEqual(out.crawl.webhook, webhook); + }, + ); +}); + +test("crawl create refuses malformed webhook flags before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + const cases = [ + ["--webhook", "http://example.com/hook"], + ["--webhook", "not-a-url"], + ["--webhook-event", "crawl.completed"], + ["--webhook-metadata", "k=v"], + ["--webhook", "https://example.com/hook", "--webhook-event", "crawl.done"], + ["--webhook", "https://example.com/hook", "--webhook-event", "crawl.failed", "--webhook-event", "crawl.failed"], + ["--webhook", "https://example.com/hook", "--webhook-metadata", "novalue"], + ["--webhook", "https://example.com/hook", "--webhook-metadata", `${"k".repeat(65)}=v`], + ["--webhook", "https://example.com/hook", "--webhook-metadata", `k=${"v".repeat(1023)}`], + ["--webhook", "https://example.com/hook", ...Array.from({ length: 21 }, (_, i) => ["--webhook-metadata", `k${i}=v`]).flat()], + ]; + for (const flags of cases) { + const { code, out } = await run(["create", "https://example.com/products/", ...flags]); + assert.equal(code, 1, flags.join(" ")); + assert.equal(out.error.code, "INVALID_USAGE", flags.join(" ")); + } + assert.equal(calls.length, 0); + }, + ); +});