From 572d7ad61a02614db3bc511b8749f5592292f980 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 30 Oct 2026 09:51:57 +0000 Subject: [PATCH] feat(crawl): get notified by webhook when a crawl starts, completes, stops or fails The Crawl API now accepts a webhook on create and sends signed events to it, so a caller no longer has to poll to learn when a crawl ends. create() takes webhook= (url, events, metadata), every returned crawl carries the webhook it runs with as Crawl.webhook, CrawlWebhookEvent models the event body, and zenrows.crawl exposes verify_webhook_signature() to check an event's X-Signature header with the account's webhook signing key. --- README.md | 59 ++++++++++++++ acceptance/crawl/README.md | 3 + acceptance/crawl/surface/test.py | 36 +++++++++ src/zenrows/crawl/__init__.py | 26 ++++++ src/zenrows/crawl/client.py | 14 ++++ src/zenrows/crawl/models.py | 105 +++++++++++++++++++++++- tests/test_crawl_client.py | 135 +++++++++++++++++++++++++++++++ 7 files changed, 376 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c2e9ec..994b5f9 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ This package ships three clients: - [Quickstart — Crawl API (new) (`ZenRowsCrawlClient`)](#quickstart--crawl-api-new-zenrowscrawlclient) - [Read the pages](#read-the-pages) - [List and stop crawls](#list-and-stop-crawls) + - [Get notified by webhook](#get-notified-by-webhook) - [Crawl errors](#crawl-errors) - [Contributing](#contributing) - [License](#license) @@ -725,6 +726,62 @@ for c in page.crawls: client.stop(crawl_id) # idempotent; read final counts with get() ``` +### Get notified by webhook + +Instead of polling, pass a `webhook` to `create` and the API POSTs an +event to your URL when the crawl starts and when it ends: +`crawl.started`, then exactly one of `crawl.completed`, `crawl.stopped` +or `crawl.failed`. `events` narrows that list (default: all four), and +`metadata` (up to 20 keys, each value at most 1,024 bytes as JSON) comes +back in every event so you can match it to your own records. The +returned crawl shows the webhook as `crawl.webhook`; it is fixed at +create. + +Every event is signed with your account's webhook signing key, so create +one first in the dashboard (Settings → API Keys); without it `create` +raises 422 `webhook_signing_key_required`. `verify_webhook_signature` +checks the `X-Signature` header (`t=...,v1=...,kid=...`): HMAC-SHA256 +over `.` with the base64-decoded key, compared in constant +time, rejecting timestamps more than 5 minutes off. + +```python +from zenrows.crawl import ( + WEBHOOK_SIGNATURE_HEADER, + CrawlWebhookEvent, + verify_webhook_signature, +) + +crawl = client.create( + start_url, + depth=1, + webhook={ + "url": "https://hooks.example.com/zenrows", # absolute https URL + "events": ["crawl.completed", "crawl.stopped", "crawl.failed"], + "metadata": {"site": "books"}, + }, +) +print(crawl.webhook) + +# kid -> your base64 webhook signing key +SECRETS = {"": ""} + + +def receive(raw_body: bytes, headers) -> int: + # Verify the raw bytes before parsing them. + if not verify_webhook_signature(raw_body, headers[WEBHOOK_SIGNATURE_HEADER], SECRETS): + return 401 + event = CrawlWebhookEvent.model_validate_json(raw_body) + print(event.event_type, event.resource.id, event.metadata) + return 200 +``` + +A failed delivery is retried with backoff for up to 3 days, so an event +can arrive more than once: deduplicate on `event.event_id` (also in the +`X-ZenRows-Event-Id` header), which is the same on every delivery of one +event. Events can arrive out of order, so a short crawl's end can come +before its `crawl.started`: order them by `occurred_at`, or read the +crawl with `client.get(event.resource.id)`. + ### Crawl errors Every non-2xx raises `CrawlAPIError`. Branch on `status_code` and `code`: @@ -754,6 +811,8 @@ except CrawlAPIError as exc: | 409 | `idempotency_request_in_flight` | Retry after the first request with this key finishes. | | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | | 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. | +| 422 | `invalid_webhook` | The `webhook` `url`, `events` or `metadata` breaks a rule; `detail` names the field. Fix it. | +| 422 | `webhook_signing_key_required` | Create a webhook signing key in the dashboard, then retry. | | 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. | | 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index bedf539..a1b53f8 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -83,6 +83,9 @@ and signal checks. | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures. +Check 25 creates a one-page crawl with a `webhook` (`url`, `events` `["crawl.completed"]`, +`metadata` `{"k": "v"}`), checks that the returned crawl carries it, and stops the crawl. The +account needs a webhook signing key for it. ## What it costs diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index eef2aba..eaacc1f 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -924,6 +924,41 @@ def c24(): c24, ) +WEBHOOK = { + "url": "https://example.com/zenrows-hook", + "events": ["crawl.completed"], + "metadata": {"k": "v"}, +} + + +def c25(): + """webhook: sent at create and carried by the returned crawl.""" + w = create(depth=1, max_pages=1, webhook=WEBHOOK) + client.stop(w.crawl_id) + got = ( + None + if w.webhook is None + else { + "url": w.webhook.url, + "events": [e.value for e in w.webhook.events], + "metadata": w.webhook.metadata, + } + ) + return ( + pass_if(got == WEBHOOK), + f"{w.crawl_id}: webhook {json.dumps(got)}", + w.model_dump_json(), + ) + + +run_check( + "25", + "Webhook", + "create(webhook={url, events, metadata}) returns a crawl whose webhook echoes them", + c25, + live=True, +) + with open(f"{RESULTS}/naming.json", "w") as f: json.dump( { @@ -942,6 +977,7 @@ def c24(): "exclude_patterns", "output_format", "discovery", + "webhook", "idempotency_key", "cursor", "limit", diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index d0e22e8..0cadf8d 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -11,8 +11,18 @@ `create(output_format="html")` or `create(output_format="markdown")` also returns each kept URL's page; without it a crawl returns URLs only. + +`create(webhook={"url": ...})` sends a signed event when the crawl starts +and when it ends; check each one with `verify_webhook_signature` and +parse it as a `CrawlWebhookEvent`. """ +from zenrows.batch._webhooks import ( + DEFAULT_WEBHOOK_TOLERANCE_SECONDS, + WEBHOOK_EVENT_ID_HEADER, + WEBHOOK_SIGNATURE_HEADER, + verify_webhook_signature, +) from zenrows.crawl.client import ( CrawlDownload, ZenRowsCrawlClient, @@ -23,10 +33,16 @@ Coverage, Crawl, CrawlErrorCode, + CrawlEventName, + CrawlEventResource, + CrawlEventType, CrawlList, CrawlResult, CrawlStatus, CrawlStop, + CrawlWebhook, + CrawlWebhookDict, + CrawlWebhookEvent, CrawlWithResults, DiscoveryStrategy, DownloadLine, @@ -37,16 +53,25 @@ __all__ = [ "CRAWL_NOT_ENABLED_CODE", + "DEFAULT_WEBHOOK_TOLERANCE_SECONDS", + "WEBHOOK_EVENT_ID_HEADER", + "WEBHOOK_SIGNATURE_HEADER", "ContentStatus", "Coverage", "Crawl", "CrawlAPIError", "CrawlDownload", "CrawlErrorCode", + "CrawlEventName", + "CrawlEventResource", + "CrawlEventType", "CrawlList", "CrawlResult", "CrawlStatus", "CrawlStop", + "CrawlWebhook", + "CrawlWebhookDict", + "CrawlWebhookEvent", "CrawlWithResults", "DiscoveryStrategy", "DownloadLine", @@ -54,4 +79,5 @@ "RunError", "StopReason", "ZenRowsCrawlClient", + "verify_webhook_signature", ] diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 181745c..c6992b5 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -37,6 +37,7 @@ CrawlResult, CrawlStatus, CrawlStop, + CrawlWebhookDict, CrawlWithResults, DownloadLine, ) @@ -175,6 +176,7 @@ def create( exclude_patterns: list[str] | None = None, output_format: Literal["html", "markdown"] | None = None, discovery: list[Literal["links"]] | None = None, + webhook: CrawlWebhookDict | None = None, idempotency_key: str | None = None, ) -> Crawl: """`POST /crawls` — start a crawl and return it, `running`. @@ -198,6 +200,17 @@ def create( - `discovery`: how the crawl finds the URLs it follows on each page it walks. `["links"]` follows the links in the page's HTML, and is what a crawl runs with when you leave it out. + - `webhook`: get notified when the crawl starts and how it + ends: `{"url": "https://...", "events": [...], "metadata": {...}}` + (see `CrawlWebhookDict`). `events` defaults to all four of + `crawl.started`, `crawl.completed`, `crawl.stopped` and + `crawl.failed`. Each event is POSTed to `url` as JSON + (`CrawlWebhookEvent`) and signed with the account's webhook + signing key; check it with `verify_webhook_signature`. The + returned crawl's `webhook` shows it. Without a signing key on + the account, `create` raises 422 + `webhook_signing_key_required`; a bad `url`, `events` or + `metadata` raises 422 `invalid_webhook`. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -215,6 +228,7 @@ def create( "exclude_patterns": exclude_patterns, "output_format": output_format, "discovery": discovery, + "webhook": webhook, } body.update({k: v for k, v in optional.items() if v is not None}) headers = {"Content-Type": "application/json"} diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index e6858cb..cb3873a 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -1,4 +1,5 @@ -"""Pydantic v2 models for the Crawl API's responses. +"""Pydantic v2 models for the Crawl API's responses, and the +`CrawlWebhookDict` that `create` takes. Hand-written: the surface is a handful of small schemas. @@ -8,12 +9,19 @@ value (see `zenrows.batch._open_enum`). """ +import sys from datetime import datetime from enum import Enum -from typing import Any +from typing import Any, Literal, TypedDict from pydantic import BaseModel, Field +if sys.version_info >= (3, 11): + from typing import NotRequired +else: + # `typing.NotRequired` only exists on 3.11+; the SDK floors at 3.10. + from typing_extensions import NotRequired + from zenrows.batch._open_enum import open_enum_missing @@ -98,6 +106,90 @@ class CrawlErrorCode(Enum): _missing_ = classmethod(open_enum_missing) +class CrawlEventType(Enum): + """A crawl event a webhook can be sent. `crawl.started` once the + crawl is created and runs; `crawl.completed`, `crawl.stopped` or + `crawl.failed` once it reaches that status, exactly one of them per + crawl. Clients must accept values not listed here.""" + + STARTED = "crawl.started" + COMPLETED = "crawl.completed" + STOPPED = "crawl.stopped" + FAILED = "crawl.failed" + + _missing_ = classmethod(open_enum_missing) + + +CrawlEventName = Literal["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"] +"""The event names `CrawlWebhookDict.events` takes.""" + + +class CrawlWebhookDict(TypedDict): + """The `webhook` to pass to `create`: where the crawl's events are + sent, and which ones. It is fixed at create; there is no update. + + - `url` (required): the absolute `https` URL each event is POSTed + to. Its host must resolve. + - `events`: the events to send, each named once. Leave it out for + all four (`crawl.started`, `crawl.completed`, `crawl.stopped`, + `crawl.failed`). + - `metadata`: sent back in every event, to match it to your own + records. Up to 20 keys of 1 to 64 characters; each value any + JSON value of at most 1,024 bytes as compact JSON. + + Every event is signed with the account's webhook signing key, so + the account needs one (Settings → API Keys in the dashboard).""" + + url: str + events: NotRequired[list[CrawlEventName]] + metadata: NotRequired[dict[str, Any]] + + +class CrawlWebhook(BaseModel): + """The webhook a crawl sends its events to, as it runs with it.""" + + url: str + events: list[CrawlEventType] + """The events the crawl sends: all four when the create left + `events` out.""" + metadata: dict[str, Any] | None = None + """Sent back in every event. None when the crawl has none.""" + + +class CrawlEventResource(BaseModel): + """The crawl a webhook event is about.""" + + type: str + """`crawl`.""" + id: str + """The crawl's `crawl_id`.""" + url: str | None = None + """The crawl on the API. Read it with `get` for its status, + coverage and results.""" + + +class CrawlWebhookEvent(BaseModel): + """The JSON body POSTed to a crawl's webhook for each event. + + Every delivery of one event carries the same `event_id` (also in + the `X-ZenRows-Event-Id` header): drop an event you have already + handled. Events can arrive out of order; order them by + `occurred_at`, or read the crawl. Verify the `X-Signature` header + with `verify_webhook_signature` before parsing the body.""" + + schema_: str = Field(alias="schema") + """`zenrows.webhook/v1`.""" + event_id: str + """Stable across retries, e.g. `ev_01M5AKXYZ_completed`.""" + event_type: CrawlEventType + occurred_at: datetime + """When the crawl was created, for `crawl.started`; when it + finished, for the others.""" + resource: CrawlEventResource + metadata: dict[str, Any] | None = None + """The webhook's `metadata`. None when the crawl has none.""" + + class RunError(BaseModel): """Why a crawl failed, or why it stopped because the account ran out of credits or the API key hit a credit cap. Present when `status` is @@ -142,6 +234,9 @@ class Crawl(BaseModel): include_patterns: list[str] | None = None exclude_patterns: list[str] | None = None output_format: OutputFormat | None = None + webhook: CrawlWebhook | None = None + """Where the crawl sends its events. None for a crawl created + without a webhook.""" duplicates_removed: int | None = None """Links met again and dropped as already seen.""" finished_at: datetime | None = None @@ -205,10 +300,16 @@ class DownloadLine(BaseModel): "Coverage", "Crawl", "CrawlErrorCode", + "CrawlEventName", + "CrawlEventResource", + "CrawlEventType", "CrawlList", "CrawlResult", "CrawlStatus", "CrawlStop", + "CrawlWebhook", + "CrawlWebhookDict", + "CrawlWebhookEvent", "CrawlWithResults", "DiscoveryStrategy", "DownloadLine", diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index e097ad0..6b6e7ee 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -6,6 +6,9 @@ stop rule, downloads with their status, retries, and wait. """ +import base64 +import hashlib +import hmac import json import pytest @@ -20,10 +23,13 @@ ContentStatus, CrawlAPIError, CrawlErrorCode, + CrawlEventType, CrawlResult, CrawlStatus, + CrawlWebhookEvent, OutputFormat, StopReason, + verify_webhook_signature, ) BASE_URL = "http://localhost:9100/v1" @@ -175,6 +181,135 @@ def test_create_sends_markdown_output_format(client: ZenRowsCrawlClient): } +@respx.mock +def test_create_sends_webhook_and_reads_it_back(client: ZenRowsCrawlClient): + webhook = { + "url": "https://hooks.example.com/zenrows", + "events": ["crawl.completed"], + "metadata": {"k": "v", "n": 1}, + } + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response(202, json=crawl_body(webhook=webhook)) + ) + + crawl = client.create( + "https://example.com/", + webhook={ + "url": "https://hooks.example.com/zenrows", + "events": ["crawl.completed"], + "metadata": {"k": "v", "n": 1}, + }, + ) + + assert json.loads(route.calls.last.request.content) == { + "url": "https://example.com/", + "webhook": webhook, + } + assert crawl.webhook is not None + assert crawl.webhook.url == "https://hooks.example.com/zenrows" + assert crawl.webhook.events == [CrawlEventType.COMPLETED] + assert crawl.webhook.metadata == {"k": "v", "n": 1} + + +@respx.mock +def test_create_sends_only_the_webhook_url_when_that_is_all_given(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response( + 202, + json=crawl_body( + webhook={ + "url": "https://hooks.example.com/zenrows", + "events": [ + "crawl.started", + "crawl.completed", + "crawl.stopped", + "crawl.failed", + "crawl.paused", + ], + "metadata": None, + } + ), + ) + ) + + crawl = client.create( + "https://example.com/", webhook={"url": "https://hooks.example.com/zenrows"} + ) + + assert json.loads(route.calls.last.request.content)["webhook"] == { + "url": "https://hooks.example.com/zenrows" + } + assert crawl.webhook is not None + assert crawl.webhook.metadata is None + events = crawl.webhook.events + assert events[:4] == [ + CrawlEventType.STARTED, + CrawlEventType.COMPLETED, + CrawlEventType.STOPPED, + CrawlEventType.FAILED, + ] + assert is_unknown(events[4]) and events[4].value == "crawl.paused" + + +@respx.mock +def test_crawl_without_webhook_reads_none(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls").mock( + return_value=Response( + 200, json={"crawls": [crawl_body(), crawl_body(crawl_id="c_2", webhook=None)]} + ) + ) + assert [c.webhook for c in client.list().crawls] == [None, None] + + +@pytest.mark.parametrize("code", ["invalid_webhook", "webhook_signing_key_required"]) +@respx.mock +def test_webhook_refusals_carry_their_code(client: ZenRowsCrawlClient, code: str): + respx.post(f"{BASE_URL}/crawls").mock(return_value=problem(422, code)) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", webhook={"url": "https://hooks.example.com/z"}) + assert exc_info.value.status_code == 422 + assert exc_info.value.code == code + + +def test_webhook_event_verifies_and_parses(): + secret = base64.b64encode(bytes(range(32))).decode() + body = ( + b'{"schema":"zenrows.webhook/v1","event_id":"ev_01M5AKXYZ_completed",' + b'"event_type":"crawl.completed","occurred_at":"2026-10-27T10:42:10Z",' + b'"resource":{"type":"crawl","id":"c_01M5AKXYZ",' + b'"url":"https://api.zenrows.com/v1/crawls/c_01M5AKXYZ"},"metadata":{"site":"golfclub"}}' + ) + t = 1790000000 + digest = hmac.new(bytes(range(32)), f"{t}.".encode() + body, hashlib.sha256).hexdigest() + header = f"t={t},v1={digest},kid=kid_live" + + assert verify_webhook_signature(body, header, {"kid_live": secret}, now=t) + assert not verify_webhook_signature(body + b" ", header, {"kid_live": secret}, now=t) + + event = CrawlWebhookEvent.model_validate_json(body) + assert event.schema_ == "zenrows.webhook/v1" + assert event.event_id == "ev_01M5AKXYZ_completed" + assert event.event_type is CrawlEventType.COMPLETED + assert event.resource.type == "crawl" + assert event.resource.id == "c_01M5AKXYZ" + assert event.metadata == {"site": "golfclub"} + + +def test_webhook_event_without_metadata_or_resource_url(): + event = CrawlWebhookEvent.model_validate( + { + "schema": "zenrows.webhook/v1", + "event_id": "ev_1_started", + "event_type": "crawl.started", + "occurred_at": "2026-10-27T10:40:00Z", + "resource": {"type": "crawl", "id": "c_1"}, + } + ) + assert event.event_type is CrawlEventType.STARTED + assert event.resource.url is None + assert event.metadata is None + + # ----- get / list / stop -----