feat(crawl): get notified by webhook when a crawl starts, completes, stops or fails

README.md

@@ -43,6 +43,7 @@ This package ships three clients:
 - [Quickstart — Crawl API (new) (`ZenRowsCrawlClient`)](#quickstart--crawl-api-new-zenrowscrawlclient)
   - [Read the pages](#read-the-pages)
   - [List and stop crawls](#list-and-stop-crawls)
+  - [Get notified by webhook](#get-notified-by-webhook)
   - [Crawl errors](#crawl-errors)
 - [Contributing](#contributing)
 - [License](#license)
@@ -725,6 +726,62 @@ for c in page.crawls:
 client.stop(crawl_id)                   # idempotent; read final counts with get()
 ```
 
+### Get notified by webhook
+
+Instead of polling, pass a `webhook` to `create` and the API POSTs an
+event to your URL when the crawl starts and when it ends:
+`crawl.started`, then exactly one of `crawl.completed`, `crawl.stopped`
+or `crawl.failed`. `events` narrows that list (default: all four), and
+`metadata` (up to 20 keys, each value at most 1,024 bytes as JSON) comes
+back in every event so you can match it to your own records. The
+returned crawl shows the webhook as `crawl.webhook`; it is fixed at
+create.
+
+Every event is signed with your account's webhook signing key, so create
+one first in the dashboard (Settings → API Keys); without it `create`
+raises 422 `webhook_signing_key_required`. `verify_webhook_signature`
+checks the `X-Signature` header (`t=...,v1=...,kid=...`): HMAC-SHA256
+over `<t>.<raw body>` with the base64-decoded key, compared in constant
+time, rejecting timestamps more than 5 minutes off.
+
+```python
+from zenrows.crawl import (
+    WEBHOOK_SIGNATURE_HEADER,
+    CrawlWebhookEvent,
+    verify_webhook_signature,
+)
+
+crawl = client.create(
+    start_url,
+    depth=1,
+    webhook={
+        "url": "https://hooks.example.com/zenrows",   # absolute https URL
+        "events": ["crawl.completed", "crawl.stopped", "crawl.failed"],
+        "metadata": {"site": "books"},
+    },
+)
+print(crawl.webhook)
+
+# kid -> your base64 webhook signing key
+SECRETS = {"<kid>": "<secret>"}
+
+
+def receive(raw_body: bytes, headers) -> int:
+    # Verify the raw bytes before parsing them.
+    if not verify_webhook_signature(raw_body, headers[WEBHOOK_SIGNATURE_HEADER], SECRETS):
+        return 401
+    event = CrawlWebhookEvent.model_validate_json(raw_body)
+    print(event.event_type, event.resource.id, event.metadata)
+    return 200
+```
+
+A failed delivery is retried with backoff for up to 3 days, so an event
+can arrive more than once: deduplicate on `event.event_id` (also in the
+`X-ZenRows-Event-Id` header), which is the same on every delivery of one
+event. Events can arrive out of order, so a short crawl's end can come
+before its `crawl.started`: order them by `occurred_at`, or read the
+crawl with `client.get(event.resource.id)`.
+
 ### Crawl errors
 
 Every non-2xx raises `CrawlAPIError`. Branch on `status_code` and `code`:
@@ -754,6 +811,8 @@ except CrawlAPIError as exc:
 | 409 | `idempotency_request_in_flight` | Retry after the first request with this key finishes. |
 | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. |
 | 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. |
+| 422 | `invalid_webhook` | The `webhook` `url`, `events` or `metadata` breaks a rule; `detail` names the field. Fix it. |
+| 422 | `webhook_signing_key_required` | Create a webhook signing key in the dashboard, then retry. |
 | 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. |
 | 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. |
 

acceptance/crawl/README.md

@@ -83,6 +83,9 @@ and signal checks.
 | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) |
 
 Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures.
+Check 25 creates a one-page crawl with a `webhook` (`url`, `events` `["crawl.completed"]`,
+`metadata` `{"k": "v"}`), checks that the returned crawl carries it, and stops the crawl. The
+account needs a webhook signing key for it.
 
 ## What it costs
 

acceptance/crawl/surface/test.py

@@ -924,6 +924,41 @@ def c24():
     c24,
 )
 
+WEBHOOK = {
+    "url": "https://example.com/zenrows-hook",
+    "events": ["crawl.completed"],
+    "metadata": {"k": "v"},
+}
+
+
+def c25():
+    """webhook: sent at create and carried by the returned crawl."""
+    w = create(depth=1, max_pages=1, webhook=WEBHOOK)
+    client.stop(w.crawl_id)
+    got = (
+        None
+        if w.webhook is None
+        else {
+            "url": w.webhook.url,
+            "events": [e.value for e in w.webhook.events],
+            "metadata": w.webhook.metadata,
+        }
+    )
+    return (
+        pass_if(got == WEBHOOK),
+        f"{w.crawl_id}: webhook {json.dumps(got)}",
+        w.model_dump_json(),
+    )
+
+
+run_check(
+    "25",
+    "Webhook",
+    "create(webhook={url, events, metadata}) returns a crawl whose webhook echoes them",
+    c25,
+    live=True,
+)
+
 with open(f"{RESULTS}/naming.json", "w") as f:
     json.dump(
         {
@@ -942,6 +977,7 @@ def c24():
                     "exclude_patterns",
                     "output_format",
                     "discovery",
+                    "webhook",
                     "idempotency_key",
                     "cursor",
                     "limit",

src/zenrows/crawl/__init__.py

@@ -11,8 +11,18 @@
 
 `create(output_format="html")` or `create(output_format="markdown")` also
 returns each kept URL's page; without it a crawl returns URLs only.
+
+`create(webhook={"url": ...})` sends a signed event when the crawl starts
+and when it ends; check each one with `verify_webhook_signature` and
+parse it as a `CrawlWebhookEvent`.
 """
 
+from zenrows.batch._webhooks import (
+    DEFAULT_WEBHOOK_TOLERANCE_SECONDS,
+    WEBHOOK_EVENT_ID_HEADER,
+    WEBHOOK_SIGNATURE_HEADER,
+    verify_webhook_signature,
+)
 from zenrows.crawl.client import (
     CrawlDownload,
     ZenRowsCrawlClient,
@@ -23,10 +33,16 @@
     Coverage,
     Crawl,
     CrawlErrorCode,
+    CrawlEventName,
+    CrawlEventResource,
+    CrawlEventType,
     CrawlList,
     CrawlResult,
     CrawlStatus,
     CrawlStop,
+    CrawlWebhook,
+    CrawlWebhookDict,
+    CrawlWebhookEvent,
     CrawlWithResults,
     DiscoveryStrategy,
     DownloadLine,
@@ -37,21 +53,31 @@
 
 __all__ = [
     "CRAWL_NOT_ENABLED_CODE",
+    "DEFAULT_WEBHOOK_TOLERANCE_SECONDS",
+    "WEBHOOK_EVENT_ID_HEADER",
+    "WEBHOOK_SIGNATURE_HEADER",
     "ContentStatus",
     "Coverage",
     "Crawl",
     "CrawlAPIError",
     "CrawlDownload",
     "CrawlErrorCode",
+    "CrawlEventName",
+    "CrawlEventResource",
+    "CrawlEventType",
     "CrawlList",
     "CrawlResult",
     "CrawlStatus",
     "CrawlStop",
+    "CrawlWebhook",
+    "CrawlWebhookDict",
+    "CrawlWebhookEvent",
     "CrawlWithResults",
     "DiscoveryStrategy",
     "DownloadLine",
     "OutputFormat",
     "RunError",
     "StopReason",
     "ZenRowsCrawlClient",
+    "verify_webhook_signature",
 ]

src/zenrows/crawl/client.py

@@ -37,6 +37,7 @@
     CrawlResult,
     CrawlStatus,
     CrawlStop,
+    CrawlWebhookDict,
     CrawlWithResults,
     DownloadLine,
 )
@@ -175,6 +176,7 @@ def create(
         exclude_patterns: list[str] | None = None,
         output_format: Literal["html", "markdown"] | None = None,
         discovery: list[Literal["links"]] | None = None,
+        webhook: CrawlWebhookDict | None = None,
         idempotency_key: str | None = None,
     ) -> Crawl:
         """`POST /crawls` — start a crawl and return it, `running`.
@@ -198,6 +200,17 @@ def create(
         - `discovery`: how the crawl finds the URLs it follows on each
           page it walks. `["links"]` follows the links in the page's
           HTML, and is what a crawl runs with when you leave it out.
+        - `webhook`: get notified when the crawl starts and how it
+          ends: `{"url": "https://...", "events": [...], "metadata": {...}}`
+          (see `CrawlWebhookDict`). `events` defaults to all four of
+          `crawl.started`, `crawl.completed`, `crawl.stopped` and
+          `crawl.failed`. Each event is POSTed to `url` as JSON
+          (`CrawlWebhookEvent`) and signed with the account's webhook
+          signing key; check it with `verify_webhook_signature`. The
+          returned crawl's `webhook` shows it. Without a signing key on
+          the account, `create` raises 422
+          `webhook_signing_key_required`; a bad `url`, `events` or
+          `metadata` raises 422 `invalid_webhook`.
         - `idempotency_key`: a retry with the same key and body answers
           with the crawl the first request created.
 
@@ -215,6 +228,7 @@ def create(
             "exclude_patterns": exclude_patterns,
             "output_format": output_format,
             "discovery": discovery,
+            "webhook": webhook,
         }
         body.update({k: v for k, v in optional.items() if v is not None})
         headers = {"Content-Type": "application/json"}

src/zenrows/crawl/models.py

@@ -1,4 +1,5 @@
-"""Pydantic v2 models for the Crawl API's responses.
+"""Pydantic v2 models for the Crawl API's responses, and the
+`CrawlWebhookDict` that `create` takes.
 
 Hand-written: the surface is a handful of small schemas.
 
@@ -8,12 +9,19 @@
 value (see `zenrows.batch._open_enum`).
 """
 
+import sys
 from datetime import datetime
 from enum import Enum
-from typing import Any
+from typing import Any, Literal, TypedDict
 
 from pydantic import BaseModel, Field
 
+if sys.version_info >= (3, 11):
+    from typing import NotRequired
+else:
+    # `typing.NotRequired` only exists on 3.11+; the SDK floors at 3.10.
+    from typing_extensions import NotRequired
+
 from zenrows.batch._open_enum import open_enum_missing
 
 
@@ -98,6 +106,90 @@ class CrawlErrorCode(Enum):
     _missing_ = classmethod(open_enum_missing)
 
 
+class CrawlEventType(Enum):
+    """A crawl event a webhook can be sent. `crawl.started` once the
+    crawl is created and runs; `crawl.completed`, `crawl.stopped` or
+    `crawl.failed` once it reaches that status, exactly one of them per
+    crawl. Clients must accept values not listed here."""
+
+    STARTED = "crawl.started"
+    COMPLETED = "crawl.completed"
+    STOPPED = "crawl.stopped"
+    FAILED = "crawl.failed"
+
+    _missing_ = classmethod(open_enum_missing)
+
+
+CrawlEventName = Literal["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"]
+"""The event names `CrawlWebhookDict.events` takes."""
+
+
+class CrawlWebhookDict(TypedDict):
+    """The `webhook` to pass to `create`: where the crawl's events are
+    sent, and which ones. It is fixed at create; there is no update.
+
+    - `url` (required): the absolute `https` URL each event is POSTed
+      to. Its host must resolve.
+    - `events`: the events to send, each named once. Leave it out for
+      all four (`crawl.started`, `crawl.completed`, `crawl.stopped`,
+      `crawl.failed`).
+    - `metadata`: sent back in every event, to match it to your own
+      records. Up to 20 keys of 1 to 64 characters; each value any
+      JSON value of at most 1,024 bytes as compact JSON.
+
+    Every event is signed with the account's webhook signing key, so
+    the account needs one (Settings → API Keys in the dashboard)."""
+
+    url: str
+    events: NotRequired[list[CrawlEventName]]
+    metadata: NotRequired[dict[str, Any]]
+
+
+class CrawlWebhook(BaseModel):
+    """The webhook a crawl sends its events to, as it runs with it."""
+
+    url: str
+    events: list[CrawlEventType]
+    """The events the crawl sends: all four when the create left
+    `events` out."""
+    metadata: dict[str, Any] | None = None
+    """Sent back in every event. None when the crawl has none."""
+
+
+class CrawlEventResource(BaseModel):
+    """The crawl a webhook event is about."""
+
+    type: str
+    """`crawl`."""
+    id: str
+    """The crawl's `crawl_id`."""
+    url: str | None = None
+    """The crawl on the API. Read it with `get` for its status,
+    coverage and results."""
+
+
+class CrawlWebhookEvent(BaseModel):
+    """The JSON body POSTed to a crawl's webhook for each event.
+
+    Every delivery of one event carries the same `event_id` (also in
+    the `X-ZenRows-Event-Id` header): drop an event you have already
+    handled. Events can arrive out of order; order them by
+    `occurred_at`, or read the crawl. Verify the `X-Signature` header
+    with `verify_webhook_signature` before parsing the body."""
+
+    schema_: str = Field(alias="schema")
+    """`zenrows.webhook/v1`."""
+    event_id: str
+    """Stable across retries, e.g. `ev_01M5AKXYZ_completed`."""
+    event_type: CrawlEventType
+    occurred_at: datetime
+    """When the crawl was created, for `crawl.started`; when it
+    finished, for the others."""
+    resource: CrawlEventResource
+    metadata: dict[str, Any] | None = None
+    """The webhook's `metadata`. None when the crawl has none."""
+
+
 class RunError(BaseModel):
     """Why a crawl failed, or why it stopped because the account ran out
     of credits or the API key hit a credit cap. Present when `status` is
@@ -142,6 +234,9 @@ class Crawl(BaseModel):
     include_patterns: list[str] | None = None
     exclude_patterns: list[str] | None = None
     output_format: OutputFormat | None = None
+    webhook: CrawlWebhook | None = None
+    """Where the crawl sends its events. None for a crawl created
+    without a webhook."""
     duplicates_removed: int | None = None
     """Links met again and dropped as already seen."""
     finished_at: datetime | None = None
@@ -205,10 +300,16 @@ class DownloadLine(BaseModel):
     "Coverage",
     "Crawl",
     "CrawlErrorCode",
+    "CrawlEventName",
+    "CrawlEventResource",
+    "CrawlEventType",
     "CrawlList",
     "CrawlResult",
     "CrawlStatus",
     "CrawlStop",
+    "CrawlWebhook",
+    "CrawlWebhookDict",
+    "CrawlWebhookEvent",
     "CrawlWithResults",
     "DiscoveryStrategy",
     "DownloadLine",