fix(webhooks): make delivery durable and authenticated

Cargo.toml

@@ -74,7 +74,7 @@ stellar-xdr = { version = "25.0.0", features = ["std", "curr"] }
 stellar_sdk = "0.1"
 base64 = "0.22"
 ed25519-dalek = "2.1"
-jsonwebtoken = "10.4"
+jsonwebtoken = { version = "10.4", features = ["aws_lc_rs"] }
 utoipa = { version = "4.2", features = ["axum_extras", "chrono", "uuid"] }
 utoipa-swagger-ui = { version = "9.0", features = ["axum"] }
 urlencoding = "2.1"

README.md

@@ -82,6 +82,72 @@ cargo test
 
 Integration/load tests live in `tests/` and `load-tests/`.
 
+## Outbound webhooks
+
+An authenticated `POST /api/webhooks` (or `POST /api/v1/webhooks`) returns the
+subscription fields plus its signing `secret` once, with `Cache-Control:
+no-store`. Save this secret in the receiving integration when you register.
+List, detail, and delivery-status responses do not return it.
+
+Each queued event has a persistent UUID. Its serialized JSON envelope (`id`,
+`event`, `timestamp`, `data`) is saved before delivery and reused byte for byte
+on retries, including after a dispatcher restart. Consumers should deduplicate
+using `X-Zapier-Delivery-ID`, which is the same value as the envelope's `id` and
+the `event_id` returned by `POST /api/webhooks/{id}/test`.
+
+Requests also include `X-Zapier-Event`, `X-Zapier-Timestamp` (the event creation
+time in Unix seconds), and `X-Zapier-Signature`. The signature is
+`sha256=<hex HMAC-SHA256>` over the exact raw request body, keyed with that
+subscription's secret. Verify the raw bytes before parsing JSON or acting on
+an event. Re-serializing JSON before checking the signature can change the
+signed bytes. The creation timestamp remains stable across retries; use the
+event ID to deduplicate an already accepted event.
+
+Any non-2xx response or transport failure counts as a failed attempt. Redirects
+are treated as failures without following another URL, and failure records do
+not include the receiver's response body. Delivery becomes terminal after
+**three recorded failures**, including the initial request, with a ten-second
+timeout on each request. Retry due times are stored
+in SQLite: the delays after the first and second failures are five and ten
+seconds. The exponential delay is capped at 300 seconds if the attempt limit
+is increased. The dispatcher polls every five seconds, so actual delivery can
+occur later than the due time. The third failure becomes terminal `failed`.
+Inactive or missing subscriptions also become terminal without another send.
+
+A thirty-second reservation prevents another worker from immediately sending
+the same due event and makes events recoverable after a worker stops. Delivery
+is at least once: an accepted HTTP request whose database update fails can be
+sent again with the same ID and exact body. Crashes and failed database writes
+can therefore cause more physical sends than the recorded failure count.
+Reservation tokens prevent a resumed, expired worker from overwriting a newer
+worker's result. Successful delivery and the
+subscription's `last_fired_at` are recorded in one transaction; database
+failures are reported by the dispatcher instead of reported as success.
+
+Authenticated subscribers can query
+`GET /api/webhooks/{webhook_id}/events/{event_id}` for `status` (`pending`,
+`delivered`, or `failed`), `retries` (failed attempts, retained after success),
+`last_error`, `created_at`, `next_attempt_at`, `last_attempt_at`, and
+`delivered_at`. The latter three timestamps use Unix seconds and are nullable;
+while a worker holds an event, `next_attempt_at` is its reservation expiry.
+Only the subscription's owner can access the record. Secrets and event data
+are excluded from this response.
+The same resource is available at
+`GET /api/v1/webhooks/{webhook_id}/events/{event_id}` with a valid access-token
+`Authorization: Bearer ...` header.
+
+Migration `035_webhook_delivery_retries.sql` preserves existing raw payloads
+and recorded errors, marks previously stranded exhausted events as `failed`,
+and freezes legacy envelopes from their stored ID and creation time on their
+next delivery. Attempts sent before this migration used a different ID and
+cannot be retroactively deduplicated with their new, stable ID.
+
+Run the focused dispatcher, signature, migration, and API tests with:
+
+```bash
+cargo test --test webhook_delivery_regression_tests --test webhook_event_service_tests
+```
+
 ## Observability
 
 - **Logs** — set `LOGSTASH_ENABLED=true` to forward structured logs to an ELK stack (see `docker-compose.elk.yml` in the core repo)

migrations/035_webhook_delivery_retries.sql

@@ -0,0 +1,18 @@
+-- Keep the serialized delivery envelope and retry schedule across restarts.
+-- Existing payloads remain unchanged; the dispatcher freezes legacy envelopes
+-- using their stored event ID and creation time before their next delivery.
+ALTER TABLE webhook_events ADD COLUMN delivery_payload TEXT;
+ALTER TABLE webhook_events ADD COLUMN next_attempt_at INTEGER;
+ALTER TABLE webhook_events ADD COLUMN last_attempt_at INTEGER;
+ALTER TABLE webhook_events ADD COLUMN delivered_at INTEGER;
+ALTER TABLE webhook_events ADD COLUMN delivery_token TEXT;
+
+-- The old dispatcher stranded the third failure as pending, although its query
+-- excluded retries >= 3. Preserve the recorded error and make those rows terminal.
+UPDATE webhook_events
+SET status = 'failed',
+    last_error = COALESCE(last_error, 'maximum_attempts_exhausted')
+WHERE status = 'pending' AND retries >= 3;
+
+CREATE INDEX IF NOT EXISTS idx_webhook_events_due
+ON webhook_events(status, next_attempt_at, created_at);

src/api/v1/mod.rs

@@ -131,9 +131,10 @@ pub fn routes(
         .with_state(app_state)
         .layer(middleware::from_fn(auth_middleware));
 
-    let protected_webhook_routes = Router::new()
-        .nest("/webhooks", webhooks::routes(pool.clone()))
-        .layer(middleware::from_fn(auth_middleware));
+    let protected_webhook_routes = webhooks::versioned_routes(pool.clone());
+
+    let unversioned_webhook_routes =
+        webhooks::routes(pool.clone()).layer(middleware::from_fn(auth_middleware));
 
     // 4. RPC routes
     let rpc_routes = Router::new()
@@ -189,6 +190,7 @@ pub fn routes(
         .route("/api/version", get(get_api_version))
         // Preserve existing unversioned endpoints for backward compatibility.
         .merge(v1_router)
+        .merge(unversioned_webhook_routes)
         .layer(cors)
         .layer(middleware::from_fn(
             crate::request_id::request_id_middleware,