diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 8cccafb0..4d42eb2c 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -178,7 +178,7 @@ jobs: - name: Run tests with coverage working-directory: backend env: - DATABASE_URL: sqlite::memory: + DATABASE_URL: "sqlite::memory:" REDIS_URL: redis://127.0.0.1:6379 run: | cargo llvm-cov \ diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index d4ce9162..f83a20c0 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -22,7 +22,7 @@ jobs: # ONLY for `--lib` (production code). Tests are allowed to keep `.unwrap()` # / `.expect()` / `panic!` because they are the idiomatic way to express # preconditions and assertions in unit tests. - - name: Backend Clippy (lib: deny panic-lints; tests: warn) + - name: "Backend Clippy (lib: deny panic-lints; tests: warn)" working-directory: ./backend run: | cargo clippy --lib -- \ diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f2c12106..b8736320 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -46,7 +46,7 @@ jobs: - name: Run tests with coverage working-directory: backend env: - DATABASE_URL: sqlite::memory: + DATABASE_URL: "sqlite::memory:" REDIS_URL: redis://127.0.0.1:6379 run: | cargo llvm-cov \ diff --git a/.github/workflows/vault-deploy-approle.yml b/.github/workflows/vault-deploy-approle.yml index 55c5de2d..b695b0a9 100644 --- a/.github/workflows/vault-deploy-approle.yml +++ b/.github/workflows/vault-deploy-approle.yml @@ -13,6 +13,8 @@ env: jobs: test-with-approle: runs-on: ubuntu-latest + env: + VAULT_AUTH_CONFIGURED: ${{ secrets.VAULT_ADDR != '' && secrets.VAULT_ROLE_ID != '' && secrets.VAULT_SECRET_ID != '' }} steps: - uses: actions/checkout@v4 @@ -23,12 +25,12 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Set fallback JWT secret when Vault credentials are unavailable - if: ${{ secrets.VAULT_ADDR == '' || secrets.VAULT_ROLE_ID == '' || secrets.VAULT_SECRET_ID == '' }} + if: ${{ env.VAULT_AUTH_CONFIGURED != 'true' }} run: | echo "JWT_SECRET=fallback-test-secret-value-at-least-32-characters" >> $GITHUB_ENV - name: Authenticate to Vault with AppRole - if: ${{ secrets.VAULT_ADDR != '' && secrets.VAULT_ROLE_ID != '' && secrets.VAULT_SECRET_ID != '' }} + if: ${{ env.VAULT_AUTH_CONFIGURED == 'true' }} id: vault-auth run: | TOKEN=$(curl -s -X POST \ diff --git a/.github/workflows/vault-deploy.yml b/.github/workflows/vault-deploy.yml index 56c9d49b..1378d30e 100644 --- a/.github/workflows/vault-deploy.yml +++ b/.github/workflows/vault-deploy.yml @@ -13,6 +13,8 @@ env: jobs: test: runs-on: ubuntu-latest + env: + VAULT_AUTH_CONFIGURED: ${{ secrets.VAULT_ADDR != '' }} permissions: contents: read id-token: write # Required for GitHub OIDC token @@ -27,13 +29,13 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Set fallback test secrets when Vault is unavailable - if: ${{ secrets.VAULT_ADDR == '' }} + if: ${{ env.VAULT_AUTH_CONFIGURED != 'true' }} run: | echo "JWT_SECRET=fallback-test-secret-value-at-least-32-characters" >> $GITHUB_ENV echo "DATABASE_URL=sqlite::memory:" >> $GITHUB_ENV - name: Get Vault Token via GitHub OIDC - if: ${{ secrets.VAULT_ADDR != '' }} + if: ${{ env.VAULT_AUTH_CONFIGURED == 'true' }} id: vault-auth run: | TOKEN=$(curl -s -X POST \ diff --git a/.gitignore b/.gitignore index 328ed907..5a1d7cbb 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,7 @@ benchmarks/baseline.json /public/worker-*.js.map /public/fallback-*.js /public/fallback-*.js.map + +# Rust backend build +backend/target/ + diff --git a/backend/Cargo.lock b/backend/Cargo.lock new file mode 100644 index 00000000..e6afe569 --- /dev/null +++ b/backend/Cargo.lock @@ -0,0 +1,398 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stellar-insights-backend" +version = "0.1.0" +dependencies = [ + "hex", + "hmac", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "uuid" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +dependencies = [ + "getrandom", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/backend/Cargo.toml b/backend/Cargo.toml new file mode 100644 index 00000000..b084b460 --- /dev/null +++ b/backend/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "stellar-insights-backend" +version = "0.1.0" +edition = "2021" +description = "Backend services for Stellar Insights (webhooks and related modules)" +license = "MIT OR Apache-2.0" + +[dependencies] +hex = "0.4" +hmac = "0.12" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" +uuid = { version = "1", features = ["v4", "serde"] } + +[dev-dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] } diff --git a/backend/WEBHOOKS.md b/backend/WEBHOOKS.md new file mode 100644 index 00000000..25352d41 --- /dev/null +++ b/backend/WEBHOOKS.md @@ -0,0 +1,109 @@ +# Webhook library + +This crate provides a signed delivery loop behind an `HttpTransport` interface. +It is not started by this repository's Next.js application. The deployed Rust +service lives in `Stellar-Analysis/backend`; its dispatcher and persistence must +be integrated there before this frontend PR can establish end-to-end delivery. + +A complete patch for the existing backend is included in +[integration/README.md](integration/README.md), with its pinned source revision, +application instructions, file digests and native test evidence. It repairs the +real service's dispatcher, persistence and authenticated API. Keeping that patch +on this contribution branch does not apply it to the backend or deploy it. + +## Delivery contract + +Create a `WebhookEvent` once for a logical event. Retain its `idempotency_key` +when recovering the event. Every attempt for that event and subscriber sends +the same JSON bytes, signature and `X-Stellar-Insights-Idempotency-Key` header. +Treat a 2xx response as accepted; transport errors and other status codes retry +until `max_attempts` is exhausted. + +`WebhookDeliverer::new` applies real blocking waits before retry attempts. With +the default policy, attempts begin after waits of 0, 100, 200, 400 and 800 ms, +plus the time spent in preceding transport calls. A successful attempt ends +the loop immediately. Each wait is capped by `max_backoff`. +The policy retains nanosecond precision: for attempt `n >= 2`, the requested +wait is `min(initial_backoff * 2^(n - 2), max_backoff)`. Large attempt numbers +saturate at the configured cap without wrapping; zero initial delay or zero +cap still requests no wait. This describes requested delays, not a guarantee +of the operating system's sleep precision. + +An embedding runtime can use `WebhookDeliverer::with_sleeper` and implement +`RetrySleeper`. That implementation must actually wait for the requested delay; +the constructor exists to integrate the runtime's clock, not to defer a retry +after it has already been sent. The caller must bound each HTTP request's own +duration through its transport. + +## Subscriber status + +Status belongs to `(event_id, subscriber_id)`. One subscriber's failed request +does not replace another subscriber's delivered result. + +- `store.by_subscriber(event_id, subscriber_id)` returns one target's status. +- `store.by_event_id(event_id)` returns all target statuses in subscriber-id order. +- `store.len()` counts target statuses, not distinct logical events. + +The event lookup now returns a collection rather than the original single-row +`Option`. This is an intentional API correction for fan-out delivery. Records +are still kept in memory only; they do not survive process restart and are not +an HTTP status endpoint or a durable queue. + +## Signature verification + +Read the exact request body bytes and the `X-Stellar-Insights-Signature` header. +Use `verify_signature(subscriber_secret, body, signature)` before parsing or +acting on the payload. Do not reserialize JSON before verification. Retain the +event id in the receiver's own deduplication record before applying an effect. + +## Validation + +Run `cargo test` from this directory. The delivery regression exercises the +actual loop with an observed clock: HTTP 503, a transport failure, then 204. +It checks attempt timing, stable body and headers, and the stop on success. +A second case sends one event to two subscribers and checks that an exhausted +delivery cannot erase the other subscriber's successful status. The original +signature-tampering regression remains in place. + +## Backoff boundary validation (2026-10-04) + +The two issue-required targets passed **6/6 tests, zero failed or ignored** +with Rust/Cargo 1.98.1 and the unchanged lockfile: + +```bash +cargo test --locked --manifest-path backend/Cargo.toml \ + --test idempotency_key_test --test tampered_payload_test -- --nocapture +``` + +[Successful run 37200239373, job 111430260057](https://github.com/woahwhattheheck/frontend/actions/runs/37200239373/job/111430260057) +compiled the actual library and executed all five delivery tests plus the +signature-tampering test. The delivery target took 0.30 seconds. Source was +product commit `499c7f6c37cb04a541e26d9ca87b503e3689077a`, with exact executed +blobs `80024ec002e6a5a07bffa451aac6ea486cecc367` (deliver.rs) and +`45ef8ef9cd4f61cb9d7d23c2f9dcf48bfd1c36e6` (idempotency_key_test.rs). +The isolated runner commit was `29fa5248754eed2e7989c52f0e1308380e65ff27`; +its temporary workflow is not part of the contribution branch. + +The new observed-clock case previously recorded four requests at zero delay; +it now records cumulative times 0, 250, 750 and 1500 microseconds. The scaling +case previously returned 65.536 seconds at attempt 19 instead of 131.072; +it now doubles until the configured 300-second cap. The same case retains +controls for extreme attempt counts, Duration overflow, zero waits and the +unchanged default schedule. These are requested-delay and library correctness +results, not measured operating-system sleep precision or deployed throughput. + +[Baseline run 37200127974, job 111429935872](https://github.com/woahwhattheheck/frontend/actions/runs/37200127974/job/111429935872) +ran those new cases against original production blob +`5718b74e2bfcb57a986e57c03dc23fb94bfde12b`: **one existing control passed and +both new cases failed**, Cargo exit 101. The substring filter also matched the +existing control, so an overly specific zero-pass log guard stopped that job +before candidate execution. The runner-only correction reused this recorded +baseline without repeating it; no source or test assertion was weakened. + +Actions retained raw logs as artifact `11301904770` for the successful run +(uploaded ZIP SHA-256 `085c8414bff1ea80a7e278c4764c2f1e9d7bb4d87a21ca58e426bc7d9eecc679`) +and `11302477232` for the baseline +(`7f37aa5b1fdbe7d109dd78dfc0e59a1df18c1f67bb124b87ee84d02e5f3ac206`). +Both were configured for seven-day artifact retention; this source-bound +summary remains in Git. No full frontend suite, backend integration replay, +external HTTP delivery, deployment or sponsor acceptance is claimed here. diff --git a/backend/integration/README.md b/backend/integration/README.md new file mode 100644 index 00000000..e4823caa --- /dev/null +++ b/backend/integration/README.md @@ -0,0 +1,182 @@ +# Stellar-Analysis backend webhook repair + +## Target and application + +This patch targets the root of **Stellar-Analysis/backend**, based on commit +`965e916227cf9173cf6ad254a6971429060c1d19`. The frontend issue and its existing +PR are coordination and transport locations; the production patch does not +belong in the frontend's standalone webhook mini-crate. + +Apply the companion `stellar-analysis-backend.patch` to that backend revision: + +```bash +git switch -c fix/durable-webhook-delivery 965e916227cf9173cf6ad254a6971429060c1d19 +git apply --check /path/to/stellar-analysis-backend.patch +git apply /path/to/stellar-analysis-backend.patch +cargo test --locked --test webhook_delivery_regression_tests --test webhook_event_service_tests +``` + +The patch changes 13 existing files and adds two files. The accompanying +`stellar-analysis-backend-manifest.json` records each file's original Git blob +SHA, final Git blob SHA, SHA-256, and size. Every preserved original was checked +against the blob SHA obtained from the pinned GitHub tree. A source audit found +exactly the claimed changes. `git apply --check` succeeds against those original +files. Formatting checks pass for every changed Rust source and test file. + +## Production behavior + +The existing `WebhookDispatcher`, `WebhookService`, event producer service, +SQLite migrations, and authenticated API implement the repair. No additional +dispatcher, standalone runtime, or test-only implementation is introduced. + +- Enqueue persists the event UUID and complete serialized envelope. Retries + reuse the same ID, event timestamp, bytes, headers, and HMAC input after a + dispatcher restart. Legacy rows derive their envelope from the stored event + ID and creation timestamp, then persist it before the first new send. +- A due event is reserved atomically for 30 seconds before delivery. A random + reservation token fences completion writes, preventing an expired worker + from overwriting the result of a newer reservation. Expired reservations can + be recovered by another worker. +- Each request has a 10-second timeout. Redirects are rejected without + contacting another destination. Any non-2xx response or transport failure is + recorded as a failure; receiver response bodies are not retained or returned + through the status API. +- Failure count, last error, last attempt time, and the next due time are + persisted. The first and second failures schedule retries after five and ten + seconds. The exponential helper saturates at 300 seconds if the attempt limit + is later increased. The third recorded failure becomes terminal `failed`. + Inactive or missing subscriptions also terminate without an HTTP request. +- Successful delivery preserves previous failure count and last error and + records the delivered timestamp. The event update and subscription's + `last_fired_at` share one transaction. Database errors propagate instead of + producing success logs. Event enqueue errors also propagate to the producer. +- Authenticated owners can query + `GET /api/webhooks/{webhook_id}/events/{event_id}` or the corresponding + `/api/v1/webhooks/...` resource. The result contains persisted delivery + metadata, excludes payload and secrets, and uses the same 404 response for + absent, mismatched, and non-owned events. Access-token JWT middleware protects + the actual mounted routes. Existing malformed nested route aliases are + preserved for compatibility. +- Successful authenticated registration returns the signing secret once with + `Cache-Control: no-store`. List, detail, and delivery status responses omit + it. The README specifies raw-body HMAC verification and event-ID deduplication. + +Migration `035_webhook_delivery_retries.sql` adds the delivery envelope, +reservation token, and nullable scheduling/timestamp columns, plus a due-event +index. It repairs exhausted legacy `pending` rows to `failed` while preserving +existing errors. Migration 019 remains byte-identical to the pinned source. + +The existing backend event types remain `corridor.health_degraded`, +`anchor.status_changed`, `payment.created`, and `corridor.liquidity_dropped`. +This patch repairs their delivery path. It does not add new anomaly or snapshot +event producers corresponding to other wording in frontend issue 336. + +## Necessary build and authentication prerequisites + +The focused native target exposed existing production prerequisites. These +small fixes are included in the same patch so the real package can compile and +its authenticated behavior can execute: + +| Files | Change and reason | +| --- | --- | +| `src/observability/job_metrics.rs` | Four `with_label_values::<&str>` annotations resolve generic inference for mixed label references. | +| `src/observability/metrics.rs` | One identical explicit type annotation resolves the same inference failure. | +| `src/main.rs` | Converts the configured compression threshold from `u16` to `u64` for the existing tower-http API. Adds the existing validated `JWT_SECRET` as the router's `JwtSecret` extension. | +| `Cargo.toml`, `Cargo.lock` | Enables `jsonwebtoken`'s `aws_lc_rs` feature at its unchanged 10.4 version. The default dependency had no crypto provider and panicked when the real JWT tests signed or verified a token. AWS-LC was already present in the dependency graph; the lockfile adds its JWT edge and required `untrusted` 0.7.1 parser dependency without unrelated version updates. | + +No Cargo target is disabled, no binary is excluded, and no stub authentication +provider replaces the production implementation. No credential values or +provider configuration are changed. + +## Verification + +**Final focused native result: 26 passed, zero failed; Cargo exited 0.** +`webhook_delivery_regression_tests` passed all 21 tests, and +`webhook_event_service_tests` passed all five tests. The production library and +binary compiled with the real JWT provider enabled. The final run retained the +same native targets and succeeded after relocating completed compiler archives +to reduce the shared machine's memory consumption. Evidence: +`stellar_backend_native_final.log`. + +The native regression target imports the exact production webhook source files +by path to execute their private inline tests. It uses the compiled backend +library's real auth, middleware, crypto, validation, and broadcast modules. This +keeps the tests runnable without compiling unrelated broken library unit tests; +the normal production library and binary remain part of the Cargo build. + +The regression target currently contains 21 tests covering: + +- Actual localhost HTTP requests and SQLite persistence for failure followed by + success, dispatcher recreation, stable headers and body, tamper rejection, + wrong-secret rejection, and retained failure metadata. +- Three recorded failures becoming terminal, with no fourth request; no early + retry before the persisted due time; inactive subscriptions sending nothing. +- Receiver redirects not followed and receiver error bodies not persisted. +- Injected SQLite failures propagating and rolling back both delivered state + and `last_fired_at`, while keeping a recoverable reservation. +- Legacy timestamp/envelope handling, preservation through retries, migration + repair, saturated backoff, concurrent selectors, and expired-worker fencing. +- An independent RFC 4231 HMAC vector and existing signature/type checks. +- Actual access-token JWT validation at the mounted versioned path, invalid, + wrong-key, and refresh-token rejection, owner isolation, and creation-only + usable signing-secret disclosure. + +The existing `webhook_event_service_tests` target contains five tests, including +the new failure-injection assertion that a rejected enqueue is not reported as +a successful trigger. + +### Captured intermediate results and limits + +1. The first native regression execution completed with **19 passed and two + failed**. Both failures were the real JWT tests panicking because no crypto + provider was enabled. This exposed the production dependency defect above; + it is not a successful final validation result. Evidence: + `stellar_backend_jwt_provider_failure.log`. +2. Two subsequent attempts after enabling the provider were terminated by + `SIGKILL` while compiling the backend library. They emitted only existing + production warnings and no source error. The shared 8 GiB memory cgroup + recorded OOM kills. Completed compiler archives were moved out of tmpfs to + reduce memory consumption; source and target selection were unchanged. + Evidence from the later failure: `stellar_backend_resource_failure.log`. +3. An earlier `cargo test --locked --lib webhook --no-run` found 41 compilation + errors across unrelated unit-test code and the five production metric + inference problems now fixed. Unrelated failures include missing `Arc` and + `mock_stellar` references, `JobHealthStatus` comparisons without `PartialEq`, + trace executor `Unpin` requirements, stale numeric assertions, and websocket + methods called without their required IP argument. Evidence: + `stellar_backend_build.log`. The full library unit-test suite and all-repo + test suite are **not claimed to pass**. + +Exact native command used in the shared environment, from the backend root: + +```bash +RUSTUP_HOME=/workspace/scratch/1b6b1b08e6b1/toolchain/rustup \ +CARGO_HOME=/workspace/scratch/1b6b1b08e6b1/toolchain/cargo \ +CARGO_BUILD_JOBS=1 \ +CARGO_TARGET_DIR=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target \ +CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 \ +OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include \ +/workspace/scratch/1b6b1b08e6b1/toolchain/cargo/bin/rustup run 1.89.0 \ + cargo test --locked \ + --test webhook_delivery_regression_tests \ + --test webhook_event_service_tests +``` + +The OpenSSL paths compensate for the shared environment lacking `pkg-config`. +The reduced debug output and disabled incremental compilation are resource +settings; neither changes the repository manifest nor removes build targets. + +## Operational boundaries + +Delivery remains at least once. If an HTTP request succeeds but the database +write fails, a later worker may resend that event with identical ID and bytes. +Crashes or failed status writes may therefore produce more physical requests +than the three recorded failure limit. Consumers must verify the raw-body HMAC +and deduplicate by event ID. The status resource exposes the persisted current +state and retained failure metadata; it is not an append-only per-attempt audit +log. Attempts made before migration used old per-attempt IDs and cannot be +retroactively deduplicated against the new stable identity. + +The patch must still be landed in the actual backend repository and its normal +migration process run before these changes affect a deployed service. A copy +carried in the frontend PR is a reviewable handoff, not a backend deployment. diff --git a/backend/integration/stellar-analysis-backend-manifest.json b/backend/integration/stellar-analysis-backend-manifest.json new file mode 100644 index 00000000..4b23b25d --- /dev/null +++ b/backend/integration/stellar-analysis-backend-manifest.json @@ -0,0 +1,146 @@ +{ + "repository": "Stellar-Analysis/backend", + "base_commit": "965e916227cf9173cf6ad254a6971429060c1d19", + "patch_file": "stellar-analysis-backend.patch", + "patch_sha256": "cc062b0ec4f8a65a7bdb4e670fffccf4e01734a97b2f6d26a22fba33fd09ffe5", + "patch_bytes": 81743, + "files": [ + { + "path": "Cargo.lock", + "change": "modified", + "base_git_blob_sha1": "5384403de335c663b745b30cf4a47d291a22c90f", + "git_blob_sha1": "bfe8f7db1c1dd536eb4bb6296654713ee39d1c5f", + "sha256": "8baf28ca19ab32207d0303a5d6218181933af9dc8797831a457fa33bf4d685c6", + "bytes": 152720 + }, + { + "path": "Cargo.toml", + "change": "modified", + "base_git_blob_sha1": "f0fe093ebbb5c3eb962b3337beb1b764afd0fd39", + "git_blob_sha1": "49e1f45338b44b0706b8387d1df7eac9f3501825", + "sha256": "62a380fb4859153e2203993a811f49b0b405d3ecaa48e0b5ee634f953f3b6693", + "bytes": 3595 + }, + { + "path": "README.md", + "change": "modified", + "base_git_blob_sha1": "160135699501a4db3ad4ab8439fc9f19625e7fe4", + "git_blob_sha1": "28fcda321a1e418d75bbbd8bf38b7e34c12f1010", + "sha256": "8168c50657896a2583bdaeb02668f29bb6db4b781aa9cdf540f7adc2a63fdd99", + "bytes": 7226 + }, + { + "path": "migrations/035_webhook_delivery_retries.sql", + "change": "added", + "base_git_blob_sha1": null, + "git_blob_sha1": "89b735a9e69df27ffafcb461ba1972efc65ab6de", + "sha256": "fd6daa775fd668812846eaf02f8b54f5d71ba0e8a6f4fc6cc579a069f3843751", + "bytes": 967 + }, + { + "path": "src/api/v1/mod.rs", + "change": "modified", + "base_git_blob_sha1": "a3677c8afa44f712a391a15ff1765b6a095b37ac", + "git_blob_sha1": "19716833a53a40c461b3ec7ee97154c98c284b90", + "sha256": "0cca91308f35b7625ae0bbf94208fe6a8d56bc468b4a70e1ea856e796ec274a3", + "bytes": 7465 + }, + { + "path": "src/api/webhooks.rs", + "change": "modified", + "base_git_blob_sha1": "9f6d2f041ed6a863b0b2737e9fdd72beeb869ba9", + "git_blob_sha1": "f6abcdcaa28edd47c205e032169d9a2d8cc8fe9e", + "sha256": "789dd22eaac008c8206ce298edd75b553aba7e92ddad63d1a7cee257e9049836", + "bytes": 21565 + }, + { + "path": "src/main.rs", + "change": "modified", + "base_git_blob_sha1": "622bfe12d04b476206149e2fbf42cb42bc922bde", + "git_blob_sha1": "f51ac143c41c8bec4031030468f3f56af7db3519", + "sha256": "266bcd88bc0ad6fb8653c089e827ae405173ecb6a9272185f2887b17e3dcb9d6", + "bytes": 23815 + }, + { + "path": "src/observability/job_metrics.rs", + "change": "modified", + "base_git_blob_sha1": "bdff8373db34997ab1c1b356357330b68c7fbd62", + "git_blob_sha1": "b4d4adc2049f0a740572c4ca09d7a5cf07d1f22c", + "sha256": "2b49a000316ffe2aec8908a3fd624633ad4aa4bb227c583b8288362ed4a7dd08", + "bytes": 15660 + }, + { + "path": "src/observability/metrics.rs", + "change": "modified", + "base_git_blob_sha1": "6e9844267d70b902b2cffcee03ca0fa70a2cc66e", + "git_blob_sha1": "be5696ba26547b216050c3284e74e071791a10e0", + "sha256": "bac2b49f5dde3edde87f9fe4421b918a9471c5956f027555604021cd0df7f82f", + "bytes": 21709 + }, + { + "path": "src/openapi.rs", + "change": "modified", + "base_git_blob_sha1": "b836d305a06ec70791eb86061ef379832efad1d0", + "git_blob_sha1": "65fdfd66e4d6e57c3f3ba1f2f942d0c15430d9ae", + "sha256": "74603fb867ba1e2a07e180fab4198d4cc45e2e421bd8ec5f624558c99b32d2ea", + "bytes": 11011 + }, + { + "path": "src/services/webhook_dispatcher.rs", + "change": "modified", + "base_git_blob_sha1": "7b507755f72dcf61367cccada4fe96bbcef47f7f", + "git_blob_sha1": "b19ab058a26191d8f17cc1cffaafb4450decd479", + "sha256": "aaefa272b3c1d1704d1b825a3b0584b001358412fa2410ad32b0088b14cc1996", + "bytes": 23398 + }, + { + "path": "src/services/webhook_event_service.rs", + "change": "modified", + "base_git_blob_sha1": "6355fb10585cc54cbbcada77714dae15be91e172", + "git_blob_sha1": "d0ff11e7932d073698e6c78a2c115820452badf0", + "sha256": "86d7b92de201cf6edd2b9bb9cb959bce5da48bdf28917ec3be1d466a0db19689", + "bytes": 7274 + }, + { + "path": "src/webhooks/mod.rs", + "change": "modified", + "base_git_blob_sha1": "f3cc3d7216050e31ecd826aa986f61454579604c", + "git_blob_sha1": "30cec2c83576dcef98a6cb12eeb9d13f3b651117", + "sha256": "34ba93c6ac8eac24e87baed6f4123da0cce4747974d1055dc9901744ff1eeb94", + "bytes": 26587 + }, + { + "path": "tests/webhook_delivery_regression_tests.rs", + "change": "added", + "base_git_blob_sha1": null, + "git_blob_sha1": "a471b31a52239d2f9aee22c1ec4fde605d9ad9c7", + "sha256": "3f45e28f73b95c5685b789459133e9931e2b21f594f7cbf95c1a0e275434e339", + "bytes": 676 + }, + { + "path": "tests/webhook_event_service_tests.rs", + "change": "modified", + "base_git_blob_sha1": "2f5e53dabcd3d81b982c58533005e53f36768105", + "git_blob_sha1": "1a0b666dc8f9f7d60c6c4d12e594b61362dd0232", + "sha256": "0509539470449ee63036e91dcc54e444f58e1ed82d9241a6127e68706857728d", + "bytes": 10558 + } + ], + "validation": { + "native_command": "cargo test --locked --test webhook_delivery_regression_tests --test webhook_event_service_tests", + "rust_version": "1.89.0", + "exit_code": 0, + "passed": 26, + "failed": 0, + "targets": { + "webhook_delivery_regression_tests": 21, + "webhook_event_service_tests": 5 + }, + "log": "stellar_backend_native_final.log", + "log_sha256": "6ce78618e42d89b63006639e6630f2be61c20b2745beb759d5e790ae0ea200c3", + "format_check": "passed for all changed Rust files", + "baseline_apply_check": "passed", + "full_unit_suite": "not passed; unrelated baseline compilation errors documented separately", + "applied_file_digest_comparison": "passed for all 15 files after applying patch to original baseline copies" + } +} diff --git a/backend/integration/stellar-analysis-backend.patch b/backend/integration/stellar-analysis-backend.patch new file mode 100644 index 00000000..fd8fbe86 --- /dev/null +++ b/backend/integration/stellar-analysis-backend.patch @@ -0,0 +1,2149 @@ +diff --git a/Cargo.lock b/Cargo.lock +index 5384403de335c663b745b30cf4a47d291a22c90f..bfe8f7db1c1dd536eb4bb6296654713ee39d1c5f 100644 +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -355,6 +355,7 @@ + checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" + dependencies = [ + "aws-lc-sys", ++ "untrusted 0.7.1", + "zeroize", + ] + +@@ -2255,6 +2256,7 @@ + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" + dependencies = [ ++ "aws-lc-rs", + "base64 0.22.1", + "getrandom 0.2.17", + "js-sys", +@@ -3589,7 +3591,7 @@ + "cfg-if", + "getrandom 0.2.17", + "libc", +- "untrusted", ++ "untrusted 0.9.0", + "windows-sys 0.52.0", + ] + +@@ -3817,7 +3819,7 @@ + "aws-lc-rs", + "ring", + "rustls-pki-types", +- "untrusted", ++ "untrusted 0.9.0", + ] + + [[package]] +@@ -5224,6 +5226,12 @@ + + [[package]] + name = "untrusted" ++version = "0.7.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" ++ ++[[package]] ++name = "untrusted" + version = "0.9.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +diff --git a/Cargo.toml b/Cargo.toml +index f0fe093ebbb5c3eb962b3337beb1b764afd0fd39..49e1f45338b44b0706b8387d1df7eac9f3501825 100644 +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -74,7 +74,7 @@ + stellar_sdk = "0.1" + base64 = "0.22" + ed25519-dalek = "2.1" +-jsonwebtoken = "10.4" ++jsonwebtoken = { version = "10.4", features = ["aws_lc_rs"] } + utoipa = { version = "4.2", features = ["axum_extras", "chrono", "uuid"] } + utoipa-swagger-ui = { version = "9.0", features = ["axum"] } + urlencoding = "2.1" +diff --git a/README.md b/README.md +index 160135699501a4db3ad4ab8439fc9f19625e7fe4..28fcda321a1e418d75bbbd8bf38b7e34c12f1010 100644 +--- a/README.md ++++ b/README.md +@@ -82,6 +82,72 @@ + + Integration/load tests live in `tests/` and `load-tests/`. + ++## Outbound webhooks ++ ++An authenticated `POST /api/webhooks` (or `POST /api/v1/webhooks`) returns the ++subscription fields plus its signing `secret` once, with `Cache-Control: ++no-store`. Save this secret in the receiving integration when you register. ++List, detail, and delivery-status responses do not return it. ++ ++Each queued event has a persistent UUID. Its serialized JSON envelope (`id`, ++`event`, `timestamp`, `data`) is saved before delivery and reused byte for byte ++on retries, including after a dispatcher restart. Consumers should deduplicate ++using `X-Zapier-Delivery-ID`, which is the same value as the envelope's `id` and ++the `event_id` returned by `POST /api/webhooks/{id}/test`. ++ ++Requests also include `X-Zapier-Event`, `X-Zapier-Timestamp` (the event creation ++time in Unix seconds), and `X-Zapier-Signature`. The signature is ++`sha256=` over the exact raw request body, keyed with that ++subscription's secret. Verify the raw bytes before parsing JSON or acting on ++an event. Re-serializing JSON before checking the signature can change the ++signed bytes. The creation timestamp remains stable across retries; use the ++event ID to deduplicate an already accepted event. ++ ++Any non-2xx response or transport failure counts as a failed attempt. Redirects ++are treated as failures without following another URL, and failure records do ++not include the receiver's response body. Delivery becomes terminal after ++**three recorded failures**, including the initial request, with a ten-second ++timeout on each request. Retry due times are stored ++in SQLite: the delays after the first and second failures are five and ten ++seconds. The exponential delay is capped at 300 seconds if the attempt limit ++is increased. The dispatcher polls every five seconds, so actual delivery can ++occur later than the due time. The third failure becomes terminal `failed`. ++Inactive or missing subscriptions also become terminal without another send. ++ ++A thirty-second reservation prevents another worker from immediately sending ++the same due event and makes events recoverable after a worker stops. Delivery ++is at least once: an accepted HTTP request whose database update fails can be ++sent again with the same ID and exact body. Crashes and failed database writes ++can therefore cause more physical sends than the recorded failure count. ++Reservation tokens prevent a resumed, expired worker from overwriting a newer ++worker's result. Successful delivery and the ++subscription's `last_fired_at` are recorded in one transaction; database ++failures are reported by the dispatcher instead of reported as success. ++ ++Authenticated subscribers can query ++`GET /api/webhooks/{webhook_id}/events/{event_id}` for `status` (`pending`, ++`delivered`, or `failed`), `retries` (failed attempts, retained after success), ++`last_error`, `created_at`, `next_attempt_at`, `last_attempt_at`, and ++`delivered_at`. The latter three timestamps use Unix seconds and are nullable; ++while a worker holds an event, `next_attempt_at` is its reservation expiry. ++Only the subscription's owner can access the record. Secrets and event data ++are excluded from this response. ++The same resource is available at ++`GET /api/v1/webhooks/{webhook_id}/events/{event_id}` with a valid access-token ++`Authorization: Bearer ...` header. ++ ++Migration `035_webhook_delivery_retries.sql` preserves existing raw payloads ++and recorded errors, marks previously stranded exhausted events as `failed`, ++and freezes legacy envelopes from their stored ID and creation time on their ++next delivery. Attempts sent before this migration used a different ID and ++cannot be retroactively deduplicated with their new, stable ID. ++ ++Run the focused dispatcher, signature, migration, and API tests with: ++ ++```bash ++cargo test --test webhook_delivery_regression_tests --test webhook_event_service_tests ++``` ++ + ## Observability + + - **Logs** — set `LOGSTASH_ENABLED=true` to forward structured logs to an ELK stack (see `docker-compose.elk.yml` in the core repo) +diff --git a/migrations/035_webhook_delivery_retries.sql b/migrations/035_webhook_delivery_retries.sql +new file mode 100644 +index 0000000000000000000000000000000000000000..89b735a9e69df27ffafcb461ba1972efc65ab6de +--- /dev/null ++++ b/migrations/035_webhook_delivery_retries.sql +@@ -0,0 +1,18 @@ ++-- Keep the serialized delivery envelope and retry schedule across restarts. ++-- Existing payloads remain unchanged; the dispatcher freezes legacy envelopes ++-- using their stored event ID and creation time before their next delivery. ++ALTER TABLE webhook_events ADD COLUMN delivery_payload TEXT; ++ALTER TABLE webhook_events ADD COLUMN next_attempt_at INTEGER; ++ALTER TABLE webhook_events ADD COLUMN last_attempt_at INTEGER; ++ALTER TABLE webhook_events ADD COLUMN delivered_at INTEGER; ++ALTER TABLE webhook_events ADD COLUMN delivery_token TEXT; ++ ++-- The old dispatcher stranded the third failure as pending, although its query ++-- excluded retries >= 3. Preserve the recorded error and make those rows terminal. ++UPDATE webhook_events ++SET status = 'failed', ++ last_error = COALESCE(last_error, 'maximum_attempts_exhausted') ++WHERE status = 'pending' AND retries >= 3; ++ ++CREATE INDEX IF NOT EXISTS idx_webhook_events_due ++ON webhook_events(status, next_attempt_at, created_at); +diff --git a/src/api/v1/mod.rs b/src/api/v1/mod.rs +index a3677c8afa44f712a391a15ff1765b6a095b37ac..19716833a53a40c461b3ec7ee97154c98c284b90 100644 +--- a/src/api/v1/mod.rs ++++ b/src/api/v1/mod.rs +@@ -131,9 +131,10 @@ + .with_state(app_state) + .layer(middleware::from_fn(auth_middleware)); + +- let protected_webhook_routes = Router::new() +- .nest("/webhooks", webhooks::routes(pool.clone())) +- .layer(middleware::from_fn(auth_middleware)); ++ let protected_webhook_routes = webhooks::versioned_routes(pool.clone()); ++ ++ let unversioned_webhook_routes = ++ webhooks::routes(pool.clone()).layer(middleware::from_fn(auth_middleware)); + + // 4. RPC routes + let rpc_routes = Router::new() +@@ -189,6 +190,7 @@ + .route("/api/version", get(get_api_version)) + // Preserve existing unversioned endpoints for backward compatibility. + .merge(v1_router) ++ .merge(unversioned_webhook_routes) + .layer(cors) + .layer(middleware::from_fn( + crate::request_id::request_id_middleware, +diff --git a/src/api/webhooks.rs b/src/api/webhooks.rs +index 9f6d2f041ed6a863b0b2737e9fdd72beeb869ba9..f6abcdcaa28edd47c205e032169d9a2d8cc8fe9e 100644 +--- a/src/api/webhooks.rs ++++ b/src/api/webhooks.rs +@@ -43,11 +43,16 @@ + + let service = WebhookService::new(db); + let response = service +- .register_webhook(&auth_user.user_id, request) ++ .register_webhook_with_secret(&auth_user.user_id, request) + .await + .map_err(|e| WebhookApiError::ServerError(e.to_string()))?; + +- Ok((StatusCode::CREATED, Json(response)).into_response()) ++ Ok(( ++ StatusCode::CREATED, ++ [("Cache-Control", "no-store")], ++ Json(response), ++ ) ++ .into_response()) + } + + /// GET /api/webhooks - List webhooks for authenticated user +@@ -181,6 +186,37 @@ + Ok((StatusCode::OK, Json(response)).into_response()) + } + ++/// GET /api/webhooks/:id/events/:event_id - Get delivery status for an owned webhook ++#[utoipa::path( ++ get, ++ path = "/api/webhooks/{id}/events/{event_id}", ++ params( ++ ("id" = String, Path, description = "Webhook ID"), ++ ("event_id" = String, Path, description = "Webhook event ID") ++ ), ++ responses( ++ (status = 200, description = "Webhook event delivery status"), ++ (status = 401, description = "Unauthorized"), ++ (status = 404, description = "Webhook event not found"), ++ (status = 500, description = "Internal server error") ++ ), ++ tag = "Webhooks" ++)] ++pub async fn get_webhook_event_status( ++ State(db): State, ++ auth_user: AuthUser, ++ Path((webhook_id, event_id)): Path<(String, String)>, ++) -> Result { ++ let service = WebhookService::new(db); ++ let event = service ++ .get_event_status(&webhook_id, &event_id, &auth_user.user_id) ++ .await ++ .map_err(|e| WebhookApiError::ServerError(e.to_string()))? ++ .ok_or_else(|| WebhookApiError::NotFound("Webhook event not found".to_string()))?; ++ ++ Ok((StatusCode::OK, Json(event)).into_response()) ++} ++ + /// POST /api/webhooks/:id/test - Queue a test event for delivery + #[utoipa::path( + post, +@@ -263,9 +299,316 @@ + + /// Create webhook routes + pub fn routes(db: SqlitePool) -> Router { ++ Router::new().nest("/api/webhooks", resource_routes(db)) ++} ++ ++/// Resource-relative routes for mounting under the versioned API. ++pub(crate) fn resource_routes(db: SqlitePool) -> Router { + Router::new() +- .route("/api/webhooks", post(register_webhook).get(list_webhooks)) +- .route("/api/webhooks/:id", get(get_webhook).delete(delete_webhook)) +- .route("/api/webhooks/:id/test", post(test_webhook)) ++ .route("/", post(register_webhook).get(list_webhooks)) ++ .route("/:id", get(get_webhook).delete(delete_webhook)) ++ .route("/:id/events/:event_id", get(get_webhook_event_status)) ++ .route("/:id/test", post(test_webhook)) + .with_state(db) + } ++ ++/// Protected routes mounted by the v1 API, with the existing nested paths ++/// retained as aliases for clients that already use them. ++pub(crate) fn versioned_routes(db: SqlitePool) -> Router { ++ Router::new() ++ .nest("/webhooks", resource_routes(db.clone()).merge(routes(db))) ++ .layer(axum::middleware::from_fn( ++ crate::auth_middleware::auth_middleware, ++ )) ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use axum::{body::Body, http::Request}; ++ use sqlx::sqlite::SqlitePoolOptions; ++ use tower::ServiceExt; ++ ++ async fn event_status_db() -> anyhow::Result { ++ let pool = SqlitePoolOptions::new() ++ .max_connections(1) ++ .connect("sqlite::memory:") ++ .await?; ++ ++ sqlx::query("CREATE TABLE users (id TEXT PRIMARY KEY)") ++ .execute(&pool) ++ .await?; ++ sqlx::raw_sql(include_str!("../../migrations/019_oauth_webhooks.sql")) ++ .execute(&pool) ++ .await?; ++ sqlx::raw_sql(include_str!( ++ "../../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&pool) ++ .await?; ++ sqlx::raw_sql( ++ "INSERT INTO users (id) VALUES ('owner'), ('other'); ++ INSERT INTO webhooks (id, user_id, url, event_types, secret) ++ VALUES ('webhook-1', 'owner', 'https://example.com/webhook', 'test', 'private-secret'), ++ ('webhook-other', 'other', 'https://example.com/other', 'test', 'other-secret'); ++ INSERT INTO webhook_events ++ (id, webhook_id, event_type, payload, status, retries, last_error, ++ created_at, next_attempt_at, last_attempt_at, delivered_at) ++ VALUES ++ ('event-1', 'webhook-1', 'test', '{\"private\":\"payload\"}', 'pending', 2, ++ 'Webhook failed with status 503', '2023-11-14T22:13:20Z', ++ 1700000090, 1700000030, NULL);", ++ ) ++ .execute(&pool) ++ .await?; ++ ++ Ok(pool) ++ } ++ ++ fn event_status_request( ++ webhook_id: &str, ++ event_id: &str, ++ user_id: Option<&str>, ++ ) -> Result, axum::http::Error> { ++ let mut request = Request::builder() ++ .uri(format!("/api/webhooks/{webhook_id}/events/{event_id}")) ++ .body(Body::empty())?; ++ ++ if let Some(user_id) = user_id { ++ // Auth middleware adds this extension only after validating the access token. ++ request.extensions_mut().insert(AuthUser { ++ user_id: user_id.to_string(), ++ username: user_id.to_string(), ++ }); ++ } ++ ++ Ok(request) ++ } ++ ++ #[tokio::test] ++ async fn event_status_requires_authentication() -> anyhow::Result<()> { ++ let app = routes(event_status_db().await?); ++ let response = app ++ .oneshot(event_status_request("webhook-1", "event-1", None)?) ++ .await?; ++ ++ assert_eq!(response.status(), StatusCode::UNAUTHORIZED); ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn event_status_returns_delivery_metadata_for_owner() -> anyhow::Result<()> { ++ let app = routes(event_status_db().await?); ++ let response = app ++ .oneshot(event_status_request("webhook-1", "event-1", Some("owner"))?) ++ .await?; ++ ++ assert_eq!(response.status(), StatusCode::OK); ++ let body = axum::body::to_bytes(response.into_body(), 4096).await?; ++ let body: serde_json::Value = serde_json::from_slice(&body)?; ++ assert_eq!( ++ body, ++ json!({ ++ "id": "event-1", ++ "webhook_id": "webhook-1", ++ "event_type": "test", ++ "status": "pending", ++ "retries": 2, ++ "last_error": "Webhook failed with status 503", ++ "created_at": "2023-11-14T22:13:20Z", ++ "next_attempt_at": 1700000090_i64, ++ "last_attempt_at": 1700000030_i64, ++ "delivered_at": null ++ }) ++ ); ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn event_status_hides_nonowned_missing_and_mismatched_events() -> anyhow::Result<()> { ++ let app = routes(event_status_db().await?); ++ ++ for (webhook_id, event_id, user_id) in [ ++ ("webhook-1", "event-1", "other"), ++ ("webhook-other", "event-1", "other"), ++ ("missing-webhook", "event-1", "owner"), ++ ("webhook-1", "missing-event", "owner"), ++ ] { ++ let response = app ++ .clone() ++ .oneshot(event_status_request(webhook_id, event_id, Some(user_id))?) ++ .await?; ++ ++ assert_eq!(response.status(), StatusCode::NOT_FOUND); ++ let body = axum::body::to_bytes(response.into_body(), 4096).await?; ++ let body: serde_json::Value = serde_json::from_slice(&body)?; ++ assert_eq!(body, json!({"error": "Webhook event not found"})); ++ } ++ ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn versioned_event_status_validates_jwt_at_the_mounted_path() -> anyhow::Result<()> { ++ use crate::auth::Claims; ++ use crate::auth_middleware::JwtSecret; ++ use jsonwebtoken::{encode, EncodingKey, Header}; ++ use std::sync::Arc; ++ ++ let secret = "webhook-status-test-key-at-least-32-bytes"; ++ let app = Router::new() ++ .nest("/api/v1", versioned_routes(event_status_db().await?)) ++ .layer(axum::Extension(JwtSecret(Arc::from(secret)))); ++ let token = |user_id: &str, token_type: &str, key: &str| { ++ let now = chrono::Utc::now().timestamp(); ++ encode( ++ &Header::default(), ++ &Claims { ++ sub: user_id.to_string(), ++ username: user_id.to_string(), ++ iat: now, ++ exp: now + 3600, ++ token_type: token_type.to_string(), ++ }, ++ &EncodingKey::from_secret(key.as_bytes()), ++ ) ++ }; ++ ++ let owner_token = token("owner", "access", secret)?; ++ for (uri, bearer, expected) in [ ++ ( ++ "/api/v1/webhooks/webhook-1/events/event-1", ++ None, ++ StatusCode::UNAUTHORIZED, ++ ), ++ ( ++ "/api/v1/webhooks/webhook-1/events/event-1", ++ Some("not-a-token".to_string()), ++ StatusCode::UNAUTHORIZED, ++ ), ++ ( ++ "/api/v1/webhooks/webhook-1/events/event-1", ++ Some(token("owner", "access", "wrong-key")?), ++ StatusCode::UNAUTHORIZED, ++ ), ++ ( ++ "/api/v1/webhooks/webhook-1/events/event-1", ++ Some(token("owner", "refresh", secret)?), ++ StatusCode::UNAUTHORIZED, ++ ), ++ ( ++ "/api/v1/webhooks/webhook-1/events/event-1", ++ Some(token("other", "access", secret)?), ++ StatusCode::NOT_FOUND, ++ ), ++ ( ++ "/api/v1/webhooks/webhook-1/events/event-1", ++ Some(owner_token.clone()), ++ StatusCode::OK, ++ ), ++ ( ++ "/api/v1/webhooks/api/webhooks/webhook-1/events/event-1", ++ Some(owner_token), ++ StatusCode::OK, ++ ), ++ ] { ++ let mut request = Request::builder().uri(uri); ++ if let Some(bearer) = bearer { ++ request = request.header("Authorization", format!("Bearer {bearer}")); ++ } ++ let response = app.clone().oneshot(request.body(Body::empty())?).await?; ++ assert_eq!(response.status(), expected, "route {uri}"); ++ } ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn registration_returns_signing_secret_once_to_authenticated_owner() -> anyhow::Result<()> ++ { ++ use crate::auth::Claims; ++ use crate::auth_middleware::JwtSecret; ++ use crate::webhooks::WebhookSignature; ++ use jsonwebtoken::{encode, EncodingKey, Header}; ++ use std::sync::Arc; ++ ++ let pool = event_status_db().await?; ++ let jwt_secret = "webhook-registration-test-key-at-least-32-bytes"; ++ let app = Router::new() ++ .nest("/api/v1", versioned_routes(pool.clone())) ++ .layer(axum::Extension(JwtSecret(Arc::from(jwt_secret)))); ++ let now = chrono::Utc::now().timestamp(); ++ let token = encode( ++ &Header::default(), ++ &Claims { ++ sub: "owner".to_string(), ++ username: "owner".to_string(), ++ iat: now, ++ exp: now + 3600, ++ token_type: "access".to_string(), ++ }, ++ &EncodingKey::from_secret(jwt_secret.as_bytes()), ++ )?; ++ let payload = json!({ ++ "url": "https://example.com/receiver", ++ "event_types": ["test"] ++ }); ++ let registration = |authorization: Option<&str>| { ++ let mut request = Request::builder() ++ .method("POST") ++ .uri("/api/v1/webhooks") ++ .header("Content-Type", "application/json"); ++ if let Some(authorization) = authorization { ++ request = request.header("Authorization", format!("Bearer {authorization}")); ++ } ++ request.body(Body::from(payload.to_string())) ++ }; ++ ++ let denied = app.clone().oneshot(registration(None)?).await?; ++ assert_eq!(denied.status(), StatusCode::UNAUTHORIZED); ++ let response = app.clone().oneshot(registration(Some(&token))?).await?; ++ assert_eq!(response.status(), StatusCode::CREATED); ++ assert_eq!(response.headers().get("Cache-Control").unwrap(), "no-store"); ++ let body = axum::body::to_bytes(response.into_body(), 4096).await?; ++ let body: serde_json::Value = serde_json::from_slice(&body)?; ++ let webhook_id = body["id"].as_str().unwrap(); ++ let secret = body["secret"].as_str().unwrap(); ++ assert!(!secret.is_empty()); ++ ++ let service = WebhookService::new(pool); ++ let stored = service.get_webhook(webhook_id).await?.unwrap(); ++ assert_eq!(secret, stored.secret); ++ let sample_body = r#"{"event":"test","data":{"value":42}}"#; ++ let signature = WebhookSignature::sign(sample_body, &stored.secret); ++ assert!(WebhookSignature::verify(sample_body, secret, &signature)); ++ let event_id = service ++ .create_webhook_event(webhook_id, "test", json!({"value": 42})) ++ .await?; ++ ++ for path in [ ++ "/api/v1/webhooks".to_string(), ++ format!("/api/v1/webhooks/{webhook_id}"), ++ format!("/api/v1/webhooks/{webhook_id}/events/{event_id}"), ++ ] { ++ let response = app ++ .clone() ++ .oneshot( ++ Request::builder() ++ .uri(&path) ++ .header("Authorization", format!("Bearer {token}")) ++ .body(Body::empty())?, ++ ) ++ .await?; ++ assert_eq!(response.status(), StatusCode::OK, "route {path}"); ++ let body = axum::body::to_bytes(response.into_body(), 8192).await?; ++ let body: serde_json::Value = serde_json::from_slice(&body)?; ++ assert!(body.get("secret").is_none()); ++ if let Some(webhooks) = body["webhooks"].as_array() { ++ assert!(webhooks ++ .iter() ++ .all(|webhook| webhook.get("secret").is_none())); ++ } ++ assert!(!body.to_string().contains(secret)); ++ } ++ Ok(()) ++ } ++} +diff --git a/src/main.rs b/src/main.rs +index 622bfe12d04b476206149e2fbf42cb42bc922bde..f51ac143c41c8bec4031030468f3f56af7db3519 100644 +--- a/src/main.rs ++++ b/src/main.rs +@@ -40,9 +40,8 @@ + concurrency_limit_middleware, panic_recovery_middleware, ApiVersioning, BatchEndpoints, + ConcurrencyLimitState, DatabaseSchemaSeparation, DeprecationWarnings, ETagCachingSupport, + FieldSelectionParameter, MobilePaginationEndpoints, MobileRequestLogging, +- NetworkAwareRpcClient, NetworkContextMiddleware, PushNotificationService, +- ResponseCompression, WebSocketRealTimeUpdates, PushNotificationRegistration, +- Sep10ForMobile, ++ NetworkAwareRpcClient, NetworkContextMiddleware, PushNotificationRegistration, ++ PushNotificationService, ResponseCompression, Sep10ForMobile, WebSocketRealTimeUpdates, + }, + observability::logging::request_response_logging_middleware, + observability::metrics as obs_metrics, +@@ -223,9 +222,8 @@ + tracing::info!("Push notification service initialized"); + + // Initialize SEP-10 for mobile (issue #1376) +- let _sep10_for_mobile = Sep10ForMobile::new( +- stellar_analysis_backend::models::sep10_for_mobile::Config::default(), +- ); ++ let _sep10_for_mobile = ++ Sep10ForMobile::new(stellar_analysis_backend::models::sep10_for_mobile::Config::default()); + tracing::info!("SEP-10 for mobile initialized"); + + // Initialize push notification registration (issue #1377) +@@ -506,11 +504,21 @@ + .route("/graphql/health", get(graphql_health_handler)) + .layer(axum::Extension(Arc::clone(&graphql_api))); + ++ // Protected routes consume the configured secret through this extension. ++ let jwt_secret = std::env::var("JWT_SECRET") ++ .context("JWT_SECRET is required for authenticated API routes")?; ++ + let app = base_routes + .nest("/admin", admin_routes) + .merge(graphql_routes) + .merge(ws_routes) +- .route("/swagger-ui/*path", get(|| async { "Swagger UI documentation" })) ++ .route( ++ "/swagger-ui/*path", ++ get(|| async { "Swagger UI documentation" }), ++ ) ++ .layer(axum::Extension( ++ stellar_analysis_backend::auth_middleware::JwtSecret(Arc::from(jwt_secret)), ++ )) + .layer(middleware::from_fn( + stellar_analysis_backend::payload_limit::payload_limit_middleware, + )) +@@ -544,7 +552,7 @@ + .br(true) + .quality(compression_level) + .compress_when( +- SizeAbove::new(compression_min_size) ++ SizeAbove::new(u64::from(compression_min_size)) + .and(NotForContentType::IMAGES) + .and(NotForContentType::SSE), + ), +diff --git a/src/observability/job_metrics.rs b/src/observability/job_metrics.rs +index bdff8373db34997ab1c1b356357330b68c7fbd62..b4d4adc2049f0a740572c4ca09d7a5cf07d1f22c 100644 +--- a/src/observability/job_metrics.rs ++++ b/src/observability/job_metrics.rs +@@ -269,7 +269,7 @@ + + // Record metrics + JOB_EXECUTIONS_TOTAL +- .with_label_values(&[job_name, "success"]) ++ .with_label_values::<&str>(&[job_name, "success"]) + .inc(); + JOB_DURATION_SECONDS + .with_label_values(&[job_name]) +@@ -296,7 +296,7 @@ + + // Record metrics + JOB_EXECUTIONS_TOTAL +- .with_label_values(&[job_name, "failure"]) ++ .with_label_values::<&str>(&[job_name, "failure"]) + .inc(); + JOB_FAILURES_TOTAL + .with_label_values(&[job_name, &error_type]) +@@ -327,10 +327,10 @@ + + // Record metrics + JOB_EXECUTIONS_TOTAL +- .with_label_values(&[job_name, "timeout"]) ++ .with_label_values::<&str>(&[job_name, "timeout"]) + .inc(); + JOB_FAILURES_TOTAL +- .with_label_values(&[job_name, "timeout"]) ++ .with_label_values::<&str>(&[job_name, "timeout"]) + .inc(); + JOB_DURATION_SECONDS + .with_label_values(&[job_name]) +diff --git a/src/observability/metrics.rs b/src/observability/metrics.rs +index 6e9844267d70b902b2cffcee03ca0fa70a2cc66e..be5696ba26547b216050c3284e74e071791a10e0 100644 +--- a/src/observability/metrics.rs ++++ b/src/observability/metrics.rs +@@ -316,7 +316,7 @@ + + pub fn record_http_error(status_code: u16, method: &str, path: &str) { + HTTP_ERRORS_TOTAL +- .with_label_values(&[&status_code.to_string(), method, path]) ++ .with_label_values::<&str>(&[&status_code.to_string(), method, path]) + .inc(); + ERRORS_TOTAL.inc(); + } +diff --git a/src/openapi.rs b/src/openapi.rs +index b836d305a06ec70791eb86061ef379832efad1d0..65fdfd66e4d6e57c3f3ba1f2f942d0c15430d9ae 100644 +--- a/src/openapi.rs ++++ b/src/openapi.rs +@@ -110,6 +110,7 @@ + crate::api::webhooks::get_webhook, + crate::api::webhooks::delete_webhook, + crate::api::webhooks::test_webhook, ++ crate::api::webhooks::get_webhook_event_status, + // Account Merges + crate::api::account_merges::get_account_merge_stats, + crate::api::account_merges::get_recent_account_merges, +diff --git a/src/services/webhook_dispatcher.rs b/src/services/webhook_dispatcher.rs +index 7b507755f72dcf61367cccada4fe96bbcef47f7f..b19ab058a26191d8f17cc1cffaafb4450decd479 100644 +--- a/src/services/webhook_dispatcher.rs ++++ b/src/services/webhook_dispatcher.rs +@@ -4,9 +4,12 @@ + use reqwest::Client; + use sqlx::SqlitePool; + use std::time::Duration; +-use uuid::Uuid; +- +-use crate::webhooks::{WebhookEventEnvelope, WebhookService, WebhookSignature}; ++ ++use crate::webhooks::{ ++ WebhookEventEnvelope, WebhookService, WebhookSignature, MAX_WEBHOOK_ATTEMPTS, ++}; ++ ++const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + + /// Webhook dispatcher - sends events to webhooks asynchronously + pub struct WebhookDispatcher { +@@ -19,7 +22,8 @@ + #[must_use] + pub fn new(db: SqlitePool) -> Self { + let http_client = Client::builder() +- .timeout(Duration::from_secs(10)) ++ .timeout(REQUEST_TIMEOUT) ++ .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap_or_else(|_| Client::new()); + +@@ -46,77 +50,108 @@ + let service = WebhookService::new(self.db.clone()); + + // Fetch pending events (max 10 per run) +- let events = service.get_pending_events(10).await?; +- +- for (event_id, webhook_id, event_type, payload_str) in events { ++ let events = service.get_pending_deliveries(10).await?; ++ ++ for event in events { ++ let Some(token) = service.reserve_event(&event).await? else { ++ continue; ++ }; ++ + // Get webhook details +- let webhook = if let Some(w) = service.get_webhook(&webhook_id).await? { ++ let webhook = if let Some(w) = service.get_webhook(&event.webhook_id).await? { + w + } else { + // Webhook was deleted, mark event as failed +- let _ = service +- .update_event_status(&event_id, "failed", Some("webhook_deleted"), 0) +- .await; ++ service ++ .finish_reserved_event( ++ &event.id, ++ &token, ++ "failed", ++ Some("webhook_deleted"), ++ event.retries, ++ ) ++ .await?; + continue; + }; + + if !webhook.is_active { +- let _ = service +- .update_event_status(&event_id, "failed", Some("webhook_inactive"), 0) +- .await; ++ service ++ .finish_reserved_event( ++ &event.id, ++ &token, ++ "failed", ++ Some("webhook_inactive"), ++ event.retries, ++ ) ++ .await?; + continue; + } + ++ let body = match event.delivery_body() { ++ Ok(body) => body, ++ Err(error) => { ++ service ++ .finish_reserved_event( ++ &event.id, ++ &token, ++ "failed", ++ Some(&format!("invalid_event_payload: {error}")), ++ event.retries, ++ ) ++ .await?; ++ continue; ++ } ++ }; ++ let body = service.persist_delivery_payload(&event.id, &body).await?; ++ if !service.owns_reservation(&event.id, &token).await? { ++ continue; ++ } ++ + // Attempt delivery + match self +- .deliver_webhook(&webhook.url, &payload_str, &webhook.secret, &event_type) ++ .deliver_webhook(&webhook.url, &body, &webhook.secret) + .await + { + Ok(()) => { + // Success +- let _ = service +- .update_event_status(&event_id, "delivered", None, 0) +- .await; +- +- // Update webhook's last_fired_at +- let _ = service.update_last_fired(&webhook_id).await; ++ service ++ .finish_reserved_event(&event.id, &token, "delivered", None, event.retries) ++ .await?; + + tracing::info!( + "Webhook delivered successfully: webhook_id={}, event={}", +- webhook_id, +- event_type ++ event.webhook_id, ++ event.event_type + ); + } + Err(e) => { +- // Determine retry count from event +- let current_retries = self.get_event_retries(&event_id).await.unwrap_or(0); +- +- if current_retries < 3 { +- // Retry later +- let _ = service +- .update_event_status( +- &event_id, +- "pending", +- Some(&e.to_string()), +- current_retries + 1, +- ) +- .await; +- ++ let failures = event.retries + 1; ++ let status = if failures < MAX_WEBHOOK_ATTEMPTS { ++ "pending" ++ } else { ++ "failed" ++ }; ++ service ++ .finish_reserved_event( ++ &event.id, ++ &token, ++ status, ++ Some(&e.to_string()), ++ failures, ++ ) ++ .await?; ++ ++ if status == "pending" { + tracing::warn!( + "Webhook delivery failed (will retry): webhook_id={}, error={}, retries={}", +- webhook_id, ++ event.webhook_id, + e, +- current_retries + 1 ++ failures + ); + } else { +- // Max retries exceeded +- let _ = service +- .update_event_status(&event_id, "failed", Some(&e.to_string()), 3) +- .await; +- + tracing::error!( + "Webhook delivery failed (max retries): webhook_id={}, error={}", +- webhook_id, ++ event.webhook_id, + e + ); + } +@@ -128,74 +163,496 @@ + } + + /// Deliver webhook to URL +- async fn deliver_webhook( +- &self, +- url: &str, +- payload: &str, +- secret: &str, +- event_type: &str, +- ) -> Result<()> { +- let delivery_id = Uuid::new_v4().to_string(); +- let timestamp = chrono::Utc::now().timestamp(); +- +- // Create envelope +- let envelope = WebhookEventEnvelope { +- id: delivery_id.clone(), +- event: event_type.to_string(), +- timestamp, +- data: serde_json::from_str(payload)?, +- }; +- +- let body = serde_json::to_string(&envelope)?; +- let signature = WebhookSignature::sign(&body, secret); ++ async fn deliver_webhook(&self, url: &str, body: &str, secret: &str) -> Result<()> { ++ // Headers and signature describe the exact envelope frozen in the DB. ++ let envelope: WebhookEventEnvelope = serde_json::from_str(body)?; ++ let signature = WebhookSignature::sign(body, secret); + + tracing::debug!( + "Sending webhook to {}: delivery_id={}, signature={}...", + url, +- delivery_id, ++ envelope.id, + &signature[..20] + ); + + let response = self + .http_client + .post(url) +- .header("X-Zapier-Event", event_type) ++ .timeout(REQUEST_TIMEOUT) ++ .header("X-Zapier-Event", envelope.event) + .header("X-Zapier-Signature", signature) +- .header("X-Zapier-Timestamp", timestamp.to_string()) +- .header("X-Zapier-Delivery-ID", delivery_id) ++ .header("X-Zapier-Timestamp", envelope.timestamp.to_string()) ++ .header("X-Zapier-Delivery-ID", envelope.id) + .header("Content-Type", "application/json") +- .body(body) ++ .body(body.to_owned()) + .send() + .await?; + + if response.status().is_success() { + Ok(()) + } else { +- anyhow::bail!( +- "Webhook failed with status {}: {}", +- response.status(), +- response.text().await.unwrap_or_default() +- ) +- } +- } +- +- /// Get current retry count for an event +- async fn get_event_retries(&self, event_id: &str) -> Result { +- let retries: Option = +- sqlx::query_scalar("SELECT retries FROM webhook_events WHERE id = ?") +- .bind(event_id) +- .fetch_optional(&self.db) +- .await?; +- +- Ok(retries.unwrap_or(0) as i32) ++ anyhow::bail!("Webhook failed with status {}", response.status()) ++ } + } + } + + #[cfg(test)] + mod tests { +- #[test] +- fn test_webhook_dispatcher_creation() { +- // This is a smoke test for basic creation +- // Full tests would require mocking the database and HTTP client ++ use super::*; ++ use anyhow::Context; ++ use axum::{ ++ extract::State, ++ http::{HeaderMap, StatusCode}, ++ routing::post, ++ Router, ++ }; ++ use serde_json::json; ++ use sqlx::sqlite::SqlitePoolOptions; ++ use std::sync::Arc; ++ use tokio::{net::TcpListener, sync::Mutex, task::JoinHandle}; ++ ++ use crate::webhooks::WebhookEventStatus; ++ ++ const WEBHOOK_ID: &str = "webhook-1"; ++ const SUBSCRIBER_SECRET: &str = "subscriber-secret"; ++ ++ #[derive(Clone)] ++ struct CapturedRequest { ++ headers: HeaderMap, ++ body: String, ++ } ++ ++ struct ReceiverState { ++ statuses: Vec, ++ requests: Mutex>, ++ redirect_to: Option, ++ } ++ ++ struct TestReceiver { ++ url: String, ++ state: Arc, ++ task: JoinHandle>, ++ } ++ ++ impl TestReceiver { ++ async fn start(statuses: Vec) -> Result { ++ Self::start_with_redirect(statuses, None).await ++ } ++ ++ async fn start_with_redirect( ++ statuses: Vec, ++ redirect_to: Option, ++ ) -> Result { ++ let listener = TcpListener::bind("127.0.0.1:0").await?; ++ let address = listener.local_addr()?; ++ let state = Arc::new(ReceiverState { ++ statuses, ++ requests: Mutex::new(Vec::new()), ++ redirect_to, ++ }); ++ let app = Router::new() ++ .route("/webhook", post(capture_request)) ++ .with_state(state.clone()); ++ let task = tokio::spawn(async move { axum::serve(listener, app).await }); ++ ++ Ok(Self { ++ url: format!("http://{address}/webhook"), ++ state, ++ task, ++ }) ++ } ++ ++ async fn requests(&self) -> Vec { ++ self.state.requests.lock().await.clone() ++ } ++ } ++ ++ impl Drop for TestReceiver { ++ fn drop(&mut self) { ++ self.task.abort(); ++ } ++ } ++ ++ async fn capture_request( ++ State(state): State>, ++ headers: HeaderMap, ++ body: String, ++ ) -> (StatusCode, HeaderMap, &'static str) { ++ let mut requests = state.requests.lock().await; ++ let status = state ++ .statuses ++ .get(requests.len()) ++ .copied() ++ .unwrap_or(StatusCode::SERVICE_UNAVAILABLE); ++ requests.push(CapturedRequest { headers, body }); ++ let mut headers = HeaderMap::new(); ++ if let Some(location) = &state.redirect_to { ++ headers.insert(axum::http::header::LOCATION, location.parse().unwrap()); ++ } ++ (status, headers, "receiver response") ++ } ++ ++ async fn dispatcher_db(url: &str, active: bool) -> Result { ++ let pool = SqlitePoolOptions::new() ++ .max_connections(1) ++ .connect("sqlite::memory:") ++ .await?; ++ sqlx::query("CREATE TABLE users (id TEXT PRIMARY KEY)") ++ .execute(&pool) ++ .await?; ++ sqlx::raw_sql(include_str!("../../migrations/019_oauth_webhooks.sql")) ++ .execute(&pool) ++ .await?; ++ sqlx::raw_sql(include_str!( ++ "../../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&pool) ++ .await?; ++ sqlx::query("INSERT INTO users (id) VALUES ('owner')") ++ .execute(&pool) ++ .await?; ++ sqlx::query( ++ "INSERT INTO webhooks (id, user_id, url, event_types, secret, is_active) ++ VALUES (?, 'owner', ?, 'test', ?, ?)", ++ ) ++ .bind(WEBHOOK_ID) ++ .bind(url) ++ .bind(SUBSCRIBER_SECRET) ++ .bind(active) ++ .execute(&pool) ++ .await?; ++ ++ Ok(pool) ++ } ++ ++ async fn queue_event(pool: &SqlitePool) -> Result { ++ WebhookService::new(pool.clone()) ++ .create_webhook_event(WEBHOOK_ID, "test", json!({"amount": 42})) ++ .await ++ } ++ ++ async fn event_state(pool: &SqlitePool, event_id: &str) -> Result { ++ Ok(sqlx::query_as::<_, WebhookEventStatus>( ++ "SELECT id, webhook_id, event_type, status, retries, last_error, created_at, ++ next_attempt_at, last_attempt_at, delivered_at ++ FROM webhook_events WHERE id = ?", ++ ) ++ .bind(event_id) ++ .fetch_one(pool) ++ .await?) ++ } ++ ++ async fn make_due(pool: &SqlitePool, event_id: &str) -> Result<()> { ++ sqlx::query("UPDATE webhook_events SET next_attempt_at = 0 WHERE id = ?") ++ .bind(event_id) ++ .execute(pool) ++ .await?; ++ Ok(()) ++ } ++ ++ async fn last_fired_at(pool: &SqlitePool) -> Result> { ++ Ok( ++ sqlx::query_scalar("SELECT last_fired_at FROM webhooks WHERE id = ?") ++ .bind(WEBHOOK_ID) ++ .fetch_one(pool) ++ .await?, ++ ) ++ } ++ ++ #[tokio::test] ++ async fn retries_survive_dispatcher_restart_with_identical_signed_delivery() -> Result<()> { ++ let receiver = ++ TestReceiver::start(vec![StatusCode::INTERNAL_SERVER_ERROR, StatusCode::OK]).await?; ++ let pool = dispatcher_db(&receiver.url, true).await?; ++ let event_id = queue_event(&pool).await?; ++ let first_dispatcher = WebhookDispatcher::new(pool.clone()); ++ let before = chrono::Utc::now().timestamp(); ++ ++ first_dispatcher.process_pending_events().await?; ++ let pending = event_state(&pool, &event_id).await?; ++ assert_eq!(pending.status, "pending"); ++ assert_eq!(pending.retries, 1); ++ assert!(pending ++ .last_error ++ .as_deref() ++ .is_some_and(|e| e.contains("500"))); ++ assert!( ++ pending ++ .next_attempt_at ++ .context("retry must have a due time")? ++ >= before + 5 ++ ); ++ assert!(pending.last_attempt_at.is_some()); ++ assert_eq!(pending.delivered_at, None); ++ assert_eq!(last_fired_at(&pool).await?, None); ++ drop(first_dispatcher); ++ ++ let restarted_dispatcher = WebhookDispatcher::new(pool.clone()); ++ restarted_dispatcher.process_pending_events().await?; ++ assert_eq!(receiver.requests().await.len(), 1); ++ assert_eq!(event_state(&pool, &event_id).await?.retries, 1); ++ ++ make_due(&pool, &event_id).await?; ++ restarted_dispatcher.process_pending_events().await?; ++ ++ let delivered = event_state(&pool, &event_id).await?; ++ assert_eq!(delivered.status, "delivered"); ++ assert_eq!(delivered.retries, 1); ++ assert_eq!(delivered.last_error, pending.last_error); ++ assert_eq!(delivered.next_attempt_at, None); ++ assert!(delivered.delivered_at.is_some()); ++ assert!(last_fired_at(&pool).await?.is_some()); ++ ++ let requests = receiver.requests().await; ++ assert_eq!(requests.len(), 2); ++ let first = &requests[0]; ++ let second = &requests[1]; ++ assert_eq!(first.body, second.body); ++ for name in [ ++ "X-Zapier-Event", ++ "X-Zapier-Signature", ++ "X-Zapier-Timestamp", ++ "X-Zapier-Delivery-ID", ++ "Content-Type", ++ ] { ++ assert_eq!(first.headers.get(name), second.headers.get(name)); ++ assert!(first.headers.contains_key(name)); ++ } ++ ++ let envelope: WebhookEventEnvelope = serde_json::from_str(&first.body)?; ++ assert_eq!(envelope.id, event_id); ++ assert_eq!(envelope.event, "test"); ++ assert_eq!(envelope.data, json!({"amount": 42})); ++ assert_eq!( ++ first ++ .headers ++ .get("X-Zapier-Delivery-ID") ++ .context("delivery ID header")? ++ .to_str()?, ++ event_id ++ ); ++ assert_eq!( ++ first ++ .headers ++ .get("X-Zapier-Timestamp") ++ .context("timestamp header")? ++ .to_str()?, ++ envelope.timestamp.to_string() ++ ); ++ let signature = first ++ .headers ++ .get("X-Zapier-Signature") ++ .context("signature header")? ++ .to_str()?; ++ assert!(WebhookSignature::verify( ++ &first.body, ++ SUBSCRIBER_SECRET, ++ signature ++ )); ++ assert!(!WebhookSignature::verify( ++ &format!("{} ", first.body), ++ SUBSCRIBER_SECRET, ++ signature ++ )); ++ assert!(!WebhookSignature::verify( ++ &first.body, ++ "different-subscriber-secret", ++ signature ++ )); ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn three_failed_attempts_become_terminal_without_a_fourth_request() -> Result<()> { ++ let receiver = TestReceiver::start(vec![StatusCode::SERVICE_UNAVAILABLE]).await?; ++ let pool = dispatcher_db(&receiver.url, true).await?; ++ let event_id = queue_event(&pool).await?; ++ ++ for attempt in 1..=3 { ++ let dispatcher = WebhookDispatcher::new(pool.clone()); ++ let before = chrono::Utc::now().timestamp(); ++ dispatcher.process_pending_events().await?; ++ ++ let state = event_state(&pool, &event_id).await?; ++ assert_eq!(state.retries, attempt); ++ assert!(state ++ .last_error ++ .as_deref() ++ .is_some_and(|e| e.contains("503"))); ++ assert_eq!(state.delivered_at, None); ++ assert_eq!(receiver.requests().await.len(), attempt as usize); ++ ++ if attempt < 3 { ++ assert_eq!(state.status, "pending"); ++ let delay = if attempt == 1 { 5 } else { 10 }; ++ assert!(state.next_attempt_at.context("retry due time")? >= before + delay); ++ dispatcher.process_pending_events().await?; ++ assert_eq!(receiver.requests().await.len(), attempt as usize); ++ make_due(&pool, &event_id).await?; ++ } else { ++ assert_eq!(state.status, "failed"); ++ assert_eq!(state.next_attempt_at, None); ++ } ++ } ++ ++ // Even an explicitly due timestamp cannot revive a terminal delivery. ++ make_due(&pool, &event_id).await?; ++ WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await?; ++ assert_eq!(receiver.requests().await.len(), 3); ++ assert_eq!(event_state(&pool, &event_id).await?.status, "failed"); ++ assert_eq!(last_fired_at(&pool).await?, None); ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn redirects_do_not_contact_another_receiver_or_store_response_bodies() -> Result<()> { ++ let destination = TestReceiver::start(vec![StatusCode::OK]).await?; ++ let redirect = TestReceiver::start_with_redirect( ++ vec![StatusCode::FOUND], ++ Some(destination.url.clone()), ++ ) ++ .await?; ++ let pool = dispatcher_db(&redirect.url, true).await?; ++ let event_id = queue_event(&pool).await?; ++ WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await?; ++ let state = event_state(&pool, &event_id).await?; ++ assert_eq!(state.status, "pending"); ++ assert_eq!(state.retries, 1); ++ let error = state ++ .last_error ++ .context("redirect failure must be recorded")?; ++ assert!(error.contains("302")); ++ assert!(!error.contains("receiver response")); ++ assert_eq!(redirect.requests().await.len(), 1); ++ assert!(destination.requests().await.is_empty()); ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn persistence_errors_propagate_and_roll_back_delivery_state() -> Result<()> { ++ for (trigger, expected_error) in [ ++ ( ++ "CREATE TRIGGER reject_delivered BEFORE UPDATE OF status ON webhook_events ++ WHEN NEW.status = 'delivered' ++ BEGIN SELECT RAISE(ABORT, 'reject delivered'); END;", ++ "reject delivered", ++ ), ++ ( ++ "CREATE TRIGGER reject_last_fired BEFORE UPDATE OF last_fired_at ON webhooks ++ BEGIN SELECT RAISE(ABORT, 'reject last_fired_at'); END;", ++ "reject last_fired_at", ++ ), ++ ] { ++ let receiver = TestReceiver::start(vec![StatusCode::OK]).await?; ++ let pool = dispatcher_db(&receiver.url, true).await?; ++ let event_id = queue_event(&pool).await?; ++ sqlx::raw_sql(trigger).execute(&pool).await?; ++ let before = chrono::Utc::now().timestamp(); ++ ++ let error = WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await ++ .err() ++ .context("injected persistence failure must propagate")?; ++ assert!(error.to_string().contains(expected_error)); ++ ++ let state = event_state(&pool, &event_id).await?; ++ assert_eq!(state.status, "pending"); ++ assert_eq!(state.retries, 0); ++ assert_eq!(state.last_error, None); ++ assert_eq!(state.delivered_at, None); ++ assert!(state.last_attempt_at.is_some()); ++ assert!(state.next_attempt_at.context("reservation must remain")? >= before + 30); ++ assert_eq!(last_fired_at(&pool).await?, None); ++ assert_eq!(receiver.requests().await.len(), 1); ++ ++ WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await?; ++ assert_eq!(receiver.requests().await.len(), 1); ++ } ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn inactive_webhook_does_not_send_or_reset_failure_count() -> Result<()> { ++ let receiver = TestReceiver::start(vec![StatusCode::OK]).await?; ++ let pool = dispatcher_db(&receiver.url, false).await?; ++ let event_id = queue_event(&pool).await?; ++ sqlx::query( ++ "UPDATE webhook_events SET retries = 2, last_error = 'previous failure' WHERE id = ?", ++ ) ++ .bind(&event_id) ++ .execute(&pool) ++ .await?; ++ ++ WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await?; ++ ++ let state = event_state(&pool, &event_id).await?; ++ assert_eq!(state.status, "failed"); ++ assert_eq!(state.retries, 2); ++ assert_eq!(state.last_error.as_deref(), Some("webhook_inactive")); ++ assert_eq!(state.next_attempt_at, None); ++ assert_eq!(state.delivered_at, None); ++ assert_eq!(last_fired_at(&pool).await?, None); ++ assert!(receiver.requests().await.is_empty()); ++ Ok(()) ++ } ++ ++ #[tokio::test] ++ async fn legacy_sqlite_timestamp_is_frozen_before_retry() -> Result<()> { ++ let receiver = ++ TestReceiver::start(vec![StatusCode::INTERNAL_SERVER_ERROR, StatusCode::OK]).await?; ++ let pool = dispatcher_db(&receiver.url, true).await?; ++ let event_id = "legacy-event"; ++ sqlx::query( ++ "INSERT INTO webhook_events ++ (id, webhook_id, event_type, payload, status, retries, created_at) ++ VALUES (?, ?, 'test', '{\"legacy\":true}', 'pending', 0, '2023-11-14 22:13:20')", ++ ) ++ .bind(event_id) ++ .bind(WEBHOOK_ID) ++ .execute(&pool) ++ .await?; ++ ++ WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await?; ++ let first = receiver.requests().await; ++ assert_eq!(first.len(), 1); ++ let envelope: WebhookEventEnvelope = serde_json::from_str(&first[0].body)?; ++ assert_eq!(envelope.id, event_id); ++ assert_eq!(envelope.timestamp, 1_700_000_000); ++ assert_eq!(envelope.data, json!({"legacy": true})); ++ let frozen_body: Option = ++ sqlx::query_scalar("SELECT delivery_payload FROM webhook_events WHERE id = ?") ++ .bind(event_id) ++ .fetch_one(&pool) ++ .await?; ++ assert_eq!(frozen_body.as_deref(), Some(first[0].body.as_str())); ++ ++ // Retries use the frozen body even if the original row is later edited. ++ sqlx::query( ++ "UPDATE webhook_events SET payload = '{\"legacy\":false}', ++ created_at = '2030-01-01 00:00:00', next_attempt_at = 0 WHERE id = ?", ++ ) ++ .bind(event_id) ++ .execute(&pool) ++ .await?; ++ WebhookDispatcher::new(pool.clone()) ++ .process_pending_events() ++ .await?; ++ ++ let requests = receiver.requests().await; ++ assert_eq!(requests.len(), 2); ++ assert_eq!(requests[0].body, requests[1].body); ++ assert_eq!(event_state(&pool, event_id).await?.status, "delivered"); ++ Ok(()) + } + } +diff --git a/src/services/webhook_event_service.rs b/src/services/webhook_event_service.rs +index 6355fb10585cc54cbbcada77714dae15be91e172..d0ff11e7932d073698e6c78a2c115820452badf0 100644 +--- a/src/services/webhook_event_service.rs ++++ b/src/services/webhook_event_service.rs +@@ -140,10 +140,9 @@ + } + + // Create webhook event for delivery +- let _ = self +- .webhook_service ++ self.webhook_service + .create_webhook_event(&webhook.id, event_type_str, payload.clone()) +- .await; ++ .await?; + + tracing::info!( + "Webhook event triggered: webhook_id={}, event_type={}", +diff --git a/src/webhooks/mod.rs b/src/webhooks/mod.rs +index f3cc3d7216050e31ecd826aa986f61454579604c..30cec2c83576dcef98a6cb12eeb9d13f3b651117 100644 +--- a/src/webhooks/mod.rs ++++ b/src/webhooks/mod.rs +@@ -12,6 +12,20 @@ + type HmacSha256 = Hmac; + + pub use channel::{WebhookChannel, WebhookEndpoint}; ++ ++/// Maximum failed delivery attempts, including the initial attempt. ++pub const MAX_WEBHOOK_ATTEMPTS: i32 = 3; ++const RETRY_BASE_SECONDS: i64 = 5; ++const RETRY_MAX_SECONDS: i64 = 300; ++// Longer than the dispatcher's per-request timeout. A failed status write or ++// worker restart must not immediately send the same event again. ++const DELIVERY_RESERVATION_SECONDS: i64 = 30; ++ ++fn retry_delay_seconds(failures: i32) -> i64 { ++ RETRY_BASE_SECONDS ++ .saturating_mul(2_i64.saturating_pow(failures.saturating_sub(1).max(0) as u32)) ++ .min(RETRY_MAX_SECONDS) ++} + + /// Webhook signature - for verifying webhook requests + pub struct WebhookSignature; +@@ -70,6 +84,14 @@ + pub created_at: String, + } + ++/// Returned once at registration so the subscriber can verify deliveries. ++#[derive(Debug, Serialize)] ++pub struct WebhookRegistrationResponse { ++ #[serde(flatten)] ++ pub webhook: WebhookResponse, ++ pub secret: String, ++} ++ + /// Webhook event envelope + #[derive(Debug, Serialize, Deserialize)] + pub struct WebhookEventEnvelope { +@@ -77,6 +99,57 @@ + pub event: String, + pub timestamp: i64, + pub data: serde_json::Value, ++} ++ ++/// Persisted delivery state; subscriber secrets and event data are not exposed. ++#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] ++pub struct WebhookEventStatus { ++ pub id: String, ++ pub webhook_id: String, ++ pub event_type: String, ++ pub status: String, ++ /// Number of failed attempts, retained after eventual success. ++ pub retries: i32, ++ pub last_error: Option, ++ pub created_at: String, ++ pub next_attempt_at: Option, ++ pub last_attempt_at: Option, ++ pub delivered_at: Option, ++} ++ ++#[derive(Debug, sqlx::FromRow)] ++pub(crate) struct PendingWebhookEvent { ++ pub id: String, ++ pub webhook_id: String, ++ pub event_type: String, ++ pub payload: String, ++ pub retries: i32, ++ pub created_at: String, ++ pub delivery_payload: Option, ++} ++ ++impl PendingWebhookEvent { ++ /// Older events have no stored envelope yet. Derive it from immutable ++ /// database fields, accepting both service and SQLite timestamp formats. ++ pub(crate) fn delivery_body(&self) -> anyhow::Result { ++ if let Some(body) = &self.delivery_payload { ++ return Ok(body.clone()); ++ } ++ ++ let timestamp = chrono::DateTime::parse_from_rfc3339(&self.created_at) ++ .map(|time| time.timestamp()) ++ .or_else(|_| { ++ chrono::NaiveDateTime::parse_from_str(&self.created_at, "%Y-%m-%d %H:%M:%S%.f") ++ .map(|time| time.and_utc().timestamp()) ++ })?; ++ ++ Ok(serde_json::to_string(&WebhookEventEnvelope { ++ id: self.id.clone(), ++ event: self.event_type.clone(), ++ timestamp, ++ data: serde_json::from_str(&self.payload)?, ++ })?) ++ } + } + + /// Event types that can trigger webhooks +@@ -132,6 +205,18 @@ + user_id: &str, + request: CreateWebhookRequest, + ) -> anyhow::Result { ++ Ok(self ++ .register_webhook_with_secret(user_id, request) ++ .await? ++ .webhook) ++ } ++ ++ /// Register a webhook and return its signing secret to its creator once. ++ pub async fn register_webhook_with_secret( ++ &self, ++ user_id: &str, ++ request: CreateWebhookRequest, ++ ) -> anyhow::Result { + let id = Uuid::new_v4().to_string(); + let secret = Uuid::new_v4().to_string(); + let event_types_str = request.event_types.join(","); +@@ -161,13 +246,16 @@ + .execute(&self.db) + .await?; + +- Ok(WebhookResponse { +- id, +- url: request.url, +- event_types: request.event_types, +- filters: request.filters, +- is_active: true, +- created_at: now, ++ Ok(WebhookRegistrationResponse { ++ webhook: WebhookResponse { ++ id, ++ url: request.url, ++ event_types: request.event_types, ++ filters: request.filters, ++ is_active: true, ++ created_at: now, ++ }, ++ secret, + }) + } + +@@ -225,11 +313,17 @@ + ) -> anyhow::Result { + let id = Uuid::new_v4().to_string(); + let payload_str = payload.to_string(); +- let now = chrono::Utc::now().to_rfc3339(); ++ let now = chrono::Utc::now(); ++ let delivery_payload = serde_json::to_string(&WebhookEventEnvelope { ++ id: id.clone(), ++ event: event_type.to_string(), ++ timestamp: now.timestamp(), ++ data: payload, ++ })?; + + sqlx::query( +- "INSERT INTO webhook_events (id, webhook_id, event_type, payload, status, retries, created_at) +- VALUES (?, ?, ?, ?, ?, ?, ?)" ++ "INSERT INTO webhook_events (id, webhook_id, event_type, payload, status, retries, created_at, delivery_payload) ++ VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) + .bind(id.clone()) + .bind(webhook_id) +@@ -237,48 +331,132 @@ + .bind(payload_str) + .bind("pending") + .bind(0) ++ .bind(now.to_rfc3339()) ++ .bind(delivery_payload) ++ .execute(&self.db) ++ .await?; ++ ++ Ok(id) ++ } ++ ++ /// Get pending events whose persisted retry time has arrived. ++ pub async fn get_pending_events( ++ &self, ++ limit: usize, ++ ) -> anyhow::Result> { ++ Ok(self ++ .get_pending_deliveries(limit) ++ .await? ++ .into_iter() ++ .map(|event| (event.id, event.webhook_id, event.event_type, event.payload)) ++ .collect()) ++ } ++ ++ pub(crate) async fn get_pending_deliveries( ++ &self, ++ limit: usize, ++ ) -> anyhow::Result> { ++ let query_limit = i64::try_from(limit)?; ++ ++ Ok(sqlx::query_as::<_, PendingWebhookEvent>( ++ "SELECT we.id, we.webhook_id, we.event_type, we.payload, ++ we.retries, we.created_at, we.delivery_payload ++ FROM webhook_events we ++ WHERE we.status = 'pending' AND we.retries < ? ++ AND (we.next_attempt_at IS NULL OR we.next_attempt_at <= ?) ++ ORDER BY we.created_at ASC, we.id ASC ++ LIMIT ?", ++ ) ++ .bind(MAX_WEBHOOK_ATTEMPTS) ++ .bind(chrono::Utc::now().timestamp()) ++ .bind(query_limit) ++ .fetch_all(&self.db) ++ .await?) ++ } ++ ++ /// Atomically reserve a due event before making an HTTP request. Competing ++ /// dispatchers that selected the same row cannot both start delivery. ++ pub(crate) async fn reserve_event( ++ &self, ++ event: &PendingWebhookEvent, ++ ) -> anyhow::Result> { ++ let now = chrono::Utc::now().timestamp(); ++ let token = Uuid::new_v4().to_string(); ++ let result = sqlx::query( ++ "UPDATE webhook_events SET next_attempt_at = ?, last_attempt_at = ?, delivery_token = ? ++ WHERE id = ? AND status = 'pending' AND retries = ? AND retries < ? ++ AND (next_attempt_at IS NULL OR next_attempt_at <= ?)", ++ ) ++ .bind(now + DELIVERY_RESERVATION_SECONDS) ++ .bind(now) ++ .bind(&token) ++ .bind(&event.id) ++ .bind(event.retries) ++ .bind(MAX_WEBHOOK_ATTEMPTS) + .bind(now) + .execute(&self.db) + .await?; + +- Ok(id) +- } +- +- /// Get pending webhook events +- pub async fn get_pending_events( +- &self, +- limit: usize, +- ) -> anyhow::Result> { +- let query_limit = limit as i64; +- +- let rows = sqlx::query( +- "SELECT we.id, we.webhook_id, we.event_type, we.payload +- FROM webhook_events we +- WHERE we.status = 'pending' AND we.retries < 3 +- ORDER BY we.created_at ASC +- LIMIT ?", +- ) +- .bind(query_limit) +- .fetch_all(&self.db) +- .await?; +- +- let events: Vec<(String, String, String, String)> = rows +- .into_iter() +- .map(|row| { +- use sqlx::Row; +- ( +- row.get::(0), +- row.get::(1), +- row.get::(2), +- row.get::(3), +- ) +- }) +- .collect(); +- +- Ok(events) +- } +- +- /// Update webhook event status ++ Ok((result.rows_affected() == 1).then_some(token)) ++ } ++ ++ /// Check ownership again after preparing the request, before contacting ++ /// the subscriber. A paused worker must not send under an expired claim. ++ pub(crate) async fn owns_reservation( ++ &self, ++ event_id: &str, ++ token: &str, ++ ) -> anyhow::Result { ++ Ok(sqlx::query_scalar( ++ "SELECT EXISTS(SELECT 1 FROM webhook_events ++ WHERE id = ? AND status = 'pending' AND delivery_token = ? AND next_attempt_at > ?)", ++ ) ++ .bind(event_id) ++ .bind(token) ++ .bind(chrono::Utc::now().timestamp()) ++ .fetch_one(&self.db) ++ .await?) ++ } ++ ++ /// Freeze legacy envelopes before sending; never overwrite a stored body. ++ pub(crate) async fn persist_delivery_payload( ++ &self, ++ event_id: &str, ++ body: &str, ++ ) -> anyhow::Result { ++ Ok(sqlx::query_scalar( ++ "UPDATE webhook_events SET delivery_payload = COALESCE(delivery_payload, ?) ++ WHERE id = ? RETURNING delivery_payload", ++ ) ++ .bind(body) ++ .bind(event_id) ++ .fetch_one(&self.db) ++ .await?) ++ } ++ ++ /// Query a delivery only when its webhook belongs to the authenticated user. ++ pub async fn get_event_status( ++ &self, ++ webhook_id: &str, ++ event_id: &str, ++ user_id: &str, ++ ) -> anyhow::Result> { ++ Ok(sqlx::query_as::<_, WebhookEventStatus>( ++ "SELECT we.id, we.webhook_id, we.event_type, we.status, we.retries, ++ we.last_error, we.created_at, we.next_attempt_at, ++ we.last_attempt_at, we.delivered_at ++ FROM webhook_events we JOIN webhooks w ON w.id = we.webhook_id ++ WHERE we.id = ? AND we.webhook_id = ? AND w.user_id = ?", ++ ) ++ .bind(event_id) ++ .bind(webhook_id) ++ .bind(user_id) ++ .fetch_optional(&self.db) ++ .await?) ++ } ++ ++ /// Persist the outcome and next due time. A success retains earlier failure ++ /// information and updates last_fired_at in the same transaction. + pub async fn update_event_status( + &self, + event_id: &str, +@@ -286,16 +464,74 @@ + error: Option<&str>, + retries: i32, + ) -> anyhow::Result<()> { +- sqlx::query( +- "UPDATE webhook_events SET status = ?, last_error = ?, retries = ? WHERE id = ?", ++ self.set_event_status(event_id, status, error, retries, None) ++ .await ++ } ++ ++ pub(crate) async fn finish_reserved_event( ++ &self, ++ event_id: &str, ++ token: &str, ++ status: &str, ++ error: Option<&str>, ++ retries: i32, ++ ) -> anyhow::Result<()> { ++ self.set_event_status(event_id, status, error, retries, Some(token)) ++ .await ++ } ++ ++ async fn set_event_status( ++ &self, ++ event_id: &str, ++ status: &str, ++ error: Option<&str>, ++ retries: i32, ++ token: Option<&str>, ++ ) -> anyhow::Result<()> { ++ anyhow::ensure!(retries >= 0, "Invalid webhook retry count"); ++ anyhow::ensure!( ++ matches!(status, "pending" | "delivered" | "failed"), ++ "Invalid webhook event status" ++ ); ++ let status = if status == "pending" && retries >= MAX_WEBHOOK_ATTEMPTS { ++ "failed" ++ } else { ++ status ++ }; ++ let now = chrono::Utc::now(); ++ let next_attempt_at = ++ (status == "pending").then(|| now.timestamp() + retry_delay_seconds(retries)); ++ let delivered_at = (status == "delivered").then(|| now.timestamp()); ++ let mut transaction = self.db.begin().await?; ++ let webhook_id: String = sqlx::query_scalar( ++ "UPDATE webhook_events ++ SET status = ?, last_error = COALESCE(?, last_error), retries = ?, ++ next_attempt_at = ?, delivered_at = ?, delivery_token = NULL ++ WHERE id = ? AND status = 'pending' ++ AND (delivery_token = ? OR (? IS NULL AND delivery_token IS NULL)) ++ RETURNING webhook_id", + ) + .bind(status) + .bind(error) + .bind(retries) ++ .bind(next_attempt_at) ++ .bind(delivered_at) + .bind(event_id) +- .execute(&self.db) ++ .bind(token) ++ .bind(token) ++ .fetch_one(&mut *transaction) + .await?; + ++ if status == "delivered" { ++ let result = sqlx::query("UPDATE webhooks SET last_fired_at = ? WHERE id = ?") ++ .bind(now.to_rfc3339()) ++ .bind(webhook_id) ++ .execute(&mut *transaction) ++ .await?; ++ anyhow::ensure!(result.rows_affected() == 1, "Webhook no longer exists"); ++ } ++ ++ transaction.commit().await?; + Ok(()) + } + +@@ -316,6 +552,210 @@ + mod tests { + use super::*; + ++ async fn legacy_database() -> SqlitePool { ++ let db = sqlx::sqlite::SqlitePoolOptions::new() ++ .max_connections(1) ++ .connect("sqlite::memory:") ++ .await ++ .unwrap(); ++ sqlx::raw_sql("CREATE TABLE users (id TEXT PRIMARY KEY);") ++ .execute(&db) ++ .await ++ .unwrap(); ++ sqlx::raw_sql(include_str!("../../migrations/019_oauth_webhooks.sql")) ++ .execute(&db) ++ .await ++ .unwrap(); ++ sqlx::raw_sql( ++ "INSERT INTO users (id) VALUES ('owner'); ++ INSERT INTO webhooks (id, user_id, url, event_types, secret) ++ VALUES ('webhook', 'owner', 'http://localhost/', 'test', 'subscriber-secret');", ++ ) ++ .execute(&db) ++ .await ++ .unwrap(); ++ db ++ } ++ ++ #[test] ++ fn retry_delay_is_exponential_and_saturates() { ++ assert_eq!( ++ (1..=8).map(retry_delay_seconds).collect::>(), ++ vec![5, 10, 20, 40, 80, 160, 300, 300] ++ ); ++ assert_eq!(retry_delay_seconds(i32::MAX), 300); ++ } ++ ++ #[tokio::test] ++ async fn migration_closes_exhausted_events_without_losing_errors() { ++ let db = legacy_database().await; ++ sqlx::raw_sql( ++ "INSERT INTO webhook_events (id, webhook_id, event_type, payload, status, retries, last_error) ++ VALUES ('exhausted', 'webhook', 'test', '{}', 'pending', 3, 'original failure'), ++ ('delivered', 'webhook', 'test', '{}', 'delivered', 2, 'prior failure'), ++ ('ready', 'webhook', 'test', '{}', 'pending', 1, 'retryable failure');", ++ ) ++ .execute(&db) ++ .await ++ .unwrap(); ++ sqlx::raw_sql(include_str!( ++ "../../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&db) ++ .await ++ .unwrap(); ++ ++ let service = WebhookService::new(db); ++ let exhausted = service ++ .get_event_status("webhook", "exhausted", "owner") ++ .await ++ .unwrap() ++ .unwrap(); ++ assert_eq!(exhausted.status, "failed"); ++ assert_eq!(exhausted.retries, 3); ++ assert_eq!(exhausted.last_error.as_deref(), Some("original failure")); ++ let delivered = service ++ .get_event_status("webhook", "delivered", "owner") ++ .await ++ .unwrap() ++ .unwrap(); ++ assert_eq!(delivered.status, "delivered"); ++ assert_eq!(delivered.retries, 2); ++ assert_eq!(delivered.last_error.as_deref(), Some("prior failure")); ++ let pending = service.get_pending_events(10).await.unwrap(); ++ assert_eq!(pending.len(), 1); ++ assert_eq!(pending[0].0, "ready"); ++ } ++ ++ #[tokio::test] ++ async fn stale_workers_cannot_reserve_the_same_due_event() { ++ let db = legacy_database().await; ++ sqlx::raw_sql(include_str!( ++ "../../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&db) ++ .await ++ .unwrap(); ++ let service = WebhookService::new(db.clone()); ++ let event_id = service ++ .create_webhook_event("webhook", "test", serde_json::json!({"value": 1})) ++ .await ++ .unwrap(); ++ let first_view = service.get_pending_deliveries(10).await.unwrap(); ++ let second_view = service.get_pending_deliveries(10).await.unwrap(); ++ let (first, second) = tokio::join!( ++ service.reserve_event(&first_view[0]), ++ service.reserve_event(&second_view[0]) ++ ); ++ assert_ne!(first.unwrap().is_some(), second.unwrap().is_some()); ++ assert!(service.get_pending_events(10).await.unwrap().is_empty()); ++ ++ // A crashed worker's reservation expires without discarding the event. ++ sqlx::query("UPDATE webhook_events SET next_attempt_at = 0 WHERE id = ?") ++ .bind(&event_id) ++ .execute(&db) ++ .await ++ .unwrap(); ++ let recovered = service.get_pending_deliveries(10).await.unwrap(); ++ assert_eq!(recovered[0].id, event_id); ++ assert_eq!( ++ recovered[0].delivery_payload, ++ first_view[0].delivery_payload ++ ); ++ assert!(service ++ .reserve_event(&recovered[0]) ++ .await ++ .unwrap() ++ .is_some()); ++ } ++ ++ #[tokio::test] ++ async fn expired_workers_cannot_overwrite_a_new_delivery_outcome() { ++ let db = legacy_database().await; ++ sqlx::raw_sql(include_str!( ++ "../../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&db) ++ .await ++ .unwrap(); ++ let service = WebhookService::new(db.clone()); ++ let event_id = service ++ .create_webhook_event("webhook", "test", serde_json::json!({})) ++ .await ++ .unwrap(); ++ let event = service.get_pending_deliveries(1).await.unwrap(); ++ let old_token = service.reserve_event(&event[0]).await.unwrap().unwrap(); ++ sqlx::query("UPDATE webhook_events SET next_attempt_at = 0 WHERE id = ?") ++ .bind(&event_id) ++ .execute(&db) ++ .await ++ .unwrap(); ++ assert!(!service ++ .owns_reservation(&event_id, &old_token) ++ .await ++ .unwrap()); ++ let new_token = service.reserve_event(&event[0]).await.unwrap().unwrap(); ++ assert_ne!(old_token, new_token); ++ service ++ .finish_reserved_event(&event_id, &new_token, "pending", Some("new failure"), 1) ++ .await ++ .unwrap(); ++ let current = service ++ .get_event_status("webhook", &event_id, "owner") ++ .await ++ .unwrap() ++ .unwrap(); ++ assert!(service ++ .finish_reserved_event(&event_id, &old_token, "pending", Some("stale failure"), 1) ++ .await ++ .is_err()); ++ let after = service ++ .get_event_status("webhook", &event_id, "owner") ++ .await ++ .unwrap() ++ .unwrap(); ++ assert_eq!(after.last_error, current.last_error); ++ assert_eq!(after.retries, current.retries); ++ assert_eq!(after.next_attempt_at, current.next_attempt_at); ++ } ++ ++ #[tokio::test] ++ async fn terminal_events_are_not_resurrected_by_status_updates() { ++ let db = legacy_database().await; ++ sqlx::raw_sql(include_str!( ++ "../../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&db) ++ .await ++ .unwrap(); ++ let service = WebhookService::new(db); ++ let event_id = service ++ .create_webhook_event("webhook", "test", serde_json::json!({})) ++ .await ++ .unwrap(); ++ service ++ .update_event_status(&event_id, "pending", Some("last failure"), 3) ++ .await ++ .unwrap(); ++ assert!(service ++ .update_event_status(&event_id, "pending", None, 0) ++ .await ++ .is_err()); ++ assert!(service ++ .update_event_status("missing", "delivered", None, 0) ++ .await ++ .is_err()); ++ let state = service ++ .get_event_status("webhook", &event_id, "owner") ++ .await ++ .unwrap() ++ .unwrap(); ++ assert_eq!(state.status, "failed"); ++ assert_eq!(state.retries, 3); ++ assert_eq!(state.last_error.as_deref(), Some("last failure")); ++ assert!(state.next_attempt_at.is_none()); ++ } ++ + #[test] + fn test_webhook_signature() { + let payload = r#"{"event":"test"}"#; +@@ -323,6 +763,15 @@ + + let signature = WebhookSignature::sign(payload, secret); + assert!(WebhookSignature::verify(payload, secret, &signature)); ++ } ++ ++ #[test] ++ fn webhook_signature_matches_hmac_sha256_test_vector() { ++ // RFC 4231, test case 1. ++ assert_eq!( ++ WebhookSignature::sign("Hi There", &"\x0b".repeat(20)), ++ "sha256=b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7" ++ ); + } + + #[test] +diff --git a/tests/webhook_delivery_regression_tests.rs b/tests/webhook_delivery_regression_tests.rs +new file mode 100644 +index 0000000000000000000000000000000000000000..a471b31a52239d2f9aee22c1ec4fde605d9ad9c7 +--- /dev/null ++++ b/tests/webhook_delivery_regression_tests.rs +@@ -0,0 +1,14 @@ ++// Compile the production webhook modules as one integration target so their ++// private dispatcher/database tests can run independently of unrelated lib ++// unit tests. Authentication, validation, encryption and broadcast types come ++// from the actual backend library; no alternate delivery implementation is used. ++pub use stellar_analysis_backend::{auth, auth_middleware, broadcast, crypto, validation}; ++ ++#[path = "../src/api/webhooks.rs"] ++pub mod webhook_api; ++#[path = "../src/services/webhook_dispatcher.rs"] ++pub mod webhook_dispatcher; ++#[path = "../src/services/webhook_event_service.rs"] ++pub mod webhook_event_service; ++#[path = "../src/webhooks/mod.rs"] ++pub mod webhooks; +diff --git a/tests/webhook_event_service_tests.rs b/tests/webhook_event_service_tests.rs +index 2f5e53dabcd3d81b982c58533005e53f36768105..1a0b666dc8f9f7d60c6c4d12e594b61362dd0232 100644 +--- a/tests/webhook_event_service_tests.rs ++++ b/tests/webhook_event_service_tests.rs +@@ -47,6 +47,13 @@ + .await + .unwrap(); + ++ sqlx::raw_sql(include_str!( ++ "../migrations/035_webhook_delivery_retries.sql" ++ )) ++ .execute(&pool) ++ .await ++ .unwrap(); ++ + pool + } + +@@ -99,6 +106,40 @@ + assert_eq!(events.len(), 1); + assert_eq!(events[0].1, webhook_id); + assert_eq!(events[0].2, "anchor.status_changed"); ++ } ++ ++ #[tokio::test] ++ async fn test_event_enqueue_failure_is_not_reported_as_success() { ++ let pool = setup_test_db().await; ++ sqlx::raw_sql( ++ "INSERT INTO webhooks (id, user_id, url, event_types, secret) ++ VALUES ('webhook', 'owner', 'https://example.com/webhook', 'payment.created', 'secret'); ++ CREATE TRIGGER reject_event BEFORE INSERT ON webhook_events ++ BEGIN SELECT RAISE(ABORT, 'event enqueue rejected'); END;", ++ ) ++ .execute(&pool) ++ .await ++ .unwrap(); ++ let result = WebhookEventService::new(pool.clone()) ++ .trigger_payment_created( ++ "payment", ++ "source", ++ "destination", ++ "XLM", ++ "native", ++ 42.0, ++ "2026-01-01T00:00:00Z", ++ ) ++ .await; ++ assert!(result ++ .unwrap_err() ++ .to_string() ++ .contains("event enqueue rejected")); ++ let events: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM webhook_events") ++ .fetch_one(&pool) ++ .await ++ .unwrap(); ++ assert_eq!(events, 0); + } + + #[tokio::test] diff --git a/backend/integration/stellar_backend_build.log b/backend/integration/stellar_backend_build.log new file mode 100644 index 00000000..e1fdcb1f --- /dev/null +++ b/backend/integration/stellar_backend_build.log @@ -0,0 +1,1005 @@ + Compiling aws-lc-rs v1.17.0 + Compiling aws-lc-sys v0.41.0 + Compiling rustls v0.23.40 + Compiling openssl-sys v0.9.116 + Compiling parking_lot v0.12.5 + Compiling rand_chacha v0.9.0 + Compiling rustls-webpki v0.103.13 + Compiling anyhow v1.0.102 + Compiling concurrent-queue v2.5.0 + Compiling webpki-roots v1.0.7 + Compiling event-listener v5.4.1 + Compiling chrono v0.4.45 + Compiling tokio v1.49.0 + Compiling crate-git-revision v0.0.6 + Compiling regex-automata v0.4.14 + Compiling tracing-core v0.1.36 + Compiling pin-project-internal v1.1.13 + Compiling alloc-no-stdlib v2.0.4 + Compiling linux-raw-sys v0.12.1 + Compiling unicode-xid v0.1.0 + Compiling rustversion v1.0.22 + Compiling winnow v1.0.3 + Compiling lazy_static v1.5.0 + Compiling time-core v0.1.8 + Compiling fastrand v2.4.1 + Compiling powerfmt v0.2.0 + Compiling num-conv v0.2.2 + Compiling time-macros v0.2.27 + Compiling toml_parser v1.1.2+spec-1.1.0 + Compiling deranged v0.5.8 + Compiling rustix v1.1.4 + Compiling proc-macro2 v0.4.30 + Compiling regex v1.12.4 + Compiling alloc-stdlib v0.2.2 + Compiling pin-project v1.1.13 + Compiling tracing v0.1.44 + Compiling webpki-roots v0.26.11 + Compiling tokio-stream v0.1.18 + Compiling rand v0.9.4 + Compiling futures-intrusive v0.5.0 + Compiling libsqlite3-sys v0.30.1 + Compiling crossbeam-queue v0.3.12 + Compiling derive_builder_core v0.20.2 + Compiling pest_meta v2.8.6 + Compiling itertools v0.14.0 + Compiling libsodium-sys v0.2.7 + Compiling opentelemetry v0.27.1 + Compiling darling_core v0.23.0 + Compiling hybrid-array v0.4.12 + Compiling num-bigint v0.3.3 + Compiling proc-macro-error-attr v1.0.4 + Compiling semver v1.0.28 + Compiling syn v1.0.109 + Compiling toml_datetime v1.1.1+spec-1.1.0 + Compiling byteorder v1.5.0 + Compiling base64 v0.22.1 + Compiling syn v0.15.44 + Compiling openssl-probe v0.2.1 + Compiling sqlx-core v0.8.6 + Compiling toml_edit v0.25.12+spec-1.1.0 + Compiling darling_macro v0.23.0 + Compiling rustc_version v0.4.1 + Compiling prost-derive v0.13.5 + Compiling pest_generator v2.8.6 + Compiling derive_builder_macro v0.20.2 + Compiling tungstenite v0.29.0 + Compiling brotli-decompressor v5.0.1 + Compiling quote v0.6.13 + Compiling time v0.3.47 + Compiling sharded-slab v0.1.7 + Compiling futures-executor v0.3.32 + Compiling atoi v2.0.0 + Compiling strum_macros v0.27.2 + Compiling zopfli v0.8.3 + Compiling flume v0.11.1 + Compiling futures-channel v0.3.32 + Compiling tower-http v0.6.11 + Compiling matchers v0.2.0 + Compiling async-graphql-value v7.2.1 + Compiling flate2 v1.1.9 + Compiling serde_urlencoded v0.7.1 + Compiling crossbeam-epoch v0.9.18 + Compiling tracing-log v0.2.0 + Compiling tracing-serde v0.2.0 + Compiling inout v0.1.4 + Compiling num-bigint v0.2.6 + Compiling num-rational v0.3.2 + Compiling async-io v2.6.0 + Compiling proc-macro-error v1.0.4 + Compiling multer v3.1.0 + Compiling serde_path_to_error v0.1.20 + Compiling thread_local v1.1.9 + Compiling encoding_rs v0.8.35 + Compiling num-modular v0.6.1 + Compiling nu-ansi-term v0.50.3 + Compiling unicase v2.9.0 + Compiling rust_decimal v1.42.0 + Compiling rayon-core v1.13.0 + Compiling cmov v0.5.4 + Compiling foreign-types-shared v0.1.1 + Compiling crc16 v0.4.0 + Compiling glob v0.3.3 + Compiling openssl v0.10.80 + Compiling opentelemetry_sdk v0.27.1 + Compiling foreign-types v0.3.2 + Compiling ctutils v0.4.2 + Compiling mime_guess v2.0.5 + Compiling tracing-subscriber v0.3.23 + Compiling num-order v1.2.0 + Compiling cipher v0.4.4 + Compiling crossbeam-deque v0.8.6 + Compiling sqlx-sqlite v0.8.6 + Compiling zip v3.0.0 + Compiling async-graphql-parser v7.2.1 + Compiling strum v0.27.2 + Compiling brotli v8.0.3 + Compiling tokio-tungstenite v0.29.0 + Compiling derive_builder v0.20.2 + Compiling pest_derive v2.8.6 + Compiling prost v0.13.5 + Compiling curve25519-dalek v4.1.3 + Compiling darling v0.23.0 + Compiling proc-macro-crate v3.5.0 + Compiling crypto-common v0.2.2 + Compiling block-buffer v0.12.1 + Compiling polling v3.11.0 + Compiling stellar-strkey v0.0.13 + Compiling uuid v1.23.3 + Compiling rust-embed-utils v8.11.0 + Compiling num-bigint v0.4.6 + Compiling axum-core v0.5.6 + Compiling half v2.7.1 + Compiling universal-hash v0.5.1 + Compiling matrixmultiply v0.3.10 + Compiling alloca v0.4.0 + Compiling futures-lite v2.6.1 + Compiling openssl-macros v0.1.1 + Compiling proc-macro-error-attr2 v2.0.0 + Compiling json v0.12.4 + Compiling ciborium-io v0.2.2 + Compiling dotenvy v0.15.7 + Compiling base64 v0.11.0 + Compiling matchit v0.8.4 + Compiling plotters-backend v0.3.7 + Compiling signature v2.2.0 + Compiling Inflector v0.11.4 + Compiling compression-core v0.4.32 + Compiling native-tls v0.2.18 + Compiling utf-8 v0.7.6 + Compiling arrayvec v0.7.6 + Compiling opaque-debug v0.3.1 + Compiling anstyle v1.0.14 + Compiling const-oid v0.10.2 + Compiling protobuf v3.7.2 + Compiling clap_lex v1.1.0 + Compiling ascii_utils v0.9.3 + Compiling fast_chemail v0.9.6 + Compiling clap_builder v4.6.0 + Compiling digest v0.11.3 + Compiling polyval v0.6.2 + Compiling tungstenite v0.24.0 + Compiling compression-codecs v0.4.38 + Compiling async-graphql-derive v7.2.1 + Compiling sodiumoxide v0.2.6 + Compiling plotters-svg v0.3.7 + Compiling axum v0.8.9 + Compiling xdr-rs-serialize v0.2.5 + Compiling sqlx-macros-core v0.8.6 + Compiling ciborium-ll v0.2.2 + Compiling proc-macro-error2 v2.0.1 + Compiling rust-embed-impl v8.11.0 + Compiling tonic v0.12.3 + Compiling handlebars v6.4.1 + Compiling xdr-rs-serialize-derive v0.2.5 + Compiling utoipa-swagger-ui v9.0.2 + Compiling reqwest v0.12.28 + Compiling rustls-native-certs v0.8.3 + Compiling tempfile v3.27.0 + Compiling stellar-xdr v25.0.1 + Compiling tokio-rustls v0.26.4 + Compiling event-listener-strategy v0.5.4 + Compiling itertools v0.13.0 + Compiling protobuf-support v3.7.2 + Compiling asynk-strim v0.1.5 + Compiling curve25519-dalek-derive v0.1.1 + Compiling utoipa-gen v5.5.0 + Compiling static_assertions_next v1.1.2 + Compiling base64 v0.13.1 + Compiling base64ct v1.8.3 + Compiling prometheus v0.14.0 + Compiling typed-path v0.12.3 + Compiling base32 v0.4.0 + Compiling bitflags v1.3.2 + Compiling rawpointer v0.2.1 + Compiling cast v0.3.0 + Compiling criterion-plot v0.8.2 + Compiling stellar-base v0.5.0 + Compiling utoipa v5.5.0 + Compiling zip v7.2.0 + Compiling password-hash v0.5.0 + Compiling async-graphql v7.2.1 + Compiling rust-embed v8.11.0 + Compiling async-lock v3.4.2 + Compiling opentelemetry-http v0.27.0 + Compiling hyper-rustls v0.27.9 + Compiling rustls-platform-verifier v0.7.0 + Compiling rayon v1.12.0 + Compiling opentelemetry-proto v0.27.0 + Compiling utoipa-gen v4.3.1 + Compiling validator_derive v0.20.0 + Compiling ciborium v0.2.2 + Compiling sqlx-macros v0.8.6 + Compiling plotters v0.3.7 + Compiling async-compression v0.4.42 + Compiling tokio-tungstenite v0.24.0 + Compiling ghash v0.5.1 + Compiling clap v4.6.1 + Compiling ed25519 v2.2.3 + Compiling simple_asn1 v0.6.4 + Compiling aes v0.8.4 + Compiling ctr v0.9.2 + Compiling arc-swap v1.9.1 + Compiling axum-core v0.4.5 + Compiling ureq v2.12.1 + Compiling backon v1.6.0 + Compiling combine v4.6.7 + Compiling crossbeam-channel v0.5.15 + Compiling blake2 v0.10.6 + Compiling pem v3.0.6 + Compiling email-encoding v0.4.1 + Compiling num-complex v0.4.6 + Compiling tinytemplate v1.2.1 + Compiling aead v0.5.2 + Compiling toml v0.5.11 + Compiling serde_spanned v1.1.1 + Compiling csv-core v0.1.13 + Compiling nom v8.0.0 + Compiling hostname v0.4.2 + Compiling page_size v0.6.0 + Compiling rust_xlsxwriter_derive v0.2.0 + Compiling cfg_eval v0.1.2 + Compiling matchit v0.7.3 + Compiling cpufeatures v0.3.0 + Compiling ethnum v1.5.3 + Compiling quoted_printable v0.5.2 + Compiling toml_writer v1.1.1+spec-1.1.0 + Compiling hashbrown v0.14.5 + Compiling xxhash-rust v0.8.15 + Compiling escape-bytes v0.1.1 + Compiling email_address v0.2.9 + Compiling oorandom v11.1.5 + Compiling anes v0.1.6 + Compiling symlink v0.1.0 + Compiling arcstr v1.2.0 + Compiling sha1_smol v1.0.1 + Compiling nacl v0.5.3 + Compiling redis v1.2.3 + Compiling stellar_sdk v0.1.4 + Compiling criterion v0.8.2 + Compiling tracing-appender v0.2.5 + Compiling dashmap v6.2.1 + Compiling lettre v0.11.22 + Compiling toml v1.1.2+spec-1.1.0 + Compiling sha2 v0.11.0 + Compiling axum v0.7.9 + Compiling rust_xlsxwriter v0.95.0 + Compiling csv v1.4.0 + Compiling aes-gcm v0.10.3 + Compiling ndarray v0.17.2 + Compiling jsonwebtoken v10.4.0 + Compiling argon2 v0.5.3 + Compiling ed25519-dalek v2.2.0 + Compiling tower-http v0.7.0 + Compiling sqlx v0.8.6 + Compiling validator v0.20.0 + Compiling utoipa v4.2.3 + Compiling opentelemetry-otlp v0.27.0 + Compiling reqwest v0.13.4 + Compiling async-graphql-axum v7.2.1 + Compiling hmac v0.13.0 + Compiling tracing-logstash v0.7.0 + Compiling tracing-opentelemetry v0.28.0 + Compiling failsafe v1.3.0 + Compiling tower v0.4.13 + Compiling futures v0.3.32 + Compiling ipnetwork v0.20.0 + Compiling md5 v0.7.0 + Compiling urlencoding v2.1.3 + Compiling stellar-analysis-backend v0.1.0 (/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend) +error[E0433]: failed to resolve: use of undeclared type `Arc` + --> src/features/websocket_streaming.rs:134:25 + | +134 | let streaming = Arc::new(WebSocketStreaming::new(config)); + | ^^^ use of undeclared type `Arc` + | + = note: struct `crate::rpc::stellar::tests::Arc` exists but is inaccessible +help: consider importing this struct + | +100 + use std::sync::Arc; + | + +error[E0433]: failed to resolve: could not find `mock_stellar` in `super` + --> src/rpc/stellar.rs:2018:29 + | +2018 | Some(super::mock_stellar::MOCK_LATEST_LEDGER.saturating_add(1)), + | ^^^^^^^^^^^^ could not find `mock_stellar` in `super` + | +help: consider importing this module + | +1903 + use crate::rpc::mock_stellar; + | +help: if you import `mock_stellar`, refer to it directly + | +2018 - Some(super::mock_stellar::MOCK_LATEST_LEDGER.saturating_add(1)), +2018 + Some(mock_stellar::MOCK_LATEST_LEDGER.saturating_add(1)), + | + +error[E0433]: failed to resolve: could not find `mock_stellar` in `super` + --> src/rpc/stellar.rs:2028:20 + | +2028 | super::mock_stellar::MOCK_LATEST_LEDGER + | ^^^^^^^^^^^^ could not find `mock_stellar` in `super` + | +help: consider importing this module + | +1903 + use crate::rpc::mock_stellar; + | +help: if you import `mock_stellar`, refer to it directly + | +2028 - super::mock_stellar::MOCK_LATEST_LEDGER +2028 + mock_stellar::MOCK_LATEST_LEDGER + | + +error[E0433]: failed to resolve: could not find `mock_stellar` in `super` + --> src/rpc/stellar.rs:2438:31 + | +2438 | let payments = super::mock_stellar::mock_payments(10); + | ^^^^^^^^^^^^ could not find `mock_stellar` in `super` + | +help: consider importing this module + | +1903 + use crate::rpc::mock_stellar; + | +help: if you import `mock_stellar`, refer to it directly + | +2438 - let payments = super::mock_stellar::mock_payments(10); +2438 + let payments = mock_stellar::mock_payments(10); + | + +warning: unused import: `Span` + --> src/observability/health_checks.rs:15:27 + | +15 | use tracing::{instrument, Span}; + | ^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:283:11 + | +283 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + | + = note: `#[warn(unused_variables)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:348:11 + | +348 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `start` + --> src/observability/health_checks.rs:175:9 + | +175 | let start = Instant::now(); + | ^^^^^ help: if this is intentional, prefix it with an underscore: `_start` + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:401:9 + | +401 | / assert_eq!( +402 | | determine_health_status(true, 5, 90.0, Some(100), Some(110)), +403 | | JobHealthStatus::Critical +404 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:406:9 + | +406 | / assert_eq!( +407 | | determine_health_status(true, 3, 40.0, Some(100), Some(110)), +408 | | JobHealthStatus::Critical +409 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:412:9 + | +412 | / assert_eq!( +413 | | determine_health_status(true, 3, 90.0, Some(100), Some(110)), +414 | | JobHealthStatus::Warning +415 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:417:9 + | +417 | / assert_eq!( +418 | | determine_health_status(true, 1, 70.0, Some(100), Some(110)), +419 | | JobHealthStatus::Warning +420 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:423:9 + | +423 | / assert_eq!( +424 | | determine_health_status(true, 0, 95.0, Some(100), None), +425 | | JobHealthStatus::Healthy +426 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:428:9 + | +428 | / assert_eq!( +429 | | determine_health_status(false, 1, 90.0, Some(100), Some(110)), +430 | | JobHealthStatus::Healthy +431 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0369]: binary operation `==` cannot be applied to type `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:434:9 + | +434 | / assert_eq!( +435 | | determine_health_status(false, 0, 0.0, None, None), +436 | | JobHealthStatus::Unknown +437 | | ); + | | ^ + | |_________| + | |_________job_monitoring::JobHealthStatus + | job_monitoring::JobHealthStatus + | +note: an implementation of `PartialEq` might be missing for `job_monitoring::JobHealthStatus` + --> src/handlers/job_monitoring.rs:63:1 + | +63 | pub enum JobHealthStatus { + | ^^^^^^^^^^^^^^^^^^^^^^^^ must implement `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) +help: consider annotating `job_monitoring::JobHealthStatus` with `#[derive(PartialEq)]` + | +63 + #[derive(PartialEq)] +64 | pub enum JobHealthStatus { + | + +error[E0277]: `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` cannot be unpinned + --> src/jobs/trace_aware_executor.rs:83:40 + | +83 | let task = TraceAwareTask::new(async { 42 }, span); + | ------------------- ^^^^^^^^^^^^ the trait `Unpin` is not implemented for `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` + | | + | required by a bound introduced by this call + | + = note: consider using the `pin!` macro + consider using `Box::pin` if you need to access the pinned value outside of the current scope +note: required by a bound in `trace_aware_executor::TraceAwareTask::::new` + --> src/jobs/trace_aware_executor.rs:27:30 + | +27 | F: std::future::Future + Unpin, + | ^^^^^ required by this bound in `TraceAwareTask::::new` +... +30 | pub fn new(future: F, span: Span) -> Self { + | --- required by a bound in this associated function + +error[E0277]: `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` cannot be unpinned + --> src/jobs/trace_aware_executor.rs:83:20 + | +83 | let task = TraceAwareTask::new(async { 42 }, span); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Unpin` is not implemented for `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` + | + = note: consider using the `pin!` macro + consider using `Box::pin` if you need to access the pinned value outside of the current scope +note: required by a bound in `trace_aware_executor::TraceAwareTask` + --> src/jobs/trace_aware_executor.rs:19:30 + | +17 | pub struct TraceAwareTask + | -------------- required by a bound in this struct +18 | where +19 | F: std::future::Future + Unpin, + | ^^^^^ required by this bound in `TraceAwareTask` + +error[E0277]: `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` cannot be unpinned + --> src/jobs/trace_aware_executor.rs:84:22 + | +84 | let result = task.await; + | ^^^^ the trait `Unpin` is not implemented for `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` + | + = note: consider using the `pin!` macro + consider using `Box::pin` if you need to access the pinned value outside of the current scope +note: required by a bound in `trace_aware_executor::TraceAwareTask` + --> src/jobs/trace_aware_executor.rs:19:30 + | +17 | pub struct TraceAwareTask + | -------------- required by a bound in this struct +18 | where +19 | F: std::future::Future + Unpin, + | ^^^^^ required by this bound in `TraceAwareTask` + +error[E0277]: `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` cannot be unpinned + --> src/jobs/trace_aware_executor.rs:84:27 + | +84 | let result = task.await; + | ^^^^^ the trait `Unpin` is not implemented for `{async block@src/jobs/trace_aware_executor.rs:83:40: 83:45}` + | + = note: consider using the `pin!` macro + consider using `Box::pin` if you need to access the pinned value outside of the current scope +note: required by a bound in `trace_aware_executor::TraceAwareTask` + --> src/jobs/trace_aware_executor.rs:19:30 + | +17 | pub struct TraceAwareTask + | -------------- required by a bound in this struct +18 | where +19 | F: std::future::Future + Unpin, + | ^^^^^ required by this bound in `TraceAwareTask` + +error[E0277]: `{async block@src/jobs/trace_aware_executor.rs:90:47: 90:52}` cannot be unpinned + --> src/jobs/trace_aware_executor.rs:90:47 + | +90 | let handle = spawn_traced("test_job", async { "success" }); + | ------------ ^^^^^^^^^^^^^^^^^^^ the trait `Unpin` is not implemented for `{async block@src/jobs/trace_aware_executor.rs:90:47: 90:52}` + | | + | required by a bound introduced by this call + | + = note: consider using the `pin!` macro + consider using `Box::pin` if you need to access the pinned value outside of the current scope +note: required by a bound in `trace_aware_executor::spawn_traced` + --> src/jobs/trace_aware_executor.rs:62:37 + | +60 | pub fn spawn_traced(_name: &str, future: F) -> tokio::task::JoinHandle + | ------------ required by a bound in this function +61 | where +62 | F: std::future::Future + Send + Unpin + 'static, + | ^^^^^ required by this bound in `spawn_traced` + +error[E0277]: `{async block@src/jobs/trace_aware_executor.rs:100:48: 100:53}` cannot be unpinned + --> src/jobs/trace_aware_executor.rs:100:48 + | +100 | let handle = spawn_traced("child_job", async { 100 }); + | ------------ ^^^^^^^^^^^^^ the trait `Unpin` is not implemented for `{async block@src/jobs/trace_aware_executor.rs:100:48: 100:53}` + | | + | required by a bound introduced by this call + | + = note: consider using the `pin!` macro + consider using `Box::pin` if you need to access the pinned value outside of the current scope +note: required by a bound in `trace_aware_executor::spawn_traced` + --> src/jobs/trace_aware_executor.rs:62:37 + | +60 | pub fn spawn_traced(_name: &str, future: F) -> tokio::task::JoinHandle + | ------------ required by a bound in this function +61 | where +62 | F: std::future::Future + Send + Unpin + 'static, + | ^^^^^ required by this bound in `spawn_traced` + +error[E0308]: mismatched types + --> src/observability/job_metrics.rs:272:32 + | +272 | .with_label_values(&[job_name, "success"]) + | ----------------- ^^^^^^^^^^^^^^^^^^^^^^ expected `&[&String]`, found `&[&str; 2]` + | | + | arguments to this method are incorrect + | + = note: expected reference `&[&std::string::String]` + found reference `&[&str; 2]` +note: method defined here + --> /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/prometheus-0.14.0/src/vec.rs:292:12 + | +292 | pub fn with_label_values(&self, vals: &[V]) -> T::M + | ^^^^^^^^^^^^^^^^^ + +error[E0308]: mismatched types + --> src/observability/job_metrics.rs:299:32 + | +299 | .with_label_values(&[job_name, "failure"]) + | ----------------- ^^^^^^^^^^^^^^^^^^^^^^ expected `&[&String]`, found `&[&str; 2]` + | | + | arguments to this method are incorrect + | + = note: expected reference `&[&std::string::String]` + found reference `&[&str; 2]` +note: method defined here + --> /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/prometheus-0.14.0/src/vec.rs:292:12 + | +292 | pub fn with_label_values(&self, vals: &[V]) -> T::M + | ^^^^^^^^^^^^^^^^^ + +error[E0308]: mismatched types + --> src/observability/job_metrics.rs:330:32 + | +330 | .with_label_values(&[job_name, "timeout"]) + | ----------------- ^^^^^^^^^^^^^^^^^^^^^^ expected `&[&String]`, found `&[&str; 2]` + | | + | arguments to this method are incorrect + | + = note: expected reference `&[&std::string::String]` + found reference `&[&str; 2]` +note: method defined here + --> /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/prometheus-0.14.0/src/vec.rs:292:12 + | +292 | pub fn with_label_values(&self, vals: &[V]) -> T::M + | ^^^^^^^^^^^^^^^^^ + +error[E0308]: mismatched types + --> src/observability/job_metrics.rs:333:32 + | +333 | .with_label_values(&[job_name, "timeout"]) + | ----------------- ^^^^^^^^^^^^^^^^^^^^^^ expected `&[&String]`, found `&[&str; 2]` + | | + | arguments to this method are incorrect + | + = note: expected reference `&[&std::string::String]` + found reference `&[&str; 2]` +note: method defined here + --> /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/prometheus-0.14.0/src/vec.rs:292:12 + | +292 | pub fn with_label_values(&self, vals: &[V]) -> T::M + | ^^^^^^^^^^^^^^^^^ + +error[E0308]: mismatched types + --> src/observability/metrics.rs:319:28 + | +319 | .with_label_values(&[&status_code.to_string(), method, path]) + | ----------------- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `&[&String]`, found `&[&str; 3]` + | | + | arguments to this method are incorrect + | + = note: expected reference `&[&std::string::String]` + found reference `&[&str; 3]` +note: method defined here + --> /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/prometheus-0.14.0/src/vec.rs:292:12 + | +292 | pub fn with_label_values(&self, vals: &[V]) -> T::M + | ^^^^^^^^^^^^^^^^^ + +error[E0277]: cannot add `{float}` to `u64` + --> src/observability/metrics.rs:477:53 + | +477 | assert!(HTTP_REQUESTS_TOTAL.get() >= before + 1.0); + | ^ no implementation for `u64 + {float}` + | + = help: the trait `Add<{float}>` is not implemented for `u64` + = help: the following other types implement trait `Add`: + `&u64` implements `Add<&num_bigint::BigInt>` + `&u64` implements `Add<&num_bigint::BigUint>` + `&u64` implements `Add<&num_bigint::bigint::BigInt>` + `&u64` implements `Add<&num_bigint::biguint::BigUint>` + `&u64` implements `Add<&num_complex::Complex>` + `&u64` implements `Add` + `&u64` implements `Add` + `&u64` implements `Add` + and 20 others + +error[E0277]: can't compare `i64` with `{float}` + --> src/observability/metrics.rs:625:9 + | +625 | assert_eq!(DB_POOL_CONNECTIONS_ACTIVE.get(), 7.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no implementation for `i64 == {float}` + | + = help: the trait `PartialEq<{float}>` is not implemented for `i64` + = help: the following other types implement trait `PartialEq`: + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0308]: mismatched types + --> src/observability/metrics.rs:625:9 + | +625 | assert_eq!(DB_POOL_CONNECTIONS_ACTIVE.get(), 7.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i64`, found floating-point number + | + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: can't compare `i64` with `{float}` + --> src/observability/metrics.rs:626:9 + | +626 | assert_eq!(DB_POOL_CONNECTIONS_IDLE.get(), 3.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no implementation for `i64 == {float}` + | + = help: the trait `PartialEq<{float}>` is not implemented for `i64` + = help: the following other types implement trait `PartialEq`: + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0308]: mismatched types + --> src/observability/metrics.rs:626:9 + | +626 | assert_eq!(DB_POOL_CONNECTIONS_IDLE.get(), 3.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i64`, found floating-point number + | + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: cannot subtract `{float}` from `i64` + --> src/observability/metrics.rs:627:44 + | +627 | assert!((DB_POOL_UTILIZATION.get() - 0.7).abs() < f64::EPSILON); + | ^ no implementation for `i64 - {float}` + | + = help: the trait `Sub<{float}>` is not implemented for `i64` + = help: the following other types implement trait `Sub`: + `&i64` implements `Sub<&num_bigint::BigInt>` + `&i64` implements `Sub<&num_bigint::bigint::BigInt>` + `&i64` implements `Sub<&num_complex::Complex>` + `&i64` implements `Sub` + `&i64` implements `Sub` + `&i64` implements `Sub` + `&i64` implements `Sub>` + `&i64` implements `Sub` + and 12 others + +error[E0277]: can't compare `i64` with `{float}` + --> src/observability/metrics.rs:629:9 + | +629 | assert_eq!(DB_POOL_ACTIVE.get(), 7.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no implementation for `i64 == {float}` + | + = help: the trait `PartialEq<{float}>` is not implemented for `i64` + = help: the following other types implement trait `PartialEq`: + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0308]: mismatched types + --> src/observability/metrics.rs:629:9 + | +629 | assert_eq!(DB_POOL_ACTIVE.get(), 7.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i64`, found floating-point number + | + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: can't compare `i64` with `{float}` + --> src/observability/metrics.rs:630:9 + | +630 | assert_eq!(DB_POOL_IDLE.get(), 3.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no implementation for `i64 == {float}` + | + = help: the trait `PartialEq<{float}>` is not implemented for `i64` + = help: the following other types implement trait `PartialEq`: + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0308]: mismatched types + --> src/observability/metrics.rs:630:9 + | +630 | assert_eq!(DB_POOL_IDLE.get(), 3.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i64`, found floating-point number + | + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: can't compare `i64` with `{float}` + --> src/observability/metrics.rs:631:9 + | +631 | assert_eq!(DB_POOL_SIZE.get(), 10.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ no implementation for `i64 == {float}` + | + = help: the trait `PartialEq<{float}>` is not implemented for `i64` + = help: the following other types implement trait `PartialEq`: + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + `i64` implements `PartialEq` + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0308]: mismatched types + --> src/observability/metrics.rs:631:9 + | +631 | assert_eq!(DB_POOL_SIZE.get(), 10.0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `i64`, found floating-point number + | + = note: this error originates in the macro `assert_eq` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0599]: no method named `name` found for struct `tracing::Span` in the current scope + --> src/observability/trace_context.rs:173:25 + | +173 | assert_eq!(span.name(), "test_job"); + | ^^^^ + | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following traits define an item `name`, perhaps you need to implement one of them: + candidate #1: `EventProcessor` + candidate #2: `PriceFeedProvider` + candidate #3: `broadcast::NotificationChannel` +help: there is a method `ne` with a similar name, but with different arguments + --> /rustc/29483883eed69d5fb4db01964cdf2af4d86e9cb2/library/core/src/cmp.rs:263:5 + +error[E0061]: this method takes 1 argument but 0 arguments were supplied + --> src/websocket.rs:1003:32 + | +1003 | permits.push(state.try_acquire_connection_permit().unwrap()); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-- argument #1 of type `std::net::IpAddr` is missing + | +note: method defined here + --> src/websocket.rs:398:8 + | +398 | fn try_acquire_connection_permit(self: &Arc, ip: IpAddr) -> Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ---------- +help: provide the argument + | +1003 | permits.push(state.try_acquire_connection_permit(/* std::net::IpAddr */).unwrap()); + | ++++++++++++++++++++++ + +error[E0061]: this method takes 1 argument but 0 arguments were supplied + --> src/websocket.rs:1007:23 + | +1007 | assert!(state.try_acquire_connection_permit().is_none()); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-- argument #1 of type `std::net::IpAddr` is missing + | +note: method defined here + --> src/websocket.rs:398:8 + | +398 | fn try_acquire_connection_permit(self: &Arc, ip: IpAddr) -> Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ---------- +help: provide the argument + | +1007 | assert!(state.try_acquire_connection_permit(/* std::net::IpAddr */).is_none()); + | ++++++++++++++++++++++ + +error[E0061]: this method takes 1 argument but 0 arguments were supplied + --> src/websocket.rs:1011:23 + | +1011 | assert!(state.try_acquire_connection_permit().is_some()); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-- argument #1 of type `std::net::IpAddr` is missing + | +note: method defined here + --> src/websocket.rs:398:8 + | +398 | fn try_acquire_connection_permit(self: &Arc, ip: IpAddr) -> Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ---------- +help: provide the argument + | +1011 | assert!(state.try_acquire_connection_permit(/* std::net::IpAddr */).is_some()); + | ++++++++++++++++++++++ + +error[E0061]: this method takes 1 argument but 0 arguments were supplied + --> src/websocket.rs:1034:28 + | +1034 | let permit = state.try_acquire_connection_permit().unwrap(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-- argument #1 of type `std::net::IpAddr` is missing + | +note: method defined here + --> src/websocket.rs:398:8 + | +398 | fn try_acquire_connection_permit(self: &Arc, ip: IpAddr) -> Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ---------- +help: provide the argument + | +1034 | let permit = state.try_acquire_connection_permit(/* std::net::IpAddr */).unwrap(); + | ++++++++++++++++++++++ + +error[E0061]: this method takes 1 argument but 0 arguments were supplied + --> src/websocket.rs:1109:32 + | +1109 | permits.push(state.try_acquire_connection_permit().unwrap()); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-- argument #1 of type `std::net::IpAddr` is missing + | +note: method defined here + --> src/websocket.rs:398:8 + | +398 | fn try_acquire_connection_permit(self: &Arc, ip: IpAddr) -> Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ---------- +help: provide the argument + | +1109 | permits.push(state.try_acquire_connection_permit(/* std::net::IpAddr */).unwrap()); + | ++++++++++++++++++++++ + +error[E0061]: this method takes 1 argument but 0 arguments were supplied + --> src/websocket.rs:1112:23 + | +1112 | assert!(state.try_acquire_connection_permit().is_none()); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^-- argument #1 of type `std::net::IpAddr` is missing + | +note: method defined here + --> src/websocket.rs:398:8 + | +398 | fn try_acquire_connection_permit(self: &Arc, ip: IpAddr) -> Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ---------- +help: provide the argument + | +1112 | assert!(state.try_acquire_connection_permit(/* std::net::IpAddr */).is_none()); + | ++++++++++++++++++++++ + +warning: unused variable: `client` + --> src/rpc/stellar.rs:517:13 + | +517 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:638:13 + | +638 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +Some errors have detailed explanations: E0061, E0277, E0308, E0369, E0433, E0599. +For more information about an error, try `rustc --explain E0061`. +warning: `stellar-analysis-backend` (lib test) generated 6 warnings +error: could not compile `stellar-analysis-backend` (lib test) due to 41 previous errors; 6 warnings emitted diff --git a/backend/integration/stellar_backend_jwt_provider_failure.log b/backend/integration/stellar_backend_jwt_provider_failure.log new file mode 100644 index 00000000..28c421a0 --- /dev/null +++ b/backend/integration/stellar_backend_jwt_provider_failure.log @@ -0,0 +1,213 @@ + Compiling stellar-analysis-backend v0.1.0 (/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend) +warning: unused import: `Span` + --> src/observability/health_checks.rs:15:27 + | +15 | use tracing::{instrument, Span}; + | ^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:283:11 + | +283 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + | + = note: `#[warn(unused_variables)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:348:11 + | +348 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `start` + --> src/observability/health_checks.rs:175:9 + | +175 | let start = Instant::now(); + | ^^^^^ help: if this is intentional, prefix it with an underscore: `_start` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:517:13 + | +517 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:638:13 + | +638 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: function `compute_transaction_hash` is never used + --> src/services/contract.rs:88:4 + | +88 | fn compute_transaction_hash(network_passphrase: &str, tx_envelope_xdr: &[u8]) -> Result<[u8; 32]> { + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(dead_code)]` on by default + +warning: comparison is useless due to type limits + --> src/analytics.rs:97:8 + | +97 | if score.total_assets < 0 { + | ^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_comparisons)]` on by default + +warning: `stellar-analysis-backend` (lib) generated 8 warnings (run `cargo fix --lib -p stellar-analysis-backend` to apply 1 suggestion) +warning: unused imports: `WebhookChannel` and `WebhookEndpoint` + --> tests/../src/webhooks/mod.rs:14:19 + | +14 | pub use channel::{WebhookChannel, WebhookEndpoint}; + | ^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: method `run` is never used + --> tests/../src/services/webhook_dispatcher.rs:34:18 + | +20 | impl WebhookDispatcher { + | ---------------------- method in this implementation +... +34 | pub async fn run(&self) -> Result<()> { + | ^^^ + | + = note: `#[warn(dead_code)]` on by default + +warning: field `webhook_service` is never read + --> tests/../src/services/webhook_event_service.rs:16:5 + | +15 | pub struct WebhookEventService { + | ------------------- field in this struct +16 | webhook_service: Arc, + | ^^^^^^^^^^^^^^^ + +warning: methods `trigger_corridor_health_degraded`, `trigger_anchor_status_changed`, `trigger_payment_created`, `trigger_corridor_liquidity_dropped`, `trigger_event`, and `get_webhooks_for_event` are never used + --> tests/../src/services/webhook_event_service.rs:28:18 + | +19 | impl WebhookEventService { + | ------------------------ methods in this implementation +... +28 | pub async fn trigger_corridor_health_degraded( + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +... +50 | pub async fn trigger_anchor_status_changed( + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +... +74 | pub async fn trigger_payment_created( + | ^^^^^^^^^^^^^^^^^^^^^^^ +... +100 | pub async fn trigger_corridor_liquidity_dropped( + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +... +122 | async fn trigger_event( + | ^^^^^^^^^^^^^ +... +158 | async fn get_webhooks_for_event( + | ^^^^^^^^^^^^^^^^^^^^^^ + +warning: methods `register_webhook` and `update_last_fired` are never used + --> tests/../src/webhooks/mod.rs:203:18 + | +193 | impl WebhookService { + | ------------------- methods in this implementation +... +203 | pub async fn register_webhook( + | ^^^^^^^^^^^^^^^^ +... +539 | pub async fn update_last_fired(&self, webhook_id: &str) -> anyhow::Result<()> { + | ^^^^^^^^^^^^^^^^^ + +warning: struct `WebhookEndpoint` is never constructed + --> tests/../src/webhooks/channel.rs:9:12 + | +9 | pub struct WebhookEndpoint { + | ^^^^^^^^^^^^^^^ + +warning: associated function `new` is never used + --> tests/../src/webhooks/channel.rs:17:12 + | +15 | impl WebhookEndpoint { + | -------------------- associated function in this implementation +16 | #[must_use] +17 | pub fn new(url: String, secret: Option) -> Self { + | ^^^ + +warning: struct `WebhookChannel` is never constructed + --> tests/../src/webhooks/channel.rs:26:12 + | +26 | pub struct WebhookChannel { + | ^^^^^^^^^^^^^^ + +warning: associated function `new` is never used + --> tests/../src/webhooks/channel.rs:33:12 + | +31 | impl WebhookChannel { + | ------------------- associated function in this implementation +32 | #[must_use] +33 | pub fn new(endpoints: Vec) -> Self { + | ^^^ + +warning: function `determine_severity` is never used + --> tests/../src/webhooks/events.rs:103:8 + | +103 | pub fn determine_severity(old: &CorridorMetrics, new: &CorridorMetrics) -> String { + | ^^^^^^^^^^^^^^^^^^ + +warning: `stellar-analysis-backend` (test "webhook_delivery_regression_tests") generated 10 warnings (run `cargo fix --test "webhook_delivery_regression_tests"` to apply 1 suggestion) + Finished `test` profile [unoptimized] target(s) in 1m 06s + Running tests/webhook_delivery_regression_tests.rs (target/debug/deps/webhook_delivery_regression_tests-a09564ed80adeaca) + +running 21 tests +test webhook_api::tests::event_status_returns_delivery_metadata_for_owner ... ok +test webhook_api::tests::registration_returns_signing_secret_once_to_authenticated_owner ... FAILED +test webhook_api::tests::event_status_requires_authentication ... ok +test webhook_api::tests::event_status_hides_nonowned_missing_and_mismatched_events ... ok +test webhook_api::tests::versioned_event_status_validates_jwt_at_the_mounted_path ... FAILED +test webhooks::events::tests::test_corridor_degradation_detection ... ok +test webhook_event_service::tests::test_filter_application ... ok +test webhooks::tests::expired_workers_cannot_overwrite_a_new_delivery_outcome ... ok +test webhooks::tests::retry_delay_is_exponential_and_saturates ... ok +test webhooks::tests::migration_closes_exhausted_events_without_losing_errors ... ok +test webhooks::tests::terminal_events_are_not_resurrected_by_status_updates ... ok +test webhooks::tests::test_event_type_conversion ... ok +test webhooks::tests::test_webhook_signature ... ok +test webhooks::tests::webhook_signature_matches_hmac_sha256_test_vector ... ok +test webhook_dispatcher::tests::inactive_webhook_does_not_send_or_reset_failure_count ... ok +test webhook_dispatcher::tests::redirects_do_not_contact_another_receiver_or_store_response_bodies ... ok +test webhooks::tests::stale_workers_cannot_reserve_the_same_due_event ... ok +test webhook_dispatcher::tests::retries_survive_dispatcher_restart_with_identical_signed_delivery ... ok +test webhook_dispatcher::tests::legacy_sqlite_timestamp_is_frozen_before_retry ... ok +test webhook_dispatcher::tests::three_failed_attempts_become_terminal_without_a_fourth_request ... ok +test webhook_dispatcher::tests::persistence_errors_propagate_and_roll_back_delivery_state ... ok + +failures: + +---- webhook_api::tests::registration_returns_signing_secret_once_to_authenticated_owner stdout ---- + +thread 'webhook_api::tests::registration_returns_signing_secret_once_to_authenticated_owner' panicked at /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/jsonwebtoken-10.4.0/src/crypto/mod.rs:124:40: + +Could not automatically determine the process-level CryptoProvider from jsonwebtoken crate features. +Call CryptoProvider::install_default() before this point to select a provider manually, or make sure exactly one of the 'rust_crypto' and 'aws_lc_rs' features is enabled. +See the documentation of the CryptoProvider type for more information. + +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- webhook_api::tests::versioned_event_status_validates_jwt_at_the_mounted_path stdout ---- + +thread 'webhook_api::tests::versioned_event_status_validates_jwt_at_the_mounted_path' panicked at /workspace/scratch/1b6b1b08e6b1/toolchain/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/jsonwebtoken-10.4.0/src/crypto/mod.rs:124:40: + +Could not automatically determine the process-level CryptoProvider from jsonwebtoken crate features. +Call CryptoProvider::install_default() before this point to select a provider manually, or make sure exactly one of the 'rust_crypto' and 'aws_lc_rs' features is enabled. +See the documentation of the CryptoProvider type for more information. + + + +failures: + webhook_api::tests::registration_returns_signing_secret_once_to_authenticated_owner + webhook_api::tests::versioned_event_status_validates_jwt_at_the_mounted_path + +test result: FAILED. 19 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.40s + +error: test failed, to rerun pass `--test webhook_delivery_regression_tests` diff --git a/backend/integration/stellar_backend_native_final.log b/backend/integration/stellar_backend_native_final.log new file mode 100644 index 00000000..79da6483 --- /dev/null +++ b/backend/integration/stellar_backend_native_final.log @@ -0,0 +1,97 @@ + Compiling stellar-analysis-backend v0.1.0 (/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend) +warning: unused import: `Span` + --> src/observability/health_checks.rs:15:27 + | +15 | use tracing::{instrument, Span}; + | ^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:283:11 + | +283 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + | + = note: `#[warn(unused_variables)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:348:11 + | +348 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `start` + --> src/observability/health_checks.rs:175:9 + | +175 | let start = Instant::now(); + | ^^^^^ help: if this is intentional, prefix it with an underscore: `_start` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:517:13 + | +517 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:638:13 + | +638 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: function `compute_transaction_hash` is never used + --> src/services/contract.rs:88:4 + | +88 | fn compute_transaction_hash(network_passphrase: &str, tx_envelope_xdr: &[u8]) -> Result<[u8; 32]> { + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(dead_code)]` on by default + +warning: comparison is useless due to type limits + --> src/analytics.rs:97:8 + | +97 | if score.total_assets < 0 { + | ^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_comparisons)]` on by default + +warning: `stellar-analysis-backend` (lib) generated 8 warnings (run `cargo fix --lib -p stellar-analysis-backend` to apply 1 suggestion) + Finished `test` profile [unoptimized] target(s) in 1m 07s + Running tests/webhook_delivery_regression_tests.rs (target/debug/deps/webhook_delivery_regression_tests-b0c910939a113d73) + +running 21 tests +test webhook_api::tests::event_status_returns_delivery_metadata_for_owner ... ok +test webhook_api::tests::event_status_requires_authentication ... ok +test webhook_api::tests::registration_returns_signing_secret_once_to_authenticated_owner ... ok +test webhook_api::tests::event_status_hides_nonowned_missing_and_mismatched_events ... ok +test webhook_event_service::tests::test_filter_application ... ok +test webhooks::events::tests::test_corridor_degradation_detection ... ok +test webhook_api::tests::versioned_event_status_validates_jwt_at_the_mounted_path ... ok +test webhooks::tests::expired_workers_cannot_overwrite_a_new_delivery_outcome ... ok +test webhooks::tests::migration_closes_exhausted_events_without_losing_errors ... ok +test webhooks::tests::retry_delay_is_exponential_and_saturates ... ok +test webhooks::tests::stale_workers_cannot_reserve_the_same_due_event ... ok +test webhooks::tests::test_event_type_conversion ... ok +test webhooks::tests::test_webhook_signature ... ok +test webhooks::tests::webhook_signature_matches_hmac_sha256_test_vector ... ok +test webhooks::tests::terminal_events_are_not_resurrected_by_status_updates ... ok +test webhook_dispatcher::tests::inactive_webhook_does_not_send_or_reset_failure_count ... ok +test webhook_dispatcher::tests::redirects_do_not_contact_another_receiver_or_store_response_bodies ... ok +test webhook_dispatcher::tests::retries_survive_dispatcher_restart_with_identical_signed_delivery ... ok +test webhook_dispatcher::tests::legacy_sqlite_timestamp_is_frozen_before_retry ... ok +test webhook_dispatcher::tests::persistence_errors_propagate_and_roll_back_delivery_state ... ok +test webhook_dispatcher::tests::three_failed_attempts_become_terminal_without_a_fourth_request ... ok + +test result: ok. 21 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.26s + + Running tests/webhook_event_service_tests.rs (target/debug/deps/webhook_event_service_tests-c992f61e9e614d62) + +running 5 tests +test webhook_integration_tests::test_payment_created_webhook ... ok +test webhook_integration_tests::test_event_enqueue_failure_is_not_reported_as_success ... ok +test webhook_integration_tests::test_webhook_filters ... ok +test webhook_integration_tests::test_webhook_registration_and_triggering ... ok +test webhook_integration_tests::test_corridor_health_webhook ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + diff --git a/backend/integration/stellar_backend_resource_failure.log b/backend/integration/stellar_backend_resource_failure.log new file mode 100644 index 00000000..a3c3208f --- /dev/null +++ b/backend/integration/stellar_backend_resource_failure.log @@ -0,0 +1,62 @@ + Compiling stellar-analysis-backend v0.1.0 (/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend) +warning: unused import: `Span` + --> src/observability/health_checks.rs:15:27 + | +15 | use tracing::{instrument, Span}; + | ^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:283:11 + | +283 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + | + = note: `#[warn(unused_variables)]` on by default + +warning: unused variable: `client` + --> src/api/rpc.rs:348:11 + | +348 | State(client): State>, + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `start` + --> src/observability/health_checks.rs:175:9 + | +175 | let start = Instant::now(); + | ^^^^^ help: if this is intentional, prefix it with an underscore: `_start` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:517:13 + | +517 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: unused variable: `client` + --> src/rpc/stellar.rs:638:13 + | +638 | let client = match Client::builder().timeout(request_timeout).build() { + | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` + +warning: function `compute_transaction_hash` is never used + --> src/services/contract.rs:88:4 + | +88 | fn compute_transaction_hash(network_passphrase: &str, tx_envelope_xdr: &[u8]) -> Result<[u8; 32]> { + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(dead_code)]` on by default + +warning: comparison is useless due to type limits + --> src/analytics.rs:97:8 + | +97 | if score.total_assets < 0 { + | ^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_comparisons)]` on by default + +warning: `stellar-analysis-backend` (lib) generated 8 warnings (run `cargo fix --lib -p stellar-analysis-backend` to apply 1 suggestion) +error: could not compile `stellar-analysis-backend` (lib); 8 warnings emitted + +Caused by: + process didn't exit successfully: `/workspace/scratch/1b6b1b08e6b1/toolchain/rustup/toolchains/1.89.0-x86_64-unknown-linux-gnu/bin/rustc --crate-name stellar_analysis_backend --edition=2021 src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no '--allow=clippy::pedantic' '--allow=clippy::nursery' '--warn=clippy::all' '--warn=clippy::unwrap_used' '--allow=clippy::type_complexity' '--allow=clippy::too_many_arguments' '--allow=clippy::should_implement_trait' '--warn=clippy::panic' '--allow=clippy::module_name_repetitions' '--allow=clippy::module_inception' '--allow=clippy::missing_panics_doc' '--allow=clippy::missing_errors_doc' '--allow=clippy::manual_clamp' '--allow=clippy::large_enum_variant' '--allow=clippy::items_after_test_module' '--warn=clippy::expect_used' '--allow=clippy::derivable_impls' --check-cfg 'cfg(docsrs,test)' --check-cfg 'cfg(feature, values("legacy_sep10_tests"))' -C metadata=600bab79758b6afa -C extra-filename=-111383f0acd43d86 --out-dir /workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps -C strip=debuginfo -L dependency=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps --extern aes_gcm=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libaes_gcm-f87b500bc6422363.rmeta --extern anyhow=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libanyhow-19db838894d4f6f4.rmeta --extern argon2=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libargon2-fc1406f967afe783.rmeta --extern async_graphql=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libasync_graphql-8dbc7a8babc6ec23.rmeta --extern async_graphql_axum=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libasync_graphql_axum-91c1841782254c65.rmeta --extern async_lock=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libasync_lock-828d535d46b56f67.rmeta --extern async_trait=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libasync_trait-99904de6cffa05ce.so --extern axum=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libaxum-a8faffd12c20fd8e.rmeta --extern base64=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libbase64-f3df1778b6728a6f.rmeta --extern chrono=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libchrono-774721e92b8e57c5.rmeta --extern csv=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libcsv-9edbfe54f6265dfb.rmeta --extern dashmap=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libdashmap-b5e1921a415c4a68.rmeta --extern data_encoding=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libdata_encoding-1cd8a1580c28a75a.rmeta --extern dotenvy=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libdotenvy-e61ab49826cf1050.rmeta --extern ed25519_dalek=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libed25519_dalek-3c4ced7de20afb5c.rmeta --extern failsafe=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libfailsafe-21bf3e88fa727b69.rmeta --extern futures=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libfutures-3a94c18dc66a637e.rmeta --extern hex=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libhex-e857c8a2edf078fb.rmeta --extern hmac=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libhmac-ba4d99e9cdba6f30.rmeta --extern http_body_util=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libhttp_body_util-331feb07d007c7f4.rmeta --extern ipnetwork=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libipnetwork-f8825c56a1b07f7c.rmeta --extern jsonwebtoken=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libjsonwebtoken-4469b9316a16b5e3.rmeta --extern lazy_static=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/liblazy_static-9ec1bdd762aba982.rmeta --extern lettre=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/liblettre-ff0e5eb01c277f14.rmeta --extern log=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/liblog-1224bdafb1f8cd4c.rmeta --extern md5=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libmd5-fa7480d1c6144b5a.rmeta --extern ndarray=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libndarray-e48dda3ece46b6d6.rmeta --extern once_cell=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libonce_cell-c1e6cb9cb0e9c149.rmeta --extern opentelemetry=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libopentelemetry-41076ee2f49eeb97.rmeta --extern opentelemetry_otlp=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libopentelemetry_otlp-3d195670407f13e7.rmeta --extern opentelemetry_sdk=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libopentelemetry_sdk-28b8a91a63bafc86.rmeta --extern prometheus=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libprometheus-4cf28eef66b0a63b.rmeta --extern rand=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/librand-156950e2331f7533.rmeta --extern redis=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libredis-83b3a67696d5bc0c.rmeta --extern regex=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libregex-c6e765ba68350673.rmeta --extern reqwest=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libreqwest-ec155b5d3176e545.rmeta --extern rust_xlsxwriter=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/librust_xlsxwriter-b0800e2b183c831d.rmeta --extern serde=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libserde-5744592ed17ef8c2.rmeta --extern serde_json=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libserde_json-4238bea3d3cb4f54.rmeta --extern sha2=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libsha2-9ad45c77bb727c33.rmeta --extern sqlx=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libsqlx-08671638537a1f8e.rmeta --extern stellar_xdr=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libstellar_xdr-00b6778a68cc9aea.rmeta --extern stellar_sdk=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libstellar_sdk-b168b82a774482a8.rmeta --extern thiserror=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libthiserror-e4f76b8b3cbd4ca7.rmeta --extern tokio=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtokio-350c09bed3605045.rmeta --extern tokio_tungstenite=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtokio_tungstenite-99b7be531df960fe.rmeta --extern toml=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtoml-4822c34dabc99fdb.rmeta --extern tower=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtower-80fa402008bb99d5.rmeta --extern tower_http=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtower_http-bea37909f26944fa.rmeta --extern tracing=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtracing-553ac499056a6916.rmeta --extern tracing_appender=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtracing_appender-274362ccc05ded1d.rmeta --extern tracing_log=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtracing_log-64fcdd65ebdd99aa.rmeta --extern tracing_logstash=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtracing_logstash-69806c2115cdb00d.rmeta --extern tracing_opentelemetry=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtracing_opentelemetry-27aceef89bb0363f.rmeta --extern tracing_subscriber=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libtracing_subscriber-ab13a0a07d6fdb00.rmeta --extern url=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/liburl-c4ee32fce07952fb.rmeta --extern urlencoding=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/liburlencoding-bd0f0600e6ed6fc5.rmeta --extern utoipa=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libutoipa-873321ff00e354d5.rmeta --extern utoipa_swagger_ui=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libutoipa_swagger_ui-2ab06d5ac2487bfa.rmeta --extern uuid=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libuuid-92aac9ccaedf92ce.rmeta --extern validator=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/deps/libvalidator-873184f7fa329dc2.rmeta -L native=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/build/aws-lc-sys-ab05c3b891813490/out -L native=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/build/ring-eb3f1d84b677badd/out -L native=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/build/libsqlite3-sys-d625df24976f0ac8/out -L native=/workspace/scratch/1b6b1b08e6b1/backlog/stellar_backend/target/debug/build/libsodium-sys-5a14c749ac72a8b8/out/installed/lib -L native=/usr/lib/x86_64-linux-gnu` (signal: 9, SIGKILL: kill) diff --git a/backend/src/lib.rs b/backend/src/lib.rs new file mode 100644 index 00000000..39298c29 --- /dev/null +++ b/backend/src/lib.rs @@ -0,0 +1,6 @@ +//! Stellar Insights backend library. +//! +//! Modules land here as focused, reviewable units. `webhooks` delivers +//! signed, idempotent outbound event notifications to subscriber URLs. + +pub mod webhooks; diff --git a/backend/src/webhooks/deliver.rs b/backend/src/webhooks/deliver.rs new file mode 100644 index 00000000..80024ec0 --- /dev/null +++ b/backend/src/webhooks/deliver.rs @@ -0,0 +1,198 @@ +use std::time::Duration; + +use serde_json::json; +use thiserror::Error; + +use super::event::WebhookEvent; +use super::sign::{self, IDEMPOTENCY_HEADER, SIGNATURE_HEADER}; +use super::status::{DeliveryRecord, DeliveryStatus, DeliveryStore}; + +/// Bounded retry policy for outbound webhook delivery. +#[derive(Debug, Clone)] +pub struct DeliveryPolicy { + pub max_attempts: u32, + pub initial_backoff: Duration, + pub max_backoff: Duration, +} + +impl Default for DeliveryPolicy { + fn default() -> Self { + Self { + max_attempts: 5, + initial_backoff: Duration::from_millis(100), + max_backoff: Duration::from_secs(30), + } + } +} + +impl DeliveryPolicy { + /// Backoff delay before the given 1-based attempt number. + pub fn backoff_for_attempt(&self, attempt: u32) -> Duration { + if attempt <= 1 { + return Duration::from_millis(0); + } + // Scale in nanoseconds so a nonzero sub-millisecond wait stays nonzero. + // Any factor beyond u128 already exceeds every representable Duration + // for a nonzero initial delay; saturation also keeps zero delays zero. + let factor = 1u128.checked_shl(attempt - 2).unwrap_or(u128::MAX); + let nanos = self + .initial_backoff + .as_nanos() + .saturating_mul(factor) + .min(self.max_backoff.as_nanos()); + // The cap is itself a Duration, so both components fit their types. + Duration::new((nanos / 1_000_000_000) as u64, (nanos % 1_000_000_000) as u32) + } +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum DeliverError { + #[error("cannot serialize webhook payload: {0}")] + Serialization(String), + #[error("subscriber returned non-success status {0}")] + NonSuccess(u16), + #[error("transport failure: {0}")] + Transport(String), + #[error("exhausted {0} delivery attempts")] + Exhausted(u32), +} + +/// One HTTP-shaped delivery attempt result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeliveryAttempt { + pub attempt: u32, + pub status_code: Option, + pub ok: bool, + pub error: Option, +} + +/// Abstraction over the HTTP client so unit tests can inject a fake transport. +pub trait HttpTransport { + fn post(&mut self, url: &str, headers: &[(&str, String)], body: &[u8]) -> Result; +} + +/// Waits between delivery attempts. The default implementation waits in real +/// time; an embedding runtime may supply its own clock-aware implementation. +pub trait RetrySleeper { + fn sleep(&mut self, delay: Duration); +} + +#[derive(Debug, Default)] +pub struct ThreadSleeper; + +impl RetrySleeper for ThreadSleeper { + fn sleep(&mut self, delay: Duration) { + std::thread::sleep(delay); + } +} + +/// Delivers signed webhook events with bounded retry-with-backoff. +pub struct WebhookDeliverer { + pub transport: T, + pub policy: DeliveryPolicy, + pub store: DeliveryStore, + sleeper: S, +} + +impl WebhookDeliverer { + pub fn new(transport: T, policy: DeliveryPolicy) -> Self { + Self::with_sleeper(transport, policy, ThreadSleeper) + } +} + +impl WebhookDeliverer { + pub fn with_sleeper(transport: T, policy: DeliveryPolicy, sleeper: S) -> Self { + Self { + transport, + policy, + store: DeliveryStore::default(), + sleeper, + } + } + + /// Deliver `event` to `url`, signing with `secret`. + /// The same `event.idempotency_key` is sent on every retry attempt. + pub fn deliver( + &mut self, + subscriber_id: &str, + url: &str, + secret: &[u8], + event: &WebhookEvent, + ) -> Result { + let body = serde_json::to_vec(&json!({ + "id": event.idempotency_key, + "type": event.kind, + "data": event.payload, + })) + .map_err(|error| DeliverError::Serialization(error.to_string()))?; + + let signature = sign::sign_payload(secret, &body); + let idem_header = event.idempotency_key.to_string(); + + let mut last_error = DeliverError::Exhausted(0); + + for attempt in 1..=self.policy.max_attempts { + let delay = self.policy.backoff_for_attempt(attempt); + if !delay.is_zero() { + self.sleeper.sleep(delay); + } + + let headers = [ + (SIGNATURE_HEADER, signature.clone()), + (IDEMPOTENCY_HEADER, idem_header.clone()), + ("Content-Type", "application/json".to_string()), + ]; + + match self.transport.post(url, &headers, &body) { + Ok(code) if (200..300).contains(&code) => { + let record = DeliveryRecord { + event_id: event.idempotency_key, + subscriber_id: subscriber_id.to_string(), + status: DeliveryStatus::Delivered, + attempts: attempt, + last_status_code: Some(code), + last_error: None, + }; + self.store.upsert(record.clone()); + return Ok(record); + } + Ok(code) => { + last_error = DeliverError::NonSuccess(code); + self.store.upsert(DeliveryRecord { + event_id: event.idempotency_key, + subscriber_id: subscriber_id.to_string(), + status: DeliveryStatus::Pending, + attempts: attempt, + last_status_code: Some(code), + last_error: Some(format!("HTTP {code}")), + }); + } + Err(msg) => { + last_error = DeliverError::Transport(msg.clone()); + self.store.upsert(DeliveryRecord { + event_id: event.idempotency_key, + subscriber_id: subscriber_id.to_string(), + status: DeliveryStatus::Pending, + attempts: attempt, + last_status_code: None, + last_error: Some(msg), + }); + } + } + } + + let failed = DeliveryRecord { + event_id: event.idempotency_key, + subscriber_id: subscriber_id.to_string(), + status: DeliveryStatus::Failed, + attempts: self.policy.max_attempts, + last_status_code: match &last_error { + DeliverError::NonSuccess(code) => Some(*code), + _ => None, + }, + last_error: Some(last_error.to_string()), + }; + self.store.upsert(failed.clone()); + Err(DeliverError::Exhausted(self.policy.max_attempts)) + } +} diff --git a/backend/src/webhooks/event.rs b/backend/src/webhooks/event.rs new file mode 100644 index 00000000..fb3ca38a --- /dev/null +++ b/backend/src/webhooks/event.rs @@ -0,0 +1,45 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Kind of outbound notification a subscriber may receive. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WebhookEventKind { + CorridorAlert, + AnomalyDetected, + SnapshotCompleted, +} + +/// A logical webhook event. The `idempotency_key` is assigned once and +/// reused for every delivery attempt of this event. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WebhookEvent { + pub idempotency_key: Uuid, + pub kind: WebhookEventKind, + pub payload: serde_json::Value, +} + +impl WebhookEvent { + /// Create a new logical event with a freshly generated idempotency key. + pub fn new(kind: WebhookEventKind, payload: serde_json::Value) -> Self { + Self { + idempotency_key: Uuid::new_v4(), + kind, + payload, + } + } + + /// Rebuild an event that must keep a previously issued idempotency key + /// (e.g. when loading from durable storage before a retry). + pub fn with_idempotency_key( + idempotency_key: Uuid, + kind: WebhookEventKind, + payload: serde_json::Value, + ) -> Self { + Self { + idempotency_key, + kind, + payload, + } + } +} diff --git a/backend/src/webhooks/mod.rs b/backend/src/webhooks/mod.rs new file mode 100644 index 00000000..3bc3c708 --- /dev/null +++ b/backend/src/webhooks/mod.rs @@ -0,0 +1,16 @@ +//! Outbound webhook delivery for subscriber integrations. +//! +//! Design follows the Stripe/GitHub convention subscribers already know: +//! every logical event gets a stable idempotency key that does not change +//! across retries, and every payload is HMAC-signed with a per-subscriber +//! secret so forged deliveries are detectable. + +pub mod deliver; +pub mod event; +pub mod sign; +pub mod status; + +pub use deliver::{DeliverError, DeliveryAttempt, DeliveryPolicy, WebhookDeliverer}; +pub use event::{WebhookEvent, WebhookEventKind}; +pub use sign::{sign_payload, verify_signature, SIGNATURE_HEADER}; +pub use status::{DeliveryRecord, DeliveryStatus, DeliveryStore}; diff --git a/backend/src/webhooks/sign.rs b/backend/src/webhooks/sign.rs new file mode 100644 index 00000000..81d1220e --- /dev/null +++ b/backend/src/webhooks/sign.rs @@ -0,0 +1,51 @@ +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +type HmacSha256 = Hmac; + +/// Header name carrying the hex-encoded HMAC-SHA256 signature. +pub const SIGNATURE_HEADER: &str = "X-Stellar-Insights-Signature"; + +/// Header name carrying the stable per-event idempotency key. +pub const IDEMPOTENCY_HEADER: &str = "X-Stellar-Insights-Idempotency-Key"; + +/// HMAC-SHA256 sign `payload` with the per-subscriber `secret`. +/// Returns a lowercase hex digest suitable for the signature header. +pub fn sign_payload(secret: &[u8], payload: &[u8]) -> String { + let mut mac = + HmacSha256::new_from_slice(secret).expect("HMAC accepts keys of any size"); + mac.update(payload); + let result = mac.finalize().into_bytes(); + hex::encode(result) +} + +/// Constant-time verification of a hex-encoded signature against `payload`. +pub fn verify_signature(secret: &[u8], payload: &[u8], signature_hex: &str) -> bool { + let expected = sign_payload(secret, payload); + // Timing-safe compare via hmac's own verifier when possible; fall back + // to length-checked equality on decoded bytes. + match (hex::decode(signature_hex), hex::decode(&expected)) { + (Ok(provided), Ok(exp)) if provided.len() == exp.len() => { + let mut mac = + HmacSha256::new_from_slice(secret).expect("HMAC accepts keys of any size"); + mac.update(payload); + mac.verify_slice(&provided).is_ok() + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_signature() { + let secret = b"subscriber-secret"; + let payload = br#"{"hello":"world"}"#; + let sig = sign_payload(secret, payload); + assert!(verify_signature(secret, payload, &sig)); + assert!(!verify_signature(secret, payload, "00")); + assert!(!verify_signature(b"other", payload, &sig)); + } +} diff --git a/backend/src/webhooks/status.rs b/backend/src/webhooks/status.rs new file mode 100644 index 00000000..4bb07e63 --- /dev/null +++ b/backend/src/webhooks/status.rs @@ -0,0 +1,62 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Outcome of an outbound webhook delivery. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DeliveryStatus { + Pending, + Delivered, + Failed, +} + +/// Queryable delivery status for one logical event and subscriber. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeliveryRecord { + pub event_id: Uuid, + pub subscriber_id: String, + pub status: DeliveryStatus, + pub attempts: u32, + pub last_status_code: Option, + pub last_error: Option, +} + +/// In-memory delivery status, scoped by logical event and subscriber. This +/// library does not persist the records across process restarts. +#[derive(Debug, Default, Clone)] +pub struct DeliveryStore { + by_event: BTreeMap>, +} + +impl DeliveryStore { + pub fn upsert(&mut self, record: DeliveryRecord) { + self.by_event + .entry(record.event_id) + .or_default() + .insert(record.subscriber_id.clone(), record); + } + + /// All subscriber outcomes for an event, ordered by subscriber id. + pub fn by_event_id(&self, event_id: Uuid) -> Vec<&DeliveryRecord> { + self.by_event + .get(&event_id) + .map(|subscribers| subscribers.values().collect()) + .unwrap_or_default() + } + + pub fn by_subscriber(&self, event_id: Uuid, subscriber_id: &str) -> Option<&DeliveryRecord> { + self.by_event + .get(&event_id) + .and_then(|subscribers| subscribers.get(subscriber_id)) + } + + pub fn len(&self) -> usize { + self.by_event.values().map(BTreeMap::len).sum() + } + + pub fn is_empty(&self) -> bool { + self.by_event.is_empty() + } +} diff --git a/backend/tests/idempotency_key_test.rs b/backend/tests/idempotency_key_test.rs new file mode 100644 index 00000000..45ef8ef9 --- /dev/null +++ b/backend/tests/idempotency_key_test.rs @@ -0,0 +1,279 @@ +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use stellar_insights_backend::webhooks::{ + deliver::{DeliverError, DeliveryPolicy, HttpTransport, RetrySleeper, WebhookDeliverer}, + event::{WebhookEvent, WebhookEventKind}, + sign::IDEMPOTENCY_HEADER, + status::DeliveryStatus, +}; +use uuid::Uuid; + +#[derive(Default)] +struct RecordingTransport { + calls: Arc, Vec)>>>, + /// Return codes per call; last code repeats if exhausted. + responses: Vec>, + clock: Arc>, + call_times: Arc>>, +} + +impl HttpTransport for RecordingTransport { + fn post(&mut self, url: &str, headers: &[(&str, String)], body: &[u8]) -> Result { + self.call_times + .lock() + .unwrap() + .push(*self.clock.lock().unwrap()); + let owned_headers: Vec<(String, String)> = headers + .iter() + .map(|(k, v)| ((*k).to_string(), v.clone())) + .collect(); + self.calls + .lock() + .unwrap() + .push((url.to_string(), owned_headers, body.to_vec())); + let idx = self.calls.lock().unwrap().len() - 1; + if idx < self.responses.len() { + self.responses[idx].clone() + } else { + self.responses.last().cloned().unwrap_or(Ok(200)) + } + } +} + +#[test] +fn idempotency_key_stable_across_retries() { + let calls = Arc::new(Mutex::new(Vec::new())); + let transport = RecordingTransport { + calls: Arc::clone(&calls), + responses: vec![Ok(500), Ok(502), Ok(200)], + ..RecordingTransport::default() + }; + let policy = DeliveryPolicy { + max_attempts: 5, + ..DeliveryPolicy::default() + }; + let mut deliverer = WebhookDeliverer::new(transport, policy); + + let event = WebhookEvent::new( + WebhookEventKind::CorridorAlert, + serde_json::json!({"corridor": "US-MX", "score": 0.12}), + ); + let key_before = event.idempotency_key; + + let record = deliverer + .deliver("sub_1", "https://example.test/hook", b"secret", &event) + .expect("eventually delivered"); + + assert_eq!(record.event_id, key_before); + assert_eq!(record.attempts, 3); + + let captured = calls.lock().unwrap(); + assert_eq!(captured.len(), 3); + let keys: Vec = captured + .iter() + .map(|(_, headers, _)| { + headers + .iter() + .find(|(k, _)| k == IDEMPOTENCY_HEADER) + .map(|(_, v)| v.clone()) + .expect("idempotency header present") + }) + .collect(); + assert_eq!(keys[0], keys[1]); + assert_eq!(keys[1], keys[2]); + assert_eq!(keys[0], key_before.to_string()); + + // Rebuilding the same logical event keeps the key. + let replayed = WebhookEvent::with_idempotency_key( + key_before, + WebhookEventKind::CorridorAlert, + serde_json::json!({"corridor": "US-MX", "score": 0.12}), + ); + assert_eq!(replayed.idempotency_key, key_before); + assert_ne!(Uuid::nil(), key_before); +} + +struct RecordingSleeper { + clock: Arc>, +} + +impl RetrySleeper for RecordingSleeper { + fn sleep(&mut self, delay: Duration) { + *self.clock.lock().unwrap() += delay; + } +} + +#[test] +fn transport_attempts_observe_backoff_and_stop_after_success() { + let clock = Arc::new(Mutex::new(Duration::ZERO)); + let call_times = Arc::new(Mutex::new(Vec::new())); + let calls = Arc::new(Mutex::new(Vec::new())); + let transport = RecordingTransport { + calls: Arc::clone(&calls), + responses: vec![Ok(503), Err("connection lost".to_string()), Ok(204)], + clock: Arc::clone(&clock), + call_times: Arc::clone(&call_times), + }; + let sleeper = RecordingSleeper { clock }; + let policy = DeliveryPolicy { + max_attempts: 5, + initial_backoff: Duration::from_millis(10), + max_backoff: Duration::from_millis(15), + }; + let mut deliverer = WebhookDeliverer::with_sleeper(transport, policy, sleeper); + let event = WebhookEvent::new( + WebhookEventKind::AnomalyDetected, + serde_json::json!({"score": 0.12}), + ); + + let record = deliverer + .deliver("sub_1", "https://example.test/hook", b"secret", &event) + .unwrap(); + + assert_eq!( + *call_times.lock().unwrap(), + vec![ + Duration::ZERO, + Duration::from_millis(10), + Duration::from_millis(25), + ] + ); + assert_eq!(record.status, DeliveryStatus::Delivered); + assert_eq!(record.attempts, 3); + let captured = calls.lock().unwrap(); + assert_eq!(captured[0].1, captured[1].1); + assert_eq!(captured[1].1, captured[2].1); + assert_eq!(captured[0].2, captured[1].2); + assert_eq!(captured[1].2, captured[2].2); +} + +#[test] +fn failed_subscriber_does_not_replace_another_subscribers_success() { + let calls = Arc::new(Mutex::new(Vec::new())); + let transport = RecordingTransport { + calls: Arc::clone(&calls), + responses: vec![Ok(200), Ok(503)], + ..RecordingTransport::default() + }; + let policy = DeliveryPolicy { + max_attempts: 2, + initial_backoff: Duration::ZERO, + max_backoff: Duration::ZERO, + }; + let mut deliverer = WebhookDeliverer::new(transport, policy); + let event = WebhookEvent::new( + WebhookEventKind::SnapshotCompleted, + serde_json::json!({"snapshot": "snapshot-1"}), + ); + let delivered = deliverer + .deliver("sub_a", "https://example.test/a", b"a-secret", &event) + .unwrap(); + assert_eq!( + deliverer.deliver("sub_b", "https://example.test/b", b"b-secret", &event), + Err(DeliverError::Exhausted(2)) + ); + + assert_eq!(calls.lock().unwrap().len(), 3); + assert_eq!( + deliverer + .store + .by_subscriber(event.idempotency_key, "sub_a"), + Some(&delivered) + ); + let failed = deliverer + .store + .by_subscriber(event.idempotency_key, "sub_b") + .unwrap(); + assert_eq!(failed.status, DeliveryStatus::Failed); + assert_eq!(failed.attempts, 2); + assert_eq!(failed.last_status_code, Some(503)); + assert_eq!(deliverer.store.len(), 2); + let outcomes = deliverer.store.by_event_id(event.idempotency_key); + assert_eq!(outcomes.len(), 2); + assert_eq!(outcomes[0].subscriber_id, "sub_a"); + assert_eq!(outcomes[1].subscriber_id, "sub_b"); + assert!(deliverer.store.by_event_id(Uuid::new_v4()).is_empty()); +} + +#[test] +fn backoff_preserves_fractional_delays_and_observed_waits() { + let clock = Arc::new(Mutex::new(Duration::ZERO)); + let call_times = Arc::new(Mutex::new(Vec::new())); + let transport = RecordingTransport { + responses: vec![Ok(503), Ok(503), Ok(503), Ok(204)], + clock: Arc::clone(&clock), + call_times: Arc::clone(&call_times), + ..RecordingTransport::default() + }; + let policy = DeliveryPolicy { + max_attempts: 4, + initial_backoff: Duration::from_micros(250), + max_backoff: Duration::from_micros(750), + }; + let mut deliverer = + WebhookDeliverer::with_sleeper(transport, policy, RecordingSleeper { clock }); + let event = WebhookEvent::new( + WebhookEventKind::SnapshotCompleted, + serde_json::json!({"snapshot": "fractional-backoff"}), + ); + let record = deliverer + .deliver("sub_1", "https://example.test/hook", b"secret", &event) + .unwrap(); + + assert_eq!(record.attempts, 4); + assert_eq!( + *call_times.lock().unwrap(), + vec![ + Duration::ZERO, + Duration::from_micros(250), + Duration::from_micros(750), + Duration::from_micros(1500), + ] + ); +} + +#[test] +fn backoff_scales_to_the_cap_without_exponent_or_duration_overflow() { + let policy = DeliveryPolicy { + max_attempts: 25, + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_secs(300), + }; + assert_eq!(policy.backoff_for_attempt(18), Duration::from_millis(65_536)); + assert_eq!(policy.backoff_for_attempt(19), Duration::from_millis(131_072)); + assert_eq!(policy.backoff_for_attempt(20), Duration::from_millis(262_144)); + assert_eq!(policy.backoff_for_attempt(21), Duration::from_secs(300)); + assert_eq!(policy.backoff_for_attempt(u32::MAX), Duration::from_secs(300)); + + let largest = DeliveryPolicy { + initial_backoff: Duration::from_nanos(1), + max_backoff: Duration::MAX, + ..policy.clone() + }; + assert_eq!(largest.backoff_for_attempt(u32::MAX), Duration::MAX); + let overflowing_product = DeliveryPolicy { + initial_backoff: Duration::MAX, + ..largest.clone() + }; + assert_eq!(overflowing_product.backoff_for_attempt(129), Duration::MAX); + let zero = DeliveryPolicy { + initial_backoff: Duration::ZERO, + ..largest + }; + assert_eq!(zero.backoff_for_attempt(u32::MAX), Duration::ZERO); + let no_wait = DeliveryPolicy { + max_backoff: Duration::ZERO, + ..policy + }; + assert_eq!(no_wait.backoff_for_attempt(u32::MAX), Duration::ZERO); + + let default_policy = DeliveryPolicy::default(); + assert_eq!(default_policy.backoff_for_attempt(0), Duration::ZERO); + for (attempt, millis) in [(1, 0), (2, 100), (3, 200), (4, 400), (5, 800)] { + assert_eq!( + default_policy.backoff_for_attempt(attempt), + Duration::from_millis(millis) + ); + } +} diff --git a/backend/tests/tampered_payload_test.rs b/backend/tests/tampered_payload_test.rs new file mode 100644 index 00000000..1a17f1cd --- /dev/null +++ b/backend/tests/tampered_payload_test.rs @@ -0,0 +1,29 @@ +use stellar_insights_backend::webhooks::sign::{sign_payload, verify_signature, SIGNATURE_HEADER}; + +#[test] +fn tampered_payload_fails_verification() { + let secret = b"per-subscriber-secret"; + let original = br#"{"id":"evt_1","type":"anomaly_detected","data":{"score":0.9}}"#; + let signature = sign_payload(secret, original); + + assert!( + verify_signature(secret, original, &signature), + "untampered payload must verify" + ); + + let mut tampered = original.to_vec(); + // Flip a byte inside the JSON body. + let idx = tampered.iter().position(|&b| b == b'9').expect("digit present"); + tampered[idx] = b'1'; + + assert!( + !verify_signature(secret, &tampered, &signature), + "tampered payload must not verify under the original signature" + ); + + // Wrong secret also fails. + assert!(!verify_signature(b"other-secret", original, &signature)); + + // Documented header name is stable for consumers. + assert_eq!(SIGNATURE_HEADER, "X-Stellar-Insights-Signature"); +}