From 119e629fb5ec756bb51c4eae81e7491c24dd2dc2 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar <65710+neekolas@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:49:00 -0700 Subject: [PATCH] ci: run five-minute Rust chaos checks on affected PRs --- .github/CI.md | 8 ++++ .github/ci-suites.json | 43 +++++++++++++++++++ .github/ci-suites.yml | 15 +++++++ .github/workflows/ci.yml | 13 ++++++ .github/workflows/test-chaos.yml | 73 ++++++++++++++++++++++++++++++++ apps/chaos/AGENTS.md | 4 +- apps/chaos/README.md | 15 +++++++ 7 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-chaos.yml diff --git a/.github/CI.md b/.github/CI.md index 6ed6565f84..83fba989b9 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -117,6 +117,14 @@ browser, and SDK platform suites run on the merge. `test_workspace` builds with the `ci-test` Cargo profile: release speed with overflow checks and debug assertions in the workspace crates. Android lint has no suite of its own: `sdk-build` in `test-android.yml` runs `gradlew build`, which includes it. +`test_chaos` blocks ready pull requests that touch `crates/xmtp_mls/` or +`apps/chaos/`, and runs on merges. It uses GitHub-hosted `ubuntu-24.04` runners, +builds the local backend, passes a 50-round baseline, then enables all fault +types for a five-minute run. Setup, builds, the baseline, and completion of the final +round are outside the five-minute window. Failed and inconclusive runs fail +the job without retries. The `chaos-report` artifact contains only bounded +status and inspection output; raw `.chaos/` files contain private test keys +and are never uploaded. Source lint does not generate SDK products or run compiler checks. Test owns full types, full lint, Clippy, SDK and runtime checks. Pull requests that only change Rust core crates omit the SDK and mobile suites; the merge run covers diff --git a/.github/ci-suites.json b/.github/ci-suites.json index 3dcc28ca64..b12ab5f15c 100644 --- a/.github/ci-suites.json +++ b/.github/ci-suites.json @@ -738,6 +738,49 @@ "labels": [] } }, + "test_chaos": { + "title": "Rust chaos", + "phase": "test", + "workflow": "test-chaos.yml", + "run_on": [ + "ready_pr_push", + "merge" + ], + "disable_on_forks": false, + "secrets": "cachix-pr", + "permissions": { + "contents": "read" + }, + "covered_by": null, + "skip_full_runs": false, + "kache": [], + "products": [], + "runners": { + "runner": { + "pr": "ubuntu-24.04", + "merge": "ubuntu-24.04" + } + }, + "with": {}, + "cargo_roots": [], + "crates": [], + "workflow_files": [ + ".github/actions/setup-nix/**", + ".github/workflows/test-chaos.yml" + ], + "workflow_tier": "blocking", + "paths": [ + "apps/chaos/**", + "crates/xmtp_mls/**" + ], + "labels": [], + "advisory": { + "cargo_roots": [], + "crates": [], + "paths": [], + "labels": [] + } + }, "docs_rust": { "title": "Rust reference", "phase": "test", diff --git a/.github/ci-suites.yml b/.github/ci-suites.yml index c1b56196ae..d8691476ae 100644 --- a/.github/ci-suites.yml +++ b/.github/ci-suites.yml @@ -546,6 +546,21 @@ suites: - "{webdriver.json,.config/nextest.toml,dev/wasm.just}" - "{nix/package/wasm-nextest.nix,nix/ci-checks.nix,nix/lib/junit.nix}" - "nix/services/**" + test_chaos: + title: Rust chaos + phase: test + workflow: test-chaos.yml + run_on: [ready_pr_push, merge] + disable_on_forks: false + secrets: cachix-pr + permissions: {contents: read} + kache: [] + products: [] + runners: + runner: {pr: ubuntu-24.04, merge: ubuntu-24.04} + pr_filters: + - "crates/xmtp_mls/**" + - "apps/chaos/**" docs_rust: title: Rust reference phase: test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3abb48fa95..0bad5d7a2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,6 +215,18 @@ jobs: secrets: inherit with: runner: ${{ github.event_name == 'pull_request' && 'blacksmith-16vcpu-ubuntu-2404' || 'ubuntu-24.04' }} + test_chaos: + name: Rust chaos + needs: + - detect-changes + if: contains(fromJSON(needs.detect-changes.outputs.test_suites), 'test_chaos') + uses: ./.github/workflows/test-chaos.yml + permissions: + contents: read + secrets: + CACHIX_PR_AUTH_TOKEN: ${{ secrets.CACHIX_PR_AUTH_TOKEN }} + with: + runner: ubuntu-24.04 docs_rust: name: Rust reference needs: @@ -504,6 +516,7 @@ jobs: - test_nix_checks - test_workspace - test_wasm + - test_chaos - docs_rust - check_types - check_sdk diff --git a/.github/workflows/test-chaos.yml b/.github/workflows/test-chaos.yml new file mode 100644 index 0000000000..5240ec5f6b --- /dev/null +++ b/.github/workflows/test-chaos.yml @@ -0,0 +1,73 @@ +name: "Test: Rust chaos" +on: + workflow_call: + inputs: + runner: + description: The GitHub-hosted runner label from .github/ci-suites.yml. + type: string + required: true + secrets: + CACHIX_PR_AUTH_TOKEN: + required: false +permissions: + contents: read +env: + NIX_DEVSHELL: rust +jobs: + test: + name: Five-minute fault run + runs-on: ${{ inputs.runner }} + # Includes the source builds and baseline before the five-minute fault run. + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - name: Set up Nix + uses: ./.github/actions/setup-nix + with: + cache-mode: "ci" + cachix-pr-auth-token: ${{ secrets.CACHIX_PR_AUTH_TOKEN }} + github-token: ${{ github.token }} + - name: Build the chaos supervisor + run: dev/nix-shell 'just chaos build' + - name: Start the local backend and fault proxy + run: dev/nix-shell 'just backend up anvil toxiproxy' + - name: Test the chaos suite + run: dev/nix-shell 'just chaos test' + - name: Run the 50-round baseline + id: baseline + run: dev/nix-shell 'just chaos baseline --rounds 50 --seed 1338 --directory .chaos/ci-baseline' + - name: Run five minutes with all fault types enabled + # Timed runs finish their current round before checking the deadline. + run: dev/nix-shell 'just chaos run --hours 0.08333333333333333 --seed 1340 --faults all --directory .chaos/ci-faults' + - name: Report bounded run evidence + if: ${{ always() && steps.baseline.outcome != 'skipped' }} + run: | + for phase in baseline faults; do + directory=".chaos/ci-$phase" + if [ -d "$directory" ]; then + dev/nix-shell "just chaos status --directory $directory" + dev/nix-shell "just chaos inspect $directory" + fi + done | tee "$RUNNER_TEMP/chaos-report.txt" + # Raw .chaos files contain wallets and database keys. Upload only the + # bounded status/inspect output, including on a failed or inconclusive run. + - name: Retain the bounded chaos report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: chaos-report + path: ${{ runner.temp }}/chaos-report.txt + if-no-files-found: ignore + retention-days: 7 + - name: Retain backend startup diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: chaos-backend-startup-logs + path: ${{ runner.temp }}/backend-startup-logs + if-no-files-found: ignore + retention-days: 7 + - name: Stop the local backend + if: always() + run: dev/nix-shell 'just backend down' diff --git a/apps/chaos/AGENTS.md b/apps/chaos/AGENTS.md index c3e19b3a67..79e022f1e6 100644 --- a/apps/chaos/AGENTS.md +++ b/apps/chaos/AGENTS.md @@ -1,7 +1,9 @@ # Chaos suite `xchaos` runs real client processes against this worktree's self-hosted backend. -It is outside `default-members` and CI. +It is outside `default-members`. The blocking `Rust chaos` CI suite runs a +50-round baseline and five minutes of faults for changes to this app or +`crates/xmtp_mls/`, and on merges. See [CI selection](../../.github/CI.md). ## Run gate diff --git a/apps/chaos/README.md b/apps/chaos/README.md index aef4da82bc..f1a55a1af6 100644 --- a/apps/chaos/README.md +++ b/apps/chaos/README.md @@ -75,6 +75,21 @@ Runs accept `--rounds`, `--hours`, `--seed`, `--faults`, `--strict`, and Use a comma-separated list to combine sets, such as `--faults network,disk`. Baseline always disables faults. +## Pull request check + +Changes to `crates/xmtp_mls/` or `apps/chaos/` select the blocking `Rust chaos` +CI suite on ready pull requests. It also runs on merges. The check uses a +GitHub-hosted `ubuntu-24.04` runner and a local backend built from the tested +code. A 50-round baseline with seed 1338 must pass before an all-fault run +with seed 1340 runs for five minutes. The final round finishes before the +time limit is checked. Setup, builds, and the baseline take additional time. +Failures and inconclusive results fail the check without retries. +Fault types are sampled; a short run need not exercise every type. + +The `chaos-report` artifact retains bounded `status` and `inspect` output for +seven days. Raw `.chaos/` files are never uploaded because they contain private +wallets and database keys. See [CI selection](../../.github/CI.md). + ## Murmur run sequence Before the approved run, test short baselines with seeds 1337 and 1338 and