ci: run five-minute Rust chaos checks on affected PRs

.github/CI.md

@@ -117,6 +117,14 @@ browser, and SDK platform suites run on the merge. `test_workspace` builds
 with the `ci-test` Cargo profile: release speed with overflow checks and
 debug assertions in the workspace crates. Android lint has no suite of its own: `sdk-build` in
 `test-android.yml` runs `gradlew build`, which includes it.
+`test_chaos` blocks ready pull requests that touch `crates/xmtp_mls/` or
+`apps/chaos/`, and runs on merges. It uses GitHub-hosted `ubuntu-24.04` runners,
+builds the local backend, passes a 50-round baseline, then enables all fault
+types for a five-minute run. Setup, builds, the baseline, and completion of the final
+round are outside the five-minute window. Failed and inconclusive runs fail
+the job without retries. The `chaos-report` artifact contains only bounded
+status and inspection output; raw `.chaos/` files contain private test keys
+and are never uploaded.
 Source lint does not generate SDK products or run compiler checks. Test owns
 full types, full lint, Clippy, SDK and runtime checks. Pull requests that only
 change Rust core crates omit the SDK and mobile suites; the merge run covers

.github/ci-suites.json

@@ -738,6 +738,49 @@
         "labels": []
       }
     },
+    "test_chaos": {
+      "title": "Rust chaos",
+      "phase": "test",
+      "workflow": "test-chaos.yml",
+      "run_on": [
+        "ready_pr_push",
+        "merge"
+      ],
+      "disable_on_forks": false,
+      "secrets": "cachix-pr",
+      "permissions": {
+        "contents": "read"
+      },
+      "covered_by": null,
+      "skip_full_runs": false,
+      "kache": [],
+      "products": [],
+      "runners": {
+        "runner": {
+          "pr": "ubuntu-24.04",
+          "merge": "ubuntu-24.04"
+        }
+      },
+      "with": {},
+      "cargo_roots": [],
+      "crates": [],
+      "workflow_files": [
+        ".github/actions/setup-nix/**",
+        ".github/workflows/test-chaos.yml"
+      ],
+      "workflow_tier": "blocking",
+      "paths": [
+        "apps/chaos/**",
+        "crates/xmtp_mls/**"
+      ],
+      "labels": [],
+      "advisory": {
+        "cargo_roots": [],
+        "crates": [],
+        "paths": [],
+        "labels": []
+      }
+    },
     "docs_rust": {
       "title": "Rust reference",
       "phase": "test",

.github/ci-suites.yml

@@ -546,6 +546,21 @@ suites:
       - "{webdriver.json,.config/nextest.toml,dev/wasm.just}"
       - "{nix/package/wasm-nextest.nix,nix/ci-checks.nix,nix/lib/junit.nix}"
       - "nix/services/**"
+  test_chaos:
+    title: Rust chaos
+    phase: test
+    workflow: test-chaos.yml
+    run_on: [ready_pr_push, merge]
+    disable_on_forks: false
+    secrets: cachix-pr
+    permissions: {contents: read}
+    kache: []
+    products: []
+    runners:
+      runner: {pr: ubuntu-24.04, merge: ubuntu-24.04}
+    pr_filters:
+      - "crates/xmtp_mls/**"
+      - "apps/chaos/**"
   docs_rust:
     title: Rust reference
     phase: test

.github/workflows/ci.yml

@@ -215,6 +215,18 @@ jobs:
     secrets: inherit
     with:
       runner: ${{ github.event_name == 'pull_request' && 'blacksmith-16vcpu-ubuntu-2404' || 'ubuntu-24.04' }}
+  test_chaos:
+    name: Rust chaos
+    needs:
+    - detect-changes
+    if: contains(fromJSON(needs.detect-changes.outputs.test_suites), 'test_chaos')
+    uses: ./.github/workflows/test-chaos.yml
+    permissions:
+      contents: read
+    secrets:
+      CACHIX_PR_AUTH_TOKEN: ${{ secrets.CACHIX_PR_AUTH_TOKEN }}
+    with:
+      runner: ubuntu-24.04
   docs_rust:
     name: Rust reference
     needs:
@@ -504,6 +516,7 @@ jobs:
     - test_nix_checks
     - test_workspace
     - test_wasm
+    - test_chaos
     - docs_rust
     - check_types
     - check_sdk

.github/workflows/test-chaos.yml

@@ -0,0 +1,73 @@
+name: "Test: Rust chaos"
+on:
+  workflow_call:
+    inputs:
+      runner:
+        description: The GitHub-hosted runner label from .github/ci-suites.yml.
+        type: string
+        required: true
+    secrets:
+      CACHIX_PR_AUTH_TOKEN:
+        required: false
+permissions:
+  contents: read
+env:
+  NIX_DEVSHELL: rust
+jobs:
+  test:
+    name: Five-minute fault run
+    runs-on: ${{ inputs.runner }}
+    # Includes the source builds and baseline before the five-minute fault run.
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - name: Set up Nix
+        uses: ./.github/actions/setup-nix
+        with:
+          cache-mode: "ci"
+          cachix-pr-auth-token: ${{ secrets.CACHIX_PR_AUTH_TOKEN }}
+          github-token: ${{ github.token }}
+      - name: Build the chaos supervisor
+        run: dev/nix-shell 'just chaos build'
+      - name: Start the local backend and fault proxy
+        run: dev/nix-shell 'just backend up anvil toxiproxy'
+      - name: Test the chaos suite
+        run: dev/nix-shell 'just chaos test'
+      - name: Run the 50-round baseline
+        id: baseline
+        run: dev/nix-shell 'just chaos baseline --rounds 50 --seed 1338 --directory .chaos/ci-baseline'
+      - name: Run five minutes with all fault types enabled
+        # Timed runs finish their current round before checking the deadline.
+        run: dev/nix-shell 'just chaos run --hours 0.08333333333333333 --seed 1340 --faults all --directory .chaos/ci-faults'
+      - name: Report bounded run evidence
+        if: ${{ always() && steps.baseline.outcome != 'skipped' }}
+        run: |
+          for phase in baseline faults; do
+            directory=".chaos/ci-$phase"
+            if [ -d "$directory" ]; then
+              dev/nix-shell "just chaos status --directory $directory"
+              dev/nix-shell "just chaos inspect $directory"
+            fi
+          done | tee "$RUNNER_TEMP/chaos-report.txt"
+      # Raw .chaos files contain wallets and database keys. Upload only the
+      # bounded status/inspect output, including on a failed or inconclusive run.
+      - name: Retain the bounded chaos report
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
+        with:
+          name: chaos-report
+          path: ${{ runner.temp }}/chaos-report.txt
+          if-no-files-found: ignore
+          retention-days: 7
+      - name: Retain backend startup diagnostics
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
+        with:
+          name: chaos-backend-startup-logs
+          path: ${{ runner.temp }}/backend-startup-logs
+          if-no-files-found: ignore
+          retention-days: 7
+      - name: Stop the local backend
+        if: always()
+        run: dev/nix-shell 'just backend down'

apps/chaos/AGENTS.md

@@ -1,7 +1,9 @@
 # Chaos suite
 
 `xchaos` runs real client processes against this worktree's self-hosted backend.
-It is outside `default-members` and CI.
+It is outside `default-members`. The blocking `Rust chaos` CI suite runs a
+50-round baseline and five minutes of faults for changes to this app or
+`crates/xmtp_mls/`, and on merges. See [CI selection](../../.github/CI.md).
 
 ## Run gate
 

apps/chaos/README.md

@@ -75,6 +75,21 @@ Runs accept `--rounds`, `--hours`, `--seed`, `--faults`, `--strict`, and
 Use a comma-separated list to combine sets, such as `--faults network,disk`.
 Baseline always disables faults.
 
+## Pull request check
+
+Changes to `crates/xmtp_mls/` or `apps/chaos/` select the blocking `Rust chaos`
+CI suite on ready pull requests. It also runs on merges. The check uses a
+GitHub-hosted `ubuntu-24.04` runner and a local backend built from the tested
+code. A 50-round baseline with seed 1338 must pass before an all-fault run
+with seed 1340 runs for five minutes. The final round finishes before the
+time limit is checked. Setup, builds, and the baseline take additional time.
+Failures and inconclusive results fail the check without retries.
+Fault types are sampled; a short run need not exercise every type.
+
+The `chaos-report` artifact retains bounded `status` and `inspect` output for
+seven days. Raw `.chaos/` files are never uploaded because they contain private
+wallets and database keys. See [CI selection](../../.github/CI.md).
+
 ## Murmur run sequence
 
 Before the approved run, test short baselines with seeds 1337 and 1338 and