feat: quarantine known flakes, webhook notifications and test-tracking docs (P4)

GETTING_STARTED.MD

@@ -299,7 +299,10 @@ For a real repository:
    dependency edges, caching inputs or outputs, aliases, or file capabilities.
 4. Start with a narrow command such as `aster test //path/to/project`, then
    expand to `--all` or `aster affected` after the graph looks correct.
-5. Add the verified command to CI.
+5. Add the verified command to CI. With `--retry-failed --test-results`,
+   Aster retries only failing tests and records every result; see
+   [Test tracking](docs/test-tracking.md) to send them to aster-srv for flake
+   detection and quarantine.
 
 From there, see the main [README](README.md) for supported languages, richer
 project configuration, affected-path exclusions, watch settings, and the

README.md

@@ -355,6 +355,28 @@ first failure (`pytest -x`, `--bail`, `mix test --max-failures`, `go test
 -failfast`), when a target runs several commands, or when the runner cannot
 select single tests.
 
+### Quarantining flaky tests
+
+With test results captured (`--test-results` or `--retry-failed`), a failed
+target reports `passed (quarantined: N)` when, after retries, its only
+failures are quarantined tests, its results are complete and it exited with
+its runner's "tests failed" code. Its dependents run, and each such failure
+gets a `::warning` annotation. The list comes from aster-srv (tests proven
+flaky in 2 runs within 7 days) and from `aster-quarantine.toml` at the
+workspace root, whose entries need a reason and an expiry date:
+
+```toml
+[[test]]
+name = "test creates an agent"
+file = "test/platform/agents_test.exs"
+reason = "DBConnection.OwnershipError under load; tracked in #4812"
+expires = 2026-11-30
+```
+
+`ASTER_QUARANTINE=off` turns it off. [Test tracking](docs/test-tracking.md)
+covers capture per runner, uploading to aster-srv, the labels in the step
+summary, and the quarantine rules in full.
+
 ## Affected projects
 
 ```console

crates/aster-results-model/src/ndjson.rs

@@ -80,6 +80,7 @@ mod tests {
                 duration_ms: Some(3),
                 failure_hash: None,
                 message: None,
+                quarantined: false,
             }),
         ]
     }

crates/aster-results-model/src/record.rs

@@ -125,6 +125,14 @@ pub struct TestRecord {
     /// [`crate::scrub_message`] of the failure message.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub message: Option<String>,
+    /// Set on a test's last attempt when it failed but the test is
+    /// quarantined and Aster let its target pass.
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub quarantined: bool,
+}
+
+fn is_false(value: &bool) -> bool {
+    !value
 }
 
 #[cfg(test)]
@@ -190,11 +198,20 @@ mod tests {
             duration_ms: Some(1840),
             failure_hash: Some("f3a9".into()),
             message: Some("** (DBConnection.OwnershipError) cannot find owner".into()),
+            quarantined: true,
         });
         let json = serde_json::to_value(&record).unwrap();
         assert_eq!(json["type"], "test");
         assert_eq!(json["outcome"], "fail");
+        assert_eq!(json["quarantined"], true);
         assert_eq!(round_trip(&record), record);
+        // Only a quarantined failure says so; older readers ignore it.
+        let Record::Test(mut plain) = record else {
+            unreachable!()
+        };
+        plain.quarantined = false;
+        let json = serde_json::to_value(Record::Test(plain)).unwrap();
+        assert!(json.get("quarantined").is_none());
     }
 
     #[test]

crates/aster-srv/Cargo.toml

@@ -27,6 +27,5 @@ tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal
 tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] }
 tokio-postgres-rustls = "0.14"
 gcp_auth = { version = "0.12.7", default-features = false, features = ["aws-lc-rs"] }
-
-[dev-dependencies]
+# HMAC for webhook signatures; already in the build through rustls.
 aws-lc-rs = "1"

crates/aster-srv/README.md

@@ -21,12 +21,14 @@ from this server's state, looked up with the same credentials.
 | `GET /v1/test-results/tests/:test_id` | One test's state and history. |
 | `GET /v1/test-results/runs/:run_id/failures` | The failing tests of a run, with their state. |
 | `GET /v1/test-results/failures?github_run=` | The failing tests of every Aster run in one GitHub Actions run. |
+| `GET /v1/test-results/quarantine` | The caller's quarantine list and its version. See [Quarantine](#quarantine). |
 | `GET /v1/test-results/durations?target=&day=` | Each test file's median duration, for splitting a partitioned suite. |
 | `GET /healthz` | The process is up. |
 | `GET /readyz` | The database answers. |
 
 Rows are kept for the retention period (30 days by default). Day partitions
-past it are dropped every six hours.
+past it are dropped every six hours, with the older quarantine log rows and
+webhook deliveries.
 
 ## Flaky and broken tests
 
@@ -71,6 +73,84 @@ branch matches it as `main` or `refs/heads/main`.
 (repo, `failure_hash`) the `failures` and distinct `tests` counts, `first_seen`
 and `last_seen` days, and an example message and test.
 
+## Quarantine
+
+At the end of each pass, in the same transaction and as of the same clock,
+the classifier updates `test_results.quarantine`, in this order:
+
+1. A quarantined test that is now **broken** leaves: broken tests are never
+   quarantined.
+2. An entry older than `ASTER_SRV_QUARANTINE_DAYS` (default 30, at most the
+   retention period) **expires**.
+3. A test whose final results since it entered end in
+   `ASTER_SRV_QUARANTINE_RELEASE_PASSES` (default 20) consecutive passes is
+   **released**. Any failure resets the count.
+4. A test that is not broken **enters** once definite flake events from
+   `ASTER_SRV_QUARANTINE_RUNS` (default 2) different runs fall within the last
+   7 days. Only events after it last left count, so a released or expired
+   test is judged afresh. `0` turns automatic quarantine off.
+
+Every change is a row in `test_results.quarantine_log` (`quarantined`,
+`released`, `expired` or `broken`, with details), kept for the retention
+period. A repository's newest log id is its list's `version`.
+
+`GET /v1/test-results/quarantine` (the upload token; `repo=` must match it)
+returns:
+
+```json
+{"repo": "ArchAstro/firstlanding", "version": 312, "generated_at": "…",
+ "truncated": false,
+ "tests": [{"test_id": "c81e…", "target": "//services/platform:test-ci",
+            "file": "test/platform/agents_test.exs", "suite": "Platform.AgentsTest",
+            "name": "test creates an agent", "since": "…", "expires_at": "…",
+            "runs": 2}]}
+```
+
+It lists at most 10,000 tests (`truncated` says when there were more), which
+only lets more tests fail. `GET /v1/test-results/tests?state=quarantined`
+filters the test list the same way. How Aster applies the list, and the
+repository's own `aster-quarantine.toml`, are in
+[docs/test-tracking.md](../../docs/test-tracking.md#5-quarantine).
+
+## Webhooks
+
+`ASTER_SRV_WEBHOOKS` is a JSON array of endpoints, each with the events it
+wants:
+
+```json
+[{"name": "flake-bot", "url": "https://hooks.example.com/aster",
+  "secret": "at-least-16-characters", "events": ["test.quarantined", "test.released"]}]
+```
+
+| Event | When |
+|---|---|
+| `test.quarantined` | A test entered quarantine. `detail`: `runs`, `events`, `expires_at`. |
+| `test.released` | It passed enough times in a row. `detail`: `entered_at`, `passes`. |
+| `test.expired` | Its entry reached the longest stay. `detail`: `entered_at`. |
+| `test.broken` | A test became broken, quarantined or not. `detail`: `since_sha`, `failure_hash`. |
+
+Each delivery is a `POST` of
+
+```json
+{"event": "test.quarantined", "occurred_at": "…", "repo": "ArchAstro/firstlanding",
+ "test": {"test_id": "…", "target": "…", "file": "…", "suite": "…", "name": "…"},
+ "detail": {…}}
+```
+
+with `X-Aster-Event`, `X-Aster-Delivery` (the same on every retry, so a
+receiver can drop repeats) and `X-Aster-Signature: sha256=<hex HMAC-SHA256 of
+the body, keyed by the secret>`. Check the signature over the raw body before
+parsing it.
+
+The classifier records deliveries in `webhooks.deliveries` in the transaction
+that makes the change, so nothing is announced that did not happen and
+nothing that happened is lost. A sender on every replica claims due rows
+every 15 seconds (`FOR UPDATE SKIP LOCKED`) and posts them with a 10-second
+timeout. A 2xx delivers; anything else is retried after 30 seconds, doubling
+up to 6 hours, and abandoned after 12 attempts (about 14.5 hours). A slow or
+failing endpoint holds up only its own deliveries. Endpoints must be `https://`
+(`http://` only for localhost). Unsubscribed events, and all events with no
+endpoints configured, record nothing. Rows are kept for the retention period.
 ## Reading results
 
 These routes take the same bearer token as uploads, and a token reads only its
@@ -83,7 +163,7 @@ last classifier pass, up to five minutes old.
 Lists are paged with `limit` and an opaque `cursor`; pass the previous page's
 `next_cursor` until it is `null`.
 
-`GET /v1/test-results/tests?state=flaky|broken|healthy&limit=100&cursor=`
+`GET /v1/test-results/tests?state=flaky|broken|healthy|quarantined&limit=100&cursor=`
 (`limit` 1 to 500, default 100), sorted by `flake_score` descending, then
 `test_id`:
 
@@ -273,7 +353,7 @@ needs no JavaScript.
 | `/ui` | Flaky leaderboard (flake score, last flake) and broken tests |
 | `/ui/tests/:test_id?repo=owner/name` | One test: state, a pass/fail strip (oldest to newest) and recent results with failure messages |
 | `/ui/clusters` | Failures of the last 7 days grouped by failure hash |
-| `/ui/quarantine` | The quarantine list; empty until quarantine exists |
+| `/ui/quarantine` | The quarantine list, its rules and its recent changes |
 
 `?repo=owner/name` picks the repository; without it the one with the newest
 run is shown. Any repository on `ASTER_SRV_REPOSITORIES` can be viewed.
@@ -319,10 +399,25 @@ Pages are sent with a strict `Content-Security-Policy`, `nosniff`,
 | `ASTER_SRV_IAP_AUDIENCE` | unset | The Google IAP audience the dashboard accepts, exact or a prefix ending in `/`. Unset turns the dashboard off. See [Dashboard](#dashboard). |
 | `ASTER_SRV_UI_EMAIL_DOMAINS` | unset | Comma-separated domains (`example.com`, `*.example.com`) whose emails may view the dashboard. Unset allows anyone who passes IAP. |
 | `ASTER_SRV_UI_DEV_EMAIL` | unset | The viewer to trust when no IAP audience is set. Development only. |
+| `ASTER_SRV_QUARANTINE_RUNS` | `2` | Different runs with a definite flake in 7 days that quarantine a test; `0` turns automatic quarantine off. |
+| `ASTER_SRV_QUARANTINE_RELEASE_PASSES` | `20` | Consecutive passes that release a quarantined test. |
+| `ASTER_SRV_QUARANTINE_DAYS` | `30` | The longest a test stays quarantined; at most `ASTER_SRV_RETENTION_DAYS`. |
+| `ASTER_SRV_WEBHOOKS` | unset | Webhook endpoints, as JSON. See [Webhooks](#webhooks). |
 | `ASTER_SRV_STATIC_TOKENS` | unset | `owner/name=token` pairs accepted instead of OIDC tokens. Development only. |
 
 Logs are one JSON object per line, with `severity` and `message`.
 
+## Runbook
+
+| Symptom | Check | Do |
+|---|---|---|
+| Uploads fail | `/readyz`; logs with `"severity":"ERROR"` | Uploads never fail CI, so results are only missing. Fix the database or the allowlist (`403`). |
+| Classification stalls | `SELECT watermark FROM test_results.classifier_state` should be minutes old; `classifier pass failed` in the logs | A pass is safe to rerun; restarting a pod starts one at once. |
+| A real regression is being let through | `/ui/quarantine`, the test's page | In CI set `ASTER_QUARANTINE=off` for the job to fail it at once. On the server, delete its row: `DELETE FROM test_results.quarantine WHERE repo = '…' AND test_id = '…'` (it can re-enter on new flakes after now). Set `ASTER_SRV_QUARANTINE_RUNS=0` to stop new entries. |
+| Too many or too few tests quarantined | `SELECT action, count(*) FROM test_results.quarantine_log WHERE at > now() - interval '1 day' GROUP BY 1` | Tune `ASTER_SRV_QUARANTINE_RUNS` and `ASTER_SRV_QUARANTINE_RELEASE_PASSES`; they apply from the next pass. |
+| Webhooks not arriving | `SELECT endpoint, event, attempts, last_status, last_error FROM webhooks.deliveries WHERE delivered_at IS NULL ORDER BY id DESC LIMIT 20` | Abandoned rows have `abandoned_at`; to resend, `UPDATE webhooks.deliveries SET abandoned_at = NULL, attempts = 0, next_attempt_at = now() WHERE id = …`. |
+| A deploy waits at startup | `migration waiting for table locks` in the logs | Expected while the old release writes; it retries for about a minute. |
+
 ## Image
 
 `ghcr.io/archastro/aster-srv:<aster version>` is published with each Aster

crates/aster-srv/src/config.rs

@@ -84,6 +84,26 @@ pub struct Config {
     /// `ASTER_SRV_UI_DEV_EMAIL`: the viewer the dashboard assumes when no IAP
     /// audience is configured. For local development only.
     pub ui_dev_email: Option<String>,
+    /// When tests enter and leave quarantine.
+    pub quarantine: QuarantineRules,
+    /// `ASTER_SRV_WEBHOOKS`: where state changes are sent. Empty sends
+    /// nothing.
+    pub webhooks: Vec<crate::webhooks::Endpoint>,
+}
+
+/// When tests enter and leave quarantine.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct QuarantineRules {
+    /// `ASTER_SRV_QUARANTINE_RUNS`: a test enters quarantine once this many
+    /// different runs in the last 7 days hold a definite flake of it. 0
+    /// turns automatic quarantine off. Default 2.
+    pub runs: u32,
+    /// `ASTER_SRV_QUARANTINE_RELEASE_PASSES`: consecutive passes, while
+    /// quarantined, that release a test. Default 20.
+    pub release_passes: u32,
+    /// `ASTER_SRV_QUARANTINE_DAYS`: the longest a test stays quarantined,
+    /// at most the retention period. Default 30.
+    pub days: u32,
 }
 
 /// A fixed bearer token standing in for one repository.
@@ -173,6 +193,31 @@ impl Config {
                  (or end in / to match any backend service of a project)"
             );
         }
+        let number = |name: &str, default: u32, min: u32, max: u32| -> Result<u32> {
+            match var(name) {
+                None => Ok(default),
+                Some(value) => value
+                    .trim()
+                    .parse()
+                    .ok()
+                    .filter(|n| (min..=max).contains(n))
+                    .with_context(|| format!("{name} must be a number from {min} to {max}")),
+            }
+        };
+        let quarantine = QuarantineRules {
+            runs: number("ASTER_SRV_QUARANTINE_RUNS", 2, 0, 1000)?,
+            release_passes: number("ASTER_SRV_QUARANTINE_RELEASE_PASSES", 20, 1, 10_000)?,
+            days: number(
+                "ASTER_SRV_QUARANTINE_DAYS",
+                30.min(retention_days),
+                1,
+                retention_days,
+            )?,
+        };
+        let webhooks = match var("ASTER_SRV_WEBHOOKS") {
+            Some(value) => crate::webhooks::Endpoint::parse_all(&value)?,
+            None => Vec::new(),
+        };
         Ok(Self {
             listen,
             database_url,
@@ -188,6 +233,8 @@ impl Config {
             iap_audience,
             ui_email_domains,
             ui_dev_email: var("ASTER_SRV_UI_DEV_EMAIL").map(|email| email.trim().to_string()),
+            quarantine,
+            webhooks,
         })
     }
 }
@@ -306,6 +353,50 @@ mod tests {
         assert!(config.iap_audience.is_none());
         assert!(config.ui_email_domains.is_none());
         assert!(config.ui_dev_email.is_none());
+        assert_eq!(
+            config.quarantine,
+            QuarantineRules {
+                runs: 2,
+                release_passes: 20,
+                days: 30
+            }
+        );
+        assert!(config.webhooks.is_empty());
+    }
+
+    #[test]
+    fn reads_the_quarantine_rules() {
+        let mut pairs = REQUIRED.to_vec();
+        pairs.push(("ASTER_SRV_QUARANTINE_RUNS", "0"));
+        pairs.push(("ASTER_SRV_QUARANTINE_RELEASE_PASSES", "5"));
+        pairs.push(("ASTER_SRV_QUARANTINE_DAYS", "14"));
+        let config = Config::from_env(env_of(&pairs)).unwrap();
+        assert_eq!(
+            config.quarantine,
+            QuarantineRules {
+                runs: 0,
+                release_passes: 5,
+                days: 14
+            }
+        );
+        // Quarantine never outlives the results that justify it.
+        let mut pairs = REQUIRED.to_vec();
+        pairs.push(("ASTER_SRV_RETENTION_DAYS", "10"));
+        assert_eq!(
+            Config::from_env(env_of(&pairs)).unwrap().quarantine.days,
+            10
+        );
+        pairs.push(("ASTER_SRV_QUARANTINE_DAYS", "11"));
+        assert!(Config::from_env(env_of(&pairs)).is_err());
+        for (name, bad) in [
+            ("ASTER_SRV_QUARANTINE_RUNS", "-1"),
+            ("ASTER_SRV_QUARANTINE_RELEASE_PASSES", "0"),
+            ("ASTER_SRV_QUARANTINE_DAYS", "0"),
+        ] {
+            let mut pairs = REQUIRED.to_vec();
+            pairs.push((name, bad));
+            assert!(Config::from_env(env_of(&pairs)).is_err(), "{name}={bad}");
+        }
     }
 
     #[test]

crates/aster-srv/src/lib.rs

@@ -15,6 +15,7 @@ pub mod log;
 pub mod raw_store;
 pub mod test_results;
 mod ui;
+pub mod webhooks;
 
 /// An HTTP client builder on aws-lc-rs, the workspace's one TLS provider.
 /// reqwest is built without its own (it would bring ring, and two
@@ -44,7 +45,10 @@ pub struct App {
 
 /// Every feature's migrations, in order.
 pub fn migrations() -> Vec<&'static db::Migration> {
-    test_results::MIGRATIONS.iter().collect()
+    test_results::MIGRATIONS
+        .iter()
+        .chain(webhooks::MIGRATIONS)
+        .collect()
 }
 
 impl App {

crates/aster-srv/src/main.rs

@@ -41,6 +41,7 @@ async fn run() -> anyhow::Result<()> {
     let app = App::start(config).await?;
     aster_srv::test_results::retention::spawn(app.clone());
     aster_srv::test_results::classifier::spawn(app.clone());
+    aster_srv::webhooks::spawn(app.clone());
     let listener = tokio::net::TcpListener::bind(listen).await?;
     log::info(
         "aster-srv listening",