isb 3.1.1: egress and metrics-history CPU fixes; inline tests don't count toward the file-size ratchet

Cargo.toml

@@ -54,20 +54,20 @@ members = ["crates/*"]
 default-members = [".", "crates/*"]
 
 [workspace.package]
-version = "3.1.0"
+version = "3.1.1"
 edition = "2024"
 rust-version = "1.87"
 license = "MIT"
 repository = "https://github.com/execution-associates/isb"
 homepage = "https://github.com/execution-associates/isb"
 
 [workspace.dependencies]
-isb-core = { path = "crates/isb-core", version = "=3.1.0" }
-isb-server = { path = "crates/isb-server", version = "=3.1.0" }
-isb-apps = { path = "crates/isb-apps", version = "=3.1.0" }
-isb-daemon = { path = "crates/isb-daemon", version = "=3.1.0" }
-isb-tui = { path = "crates/isb-tui", version = "=3.1.0" }
-isb-egress = { path = "crates/isb-egress", version = "=3.1.0" }
+isb-core = { path = "crates/isb-core", version = "=3.1.1" }
+isb-server = { path = "crates/isb-server", version = "=3.1.1" }
+isb-apps = { path = "crates/isb-apps", version = "=3.1.1" }
+isb-daemon = { path = "crates/isb-daemon", version = "=3.1.1" }
+isb-tui = { path = "crates/isb-tui", version = "=3.1.1" }
+isb-egress = { path = "crates/isb-egress", version = "=3.1.1" }
 
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

crates/isb-core/src/egress/plumb.rs

@@ -404,22 +404,92 @@ pub fn teardown(client: &Client, instance: &str) -> Result<()> {
 /// The egress bridges incus holds, each with its policy: what the proxy
 /// manager reconciles against.
 pub fn list(client: &Client) -> Result<Vec<Value>> {
-    let v = host(client).get("/1.0/networks?recursion=1")?;
-    Ok(v.as_array()
-        .map(|a| {
-            a.iter()
-                .filter(|n| n["config"][KEY_FOR].is_string())
-                .cloned()
-                .collect()
-        })
-        .unwrap_or_default())
+    // Names, then each egress bridge on its own. With recursion incus reads
+    // every host interface once per network it lists (a full netlink dump
+    // each), and the proxy manager asks every few seconds.
+    let h = host(client);
+    let urls = h.get("/1.0/networks")?;
+    let mut out = Vec::new();
+    for url in urls
+        .as_array()
+        .into_iter()
+        .flatten()
+        .filter_map(Value::as_str)
+    {
+        let Some(name) = url.strip_prefix("/1.0/networks/") else {
+            continue;
+        };
+        let name = name.split('?').next().unwrap_or_default();
+        if !name.starts_with(NET_PREFIX) {
+            continue;
+        }
+        // Gone since the listing: nothing to reconcile.
+        let Some(n) = h.get_opt(&format!("/1.0/networks/{}", encode_segment(name)))? else {
+            continue;
+        };
+        if n["config"][KEY_FOR].is_string() {
+            out.push(n);
+        }
+    }
+    Ok(out)
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::client::fake::{Route, serve};
     use crate::egress::policy::EgressSpec;
 
+    #[test]
+    fn listing_fetches_only_egress_bridges() {
+        // A recursive listing costs incus a host interface dump per network:
+        // the fake refuses it, and any network but the egress bridges.
+        let (_d, c) = serve(vec![
+            Route {
+                prefix: "GET /1.0/networks?recursion=1",
+                status: 500,
+                body: json!("recursive listing"),
+            },
+            Route {
+                prefix: "GET /1.0/networks/isbbrxaaaa",
+                status: 200,
+                body: json!({"name": "isbbrxaaaa", "config": {KEY_FOR: "isb-acme/web-1"}}),
+            },
+            Route {
+                prefix: "GET /1.0/networks/isbbrxbbbb",
+                status: 200,
+                body: json!({"name": "isbbrxbbbb", "config": {}}),
+            },
+            Route {
+                prefix: "GET /1.0/networks/isbbrxgone",
+                status: 404,
+                body: json!("not found"),
+            },
+            Route {
+                prefix: "GET /1.0/networks/",
+                status: 500,
+                body: json!("not an egress bridge"),
+            },
+            Route {
+                prefix: "GET /1.0/networks",
+                status: 200,
+                body: json!([
+                    "/1.0/networks/incusbr0",
+                    "/1.0/networks/isbbrxaaaa",
+                    "/1.0/networks/isbbrxbbbb",
+                    "/1.0/networks/isbbrxgone",
+                    "/1.0/networks/eth0",
+                ]),
+            },
+        ]);
+        let names: Vec<String> = list(&c)
+            .unwrap()
+            .iter()
+            .map(|n| n["name"].as_str().unwrap().to_string())
+            .collect();
+        assert_eq!(names, ["isbbrxaaaa"]);
+    }
+
     fn plumbing(spec: &EgressSpec) -> Plumbing {
         let n = network_name("default", "plugin");
         Plumbing::new("default", "plugin", Policy::from_spec(spec, &n).unwrap())

crates/isb-core/src/metrics_history.rs

@@ -245,6 +245,12 @@ CREATE TABLE IF NOT EXISTS samples(
 CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v INTEGER NOT NULL);
 ";
 
+/// Drops series with no samples left. The tiers are named because the key
+/// starts with `tier`: with `sid` alone each series is a scan of the whole
+/// table. Keep the list in step with [`TIERS`].
+const PRUNE_SERIES: &str = "DELETE FROM series WHERE NOT EXISTS
+    (SELECT 1 FROM samples WHERE tier IN (0, 1, 2) AND sid = series.id)";
+
 fn db_err(step: &str, e: rusqlite::Error) -> Error {
     Error::invalid(format!("metrics history: {step}: {e}"))
 }
@@ -347,11 +353,8 @@ impl OrgDb {
             )
             .map_err(|e| db_err("retention", e))?;
         }
-        tx.execute(
-            "DELETE FROM series WHERE NOT EXISTS (SELECT 1 FROM samples WHERE sid = series.id)",
-            [],
-        )
-        .map_err(|e| db_err("prune series", e))?;
+        tx.execute(PRUNE_SERIES, [])
+            .map_err(|e| db_err("prune series", e))?;
         tx.commit().map_err(|e| db_err("commit", e))?;
         let _ = self.conn.execute_batch("PRAGMA incremental_vacuum;");
         Ok(())
@@ -870,6 +873,63 @@ mod tests {
         assert_eq!(a.series[0].name, "web-2");
     }
 
+    /// `maintain` drops a series once its last sample is gone, keeps one
+    /// whose samples survive in any tier, and finds each by key instead of
+    /// scanning the table once per series (seconds per org, every minute,
+    /// once replicas have come and gone).
+    #[test]
+    fn maintain_prunes_empty_series_by_key() {
+        let dir = tempfile::tempdir().unwrap();
+        let mut db = OrgDb::open(&dir.path().join("m.db")).unwrap();
+        let plan: Vec<String> = db
+            .conn
+            .prepare(&format!("EXPLAIN QUERY PLAN {PRUNE_SERIES}"))
+            .unwrap()
+            .query_map([], |r| r.get(3))
+            .unwrap()
+            .map(Result::unwrap)
+            .collect();
+        assert!(
+            plan.iter().any(|d| d.contains("samples USING PRIMARY KEY")),
+            "{plan:?}"
+        );
+
+        // One series whose only samples are in each tier, and one with none.
+        let now: u64 = 1_800_000_000 / 600 * 600;
+        let mut rows: Vec<Row> = (0..TIERS.len())
+            .map(|t| row(&format!("tier-{t}"), now - 600, 1.0, 1.0))
+            .collect();
+        rows.push(row("empty", now - 600, 1.0, 1.0));
+        db.insert(&rows).unwrap();
+        for t in 0..TIERS.len() {
+            db.conn
+                .execute(
+                    "UPDATE samples SET tier = ?1
+                     WHERE sid = (SELECT id FROM series WHERE instance = ?2)",
+                    params![t as i64, format!("tier-{t}")],
+                )
+                .unwrap();
+        }
+        db.conn
+            .execute(
+                "DELETE FROM samples WHERE sid = (SELECT id FROM series WHERE instance = 'empty')",
+                [],
+            )
+            .unwrap();
+
+        db.maintain(now).unwrap();
+        let left: Vec<String> = db
+            .conn
+            .prepare("SELECT instance FROM series ORDER BY instance")
+            .unwrap()
+            .query_map([], |r| r.get(0))
+            .unwrap()
+            .map(Result::unwrap)
+            .collect();
+        let want: Vec<String> = (0..TIERS.len()).map(|t| format!("tier-{t}")).collect();
+        assert_eq!(left, want);
+    }
+
     #[test]
     fn history_survives_reopening() {
         let dir = tempfile::tempdir().unwrap();

docs/contributing/index.md

@@ -180,6 +180,8 @@ but never worse.
   `scripts/ratchet.txt`, so a new exemption cannot slip in unnoticed.
 - **File size**: the same script fails any `.rs` file over 1000 lines, except
   the files listed in `scripts/ratchet.txt`, each capped at its listed size.
+  Inline `#[cfg(test)] mod` blocks don't count, so a test never has to be cut
+  short to fit.
 
 Both budgets go down only. After splitting a long function or file, run
 `scripts/ratchet.sh --update` to lower them (it never raises one) and commit

docs/reference/rpc.md

@@ -19,7 +19,7 @@ it; relative paths resolve against that directory.
 When it starts, the server writes one hello line:
 
 ```json
-{"isb": "3.1.0", "protocol": 1}
+{"isb": "3.1.1", "protocol": 1}
 ```
 
 A client checks `protocol` and refuses versions it does not know.
@@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present),
 ## Example session
 
 ```text
-<- {"isb":"3.1.0","protocol":1}
+<- {"isb":"3.1.1","protocol":1}
 -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}}
 <- {"id":1,"event":"progress","data":"web: creating from dev-base"}
 <- {"id":1,"event":"progress","data":"web: starting"}

scripts/ratchet.sh

@@ -4,7 +4,9 @@
 #
 #  - File size: no Rust file may exceed LIMIT lines, except the files listed
 #    (`path lines`), each capped at its listed size. Once a listed file is
-#    under LIMIT it must leave the list.
+#    under LIMIT it must leave the list. A file's inline `#[cfg(test)] mod`
+#    block is not counted: the limit keeps code legible, and a test should
+#    never be cut short to fit under it.
 #  - Lint exemptions: clippy enforces the thresholds in clippy.toml, and an
 #    existing offender carries an `#[expect(clippy::<lint>, reason = ...)]`.
 #    The number of times each ratcheted lint is named in the source
@@ -35,7 +37,14 @@ while read -r f; do [ -f "$f" ] && files+=("$f"); done < <(sort -u <<<"$list")
 
 # `path lines` for every file, then `lint:<name> count` for every lint.
 measure() {
-  wc -l "${files[@]}" | awk '$2 != "total" { print $2, $1 }'
+  # Every line outside a top-level `#[cfg(test)] mod name { ... }` block.
+  awk '
+    function emit() { if (f != "") print f, n }
+    FNR == 1 { emit(); f = FILENAME; n = 0; test = 0; prev = "" }
+    test { if (/^}/) test = 0; next }
+    prev == "#[cfg(test)]" && /^mod [A-Za-z_]+ *\{/ { test = 1; n--; prev = ""; next }
+    { n++; prev = $0 }
+    END { emit() }' "${files[@]}"
   for l in $LINTS; do
     printf 'lint:%s %s\n' "$l" "$(cat "${files[@]}" | grep -o "clippy::$l\b" | wc -l)"
   done

scripts/ratchet.txt

@@ -1,27 +1,24 @@
 # The ratchet budgets checked by scripts/ratchet.sh: Rust files allowed over
-# its line LIMIT (`path max-lines`), and how many times each ratcheted clippy
-# lint may be named in an exemption (`lint:<name> max-count`). Numbers only go
-# down: run `scripts/ratchet.sh --update` after shrinking something.
+# its line LIMIT (`path max-lines`, not counting inline test modules), and
+# how many times each ratcheted clippy lint may be named in an exemption
+# (`lint:<name> max-count`). Numbers only go down: run
+# `scripts/ratchet.sh --update` after shrinking something.
 crates/isb-apps/src/app/deploy.rs 1736
-crates/isb-apps/src/app/preview.rs 1812
-crates/isb-apps/src/backup.rs 1675
-crates/isb-apps/src/notify/mod.rs 1263
+crates/isb-apps/src/app/preview.rs 1574
+crates/isb-apps/src/backup.rs 1501
 crates/isb-apps/src/s3.rs 1263
-crates/isb-core/src/ingress/caddy.rs 1130
-crates/isb-core/src/machine.rs 1097
 crates/isb-core/src/plan.rs 1517
 crates/isb-core/src/registry/oci.rs 1081
 crates/isb-core/src/sandbox.rs 1313
 crates/isb-core/src/spec.rs 1518
 crates/isb-core/src/stack/controller.rs 2325
 crates/isb-daemon/src/daemon/mod.rs 1089
-crates/isb-server/src/audit.rs 1153
 crates/isb-server/src/auth/http.rs 1040
 crates/isb-server/src/auth/mod.rs 1658
-crates/isb-server/src/history.rs 1581
+crates/isb-server/src/history.rs 1305
 crates/isb-server/src/server/http.rs 1085
-crates/isb-server/src/server/mcp.rs 2120
-crates/isb-tui/src/tui/ui.rs 1564
+crates/isb-server/src/server/mcp.rs 1304
+crates/isb-tui/src/tui/ui.rs 1259
 lint:cognitive_complexity 14
 lint:excessive_nesting 8
 lint:too_many_arguments 12

sdk/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "isb-sdk"
-version = "3.1.0"
+version = "3.1.1"
 description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)"
 readme = "README.md"
 license = "MIT"

sdk/python/src/isb/__init__.py

@@ -62,7 +62,7 @@ async def main():
 from ._util import Duration
 from .volumes import Volumes
 
-__version__ = "3.1.0"
+__version__ = "3.1.1"
 
 __all__ = [
     "PROTOCOL",