diff --git a/Cargo.lock b/Cargo.lock index 9909cff1..2651c5e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,7 +1329,7 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "isb" -version = "3.1.0" +version = "3.1.1" dependencies = [ "age", "clap", @@ -1351,7 +1351,7 @@ dependencies = [ [[package]] name = "isb-apps" -version = "3.1.0" +version = "3.1.1" dependencies = [ "age", "base64 0.23.1", @@ -1375,7 +1375,7 @@ dependencies = [ [[package]] name = "isb-core" -version = "3.1.0" +version = "3.1.1" dependencies = [ "age", "base64 0.23.1", @@ -1402,7 +1402,7 @@ dependencies = [ [[package]] name = "isb-daemon" -version = "3.1.0" +version = "3.1.1" dependencies = [ "age", "httparse", @@ -1422,7 +1422,7 @@ dependencies = [ [[package]] name = "isb-egress" -version = "3.1.0" +version = "3.1.1" dependencies = [ "base64 0.23.1", "httparse", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "isb-server" -version = "3.1.0" +version = "3.1.1" dependencies = [ "argon2", "base64 0.23.1", @@ -1458,7 +1458,7 @@ dependencies = [ [[package]] name = "isb-tui" -version = "3.1.0" +version = "3.1.1" dependencies = [ "isb-core", "libc", diff --git a/Cargo.toml b/Cargo.toml index 13f89174..186cbf25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ members = ["crates/*"] default-members = [".", "crates/*"] [workspace.package] -version = "3.1.0" +version = "3.1.1" edition = "2024" rust-version = "1.87" license = "MIT" @@ -62,12 +62,12 @@ repository = "https://github.com/execution-associates/isb" homepage = "https://github.com/execution-associates/isb" [workspace.dependencies] -isb-core = { path = "crates/isb-core", version = "=3.1.0" } -isb-server = { path = "crates/isb-server", version = "=3.1.0" } -isb-apps = { path = "crates/isb-apps", version = "=3.1.0" } -isb-daemon = { path = "crates/isb-daemon", version = "=3.1.0" } -isb-tui = { path = "crates/isb-tui", version = "=3.1.0" } -isb-egress = { path = "crates/isb-egress", version = "=3.1.0" } +isb-core = { path = "crates/isb-core", version = "=3.1.1" } +isb-server = { path = "crates/isb-server", version = "=3.1.1" } +isb-apps = { path = "crates/isb-apps", version = "=3.1.1" } +isb-daemon = { path = "crates/isb-daemon", version = "=3.1.1" } +isb-tui = { path = "crates/isb-tui", version = "=3.1.1" } +isb-egress = { path = "crates/isb-egress", version = "=3.1.1" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/isb-core/src/egress/plumb.rs b/crates/isb-core/src/egress/plumb.rs index 23243c56..3c097050 100644 --- a/crates/isb-core/src/egress/plumb.rs +++ b/crates/isb-core/src/egress/plumb.rs @@ -404,22 +404,92 @@ pub fn teardown(client: &Client, instance: &str) -> Result<()> { /// The egress bridges incus holds, each with its policy: what the proxy /// manager reconciles against. pub fn list(client: &Client) -> Result> { - let v = host(client).get("/1.0/networks?recursion=1")?; - Ok(v.as_array() - .map(|a| { - a.iter() - .filter(|n| n["config"][KEY_FOR].is_string()) - .cloned() - .collect() - }) - .unwrap_or_default()) + // Names, then each egress bridge on its own. With recursion incus reads + // every host interface once per network it lists (a full netlink dump + // each), and the proxy manager asks every few seconds. + let h = host(client); + let urls = h.get("/1.0/networks")?; + let mut out = Vec::new(); + for url in urls + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + { + let Some(name) = url.strip_prefix("/1.0/networks/") else { + continue; + }; + let name = name.split('?').next().unwrap_or_default(); + if !name.starts_with(NET_PREFIX) { + continue; + } + // Gone since the listing: nothing to reconcile. + let Some(n) = h.get_opt(&format!("/1.0/networks/{}", encode_segment(name)))? else { + continue; + }; + if n["config"][KEY_FOR].is_string() { + out.push(n); + } + } + Ok(out) } #[cfg(test)] mod tests { use super::*; + use crate::client::fake::{Route, serve}; use crate::egress::policy::EgressSpec; + #[test] + fn listing_fetches_only_egress_bridges() { + // A recursive listing costs incus a host interface dump per network: + // the fake refuses it, and any network but the egress bridges. + let (_d, c) = serve(vec![ + Route { + prefix: "GET /1.0/networks?recursion=1", + status: 500, + body: json!("recursive listing"), + }, + Route { + prefix: "GET /1.0/networks/isbbrxaaaa", + status: 200, + body: json!({"name": "isbbrxaaaa", "config": {KEY_FOR: "isb-acme/web-1"}}), + }, + Route { + prefix: "GET /1.0/networks/isbbrxbbbb", + status: 200, + body: json!({"name": "isbbrxbbbb", "config": {}}), + }, + Route { + prefix: "GET /1.0/networks/isbbrxgone", + status: 404, + body: json!("not found"), + }, + Route { + prefix: "GET /1.0/networks/", + status: 500, + body: json!("not an egress bridge"), + }, + Route { + prefix: "GET /1.0/networks", + status: 200, + body: json!([ + "/1.0/networks/incusbr0", + "/1.0/networks/isbbrxaaaa", + "/1.0/networks/isbbrxbbbb", + "/1.0/networks/isbbrxgone", + "/1.0/networks/eth0", + ]), + }, + ]); + let names: Vec = list(&c) + .unwrap() + .iter() + .map(|n| n["name"].as_str().unwrap().to_string()) + .collect(); + assert_eq!(names, ["isbbrxaaaa"]); + } + fn plumbing(spec: &EgressSpec) -> Plumbing { let n = network_name("default", "plugin"); Plumbing::new("default", "plugin", Policy::from_spec(spec, &n).unwrap()) diff --git a/crates/isb-core/src/metrics_history.rs b/crates/isb-core/src/metrics_history.rs index b5820c63..9959ad7c 100644 --- a/crates/isb-core/src/metrics_history.rs +++ b/crates/isb-core/src/metrics_history.rs @@ -245,6 +245,12 @@ CREATE TABLE IF NOT EXISTS samples( CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v INTEGER NOT NULL); "; +/// Drops series with no samples left. The tiers are named because the key +/// starts with `tier`: with `sid` alone each series is a scan of the whole +/// table. Keep the list in step with [`TIERS`]. +const PRUNE_SERIES: &str = "DELETE FROM series WHERE NOT EXISTS + (SELECT 1 FROM samples WHERE tier IN (0, 1, 2) AND sid = series.id)"; + fn db_err(step: &str, e: rusqlite::Error) -> Error { Error::invalid(format!("metrics history: {step}: {e}")) } @@ -347,11 +353,8 @@ impl OrgDb { ) .map_err(|e| db_err("retention", e))?; } - tx.execute( - "DELETE FROM series WHERE NOT EXISTS (SELECT 1 FROM samples WHERE sid = series.id)", - [], - ) - .map_err(|e| db_err("prune series", e))?; + tx.execute(PRUNE_SERIES, []) + .map_err(|e| db_err("prune series", e))?; tx.commit().map_err(|e| db_err("commit", e))?; let _ = self.conn.execute_batch("PRAGMA incremental_vacuum;"); Ok(()) @@ -870,6 +873,63 @@ mod tests { assert_eq!(a.series[0].name, "web-2"); } + /// `maintain` drops a series once its last sample is gone, keeps one + /// whose samples survive in any tier, and finds each by key instead of + /// scanning the table once per series (seconds per org, every minute, + /// once replicas have come and gone). + #[test] + fn maintain_prunes_empty_series_by_key() { + let dir = tempfile::tempdir().unwrap(); + let mut db = OrgDb::open(&dir.path().join("m.db")).unwrap(); + let plan: Vec = db + .conn + .prepare(&format!("EXPLAIN QUERY PLAN {PRUNE_SERIES}")) + .unwrap() + .query_map([], |r| r.get(3)) + .unwrap() + .map(Result::unwrap) + .collect(); + assert!( + plan.iter().any(|d| d.contains("samples USING PRIMARY KEY")), + "{plan:?}" + ); + + // One series whose only samples are in each tier, and one with none. + let now: u64 = 1_800_000_000 / 600 * 600; + let mut rows: Vec = (0..TIERS.len()) + .map(|t| row(&format!("tier-{t}"), now - 600, 1.0, 1.0)) + .collect(); + rows.push(row("empty", now - 600, 1.0, 1.0)); + db.insert(&rows).unwrap(); + for t in 0..TIERS.len() { + db.conn + .execute( + "UPDATE samples SET tier = ?1 + WHERE sid = (SELECT id FROM series WHERE instance = ?2)", + params![t as i64, format!("tier-{t}")], + ) + .unwrap(); + } + db.conn + .execute( + "DELETE FROM samples WHERE sid = (SELECT id FROM series WHERE instance = 'empty')", + [], + ) + .unwrap(); + + db.maintain(now).unwrap(); + let left: Vec = db + .conn + .prepare("SELECT instance FROM series ORDER BY instance") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + let want: Vec = (0..TIERS.len()).map(|t| format!("tier-{t}")).collect(); + assert_eq!(left, want); + } + #[test] fn history_survives_reopening() { let dir = tempfile::tempdir().unwrap(); diff --git a/docs/contributing/index.md b/docs/contributing/index.md index d44cb5ff..ea3e2b6c 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -180,6 +180,8 @@ but never worse. `scripts/ratchet.txt`, so a new exemption cannot slip in unnoticed. - **File size**: the same script fails any `.rs` file over 1000 lines, except the files listed in `scripts/ratchet.txt`, each capped at its listed size. + Inline `#[cfg(test)] mod` blocks don't count, so a test never has to be cut + short to fit. Both budgets go down only. After splitting a long function or file, run `scripts/ratchet.sh --update` to lower them (it never raises one) and commit diff --git a/docs/reference/rpc.md b/docs/reference/rpc.md index 3cefa44f..4b6a8053 100644 --- a/docs/reference/rpc.md +++ b/docs/reference/rpc.md @@ -19,7 +19,7 @@ it; relative paths resolve against that directory. When it starts, the server writes one hello line: ```json -{"isb": "3.1.0", "protocol": 1} +{"isb": "3.1.1", "protocol": 1} ``` A client checks `protocol` and refuses versions it does not know. @@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present), ## Example session ```text -<- {"isb":"3.1.0","protocol":1} +<- {"isb":"3.1.1","protocol":1} -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}} <- {"id":1,"event":"progress","data":"web: creating from dev-base"} <- {"id":1,"event":"progress","data":"web: starting"} diff --git a/scripts/ratchet.sh b/scripts/ratchet.sh index 0244d3b2..84ed9f85 100755 --- a/scripts/ratchet.sh +++ b/scripts/ratchet.sh @@ -4,7 +4,9 @@ # # - File size: no Rust file may exceed LIMIT lines, except the files listed # (`path lines`), each capped at its listed size. Once a listed file is -# under LIMIT it must leave the list. +# under LIMIT it must leave the list. A file's inline `#[cfg(test)] mod` +# block is not counted: the limit keeps code legible, and a test should +# never be cut short to fit under it. # - Lint exemptions: clippy enforces the thresholds in clippy.toml, and an # existing offender carries an `#[expect(clippy::, reason = ...)]`. # The number of times each ratcheted lint is named in the source @@ -35,7 +37,14 @@ while read -r f; do [ -f "$f" ] && files+=("$f"); done < <(sort -u <<<"$list") # `path lines` for every file, then `lint: count` for every lint. measure() { - wc -l "${files[@]}" | awk '$2 != "total" { print $2, $1 }' + # Every line outside a top-level `#[cfg(test)] mod name { ... }` block. + awk ' + function emit() { if (f != "") print f, n } + FNR == 1 { emit(); f = FILENAME; n = 0; test = 0; prev = "" } + test { if (/^}/) test = 0; next } + prev == "#[cfg(test)]" && /^mod [A-Za-z_]+ *\{/ { test = 1; n--; prev = ""; next } + { n++; prev = $0 } + END { emit() }' "${files[@]}" for l in $LINTS; do printf 'lint:%s %s\n' "$l" "$(cat "${files[@]}" | grep -o "clippy::$l\b" | wc -l)" done diff --git a/scripts/ratchet.txt b/scripts/ratchet.txt index a38bc8ed..3c8ac075 100644 --- a/scripts/ratchet.txt +++ b/scripts/ratchet.txt @@ -1,27 +1,24 @@ # The ratchet budgets checked by scripts/ratchet.sh: Rust files allowed over -# its line LIMIT (`path max-lines`), and how many times each ratcheted clippy -# lint may be named in an exemption (`lint: max-count`). Numbers only go -# down: run `scripts/ratchet.sh --update` after shrinking something. +# its line LIMIT (`path max-lines`, not counting inline test modules), and +# how many times each ratcheted clippy lint may be named in an exemption +# (`lint: max-count`). Numbers only go down: run +# `scripts/ratchet.sh --update` after shrinking something. crates/isb-apps/src/app/deploy.rs 1736 -crates/isb-apps/src/app/preview.rs 1812 -crates/isb-apps/src/backup.rs 1675 -crates/isb-apps/src/notify/mod.rs 1263 +crates/isb-apps/src/app/preview.rs 1574 +crates/isb-apps/src/backup.rs 1501 crates/isb-apps/src/s3.rs 1263 -crates/isb-core/src/ingress/caddy.rs 1130 -crates/isb-core/src/machine.rs 1097 crates/isb-core/src/plan.rs 1517 crates/isb-core/src/registry/oci.rs 1081 crates/isb-core/src/sandbox.rs 1313 crates/isb-core/src/spec.rs 1518 crates/isb-core/src/stack/controller.rs 2325 crates/isb-daemon/src/daemon/mod.rs 1089 -crates/isb-server/src/audit.rs 1153 crates/isb-server/src/auth/http.rs 1040 crates/isb-server/src/auth/mod.rs 1658 -crates/isb-server/src/history.rs 1581 +crates/isb-server/src/history.rs 1305 crates/isb-server/src/server/http.rs 1085 -crates/isb-server/src/server/mcp.rs 2120 -crates/isb-tui/src/tui/ui.rs 1564 +crates/isb-server/src/server/mcp.rs 1304 +crates/isb-tui/src/tui/ui.rs 1259 lint:cognitive_complexity 14 lint:excessive_nesting 8 lint:too_many_arguments 12 diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 6fb45cf0..fb90e60a 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "isb-sdk" -version = "3.1.0" +version = "3.1.1" description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)" readme = "README.md" license = "MIT" diff --git a/sdk/python/src/isb/__init__.py b/sdk/python/src/isb/__init__.py index c3d29e98..58c40b99 100644 --- a/sdk/python/src/isb/__init__.py +++ b/sdk/python/src/isb/__init__.py @@ -62,7 +62,7 @@ async def main(): from ._util import Duration from .volumes import Volumes -__version__ = "3.1.0" +__version__ = "3.1.1" __all__ = [ "PROTOCOL", diff --git a/sdk/python/tests/test_unit.py b/sdk/python/tests/test_unit.py index 8999ac4b..87da28dc 100644 --- a/sdk/python/tests/test_unit.py +++ b/sdk/python/tests/test_unit.py @@ -222,7 +222,7 @@ async def test_death_fails_pending_requests(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"3.1.0","protocol":1}' + echo '{"isb":"3.1.1","protocol":1}' read line echo "boom" >&2 exit 3 @@ -243,7 +243,7 @@ async def test_close_fails_pending(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"3.1.0","protocol":1}' + echo '{"isb":"3.1.1","protocol":1}' cat >/dev/null """, ) @@ -261,7 +261,7 @@ async def test_events_and_out_of_order_replies(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"3.1.0","protocol":1}' + echo '{"isb":"3.1.1","protocol":1}' read a read b echo 'not json' diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index cb05b37f..2863153b 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -72,7 +72,7 @@ wheels = [ [[package]] name = "isb-sdk" -version = "3.1.0" +version = "3.1.1" source = { editable = "." } [package.dev-dependencies] diff --git a/sdk/typescript/bun.lock b/sdk/typescript/bun.lock index 16fe38ca..5c829934 100644 --- a/sdk/typescript/bun.lock +++ b/sdk/typescript/bun.lock @@ -12,8 +12,8 @@ "typescript": "^5.9.0", }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "3.1.0", - "@execution-associates/isb-linux-x64": "3.1.0", + "@execution-associates/isb-linux-arm64": "3.1.1", + "@execution-associates/isb-linux-x64": "3.1.1", }, }, }, diff --git a/sdk/typescript/npm/linux-arm64/package.json b/sdk/typescript/npm/linux-arm64/package.json index c8042810..3195bed8 100644 --- a/sdk/typescript/npm/linux-arm64/package.json +++ b/sdk/typescript/npm/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-arm64", - "version": "3.1.0", + "version": "3.1.1", "description": "The static isb binary for Linux arm64 (aarch64-unknown-linux-musl), used by @execution-associates/isb", "license": "MIT", "repository": { diff --git a/sdk/typescript/npm/linux-x64/package.json b/sdk/typescript/npm/linux-x64/package.json index 1fd80229..d65323e2 100644 --- a/sdk/typescript/npm/linux-x64/package.json +++ b/sdk/typescript/npm/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-x64", - "version": "3.1.0", + "version": "3.1.1", "description": "The static isb binary for Linux x64 (x86_64-unknown-linux-musl), used by @execution-associates/isb", "license": "MIT", "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 51842651..a1a5b167 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb", - "version": "3.1.0", + "version": "3.1.1", "description": "TypeScript SDK for isb: declarative incus sandboxes (containers and VMs)", "license": "MIT", "type": "module", @@ -51,8 +51,8 @@ "prepack": "bun run build" }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "3.1.0", - "@execution-associates/isb-linux-x64": "3.1.0" + "@execution-associates/isb-linux-arm64": "3.1.1", + "@execution-associates/isb-linux-x64": "3.1.1" }, "devDependencies": { "@biomejs/biome": "^2.2.0",