isb 3.3.0: a host reboot comes back in waves: staggered autostart, paced controller starts, dev sandbox disk caps

Cargo.toml

@@ -54,20 +54,20 @@ members = ["crates/*"]
 default-members = [".", "crates/*"]
 
 [workspace.package]
-version = "3.2.0"
+version = "3.3.0"
 edition = "2024"
 rust-version = "1.87"
 license = "MIT"
 repository = "https://github.com/execution-associates/isb"
 homepage = "https://github.com/execution-associates/isb"
 
 [workspace.dependencies]
-isb-core = { path = "crates/isb-core", version = "=3.2.0" }
-isb-server = { path = "crates/isb-server", version = "=3.2.0" }
-isb-apps = { path = "crates/isb-apps", version = "=3.2.0" }
-isb-daemon = { path = "crates/isb-daemon", version = "=3.2.0" }
-isb-tui = { path = "crates/isb-tui", version = "=3.2.0" }
-isb-egress = { path = "crates/isb-egress", version = "=3.2.0" }
+isb-core = { path = "crates/isb-core", version = "=3.3.0" }
+isb-server = { path = "crates/isb-server", version = "=3.3.0" }
+isb-apps = { path = "crates/isb-apps", version = "=3.3.0" }
+isb-daemon = { path = "crates/isb-daemon", version = "=3.3.0" }
+isb-tui = { path = "crates/isb-tui", version = "=3.3.0" }
+isb-egress = { path = "crates/isb-egress", version = "=3.3.0" }
 
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

crates/isb-core/src/plan.rs

@@ -22,14 +22,15 @@ use crate::error::{Error, Result};
 use crate::flex::parse_duration;
 use crate::idmap::{self, SubIds};
 use crate::sandbox::expiry::Expires;
-use crate::spec::{
-    ExecDefaults, InstanceType, MountType, PortBind, ReadyCheck, RestartMode, SandboxSpec,
-};
+use crate::spec::{ExecDefaults, InstanceType, MountType, PortBind, ReadyCheck, SandboxSpec};
 
+mod boot;
 mod names;
 mod owner;
 pub use names::{device_name_for_path, validate_instance_name};
 
+pub use boot::{BOOT_DELAY_CONTAINER, BOOT_DELAY_VM, boot_config, dev_disk_limits};
+
 pub type Props = BTreeMap<String, String>;
 
 /// Facts about the host that resolution depends on.
@@ -621,15 +622,9 @@ pub fn resolve(
     for (k, v) in spec.env.file_vars() {
         config.insert(format!("environment.{k}"), v);
     }
-    if let Some(r) = spec.restart {
-        // incus' default (no boot.autostart) already restores the state the
-        // instance had at shutdown, which is exactly unless-stopped.
-        if matches!(r, RestartMode::Always | RestartMode::OnFailure) {
-            config.insert("boot.autostart".into(), "true".into());
-        }
-        if r.is_long_running() {
-            config.insert("boot.autorestart".into(), "true".into());
-        }
+    config.extend(boot_config(spec, vm).map_err(|e| Error::invalid(format!("{name}: {e}")))?);
+    if spec.restart.is_some_and(|r| r.is_long_running()) {
+        config.insert("boot.autorestart".into(), "true".into());
     }
     if oci {
         let mut line: Vec<String> = spec.entrypoint.clone().unwrap_or_default();
@@ -859,7 +854,9 @@ pub fn resolve(
         }
         None => None,
     };
-    let mut root_extra = Props::new();
+    let mut root_extra = (!boot::is_replica(spec))
+        .then(dev_disk_limits)
+        .unwrap_or_default();
     for (dname, props) in &spec.raw_devices {
         if dname == "root" {
             // Tune the root disk (size, ...): merged over the generated one below.

crates/isb-core/src/plan/boot.rs

@@ -0,0 +1,73 @@
+//! What a spec says about the host's boot and disk: the `boot.autostart`
+//! keys an instance gets, and a dev sandbox's root disk I/O caps.
+
+use super::Props;
+use crate::flex::parse_duration;
+use crate::spec::{RestartMode, SandboxSpec};
+
+/// Seconds incus waits after autostarting a container, or a VM, before it
+/// starts the next one. Incus starts autostart instances one at a time, but
+/// a start returns as soon as the guest's init runs, so with no delay a host
+/// boot wakes every instance in the same few seconds (2026-10-24 on titan:
+/// 154 at once, io pressure at 98% for 85 minutes).
+pub const BOOT_DELAY_CONTAINER: u64 = 2;
+pub const BOOT_DELAY_VM: u64 = 10;
+
+/// The `boot.autostart` keys an instance gets: the spec's `boot`, over the
+/// defaults. `always` and `on-failure` start with the host, spaced by
+/// [`BOOT_DELAY_CONTAINER`] (or `_VM`); `unless-stopped` leaves autostart
+/// unset, which in incus restores the state the instance had at shutdown;
+/// anything else is `false`, so a reboot does not bring back every dev
+/// sandbox that happened to be running. Applied in place by the stack
+/// controller too, so a replica is never replaced for these keys.
+pub fn boot_config(spec: &SandboxSpec, vm: bool) -> Result<Props, String> {
+    let b = spec.boot.clone().unwrap_or_default();
+    let autostart = b.autostart.or(match spec.restart.unwrap_or_default() {
+        RestartMode::Always | RestartMode::OnFailure => Some(true),
+        RestartMode::UnlessStopped => None,
+        RestartMode::No => Some(false),
+    });
+    let mut out = Props::new();
+    if let Some(a) = autostart {
+        out.insert("boot.autostart".into(), a.to_string());
+    }
+    if let Some(p) = b.priority {
+        out.insert("boot.autostart.priority".into(), p.to_string());
+    }
+    let delay = match &b.delay {
+        Some(d) => Some(
+            parse_duration(d)
+                .map_err(|e| format!("boot.delay: {e}"))?
+                .as_secs(),
+        ),
+        None if autostart == Some(true) => Some(if vm {
+            BOOT_DELAY_VM
+        } else {
+            BOOT_DELAY_CONTAINER
+        }),
+        None => None,
+    };
+    if let Some(d) = delay {
+        out.insert("boot.autostart.delay".into(), d.to_string());
+    }
+    Ok(out)
+}
+
+/// A stack replica: the controller labels every instance it creates.
+pub(super) fn is_replica(spec: &SandboxSpec) -> bool {
+    spec.labels.contains_key(crate::stack::LABEL_STACK)
+}
+
+/// The root disk's default I/O caps for a sandbox that is not a stack
+/// replica, merged under `raw_devices.root`. They bound one runaway sandbox
+/// (a build, a test suite, a stuck loop) on a host whose instances share one
+/// disk, without starving it: titan-iac measured a 2MiB/s cap breaking a
+/// k3s control plane, and caps its org guests at 128MiB/s read and 32MiB/s
+/// write. A dev sandbox writes more (cargo targets, package installs), so
+/// its write cap is twice that.
+pub fn dev_disk_limits() -> Props {
+    Props::from([
+        ("limits.read".into(), "128MiB".into()),
+        ("limits.write".into(), "64MiB".into()),
+    ])
+}

crates/isb-core/src/plan/tests.rs

@@ -1,5 +1,5 @@
 use super::*;
-use crate::spec::{IdmapMode, IdmapSpec, NamedVolumeSpec, PortBinding, Volume};
+use crate::spec::{IdmapMode, IdmapSpec, NamedVolumeSpec, PortBinding, RestartMode, Volume};
 
 const TITAN: &str = "root:1000000:1000000000\nroot:1000:1\n";
 
@@ -903,3 +903,98 @@ fn a_new_volume_belongs_to_the_service_user() {
     let (d, fixes) = owner_plan(Volume::named("v"), None, &h, true);
     assert!(d.volumes[0].config.is_empty() && fixes.is_empty());
 }
+
+fn resolved(spec: &SandboxSpec) -> Desired {
+    resolve(spec, &VolumeDefs::new(), &host(), Path::new("/")).unwrap()
+}
+
+fn boot_keys(d: &Desired) -> Vec<(String, String)> {
+    d.config
+        .iter()
+        .filter(|(k, _)| k.starts_with("boot."))
+        .map(|(k, v)| (k.clone(), v.clone()))
+        .collect()
+}
+
+fn kv(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
+    pairs
+        .iter()
+        .map(|(k, v)| (k.to_string(), v.to_string()))
+        .collect()
+}
+
+#[test]
+fn a_plain_sandbox_does_not_come_back_after_a_reboot() {
+    // incus would restore an unset autostart from the last state.
+    let s = SandboxSpec::new("dev", "dev-base");
+    assert_eq!(boot_keys(&resolved(&s)), kv(&[("boot.autostart", "false")]));
+    let mut s = s;
+    s.restart = Some(RestartMode::No);
+    assert_eq!(boot_keys(&resolved(&s)), kv(&[("boot.autostart", "false")]));
+}
+
+#[test]
+fn a_long_running_sandbox_autostarts_spaced() {
+    let mut s = SandboxSpec::new("dev", "dev-base");
+    s.restart = Some(RestartMode::Always);
+    assert_eq!(
+        boot_keys(&resolved(&s)),
+        kv(&[
+            ("boot.autorestart", "true"),
+            ("boot.autostart", "true"),
+            ("boot.autostart.delay", "2"),
+        ])
+    );
+    s.instance_type = InstanceType::VirtualMachine;
+    assert_eq!(boot_config(&s, true).unwrap()["boot.autostart.delay"], "10");
+    // unless-stopped is incus' restore-the-last-state.
+    s.restart = Some(RestartMode::UnlessStopped);
+    assert_eq!(
+        boot_keys(&resolved(&s)),
+        kv(&[("boot.autorestart", "true")])
+    );
+}
+
+#[test]
+fn boot_is_overridable_in_the_spec() {
+    let mut s = SandboxSpec::new("dev", "dev-base");
+    s.boot = Some(crate::spec::BootSpec {
+        autostart: Some(true),
+        priority: Some(7),
+        delay: Some("1m".into()),
+    });
+    assert_eq!(
+        boot_keys(&resolved(&s)),
+        kv(&[
+            ("boot.autostart", "true"),
+            ("boot.autostart.delay", "60"),
+            ("boot.autostart.priority", "7"),
+        ])
+    );
+    // raw_config still has the last word.
+    s.raw_config
+        .insert("boot.autostart.delay".into(), "0".into());
+    assert_eq!(resolved(&s).config["boot.autostart.delay"], "0");
+    s.boot.as_mut().unwrap().delay = Some("soon".into());
+    let e = resolve(&s, &VolumeDefs::new(), &host(), Path::new("/")).unwrap_err();
+    assert!(e.to_string().contains("boot.delay"), "{e}");
+}
+
+#[test]
+fn a_dev_sandbox_root_disk_is_io_capped_and_a_replica_is_not() {
+    let s = SandboxSpec::new("dev", "dev-base");
+    let root = &resolved(&s).devices["root"].props;
+    assert_eq!(root["limits.read"], "128MiB");
+    assert_eq!(root["limits.write"], "64MiB");
+    // Overridden through raw_devices.root, like its size.
+    let s = SandboxSpec::new("dev", "dev-base").raw_device(
+        "root",
+        BTreeMap::from([("limits.write".into(), "256MiB".into())]),
+    );
+    let root = &resolved(&s).devices["root"].props;
+    assert_eq!(root["limits.read"], "128MiB");
+    assert_eq!(root["limits.write"], "256MiB");
+    let s = SandboxSpec::new("web-web-1-abcd", "dev-base").label(crate::stack::LABEL_STACK, "web");
+    let root = &resolved(&s).devices["root"].props;
+    assert!(!root.contains_key("limits.read") && !root.contains_key("limits.write"));
+}

crates/isb-core/src/spec.rs

@@ -16,8 +16,10 @@ pub use secret::{DEFAULT_SECRET_REFRESH, OnChange, SecretAs, SecretDef};
 mod env;
 pub use env::Environment;
 mod mount;
+mod restart;
 pub(crate) use mount::is_host_path;
 pub use mount::{MountType, VolumeOptions, VolumeSpec};
+pub use restart::{BootSpec, RestartMode};
 mod ready;
 pub use ready::ReadyCheck;
 
@@ -324,6 +326,13 @@ pub struct SandboxSpec {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub restart: Option<RestartMode>,
 
+    /// Whether and in what order incus starts the instance with the host
+    /// (`boot.autostart`, `.priority`, `.delay`). Default: from `restart`,
+    /// with a stack replica in the first start wave and every autostarting
+    /// instance spaced from the next (docs/reference/compose.md#boot).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub boot: Option<BootSpec>,
+
     /// A recurring health test, as in docker compose. `isb stack deploy`
     /// routes traffic only to healthy replicas and replaces unhealthy ones;
     /// `depends_on` can wait for it.
@@ -868,42 +877,6 @@ impl ExecDefaults {
 // Long-running services: restart, healthcheck, depends_on, deploy, secrets.
 // ----------------------------------------------------------------------------
 
-/// docker's `restart:`.
-#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, JsonSchema)]
-#[serde(rename_all = "kebab-case")]
-pub enum RestartMode {
-    #[default]
-    No,
-    Always,
-    OnFailure,
-    UnlessStopped,
-}
-
-// By hand so that YAML 1.1 habits (`restart: no` read as false) still work.
-impl<'de> Deserialize<'de> for RestartMode {
-    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
-        use serde::de::Error as _;
-        let s = flex::Scalar::deserialize(d)?.into_string();
-        Ok(match s.as_str() {
-            "no" | "false" | "" => RestartMode::No,
-            "always" => RestartMode::Always,
-            "on-failure" => RestartMode::OnFailure,
-            "unless-stopped" => RestartMode::UnlessStopped,
-            other => {
-                return Err(D::Error::custom(format!(
-                    "unknown restart {other:?} (no, always, on-failure, unless-stopped)"
-                )));
-            }
-        })
-    }
-}
-
-impl RestartMode {
-    pub fn is_long_running(&self) -> bool {
-        *self != RestartMode::No
-    }
-}
-
 /// docker compose's `healthcheck:`. Durations are strings (`30s`, `1m30s`).
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
 #[serde(deny_unknown_fields)]