From 60e32ecb622fa66983c15eb5328bc16abb8eb4fd Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 26 Oct 2026 00:24:36 +0000 Subject: [PATCH 1/3] Boot policy: staggered autostart, paced controller starts, dev sandbox disk caps Stack replicas autostart in waves (slot 1 at priority 2, the rest at 1, 2s apart, a VM 10s), set on running replicas in place and kept out of the revision. Plain sandboxes default to boot.autostart=false and get root disk caps (128MiB/s read, 64MiB/s write). The controller paces instance starts and creates across every stack (--start-concurrency, --start-io-pressure on /proc/pressure/io). Claude-Session: https://claude.ai/code/session_01XJek6YFHRCRmXsmJ3v1WV4 --- crates/isb-core/src/plan.rs | 23 +- crates/isb-core/src/plan/boot.rs | 73 ++++ crates/isb-core/src/plan/tests.rs | 97 ++++- crates/isb-core/src/spec.rs | 45 +-- crates/isb-core/src/spec/restart.rs | 72 ++++ crates/isb-core/src/stack/controller.rs | 129 ++++--- crates/isb-core/src/stack/controller/boot.rs | 76 ++++ .../src/stack/controller/instances.rs | 85 ++++- crates/isb-core/src/stack/controller_tests.rs | 21 ++ crates/isb-core/src/stack/mod.rs | 12 + crates/isb-core/src/stack/pacing.rs | 334 ++++++++++++++++++ crates/isb-daemon/src/daemon/mod.rs | 21 +- crates/isb-daemon/src/daemon/sandboxes.rs | 13 + docs/concepts/stacks.md | 25 +- docs/reference/compose.md | 38 +- docs/reference/configuration.md | 12 + scripts/ratchet.txt | 8 +- sdk/python/src/isb/_spec.py | 24 ++ sdk/typescript/src/spec.ts | 39 +- src/bin/isb/serve.rs | 12 + 20 files changed, 1010 insertions(+), 149 deletions(-) create mode 100644 crates/isb-core/src/plan/boot.rs create mode 100644 crates/isb-core/src/spec/restart.rs create mode 100644 crates/isb-core/src/stack/controller/boot.rs create mode 100644 crates/isb-core/src/stack/pacing.rs diff --git a/crates/isb-core/src/plan.rs b/crates/isb-core/src/plan.rs index 1b2d3f89..ea8736ca 100644 --- a/crates/isb-core/src/plan.rs +++ b/crates/isb-core/src/plan.rs @@ -21,12 +21,13 @@ use serde_json::{Value, json}; use crate::error::{Error, Result}; use crate::flex::parse_duration; use crate::idmap::{self, SubIds}; -use crate::spec::{ - ExecDefaults, InstanceType, MountType, PortBind, ReadyCheck, RestartMode, SandboxSpec, -}; +use crate::spec::{ExecDefaults, InstanceType, MountType, PortBind, ReadyCheck, SandboxSpec}; +mod boot; mod owner; +pub use boot::{BOOT_DELAY_CONTAINER, BOOT_DELAY_VM, boot_config, dev_disk_limits}; + pub type Props = BTreeMap; /// Facts about the host that resolution depends on. @@ -663,15 +664,9 @@ pub fn resolve( for (k, v) in spec.env.file_vars() { config.insert(format!("environment.{k}"), v); } - if let Some(r) = spec.restart { - // incus' default (no boot.autostart) already restores the state the - // instance had at shutdown, which is exactly unless-stopped. - if matches!(r, RestartMode::Always | RestartMode::OnFailure) { - config.insert("boot.autostart".into(), "true".into()); - } - if r.is_long_running() { - config.insert("boot.autorestart".into(), "true".into()); - } + config.extend(boot_config(spec, vm).map_err(|e| Error::invalid(format!("{name}: {e}")))?); + if spec.restart.is_some_and(|r| r.is_long_running()) { + config.insert("boot.autorestart".into(), "true".into()); } if oci { let mut line: Vec = spec.entrypoint.clone().unwrap_or_default(); @@ -901,7 +896,9 @@ pub fn resolve( } None => None, }; - let mut root_extra = Props::new(); + let mut root_extra = (!boot::is_replica(spec)) + .then(dev_disk_limits) + .unwrap_or_default(); for (dname, props) in &spec.raw_devices { if dname == "root" { // Tune the root disk (size, ...): merged over the generated one below. diff --git a/crates/isb-core/src/plan/boot.rs b/crates/isb-core/src/plan/boot.rs new file mode 100644 index 00000000..185abb02 --- /dev/null +++ b/crates/isb-core/src/plan/boot.rs @@ -0,0 +1,73 @@ +//! What a spec says about the host's boot and disk: the `boot.autostart` +//! keys an instance gets, and a dev sandbox's root disk I/O caps. + +use super::Props; +use crate::flex::parse_duration; +use crate::spec::{RestartMode, SandboxSpec}; + +/// Seconds incus waits after autostarting a container, or a VM, before it +/// starts the next one. Incus starts autostart instances one at a time, but +/// a start returns as soon as the guest's init runs, so with no delay a host +/// boot wakes every instance in the same few seconds (2026-10-24 on titan: +/// 154 at once, io pressure at 98% for 85 minutes). +pub const BOOT_DELAY_CONTAINER: u64 = 2; +pub const BOOT_DELAY_VM: u64 = 10; + +/// The `boot.autostart` keys an instance gets: the spec's `boot`, over the +/// defaults. `always` and `on-failure` start with the host, spaced by +/// [`BOOT_DELAY_CONTAINER`] (or `_VM`); `unless-stopped` leaves autostart +/// unset, which in incus restores the state the instance had at shutdown; +/// anything else is `false`, so a reboot does not bring back every dev +/// sandbox that happened to be running. Applied in place by the stack +/// controller too, so a replica is never replaced for these keys. +pub fn boot_config(spec: &SandboxSpec, vm: bool) -> Result { + let b = spec.boot.clone().unwrap_or_default(); + let autostart = b.autostart.or(match spec.restart.unwrap_or_default() { + RestartMode::Always | RestartMode::OnFailure => Some(true), + RestartMode::UnlessStopped => None, + RestartMode::No => Some(false), + }); + let mut out = Props::new(); + if let Some(a) = autostart { + out.insert("boot.autostart".into(), a.to_string()); + } + if let Some(p) = b.priority { + out.insert("boot.autostart.priority".into(), p.to_string()); + } + let delay = match &b.delay { + Some(d) => Some( + parse_duration(d) + .map_err(|e| format!("boot.delay: {e}"))? + .as_secs(), + ), + None if autostart == Some(true) => Some(if vm { + BOOT_DELAY_VM + } else { + BOOT_DELAY_CONTAINER + }), + None => None, + }; + if let Some(d) = delay { + out.insert("boot.autostart.delay".into(), d.to_string()); + } + Ok(out) +} + +/// A stack replica: the controller labels every instance it creates. +pub(super) fn is_replica(spec: &SandboxSpec) -> bool { + spec.labels.contains_key(crate::stack::LABEL_STACK) +} + +/// The root disk's default I/O caps for a sandbox that is not a stack +/// replica, merged under `raw_devices.root`. They bound one runaway sandbox +/// (a build, a test suite, a stuck loop) on a host whose instances share one +/// disk, without starving it: titan-iac measured a 2MiB/s cap breaking a +/// k3s control plane, and caps its org guests at 128MiB/s read and 32MiB/s +/// write. A dev sandbox writes more (cargo targets, package installs), so +/// its write cap is twice that. +pub fn dev_disk_limits() -> Props { + Props::from([ + ("limits.read".into(), "128MiB".into()), + ("limits.write".into(), "64MiB".into()), + ]) +} diff --git a/crates/isb-core/src/plan/tests.rs b/crates/isb-core/src/plan/tests.rs index 01895878..046f3e9d 100644 --- a/crates/isb-core/src/plan/tests.rs +++ b/crates/isb-core/src/plan/tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::spec::{IdmapMode, IdmapSpec, NamedVolumeSpec, PortBinding, Volume}; +use crate::spec::{IdmapMode, IdmapSpec, NamedVolumeSpec, PortBinding, RestartMode, Volume}; const TITAN: &str = "root:1000000:1000000000\nroot:1000:1\n"; @@ -903,3 +903,98 @@ fn a_new_volume_belongs_to_the_service_user() { let (d, fixes) = owner_plan(Volume::named("v"), None, &h, true); assert!(d.volumes[0].config.is_empty() && fixes.is_empty()); } + +fn resolved(spec: &SandboxSpec) -> Desired { + resolve(spec, &VolumeDefs::new(), &host(), Path::new("/")).unwrap() +} + +fn boot_keys(d: &Desired) -> Vec<(String, String)> { + d.config + .iter() + .filter(|(k, _)| k.starts_with("boot.")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +fn kv(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[test] +fn a_plain_sandbox_does_not_come_back_after_a_reboot() { + // incus would restore an unset autostart from the last state. + let s = SandboxSpec::new("dev", "dev-base"); + assert_eq!(boot_keys(&resolved(&s)), kv(&[("boot.autostart", "false")])); + let mut s = s; + s.restart = Some(RestartMode::No); + assert_eq!(boot_keys(&resolved(&s)), kv(&[("boot.autostart", "false")])); +} + +#[test] +fn a_long_running_sandbox_autostarts_spaced() { + let mut s = SandboxSpec::new("dev", "dev-base"); + s.restart = Some(RestartMode::Always); + assert_eq!( + boot_keys(&resolved(&s)), + kv(&[ + ("boot.autorestart", "true"), + ("boot.autostart", "true"), + ("boot.autostart.delay", "2"), + ]) + ); + s.instance_type = InstanceType::VirtualMachine; + assert_eq!(boot_config(&s, true).unwrap()["boot.autostart.delay"], "10"); + // unless-stopped is incus' restore-the-last-state. + s.restart = Some(RestartMode::UnlessStopped); + assert_eq!( + boot_keys(&resolved(&s)), + kv(&[("boot.autorestart", "true")]) + ); +} + +#[test] +fn boot_is_overridable_in_the_spec() { + let mut s = SandboxSpec::new("dev", "dev-base"); + s.boot = Some(crate::spec::BootSpec { + autostart: Some(true), + priority: Some(7), + delay: Some("1m".into()), + }); + assert_eq!( + boot_keys(&resolved(&s)), + kv(&[ + ("boot.autostart", "true"), + ("boot.autostart.delay", "60"), + ("boot.autostart.priority", "7"), + ]) + ); + // raw_config still has the last word. + s.raw_config + .insert("boot.autostart.delay".into(), "0".into()); + assert_eq!(resolved(&s).config["boot.autostart.delay"], "0"); + s.boot.as_mut().unwrap().delay = Some("soon".into()); + let e = resolve(&s, &VolumeDefs::new(), &host(), Path::new("/")).unwrap_err(); + assert!(e.to_string().contains("boot.delay"), "{e}"); +} + +#[test] +fn a_dev_sandbox_root_disk_is_io_capped_and_a_replica_is_not() { + let s = SandboxSpec::new("dev", "dev-base"); + let root = &resolved(&s).devices["root"].props; + assert_eq!(root["limits.read"], "128MiB"); + assert_eq!(root["limits.write"], "64MiB"); + // Overridden through raw_devices.root, like its size. + let s = SandboxSpec::new("dev", "dev-base").raw_device( + "root", + BTreeMap::from([("limits.write".into(), "256MiB".into())]), + ); + let root = &resolved(&s).devices["root"].props; + assert_eq!(root["limits.read"], "128MiB"); + assert_eq!(root["limits.write"], "256MiB"); + let s = SandboxSpec::new("web-web-1-abcd", "dev-base").label(crate::stack::LABEL_STACK, "web"); + let root = &resolved(&s).devices["root"].props; + assert!(!root.contains_key("limits.read") && !root.contains_key("limits.write")); +} diff --git a/crates/isb-core/src/spec.rs b/crates/isb-core/src/spec.rs index 51d47c46..65664212 100644 --- a/crates/isb-core/src/spec.rs +++ b/crates/isb-core/src/spec.rs @@ -16,8 +16,10 @@ pub use secret::{DEFAULT_SECRET_REFRESH, OnChange, SecretAs, SecretDef}; mod env; pub use env::Environment; mod mount; +mod restart; pub(crate) use mount::is_host_path; pub use mount::{MountType, VolumeOptions, VolumeSpec}; +pub use restart::{BootSpec, RestartMode}; /// A compose file: named volumes plus any number of services, each one /// sandbox. Mirrors docker compose wherever incus allows. @@ -322,6 +324,13 @@ pub struct SandboxSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub restart: Option, + /// Whether and in what order incus starts the instance with the host + /// (`boot.autostart`, `.priority`, `.delay`). Default: from `restart`, + /// with a stack replica in the first start wave and every autostarting + /// instance spaced from the next (docs/reference/compose.md#boot). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub boot: Option, + /// A recurring health test, as in docker compose. `isb stack deploy` /// routes traffic only to healthy replicas and replaces unhealthy ones; /// `depends_on` can wait for it. @@ -944,42 +953,6 @@ impl ExecDefaults { // Long-running services: restart, healthcheck, depends_on, deploy, secrets. // ---------------------------------------------------------------------------- -/// docker's `restart:`. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, JsonSchema)] -#[serde(rename_all = "kebab-case")] -pub enum RestartMode { - #[default] - No, - Always, - OnFailure, - UnlessStopped, -} - -// By hand so that YAML 1.1 habits (`restart: no` read as false) still work. -impl<'de> Deserialize<'de> for RestartMode { - fn deserialize>(d: D) -> Result { - use serde::de::Error as _; - let s = flex::Scalar::deserialize(d)?.into_string(); - Ok(match s.as_str() { - "no" | "false" | "" => RestartMode::No, - "always" => RestartMode::Always, - "on-failure" => RestartMode::OnFailure, - "unless-stopped" => RestartMode::UnlessStopped, - other => { - return Err(D::Error::custom(format!( - "unknown restart {other:?} (no, always, on-failure, unless-stopped)" - ))); - } - }) - } -} - -impl RestartMode { - pub fn is_long_running(&self) -> bool { - *self != RestartMode::No - } -} - /// docker compose's `healthcheck:`. Durations are strings (`30s`, `1m30s`). #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/isb-core/src/spec/restart.rs b/crates/isb-core/src/spec/restart.rs new file mode 100644 index 00000000..c6a1bf26 --- /dev/null +++ b/crates/isb-core/src/spec/restart.rs @@ -0,0 +1,72 @@ +//! `restart:` and `boot:`: whether an instance is long-running, and +//! whether and in what order incus starts it with the host. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::flex; + +/// docker's `restart:`. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub enum RestartMode { + #[default] + No, + Always, + OnFailure, + UnlessStopped, +} + +// By hand so that YAML 1.1 habits (`restart: no` read as false) still work. +impl<'de> Deserialize<'de> for RestartMode { + fn deserialize>(d: D) -> Result { + use serde::de::Error as _; + let s = flex::Scalar::deserialize(d)?.into_string(); + Ok(match s.as_str() { + "no" | "false" | "" => RestartMode::No, + "always" => RestartMode::Always, + "on-failure" => RestartMode::OnFailure, + "unless-stopped" => RestartMode::UnlessStopped, + other => { + return Err(D::Error::custom(format!( + "unknown restart {other:?} (no, always, on-failure, unless-stopped)" + ))); + } + }) + } +} + +impl RestartMode { + pub fn is_long_running(&self) -> bool { + *self != RestartMode::No + } +} + +/// How incus starts an instance with the host. Every field left out takes +/// its default ([`crate::plan::boot_config`]). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BootSpec { + /// Start with the host (`boot.autostart`). + #[serde( + default, + deserialize_with = "flex::opt_bool", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "Option")] + pub autostart: Option, + + /// Start order (`boot.autostart.priority`): higher starts first. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// How long incus waits after starting this instance before it starts + /// the next (`boot.autostart.delay`), e.g. `5s`; whole seconds. + #[serde( + default, + deserialize_with = "flex::opt_string", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "Option")] + pub delay: Option, +} diff --git a/crates/isb-core/src/stack/controller.rs b/crates/isb-core/src/stack/controller.rs index 1232178c..1ed30785 100644 --- a/crates/isb-core/src/stack/controller.rs +++ b/crates/isb-core/src/stack/controller.rs @@ -18,10 +18,7 @@ use serde::Serialize; use super::changes::diff; use super::ports::{Published, published}; use super::secrets::{Cycle, LABEL_SECRETS, StaleSecret}; -use super::{ - LABEL_REV, LABEL_SERVICE, LABEL_SLOT, LABEL_STACK, StackDef, Store, instance_name, new_id, - now_secs, validate_stack_name, -}; +use super::{StackDef, Store, instance_name, new_id, now_secs, validate_stack_name}; use crate::balance::Balancer; use crate::client::{Client, encode_segment}; use crate::error::{Error, Result}; @@ -35,13 +32,14 @@ use crate::spec::{ }; use crate::supervise; +mod boot; mod dns; mod health; mod instances; pub use dns::{DnsScope, DnsScopeFn}; use health::InstRt; -use instances::instance_state; -pub use instances::list_instances; +pub use instances::{Inst, list_instances}; +use instances::{instance_spec, instance_state, live_versions}; /// How long a replaced instance's connections may drain before it is stopped. const DRAIN: Duration = Duration::from_secs(10); @@ -290,6 +288,8 @@ struct Inner { metrics_sink: Mutex>>, /// Which project environment a stack's services are also named in. dns_scope: Mutex>, + /// Paces instance starts and creates across every stack. + gate: super::pacing::StartGate, } impl Inner { @@ -367,6 +367,25 @@ impl Controller { interval: Duration, secrets: Arc, observer: Option>, + ) -> Result { + Controller::start_paced( + client, + store, + interval, + secrets, + observer, + Default::default(), + ) + } + + /// [`Controller::start_with`], pacing instance starts by `pacing`. + pub fn start_paced( + client: Client, + store: Store, + interval: Duration, + secrets: Arc, + observer: Option>, + pacing: super::pacing::Pacing, ) -> Result { let c = Controller { inner: Arc::new(Inner { @@ -387,6 +406,7 @@ impl Controller { event_sink: Mutex::new(None), metrics_sink: Mutex::new(None), dns_scope: Mutex::new(None), + gate: super::pacing::StartGate::host(pacing), }), }; // Driver-backed secrets are polled on their refresh intervals; the @@ -952,69 +972,6 @@ fn limit_error(msg: &str) -> bool { msg.contains(" quota (") || msg.contains(" limit (") } -/// The spec an instance of `service` is created from: labelled, with its -/// published TCP ports removed (the balancer serves them) and its UDP ports -/// as NAT proxies (see [`super::ports`]), and always long-running, as swarm -/// ignores `restart` in favour of `restart_policy`. -fn instance_spec( - def: &StackDef, - service: &str, - spec: &SandboxSpec, - slot: u32, - rev: &str, -) -> Result { - let mut s = spec.clone(); - s.image = def.instance_image(service, &spec.image); - s.restart = Some(RestartMode::Always); - (s.ports, s.stack_udp) = super::ports::instance_ports(spec)?; - s.domains.clear(); - if let Some(d) = &s.deploy { - s.labels.extend(d.labels.clone()); - } - s.labels.insert(LABEL_STACK.into(), def.name.clone()); - s.labels.insert(LABEL_SERVICE.into(), service.into()); - s.labels.insert(LABEL_SLOT.into(), slot.to_string()); - s.labels.insert(LABEL_REV.into(), rev.into()); - let live = live_versions(def, service); - if !live.is_empty() { - s.labels - .insert(LABEL_SECRETS.into(), super::secrets::versions_label(&live)); - } - Ok(s) -} - -/// The versions bound now of the secrets `service` takes in place. -fn live_versions(def: &StackDef, service: &str) -> BTreeMap { - def.live_secrets(service) - .into_iter() - .map(|(k, (_, v))| (k, v)) - .collect() -} - -/// A stack's instance as listed. -#[derive(Debug, Clone)] -#[doc(hidden)] -pub struct Inst { - pub name: String, - pub(super) slot: u32, - pub rev: String, - status: String, - /// [`LABEL_SECRETS`]: the `restart`/`none` secret versions its app last - /// started with; `None` on an instance from before the label. - secrets: Option>, -} - -impl Inst { - #[doc(hidden)] - pub fn is_running(&self) -> bool { - self.running() - } - - fn running(&self) -> bool { - self.status.eq_ignore_ascii_case("running") - } -} - /// A service's worker. struct Worker { inner: Arc, @@ -1362,6 +1319,7 @@ impl Worker { // Keep what exists running, set up and healthy. for i in &insts { + self.set_boot_order(def, i, &spec, &rev); self.maintain(def, i, &spec, oci, probe.as_ref())?; } // An interrupted rollout can leave an old instance next to a current @@ -1674,6 +1632,8 @@ impl Worker { .and_then(|p| p.condition) .unwrap_or_default(); let sb = self.handle(&i.name); + let inner = self.inner.clone(); + let mut permit = None; if !i.running() { self.set_rotation(&i.name, false); if condition == RestartCondition::None { @@ -1689,6 +1649,12 @@ impl Worker { &format!("{} is {}; starting it", i.name, i.status), ); self.count_restart(&i.name); + // Held through the setup below: the guest's boot is the load. + let held = &mut |p: f64| self.held(&i.name, p); + let Some(p) = inner.gate.acquire(&self.shared.stop, held) else { + return Ok(()); + }; + permit = Some(p); if let Err(e) = sb.start() { self.log(&format!("cannot start {}: {e}", i.name)); return Ok(()); @@ -1708,6 +1674,13 @@ impl Worker { rt.pid = pid; rt.started(Instant::now()); rt.next_probe = None; + if permit.is_none() { + // Booted by incus or found running at startup: as many as + // there are replicas at once, so wait out io pressure. + inner + .gate + .wait_pressure(&self.shared.stop, &mut |p| self.held(&i.name, p)); + } let r = self.setup(def, &sb, spec, oci); if let Err(e) = r { // Leave pid unset so the next pass tries again. @@ -1725,6 +1698,7 @@ impl Worker { self.mark_started(def, &i.name)?; } } + drop(permit); let alive = self.alive(&sb, spec, oci); let healthy = match probe { None => alive, @@ -1927,6 +1901,16 @@ impl Worker { } else { None }; + // Before a stop-first retire, so the old instance serves while it + // waits; held through the create, start and readiness checks. + let inner = self.inner.clone(); + let held = &mut |p: f64| self.held(&format!("slot {slot}"), p); + let Some(permit) = inner.gate.acquire(&self.shared.stop, held) else { + return Err(Error::OperationFailed { + step: format!("slot {slot}"), + message: "isb serve is stopping".into(), + }); + }; if let (Some(o), UpdateOrder::StopFirst) = (old, order) { self.log(&format!("slot {slot}: replacing {o} (stop-first)")); self.slot_state(def, slot, Some("draining"), None, None); @@ -1946,6 +1930,7 @@ impl Worker { let mut report = |m: &str| eprintln!("isb serve: {stack}: {m}"); let created = crate::sandbox::ensure(self.client(), &d, EnsureOptions::default(), &mut report); + drop(permit); let result = created.and_then(|_| { let inst = Inst { name: name.clone(), @@ -1953,6 +1938,12 @@ impl Worker { rev: rev.to_string(), status: "Running".into(), secrets: Some(live_versions(def, &self.service)), + boot: d + .config + .iter() + .filter(|(k, _)| k.starts_with("boot.autostart")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), }; self.insts.push(inst.clone()); self.slot_state(def, slot, None, None, Some("probing")); diff --git a/crates/isb-core/src/stack/controller/boot.rs b/crates/isb-core/src/stack/controller/boot.rs new file mode 100644 index 00000000..e504a7da --- /dev/null +++ b/crates/isb-core/src/stack/controller/boot.rs @@ -0,0 +1,76 @@ +//! A replica's boot order, set in place, and what a start waits for. + +use super::{Inst, Worker, instance_spec}; +use crate::error::Error; +use crate::spec::SandboxSpec; +use crate::stack::StackDef; + +/// A replica's default `boot.autostart.priority`: every service's first +/// slot starts with the host before any service's second, so after a reboot +/// each service serves again before the extra replicas add to the load. +/// Above incus' default (0), where sandboxes that autostart sit. +pub(super) fn boot_priority(slot: u32) -> i32 { + if slot <= 1 { 2 } else { 1 } +} + +impl Worker { + /// Bring an instance's `boot.autostart*` keys to what its slot should + /// have, in place: incus applies them to a running instance without a + /// restart, and they are not part of the revision, so a replica made + /// before the boot policy gets it without being replaced. Only the keys + /// the policy sets are written; a failure is logged and tried again on + /// the next pass. + pub(super) fn set_boot_order( + &mut self, + def: &StackDef, + i: &Inst, + spec: &SandboxSpec, + rev: &str, + ) { + let want = instance_spec(def, &self.service, spec, i.slot, rev).and_then(|s| { + let vm = s.instance_type == crate::spec::InstanceType::VirtualMachine; + crate::plan::boot_config(&s, vm).map_err(Error::invalid) + }); + let want = match want { + Ok(w) => w, + Err(e) => { + self.log(&format!("boot order for {}: {e}", i.name)); + return; + } + }; + let stale: crate::plan::Props = want + .into_iter() + .filter(|(k, v)| i.boot.get(k) != Some(v)) + .collect(); + if stale.is_empty() { + return; + } + let r = crate::sandbox::update_instance( + self.client(), + &i.name, + &format!("set the boot order of {}", i.name), + &mut |config, _| { + for (k, v) in &stale { + config.insert(k.clone(), serde_json::Value::String(v.clone())); + } + Ok(()) + }, + ); + match r { + Ok(()) => { + let keys: Vec = stale.iter().map(|(k, v)| format!("{k}={v}")).collect(); + self.log(&format!("{}: set {}", i.name, keys.join(" "))); + } + Err(e) => self.log(&format!("cannot set the boot order of {}: {e}", i.name)), + } + } + + /// Say once that host io pressure holds `name`'s start or setup. + pub(super) fn held(&self, name: &str, pressure: f64) { + self.event( + "warn", + Some(name), + &format!("host io pressure is {pressure:.0}%; {name} waits for it to fall"), + ); + } +} diff --git a/crates/isb-core/src/stack/controller/instances.rs b/crates/isb-core/src/stack/controller/instances.rs index fbded6f8..0e793d31 100644 --- a/crates/isb-core/src/stack/controller/instances.rs +++ b/crates/isb-core/src/stack/controller/instances.rs @@ -1,15 +1,16 @@ -//! Reading a stack's instances from incus: which exist (by their labels) -//! and an instance's init pid and address. +//! A stack's instances: the spec one is created from, which exist in incus +//! (by their labels), and an instance's init pid and address. +use std::collections::BTreeMap; use std::net::IpAddr; use serde_json::Value; use super::super::secrets::LABEL_SECRETS; -use super::super::{LABEL_REV, LABEL_SERVICE, LABEL_SLOT, LABEL_STACK}; -use super::Inst; +use super::super::{LABEL_REV, LABEL_SERVICE, LABEL_SLOT, LABEL_STACK, StackDef}; use crate::client::{Client, encode_query, encode_segment}; use crate::error::Result; +use crate::spec::{RestartMode, SandboxSpec}; /// A stack's instances (of one service), using incus' server-side filter. #[doc(hidden)] @@ -51,6 +52,11 @@ pub fn list_instances(client: &Client, stack: &str, service: Option<&str>) -> Re secrets: super::super::secrets::parse_versions_label( c.get(&format!("user.{LABEL_SECRETS}")).map(String::as_str), ), + boot: c + .iter() + .filter(|(k, _)| k.starts_with("boot.autostart")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), }); } out.sort_by(|a, b| (a.slot, &a.name).cmp(&(b.slot, &b.name))); @@ -95,3 +101,74 @@ pub(super) fn instance_state(client: &Client, name: &str) -> Result<(i64, Option } Ok((pid, v4.or(v6))) } + +/// The spec an instance of `service` is created from: labelled, with its +/// published TCP ports removed (the balancer serves them) and its UDP ports +/// as NAT proxies (see [`super::ports`]), and always long-running, as swarm +/// ignores `restart` in favour of `restart_policy`. +pub(super) fn instance_spec( + def: &StackDef, + service: &str, + spec: &SandboxSpec, + slot: u32, + rev: &str, +) -> Result { + let mut s = spec.clone(); + s.image = def.instance_image(service, &spec.image); + s.restart = Some(RestartMode::Always); + let boot = s.boot.get_or_insert_with(Default::default); + if boot.priority.is_none() { + boot.priority = Some(super::boot::boot_priority(slot)); + } + (s.ports, s.stack_udp) = super::super::ports::instance_ports(spec)?; + s.domains.clear(); + if let Some(d) = &s.deploy { + s.labels.extend(d.labels.clone()); + } + s.labels.insert(LABEL_STACK.into(), def.name.clone()); + s.labels.insert(LABEL_SERVICE.into(), service.into()); + s.labels.insert(LABEL_SLOT.into(), slot.to_string()); + s.labels.insert(LABEL_REV.into(), rev.into()); + let live = live_versions(def, service); + if !live.is_empty() { + s.labels.insert( + LABEL_SECRETS.into(), + super::super::secrets::versions_label(&live), + ); + } + Ok(s) +} + +/// The versions bound now of the secrets `service` takes in place. +pub(super) fn live_versions(def: &StackDef, service: &str) -> BTreeMap { + def.live_secrets(service) + .into_iter() + .map(|(k, (_, v))| (k, v)) + .collect() +} + +/// A stack's instance as listed. +#[derive(Debug, Clone)] +#[doc(hidden)] +pub struct Inst { + pub name: String, + pub(in crate::stack) slot: u32, + pub rev: String, + pub(super) status: String, + /// [`LABEL_SECRETS`]: the `restart`/`none` secret versions its app last + /// started with; `None` on an instance from before the label. + pub(super) secrets: Option>, + /// Its `boot.autostart*` config. + pub(super) boot: crate::plan::Props, +} + +impl Inst { + #[doc(hidden)] + pub fn is_running(&self) -> bool { + self.running() + } + + pub(super) fn running(&self) -> bool { + self.status.eq_ignore_ascii_case("running") + } +} diff --git a/crates/isb-core/src/stack/controller_tests.rs b/crates/isb-core/src/stack/controller_tests.rs index 22d33bbe..f3cf2958 100644 --- a/crates/isb-core/src/stack/controller_tests.rs +++ b/crates/isb-core/src/stack/controller_tests.rs @@ -234,6 +234,27 @@ fn instance_specs_are_labelled_and_unpublished() { assert_eq!(s.restart, Some(RestartMode::Always)); } +#[test] +fn first_slots_start_with_the_host_before_the_rest() { + let d = def("services:\n web: {image: x, deploy: {replicas: 3}}\n"); + let web = d.service("web").unwrap(); + let boot = |slot| { + let s = instance_spec(&d, "web", web, slot, "abcd").unwrap(); + crate::plan::boot_config(&s, false).unwrap() + }; + assert_eq!(boot(1)["boot.autostart"], "true"); + assert_eq!(boot(1)["boot.autostart.priority"], "2"); + assert_eq!(boot(1)["boot.autostart.delay"], "2"); + assert_eq!(boot(2)["boot.autostart.priority"], "1"); + assert_eq!(boot(3)["boot.autostart.priority"], "1"); + // The spec's own order wins. + let d = def("services:\n web: {image: x, boot: {priority: 9, delay: 5s}}\n"); + let s = instance_spec(&d, "web", d.service("web").unwrap(), 2, "abcd").unwrap(); + let b = crate::plan::boot_config(&s, false).unwrap(); + assert_eq!(b["boot.autostart.priority"], "9"); + assert_eq!(b["boot.autostart.delay"], "5"); +} + #[test] fn deploy_diff() { let a = def("services:\n web: {image: x}\n db: {image: y}\n"); diff --git a/crates/isb-core/src/stack/mod.rs b/crates/isb-core/src/stack/mod.rs index 49be2e41..e4898dc6 100644 --- a/crates/isb-core/src/stack/mod.rs +++ b/crates/isb-core/src/stack/mod.rs @@ -22,6 +22,7 @@ pub mod controller; pub mod deployments; pub mod failure; pub mod migrate; +pub mod pacing; mod ports; pub mod secrets; pub mod source; @@ -187,6 +188,8 @@ impl StackDef { s.env.on_change.clear(); // Domains are the ingress's: changing them never replaces an instance. s.domains.clear(); + // Boot order is set on the running instances in place. + s.boot = None; // Published ports are the balancer's, not the instance's (UDP: below). s.ports.retain(|p| p.bind == crate::spec::PortBind::Guest); if let Some(d) = &mut s.deploy { @@ -570,6 +573,15 @@ mod tests { assert_ne!(a.revision("web").unwrap(), d.revision("web").unwrap()); } + #[test] + fn revision_ignores_boot_order() { + // Set on running replicas in place, so changing it rolls nothing. + let a = def("services:\n web: {image: x}\n"); + let b = + def("services:\n web: {image: x, boot: {autostart: false, priority: 3, delay: 9s}}\n"); + assert_eq!(a.revision("web").unwrap(), b.revision("web").unwrap()); + } + #[test] fn revision_of_an_existing_spec_is_pinned() { // Upgrading isb must not roll a service whose spec did not change: new diff --git a/crates/isb-core/src/stack/pacing.rs b/crates/isb-core/src/stack/pacing.rs new file mode 100644 index 00000000..bdd4c39f --- /dev/null +++ b/crates/isb-core/src/stack/pacing.rs @@ -0,0 +1,334 @@ +//! Host-wide pacing of the instance starts and creates the stack controller +//! makes, across every stack and org. +//! +//! Each service has its own worker thread, so after a host boot (or an +//! `isb serve` restart beside stopped replicas) every worker would start its +//! instances in the same second. On a host whose instances share one disk +//! that is an I/O storm: on 2026-10-24 titan sat at io pressure 98% for 85 +//! minutes, with sshd and tailscaled starved. The gate bounds how many starts +//! run at once, spaces them, and holds new ones while the host's io pressure +//! (`/proc/pressure/io`) is high. A setup after a boot it did not cause (the +//! instance was already running) waits out the pressure only. +//! +//! `isb serve --start-concurrency` sets how many starts run at once (default +//! 4), and `--start-io-pressure` the `full avg10` percentage at or above +//! which new ones wait (default 40; `0` turns the check off). A start waits +//! at most [`MAX_HOLD`] for pressure to fall, so a host that stays busy +//! still recovers. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Condvar, Mutex}; +use std::time::{Duration, Instant}; + +/// The default starts in flight at once. +pub const DEFAULT_CONCURRENCY: u32 = 4; +/// The default io pressure (`full avg10`, percent) that holds new starts. +pub const DEFAULT_IO_PRESSURE: f64 = 40.0; +/// The least time between two starts. +pub const SPACING: Duration = Duration::from_secs(1); +/// The longest a start waits for io pressure to fall. +pub const MAX_HOLD: Duration = Duration::from_secs(300); +/// How often a held start looks again. +const POLL: Duration = Duration::from_millis(500); + +type PressureFn = Box Option + Send + Sync>; + +/// How the controller paces starts. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Pacing { + /// Starts in flight at once. + pub concurrency: usize, + /// The io pressure that holds new starts; `None` never holds them. + pub io_pressure: Option, +} + +impl Default for Pacing { + fn default() -> Pacing { + Pacing { + concurrency: DEFAULT_CONCURRENCY as usize, + io_pressure: Some(DEFAULT_IO_PRESSURE), + } + } +} + +/// The gate. One per controller. +pub struct StartGate { + max: usize, + spacing: Duration, + limit: Option, + max_hold: Duration, + pressure: PressureFn, + state: Mutex, + cv: Condvar, +} + +#[derive(Default)] +struct State { + in_flight: usize, + last: Option, +} + +/// A start in flight: the slot frees when it drops. +pub struct Permit<'a> { + gate: &'a StartGate, +} + +impl Drop for Permit<'_> { + fn drop(&mut self) { + let mut st = self.gate.state.lock().unwrap(); + st.in_flight -= 1; + self.gate.cv.notify_all(); + } +} + +impl std::fmt::Debug for StartGate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StartGate") + .field("max", &self.max) + .field("limit", &self.limit) + .finish_non_exhaustive() + } +} + +impl StartGate { + /// A gate on this host's `/proc/pressure/io`. + pub fn host(p: Pacing) -> StartGate { + StartGate::new( + p.concurrency, + SPACING, + p.io_pressure, + MAX_HOLD, + Box::new(host_io_pressure), + ) + } + + pub fn new( + max: usize, + spacing: Duration, + limit: Option, + max_hold: Duration, + pressure: PressureFn, + ) -> StartGate { + StartGate { + max: max.max(1), + spacing, + limit, + max_hold, + pressure, + state: Mutex::new(State::default()), + cv: Condvar::new(), + } + } + + /// Whether io pressure is at or above the limit now. + fn pressured(&self) -> Option { + let limit = self.limit?; + (self.pressure)().filter(|p| *p >= limit) + } + + /// Wait for a start slot: fewer than the maximum in flight, the spacing + /// since the last start passed, and io pressure under the limit (or + /// held for [`MAX_HOLD`] already). `held` is called once, with the + /// pressure, the first time pressure holds the start. `None` when + /// `stop` was set while waiting. + pub fn acquire(&self, stop: &AtomicBool, held: &mut dyn FnMut(f64)) -> Option> { + let since = Instant::now(); + let mut told = false; + loop { + if stop.load(Ordering::Relaxed) { + return None; + } + // Read outside the lock: it is a file read. + let pressure = if since.elapsed() < self.max_hold { + self.pressured() + } else { + None + }; + if let Some(p) = pressure { + if !told { + told = true; + held(p); + } + std::thread::sleep(POLL); + continue; + } + let mut st = self.state.lock().unwrap(); + let spaced = st.last.is_none_or(|t| t.elapsed() >= self.spacing); + if st.in_flight < self.max && spaced { + st.in_flight += 1; + st.last = Some(Instant::now()); + return Some(Permit { gate: self }); + } + let wait = match st.last { + Some(t) if st.in_flight < self.max => self.spacing.saturating_sub(t.elapsed()), + _ => POLL, + }; + let _ = self + .cv + .wait_timeout(st, wait.clamp(Duration::from_millis(10), POLL)); + } + } + + /// Wait (up to [`MAX_HOLD`]) for io pressure to fall under the limit, + /// without taking a slot. `held` as for [`StartGate::acquire`]. + pub fn wait_pressure(&self, stop: &AtomicBool, held: &mut dyn FnMut(f64)) { + let since = Instant::now(); + let mut told = false; + while since.elapsed() < self.max_hold && !stop.load(Ordering::Relaxed) { + let Some(p) = self.pressured() else { return }; + if !told { + told = true; + held(p); + } + std::thread::sleep(POLL); + } + } +} + +/// The host's io pressure: `full avg10` from `/proc/pressure/io`, the share +/// of the last ten seconds in which every runnable task waited on I/O. +/// `None` without PSI (an old kernel, or not Linux). +pub fn host_io_pressure() -> Option { + parse_full_avg10(&std::fs::read_to_string("/proc/pressure/io").ok()?) +} + +fn parse_full_avg10(s: &str) -> Option { + let line = s.lines().find(|l| l.starts_with("full "))?; + line.split_whitespace() + .find_map(|f| f.strip_prefix("avg10="))? + .parse() + .ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + fn gate(max: usize, spacing: Duration, pressure: Arc>) -> StartGate { + StartGate::new( + max, + spacing, + Some(40.0), + Duration::from_secs(5), + Box::new(move || Some(*pressure.lock().unwrap())), + ) + } + + #[test] + fn parses_psi() { + let s = "some avg10=10.61 avg60=5.87 avg300=1.75 total=2985247473\nfull avg10=9.04 avg60=4.92 avg300=1.44 total=2430035415\n"; + assert_eq!(parse_full_avg10(s), Some(9.04)); + assert_eq!(parse_full_avg10("some avg10=1.00\n"), None); + } + + #[test] + fn bounds_starts_in_flight() { + let g = Arc::new(gate(2, Duration::ZERO, Arc::new(Mutex::new(0.0)))); + let stop = Arc::new(AtomicBool::new(false)); + let (now, peak) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let threads: Vec<_> = (0..6) + .map(|_| { + let (g, stop, now, peak) = (g.clone(), stop.clone(), now.clone(), peak.clone()); + std::thread::spawn(move || { + let _p = g.acquire(&stop, &mut |_| {}).unwrap(); + let n = now.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(n, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(50)); + now.fetch_sub(1, Ordering::SeqCst); + }) + }) + .collect(); + for t in threads { + t.join().unwrap(); + } + assert_eq!(peak.load(Ordering::SeqCst), 2); + } + + #[test] + fn spaces_starts() { + let g = gate(8, Duration::from_millis(100), Arc::new(Mutex::new(0.0))); + let stop = AtomicBool::new(false); + let t = Instant::now(); + for _ in 0..3 { + drop(g.acquire(&stop, &mut |_| {}).unwrap()); + } + assert!( + t.elapsed() >= Duration::from_millis(200), + "{:?}", + t.elapsed() + ); + } + + #[test] + fn holds_while_io_pressure_is_high() { + let p = Arc::new(Mutex::new(90.0)); + let g = Arc::new(gate(4, Duration::ZERO, p.clone())); + let stop = Arc::new(AtomicBool::new(false)); + let held = Arc::new(Mutex::new(Vec::new())); + let t = { + let (g, stop, held) = (g.clone(), stop.clone(), held.clone()); + std::thread::spawn(move || { + let at = Instant::now(); + let ok = g + .acquire(&stop, &mut |p| held.lock().unwrap().push(p)) + .is_some(); + (ok, at.elapsed()) + }) + }; + std::thread::sleep(Duration::from_millis(700)); + *p.lock().unwrap() = 5.0; + let (ok, waited) = t.join().unwrap(); + assert!(ok); + assert!(waited >= Duration::from_millis(700), "{waited:?}"); + // Told once, with the pressure that held it. + assert_eq!(*held.lock().unwrap(), vec![90.0]); + } + + #[test] + fn pressure_holds_at_most_max_hold() { + let g = StartGate::new( + 4, + Duration::ZERO, + Some(40.0), + Duration::from_millis(600), + Box::new(|| Some(99.0)), + ); + let t = Instant::now(); + assert!(g.acquire(&AtomicBool::new(false), &mut |_| {}).is_some()); + assert!(t.elapsed() >= Duration::from_millis(600)); + let t = Instant::now(); + g.wait_pressure(&AtomicBool::new(false), &mut |_| {}); + assert!(t.elapsed() >= Duration::from_millis(600)); + } + + #[test] + fn stop_ends_the_wait() { + let g = StartGate::new( + 1, + Duration::ZERO, + None, + Duration::from_secs(5), + Box::new(|| None), + ); + let stop = AtomicBool::new(false); + let _held = g.acquire(&stop, &mut |_| {}).unwrap(); + stop.store(true, Ordering::Relaxed); + assert!(g.acquire(&stop, &mut |_| {}).is_none()); + } + + #[test] + fn no_limit_never_holds() { + let g = StartGate::new( + 1, + Duration::ZERO, + None, + Duration::from_secs(5), + Box::new(|| Some(100.0)), + ); + let t = Instant::now(); + drop(g.acquire(&AtomicBool::new(false), &mut |_| {}).unwrap()); + assert!(t.elapsed() < Duration::from_millis(400)); + } +} diff --git a/crates/isb-daemon/src/daemon/mod.rs b/crates/isb-daemon/src/daemon/mod.rs index dd1276b7..2fa7a434 100644 --- a/crates/isb-daemon/src/daemon/mod.rs +++ b/crates/isb-daemon/src/daemon/mod.rs @@ -105,6 +105,8 @@ pub struct ServeConfig { pub policy: RemotePolicy, pub state_dir: PathBuf, pub interval: Duration, + /// How the stack controller paces instance starts. + pub start_pacing: crate::stack::pacing::Pacing, /// Where the secrets key is looked for (and generated). pub keys: crate::secrets::KeySources, /// `~/.config/isb/secrets.toml`: break-glass recipients. @@ -345,12 +347,13 @@ pub fn serve(client: Client, cfg: ServeConfig) -> Result<()> { let observer = ingress .clone() .map(|m| m as Arc); - let ctl = Controller::start_with( + let ctl = Controller::start_paced( client.clone(), store, cfg.interval, secrets.clone(), observer, + cfg.start_pacing, )?; if let Some(m) = &ingress { m.start(ctl.clone())?; @@ -1013,18 +1016,6 @@ fn sandbox_extend(d: &Daemon, a: Value, c: &Caller) -> Result { })) } -const OUTPUT_CAP: usize = 256 * 1024; - -fn cap(b: &[u8]) -> (String, bool) { - if b.len() <= OUTPUT_CAP { - return (String::from_utf8_lossy(b).into_owned(), false); - } - ( - String::from_utf8_lossy(&b[b.len() - OUTPUT_CAP..]).into_owned(), - true, - ) -} - fn sandbox_exec(d: &Daemon, a: Value, c: &Caller) -> Result { #[derive(Deserialize)] struct A { @@ -1064,8 +1055,8 @@ fn sandbox_exec(d: &Daemon, a: Value, c: &Caller) -> Result { } r => r?, }; - let (stdout, t1) = cap(&out.stdout); - let (stderr, t2) = cap(&out.stderr); + let (stdout, t1) = sandboxes::cap(&out.stdout); + let (stderr, t2) = sandboxes::cap(&out.stderr); Ok( json!({"exit_code": out.exit_code, "stdout": stdout, "stderr": stderr, "truncated": t1 || t2}), ) diff --git a/crates/isb-daemon/src/daemon/sandboxes.rs b/crates/isb-daemon/src/daemon/sandboxes.rs index 20dca346..9dbd87b3 100644 --- a/crates/isb-daemon/src/daemon/sandboxes.rs +++ b/crates/isb-daemon/src/daemon/sandboxes.rs @@ -32,6 +32,19 @@ const ISBX_IDLE: Duration = Duration::from_secs(30 * 60); /// quota. pub const SANDBOX_ROOT_SIZE: &str = crate::org::DEFAULT_ROOT_SIZE; +const OUTPUT_CAP: usize = 256 * 1024; + +/// The last [`OUTPUT_CAP`] bytes of an exec's output, and whether it was cut. +pub(super) fn cap(b: &[u8]) -> (String, bool) { + if b.len() <= OUTPUT_CAP { + return (String::from_utf8_lossy(b).into_owned(), false); + } + ( + String::from_utf8_lossy(&b[b.len() - OUTPUT_CAP..]).into_owned(), + true, + ) +} + fn now() -> u64 { crate::stack::now_secs() } diff --git a/docs/concepts/stacks.md b/docs/concepts/stacks.md index 042a462b..a24b3d02 100644 --- a/docs/concepts/stacks.md +++ b/docs/concepts/stacks.md @@ -65,7 +65,7 @@ Three layers, each working without the ones above it: nothing else involved. 2. **incus starts the instances with the host** (`boot.autostart`), and the unit restores the app's secrets before every start, so a reboot needs no - isb. + isb ([After a host reboot](#after-a-host-reboot)). 3. **The daemon heals what the guest cannot see:** it runs each `healthcheck`, takes failing replicas out of the load balancer and restarts their app (and replaces the instance after three restarts that @@ -77,6 +77,29 @@ the load balancer: published ports close until it starts again (systemd restarts it within seconds), and it resumes every stack from its state directory without touching healthy instances. +## After a host reboot + +incus starts autostart instances one at a time, highest +`boot.autostart.priority` first, waiting `boot.autostart.delay` seconds after +each. Every replica gets both ([`boot`](../reference/compose.md#boot)): slot +1 of every service at priority 2, the other slots at 1, each 2 s (a VM 10 s) +from the next. A host with a hundred replicas brings them back over a few +minutes, each service serving again before its extra replicas start, instead +of booting them all against the same disk in one second. Dev sandboxes +(`restart: no`) do not come back at all. + +The daemon reconciles those keys on running replicas in place, with a config +change incus applies without a restart; they are not part of the revision, +so a replica made before them is updated, never replaced. + +The daemon's own starts and creates are paced the same way, across every +stack and org: at most `--start-concurrency` (4) at once, a second apart, and +none while the host's io pressure is at or above `--start-io-pressure` (40% +`full avg10`), each waiting at most 5 minutes for it to fall. So `isb serve` +coming up right after a boot, beside replicas incus has not reached yet, +adds to the waves rather than starting everything itself +([Configuration](../reference/configuration.md#stack-controller)). + ## Replicas and names Each service has `deploy.replicas` slots. A slot holds one instance named diff --git a/docs/reference/compose.md b/docs/reference/compose.md index 00a8db74..b15acb98 100644 --- a/docs/reference/compose.md +++ b/docs/reference/compose.md @@ -888,6 +888,8 @@ makes the service long-running: it is meant to outlive `isb up`. `boot.autostart: true`; `unless-stopped` leaves it unset, which in incus restores whatever state the instance had at shutdown. All three set `boot.autorestart: true`, so incus restarts an instance whose init dies. + `no` sets `boot.autostart: false`: a dev sandbox stays down after a host + reboot even if it was running. [`boot`](#boot) changes any of this. - On a system image, `command` is installed as a systemd unit, `isb-.service`, with `Restart=always` (`on-failure` for `on-failure`) after 5 s, its environment (`environment` plus `exec.env`) in a @@ -907,6 +909,35 @@ restart: always command: [node, server.js] ``` +### `boot` + +Whether and in what order incus starts the instance when the host boots: +`autostart` (bool), `priority` (integer, higher first) and `delay` (a +duration, whole seconds: how long incus waits after starting this instance +before it starts the next). They set `boot.autostart`, +`boot.autostart.priority` and `boot.autostart.delay`; `raw_config` still wins +over them. + +Incus starts autostart instances one at a time, highest priority first, but a +start returns as soon as the guest's init runs, so without a delay a reboot +wakes every instance within seconds and they all boot against the same disk +at once. The defaults space them: + +| | `autostart` | `priority` | `delay` | +|---|---|---|---| +| `restart: always` or `on-failure` | `true` | unset (0) | `2s`, a VM `10s` | +| `restart: unless-stopped` | unset: the state at shutdown | unset | unset | +| `restart: no` (the default) | `false` | unset | unset | +| a stack replica | `true` | `2` in slot 1, `1` in the others | `2s`, a VM `10s` | + +So after a reboot every service's first replica comes back before any extra +one. A stack's `boot` is set on its running replicas in place, never by +replacing them: it is not part of the service's revision. + +```yaml +boot: {priority: 5, delay: 10s} # a database the other services wait for +``` + ### `healthcheck` docker compose's healthcheck. `isb up` uses it for `depends_on` with @@ -1074,7 +1105,12 @@ Reconciled per the device rules. `root` is special: its properties are merged over the generated root disk (`{type: disk, path: /, pool: }`), for example to set `size`, and -need no `type`. `root: {size: ...}` alone is allowed for remote callers without +need no `type`. A sandbox that is not a stack replica gets I/O caps on it by +default, `limits.read: 128MiB` and `limits.write: 64MiB` (bytes per second), +so one busy sandbox cannot queue every other instance's I/O behind its own on +a host where they share a disk; set either here to change it (a stack replica +has no caps unless it sets them). Caps far lower than these starve the +guest rather than bound it. `root: {size: ...}` alone is allowed for remote callers without `--allow-raw`: it is how a stack service sizes its root disk against an org's disk limit ([Orgs](../concepts/orgs.md#limits-are-budgets)); in such an org a root disk without one gets 10GiB at creation, outside the spec (a stack diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index bb262cad..57def0b7 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -126,6 +126,18 @@ settings for it. | `--egress-pin` | `ISB_EGRESS_PINS` | none | `NAME=IP[:PORT]`, comma-separated: the proxy connects to `NAME` at that address instead of resolving it, and may reach a private one. The only way a name that resolves to a private address is reachable. | | `--egress-ca` | `ISB_EGRESS_CA` | none | PEM files of certificates the proxy trusts for the hosts it connects to, besides the system's (a private CA) | +### Stack controller + +| Flag | Environment | Default | | +|---|---|---|---| +| `--interval` | none | `5s` | how often each service is reconciled and health-checked | +| `--start-concurrency` | `ISB_START_CONCURRENCY` | `4` | instance starts and creates the controller runs at once, across every stack and org | +| `--start-io-pressure` | `ISB_START_IO_PRESSURE` | `40` | hold new starts while the host's io pressure (`full avg10` in `/proc/pressure/io`, percent) is at or above this, for at most 5 minutes each; `0` turns it off | + +Starts are also spaced a second apart. Together they keep a host boot, or a +daemon restart beside many stopped replicas, from starting everything in the +same second ([Stacks](../concepts/stacks.md#after-a-host-reboot)). + ### Audit and history | Flag | Environment | Default | | diff --git a/scripts/ratchet.txt b/scripts/ratchet.txt index a38bc8ed..e9f5131c 100644 --- a/scripts/ratchet.txt +++ b/scripts/ratchet.txt @@ -9,12 +9,12 @@ crates/isb-apps/src/notify/mod.rs 1263 crates/isb-apps/src/s3.rs 1263 crates/isb-core/src/ingress/caddy.rs 1130 crates/isb-core/src/machine.rs 1097 -crates/isb-core/src/plan.rs 1517 +crates/isb-core/src/plan.rs 1514 crates/isb-core/src/registry/oci.rs 1081 crates/isb-core/src/sandbox.rs 1313 -crates/isb-core/src/spec.rs 1518 -crates/isb-core/src/stack/controller.rs 2325 -crates/isb-daemon/src/daemon/mod.rs 1089 +crates/isb-core/src/spec.rs 1491 +crates/isb-core/src/stack/controller.rs 2316 +crates/isb-daemon/src/daemon/mod.rs 1080 crates/isb-server/src/audit.rs 1153 crates/isb-server/src/auth/http.rs 1040 crates/isb-server/src/auth/mod.rs 1658 diff --git a/sdk/python/src/isb/_spec.py b/sdk/python/src/isb/_spec.py index 8d6b90b0..8c14ea50 100644 --- a/sdk/python/src/isb/_spec.py +++ b/sdk/python/src/isb/_spec.py @@ -25,6 +25,19 @@ UpdateOrder = Literal["stop-first", "start-first"] +class BootSpec(TypedDict, total=False): + """How incus starts an instance with the host. Every field left out takes + its default ([`crate::plan::boot_config`]). + """ + #: Start with the host (`boot.autostart`). + autostart: Optional[BoolOrString] + #: How long incus waits after starting this instance before it starts + #: the next (`boot.autostart.delay`), e.g. `5s`; whole seconds. + delay: Optional[IntOrString] + #: Start order (`boot.autostart.priority`): higher starts first. + priority: Optional[int] + + class Dependency(TypedDict, total=False): condition: DependCondition @@ -268,6 +281,11 @@ class RestartPolicy(TypedDict, total=False): class SandboxSpec(TypedDict, total=False): """Everything about one sandbox: a compose service.""" + #: Whether and in what order incus starts the instance with the host + #: (`boot.autostart`, `.priority`, `.delay`). Default: from `restart`, + #: with a stack replica in the first start wave and every autostarting + #: instance spaced from the next (docs/reference/compose.md#boot). + boot: Optional[BootSpec] #: The sandbox's main command, run by a foreground `isb up` once the #: sandbox is ready, as `user` in `working_dir`. Its output is streamed, #: and `up` stops the sandbox when every command has exited. A list is @@ -505,6 +523,11 @@ class ComposeFile(TypedDict, total=False): class SandboxSpecFields(TypedDict, total=False): """SandboxSpec without `container_name` and `image`: the keyword arguments of Sandbox.create.""" + #: Whether and in what order incus starts the instance with the host + #: (`boot.autostart`, `.priority`, `.delay`). Default: from `restart`, + #: with a stack replica in the first start wave and every autostarting + #: instance spaced from the next (docs/reference/compose.md#boot). + boot: Optional[BootSpec] #: The sandbox's main command, run by a foreground `isb up` once the #: sandbox is ready, as `user` in `working_dir`. Its output is streamed, #: and `up` stops the sandbox when every command has exited. A list is @@ -610,6 +633,7 @@ class SandboxSpecFields(TypedDict, total=False): __all__ = [ "BoolOrString", + "BootSpec", "Command", "ComposeFile", "DependCondition", diff --git a/sdk/typescript/src/spec.ts b/sdk/typescript/src/spec.ts index 5e08c7d6..692cc8bd 100644 --- a/sdk/typescript/src/spec.ts +++ b/sdk/typescript/src/spec.ts @@ -15,6 +15,11 @@ export type BoolOrString = boolean | string; * via the `definition` "OnChange". */ export type OnChange = "none" | "restart" | "roll"; +/** + * This interface was referenced by `ComposeFile`'s JSON-Schema + * via the `definition` "IntOrString". + */ +export type IntOrString = number | string; /** * A command: argv, or a string split the way a shell splits words. * @@ -29,11 +34,6 @@ export type Command = string | Scalar[]; * via the `definition` "Scalar". */ export type Scalar = string | boolean | number; -/** - * This interface was referenced by `ComposeFile`'s JSON-Schema - * via the `definition` "IntOrString". - */ -export type IntOrString = number | string; /** * This interface was referenced by `ComposeFile`'s JSON-Schema * via the `definition` "DependsOnRepr". @@ -352,6 +352,13 @@ export interface SecretDef { * via the `definition` "SandboxSpec". */ export interface SandboxSpec { + /** + * Whether and in what order incus starts the instance with the host + * (`boot.autostart`, `.priority`, `.delay`). Default: from `restart`, + * with a stack replica in the first start wave and every autostarting + * instance spaced from the next (docs/reference/compose.md#boot). + */ + boot?: BootSpec | null; /** * The sandbox's main command, run by a foreground `isb up` once the * sandbox is ready, as `user` in `working_dir`. Its output is streamed, @@ -496,6 +503,28 @@ export interface SandboxSpec { */ working_dir?: string | null; } +/** + * How incus starts an instance with the host. Every field left out takes + * its default ([`crate::plan::boot_config`]). + * + * This interface was referenced by `ComposeFile`'s JSON-Schema + * via the `definition` "BootSpec". + */ +export interface BootSpec { + /** + * Start with the host (`boot.autostart`). + */ + autostart?: BoolOrString | null; + /** + * How long incus waits after starting this instance before it starts + * the next (`boot.autostart.delay`), e.g. `5s`; whole seconds. + */ + delay?: IntOrString | null; + /** + * Start order (`boot.autostart.priority`): higher starts first. + */ + priority?: number | null; +} /** * This interface was referenced by `ComposeFile`'s JSON-Schema * via the `definition` "Dependency". diff --git a/src/bin/isb/serve.rs b/src/bin/isb/serve.rs index bff556be..6e1b113d 100644 --- a/src/bin/isb/serve.rs +++ b/src/bin/isb/serve.rs @@ -23,6 +23,14 @@ pub(crate) struct ServeArgs { /// How often each service is reconciled and health-checked. #[arg(long, value_parser = dur, default_value = "5s")] pub(crate) interval: Duration, + /// Instance starts and creates the stack controller runs at once, + /// across every stack. + #[arg(long, env = "ISB_START_CONCURRENCY", default_value_t = isb::stack::pacing::DEFAULT_CONCURRENCY, value_parser = clap::value_parser!(u32).range(1..))] + pub(crate) start_concurrency: u32, + /// Hold new starts while the host's io pressure (/proc/pressure/io, + /// full avg10) is at or above this percentage; 0 turns it off. + #[arg(long, env = "ISB_START_IO_PRESSURE", default_value_t = isb::stack::pacing::DEFAULT_IO_PRESSURE)] + pub(crate) start_io_pressure: f64, /// Cloudflare Access team domain (https://TEAM.cloudflareaccess.com). #[arg(long, env = "CF_ACCESS_TEAM_DOMAIN")] pub(crate) access_team_domain: Option, @@ -313,6 +321,10 @@ pub(crate) fn serve(ctx: &Ctx, a: ServeArgs) -> Result { }, state_dir: a.state_dir.unwrap_or_else(isb::daemon::default_state_dir), interval: a.interval, + start_pacing: isb::stack::pacing::Pacing { + concurrency: a.start_concurrency as usize, + io_pressure: Some(a.start_io_pressure).filter(|p| *p > 0.0), + }, keys: isb::secrets::KeySources::from_env(), secrets_config: isb::secrets::SecretsConfig::default_path(), auth: isb::auth::AuthConfig { From c47dee4d2b70248f77daa8b579da8e7fb78a2b6d Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 26 Oct 2026 00:35:15 +0000 Subject: [PATCH 2/3] Log a paced start once it has its slot Claude-Session: https://claude.ai/code/session_01XJek6YFHRCRmXsmJ3v1WV4 --- crates/isb-core/src/stack/controller.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/isb-core/src/stack/controller.rs b/crates/isb-core/src/stack/controller.rs index 1ed30785..4dcf27d3 100644 --- a/crates/isb-core/src/stack/controller.rs +++ b/crates/isb-core/src/stack/controller.rs @@ -1643,18 +1643,18 @@ impl Worker { self.message = Some(format!("{}: restart limit reached", i.name)); return Ok(()); } - self.event( - "warn", - Some(&i.name), - &format!("{} is {}; starting it", i.name, i.status), - ); - self.count_restart(&i.name); // Held through the setup below: the guest's boot is the load. let held = &mut |p: f64| self.held(&i.name, p); let Some(p) = inner.gate.acquire(&self.shared.stop, held) else { return Ok(()); }; permit = Some(p); + self.event( + "warn", + Some(&i.name), + &format!("{} is {}; starting it", i.name, i.status), + ); + self.count_restart(&i.name); if let Err(e) = sb.start() { self.log(&format!("cannot start {}: {e}", i.name)); return Ok(()); From d9ed25d6272050869b13e06dcfddd5cde7ec6cee Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 26 Oct 2026 01:46:42 +0000 Subject: [PATCH 3/3] isb 3.3.0 Claude-Session: https://claude.ai/code/session_01XJek6YFHRCRmXsmJ3v1WV4 --- Cargo.lock | 14 +++++++------- Cargo.toml | 14 +++++++------- docs/reference/rpc.md | 4 ++-- sdk/python/pyproject.toml | 2 +- sdk/python/src/isb/__init__.py | 2 +- sdk/python/tests/test_unit.py | 6 +++--- sdk/python/uv.lock | 2 +- sdk/typescript/bun.lock | 4 ++-- sdk/typescript/npm/linux-arm64/package.json | 2 +- sdk/typescript/npm/linux-x64/package.json | 2 +- sdk/typescript/package.json | 6 +++--- 11 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 023933f0..fa646333 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,7 +1329,7 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "isb" -version = "3.2.0" +version = "3.3.0" dependencies = [ "age", "clap", @@ -1351,7 +1351,7 @@ dependencies = [ [[package]] name = "isb-apps" -version = "3.2.0" +version = "3.3.0" dependencies = [ "age", "base64 0.23.1", @@ -1375,7 +1375,7 @@ dependencies = [ [[package]] name = "isb-core" -version = "3.2.0" +version = "3.3.0" dependencies = [ "age", "base64 0.23.1", @@ -1402,7 +1402,7 @@ dependencies = [ [[package]] name = "isb-daemon" -version = "3.2.0" +version = "3.3.0" dependencies = [ "age", "httparse", @@ -1422,7 +1422,7 @@ dependencies = [ [[package]] name = "isb-egress" -version = "3.2.0" +version = "3.3.0" dependencies = [ "base64 0.23.1", "httparse", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "isb-server" -version = "3.2.0" +version = "3.3.0" dependencies = [ "argon2", "base64 0.23.1", @@ -1458,7 +1458,7 @@ dependencies = [ [[package]] name = "isb-tui" -version = "3.2.0" +version = "3.3.0" dependencies = [ "isb-core", "libc", diff --git a/Cargo.toml b/Cargo.toml index c8a83e9a..107db17d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ members = ["crates/*"] default-members = [".", "crates/*"] [workspace.package] -version = "3.2.0" +version = "3.3.0" edition = "2024" rust-version = "1.87" license = "MIT" @@ -62,12 +62,12 @@ repository = "https://github.com/execution-associates/isb" homepage = "https://github.com/execution-associates/isb" [workspace.dependencies] -isb-core = { path = "crates/isb-core", version = "=3.2.0" } -isb-server = { path = "crates/isb-server", version = "=3.2.0" } -isb-apps = { path = "crates/isb-apps", version = "=3.2.0" } -isb-daemon = { path = "crates/isb-daemon", version = "=3.2.0" } -isb-tui = { path = "crates/isb-tui", version = "=3.2.0" } -isb-egress = { path = "crates/isb-egress", version = "=3.2.0" } +isb-core = { path = "crates/isb-core", version = "=3.3.0" } +isb-server = { path = "crates/isb-server", version = "=3.3.0" } +isb-apps = { path = "crates/isb-apps", version = "=3.3.0" } +isb-daemon = { path = "crates/isb-daemon", version = "=3.3.0" } +isb-tui = { path = "crates/isb-tui", version = "=3.3.0" } +isb-egress = { path = "crates/isb-egress", version = "=3.3.0" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/docs/reference/rpc.md b/docs/reference/rpc.md index 42150752..f38eca22 100644 --- a/docs/reference/rpc.md +++ b/docs/reference/rpc.md @@ -19,7 +19,7 @@ it; relative paths resolve against that directory. When it starts, the server writes one hello line: ```json -{"isb": "3.2.0", "protocol": 1} +{"isb": "3.3.0", "protocol": 1} ``` A client checks `protocol` and refuses versions it does not know. @@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present), ## Example session ```text -<- {"isb":"3.2.0","protocol":1} +<- {"isb":"3.3.0","protocol":1} -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}} <- {"id":1,"event":"progress","data":"web: creating from dev-base"} <- {"id":1,"event":"progress","data":"web: starting"} diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 51c61865..4f82a717 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "isb-sdk" -version = "3.2.0" +version = "3.3.0" description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)" readme = "README.md" license = "MIT" diff --git a/sdk/python/src/isb/__init__.py b/sdk/python/src/isb/__init__.py index fdac2f8b..25218985 100644 --- a/sdk/python/src/isb/__init__.py +++ b/sdk/python/src/isb/__init__.py @@ -62,7 +62,7 @@ async def main(): from ._util import Duration from .volumes import Volumes -__version__ = "3.2.0" +__version__ = "3.3.0" __all__ = [ "PROTOCOL", diff --git a/sdk/python/tests/test_unit.py b/sdk/python/tests/test_unit.py index 4ffd8c70..b70a9c1c 100644 --- a/sdk/python/tests/test_unit.py +++ b/sdk/python/tests/test_unit.py @@ -222,7 +222,7 @@ async def test_death_fails_pending_requests(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"3.2.0","protocol":1}' + echo '{"isb":"3.3.0","protocol":1}' read line echo "boom" >&2 exit 3 @@ -243,7 +243,7 @@ async def test_close_fails_pending(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"3.2.0","protocol":1}' + echo '{"isb":"3.3.0","protocol":1}' cat >/dev/null """, ) @@ -261,7 +261,7 @@ async def test_events_and_out_of_order_replies(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"3.2.0","protocol":1}' + echo '{"isb":"3.3.0","protocol":1}' read a read b echo 'not json' diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 2a153183..f4ef4796 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -72,7 +72,7 @@ wheels = [ [[package]] name = "isb-sdk" -version = "3.2.0" +version = "3.3.0" source = { editable = "." } [package.dev-dependencies] diff --git a/sdk/typescript/bun.lock b/sdk/typescript/bun.lock index 18eab9a2..bc6ebefb 100644 --- a/sdk/typescript/bun.lock +++ b/sdk/typescript/bun.lock @@ -12,8 +12,8 @@ "typescript": "^5.9.0", }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "3.2.0", - "@execution-associates/isb-linux-x64": "3.2.0", + "@execution-associates/isb-linux-arm64": "3.3.0", + "@execution-associates/isb-linux-x64": "3.3.0", }, }, }, diff --git a/sdk/typescript/npm/linux-arm64/package.json b/sdk/typescript/npm/linux-arm64/package.json index 0cc2864d..c3383efa 100644 --- a/sdk/typescript/npm/linux-arm64/package.json +++ b/sdk/typescript/npm/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-arm64", - "version": "3.2.0", + "version": "3.3.0", "description": "The static isb binary for Linux arm64 (aarch64-unknown-linux-musl), used by @execution-associates/isb", "license": "MIT", "repository": { diff --git a/sdk/typescript/npm/linux-x64/package.json b/sdk/typescript/npm/linux-x64/package.json index 17004ce4..2a5ee8d4 100644 --- a/sdk/typescript/npm/linux-x64/package.json +++ b/sdk/typescript/npm/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-x64", - "version": "3.2.0", + "version": "3.3.0", "description": "The static isb binary for Linux x64 (x86_64-unknown-linux-musl), used by @execution-associates/isb", "license": "MIT", "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 9df1f2d9..4aebff59 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb", - "version": "3.2.0", + "version": "3.3.0", "description": "TypeScript SDK for isb: declarative incus sandboxes (containers and VMs)", "license": "MIT", "type": "module", @@ -51,8 +51,8 @@ "prepack": "bun run build" }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "3.2.0", - "@execution-associates/isb-linux-x64": "3.2.0" + "@execution-associates/isb-linux-arm64": "3.3.0", + "@execution-associates/isb-linux-x64": "3.3.0" }, "devDependencies": { "@biomejs/biome": "^2.2.0",