fix(sandbox): negotiate the Docker Engine API version (#457)

api/README.md

@@ -77,9 +77,11 @@ place, avoiding stale macOS executable-page signature state. `CARGO_TARGET_DIR`
 must stay inside this worktree; `LAYERFS_NATIVE_OUTPUT` can select a separate
 platform artifact. Keep Linux addons separate from the host addon.
 
-At runtime, the Docker Engine server selected by `LAYERFS_DOCKER` must accept
-Engine API v1.54 requests. The SDK always uses explicit `/v1.54/...` paths and
-does not negotiate from the server's maximum API version; see the [agent settings](docs/agent.md#settings).
+At runtime, the Docker Engine server selected by `LAYERFS_DOCKER` must speak
+Engine API v1.48 (Docker 28.0) or newer. Before its first request the SDK reads
+the server's version once (`GET /version`) and uses `/v1.<minor>/...` paths at
+that version, up to v1.54; an older server is refused up front, naming both
+versions. See the [agent settings](docs/agent.md#settings).
 
 New Store schema versions are 16–18 (pack layouts) and 22–24 (with acquisition),
 with history identity format 3. Old Disposable Stores are rejected without

api/docs/agent.md

@@ -149,9 +149,15 @@ Keep each agent's operations on a Workspace sequential.
 | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. |
 | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. |
 
-The Docker Engine server selected by `LAYERFS_DOCKER` must accept Engine API
-v1.54 requests. The SDK always uses explicit `/v1.54/...` paths and does not
-negotiate from the server's maximum API version. Check the server's API range
+The Docker Engine server selected by `LAYERFS_DOCKER` must speak Engine API
+v1.48 (Docker 28.0) or newer. Before its first request the SDK reads the
+server's API version once with an unversioned `GET /version` and uses
+`/v1.<minor>/...` paths at that version, up to v1.54 (a newer server is
+addressed as v1.54). A server older than v1.48 is refused before any other
+request, with an error naming the server's version and the supported range:
+earlier Engines fail to upload the daemon. If `GET /version` cannot be read,
+the SDK uses v1.54, and the server's own refusal of it, if any, is the error. A
+failed request is never resent at another version. Check the server's API range
 with `docker version`.
 
 The SDK and the `layerfs` command read only these `LAYERFS_` names. An

core/docs/architecture/71-ordinary-engine-runtime-foundation.md

@@ -7,7 +7,7 @@
 > and command-client provenance are deferred by owner direction. Native FUSE
 > remains R2; ordinary execution adds no filesystem registration.
 
-The active replacement Sandbox provides a concrete Docker Engine API1.54 Unix
+The active replacement Sandbox provides a concrete Docker Engine API1.48 to 1.54 Unix
 adapter for ordinary command creation, non-TTY launch, independent stdin/output
 owners and coherent runtime inspection. It uses exact full acknowledged container
 and Exec IDs, not names, prefixes, host/VM PID guesses or filesystem registration.

engine/crates/layerfs-sandbox/src/backend/docker/archive.rs

@@ -71,7 +71,7 @@ impl Sandbox {
             }
             self.upload_attempted = true;
             let path = format!(
-                "/v1.54/containers/{}/archive?path=/&noOverwriteDirNonDir=true&copyUIDGID=true",
+                "/containers/{}/archive?path=/&noOverwriteDirNonDir=true&copyUIDGID=true",
                 self.id
             );
             let mut response = self

engine/crates/layerfs-sandbox/src/backend/docker/container.rs

@@ -43,8 +43,7 @@ impl SandboxRequest {
 impl Docker {
     /// One read of an existing exact plain local volume; it has no create effect.
     fn plain_volume(&self, name: &str) -> Result<(), Box<WireFailure>> {
-        let mut response =
-            self.exchange("GET", &format!("/v1.54/volumes/{name}"), false, |_| Ok(()))?;
+        let mut response = self.exchange("GET", &format!("/volumes/{name}"), false, |_| Ok(()))?;
         if response.status != 200 {
             return Err(response.fail(RuntimeError::Http(response.status), None));
         }
@@ -103,8 +102,8 @@ impl Docker {
             phase = CreatePhase::Create;
             // `naming` admitted only characters that need no URL escape.
             let path = match &self.container_name {
-                Some(name) => format!("/v1.54/containers/create?name={name}"),
-                None => "/v1.54/containers/create".to_owned(),
+                Some(name) => format!("/containers/create?name={name}"),
+                None => "/containers/create".to_owned(),
             };
             let mut response = self.exchange("POST", &path, false, |w| request.encode(w))?;
             if response.status != 201 {
@@ -181,7 +180,7 @@ impl Sandbox {
             return Err(self.invalid("original container Start admission"));
         }
         self.start_attempted = true;
-        self.empty("POST", &format!("/v1.54/containers/{}/start", self.id), 204)?;
+        self.empty("POST", &format!("/containers/{}/start", self.id), 204)?;
         self.started = true;
         Ok(())
     }
@@ -193,7 +192,7 @@ impl Sandbox {
             return Err(self.invalid("original container Stop admission"));
         }
         self.stop_attempted = true;
-        let path = format!("/v1.54/containers/{}/stop?t={seconds}", self.id);
+        let path = format!("/containers/{}/stop?t={seconds}", self.id);
         self.empty("POST", &path, 204)
             .or_else(|failure| match failure.cause {
                 RuntimeError::Http(304) => Ok(()),
@@ -210,7 +209,7 @@ impl Sandbox {
         self.delete_attempted = true;
         self.empty(
             "DELETE",
-            &format!("/v1.54/containers/{}?force=false&v=false", self.id),
+            &format!("/containers/{}?force=false&v=false", self.id),
             204,
         )?;
         self.deleted = true;

engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs

@@ -65,7 +65,7 @@ impl Sandbox {
             .docker
             .exchange(
                 "GET",
-                &format!("/v1.54/containers/{}/json", self.id),
+                &format!("/containers/{}/json", self.id),
                 false,
                 |_| Ok(()),
             )

engine/crates/layerfs-sandbox/src/backend/docker/http.rs

@@ -10,7 +10,7 @@ use std::{
     io::{self, BufReader, Read, Write},
     net::Shutdown,
     path::{Path, PathBuf},
-    sync::{Arc, Mutex},
+    sync::{Arc, Mutex, OnceLock},
     time::{Duration, Instant},
 };
 /// How long an idle kept Engine connection may wait for its next request.
@@ -28,6 +28,9 @@ pub struct Docker {
     /// Exec create and inspect (#352): each takes it, or connects when it is
     /// absent, stale or in use, and returns it after reading its whole body.
     pub(super) kept: Kept,
+    /// The Engine API minor version every request of this client and its
+    /// clones uses, set by the first request (see `version.rs`).
+    pub(super) minor: Arc<OnceLock<u32>>,
 }
 #[derive(Clone, Default)]
 pub(super) struct Kept(Arc<Mutex<Option<(Socket, Instant)>>>);
@@ -75,6 +78,7 @@ impl Docker {
             deadline: None,
             container_name: None,
             kept: Kept::default(),
+            minor: Arc::default(),
         })
     }
     /// Names the containers this client creates `name`: an ASCII letter or
@@ -142,6 +146,18 @@ impl Docker {
         media: &str,
         length: u64,
         encode: impl FnOnce(&mut dyn Write) -> io::Result<()>,
+    ) -> Result<Response, Box<WireFailure>> {
+        let path = format!("/v1.{}{path}", self.minor()?);
+        self.transmit(method, &path, mode, media, length, encode)
+    }
+    pub(super) fn transmit(
+        &self,
+        method: &str,
+        path: &str,
+        mode: Mode,
+        media: &str,
+        length: u64,
+        encode: impl FnOnce(&mut dyn Write) -> io::Result<()>,
     ) -> Result<Response, Box<WireFailure>> {
         let upgrade = matches!(mode, Mode::Upgrade);
         let reused = match mode {
@@ -235,7 +251,7 @@ impl Docker {
 }
 /// The connection header a request sends.
 #[derive(Clone, Copy)]
-enum Mode {
+pub(super) enum Mode {
     Close,
     Upgrade,
     Keep,

engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs

@@ -12,7 +12,7 @@ impl Docker {
         exec: ExecId,
     ) -> Result<ExecInspection, Box<WireFailure>> {
         let mut response = self
-            .exchange_kept("GET", &format!("/v1.54/exec/{exec}/json"), |_| Ok(()))
+            .exchange_kept("GET", &format!("/exec/{exec}/json"), |_| Ok(()))
             .map_err(|mut failure| {
                 failure.requested_container = Some(container);
                 failure.requested_exec = Some(exec);

engine/crates/layerfs-sandbox/src/backend/docker/listener.rs

@@ -47,7 +47,7 @@ impl Sandbox {
             let mut r = pending.exchange(
                     "GET",
                     &format!(
-                        "/v1.54/containers/{}/logs?stdout=true&stderr=true&follow=true&tail=all&timestamps=false",
+                        "/containers/{}/logs?stdout=true&stderr=true&follow=true&tail=all&timestamps=false",
                         self.id
                     ),
                     false,
@@ -145,7 +145,7 @@ impl Sandbox {
             .exchange(
                 "GET",
                 &format!(
-                    "/v1.54/containers/{}/logs?stdout=true&stderr=true&follow=false&tail={lines}&timestamps=false",
+                    "/containers/{}/logs?stdout=true&stderr=true&follow=false&tail={lines}&timestamps=false",
                     self.id
                 ),
                 false,

engine/crates/layerfs-sandbox/src/backend/docker/mod.rs

@@ -1,4 +1,4 @@
-//! Local Docker Engine API1.54, with exact IDs and no command replay.
+//! Local Docker Engine API1.48 to 1.54, negotiated once, with exact IDs and no command replay.
 mod archive;
 mod body;
 mod container;
@@ -15,6 +15,7 @@ mod socket;
 mod streams;
 mod strings;
 mod topology;
+mod version;
 pub use archive::{UploadFailure, UploadWork};
 pub use container_types::{
     CreatePhase, Sandbox, SandboxCreateFailure, SandboxDisposition, SandboxRequest,

engine/crates/layerfs-sandbox/src/backend/docker/request.rs

@@ -119,11 +119,10 @@ impl Docker {
                     error_body: None,
                 })
             })?;
-            let mut response = self.exchange_kept(
-                "POST",
-                &format!("/v1.54/containers/{container}/exec"),
-                |w| request.encode(w),
-            )?;
+            let mut response =
+                self.exchange_kept("POST", &format!("/containers/{container}/exec"), |w| {
+                    request.encode(w)
+                })?;
             if response.status != 201 {
                 return Err(response.fail(RuntimeError::Http(response.status), None));
             }

engine/crates/layerfs-sandbox/src/backend/docker/streams.rs

@@ -97,12 +97,11 @@ impl ExecCreated {
             }));
         }
         self.start_attempted = true;
-        let result = self.docker.exchange(
-            "POST",
-            &format!("/v1.54/exec/{}/start", self.exec),
-            true,
-            |w| w.write_all(b"{\"Detach\":false,\"Tty\":false}"),
-        );
+        let result =
+            self.docker
+                .exchange("POST", &format!("/exec/{}/start", self.exec), true, |w| {
+                    w.write_all(b"{\"Detach\":false,\"Tty\":false}")
+                });
         match result {
             Ok(response) if response.status == 101 => Ok(Attached {
                 created: self,

engine/crates/layerfs-sandbox/src/backend/docker/version.rs

@@ -0,0 +1,87 @@
+//! The Engine API version a client speaks: the daemon's, up to the newest this client targets.
+use super::{
+    http::{failure, Mode},
+    json::{once, Json},
+    Docker,
+};
+use crate::{ErrorDiagnostic, RuntimeError, WireFailure};
+/// The oldest Engine API minor version (1.N) this client speaks: Engine 28.0.
+/// Before it, uploading the daemon fails (`getent unable to find entry "0:0"`:
+/// the archive's `copyUIDGID` cannot resolve the container's `0:0` user).
+const OLDEST: u32 = 48;
+/// The newest Engine API minor version this client speaks.
+const NEWEST: u32 = 54;
+impl Docker {
+    /// The minor version every request of this client and its clones uses. The
+    /// first request reads the daemon's with one unversioned `GET /version`; a
+    /// daemon older than [`OLDEST`] is refused before the request is sent. A
+    /// daemon that does not answer it readably gets [`NEWEST`], as before
+    /// negotiation, and its own refusal of that version reaches the caller.
+    pub(super) fn minor(&self) -> Result<u32, Box<WireFailure>> {
+        if let Some(minor) = self.minor.get() {
+            return Ok(*minor);
+        }
+        let minor = match self.daemon_minor() {
+            Some(daemon) if daemon < OLDEST => return Err(refusal(daemon)),
+            Some(daemon) => daemon.min(NEWEST),
+            None => NEWEST,
+        };
+        Ok(*self.minor.get_or_init(|| minor))
+    }
+    /// One read of the daemon's newest API minor version; never resent or retried.
+    fn daemon_minor(&self) -> Option<u32> {
+        let mut response = self
+            .transmit(
+                "GET",
+                "/version",
+                Mode::Close,
+                "application/json",
+                0,
+                |_| Ok(()),
+            )
+            .ok()?;
+        if response.status != 200 {
+            return None;
+        }
+        let mut bits = 0;
+        let mut minor = None;
+        let mut j = Json::new(&mut response.body);
+        j.object(|j, key| {
+            if key.equals("ApiVersion") {
+                once(&mut bits, 1)?;
+                let version = j.string(true)?;
+                minor = version
+                    .text()
+                    .strip_prefix("1.")
+                    .and_then(|minor| minor.parse().ok())
+                    .filter(|_| !version.overflow);
+            } else {
+                j.skip(1)?;
+            }
+            Ok(())
+        })
+        .ok()?;
+        j.finish().ok()?;
+        minor
+    }
+}
+fn refusal(daemon: u32) -> Box<WireFailure> {
+    let message = format!(
+        "Engine API 1.{daemon} is older than the 1.{OLDEST} to 1.{NEWEST} this client speaks"
+    )
+    .into_bytes();
+    let mut refused = failure(
+        false,
+        0,
+        None,
+        RuntimeError::Unsupported("Engine API version"),
+        None,
+    );
+    refused.error_body = Some(ErrorDiagnostic {
+        observed_bytes: message.len() as u64,
+        prefix: message,
+        complete: true,
+        read_error: None,
+    });
+    refused
+}

engine/crates/layerfs-sandbox/tests/engine_keepalive.rs

@@ -70,7 +70,13 @@ struct Engine {
     worker: Option<JoinHandle<Vec<Vec<String>>>>,
 }
 impl Engine {
-    fn new(script: Vec<Connection>) -> (Docker, Self) {
+    /// The client's first connection asks the version, so the script follows
+    /// an Engine that answers it; `finish` leaves that connection out.
+    fn new(mut script: Vec<Connection>) -> (Docker, Self) {
+        script.insert(
+            0,
+            (vec![response(200, r#"{"ApiVersion":"1.54"}"#, true)], None),
+        );
         let directory = std::env::temp_dir().join(format!(
             "layerfs-keepalive-{}-{}",
             std::process::id(),
@@ -118,7 +124,9 @@ impl Engine {
     }
     /// The request heads each connection received.
     fn finish(mut self) -> Vec<Vec<String>> {
-        self.worker.take().unwrap().join().unwrap()
+        let mut connections = self.worker.take().unwrap().join().unwrap();
+        connections.remove(0);
+        connections
     }
 }
 impl Drop for Engine {