From cc1caf646f2c055cada13425c00e415c0e967478 Mon Sep 17 00:00:00 2001 From: Seongjae Date: Sat, 10 Oct 2026 00:02:57 +0900 Subject: [PATCH 1/5] runner: flood the sandbox prepare test with the system's yes prepare_stdout_flood_is_too_large expects "output too large" from a helper that floods its output, before the 2 s prepare deadline. Its stand-in wrote one byte per printf, so under load the deadline could come first: with 16 CPU-bound processes on this 8-CPU Mac it timed out in 7 runs of 50. Writing in large blocks (exec yes x) left 3 timeouts, all in the first runs: macOS checks a new executable file on its first exec, and the stand-in was a script written anew for every run (a never seen script took 0.36-0.62 s to run once without load). The test now spawns /usr/bin/yes, on macOS and Linux, and passed in 50 runs of 50 under the same load. Test code only. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/runner/src/linux_sandbox.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/runner/src/linux_sandbox.rs b/crates/runner/src/linux_sandbox.rs index 6b0e2d7..234b931 100644 --- a/crates/runner/src/linux_sandbox.rs +++ b/crates/runner/src/linux_sandbox.rs @@ -609,9 +609,11 @@ mod tests { #[test] fn prepare_stdout_flood_is_too_large() { - let dir = tempfile::tempdir().unwrap(); - let helper = write_script(dir.path(), "#!/bin/sh\nwhile :; do printf x; done\n"); - let spawned = Command::new(&helper) + // The system's `yes` floods in large writes, from a file executed before. Under load, + // a byte per write, or the first exec of a script written for the test, could take + // long enough to reach the deadline before the limit. + let helper = Path::new("/usr/bin/yes"); + let spawned = Command::new(helper) .arg("prepare") .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -619,7 +621,7 @@ mod tests { .spawn() .unwrap(); let start = Instant::now(); - let err = prepare_from_child(&helper, spawned, b"{}", Duration::from_secs(2), 64 * 1024) + let err = prepare_from_child(helper, spawned, b"{}", Duration::from_secs(2), 64 * 1024) .unwrap_err(); assert_eq!(err.code, ErrorCode::ProcessSpawnFailed); assert!(err.message.contains("output too large"), "{}", err.message); From 58ecec4761d8fbc67ddb603047d81bc0d79fd83f Mon Sep 17 00:00:00 2001 From: Seongjae Date: Sat, 10 Oct 2026 00:11:07 +0900 Subject: [PATCH 2/5] codex-runtime: give the worker binary tests a 10 s deadline for its socket runtime_binary.rs waited for a started worker's socket for 100 tries of 20 ms, about 2 s. Every cargo command that includes the worker places the binary anew, and macOS checks a new executable file on its first exec: on this Mac the first launch takes 2.2-2.8 s, so these tests failed or passed by chance (U4's local runs recorded the failures; with 8 CPU-bound processes running, 2 runs of 10 failed, every test at the wait). wait_connect now waits up to 10 s, and passed in 10 runs of 10 under the same load. Test code only. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/codex-runtime/tests/runtime_binary.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/codex-runtime/tests/runtime_binary.rs b/crates/codex-runtime/tests/runtime_binary.rs index 1e3b73d..073855b 100644 --- a/crates/codex-runtime/tests/runtime_binary.rs +++ b/crates/codex-runtime/tests/runtime_binary.rs @@ -20,14 +20,20 @@ fn runner_socket(dir: &Path) -> PathBuf { dir.join("runner.sock") } +/// The worker's socket, once it accepts. A worker binary never run before can take macOS a few +/// seconds to start: it checks a new executable file on its first exec, and cargo places the +/// binary anew for each test run. async fn wait_connect(socket: &Path) -> UnixStream { - for _ in 0..100 { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { if let Ok(stream) = UnixStream::connect(socket).await { return stream; } + if tokio::time::Instant::now() >= deadline { + panic!("worker socket not ready at {}", socket.display()); + } tokio::time::sleep(Duration::from_millis(20)).await; } - panic!("worker socket not ready at {}", socket.display()); } async fn spawn_worker(socket: &Path) -> tokio::process::Child { From 5787a9264e723460776bb711914a2648b114844b Mon Sep 17 00:00:00 2001 From: Seongjae Date: Sat, 10 Oct 2026 00:16:09 +0900 Subject: [PATCH 3/5] server: wait up to 10 s for a started worker's socket Product code. The gateway waited for the worker it starts (`--runtime-bin`) for 100 tries of 20 ms, about 2 s. macOS checks a new executable file on its first exec, and on this Mac a worker binary never run before took 2.2-2.8 s to accept on its socket, so a gateway's first start after the worker binary was built or installed could fail with "worker socket not ready" (the gateway's own start test failed that way in U4's local runs, and in 1 run of 10 with the worker placed anew and 8 CPU-bound processes running). wait_for_socket now waits up to 10 s, and a worker that exits first still fails the start at once, which a new test checks with /usr/bin/false; the start test then passed in 10 runs of 10 under the same load. Only this function changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/server/src/runtime.rs | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/crates/server/src/runtime.rs b/crates/server/src/runtime.rs index a871f1a..07fbc1b 100644 --- a/crates/server/src/runtime.rs +++ b/crates/server/src/runtime.rs @@ -178,17 +178,25 @@ pub(crate) fn worker_command(bin: &Path, socket: &Path) -> Command { command } +/// How long the gateway waits for a started worker's socket. A worker binary never run before +/// can take macOS a few seconds to start, as it checks a new executable file on its first exec. +const WORKER_READY_BOUND: Duration = Duration::from_secs(10); + +/// The worker's socket once it accepts; an error at once if the worker exits first. async fn wait_for_socket(socket: &Path, wait: &JoinHandle<()>) -> Result { - for _ in 0..100 { + let deadline = tokio::time::Instant::now() + WORKER_READY_BOUND; + loop { if wait.is_finished() { return Err(anyhow!("worker exited before the runner socket was ready")); } match UnixStream::connect(socket).await { Ok(stream) => return Ok(stream), - Err(_) => tokio::time::sleep(Duration::from_millis(20)).await, + Err(_) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(20)).await + } + Err(_) => return Err(anyhow!("worker socket not ready at {}", socket.display())), } } - Err(anyhow!("worker socket not ready at {}", socket.display())) } #[cfg(test)] @@ -498,4 +506,26 @@ mod tests { .expect("process leases released after worker death"); assert!(!dir.exists() || std::fs::read_dir(&dir).map(|d| d.count()).unwrap_or(0) == 0); } + + /// The wait for a worker's socket is long enough for a first exec, but a worker that exits + /// first fails the start at once. + #[tokio::test] + async fn a_worker_that_exits_before_its_socket_fails_at_once() { + let store = Arc::new(Store::memory().unwrap()); + let started = std::time::Instant::now(); + let error = + RuntimeProcess::spawn(Path::new("/usr/bin/false"), None, Arc::new(|_| {}), store) + .await + .err() + .expect("a worker that exits cannot serve"); + assert!( + format!("{error:#}").contains("worker exited before"), + "{error:#}" + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + } } From 7ea7fdb6d3572b3b6c131cb7539a7a411b4e4611 Mon Sep 17 00:00:00 2001 From: Seongjae Date: Sat, 10 Oct 2026 00:40:15 +0900 Subject: [PATCH 4/5] runner: run each launch stand-in once when it is written a_governed_process_times_out_like_any_other failed again in a local macos-core run, the launch reported as not started (HelperEnded) under its 2 s timeout. The launch tests' stand-in helper is a bash script written to a new path for every test, and macOS checks a new executable file on its first exec; while the runner's tests start many new files at once, that first exec can take longer than the bounds of the launch it serves, which run from the spawn. With 16 CPU-bound processes on this 8-CPU Mac the test failed in 5 runs of 20 that way. The stand-in now exits at once when called with --warm, and stand_in() runs it so once when it writes it; the test then passed in 20 runs of 20 under the same load. Test code only. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/runner/src/launch.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/runner/src/launch.rs b/crates/runner/src/launch.rs index d20b843..d38c454 100644 --- a/crates/runner/src/launch.rs +++ b/crates/runner/src/launch.rs @@ -688,10 +688,13 @@ mod tests { /// consumes the permit, then runs `behavior`, which writes transcript lines with `report` /// as the real helper does. It cannot present a grant, so it tests CodeSpace's side only. /// It runs under bash: the carriers' numbers can exceed 9, which dash, Linux's `sh`, cannot - /// redirect. + /// redirect. It is run once when written, with `--warm`: macOS checks a new executable file + /// on its first exec, which can take seconds while other tests start theirs, and that time + /// must not count against the launches it serves. fn stand_in(dir: &Path, name: &str, entry: &str, behavior: &str) -> PathBuf { let script = format!( r#"#!/bin/bash +if [ "$1" = --warm ]; then exit 0; fi while [ "$1" != "--" ]; do case "$1" in --permit-fd) permit="$2" ;; @@ -710,6 +713,11 @@ report() {{ eval "printf '%s\n' \"\$1\" >&$report"; }} let path = dir.join(name); std::fs::write(&path, script).unwrap(); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + let warmed = std::process::Command::new(&path) + .arg("--warm") + .status() + .unwrap(); + assert!(warmed.success(), "{warmed}"); path } From f91380a9cfe91f874bd05e700ed00992113be10b Mon Sep 17 00:00:00 2001 From: Seongjae Date: Sat, 10 Oct 2026 00:48:24 +0900 Subject: [PATCH 5/5] runner: warm the launch stand-ins on macOS only The stand-in is written by this process and, since 7ea7fdb, run at once with --warm. On Linux that buys nothing, as the first-exec check is macOS's, and it widens a window the repository already documents: Linux refuses to execute a file that any process holds open for writing (ETXTBSY), and a child another test thread forks while the file is being written keeps that descriptor until it execs (see write_script in linux_sandbox, which writes its scripts through a short-lived /bin/sh for this reason). The warm run now happens on macOS only; macOS is unchanged. Test code only. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/runner/src/launch.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/runner/src/launch.rs b/crates/runner/src/launch.rs index d38c454..c5d9b56 100644 --- a/crates/runner/src/launch.rs +++ b/crates/runner/src/launch.rs @@ -688,9 +688,11 @@ mod tests { /// consumes the permit, then runs `behavior`, which writes transcript lines with `report` /// as the real helper does. It cannot present a grant, so it tests CodeSpace's side only. /// It runs under bash: the carriers' numbers can exceed 9, which dash, Linux's `sh`, cannot - /// redirect. It is run once when written, with `--warm`: macOS checks a new executable file - /// on its first exec, which can take seconds while other tests start theirs, and that time - /// must not count against the launches it serves. + /// redirect. On macOS it is run once when written, with `--warm`: macOS checks a new + /// executable file on its first exec, which can take seconds while other tests start theirs, + /// and that time must not count against the launches it serves. Not on Linux, where the + /// check does not exist and running a file this process has just written can fail with + /// `ETXTBSY` (see `write_script` in `linux_sandbox`). fn stand_in(dir: &Path, name: &str, entry: &str, behavior: &str) -> PathBuf { let script = format!( r#"#!/bin/bash @@ -713,11 +715,13 @@ report() {{ eval "printf '%s\n' \"\$1\" >&$report"; }} let path = dir.join(name); std::fs::write(&path, script).unwrap(); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); - let warmed = std::process::Command::new(&path) - .arg("--warm") - .status() - .unwrap(); - assert!(warmed.success(), "{warmed}"); + if cfg!(target_os = "macos") { + let warmed = std::process::Command::new(&path) + .arg("--warm") + .status() + .unwrap(); + assert!(warmed.success(), "{warmed}"); + } path }