From c7592e89699991704065efadc58c9e6146c45db6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 11:50:25 +0000 Subject: [PATCH 1/2] fix: exit 75 only when the installed binary is replaced The install transaction renames the staged binary onto the installed path. On Linux the running path then ends in " (deleted)", and the watch exits 75 so a supervisor can start the gateway again. A move, an unreadable path, or a failed re-hash waits for one more sample. A second sample with the same failure exits 1 and names the recovery command. The failure text does not contain "changed on disk". Co-authored-by: Sash --- docs/LAUNCHER_CONTRACT.md | 19 +- .../crates/elastos-server/src/host_lock.rs | 226 ++++++++++++++++-- 2 files changed, 213 insertions(+), 32 deletions(-) diff --git a/docs/LAUNCHER_CONTRACT.md b/docs/LAUNCHER_CONTRACT.md index b079a7c1c..11f69dc45 100644 --- a/docs/LAUNCHER_CONTRACT.md +++ b/docs/LAUNCHER_CONTRACT.md @@ -56,19 +56,26 @@ The gateway handles `SIGTERM` and `SIGINT` by signalling its process group, waiting, and then signalling that group again. Exit 75 comes from `std::process::exit` in the binary watch and skips that handler. -While the gateway is running, the binary watch exits 75 when the installed -binary is replaced, when its path changes, or when the watch cannot read it. -The install transaction replaces that binary by renaming the staged file onto -the installed path. On Linux that running path then ends in ` (deleted)`. +While the gateway is running, a replaced installed binary or a changed binary +hash makes that process exit 75. The install transaction replaces the binary +by renaming the staged file onto the installed path. On Linux that running +path then ends in ` (deleted)`, and the watch exits 75. A supervisor starts +`elastos gateway --addr localhost:8090` again. + +A moved path, a path that cannot be read, and a file that could not be +re-verified wait for one more sample. A replacement on that next sample exits +75. A second sample with the same failure exits 1. The message names +`elastos gateway --addr localhost:8090` as the recovery command. A supervisor +leaves that process stopped. ## Exit codes | Code | Meaning | | --- | --- | | 0 | The command finished. | -| 1 | The command failed. A second host for the same data directory is one case. | +| 1 | The command failed. This includes a second host for the same data directory and a binary watch that confirmed a move, an unreadable path, or a failed re-verify. A supervisor leaves the gateway stopped. | | 2 | The `elastos` command line is invalid. | -| 75 | The running gateway's binary watch exited. | +| 75 | The running gateway exited because the installed binary was replaced or its hash changed. A supervisor starts the gateway again. | A supervisor uses these exit codes. It does not parse English sentences from stdout. Exit 0 covers more than one successful update outcome. A later issue diff --git a/elastos/crates/elastos-server/src/host_lock.rs b/elastos/crates/elastos-server/src/host_lock.rs index 40bb32c05..c9839702b 100644 --- a/elastos/crates/elastos-server/src/host_lock.rs +++ b/elastos/crates/elastos-server/src/host_lock.rs @@ -214,6 +214,18 @@ struct BinarySupersessionWatch { current_exe_path: PathBuf, current_exe_stamp: BinaryFileStamp, current_binary_sha256: String, + /// A move, an unreadable path, or a failed re-hash waits for one more sample. + /// The install rename onto the running path is a replacement and does not wait. + pending_confirm: bool, +} + +const EXIT_LAUNCHER_RESTART: i32 = 75; +const EXIT_LAUNCHER_STAYS_STOPPED: i32 = 1; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Supersession { + reason: String, + exit_code: i32, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -251,9 +263,9 @@ pub fn spawn_installed_binary_supersession_watch(data_dir: &Path, role: &str) { interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { interval.tick().await; - if let Some(reason) = watch.superseded_reason() { - eprintln!("[{}] {}", role, reason); - std::process::exit(75); + if let Some(event) = watch.superseded_reason() { + eprintln!("[{}] {}", role, event.reason); + std::process::exit(event.exit_code); } } }); @@ -272,19 +284,45 @@ impl BinarySupersessionWatch { current_exe_path: current_exe, current_exe_stamp: snapshot.stamp, current_binary_sha256: snapshot.sha256, + pending_confirm: false, }) } - fn superseded_reason(&mut self) -> Option { + fn superseded_reason(&mut self) -> Option { self.superseded_reason_with(&RealBinarySnapshotReader) } - fn superseded_reason_with(&mut self, reader: &impl BinarySnapshotReader) -> Option { + fn restart(&mut self, reason: String) -> Option { + self.pending_confirm = false; + Some(Supersession { + reason, + exit_code: EXIT_LAUNCHER_RESTART, + }) + } + + /// A single failed sample can be the middle of an install rename. + /// The next sample decides. A replacement on that sample restarts. + fn confirm_stop(&mut self, reason: String) -> Option { + if !self.pending_confirm { + self.pending_confirm = true; + return None; + } + self.pending_confirm = false; + Some(Supersession { + reason, + exit_code: EXIT_LAUNCHER_STAYS_STOPPED, + }) + } + + fn superseded_reason_with( + &mut self, + reader: &impl BinarySnapshotReader, + ) -> Option { let current_exe_raw = match reader.current_exe_path_raw() { Ok(path) => path, Err(err) => { - return Some(format!( - "host executable path could not be read for {}: {err:#}. Exiting stale host.", + return self.confirm_stop(format!( + "host executable path could not be read for {}: {err:#}. Start `elastos gateway --addr localhost:8090` after the path can be read.", self.current_exe_path.display() )); } @@ -293,15 +331,15 @@ impl BinarySupersessionWatch { let current_exe = normalize_deleted_exe_path(¤t_exe_raw); if raw_text.ends_with(" (deleted)") { - return Some(format!( + return self.restart(format!( "host executable {} was replaced on disk. Exiting stale host.", self.current_exe_path.display() )); } if current_exe != self.current_exe_path { - return Some(format!( - "host executable moved from {} to {}. Exiting stale host.", + return self.confirm_stop(format!( + "host executable moved from {} to {}. Start `elastos gateway --addr localhost:8090` if this Home should run.", self.current_exe_path.display(), current_exe.display() )); @@ -310,34 +348,36 @@ impl BinarySupersessionWatch { let current_stamp = match reader.metadata_stamp(&self.current_exe_path) { Ok(stamp) => stamp, Err(err) => { - return Some(format!( - "host executable {} could not be verified on disk: {err:#}. Exiting stale host.", + return self.confirm_stop(format!( + "host executable {} could not be verified on disk: {err:#}. Start `elastos gateway --addr localhost:8090` after the file can be read.", self.current_exe_path.display() )); } }; if current_stamp == self.current_exe_stamp { + self.pending_confirm = false; return None; } let current_snapshot = match reader.digest_snapshot(&self.current_exe_path) { Ok(snapshot) => snapshot, Err(err) => { - return Some(format!( - "host executable {} changed on disk and could not be re-verified: {err:#}. Exiting stale host.", + return self.confirm_stop(format!( + "host executable {} could not be re-verified: {err:#}. Start `elastos gateway --addr localhost:8090` after the file can be read.", self.current_exe_path.display() )); } }; if current_snapshot.sha256 != self.current_binary_sha256 { - return Some(format!( + return self.restart(format!( "host binary {} changed on disk. Exiting stale host so the newer binary can take over.", self.current_exe_path.display() )); } self.current_exe_stamp = current_snapshot.stamp; + self.pending_confirm = false; None } } @@ -502,6 +542,8 @@ mod tests { struct TestBinarySnapshotReaderState { current_exe_raw: RefCell, digest_reads: Cell, + metadata_failures: Cell, + digest_failures: Cell, } impl TestBinarySnapshotReader { @@ -510,10 +552,20 @@ mod tests { state: Rc::new(TestBinarySnapshotReaderState { current_exe_raw: RefCell::new(path), digest_reads: Cell::new(0), + metadata_failures: Cell::new(0), + digest_failures: Cell::new(0), }), } } + fn fail_metadata_once(&self) { + self.state.metadata_failures.set(1); + } + + fn fail_digest(&self, times: usize) { + self.state.digest_failures.set(times); + } + fn set_current_exe_raw(&self, path: PathBuf) { *self.state.current_exe_raw.borrow_mut() = path; } @@ -529,10 +581,20 @@ mod tests { } fn metadata_stamp(&self, path: &Path) -> anyhow::Result { + let remaining = self.state.metadata_failures.get(); + if remaining > 0 { + self.state.metadata_failures.set(remaining - 1); + anyhow::bail!("metadata unavailable during replace"); + } metadata_stamp(path) } fn digest_snapshot(&self, path: &Path) -> anyhow::Result { + let remaining = self.state.digest_failures.get(); + if remaining > 0 { + self.state.digest_failures.set(remaining - 1); + anyhow::bail!("digest unavailable during replace"); + } self.state .digest_reads .set(self.state.digest_reads.get().saturating_add(1)); @@ -686,8 +748,9 @@ mod tests { let initial_reads = reader.digest_reads(); write_file(&exe, b"changed content with a different length"); - let reason = watch.superseded_reason_with(&reader).expect("superseded"); - assert!(reason.contains("changed on disk"), "{reason}"); + let event = watch.superseded_reason_with(&reader).expect("superseded"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_RESTART); + assert!(event.reason.contains("changed on disk"), "{}", event.reason); assert_eq!(reader.digest_reads(), initial_reads + 1); } @@ -703,8 +766,9 @@ mod tests { write_file(&replacement, b"modified"); fs::rename(&replacement, &exe).expect("replace file"); - let reason = watch.superseded_reason_with(&reader).expect("superseded"); - assert!(reason.contains("changed on disk"), "{reason}"); + let event = watch.superseded_reason_with(&reader).expect("superseded"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_RESTART); + assert!(event.reason.contains("changed on disk"), "{}", event.reason); assert_eq!(reader.digest_reads(), initial_reads + 1); } @@ -718,9 +782,15 @@ mod tests { let moved = temp.path().join("elastos-new"); reader.set_current_exe_raw(moved.clone()); - let reason = watch.superseded_reason_with(&reader).expect("superseded"); - assert!(reason.contains("moved"), "{reason}"); - assert!(reason.contains(&moved.display().to_string()), "{reason}"); + assert!(watch.superseded_reason_with(&reader).is_none()); + let event = watch.superseded_reason_with(&reader).expect("confirmed"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_STAYS_STOPPED); + assert!(event.reason.contains("moved"), "{}", event.reason); + assert!( + event.reason.contains(&moved.display().to_string()), + "{}", + event.reason + ); } #[test] @@ -732,8 +802,14 @@ mod tests { let (mut watch, reader) = test_watch(&exe); fs::remove_file(&exe).expect("remove file"); - let reason = watch.superseded_reason_with(&reader).expect("superseded"); - assert!(reason.contains("could not be verified on disk"), "{reason}"); + assert!(watch.superseded_reason_with(&reader).is_none()); + let event = watch.superseded_reason_with(&reader).expect("confirmed"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_STAYS_STOPPED); + assert!( + event.reason.contains("could not be verified on disk"), + "{}", + event.reason + ); } #[test] @@ -745,7 +821,105 @@ mod tests { let (mut watch, reader) = test_watch(&exe); reader.set_current_exe_raw(PathBuf::from(format!("{} (deleted)", exe.display()))); - let reason = watch.superseded_reason_with(&reader).expect("superseded"); - assert!(reason.contains("replaced on disk"), "{reason}"); + let event = watch.superseded_reason_with(&reader).expect("superseded"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_RESTART); + assert!( + event.reason.contains("replaced on disk"), + "{}", + event.reason + ); + } + + #[test] + fn supersession_watch_retries_a_metadata_miss_then_restarts_for_new_bytes() { + let temp = tempfile::tempdir().expect("tempdir"); + let exe = temp.path().join("elastos"); + write_file(&exe, b"original"); + let (mut watch, reader) = test_watch(&exe); + reader.fail_metadata_once(); + assert!(watch.superseded_reason_with(&reader).is_none()); + write_file(&exe, b"replacement bytes"); + let event = watch.superseded_reason_with(&reader).expect("replacement"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_RESTART); + assert!(event.reason.contains("changed on disk"), "{}", event.reason); + } + + #[test] + fn supersession_watch_confirms_a_failed_rehash_without_a_restart_word() { + let temp = tempfile::tempdir().expect("tempdir"); + let exe = temp.path().join("elastos"); + write_file(&exe, b"original"); + let (mut watch, reader) = test_watch(&exe); + let replacement = temp.path().join("replacement"); + write_file(&replacement, b"different"); + fs::rename(&replacement, &exe).expect("replace file"); + reader.fail_digest(2); + + assert!(watch.superseded_reason_with(&reader).is_none()); + let event = watch.superseded_reason_with(&reader).expect("confirmed"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_STAYS_STOPPED); + assert!( + event.reason.contains("could not be re-verified"), + "{}", + event.reason + ); + assert!( + !event.reason.contains("changed on disk"), + "{}", + event.reason + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn install_rename_onto_running_binary_exits_75() { + if std::env::var("ELASTOS_SUPERSESSION_CHILD").as_deref() == Ok("1") { + let ready = std::env::var("ELASTOS_SUPERSESSION_READY").expect("ready path"); + let mut watch = BinarySupersessionWatch::from_data_dir(std::env::temp_dir().as_path()) + .expect("child watch"); + fs::write(&ready, b"ready").expect("ready"); + for _ in 0..200 { + if let Some(event) = watch.superseded_reason() { + std::process::exit(event.exit_code); + } + std::thread::sleep(Duration::from_millis(20)); + } + std::process::exit(2); + } + + let temp = tempfile::tempdir().expect("tempdir"); + let exe = temp.path().join("elastos"); + fs::copy(std::env::current_exe().expect("test binary"), &exe).expect("copy binary"); + let ready = temp.path().join("ready"); + let mut child = std::process::Command::new(&exe) + .arg("--exact") + .arg("host_lock::tests::install_rename_onto_running_binary_exits_75") + .env("ELASTOS_SUPERSESSION_CHILD", "1") + .env("ELASTOS_SUPERSESSION_READY", &ready) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn child"); + let deadline = Instant::now() + Duration::from_secs(60); + while !ready.exists() { + if Instant::now() > deadline { + let _ = child.kill(); + panic!("child did not become ready"); + } + if let Some(status) = child.try_wait().expect("poll child") { + let mut err = String::new(); + if let Some(stderr) = child.stderr.as_mut() { + use std::io::Read as _; + let _ = stderr.read_to_string(&mut err); + } + panic!("child exited before ready: {status}: {err}"); + } + std::thread::sleep(Duration::from_millis(20)); + } + let staged = temp.path().join("staged"); + write_file(&staged, b"staged replacement"); + fs::rename(&staged, &exe).expect("install rename"); + let status = child.wait().expect("child exit"); + assert_eq!(status.code(), Some(EXIT_LAUNCHER_RESTART)); } } From 2efbfbf3da2f3051f2c3a0ed800452f3704d94ce Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 13:44:07 +0000 Subject: [PATCH 2/2] fix: confirm a binary watch stop only on the same failure twice The contract says a second sample with the same failure exits 1, but the watch kept one shared pending flag, so an unreadable file followed by a move, or moves to two different paths, also exited 1. The watch now records which failure it saw (and, for a move, the new path). Only the same failure on the next sample stops; a different one waits again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- docs/LAUNCHER_CONTRACT.md | 4 +- .../crates/elastos-server/src/host_lock.rs | 77 +++++++++++++++---- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docs/LAUNCHER_CONTRACT.md b/docs/LAUNCHER_CONTRACT.md index 832a923a5..61319464e 100644 --- a/docs/LAUNCHER_CONTRACT.md +++ b/docs/LAUNCHER_CONTRACT.md @@ -115,7 +115,9 @@ path then ends in ` (deleted)`, and the watch exits 75. A supervisor starts A moved path, a path that cannot be read, and a file that could not be re-verified wait for one more sample. A replacement on that next sample exits -75. A second sample with the same failure exits 1. The message names +75. A second sample with the same failure exits 1. A different failure on +that next sample, or a move to a different path, waits for one more sample +instead. The message names `elastos gateway --addr localhost:8090` as the recovery command. A supervisor leaves that process stopped. diff --git a/elastos/crates/elastos-server/src/host_lock.rs b/elastos/crates/elastos-server/src/host_lock.rs index 0578af0f5..b4ae66664 100644 --- a/elastos/crates/elastos-server/src/host_lock.rs +++ b/elastos/crates/elastos-server/src/host_lock.rs @@ -315,8 +315,19 @@ struct BinarySupersessionWatch { current_exe_stamp: BinaryFileStamp, current_binary_sha256: String, /// A move, an unreadable path, or a failed re-hash waits for one more sample. + /// Only the same failure on the next sample confirms it. /// The install rename onto the running path is a replacement and does not wait. - pending_confirm: bool, + pending_failure: Option, +} + +/// A failed sample of the binary watch. A move names where the path now points, +/// so moves to two different paths are two different failures. +#[derive(Debug, Clone, PartialEq, Eq)] +enum WatchFailure { + PathUnreadable, + Moved(PathBuf), + Unverifiable, + ReverifyFailed, } const EXIT_LAUNCHER_RESTART: i32 = 75; @@ -384,7 +395,7 @@ impl BinarySupersessionWatch { current_exe_path: current_exe, current_exe_stamp: snapshot.stamp, current_binary_sha256: snapshot.sha256, - pending_confirm: false, + pending_failure: None, }) } @@ -393,7 +404,7 @@ impl BinarySupersessionWatch { } fn restart(&mut self, reason: String) -> Option { - self.pending_confirm = false; + self.pending_failure = None; Some(Supersession { reason, exit_code: EXIT_LAUNCHER_RESTART, @@ -402,12 +413,14 @@ impl BinarySupersessionWatch { /// A single failed sample can be the middle of an install rename. /// The next sample decides. A replacement on that sample restarts. - fn confirm_stop(&mut self, reason: String) -> Option { - if !self.pending_confirm { - self.pending_confirm = true; + /// A different failure on that sample waits again; only the same failure + /// on two samples in a row stops. + fn confirm_stop(&mut self, failure: WatchFailure, reason: String) -> Option { + if self.pending_failure.as_ref() != Some(&failure) { + self.pending_failure = Some(failure); return None; } - self.pending_confirm = false; + self.pending_failure = None; Some(Supersession { reason, exit_code: EXIT_LAUNCHER_STAYS_STOPPED, @@ -421,7 +434,7 @@ impl BinarySupersessionWatch { let current_exe_raw = match reader.current_exe_path_raw() { Ok(path) => path, Err(err) => { - return self.confirm_stop(format!( + return self.confirm_stop(WatchFailure::PathUnreadable, format!( "host executable path could not be read for {}: {err:#}. Start `elastos gateway --addr localhost:8090` after the path can be read.", self.current_exe_path.display() )); @@ -438,7 +451,7 @@ impl BinarySupersessionWatch { } if current_exe != self.current_exe_path { - return self.confirm_stop(format!( + return self.confirm_stop(WatchFailure::Moved(current_exe.clone()), format!( "host executable moved from {} to {}. Start `elastos gateway --addr localhost:8090` if this Home should run.", self.current_exe_path.display(), current_exe.display() @@ -448,7 +461,7 @@ impl BinarySupersessionWatch { let current_stamp = match reader.metadata_stamp(&self.current_exe_path) { Ok(stamp) => stamp, Err(err) => { - return self.confirm_stop(format!( + return self.confirm_stop(WatchFailure::Unverifiable, format!( "host executable {} could not be verified on disk: {err:#}. Start `elastos gateway --addr localhost:8090` after the file can be read.", self.current_exe_path.display() )); @@ -456,14 +469,14 @@ impl BinarySupersessionWatch { }; if current_stamp == self.current_exe_stamp { - self.pending_confirm = false; + self.pending_failure = None; return None; } let current_snapshot = match reader.digest_snapshot(&self.current_exe_path) { Ok(snapshot) => snapshot, Err(err) => { - return self.confirm_stop(format!( + return self.confirm_stop(WatchFailure::ReverifyFailed, format!( "host executable {} could not be re-verified: {err:#}. Start `elastos gateway --addr localhost:8090` after the file can be read.", self.current_exe_path.display() )); @@ -477,7 +490,7 @@ impl BinarySupersessionWatch { } self.current_exe_stamp = current_snapshot.stamp; - self.pending_confirm = false; + self.pending_failure = None; None } } @@ -990,6 +1003,44 @@ mod tests { assert!(event.reason.contains("changed on disk"), "{}", event.reason); } + /// Only the same failure on two samples in a row exits 1. A different + /// failure on the next sample waits for one more sample. + #[test] + fn supersession_watch_confirms_only_the_same_failure_twice() { + let temp = tempfile::tempdir().expect("tempdir"); + let exe = temp.path().join("elastos"); + write_file(&exe, b"original"); + let (mut watch, reader) = test_watch(&exe); + + reader.fail_metadata_once(); + assert!( + watch.superseded_reason_with(&reader).is_none(), + "unreadable file waits" + ); + let moved = temp.path().join("elastos-moved"); + reader.set_current_exe_raw(moved.clone()); + assert!( + watch.superseded_reason_with(&reader).is_none(), + "a move after an unreadable file is a different failure" + ); + let moved_again = temp.path().join("elastos-moved-again"); + reader.set_current_exe_raw(moved_again.clone()); + assert!( + watch.superseded_reason_with(&reader).is_none(), + "a move to another path is a different failure" + ); + + let event = watch + .superseded_reason_with(&reader) + .expect("the same move on the next sample confirms"); + assert_eq!(event.exit_code, EXIT_LAUNCHER_STAYS_STOPPED); + assert!( + event.reason.contains(&moved_again.display().to_string()), + "{}", + event.reason + ); + } + #[test] fn supersession_watch_confirms_a_failed_rehash_without_a_restart_word() { let temp = tempfile::tempdir().expect("tempdir");