fix(bin): stop lock waits in a removed home from spinning, and order pi-watch recovery tests deterministically

bin/fm-lock.sh

@@ -73,7 +73,10 @@ if ! fm_lock_try_acquire "$CLAIM_LOCK"; then
     echo "error: the prior session's bounded startup sweep is finishing; operate read-only until it releases the fleet lock" >&2
     exit 1
   fi
-  fm_lock_acquire_wait "$CLAIM_LOCK"
+  if ! fm_lock_acquire_wait "$CLAIM_LOCK"; then
+    echo "error: cannot take the fleet lock claim mutex; operate read-only until resolved" >&2
+    exit 1
+  fi
 fi
 CLAIM_LOCK_HELD=1
 

bin/fm-startup-network.sh

@@ -200,7 +200,7 @@ cmd_start() {  # <locked> <harvest-pid>
     return 1
   fi
 
-  fm_lock_acquire_wait "$PUBLISH_LOCK"
+  fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1
   if [ "$(status_get state)" = running ] && worker_alive \
     && { [ "$locked" != 1 ] || [ "$(status_get lock_pid)" = "$lock_pid" ]; }; then
     # A worker from this or a previous session is still going. Starting a second
@@ -299,7 +299,7 @@ await_delivery() {  # <generation> <state>
   limit=$(( $(delivery_budget) * 10 ))
   while [ "$waited" -lt "$limit" ]; do
     claim_live=0
-    fm_lock_acquire_wait "$PUBLISH_LOCK"
+    fm_lock_acquire_wait "$PUBLISH_LOCK" || return 0
     if [ "$(status_get generation)" != "$generation" ]; then
       fm_lock_release "$PUBLISH_LOCK"
       return 0
@@ -332,7 +332,7 @@ EOF
     sleep 0.1
     waited=$((waited + 1))
   done
-  fm_lock_acquire_wait "$PUBLISH_LOCK"
+  fm_lock_acquire_wait "$PUBLISH_LOCK" || return 0
   if [ "$(status_get generation)" != "$generation" ] || [ -f "$DELIVERED_FILE" ]; then
     fm_lock_release "$PUBLISH_LOCK"
     return 0
@@ -345,7 +345,7 @@ EOF
 
 publish() {  # <generation> <state> <phases> <locked> <started> <rc> <output-file> <timing-file>
   local generation=$1 state=$2 phases=$3 locked=$4 started=$5 rc=$6 out=$7 timings=${8:-} report_published=1
-  fm_lock_acquire_wait "$PUBLISH_LOCK"
+  fm_lock_acquire_wait "$PUBLISH_LOCK" || return 0
   if [ "$(status_get generation)" != "$generation" ]; then
     fm_lock_release "$PUBLISH_LOCK"
     return 0
@@ -387,7 +387,7 @@ cmd_run() {  # <locked> <lock-pid> <generation>
   budget=$(stage_budget)
   phases=probe
   if [ -n "$generation" ]; then
-    fm_lock_acquire_wait "$PUBLISH_LOCK"
+    fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1
     if [ "$(status_get generation)" = "$generation" ] && [ "$(status_get pid)" = "$$" ]; then
       internal=1
       started=$(status_get started)
@@ -410,7 +410,7 @@ cmd_run() {  # <locked> <lock-pid> <generation>
 
   if [ "$internal" -eq 0 ]; then
     generation="$(now).$$.manual"
-    fm_lock_acquire_wait "$PUBLISH_LOCK"
+    fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1
     if [ "$(status_get state)" = running ] && worker_alive; then
       fm_lock_release "$PUBLISH_LOCK"
       return 1
@@ -438,7 +438,11 @@ EOF
   stage_started=$(fm_timing_now_ms)
   rc=0
   if [ "$sweep_locked" -eq 1 ]; then
-    fm_lock_acquire_wait "$STATE/.lock.acquire"
+    if ! fm_lock_acquire_wait "$STATE/.lock.acquire"; then
+      rm -f "$out" 2>/dev/null || true
+      [ -z "$timings" ] || rm -f "$timings" 2>/dev/null || true
+      return 1
+    fi
     lease_held=1
     if ! lock_unchanged "$lock_pid"; then
       sweep_locked=0
@@ -544,7 +548,7 @@ print_state() {
 
 cmd_harvest() {  # <pid>
   local pid=$1 generation state claim_record claim_generation claim_pid
-  fm_lock_acquire_wait "$PUBLISH_LOCK"
+  fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1
   generation=$(status_get generation)
   # Another session's live claim is left alone; the worker reaps a dead one.
   if [ -f "$CLAIM_FILE" ]; then

bin/fm-wake-lib.sh

@@ -732,6 +732,19 @@ fm_lock_try_acquire() {
   if fm_lock_try_create "$lockdir"; then
     return 0
   fi
+  # A lock that does not exist has no stale holder to recover. Either its holder
+  # released it just after creation failed, which one more creation attempt
+  # settles, or the lock's directory is gone or unwritable. Stealing it then
+  # would fail the same way one "$lockdir.steal" deeper, recursing without bound
+  # and burning a CPU core for as long as the directory stays gone.
+  if [ ! -e "$lockdir" ] && [ ! -L "$lockdir" ]; then
+    if fm_lock_try_create "$lockdir"; then
+      return 0
+    fi
+    if [ ! -e "$lockdir" ] && [ ! -L "$lockdir" ]; then
+      return 1
+    fi
+  fi
 
   # Compare against ${BASHPID:-$$} inline, never via a command substitution:
   # $() forks a subshell whose BASHPID is not this frame's pid.
@@ -824,9 +837,17 @@ fm_lock_try_acquire() {
   return "$rc"
 }
 
+# Waits until the lock is held, and fails only once the lock's directory no
+# longer exists: a lock can never be taken there again, as when a process
+# outlives the home it was working in.
 fm_lock_acquire_wait() {
-  local lockdir=$1
+  local lockdir=$1 parent
+  case "$lockdir" in
+    */*) parent=${lockdir%/*}; parent=${parent:-/} ;;
+    *) parent=. ;;
+  esac
   while ! fm_lock_try_acquire "$lockdir"; do
+    [ -d "$parent" ] || return 1
     sleep 0.1
   done
 }
@@ -942,7 +963,7 @@ fm_wake_append() {
   recovery_marker="$STATE/.watcher-down"
   status=0
 
-  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1
   _fm_recovery_marker_publish "$recovery_marker" downtime || status=$?
   if [ "$status" -eq 0 ]; then
     seq=$(cat "$seq_file" 2>/dev/null || echo 0)