fix(daemon): end a normal unmount stalled at Join with one forced unmount (#426, #460)

.agents/skills/layerfs-js-sdk/SKILL.md

@@ -373,7 +373,7 @@ method, its arguments, result fields, refusals with what to do, limits and examp
 | [fs writes](references/methods/fs-write.md) | `write(branch, path, content, message?)`, `edit(branch, path, oldText, newText, message?)`, `apply(branch, changes, message?)`, `mkdir(branch, path, message?)`, `rm(branch, path, message?)`, `cp(branch, from, to, message?)`, `mv(branch, from, to, message?)` |
 | [branch](references/methods/branch.md) | `fork(name, fromRef?)`, `list_commits(branch, pageSize?, pagePos?)`, `diff_commits(from, to, pageSize?, pagePos?)`, `diff_branches(from, to, pageSize?, pagePos?)`, `merge(a, b, name, message?)`, `publish(branch, commit?)` |
 | [sandbox](references/methods/sandbox.md) | `create(options?)`, `list_workspaces(sandbox, pageSize?, pagePos?)`, `close(sandbox)` |
-| [workspace](references/methods/workspace.md) | `mount(sandbox, branch)` → `{ sandbox, path, image }`, `exec(ws, command, message?)`, `commit(ws, message?, force?)`, `unmount(ws, discard?)` |
+| [workspace](references/methods/workspace.md) | `mount(sandbox, branch)` → `{ sandbox, path, image }`, `exec(ws, command, message?)`, `commit(ws, message?, force?)`, `unmount(ws, discard?, force?)` |
 
 A `ref` is `main`, a Branch name, a Commit id or a Layer id. Paths are absolute in the
 view (`/src/index.js`). Pages are `{ items, hasMore }`; advance with

.agents/skills/layerfs-js-sdk/references/methods/workspace.md

@@ -274,16 +274,18 @@ conflict detail also exposes `conflictsTruncated`, and individual conflicts mark
 their text. Inspect those files with bounded `workspace.exec` commands and Commit
 each command. The omission flags never mean the unresolved source is safe to drop.
 
-## `workspace.unmount(ws, discard?)`
+## `workspace.unmount(ws, discard?, force?)`
 
 Unmounts a Workspace. A Sandbox left with no Workspace closes itself two minutes later.
 
 | Arg | Meaning |
 | --- | --- |
 | `ws` | The Workspace from `mount` |
 | `discard` | `true` to drop uncommitted changes instead of refusing |
+| `force` | `true` to abort the Workspace's kernel connection instead; drops uncommitted changes. Use it only for an unmount that stalled (below) |
 
-**Returns:** `{ kind: "Unmounted" }`.
+**Returns:** `{ kind: "Unmounted" }`, or with `force`
+`{ kind: "ForceUnmounted", abort, detach, cleanup }`.
 
 **Refusals**
 
@@ -292,6 +294,7 @@ Unmounts a Workspace. A Sandbox left with no Workspace closes itself two minutes
 | `Dirty` | Uncommitted changes; `detail.changed` lists paths | Commit them, so the step is recorded; discard only a Workspace you're abandoning |
 | `Busy` | A background process still runs there | Stop it (`backend.daemon.process.cancel`, in the observe skill), or wait for it |
 | `NotMounted` | Already unmounted, or the wrong Workspace | Use the Workspace from `mount` |
+| `Failed` with `TeardownCustody { stage: Join` in the message | The mount is gone but its connection did not end, so the unmount stalled; every later plain unmount answers the same | `workspace.unmount(ws, true, true)` once |
 
 ```js
 // ws: from workspace.mount. Every mount creates a new Workspace; never hard-code one.

api/agent/src/workspace/unmount.js

@@ -1,6 +1,8 @@
 // workspace.unmount: ends the Workspace; `discard` drops its uncommitted
-// changes. A Sandbox left with no Workspace closes once it has been idle.
+// changes. `force` aborts its kernel connection instead, also after a normal
+// unmount stalled at Join, and drops uncommitted changes. A Sandbox left with
+// no Workspace closes once it has been idle.
 import { invokeClient, workspaceArgs } from "../client.js";
-export function unmount(client, ws, discard = false) {
-  return invokeClient(client, "workspace.unmount", { ...workspaceArgs(ws), discard });
+export function unmount(client, ws, discard = false, force = false) {
+  return invokeClient(client, "workspace.unmount", { ...workspaceArgs(ws), discard, force });
 }

api/agent/test/fixture/cases.mjs

@@ -107,6 +107,8 @@ export const CASES = [
     op: "workspace.commit", sent: { ...WS, force: true }, result: committed },
   { name: "workspace unmount", argv: ["workspace", "unmount", HANDLE, "--discard"], sdk: (lf) => lf.workspace.unmount(WS, true),
     op: "workspace.unmount", sent: { ...WS, discard: true }, result: { kind: "Unmounted" } },
+  { name: "workspace unmount (force)", argv: ["workspace", "unmount", HANDLE, "--force"], sdk: (lf) => lf.workspace.unmount(WS, false, true),
+    op: "workspace.unmount", sent: { ...WS, force: true }, result: { kind: "ForceUnmounted", abort: "Written", detach: "Detached", cleanup: "Queued" } },
   { name: "sandbox create", argv: ["sandbox", "create", "--image", "sha256:abc"], sdk: (lf) => lf.sandbox.create({ image: "sha256:abc" }),
     op: "sandbox.create", sent: { deploy: { image: "sha256:abc" } }, result: { sandbox: "2", image: "sha256:abc" } },
   { name: "sandbox list_workspaces", argv: ["sandbox", "list_workspaces", "2"], sdk: (lf) => lf.sandbox.list_workspaces("2"),

api/agent/test/fixture/fake-native.cjs

@@ -60,7 +60,7 @@ function answer(op, args) {
       return { value: { kind: `exit:${code}`, exitCode: code, stdout: `out:${args.command}`, stderr: "err", truncated: false } };
     }
     case "workspace.commit": return { value: committed };
-    case "workspace.unmount": return { value: { kind: "Unmounted" } };
+    case "workspace.unmount": return { value: args.force ? { kind: "ForceUnmounted", abort: "Written", detach: "Detached", cleanup: "Queued" } : { kind: "Unmounted" } };
     case "backend.daemon.process.start": return { value: { kind: "running", process: "proc_1_ab", command: args.command } };
     case "backend.store.project.get_history": return { value: exports.history ?? { kind: "ok", version: "h1", project: "website", branches: [], commits: [], more: {}, layers: [], workspaces: [] } };
     case "backend.store.project.get_storage": return { value: { dedupe: "2", compression: null } };

api/cli/src/workspace/unmount.js

@@ -1,8 +1,8 @@
-// layerfs workspace unmount: workspace.unmount(ws, discard?).
+// layerfs workspace unmount: workspace.unmount(ws, discard?, force?).
 
 export default {
   args: ["ws"],
-  options: { discard: "bool" },
-  summary: "end the Workspace; --discard drops uncommitted changes",
-  run: (lf, a, o) => lf.workspace.unmount(a.ws, o.discard),
+  options: { discard: "bool", force: "bool" },
+  summary: "end the Workspace; --discard drops uncommitted changes, --force aborts its connection",
+  run: (lf, a, o) => lf.workspace.unmount(a.ws, o.discard, o.force),
 };

api/docs/agent.md

@@ -273,7 +273,7 @@ trace line covers the command's run.
 | fs writes (7) | `write(branch, path, content, message? \| { message?, mode? })`, `edit(branch, path, oldText, newText, message?)`, `apply(branch, changes, message?)`, `mkdir(branch, path, message? \| { message?, mode? })`, `rm(branch, path, message?)`, `cp(branch, from, to, message?)`, `mv(branch, from, to, message?)` |
 | branch (6) | `fork(name, fromRef?)`, `list_commits(branch, pageSize?, pagePos?)`, `diff_commits(from, to, pageSize?, pagePos?, { cursor?, hunks? }?)`, `diff_branches(from, to, pageSize?, pagePos?, { cursor?, hunks? }?)`, `merge(a, b, name, message?, units?)`, `publish(branch, commit?)` |
 | sandbox (3) | `create({ image }?)` → `{ sandbox, image }`, `list_workspaces(sandbox, pageSize?, pagePos?)`, `close(sandbox)` |
-| workspace (4) | `mount(sandbox, branch)` → `{ sandbox, path, image }`, `exec(ws, command, message?, { stdout?, stderr? }?)`, `commit(ws, message?, force?)`, `unmount(ws, discard?)` |
+| workspace (4) | `mount(sandbox, branch)` → `{ sandbox, path, image }`, `exec(ws, command, message?, { stdout?, stderr? }?)`, `commit(ws, message?, force?)`, `unmount(ws, discard?, force?)` |
 
 A Workspace's status is not an agent call: it is `backend.daemon.workspace.status(ws)`
 in [`@layerfs/backend`](backend.md).
@@ -518,7 +518,7 @@ table differs, and when a family's files stop matching the SDK's.
 | `workspace.mount` | `layerfs workspace mount <sandbox> <branch>` |
 | `workspace.exec` | `layerfs workspace exec <ws> <command> [-m <text>] [--stream]` |
 | `workspace.commit` | `layerfs workspace commit <ws> [-m <text>] [--force]` |
-| `workspace.unmount` | `layerfs workspace unmount <ws> [--discard]` |
+| `workspace.unmount` | `layerfs workspace unmount <ws> [--discard] [--force]` |
 | `sandbox.create` | `layerfs sandbox create [--image <text>]` |
 | `sandbox.list_workspaces` | `layerfs sandbox list_workspaces <sandbox> [--page-size <n>] [--page-pos <n>]` |
 | `sandbox.close` | `layerfs sandbox close <sandbox>` |

core/docs/architecture/76-native-mount-session.md

@@ -197,9 +197,11 @@ Any step that is not established leaves the entry `Retained` with its stage
 (`Detach`, `Join`, `Owner`, `Requests`, `Lane`, `Revoke`, `Close` or `Registry`),
 the observer and the original evidence. The reply is `Retained` with
 `TeardownCustody`; it is not reported as an uncertain mutation and nothing is
-replayed. Later Attach, Unmount or ForceUnmount on that entry returns the same
-custody. The normal path never uses the abort control and never sets the
-terminal fence.
+replayed. Later Attach or Unmount on that entry returns the same custody, and
+so does ForceUnmount, except after a detached drain stopped at `Join`: that one
+[stalled normal unmount](80-forced-teardown.md#stalled-normal-unmount) is ended
+by one forced unmount. The normal path never uses the abort control and never
+sets the terminal fence.
 
 ## Forced unmount
 

core/docs/architecture/80-forced-teardown.md

@@ -47,7 +47,8 @@ application takes it, although such a refusal carries no custody of its own.
 | Condition | Answer |
 | --- | --- |
 | The token names no entry, another namespace, or a binding still in progress | `Missing`, `Invalid` or `Busy` at `admission`, as for every control operation |
-| The entry is `Retained` | The stored custody, as `Retained`. No second abort or detach ever follows it |
+| The entry is `Retained` by a normal unmount stopped at `Join`, its abort control bound and unused | Admitted as stalled: native phase `Stopping`, Commit knowledge `Absent` (a normal unmount is admitted only idle). See [stalled normal unmount](#stalled-normal-unmount) |
+| The entry is `Retained` otherwise | The stored custody, as `Retained`. No second abort or detach ever follows it |
 | Activity is `Committing`, `Attaching` or `Closing` | `Busy` at `force:admission` |
 | No native connection (`Unattached`) | `Invalid` at `force:admission` |
 | An Attach outcome is unknown (`Attaching`) | `Unknown` at `force:admission` |
@@ -113,7 +114,29 @@ After admission, outside the registry lock, each step is made once:
 6. **Routing removal and `ForceUnmounted`.** The token is `Missing` afterwards.
 
 No `MNT_FORCE`, lazy detach, second abort, second detach, retry, process
-signal or wait on a caller's exit or output exists on this path. Revoke, Close
+signal or wait on a caller's exit or output exists on this path.
+
+### Stalled normal unmount
+
+A normal unmount can detach successfully and still stop at `Join`: when
+another mount of the same superblock outlives the detach, for example a copy
+in a mount namespace made by a process outside the Sandbox, the kernel never
+ends the connection, so its receive loops stay blocked in their reads
+(issues #426 and #460). The entry keeps that undrained session as `Retained`
+at `Join` with no forced facts, and every later normal `Unmount` returns that
+custody unchanged.
+
+`ForceUnmount` is the one way out. It takes the session back and makes the
+same sequence with two differences. Step 1 is `NativeSession::abort_detached`,
+which refuses unless the connection is detached and not yet aborted. Step 2
+makes no detach, because the normal unmount's plain detach is the
+connection's one detach, so the facts report `detach: Detached` from the
+start. A stop after the abort is ordinary forced custody and is never forced
+again. After the abort the drain completes and the mount directory is removed,
+and Linux then detaches the mounts on that removed mount point in every
+namespace, so a copy at the same path goes with it and the connection ends.
+A copy mounted anywhere else stays, and its calls get the aborted connection's
+answer until it is unmounted. Revoke, Close
 and the observation are submitted without waiting for admission, as on the
 normal path, and at most one Lifecycle completion is held while another job is
 submitted.
@@ -139,12 +162,26 @@ returned only by a later `Unmount`, `ForceUnmount` or `Attach`.
 
 ## Limits
 
-- Forced teardown needs the abort control, which is bound at Attach only when
-  the control filesystem is mounted where the daemon runs. The Sandbox
-  container does not mount it, so in that topology every Force is refused at
-  `force:capability`. Mounting it there is an owner decision.
-- There is no way out of `Retained`. A request that ended retained keeps the
-  drain at `Requests` with no detach; daemon stop is the exit.
+- Forced teardown needs the abort control, which is bound at Attach from the
+  control filesystem. A daemon uses the shared `/sys/fs/fuse/connections`
+  when it is mounted there. Otherwise (a Sandbox container mounts none) it
+  makes one mount of it at startup in `fusectl/` beside its Overlay file, a
+  directory only the daemon can reach, because the tree lists every
+  connection of the kernel, other containers' included. The daemon still
+  binds only its own connection's row. The mount is best effort: where it is
+  refused, as inside a user namespace (rootless Docker or Podman), which
+  refuses it though not FUSE mounts, the daemon starts anyway with no abort
+  control bound, writes one `LAYERFS_DAEMON_ABORT_UNBOUND <error>` line to
+  stderr, and every Force is refused at `force:capability`, as before the
+  mount existed. A refused mount leaves no directory behind. An existing
+  `fusectl/` without the mount is not adopted, so it also leaves the control
+  unbound.
+- The private mount lasts as long as the daemon's mount namespace. In a
+  Sandbox that ends with the container; a daemon run elsewhere leaves it
+  mounted after it exits.
+- Apart from a [stalled normal unmount](#stalled-normal-unmount), there is no
+  way out of `Retained`. A request that ended retained keeps the drain at
+  `Requests` with no detach; daemon stop is the exit.
 - A Workspace that is `Unattached` with unknown Commit custody has no terminal
   exit: Unmount is `Unknown` and Force is `Invalid`.
 - A retained Commit failure can keep Lifecycle completions (its local

engine/crates/layerfs-api/src/sandbox/values.rs

@@ -6,7 +6,7 @@
 //! refusals with stable codes.
 use super::{CommitReply, MountedWorkspace, Unmounted};
 use crate::{OperationCause, OperationFailure};
-use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceStatus};
+use layerfs_bridge::control::{ForcedOutcome, SdkRefusal, SdkValue, WorkspaceStatus};
 use layerfs_bridge::sdk_values::{
     commit_value, conflicts, conflicts_truncated, hex, lww, lww_truncated,
 };
@@ -171,6 +171,17 @@ pub fn commit(reply: &CommitReply) -> Result<SdkValue, SdkRefusal> {
     }
 }
 
+/// `sandbox.unmount` with `force`: what the forced teardown established.
+pub fn force_unmounted(outcome: &ForcedOutcome) -> SdkValue {
+    let text = |value: &dyn std::fmt::Debug| SdkValue::text(format!("{value:?}"));
+    SdkValue::map([
+        ("kind", SdkValue::text("ForceUnmounted")),
+        ("abort", text(&outcome.facts.abort)),
+        ("detach", text(&outcome.facts.detach)),
+        ("cleanup", text(&outcome.cleanup)),
+    ])
+}
+
 /// `sandbox.unmount`: Unmounted, or the `Dirty` refusal naming the paths.
 pub fn unmounted(answer: &Unmounted) -> Result<SdkValue, SdkRefusal> {
     match answer {

engine/crates/layerfs-api/tests/kernel_application.rs

@@ -39,6 +39,7 @@ fn connect(address: SocketAddr) -> duplex::Connection {
 }
 #[test]
 fn actual_daemon_install_hello_bind_status_no_constructor_and_session_end() {
+    support::shared_control();
     let f = support::Fixture::new("application", None);
     fs::set_permissions(&f.directory, fs::Permissions::from_mode(0o700)).unwrap();
     let setup = DaemonSetup {

engine/crates/layerfs-api/tests/observed_application.rs

@@ -234,6 +234,7 @@ fn task_ids(process: &Process) -> BTreeSet<u32> {
 }
 #[test]
 fn equal_calls_and_reused_slot_have_distinct_public_observation_identity() {
+    support::shared_control();
     let f = support::Fixture::new("observed-correlation", None);
     let log = f.directory.join("stderr.log");
     let (process, address, mut initial) = start(&f, File::create(&log).unwrap());
@@ -334,6 +335,7 @@ fn equal_calls_and_reused_slot_have_distinct_public_observation_identity() {
 }
 #[test]
 fn diagnostic_sink_failure_does_not_repeat_or_hide_successful_bind() {
+    support::shared_control();
     let f = support::Fixture::new("observed-failed-sink", None);
     let (process, address, mut control) =
         start(&f, File::options().write(true).open("/dev/full").unwrap());
@@ -393,6 +395,7 @@ fn timing_names(telemetry: &layerfs_bridge::control::SdkValue) -> Vec<String> {
 }
 #[test]
 fn traced_diagnostic_failure_is_reported_and_keeps_the_connection() {
+    support::shared_control();
     let f = support::Fixture::new("observed-traced-sink", None);
     let (process, _address, mut control) =
         start(&f, File::options().write(true).open("/dev/full").unwrap());
@@ -441,6 +444,7 @@ fn traced_diagnostic_failure_is_reported_and_keeps_the_connection() {
 }
 #[test]
 fn traced_records_share_the_scope_and_untraced_replies_stay_plain() {
+    support::shared_control();
     let f = support::Fixture::new("observed-traced-records", None);
     let log = f.directory.join("stderr.log");
     let (process, _address, mut control) = start(&f, File::create(&log).unwrap());
@@ -503,6 +507,7 @@ fn width(telemetry: &layerfs_bridge::control::SdkValue, path: &[&str]) -> usize
 /// carries its delta selection work.
 #[test]
 fn traced_calls_carry_sql_work_and_status_unmount_carry_opcodes() {
+    support::shared_control();
     let f = support::Fixture::new("observed-traced-work", None);
     let log = f.directory.join("stderr.log");
     let (process, _address, mut control) = start(&f, File::create(&log).unwrap());
@@ -599,6 +604,7 @@ fn traced_calls_carry_sql_work_and_status_unmount_carry_opcodes() {
 }
 #[test]
 fn resource_and_output_failure_preserve_original_status_reply() {
+    support::shared_control();
     let f = support::Fixture::new("observed-resource-and-sink", None);
     let (process, address, mut control) =
         start(&f, File::options().write(true).open("/dev/full").unwrap());
@@ -676,6 +682,7 @@ impl Drop for MountGuard {
 /// it makes none. Creates are ordinary processes of the command identity.
 #[test]
 fn configured_serial_low_water_reaches_a_mounted_create_on_both_startup_routes() {
+    support::shared_control();
     use std::os::unix::{fs::MetadataExt, process::CommandExt};
     const LOW: u64 = layerfs_workspace::SERIAL_REFILL - 1;
     let f = support::Fixture::new("observed-low-water", None);
@@ -812,6 +819,7 @@ fn configured_serial_low_water_reaches_a_mounted_create_on_both_startup_routes()
 /// one the test above starts with.
 #[test]
 fn startup_refuses_a_serial_low_water_at_or_above_the_refill_window() {
+    support::shared_control();
     let f = support::Fixture::new("observed-low-water-refused", None);
     let overlay = f.directory.join("overlay.sqlite");
     for refused in [
@@ -881,6 +889,7 @@ fn startup_refuses_a_serial_low_water_at_or_above_the_refill_window() {
 /// it stays; an open connection does not.
 #[test]
 fn an_idle_daemon_exits_and_a_workspace_keeps_it() {
+    support::shared_control();
     let f = support::Fixture::new("observed-idle-exit", None);
     let log = f.directory.join("stderr.log");
     let mounts = f.directory.join("mounts");