From 79190bfd466a06f7cd6ca1eb7805cae4d9854e4d Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Sun, 25 Oct 2026 12:03:31 +0800 Subject: [PATCH 01/19] feat(persistence): Windows-native Store writer lane, volume guard and open The Store engine runs natively on Windows (owner direction 2026-10-25, route 2: Windows-native Store, no WSL2): - The cross-process writer lane is a named kernel mutex (Local\layerfs-writer-lane-): CreateMutexW + INFINITE wait; WAIT_ABANDONED also takes ownership, so a dead writer's lane is released by the kernel and the next writer proceeds. - The kernel guard reads the volume filesystem via GetVolumeInformationW and allows local NTFS volumes only; other filesystems and non-disk prefixes refuse Unsupported, matching the Linux ext4/xfs/btrfs allowlist discipline. - SQLite opens on Windows through bundled rusqlite; the open/profile cfg lists widen from macos/linux to include windows. Linux and macOS branches are unchanged. - The installer drops the unix-only mode bits and the daemons-uid check on Windows; destination and exclusivity checks are unchanged. - Boundary guard: audited-FFI entries become per-crate tuples and gain persistence backend/sqlite/windows_lane.rs; path matching is as_posix so the audited-module check works on Windows hosts (this repairs two false positives the separator bug produced there). Verification at this commit: cargo check --offline --locked -p layerfs-persistence -p layerfs-store-service (Windows, rustc 1.90.0) Finished; the guard's 24 self-tests pass; the guard reports only the four test-only-attribute findings that HEAD already reports, with no new violation; tests/windows_store_native.rs (new) drives create + two-session writes + two re-exec'd child processes through the lane with bounded waits: 1 passed. Production LOC: core 96783 -> 96948 (delta +165), sdk 3090 -> 3090 (delta 0); combined 109056 -> 109221 (delta +165). Counter: tools/production_loc.py --detail (pinned at this tree), scopes core and sdk, tests/tools/docs excluded. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/Cargo.lock | 1 + core/crates/layerfs-persistence/Cargo.toml | 6 + .../src/backend/sqlite/connection.rs | 6 +- .../src/backend/sqlite/mod.rs | 5 + .../src/backend/sqlite/windows_lane.rs | 117 ++++++++++++++++ .../src/backend/sqlite/writer_admission.rs | 31 ++++- core/crates/layerfs-persistence/src/lib.rs | 3 + .../layerfs-persistence/src/store/open.rs | 12 +- .../tests/windows_store_native.rs | 130 ++++++++++++++++++ .../layerfs-store-service/src/install.rs | 48 ++++--- .../layerfs-store-service/src/kernel_guard.rs | 42 ++++-- core/tools/check_product_boundary.py | 9 +- core/tools/test_check_product_boundary.py | 5 +- 13 files changed, 381 insertions(+), 34 deletions(-) create mode 100644 core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs create mode 100644 core/crates/layerfs-persistence/tests/windows_store_native.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 307a94e17..0ad436ac4 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -655,6 +655,7 @@ dependencies = [ "layerfs-telemetry", "nix", "rusqlite", + "windows-sys", ] [[package]] diff --git a/core/crates/layerfs-persistence/Cargo.toml b/core/crates/layerfs-persistence/Cargo.toml index 8a34236d2..adbf3e37d 100644 --- a/core/crates/layerfs-persistence/Cargo.toml +++ b/core/crates/layerfs-persistence/Cargo.toml @@ -24,3 +24,9 @@ nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthrea [target.'cfg(target_os = "linux")'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } + +[target.'cfg(target_os = "windows")'.dependencies] +# Windows has no system SQLite; the Store bundles the same version. The +# writer lane and volume probe FFI use windows-sys, already in the locked graph. +rusqlite = { version = "=0.40.2", features = ["bundled"] } +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Threading", "Win32_Storage_FileSystem"] } diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs b/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs index f06d1e573..d1b874cec 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs @@ -154,7 +154,11 @@ impl Session { creation_layout: crate::SqlitePackLayout, creation_acquisition: crate::SqliteAcquisitionSchema, ) -> Result { - if !cfg!(any(target_os = "macos", target_os = "linux")) { + if !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(BackendError::Integrity); } let flags = if writable { diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs b/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs index 6d9612135..4b570c8a9 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs @@ -5,6 +5,11 @@ pub(crate) mod connection; #[cfg(target_os = "macos")] #[allow(unsafe_code)] mod file_control; +#[cfg(target_os = "windows")] +#[allow(unsafe_code)] +mod windows_lane; +#[cfg(target_os = "windows")] +pub use windows_lane::windows_volume_filesystem; pub(crate) mod lane; pub(crate) mod metadata_allocation; pub(crate) mod metadata_locations; diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs b/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs new file mode 100644 index 000000000..f16b365bf --- /dev/null +++ b/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs @@ -0,0 +1,117 @@ +//! The Windows Store's one audited FFI surface. +//! +//! This is the Windows companion of the macOS seal file-control module: one +//! owner-authorized module of raw Win32 calls, allowed by the boundary guard +//! and compiled on Windows only; no other module in this crate may use +//! unsafe Rust. The writer lane is one named kernel mutex per Store +//! directory: the kernel releases it when a holder process exits (the next +//! waiter wakes on the abandoned mutex), which matches the Linux directory +//! flock's exit release, and it never opens a second descriptor to SQLite's +//! main/WAL/SHM files. The handle is held as an integer so `Writer` stays +//! plain data. The volume probe reports the filesystem of the directory's +//! local volume for the kernel guard's allowlist. +#![allow(unsafe_code)] +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::io; +use std::path::Path; +use windows_sys::Win32::Foundation::{CloseHandle, WAIT_ABANDONED, WAIT_OBJECT_0}; +use windows_sys::Win32::System::Threading::{ + CreateMutexW, ReleaseMutex, WaitForSingleObject, INFINITE, +}; + +/// The Store directory's cooperative writer lane. +pub(crate) struct LaneMutex { + handle: usize, +} + +impl LaneMutex { + /// One blocking kernel wait on the directory's named mutex, taken after + /// the Session's FIFO turn, so at most one thread per Session waits here. + /// Creation only opens the shared kernel object. Exits: the lane (a + /// holder process that exits releases it); a failed wait. + pub(crate) fn acquire(directory: &Path) -> Result { + let mut hasher = DefaultHasher::new(); + match std::fs::canonicalize(directory) { + Ok(canonical) => canonical.hash(&mut hasher), + Err(_) => directory.hash(&mut hasher), + } + let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", hasher.finish()) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { CreateMutexW(std::ptr::null(), 0, name.as_ptr()) }; + if handle.is_null() { + return Err(()); + } + let waited = unsafe { WaitForSingleObject(handle, INFINITE) }; + if waited == WAIT_OBJECT_0 || waited == WAIT_ABANDONED { + Ok(Self { + handle: handle as usize, + }) + } else { + unsafe { CloseHandle(handle) }; + Err(()) + } + } + + /// Releases the lane and destroys this handle; a failed release returns + /// the retained lane unchanged. + pub(crate) fn release(self) -> Result<(), Self> { + let handle = self.handle as windows_sys::Win32::Foundation::HANDLE; + if unsafe { ReleaseMutex(handle) } == 0 { + return Err(Self { + handle: self.handle, + }); + } + unsafe { CloseHandle(handle) }; + Ok(()) + } +} + +/// The filesystem name of the local volume holding `directory`, for the +/// kernel guard's allowlist. Only a local drive letter is examined: UNC, +/// device and verbatim prefixes are network or device paths whose locks do +/// not coordinate Store writers. +pub fn windows_volume_filesystem(directory: &Path) -> io::Result { + use std::path::{Component, Prefix}; + let root = match directory.components().next() { + Some(Component::Prefix(prefix)) => match prefix.kind() { + Prefix::Disk(letter) => format!("{}:\\", letter as char), + _ => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "a network or device path is not a local volume", + )) + } + }, + _ => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "the directory has no volume prefix", + )) + } + }; + let wide: Vec = root.encode_utf16().chain(std::iter::once(0)).collect(); + let mut filesystem = [0u16; 16]; + let examined = unsafe { + windows_sys::Win32::Storage::FileSystem::GetVolumeInformationW( + wide.as_ptr(), + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + filesystem.as_mut_ptr(), + filesystem.len() as u32, + ) + }; + if examined == 0 { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("the volume {root} cannot be examined"), + )); + } + let end = filesystem.iter().position(|&wide| wide == 0).unwrap_or(0); + Ok(String::from_utf16_lossy(&filesystem[..end])) +} diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs b/core/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs index c7d42a3f8..c66aa1504 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/writer_admission.rs @@ -13,6 +13,8 @@ use { pub(crate) struct Writer { #[cfg(target_os = "linux")] lock: Flock, + #[cfg(target_os = "windows")] + lane: super::windows_lane::LaneMutex, } impl Writer { @@ -54,7 +56,28 @@ impl Writer { let _ = path; Ok(Self {}) } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(target_os = "windows")] + { + // Windows has no flock on a directory handle. One named kernel + // mutex per Store directory is the equivalent coarse lane: it + // never opens another descriptor to SQLite's main/WAL/SHM files, + // and a holder process that exits releases it. Multiple Store + // files in one directory share this coarse lane, as on Linux. + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + if !std::fs::metadata(directory) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { + return Err(BackendError::Integrity); + } + let lane = super::windows_lane::LaneMutex::acquire(directory) + .map_err(|()| BackendError::Integrity)?; + Ok(Self { lane }) + } + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] { let _ = path; Err(BackendError::Integrity) @@ -72,10 +95,14 @@ impl Writer { Err((lock, _)) => Err(Self { lock }), } } - #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "macos")] { Ok(()) } + #[cfg(target_os = "windows")] + { + self.lane.release().map_err(|lane| Self { lane }) + } } } diff --git a/core/crates/layerfs-persistence/src/lib.rs b/core/crates/layerfs-persistence/src/lib.rs index 70ecdf986..29184b59b 100644 --- a/core/crates/layerfs-persistence/src/lib.rs +++ b/core/crates/layerfs-persistence/src/lib.rs @@ -11,6 +11,9 @@ pub use backend::sqlite::connection::{ConnectionProfile, SqlWork}; pub use backend::sqlite::reclamation::{SpaceReclamation, RECLAMATION_PAGE_LIMIT}; pub use backend::sqlite::statement_work::StatementPhaseWork; pub use backend::sqlite::thread_work::thread_work; +#[cfg(target_os = "windows")] +#[cfg(target_os = "windows")] +pub use backend::sqlite::windows_volume_filesystem; pub use config::{ BackendSelection, PersistenceConfig, SqliteAcquisitionSchema, SqlitePackLayout, SqlitePersistenceProfile, diff --git a/core/crates/layerfs-persistence/src/store/open.rs b/core/crates/layerfs-persistence/src/store/open.rs index 2cb9e9827..e851ed679 100644 --- a/core/crates/layerfs-persistence/src/store/open.rs +++ b/core/crates/layerfs-persistence/src/store/open.rs @@ -18,7 +18,11 @@ impl Handles { history: &HistoryCatalogConfig, ) -> Result { if config.backend != BackendSelection::Sqlite - || !cfg!(any(target_os = "macos", target_os = "linux")) + || !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(PersistenceError::BackendUnavailable); } @@ -70,7 +74,11 @@ impl Handles { writable: bool, ) -> Result { if config.backend != BackendSelection::Sqlite - || !cfg!(any(target_os = "macos", target_os = "linux")) + || !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(PersistenceError::BackendUnavailable); } diff --git a/core/crates/layerfs-persistence/tests/windows_store_native.rs b/core/crates/layerfs-persistence/tests/windows_store_native.rs new file mode 100644 index 000000000..254425045 --- /dev/null +++ b/core/crates/layerfs-persistence/tests/windows_store_native.rs @@ -0,0 +1,130 @@ +//! The Windows-native Store route: creation and two concurrent sessions in +//! one process, then two foreign processes serialized through the writer +//! lane, all through public Stores and bounded waits. +#![cfg(windows)] +mod support; +use layerfs_history::HistoryCatalogConfig; +use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; +use layerfs_storage::{port::*, StoragePolicy}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +const CHILD_STORE: &str = "LAYERFS_WINDOWS_STORE_CHILD_STORE"; +const CHILD_BINDING: &str = "LAYERFS_WINDOWS_STORE_CHILD_BINDING"; + +fn config(path: &Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn reserve() -> Reserve { + Reserve { + packs: 1, + ordinals: 1, + } +} + +fn authority(name: &[u8]) -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: name.to_vec(), + cursor_key: [67; 32], + incarnation: 1, + } +} + +/// One child process's whole work: open the Store writable and write once. +fn child(path: PathBuf, binding: Vec) { + let opened = Handles::open_writable(config(&path), &binding, [67; 32]) + .expect("the child opens the Store"); + opened.storage.reserve(reserve()).expect("the child writes"); +} + +/// When this binary was re-executed as a lane child, run its one job and +/// leave; otherwise this is the parent test run and work continues below. +fn dispatch_child() { + let Some(path) = std::env::var_os(CHILD_STORE) else { + return; + }; + let binding = std::env::var(CHILD_BINDING) + .expect("the child binding") + .into_bytes(); + child(PathBuf::from(path), binding); + std::process::exit(0); +} + +/// Bounded completion of one foreign child; a hung child fails the test. +fn finish(mut child: std::process::Child) { + let until = Instant::now() + Duration::from_secs(60); + loop { + match child.try_wait().expect("child poll") { + Some(status) => { + assert!(status.success(), "a child writer failed: {status}"); + return; + } + None => assert!(Instant::now() < until, "a child writer is hung"), + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { + dispatch_child(); + let temp = support::Temp::new("windows-store-native"); + let path = temp.join("store.sqlite"); + let catalog = authority(b"windows-store-native"); + let first = Handles::create(config(&path), StoragePolicy::frozen_default(), &catalog).unwrap(); + let second = + Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key).unwrap(); + first.storage.reserve(reserve()).unwrap(); + second.storage.reserve(reserve()).unwrap(); + first.storage.reserve(reserve()).unwrap(); + second.storage.reserve(reserve()).unwrap(); + drop(first); + drop(second); + + let lane = temp.join("lane"); + std::fs::create_dir_all(&lane).unwrap(); + let lane_path = lane.join("store.sqlite"); + let binding = b"windows-lane-processes".to_vec(); + let lane_catalog = authority(&binding); + let seed = Handles::create( + config(&lane_path), + StoragePolicy::frozen_default(), + &lane_catalog, + ) + .unwrap(); + seed.storage.reserve(reserve()).unwrap(); + drop(seed); + let executable = std::env::current_exe().unwrap(); + let mut children = Vec::new(); + for _ in 0..2 { + children.push( + Command::new(&executable) + .env(CHILD_STORE, &lane_path) + .env(CHILD_BINDING, "windows-lane-processes") + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + } + for child in children { + finish(child); + } + let reopened = + Handles::open_writable(config(&lane_path), &binding, lane_catalog.cursor_key).unwrap(); + reopened.storage.reserve(reserve()).unwrap(); + if let Some(snapshot) = std::env::var_os("LAYERFS_WINDOWS_STORE_SNAPSHOT_DIR") { + // Copies the lane Store for external verification runs; the sessions + // above are all dropped, so the database is closed when this runs. + let snapshot = PathBuf::from(snapshot); + std::fs::create_dir_all(&snapshot).unwrap(); + std::fs::copy(&lane_path, snapshot.join("store.sqlite")).unwrap(); + for side in ["store.sqlite-wal", "store.sqlite-shm"] { + if let Ok(bytes) = std::fs::read(lane.with_file_name(side)) { + std::fs::write(snapshot.join(side), bytes).unwrap(); + } + } + } +} diff --git a/core/crates/layerfs-store-service/src/install.rs b/core/crates/layerfs-store-service/src/install.rs index 850570a33..bde143c77 100644 --- a/core/crates/layerfs-store-service/src/install.rs +++ b/core/crates/layerfs-store-service/src/install.rs @@ -21,6 +21,7 @@ use std::sync::Arc; /// The daemon's user inside its Sandbox (the runtime's `User` is `0:0`). A /// daemon opens only a Store file and directory it owns, so an installer /// must run as this user too. +#[cfg(not(target_os = "windows"))] const DAEMON_UID: u32 = 0; /// The destination and its exclusive temporary name. @@ -162,11 +163,18 @@ pub fn install_local( installer_user()?; let text = |error: io::Error| error.to_string(); if !directory.exists() { - use std::os::unix::fs::DirBuilderExt; - fs::DirBuilder::new() - .mode(0o700) - .create(directory) - .map_err(text)?; + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new() + .mode(0o700) + .create(directory) + .map_err(text)?; + } + #[cfg(not(unix))] + { + fs::DirBuilder::new().create(directory).map_err(text)?; + } } let (target, temporary) = paths(destination).map_err(text)?; let mut file = claim(&target, &temporary).map_err(text)?; @@ -204,16 +212,24 @@ pub fn install_local( /// Refuses an installer that is not the daemons' user. fn installer_user() -> Result<(), String> { - use std::os::unix::fs::MetadataExt; - // the owner of /proc/self is this process's effective user - let uid = fs::metadata("/proc/self") - .map_err(|error| format!("this process's user is unknown: {error}"))? - .uid(); - if uid != DAEMON_UID { - return Err(format!( - "the Store install runs as user {uid}; it must run as user {DAEMON_UID}, the \ - daemons' user, which opens only a Store it owns" - )); + #[cfg(target_os = "windows")] + { + // Windows has no daemons' uid to check here; the destination's + // local-volume discipline is the kernel guard's, which still applies. + Ok(()) + } + #[cfg(not(target_os = "windows"))] + { + use std::os::unix::fs::MetadataExt; + // the owner of /proc/self is this process's effective user + let uid = fs::metadata("/proc/self") + .map_err(|error| format!("this process's user is unknown: {error}"))? + .uid(); + if uid != DAEMON_UID { + return Err(format!( + "the Store install runs as user {uid}; it must run as user {DAEMON_UID}, the daemons' user, which opens only a Store it owns" + )); + } + Ok(()) } - Ok(()) } diff --git a/core/crates/layerfs-store-service/src/kernel_guard.rs b/core/crates/layerfs-store-service/src/kernel_guard.rs index 49489cac1..3858730bf 100644 --- a/core/crates/layerfs-store-service/src/kernel_guard.rs +++ b/core/crates/layerfs-store-service/src/kernel_guard.rs @@ -1,12 +1,14 @@ //! Whether this process may open the global Store directly (#352 F-020). //! -//! Every opener of one Store must share one Linux kernel and the local -//! filesystem that holds it: SQLite's WAL locks and the writers' admission -//! lane (a `flock` on the Store's directory) coordinate only within one -//! kernel. The guard checks what this process can verify before it opens any -//! file: the platform, and the type of the filesystem holding the Store's -//! directory, from an allowlist of local filesystems. Anything else is -//! `Unsupported` with its reason; there is no other route to fall back to. +//! Every opener of one Store must share one kernel and the local filesystem +//! that holds it: SQLite's WAL locks and the writers' admission lane +//! coordinate only within one kernel. On Linux that is the daemons' kernel +//! over ext4, xfs or btrfs; on Windows it is this machine's kernel over +//! NTFS on a local volume. The guard checks what this process can verify +//! before it opens any file: the platform, and the type of the filesystem +//! holding the Store's directory, from an allowlist of local filesystems. +//! Anything else is `Unsupported` with its reason; there is no other route +//! to fall back to. use std::path::Path; /// Filesystems that carry the Store's locks: ext4, xfs and btrfs. @@ -56,7 +58,31 @@ pub fn check(store: &Path) -> Result<&'static str, Unsupported> { )) }) } - #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "windows")] + { + // The allowlist is the Store's own discipline on Windows: only NTFS + // on a local volume carries the byte-range locks SQLite's WAL needs + // between processes. The audited probe lives in the persistence + // crate's Windows FFI module; network and device paths are refused + // there before a volume is examined. + let directory = store + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + match layerfs_persistence::windows_volume_filesystem(directory) { + Ok(filesystem) if filesystem == "NTFS" => Ok("ntfs"), + Ok(filesystem) => Err(Unsupported(format!( + "filesystem {filesystem} at {} does not carry the Store's locks between \ + processes; direct Store access needs NTFS", + directory.display() + ))), + Err(error) => Err(Unsupported(format!( + "the Store's directory {} cannot be examined: {error}", + directory.display() + ))), + } + } + #[cfg(not(any(target_os = "linux", target_os = "windows")))] { let _ = store; Err(Unsupported( diff --git a/core/tools/check_product_boundary.py b/core/tools/check_product_boundary.py index 7b761f3ce..ef0f35a1d 100644 --- a/core/tools/check_product_boundary.py +++ b/core/tools/check_product_boundary.py @@ -20,8 +20,11 @@ # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { - "layerfs-storage": "src/encoding/codec.rs", - "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-storage": ("src/encoding/codec.rs",), + "layerfs-persistence": ( + "src/backend/sqlite/file_control.rs", + "src/backend/sqlite/windows_lane.rs", + ), } UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") UNSAFE_ROOT_ATTR = { @@ -165,7 +168,7 @@ def unsafe_violations(path, source): relative = Path(*path.parts[path.parts.index("src"):]) code = "\n".join(line.split("//", 1)[0] for line in source.splitlines()) if UNSAFE.search(code): - if crate in UNSAFE_FREE_CRATES or str(relative) != UNSAFE_AUDITED_MODULE[crate]: + if crate in UNSAFE_FREE_CRATES or relative.as_posix() not in UNSAFE_AUDITED_MODULE[crate]: found.append((1, "unsafe outside the audited module boundary; see core/AGENTS.md")) if relative == Path("src/lib.rs") and UNSAFE_ROOT_ATTR[crate] not in source: found.append((1, f"crate root must declare {UNSAFE_ROOT_ATTR[crate]}")) diff --git a/core/tools/test_check_product_boundary.py b/core/tools/test_check_product_boundary.py index 9769a3178..955f18349 100644 --- a/core/tools/test_check_product_boundary.py +++ b/core/tools/test_check_product_boundary.py @@ -50,6 +50,7 @@ def test_persistence_seal_ffi_is_limited_to_the_authorized_module(self): base = Path("core/crates/layerfs-persistence/src") ffi = "fn seal() { unsafe { file_control(); } }" self.assertFalse(unsafe_violations(base / "backend/sqlite/file_control.rs", ffi)) + self.assertFalse(unsafe_violations(base / "backend/sqlite/windows_lane.rs", ffi)) for sibling in ["backend/sqlite/seal.rs", "store/seal.rs", "backend/sqlite/connection.rs"]: self.assertTrue(unsafe_violations(base / sibling, ffi)) self.assertFalse(unsafe_violations(base / "lib.rs", "#![deny(unsafe_code)]")) @@ -279,7 +280,7 @@ def test_shipped_sql_and_external_tests_scope(self): path = core / "crates" / "store" / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") - self.assertEqual([str(p.relative_to(core / "crates" / "store")) + self.assertEqual([p.relative_to(core / "crates" / "store").as_posix() for p in production_files(core)], ["sql/schema.sql", "src/lib.rs", "src/sql/query.sql"]) @@ -290,7 +291,7 @@ def test_flat_api_product_scope(self): path = core / "crates/layerfs-api" / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text("") - self.assertEqual([str(p.relative_to(core / "crates/layerfs-api")) + self.assertEqual([p.relative_to(core / "crates/layerfs-api").as_posix() for p in production_files(core)], ["src/fs/read.rs", "src/lib.rs"]) From 0becc9bc2d869b0c92fbaaca6d9252e343cd3727 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Sun, 25 Oct 2026 12:05:25 +0800 Subject: [PATCH 02/19] feat(sdk): Windows-native project Init, overlay identity and the Store face The SDK's in-process Store face and the construction libraries it needs run natively on Windows (route 2, owner direction 2026-10-25): - layerfs-project Init: host byte encoding moves to import/host.rs; Windows carries names as UTF-8 bytes (lossy outside UTF-8, a recorded limitation) and synthesizes the mode from the read-only attribute. Acquisition identity is the real volume serial and file index from GetFileInformationByHandle in the audited import/windows_identity.rs; zeroed identities made every entry collide onto one dev+ino key and Init refused with "the entry changed while Init read it". The cfg(unix) raw-byte and stat paths are unchanged. - layerfs-overlay: audited database/windows_identity.rs provides handle and path identity (volume serial, file index, link count) and GetDiskFreeSpaceExW free space; physical allocation checks and the startup gate widen to windows. store-service reads overlay backing identity through it instead of the (0,0,0) placeholder. - The napi addon builds on Windows: Store discovery (discovery_windows) uses a named kernel mutex with save/load record files; fs, branch, project and operator calls run in-process through the #352 direct route; workspace, exec and daemon calls refuse Unsupported - FUSE and the daemon stay Linux-only. exec/output/pool/sandbox/workspace stay unix modules; the unix dispatch table is unchanged. - Boundary guard: audited FFI adds project import/windows_identity.rs and overlay database/windows_identity.rs; both crate roots move forbid->deny. Guard self-tests cover both. No tests were deleted or weakened; store-service keeps HEAD's Windows cfg asymmetry warnings no longer (backing delete lets and identity fields are cfg-scoped). Verification at this commit: cargo check --offline --locked for layerfs-persistence, -store-service, -project, -overlay and -node passes on Windows (rustc 1.90.0, only HEAD's pre-existing warnings in untouched crates) and in the rust:1.85.1-bookworm container (zero warnings for all five); tests/windows_store_native.rs 1 passed; guard self-tests 24/24; the lifecycle smoke (init -> fork dev -> one-Commit fs.write -> read back on dev and main -> list_commits -> close, no WSL2, no daemon) passes against this commit's addon. Production LOC: core 96948 -> 97232 (delta +284), sdk 3090 -> 3455 (delta +365); combined 109221 -> 109870 (delta +649). Counter: tools/production_loc.py --detail (pinned at this tree), scopes core and sdk; overlay +146, project +126, store-service +12, layerfs-node +365. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/Cargo.lock | 3 + core/crates/layerfs-overlay/Cargo.toml | 6 + .../layerfs-overlay/src/database/mod.rs | 3 + .../layerfs-overlay/src/database/physical.rs | 65 +++- .../layerfs-overlay/src/database/startup.rs | 6 +- .../src/database/windows_identity.rs | 99 ++++++ core/crates/layerfs-overlay/src/lib.rs | 4 +- core/crates/layerfs-project/Cargo.toml | 4 + .../layerfs-project/src/import/backing.rs | 36 +-- .../layerfs-project/src/import/error.rs | 6 - .../layerfs-project/src/import/files.rs | 18 +- .../crates/layerfs-project/src/import/host.rs | 111 +++++++ .../crates/layerfs-project/src/import/init.rs | 18 +- core/crates/layerfs-project/src/import/mod.rs | 10 +- .../crates/layerfs-project/src/import/scan.rs | 35 ++- .../layerfs-project/src/import/source.rs | 49 +-- .../src/import/windows_identity.rs | 70 +++++ core/crates/layerfs-project/src/lib.rs | 7 +- .../src/overlay/backing.rs | 30 +- core/tools/check_product_boundary.py | 8 +- core/tools/test_check_product_boundary.py | 2 + sdk/js/native/Cargo.toml | 4 + sdk/js/native/src/direct.rs | 2 + sdk/js/native/src/discovery_windows.rs | 281 ++++++++++++++++++ sdk/js/native/src/dispatch.rs | 166 ++++++++--- sdk/js/native/src/lib.rs | 9 + sdk/js/native/src/project.rs | 1 + sdk/js/native/src/registry.rs | 19 +- sdk/js/native/src/trace.rs | 16 +- 29 files changed, 932 insertions(+), 156 deletions(-) create mode 100644 core/crates/layerfs-overlay/src/database/windows_identity.rs create mode 100644 core/crates/layerfs-project/src/import/host.rs create mode 100644 core/crates/layerfs-project/src/import/windows_identity.rs create mode 100644 sdk/js/native/src/discovery_windows.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 0ad436ac4..1a06122f7 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -633,6 +633,7 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "windows-sys", ] [[package]] @@ -642,6 +643,7 @@ dependencies = [ "blake3", "nix", "rusqlite", + "windows-sys", ] [[package]] @@ -668,6 +670,7 @@ dependencies = [ "layerfs-storage", "layerfs-telemetry", "sha2 0.11.0", + "windows-sys", ] [[package]] diff --git a/core/crates/layerfs-overlay/Cargo.toml b/core/crates/layerfs-overlay/Cargo.toml index a0de58150..ba602451a 100644 --- a/core/crates/layerfs-overlay/Cargo.toml +++ b/core/crates/layerfs-overlay/Cargo.toml @@ -13,3 +13,9 @@ rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } [target.'cfg(target_os = "linux")'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } + +[target.'cfg(target_os = "windows")'.dependencies] +# Windows has no system SQLite, and the audited identity module needs the +# Win32 file-information calls; windows-sys is already in the locked graph. +rusqlite = { version = "=0.40.2", features = ["bundled"] } +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } diff --git a/core/crates/layerfs-overlay/src/database/mod.rs b/core/crates/layerfs-overlay/src/database/mod.rs index 17948fa35..9da175d21 100644 --- a/core/crates/layerfs-overlay/src/database/mod.rs +++ b/core/crates/layerfs-overlay/src/database/mod.rs @@ -5,5 +5,8 @@ pub(crate) mod startup; pub(crate) mod statements; pub(crate) mod physical; +#[cfg(target_os = "windows")] +#[allow(unsafe_code)] +pub(crate) mod windows_identity; pub(crate) mod accounting; diff --git a/core/crates/layerfs-overlay/src/database/physical.rs b/core/crates/layerfs-overlay/src/database/physical.rs index 65b7e9b7e..0d1601397 100644 --- a/core/crates/layerfs-overlay/src/database/physical.rs +++ b/core/crates/layerfs-overlay/src/database/physical.rs @@ -94,11 +94,56 @@ impl Allocation { work: self.work.get(), }) } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(target_os = "windows")] + pub(crate) fn new(file: File, path: &Path) -> OverlayResult { + let identity = super::windows_identity::windows_handle_identity(&file)?; + if identity.links != 1 { + return Err(OverlayError::Invalid("allocation file identity")); + } + let len = file.metadata()?.len(); + Ok(Self { + file, + device: identity.device, + inode: identity.index, + path: path.to_owned(), + work: Cell::new(AllocationWork::default()), + high_water: Cell::new(len), + }) + } + #[cfg(target_os = "windows")] + pub(crate) fn state(&self) -> OverlayResult { + let mut work = self.work.get(); + work.observations = work.observations.saturating_add(1); + self.work.set(work); + let meta = self.file.metadata()?; + let by_path = super::windows_identity::windows_path_identity(&self.path)?; + let by_handle = super::windows_identity::windows_handle_identity(&self.file)?; + if by_path.device != by_handle.device + || by_path.index != by_handle.index + || by_handle.device != self.device + || by_handle.index != self.inode + || by_handle.links != 1 + { + return Err(OverlayError::Invalid("allocation file identity")); + } + // Windows exposes no allocated-blocks count; the logical length + // stands in for it (the disposable overlay never preallocates). + let allocated = meta.len(); + self.high_water.set(self.high_water.get().max(allocated)); + Ok(AllocationState { + logical_bytes: meta.len(), + allocated_bytes: allocated, + high_water_allocated_bytes: self.high_water.get(), + reserved_tail_bytes: 0, + cleanup_headroom_bytes: 0, + work: self.work.get(), + }) + } + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] pub(crate) fn new(_file: File, _path: &Path) -> OverlayResult { Err(OverlayError::UnsupportedPlatform) } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] pub(crate) fn state(&self) -> OverlayResult { Err(OverlayError::UnsupportedPlatform) } @@ -119,7 +164,21 @@ impl Allocation { // those are not ordinary writable capacity. Retain the original cause. Ok(filesystem.blocks_available() == 0) } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] + #[cfg(target_os = "windows")] + pub(crate) fn device_full(&self) -> OverlayResult { + self.state()?; + let mut work = self.work.get(); + work.capacity_checks = work.capacity_checks.saturating_add(1); + self.work.set(work); + let directory = self + .path + .parent() + .unwrap_or_else(|| std::path::Path::new(".")); + let free = super::windows_identity::windows_free_bytes(directory) + .map_err(|_| OverlayError::Invalid("capacity observation"))?; + Ok(free == 0) + } + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] pub(crate) fn device_full(&self) -> OverlayResult { Err(OverlayError::UnsupportedPlatform) } diff --git a/core/crates/layerfs-overlay/src/database/startup.rs b/core/crates/layerfs-overlay/src/database/startup.rs index 2e7a33f8a..db39ac8e3 100644 --- a/core/crates/layerfs-overlay/src/database/startup.rs +++ b/core/crates/layerfs-overlay/src/database/startup.rs @@ -66,7 +66,11 @@ fn initialize(path: &Path, config: ProfileConfig, template: Option<&Path>) -> Cr let mut work = CreationWork::default(); let mut allocation = None; let result = (|| { - if !cfg!(any(target_os = "macos", target_os = "linux")) { + if !cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )) { return Err(crate::OverlayError::UnsupportedPlatform); } let mut options = OpenOptions::new(); diff --git a/core/crates/layerfs-overlay/src/database/windows_identity.rs b/core/crates/layerfs-overlay/src/database/windows_identity.rs new file mode 100644 index 000000000..87fe4ba32 --- /dev/null +++ b/core/crates/layerfs-overlay/src/database/windows_identity.rs @@ -0,0 +1,99 @@ +//! The overlay's one audited Windows FFI surface: a file's real identity. +//! +//! This follows the audited-module precedent of the storage codec, the +//! persistence file-control and Windows lanes, and the project host probe: +//! one owner-authorized module of raw Win32 calls, allowed by the boundary +//! guard and compiled on Windows only; no other module in this crate may +//! use unsafe Rust. The allocation's identity checks need the volume serial +//! number, the file index and the link count, which std does not expose. +#![allow(unsafe_code)] +use std::fs::File; +use std::io; +use std::os::windows::io::AsRawHandle; +use std::path::Path; +use windows_sys::Win32::Storage::FileSystem::BY_HANDLE_FILE_INFORMATION; + +/// One entry's Windows identity. +pub struct WindowsIdentity { + pub device: u64, + pub index: u64, + pub links: u64, +} + +/// The identity of the entry an open file already holds, without opening +/// anything new. +pub fn windows_handle_identity(file: &File) -> io::Result { + use windows_sys::Win32::Storage::FileSystem::GetFileInformationByHandle; + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + let read = unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut information) }; + finish(read, information) +} + +/// The identity of `path`'s entry, read as attributes only: no data access, +/// no link traversal; directories need the backup-semantics flag. +pub(crate) fn windows_path_identity(path: &Path) -> io::Result { + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, GetFileInformationByHandle, FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, + }; + let wide: Vec = path + .as_os_str() + .to_string_lossy() + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + let read = unsafe { GetFileInformationByHandle(handle, &mut information) }; + unsafe { CloseHandle(handle) }; + finish(read, information) +} + +fn finish(read: i32, information: BY_HANDLE_FILE_INFORMATION) -> io::Result { + if read == 0 { + return Err(io::Error::last_os_error()); + } + Ok(WindowsIdentity { + device: u64::from(information.dwVolumeSerialNumber), + index: (u64::from(information.nFileIndexHigh) << 32) | u64::from(information.nFileIndexLow), + links: u64::from(information.nNumberOfLinks), + }) +} + +/// The bytes available to the caller on `directory`'s volume. +pub(crate) fn windows_free_bytes(directory: &Path) -> io::Result { + use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW; + let wide: Vec = directory + .as_os_str() + .to_string_lossy() + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let mut free: u64 = 0; + let read = unsafe { + GetDiskFreeSpaceExW( + wide.as_ptr(), + &mut free, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if read == 0 { + return Err(io::Error::last_os_error()); + } + Ok(free) +} diff --git a/core/crates/layerfs-overlay/src/lib.rs b/core/crates/layerfs-overlay/src/lib.rs index 1c38d3418..f4be6e390 100644 --- a/core/crates/layerfs-overlay/src/lib.rs +++ b/core/crates/layerfs-overlay/src/lib.rs @@ -3,7 +3,7 @@ //! The engine owns SQL and profiling. Filesystem validation, scheduling, kernel //! replies and process lifetimes belong to its callers. Windows bound each job; //! they do not limit total files, changes, payload or Workspace lifetime. -#![forbid(unsafe_code)] +#![deny(unsafe_code)] mod contract; mod database; @@ -46,6 +46,8 @@ pub use contract::kernel_directory::{ pub use database::accounting::{Resources, StoredCounts}; pub use database::physical::{AllocationState, AllocationWork, ClosedFile}; pub use database::startup::{Creation, CreationWork}; +#[cfg(target_os = "windows")] +pub use database::windows_identity::windows_handle_identity; pub use db::Overlay; pub use diagnostics::payload::PayloadWork; pub use error::{CloseFailure, OverlayError, OverlayResult}; diff --git a/core/crates/layerfs-project/Cargo.toml b/core/crates/layerfs-project/Cargo.toml index 36cc2a5fc..8338c36bd 100644 --- a/core/crates/layerfs-project/Cargo.toml +++ b/core/crates/layerfs-project/Cargo.toml @@ -12,6 +12,10 @@ layerfs-storage = { path = "../layerfs-storage" } layerfs-history = { path = "../layerfs-history", default-features = false } layerfs-telemetry = { path = "../layerfs-telemetry" } +[target.'cfg(target_os = "windows")'.dependencies] +# The audited host-identity probe; already in the locked graph. +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } + [dev-dependencies] layerfs-persistence = { path = "../layerfs-persistence" } sha2 = "=0.11.0" diff --git a/core/crates/layerfs-project/src/import/backing.rs b/core/crates/layerfs-project/src/import/backing.rs index 0684a071d..31906e98b 100644 --- a/core/crates/layerfs-project/src/import/backing.rs +++ b/core/crates/layerfs-project/src/import/backing.rs @@ -13,10 +13,7 @@ use layerfs_storage::port::acquisition::{ WRITE_ROW_BYTES, WRITE_WINDOW_BYTES, WRITE_WINDOW_ROWS, }; use layerfs_storage::port::PersistenceError; -use std::{ - cell::Cell, collections::VecDeque, ffi::OsString, fs::Metadata, os::unix::ffi::OsStringExt, - os::unix::fs::MetadataExt, path::PathBuf, -}; +use std::{cell::Cell, collections::VecDeque, fs::Metadata}; /// Rows the job queue or one resident child buffer may hold. pub(crate) const WINDOW_ROWS: usize = 512; @@ -34,21 +31,24 @@ pub(crate) struct Stamp { pub ctime_nsec: i64, } impl Stamp { - pub(crate) fn of(metadata: &Metadata) -> Self { - Self { - dev: metadata.dev(), - ino: metadata.ino(), - len: metadata.len(), - mode: metadata.mode(), - mtime: metadata.mtime(), - mtime_nsec: metadata.mtime_nsec(), - ctime: metadata.ctime(), - ctime_nsec: metadata.ctime_nsec(), - } + pub(crate) fn of(path: &std::path::Path, metadata: &Metadata) -> std::io::Result { + let host = super::host::HostStat::of(path, metadata)?; + Ok(Self { + dev: host.dev, + ino: host.ino, + len: host.len, + mode: host.mode, + mtime: host.mtime, + mtime_nsec: host.mtime_nsec, + ctime: host.ctime, + ctime_nsec: host.ctime_nsec, + }) } /// True while `metadata` is still the same unchanged regular file. - pub(crate) fn matches_file(&self, metadata: &Metadata) -> bool { - metadata.is_file() && !metadata.file_type().is_symlink() && *self == Self::of(metadata) + pub(crate) fn matches_file(&self, path: &std::path::Path, metadata: &Metadata) -> bool { + metadata.is_file() + && !metadata.file_type().is_symlink() + && Self::of(path, metadata).is_ok_and(|now| now == *self) } /// The identity and byte-exact evidence every path of this inode must share. pub(crate) fn identity(&self) -> HostIdentity { @@ -377,7 +377,7 @@ pub(crate) fn window_full(rows: usize, held: usize, bytes: usize) -> bool { fn changed_source(error: AcquisitionError, path: impl Fn(u64) -> Option>) -> Failure { if let AcquisitionError::Changed { position } = error { if let Some(path) = path(position) { - let path = PathBuf::from(OsString::from_vec(path)); + let path = super::host::bytes_to_path(&path); return Failure::refused(SourceRefusal::Changed, path); } } diff --git a/core/crates/layerfs-project/src/import/error.rs b/core/crates/layerfs-project/src/import/error.rs index d2d75e828..5d46b175c 100644 --- a/core/crates/layerfs-project/src/import/error.rs +++ b/core/crates/layerfs-project/src/import/error.rs @@ -115,7 +115,6 @@ impl From for ProjectError { Self::Io(error) } } -#[cfg(unix)] impl ProjectError { /// True when a persistence outcome behind this failure is unknown. pub(crate) fn unknown_outcome(&self) -> bool { @@ -129,7 +128,6 @@ impl ProjectError { } } } -#[cfg(unix)] fn storage_unknown(error: &StorageError) -> bool { match error { StorageError::UnknownOutcome { .. } => true, @@ -139,20 +137,16 @@ fn storage_unknown(error: &StorageError) -> bool { _ => false, } } -#[cfg(unix)] pub(crate) fn content(error: ContentError) -> ProjectError { ProjectError::Content(error) } -#[cfg(unix)] pub(crate) fn storage(error: StorageError) -> ProjectError { ProjectError::Storage(error) } -#[cfg(unix)] pub(crate) fn acquisition(error: AcquisitionError) -> ProjectError { ProjectError::Acquisition(error) } /// A backing answer that contradicts what this operation itself wrote. -#[cfg(unix)] pub(crate) fn malformed() -> ProjectError { ProjectError::Acquisition(AcquisitionError::Persistence(PersistenceError::Malformed)) } diff --git a/core/crates/layerfs-project/src/import/files.rs b/core/crates/layerfs-project/src/import/files.rs index 956402b04..bbd6e2ca5 100644 --- a/core/crates/layerfs-project/src/import/files.rs +++ b/core/crates/layerfs-project/src/import/files.rs @@ -17,9 +17,7 @@ use layerfs_storage::port::acquisition::{self, WRITE_WINDOW_ROWS}; use layerfs_storage::{Save, Storage}; use layerfs_telemetry::timer::Timing; use std::{ - ffi::{OsStr, OsString}, fs::{self, File}, - os::unix::ffi::{OsStrExt, OsStringExt}, path::PathBuf, sync::{mpsc, Arc, Mutex}, time::{Duration, Instant}, @@ -39,7 +37,7 @@ impl Job { Self { id: job.position, stamp: Stamp::from_identity(&job.host), - path: PathBuf::from(OsString::from_vec(job.host_path)), + path: super::host::bytes_to_path(&job.host_path), } } } @@ -171,7 +169,7 @@ fn check_aliases(backing: &Backing<'_>, progress: &mut ImportProgress) -> Result }; let parent = entry.key.parent.ok_or_else(malformed)?; if directory.as_ref().is_none_or(|(held, _)| *held != parent) { - let path = OsString::from_vec(backing.directory_path(parent)?); + let path = super::host::bytes_to_os(&backing.directory_path(parent)?); directory = Some((parent, PathBuf::from(path))); } if first.is_none_or(|(held, _)| held != canonical) { @@ -181,8 +179,8 @@ fn check_aliases(backing: &Backing<'_>, progress: &mut ImportProgress) -> Result let (Some((_, directory)), Some((_, stamp))) = (&directory, &first) else { return Err(malformed()); }; - let path = directory.join(OsStr::from_bytes(&entry.key.name)); - if !stamp.matches_file(&fs::symlink_metadata(&path)?) { + let path = directory.join(super::host::bytes_to_os(&entry.key.name)); + if !stamp.matches_file(&path, &fs::symlink_metadata(&path)?) { return Err(Failure::refused(SourceRefusal::Changed, path)); } } @@ -200,7 +198,7 @@ fn construct_file( return Err(Failure::Deadline); } let mut file = File::open(&job.path)?; - if !job.stamp.matches_file(&file.metadata()?) { + if !job.stamp.matches_file(&job.path, &file.metadata()?) { return Err(Failure::refused(SourceRefusal::Changed, &job.path)); } let (result, _) = Timing::disabled("history.import_file", |scope| { @@ -214,8 +212,10 @@ fn construct_file( }); let built = result.map_err(content)?; if built.logical_len != job.stamp.len - || !job.stamp.matches_file(&file.metadata()?) - || !job.stamp.matches_file(&fs::symlink_metadata(&job.path)?) + || !job.stamp.matches_file(&job.path, &file.metadata()?) + || !job + .stamp + .matches_file(&job.path, &fs::symlink_metadata(&job.path)?) { return Err(Failure::refused(SourceRefusal::Changed, &job.path)); } diff --git a/core/crates/layerfs-project/src/import/host.rs b/core/crates/layerfs-project/src/import/host.rs new file mode 100644 index 000000000..7e470adea --- /dev/null +++ b/core/crates/layerfs-project/src/import/host.rs @@ -0,0 +1,111 @@ +//! Host byte encoding of paths and names, and the metadata identity Init +//! reads. Unix uses the raw bytes and the stat identity; Windows has no raw +//! byte spelling, so names are carried as their UTF-8 bytes (a name outside +//! UTF-8 becomes lossy — a recorded limitation of this route), and the +//! identity is the entry's real volume serial number and file index, its +//! length, timestamps and a mode synthesized from the read-only attribute. +//! Hard-link membership is not detected on Windows (the format supports it; +//! each link acquires as its own file). +use std::ffi::{OsStr, OsString}; +use std::fs::Metadata; +use std::io; +use std::path::{Path, PathBuf}; + +/// One host name's bytes. +pub(super) fn os_to_bytes(value: &OsStr) -> Vec { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + value.as_bytes().to_vec() + } + #[cfg(windows)] + { + value.to_string_lossy().into_owned().into_bytes() + } +} + +/// The host name of `value`'s bytes. +pub(super) fn bytes_to_os(value: &[u8]) -> OsString { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStringExt; + OsString::from_vec(value.to_vec()) + } + #[cfg(windows)] + { + OsString::from(String::from_utf8_lossy(value).into_owned()) + } +} + +/// The host path of `value`'s bytes. +pub(super) fn bytes_to_path(value: &[u8]) -> PathBuf { + PathBuf::from(bytes_to_os(value)) +} + +/// The host identity fields Init observes. +#[derive(Clone, Copy, Eq, PartialEq)] +pub(crate) struct HostStat { + pub dev: u64, + pub ino: u64, + pub len: u64, + pub mode: u32, + pub mtime: i64, + pub mtime_nsec: i64, + pub ctime: i64, + pub ctime_nsec: i64, +} + +impl HostStat { + /// Reads `path`'s identity. The acquisition keys rows by device and file + /// index: a real identity is required, never a silent zero that would + /// collide every file onto one key. + pub(super) fn of(path: &Path, metadata: &Metadata) -> io::Result { + #[cfg(unix)] + { + let _ = path; + use std::os::unix::fs::MetadataExt; + Ok(Self { + dev: metadata.dev(), + ino: metadata.ino(), + len: metadata.len(), + mode: metadata.mode(), + mtime: metadata.mtime(), + mtime_nsec: metadata.mtime_nsec(), + ctime: metadata.ctime(), + ctime_nsec: metadata.ctime_nsec(), + }) + } + #[cfg(windows)] + { + // The format stores no ownership; mode is synthesized from the + // read-only attribute, and creation time stands in for ctime. + let base = if metadata.is_dir() { 0o555 } else { 0o644 }; + let mode = if metadata.permissions().readonly() { + base + } else { + base | 0o200 + }; + let (mtime, mtime_nsec) = match metadata.modified() { + Ok(time) => match time.duration_since(std::time::UNIX_EPOCH) { + Ok(span) => (span.as_secs() as i64, i64::from(span.subsec_nanos())), + Err(before) => ( + -(before.duration().as_secs() as i64), + -(i64::from(before.duration().subsec_nanos())), + ), + }, + Err(_) => (0, 0), + }; + let identity = super::windows_identity::windows_file_identity(path)?; + Ok(Self { + dev: identity.device, + ino: identity.index, + len: metadata.len(), + mode, + mtime, + mtime_nsec, + ctime: identity.created, + ctime_nsec: identity.created_nsec, + }) + } + } +} diff --git a/core/crates/layerfs-project/src/import/init.rs b/core/crates/layerfs-project/src/import/init.rs index 3a068b58d..7eaba3e09 100644 --- a/core/crates/layerfs-project/src/import/init.rs +++ b/core/crates/layerfs-project/src/import/init.rs @@ -1,8 +1,6 @@ //! Scan, construct/save, consume inode identities and publish one genesis stack. use crate::import::observe::InitCounters; -#[cfg(unix)] use crate::import::observe::InitStep; -#[cfg(unix)] use crate::{ backing::Backing, error::{acquisition, storage}, @@ -11,16 +9,12 @@ use crate::{ scan, }; use crate::{ProjectError, ProjectResult}; -#[cfg(unix)] use layerfs_content::filesystem::{profile_id, scope_for_seed, FilesystemObjects}; use layerfs_content::ObjectId; use layerfs_history::{HistoryCatalog, HistoryName, LayerStackId, LayerStackRecord}; -#[cfg(unix)] use layerfs_history::{ReserveRequest, StackInitialization}; use layerfs_storage::port::acquisition::Acquisition; -#[cfg(unix)] use layerfs_storage::port::acquisition::Begin; -#[cfg(unix)] use layerfs_storage::Save; use layerfs_storage::{read::Diagnostics, Storage}; use layerfs_telemetry::timer::{Active, TimingScope}; @@ -88,7 +82,6 @@ pub fn init( /// published. A failure of that removal is [`ProjectError::Cleanup`]; an /// unknown Store, history or backing outcome is [`ProjectError::Uncertain`] /// and leaves the operation untouched for its owner. -#[cfg(unix)] pub fn init_observed( store: &Storage, catalog: &dyn HistoryCatalog, @@ -96,7 +89,6 @@ pub fn init_observed( timer: &TimingScope<'_, Active>, counters: Arc, ) -> ProjectResult { - use std::os::unix::fs::MetadataExt; counters.step(InitStep::Scan); let mut progress = ImportProgress::observed(request.deadline, counters); progress.tick()?; @@ -107,9 +99,10 @@ pub fn init_observed( let scope = scope_for_seed(request.scope_seed); let mut stack = [0; 16]; stack.copy_from_slice(&request.stack.to_bytes()[1..]); + let host = super::host::HostStat::of(&request.source, &source)?; let begun = request.acquisition.begin(&Begin { - source_device: source.dev(), - source_inode: source.ino(), + source_device: host.dev, + source_inode: host.ino, stack, scope: scope.object(), }); @@ -176,7 +169,6 @@ pub fn init_observed( /// Scans with attributes, constructs each file once, consumes serials and /// builds the tree. The returned tree save is unfinished. -#[cfg(unix)] fn acquire<'a>( store: &'a Storage, catalog: &dyn HistoryCatalog, @@ -237,8 +229,8 @@ fn acquire<'a>( Ok((root, reservation.start, scanned.entries, save)) } -/// Refuses host Init where the required Unix metadata contract is unavailable. -#[cfg(not(unix))] +/// Refuses host Init where the host metadata contract is unavailable. +#[cfg(not(any(unix, target_os = "windows")))] pub fn init_observed( _storage: &Storage, _catalog: &dyn HistoryCatalog, diff --git a/core/crates/layerfs-project/src/import/mod.rs b/core/crates/layerfs-project/src/import/mod.rs index f1093b1a2..bdd446f66 100644 --- a/core/crates/layerfs-project/src/import/mod.rs +++ b/core/crates/layerfs-project/src/import/mod.rs @@ -1,18 +1,16 @@ //! Responsibility-scoped implementation modules and reexports. -#[cfg(unix)] pub(crate) mod backing; -#[cfg(unix)] pub(crate) mod batch; pub(crate) mod error; -#[cfg(unix)] pub(crate) mod files; pub(crate) mod hints; +pub(crate) mod host; pub(crate) mod init; -#[cfg(unix)] pub(crate) mod namespace; pub(crate) mod observe; -#[cfg(unix)] pub(crate) mod scan; -#[cfg(unix)] pub(crate) mod source; +#[cfg(target_os = "windows")] +#[allow(unsafe_code)] +mod windows_identity; pub(crate) mod work; diff --git a/core/crates/layerfs-project/src/import/scan.rs b/core/crates/layerfs-project/src/import/scan.rs index 3ee3621b7..03458a6ba 100644 --- a/core/crates/layerfs-project/src/import/scan.rs +++ b/core/crates/layerfs-project/src/import/scan.rs @@ -17,12 +17,7 @@ use layerfs_content::filesystem::symlink::{emit_symlink, SymlinkTarget}; use layerfs_content::filesystem::FilesystemObjects; use layerfs_content::{inode_leaf::InodeKind, PathName}; use layerfs_storage::port::acquisition::{EntryKey, NewEntry, Placed}; -use std::{ - ffi::{OsStr, OsString}, - fs::{self, Metadata}, - os::unix::ffi::{OsStrExt, OsStringExt}, - path::{Path, PathBuf}, -}; +use std::{fs::{self, Metadata}, path::Path}; /// What one completed scan placed in backing. pub(crate) struct Scanned { @@ -84,11 +79,11 @@ impl Roots<'_, '_> { ) -> Result { let path = || match parent { Some(_) => child_path(directory, &child.name), - None => directory.as_os_str().as_bytes().to_vec(), + None => super::host::os_to_bytes(directory.as_os_str()), }; - let mode = child.portable_mode().map_err(|reason| { - Failure::refused(reason, PathBuf::from(OsString::from_vec(path()))) - })?; + let mode = child + .portable_mode() + .map_err(|reason| Failure::refused(reason, super::host::bytes_to_path(&path())))?; let nanos = child.stamp.mtime_nsec as u32; let key = (child.kind.code(), mode, child.stamp.mtime, nanos); let metadata = PortableMetadata { @@ -125,10 +120,11 @@ impl Roots<'_, '_> { } fn child_path(directory: &Path, name: &[u8]) -> Vec { - directory - .join(OsStr::from_bytes(name)) - .into_os_string() - .into_vec() + super::host::os_to_bytes( + &directory + .join(super::host::bytes_to_os(name)) + .into_os_string(), + ) } /// Positions being assigned while directories are read. @@ -170,7 +166,7 @@ pub(crate) fn scan( name: Vec::new(), kind: InodeKind::Directory, target: None, - stamp: Stamp::of(root), + stamp: Stamp::of(source, root)?, }; let root = roots.entry(root, None, Some(0), source)?; placement @@ -191,12 +187,15 @@ pub(crate) fn scan( placement.frontier -= 1; progress.tick()?; let parent = directory.position; - let path = PathBuf::from(OsString::from_vec(directory.host_path)); + let path = super::host::bytes_to_path(&directory.host_path); let (mut count, mut wide) = (0usize, false); for child in fs::read_dir(&path)? { progress.tick()?; let child = child?; - buffer.push(observe(&child.path(), child.file_name().as_bytes())?); + buffer.push(observe( + &child.path(), + &super::host::os_to_bytes(&child.file_name()), + )?); count += 1; progress.counters.entry(); if buffer.len() == WINDOW_ROWS { @@ -305,6 +304,6 @@ fn observe(path: &Path, name: &[u8]) -> Result { name: name.to_vec(), kind, target, - stamp: Stamp::of(&metadata), + stamp: Stamp::of(path, &metadata)?, }) } diff --git a/core/crates/layerfs-project/src/import/source.rs b/core/crates/layerfs-project/src/import/source.rs index 4640861a5..1c9443fbc 100644 --- a/core/crates/layerfs-project/src/import/source.rs +++ b/core/crates/layerfs-project/src/import/source.rs @@ -1,9 +1,9 @@ //! Host source observations; symbolic links are stored without traversal. +use super::host::{os_to_bytes, HostStat}; use crate::{ProjectError, SourceRefusal}; use layerfs_content::filesystem::symlink::SymlinkTarget; use std::{ fs::{self, Metadata}, - os::unix::{ffi::OsStrExt, fs::MetadataExt}, path::Path, }; @@ -13,20 +13,13 @@ pub(super) fn read_link(path: &Path, scanned: &Metadata) -> Result, Proj let target = fs::read_link(path)?; let after = fs::symlink_metadata(path)?; if !after.file_type().is_symlink() - || after.dev() != scanned.dev() - || after.ino() != scanned.ino() - || after.len() != scanned.len() - || after.mode() != scanned.mode() - || after.mtime() != scanned.mtime() - || after.mtime_nsec() != scanned.mtime_nsec() - || after.ctime() != scanned.ctime() - || after.ctime_nsec() != scanned.ctime_nsec() + || HostStat::of(path, &after)? != HostStat::of(path, scanned)? { return Err(ProjectError::refused(SourceRefusal::Changed, path)); } // Keep the canonical target bound/grammar: no normalization or UTF-8 cast. - let target = SymlinkTarget::new(target.as_os_str().as_bytes().to_vec()) - .map_err(ProjectError::Content)?; + let target = + SymlinkTarget::new(os_to_bytes(target.as_os_str())).map_err(ProjectError::Content)?; Ok(target.as_bytes().to_vec()) } @@ -34,18 +27,34 @@ pub(super) fn read_link(path: &Path, scanned: &Metadata) -> Result, Proj /// /// The placement is resolved through its links first, then each ancestor's /// host identity is compared with the source's, so a symlinked or otherwise -/// aliased spelling of a directory inside the source is recognised. Nothing is -/// begun or changed: a backing inside the source would both alter the source +/// aliased spelling of a directory inside the source is recognised. Nothing +/// is begun or changed: a backing inside the source would both alter the source /// while it is read and be acquired as part of it. pub(super) fn outside_source(source: &Metadata, placement: &Path) -> Result<(), ProjectError> { + #[cfg(unix)] let mut ancestor = fs::canonicalize(placement)?; - loop { - let metadata = fs::symlink_metadata(&ancestor)?; - if metadata.dev() == source.dev() && metadata.ino() == source.ino() { - return Err(ProjectError::BackingInsideSource); - } - if !ancestor.pop() { - return Ok(()); + #[cfg(not(unix))] + let ancestor = fs::canonicalize(placement)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + loop { + let metadata = fs::symlink_metadata(&ancestor)?; + if metadata.dev() == source.dev() && metadata.ino() == source.ino() { + return Err(ProjectError::BackingInsideSource); + } + if !ancestor.pop() { + return Ok(()); + } } } + #[cfg(windows)] + { + // Windows std carries no path-independent identity here, and the + // entry's own identity probe is per-path; an aliased spelling of + // the source is not recognised by this check (a recorded + // limitation). + let _ = (source, &ancestor); + Ok(()) + } } diff --git a/core/crates/layerfs-project/src/import/windows_identity.rs b/core/crates/layerfs-project/src/import/windows_identity.rs new file mode 100644 index 000000000..7649ce860 --- /dev/null +++ b/core/crates/layerfs-project/src/import/windows_identity.rs @@ -0,0 +1,70 @@ +//! The Windows host-identity probe of namespace Init. +//! +//! This is layerfs-project's one owner-authorized module of raw Win32 +//! calls, allowed by the boundary guard and compiled on Windows only; no +//! other module in this crate may use unsafe Rust. The acquisition keys +//! rows by device and file index; the volume serial number and file index +//! are the entry's real identity. +#![allow(unsafe_code)] +use std::io; +use std::path::Path; + +/// The Windows host identity of one entry: its volume serial number, its +/// file index and its creation time as Unix seconds and nanoseconds. +pub(crate) struct WindowsFileIdentity { + pub(crate) device: u64, + pub(crate) index: u64, + pub(crate) created: i64, + pub(crate) created_nsec: i64, +} + +/// Reads `path`'s volume, file index and creation time without opening the +/// entry's data: attributes only, with traversal of neither links nor +/// content. Directories need the backup-semantics flag. +pub(crate) fn windows_file_identity(path: &Path) -> io::Result { + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, + FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + OPEN_EXISTING, + }; + let wide: Vec = path + .as_os_str() + .to_string_lossy() + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + let read = unsafe { GetFileInformationByHandle(handle, &mut information) }; + unsafe { CloseHandle(handle) }; + if read == 0 { + return Err(io::Error::last_os_error()); + } + // FILETIME is 100ns ticks since 1601-01-01; Unix time starts + // 11644473600 seconds later. + const EPOCH_DELTA_SECONDS: u64 = 11_644_473_600; + let created_ticks = (u64::from(information.ftCreationTime.dwHighDateTime) << 32) + | u64::from(information.ftCreationTime.dwLowDateTime); + let created_unix = (created_ticks / 10_000_000).saturating_sub(EPOCH_DELTA_SECONDS); + let created_subsec = (created_ticks % 10_000_000) * 100; + Ok(WindowsFileIdentity { + device: u64::from(information.dwVolumeSerialNumber), + index: (u64::from(information.nFileIndexHigh) << 32) | u64::from(information.nFileIndexLow), + created: created_unix as i64, + created_nsec: created_subsec as i64, + }) +} diff --git a/core/crates/layerfs-project/src/lib.rs b/core/crates/layerfs-project/src/lib.rs index d80033f92..54f188e31 100644 --- a/core/crates/layerfs-project/src/lib.rs +++ b/core/crates/layerfs-project/src/lib.rs @@ -1,20 +1,15 @@ //! Namespace Init over the content, storage and history contracts. -#![forbid(unsafe_code)] +#![deny(unsafe_code)] #![deny(missing_docs)] mod import; pub use error::{ProjectError, ProjectResult, RetainedAcquisition, SourceRefusal}; -#[cfg(unix)] pub(crate) use import::backing; -#[cfg(unix)] pub(crate) use import::batch; pub(crate) use import::error; -#[cfg(unix)] pub(crate) use import::files; pub(crate) use import::init; -#[cfg(unix)] pub(crate) use import::namespace; pub use import::observe::{InitCounters, InitCounts, InitStep}; -#[cfg(unix)] pub(crate) use import::scan; pub use init::{init, init_observed, InitRequest, Initialized}; diff --git a/core/crates/layerfs-store-service/src/overlay/backing.rs b/core/crates/layerfs-store-service/src/overlay/backing.rs index de78d4787..738be4aac 100644 --- a/core/crates/layerfs-store-service/src/overlay/backing.rs +++ b/core/crates/layerfs-store-service/src/overlay/backing.rs @@ -14,7 +14,11 @@ pub struct Backing { file: Option, namespace: i64, incarnation: [u8; 32], + // The deletion identity check reads these on unix only; Windows and the + // other platforms refuse deletion before reaching them. + #[cfg_attr(not(unix), allow(dead_code))] device: u64, + #[cfg_attr(not(unix), allow(dead_code))] inode: u64, pool: Weak, terminal: AtomicBool, @@ -42,7 +46,19 @@ impl Backing { } (metadata.dev(), metadata.ino(), metadata.nlink()) }; - #[cfg(not(unix))] + #[cfg(target_os = "windows")] + let (device, inode, links) = { + // The audited probe in the overlay engine reads the entry's real + // volume, file index and link count. + let identity = layerfs_overlay::windows_handle_identity(&file).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("overlay file identity: {error}"), + ) + })?; + (identity.device, identity.index, identity.links) + }; + #[cfg(not(any(unix, target_os = "windows")))] let (device, inode, links) = (0, 0, 0); if !metadata.is_file() || links != 1 { return Err(io::Error::new( @@ -69,14 +85,14 @@ impl Backing { self.closed.store(true, Ordering::Release); } fn delete(&self) -> io::Result<()> { - let metadata = std::fs::symlink_metadata(&self.path)?; - let original = self - .file - .as_ref() - .expect("retained identity file") - .metadata()?; #[cfg(unix)] { + let metadata = std::fs::symlink_metadata(&self.path)?; + let original = self + .file + .as_ref() + .expect("retained identity file") + .metadata()?; use std::os::unix::fs::MetadataExt; if !metadata.is_file() || metadata.nlink() != 1 diff --git a/core/tools/check_product_boundary.py b/core/tools/check_product_boundary.py index ef0f35a1d..0790fb25e 100644 --- a/core/tools/check_product_boundary.py +++ b/core/tools/check_product_boundary.py @@ -25,15 +25,17 @@ "src/backend/sqlite/file_control.rs", "src/backend/sqlite/windows_lane.rs", ), + "layerfs-project": ("src/import/windows_identity.rs",), + "layerfs-overlay": ("src/database/windows_identity.rs",), } -UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") +UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", "layerfs-content": "#![forbid(unsafe_code)]", "layerfs-telemetry": "#![forbid(unsafe_code)]", "layerfs-persistence": "#![deny(unsafe_code)]", - "layerfs-project": "#![forbid(unsafe_code)]", - "layerfs-overlay": "#![forbid(unsafe_code)]", + "layerfs-project": "#![deny(unsafe_code)]", + "layerfs-overlay": "#![deny(unsafe_code)]", "layerfs-workspace": "#![forbid(unsafe_code)]", "layerfs-store-service": "#![forbid(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", diff --git a/core/tools/test_check_product_boundary.py b/core/tools/test_check_product_boundary.py index 955f18349..e9e41cff5 100644 --- a/core/tools/test_check_product_boundary.py +++ b/core/tools/test_check_product_boundary.py @@ -51,6 +51,8 @@ def test_persistence_seal_ffi_is_limited_to_the_authorized_module(self): ffi = "fn seal() { unsafe { file_control(); } }" self.assertFalse(unsafe_violations(base / "backend/sqlite/file_control.rs", ffi)) self.assertFalse(unsafe_violations(base / "backend/sqlite/windows_lane.rs", ffi)) + self.assertFalse(unsafe_violations(Path("core/crates/layerfs-project/src/import/windows_identity.rs"), ffi)) + self.assertFalse(unsafe_violations(Path("core/crates/layerfs-overlay/src/database/windows_identity.rs"), ffi)) for sibling in ["backend/sqlite/seal.rs", "store/seal.rs", "backend/sqlite/connection.rs"]: self.assertTrue(unsafe_violations(base / sibling, ffi)) self.assertFalse(unsafe_violations(base / "lib.rs", "#![deny(unsafe_code)]")) diff --git a/sdk/js/native/Cargo.toml b/sdk/js/native/Cargo.toml index 9dcb183b9..2179bfac6 100644 --- a/sdk/js/native/Cargo.toml +++ b/sdk/js/native/Cargo.toml @@ -26,3 +26,7 @@ napi-derive = "=2.16.13" [build-dependencies] napi-build = "=2.1.6" + +[target.'cfg(target_os = "windows")'.dependencies] +# The Windows discovery guard's named kernel mutex; already in the locked graph. +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_System_Threading"] } diff --git a/sdk/js/native/src/direct.rs b/sdk/js/native/src/direct.rs index a7dcb5895..b07102a67 100644 --- a/sdk/js/native/src/direct.rs +++ b/sdk/js/native/src/direct.rs @@ -173,6 +173,7 @@ pub(crate) fn call(method: &str, args: SdkValue, operator: bool) -> Outcome { } /// The Branch id of `name` in the project a call names, read in this process. +#[cfg(unix)] pub(crate) fn branch_id( args: &SdkValue, name: &str, @@ -183,6 +184,7 @@ pub(crate) fn branch_id( } /// The newest Commit of `branch`, read in this process; `None` when it has none. +#[cfg(unix)] pub(crate) fn head( args: &SdkValue, branch: layerfs_history::BranchId, diff --git a/sdk/js/native/src/discovery_windows.rs b/sdk/js/native/src/discovery_windows.rs new file mode 100644 index 000000000..587f5bdf4 --- /dev/null +++ b/sdk/js/native/src/discovery_windows.rs @@ -0,0 +1,281 @@ +//! Private, permission-authorized project discovery on Windows: the Store +//! face of the unix `discovery` module. Sandbox discovery and daemon +//! operation are refused `Unsupported`; the native Store route serves +//! project, fs and branch calls in-process. +//! +//! Windows cannot express the unix entry's owner/mode privacy check; a +//! symlink and an unexpected entry kind are still refused, and hardening the +//! home tree's ACLs is an open owner question. The discovery turn keeps the +//! unix discipline: this process's calls take turns on one directory, and +//! between processes one named kernel mutex per directory is held, released +//! by the kernel when its holder exits. +use crate::{ + job::{failed, refused}, + registry::{self, Project}, +}; +use layerfs_bridge::{ + control::{Reply, SdkRefusal, SdkValue}, + provision::StoreManifest, +}; +use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; +use std::{ + collections::BTreeSet, + fs::{self, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::{Condvar, Mutex}, + time::{Duration, Instant}, +}; + +type Result = std::result::Result; + +pub(crate) fn home(args: &SdkValue) -> Result { + let home = args + .get_text("home") + .map(PathBuf::from) + .or_else(|| std::env::var_os("LAYERFS_HOME").map(PathBuf::from)) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_HOME is required"))?; + if !home.is_absolute() { + return Err(refused("InvalidInput", "LAYERFS_HOME must be absolute")); + } + directory(&home)?; + Ok(home) +} + +/// A private entry on Windows: no symlink and the expected kind. Owner and +/// mode bits have no Windows reading here; hardening the home tree's ACLs is +/// an open owner question. +fn private(path: &Path, dir: bool) -> Result<()> { + let meta = fs::symlink_metadata(path).map_err(failed)?; + if meta.file_type().is_symlink() || meta.is_dir() != dir { + return Err(refused( + "Unauthorized", + format!( + "{} must be a private {} entry", + path.display(), + if dir { "directory" } else { "file" } + ), + )); + } + Ok(()) +} + +pub(crate) fn directory(path: &Path) -> Result<()> { + match fs::DirBuilder::new().create(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(failed(e)), + } + private(path, true) +} + +pub(crate) fn project_dir(args: &SdkValue) -> Result { + let name = registry::text(args, "project")?; + layerfs_history::HistoryName::new(name).map_err(failed)?; + let path = home(args)?.join(name); + private(&path, true)?; + Ok(path) +} + +/// Calls take turns on one directory: in this process before its mutex, and +/// between processes on the mutex itself. The wait is readiness, bounded, +/// and nothing has been attempted before it ends. +const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); +static HELD: Mutex> = Mutex::new(BTreeSet::new()); +static FREED: Condvar = Condvar::new(); + +/// One discovery mutation's ownership of a directory: this process's turn +/// and the named kernel mutex. Dropping it releases both. +pub(crate) struct Guard { + handle: usize, + directory: PathBuf, +} +impl Drop for Guard { + fn drop(&mut self) { + unsafe { + windows_sys::Win32::System::Threading::ReleaseMutex(self.handle as _); + windows_sys::Win32::Foundation::CloseHandle(self.handle as _); + } + let mut held = HELD.lock().unwrap_or_else(|error| error.into_inner()); + held.remove(&self.directory); + FREED.notify_all(); + } +} + +/// One mutex name per directory; the canonical path identifies it between +/// processes. +fn mutex_name(directory: &Path) -> String { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut hasher = DefaultHasher::new(); + match fs::canonicalize(directory) { + Ok(canonical) => canonical.hash(&mut hasher), + Err(_) => directory.hash(&mut hasher), + } + format!("Local\\layerfs-discovery-{:016x}", hasher.finish()) +} + +pub(crate) fn guard(directory: &Path) -> Result { + let deadline = Instant::now() + IN_PROCESS_WAIT; + let mut held = registry::lock(&HELD)?; + while held.contains(directory) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(refused( + "Busy", + "another call of this process owns this discovery mutation", + )); + } + held = FREED + .wait_timeout(held, remaining) + .map_err(|_| refused("Failed", "discovery turn poisoned"))? + .0; + } + held.insert(directory.to_owned()); + drop(held); + match os_lock(directory, deadline) { + Ok(handle) => Ok(Guard { + handle, + directory: directory.to_owned(), + }), + Err(refusal) => { + let mut held = HELD.lock().unwrap_or_else(|error| error.into_inner()); + held.remove(directory); + FREED.notify_all(); + Err(refusal) + } + } +} + +/// The named kernel mutex of a directory, waited for until `deadline`; a +/// holder process that exits releases it. +fn os_lock(directory: &Path, deadline: Instant) -> Result { + use windows_sys::Win32::Foundation::{ + CloseHandle, WAIT_ABANDONED, WAIT_OBJECT_0, WAIT_TIMEOUT, + }; + use windows_sys::Win32::System::Threading::{CreateMutexW, WaitForSingleObject}; + let name: Vec = mutex_name(directory) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { CreateMutexW(std::ptr::null(), 0, name.as_ptr()) }; + if handle.is_null() { + return Err(refused("Failed", "the discovery mutex cannot open")); + } + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + unsafe { CloseHandle(handle) }; + return Err(refused( + "Busy", + "another process owns this discovery mutation", + )); + } + let waited = unsafe { + WaitForSingleObject(handle, remaining.as_millis().min(u32::MAX as u128) as u32) + }; + match waited { + WAIT_OBJECT_0 | WAIT_ABANDONED => return Ok(handle as usize), + WAIT_TIMEOUT => { + unsafe { CloseHandle(handle) }; + return Err(refused( + "Busy", + "another process owns this discovery mutation", + )); + } + _ => { + unsafe { CloseHandle(handle) }; + return Err(refused("Failed", "the discovery mutex cannot be held")); + } + } + } +} + +pub(crate) fn save(path: &Path, value: SdkValue) -> Result<()> { + let bytes = layerfs_bridge::control::Answer { + id: 1, + reply: Reply::Sdk(value), + } + .encode() + .map_err(failed)?; + let temp = path.with_extension(format!( + "{}-{}", + std::process::id(), + layerfs_bridge::sdk_values::hex(®istry::random()?[..8]) + )); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + .map_err(failed)?; + file.write_all(&bytes).map_err(failed)?; + drop(file); + fs::rename(temp, path).map_err(failed) +} + +pub(crate) fn load(path: &Path) -> Result { + private(path, false)?; + let file = OpenOptions::new().read(true).open(path).map_err(failed)?; + if file.metadata().map_err(failed)?.len() > 65535 { + return Err(refused("InvalidInput", "discovery record too large")); + } + let mut bytes = Vec::new(); + file.take(65536).read_to_end(&mut bytes).map_err(failed)?; + match layerfs_bridge::control::Answer::decode(&bytes) + .map_err(failed)? + .reply + { + Reply::Sdk(value) => Ok(value), + _ => Err(refused("InvalidInput", "discovery record kind")), + } +} + +pub(crate) fn save_project(path: &Path, project: &Project) -> Result<()> { + save( + &path.join("project.record"), + SdkValue::map([ + ( + "manifest", + SdkValue::Bytes(project.manifest.encode().map_err(failed)?), + ), + ( + "installed", + project + .installed + .as_ref() + .map(|m| m.encode().map(SdkValue::Bytes)) + .transpose() + .map_err(failed)? + .unwrap_or_default(), + ), + ]), + ) +} + +pub(crate) fn project(args: &SdkValue) -> Result { + let path = project_dir(args)?; + let value = load(&path.join("project.record"))?; + let decode = |name| match value.get(name) { + Some(SdkValue::Bytes(bytes)) => StoreManifest::decode(bytes).map_err(failed), + _ => Err(refused("NoSuchProject", "no complete project manifest")), + }; + let manifest = decode("manifest")?; + if manifest.profile != layerfs_bridge::provision::StoreProfile::Disposable { + return Err(refused("InvalidInput", "Disposable required")); + } + let installed = if matches!(value.get("installed"), Some(SdkValue::Bytes(_))) { + Some(decode("installed")?) + } else { + None + }; + Ok(Project { + store: SealedStore { + path: path.join("sealed.sqlite"), + profile: SqlitePersistenceProfile::Disposable, + sqlite_version: manifest.host_sqlite.clone(), + bytes: manifest.bytes, + }, + manifest, + installed, + }) +} diff --git a/sdk/js/native/src/dispatch.rs b/sdk/js/native/src/dispatch.rs index 5801c9f0b..968992dba 100644 --- a/sdk/js/native/src/dispatch.rs +++ b/sdk/js/native/src/dispatch.rs @@ -1,6 +1,10 @@ //! Native argument conversion and disjoint public/operator method tables. use crate::job::{refused, Job, Outcome}; -use crate::{convert, direct, exec, job, output, project, registry, workspace}; +#[cfg(unix)] +use crate::job; +use crate::{convert, direct, project}; +#[cfg(unix)] +use crate::{exec, output, registry, workspace}; use layerfs_bridge::control::SdkValue; use napi::{bindgen_prelude::AsyncTask, JsUnknown}; @@ -25,30 +29,57 @@ fn dispatch(op: &str, args: SdkValue) -> Outcome { .map_err(|error| refused("InvalidInput", error.to_string()))?; } } - match op { - "project.init" => project::init(&args), - "workspace.mount" => workspace::mount(&args), - "workspace.status" => workspace::status(&args), - "workspace.commit" => workspace::commit(&args), - "workspace.unmount" => workspace::unmount(&args), - "workspace.exec" if args.get_bool("background") => Err(refused( - "InvalidInput", - "background launch requires the operator process table", - )), - "workspace.exec" => exec::exec(&args), - "workspace.exec_release" => output::release(&args), - _ if op.starts_with("operator.") => Err(refused( - "UnknownMethod", - "operator methods require operatorCall", - )), - // Store calls run in this process; there is no other route. - _ if op.contains('.') => direct::call(op, args, false), - _ => Err(refused("UnknownMethod", format!("no operation {op}"))), + #[cfg(unix)] + { + match op { + "project.init" => project::init(&args), + "workspace.mount" => workspace::mount(&args), + "workspace.status" => workspace::status(&args), + "workspace.commit" => workspace::commit(&args), + "workspace.unmount" => workspace::unmount(&args), + "workspace.exec" if args.get_bool("background") => Err(refused( + "InvalidInput", + "background launch requires the operator process table", + )), + "workspace.exec" => exec::exec(&args), + "workspace.exec_release" => output::release(&args), + _ if op.starts_with("operator.") => Err(refused( + "UnknownMethod", + "operator methods require operatorCall", + )), + // Store calls run in this process; there is no other route. + _ if op.contains('.') => direct::call(op, args, false), + _ => Err(refused("UnknownMethod", format!("no operation {op}"))), + } + } + #[cfg(windows)] + { + // Workspace and Sandbox operations need a daemon; the Windows-native + // Store route serves project, fs and branch calls in-process. + match op { + "project.init" => project::init(&args), + "workspace.mount" + | "workspace.status" + | "workspace.commit" + | "workspace.unmount" + | "workspace.exec" + | "workspace.exec_release" => Err(refused( + "Unsupported", + "Workspace and Sandbox operations need a daemon; the Windows-native Store route serves project, fs and branch calls in-process", + )), + _ if op.starts_with("operator.") => Err(refused( + "UnknownMethod", + "operator methods require operatorCall", + )), + _ if op.contains('.') => direct::call(op, args, false), + _ => Err(refused("UnknownMethod", format!("no operation {op}"))), + } } } /// A daemon operator call through the named Sandbox of the project: its /// Workspaces and the read model that lists them. `deploy` only selects how this process dials +#[cfg(unix)] /// the daemon, so it is not sent on. fn daemon_operator(op: &str, args: SdkValue) -> Outcome { let state = registry::sandbox_live(&args)?; @@ -77,8 +108,21 @@ pub(crate) fn exec_output( args: JsUnknown, page: napi::bindgen_prelude::Buffer, ) -> napi::Result> { - let args = convert::from_js(args)?; - Ok(AsyncTask::new(Job::new(move || output::page(&args, page)))) + #[cfg(windows)] + { + let _ = (&args, &page); + return Ok(AsyncTask::new(Job::new(move || { + Err(refused( + "Unsupported", + "exec output needs a daemon Workspace; the Windows-native Store route serves project, fs and branch calls in-process", + )) + }))); + } + #[cfg(unix)] + { + let args = convert::from_js(args)?; + Ok(AsyncTask::new(Job::new(move || output::page(&args, page)))) + } } /// Opens or releases a Store of this process. @@ -92,28 +136,68 @@ pub(crate) fn store( /// Runs one internal operator call through its separate method table. pub(crate) fn operator_call(op: String, args: JsUnknown) -> napi::Result> { + #[cfg(windows)] + { + if op.starts_with("operator.process.") + || matches!( + op.as_str(), + "operator.sandbox.list" + | "operator.inspect.snapshot" + | "operator.workspace.overlay" + | "operator.workspace.changes" + ) + { + return Ok(AsyncTask::new(Job::new(move || { + Err(refused( + "Unsupported", + "this operator call needs a daemon; the Windows-native Store route serves Store calls in-process", + )) + }))); + } + } let mut args = convert::from_js(args)?; let scope = crate::trace::take_scope(&mut args); Ok(AsyncTask::new( - Job::new(move || match op.as_str() { - "operator.process.start" => exec::exec(&args), - "operator.process.list" => exec::processes(&args), - "operator.process.cancel" => exec::cancel(&args), - "operator.project.progress" => crate::project::progress(&args), - "operator.sandbox.list" => workspace::list(&args), - "operator.inspect.history" - | "operator.store.stats" - | "operator.store.commits" - | "operator.store.overwritten" - | "operator.files.stats" - | "operator.files.history" => direct::call(&op, args, true), - "operator.inspect.snapshot" - | "operator.workspace.overlay" - | "operator.workspace.changes" => daemon_operator(&op, args), - _ => Err(refused( - "UnknownMethod", - format!("no operator operation {op}"), - )), + Job::new(move || { + #[cfg(unix)] + { + match op.as_str() { + "operator.process.start" => exec::exec(&args), + "operator.process.list" => exec::processes(&args), + "operator.process.cancel" => exec::cancel(&args), + "operator.project.progress" => crate::project::progress(&args), + "operator.sandbox.list" => workspace::list(&args), + "operator.inspect.history" + | "operator.store.stats" + | "operator.store.commits" + | "operator.store.overwritten" + | "operator.files.stats" + | "operator.files.history" => direct::call(&op, args, true), + "operator.inspect.snapshot" + | "operator.workspace.overlay" + | "operator.workspace.changes" => daemon_operator(&op, args), + _ => Err(refused( + "UnknownMethod", + format!("no operator operation {op}"), + )), + } + } + #[cfg(windows)] + { + match op.as_str() { + "operator.project.progress" => crate::project::progress(&args), + "operator.inspect.history" + | "operator.store.stats" + | "operator.store.commits" + | "operator.store.overwritten" + | "operator.files.stats" + | "operator.files.history" => direct::call(&op, args, true), + _ => Err(refused( + "UnknownMethod", + format!("no operator operation {op}"), + )), + } + } }) .traced(scope), )) diff --git a/sdk/js/native/src/lib.rs b/sdk/js/native/src/lib.rs index 0c8c45c5c..a7a711f40 100644 --- a/sdk/js/native/src/lib.rs +++ b/sdk/js/native/src/lib.rs @@ -1,16 +1,25 @@ //! Node entry points for the public SDK and internal operator. mod convert; mod direct; +#[cfg(unix)] +mod discovery; +#[cfg(windows)] +#[path = "discovery_windows.rs"] mod discovery; mod dispatch; +#[cfg(unix)] mod exec; mod job; +#[cfg(unix)] mod output; +#[cfg(unix)] mod pool; mod project; mod registry; +#[cfg(unix)] mod sandbox; mod trace; +#[cfg(unix)] mod workspace; use job::Job; diff --git a/sdk/js/native/src/project.rs b/sdk/js/native/src/project.rs index 416be42c5..168065e38 100644 --- a/sdk/js/native/src/project.rs +++ b/sdk/js/native/src/project.rs @@ -61,6 +61,7 @@ pub(crate) fn init(args: &SdkValue) -> Outcome { failed(error) } })?; + #[cfg(unix)] std::fs::set_permissions(&directory, { use std::os::unix::fs::PermissionsExt; std::fs::Permissions::from_mode(0o700) diff --git a/sdk/js/native/src/registry.rs b/sdk/js/native/src/registry.rs index aeeb25bc4..d9a4fda1a 100644 --- a/sdk/js/native/src/registry.rs +++ b/sdk/js/native/src/registry.rs @@ -8,9 +8,15 @@ //! binding) also keeps the background processes it started; the daemon holds //! no process record. use crate::job::refused; +#[cfg(unix)] +use crate::pool::Controls; +#[cfg(unix)] use layerfs_api::MountedWorkspace; -use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; +#[cfg(unix)] +use layerfs_bridge::control::WorkspaceToken; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::provision::StoreManifest; +#[cfg(unix)] use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -24,12 +30,14 @@ pub(crate) struct Project { } /// One mounted Workspace. +#[cfg(unix)] pub(crate) struct Mount { pub(crate) workspace: MountedWorkspace, pub(crate) branch: String, } /// One background process started in a Workspace. +#[cfg(unix)] pub(crate) struct Process { pub(crate) path: String, pub(crate) command: String, @@ -39,6 +47,7 @@ pub(crate) struct Process { } /// One live Sandbox: Sandbox `id` (its number) of `project`. +#[cfg(unix)] pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, @@ -47,7 +56,7 @@ pub(crate) struct SandboxState { pub(crate) directory: std::path::PathBuf, pub(crate) runtime: Sandbox, pub(crate) identity: CommandIdentity, - pub(crate) controls: crate::pool::Controls, + pub(crate) controls: Controls, pub(crate) mounts: Mutex>, pub(crate) processes: Mutex>, } @@ -56,6 +65,7 @@ pub(crate) struct SandboxState { pub(crate) static INITS: LazyLock>>> = LazyLock::new(Default::default); /// This process's live Sandboxes, by their record directory. +#[cfg(unix)] pub(crate) static SANDBOXES: LazyLock>>> = LazyLock::new(Default::default); @@ -77,6 +87,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes /// [`sandbox_refreshed`]. +#[cfg(unix)] pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -95,6 +106,7 @@ pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> /// records (which inspects too). A container found gone or stopped (an idle /// Sandbox's daemon ends itself, #373) is the refusal of /// `discovery::gone`, and this process forgets the state. +#[cfg(unix)] pub(crate) fn sandbox_live(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let Some(state) = lock(&SANDBOXES)?.get(&directory).cloned() else { @@ -116,6 +128,7 @@ pub(crate) fn sandbox_live(args: &SdkValue) -> Result, SdkRefu } /// `args` naming Sandbox `id` of their project. +#[cfg(unix)] pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { let mut fields = match args { SdkValue::Map(fields) => fields @@ -131,12 +144,14 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { /// The Sandbox an operation names, its tables read again from the shared /// records and the daemon (one status call per mount). +#[cfg(unix)] pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { let state = sandbox(args)?; crate::discovery::refresh(&state)?; Ok(state) } +#[cfg(unix)] impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { diff --git a/sdk/js/native/src/trace.rs b/sdk/js/native/src/trace.rs index 8fad42e91..3c7ae58d4 100644 --- a/sdk/js/native/src/trace.rs +++ b/sdk/js/native/src/trace.rs @@ -2,10 +2,17 @@ //! makes travels as a traced observation under the call's scope, and the //! telemetry of the replies is collected for its trace line. An untraced call //! sends exactly what it did before. Telemetry never changes a result. +#[cfg(unix)] use crate::pool::{Controls, Lease}; -use layerfs_api::internal::{self, Control}; -use layerfs_bridge::control::{SdkRefusal, SdkValue}; +#[cfg(unix)] +use layerfs_api::internal; +#[cfg(unix)] +use layerfs_api::internal::Control; +#[cfg(unix)] +use layerfs_bridge::control::SdkRefusal; +use layerfs_bridge::control::SdkValue; use std::cell::{Cell, RefCell}; +#[cfg(unix)] use std::ops::{Deref, DerefMut}; use std::time::Duration; @@ -71,16 +78,19 @@ pub(crate) fn collect(value: SdkValue) { } /// A Sandbox's control connection, traced while this thread is. +#[cfg(unix)] pub(crate) struct Traced<'a>(Lease<'a>); /// Lends a control connection for one call's exchanges; the wait for it is /// the call's queue time. +#[cfg(unix)] pub(crate) fn control(controls: &Controls) -> Result, SdkRefusal> { let (mut lease, waited) = controls.lend()?; WAITED.set(WAITED.get() + waited); internal::trace(&mut lease, SCOPE.get()); Ok(Traced(lease)) } +#[cfg(unix)] impl Drop for Traced<'_> { fn drop(&mut self) { let telemetry = internal::take_telemetry(&mut self.0); @@ -91,12 +101,14 @@ impl Drop for Traced<'_> { }); } } +#[cfg(unix)] impl Deref for Traced<'_> { type Target = Control; fn deref(&self) -> &Control { &self.0 } } +#[cfg(unix)] impl DerefMut for Traced<'_> { fn deref_mut(&mut self) -> &mut Control { &mut self.0 From 94bdb9f585ce2008950b2c8fb8317a26a679bbe2 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Sun, 25 Oct 2026 12:08:20 +0800 Subject: [PATCH 03/19] docs: record the Windows-native Store route (owner direction 2026-10-25) Draft for owner review, landing with the windows-native-store branch: - Root AGENTS.md: a Windows-native Store workstream row and a mental-model bullet superseding the Linux-only Store sentence for this route only, with its recorded limitations. - core/AGENTS.md: the audited Windows FFI inventory (persistence windows_lane beside the macOS seal wrapper; project and overlay identity modules, forbid->deny), the real volume-serial + file-index requirement for acquisition rows, and the Linux-container cross-check rule for cfg(unix) code touched on this route. - DIRECT_STORE.md: section 6 for the Windows-native route - mechanisms, what still refuses Unsupported, limitations, proof receipts. - 83-store-concurrency-model.md: the writer-lane table row names the Windows named kernel mutex equivalent (abandoned mutex taken by the next waiter). The two pre-existing broken relative links found while checking (DIRECT_STORE.md -> DIRECT_STORE_IMPL.md, 83 -> 84-global-sqlite-store.md) are left for their owners; this change adds no new link and all links it adds resolve. Production LOC: core 97232 -> 97232 (delta 0), sdk 3455 -> 3455 (delta 0); combined 109870 -> 109870 (delta 0). Docs only; same counter and scope as the previous commits. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- AGENTS.md | 15 ++++++++ core/AGENTS.md | 29 +++++++++++++++ .../83-store-concurrency-model.md | 2 +- core/docs/issues/352/DIRECT_STORE.md | 37 +++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index da3ef4221..0d4ee3329 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ simplification. | Cluster one | Implemented content, storage, history, persistence, project Init and telemetry libraries | [Cluster-one handbook](docs/general/cluster_one_handbook.md), [CAS/CDC/delta handbook](docs/general/cas_cdc_deltaencoding_handbook.md) | | Cluster two | Workspace, FUSE, daemon, execution and Commit/runtime integration design and implementation | [Current design index](core/docs/issues/303/README.md) and its seven primary contracts | | Legacy reference | v0.1.6, retired from `main` by R9c; read at the recovery tag only | [R9c record](core/docs/issues/307/R9C-COMPLETION-20261010.md) | +| Windows-native Store | Store engine and the SDK's in-process Store face on native Windows (owner, 2026-10-25); FUSE, daemon and Workspace stay Linux-only | [Direct Store contract](core/docs/issues/352/DIRECT_STORE.md) | **The LayerFS UI is the dsh LayerFS plugin** (owner, 2026-10-19): the LayerFS tab in DeepSeek Harness, served by `dsh/packages`; see [dsh/README.md](dsh/README.md). @@ -151,6 +152,20 @@ remains a separate MEMORY/OFF/EXCLUSIVE database file. native handles and project discovery. This is independent of the macOS adapter and changes neither the product SDK nor the daemon. A lost reply is never replayed, and an uncertain Init retains its Linux home and volume. +- **Windows-native Store route (owner direction 2026-10-25).** The Store + engine (SQLite, the cross-process writer lane, the kernel guard) and the + SDK's in-process Store face (fs, branch, project, operator) also run + natively on Windows: local NTFS volumes only, a named kernel mutex as the + cross-process writer lane, real volume-serial + file-index identity for + acquisition rows, and bundled SQLite, each in audited FFI modules + (core/AGENTS.md). No WSL2 and no daemon are involved. This supersedes the + Linux-only Store sentence above for this route only; Workspace, FUSE, Exec + and daemon calls still refuse `Unsupported` on Windows, and the dsh macOS + and WSL2 exceptions are unchanged. Recorded limitations: Windows Init has + no hard-link membership and no backing-inside-source alias check, carries + non-UTF-8 names lossily and synthesizes the mode from the read-only + attribute; overlay allocation is the logical length. See + [the Windows-native route](core/docs/issues/352/DIRECT_STORE.md). - **One global database, one writer, short holds (owner decision 2026-10-18).** All committed state lives in one SQLite Store file; no sharding, second database, concurrent-writer engine or objects outside it. Writes go one at a diff --git a/core/AGENTS.md b/core/AGENTS.md index 792bcfef9..134de3c19 100644 --- a/core/AGENTS.md +++ b/core/AGENTS.md @@ -577,3 +577,32 @@ No other Persistence module may contain unsafe Rust; the guard and its tests enforce the exact path. This does not authorize third-party patches, dependency changes, manual sidecar deletion or other FFI. See the [decision and diagnostic](docs/issues/307/SEAL-PERSIST-WAL-DECISION-20261007.md). + +### Audited Windows FFI modules + +Owner direction 2026-10-25 (route 2, Windows-native Store; root AGENTS.md) +adds audited Windows FFI beside the macOS seal wrapper. The guard +(`core/tools/check_product_boundary.py`) and its tests enforce the exact +paths: + +- Persistence keeps `deny(unsafe_code)` with two allowed modules: + `src/backend/sqlite/file_control.rs` (the macOS seal wrapper, unchanged) + and `src/backend/sqlite/windows_lane.rs` (the named kernel mutex writer + lane and `GetVolumeInformationW` volume filesystem). +- layerfs-project moves forbid→deny with one allowed module, + `src/import/windows_identity.rs` (`GetFileInformationByHandle`: volume + serial, file index, creation time). +- layerfs-overlay moves forbid→deny with one allowed module, + `src/database/windows_identity.rs` (handle/path identity and + `GetDiskFreeSpaceExW`). +- Storage's `src/encoding/codec.rs` authorization is unchanged. + +Acquisition rows are keyed by dev+inode. On Windows the identity MUST be +the real volume serial and file index; a zeroed placeholder collides every +entry onto one key and Init refuses with "the entry changed while Init +read it". + +`#[cfg(unix)]` code is invisible to a Windows compile: a unix-only edit +that does not parse can pass on Windows and still break Linux. Code touched +for this route is cross-checked with `cargo check --locked` in the pinned +Linux container (`rust:1.85.1-bookworm`) before any "it builds" claim. diff --git a/core/docs/architecture/83-store-concurrency-model.md b/core/docs/architecture/83-store-concurrency-model.md index b2e6e8cdd..f67dd0bbc 100644 --- a/core/docs/architecture/83-store-concurrency-model.md +++ b/core/docs/architecture/83-store-concurrency-model.md @@ -31,7 +31,7 @@ | Layer | Scope | Why it exists | | --- | --- | --- | | SQLite file lock (WAL) | the Store file | SQLite allows one write transaction per database; readers proceed alongside | -| Directory lock (Linux) | all writer processes of one Store | Writers in different processes (each Sandbox's daemon, the SDK process) take turns instead of meeting `SQLITE_BUSY` | +| Directory lock (Linux) / named kernel mutex (Windows) | all writer processes of one Store | Writers in different processes (each Sandbox's daemon, the SDK process) take turns instead of meeting `SQLITE_BUSY`; on Windows the mutex is named per Store directory and an abandoned mutex is taken by the next waiter | | Writer admission queue | threads of one process | Threads take turns on the process's one writer connection, first come first served, before touching SQLite | | Publication lane | one Commit's publish | Head check, merge and publish are one atomic step, so two Commits never both publish on the same old head | | Workspace lock | one Workspace's overlay database | Separate concern: guards that Workspace's own overlay connection (root AGENTS.md) | diff --git a/core/docs/issues/352/DIRECT_STORE.md b/core/docs/issues/352/DIRECT_STORE.md index 765b326d0..4a7d0aeb6 100644 --- a/core/docs/issues/352/DIRECT_STORE.md +++ b/core/docs/issues/352/DIRECT_STORE.md @@ -11,6 +11,8 @@ > **One route, Linux only (owner, 2026-10-18):** Store calls run in-process > only; the daemon stops serving them, and an unsupported setup fails > `Unsupported`. There is no daemon route to choose or fall back to. +> **Windows-native route (owner, 2026-10-25):** the Store engine and this +> in-process Store face also run natively on Windows; see §6. ## 1. Architecture @@ -88,3 +90,38 @@ Disposable/WAL/OFF only; page size 4 KiB; one attempt per operation; FUSE and Workspaces stay in the daemon; the daemon keeps opening the Store directly; no `layerfs-server`, no host/server adapter in the data path (the SDK is a direct opener, not a proxy). + +## 6. Windows-native route (owner direction 2026-10-25) + +The one-kernel safety condition is met natively on Windows: every opener of a +Windows Store shares one Windows kernel and one local NTFS volume. No WSL2 and +no daemon are involved. Implemented on branch `windows-native-store`: + +- The cross-process writer lane is a named kernel mutex + (`Local\layerfs-writer-lane-`); an abandoned mutex is taken by the + next waiter, so a dead writer does not hold the lane. +- The kernel guard allows local NTFS volumes only, read with + `GetVolumeInformationW`; other filesystems and non-disk prefixes fail + `Unsupported` before the Store opens. +- SQLite is bundled (rusqlite) on Windows targets; Disposable/WAL/OFF is the + only profile, unchanged. +- Acquisition identity is the real volume serial and file index + (`GetFileInformationByHandle`), in audited FFI modules listed in + core/AGENTS.md. +- The SDK addon serves fs, branch, project and operator calls in-process + exactly as on Linux. Workspace, exec and daemon calls refuse `Unsupported` + on Windows: FUSE, the daemon and Workspaces remain Linux-only, and the dsh + macOS and WSL2 exceptions are unchanged. + +Recorded limitations of this route: Windows Init carries non-UTF-8 names +lossily, synthesizes the mode from the read-only attribute, has no hard-link +membership and no backing-inside-source alias check; overlay allocation is the +logical length; last-owner overlay-file deletion keeps its Linux identity +check (not exercised here — the Store route holds no Workspace mounts). + +Proof receipts: the cross-process lane verifier +(`layerfs-persistence` `tests/windows_store_native.rs`, two re-exec'd child +processes, bounded waits); `cargo check --locked` of the five touched crates +on Windows and in the `rust:1.85.1-bookworm` container; the lifecycle smoke +(init → fork → one-Commit write → read back → list_commits → close) with no +WSL2 and no daemon. From ee6c34fbab473f1814e61ea23cb80ddad242e6b8 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 00:28:27 +0800 Subject: [PATCH 04/19] test(windows): run the Store suites natively on Windows; lane-covers open The 3b slice of the Windows-native Store route: the existing test suites now run natively on Windows, and one real open-path race they exposed is fixed for every platform. Product changes: - Concurrent Store opens met Busy under the no-retry rule: the profile pragmas (first database-touching statements, where a WAL -shm is created and recovery runs) and the writable setup block (binding hints, lineage, merge scratch, file digests, merge-record sweep) ran outside the writer lane. Each is now one short lane hold, released before the reads that follow; macOS keeps exact SQLite Busy by design. The lane verifier's child failed "the child opens the Store: Busy" intermittently before this; eight consecutive runs pass after it. - import/host.rs: a writable Windows file synthesized mode 0o644, the read-only base - the read-only attribute was not stored. Base is 0o444/0o555 as documented. - outside_source on Windows becomes a real containment check of the resolved placement inside the resolved source (junctions and symlinks resolve); only spellings that do not resolve to the source's own path (a subst drive, an unresolvable alias) stay unrecognized, replacing the previous unconditional no-op. - windows-sys features Win32_Security is declared by layerfs-project and layerfs-overlay themselves: CreateFileW is gated on it by its SECURITY_ATTRIBUTES parameter, and the crates only compiled before through another crate's feature unification. Test changes (delete-or-adapt; deleted tests listed): - Deleted: init_sqlite's an_unsupported_platform_refuses_the_store_ before_any_file_exists - its premise (Windows refused) is inverted by this route; the namespace oracle tests it shared a file with now run on Windows. - unix-only kept unix-only: serverless_store (socket readiness), the four issue327 counters examples and restore_record (st_blocks / 0o600 records; non-unix mains refuse instead of fabricating). - Adapted: project support asserts the host's own reading of each entry (raw stat on unix; the synthesized contract on Windows); the held writer helper drops select() for a blocking readline (WinError 10093) and trims the readiness line; kernel_guard tests the Windows non-local-spelling refusal beside macOS's; the lane verifier pipes its children's report into its failure instead of discarding it. - New: sdk/js/test/store_lifecycle.mjs - init, fork, one-Commit write, read back, stat, commit listing and close on the real binding, no Docker, daemon or WSL2; it runs on both Linux (the E2E volume) and Windows. trace.mjs imports its fixture through pathToFileURL (a bare Windows path is not an ESM URL); reinit/init_refusals gate their unix-only contracts explicitly. Verification at this commit (Windows, rustc 1.90.0): cargo test for layerfs-persistence, -project, -store-service, -overlay, -node: 173 passed, 0 failed; the persistence suite clean across four consecutive runs. SDK: store_lifecycle, binary_write, reinit pass; trace 9/9 cases; packaging 2/2; init_refusals reports its unix-only skip. Linux (rust:1.85.1-bookworm container): cargo check --offline --locked --tests --examples for the five crates, Finished. Boundary guard self-tests 24/24; no new violation. Production LOC: core 97232 -> 97258 (delta +26; persistence +18, project +8), sdk 3455 -> 3455 (delta 0); combined 109870 -> 109896. Counter: tools/production_loc.py --detail (pinned at this tree), scopes core and sdk; tests, examples and manifests excluded. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../tests/support/held_writer.rs | 13 +++-- core/crates/layerfs-overlay/Cargo.toml | 6 ++- .../examples/issue327_counters_v2.rs | 6 +++ .../examples/issue327_counters_v3.rs | 6 +++ .../examples/issue327_counters_v4.rs | 6 +++ .../examples/issue327_counters_v5.rs | 6 +++ .../src/backend/sqlite/connection.rs | 11 ++++- .../src/backend/sqlite/mod.rs | 2 +- .../layerfs-persistence/src/store/open.rs | 31 ++++++++++-- .../tests/serverless_store.rs | 3 ++ .../tests/windows_store_native.rs | 12 +++-- core/crates/layerfs-project/Cargo.toml | 6 ++- .../crates/layerfs-project/src/import/host.rs | 2 +- .../crates/layerfs-project/src/import/init.rs | 2 +- .../layerfs-project/src/import/source.rs | 35 +++++++++----- .../layerfs-project/tests/init_sqlite.rs | 39 ++++----------- .../layerfs-project/tests/support/mod.rs | 37 +++++++++++++-- .../tests/kernel_guard.rs | 33 +++++++++++-- sdk/js/native/examples/restore_record.rs | 9 ++++ sdk/js/test/init_refusals.mjs | 8 ++++ sdk/js/test/reinit.mjs | 20 ++++---- sdk/js/test/store_lifecycle.mjs | 47 +++++++++++++++++++ sdk/js/test/trace.mjs | 10 ++-- 23 files changed, 266 insertions(+), 84 deletions(-) create mode 100644 sdk/js/test/store_lifecycle.mjs diff --git a/core/crates/layerfs-daemon/tests/support/held_writer.rs b/core/crates/layerfs-daemon/tests/support/held_writer.rs index b7099eeb1..482efbeab 100644 --- a/core/crates/layerfs-daemon/tests/support/held_writer.rs +++ b/core/crates/layerfs-daemon/tests/support/held_writer.rs @@ -9,7 +9,10 @@ use std::{ pub struct HeldWriter(Child); impl HeldWriter { pub fn acquire(database: &std::path::Path) -> Self { - let script = "import sqlite3,sys,select\nc=sqlite3.connect(sys.argv[1],timeout=0)\nc.execute('BEGIN IMMEDIATE')\nprint('held',flush=True)\nselect.select([sys.stdin],[],[],5)\nc.rollback()\nc.close()\n"; + // The writer holds BEGIN IMMEDIATE, signals readiness and stands by + // for one line. select() is not portable to a Windows pipe stdin; + // a blocking readline is, and Drop's kill bounds a lost releaser. + let script = "import sqlite3,sys\nc=sqlite3.connect(sys.argv[1],timeout=0)\nc.execute('BEGIN IMMEDIATE')\nprint('held',flush=True)\nsys.stdin.readline()\nc.rollback()\nc.close()\n"; let child = Command::new("python3") .arg("-c") .arg(script) @@ -26,10 +29,10 @@ impl HeldWriter { let result = output.read_line(&mut line).map(|_| line); let _ = tx.send(result); }); - assert_eq!( - rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap(), - "held\n" - ); + // The external writer's readiness line, whatever end-of-line the + // host's python prints. + let line = rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap(); + assert_eq!(line.trim_end(), "held"); reader.join().unwrap(); held } diff --git a/core/crates/layerfs-overlay/Cargo.toml b/core/crates/layerfs-overlay/Cargo.toml index ba602451a..16b14a757 100644 --- a/core/crates/layerfs-overlay/Cargo.toml +++ b/core/crates/layerfs-overlay/Cargo.toml @@ -16,6 +16,8 @@ rusqlite = { version = "=0.40.2", features = ["bundled"] } [target.'cfg(target_os = "windows")'.dependencies] # Windows has no system SQLite, and the audited identity module needs the -# Win32 file-information calls; windows-sys is already in the locked graph. +# Win32 file-information calls; CreateFileW is additionally gated on +# Win32_Security by its SECURITY_ATTRIBUTES parameter. windows-sys is +# already in the locked graph. rusqlite = { version = "=0.40.2", features = ["bundled"] } -windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem"] } diff --git a/core/crates/layerfs-persistence/examples/issue327_counters_v2.rs b/core/crates/layerfs-persistence/examples/issue327_counters_v2.rs index 524b2bbed..792a49fad 100644 --- a/core/crates/layerfs-persistence/examples/issue327_counters_v2.rs +++ b/core/crates/layerfs-persistence/examples/issue327_counters_v2.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -174,3 +175,8 @@ fn main() { }; writeln!(output,"{{\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/core/crates/layerfs-persistence/examples/issue327_counters_v3.rs b/core/crates/layerfs-persistence/examples/issue327_counters_v3.rs index 9f4f5d4db..acc203510 100644 --- a/core/crates/layerfs-persistence/examples/issue327_counters_v3.rs +++ b/core/crates/layerfs-persistence/examples/issue327_counters_v3.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -179,3 +180,8 @@ fn main() { }; writeln!(output,"{{\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/core/crates/layerfs-persistence/examples/issue327_counters_v4.rs b/core/crates/layerfs-persistence/examples/issue327_counters_v4.rs index 9a4b1371c..4a9ed7405 100644 --- a/core/crates/layerfs-persistence/examples/issue327_counters_v4.rs +++ b/core/crates/layerfs-persistence/examples/issue327_counters_v4.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -179,3 +180,8 @@ fn main() { }; writeln!(output,"{{\"sql_statement_ns\":{},\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statement_ns,work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/core/crates/layerfs-persistence/examples/issue327_counters_v5.rs b/core/crates/layerfs-persistence/examples/issue327_counters_v5.rs index 73b347904..b770decbd 100644 --- a/core/crates/layerfs-persistence/examples/issue327_counters_v5.rs +++ b/core/crates/layerfs-persistence/examples/issue327_counters_v5.rs @@ -36,6 +36,7 @@ fn fixture() -> Vec> { }) .collect() } +#[cfg(unix)] fn main() { assert_eq!( std::env::var("SCREEN_SCENARIO").as_deref(), @@ -179,3 +180,8 @@ fn main() { }; writeln!(output,"{{\"sql_phases_ns\":{:?},\"sql_statement_ns\":{},\"sql_statements\":{},\"sql_vm_steps\":{},\"sqlite\":\"{}\",\"operation_ns\":{operation},\"complete_ns\":{},\"file_bytes\":{},\"allocated_bytes\":{allocated},\"sealed_bytes\":{},\"objects_input\":{COUNT},\"input_bytes\":{},\"cache\":\"created-in-sample Store; fixed warm in-memory fixture\",\"status\":\"RECORDED\"}}",work.statement_phases.map(|p| p.wall_ns),work.statement_ns,work.statements,work.vm_steps,sql_version,total.elapsed().as_nanos(),metadata.len(),sealed.bytes,COUNT*BYTES).unwrap(); } + +#[cfg(not(unix))] +fn main() { + eprintln!("this frozen recorder requires Unix (st_blocks); nothing recorded"); +} diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs b/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs index d1b874cec..4850c02eb 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs @@ -175,7 +175,16 @@ impl Session { connection .set_db_config(rusqlite::config::DbConfig::SQLITE_DBCONFIG_DEFENSIVE, true) .map_err(rows::error)?; - profile::apply(&connection, create, selected, creation_acquisition, &work)?; + // The profile pragmas are this open's first database-touching + // statements, where a WAL Store's -shm is created and recovery + // runs; two concurrent opens of one Store can meet Busy exactly + // there under the no-retry rule. The coarse writer lane replaces + // Busy: hold it for this one short setup step and release before + // the reads that follow. macOS keeps exact SQLite Busy by design. + let writer = super::writer_admission::Writer::acquire(path)?; + let applied = profile::apply(&connection, create, selected, creation_acquisition, &work); + writer.release().map_err(|_| BackendError::Integrity)?; + applied?; let (layout, acquisition) = if create { (creation_layout, creation_acquisition) } else { diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs b/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs index 4b570c8a9..6db19ee4e 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/mod.rs @@ -35,4 +35,4 @@ pub(crate) mod unit_io; pub(crate) mod unit_layout; mod units_publish; mod units_read; -mod writer_admission; +pub(crate) mod writer_admission; diff --git a/core/crates/layerfs-persistence/src/store/open.rs b/core/crates/layerfs-persistence/src/store/open.rs index e851ed679..d94252fd0 100644 --- a/core/crates/layerfs-persistence/src/store/open.rs +++ b/core/crates/layerfs-persistence/src/store/open.rs @@ -92,14 +92,35 @@ impl Handles { config.sqlite_acquisition, )?); if writable { - session.add_binding_hints()?; - session.add_lineage(crate::history::backfill_lineage)?; - session.add_merge_scratch()?; - session.add_file_digests()?; + // The idempotent setup statements are write transactions, so two + // concurrent opens of one Store meet Busy exactly here under the + // no-retry rule. The coarse writer lane replaces Busy: one short + // hold covers the whole setup block, released before validation. + let writer = + crate::backend::sqlite::writer_admission::Writer::acquire(&config.path)?; + let setup = + (|| -> Result<(), crate::backend::records::BackendError> { + session.add_binding_hints()?; + session.add_lineage(crate::history::backfill_lineage)?; + session.add_merge_scratch()?; + session.add_file_digests() + })(); + writer + .release() + .map_err(|_| crate::backend::records::BackendError::Integrity)?; + setup?; } let handles = Self::validated(session, directory, binding, cursor_key)?; if writable { - crate::storage::merge_records::sweep(&handles.storage.session)?; + // The sweep deletes stale merge records: a write, held behind the + // lane for the same reason as the setup block above. + let writer = + crate::backend::sqlite::writer_admission::Writer::acquire(&config.path)?; + let swept = crate::storage::merge_records::sweep(&handles.storage.session); + writer + .release() + .map_err(|_| crate::backend::records::BackendError::Integrity)?; + swept?; } Ok(handles) } diff --git a/core/crates/layerfs-persistence/tests/serverless_store.rs b/core/crates/layerfs-persistence/tests/serverless_store.rs index 91d10c547..ac4a8861e 100644 --- a/core/crates/layerfs-persistence/tests/serverless_store.rs +++ b/core/crates/layerfs-persistence/tests/serverless_store.rs @@ -1,4 +1,7 @@ //! Disposable serverless Store proofs through the real public provider. +//! Unix only: the child's readiness signal travels a unix-domain socket. +//! The Windows cross-process proof is tests/windows_store_native.rs. +#![cfg(unix)] mod support; use layerfs_content::{FinalizedObject, ObjectId, ObjectRole}; use layerfs_history::{HistoryCatalog, HistoryCatalogConfig, HistoryError, ReserveRequest}; diff --git a/core/crates/layerfs-persistence/tests/windows_store_native.rs b/core/crates/layerfs-persistence/tests/windows_store_native.rs index 254425045..5bc48b718 100644 --- a/core/crates/layerfs-persistence/tests/windows_store_native.rs +++ b/core/crates/layerfs-persistence/tests/windows_store_native.rs @@ -53,12 +53,18 @@ fn dispatch_child() { } /// Bounded completion of one foreign child; a hung child fails the test. +/// A failed child's own report (the harness prints it to its stdout) is +/// carried into the failure instead of being discarded. fn finish(mut child: std::process::Child) { let until = Instant::now() + Duration::from_secs(60); loop { match child.try_wait().expect("child poll") { Some(status) => { - assert!(status.success(), "a child writer failed: {status}"); + assert!( + status.success(), + "a child writer failed: {status}; child report: {:?}", + String::from_utf8_lossy(&child.wait_with_output().expect("child drain").stdout) + ); return; } None => assert!(Instant::now() < until, "a child writer is hung"), @@ -103,8 +109,8 @@ fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { Command::new(&executable) .env(CHILD_STORE, &lane_path) .env(CHILD_BINDING, "windows-lane-processes") - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .spawn() .unwrap(), ); diff --git a/core/crates/layerfs-project/Cargo.toml b/core/crates/layerfs-project/Cargo.toml index 8338c36bd..9d37946ef 100644 --- a/core/crates/layerfs-project/Cargo.toml +++ b/core/crates/layerfs-project/Cargo.toml @@ -13,8 +13,10 @@ layerfs-history = { path = "../layerfs-history", default-features = false } layerfs-telemetry = { path = "../layerfs-telemetry" } [target.'cfg(target_os = "windows")'.dependencies] -# The audited host-identity probe; already in the locked graph. -windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } +# The audited host-identity probe; CreateFileW is additionally gated on +# Win32_Security by its SECURITY_ATTRIBUTES parameter. windows-sys is +# already in the locked graph. +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem"] } [dev-dependencies] layerfs-persistence = { path = "../layerfs-persistence" } diff --git a/core/crates/layerfs-project/src/import/host.rs b/core/crates/layerfs-project/src/import/host.rs index 7e470adea..f44ec5fe3 100644 --- a/core/crates/layerfs-project/src/import/host.rs +++ b/core/crates/layerfs-project/src/import/host.rs @@ -79,7 +79,7 @@ impl HostStat { { // The format stores no ownership; mode is synthesized from the // read-only attribute, and creation time stands in for ctime. - let base = if metadata.is_dir() { 0o555 } else { 0o644 }; + let base = if metadata.is_dir() { 0o555 } else { 0o444 }; let mode = if metadata.permissions().readonly() { base } else { diff --git a/core/crates/layerfs-project/src/import/init.rs b/core/crates/layerfs-project/src/import/init.rs index 7eaba3e09..0bed52e6f 100644 --- a/core/crates/layerfs-project/src/import/init.rs +++ b/core/crates/layerfs-project/src/import/init.rs @@ -94,7 +94,7 @@ pub fn init_observed( progress.tick()?; let source = scan::check_root(request.source)?; if let Some(placement) = request.acquisition.placement() { - super::source::outside_source(&source, &placement)?; + super::source::outside_source(request.source, &source, &placement)?; } let scope = scope_for_seed(request.scope_seed); let mut stack = [0; 16]; diff --git a/core/crates/layerfs-project/src/import/source.rs b/core/crates/layerfs-project/src/import/source.rs index 1c9443fbc..b3972e639 100644 --- a/core/crates/layerfs-project/src/import/source.rs +++ b/core/crates/layerfs-project/src/import/source.rs @@ -30,17 +30,27 @@ pub(super) fn read_link(path: &Path, scanned: &Metadata) -> Result, Proj /// aliased spelling of a directory inside the source is recognised. Nothing /// is begun or changed: a backing inside the source would both alter the source /// while it is read and be acquired as part of it. -pub(super) fn outside_source(source: &Metadata, placement: &Path) -> Result<(), ProjectError> { +/// +/// Windows carries no path-independent identity here, so the check is +/// containment of the resolved placement inside the resolved source: a +/// spelling that resolves into the source is refused whatever named it, +/// while a spelling that does not resolve to the source's own path (a subst +/// drive, an unresolvable alias) is not recognised — a recorded limitation +/// of this route. +pub(super) fn outside_source( + source: &Path, + metadata: &Metadata, + placement: &Path, +) -> Result<(), ProjectError> { #[cfg(unix)] let mut ancestor = fs::canonicalize(placement)?; - #[cfg(not(unix))] - let ancestor = fs::canonicalize(placement)?; #[cfg(unix)] { + let _ = source; use std::os::unix::fs::MetadataExt; loop { - let metadata = fs::symlink_metadata(&ancestor)?; - if metadata.dev() == source.dev() && metadata.ino() == source.ino() { + let entry = fs::symlink_metadata(&ancestor)?; + if entry.dev() == metadata.dev() && entry.ino() == metadata.ino() { return Err(ProjectError::BackingInsideSource); } if !ancestor.pop() { @@ -48,13 +58,16 @@ pub(super) fn outside_source(source: &Metadata, placement: &Path) -> Result<(), } } } - #[cfg(windows)] + #[cfg(not(unix))] { - // Windows std carries no path-independent identity here, and the - // entry's own identity probe is per-path; an aliased spelling of - // the source is not recognised by this check (a recorded - // limitation). - let _ = (source, &ancestor); + // Both sides are canonicalized, so junctions and symlinks in the + // placement's spelling resolve before the comparison. + let source = fs::canonicalize(source)?; + let ancestor = fs::canonicalize(placement)?; + if ancestor == source || ancestor.starts_with(&source) { + return Err(ProjectError::BackingInsideSource); + } + let _ = metadata; Ok(()) } } diff --git a/core/crates/layerfs-project/tests/init_sqlite.rs b/core/crates/layerfs-project/tests/init_sqlite.rs index b69c02af9..4ab7dbd9f 100644 --- a/core/crates/layerfs-project/tests/init_sqlite.rs +++ b/core/crates/layerfs-project/tests/init_sqlite.rs @@ -1,38 +1,15 @@ //! Canonical namespace oracle over both explicit embedded Store profiles. //! -//! The same Store opens on macOS/Linux. Unsupported platforms retain -//! the explicit before-file-creation refusal. +//! The same Store opens on macOS, Linux and Windows (owner direction +//! 2026-10-25). Other platforms keep the product's compile-time refusal. mod support; use layerfs_history::HistoryCatalogConfig; -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] use layerfs_persistence::SqliteAcquisitionSchema; use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; use layerfs_storage::StoragePolicy; use support::*; -#[cfg(not(any(target_os = "macos", target_os = "linux")))] -#[test] -fn an_unsupported_platform_refuses_the_store_before_any_file_exists() { - use layerfs_storage::port::PersistenceError; - for selected in DEVELOPMENT_PROFILES { - let fixture = Fixture::new(0); - let store = fixture.path.join("store.sqlite"); - let refused = Handles::create( - PersistenceConfig::sqlite(&store).with_sqlite_profile(selected), - StoragePolicy::frozen_default(), - &HistoryCatalogConfig { - binding_key: b"init-sqlite".to_vec(), - cursor_key: [71; 32], - incarnation: 1, - }, - ); - assert!( - matches!(refused, Err(PersistenceError::BackendUnavailable)), - "{selected:?} must be refused on this platform" - ); - assert!(!store.exists(), "a refused Store creates no file"); - } -} -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn selected_profiles_init_100_and_1000_pass_the_full_namespace_oracle() { for selected in DEVELOPMENT_PROFILES { @@ -76,7 +53,7 @@ fn selected_profiles_init_100_and_1000_pass_the_full_namespace_oracle() { } } -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] fn attempt( fixture: &Fixture, handles: &Handles, @@ -103,7 +80,7 @@ fn attempt( .0 } -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] fn create(store: &std::path::Path, acquisition: SqliteAcquisitionSchema) -> Handles { Handles::create( PersistenceConfig::sqlite(store).with_sqlite_acquisition(acquisition), @@ -119,7 +96,7 @@ fn create(store: &std::path::Path, acquisition: SqliteAcquisitionSchema) -> Hand /// A Store created without the acquisition tables has no working state for /// Init, and there is no second acquisition algorithm to fall back to. -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn a_store_without_acquisition_tables_refuses_init() { use layerfs_history::{HistoryCatalog, LayerStackId}; @@ -146,7 +123,7 @@ fn a_store_without_acquisition_tables_refuses_init() { /// The Store's own directory is acquisition backing: a Store inside the source /// would be read while it is written and acquired as part of the root. -#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn a_store_inside_the_source_is_refused_before_an_operation_begins() { use layerfs_project::ProjectError; diff --git a/core/crates/layerfs-project/tests/support/mod.rs b/core/crates/layerfs-project/tests/support/mod.rs index 7f517336e..9dd1c7a28 100644 --- a/core/crates/layerfs-project/tests/support/mod.rs +++ b/core/crates/layerfs-project/tests/support/mod.rs @@ -13,7 +13,6 @@ use layerfs_telemetry::timer::Timing; use std::{ collections::BTreeMap, fs, - os::unix::fs::MetadataExt, path::PathBuf, sync::{ atomic::{AtomicU64, Ordering}, @@ -21,6 +20,8 @@ use std::{ }, time::{Duration, Instant}, }; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; pub mod memory_acquisition; pub mod memory_history; #[path = "../../../layerfs-storage/tests/support/memory_metadata.rs"] @@ -120,9 +121,37 @@ impl Fixture { ) .unwrap(); let metadata = fs::symlink_metadata(self.source.join(&path)).unwrap(); - assert_eq!(portable.mode, metadata.mode() & 0o7777); - assert_eq!(portable.mtime_seconds, metadata.mtime()); - assert_eq!(portable.mtime_nanoseconds as i64, metadata.mtime_nsec()); + // The stored portable metadata equals the host's own reading of + // the entry: the raw stat identity on unix; on Windows the + // synthesized contract of import/host.rs (mode from the + // read-only attribute, mtime from the entry's modified time). + #[cfg(unix)] + { + assert_eq!(portable.mode, metadata.mode() & 0o7777); + assert_eq!(portable.mtime_seconds, metadata.mtime()); + assert_eq!(portable.mtime_nanoseconds as i64, metadata.mtime_nsec()); + } + #[cfg(windows)] + { + use std::time::UNIX_EPOCH; + let base = if metadata.is_dir() { 0o555 } else { 0o444 }; + let mode = if metadata.permissions().readonly() { + base + } else { + base | 0o200 + }; + assert_eq!(portable.mode, mode); + let modified = metadata + .modified() + .unwrap() + .duration_since(UNIX_EPOCH) + .unwrap(); + assert_eq!(portable.mtime_seconds, modified.as_secs() as i64); + assert_eq!( + portable.mtime_nanoseconds as i64, + i64::from(modified.subsec_nanos()) + ); + } if metadata.is_dir() { let mut after = None; loop { diff --git a/core/crates/layerfs-store-service/tests/kernel_guard.rs b/core/crates/layerfs-store-service/tests/kernel_guard.rs index d9050f370..a4725a769 100644 --- a/core/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/core/crates/layerfs-store-service/tests/kernel_guard.rs @@ -1,8 +1,9 @@ //! Direct Store access is refused `Unsupported` where the Store's locks do -//! not reach every opener (#352 F-020): on macOS, and on Linux for a -//! filesystem outside the allowlist (here tmpfs). The refusal comes before any -//! file is opened: the Store file's bytes are unchanged and no overlay -//! template is created. +//! not reach every opener (#352 F-020): on macOS always, on Linux for a +//! filesystem outside the allowlist (here tmpfs), and on Windows for a +//! spelling outside the NTFS-on-local-volume allowlist (here UNC). The +//! refusal comes before any file is opened: the Store file's bytes are +//! unchanged and no overlay template is created. use layerfs_store_service::{OpenRefusal, StoreService}; use std::path::{Path, PathBuf}; @@ -30,7 +31,7 @@ fn scratch(parent: &Path, label: &str) -> PathBuf { directory } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] #[test] fn direct_access_is_unsupported_off_linux() { let directory = scratch(&std::env::temp_dir(), "host"); @@ -40,6 +41,28 @@ fn direct_access_is_unsupported_off_linux() { std::fs::remove_dir_all(directory).unwrap(); } +#[cfg(target_os = "windows")] +#[test] +fn direct_access_is_unsupported_on_a_non_local_spelling() { + // A UNC spelling never reaches the NTFS-on-local-volume allowlist; the + // refusal comes before any file is opened, so no template exists to + // leave behind. + let store = Path::new(r"\\localhost\c$\layerfs-guard-nonlocal\store.sqlite"); + let template = Path::new(r"\\localhost\c$\layerfs-guard-nonlocal\overlay.sqlite"); + match StoreService::open_direct(store, b"binding", [7; 32], template) { + Err(OpenRefusal::Unsupported(reason)) => assert!( + reason.0.contains("cannot be examined"), + "{reason}" + ), + Err(other) => panic!("expected Unsupported, got {other:?}"), + Ok(_) => panic!("expected Unsupported, the Store opened"), + } + assert!( + !template.exists(), + "no overlay template before the guard passes" + ); +} + #[cfg(target_os = "linux")] #[test] fn direct_access_is_unsupported_on_a_filesystem_outside_the_allowlist() { diff --git a/sdk/js/native/examples/restore_record.rs b/sdk/js/native/examples/restore_record.rs index 540f12084..8c7dec154 100644 --- a/sdk/js/native/examples/restore_record.rs +++ b/sdk/js/native/examples/restore_record.rs @@ -13,6 +13,7 @@ use layerfs_bridge::control::{Answer, Reply, SdkValue}; use layerfs_bridge::provision::{ProviderKind, StoreManifest, StoreProfile}; use std::io::Write; +#[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; fn hex(text: &str) -> [u8; N] { @@ -23,6 +24,7 @@ fn hex(text: &str) -> [u8; N] { bytes.try_into().expect("length") } +#[cfg(unix)] fn main() { let args: Vec = std::env::args().skip(1).collect(); let [home, project, binding, stack, branch, root, bytes, host, daemon] = &args[..] else { @@ -78,3 +80,10 @@ fn main() { file.write_all(&record).expect("write project.record"); println!("{}", path.display()); } + +// The record format this writes (0o600 mode, unix paths) belongs to the +// Linux restore runbook; Windows discovery saves its own records. +#[cfg(not(unix))] +fn main() { + eprintln!("restore_record is the Linux restore runbook; not run here"); +} diff --git a/sdk/js/test/init_refusals.mjs b/sdk/js/test/init_refusals.mjs index 1dda06d03..e82148bcf 100644 --- a/sdk/js/test/init_refusals.mjs +++ b/sdk/js/test/init_refusals.mjs @@ -14,6 +14,14 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +if (process.platform === "win32") { + // Every refusal this test drives is a unix mode-bit or filesystem-type + // contract (setgid, setuid, sticky, overlayfs, FIFO); Windows carries + // none of them. The Windows Store-face lifecycle is store_lifecycle.mjs. + console.log("init_refusals: unix-only contracts, not run on Windows"); + process.exit(0); +} + const OVERLAYFS = 0x794c7630; const work = mkdtempSync(join(tmpdir(), "layerfs-init-refusals-")); const home = join(work, "home"); diff --git a/sdk/js/test/reinit.mjs b/sdk/js/test/reinit.mjs index b3c2d4470..36b85ea19 100644 --- a/sdk/js/test/reinit.mjs +++ b/sdk/js/test/reinit.mjs @@ -28,14 +28,18 @@ try { }; const layerfs = client(); - // A set-group-ID source directory has a mode LayerFS cannot store. - chmodSync(source, 0o2755); - await assert.rejects(layerfs.project.init(source), { - code: "InvalidInput", - detail: { reason: "Mode", path: source, mode: "2755" }, - }); - assert.ok(!existsSync(join(work, "home", "website")), "a refused Init leaves no entry"); - chmodSync(source, 0o755); + // A set-group-ID source directory has a mode LayerFS cannot store. The + // bit does not exist on Windows, whose chmod maps only the read-only + // attribute; the refusal contract is checked on unix platforms. + if (process.platform !== "win32") { + chmodSync(source, 0o2755); + await assert.rejects(layerfs.project.init(source), { + code: "InvalidInput", + detail: { reason: "Mode", path: source, mode: "2755" }, + }); + assert.ok(!existsSync(join(work, "home", "website")), "a refused Init leaves no entry"); + chmodSync(source, 0o755); + } const first = await layerfs.project.init(source); assert.equal(first.kind, "Initialized"); diff --git a/sdk/js/test/store_lifecycle.mjs b/sdk/js/test/store_lifecycle.mjs new file mode 100644 index 000000000..7e375b632 --- /dev/null +++ b/sdk/js/test/store_lifecycle.mjs @@ -0,0 +1,47 @@ +// The Store-face lifecycle on the real native binding (owner direction +// 2026-10-25, route 2): project init, fork, a one-Commit fs.write, reading +// back on the new branch and on main, stat, the Commit listing and close — +// all in this process through the SDK's direct Store route. No Docker, no +// daemon, no WSL2. The Store's volume is LAYERFS_STORE_PATH's parent when +// the E2E launcher provides one (Linux); otherwise this machine's temporary +// volume, which the Windows route guards as NTFS: +// node sdk/js/test/store_lifecycle.mjs +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +const work = mkdtempSync(join(tmpdir(), "layerfs-lifecycle-")); +const volume = process.env.LAYERFS_STORE_PATH + ? join(dirname(dirname(process.env.LAYERFS_STORE_PATH)), `lifecycle-${process.pid}`) + : join(work, "volume"); +const store = join(volume, "global", "store.sqlite"); +try { + const source = join(work, "source"); + mkdirSync(source); + writeFileSync(join(source, "a.txt"), "hello windows-native store\n"); + process.env.LAYERFS_HOME = join(work, "home"); + mkdirSync(join(volume, "global"), { recursive: true }); + const { LayerFS } = await import("@layerfs/sdk"); + const layerfs = new LayerFS("lifecycle", { storePath: store }); + + assert.equal((await layerfs.project.init(source)).kind, "Initialized"); + await layerfs.branch.fork("dev"); + + const body = "written by the native Store face\n"; + assert.equal((await layerfs.fs.write("dev", "/windows.md", body, "lifecycle write")).kind, "Committed"); + + assert.equal((await layerfs.fs.read("dev", "/windows.md")).text, body); + assert.equal((await layerfs.fs.read("main", "/a.txt")).text, "hello windows-native store\n"); + assert.equal((await layerfs.fs.stat("dev", "/windows.md")).size, Buffer.byteLength(body)); + + const commits = await layerfs.branch.list_commits("dev"); + const records = commits.records ?? commits.items ?? commits; + assert.ok(Array.isArray(records) && records.length >= 1, JSON.stringify(commits).slice(0, 200)); + + await layerfs.close(); + console.log(`store_lifecycle: ok (${records.length} commit listed)`); +} finally { + rmSync(work, { recursive: true, force: true }); + rmSync(volume, { recursive: true, force: true }); +} diff --git a/sdk/js/test/trace.mjs b/sdk/js/test/trace.mjs index 4b92cec2e..ffcf2546f 100644 --- a/sdk/js/test/trace.mjs +++ b/sdk/js/test/trace.mjs @@ -5,9 +5,11 @@ import assert from "node:assert/strict"; import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const FAKE = fileURLToPath(new URL("./fixture/fake-native.cjs", import.meta.url)); +// import() needs a file:// URL; a bare Windows path is not one. +const FAKE_URL = pathToFileURL(FAKE).href; Object.assign(process.env, { LAYERFS_NATIVE: FAKE, LAYERFS_HOME: "/private/trace-home", LAYERFS_DOCKER: "/fake.sock", LAYERFS_IMAGE: "fake-image", LAYERFS_VOLUME: "fake-volume", LAYERFS_DAEMON: "/fake/daemon", @@ -80,7 +82,7 @@ const selected = process.argv[2]; try { if (selected === "on") { const reference = await script(new LayerFS("website")); - const native = (await import(FAKE)).default; + const native = (await import(FAKE_URL)).default; const before = native.calls.length; assert.ok(native.observed.every((scope) => scope === null), "untraced calls carry no scope"); process.env.LAYERFS_TRACE = file; @@ -253,7 +255,7 @@ try { } else if (selected === "store") { // Store calls open the client's Store in this process once, and close() // releases it; no Sandbox is started for them (#352 F-020). - const native = (await import(FAKE)).default; + const native = (await import(FAKE_URL)).default; const client = new LayerFS("website"); await client.project.list_branches(); await client.fs.read("main", "/README.md"); @@ -286,7 +288,7 @@ try { // A client in a container dials its daemonHost on every call that // reaches a Sandbox, operator calls included: the binding reopens or // starts the Sandbox with the call's deploy. - const native = (await import(FAKE)).default; + const native = (await import(FAKE_URL)).default; await script(new LayerFS("website", { daemonHost: "host.docker.internal" })); const named = native.calls.filter((_, index) => "sandbox" in native.args[index]); for (const method of ["workspace.exec", "workspace.status", "workspace.commit", "workspace.unmount", "operator.workspace.overlay", "operator.workspace.changes", "operator.inspect.snapshot", "operator.process.start"]) { From 843d22f3a7c17acba2bcda8eb3aa13a3ee2e4fde Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 00:53:52 +0800 Subject: [PATCH 05/19] test(sdk): the full Store method matrix on the native route Slice 4 of the Windows-native Store route: sdk/js/test/store_matrix.mjs drives the Store face's whole public method table through the real binding on the direct route - writes (write, edit, mkdir, cp, mv, rm), reads (read with a line range, list, stat, grep, glob, blame), comparisons (diff_files, diff_commits, diff_branches) and combination (merge with its merged-branch readback, publish with main's readback and the branch listing) - plus two contracts worth pinning: main is the project's main line, not a Branch, so branch.list_commits("main") is NoSuchBranch and main moves only through branch.publish; and on Windows the daemon-only workspace surface refuses Unsupported once the client's own deploy validation is satisfied. No Docker, no daemon, no WSL2; the same file runs on Linux against the E2E volume. Verification: three consecutive rounds of store_matrix, store_lifecycle, binary_write and reinit all pass on Windows against this tree's addon; trace (9 cases) and packaging (2 cases) unchanged. Production LOC: core 97258 -> 97258 (delta 0), sdk 3455 -> 3455 (delta 0); combined 109896 -> 109896. Test-only change: the sdk scope counts sdk/js/src and sdk/js/operator, so the new test contributes no production lines. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- sdk/js/test/store_matrix.mjs | 110 +++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 sdk/js/test/store_matrix.mjs diff --git a/sdk/js/test/store_matrix.mjs b/sdk/js/test/store_matrix.mjs new file mode 100644 index 000000000..758ea6c8a --- /dev/null +++ b/sdk/js/test/store_matrix.mjs @@ -0,0 +1,110 @@ +// The full Store method matrix on the real native binding (owner direction +// 2026-10-25, route 2): every fs, branch and project operation the direct +// route serves — writes (write, edit, mkdir, cp, mv, rm, apply), reads +// (read with a line range, list, stat, grep, glob, blame), comparisons +// (diff_files, diff_commits, diff_branches) and branch combination +// (merge, publish) — plus the Windows refusal of the daemon-only workspace +// surface. No Docker, no daemon, no WSL2; on Linux the Store sits on the +// E2E launcher's volume (LAYERFS_STORE_PATH): +// node sdk/js/test/store_matrix.mjs +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +const work = mkdtempSync(join(tmpdir(), "layerfs-matrix-")); +const volume = process.env.LAYERFS_STORE_PATH + ? join(dirname(dirname(process.env.LAYERFS_STORE_PATH)), `matrix-${process.pid}`) + : join(work, "volume"); +const store = join(volume, "global", "store.sqlite"); +const retryRemove = (path) => { + for (let attempt = 0; attempt < 5; attempt++) { + try { rmSync(path, { recursive: true, force: true }); return; } catch { } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + } +}; +try { + const source = join(work, "source"); + mkdirSync(join(source, "lib"), { recursive: true }); + writeFileSync(join(source, "README.md"), "one\ntwo\nthree\n"); + writeFileSync(join(source, "lib/app.js"), "const a = 1;\nconst b = 2;\n"); + process.env.LAYERFS_HOME = join(work, "home"); + mkdirSync(join(volume, "global"), { recursive: true }); + const { LayerFS } = await import("@layerfs/sdk"); + const layerfs = new LayerFS("matrix", { storePath: store }); + + assert.equal((await layerfs.project.init(source)).kind, "Initialized"); + const branches = await layerfs.project.list_branches(); + assert.deepEqual(branches.items.map((item) => item.name), ["main"]); + await layerfs.branch.fork("work"); + + // Writes: each call is one Commit; the surface changes shape together. + await layerfs.fs.mkdir("work", "/docs", "make docs"); + await layerfs.fs.write("work", "/docs/note.md", "alpha\n", "first note"); + const second = await layerfs.fs.write("work", "/docs/note.md", "alpha\nbeta\n", "second note"); + assert.equal(second.kind, "Committed"); + const edited = await layerfs.fs.edit("work", "/docs/note.md", "alpha", "gamma", "reword"); + assert.equal(edited.kind, "Committed"); + assert.notEqual(edited.head, second.head, "each write call is one new Commit"); + await layerfs.fs.cp("work", "/docs/note.md", "/docs/copy.md", "copy"); + await layerfs.fs.mv("work", "/docs/copy.md", "/docs/moved.md", "move"); + await layerfs.fs.rm("work", "/docs/moved.md", "remove"); + const listing = await layerfs.fs.list("work", "/docs"); + assert.deepEqual(listing.items.map((item) => item.name).sort(), ["note.md"]); + + // Reads: content, shape and search over the written tree. + assert.equal((await layerfs.fs.read("work", "/docs/note.md")).text, "gamma\nbeta\n"); + const ranged = await layerfs.fs.read("work", "/README.md", 1, 2); + assert.equal(ranged.text, "one\ntwo\n"); + assert.equal((await layerfs.fs.stat("work", "/docs/note.md")).size, Buffer.byteLength("gamma\nbeta\n")); + const grep = await layerfs.fs.grep("work", "gamma"); + assert.deepEqual(grep.items.map((hit) => hit.path), ["/docs/note.md"]); + assert.deepEqual((await layerfs.fs.glob("work", "**/*.js")).items, ["/lib/app.js"]); + const blame = await layerfs.fs.blame("work", "/docs/note.md"); + assert.equal(blame.lines.length, 2); + assert.ok(blame.lines.every((line) => typeof line.commit === "string")); + + // Comparisons: file, commit and branch. + assert.equal((await layerfs.fs.diff_files("main", "/README.md", "work", "/README.md")).kind, "Equal"); + const changed = await layerfs.fs.diff_files("main", "/README.md", "work", "/docs/note.md"); + assert.notEqual(changed.kind, "Equal"); + const commits = await layerfs.branch.list_commits("work"); + assert.ok(commits.items.length >= 6, `${commits.items.length} commits listed`); + // main is the project's main line, not a Branch: it moves only through + // branch.publish and is refused by the Branch name index. + await assert.rejects(layerfs.branch.list_commits("main"), { code: "NoSuchBranch" }); + const oldest = commits.items[commits.items.length - 1]; + const commitDiff = await layerfs.branch.diff_commits(oldest.id, commits.items[0].id); + assert.ok(JSON.stringify(commitDiff).includes("/docs/note.md"), JSON.stringify(commitDiff).slice(0, 200)); + const branchDiff = await layerfs.branch.diff_branches("main", "work"); + assert.equal(branchDiff.kind, "ok"); + + // Combination: merge reads both lines; publish moves main's head. + const merged = await layerfs.branch.merge("main", "work", "merged", "combine"); + assert.equal(merged.kind, "Merged"); + assert.equal((await layerfs.fs.read("merged", "/docs/note.md")).text, "gamma\nbeta\n"); + const published = await layerfs.branch.publish("work"); + assert.ok(published.kind, JSON.stringify(published).slice(0, 200)); + assert.equal((await layerfs.fs.read("main", "/docs/note.md")).text, "gamma\nbeta\n"); + const after = await layerfs.project.list_branches(); + assert.deepEqual(after.items.map((item) => item.name).sort(), ["main", "merged", "work"]); + + // The daemon-only surface refuses Unsupported on the Windows route; on + // Linux the workspace route is the daemon's and is not exercised here. + if (process.platform === "win32") { + // The client's own deploy validation must pass first; the refusal under + // test is the native layer's, for the daemon-only surface. + process.env.LAYERFS_IMAGE = "fake-image"; + process.env.LAYERFS_VOLUME = "fake-volume"; + process.env.LAYERFS_DAEMON = "/fake/daemon"; + await assert.rejects(layerfs.workspace.mount("work", join(work, "mount")), { + code: "Unsupported", + }, "the Windows route refuses the daemon surface"); + } + + await layerfs.close(); + console.log(`store_matrix: ok (${commits.items.length} commits, ${after.items.length} branches)`); +} finally { + retryRemove(work); + retryRemove(volume); +} From ada6f39f764302059f19b78d730bc1acbecec7f7 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 01:15:59 +0800 Subject: [PATCH 06/19] test(sdk): cross-process concurrent Store writers on the native route Four re-exec'd child processes write distinct files to one branch of one Store concurrently, each through its own client of the direct route: the named-mutex writer lane serializes the publishes and the discovery record guard serializes the shared home's project record, so every write must land. The parent reads all four files back and counts the Commits exactly (base + 4). Bounded: each child is killed at 30 s and its captured output is carried into the parent's failure. No Docker, no daemon, no WSL2; the same file runs on Linux against the E2E volume. Verification: four consecutive runs pass on Windows against this tree's addon (4 writers, base+4 commits each time). Production LOC: core 97258 -> 97258 (delta 0), sdk 3455 -> 3455 (delta 0); combined 109896 -> 109896. Test-only change. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- sdk/js/test/store_concurrent.mjs | 91 ++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 sdk/js/test/store_concurrent.mjs diff --git a/sdk/js/test/store_concurrent.mjs b/sdk/js/test/store_concurrent.mjs new file mode 100644 index 000000000..e059d3d3e --- /dev/null +++ b/sdk/js/test/store_concurrent.mjs @@ -0,0 +1,91 @@ +// Cross-process Store writers on the real native binding (owner direction +// 2026-10-25, route 2): the parent initializes a project and forks one +// branch, then four re-exec'd child processes write distinct files to that +// branch concurrently, each through its own client of the same Store - the +// direct route's cross-process writer lane and the discovery record guard +// serialize them; every write must land. The parent reads all four files +// back and counts the Commits. No Docker, no daemon, no WSL2; on Linux the +// Store sits on the E2E launcher's volume (LAYERFS_STORE_PATH): +// node sdk/js/test/store_concurrent.mjs # parent run +// node sdk/js/test/store_concurrent.mjs --child +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +// One child writer: open the project's Store and write its own file once. +if (process.argv[2] === "--child") { + const [, , , home, store, name] = process.argv; + process.env.LAYERFS_HOME = home; + const { LayerFS } = await import("@layerfs/sdk"); + const writer = new LayerFS("concurrent", { storePath: store }); + const body = `written by ${name} in pid ${process.pid}\n`; + const done = await writer.fs.write("load", `/${name}.md`, body, `write ${name}`); + await writer.close(); + assert.equal(done.kind, "Committed"); + process.exit(0); +} + +const WRITERS = 4; +const work = mkdtempSync(join(tmpdir(), "layerfs-concurrent-")); +const volume = process.env.LAYERFS_STORE_PATH + ? join(dirname(dirname(process.env.LAYERFS_STORE_PATH)), `concurrent-${process.pid}`) + : join(work, "volume"); +const store = join(volume, "global", "store.sqlite"); +const home = join(work, "home"); +const retryRemove = (path) => { + for (let attempt = 0; attempt < 5; attempt++) { + try { rmSync(path, { recursive: true, force: true }); return; } catch { } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + } +}; +try { + const source = join(work, "source"); + mkdirSync(source); + writeFileSync(join(source, "README.md"), "concurrent writers\n"); + process.env.LAYERFS_HOME = home; + mkdirSync(join(volume, "global"), { recursive: true }); + const { LayerFS } = await import("@layerfs/sdk"); + const parent = new LayerFS("concurrent", { storePath: store }); + await parent.project.init(source); + await parent.branch.fork("load"); + const base = (await parent.branch.list_commits("load")).items.length; + + const children = []; + for (let n = 1; n <= WRITERS; n++) { + children.push(spawn(process.execPath, [import.meta.url.pathname ?? process.argv[1], "--child", home, store, `w${n}`], { + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, LAYERFS_STORE_PATH: store }, + })); + } + const reports = await Promise.all(children.map(async (child, index) => { + const [code, output] = await new Promise((resolve) => { + let captured = ""; + child.stdout.on("data", (chunk) => { captured += chunk; }); + child.stderr.on("data", (chunk) => { captured += chunk; }); + const timer = setTimeout(() => { + child.kill(); + resolve([-1, `${captured} (killed: 30s ceiling)`]); + }, 30_000); + child.on("exit", (exitCode) => { clearTimeout(timer); resolve([exitCode, captured]); }); + }); + return { index, code, output }; + })); + for (const { index, code, output } of reports) { + assert.equal(code, 0, `writer w${index + 1} failed (${code}): ${output.slice(0, 400)}`); + } + + // Every writer's file is committed and readable; the branch carries them all. + for (let n = 1; n <= WRITERS; n++) { + const read = await parent.fs.read("load", `/w${n}.md`); + assert.ok(read.text.startsWith(`written by w${n} in pid `), read.text); + } + const commits = await parent.branch.list_commits("load", 50); + assert.equal(commits.items.length, base + WRITERS, `${commits.items.length} commits, expected ${base + WRITERS}`); + await parent.close(); + console.log(`store_concurrent: ok (${WRITERS} writers, ${commits.items.length} commits)`); +} finally { + retryRemove(work); + retryRemove(volume); +} From 5a41b5f11d65df53b1d455cd38f4e11e2bb8d097 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 01:42:13 +0800 Subject: [PATCH 07/19] test(windows): abandoned-lane acquisition and divergent merge readback Two coverage pins from the confidence review: - A dead lane holder does not block the next writer: a thread ends while owning the product's own named mutex (the name derived exactly as windows_lane derives it), the kernel marks it abandoned, and the Store still opens and writes through that WAIT_ABANDONED acquisition. Deterministic - no timing, no kill race. - The matrix test's merge becomes a divergent one: both sources advanced past their common base, so fastForward must be false, the merged head carries two parents, and both sides' files are read back on the merged Branch and on main after publish - the merge's content correctness, which the earlier fast-forward case could not show. Verification: windows_store_native 2 passed; store_matrix passes with the five-branch end state on Windows. Production LOC: core 97258 -> 97258 (delta 0), sdk 3455 -> 3455 (delta 0). Test-only change. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../tests/windows_store_native.rs | 53 ++++++++++++++++++- sdk/js/test/store_matrix.mjs | 19 ++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/core/crates/layerfs-persistence/tests/windows_store_native.rs b/core/crates/layerfs-persistence/tests/windows_store_native.rs index 5bc48b718..5af13e74c 100644 --- a/core/crates/layerfs-persistence/tests/windows_store_native.rs +++ b/core/crates/layerfs-persistence/tests/windows_store_native.rs @@ -73,9 +73,58 @@ fn finish(mut child: std::process::Child) { } } +/// The lane's dead-holder path: a thread that ends while owning the named +/// mutex leaves it abandoned, and the kernel hands ownership to the next +/// waiter. The abandoned state is produced deterministically here - a +/// thread waits on the product's own mutex name and returns without +/// releasing - and the Store must still open and write through that +/// `WAIT_ABANDONED` acquisition. #[test] -fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { - dispatch_child(); +fn a_dead_lane_holder_does_not_block_the_next_writer() { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let temp = support::Temp::new("windows-store-abandoned"); + let path = temp.join("store.sqlite"); + let catalog = authority(b"windows-store-abandoned"); + let seed = Handles::create(config(&path), StoragePolicy::frozen_default(), &catalog).unwrap(); + seed.storage.reserve(reserve()).unwrap(); + drop(seed); + + let directory = temp.path().to_path_buf(); + let holder = std::thread::spawn(move || { + // The product's own lane name, derived exactly as windows_lane does. + let mut hasher = DefaultHasher::new(); + match std::fs::canonicalize(&directory) { + Ok(canonical) => canonical.hash(&mut hasher), + Err(_) => directory.hash(&mut hasher), + } + let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", hasher.finish()) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { + windows_sys::Win32::System::Threading::CreateMutexW( + std::ptr::null(), + 0, + name.as_ptr(), + ) + }; + assert!(!handle.is_null()); + let waited = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(handle, windows_sys::Win32::System::Threading::INFINITE) }; + assert_eq!(waited, windows_sys::Win32::Foundation::WAIT_OBJECT_0); + // The thread owns the mutex and now ends without ReleaseMutex: the + // kernel marks it abandoned for the next waiter. + }); + holder.join().unwrap(); + + let reopened = Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key) + .expect("an abandoned lane is taken by the next writer"); + reopened.storage.reserve(reserve()).unwrap(); + drop(reopened); +} + +#[test] +fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { dispatch_child(); let temp = support::Temp::new("windows-store-native"); let path = temp.join("store.sqlite"); let catalog = authority(b"windows-store-native"); diff --git a/sdk/js/test/store_matrix.mjs b/sdk/js/test/store_matrix.mjs index 758ea6c8a..b6b00321c 100644 --- a/sdk/js/test/store_matrix.mjs +++ b/sdk/js/test/store_matrix.mjs @@ -86,8 +86,25 @@ try { const published = await layerfs.branch.publish("work"); assert.ok(published.kind, JSON.stringify(published).slice(0, 200)); assert.equal((await layerfs.fs.read("main", "/docs/note.md")).text, "gamma\nbeta\n"); + // A divergent merge: both sides advanced past their common base, so this + // is a real merge Commit (two parents), and both sides' files must be + // readable on the merged Branch and on main after its publish. + await layerfs.fs.write("work", "/later.md", "later\n", "work advanced"); + await layerfs.branch.fork("feat2"); + await layerfs.fs.write("feat2", "/feat2.md", "feat2 line\n", "feat2 advanced"); + const combined = await layerfs.branch.merge("work", "feat2", "combined", "combine diverged"); + assert.equal(combined.kind, "Merged"); + assert.equal(combined.fastForward, false, "diverged sources cannot fast-forward"); + assert.equal((await layerfs.fs.read("combined", "/later.md")).text, "later\n"); + assert.equal((await layerfs.fs.read("combined", "/feat2.md")).text, "feat2 line\n"); + const combinedHead = (await layerfs.branch.list_commits("combined", 1)).items[0]; + assert.ok(combinedHead.secondParent, "a divergent merge Commit carries two parents"); + await layerfs.branch.publish("combined"); + assert.equal((await layerfs.fs.read("main", "/later.md")).text, "later\n"); + assert.equal((await layerfs.fs.read("main", "/feat2.md")).text, "feat2 line\n"); + const after = await layerfs.project.list_branches(); - assert.deepEqual(after.items.map((item) => item.name).sort(), ["main", "merged", "work"]); + assert.deepEqual(after.items.map((item) => item.name).sort(), ["combined", "feat2", "main", "merged", "work"]); // The daemon-only surface refuses Unsupported on the Windows route; on // Linux the workspace route is the daemon's and is not exercised here. From 3d90143b15a8bc19d80126de73001fdf3f3e1468 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 02:43:33 +0800 Subject: [PATCH 08/19] fix(persistence): lane-hold open without Linux self-deadlock; dead-lane test The 3b lane wraps deadlocked Linux opens: the setup block's raw second Writer::acquire held the directory flock on one open file description while Session::run's admission inside acquired it on another - flock is per description, so the opening thread waited on itself. Windows passed only because its named kernel mutex is thread-recursive; the Linux container suite caught it (concurrent_first_installs: 0.4s at the parent commit, >115s hung at 3b, diagnosed by /proc/locks, per-acquire backtraces and a parent-commit rerun). The corrected shape, verified on both platforms: - Session::connect holds the writer lane for the whole connect-time statement block (profile pragmas, stored-format and profile reads) - the open's first database-touching statements, where a cleaned -shm is rebuilt under an exclusive recovery hold. - The writable setup block (binding hints, lineage, merge scratch, file digests) runs inside one held publication lane: its statements enter as nested turns and take no second lock; cross-process opens are serialized by the lane itself. - The merge-record sweep stays plain: its admission takes the lane. Also: a_dead_lane_holder_does_not_block_the_next_writer moves to its own binary (windows_lane_abandoned.rs) so it cannot run beside the foreign-process test's re-exec'd children, whose spawn now filters --exact; the persistence test support's temporary directory names gain nanoseconds (the project crate's already had them), removing cross-run name collisions on Windows pid reuse. Verification: Linux container - concurrent_first_installs 0.37s; persistence 99 passed 0 failed, project 22/0, store-service 3/0, overlay 74/0. Windows - the lane binary 50/50 clean isolated runs; the SDK suite (store_matrix, store_concurrent, store_lifecycle) passes on a rebuilt addon. One residual, documented openly: under cargo test the lane test's re-exec'd child still meets Busy roughly once per 25 suite runs (never in 50 isolated binary runs); its report is carried into the failure. The busy source inside a mutex-held open remains undiagnosed - recorded as an open item, not fixed by this commit. Production LOC: core 97258 -> 97255 (delta -3; persistence -3: the closed-over profile tail moves lines, the lane-held setup comment replaces the raw wrap), sdk 3455 -> 3455 (delta 0). Counter: tools/production_loc.py (pinned at this tree), scopes core and sdk; tests excluded. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../src/backend/sqlite/connection.rs | 12 ++- .../layerfs-persistence/src/store/open.rs | 32 ++++---- .../layerfs-persistence/tests/support/mod.rs | 6 +- .../tests/windows_lane_abandoned.rs | 78 +++++++++++++++++++ .../tests/windows_store_native.rs | 57 ++------------ 5 files changed, 113 insertions(+), 72 deletions(-) create mode 100644 core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs b/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs index 4850c02eb..d7598adfa 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/connection.rs @@ -182,9 +182,11 @@ impl Session { // Busy: hold it for this one short setup step and release before // the reads that follow. macOS keeps exact SQLite Busy by design. let writer = super::writer_admission::Writer::acquire(path)?; - let applied = profile::apply(&connection, create, selected, creation_acquisition, &work); - writer.release().map_err(|_| BackendError::Integrity)?; - applied?; + let connected = (|| -> Result< + (ConnectionProfile, crate::SqlitePackLayout, crate::SqliteAcquisitionSchema), + BackendError, + > { + profile::apply(&connection, create, selected, creation_acquisition, &work)?; let (layout, acquisition) = if create { (creation_layout, creation_acquisition) } else { @@ -257,6 +259,10 @@ impl Session { .map_err(rows::error)? as usize, }; profile::check(&profile)?; + Ok((profile, layout, acquisition)) + })(); + writer.release().map_err(|_| BackendError::Integrity)?; + let (profile, layout, acquisition) = connected?; super::thread_work::add(&work.borrow()); Ok(Self { admission: super::admission::Admission::default(), diff --git a/core/crates/layerfs-persistence/src/store/open.rs b/core/crates/layerfs-persistence/src/store/open.rs index d94252fd0..328d0c531 100644 --- a/core/crates/layerfs-persistence/src/store/open.rs +++ b/core/crates/layerfs-persistence/src/store/open.rs @@ -92,12 +92,15 @@ impl Handles { config.sqlite_acquisition, )?); if writable { - // The idempotent setup statements are write transactions, so two - // concurrent opens of one Store meet Busy exactly here under the - // no-retry rule. The coarse writer lane replaces Busy: one short - // hold covers the whole setup block, released before validation. - let writer = - crate::backend::sqlite::writer_admission::Writer::acquire(&config.path)?; + // The setup statements are write transactions that two + // concurrent opens of one Store can interleave under the + // no-retry rule, so the whole block runs inside one held + // publication lane: its writer lock serializes the opens + // cross-process, and the block's own statements enter as + // nested turns, which do not take the lock a second time - + // a raw second hold here would self-deadlock on Linux, where + // flock is held per open file description. + let lane = session.hold_lane()?; let setup = (|| -> Result<(), crate::backend::records::BackendError> { session.add_binding_hints()?; @@ -105,22 +108,15 @@ impl Handles { session.add_merge_scratch()?; session.add_file_digests() })(); - writer - .release() - .map_err(|_| crate::backend::records::BackendError::Integrity)?; + drop(lane); setup?; } let handles = Self::validated(session, directory, binding, cursor_key)?; if writable { - // The sweep deletes stale merge records: a write, held behind the - // lane for the same reason as the setup block above. - let writer = - crate::backend::sqlite::writer_admission::Writer::acquire(&config.path)?; - let swept = crate::storage::merge_records::sweep(&handles.storage.session); - writer - .release() - .map_err(|_| crate::backend::records::BackendError::Integrity)?; - swept?; + // The sweep runs as one write transaction whose admission takes + // the writer lane itself; the setup block's held lane above is + // released by then. + crate::storage::merge_records::sweep(&handles.storage.session)?; } Ok(handles) } diff --git a/core/crates/layerfs-persistence/tests/support/mod.rs b/core/crates/layerfs-persistence/tests/support/mod.rs index 86b851581..419a11953 100644 --- a/core/crates/layerfs-persistence/tests/support/mod.rs +++ b/core/crates/layerfs-persistence/tests/support/mod.rs @@ -20,9 +20,13 @@ impl Temp { pub fn new(name: &str) -> Self { static NEXT: AtomicU64 = AtomicU64::new(0); let path = std::env::temp_dir().join(format!( - "layerfs-history-{}-{}-{}", + "layerfs-history-{}-{}-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|span| span.as_nanos()) + .unwrap_or(0), name )); std::fs::create_dir_all(&path).expect("temporary directory"); diff --git a/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs b/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs new file mode 100644 index 000000000..13658a85a --- /dev/null +++ b/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs @@ -0,0 +1,78 @@ +//! The Windows lane's dead-holder path, in its own binary so it never runs +//! beside the foreign-process test's re-exec'd children. +#![cfg(windows)] +mod support; +use layerfs_history::HistoryCatalogConfig; +use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; +use layerfs_storage::{port::*, StoragePolicy}; +use std::path::{Path, PathBuf}; + +fn config(path: &Path) -> PersistenceConfig { + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable) +} + +fn reserve() -> Reserve { + Reserve { + packs: 1, + ordinals: 1, + } +} + +fn authority(name: &[u8]) -> HistoryCatalogConfig { + HistoryCatalogConfig { + binding_key: name.to_vec(), + cursor_key: [67; 32], + incarnation: 1, + } +} + +/// The lane's dead-holder path: a thread that ends while owning the named +/// mutex leaves it abandoned, and the kernel hands ownership to the next +/// waiter. The abandoned state is produced deterministically here - a +/// thread waits on the product's own mutex name and returns without +/// releasing - and the Store must still open and write through that +/// `WAIT_ABANDONED` acquisition. +#[test] +fn a_dead_lane_holder_does_not_block_the_next_writer() { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let temp = support::Temp::new("windows-store-abandoned"); + let path = temp.join("store.sqlite"); + let catalog = authority(b"windows-store-abandoned"); + let seed = Handles::create(config(&path), StoragePolicy::frozen_default(), &catalog).unwrap(); + seed.storage.reserve(reserve()).unwrap(); + drop(seed); + + let directory = temp.path().to_path_buf(); + let holder = std::thread::spawn(move || { + // The product's own lane name, derived exactly as windows_lane does. + let mut hasher = DefaultHasher::new(); + match std::fs::canonicalize(&directory) { + Ok(canonical) => canonical.hash(&mut hasher), + Err(_) => directory.hash(&mut hasher), + } + let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", hasher.finish()) + .encode_utf16() + .chain(std::iter::once(0)) + .collect(); + let handle = unsafe { + windows_sys::Win32::System::Threading::CreateMutexW( + std::ptr::null(), + 0, + name.as_ptr(), + ) + }; + assert!(!handle.is_null()); + let waited = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(handle, windows_sys::Win32::System::Threading::INFINITE) }; + assert_eq!(waited, windows_sys::Win32::Foundation::WAIT_OBJECT_0); + // The thread owns the mutex and now ends without ReleaseMutex: the + // kernel marks it abandoned for the next waiter. + }); + holder.join().unwrap(); + + let reopened = Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key) + .expect("an abandoned lane is taken by the next writer"); + reopened.storage.reserve(reserve()).unwrap(); + drop(reopened); +} + diff --git a/core/crates/layerfs-persistence/tests/windows_store_native.rs b/core/crates/layerfs-persistence/tests/windows_store_native.rs index 5af13e74c..ec6412df1 100644 --- a/core/crates/layerfs-persistence/tests/windows_store_native.rs +++ b/core/crates/layerfs-persistence/tests/windows_store_native.rs @@ -73,56 +73,6 @@ fn finish(mut child: std::process::Child) { } } -/// The lane's dead-holder path: a thread that ends while owning the named -/// mutex leaves it abandoned, and the kernel hands ownership to the next -/// waiter. The abandoned state is produced deterministically here - a -/// thread waits on the product's own mutex name and returns without -/// releasing - and the Store must still open and write through that -/// `WAIT_ABANDONED` acquisition. -#[test] -fn a_dead_lane_holder_does_not_block_the_next_writer() { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let temp = support::Temp::new("windows-store-abandoned"); - let path = temp.join("store.sqlite"); - let catalog = authority(b"windows-store-abandoned"); - let seed = Handles::create(config(&path), StoragePolicy::frozen_default(), &catalog).unwrap(); - seed.storage.reserve(reserve()).unwrap(); - drop(seed); - - let directory = temp.path().to_path_buf(); - let holder = std::thread::spawn(move || { - // The product's own lane name, derived exactly as windows_lane does. - let mut hasher = DefaultHasher::new(); - match std::fs::canonicalize(&directory) { - Ok(canonical) => canonical.hash(&mut hasher), - Err(_) => directory.hash(&mut hasher), - } - let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", hasher.finish()) - .encode_utf16() - .chain(std::iter::once(0)) - .collect(); - let handle = unsafe { - windows_sys::Win32::System::Threading::CreateMutexW( - std::ptr::null(), - 0, - name.as_ptr(), - ) - }; - assert!(!handle.is_null()); - let waited = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(handle, windows_sys::Win32::System::Threading::INFINITE) }; - assert_eq!(waited, windows_sys::Win32::Foundation::WAIT_OBJECT_0); - // The thread owns the mutex and now ends without ReleaseMutex: the - // kernel marks it abandoned for the next waiter. - }); - holder.join().unwrap(); - - let reopened = Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key) - .expect("an abandoned lane is taken by the next writer"); - reopened.storage.reserve(reserve()).unwrap(); - drop(reopened); -} - #[test] fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { dispatch_child(); let temp = support::Temp::new("windows-store-native"); @@ -156,6 +106,13 @@ fn windows_native_store_serves_sessions_and_serializes_foreign_processes() { for _ in 0..2 { children.push( Command::new(&executable) + // The child runs only this test; without the filter the + // harness would run every test in the binary (and their + // dispatch guards) beside it. + .args([ + "--exact", + "windows_native_store_serves_sessions_and_serializes_foreign_processes", + ]) .env(CHILD_STORE, &lane_path) .env(CHILD_BINDING, "windows-lane-processes") .stdout(Stdio::piped()) From 27ace1469b546a871fd09fb28cb4532d43c46c20 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 04:58:38 +0800 Subject: [PATCH 09/19] fix(persistence): the close-time checkpoint runs under the writer lane The residual lane-test flake is diagnosed and closed. Instrumented runs pinned it: a re-exec'd child met SQLITE_BUSY(5) on its first statement (PRAGMA synchronous=0, where a cleaned -shm is attached and possibly rebuilt) while holding the named writer mutex - the contender was the sibling process's connection teardown, whose close-time WAL checkpoint is SQLite work that no lane had ever covered. The same window exists on Linux through flock and in the daemon SDK's own direct-route close. The fix coordinates the close instead of the drop: layerfs-persistence exports StoreLane (a best-effort hold of the Store directory's writer lane, released on drop) and Handles::shutdown, which drops the session under that lane; the SDK's direct route holds the lane while the service's sessions close on storeClose; the lane tests' children and the abandoned-lane test close through shutdown. A plain drop keeps its old bare semantics - an uncoordinated close, like any error-path teardown, never waits behind the lane. An intermediate design (acquiring the lane in State::drop) is reverted: it deadlocked multi-session teardown in-process, caught by the SDK lifecycle (init+fork+write+close hung). Drop-time blocking locks stay out of the product. Verification: Windows - the persistence suite 25 consecutive rounds clean under cargo (the flake reproduced every 2-9 rounds before), the lane binary's children closing through shutdown; SDK store_matrix, store_concurrent, store_lifecycle and a minimal init-fork-write-close all pass on the rebuilt addon. Linux container - persistence 99/0, project 22/0, store-service 3/0, overlay 74/0. Production LOC: core 97255 -> 97271 (delta +16; persistence +14, node binding +2), sdk 3455 -> 3457 (delta +2); combined 109893 -> 109911. Counter: tools/production_loc.py (pinned at this tree). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/crates/layerfs-persistence/src/lib.rs | 17 +++++++++++++++++ .../layerfs-persistence/src/store/handles.rs | 8 ++++++++ .../tests/windows_lane_abandoned.rs | 2 +- .../tests/windows_store_native.rs | 3 +++ sdk/js/native/src/direct.rs | 4 ++++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/core/crates/layerfs-persistence/src/lib.rs b/core/crates/layerfs-persistence/src/lib.rs index 29184b59b..282adf4a4 100644 --- a/core/crates/layerfs-persistence/src/lib.rs +++ b/core/crates/layerfs-persistence/src/lib.rs @@ -19,6 +19,23 @@ pub use config::{ SqlitePersistenceProfile, }; pub use handles::Handles; + +/// The Store directory's writer lane, held for a coordinated close: the +/// connection teardown's close-time checkpoint runs inside it, never beside +/// another opener's first attach. Best-effort - a failed acquisition closes +/// bare, as an uncoordinated drop always has. +pub struct StoreLane(Option); +impl Drop for StoreLane { + fn drop(&mut self) { + if let Some(writer) = self.0.take() { + let _ = writer.release(); + } + } +} +/// Holds the writer lane of the Store directory holding `store`. +pub fn store_lane(store: &std::path::Path) -> StoreLane { + StoreLane(crate::backend::sqlite::writer_admission::Writer::acquire(store).ok()) +} pub use history::HistoryProvider; pub use storage::acquisition::AcquisitionProvider; pub use storage::merge_records::{ diff --git a/core/crates/layerfs-persistence/src/store/handles.rs b/core/crates/layerfs-persistence/src/store/handles.rs index 5a667a288..00f20b3d9 100644 --- a/core/crates/layerfs-persistence/src/store/handles.rs +++ b/core/crates/layerfs-persistence/src/store/handles.rs @@ -35,6 +35,14 @@ impl Handles { pub fn close(&self) { self.storage.session.admission.close(); } + /// Drops this session under the writer lane, so its connection's + /// close-time checkpoint never runs beside another opener's first + /// attach. A failed lane acquisition closes bare. + pub fn shutdown(self) { + let lane = crate::store_lane(&self.storage.session.path); + drop(self); + drop(lane); + } /// Read-back settings from the actual shared connection. pub fn profile(&self) -> &ConnectionProfile { &self.storage.session.profile diff --git a/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs b/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs index 13658a85a..ebd3e9855 100644 --- a/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs +++ b/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs @@ -73,6 +73,6 @@ fn a_dead_lane_holder_does_not_block_the_next_writer() { let reopened = Handles::open_writable(config(&path), &catalog.binding_key, catalog.cursor_key) .expect("an abandoned lane is taken by the next writer"); reopened.storage.reserve(reserve()).unwrap(); - drop(reopened); + reopened.shutdown(); } diff --git a/core/crates/layerfs-persistence/tests/windows_store_native.rs b/core/crates/layerfs-persistence/tests/windows_store_native.rs index ec6412df1..439f1da7c 100644 --- a/core/crates/layerfs-persistence/tests/windows_store_native.rs +++ b/core/crates/layerfs-persistence/tests/windows_store_native.rs @@ -37,6 +37,9 @@ fn child(path: PathBuf, binding: Vec) { let opened = Handles::open_writable(config(&path), &binding, [67; 32]) .expect("the child opens the Store"); opened.storage.reserve(reserve()).expect("the child writes"); + // A clean close: the child's Store is not crashed, so the sibling's + // next open does not pay WAL recovery beside the lane under test. + opened.shutdown(); } /// When this binary was re-executed as a lane child, run its one job and diff --git a/sdk/js/native/src/direct.rs b/sdk/js/native/src/direct.rs index b07102a67..e0f4a043e 100644 --- a/sdk/js/native/src/direct.rs +++ b/sdk/js/native/src/direct.rs @@ -109,10 +109,14 @@ pub(crate) fn release(args: &SdkValue) -> Outcome { } let opened = stores.remove(&store).expect("present above"); drop(stores); + // The service's sessions close under the Store's writer lane: their + // close-time checkpoints never run beside another opener's first attach. + let lane = layerfs_persistence::store_lane(&store); let closed = match Arc::try_unwrap(opened.service) { Ok(service) => service.close().map_err(failed), Err(_) => Ok(()), }; + drop(lane); let removed = std::fs::remove_dir_all(&opened.directory).map_err(failed); closed.and(removed)?; Ok(SdkValue::map([("clients", SdkValue::Int(0))])) From 27f6cf89aed9c4410fef61d9409b586f6e3ad174 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 07:34:46 +0800 Subject: [PATCH 10/19] fix(store-service): the service close coordinates its Store teardown StoreService::close now records the Store file (open_direct populates it through the new StoreService::at) and, after the overlay owner stops - its final Commits take the writer lane themselves, so no outer hold may surround that - drops the service under the Store's writer lane: the sessions' close-time checkpoints never run beside another opener's first attach. This is the daemon/SDK-symmetric completion of the coordinated close the SDK's direct route received; direct.rs drops its own manual hold in favor of the service's (a second hold on another descriptor would self-deadlock the closing thread on Linux). A service built with new() keeps an empty path and closes bare, as before - only open_direct records the file. Verification: Windows store-service tests 3/0, SDK store_matrix, store_concurrent, store_lifecycle pass on the rebuilt addon (the service close is in storeClose's path). Linux container: store-service 3/0; persistence 99/0, project 22/0, overlay 74/0 at this tree. The daemon suite needs a privileged container (/dev/fuse, fusectl, cgroup mounts): 62 non-mount tests pass; the 99 mount-path failures are environmental (attach:mount No such file, permission denied), the same at the parent commit - recorded as a pre-existing container limitation, not a regression of this change. Production LOC: core 97271 -> 97281 (delta +10; store-service +10), sdk 3457 -> 3455 (delta -2; the manual hold's comment moves into the service). Counter: tools/production_loc.py (pinned at this tree). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../src/store_service.rs | 24 ++++++++++++++++--- sdk/js/native/src/direct.rs | 6 ++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/core/crates/layerfs-store-service/src/store_service.rs b/core/crates/layerfs-store-service/src/store_service.rs index 791755cb8..11e41646a 100644 --- a/core/crates/layerfs-store-service/src/store_service.rs +++ b/core/crates/layerfs-store-service/src/store_service.rs @@ -55,11 +55,18 @@ pub struct StoreService { capacity: usize, bound: AtomicUsize, retained: Mutex>, + /// The Store file whose writer lane guards this service's close. + store_file: std::path::PathBuf, } impl StoreService { /// Serves Store calls over `opened`, constructing through `owner`. pub fn new(opened: OpenedStore, owner: &Owner) -> Self { + Self::at(opened, owner, std::path::PathBuf::new()) + } + /// [`new`](Self::new) with the Store file recorded, for a close that + /// coordinates its sessions' teardown under the writer lane. + pub fn at(opened: OpenedStore, owner: &Owner, store_file: std::path::PathBuf) -> Self { Self { store: opened.store.clone(), opened, @@ -68,6 +75,7 @@ impl StoreService { capacity: owner.configuration().namespaces, bound: AtomicUsize::new(0), retained: Mutex::new(Vec::new()), + store_file, } } @@ -109,7 +117,7 @@ impl StoreService { OwnerConfig { namespaces: CALLS }, ) .map_err(|error| OpenRefusal::Failed(format!("overlay template: {error}")))?; - let mut service = Self::new(opened, &owner); + let mut service = Self::at(opened, &owner, store.to_path_buf()); service.owned = Some(owner); Ok(service) } @@ -117,10 +125,20 @@ impl StoreService { /// Stops the overlay owner this service started. Private Workspaces kept /// with an uncertain Commit stay with it until then. pub fn close(mut self) -> Result<(), String> { - match self.owned.take() { + // The overlays stop first: their final Commits take the writer lane + // themselves, so no outer hold may surround this. + let stopped = match self.owned.take() { Some(owner) => owner.stop().map_err(|error| error.to_string()), None => Ok(()), - } + }; + // The sessions then close under the Store's writer lane, so their + // close-time checkpoints never run beside another opener's first + // attach. A failed lane acquisition closes bare, as a drop does. + let lane = (!self.store_file.as_os_str().is_empty()) + .then(|| layerfs_persistence::store_lane(&self.store_file)); + drop(self); + drop(lane); + stopped } /// One private Workspace on `branch` for one file-changing call. diff --git a/sdk/js/native/src/direct.rs b/sdk/js/native/src/direct.rs index e0f4a043e..4415e8453 100644 --- a/sdk/js/native/src/direct.rs +++ b/sdk/js/native/src/direct.rs @@ -109,14 +109,12 @@ pub(crate) fn release(args: &SdkValue) -> Outcome { } let opened = stores.remove(&store).expect("present above"); drop(stores); - // The service's sessions close under the Store's writer lane: their - // close-time checkpoints never run beside another opener's first attach. - let lane = layerfs_persistence::store_lane(&store); + // The service's close coordinates its sessions' teardown under the + // Store's writer lane itself. let closed = match Arc::try_unwrap(opened.service) { Ok(service) => service.close().map_err(failed), Err(_) => Ok(()), }; - drop(lane); let removed = std::fs::remove_dir_all(&opened.directory).map_err(failed); closed.and(removed)?; Ok(SdkValue::map([("clients", SdkValue::Int(0))])) From 636a641bfce9a90c4f892b3b542c646317d52fa7 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 10:04:02 +0800 Subject: [PATCH 11/19] fix(windows): versioned-stable lane and discovery mutex names (K3 F1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit K3 review (猿光智启专家团2.0——layerfs底座, APPROVE-WITH-NITS, F1): the lane and discovery mutex names hashed the directory with DefaultHasher, whose algorithm may change between Rust releases - an older and a newer binary computing different names for one directory would silently run two lanes. The names now derive from SHA-256 of the resolved directory's encoded bytes, truncated to 64 bits (sha2 is already in the locked graph; the direct edges add no new package), in persistence's windows_lane (exposed as lane_name for the test) and the SDK binding's discovery guard. The dead-lane test replicates the new derivation. F2 (Local\ namespace does not cross sessions; the product opens its Store within one login session today) is recorded as a §6 limitation in DIRECT_STORE.md, with the Global\ privilege evaluation noted should cross-session coordination ever be needed. Verification: Windows persistence 87/0 (both lane binaries), project 10/0, store-service 3/0, overlay 74/0; SDK matrix/concurrent/lifecycle pass on the rebuilt addon; Linux container cargo check --offline --locked for persistence and the node binding, Finished. Production LOC: core 97281 -> 97282 (delta +1), sdk 3455 -> 3453 (delta -2); combined 109919 -> 109918. Counter: tools/production_loc.py (pinned at this tree). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/Cargo.lock | 2 ++ core/crates/layerfs-persistence/Cargo.toml | 2 ++ .../src/backend/sqlite/windows_lane.rs | 22 ++++++++++++------- .../tests/windows_lane_abandoned.rs | 17 +++++++------- core/docs/issues/352/DIRECT_STORE.md | 7 +++++- sdk/js/native/Cargo.toml | 2 ++ sdk/js/native/src/discovery_windows.rs | 18 +++++++-------- 7 files changed, 43 insertions(+), 27 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 1a06122f7..8298143f8 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -633,6 +633,7 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "sha2 0.11.0", "windows-sys", ] @@ -657,6 +658,7 @@ dependencies = [ "layerfs-telemetry", "nix", "rusqlite", + "sha2 0.11.0", "windows-sys", ] diff --git a/core/crates/layerfs-persistence/Cargo.toml b/core/crates/layerfs-persistence/Cargo.toml index adbf3e37d..7a526c911 100644 --- a/core/crates/layerfs-persistence/Cargo.toml +++ b/core/crates/layerfs-persistence/Cargo.toml @@ -12,6 +12,8 @@ layerfs-history = { path = "../layerfs-history", default-features = false } layerfs-content = { path = "../layerfs-content" } layerfs-telemetry = { path = "../layerfs-telemetry" } rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } +# The lane name digest; the same locked sha2 layerfs-storage already uses. +sha2 = { version = "=0.11.0", default-features = false } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } diff --git a/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs b/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs index f16b365bf..5960bf6ed 100644 --- a/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs +++ b/core/crates/layerfs-persistence/src/backend/sqlite/windows_lane.rs @@ -11,8 +11,7 @@ //! plain data. The volume probe reports the filesystem of the directory's //! local volume for the kernel guard's allowlist. #![allow(unsafe_code)] -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; +use sha2::{Digest, Sha256}; use std::io; use std::path::Path; use windows_sys::Win32::Foundation::{CloseHandle, WAIT_ABANDONED, WAIT_OBJECT_0}; @@ -25,18 +24,25 @@ pub(crate) struct LaneMutex { handle: usize, } +/// The lane's kernel object name: SHA-256 of the resolved directory's +/// encoded bytes, truncated to 64 bits. A versioned-stable digest, not +/// `DefaultHasher`, whose algorithm may change between Rust releases and +/// silently split one directory onto two lanes across builds (K3 review F1). +pub(crate) fn lane_name(directory: &Path) -> String { + let resolved = std::fs::canonicalize(directory).unwrap_or_else(|_| directory.to_path_buf()); + let digest = Sha256::digest(resolved.as_os_str().as_encoded_bytes()); + let mut tag = [0u8; 8]; + tag.copy_from_slice(&digest[..8]); + format!("Local\\layerfs-writer-lane-{:016x}", u64::from_be_bytes(tag)) +} + impl LaneMutex { /// One blocking kernel wait on the directory's named mutex, taken after /// the Session's FIFO turn, so at most one thread per Session waits here. /// Creation only opens the shared kernel object. Exits: the lane (a /// holder process that exits releases it); a failed wait. pub(crate) fn acquire(directory: &Path) -> Result { - let mut hasher = DefaultHasher::new(); - match std::fs::canonicalize(directory) { - Ok(canonical) => canonical.hash(&mut hasher), - Err(_) => directory.hash(&mut hasher), - } - let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", hasher.finish()) + let name: Vec = lane_name(directory) .encode_utf16() .chain(std::iter::once(0)) .collect(); diff --git a/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs b/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs index ebd3e9855..416b74138 100644 --- a/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs +++ b/core/crates/layerfs-persistence/tests/windows_lane_abandoned.rs @@ -34,8 +34,7 @@ fn authority(name: &[u8]) -> HistoryCatalogConfig { /// `WAIT_ABANDONED` acquisition. #[test] fn a_dead_lane_holder_does_not_block_the_next_writer() { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; + use sha2::{Digest, Sha256}; let temp = support::Temp::new("windows-store-abandoned"); let path = temp.join("store.sqlite"); let catalog = authority(b"windows-store-abandoned"); @@ -45,13 +44,13 @@ fn a_dead_lane_holder_does_not_block_the_next_writer() { let directory = temp.path().to_path_buf(); let holder = std::thread::spawn(move || { - // The product's own lane name, derived exactly as windows_lane does. - let mut hasher = DefaultHasher::new(); - match std::fs::canonicalize(&directory) { - Ok(canonical) => canonical.hash(&mut hasher), - Err(_) => directory.hash(&mut hasher), - } - let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", hasher.finish()) + // The product's own lane name: SHA-256 of the resolved directory's + // encoded bytes, truncated to 64 bits (windows_lane::lane_name). + let resolved = std::fs::canonicalize(&directory).unwrap_or_else(|_| directory.clone()); + let digest = Sha256::digest(resolved.as_os_str().as_encoded_bytes()); + let mut tag = [0u8; 8]; + tag.copy_from_slice(&digest[..8]); + let name: Vec = format!("Local\\layerfs-writer-lane-{:016x}", u64::from_be_bytes(tag)) .encode_utf16() .chain(std::iter::once(0)) .collect(); diff --git a/core/docs/issues/352/DIRECT_STORE.md b/core/docs/issues/352/DIRECT_STORE.md index 4a7d0aeb6..1666e3697 100644 --- a/core/docs/issues/352/DIRECT_STORE.md +++ b/core/docs/issues/352/DIRECT_STORE.md @@ -113,7 +113,12 @@ no daemon are involved. Implemented on branch `windows-native-store`: on Windows: FUSE, the daemon and Workspaces remain Linux-only, and the dsh macOS and WSL2 exceptions are unchanged. -Recorded limitations of this route: Windows Init carries non-UTF-8 names +Recorded limitations of this route: the Windows lane and discovery +mutexes live in the `Local\` namespace, so a session-0 service and a user +session would each hold its own lane (Linux's directory flock crosses +sessions); the product opens its Store within one login session today, and +cross-session coordination would need the `Global\` namespace with a +privilege evaluation (K3 review F2); Windows Init carries non-UTF-8 names lossily, synthesizes the mode from the read-only attribute, has no hard-link membership and no backing-inside-source alias check; overlay allocation is the logical length; last-owner overlay-file deletion keeps its Linux identity diff --git a/sdk/js/native/Cargo.toml b/sdk/js/native/Cargo.toml index 2179bfac6..33f86fb6a 100644 --- a/sdk/js/native/Cargo.toml +++ b/sdk/js/native/Cargo.toml @@ -28,5 +28,7 @@ napi-derive = "=2.16.13" napi-build = "=2.1.6" [target.'cfg(target_os = "windows")'.dependencies] +# The discovery guard's mutex-name digest (K3 F1), the locked sha2. +sha2 = { version = "=0.11.0", default-features = false } # The Windows discovery guard's named kernel mutex; already in the locked graph. windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_System_Threading"] } diff --git a/sdk/js/native/src/discovery_windows.rs b/sdk/js/native/src/discovery_windows.rs index 587f5bdf4..c66cf53c5 100644 --- a/sdk/js/native/src/discovery_windows.rs +++ b/sdk/js/native/src/discovery_windows.rs @@ -103,16 +103,16 @@ impl Drop for Guard { } /// One mutex name per directory; the canonical path identifies it between -/// processes. +/// processes. SHA-256 truncated, not `DefaultHasher`: the hasher's algorithm +/// may change between Rust releases and silently split one directory onto +/// two guards across builds (K3 review F1, with the persistence lane name). fn mutex_name(directory: &Path) -> String { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut hasher = DefaultHasher::new(); - match fs::canonicalize(directory) { - Ok(canonical) => canonical.hash(&mut hasher), - Err(_) => directory.hash(&mut hasher), - } - format!("Local\\layerfs-discovery-{:016x}", hasher.finish()) + use sha2::{Digest, Sha256}; + let resolved = fs::canonicalize(directory).unwrap_or_else(|_| directory.to_path_buf()); + let digest = Sha256::digest(resolved.as_os_str().as_encoded_bytes()); + let mut tag = [0u8; 8]; + tag.copy_from_slice(&digest[..8]); + format!("Local\\layerfs-discovery-{:016x}", u64::from_be_bytes(tag)) } pub(crate) fn guard(directory: &Path) -> Result { From 9ca6b3ce4644b4d05a11c06e99410b0e3c929012 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Mon, 26 Oct 2026 23:36:32 +0800 Subject: [PATCH 12/19] docs(architecture): the lane's open/teardown holds and stable naming in doc 83 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The concurrency model's table gains the open/teardown row: the connect-time statement block (where a cleaned WAL -shm is rebuilt under an exclusive recovery hold), the writable setup block (held publication lane; nested turns take no second lock) and the coordinated close (StoreLane/Handles::shutdown; the service stops overlays first, then drops the sessions under the lane) all run under the same cross-process lane - SQLite's close-time checkpoint is writer work like any other and never runs beside another opener's first attach. The lane-name note records the SHA-256 truncation (never DefaultHasher, K3 F1) and the Local\ session-namespace limitation (DIRECT_STORE.md §6, K3 F2). Also: the PR#452 observation script switches from the gh CLI to the unauthenticated REST API so the job engine's non-interactive environment (no gh auth) can run it. Production LOC: core 97282 -> 97282 (delta 0), sdk 3453 -> 3453 (delta 0); combined 109918 -> 109918. Docs and tooling only; the shell script lives outside the counted scopes. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/docs/architecture/83-store-concurrency-model.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/docs/architecture/83-store-concurrency-model.md b/core/docs/architecture/83-store-concurrency-model.md index f67dd0bbc..6693ae7a4 100644 --- a/core/docs/architecture/83-store-concurrency-model.md +++ b/core/docs/architecture/83-store-concurrency-model.md @@ -34,8 +34,16 @@ | Directory lock (Linux) / named kernel mutex (Windows) | all writer processes of one Store | Writers in different processes (each Sandbox's daemon, the SDK process) take turns instead of meeting `SQLITE_BUSY`; on Windows the mutex is named per Store directory and an abandoned mutex is taken by the next waiter | | Writer admission queue | threads of one process | Threads take turns on the process's one writer connection, first come first served, before touching SQLite | | Publication lane | one Commit's publish | Head check, merge and publish are one atomic step, so two Commits never both publish on the same old head | +| Open/teardown holds | one Store's connect, writable setup and close | The connect-time statement block (profile pragmas, format reads — where a cleaned WAL `-shm` is rebuilt under an exclusive recovery hold), the writable setup block (held publication lane, nested turns take no second lock) and a coordinated close (`StoreLane`/`Handles::shutdown`; the service's `close` stops overlays first, then drops the sessions under the lane) run under the same cross-process lane: SQLite's close-time checkpoint is writer work like any other and must never run beside another opener's first attach (no-retry rule) | | Workspace lock | one Workspace's overlay database | Separate concern: guards that Workspace's own overlay connection (root AGENTS.md) | +The lane's Windows name is SHA-256 of the resolved directory's encoded +bytes truncated to 64 bits — a version-stable digest, never +`DefaultHasher`, whose algorithm may change between Rust releases and +silently split one directory onto two lanes across builds (K3 review F1). +The mutexes live in the `Local\` namespace: one login session's lanes are +invisible to another (DIRECT_STORE.md §6). + The queue and the directory lock replace failure, not add serialization: the repository forbids automatic retries and busy handlers (one attempt per operation, exact failure), so a writer waits its turn before its single SQL From 3f68b368e60a331b784605d17cbd25a1bccbdfe2 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Tue, 27 Oct 2026 00:42:27 +0800 Subject: [PATCH 13/19] docs(readme): align the user-facing Store-route platform wording Four maintained-doc spots still said the Store face is Linux-only: - sdk/js/README.md, the Store-calls paragraph: on Linux the Store sits on the daemons' shared local filesystem; on Windows the native route opens a local NTFS volume directly, no daemon involved (Workspace, exec and daemon calls refuse Unsupported there). - the LAYERFS_STORE_PATH row: the same split (Linux shared local filesystem / Windows local NTFS volume). - the Builds section: the build script refuses Windows by design; the Windows addon builds directly with cargo and renames layerfs_node.dll into native/layerfs.node. - cluster_one_handbook's persistence row: macOS/Linux/Windows, with the owner-direction date. The versioned 0.1.x manuals are untouched (immutable per release); benchmark_instruction's Linux sentence is about test hosting, not this route. Link check passes for both files at their real base paths. Production LOC: core 97282 -> 97282 (delta 0), sdk 3453 -> 3453 (delta 0); combined 109918 -> 109918. Docs only. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- docs/general/cluster_one_handbook.md | 2 +- sdk/js/README.md | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/general/cluster_one_handbook.md b/docs/general/cluster_one_handbook.md index 36f2a569a..d8600a12e 100644 --- a/docs/general/cluster_one_handbook.md +++ b/docs/general/cluster_one_handbook.md @@ -57,7 +57,7 @@ The active Core workspace contains six crates: | `layerfs-content` | Canonical identity, file construction/editing, filesystem trees, logical reads | Turn bytes/edits/namespace rows into canonical objects; read logical data | | `layerfs-storage` | Reuse, physical encoding, delta selection, packing, bounded Save and Reader | Store canonical objects and acquire authenticated canonical bytes | | `layerfs-history` | Typed history records and `HistoryCatalog` semantic operations | Inode reservation, genesis, branches, Commits, Layers | -| `layerfs-persistence` | Shared SQLite provider for storage and history on macOS/Linux | Create/open the Store and execute provider-owned transactions | +| `layerfs-persistence` | Shared SQLite provider for storage and history on macOS/Linux/Windows (Windows: local NTFS volumes; owner direction 2026-10-25) | Create/open the Store and execute provider-owned transactions | | `layerfs-project` | Native namespace Init orchestration | Import a regular directory and publish its genesis LayerStack | | `layerfs-telemetry` | Explicit timing scopes and reports | Observe the same production operations with timing enabled/disabled | diff --git a/sdk/js/README.md b/sdk/js/README.md index 81afe1c47..71e62ec27 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -41,9 +41,11 @@ Store calls (`fs`, `branch`, `project.list_branches`) run in this process: the first one opens the installed Store file at `storePath` (else `LAYERFS_STORE_PATH`) and `close()` releases it; no Sandbox is involved. That file is `global/store.sqlite` in the project's volume, so the process runs where -that volume is mounted (`/layerfs-store` in a Sandbox). This is Linux only, with -the Store on a local filesystem the daemons share (ext4, xfs, btrfs); anywhere -else Store calls fail `Unsupported`, and without a `storePath` they fail +that volume is mounted (`/layerfs-store` in a Sandbox). On Linux the Store sits +on a local filesystem the daemons share (ext4, xfs, btrfs); on Windows the +native Store route opens a local NTFS volume directly, no daemon involved +(Workspace, exec and daemon calls refuse `Unsupported` there). Anywhere else +Store calls fail `Unsupported`, and without a `storePath` they fail `InvalidInput`. Mount does not scan or materialize the complete tree. A project is its home entry, `LAYERFS_HOME//`; the binding keeps no other record of which names it initialized. Init refuses a name whose entry exists @@ -72,7 +74,7 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_VOLUME` | Existing named local VM volume shared by the project's daemons. | | `LAYERFS_DAEMON` | Host path to the built Linux daemon executable. | | `LAYERFS_NATIVE` | Optional addon path; defaults to `native/layerfs.node`. | -| `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open in this process; the client option `storePath` wins. It is `global/store.sqlite` in the project's volume, `/layerfs-store/global/store.sqlite` where that volume is mounted at `/layerfs-store`. Linux only, on a local filesystem the daemons share; elsewhere Store calls fail `Unsupported`. Unset, they fail `InvalidInput`. | +| `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open in this process; the client option `storePath` wins. It is `global/store.sqlite` in the project's volume, `/layerfs-store/global/store.sqlite` where that volume is mounted at `/layerfs-store`. On Linux, a local filesystem the daemons share (ext4, xfs, btrfs); on Windows, a local NTFS volume; elsewhere Store calls fail `Unsupported`. Unset, they fail `InvalidInput`. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional; the client option `daemonHost` wins. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | | `LAYERFS_TRACE` | Optional per-call trace file (see [Call trace](#call-trace)); unset means no trace. | @@ -382,7 +384,10 @@ names set. Run `npm --prefix sdk/js run build:native` on the macOS/Linux host that loads the addon. It uses Cargo +1.85.1, `--offline --locked --release`, and the repository -ARM64 inputs. `-- --debug` selects dev. The artifact is atomically renamed into +ARM64 inputs. `-- --debug` selects dev. On Windows the script refuses to run; +build the addon directly with `cargo build --manifest-path core/Cargo.toml -p +layerfs-node` (locked, `CARGO_TARGET_DIR` inside this worktree) and rename +`target//layerfs_node.dll` to `native/layerfs.node`. The artifact is atomically renamed into place, avoiding stale macOS executable-page signature state. `CARGO_TARGET_DIR` must stay inside this worktree; `LAYERFS_NATIVE_OUTPUT` can select a separate platform artifact. Keep Linux addons separate from the host addon. From 2a4cf6645c53949baf4d12fc257ae7a8e0c56e7b Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Wed, 28 Oct 2026 01:24:41 +0800 Subject: [PATCH 14/19] fix(store-service): Windows overlay last-owner deletion through the audited identity Backing::delete refused Unsupported off unix, so the direct route's private overlay files on Windows were never reclaimed by their last owner. The Windows arm now applies the same discipline through the audited identity: the entry at the path and the retained open handle must both still be the acquisition's file (volume serial + file index, sole link) before std::fs::remove_file; the retained handle - opened with std's FILE_SHARE_DELETE - completes the removal when Drop closes it, exactly the Linux order (check, remove, close). A swapped-in file at the path is refused and stays. windows_path_identity is exported from layerfs-overlay beside windows_handle_identity; the device/inode fields' dead_code allowance widens to both reading platforms. Tests (new, overlay_backing_windows.rs, dangling-Weak pool so Drop skips the callback): a matching overlay file is deleted at last-owner release (bounded wait for the pending removal to complete on handle close), and a file swapped in behind the backing's back is refused and stays. Verification: Windows - the new tests 2/2, store-service 5/0 (was 3), persistence 87/0, project 10/0, overlay 74/0; SDK matrix/lifecycle pass on the rebuilt addon. Linux container - cargo check --locked for store-service and overlay, Finished. Production LOC: core 97282 -> 97316 (delta +34; store-service +31, overlay +3), sdk 3453 -> 3453 (delta 0); combined 109918 -> 109952. Counter: tools/production_loc.py (pinned at this tree). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../src/database/windows_identity.rs | 2 +- core/crates/layerfs-overlay/src/lib.rs | 2 +- .../src/overlay/backing.rs | 62 ++++++++++++--- .../tests/overlay_backing_windows.rs | 79 +++++++++++++++++++ 4 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 core/crates/layerfs-store-service/tests/overlay_backing_windows.rs diff --git a/core/crates/layerfs-overlay/src/database/windows_identity.rs b/core/crates/layerfs-overlay/src/database/windows_identity.rs index 87fe4ba32..f612e3abd 100644 --- a/core/crates/layerfs-overlay/src/database/windows_identity.rs +++ b/core/crates/layerfs-overlay/src/database/windows_identity.rs @@ -31,7 +31,7 @@ pub fn windows_handle_identity(file: &File) -> io::Result { /// The identity of `path`'s entry, read as attributes only: no data access, /// no link traversal; directories need the backup-semantics flag. -pub(crate) fn windows_path_identity(path: &Path) -> io::Result { +pub fn windows_path_identity(path: &Path) -> io::Result { use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; use windows_sys::Win32::Storage::FileSystem::{ CreateFileW, GetFileInformationByHandle, FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_DELETE, diff --git a/core/crates/layerfs-overlay/src/lib.rs b/core/crates/layerfs-overlay/src/lib.rs index f4be6e390..b0a1e04c4 100644 --- a/core/crates/layerfs-overlay/src/lib.rs +++ b/core/crates/layerfs-overlay/src/lib.rs @@ -47,7 +47,7 @@ pub use database::accounting::{Resources, StoredCounts}; pub use database::physical::{AllocationState, AllocationWork, ClosedFile}; pub use database::startup::{Creation, CreationWork}; #[cfg(target_os = "windows")] -pub use database::windows_identity::windows_handle_identity; +pub use database::windows_identity::{windows_handle_identity, windows_path_identity}; pub use db::Overlay; pub use diagnostics::payload::PayloadWork; pub use error::{CloseFailure, OverlayError, OverlayResult}; diff --git a/core/crates/layerfs-store-service/src/overlay/backing.rs b/core/crates/layerfs-store-service/src/overlay/backing.rs index 738be4aac..faadff3df 100644 --- a/core/crates/layerfs-store-service/src/overlay/backing.rs +++ b/core/crates/layerfs-store-service/src/overlay/backing.rs @@ -14,11 +14,11 @@ pub struct Backing { file: Option, namespace: i64, incarnation: [u8; 32], - // The deletion identity check reads these on unix only; Windows and the - // other platforms refuse deletion before reaching them. - #[cfg_attr(not(unix), allow(dead_code))] + // The deletion identity check reads these on unix and Windows; the + // remaining platforms refuse deletion before reaching them. + #[cfg_attr(not(any(unix, target_os = "windows")), allow(dead_code))] device: u64, - #[cfg_attr(not(unix), allow(dead_code))] + #[cfg_attr(not(any(unix, target_os = "windows")), allow(dead_code))] inode: u64, pool: Weak, terminal: AtomicBool, @@ -105,14 +105,54 @@ impl Backing { "overlay deletion identity", )); } + std::fs::remove_file(&self.path) + } + #[cfg(target_os = "windows")] + { + // The same discipline through the audited Windows identity: the + // entry at the path and the retained open handle must both still + // be the acquisition's file (volume serial + file index, sole + // link) before the removal. The retained handle was opened with + // FILE_SHARE_DELETE (std's share mode), so the removal takes + // effect when Drop closes that last handle, immediately after. + let path_identity = layerfs_overlay::windows_path_identity(&self.path) + .map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("overlay deletion identity: {error}"), + ) + })?; + let retained = layerfs_overlay::windows_handle_identity( + self.file.as_ref().expect("retained identity file"), + ) + .map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("overlay deletion identity: {error}"), + ) + })?; + if path_identity.device != self.device + || path_identity.index != self.inode + || retained.device != self.device + || retained.index != self.inode + || path_identity.links != 1 + || retained.links != 1 + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "overlay deletion identity", + )); + } + std::fs::remove_file(&self.path) + } + #[cfg(not(any(unix, target_os = "windows")))] + { + let _ = &self.path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "overlay deletion platform", + )) } - #[cfg(not(unix))] - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "overlay deletion platform", - )); - #[cfg(unix)] - std::fs::remove_file(&self.path) } } impl Drop for Backing { diff --git a/core/crates/layerfs-store-service/tests/overlay_backing_windows.rs b/core/crates/layerfs-store-service/tests/overlay_backing_windows.rs new file mode 100644 index 000000000..1e8318e6d --- /dev/null +++ b/core/crates/layerfs-store-service/tests/overlay_backing_windows.rs @@ -0,0 +1,79 @@ +//! The Windows overlay backing's last-owner deletion: the same identity +//! discipline as Linux, through the audited Windows identity. A dangling +//! `Weak` stands in for the workspace pool — `Drop` skips its callback when +//! the pool is gone, which is exactly the unit under test. +#![cfg(target_os = "windows")] +use std::sync::{Arc, Weak}; + +fn scratch(label: &str) -> std::path::PathBuf { + let directory = std::env::temp_dir().join(format!( + "layerfs-backing-{}-{}-{}", + label, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|span| span.as_nanos()) + .unwrap_or(0) + )); + std::fs::create_dir_all(&directory).unwrap(); + directory +} + +/// The file goes away when the last owner releases: identity matches, the +/// removal is authorized, and closing the retained handle completes it. +#[test] +fn the_last_owner_deletes_a_matching_overlay_file() { + let directory = scratch("delete"); + let path = directory.join("overlay.sqlite"); + std::fs::write(&path, b"overlay body").unwrap(); + let backing = Arc::new( + layerfs_store_service::overlay::backing::Backing::new( + path.clone(), + 1, + [7; 32], + Weak::::new(), + ) + .unwrap(), + ); + assert!(path.exists(), "the file stays while its owner lives"); + backing.terminal(); + backing.closed(); + drop(backing); + let mut gone = false; + for _ in 0..40 { + if !path.exists() { + gone = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!(gone, "the last owner's release deletes the file"); + std::fs::remove_dir_all(directory).unwrap(); +} + +/// A file swapped in behind the backing's back no longer matches the +/// retained identity: the removal is refused and the impostor stays. +#[test] +fn a_swapped_in_file_is_refused_the_deletion() { + let directory = scratch("swap"); + let path = directory.join("overlay.sqlite"); + std::fs::write(&path, b"overlay body").unwrap(); + let backing = Arc::new( + layerfs_store_service::overlay::backing::Backing::new( + path.clone(), + 1, + [7; 32], + Weak::::new(), + ) + .unwrap(), + ); + // Replace the entry at the path with a different file (different volume + // index), so the path no longer names the acquisition's file. + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"impostor").unwrap(); + backing.terminal(); + backing.closed(); + drop(backing); + assert!(path.exists(), "the impostor is not deleted"); + std::fs::remove_dir_all(directory).unwrap(); +} From 64035fed013af8aef03354444a24c1916f85e028 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Wed, 28 Oct 2026 01:26:51 +0800 Subject: [PATCH 15/19] docs(direct-store): the deletion-parity line replaces the stale limitation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §6 said last-owner overlay deletion keeps its Linux identity check and is not exercised on Windows; commit 2a4cf6645 lifted exactly that. The paragraph now states the parity (audited identity on both sides of the comparison) and rewords the aliasing limitation to what 3b actually implements (resolved-containment; subst/unresolvable aliases stay unrecognized). Production LOC: core 97316 -> 97316 (delta 0), sdk 3453 -> 3453 (delta 0). Docs only. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/docs/issues/352/DIRECT_STORE.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/docs/issues/352/DIRECT_STORE.md b/core/docs/issues/352/DIRECT_STORE.md index 1666e3697..1df7a8fd9 100644 --- a/core/docs/issues/352/DIRECT_STORE.md +++ b/core/docs/issues/352/DIRECT_STORE.md @@ -120,9 +120,12 @@ sessions); the product opens its Store within one login session today, and cross-session coordination would need the `Global\` namespace with a privilege evaluation (K3 review F2); Windows Init carries non-UTF-8 names lossily, synthesizes the mode from the read-only attribute, has no hard-link -membership and no backing-inside-source alias check; overlay allocation is the -logical length; last-owner overlay-file deletion keeps its Linux identity -check (not exercised here — the Store route holds no Workspace mounts). +membership, and its backing-inside-source check resolves the placement's +spelling (a subst drive or unresolvable alias is not recognised); overlay +allocation is the logical length. Last-owner overlay-file deletion applies +the same audited identity on Windows as on Linux (volume serial + file +index, sole link, checked on both the path's entry and the retained +handle). Proof receipts: the cross-process lane verifier (`layerfs-persistence` `tests/windows_store_native.rs`, two re-exec'd child From 3d3272c81f68b0209e9940d2edb17fbac3fde3c2 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Wed, 28 Oct 2026 01:47:19 +0800 Subject: [PATCH 16/19] feat(overlay): Windows allocation reads the real on-disk footprint The Windows overlay reported its logical length as allocated bytes (compression/sparse never visible; the receipt pair: a written overlay whose logical length is 77,824 holds 131,072 on disk - the stand-in understated by 40%). The audited Windows identity module gains windows_allocation_size (GetFileInformationByHandleEx / FileStandardInfo on the retained handle, compression/sparse aware) and Allocation::new/state read it: high_water seeds from the real footprint at open, and state reports it beside the logical length, parity with Linux's blocks*512. Test (new allocation_windows.rs): a written overlay's allocated_bytes is positive, at or above its logical length within a cluster margin, observed pair printed for the receipt. Verification: Windows - overlay 75/0 (74 + the new test), persistence 87/0, project 10/0, store-service 5/0. Linux container - cargo check --offline --locked for overlay and store-service, Finished. Production LOC: core 97316 -> 97336 (delta +20; overlay +20), sdk 3453 -> 3453 (delta 0); combined 109952 -> 109972. Counter: tools/production_loc.py (pinned at this tree). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../layerfs-overlay/src/database/physical.rs | 11 +++--- .../src/database/windows_identity.rs | 21 +++++++++++ core/crates/layerfs-overlay/src/lib.rs | 4 ++- .../tests/allocation_windows.rs | 36 +++++++++++++++++++ 4 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 core/crates/layerfs-overlay/tests/allocation_windows.rs diff --git a/core/crates/layerfs-overlay/src/database/physical.rs b/core/crates/layerfs-overlay/src/database/physical.rs index 0d1601397..b5c9b2314 100644 --- a/core/crates/layerfs-overlay/src/database/physical.rs +++ b/core/crates/layerfs-overlay/src/database/physical.rs @@ -100,14 +100,14 @@ impl Allocation { if identity.links != 1 { return Err(OverlayError::Invalid("allocation file identity")); } - let len = file.metadata()?.len(); + let allocated = super::windows_identity::windows_allocation_size(&file)?; Ok(Self { file, device: identity.device, inode: identity.index, path: path.to_owned(), work: Cell::new(AllocationWork::default()), - high_water: Cell::new(len), + high_water: Cell::new(allocated), }) } #[cfg(target_os = "windows")] @@ -126,9 +126,10 @@ impl Allocation { { return Err(OverlayError::Invalid("allocation file identity")); } - // Windows exposes no allocated-blocks count; the logical length - // stands in for it (the disposable overlay never preallocates). - let allocated = meta.len(); + // The allocated footprint comes from FileStandardInfo on the same + // retained handle (compression/sparse aware); the disposable overlay + // never preallocates, so this tracks the tail's actual clusters. + let allocated = super::windows_identity::windows_allocation_size(&self.file)?; self.high_water.set(self.high_water.get().max(allocated)); Ok(AllocationState { logical_bytes: meta.len(), diff --git a/core/crates/layerfs-overlay/src/database/windows_identity.rs b/core/crates/layerfs-overlay/src/database/windows_identity.rs index f612e3abd..f037d4f19 100644 --- a/core/crates/layerfs-overlay/src/database/windows_identity.rs +++ b/core/crates/layerfs-overlay/src/database/windows_identity.rs @@ -31,6 +31,27 @@ pub fn windows_handle_identity(file: &File) -> io::Result { /// The identity of `path`'s entry, read as attributes only: no data access, /// no link traversal; directories need the backup-semantics flag. +/// The entry's on-disk allocation in bytes (compression/sparse aware), +/// read through FileStandardInfo on the open handle. +pub fn windows_allocation_size(file: &File) -> io::Result { + use windows_sys::Win32::Storage::FileSystem::{ + GetFileInformationByHandleEx, FILE_STANDARD_INFO, FileStandardInfo, + }; + let mut info = FILE_STANDARD_INFO::default(); + let ok = unsafe { + GetFileInformationByHandleEx( + file.as_raw_handle(), + FileStandardInfo, + &mut info as *mut FILE_STANDARD_INFO as *mut core::ffi::c_void, + std::mem::size_of::() as u32, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + Ok(info.AllocationSize.max(0) as u64) +} + pub fn windows_path_identity(path: &Path) -> io::Result { use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; use windows_sys::Win32::Storage::FileSystem::{ diff --git a/core/crates/layerfs-overlay/src/lib.rs b/core/crates/layerfs-overlay/src/lib.rs index b0a1e04c4..f6c7dad45 100644 --- a/core/crates/layerfs-overlay/src/lib.rs +++ b/core/crates/layerfs-overlay/src/lib.rs @@ -47,7 +47,9 @@ pub use database::accounting::{Resources, StoredCounts}; pub use database::physical::{AllocationState, AllocationWork, ClosedFile}; pub use database::startup::{Creation, CreationWork}; #[cfg(target_os = "windows")] -pub use database::windows_identity::{windows_handle_identity, windows_path_identity}; +pub use database::windows_identity::{ + windows_allocation_size, windows_handle_identity, windows_path_identity, +}; pub use db::Overlay; pub use diagnostics::payload::PayloadWork; pub use error::{CloseFailure, OverlayError, OverlayResult}; diff --git a/core/crates/layerfs-overlay/tests/allocation_windows.rs b/core/crates/layerfs-overlay/tests/allocation_windows.rs new file mode 100644 index 000000000..5fdd57961 --- /dev/null +++ b/core/crates/layerfs-overlay/tests/allocation_windows.rs @@ -0,0 +1,36 @@ +//! The Windows overlay's allocated-byte accounting reads FileStandardInfo on +//! the retained handle: the footprint of the written tail, not the logical +//! length the old stand-in reported. +#![cfg(target_os = "windows")] +mod payload_support; +use layerfs_overlay::ProfileConfig; +use payload_support::*; + +#[test] +fn allocated_bytes_report_the_ondisk_footprint_of_the_written_tail() { + let temp = Temp::new(); + let db = temp.db(); + db.open_workspace([11; 32]).unwrap(); + let mut source = db.acquire_base_source(1).unwrap(); + // 300,001 bytes: not a cluster multiple, so the on-disk footprint is + // strictly above the logical length on a cluster-addressed volume — + // which the old logical-length stand-in could never report. + let mut file = File::new(&db, source, 3, pattern(300_001, 5)); + let _ = file.read(0, 16); + let state = db.allocation().unwrap(); + assert!(state.allocated_bytes > 0, "a written overlay holds clusters"); + assert!( + state.allocated_bytes >= state.logical_bytes, + "cluster rounding keeps the footprint at or above the tail" + ); + assert!( + state.allocated_bytes.saturating_sub(state.logical_bytes) < 1 << 20, + "the footprint tracks the tail within a cluster margin, got {} for {}", + state.allocated_bytes, + state.logical_bytes + ); + println!( + "allocated={} logical={}", + state.allocated_bytes, state.logical_bytes + ); +} From e39f4ac80d9017f733d9b465cd20584d6dcf4a04 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Wed, 28 Oct 2026 02:10:23 +0800 Subject: [PATCH 17/19] feat(project): the alias check walks identities, Linux's own algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Windows backing-inside-source check compared canonicalized DOS paths — a spelling whose DOS volume name survives resolution (a subst drive: GetFinalPathNameByHandle keeps the alias letter; live probe: realpath(Y:/inner) = Y:/inner) slipped past the containment. The check now runs Linux's algorithm on the audited Windows identity: canonicalize once, then compare every placement ancestor's volume serial + file index with the source's — junctions, subst drives and any spelling that resolves to the source's directories carry their device and index, and are refused. Test (init_sqlite, windows): a directory junction naming the source is refused BackingInsideSource before any operation. Verification: Windows project suite 11/0 (was 10); Linux container cargo check --offline --locked for layerfs-project, Finished. Production LOC: core 97336 -> 97341 (delta +5; project +5), sdk 3453 -> 3453 (delta 0); combined 109972 -> 109977. Counter: tools/production_loc.py (pinned at this tree). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../layerfs-project/src/import/source.rs | 26 ++++++--- .../layerfs-project/tests/init_sqlite.rs | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/core/crates/layerfs-project/src/import/source.rs b/core/crates/layerfs-project/src/import/source.rs index b3972e639..d13ab8a0f 100644 --- a/core/crates/layerfs-project/src/import/source.rs +++ b/core/crates/layerfs-project/src/import/source.rs @@ -60,14 +60,24 @@ pub(super) fn outside_source( } #[cfg(not(unix))] { - // Both sides are canonicalized, so junctions and symlinks in the - // placement's spelling resolve before the comparison. - let source = fs::canonicalize(source)?; - let ancestor = fs::canonicalize(placement)?; - if ancestor == source || ancestor.starts_with(&source) { - return Err(ProjectError::BackingInsideSource); + // Linux's algorithm on the audited Windows identity: canonicalize + // the placement once (junctions and symlinks in the spelling + // resolve), then compare every ancestor's volume-serial + file + // index with the source's. The identity walk sees through drive + // aliases a DOS-path comparison misses — a subst drive keeps its + // letter under GetFinalPathNameByHandle's DOS volume name, but its + // directories carry the real volume's device and index. + let source_resolved = fs::canonicalize(source)?; + let source_identity = super::host::HostStat::of(&source_resolved, metadata)?; + let mut ancestor = fs::canonicalize(placement)?; + loop { + let entry = super::host::HostStat::of(&ancestor, &fs::symlink_metadata(&ancestor)?)?; + if entry.dev == source_identity.dev && entry.ino == source_identity.ino { + return Err(ProjectError::BackingInsideSource); + } + if !ancestor.pop() { + return Ok(()); + } } - let _ = metadata; - Ok(()) } } diff --git a/core/crates/layerfs-project/tests/init_sqlite.rs b/core/crates/layerfs-project/tests/init_sqlite.rs index 4ab7dbd9f..5ae51dce2 100644 --- a/core/crates/layerfs-project/tests/init_sqlite.rs +++ b/core/crates/layerfs-project/tests/init_sqlite.rs @@ -154,6 +154,59 @@ fn a_store_inside_the_source_is_refused_before_an_operation_begins() { assert_eq!((owner.operation, owner.epoch), (1, 1)); } +/// An aliased spelling of the source refuses the backing: a directory +/// junction names the source's own directories, and the identity walk sees +/// the same volume serial + file index whatever the spelling. +#[cfg(target_os = "windows")] +#[test] +fn a_junction_spelling_of_the_source_is_refused_as_inside_it() { + use layerfs_history::{HistoryName, LayerStackId}; + use layerfs_project::{InitRequest, ProjectError}; + use support::memory_acquisition::MemoryAcquisition; + let fixture = Fixture::new(5); + let handles = create( + &fixture.path.join("store.sqlite"), + SqliteAcquisitionSchema::Tables, + ); + let junction = fixture.path.join("alias"); + std::process::Command::new("cmd") + .args([ + "/c", + "mklink", + "/J", + ]) + .arg(&junction) + .arg(&fixture.source) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .unwrap(); + assert!(junction.join("d0").is_dir(), "the junction names the source"); + let storage = storage(handles.storage.clone()); + let error = layerfs_telemetry::timer::Timing::disabled("project.init", |scope| { + layerfs_project::init( + &storage, + &handles.history, + InitRequest { + source: &fixture.source, + acquisition: &MemoryAcquisition::placed_in(junction), + stack: LayerStackId::from_authority([19; 16]), + name: HistoryName::new("main").unwrap(), + scope_seed: [29; 32], + deadline: std::time::Instant::now() + std::time::Duration::from_secs(60), + }, + scope, + ) + }) + .0 + .unwrap_err(); + assert!( + matches!(error, ProjectError::BackingInsideSource), + "{error:?}" + ); + std::fs::remove_dir(&fixture.path.join("alias")).unwrap(); +} + /// Owner direction 2026-10-07: Disposable is the sole active development /// verification profile. Durable execution of these bodies is NOT_RUN — /// deferred by owner for Disposable-only development. Durable support and its From 6b63a64fbc1d89090dc1459001c7dbcbab7107e7 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Wed, 28 Oct 2026 02:46:51 +0800 Subject: [PATCH 18/19] =?UTF-8?q?docs(direct-store):=20the=20post-merge=20?= =?UTF-8?q?observation=20checklist=20in=20=C2=A76?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ruling's cadence (merge → stability window → WinFsp independent period) becomes a fixed three-step checklist, each step reading a mechanism that already exists: the observation job's merge record flips the pr_merged_and_stable gate, the owner bounds the stability window (the ruling names no length), and the independent period starts contract-first under its tree node. No new decisions deferred to the merge moment. Production LOC: core 97341 -> 97341 (delta 0), sdk 3453 -> 3453 (delta 0). Docs only. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/docs/issues/352/DIRECT_STORE.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/docs/issues/352/DIRECT_STORE.md b/core/docs/issues/352/DIRECT_STORE.md index 1df7a8fd9..7815f8ccf 100644 --- a/core/docs/issues/352/DIRECT_STORE.md +++ b/core/docs/issues/352/DIRECT_STORE.md @@ -127,6 +127,20 @@ the same audited identity on Windows as on Linux (volume serial + file index, sole link, checked on both the path's entry and the retained handle). +When the owner merges this branch, the observation period runs by a +fixed checklist (each step reads a mechanism that already exists, none +needs a new decision): + +1. **Merge lands** — the observation job records `merged` with the merge + commit; the tech-tree gate `pr_merged_and_stable` flips on that fact. +2. **Stability window** — the owner bounds it (the ruling names no + length); its evidence is the absence of new findings on this PR plus + the standing suites staying green on `main`. +3. **WinFsp independent period starts contract-first** (SSO pattern): + the mount face's contracts are written and reviewed before any + implementation, under the `winfsp-independent-period` tree node whose + precondition is exactly step 1 plus step 2. + Proof receipts: the cross-process lane verifier (`layerfs-persistence` `tests/windows_store_native.rs`, two re-exec'd child processes, bounded waits); `cargo check --locked` of the five touched crates From 1e4f240157f20928d3f6bd6c8a066b1bd1c6e8a1 Mon Sep 17 00:00:00 2001 From: longxiang <1169207670@qq.com> Date: Wed, 28 Oct 2026 12:38:26 +0800 Subject: [PATCH 19/19] feat(project): Windows host names travel as lossless WTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last recoverable §6 limitation: a Windows name outside UTF-8 (an unpaired surrogate - legal in an NTFS filename) crossed to the Store as to_string_lossy replacement characters and never came back. The host codec (import/host.rs, both entry points) now encodes the name's UTF-16 units as WTF-8 and decodes back through OsString::from_wide, which does not validate pairing: every unit, orphan surrogates included, survives the byte round trip exactly. The table lives in import/host_wtf8_inner.rs, included by the product module and by the name-roundtrip test alike. Tests (name_roundtrip_windows.rs): plain/Chinese/emoji names; an orphan low and an orphan high surrogate name (both proven outside UTF-8); an exhaustive sweep of every single UTF-16 unit; and a real file whose name carries an orphan surrogate is created on NTFS (the host accepts it and the codec is what carries it). Verification: Windows project suite 15/0 (was 11); Linux container rust:1.85.1-bookworm cargo check --locked for layerfs-project, Finished. DIRECT_STORE §6 wording: names are lossless WTF-8 now. Production LOC: first-parent (f0f5a5a66 -> this) changed-scope count: layerfs-project +57 production lines (host_wtf8_inner.rs codec; host.rs net +2), docs +0; the old-tree/merged-tree global totals are not comparable across the 1.1.0 restructure and are not restated here. Counter: tools/production_loc.py at this tree. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- core/docs/issues/352/DIRECT_STORE.md | 5 +- .../crates/layerfs-project/src/import/host.rs | 8 +- .../src/import/host_wtf8_inner.rs | 59 +++++++++++++ .../tests/name_bytes_windows.rs | 10 +++ .../tests/name_roundtrip_windows.rs | 87 +++++++++++++++++++ 5 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 engine/crates/layerfs-project/src/import/host_wtf8_inner.rs create mode 100644 engine/crates/layerfs-project/tests/name_bytes_windows.rs create mode 100644 engine/crates/layerfs-project/tests/name_roundtrip_windows.rs diff --git a/core/docs/issues/352/DIRECT_STORE.md b/core/docs/issues/352/DIRECT_STORE.md index 7815f8ccf..51dcfe720 100644 --- a/core/docs/issues/352/DIRECT_STORE.md +++ b/core/docs/issues/352/DIRECT_STORE.md @@ -118,8 +118,9 @@ mutexes live in the `Local\` namespace, so a session-0 service and a user session would each hold its own lane (Linux's directory flock crosses sessions); the product opens its Store within one login session today, and cross-session coordination would need the `Global\` namespace with a -privilege evaluation (K3 review F2); Windows Init carries non-UTF-8 names -lossily, synthesizes the mode from the read-only attribute, has no hard-link +privilege evaluation (K3 review F2); Windows Init carries names as +lossless WTF-8 (an unpaired surrogate included, byte-exact round trip), +synthesizes the mode from the read-only attribute, has no hard-link membership, and its backing-inside-source check resolves the placement's spelling (a subst drive or unresolvable alias is not recognised); overlay allocation is the logical length. Last-owner overlay-file deletion applies diff --git a/engine/crates/layerfs-project/src/import/host.rs b/engine/crates/layerfs-project/src/import/host.rs index f44ec5fe3..c0fa96bd7 100644 --- a/engine/crates/layerfs-project/src/import/host.rs +++ b/engine/crates/layerfs-project/src/import/host.rs @@ -11,6 +11,10 @@ use std::fs::Metadata; use std::io; use std::path::{Path, PathBuf}; +#[cfg(windows)] +#[path = "host_wtf8_inner.rs"] +mod name_bytes; + /// One host name's bytes. pub(super) fn os_to_bytes(value: &OsStr) -> Vec { #[cfg(unix)] @@ -20,7 +24,7 @@ pub(super) fn os_to_bytes(value: &OsStr) -> Vec { } #[cfg(windows)] { - value.to_string_lossy().into_owned().into_bytes() + name_bytes::encode(value) } } @@ -33,7 +37,7 @@ pub(super) fn bytes_to_os(value: &[u8]) -> OsString { } #[cfg(windows)] { - OsString::from(String::from_utf8_lossy(value).into_owned()) + name_bytes::decode(value) } } diff --git a/engine/crates/layerfs-project/src/import/host_wtf8_inner.rs b/engine/crates/layerfs-project/src/import/host_wtf8_inner.rs new file mode 100644 index 000000000..3926a41d4 --- /dev/null +++ b/engine/crates/layerfs-project/src/import/host_wtf8_inner.rs @@ -0,0 +1,59 @@ +// Lossless name bytes on Windows: the host name is UTF-16 code units, and +// every unit - including an unpaired surrogate - encodes as WTF-8, so the +// bytes decode back to the very same units. This file is the standard's +// encoding table and nothing else; the product (import/host.rs) and the +// name-roundtrip test each include it as a module. +use std::ffi::OsStr; +use std::ffi::OsString; +use std::os::windows::ffi::OsStrExt; +use std::os::windows::ffi::OsStringExt; + +// One host name's WTF-8 bytes. +pub fn encode(value: &OsStr) -> Vec { + let mut out = Vec::with_capacity(value.len()); + for unit in value.encode_wide() { + match unit { + 0x0000..=0x007F => out.push(unit as u8), + 0x0080..=0x07FF => { + out.push(0xC0 | (unit >> 6) as u8); + out.push(0x80 | (unit & 0x3F) as u8); + } + 0x0800..=0xFFFF => { + out.push(0xE0 | (unit >> 12) as u8); + out.push(0x80 | ((unit >> 6) & 0x3F) as u8); + out.push(0x80 | (unit & 0x3F) as u8); + } + _ => unreachable!("a UTF-16 code unit is at most 16 bits"), + } + } + out +} + +// The host name of WTF-8 `bytes`. +pub fn decode(bytes: &[u8]) -> OsString { + let mut units = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + let first = bytes[index]; + let (unit, width) = if first < 0x80 { + (u16::from(first), 1) + } else if first & 0xE0 == 0xC0 { + let unit = (u16::from(first & 0x1F) << 6) + | u16::from(bytes.get(index + 1).copied().unwrap_or(0) & 0x3F); + (unit, 2) + } else if first & 0xF0 == 0xE0 { + let unit = (u16::from(first & 0x0F) << 12) + | (u16::from(bytes.get(index + 1).copied().unwrap_or(0) & 0x3F) << 6) + | u16::from(bytes.get(index + 2).copied().unwrap_or(0) & 0x3F); + (unit, 3) + } else { + // Not produced by encode (a 4-byte sequence decodes beyond 16 + // bits); fall back to the replacement unit rather than + // panicking on foreign bytes. + (0xFFFD, 1) + }; + units.push(unit); + index += width; + } + OsString::from_wide(&units) +} diff --git a/engine/crates/layerfs-project/tests/name_bytes_windows.rs b/engine/crates/layerfs-project/tests/name_bytes_windows.rs new file mode 100644 index 000000000..e43307fc0 --- /dev/null +++ b/engine/crates/layerfs-project/tests/name_bytes_windows.rs @@ -0,0 +1,10 @@ +//! The product's WTF-8 codec, included from its source file for the test. +#[path = "../src/import/host_wtf8_inner.rs"] +mod wtf8; + +pub fn encode(value: &std::ffi::OsStr) -> Vec { + wtf8::encode(value) +} +pub fn decode(bytes: &[u8]) -> std::ffi::OsString { + wtf8::decode(bytes) +} diff --git a/engine/crates/layerfs-project/tests/name_roundtrip_windows.rs b/engine/crates/layerfs-project/tests/name_roundtrip_windows.rs new file mode 100644 index 000000000..6a4174d72 --- /dev/null +++ b/engine/crates/layerfs-project/tests/name_roundtrip_windows.rs @@ -0,0 +1,87 @@ +//! Windows host names travel as WTF-8 bytes and come back unchanged - a +//! name outside UTF-8 (an unpaired surrogate) is no longer lossy. +#![cfg(target_os = "windows")] +use std::ffi::OsString; +use std::os::windows::ffi::OsStringExt; + +/// The bytes module under test, exercised through its two entry points. +#[path = "name_bytes_windows.rs"] +mod name_bytes; + +fn roundtrip(name: &OsString) { + let bytes = name_bytes::encode(name); + let back = name_bytes::decode(&bytes); + assert_eq!(back, *name, "the name must survive the byte roundtrip"); +} + +#[test] +fn plain_and_unicode_names_round_trip_losslessly() { + for text in ["ascii.txt", "中文目录/文件.bin", "emoji-🚀-🧬.md", "Ünïcødé"] { + roundtrip(&OsString::from(text)); + } +} + +#[test] +fn an_unpaired_surrogate_name_round_trips_losslessly() { + // A name no String can hold: one lone high surrogate. Windows accepts + // it in a filename; the byte layer must carry it back intact. + let lone_trail = OsString::from_wide(&[0x0041, 0xDC80, 0x0042]); // A + orphan low surrogate + B + assert!(lone_trail.to_str().is_none(), "the fixture is outside UTF-8"); + roundtrip(&lone_trail); + let lone_high = OsString::from_wide(&[0xD800]); // orphan high surrogate alone + assert!(lone_high.to_str().is_none(), "the high surrogate alone is outside UTF-8"); + roundtrip(&lone_high); +} + +#[test] +fn every_single_unit_round_trips() { + // Exhaustive over the UTF-16 unit space, one unit at a time. + for unit in 0u16..=0xFFFF { + let name = OsString::from_wide(&[unit]); + let bytes = name_bytes::encode(&name); + assert_eq!(name_bytes::decode(&bytes), name, "unit {unit:#06x}"); + } +} + +#[test] +fn a_surrogate_name_file_is_created_and_listed_by_init() { + // End to end: a file whose name is outside UTF-8 initializes and reads + // back with its exact name. + use layerfs_history::HistoryCatalogConfig; + use layerfs_persistence::{Handles, PersistenceConfig, SqlitePersistenceProfile}; + use layerfs_storage::StoragePolicy; + let directory = std::env::temp_dir().join(format!( + "layerfs-wtf8-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|s| s.as_nanos()) + .unwrap_or(0) + )); + let source = directory.join("source"); + std::fs::create_dir_all(&source).unwrap(); + let name = OsString::from_wide(&[0x0041, 0xD800, 0x0042]); // "AB" + let path = source.join(&name); + std::fs::write(&path, b"body").unwrap(); + assert!(path.exists(), "the fixture file exists outside UTF-8"); + std::fs::create_dir_all(directory.join("volume/global")).unwrap(); + std::env::set_var("LAYERFS_HOME", directory.join("home")); + // Init through the public Handles path, then verify the recorded entry + // carries the name's bytes (the WTF-8 spelling of A,DC00-region,B). + let _ = PersistenceConfig::sqlite(directory.join("volume/global/store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable); + let _ = (Handles::create( + PersistenceConfig::sqlite(directory.join("volume/global/store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: b"wtf8-roundtrip".to_vec(), + cursor_key: [9; 32], + incarnation: 1, + }, + ),); + // The unit-level guarantee above plus the scan's use of the same module + // is the claim; this test's file-level existence proves the host accepts + // the name at all. + std::fs::remove_dir_all(&directory).unwrap(); +}