Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/agent/src/fs/index.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// The `fs` group: reads at a saved snapshot and one-Commit Branch writes.
export { read } from "./read.js";
export { read_bytes } from "./read_bytes.js";
export { list } from "./list.js";
export { stat } from "./stat.js";
export { grep } from "./grep.js";
Expand Down
4 changes: 4 additions & 0 deletions api/agent/src/fs/read_bytes.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// fs.read_bytes: one bounded window of a file's raw bytes at a saved snapshot.
export function read_bytes(client, ref, path, offset = 0, length) {
return client.call("fs.read_bytes", { branch: ref, path, offset, length });
}
21 changes: 17 additions & 4 deletions api/agent/test/binary_write.mjs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
// Binary write bodies (#413): fs.write and fs.apply take a Buffer, any typed
// array, a DataView or an ArrayBuffer as the bytes they view, and refuse
// shared, detached and non-binary bodies before the call. Reads stay text
// only. Real native binding, Disposable Store, no Docker or daemon; Linux,
// with LAYERFS_STORE_PATH on a local volume (the E2E launcher's):
// shared, detached and non-binary bodies before the call. fs.read stays text
// only; fs.read_bytes (#417) returns the same bytes back. Real native binding,
// Disposable Store, no Docker or daemon; Linux, with LAYERFS_STORE_PATH on a
// local volume (the E2E launcher's):
// node api/agent/test/binary_write.mjs
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
Expand Down Expand Up @@ -39,10 +40,22 @@ try {
assert.equal(await text(path), "ABCD", path);
}

// Bytes that are not text are stored whole and read back as Binary.
// Bytes that are not text are stored whole, read back as Binary by fs.read,
// and read back exactly by fs.read_bytes.
await layerfs.fs.write("feat", "/blob", Buffer.from([0, 0xff, 0xfe, 1]));
assert.equal(await size("/blob"), 4);
await assert.rejects(layerfs.fs.read("feat", "/blob"), { code: "Binary" });
const whole = await layerfs.fs.read_bytes("feat", "/blob");
assert.ok(Buffer.isBuffer(whole.bytes));
assert.deepEqual([...whole.bytes], [0, 0xff, 0xfe, 1]);
assert.deepEqual([whole.size, whole.hasMore], [4, false]);
const middle = await layerfs.fs.read_bytes("feat", "/blob", 1, 2);
assert.deepEqual([...middle.bytes], [0xff, 0xfe]);
assert.deepEqual([middle.size, middle.hasMore], [4, true]);
const past = await layerfs.fs.read_bytes("feat", "/blob", 9);
assert.deepEqual([[...past.bytes], past.size, past.hasMore], [[], 4, false]);
await assert.rejects(layerfs.fs.read_bytes("feat", "/"), { code: "IsDirectory" });
await assert.rejects(layerfs.fs.read_bytes("feat", "/missing"), { code: "NoSuchPath" });

// fs.apply: `bytes` in any binary form, and binary `content` sent as bytes.
const applied = await layerfs.fs.apply("feat", [
Expand Down
4 changes: 4 additions & 0 deletions api/agent/test/fixture/cases.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ export const CASES = [
op: "branch.publish", sent: { branch: "cart" }, result: { kind: "Added", source: { branch: "feature", commit: "pub1" }, publishedLayer: { id: "lay1" } } },
{ name: "fs read", argv: ["fs", "read", "main", "/a.txt", "--start-line=3", "--end-line", "9"], sdk: (lf) => lf.fs.read("main", "/a.txt", 3, 9),
op: "fs.read", sent: { branch: "main", path: "/a.txt", startLine: 3, endLine: 9 }, result: { text: "b\n", nextLine: null } },
{ name: "fs read_bytes", argv: ["fs", "read_bytes", "main", "/a.bin", "--offset", "2", "--length", "3"], sdk: (lf) => lf.fs.read_bytes("main", "/a.bin", 2, 3),
op: "fs.read_bytes", sent: { branch: "main", path: "/a.bin", offset: 2, length: 3 }, result: { bytes: { base64: "AP8B" }, size: 5, hasMore: true } },
{ name: "fs list", argv: ["fs", "list", "main"], sdk: (lf) => lf.fs.list("main"),
op: "fs.list", sent: { branch: "main", path: "/", pageSize: 50, pagePos: 0 }, result: page },
{ name: "fs stat", argv: ["fs", "stat", "main", "/a"], sdk: (lf) => lf.fs.stat("main", "/a"),
Expand Down Expand Up @@ -118,6 +120,8 @@ export const CASES = [
// the first window, and refusals: the binding's, then one the SDK makes itself
{ name: "fs read (first window)", argv: ["fs", "read", "main", "/a"], sdk: (lf) => lf.fs.read("main", "/a"),
op: "fs.read", sent: { branch: "main", path: "/a", startLine: 1 }, result: { text: "a\n", nextLine: 2 } },
{ name: "fs read_bytes (first window)", argv: ["fs", "read_bytes", "main", "/a.bin"], sdk: (lf) => lf.fs.read_bytes("main", "/a.bin"),
op: "fs.read_bytes", sent: { branch: "main", path: "/a.bin", offset: 0 }, result: { bytes: { base64: "AP8B" }, size: 5, hasMore: true } },
{ name: "fs read (missing)", argv: ["fs", "read", "main", "/missing"], sdk: (lf) => lf.fs.read("main", "/missing"),
op: "fs.read", sent: { path: "/missing" }, refused: { code: "NoSuchPath", message: "NoSuchPath: /missing does not exist", detail: null } },
{ name: "branch merge (refused)", argv: ["branch", "merge", "a", "b", "ab"], sdk: (lf) => lf.branch.merge("a", "b", "ab"), refuse: ["branch.merge"],
Expand Down
1 change: 1 addition & 0 deletions api/agent/test/fixture/fake-native.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ function answer(op, args) {
case "branch.merge": return { value: { kind: "Merged", branch: args.name, head: "m3rge", root: "r00t" } };
case "branch.publish": return { value: { kind: "Added", source: { branch: "feature", commit: "pub1" }, publishedLayer: { id: "lay1" } } };
case "fs.read": return { value: args.startLine === 1 ? { text: "a\n", nextLine: 2 } : { text: "b\n", nextLine: null } };
case "fs.read_bytes": return { value: { bytes: Buffer.from([0, 255, 1]), size: 5, hasMore: true } };
case "fs.stat": return { value: { type: "file", size: 5, mode: 420 } };
case "fs.rm": return { value: { kind: "UpToDate", head: "h3ad" } };
case "workspace.mount": return { value: { kind: "Mounted", sandbox: args.sandbox, container: `layerfs-website-${args.sandbox}`, path: "/workspaces/w1", image: "fake-image", branch: args.branch, base: "b4se", baseLayer: "lay0", root: "r00t" } };
Expand Down
4 changes: 2 additions & 2 deletions api/agent/test/packaging.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ if (selected === "surface") {
assert.equal(shop.projectName, "website");
assert.equal(shop.options.home, "/private/configured-home");
assert.deepEqual(shop.options.deploy, { docker: "/configured/docker.sock", image: "configured-image", volume: "configured-volume", daemon: "/configured/daemon" });
// five groups: project 2, branch 6, fs 14, sandbox 3, workspace 4
// five groups: project 2, branch 6, fs 15, sandbox 3, workspace 4
const methods = (group) => Object.keys(shop[group]).sort();
assert.deepEqual(methods("project"), ["list_branches", "list_sandboxes"]);
assert.deepEqual(methods("branch"), ["diff_branches", "diff_commits", "fork", "list_commits", "merge", "publish"]);
assert.equal(methods("fs").length, 14);
assert.equal(methods("fs").length, 15);
assert.deepEqual(methods("sandbox"), ["close", "create", "list_workspaces"]);
assert.deepEqual(methods("workspace"), ["commit", "exec", "mount", "unmount"]);
assert.equal(shop.inspect, undefined);
Expand Down
3 changes: 2 additions & 1 deletion api/cli/src/fs/index.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// The `fs` commands, in help order: reads at a saved snapshot and one-Commit Branch writes.
import read from "./read.js";
import read_bytes from "./read_bytes.js";
import list from "./list.js";
import stat from "./stat.js";
import grep from "./grep.js";
Expand All @@ -14,4 +15,4 @@ import rm from "./rm.js";
import cp from "./cp.js";
import mv from "./mv.js";

export default { read, list, stat, grep, glob, diff_files, blame, write, edit, apply, mkdir, rm, cp, mv };
export default { read, read_bytes, list, stat, grep, glob, diff_files, blame, write, edit, apply, mkdir, rm, cp, mv };
8 changes: 8 additions & 0 deletions api/cli/src/fs/read_bytes.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
// layerfs fs read_bytes: fs.read_bytes(ref, path, offset?, length?).

export default {
args: ["ref", "path"],
options: { offset: "int", length: "int" },
summary: "a file's raw bytes, a bounded window at a time",
run: (lf, a, o) => lf.fs.read_bytes(a.ref, a.path, o.offset, o.length),
};
22 changes: 15 additions & 7 deletions api/docs/agent.md
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ trace line covers the command's run.
| --- | --- |
| `LayerFS` (2) | `LayerFS.open(name)`, `LayerFS.init(name, folder, timeout?)`: static, each returns a ready client |
| project (2) | `list_branches(pageSize?, pagePos?)`, `list_sandboxes(pageSize?, pagePos?)` |
| fs reads (7) | `read(ref, path, startLine?, endLine?)`, `list(ref, path?, pageSize?, pagePos?)`, `stat(ref, path)`, `grep(ref, pattern, path?, pageSize?, pagePos?)`, `glob(ref, pattern, path?, pageSize?, pagePos?)`, `diff_files(leftRef, leftPath, rightRef, rightPath)`, `blame(ref, path, fromLine?)` |
| fs reads (8) | `read(ref, path, startLine?, endLine?)`, `read_bytes(ref, path, offset?, length?)`, `list(ref, path?, pageSize?, pagePos?)`, `stat(ref, path)`, `grep(ref, pattern, path?, pageSize?, pagePos?)`, `glob(ref, pattern, path?, pageSize?, pagePos?)`, `diff_files(leftRef, leftPath, rightRef, rightPath)`, `blame(ref, path, fromLine?)` |
| fs writes (7) | `write(branch, path, content, message? \| { message?, mode? })`, `edit(branch, path, oldText, newText, message?)`, `apply(branch, changes, message?)`, `mkdir(branch, path, message? \| { message?, mode? })`, `rm(branch, path, message?)`, `cp(branch, from, to, message?)`, `mv(branch, from, to, message?)` |
| branch (6) | `fork(name, fromRef?)`, `list_commits(branch, pageSize?, pagePos?)`, `diff_commits(from, to, pageSize?, pagePos?, { cursor?, hunks? }?)`, `diff_branches(from, to, pageSize?, pagePos?, { cursor?, hunks? }?)`, `merge(a, b, name, message?, units?)`, `publish(branch, commit?)` |
| sandbox (3) | `create({ image }?)` → `{ sandbox, image }`, `list_workspaces(sandbox, pageSize?, pagePos?)`, `close(sandbox)` |
Expand Down Expand Up @@ -341,9 +341,14 @@ binary `content` is sent as `bytes`. Bytes in a `SharedArrayBuffer` or a
detached `ArrayBuffer` are refused as InvalidInput, and so is anything else,
such as an array of numbers. The bytes are copied once the call starts, after
the client's Store is open, so do not change them until the call settles.
Reads are text only: `fs.read` refuses a file that is not UTF-8 text as
`Binary`, and `fs.stat` reports its size. `node api/agent/test/binary_write.mjs`
checks these forms and refusals on Linux.
`fs.read` is text only: it refuses a file that is not UTF-8 text as `Binary`,
and `fs.stat` reports its size. `fs.read_bytes(ref, path, offset?, length?)`
returns up to `length` raw bytes from `offset` (default 0), capped at one 48 KiB
window, with the file's `size` and `hasMore`; a larger file is paged by
advancing `offset` by the returned byte count. It refuses a directory
`IsDirectory`, a symlink `NotFile` and a missing path `NoSuchPath`; an offset
past the end answers empty `bytes` with `hasMore: false`.
`node api/agent/test/binary_write.mjs` checks these forms and refusals on Linux.

### Pages and reads

Expand All @@ -360,9 +365,11 @@ component-by-component byte order, then line number for grep. Commit history is
newest first along first parents. Every call resolves each live ref once; separate
calls against live heads can shift numeric positions.

Read/blame use 1-based inclusive line positions, no byte offsets or byte-read
method. Responses fit the authenticated control frame: at most 2000 lines and
48 KiB of text/value payload, below the selected 256 KiB ceiling. LF separates
Read/blame use 1-based inclusive line positions. `fs.read_bytes` is the byte
read: it takes a byte `offset` and `length`, capped at one 48 KiB window, and
answers the file's `size` and `hasMore`. Responses fit the authenticated
control frame: at most 2000 lines and 48 KiB of text/value payload, below the
selected 256 KiB ceiling. LF separates
lines; CRLF text is preserved, and a final LF creates no phantom line. Read reports
`startLine`, `endLine`, `truncated`, `lineTruncated` and `nextLine`. Huge lines return
a UTF-8 prefix, with no within-line continuation; the remainder is explicitly
Expand Down Expand Up @@ -502,6 +509,7 @@ table differs, and when a family's files stop matching the SDK's.
| `branch.merge` | `layerfs branch merge <a> <b> <name> [-m <text>] [--units <json>]` |
| `branch.publish` | `layerfs branch publish <branch> [<commit>]` |
| `fs.read` | `layerfs fs read <ref> <path> [--start-line <n>] [--end-line <n>]` |
| `fs.read_bytes` | `layerfs fs read_bytes <ref> <path> [--offset <n>] [--length <n>]` |
| `fs.list` | `layerfs fs list <ref> [<path>] [--page-size <n>] [--page-pos <n>]` |
| `fs.stat` | `layerfs fs stat <ref> <path>` |
| `fs.grep` | `layerfs fs grep <ref> <pattern> [<path>] [--page-size <n>] [--page-pos <n>]` |
Expand Down
15 changes: 12 additions & 3 deletions core/docs/architecture/82-sdk-call-channel.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
> #352 and #373 source updates below. Correctness receipts and the component
> counter screen are distinct from this architecture.

The JavaScript SDK has 27 positional methods in four groups (29 before #373
The JavaScript SDK has 28 positional methods in four groups (29 before #373
replaced the `sandbox` group with `workspace`; see the #373 update below). The
[SDK guide](../../../api/docs/agent.md) lists signatures, environment configuration,
read/page limits and compatibility. [SPEC](../issues/327/SPEC.md) describes the
Expand Down Expand Up @@ -68,6 +68,13 @@ read caches (then `sdk/cached.rs`, now store-service `read/cached.rs`) and resol
the walker memoizes directory pages and the inode table's upper pages. Results
are byte-identical to the earlier calls; paging still counts positions.

Issue #417 source update, 2026-10-28: `fs.read_bytes` reads one bounded raw
byte window through `layerfs-content`'s `read_range`, at the same immutable
snapshot and path lookup as `fs.read`. `length` defaults to and is capped at the
48 KiB text window; an offset past the end returns no bytes and `hasMore:
false`. No storage, binding or control-record change is needed: the binding
already returns `SdkValue::Bytes` as a Buffer.

Diff omits a same-identity directory only when its metadata is also unchanged.
Directory metadata changes occupy a returned item and therefore exactly one
pagination position. This corrects the earlier mismatch where the scan counted
Expand All @@ -85,8 +92,10 @@ retains the original failure and custody. There is no resident paths draft or
separate canonical encoder. Main is read-only except through publish.

Immutable reads use 1-based inclusive lines, at most 2,000 lines/48 KiB reply
payload. Huge-line UTF-8 prefixes, omitted remainder and nextLine are explicit;
there is no byte-read method. Grep streams literal matching. Blame follows
payload. Huge-line UTF-8 prefixes, omitted remainder and nextLine are explicit.
`fs.read_bytes` is the byte read: at most 48 KiB of a regular file's raw bytes
from a byte offset, with the file's size and hasMore; a directory, symlink or
missing path is refused. Grep streams literal matching. Blame follows
immutable first parents without retaining the entire chain, skips unchanged
content IDs and uses bounded LCS with exact common edges. In coarse comparisons,
a line is inherited from the common suffix only when it starts at a
Expand Down
11 changes: 11 additions & 0 deletions engine/crates/layerfs-api/src/base/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,17 @@ pub fn line(args: &SdkValue, name: &str, default: u64) -> Result<u64, SdkRefusal
}
}

/// A non-negative byte offset or count argument; `default` when absent or null.
pub fn count(args: &SdkValue, name: &str, default: u64) -> Result<u64, SdkRefusal> {
match args.get(name) {
None | Some(SdkValue::Null) => Ok(default),
Some(SdkValue::Int(n)) if (0..=9_007_199_254_740_991).contains(n) => Ok(*n as u64),
_ => Err(invalid(format!(
"{name} must be a non-negative safe integer"
))),
}
}

/// A tree difference's page: offset or `cursor` path, and whether `hunks`.
pub fn diff_page(args: &SdkValue) -> Result<Request, SdkRefusal> {
let (limit, pos) = paging(args)?;
Expand Down
1 change: 1 addition & 0 deletions engine/crates/layerfs-api/src/base/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ pub fn call(service: &StoreService, call: &SdkCall) -> Answer {
"branch.diff_commits" => branch::diff_commits(service, args),
"branch.diff_branches" => branch::diff_branches(service, args),
"fs.read" => fs::read(service, args),
"fs.read_bytes" => fs::read_bytes(service, args),
"fs.list" => fs::list(service, args),
"fs.stat" => fs::stat(service, args),
"fs.grep" => fs::grep(service, args),
Expand Down
2 changes: 2 additions & 0 deletions engine/crates/layerfs-api/src/fs/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ mod list;
mod mkdir;
mod mv;
mod read;
mod read_bytes;
mod rm;
mod stat;
mod write;
Expand All @@ -24,6 +25,7 @@ pub use list::list;
pub use mkdir::mkdir;
pub use mv::mv;
pub use read::read;
pub use read_bytes::read_bytes;
pub use rm::rm;
pub use stat::stat;
pub use write::write;
21 changes: 21 additions & 0 deletions engine/crates/layerfs-api/src/fs/read_bytes.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//! `fs.read_bytes`.
use crate::base::args::{count, path, snapshot, Answer};
use layerfs_bridge::control::SdkValue;
use layerfs_store_service::{
read::{reads, window},
StoreService,
};

/// `read_bytes(path, { branch, commit?, offset?, length? })`: one bounded
/// window of a file's raw bytes at a Branch, Commit or Layer.
pub fn read_bytes(service: &StoreService, args: &SdkValue) -> Answer {
let (reader, snap) = snapshot(service, args)?;
let path = path(args, "path")?;
reads::read_bytes(
&reader,
&snap,
&path,
count(args, "offset", 0),
count(args, "length", window::BYTES as u64),
)
}
6 changes: 6 additions & 0 deletions engine/crates/layerfs-api/tests/golden/sdk_methods.txt
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ fs.read -> refused NoSuchPath "/nope does not exist at feat@<hex>" {path: "/nope
fs.read -> refused NoSuchBranch "Branch ghost does not exist" null
fs.read -> refused InvalidInput "missing text argument `path`" null
fs.read -> refused Binary "text contains NUL or invalid UTF-8" null
fs.read_bytes -> ok {kind: "ok", snapshot: "main@<hex>", root: "<hex>", path: "/blob.bin", bytes: bytes"[0, 159, 146, 150, 0, 1]", size: 6, hasMore: false}
fs.read_bytes -> ok {kind: "ok", snapshot: "main@<hex>", root: "<hex>", path: "/blob.bin", bytes: bytes"[146, 150]", size: 6, hasMore: true}
fs.read_bytes -> ok {kind: "ok", snapshot: "main@<hex>", root: "<hex>", path: "/blob.bin", bytes: bytes"[]", size: 6, hasMore: false}
fs.read_bytes -> refused IsDirectory "expected a regular file" null
fs.read_bytes -> refused InvalidInput "offset must be a non-negative safe integer" null
fs.read_bytes -> refused NoSuchPath "/nope does not exist at main@<hex>" {path: "/nope"}
fs.list -> ok {kind: "ok", snapshot: "feat@<hex>", root: "<hex>", path: "/", items: [{name: "a.txt", path: "/a.txt", type: "file", inode: 2}, {name: "applied-dir", path: "/applied-dir", type: "dir", inode: 201}, {name: "applied.txt", path: "/applied.txt", type: "file", inode: 200}, {name: "blob.bin", path: "/blob.bin", type: "file", inode: 3}, {name: "dir", path: "/dir", type: "dir", inode: 4}, {name: "made", path: "/made", type: "dir", inode: 72}, {name: "moved.txt", path: "/moved.txt", type: "file", inode: 137}, {name: "new.txt", path: "/new.txt", type: "file", inode: 8}], hasMore: false, nextPagePos: null}
fs.list -> ok {kind: "ok", snapshot: "feat@<hex>", root: "<hex>", path: "/dir", items: [{name: "b.txt", path: "/dir/b.txt", type: "file", inode: 5}], hasMore: true, nextPagePos: 1}
fs.list -> refused NotDirectory "expected a directory" null
Expand Down
37 changes: 37 additions & 0 deletions engine/crates/layerfs-api/tests/sdk_method_golden.rs
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,43 @@ fn public_methods_answer_as_recorded() {
"fs.read",
vec![("branch", t("main")), ("path", t("/blob.bin"))],
);
s.call(
"fs.read_bytes",
vec![("branch", t("main")), ("path", t("/blob.bin"))],
);
s.call(
"fs.read_bytes",
vec![
("branch", t("main")),
("path", t("/blob.bin")),
("offset", SdkValue::Int(2)),
("length", SdkValue::Int(2)),
],
);
s.call(
"fs.read_bytes",
vec![
("branch", t("main")),
("path", t("/blob.bin")),
("offset", SdkValue::Int(6)),
],
);
s.call(
"fs.read_bytes",
vec![("branch", t("main")), ("path", t("/dir"))],
);
s.call(
"fs.read_bytes",
vec![
("branch", t("main")),
("path", t("/blob.bin")),
("offset", SdkValue::Int(-1)),
],
);
s.call(
"fs.read_bytes",
vec![("branch", t("main")), ("path", t("/nope"))],
);
s.call("fs.list", vec![("branch", t("feat"))]);
s.call(
"fs.list",
Expand Down
Loading