From 3b10fef9cb5ec58b56ee5efbd902b6c2720e6b63 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Wed, 28 Oct 2026 02:11:29 +0800 Subject: [PATCH] fix(storage): scale the payload chain budgets with the cutoff (#472) The payload chain-work budgets were fixed at 512 KiB canonical and 256 KiB encoded while the construction cutoff may be raised to 1 MiB. A whole file above those sizes then failed the next Save (#472); after #473 it stores FULL, so a Store with a raised cutoff gets no whole-file deltas for files of about 256 KiB or more. The budgets are now 4 x cutoff canonical and 2 x cutoff encoded. At the default 128 KiB cutoff they are exactly 512 KiB / 256 KiB, so default Stores do not change. At any supported cutoff the largest whole file fits a chain alone, so the producer never stores a base a read could not charge, and large whole files keep deltas. The cost is read work: one read may reconstruct up to 4 x cutoff (4 MiB at 1 MiB). A depth never raises the budgets; chunk chains stay depth-bound. #473's early refusal stays as a guard; valid output under a Store's own policy no longer reaches it. Tests: chain_budget_scaling (new) covers the derived budgets, the largest-file-fits invariant, #472's shapes now chaining, the largest file at each cutoff and a chain restarting at the scaled budget. delta_base_budget moves its still-valid prospective boundaries, one-save parity and corruption guard to the default cutoff, where the same numbers hold, and reaches the early refusal by lowering the fixture's cutoff after the base is stored. Its five 1 MiB "base alone over budget" cases are deleted: under scaled budgets they now chain, re-covered positively. sdk_large_file_budget now asserts a delta for fs.apply, fs.write and a Workspace Commit (CSV-like text; word salad from eight words loses to FULL at zstd level 3), keeps Init and the default-cutoff control. Validation, macOS arm64, Rust 1.85.1, worktree launcher, Disposable only: - Against main 924f782b9d production with these tests: chain_budget_scaling 1 passed / 5 failed; sdk_large_file_budget 2 passed / 3 failed (work_exceeded FULL, no delta). With the fix all pass. - layerfs-storage 93 passed; persistence 118 passed, 2 ignored; project 23; store-service 6; fast-lane binaries of layerfs-api (45: 138 passed, 6 ignored) and layerfs-daemon (9: 24 passed, 1 ignored). - Clippy -D warnings (storage, api), fmt, boundary guard, 155 tool tests. Not run: Linux, mounted FUSE Commit, Windows, read-latency/memory measurement at the 4 MiB chain. Durable: NOT_RUN - disabled by owner until explicit reauthorization. Production LOC: 102580 -> 102588 (delta +8) core 38387 -> 38395; layerfs-storage 9060 -> 9068 (store/policy.rs 179 -> 187). engine 61534 and api 2659 unchanged. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py (blob 3341e4d036) --detail/--files --root on a git archive of 924f782b9d and on this tree. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- core/crates/layerfs-storage/README.md | 2 +- .../layerfs-storage/src/store/policy.rs | 31 ++- .../tests/chain_budget_scaling.rs | 214 ++++++++++++++++++ .../tests/delta_base_budget.rs | 167 +++++++------- core/docs/architecture/05-storage.md | 25 +- .../general/cas_cdc_deltaencoding_handbook.md | 19 +- .../tests/sdk_large_file_budget.rs | 152 +++++++++---- 7 files changed, 457 insertions(+), 153 deletions(-) create mode 100644 core/crates/layerfs-storage/tests/chain_budget_scaling.rs diff --git a/core/crates/layerfs-storage/README.md b/core/crates/layerfs-storage/README.md index 1fb2ef0da..7912c09a8 100644 --- a/core/crates/layerfs-storage/README.md +++ b/core/crates/layerfs-storage/README.md @@ -31,7 +31,7 @@ needs no Workspace, branch, commit, mount or daemon. | Group body | `<= 64 KiB` (v1/v2/v4), `<= 16 KiB` (v6) | | Records per group | `<= 8191`; `1` in the compact and singleton lanes | | Groups per pack | `<= 256`; `1` in the singleton lane | -| Chain budgets | `512 KiB` canonical and `256 KiB` encoded per chain, independent of the accepted depths | +| Chain budgets | `4 × cutoff` canonical and `2 × cutoff` encoded per chain (`512 KiB` / `256 KiB` at the default cutoff), so the largest whole file fits a chain alone; independent of the accepted depths (#472) | | Metadata pooling | implemented: supplied canonical inode leaves are stored pooled; each 73-byte inode value is one stored row under its ordinal (issue 320 design B, 2026-10-13), 100 rows per leaf page; a row keeps the ordinal its value has in a named base leaf, a new value takes the next ordinal of the Save's reserved block | | Pooled record grammar | `0 | physical body` (FULL) or `1 | base id | output length | instruction count | program` (COPY/INSERT), the ordinary lane's tags read under the `InodeLeaf` role; an `InodeBranch` uses the same program against its canonical predecessor page (#360 item 3) | | Pooled chain budgets | depth `0..=50` (own persisted field); 64 KiB canonical and 139 KiB encoded per chain; 8 KiB per pooled record; match budget 128 KiB per trial | diff --git a/core/crates/layerfs-storage/src/store/policy.rs b/core/crates/layerfs-storage/src/store/policy.rs index ab51ea633..df94cab18 100644 --- a/core/crates/layerfs-storage/src/store/policy.rs +++ b/core/crates/layerfs-storage/src/store/policy.rs @@ -4,8 +4,9 @@ //! Open validates the row before any work and refuses conflicting overrides; a //! Store is never silently migrated, and objects are never re-interpreted against //! another construction cutoff. Threshold selection, maximum stored-object -//! validity, chain-work budgets and live-memory budgets are separate: raising the -//! cutoff or a depth never raises the others. +//! validity and live-memory budgets are separate: raising the cutoff or a depth +//! never raises the others. Chain-work budgets scale with the cutoff alone, so +//! the largest whole file fits a chain; a depth never raises them. use layerfs_content::{ConstructionPolicy, ContentError}; @@ -84,12 +85,14 @@ pub const SINGLETON_FRAMING_SLACK: usize = 4_096; /// Largest assembled singleton pack: one maximum canonical record plus framing. pub const SINGLETON_PACK_LIMIT: usize = CANONICAL_LIMIT + SINGLETON_FRAMING_SLACK; -/// Chain-work budgets. These are separate from the cutoff and the depths: a -/// deeper chain is still bounded by the bytes it may decode and encode. -/// Canonical bytes one dependency chain may reconstruct. -pub const CHAIN_CANONICAL_LIMIT: u64 = 512 * 1024; -/// Encoded bytes one dependency chain may read. -pub const CHAIN_ENCODED_LIMIT: u64 = 256 * 1024; +/// Chain-work budgets, in multiples of the construction cutoff, so the largest +/// whole file always fits a chain alone. A depth never raises them: a deeper +/// chain is still bounded by the bytes it may decode and encode. At the default +/// 128 KiB cutoff they are 512 KiB canonical and 256 KiB encoded. +/// Cutoffs of canonical bytes one dependency chain may reconstruct. +pub const CHAIN_CANONICAL_CUTOFFS: u64 = 4; +/// Cutoffs of encoded bytes one dependency chain may read. +pub const CHAIN_ENCODED_CUTOFFS: u64 = 2; /// Complete bodies or selected encoded groups one operation may retain. /// /// Owner: the save operation that fills it. Bound: this many bytes of pack bodies. @@ -315,6 +318,14 @@ impl StorageCapacities { pub fn from_policy(policy: StoragePolicy) -> StorageResult { let policy = policy.validated()?; let construction = policy.construction().capacities(); + let chain = |cutoffs: u64| { + policy + .small_file_threshold_bytes() + .checked_mul(cutoffs) + .ok_or(StorageError::UnsupportedPolicy { + field: "small_file_threshold_bytes", + }) + }; Ok(Self { small_file_threshold_bytes: policy.small_file_threshold_bytes(), whole_file_canonical_limit: construction.whole_file_canonical_limit, @@ -328,8 +339,8 @@ impl StorageCapacities { metadata_delta_max_depth: policy.metadata_delta_max_depth(), metadata_chain_canonical_limit: METADATA_CHAIN_CANONICAL_LIMIT, metadata_chain_encoded_limit: METADATA_CHAIN_ENCODED_LIMIT, - chain_canonical_limit: CHAIN_CANONICAL_LIMIT, - chain_encoded_limit: CHAIN_ENCODED_LIMIT, + chain_canonical_limit: chain(CHAIN_CANONICAL_CUTOFFS)?, + chain_encoded_limit: chain(CHAIN_ENCODED_CUTOFFS)?, batch_objects: BATCH_OBJECT_LIMIT, batch_bytes: WAVE_CANONICAL_BYTES_LIMIT, transaction_rows: TRANSACTION_ROW_LIMIT, diff --git a/core/crates/layerfs-storage/tests/chain_budget_scaling.rs b/core/crates/layerfs-storage/tests/chain_budget_scaling.rs new file mode 100644 index 000000000..c2f2133ff --- /dev/null +++ b/core/crates/layerfs-storage/tests/chain_budget_scaling.rs @@ -0,0 +1,214 @@ +//! The payload chain-work budgets scale with the Store's construction cutoff +//! (#472): four cutoffs of canonical bytes and two of encoded bytes, which are +//! 512 KiB and 256 KiB at the default 128 KiB cutoff. The largest whole file a +//! cutoff admits therefore always fits a chain alone, so a Save never meets a +//! declared base whose own chain a read could not charge; a larger cutoff keeps +//! whole-file deltas for the larger whole files it creates. +#[path = "support/memory_metadata.rs"] +mod metadata; +mod support; + +use layerfs_content::{AdvisoryPredecessors, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_storage::policy::SINGLETON_FRAMING_SLACK; +use layerfs_storage::{Storage, StorageCapacities, StoragePolicy, WriteOutcome}; +use std::sync::Arc; + +const KIB: u64 = 1024; +const CUTOFFS: [u64; 4] = [128 * KIB, 256 * KIB, 512 * KIB, 1024 * KIB]; +/// Bytes the whole-file canonical envelope adds to a payload. +const ENVELOPE: usize = 23; + +fn policy(cutoff: u64) -> StoragePolicy { + StoragePolicy::new(1, cutoff, 16, 8) +} + +fn capacities(cutoff: u64) -> StorageCapacities { + StorageCapacities::from_policy(policy(cutoff)).unwrap() +} + +/// Version `version` of a `len`-byte payload: one byte of it differs from +/// every other version's. +fn payload(len: usize, compressible: bool, version: usize) -> Vec { + let mut raw = if compressible { + support::patterned(len) + } else { + support::noise(len) + }; + raw[version * 7] ^= 0x5a; + raw +} + +struct Saved { + metadata: Arc, + objects: Vec, + outcomes: Vec, +} + +impl Saved { + /// Each version's stored chain depth. + fn depths(&self) -> Vec { + self.outcomes + .iter() + .map(|o| { + assert_eq!(o.delta.whole_file_depths.iter().sum::(), 1, "{o:?}"); + o.delta + .whole_file_depths + .iter() + .position(|n| *n == 1) + .unwrap() + }) + .collect() + } + + /// Every version reads back exactly, through a fresh reader. + fn assert_reads_back(&self) { + let reader = Storage::new(self.metadata.clone()).unwrap(); + let reader = reader.reader().unwrap(); + for object in &self.objects { + assert_eq!( + reader.read_objects(&[object.id()]).unwrap(), + vec![object.canonical().to_vec()] + ); + } + } +} + +/// Saves one payload per entry of `versions` (`(len, compressible)`), each in its +/// own Save and each declaring the previous one as its base. +fn chain(cutoff: u64, versions: &[(usize, bool)]) -> Saved { + let metadata = Arc::new(metadata::MemoryMetadata::default()); + metadata.state.lock().unwrap().policy = policy(cutoff); + let storage = Storage::new(metadata.clone()).unwrap(); + let mut previous: Option = None; + let mut objects = Vec::new(); + let mut outcomes = Vec::new(); + for (version, &(len, compressible)) in versions.iter().enumerate() { + let raw = payload(len, compressible, version); + let mut object = + FinalizedObject::new(ObjectRole::WholeFile, support::assembled_small_object(&raw)) + .unwrap(); + if let Some(base) = previous { + object = object.with_predecessors(AdvisoryPredecessors::explicit(base).unwrap()); + } + previous = Some(object.id()); + let save = storage.begin_save().unwrap(); + save.accept(object.clone()).unwrap(); + outcomes.push(save.finish().unwrap()); + objects.push(object); + } + Saved { + metadata, + objects, + outcomes, + } +} + +#[test] +fn default_store_budgets_are_unchanged() { + let default = StorageCapacities::from_policy(StoragePolicy::frozen_default()).unwrap(); + assert_eq!(default.small_file_threshold_bytes, 131_072); + assert_eq!(default.chain_canonical_limit, 524_288); + assert_eq!(default.chain_encoded_limit, 262_144); +} + +#[test] +fn budgets_scale_with_every_cutoff() { + for cutoff in CUTOFFS { + let capacities = capacities(cutoff); + assert_eq!(capacities.chain_canonical_limit, 4 * cutoff, "{cutoff}"); + assert_eq!(capacities.chain_encoded_limit, 2 * cutoff, "{cutoff}"); + // A depth never raises them. + let shallow = StorageCapacities::from_policy(StoragePolicy::new(1, cutoff, 1, 1)).unwrap(); + assert_eq!( + (shallow.chain_canonical_limit, shallow.chain_encoded_limit), + ( + capacities.chain_canonical_limit, + capacities.chain_encoded_limit + ) + ); + } +} + +#[test] +fn largest_whole_file_fits_a_chain_alone_at_every_cutoff() { + // The property #472 needs: a stored whole file is never a base that a read + // could not charge by itself, whatever the cutoff. + for cutoff in CUTOFFS { + let capacities = capacities(cutoff); + assert!( + capacities.whole_file_canonical_limit as u64 <= capacities.chain_canonical_limit, + "{cutoff}: {capacities:?}" + ); + assert!( + (capacities.whole_file_frame_limit + SINGLETON_FRAMING_SLACK) as u64 + <= capacities.chain_encoded_limit, + "{cutoff}: {capacities:?}" + ); + } +} + +#[test] +fn issue_472_second_versions_are_deltas() { + // At a 1 MiB cutoff the first version alone was above the fixed budgets: + // 300,000 incompressible bytes above 256 KiB encoded, 600,000 compressible + // bytes above 512 KiB canonical. The second version failed the Save. + for (len, compressible) in [(300_000, false), (600_000, true)] { + let saved = chain(1024 * KIB, &[(len, compressible), (len, compressible)]); + assert_eq!(saved.depths(), [0, 1], "{len}"); + assert_eq!(saved.outcomes[1].delta.prefix_selected, 1, "{len}"); + assert_eq!(saved.outcomes[1].delta.work_exceeded, 0, "{len}"); + assert!( + saved.outcomes[1].stored_bytes < 4 * KIB, + "{len}: {:?}", + saved.outcomes[1] + ); + saved.assert_reads_back(); + } +} + +#[test] +fn largest_file_at_each_cutoff_saves_its_next_version() { + for cutoff in CUTOFFS { + let len = cutoff as usize - 1; + // A compressible base stores few encoded bytes: the next version chains. + let saved = chain(cutoff, &[(len, true), (len, true)]); + assert_eq!(saved.depths(), [0, 1], "{cutoff}"); + saved.assert_reads_back(); + // An incompressible base stores about one cutoff, and the prospective + // charge adds the dependent's canonical length: about 22 bytes above two + // cutoffs, so the next version is stored independently, by policy. + let saved = chain(cutoff, &[(len, false), (len, false)]); + assert_eq!(saved.depths(), [0, 0], "{cutoff}"); + assert_eq!(saved.outcomes[1].delta.work_exceeded, 1, "{cutoff}"); + saved.assert_reads_back(); + } +} + +#[test] +fn a_chain_restarts_at_the_scaled_canonical_budget() { + // 600,000-byte versions at a 1 MiB cutoff: the 4 MiB canonical budget holds + // six of them, the base chain and the dependent together. + let cutoff = 1024 * KIB; + let len = 600_000; + let canonical = (len + ENVELOPE) as u64; + let fit = (capacities(cutoff).chain_canonical_limit / canonical) as usize; + assert_eq!(fit, 6); + let saved = chain(cutoff, &vec![(len, true); fit + 2]); + let mut expected: Vec = (0..fit).collect(); + expected.extend([0, 1]); + assert_eq!(saved.depths(), expected); + assert_eq!(saved.outcomes[fit].delta.work_exceeded, 1); + assert_eq!( + saved + .outcomes + .iter() + .map(|o| o.delta.work_exceeded) + .sum::(), + 1 + ); + // The deepest version reconstructs six objects, far above the fixed 512 KiB, + // and within the scaled budget the reader enforces. + assert!(fit as u64 * canonical > 512 * KIB); + assert!(fit as u64 * canonical <= capacities(cutoff).chain_canonical_limit); + saved.assert_reads_back(); +} diff --git a/core/crates/layerfs-storage/tests/delta_base_budget.rs b/core/crates/layerfs-storage/tests/delta_base_budget.rs index bcfc6fe66..8a7ffbdd1 100644 --- a/core/crates/layerfs-storage/tests/delta_base_budget.rs +++ b/core/crates/layerfs-storage/tests/delta_base_budget.rs @@ -1,13 +1,14 @@ -//! A declared base the chain budgets cannot hold is refused, not fatal. +//! The chain-work budgets decide whether a declared base is used. //! -//! Reading a stored object as a dependency charges the chain budgets (512 KiB -//! canonical, 256 KiB encoded) and fails the read when that object's own chain -//! exceeds them. A Save whose declared base is such an object must not fail: the -//! prospective chain exceeds its budget, so the new object is stored -//! independently and counted as `work_exceeded`, exactly as for a chain that -//! outgrows the budget. A base whose own chain fits is still acquired and -//! authenticated before the dependent is judged against the budget, so a corrupt -//! one fails the Save even when the dependent then tips the chain over. +//! Reading a stored object as a dependency charges the chain budgets (4 and 2 +//! cutoffs: 512 KiB canonical and 256 KiB encoded at the default cutoff) and +//! fails the read when that object's own chain exceeds them. Under its own +//! policy a Store never holds such a chain, since the largest whole file fits +//! alone (#472); a base whose chain exceeds them anyway is refused, not fatal: the +//! new object is stored independently and counted as `work_exceeded`, exactly as +//! for a chain that outgrows the budget. A base whose own chain fits is acquired +//! and authenticated before the dependent is judged against the budget, so a +//! corrupt one fails the Save even when the dependent then tips the chain over. #[path = "support/memory_metadata.rs"] mod metadata; mod support; @@ -19,6 +20,7 @@ use metadata::MemoryMetadata; use std::sync::Arc; const ONE_MIB: u64 = 1_048_576; +const DEFAULT_CUTOFF: u64 = 131_072; /// A canonical whole-file object has 23 bytes of envelope around its payload. const ENVELOPE: usize = 23; @@ -56,6 +58,12 @@ fn mebibyte_cutoff() -> StoragePolicy { StoragePolicy::new(1, ONE_MIB, 16, 8).validated().unwrap() } +fn default_cutoff() -> StoragePolicy { + StoragePolicy::new(1, DEFAULT_CUTOFF, 16, 8) + .validated() + .unwrap() +} + fn store(policy: StoragePolicy) -> Arc { let metadata = Arc::new(MemoryMetadata::default()); metadata.state.lock().unwrap().policy = policy; @@ -86,13 +94,13 @@ fn chain(policy: StoragePolicy, versions: &[(usize, bool)]) -> Chain { } } -/// The same chain of incompressible versions, all in one Save. -fn chain_in_one_save(policy: StoragePolicy, lengths: &[usize]) -> WriteOutcome { +/// The same chain of versions, all in one Save. +fn chain_in_one_save(policy: StoragePolicy, versions: &[(usize, bool)]) -> WriteOutcome { let storage = Storage::new(store(policy)).unwrap(); let save = storage.begin_save().unwrap(); let mut base = None; - for (index, len) in lengths.iter().enumerate() { - let object = version(*len, false, index, base); + for (index, (len, compressible)) in versions.iter().enumerate() { + let object = version(*len, *compressible, index, base); base = Some(object.id()); save.accept(object).unwrap(); } @@ -141,105 +149,98 @@ impl Chain { } #[test] -fn a_base_above_the_encoded_budget_is_refused_not_fatal() { - // 300,000 incompressible bytes are stored verbatim: a record above the 256 KiB - // encoded budget, while the canonical object is far below the 512 KiB one. - let saved = chain(mebibyte_cutoff(), &[(300_000, false); 2]); - saved.assert_refused_by_budget(1, "300,000 incompressible bytes"); - saved.assert_reads_back(); -} - -#[test] -fn a_base_above_the_canonical_budget_is_refused_not_fatal() { - // 600,000 compressible bytes: a record far below 256 KiB, a canonical object - // above the 512 KiB budget on its own. - let saved = chain(mebibyte_cutoff(), &[(600_000, true); 2]); - saved.assert_refused_by_budget(1, "600,000 compressible bytes"); - saved.assert_reads_back(); -} - -#[test] -fn a_small_dependent_does_not_make_an_over_budget_base_fit() { - let saved = chain(mebibyte_cutoff(), &[(300_000, false), (10_000, false)]); - saved.assert_refused_by_budget(1, "10,000 bytes after 300,000 incompressible bytes"); - saved.assert_reads_back(); -} - -#[test] -fn the_refusal_covers_the_canonical_boundary() { - // A 524,265-byte payload is 524,288 canonical bytes: the budget itself, so its - // chain fits alone and only the dependent tips it over. One byte more does not - // fit alone. Both are the same refusal. - for len in [524_265, 524_266] { - let saved = chain(mebibyte_cutoff(), &[(len, true); 2]); - saved.assert_refused_by_budget(1, &format!("{len}-byte payload")); - } -} - -#[test] -fn the_refusal_covers_the_encoded_boundary() { - // At this size a verbatim record is its payload plus 9 bytes: 262,135 payload - // bytes are exactly the 256 KiB encoded budget, so the chain fits alone and - // only the dependent tips it over. One byte more does not fit alone. - for len in [262_135, 262_136] { - let saved = chain(mebibyte_cutoff(), &[(len, false); 2]); - saved.assert_refused_by_budget(1, &format!("{len}-byte payload")); +fn a_stored_chain_above_the_budgets_is_refused_not_fatal() { + // A Store's policy never changes, and under its own budgets the producer + // never stores a chain a read could not charge. To reach the refusal, the + // base is stored under a 1 MiB cutoff and the dependent is saved after the + // fixture's policy drops to the default cutoff, whose budgets the base alone + // exceeds: 300,000 incompressible bytes above 256 KiB encoded, 600,000 + // compressible bytes above 512 KiB canonical. + for (len, compressible) in [(300_000, false), (600_000, true)] { + let metadata = store(mebibyte_cutoff()); + let base = version(len, compressible, 0, None); + save_one(&metadata, base.clone()).unwrap(); + metadata.state.lock().unwrap().policy = default_cutoff(); + let dependent = version(10_000, compressible, 1, Some(base.id())); + let outcome = save_one(&metadata, dependent.clone()).unwrap(); + assert_eq!(outcome.delta.work_exceeded, 1, "{len}: {outcome:?}"); + assert_eq!(outcome.delta.trials, 0, "{len}: {outcome:?}"); + assert_eq!( + (outcome.full_records, outcome.prefix_records), + (1, 0), + "{len}: {outcome:?}" + ); + let storage = Storage::new(metadata.clone()).unwrap(); + assert_eq!( + storage + .reader() + .unwrap() + .read_objects(&[dependent.id()]) + .unwrap(), + vec![dependent.canonical().to_vec()] + ); } } #[test] fn a_dependent_that_fills_the_canonical_budget_exactly_is_admitted() { - // Base 400,000 canonical bytes + dependent 124,288 = 524,288, the budget. - // One dependent byte more is refused. - let base = 400_000 - ENVELOPE; - let fits = chain( - mebibyte_cutoff(), - &[(base, true), (124_288 - ENVELOPE, true)], - ); - fits.assert_admitted(1, "canonical budget filled exactly"); + // At the default cutoff: three 131,072-byte canonical objects (131,049 + // payload bytes) chained, and a fourth of the same size, fill the 524,288-byte + // budget exactly. One dependent byte more is refused. + let len = 131_072 - ENVELOPE; + let fits = chain(default_cutoff(), &[(len, true); 4]); + let depth = fits.save(2, "third version").delta.whole_file_depths; + assert_eq!(depth.iter().position(|n| *n == 1), Some(2), "{depth:?}"); + fits.assert_admitted(3, "canonical budget filled exactly"); fits.assert_reads_back(); let over = chain( - mebibyte_cutoff(), - &[(base, true), (124_289 - ENVELOPE, true)], + default_cutoff(), + &[(len, true), (len, true), (len, true), (len + 1, true)], ); - over.assert_refused_by_budget(1, "one canonical byte past the budget"); + over.assert_refused_by_budget(3, "one canonical byte past the budget"); } #[test] fn a_dependent_that_fills_the_encoded_budget_exactly_is_admitted() { // A verbatim record in the compact whole-file lane is its payload plus a // 1-byte tag. Base 131,056 + 1, dependent 131,064 + 23 canonical = 262,144, - // the budget. One dependent byte more is refused. - let fits = chain(mebibyte_cutoff(), &[(131_056, false), (131_064, false)]); + // the default cutoff's budget. One dependent byte more is refused. + let fits = chain(default_cutoff(), &[(131_056, false), (131_064, false)]); fits.assert_admitted(1, "encoded budget filled exactly"); fits.assert_reads_back(); - let over = chain(mebibyte_cutoff(), &[(131_056, false), (131_065, false)]); + let over = chain(default_cutoff(), &[(131_056, false), (131_065, false)]); over.assert_refused_by_budget(1, "one encoded byte past the budget"); } #[test] fn a_chain_built_inside_one_save_is_judged_like_one_built_over_several() { - // Two 100,000-byte versions, then a third. While the Save still holds the - // second, its chain cost is the one the Save recorded when it chose its record; - // once it is stored, the cost is read back from the stored record. Both must - // put the third version's refusal at the same size. + // Four 130,000-byte versions, then a fifth, at the default cutoff. While the + // Save still holds the fourth, its chain cost is the one the Save recorded + // when it chose its record; once it is stored, the cost is read back from the + // stored record. Both must put the fifth version's refusal at the same size. let refused = |len: usize, together: bool| { + let versions = [ + (130_000, true), + (130_000, true), + (130_000, true), + (130_000, true), + (len, true), + ]; let work_exceeded = if together { - chain_in_one_save(mebibyte_cutoff(), &[100_000, 100_000, len]) + chain_in_one_save(default_cutoff(), &versions) .delta .work_exceeded } else { - let versions = [(100_000, false), (100_000, false), (len, false)]; - chain(mebibyte_cutoff(), &versions) - .save(2, "third version") + chain(default_cutoff(), &versions) + .save(4, "fifth version") .delta .work_exceeded }; work_exceeded == 1 }; - let (mut fits, mut over) = (150_000, 175_000); + let (mut fits, mut over) = (1_000, 10_000); assert!(!refused(fits, false) && refused(over, false)); while over - fits > 1 { let middle = (fits + over) / 2; @@ -248,6 +249,8 @@ fn a_chain_built_inside_one_save_is_judged_like_one_built_over_several() { false => fits = middle, } } + // 4 x 130,023 + (fits + 23) canonical bytes fill the 524,288-byte budget. + assert_eq!(fits, 524_288 - 4 * 130_023 - ENVELOPE); assert!(!refused(fits, true), "{fits} bytes fit when stored"); assert!(refused(over, true), "{over} bytes do not fit when stored"); } @@ -331,15 +334,15 @@ fn control_a_corrupt_base_inside_both_budgets_still_fails_the_save() { #[test] fn a_corrupt_base_that_fits_is_authenticated_even_when_the_dependent_tips_the_chain_over() { - // The base fits both budgets alone: 131,057 encoded and 131,079 canonical + // At the default cutoff the base fits both budgets alone: 131,057 encoded and 131,079 canonical // bytes. The dependent is 131,088 canonical bytes, so its prospective encoded // charge is 131,057 + 131,088 = 262,145, one byte over the budget: with an // intact base the Save is the budget refusal. The base is acquired and // authenticated before that is decided, so a corrupt one fails the Save. - let intact = chain(mebibyte_cutoff(), &[(131_056, false), (131_065, false)]); + let intact = chain(default_cutoff(), &[(131_056, false), (131_065, false)]); intact.assert_refused_by_budget(1, "intact base, one encoded byte past the budget"); - let metadata = store(mebibyte_cutoff()); + let metadata = store(default_cutoff()); let first = version(131_056, false, 0, None); save_one(&metadata, first.clone()).unwrap(); let raw = support::noise(131_056); diff --git a/core/docs/architecture/05-storage.md b/core/docs/architecture/05-storage.md index e048772a9..c7f574b22 100644 --- a/core/docs/architecture/05-storage.md +++ b/core/docs/architecture/05-storage.md @@ -627,10 +627,21 @@ was chosen rather than only that it was. **Chain bounds are separate from depths**, and the source says so explicitly: "a deeper chain is still bounded by the bytes it may decode and encode." +The payload chain bounds scale with the Store's construction cutoff `T` (#472): +`StorageCapacities::from_policy` multiplies `T` by `CHAIN_CANONICAL_CUTOFFS = 4` +and `CHAIN_ENCODED_CUTOFFS = 2`. At the default 128 KiB cutoff they are 512 KiB +and 256 KiB. The largest whole file a cutoff admits (`T + 22` canonical bytes, a +frame under about `1.01 × T + 1 KiB`) therefore always fits a chain alone, so a +Save never meets a declared base whose own chain a read could not charge, and a +larger cutoff keeps whole-file deltas for the larger whole files it creates. The +cost is read work: one read may reconstruct up to `4 × T` canonical bytes, 4 MiB +at the 1 MiB cutoff. A depth never raises the bounds. Chunk chains are bounded by +their depth first (32 KiB chunks). + | Bound | Value | | --- | ---: | -| `CHAIN_CANONICAL_LIMIT` | 512 KiB | -| `CHAIN_ENCODED_LIMIT` | 256 KiB | +| `chain_canonical_limit` | `4 × T` (512 KiB at the default cutoff) | +| `chain_encoded_limit` | `2 × T` (256 KiB at the default cutoff) | | `DEPENDENCY_PACK_CACHE_BYTES` | 4 MiB | | `METADATA_DECODED_WORK_LIMIT` | 32 MiB | | `METADATA_CHAIN_CANONICAL_LIMIT` | 8 × 8,192 | @@ -647,10 +658,12 @@ chain keeps its own account in its reader. The budget is judged in two steps. First, from that cost and **before dependency reconstruction/authentication**: reading a base as a dependency charges both bounds for its whole chain and fails when the base's own chain does not fit them, so a -candidate that cannot fit alone, such as a stored whole file above 512 KiB canonical -or 256 KiB encoded under a construction cutoff above the default, is a -`work_exceeded` FULL like any chain that outgrows its budget and not a failed Save. -Such a candidate is neither reconstructed nor authenticated. Second, a candidate +candidate that cannot fit alone is a `work_exceeded` FULL like any chain that +outgrows its budget and not a failed Save. Such a candidate is neither +reconstructed nor authenticated. Since the bounds scale with the cutoff, a Store's +own producer never stores such a chain; under the earlier fixed bounds a whole file +above 512 KiB canonical or 256 KiB encoded, possible only above the default cutoff, +was one (#472, #473). Second, a candidate whose chain fits is acquired and authenticated, and then the prospective chain, the base chain plus the dependent's canonical length, is tested against both bounds; a chain that fills a bound exactly is admitted. A corrupt base whose own chain fits diff --git a/docs/general/cas_cdc_deltaencoding_handbook.md b/docs/general/cas_cdc_deltaencoding_handbook.md index 0469a74b4..c5364013a 100644 --- a/docs/general/cas_cdc_deltaencoding_handbook.md +++ b/docs/general/cas_cdc_deltaencoding_handbook.md @@ -466,15 +466,19 @@ edges; an independent object has depth zero. | Default whole-file maximum depth | 16 edges (8 before #352; a Store keeps the depth it was created with) | | Default chunk maximum depth | 8 edges (4 before #352) | | Accepted configurable depth range | 0–50; zero disables prospective deltas for the role | -| Chain canonical work budget | 512 KiB | -| Chain encoded work budget | 256 KiB | +| Chain canonical work budget | 4 × cutoff: 512 KiB at the default 128 KiB cutoff | +| Chain encoded work budget | 2 × cutoff: 256 KiB at the default 128 KiB cutoff | -Increasing a supported depth or cutoff does not increase these separate work -budgets. Payload selection conservatively tests: +The work budgets scale with the Store's cutoff `T` and nothing else (#472): +increasing a supported depth does not increase them. Scaling keeps the largest +whole file a cutoff admits inside both budgets on its own, so a larger cutoff +still gives its larger whole files deltas; one read may then reconstruct up to +`4 × T`, 4 MiB at the 1 MiB cutoff. Chunk chains stay bounded by their depth +first. Payload selection conservatively tests: ```text - base-chain decoded canonical bytes + target canonical length <= 512 KiB - base-chain encoded record bytes + target canonical length <= 256 KiB + base-chain decoded canonical bytes + target canonical length <= 4 × T + base-chain encoded record bytes + target canonical length <= 2 × T ``` The second test adds target canonical length, not a predicted delta frame size. @@ -484,7 +488,8 @@ exceeds either budget is decided earlier, from the stored sizes of its records a before dependency reconstruction/authentication, because reading it as a dependency charges the same budgets and fails when the chain alone exceeds them: such a base is an independent result, not a failed Save, and is neither reconstructed nor -authenticated. A base whose chain fits is authenticated first, so a corrupt one +authenticated; with budgets scaled to the cutoff a Store's own producer never +stores such a chain. A base whose chain fits is authenticated first, so a corrupt one fails the Save even when the dependent then tips the chain over. Reader also enforces its own declared chain limits. These are dependency-work constraints, not a blanket 512 KiB maximum on every standalone FULL object; the larger canonical diff --git a/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs b/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs index 7800221f2..ff2d424c5 100644 --- a/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs +++ b/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs @@ -1,8 +1,11 @@ //! A Store created with a larger construction cutoff (`LAYERFS_STORE_POLICY` -//! `smallFileThreshold`, 128 KiB to 1 MiB) holds whole files above the default -//! size, and a second version of such a file is saved even when the first is -//! larger than the delta chain budgets (512 KiB canonical, 256 KiB encoded): the -//! new version is stored independently. +//! `smallFileThreshold`, up to 1 MiB) keeps whole files above the default size, +//! and its payload chain budgets scale with that cutoff (#472). The second +//! version of such a file is saved, as a delta against the first, through +//! `fs.apply`, `fs.write` and a Workspace Commit; Project Init of two similar +//! large files succeeds. Under the fixed 512 KiB / 256 KiB budgets these Saves +//! failed with a dependency-work `Integrity` error, and after #473 stored every +//! second version whole. #[path = "support/installed_store.rs"] #[allow(dead_code)] mod installed; @@ -19,11 +22,35 @@ fn mebibyte_cutoff() -> StoragePolicy { StoragePolicy::new(1, 1_048_576, 16, 8).validated().unwrap() } -fn text(value: &str) -> SdkValue { - SdkValue::text(value) +/// CSV text of `len` bytes, one record per line: it compresses to about half, +/// like a data export or log. +fn rows(len: usize) -> Vec { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut out = Vec::with_capacity(len + 64); + let mut row = 0_u64; + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + row += 1; + out.extend_from_slice( + format!( + "{row},2024-{:02}-{:02},{},{:016x}\n", + state % 12 + 1, + state % 28 + 1, + state % 1_000_000, + state + ) + .as_bytes(), + ); + } + out.truncate(len); + out } -/// Word-salad text of `len` bytes: it compresses like source code or logs. +/// Word-salad text of `len` bytes from eight words. Init's similarity index +/// pairs two such files; the codec finds the nearby repeats first, so a delta +/// need not beat FULL, but the Save must not fail. fn words(len: usize) -> Vec { const WORDS: [&str; 8] = [ "alpha", "beta", "gamma", "delta", "layer", "branch", "commit", "store", @@ -74,89 +101,119 @@ fn with_first_version(label: &str, policy: StoragePolicy, first: &[u8]) -> Fixtu fixture } -/// The line of `bytes` holding `offset`: its 1-based number and its text. -fn line_at(bytes: &[u8], offset: usize) -> (i64, String) { - let number = bytes[..offset].iter().filter(|b| **b == b'\n').count(); - let line = bytes.split(|b| *b == b'\n').nth(number).unwrap(); - (number as i64 + 1, String::from_utf8(line.to_vec()).unwrap()) +/// Whole-file objects this Store's Saves stored at a positive depth so far. +fn whole_file_deltas(fixture: &Fixture) -> u64 { + let depths = fixture + .service + .store + .save_counters() + .delta + .whole_file_depths; + depths[1..].iter().sum() } -/// Branch `w` holds `expected` at the line that holds `offset`. -fn assert_line_at(fixture: &Fixture, expected: &[u8], offset: usize) { - let (number, line) = line_at(expected, offset); - let reply = fixture.call( - "fs.read", +fn assert_size(fixture: &Fixture, path: &str, size: usize) { + let stat = fixture.call( + "fs.stat", vec![ - ("branch", text("w")), - ("path", text("/big")), - ("startLine", SdkValue::Int(number)), - ("endLine", SdkValue::Int(number)), + ("branch", SdkValue::text("w")), + ("path", SdkValue::text(path)), ], ); assert_eq!( - reply.get_text("text").map(str::trim_end), - Some(line.as_str()) + stat.get("size"), + Some(&SdkValue::Int(size as i64)), + "{path}" ); } -fn assert_size(fixture: &Fixture, path: &str, size: usize) { - let stat = fixture.call("fs.stat", vec![("branch", text("w")), ("path", text(path))]); +/// Branch `w` holds `expected`'s size at `path`, and its line that holds `offset`. +fn assert_text_at(fixture: &Fixture, path: &str, expected: &[u8], offset: usize) { + assert_size(fixture, path, expected.len()); + let number = expected[..offset].iter().filter(|b| **b == b'\n').count(); + let line = expected.split(|b| *b == b'\n').nth(number).unwrap(); + let reply = fixture.call( + "fs.read", + vec![ + ("branch", SdkValue::text("w")), + ("path", SdkValue::text(path)), + ("startLine", SdkValue::Int(number as i64 + 1)), + ("endLine", SdkValue::Int(number as i64 + 1)), + ], + ); assert_eq!( - stat.get("size"), - Some(&SdkValue::Int(size as i64)), + reply.get_text("text").map(str::trim_end), + Some(std::str::from_utf8(line).unwrap()), "{path}" ); } #[test] -fn fs_apply_saves_a_second_version_of_a_large_text_file() { - // 600,000 compressible bytes: a canonical object above the 512 KiB budget. - let first = words(600_000); +fn fs_apply_stores_a_second_large_text_version_as_a_delta() { + // 600,000 bytes of text: a canonical object above the fixed 512 KiB. + let first = rows(600_000); let second = edited(&first); let fixture = with_first_version("budget-apply", mebibyte_cutoff(), &first); + let before = whole_file_deltas(&fixture); fixture.apply("w", vec![write("/big", second.as_slice())]); - assert_size(&fixture, "/big", second.len()); - assert_line_at(&fixture, &second, second.len() / 2); + assert_eq!( + whole_file_deltas(&fixture) - before, + 1, + "{:?}", + fixture.service.store.save_counters().delta + ); + assert_text_at(&fixture, "/big", &second, second.len() / 2); fixture.cleanup(); } #[test] -fn fs_write_saves_a_second_version_of_a_large_incompressible_file() { - // 300,000 incompressible bytes: a record above the 256 KiB encoded budget. +fn fs_write_stores_a_second_large_incompressible_version_as_a_delta() { + // 300,000 incompressible bytes: a record above the fixed 256 KiB encoded. let first = noise(300_000); let second = edited(&first); let fixture = with_first_version("budget-write", mebibyte_cutoff(), &first); + let before = whole_file_deltas(&fixture); let reply = fixture.call( "fs.write", vec![ - ("branch", text("w")), - ("path", text("/big")), + ("branch", SdkValue::text("w")), + ("path", SdkValue::text("/big")), ("bytes", SdkValue::Bytes(second.clone())), ], ); assert_eq!(reply.get_text("kind"), Some("Committed"), "{reply:?}"); + assert_eq!( + whole_file_deltas(&fixture) - before, + 1, + "{:?}", + fixture.service.store.save_counters().delta + ); assert_size(&fixture, "/big", second.len()); fixture.cleanup(); } #[test] -fn a_workspace_commit_saves_a_second_version_of_a_large_text_file() { - // A Workspace overwrites the first ten bytes and its captured producer builds - // the new version through the same Content and Storage code as `fs.apply`; - // no FUSE mount is involved. - let first = words(600_000); +fn a_workspace_commit_stores_a_second_large_text_version_as_a_delta() { + // A private Workspace overwrites the first ten bytes and its captured + // producer builds the new version; no FUSE mount is involved. + let first = rows(600_000); let mut second = first.clone(); second[..10].fill(b'#'); let fixture = with_first_version("budget-commit", mebibyte_cutoff(), &first); + let before = whole_file_deltas(&fixture); fixture.workspace_commit("w", &[Change::Write("/big", &second[..10])], 1, || {}); - assert_size(&fixture, "/big", second.len()); - assert_line_at(&fixture, &second, 0); + assert_eq!( + whole_file_deltas(&fixture) - before, + 1, + "{:?}", + fixture.service.store.save_counters().delta + ); + assert_text_at(&fixture, "/big", &second, 0); fixture.cleanup(); } #[test] -fn init_of_a_directory_with_two_similar_large_text_files_stores_both() { - // Project Init of a directory that contains two similar files. +fn init_of_two_similar_large_text_files_stores_both() { let first = words(600_000); let second = edited(&first); let host = installed::Fixture::built_with("budget-init", mebibyte_cutoff(), |source| { @@ -166,13 +223,14 @@ fn init_of_a_directory_with_two_similar_large_text_files_stores_both() { }); let fixture = Fixture::open(host); fixture.fork("w"); - assert_size(&fixture, "/data-2024.csv", first.len()); - assert_size(&fixture, "/data-2025.csv", second.len()); + assert_text_at(&fixture, "/data-2024.csv", &first, first.len() / 2); + assert_text_at(&fixture, "/data-2025.csv", &second, second.len() / 2); fixture.cleanup(); } #[test] fn control_the_default_cutoff_saves_the_same_two_versions() { + // At the default cutoff the 600,000-byte file is chunked. let first = words(600_000); let second = edited(&first); let fixture = with_first_version("budget-default", StoragePolicy::frozen_default(), &first);