diff --git a/core/crates/layerfs-storage/README.md b/core/crates/layerfs-storage/README.md index 1fb2ef0da..7912c09a8 100644 --- a/core/crates/layerfs-storage/README.md +++ b/core/crates/layerfs-storage/README.md @@ -31,7 +31,7 @@ needs no Workspace, branch, commit, mount or daemon. | Group body | `<= 64 KiB` (v1/v2/v4), `<= 16 KiB` (v6) | | Records per group | `<= 8191`; `1` in the compact and singleton lanes | | Groups per pack | `<= 256`; `1` in the singleton lane | -| Chain budgets | `512 KiB` canonical and `256 KiB` encoded per chain, independent of the accepted depths | +| Chain budgets | `4 × cutoff` canonical and `2 × cutoff` encoded per chain (`512 KiB` / `256 KiB` at the default cutoff), so the largest whole file fits a chain alone; independent of the accepted depths (#472) | | Metadata pooling | implemented: supplied canonical inode leaves are stored pooled; each 73-byte inode value is one stored row under its ordinal (issue 320 design B, 2026-10-13), 100 rows per leaf page; a row keeps the ordinal its value has in a named base leaf, a new value takes the next ordinal of the Save's reserved block | | Pooled record grammar | `0 | physical body` (FULL) or `1 | base id | output length | instruction count | program` (COPY/INSERT), the ordinary lane's tags read under the `InodeLeaf` role; an `InodeBranch` uses the same program against its canonical predecessor page (#360 item 3) | | Pooled chain budgets | depth `0..=50` (own persisted field); 64 KiB canonical and 139 KiB encoded per chain; 8 KiB per pooled record; match budget 128 KiB per trial | diff --git a/core/crates/layerfs-storage/src/store/policy.rs b/core/crates/layerfs-storage/src/store/policy.rs index ab51ea633..df94cab18 100644 --- a/core/crates/layerfs-storage/src/store/policy.rs +++ b/core/crates/layerfs-storage/src/store/policy.rs @@ -4,8 +4,9 @@ //! Open validates the row before any work and refuses conflicting overrides; a //! Store is never silently migrated, and objects are never re-interpreted against //! another construction cutoff. Threshold selection, maximum stored-object -//! validity, chain-work budgets and live-memory budgets are separate: raising the -//! cutoff or a depth never raises the others. +//! validity and live-memory budgets are separate: raising the cutoff or a depth +//! never raises the others. Chain-work budgets scale with the cutoff alone, so +//! the largest whole file fits a chain; a depth never raises them. use layerfs_content::{ConstructionPolicy, ContentError}; @@ -84,12 +85,14 @@ pub const SINGLETON_FRAMING_SLACK: usize = 4_096; /// Largest assembled singleton pack: one maximum canonical record plus framing. pub const SINGLETON_PACK_LIMIT: usize = CANONICAL_LIMIT + SINGLETON_FRAMING_SLACK; -/// Chain-work budgets. These are separate from the cutoff and the depths: a -/// deeper chain is still bounded by the bytes it may decode and encode. -/// Canonical bytes one dependency chain may reconstruct. -pub const CHAIN_CANONICAL_LIMIT: u64 = 512 * 1024; -/// Encoded bytes one dependency chain may read. -pub const CHAIN_ENCODED_LIMIT: u64 = 256 * 1024; +/// Chain-work budgets, in multiples of the construction cutoff, so the largest +/// whole file always fits a chain alone. A depth never raises them: a deeper +/// chain is still bounded by the bytes it may decode and encode. At the default +/// 128 KiB cutoff they are 512 KiB canonical and 256 KiB encoded. +/// Cutoffs of canonical bytes one dependency chain may reconstruct. +pub const CHAIN_CANONICAL_CUTOFFS: u64 = 4; +/// Cutoffs of encoded bytes one dependency chain may read. +pub const CHAIN_ENCODED_CUTOFFS: u64 = 2; /// Complete bodies or selected encoded groups one operation may retain. /// /// Owner: the save operation that fills it. Bound: this many bytes of pack bodies. @@ -315,6 +318,14 @@ impl StorageCapacities { pub fn from_policy(policy: StoragePolicy) -> StorageResult { let policy = policy.validated()?; let construction = policy.construction().capacities(); + let chain = |cutoffs: u64| { + policy + .small_file_threshold_bytes() + .checked_mul(cutoffs) + .ok_or(StorageError::UnsupportedPolicy { + field: "small_file_threshold_bytes", + }) + }; Ok(Self { small_file_threshold_bytes: policy.small_file_threshold_bytes(), whole_file_canonical_limit: construction.whole_file_canonical_limit, @@ -328,8 +339,8 @@ impl StorageCapacities { metadata_delta_max_depth: policy.metadata_delta_max_depth(), metadata_chain_canonical_limit: METADATA_CHAIN_CANONICAL_LIMIT, metadata_chain_encoded_limit: METADATA_CHAIN_ENCODED_LIMIT, - chain_canonical_limit: CHAIN_CANONICAL_LIMIT, - chain_encoded_limit: CHAIN_ENCODED_LIMIT, + chain_canonical_limit: chain(CHAIN_CANONICAL_CUTOFFS)?, + chain_encoded_limit: chain(CHAIN_ENCODED_CUTOFFS)?, batch_objects: BATCH_OBJECT_LIMIT, batch_bytes: WAVE_CANONICAL_BYTES_LIMIT, transaction_rows: TRANSACTION_ROW_LIMIT, diff --git a/core/crates/layerfs-storage/tests/chain_budget_scaling.rs b/core/crates/layerfs-storage/tests/chain_budget_scaling.rs new file mode 100644 index 000000000..c2f2133ff --- /dev/null +++ b/core/crates/layerfs-storage/tests/chain_budget_scaling.rs @@ -0,0 +1,214 @@ +//! The payload chain-work budgets scale with the Store's construction cutoff +//! (#472): four cutoffs of canonical bytes and two of encoded bytes, which are +//! 512 KiB and 256 KiB at the default 128 KiB cutoff. The largest whole file a +//! cutoff admits therefore always fits a chain alone, so a Save never meets a +//! declared base whose own chain a read could not charge; a larger cutoff keeps +//! whole-file deltas for the larger whole files it creates. +#[path = "support/memory_metadata.rs"] +mod metadata; +mod support; + +use layerfs_content::{AdvisoryPredecessors, FinalizedObject, ObjectId, ObjectRole}; +use layerfs_storage::policy::SINGLETON_FRAMING_SLACK; +use layerfs_storage::{Storage, StorageCapacities, StoragePolicy, WriteOutcome}; +use std::sync::Arc; + +const KIB: u64 = 1024; +const CUTOFFS: [u64; 4] = [128 * KIB, 256 * KIB, 512 * KIB, 1024 * KIB]; +/// Bytes the whole-file canonical envelope adds to a payload. +const ENVELOPE: usize = 23; + +fn policy(cutoff: u64) -> StoragePolicy { + StoragePolicy::new(1, cutoff, 16, 8) +} + +fn capacities(cutoff: u64) -> StorageCapacities { + StorageCapacities::from_policy(policy(cutoff)).unwrap() +} + +/// Version `version` of a `len`-byte payload: one byte of it differs from +/// every other version's. +fn payload(len: usize, compressible: bool, version: usize) -> Vec { + let mut raw = if compressible { + support::patterned(len) + } else { + support::noise(len) + }; + raw[version * 7] ^= 0x5a; + raw +} + +struct Saved { + metadata: Arc, + objects: Vec, + outcomes: Vec, +} + +impl Saved { + /// Each version's stored chain depth. + fn depths(&self) -> Vec { + self.outcomes + .iter() + .map(|o| { + assert_eq!(o.delta.whole_file_depths.iter().sum::(), 1, "{o:?}"); + o.delta + .whole_file_depths + .iter() + .position(|n| *n == 1) + .unwrap() + }) + .collect() + } + + /// Every version reads back exactly, through a fresh reader. + fn assert_reads_back(&self) { + let reader = Storage::new(self.metadata.clone()).unwrap(); + let reader = reader.reader().unwrap(); + for object in &self.objects { + assert_eq!( + reader.read_objects(&[object.id()]).unwrap(), + vec![object.canonical().to_vec()] + ); + } + } +} + +/// Saves one payload per entry of `versions` (`(len, compressible)`), each in its +/// own Save and each declaring the previous one as its base. +fn chain(cutoff: u64, versions: &[(usize, bool)]) -> Saved { + let metadata = Arc::new(metadata::MemoryMetadata::default()); + metadata.state.lock().unwrap().policy = policy(cutoff); + let storage = Storage::new(metadata.clone()).unwrap(); + let mut previous: Option = None; + let mut objects = Vec::new(); + let mut outcomes = Vec::new(); + for (version, &(len, compressible)) in versions.iter().enumerate() { + let raw = payload(len, compressible, version); + let mut object = + FinalizedObject::new(ObjectRole::WholeFile, support::assembled_small_object(&raw)) + .unwrap(); + if let Some(base) = previous { + object = object.with_predecessors(AdvisoryPredecessors::explicit(base).unwrap()); + } + previous = Some(object.id()); + let save = storage.begin_save().unwrap(); + save.accept(object.clone()).unwrap(); + outcomes.push(save.finish().unwrap()); + objects.push(object); + } + Saved { + metadata, + objects, + outcomes, + } +} + +#[test] +fn default_store_budgets_are_unchanged() { + let default = StorageCapacities::from_policy(StoragePolicy::frozen_default()).unwrap(); + assert_eq!(default.small_file_threshold_bytes, 131_072); + assert_eq!(default.chain_canonical_limit, 524_288); + assert_eq!(default.chain_encoded_limit, 262_144); +} + +#[test] +fn budgets_scale_with_every_cutoff() { + for cutoff in CUTOFFS { + let capacities = capacities(cutoff); + assert_eq!(capacities.chain_canonical_limit, 4 * cutoff, "{cutoff}"); + assert_eq!(capacities.chain_encoded_limit, 2 * cutoff, "{cutoff}"); + // A depth never raises them. + let shallow = StorageCapacities::from_policy(StoragePolicy::new(1, cutoff, 1, 1)).unwrap(); + assert_eq!( + (shallow.chain_canonical_limit, shallow.chain_encoded_limit), + ( + capacities.chain_canonical_limit, + capacities.chain_encoded_limit + ) + ); + } +} + +#[test] +fn largest_whole_file_fits_a_chain_alone_at_every_cutoff() { + // The property #472 needs: a stored whole file is never a base that a read + // could not charge by itself, whatever the cutoff. + for cutoff in CUTOFFS { + let capacities = capacities(cutoff); + assert!( + capacities.whole_file_canonical_limit as u64 <= capacities.chain_canonical_limit, + "{cutoff}: {capacities:?}" + ); + assert!( + (capacities.whole_file_frame_limit + SINGLETON_FRAMING_SLACK) as u64 + <= capacities.chain_encoded_limit, + "{cutoff}: {capacities:?}" + ); + } +} + +#[test] +fn issue_472_second_versions_are_deltas() { + // At a 1 MiB cutoff the first version alone was above the fixed budgets: + // 300,000 incompressible bytes above 256 KiB encoded, 600,000 compressible + // bytes above 512 KiB canonical. The second version failed the Save. + for (len, compressible) in [(300_000, false), (600_000, true)] { + let saved = chain(1024 * KIB, &[(len, compressible), (len, compressible)]); + assert_eq!(saved.depths(), [0, 1], "{len}"); + assert_eq!(saved.outcomes[1].delta.prefix_selected, 1, "{len}"); + assert_eq!(saved.outcomes[1].delta.work_exceeded, 0, "{len}"); + assert!( + saved.outcomes[1].stored_bytes < 4 * KIB, + "{len}: {:?}", + saved.outcomes[1] + ); + saved.assert_reads_back(); + } +} + +#[test] +fn largest_file_at_each_cutoff_saves_its_next_version() { + for cutoff in CUTOFFS { + let len = cutoff as usize - 1; + // A compressible base stores few encoded bytes: the next version chains. + let saved = chain(cutoff, &[(len, true), (len, true)]); + assert_eq!(saved.depths(), [0, 1], "{cutoff}"); + saved.assert_reads_back(); + // An incompressible base stores about one cutoff, and the prospective + // charge adds the dependent's canonical length: about 22 bytes above two + // cutoffs, so the next version is stored independently, by policy. + let saved = chain(cutoff, &[(len, false), (len, false)]); + assert_eq!(saved.depths(), [0, 0], "{cutoff}"); + assert_eq!(saved.outcomes[1].delta.work_exceeded, 1, "{cutoff}"); + saved.assert_reads_back(); + } +} + +#[test] +fn a_chain_restarts_at_the_scaled_canonical_budget() { + // 600,000-byte versions at a 1 MiB cutoff: the 4 MiB canonical budget holds + // six of them, the base chain and the dependent together. + let cutoff = 1024 * KIB; + let len = 600_000; + let canonical = (len + ENVELOPE) as u64; + let fit = (capacities(cutoff).chain_canonical_limit / canonical) as usize; + assert_eq!(fit, 6); + let saved = chain(cutoff, &vec![(len, true); fit + 2]); + let mut expected: Vec = (0..fit).collect(); + expected.extend([0, 1]); + assert_eq!(saved.depths(), expected); + assert_eq!(saved.outcomes[fit].delta.work_exceeded, 1); + assert_eq!( + saved + .outcomes + .iter() + .map(|o| o.delta.work_exceeded) + .sum::(), + 1 + ); + // The deepest version reconstructs six objects, far above the fixed 512 KiB, + // and within the scaled budget the reader enforces. + assert!(fit as u64 * canonical > 512 * KIB); + assert!(fit as u64 * canonical <= capacities(cutoff).chain_canonical_limit); + saved.assert_reads_back(); +} diff --git a/core/crates/layerfs-storage/tests/delta_base_budget.rs b/core/crates/layerfs-storage/tests/delta_base_budget.rs index bcfc6fe66..8a7ffbdd1 100644 --- a/core/crates/layerfs-storage/tests/delta_base_budget.rs +++ b/core/crates/layerfs-storage/tests/delta_base_budget.rs @@ -1,13 +1,14 @@ -//! A declared base the chain budgets cannot hold is refused, not fatal. +//! The chain-work budgets decide whether a declared base is used. //! -//! Reading a stored object as a dependency charges the chain budgets (512 KiB -//! canonical, 256 KiB encoded) and fails the read when that object's own chain -//! exceeds them. A Save whose declared base is such an object must not fail: the -//! prospective chain exceeds its budget, so the new object is stored -//! independently and counted as `work_exceeded`, exactly as for a chain that -//! outgrows the budget. A base whose own chain fits is still acquired and -//! authenticated before the dependent is judged against the budget, so a corrupt -//! one fails the Save even when the dependent then tips the chain over. +//! Reading a stored object as a dependency charges the chain budgets (4 and 2 +//! cutoffs: 512 KiB canonical and 256 KiB encoded at the default cutoff) and +//! fails the read when that object's own chain exceeds them. Under its own +//! policy a Store never holds such a chain, since the largest whole file fits +//! alone (#472); a base whose chain exceeds them anyway is refused, not fatal: the +//! new object is stored independently and counted as `work_exceeded`, exactly as +//! for a chain that outgrows the budget. A base whose own chain fits is acquired +//! and authenticated before the dependent is judged against the budget, so a +//! corrupt one fails the Save even when the dependent then tips the chain over. #[path = "support/memory_metadata.rs"] mod metadata; mod support; @@ -19,6 +20,7 @@ use metadata::MemoryMetadata; use std::sync::Arc; const ONE_MIB: u64 = 1_048_576; +const DEFAULT_CUTOFF: u64 = 131_072; /// A canonical whole-file object has 23 bytes of envelope around its payload. const ENVELOPE: usize = 23; @@ -56,6 +58,12 @@ fn mebibyte_cutoff() -> StoragePolicy { StoragePolicy::new(1, ONE_MIB, 16, 8).validated().unwrap() } +fn default_cutoff() -> StoragePolicy { + StoragePolicy::new(1, DEFAULT_CUTOFF, 16, 8) + .validated() + .unwrap() +} + fn store(policy: StoragePolicy) -> Arc { let metadata = Arc::new(MemoryMetadata::default()); metadata.state.lock().unwrap().policy = policy; @@ -86,13 +94,13 @@ fn chain(policy: StoragePolicy, versions: &[(usize, bool)]) -> Chain { } } -/// The same chain of incompressible versions, all in one Save. -fn chain_in_one_save(policy: StoragePolicy, lengths: &[usize]) -> WriteOutcome { +/// The same chain of versions, all in one Save. +fn chain_in_one_save(policy: StoragePolicy, versions: &[(usize, bool)]) -> WriteOutcome { let storage = Storage::new(store(policy)).unwrap(); let save = storage.begin_save().unwrap(); let mut base = None; - for (index, len) in lengths.iter().enumerate() { - let object = version(*len, false, index, base); + for (index, (len, compressible)) in versions.iter().enumerate() { + let object = version(*len, *compressible, index, base); base = Some(object.id()); save.accept(object).unwrap(); } @@ -141,105 +149,98 @@ impl Chain { } #[test] -fn a_base_above_the_encoded_budget_is_refused_not_fatal() { - // 300,000 incompressible bytes are stored verbatim: a record above the 256 KiB - // encoded budget, while the canonical object is far below the 512 KiB one. - let saved = chain(mebibyte_cutoff(), &[(300_000, false); 2]); - saved.assert_refused_by_budget(1, "300,000 incompressible bytes"); - saved.assert_reads_back(); -} - -#[test] -fn a_base_above_the_canonical_budget_is_refused_not_fatal() { - // 600,000 compressible bytes: a record far below 256 KiB, a canonical object - // above the 512 KiB budget on its own. - let saved = chain(mebibyte_cutoff(), &[(600_000, true); 2]); - saved.assert_refused_by_budget(1, "600,000 compressible bytes"); - saved.assert_reads_back(); -} - -#[test] -fn a_small_dependent_does_not_make_an_over_budget_base_fit() { - let saved = chain(mebibyte_cutoff(), &[(300_000, false), (10_000, false)]); - saved.assert_refused_by_budget(1, "10,000 bytes after 300,000 incompressible bytes"); - saved.assert_reads_back(); -} - -#[test] -fn the_refusal_covers_the_canonical_boundary() { - // A 524,265-byte payload is 524,288 canonical bytes: the budget itself, so its - // chain fits alone and only the dependent tips it over. One byte more does not - // fit alone. Both are the same refusal. - for len in [524_265, 524_266] { - let saved = chain(mebibyte_cutoff(), &[(len, true); 2]); - saved.assert_refused_by_budget(1, &format!("{len}-byte payload")); - } -} - -#[test] -fn the_refusal_covers_the_encoded_boundary() { - // At this size a verbatim record is its payload plus 9 bytes: 262,135 payload - // bytes are exactly the 256 KiB encoded budget, so the chain fits alone and - // only the dependent tips it over. One byte more does not fit alone. - for len in [262_135, 262_136] { - let saved = chain(mebibyte_cutoff(), &[(len, false); 2]); - saved.assert_refused_by_budget(1, &format!("{len}-byte payload")); +fn a_stored_chain_above_the_budgets_is_refused_not_fatal() { + // A Store's policy never changes, and under its own budgets the producer + // never stores a chain a read could not charge. To reach the refusal, the + // base is stored under a 1 MiB cutoff and the dependent is saved after the + // fixture's policy drops to the default cutoff, whose budgets the base alone + // exceeds: 300,000 incompressible bytes above 256 KiB encoded, 600,000 + // compressible bytes above 512 KiB canonical. + for (len, compressible) in [(300_000, false), (600_000, true)] { + let metadata = store(mebibyte_cutoff()); + let base = version(len, compressible, 0, None); + save_one(&metadata, base.clone()).unwrap(); + metadata.state.lock().unwrap().policy = default_cutoff(); + let dependent = version(10_000, compressible, 1, Some(base.id())); + let outcome = save_one(&metadata, dependent.clone()).unwrap(); + assert_eq!(outcome.delta.work_exceeded, 1, "{len}: {outcome:?}"); + assert_eq!(outcome.delta.trials, 0, "{len}: {outcome:?}"); + assert_eq!( + (outcome.full_records, outcome.prefix_records), + (1, 0), + "{len}: {outcome:?}" + ); + let storage = Storage::new(metadata.clone()).unwrap(); + assert_eq!( + storage + .reader() + .unwrap() + .read_objects(&[dependent.id()]) + .unwrap(), + vec![dependent.canonical().to_vec()] + ); } } #[test] fn a_dependent_that_fills_the_canonical_budget_exactly_is_admitted() { - // Base 400,000 canonical bytes + dependent 124,288 = 524,288, the budget. - // One dependent byte more is refused. - let base = 400_000 - ENVELOPE; - let fits = chain( - mebibyte_cutoff(), - &[(base, true), (124_288 - ENVELOPE, true)], - ); - fits.assert_admitted(1, "canonical budget filled exactly"); + // At the default cutoff: three 131,072-byte canonical objects (131,049 + // payload bytes) chained, and a fourth of the same size, fill the 524,288-byte + // budget exactly. One dependent byte more is refused. + let len = 131_072 - ENVELOPE; + let fits = chain(default_cutoff(), &[(len, true); 4]); + let depth = fits.save(2, "third version").delta.whole_file_depths; + assert_eq!(depth.iter().position(|n| *n == 1), Some(2), "{depth:?}"); + fits.assert_admitted(3, "canonical budget filled exactly"); fits.assert_reads_back(); let over = chain( - mebibyte_cutoff(), - &[(base, true), (124_289 - ENVELOPE, true)], + default_cutoff(), + &[(len, true), (len, true), (len, true), (len + 1, true)], ); - over.assert_refused_by_budget(1, "one canonical byte past the budget"); + over.assert_refused_by_budget(3, "one canonical byte past the budget"); } #[test] fn a_dependent_that_fills_the_encoded_budget_exactly_is_admitted() { // A verbatim record in the compact whole-file lane is its payload plus a // 1-byte tag. Base 131,056 + 1, dependent 131,064 + 23 canonical = 262,144, - // the budget. One dependent byte more is refused. - let fits = chain(mebibyte_cutoff(), &[(131_056, false), (131_064, false)]); + // the default cutoff's budget. One dependent byte more is refused. + let fits = chain(default_cutoff(), &[(131_056, false), (131_064, false)]); fits.assert_admitted(1, "encoded budget filled exactly"); fits.assert_reads_back(); - let over = chain(mebibyte_cutoff(), &[(131_056, false), (131_065, false)]); + let over = chain(default_cutoff(), &[(131_056, false), (131_065, false)]); over.assert_refused_by_budget(1, "one encoded byte past the budget"); } #[test] fn a_chain_built_inside_one_save_is_judged_like_one_built_over_several() { - // Two 100,000-byte versions, then a third. While the Save still holds the - // second, its chain cost is the one the Save recorded when it chose its record; - // once it is stored, the cost is read back from the stored record. Both must - // put the third version's refusal at the same size. + // Four 130,000-byte versions, then a fifth, at the default cutoff. While the + // Save still holds the fourth, its chain cost is the one the Save recorded + // when it chose its record; once it is stored, the cost is read back from the + // stored record. Both must put the fifth version's refusal at the same size. let refused = |len: usize, together: bool| { + let versions = [ + (130_000, true), + (130_000, true), + (130_000, true), + (130_000, true), + (len, true), + ]; let work_exceeded = if together { - chain_in_one_save(mebibyte_cutoff(), &[100_000, 100_000, len]) + chain_in_one_save(default_cutoff(), &versions) .delta .work_exceeded } else { - let versions = [(100_000, false), (100_000, false), (len, false)]; - chain(mebibyte_cutoff(), &versions) - .save(2, "third version") + chain(default_cutoff(), &versions) + .save(4, "fifth version") .delta .work_exceeded }; work_exceeded == 1 }; - let (mut fits, mut over) = (150_000, 175_000); + let (mut fits, mut over) = (1_000, 10_000); assert!(!refused(fits, false) && refused(over, false)); while over - fits > 1 { let middle = (fits + over) / 2; @@ -248,6 +249,8 @@ fn a_chain_built_inside_one_save_is_judged_like_one_built_over_several() { false => fits = middle, } } + // 4 x 130,023 + (fits + 23) canonical bytes fill the 524,288-byte budget. + assert_eq!(fits, 524_288 - 4 * 130_023 - ENVELOPE); assert!(!refused(fits, true), "{fits} bytes fit when stored"); assert!(refused(over, true), "{over} bytes do not fit when stored"); } @@ -331,15 +334,15 @@ fn control_a_corrupt_base_inside_both_budgets_still_fails_the_save() { #[test] fn a_corrupt_base_that_fits_is_authenticated_even_when_the_dependent_tips_the_chain_over() { - // The base fits both budgets alone: 131,057 encoded and 131,079 canonical + // At the default cutoff the base fits both budgets alone: 131,057 encoded and 131,079 canonical // bytes. The dependent is 131,088 canonical bytes, so its prospective encoded // charge is 131,057 + 131,088 = 262,145, one byte over the budget: with an // intact base the Save is the budget refusal. The base is acquired and // authenticated before that is decided, so a corrupt one fails the Save. - let intact = chain(mebibyte_cutoff(), &[(131_056, false), (131_065, false)]); + let intact = chain(default_cutoff(), &[(131_056, false), (131_065, false)]); intact.assert_refused_by_budget(1, "intact base, one encoded byte past the budget"); - let metadata = store(mebibyte_cutoff()); + let metadata = store(default_cutoff()); let first = version(131_056, false, 0, None); save_one(&metadata, first.clone()).unwrap(); let raw = support::noise(131_056); diff --git a/core/docs/architecture/05-storage.md b/core/docs/architecture/05-storage.md index e048772a9..c7f574b22 100644 --- a/core/docs/architecture/05-storage.md +++ b/core/docs/architecture/05-storage.md @@ -627,10 +627,21 @@ was chosen rather than only that it was. **Chain bounds are separate from depths**, and the source says so explicitly: "a deeper chain is still bounded by the bytes it may decode and encode." +The payload chain bounds scale with the Store's construction cutoff `T` (#472): +`StorageCapacities::from_policy` multiplies `T` by `CHAIN_CANONICAL_CUTOFFS = 4` +and `CHAIN_ENCODED_CUTOFFS = 2`. At the default 128 KiB cutoff they are 512 KiB +and 256 KiB. The largest whole file a cutoff admits (`T + 22` canonical bytes, a +frame under about `1.01 × T + 1 KiB`) therefore always fits a chain alone, so a +Save never meets a declared base whose own chain a read could not charge, and a +larger cutoff keeps whole-file deltas for the larger whole files it creates. The +cost is read work: one read may reconstruct up to `4 × T` canonical bytes, 4 MiB +at the 1 MiB cutoff. A depth never raises the bounds. Chunk chains are bounded by +their depth first (32 KiB chunks). + | Bound | Value | | --- | ---: | -| `CHAIN_CANONICAL_LIMIT` | 512 KiB | -| `CHAIN_ENCODED_LIMIT` | 256 KiB | +| `chain_canonical_limit` | `4 × T` (512 KiB at the default cutoff) | +| `chain_encoded_limit` | `2 × T` (256 KiB at the default cutoff) | | `DEPENDENCY_PACK_CACHE_BYTES` | 4 MiB | | `METADATA_DECODED_WORK_LIMIT` | 32 MiB | | `METADATA_CHAIN_CANONICAL_LIMIT` | 8 × 8,192 | @@ -647,10 +658,12 @@ chain keeps its own account in its reader. The budget is judged in two steps. First, from that cost and **before dependency reconstruction/authentication**: reading a base as a dependency charges both bounds for its whole chain and fails when the base's own chain does not fit them, so a -candidate that cannot fit alone, such as a stored whole file above 512 KiB canonical -or 256 KiB encoded under a construction cutoff above the default, is a -`work_exceeded` FULL like any chain that outgrows its budget and not a failed Save. -Such a candidate is neither reconstructed nor authenticated. Second, a candidate +candidate that cannot fit alone is a `work_exceeded` FULL like any chain that +outgrows its budget and not a failed Save. Such a candidate is neither +reconstructed nor authenticated. Since the bounds scale with the cutoff, a Store's +own producer never stores such a chain; under the earlier fixed bounds a whole file +above 512 KiB canonical or 256 KiB encoded, possible only above the default cutoff, +was one (#472, #473). Second, a candidate whose chain fits is acquired and authenticated, and then the prospective chain, the base chain plus the dependent's canonical length, is tested against both bounds; a chain that fills a bound exactly is admitted. A corrupt base whose own chain fits diff --git a/docs/general/cas_cdc_deltaencoding_handbook.md b/docs/general/cas_cdc_deltaencoding_handbook.md index 0469a74b4..c5364013a 100644 --- a/docs/general/cas_cdc_deltaencoding_handbook.md +++ b/docs/general/cas_cdc_deltaencoding_handbook.md @@ -466,15 +466,19 @@ edges; an independent object has depth zero. | Default whole-file maximum depth | 16 edges (8 before #352; a Store keeps the depth it was created with) | | Default chunk maximum depth | 8 edges (4 before #352) | | Accepted configurable depth range | 0–50; zero disables prospective deltas for the role | -| Chain canonical work budget | 512 KiB | -| Chain encoded work budget | 256 KiB | +| Chain canonical work budget | 4 × cutoff: 512 KiB at the default 128 KiB cutoff | +| Chain encoded work budget | 2 × cutoff: 256 KiB at the default 128 KiB cutoff | -Increasing a supported depth or cutoff does not increase these separate work -budgets. Payload selection conservatively tests: +The work budgets scale with the Store's cutoff `T` and nothing else (#472): +increasing a supported depth does not increase them. Scaling keeps the largest +whole file a cutoff admits inside both budgets on its own, so a larger cutoff +still gives its larger whole files deltas; one read may then reconstruct up to +`4 × T`, 4 MiB at the 1 MiB cutoff. Chunk chains stay bounded by their depth +first. Payload selection conservatively tests: ```text - base-chain decoded canonical bytes + target canonical length <= 512 KiB - base-chain encoded record bytes + target canonical length <= 256 KiB + base-chain decoded canonical bytes + target canonical length <= 4 × T + base-chain encoded record bytes + target canonical length <= 2 × T ``` The second test adds target canonical length, not a predicted delta frame size. @@ -484,7 +488,8 @@ exceeds either budget is decided earlier, from the stored sizes of its records a before dependency reconstruction/authentication, because reading it as a dependency charges the same budgets and fails when the chain alone exceeds them: such a base is an independent result, not a failed Save, and is neither reconstructed nor -authenticated. A base whose chain fits is authenticated first, so a corrupt one +authenticated; with budgets scaled to the cutoff a Store's own producer never +stores such a chain. A base whose chain fits is authenticated first, so a corrupt one fails the Save even when the dependent then tips the chain over. Reader also enforces its own declared chain limits. These are dependency-work constraints, not a blanket 512 KiB maximum on every standalone FULL object; the larger canonical diff --git a/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs b/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs index 7800221f2..ff2d424c5 100644 --- a/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs +++ b/engine/crates/layerfs-api/tests/sdk_large_file_budget.rs @@ -1,8 +1,11 @@ //! A Store created with a larger construction cutoff (`LAYERFS_STORE_POLICY` -//! `smallFileThreshold`, 128 KiB to 1 MiB) holds whole files above the default -//! size, and a second version of such a file is saved even when the first is -//! larger than the delta chain budgets (512 KiB canonical, 256 KiB encoded): the -//! new version is stored independently. +//! `smallFileThreshold`, up to 1 MiB) keeps whole files above the default size, +//! and its payload chain budgets scale with that cutoff (#472). The second +//! version of such a file is saved, as a delta against the first, through +//! `fs.apply`, `fs.write` and a Workspace Commit; Project Init of two similar +//! large files succeeds. Under the fixed 512 KiB / 256 KiB budgets these Saves +//! failed with a dependency-work `Integrity` error, and after #473 stored every +//! second version whole. #[path = "support/installed_store.rs"] #[allow(dead_code)] mod installed; @@ -19,11 +22,35 @@ fn mebibyte_cutoff() -> StoragePolicy { StoragePolicy::new(1, 1_048_576, 16, 8).validated().unwrap() } -fn text(value: &str) -> SdkValue { - SdkValue::text(value) +/// CSV text of `len` bytes, one record per line: it compresses to about half, +/// like a data export or log. +fn rows(len: usize) -> Vec { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut out = Vec::with_capacity(len + 64); + let mut row = 0_u64; + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + row += 1; + out.extend_from_slice( + format!( + "{row},2024-{:02}-{:02},{},{:016x}\n", + state % 12 + 1, + state % 28 + 1, + state % 1_000_000, + state + ) + .as_bytes(), + ); + } + out.truncate(len); + out } -/// Word-salad text of `len` bytes: it compresses like source code or logs. +/// Word-salad text of `len` bytes from eight words. Init's similarity index +/// pairs two such files; the codec finds the nearby repeats first, so a delta +/// need not beat FULL, but the Save must not fail. fn words(len: usize) -> Vec { const WORDS: [&str; 8] = [ "alpha", "beta", "gamma", "delta", "layer", "branch", "commit", "store", @@ -74,89 +101,119 @@ fn with_first_version(label: &str, policy: StoragePolicy, first: &[u8]) -> Fixtu fixture } -/// The line of `bytes` holding `offset`: its 1-based number and its text. -fn line_at(bytes: &[u8], offset: usize) -> (i64, String) { - let number = bytes[..offset].iter().filter(|b| **b == b'\n').count(); - let line = bytes.split(|b| *b == b'\n').nth(number).unwrap(); - (number as i64 + 1, String::from_utf8(line.to_vec()).unwrap()) +/// Whole-file objects this Store's Saves stored at a positive depth so far. +fn whole_file_deltas(fixture: &Fixture) -> u64 { + let depths = fixture + .service + .store + .save_counters() + .delta + .whole_file_depths; + depths[1..].iter().sum() } -/// Branch `w` holds `expected` at the line that holds `offset`. -fn assert_line_at(fixture: &Fixture, expected: &[u8], offset: usize) { - let (number, line) = line_at(expected, offset); - let reply = fixture.call( - "fs.read", +fn assert_size(fixture: &Fixture, path: &str, size: usize) { + let stat = fixture.call( + "fs.stat", vec![ - ("branch", text("w")), - ("path", text("/big")), - ("startLine", SdkValue::Int(number)), - ("endLine", SdkValue::Int(number)), + ("branch", SdkValue::text("w")), + ("path", SdkValue::text(path)), ], ); assert_eq!( - reply.get_text("text").map(str::trim_end), - Some(line.as_str()) + stat.get("size"), + Some(&SdkValue::Int(size as i64)), + "{path}" ); } -fn assert_size(fixture: &Fixture, path: &str, size: usize) { - let stat = fixture.call("fs.stat", vec![("branch", text("w")), ("path", text(path))]); +/// Branch `w` holds `expected`'s size at `path`, and its line that holds `offset`. +fn assert_text_at(fixture: &Fixture, path: &str, expected: &[u8], offset: usize) { + assert_size(fixture, path, expected.len()); + let number = expected[..offset].iter().filter(|b| **b == b'\n').count(); + let line = expected.split(|b| *b == b'\n').nth(number).unwrap(); + let reply = fixture.call( + "fs.read", + vec![ + ("branch", SdkValue::text("w")), + ("path", SdkValue::text(path)), + ("startLine", SdkValue::Int(number as i64 + 1)), + ("endLine", SdkValue::Int(number as i64 + 1)), + ], + ); assert_eq!( - stat.get("size"), - Some(&SdkValue::Int(size as i64)), + reply.get_text("text").map(str::trim_end), + Some(std::str::from_utf8(line).unwrap()), "{path}" ); } #[test] -fn fs_apply_saves_a_second_version_of_a_large_text_file() { - // 600,000 compressible bytes: a canonical object above the 512 KiB budget. - let first = words(600_000); +fn fs_apply_stores_a_second_large_text_version_as_a_delta() { + // 600,000 bytes of text: a canonical object above the fixed 512 KiB. + let first = rows(600_000); let second = edited(&first); let fixture = with_first_version("budget-apply", mebibyte_cutoff(), &first); + let before = whole_file_deltas(&fixture); fixture.apply("w", vec![write("/big", second.as_slice())]); - assert_size(&fixture, "/big", second.len()); - assert_line_at(&fixture, &second, second.len() / 2); + assert_eq!( + whole_file_deltas(&fixture) - before, + 1, + "{:?}", + fixture.service.store.save_counters().delta + ); + assert_text_at(&fixture, "/big", &second, second.len() / 2); fixture.cleanup(); } #[test] -fn fs_write_saves_a_second_version_of_a_large_incompressible_file() { - // 300,000 incompressible bytes: a record above the 256 KiB encoded budget. +fn fs_write_stores_a_second_large_incompressible_version_as_a_delta() { + // 300,000 incompressible bytes: a record above the fixed 256 KiB encoded. let first = noise(300_000); let second = edited(&first); let fixture = with_first_version("budget-write", mebibyte_cutoff(), &first); + let before = whole_file_deltas(&fixture); let reply = fixture.call( "fs.write", vec![ - ("branch", text("w")), - ("path", text("/big")), + ("branch", SdkValue::text("w")), + ("path", SdkValue::text("/big")), ("bytes", SdkValue::Bytes(second.clone())), ], ); assert_eq!(reply.get_text("kind"), Some("Committed"), "{reply:?}"); + assert_eq!( + whole_file_deltas(&fixture) - before, + 1, + "{:?}", + fixture.service.store.save_counters().delta + ); assert_size(&fixture, "/big", second.len()); fixture.cleanup(); } #[test] -fn a_workspace_commit_saves_a_second_version_of_a_large_text_file() { - // A Workspace overwrites the first ten bytes and its captured producer builds - // the new version through the same Content and Storage code as `fs.apply`; - // no FUSE mount is involved. - let first = words(600_000); +fn a_workspace_commit_stores_a_second_large_text_version_as_a_delta() { + // A private Workspace overwrites the first ten bytes and its captured + // producer builds the new version; no FUSE mount is involved. + let first = rows(600_000); let mut second = first.clone(); second[..10].fill(b'#'); let fixture = with_first_version("budget-commit", mebibyte_cutoff(), &first); + let before = whole_file_deltas(&fixture); fixture.workspace_commit("w", &[Change::Write("/big", &second[..10])], 1, || {}); - assert_size(&fixture, "/big", second.len()); - assert_line_at(&fixture, &second, 0); + assert_eq!( + whole_file_deltas(&fixture) - before, + 1, + "{:?}", + fixture.service.store.save_counters().delta + ); + assert_text_at(&fixture, "/big", &second, 0); fixture.cleanup(); } #[test] -fn init_of_a_directory_with_two_similar_large_text_files_stores_both() { - // Project Init of a directory that contains two similar files. +fn init_of_two_similar_large_text_files_stores_both() { let first = words(600_000); let second = edited(&first); let host = installed::Fixture::built_with("budget-init", mebibyte_cutoff(), |source| { @@ -166,13 +223,14 @@ fn init_of_a_directory_with_two_similar_large_text_files_stores_both() { }); let fixture = Fixture::open(host); fixture.fork("w"); - assert_size(&fixture, "/data-2024.csv", first.len()); - assert_size(&fixture, "/data-2025.csv", second.len()); + assert_text_at(&fixture, "/data-2024.csv", &first, first.len() / 2); + assert_text_at(&fixture, "/data-2025.csv", &second, second.len() / 2); fixture.cleanup(); } #[test] fn control_the_default_cutoff_saves_the_same_two_versions() { + // At the default cutoff the 600,000-byte file is chunked. let first = words(600_000); let second = edited(&first); let fixture = with_first_version("budget-default", StoragePolicy::frozen_default(), &first);