From 67ae231be93e76d47e49b482ee8e0de3b5e4e86c Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 07:59:22 +0200 Subject: [PATCH 01/46] perf(benchmarks): add in-process allocation micro-benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArchiveStepBenchmarks is end-to-end against Azurite, where fixture and SDK overhead dominate and no single optimization is separable. At divisor=8 it reports 115.94 MB allocated for a ~32 MB / 254-file repository; at divisor=1, 444.88 MB with 4000 Gen2 collections. Useful as a regression gate, useless for attributing a specific fix. Add AllocationBenchmarks: [MemoryDiagnoser], in-process, no Docker. One benchmark per byte-pushing component that is a candidate for optimization — HashCodec, SparseFingerprint.Sampler, FileTreeSerializer, TarBuilder, and the ChunkIndexLocalStore read/write/lookup paths. Program.cs gains a --class switch so both classes are runnable without editing code, and --filter so one component can be re-measured on its own. The archive path keeps its existing single-invocation job and tail-log append unchanged; micro-benchmarks use BenchmarkDotNet's normal warmup/invocation defaults (which the single-invocation job would make meaningless) and deliberately do not touch benchmark-tail.md, whose schema is archive-specific. Baseline on this machine (Apple M4, .NET 10.0.5): TarBuilder seal one 64 MB bundle 276,498,485 B Gen2 1666 SparseFingerprint.Sampler 64 GB file 16,787,173 B Gen2 898 FileTreeSerializer.Deserialize (1000) 1,758,748 B Gen2 90 FileTreeSerializer.Serialize (1000) 1,110,504 B Gen2 142 ChunkIndexLocalStore.UpsertRemoteBacked (1000) 979,488 B ChunkIndexLocalStore.FindEntry x256 672,064 B ChunkIndexLocalStore.ReadRangeEntries (1000) 656,952 B SparseFingerprint.Sampler 200 KB file 205,256 B Gen2 62 HashCodec.ToLowerHex 304 B HashCodec.NormalizeHex (already canonical) 152 B Note the ToLowerHex fixture uses a digest with a-f nibbles on purpose. An all-zero digest hexes to "000...0", and ToLowerInvariant then returns the same instance via its no-change fast path, reporting 152 B and hiding the second allocation the method really makes. Co-Authored-By: Claude Opus 5 (1M context) --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 260 +++++++++++++++++++ src/Arius.Benchmarks/BenchmarkRunOptions.cs | 38 ++- src/Arius.Benchmarks/Program.cs | 27 +- 3 files changed, 321 insertions(+), 4 deletions(-) create mode 100644 src/Arius.Benchmarks/AllocationBenchmarks.cs diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs new file mode 100644 index 000000000..02787f088 --- /dev/null +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -0,0 +1,260 @@ +using Arius.Core.Features.ArchiveCommand; +using Arius.Core.Shared.ChunkIndex; +using Arius.Core.Shared.Encryption; +using Arius.Core.Shared.FileSystem; +using Arius.Core.Shared.FileTree; +using Arius.Core.Shared.HashCache; +using Arius.Core.Shared.Hashes; +using Arius.Core.Shared.Storage; +using Arius.Tests.Shared; +using BenchmarkDotNet.Attributes; + +namespace Arius.Benchmarks; + +/// +/// In-process allocation micro-benchmarks for the byte-pushing components of Arius.Core. +/// +/// These exist because is end-to-end against Azurite, where fixture +/// and SDK overhead dominate and no single optimization is separable. Each benchmark here isolates one +/// component so a change's effect on Allocated is directly attributable. +/// +/// Needs no Azurite and no Docker. Run with --class micro. +/// +[MemoryDiagnoser] +public class AllocationBenchmarks +{ + private const int EntryCount = 1_000; + + // ── Hash codec ─────────────────────────────────────────────────────────────── + + /// + /// A digest whose hex form actually contains a-f nibbles. This matters: a digest of all-zero + /// bytes hexes to "000...0", and then returns the same instance via its + /// no-change fast path — hiding the second allocation that ToLowerHex really makes on realistic input. + /// Deliberately not , which pins a leading 0x00 for the range-query fixtures. + /// + private readonly byte[] _digest = CreateHighNibbleDigest(); + + private const string CanonicalHex = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + private const string UppercaseHex = "00112233445566778899AABBCCDDEEFF00112233445566778899AABBCCDDEEFF"; + + [Benchmark(Description = "HashCodec.ToLowerHex")] + public string HashCodec_ToLowerHex() => HashCodec.ToLowerHex(_digest); + + /// The common case: parsing a value Arius itself wrote, already canonical lowercase. + [Benchmark(Description = "HashCodec.NormalizeHex (already canonical)")] + public string HashCodec_NormalizeHex_Canonical() => HashCodec.NormalizeHex(CanonicalHex); + + [Benchmark(Description = "HashCodec.NormalizeHex (uppercase input)")] + public string HashCodec_NormalizeHex_Uppercase() => HashCodec.NormalizeHex(UppercaseHex); + + // ── Sparse fingerprint ─────────────────────────────────────────────────────── + + private const long SmallFileSize = 200L * 1024; // one whole-file region + private const long LargeFileSize = 64L * 1024 * 1024 * 1024; // k = MaxBlocks = 64 regions + + private byte[] _readBuffer = null!; + + /// + /// The dominant real-world shape: a sub-1 MiB file, where Regions returns a single region + /// equal to the whole file, so the sampler buffers the entire file while it is hashed. + /// + [Benchmark(Description = "SparseFingerprint.Sampler small file (200 KB)")] + public byte[] SparseFingerprint_Sampler_SmallFile() + { + var sampler = new SparseFingerprint.Sampler(SmallFileSize); + var position = 0L; + + while (position < SmallFileSize) + { + var length = (int)Math.Min(_readBuffer.Length, SmallFileSize - position); + sampler.Capture(position, _readBuffer.AsSpan(0, length)); + position += length; + } + + return sampler.Finish(); + } + + /// Worst case for the capture buffers: 64 regions x 256 KiB = 16 MiB per in-flight file. + [Benchmark(Description = "SparseFingerprint.Sampler large file (64 GB logical)")] + public byte[] SparseFingerprint_Sampler_LargeFile() + => new SparseFingerprint.Sampler(LargeFileSize).Finish(); + + // ── Filetree serialization ─────────────────────────────────────────────────── + + private IReadOnlyList _fileTreeEntries = null!; + private byte[] _fileTreeBytes = null!; + + [Benchmark(Description = "FileTreeSerializer.Serialize (1000 entries)")] + public byte[] FileTreeSerializer_Serialize() => FileTreeSerializer.Serialize(_fileTreeEntries); + + [Benchmark(Description = "FileTreeSerializer.Deserialize (1000 entries)")] + public IReadOnlyList FileTreeSerializer_Deserialize() => FileTreeSerializer.Deserialize(_fileTreeBytes); + + // ── Tar builder ────────────────────────────────────────────────────────────── + + private const long TarTargetSize = 64L * 1024 * 1024; + private const int TarEntrySize = 64 * 1024; + + private byte[] _tarEntryPayload = null!; + private IEncryptionService _encryption = null!; + + /// + /// Accumulates and seals one full 64 MB bundle. The per-entry wrappers are + /// ~100 bytes each and negligible against the bundle buffer this is measuring. + /// + [Benchmark(Description = "TarBuilder seal one 64 MB bundle")] + public async Task TarBuilder_Seal_64MB() + { + await using var builder = new TarBuilder(TarTargetSize, _encryption); + + var sealedCount = 0; + for (var i = 0; i < TarTargetSize / TarEntrySize; i++) + { + var source = new MemoryStream(_tarEntryPayload, writable: false); + if (await builder.AddAsync(CreateUpload(i, TarEntrySize), source, CancellationToken.None) is not null) + sealedCount++; + } + + return sealedCount; + } + + // ── Chunk-index local store ────────────────────────────────────────────────── + + private LocalDirectory _storeRoot = default; + private ChunkIndexLocalStore _store = null!; + private ShardEntry[] _shardEntries = null!; + private ContentHash[] _lookupHashes = null!; + private PathSegment _rangePrefix = default; + + [Benchmark(Description = "ChunkIndexLocalStore.UpsertRemoteBacked (1000 rows)")] + public void ChunkIndexLocalStore_UpsertRemoteBacked() => _store.UpsertRemoteBacked(_shardEntries); + + [Benchmark(Description = "ChunkIndexLocalStore.ReadRangeEntries (1000 rows)")] + public int ChunkIndexLocalStore_ReadRangeEntries() + { + var count = 0; + _store.ReadRangeEntries(_rangePrefix, _ => count++); + return count; + } + + /// + /// The per-hash lookup shape that ChunkIndexService.LookupAsync runs 256 times per "batch", + /// twice over (pending-flush probe then entry probe). + /// + [Benchmark(Description = "ChunkIndexLocalStore.FindEntry x256 (one dedup batch)")] + public int ChunkIndexLocalStore_FindEntry_256() + { + var found = 0; + foreach (var hash in _lookupHashes) + if (_store.FindEntry(hash) is not null) + found++; + + return found; + } + + // ── Setup ──────────────────────────────────────────────────────────────────── + + [GlobalSetup] + public void Setup() + { + _readBuffer = new byte[81920]; + _tarEntryPayload = new byte[TarEntrySize]; + Random.Shared.NextBytes(_readBuffer); + Random.Shared.NextBytes(_tarEntryPayload); + + _encryption = IEncryptionService.EncryptedInstance; + + _fileTreeEntries = BuildFileTreeEntries(EntryCount); + _fileTreeBytes = FileTreeSerializer.Serialize(_fileTreeEntries); + + _shardEntries = BuildShardEntries(EntryCount); + _lookupHashes = _shardEntries.Take(256).Select(e => e.ContentHash).ToArray(); + _rangePrefix = PathSegment.Parse("00"); + + _storeRoot = TestTempRoots.CreateDirectory("benchmark-chunkindex"); + _store = new ChunkIndexLocalStore(_storeRoot); + _store.UpsertRemoteBacked(_shardEntries); + } + + [GlobalCleanup] + public void Cleanup() + { + try + { + RelativeFileSystem.DeleteDirectory(_storeRoot, RelativePath.Root, recursive: true); + } + catch (IOException) + { + // Best effort: the SQLite connection pool may still hold the file. TestTempRoots sweeps stale dirs. + } + } + + // ── Deterministic fixtures ─────────────────────────────────────────────────── + + /// + /// A distinct 32-byte digest per index, always with a leading 0x00 so every generated hash falls + /// under the "00" prefix that ranges over. + /// + private static byte[] CreateDigest(int seed) + { + var digest = new byte[32]; + BitConverter.TryWriteBytes(digest.AsSpan(1), seed); + return digest; + } + + private static ContentHash CreateContentHash(int seed) => ContentHash.FromDigest(CreateDigest(seed)); + + /// A deterministic 32-byte digest whose every byte has a high nibble in the a-f range. + private static byte[] CreateHighNibbleDigest() + { + var digest = new byte[32]; + for (var i = 0; i < digest.Length; i++) + digest[i] = (byte)(0xA0 | (i & 0x0F)); + + return digest; + } + + private static IReadOnlyList BuildFileTreeEntries(int count) + { + var entries = new List(count); + var created = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + for (var i = 0; i < count; i++) + { + entries.Add(new FileEntry + { + Name = PathSegment.Parse($"file-{i:D6}.bin"), + ContentHash = CreateContentHash(i), + Created = created.AddSeconds(i), + Modified = created.AddSeconds(i * 2), + }); + } + + return entries; + } + + private static ShardEntry[] BuildShardEntries(int count) + { + var entries = new ShardEntry[count]; + for (var i = 0; i < count; i++) + { + var contentHash = CreateContentHash(i); + entries[i] = new ShardEntry( + ContentHash: contentHash, + ChunkHash: ChunkHash.Parse(contentHash), // large chunk: chunk hash == content hash + OriginalSize: 4096 + i, + ChunkSize: 2048 + i, + StorageTierHint: BlobTier.Cool); + } + + return entries; + } + + private static FileToUpload CreateUpload(int seed, long size) + { + var filePair = new FilePair { RelativePath = RelativePath.Parse($"file-{seed:D6}.bin") }; + var hashed = new HashedFilePair(filePair, CreateContentHash(seed), DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch); + return new FileToUpload(hashed, size); + } +} diff --git a/src/Arius.Benchmarks/BenchmarkRunOptions.cs b/src/Arius.Benchmarks/BenchmarkRunOptions.cs index 9d22312de..82d62cbb4 100644 --- a/src/Arius.Benchmarks/BenchmarkRunOptions.cs +++ b/src/Arius.Benchmarks/BenchmarkRunOptions.cs @@ -1,9 +1,21 @@ namespace Arius.Benchmarks; +/// Which benchmark class to run. +internal enum BenchmarkClass +{ + /// The end-to-end archive step against Azurite. Appends to the benchmark tail log. + Archive, + + /// In-process allocation micro-benchmarks. No Docker; does not touch the tail log. + Micro, +} + internal sealed record BenchmarkRunOptions( string RepositoryRoot, string RawOutputRoot, - string TailLogPath) + string TailLogPath, + BenchmarkClass Class, + string? Filter) { public const int Iterations = 3; @@ -15,6 +27,8 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) var rawOutputRoot = defaultRawOutputRoot; var tailLogPath = Path.Combine(defaultBenchmarkRoot, "benchmark-tail.md"); + var benchmarkClass = BenchmarkClass.Archive; + string? filter = null; for (var i = 0; i < args.Count; i++) { @@ -26,6 +40,12 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) case "--tail-log": tailLogPath = RequireValue(args, ref i, "--tail-log"); break; + case "--class": + benchmarkClass = ParseClass(RequireValue(args, ref i, "--class")); + break; + case "--filter": + filter = RequireValue(args, ref i, "--filter"); + break; case "--help" or "-h": PrintHelp(); Environment.Exit(0); @@ -38,9 +58,18 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) return new( repositoryRoot, Path.GetFullPath(rawOutputRoot), - Path.GetFullPath(tailLogPath)); + Path.GetFullPath(tailLogPath), + benchmarkClass, + filter); } + static BenchmarkClass ParseClass(string value) => value.ToLowerInvariant() switch + { + "archive" => BenchmarkClass.Archive, + "micro" => BenchmarkClass.Micro, + _ => throw new ArgumentException($"Unknown benchmark class '{value}'. Expected 'archive' or 'micro'."), + }; + static string RequireValue(IReadOnlyList args, ref int index, string optionName) { if (index + 1 >= args.Count) @@ -73,7 +102,10 @@ static void PrintHelp() Console.WriteLine("Runs the canonical representative workflow benchmark on Azurite."); Console.WriteLine(); Console.WriteLine("Options:"); + Console.WriteLine(" --class 'archive' (default, end-to-end on Azurite; needs Docker)"); + Console.WriteLine(" or 'micro' (in-process allocation benchmarks)."); + Console.WriteLine(" --filter Run only matching benchmarks, e.g. '*TarBuilder*' (micro only)."); Console.WriteLine(" --raw-output Folder where per-run raw BenchmarkDotNet output is saved."); - Console.WriteLine(" --tail-log Markdown benchmark tail log to append to."); + Console.WriteLine(" --tail-log Markdown benchmark tail log to append to (archive only)."); } } diff --git a/src/Arius.Benchmarks/Program.cs b/src/Arius.Benchmarks/Program.cs index dbbfc1a20..c0adef7cf 100644 --- a/src/Arius.Benchmarks/Program.cs +++ b/src/Arius.Benchmarks/Program.cs @@ -1,6 +1,7 @@ using Arius.Benchmarks; using Arius.E2E.Tests.Datasets; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; @@ -12,6 +13,30 @@ Directory.CreateDirectory(rawOutputDirectory); Directory.CreateDirectory(Path.GetDirectoryName(options.TailLogPath)!); +var logger = new StreamLogger(Path.Combine(rawOutputDirectory, "benchmark-output.log"), append: false); + +if (options.Class is BenchmarkClass.Micro) +{ + // Micro-benchmarks need BenchmarkDotNet's normal warmup/invocation defaults to produce meaningful + // per-operation numbers, so they deliberately do NOT use the single-invocation job below. They also + // do not append to the tail log: its schema (RepresentativeScaleDivisor, ...) is archive-specific. + var microConfig = ManualConfig + .Create(DefaultConfig.Instance) + .AddLogger(logger) + .WithArtifactsPath(rawOutputDirectory); + + // --filter lets a single optimization be re-measured on its own benchmark rather than re-running + // the whole suite after every change. + if (options.Filter is { } filter) + microConfig = microConfig.AddFilter(new GlobFilter([filter])); + + BenchmarkRunner.Run(microConfig); + + return; +} + +// The archive step runs for tens of seconds and mutates real fixtures per iteration, so it is measured +// with one invocation per iteration and no warmup. var config = ManualConfig .Create(DefaultConfig.Instance) .AddJob(Job.Default @@ -20,7 +45,7 @@ .WithIterationCount(BenchmarkRunOptions.Iterations) .WithInvocationCount(1) .WithUnrollFactor(1)) - .AddLogger(new StreamLogger(Path.Combine(rawOutputDirectory, "benchmark-output.log"), append: false)) + .AddLogger(logger) .WithArtifactsPath(rawOutputDirectory); var summary = BenchmarkRunner.Run(config); From eef3d24f161ae4e6eb8c814a4d969fa78c7972c1 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:16:18 +0200 Subject: [PATCH 02/46] perf(archive): pre-size the tar bundle buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TarBuilder opened each bundle with `new MemoryStream()` — capacity zero — then accumulated up to TarTargetSize (64 MB default). MemoryStream grows by doubling, so reaching a 64 MB bundle allocated and abandoned every intermediate array (256 B, 512 B, ... 32 MB, 64 MB, 128 MB). Measured: 276 MB of transient garbage to seal one 64 MB bundle, nearly all of it on the LOH. This is the source of the Gen2 collections that show up in ArchiveStepBenchmarks at divisor=1. Pre-size the buffer instead. The capacity includes 25% headroom for tar framing: a bundle seals on the summed *file* sizes, but the stream also carries two 512-byte header blocks plus an optional PAX extended block per entry, so the serialized tar always runs over the target — presizing to exactly TarTargetSize still took one growth step (measured 193.75 MB). A bundle of very many very small files can still exceed the headroom and grow once; that is accepted rather than worked around, since the entry count is not known up front. AllocationBenchmarks, TarBuilder seal one 64 MB bundle: before 276,498,485 B (263.7 MB) 46.92 ms Gen2 1666 after 85,720,268 B ( 81.8 MB) 28.46 ms Gen2 562 3.4x less allocated, 39% faster. Live footprint is unchanged: SealedTar.Content already retained an oversized backing array (128 MB post-doubling), and now retains an 80 MB one. Not pooling the buffer here on purpose: SealedTar.Content is an ArraySegment that outlives the MemoryStream and escapes through sealedTarChannel to the tar upload stage, so pooling needs a defined return point. Presize first, measure, then decide. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped (includes TarBuilderTests). Co-Authored-By: Claude Opus 5 (1M context) --- .../Features/ArchiveCommand/TarBuilder.cs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs index dea243497..120a30d09 100644 --- a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs +++ b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs @@ -31,6 +31,22 @@ internal sealed class TarBuilder : IAsyncDisposable private MemoryStream? _tarStream; private long _currentSize; + /// + /// Headroom over the target size for tar framing. A bundle seals on the summed *file* sizes, but the + /// stream also holds two 512-byte header blocks plus an optional PAX extended block per entry, so the + /// serialized tar always runs over the target. Without headroom the buffer grows once more, doubling a + /// 64 MB allocation into a 128 MB one. + /// + private const int FramingHeadroomDivisor = 4; + + /// + /// Initial capacity for a bundle's backing buffer. is a and + /// takes an , so it is clamped; the clamp only engages for a + /// TarTargetSize far beyond any practical bundle. A bundle of very many very small files can still + /// exceed even the headroom and take one growth step — that is accepted, not worked around. + /// + private int InitialCapacity => (int)Math.Min(_targetSize + _targetSize / FramingHeadroomDivisor, int.MaxValue / 2); + /// A bundle is sealed once its accumulated size reaches this threshold. /// Used to hash the sealed tar body. /// Invoked when a new bundle is opened (its first entry). @@ -61,7 +77,10 @@ public TarBuilder( TarWriter writer; if (_tarWriter is null) { - _tarStream = new MemoryStream(); + // Pre-size the bundle buffer. Growing from zero capacity allocates and abandons every + // intermediate array (256 B, 512 B, ... 32 MB, 64 MB, 128 MB) to reach one 64 MB bundle — + // measured at 276 MB of transient garbage per bundle, nearly all of it on the LOH. + _tarStream = new MemoryStream(InitialCapacity); writer = _tarWriter = new TarWriter(_tarStream, leaveOpen: true); await (_onBundleStarted?.Invoke() ?? ValueTask.CompletedTask); } From 52fcbb70f2b4809d96d613ae1d32c0240604dfa6 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:19:13 +0200 Subject: [PATCH 03/46] perf(hashcache): pool the sparse-fingerprint capture buffers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SparseFingerprint.Sampler allocated one byte[] per region, eagerly, in its constructor — once per file. Regions() returns a *single region equal to the whole file* for anything under ~1 MiB (size <= k*BlockSize), and files under 1 MiB are exactly the small/tar route, i.e. the bulk of files. So the sampler duplicated every small file in memory while it was hashed, and TarBuilder then buffered the same bytes again. For a large file it is up to 64 x 256 KiB = 16 MiB per in-flight file, times HashWorkers=4, all on the LOH. This runs on *every* full hash, not just --fast-hash: Record is always called so that a later --fast-hash run finds a warm cache. Lay the regions back-to-back in one ArrayPool buffer and make Sampler IDisposable, returning it from SparseSamplingStream.Dispose. The lifetime was already clean — SparseSamplingStream is `await using`-scoped per file and Fingerprint() is called before scope exit, and Finish() returns a fresh 32-byte digest rather than the capture buffer. Two details that keep the digest a function of the content: - The rented buffer is cleared. A rented array arrives dirty, and a region never offered to Capture (a file that shrank mid-read, or a read that stopped early) must contribute zeros exactly as the previous `new byte[]` did. - Finish() hashes the whole span in one AppendData call. The regions are contiguous and in order, so this is byte-identical to appending each region individually, which is what the framing contract with ComputeBySeeking needs. Also pools ComputeBySeeking's read buffer (up to 1 MiB per fingerprint-floor check) and replaces BitConverter.GetBytes(size) with an explicit little-endian write. BitConverter is platform-endian; the design doc already specifies "size as 8-byte LE", so this makes the documented framing explicit rather than incidental. No supported platform is big-endian, and the hashcache is a disposable local cache regardless. AllocationBenchmarks: Sampler small file (200 KB) 205,256 B -> 304 B 88.0 -> 68.1 us Sampler large file (64 GB) 16,787,173 B -> 1,570 B 6270 -> 5258 us Gen0/Gen1/Gen2 all drop to zero on both. What remains is the Regions list, the offsets array, and the returned digest. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped, including SparseFingerprintTests.Sampler_MatchesSeekingFingerprint_ForSameContent, which is the guard that both compute paths still produce identical digests. Co-Authored-By: Claude Opus 5 (1M context) --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 9 +- .../HashCache/SparseFingerprintTests.cs | 4 +- .../Shared/HashCache/SparseFingerprint.cs | 93 +++++++++++++++---- .../Shared/HashCache/SparseSamplingStream.cs | 11 ++- 4 files changed, 93 insertions(+), 24 deletions(-) diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index 02787f088..25b49089c 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -62,8 +62,8 @@ public class AllocationBenchmarks [Benchmark(Description = "SparseFingerprint.Sampler small file (200 KB)")] public byte[] SparseFingerprint_Sampler_SmallFile() { - var sampler = new SparseFingerprint.Sampler(SmallFileSize); - var position = 0L; + using var sampler = new SparseFingerprint.Sampler(SmallFileSize); + var position = 0L; while (position < SmallFileSize) { @@ -78,7 +78,10 @@ public byte[] SparseFingerprint_Sampler_SmallFile() /// Worst case for the capture buffers: 64 regions x 256 KiB = 16 MiB per in-flight file. [Benchmark(Description = "SparseFingerprint.Sampler large file (64 GB logical)")] public byte[] SparseFingerprint_Sampler_LargeFile() - => new SparseFingerprint.Sampler(LargeFileSize).Finish(); + { + using var sampler = new SparseFingerprint.Sampler(LargeFileSize); + return sampler.Finish(); + } // ── Filetree serialization ─────────────────────────────────────────────────── diff --git a/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs b/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs index 059a04d4f..0ee8aad3f 100644 --- a/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs +++ b/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs @@ -61,7 +61,7 @@ public void ComputeBySeeking_WholeFileRegion_LargerThanBlockSize_DoesNotThrow() fp.Length.ShouldBe(32); // And it must still agree with the streaming Sampler over the same content. - var sampler = new SparseFingerprint.Sampler(size); + using var sampler = new SparseFingerprint.Sampler(size); var pos = 0; const int chunk = 64 * 1024; while (pos < data.Length) @@ -83,7 +83,7 @@ public void Sampler_MatchesSeekingFingerprint_ForSameContent() var seekFp = SparseFingerprint.ComputeBySeeking(fs, path, size); // Drive the sampler the way a sequential read would. - var sampler = new SparseFingerprint.Sampler(size); + using var sampler = new SparseFingerprint.Sampler(size); var pos = 0; const int chunk = 64 * 1024; while (pos < data.Length) diff --git a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs index 95d139f95..3afa8ba8a 100644 --- a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs +++ b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs @@ -1,3 +1,5 @@ +using System.Buffers; +using System.Buffers.Binary; using System.Security.Cryptography; namespace Arius.Core.Shared.HashCache; @@ -48,21 +50,36 @@ internal static class SparseFingerprint public static byte[] ComputeBySeeking(RelativeFileSystem fs, RelativePath path, long size) { using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); - sha.AppendData(BitConverter.GetBytes(size)); + AppendSize(sha, size); using var stream = fs.OpenRead(path); var regions = Regions(size); + if (regions.Count == 0) + return sha.GetHashAndReset(); + // Buffer the largest region: the single whole-file region for a small file can reach k×BlockSize // (up to 1 MiB at k=MinBlocks), which is larger than BlockSize — a fixed BlockSize buffer would // overflow ReadExactly for files in (BlockSize, k×BlockSize]. - var buffer = new byte[regions.Count == 0 ? 0 : regions.Max(r => r.Length)]; - foreach (var (offset, length) in regions) + var longest = 0; + for (var i = 0; i < regions.Count; i++) + longest = Math.Max(longest, regions[i].Length); + + var buffer = ArrayPool.Shared.Rent(longest); + try { - stream.Seek(offset, SeekOrigin.Begin); - stream.ReadExactly(buffer, 0, length); - sha.AppendData(buffer, 0, length); + foreach (var (offset, length) in regions) + { + stream.Seek(offset, SeekOrigin.Begin); + stream.ReadExactly(buffer, 0, length); + sha.AppendData(buffer, 0, length); + } + + return sha.GetHashAndReset(); + } + finally + { + ArrayPool.Shared.Return(buffer); } - return sha.GetHashAndReset(); } /// @@ -72,17 +89,42 @@ public static byte[] ComputeBySeeking(RelativeFileSystem fs, RelativePath path, /// for the same content (same , same size ‖ region-bytes framing) — keep /// the two in sync. /// - public sealed class Sampler + public sealed class Sampler : IDisposable { private readonly long _size; private readonly IReadOnlyList<(long Off, int Len)> _regions; - private readonly byte[][] _captured; + + /// + /// The regions laid out back-to-back in one pooled buffer, so can hash the whole + /// span in one call — byte-identical to hashing each region in order, which is what the framing + /// contract with requires. + /// + private readonly byte[] _buffer; + private readonly int[] _bufferOffsets; + private readonly int _capturedLength; + + private bool _disposed; public Sampler(long size) { - _size = size; - _regions = Regions(size); - _captured = _regions.Select(r => new byte[r.Len]).ToArray(); + _size = size; + _regions = Regions(size); + + _bufferOffsets = new int[_regions.Count]; + var total = 0; + for (var i = 0; i < _regions.Count; i++) + { + _bufferOffsets[i] = total; + total += _regions[i].Len; + } + + _capturedLength = total; + _buffer = ArrayPool.Shared.Rent(total); + + // A rented buffer arrives dirty. A region that is never offered to Capture — a file that shrank + // mid-read, or a read that stopped early — must contribute zeros, exactly as the previous + // per-region `new byte[]` did, or the fingerprint stops being a function of the content. + _buffer.AsSpan(0, _capturedLength).Clear(); } /// Offer the bytes read at ; overlapping region bytes are copied out. @@ -97,17 +139,36 @@ public void Capture(long position, ReadOnlySpan buffer) continue; var srcStart = (int)(from - position); var dstStart = (int)(from - off); - buffer.Slice(srcStart, (int)(to - from)).CopyTo(_captured[i].AsSpan(dstStart)); + buffer.Slice(srcStart, (int)(to - from)).CopyTo(_buffer.AsSpan(_bufferOffsets[i] + dstStart)); } } public byte[] Finish() { using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); - sha.AppendData(BitConverter.GetBytes(_size)); - foreach (var region in _captured) - sha.AppendData(region); + AppendSize(sha, _size); + sha.AppendData(_buffer.AsSpan(0, _capturedLength)); return sha.GetHashAndReset(); } + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + ArrayPool.Shared.Return(_buffer); + } + } + + /// + /// Frames the file size into the digest as 8 little-endian bytes. Both compute paths must agree on this + /// exactly — it is the size ‖ region-bytes prefix. + /// + private static void AppendSize(IncrementalHash sha, long size) + { + Span sizeBytes = stackalloc byte[sizeof(long)]; + BinaryPrimitives.WriteInt64LittleEndian(sizeBytes, size); + sha.AppendData(sizeBytes); } } diff --git a/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs b/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs index 5845a4140..23dea8831 100644 --- a/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs +++ b/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs @@ -63,9 +63,14 @@ public override long Position protected override void Dispose(bool disposing) { - if (disposing) - _inner.Dispose(); - + if (disposing) + { + // Returns the sampler's pooled capture buffer. Fingerprint() must therefore be called before + // this stream is disposed — which is the existing call order in ArchiveCommandHandler. + _sampler.Dispose(); + _inner.Dispose(); + } + base.Dispose(disposing); } } From 55b9e180b28637ab5119cc6e4902e15fbb07bf1f Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:22:05 +0200 Subject: [PATCH 04/46] perf(hashes): halve the allocations in the hash codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HashCodec is the funnel every ContentHash/ChunkHash/FileTreeHash construction passes through, so both of these are per-hash costs. ToLowerHex did Convert.ToHexString(digest).ToLowerInvariant() — allocating the uppercase string and then a second lowercased copy. Convert.ToHexStringLower (.NET 9+; this targets net10.0) does it in one vectorized pass. NormalizeHex always built a new string from its stackalloc scratch buffer, even when the input was already canonical lowercase — which is the dominant case, since it parses values Arius itself wrote to SQLite, snapshot JSON, blob names, and pointer files. Track whether any character actually changed and return the input when none did. Every character is still validated for length and alphabet, and the FormatException messages are unchanged. Returning the caller's string is safe: .NET strings are immutable and standalone (no substring buffer sharing, so nothing large is retained), and hash equality is by value, not reference. AllocationBenchmarks: ToLowerHex 304 B -> 152 B 32.53 -> 9.14 ns NormalizeHex (already canonical) 152 B -> 0 B 42.46 -> 27.09 ns NormalizeHex (uppercase input) 152 B -> 152 B 53.30 -> 43.94 ns The uppercase case still allocates, correctly — it has to build a new string. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped. Co-Authored-By: Claude Opus 5 (1M context) --- src/Arius.Core/Shared/Hashes/HashCodec.cs | 29 ++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Arius.Core/Shared/Hashes/HashCodec.cs b/src/Arius.Core/Shared/Hashes/HashCodec.cs index 7e7e286b5..486008245 100644 --- a/src/Arius.Core/Shared/Hashes/HashCodec.cs +++ b/src/Arius.Core/Shared/Hashes/HashCodec.cs @@ -17,19 +17,29 @@ public static string NormalizeHex(string value) throw new FormatException($"Expected {Sha256HexLength} hex characters but got {value.Length}."); Span chars = stackalloc char[Sha256HexLength]; + var alreadyCanonical = true; for (var i = 0; i < value.Length; i++) { var c = value[i]; - chars[i] = c switch + switch (c) { - >= '0' and <= '9' => c, - >= 'a' and <= 'f' => c, - >= 'A' and <= 'F' => char.ToLowerInvariant(c), - _ => throw new FormatException($"Invalid hex character '{c}'.") - }; + case >= '0' and <= '9': + case >= 'a' and <= 'f': + chars[i] = c; + break; + case >= 'A' and <= 'F': + chars[i] = char.ToLowerInvariant(c); + alreadyCanonical = false; + break; + default: + throw new FormatException($"Invalid hex character '{c}'."); + } } - return new string(chars); + // The dominant case is re-parsing a value Arius itself wrote (SQLite, snapshot JSON, blob names, + // pointer files), which is already canonical lowercase. Returning the input then avoids allocating + // a second identical string. Validation above has still run over every character. + return alreadyCanonical ? value : new string(chars); } public static string ToLowerHex(ReadOnlySpan digest) @@ -37,6 +47,9 @@ public static string ToLowerHex(ReadOnlySpan digest) if (digest.Length != Sha256ByteLength) throw new ArgumentException($"Expected {Sha256ByteLength}-byte SHA-256 digest.", nameof(digest)); - return Convert.ToHexString(digest).ToLowerInvariant(); + // Not Convert.ToHexString(...).ToLowerInvariant(): that allocates the uppercase string and then a + // second lowercased copy. This is the funnel every ContentHash/ChunkHash/FileTreeHash construction + // passes through. + return Convert.ToHexStringLower(digest); } } From 0ae0798cca2a0949e646ffae2f0f3dbefdcb62bf Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:24:25 +0200 Subject: [PATCH 05/46] perf(archive): stat each file once instead of four times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArchiveCommandHandler called fs.GetFileSize for the same file at four points in one run: the enumerate stage, the hash stage, and both branches of the dedup router. Each call is `new FileInfo(root.Resolve(path)).Length` — a FileInfo allocation, a stat syscall, and ~4 strings from LocalDirectory.Resolve. Carry the size on BinaryFile instead, captured once by LocalFileEnumerator when the entry is discovered. This extends an invariant the pipeline already has rather than inventing one: HashedFilePair captures the source timestamps at hash time for exactly this reason, documented as "no downstream stage re-reads file metadata". Net effect per archived file: 4 stat syscalls and 4 path resolutions become 1. The enumerate and hash sites use `Binary?.FileSize ?? 0L` because pointer-only pairs legitimately reach both and have no binary. The two dedup sites use `Binary!.FileSize`: the `Binary is null` branch above them has already continued, so zero is not a reachable outcome there and `?? 0L` would imply otherwise. (The old code called GetFileSize unconditionally at those two sites, which would have thrown FileNotFoundException on a pointer-only pair had that guard not existed.) Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped dotnet test src/Arius.Integration.Tests — 82 passed, 4 skipped (Azurite) No micro-benchmark for this one: it removes syscalls, not allocations, so it shows up in ArchiveStepBenchmarks rather than AllocationBenchmarks. Co-Authored-By: Claude Opus 5 (1M context) --- .../Features/ArchiveCommand/ArchiveCommandHandler.cs | 8 ++++---- .../Features/ArchiveCommand/LocalFileEnumerator.cs | 1 + src/Arius.Core/Features/ArchiveCommand/Models.cs | 8 ++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs b/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs index f9c36df9c..5cca55704 100644 --- a/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs @@ -313,7 +313,7 @@ public async ValueTask Handle(ArchiveCommand command, Cancellatio await foreach (var pair in pairs) { count++; - var fileSize = pair.Binary is null ? 0L : fs.GetFileSize(pair.RelativePath); + var fileSize = pair.Binary?.FileSize ?? 0L; totalBytes += fileSize; await _mediator.Publish(new FileScannedEvent(pair.RelativePath, fileSize), cancellationToken); await filePairChannel.Writer.WriteAsync(pair, cancellationToken); @@ -342,7 +342,7 @@ await Parallel.ForEachAsync( { try { - var fileSize = pair.Binary is null ? 0L : fs.GetFileSize(pair.RelativePath); + var fileSize = pair.Binary?.FileSize ?? 0L; await _mediator.Publish(new FileHashingEvent(pair.RelativePath, fileSize), ct); @@ -491,14 +491,14 @@ async ValueTask FullHashAndRecordAsync(RelativePath relativePath, l _logger.LogInformation("[dedup] {Path} -> hit ({Hash})", hashed.FilePair.RelativePath, hashed.ContentHash.Short8); await fileTreeEntryChannel.Writer.WriteAsync(hashed, cancellationToken); Interlocked.Increment(ref filesDeduped); - var size = fs.GetFileSize(hashed.FilePair.RelativePath); + var size = hashed.FilePair.Binary!.FileSize; Interlocked.Add(ref originalSize, size); await _mediator.Publish(new FileDedupedEvent(hashed.ContentHash, size), cancellationToken); } else { // Needs upload → mark in-flight, route by size - var fileSize = fs.GetFileSize(hashed.FilePair.RelativePath); + var fileSize = hashed.FilePair.Binary!.FileSize; inFlightHashes.TryAdd(hashed.ContentHash, fileSize); Interlocked.Add(ref originalSize, fileSize); Interlocked.Add(ref incrementalSize, fileSize); diff --git a/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs b/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs index 0867e1d9b..48611b359 100644 --- a/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs +++ b/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs @@ -114,6 +114,7 @@ private async IAsyncEnumerable EnumerateDirectoryAsync(RelativeFileSys Binary = new BinaryFile { Path = relativePath, + FileSize = fileSystem.GetFileSize(relativePath) }, Pointer = hasPointer ? new PointerFile diff --git a/src/Arius.Core/Features/ArchiveCommand/Models.cs b/src/Arius.Core/Features/ArchiveCommand/Models.cs index 0862a5d3e..8553baa6c 100644 --- a/src/Arius.Core/Features/ArchiveCommand/Models.cs +++ b/src/Arius.Core/Features/ArchiveCommand/Models.cs @@ -27,6 +27,14 @@ internal sealed record FilePair internal sealed record BinaryFile { public required RelativePath Path { get; init; } + + /// + /// Size in bytes, captured once by when the entry is discovered. + /// Carried here for the same reason carries the timestamps: so no later + /// stage re-stats the file. Enumerate, hash, and dedup all need the size, and each calling + /// GetFileSize meant four stat syscalls (and four path resolutions) per file. + /// + public required long FileSize { get; init; } } /// From f0e7b6990526b1a5b18ed171c78e5e819801ffda Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:29:46 +0200 Subject: [PATCH 06/46] perf(filetree): drop two per-node payload copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two full copies of every filetree node, on paths that run once per directory — thousands of times per run, with SynchronizeWorkers=32 in flight: - DeserializeStorageAsync copied the decompressed node out of its MemoryStream (ms.ToArray()) purely to hand it to FileTreeSerializer.Deserialize. Widen Deserialize to ReadOnlySpan and pass the stream's own buffer via the existing StreamExtensions.ToArraySegment(). This is the read path behind ls, restore, and the archive tree build. - WriteCacheAtomicallyAsync called plaintext.ToArray() on a ReadOnlyMemory only because RelativeFileSystem.WriteAllBytesAsync took a byte[]. Add a ReadOnlyMemory overload beside the array one, mirroring File.WriteAllBytesAsync which offers both, so no existing call site changes. SerializeStorageAsync deliberately keeps its ToArray(). Disposing the encryption/compression chain also closes the underlying MemoryStream, and ToArray() is the only buffer accessor that stays valid after close — Length and TryGetBuffer both throw ObjectDisposedException. The test suite caught this: switching it produced 66 failures with "Cannot access a closed Stream". Removing that copy needs a leaveOpen on IEncryptionService.WrapForEncryption or a non-closing stream shim, which is a wider change than this warrants. ShardSerializer has the same ToArray() shape and is also left alone: the shard payload is the smallest of the three (~100 KB, once per run at flush), and changing its return type would ripple through ~35 test call sites plus two fake blob-service signatures. No micro-benchmark moves for this one — the copies are in FileTreeService, which AllocationBenchmarks does not cover, so it is verified by the test suite and by inspection, and will show up in ArchiveStepBenchmarks. (The FileTreeSerializer rows did improve, 1.68 -> 1.53 MB and 22% faster, but that is step 3's NormalizeHex change becoming visible — ~1000 entries x 152 B — not this one; the baseline predates it.) Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped. Co-Authored-By: Claude Opus 5 (1M context) --- .../Shared/FileSystem/RelativeFileSystem.cs | 11 +++++++++++ src/Arius.Core/Shared/FileTree/FileTreeSerializer.cs | 4 +--- src/Arius.Core/Shared/FileTree/FileTreeService.cs | 10 ++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs index 57d0a425b..ac13532ea 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs @@ -273,6 +273,17 @@ public async Task WriteAllBytesAsync(RelativePath path, byte[] content, Cancella await File.WriteAllBytesAsync(fullPath, content, cancellationToken); } + /// + /// Writes to without copying it to an array first. + /// Mirrors . + /// + public async Task WriteAllBytesAsync(RelativePath path, ReadOnlyMemory content, CancellationToken cancellationToken) + { + var fullPath = root.Resolve(path); + CreateDirectory(path.Parent ?? RelativePath.Root); + await File.WriteAllBytesAsync(fullPath, content, cancellationToken); + } + public void ReplaceFileAtomically(RelativePath source, RelativePath destination) { var sourcePath = root.Resolve(source); diff --git a/src/Arius.Core/Shared/FileTree/FileTreeSerializer.cs b/src/Arius.Core/Shared/FileTree/FileTreeSerializer.cs index 1869f8e45..adf84b8c8 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeSerializer.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeSerializer.cs @@ -11,10 +11,8 @@ internal static class FileTreeSerializer { private static readonly Encoding s_utf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); - public static IReadOnlyList Deserialize(byte[] bytes) + public static IReadOnlyList Deserialize(ReadOnlySpan bytes) { - ArgumentNullException.ThrowIfNull(bytes); - var text = s_utf8.GetString(bytes); return ParsePersistedLines(text.Split('\n')); } diff --git a/src/Arius.Core/Shared/FileTree/FileTreeService.cs b/src/Arius.Core/Shared/FileTree/FileTreeService.cs index 694915392..d12ca552d 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeService.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeService.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Arius.Core.Shared.Compression; using Arius.Core.Shared.Encryption; +using Arius.Core.Shared.Extensions; using Arius.Core.Shared.Snapshot; using Arius.Core.Shared.Storage; using Microsoft.Extensions.Logging; @@ -243,6 +244,10 @@ private async Task SerializeStorageAsync(ReadOnlyMemory plaintext, await compressionStream.WriteAsync(plaintext, cancellationToken); } + // ToArray() and not ToArraySegment(): disposing the encryption/compression chain above also closes + // `ms`, and ToArray() is the only MemoryStream buffer accessor that stays valid after close + // (Length and TryGetBuffer both throw ObjectDisposedException). Avoiding this copy would need a + // leaveOpen on IEncryptionService.WrapForEncryption, or a non-closing stream shim. return ms.ToArray(); } @@ -252,7 +257,8 @@ private async Task> DeserializeStorageAsync(Stream await using var decompressStream = _compression.WrapForDecompression(decStream); using var ms = new MemoryStream(); await decompressStream.CopyToAsync(ms, cancellationToken); - return FileTreeSerializer.Deserialize(ms.ToArray()); + var buffer = ms.ToArraySegment(); + return FileTreeSerializer.Deserialize(buffer.AsSpan()); } private async Task WriteCacheAtomicallyAsync(RelativePath diskPath, ReadOnlyMemory plaintext, CancellationToken cancellationToken) @@ -261,7 +267,7 @@ private async Task WriteCacheAtomicallyAsync(RelativePath diskPath, ReadOnlyMemo try { - await _diskCacheFileSystem.WriteAllBytesAsync(tempPath, plaintext.ToArray(), cancellationToken); + await _diskCacheFileSystem.WriteAllBytesAsync(tempPath, plaintext, cancellationToken); _diskCacheFileSystem.ReplaceFileAtomically(tempPath, diskPath); } finally From 8fc47fe5de525701deb6c7c68f86b253d6eb1998 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:35:53 +0200 Subject: [PATCH 07/46] perf(chunk-index): bind hash BLOBs from reusable buffers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BindEntry called Convert.FromHexString(hash.ToString()) for both hash columns of every row, allocating a fresh byte[32] per hash. The upsert commands are already reused across a batch (CreateUpsertCommand is hoisted out of the loop), so the buffers can be too: allocate two per command and overwrite them in place. Each ExecuteNonQuery completes before the next row rewrites them, so a 256-row batch binds 2 arrays instead of 512. Same treatment for the EnrichThinChunks bind loop. ChunkIndexLocalStore.UpsertRemoteBacked (1000 rows) before 956.5 KB 1944 us after 847.3 KB 1475 us The 109 KB delta is exactly 1000 rows x 2 x (32 B + 24 B array header), so it is attributable rather than coincidental. 24% faster as a bonus. The read path is deliberately left on (byte[])reader.GetValue(...). Reading into a reusable buffer via SqliteDataReader.GetBytes was tried and measured 5.6x WORSE — ReadRangeEntries went 641 KB -> 3620 KB per 1000 rows and 3.4x slower — because the provider routes GetBytes through a SqliteBlob. A comment records this so it is not retried. Two corrections to earlier assumptions, for the record: - (byte[])reader.GetValue(...) does not box. byte[] is a reference type, so the cast is free; the only allocation is the array itself. - The remaining per-row read cost is dominated by the two hash *strings*, not the arrays (~304 B vs ~112 B per row). Eliminating it needs the deferred inline-digest hash representation, not a better reader accessor. That is the measurement the hash-representation decision was waiting on. The read rows did improve in this run (ReadRangeEntries 641 -> 463 KB, FindEntry x256 656 -> 610 KB) but that is the earlier ToLowerHex change becoming visible, not this commit — the recorded baseline predates it. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped. Co-Authored-By: Claude Opus 5 (1M context) --- .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 53 +++++++++++++++---- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index 2ea87f55d..c1e78529d 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -89,7 +89,9 @@ private void Initialize() command.CommandText = pendingFlushOnly ? "SELECT content_hash, chunk_hash, original_size, chunk_size, storage_tier_hint FROM chunk_index_entries WHERE content_hash = $contentHash AND pending_flush = 1;" : "SELECT content_hash, chunk_hash, original_size, chunk_size, storage_tier_hint FROM chunk_index_entries WHERE content_hash = $contentHash;"; - command.Parameters.Add("$contentHash", SqliteType.Blob).Value = ParseHashBytes(contentHash.ToString()); + var digest = CreateDigestBuffer(); + WriteDigest(contentHash.ToString(), digest); + command.Parameters.Add("$contentHash", SqliteType.Blob).Value = digest; using var reader = command.ExecuteReader(); var entry = reader.Read() ? ReadEntry(reader) : null; _logger.LogDebug("[chunk-index-local] FindEntry: contentHash={ContentHash} pendingFlushOnly={PendingFlushOnly} found={Found}", contentHash.Short8, pendingFlushOnly, entry is not null); @@ -385,7 +387,7 @@ public void UpsertPendingFlush(ShardEntry entry) using var connection = OpenConnection(); using var transaction = connection.BeginTransaction(); using var command = CreateUpsertCommand(connection, transaction, pendingFlush: true, preservePendingFlushRows: false); - BindEntry(command, entry); + BindEntry(command, entry, CreateDigestBuffer(), CreateDigestBuffer()); var rowsAffected = command.ExecuteNonQuery(); transaction.Commit(); @@ -415,10 +417,12 @@ public void UpsertPendingFlush(IEnumerable entries) using var connection = OpenConnection(); using var transaction = connection.BeginTransaction(); using var command = CreateUpsertCommand(connection, transaction, pendingFlush: true, preservePendingFlushRows: false); + var contentDigest = CreateDigestBuffer(); + var chunkDigest = CreateDigestBuffer(); var rowsAffected = 0; foreach (var entry in materialized) { - BindEntry(command, entry); + BindEntry(command, entry, contentDigest, chunkDigest); rowsAffected += command.ExecuteNonQuery(); } @@ -452,10 +456,12 @@ public void UpsertRemoteBacked(IEnumerable entries) using var connection = OpenConnection(); using var transaction = connection.BeginTransaction(); using var command = CreateUpsertCommand(connection, transaction, pendingFlush: false, preservePendingFlushRows: false); + var contentDigest = CreateDigestBuffer(); + var chunkDigest = CreateDigestBuffer(); var rowsAffected = 0; foreach (var entry in batch) { - BindEntry(command, entry); + BindEntry(command, entry, contentDigest, chunkDigest); rowsAffected += command.ExecuteNonQuery(); } @@ -497,9 +503,11 @@ public void EnrichThinChunks(IReadOnlyDictionary + /// Binds one entry onto a command reused across a batch. and + /// are the command's already-bound BLOB buffers, overwritten in place: + /// each ExecuteNonQuery completes before the next row rewrites them, so a batch of 256 rows binds + /// 2 arrays rather than 512. + /// + private static void BindEntry(SqliteCommand command, ShardEntry entry, byte[] contentDigest, byte[] chunkDigest) { - command.Parameters["$contentHash"].Value = ParseHashBytes(entry.ContentHash.ToString()); - command.Parameters["$chunkHash"].Value = ParseHashBytes(entry.ChunkHash.ToString()); + WriteDigest(entry.ContentHash.ToString(), contentDigest); + WriteDigest(entry.ChunkHash.ToString(), chunkDigest); + command.Parameters["$contentHash"].Value = contentDigest; + command.Parameters["$chunkHash"].Value = chunkDigest; command.Parameters["$originalSize"].Value = entry.OriginalSize; command.Parameters["$chunkSize"].Value = entry.ChunkSize; command.Parameters["$storageTierHint"].Value = ShardEntry.SerializeTier(entry.StorageTierHint); @@ -899,6 +917,11 @@ ON CONFLICT(prefix) DO UPDATE SET return command.ExecuteNonQuery(); } + // Reads the hash columns with (byte[])reader.GetValue(...), which allocates one byte[32] per hash per + // row. That is deliberate: reading into a reusable buffer via SqliteDataReader.GetBytes was measured + // 5.6x WORSE on ReadRangeEntries (641 KB -> 3620 KB per 1000 rows), because the provider routes + // GetBytes through a SqliteBlob. The cast itself is free — byte[] is a reference type, so nothing is + // boxed. The remaining per-row cost here is dominated by the two hash *strings*, not the arrays. private static ShardEntry ReadEntry(SqliteDataReader reader) => new( ContentHash.FromDigest((byte[])reader.GetValue(0)), @@ -907,6 +930,14 @@ private static ShardEntry ReadEntry(SqliteDataReader reader) reader.GetInt64(3), ShardEntry.DeserializeTier(reader.GetInt32(4))); - private static byte[] ParseHashBytes(string value) - => Convert.FromHexString(value); + /// + /// Writes the 32 digest bytes of a canonical-hex hash into . + /// The hash types guarantee exactly 64 canonical hex characters by construction, so this cannot fail. + /// + private static void WriteDigest(string hex, byte[] destination) + => Convert.FromHexString(hex, destination, out _, out _); + + /// Rents a reusable 32-byte digest buffer for one command or one read loop. + private static byte[] CreateDigestBuffer() => new byte[HashCodec.Sha256ByteLength]; + } From a3e957a64baf3099fae08226dd6917807296217c Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:38:51 +0200 Subject: [PATCH 08/46] perf(chunk-index): look up dedup batches with one query, not 256 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChunkIndexService.LookupAsync is documented as doing "one batched lookup per batch instead of a round-trip per file". That is true of *remote* round-trips but not of local ones: it looped over the 256 hashes calling FindPendingFlushEntry, then looped again calling FindEntry, so a single batch issued up to 512 statements — each with its own pooled connection, its own PRAGMA synchronous round-trip, its own command, and its own parameter bind. Add FindEntries / FindPendingFlushEntries to ChunkIndexLocalStore, each issuing one `content_hash IN (...)` query, and call those instead. The two phases stay separate because they have to: the dirty-row probe decides which hashes need remote coverage, and the entry probe must run after coverage completes. So this is two set queries per batch rather than one, down from 512 single-row ones. The parameter count is padded to fixed buckets (1/4/16/64/256) so the command text repeats across calls and SQLite can reuse the prepared statement instead of compiling fresh SQL for every distinct batch size. Padding slots repeat the first hash, which is safe because this is a set membership test — duplicates cannot add rows. SQLITE_MAX_VARIABLE_NUMBER is 32766, well above the top bucket. Semantics are unchanged: dirty rows still win over remote-backed rows, hashes absent from both are absent from the result, and the caller already deduped via Distinct(). AllocationBenchmarks, one 256-hash dedup batch: FindEntry x256 (per-hash) 610.00 KB 1002.0 us FindEntries x1 (batched) 205.48 KB 381.8 us 3x less allocated, 2.6x faster — and LookupAsync performs this twice per batch, so the per-run saving is roughly double that. The single-hash FindEntry/FindPendingFlushEntry remain for the single-hash LookupAsync overload, which IChunkIndexService already flags with a TODO as having no production callers. Left in place rather than removed, per AGENTS.md on pre-existing dead code. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped dotnet test src/Arius.Integration.Tests — 82 passed, 4 skipped (Azurite) Co-Authored-By: Claude Opus 5 (1M context) --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 6 + .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 109 ++++++++++++++++++ .../Shared/ChunkIndex/ChunkIndexService.cs | 17 ++- 3 files changed, 123 insertions(+), 9 deletions(-) diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index 25b49089c..eaade8ec5 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -156,6 +156,12 @@ public int ChunkIndexLocalStore_FindEntry_256() return found; } + /// + /// The batched replacement for the above: one IN (...) query for the whole 256-hash dedup batch. + /// + [Benchmark(Description = "ChunkIndexLocalStore.FindEntries x1 (one dedup batch)")] + public int ChunkIndexLocalStore_FindEntries_Batch() => _store.FindEntries(_lookupHashes).Count; + // ── Setup ──────────────────────────────────────────────────────────────────── [GlobalSetup] diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index c1e78529d..69e55e1a0 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -1,3 +1,5 @@ +using System.Collections.ObjectModel; +using System.Text; using System.Text.Json; using Arius.Core.Shared.Storage; using Microsoft.Data.Sqlite; @@ -80,6 +82,113 @@ private void Initialize() /// public ShardEntry? FindPendingFlushEntry(ContentHash contentHash) => FindEntryCore(contentHash, pendingFlushOnly: true); + /// + /// Batched twin of : one query for a whole set of hashes. + /// Hashes with no stored entry are simply absent from the result. + /// + public IReadOnlyDictionary FindEntries(IReadOnlyCollection contentHashes) + => FindEntriesCore(contentHashes, pendingFlushOnly: false); + + /// + /// Batched twin of : one query for a whole set of hashes. + /// Returns only entries still pending local flush. + /// + public IReadOnlyDictionary FindPendingFlushEntries(IReadOnlyCollection contentHashes) + => FindEntriesCore(contentHashes, pendingFlushOnly: true); + + /// + /// Looks up a set of hashes with a single IN (...) query instead of one query per hash. + /// A dedup batch is 256 hashes and each per-hash lookup previously cost its own pooled connection, its + /// own PRAGMA synchronous round-trip, and its own command — so a "batched" lookup was issuing + /// 512 statements. + /// + /// + /// The parameter count is padded to a fixed bucket so the command text repeats across calls (batches + /// are almost always full, with one ragged tail), letting SQLite reuse the prepared statement instead + /// of compiling fresh SQL per distinct batch size. Padding slots repeat the first hash, which is + /// harmless: this is a set membership test, so duplicates cannot add rows. + /// SQLITE_MAX_VARIABLE_NUMBER is 32766 on modern SQLite, well above the largest bucket. + /// + private IReadOnlyDictionary FindEntriesCore(IReadOnlyCollection contentHashes, bool pendingFlushOnly) + { + if (contentHashes.Count == 0) + return ReadOnlyDictionary.Empty; + + try + { + var slots = LookupBucketSize(contentHashes.Count); + var results = new Dictionary(contentHashes.Count); + + using var connection = OpenConnection(); + using var command = connection.CreateCommand(); + command.CommandText = BuildFindEntriesSql(slots, pendingFlushOnly); + + var digests = new byte[slots][]; + for (var i = 0; i < slots; i++) + { + digests[i] = CreateDigestBuffer(); + command.Parameters.Add($"$h{i}", SqliteType.Blob).Value = digests[i]; + } + + var slot = 0; + var first = string.Empty; + foreach (var contentHash in contentHashes) + { + var hex = contentHash.ToString(); + if (slot == 0) + first = hex; + + WriteDigest(hex, digests[slot++]); + } + + // Pad the unused slots with a repeat of the first hash. + for (; slot < slots; slot++) + WriteDigest(first, digests[slot]); + + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var entry = ReadEntry(reader); + results[entry.ContentHash] = entry; + } + + _logger.LogDebug("[chunk-index-local] FindEntries: requested={Requested} slots={Slots} pendingFlushOnly={PendingFlushOnly} found={Found}", contentHashes.Count, slots, pendingFlushOnly, results.Count); + return results; + } + catch (SqliteException ex) + { + throw CreateLocalStoreException(ex); + } + } + + /// Fixed parameter-count buckets, so the generated SQL — and its prepared statement — repeats. + private static int LookupBucketSize(int count) => count switch + { + <= 1 => 1, + <= 4 => 4, + <= 16 => 16, + <= 64 => 64, + <= 256 => 256, + _ => count, + }; + + private static string BuildFindEntriesSql(int slots, bool pendingFlushOnly) + { + var sql = new StringBuilder("SELECT content_hash, chunk_hash, original_size, chunk_size, storage_tier_hint FROM chunk_index_entries WHERE content_hash IN ("); + for (var i = 0; i < slots; i++) + { + if (i > 0) + sql.Append(", "); + sql.Append("$h").Append(i); + } + + sql.Append(')'); + if (pendingFlushOnly) + sql.Append(" AND pending_flush = 1"); + + return sql.Append(';').ToString(); + } + private ShardEntry? FindEntryCore(ContentHash contentHash, bool pendingFlushOnly) { try diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs index d6a2bc0e3..997f000af 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs @@ -104,14 +104,17 @@ public async Task> LookupAsync(IEnu if (hashes.Length == 0) return result; + // One set query for the whole batch rather than one per hash. Dirty (pending-flush) rows still win + // over remote-backed rows, which is why this probe runs before validation. + var pendingFlush = _localStore.FindPendingFlushEntries(hashes); + var validationWork = new List<(PathSegment Root, List Hashes)>(); foreach (var rootGroup in hashes.GroupBy(ChunkIndexRouter.GetRootPrefix)) { var hashesNeedingValidation = new List(); foreach (var contentHash in rootGroup) { - var pendingFlushEntry = _localStore.FindPendingFlushEntry(contentHash); - if (pendingFlushEntry is not null) + if (pendingFlush.TryGetValue(contentHash, out var pendingFlushEntry)) { // Entry is local-only / dirty result[contentHash] = pendingFlushEntry; @@ -143,15 +146,11 @@ await Parallel.ForEachAsync( await EnsureCoverageForHashesAsync(item.Root, item.Hashes, latestSnapshotName, ct); }); - // Construct the result from the validated shards + // Construct the result from the validated shards, one set query per root rather than one per hash. foreach (var item in validationWork) { - foreach (var contentHash in item.Hashes) - { - var entry = _localStore.FindEntry(contentHash); - if (entry is not null) - result[contentHash] = entry; - } + foreach (var (contentHash, entry) in _localStore.FindEntries(item.Hashes)) + result[contentHash] = entry; } return result; From 12659ab83d9bbdfd4b27ab2012e9a842fb570eda Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:39:47 +0200 Subject: [PATCH 09/46] perf(restore): restore the async read path on ChunkDownloadStream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChunkDownloadStream overrode only Read(byte[], int, int). Stream's base implementations therefore routed ReadAsync(Memory) back through ReadAsync(byte[], ...) and on to BeginEndReadAsync, which blocks a thread-pool thread on the synchronous Read for every buffer of every restored byte — while every stream underneath it (AesGcmDecryptingStream, AutoDetectDecompressionStream, PrefixedStream, ProgressStream) implements the async path correctly. Add the span/memory/async overrides as straight delegation, plus CopyToAsync so the restore pump does not re-enter the base copy loop. This is the whole restore download path: blob -> progress -> decrypt -> decompress -> here. Pure delegation, no behavioural change. The Interlocked.Exchange double-dispose guard is untouched. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped dotnet test src/Arius.Integration.Tests — 82 passed, 4 skipped (Azurite, includes restore round trips) No AllocationBenchmarks row: this frees thread-pool threads rather than reducing allocation, so it shows in ArchiveStepBenchmarks' "Completed Work Items" and in restore wall-clock. Co-Authored-By: Claude Opus 5 (1M context) --- .../Shared/ChunkStorage/ChunkStorageService.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs b/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs index 987967b62..4459a5408 100644 --- a/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs +++ b/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs @@ -306,6 +306,17 @@ private sealed class ChunkDownloadStream(Stream inner) : Stream public override long Position { get => inner.Position; set => inner.Position = value; } public override void Flush() => inner.Flush(); public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + + // The span/memory overrides are load-bearing, not tidiness. Without them Stream's base + // implementations route ReadAsync(Memory) back through ReadAsync(byte[], ...) and on to + // BeginEndReadAsync, which blocks a thread-pool thread on the synchronous Read for every buffer of + // every restored byte — even though every stream underneath (AesGcmDecryptingStream, + // AutoDetectDecompressionStream, PrefixedStream, ProgressStream) implements the async path properly. + public override int Read(Span buffer) => inner.Read(buffer); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => inner.ReadAsync(buffer, offset, count, cancellationToken); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => inner.ReadAsync(buffer, cancellationToken); + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) => inner.CopyToAsync(destination, bufferSize, cancellationToken); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); public override void SetLength(long value) => inner.SetLength(value); public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count); From 8f367c6f525ddd0afc07663975c6e5f536a48d85 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:43:11 +0200 Subject: [PATCH 10/46] perf(streaming): coalesce ProgressStream reports to 500 ms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProgressStream reported after every read, and its consumers wrap the callback in Progress (ArchiveVerb, JobRunner), which posts a thread-pool work item per report. That is one queued work item per ~64-80 KiB of every archived and restored byte, and it is what ArchiveStepBenchmarks' "Completed Work Items" counts — 39,835 at divisor=1, 506,427 on the full workflow. Throttle to at most one report per 500 ms, using Stopwatch (monotonic) rather than wall-clock time. Three properties are preserved deliberately, each of which the test suite insisted on: - The first read always reports, so a progress bar moves immediately instead of after a blank interval — and a stream consumed in a single read still reports before it ever reaches EOF. - The true total is emitted at EOF (a read returning 0) and again on dispose for a stream abandoned early. Flushing only on dispose was my first attempt and it was wrong: a consumer that reads to the end and then inspects progress before disposing would see a stale figure, which is exactly what ReadAsync_FinalProgressEqualsLength and ReadSpan_ReportsProgress caught. - A zero-length source still reports nothing (Read_ZeroLengthSource_NoProgressReported), so an empty stream does not emit a spurious 0. Two tests encoded the old per-read contract and are updated to the new one, keeping their intent: - Read_ReportsProgressAfterEachChunk asserted exactly 4 reports for 4 reads. Renamed to Read_CoalescesReports_ButStillReportsTheTotal, asserting progress starts, never goes backwards, and ends at the total. - UploadLargeAsync_RetryAfterMetadataConflict_ReportsSingleProgressSequence asserted the literal sequence [512, 1024, 1536, 2048]. It is named for a property — a retry must not restart or duplicate the sequence — so it now asserts that property: strictly increasing, no repeats, finishing on the true total. streaming.md is updated: its "Open seams" previously said smoothing was left to the consumer, which is no longer true. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/core/shared/streaming.md | 7 +- .../ChunkStorageServiceUploadTests.cs | 9 +- .../Shared/Streaming/ProgressStreamTests.cs | 10 ++- .../Shared/Streaming/ProgressStream.cs | 90 +++++++++++++++++-- 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/docs/design/core/shared/streaming.md b/docs/design/core/shared/streaming.md index 3f8494b6e..6d047d1b4 100644 --- a/docs/design/core/shared/streaming.md +++ b/docs/design/core/shared/streaming.md @@ -21,14 +21,15 @@ flowchart LR tee -. "tee for inline verify" .-> ver[RoundTripVerifier] ``` -- **`ProgressStream(inner, IProgress)`** wraps the *source* at the top of the chain so progress tracks logical bytes read off disk, not compressed bytes on the wire. `Length` is delegated to the inner `FileStream`, so the consumer knows the total up front and can compute a percentage. It reports after each read; a zero-length read reports nothing. The download path reuses it the same way, wrapping the blob's read stream. +- **`ProgressStream(inner, IProgress)`** wraps the *source* at the top of the chain so progress tracks logical bytes read off disk, not compressed bytes on the wire. `Length` is delegated to the inner `FileStream`, so the consumer knows the total up front and can compute a percentage. Reports are **coalesced to at most one per 500 ms**: the first read reports immediately (so a progress bar moves at once), later reads report only once the interval has elapsed, and the true total is always emitted at EOF and again on dispose. A zero-length source still reports nothing. The download path reuses it the same way, wrapping the blob's read stream. - **`CountingStream(inner)`** sits at the *bottom* of the chain, directly above `OpenWriteAsync`, and increments `BytesWritten` on every write. It is read *after* the chain is disposed to capture the final compressed-and-encrypted blob size, which `UploadChunkAsync` then writes into blob metadata (`chunk-size`). The note in `UploadChunkAsync` is load-bearing here: the encryption stream is disposed *explicitly* before reading `BytesWritten`, because GCM flushes its final auth tag on dispose — reading the count earlier would undercount by the tag bytes. ## Key invariants -- **No buffering proportional to file size.** Both wrappers hold only a `long` counter; the chain streams a multi-GB file without an O(file-size) allocation. See [memory-boundedness](../../cross-cutting/memory-boundedness.md). +- **No buffering proportional to file size.** Both wrappers hold only a few `long` counters; the chain streams a multi-GB file without an O(file-size) allocation. See [memory-boundedness](../../cross-cutting/memory-boundedness.md). +- **A coalesced `ProgressStream` still ends on the true total.** Throttling may drop intermediate reports, but the EOF and dispose flushes must stay, or a consumer is left short of 100% by up to one interval's worth of bytes. - **`CountingStream.BytesWritten` is valid only after the whole chain is finalized.** Every layer above it (compression frame, GCM tag) must be flushed/disposed first, or the recorded `chunk-size` is short. - **`ProgressStream` is read-only, `CountingStream` is write-only.** They guard their unsupported direction with `NotSupportedException` rather than silently no-op'ing, so a misuse fails loudly. - **Counting reflects what was actually persisted.** `CountingStream` wraps the Azure write stream, so its total is the blob's real stored size, not a pre-computed estimate. @@ -39,5 +40,5 @@ Splitting "report read progress" and "count written bytes" into separate one-lin ## Open seams / future -- `ProgressStream` reports raw read counts; smoothing/throttling of the `IProgress` callback is left to the consumer (`UploadChunkAsync` already de-dupes non-increasing reports via a `CallbackProgress`). +- `ProgressStream` throttles to one report per 500 ms at the source, because consumers wrap the callback in `Progress`, which posts a thread-pool work item per report — one per ~64-80 KiB of every archived and restored byte before this. `UploadChunkAsync`'s `CallbackProgress` still de-dupes non-increasing reports on top of it. The interval is a fixed constant, not a per-consumer setting. - Any future upload layer (e.g. a second integrity tee) slots into the same push chain in `UploadChunkAsync` between source and `OpenWriteAsync`; these two wrappers stay unchanged as the progress/size endpoints. diff --git a/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs b/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs index d4d4f29ff..c5640c6d0 100644 --- a/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs +++ b/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs @@ -223,7 +223,14 @@ public async Task UploadLargeAsync_RetryAfterMetadataConflict_ReportsSingleProgr result.AlreadyExisted.ShouldBeFalse(); blobs.DeletedBlobNames.ShouldContain(blobName); - reports.ShouldBe([512L, 1024L, 1536L, 2048L]); + + // The point of this test is that the retry does not restart or duplicate the progress sequence. + // ProgressStream coalesces reports, so assert that property rather than an exact byte sequence: + // strictly increasing (no repeat, no reset) and finishing at the true total. + reports.ShouldNotBeEmpty(); + reports.ShouldBeInOrder(); + reports.Distinct().Count().ShouldBe(reports.Count); + reports[^1].ShouldBe((long)content.Length); } [Test] diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index 9e2094e14..8ae21dc06 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -5,7 +5,7 @@ namespace Arius.Core.Tests.Shared.Streaming; public class ProgressStreamTests { [Test] - public void Read_ReportsProgressAfterEachChunk() + public void Read_CoalescesReports_ButStillReportsTheTotal() { var data = new byte[1024]; Random.Shared.NextBytes(data); @@ -17,9 +17,13 @@ public void Read_ReportsProgressAfterEachChunk() var buf = new byte[256]; while (ps.Read(buf, 0, buf.Length) > 0) { } - reports.Count.ShouldBe(4); - reports[^1].ShouldBe(1024); + // Reports are throttled to one per 500 ms (plus the first read and the EOF total), so four + // in-memory reads no longer produce four reports. What must hold is that progress starts + // promptly, never goes backwards, and ends at the true total. + reports.ShouldNotBeEmpty(); + reports.Count.ShouldBeLessThanOrEqualTo(4); reports.ShouldBeInOrder(); + reports[^1].ShouldBe(1024); } [Test] diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index 701079d8e..d65e29dce 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -1,15 +1,29 @@ +using System.Diagnostics; + namespace Arius.Core.Shared.Streaming; /// /// Read-mode stream wrapper that reports cumulative bytes read via . -/// Delegates all reads to the inner stream and reports progress after each read operation. -/// Does not buffer any data. +/// Delegates all reads to the inner stream and does not buffer any data. +/// +/// The first read reports immediately; after that reports are coalesced to at most one per +/// . Consumers wrap the callback in +/// , which posts a thread-pool work item per report, so reporting after every +/// read queued one work item per buffer — roughly one per 64-80 KiB of every archived and restored byte. +/// The true total is always emitted once the source reaches EOF (a read returning 0), and again on +/// dispose for a stream abandoned before EOF, so a consumer never ends up short of the real figure. /// public sealed class ProgressStream : Stream { + /// Minimum wall-clock gap between two progress callbacks. + private static readonly TimeSpan ReportInterval = TimeSpan.FromMilliseconds(500); + private readonly Stream _inner; private readonly IProgress _progress; private long _bytesRead; + private long _reportedBytes; + private long _lastReportTimestamp; + private bool _hasReported; /// The readable source stream. /// Receives cumulative bytes read after each read call. @@ -24,6 +38,43 @@ public ProgressStream(Stream inner, IProgress progress) _progress = progress; } + /// + /// Reports the running total, but at most once per . + /// Uses rather than wall-clock time so it is monotonic. + /// + private void ReportThrottled() + { + var now = Stopwatch.GetTimestamp(); + + // The first read always reports, so a consumer sees work start immediately rather than after a + // blank interval — and so a stream consumed in a single read still reports before EOF. + if (_hasReported && Stopwatch.GetElapsedTime(_lastReportTimestamp, now) < ReportInterval) + return; + + _hasReported = true; + _lastReportTimestamp = now; + _reportedBytes = _bytesRead; + _progress.Report(_bytesRead); + } + + /// + /// Emits the running total if the throttle has held anything back. Called at EOF and on dispose, so a + /// consumer is never left short of the real figure by up to one interval's worth of bytes. + /// + private void ReportFinal() + { + // A source that yielded nothing reports nothing — an empty stream must not emit a spurious 0. + if (_bytesRead == 0) + return; + + if (_hasReported && _reportedBytes == _bytesRead) + return; + + _hasReported = true; + _reportedBytes = _bytesRead; + _progress.Report(_bytesRead); + } + public override bool CanRead => true; public override bool CanWrite => false; public override bool CanSeek => false; @@ -34,8 +85,13 @@ public override int Read(byte[] buffer, int offset, int count) if (n > 0) { _bytesRead += n; - _progress.Report(_bytesRead); + ReportThrottled(); + } + else + { + ReportFinal(); } + return n; } @@ -45,8 +101,13 @@ public override int Read(Span buffer) if (n > 0) { _bytesRead += n; - _progress.Report(_bytesRead); + ReportThrottled(); + } + else + { + ReportFinal(); } + return n; } @@ -56,8 +117,13 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, if (n > 0) { _bytesRead += n; - _progress.Report(_bytesRead); + ReportThrottled(); } + else + { + ReportFinal(); + } + return n; } @@ -67,8 +133,13 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation if (n > 0) { _bytesRead += n; - _progress.Report(_bytesRead); + ReportThrottled(); } + else + { + ReportFinal(); + } + return n; } @@ -76,7 +147,12 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation protected override void Dispose(bool disposing) { - if (disposing) _inner.Dispose(); + if (disposing) + { + ReportFinal(); + _inner.Dispose(); + } + base.Dispose(disposing); } From 63acf128ba58328ec632477664433db43a1f5dd4 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:44:46 +0200 Subject: [PATCH 11/46] perf: three contained allocation fixes on hot paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BlobPaths prefixes were expression-bodied properties, so every blob operation re-ran `RelativePath.Root / PathSegment.Parse("chunks")` and allocated a fresh path. ChunkPath/ThinChunkPath/FileTreePath compound two of them, and ThinChunkPath runs 64-wide per tar bundle. Make them static readonly fields — the sibling RepositoryLocalStatePaths already does exactly this, so this makes the two consistent. PathSegment.TryParse and RelativePath.TryParse used value.Any(char.IsControl). LINQ over a string allocates a CharEnumerator (a class) per call, and Parse runs for every blob path and every filetree entry. Replaced with a foreach over the string, which the compiler lowers to indexer access and allocates nothing. char.IsControl is still the predicate, so the accepted character set is unchanged — this is not a hand-rolled range check. And the referenced article's actual pattern: three static readonly byte[] magic constants become static ReadOnlySpan properties, so the u8 literals point at assembly RVA data instead of being defensively copied to the heap at static init. Every use site was already .Length, SequenceEqual, or Stream.Write, all of which take a span. Worth being straight about the size of that last one: it removes three process-lifetime allocations totalling ~30 bytes. It is correct and idiomatic, and it is what the article asks for, but it is not where the memory was — the preceding commits are. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped dotnet test src/Arius.Integration.Tests — 82 passed, 4 skipped, including GoldenFileDecryptionTests and RecoveryScriptTests, which are the guards that the ArGCM1 bytes and recover-chunk.py compatibility are untouched. Co-Authored-By: Claude Opus 5 (1M context) --- .../Shared/Compression/ZstdCompressionService.cs | 2 +- .../Encryption/PassphraseEncryptionService.cs | 4 ++-- src/Arius.Core/Shared/FileSystem/PathSegment.cs | 16 +++++++++++++++- src/Arius.Core/Shared/FileSystem/RelativePath.cs | 16 +++++++++++++++- src/Arius.Core/Shared/Storage/BlobConstants.cs | 12 ++++++------ 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/Arius.Core/Shared/Compression/ZstdCompressionService.cs b/src/Arius.Core/Shared/Compression/ZstdCompressionService.cs index e63ebc6c8..0b69d6dd4 100644 --- a/src/Arius.Core/Shared/Compression/ZstdCompressionService.cs +++ b/src/Arius.Core/Shared/Compression/ZstdCompressionService.cs @@ -13,7 +13,7 @@ internal sealed class ZstdCompressionService(int compressionLevel = ZstdCompress : ICompressionService { // Magic number at the start of a zstd frame (little-endian on disk). - private static readonly byte[] ZstdMagic = [0x28, 0xB5, 0x2F, 0xFD]; // 0xFD2FB528 + private static ReadOnlySpan ZstdMagic => [0x28, 0xB5, 0x2F, 0xFD]; // 0xFD2FB528 // Legacy "+gzip" blobs are decoded by the gzip codec; reads delegate to it when a gzip frame is detected. private static readonly GZipCompressionService LegacyGzip = new(); diff --git a/src/Arius.Core/Shared/Encryption/PassphraseEncryptionService.cs b/src/Arius.Core/Shared/Encryption/PassphraseEncryptionService.cs index cbe638ae3..b35048197 100644 --- a/src/Arius.Core/Shared/Encryption/PassphraseEncryptionService.cs +++ b/src/Arius.Core/Shared/Encryption/PassphraseEncryptionService.cs @@ -35,7 +35,7 @@ internal sealed class PassphraseEncryptionService : IEncryptionService // key for anything written today (production writes AES-GCM, see GcmPbkdf2Iter = 100_000). It // must match the blobs on disk, so it cannot be raised. private const int CbcPbkdf2Iter = 10_000; - private static readonly byte[] SaltedMagic = "Salted__"u8.ToArray(); + private static ReadOnlySpan SaltedMagic => "Salted__"u8; // ── GCM constants ──────────────────────────────────────────────────────────── private const int GcmSaltSize = 16; @@ -45,7 +45,7 @@ internal sealed class PassphraseEncryptionService : IEncryptionService private const int GcmBlockSize = 64 * 1024; // 64 KiB private const int GcmPbkdf2Iter = 100_000; private const uint GcmMaxPbkdf2Iter = 10_000_000; // sanity cap: reject crafted blobs - private static readonly byte[] GcmMagic = "ArGCM1"u8.ToArray(); // 6 bytes + private static ReadOnlySpan GcmMagic => "ArGCM1"u8; // 6 bytes private readonly byte[] _passphraseBytes; diff --git a/src/Arius.Core/Shared/FileSystem/PathSegment.cs b/src/Arius.Core/Shared/FileSystem/PathSegment.cs index 1e925ed99..9ed801bbd 100644 --- a/src/Arius.Core/Shared/FileSystem/PathSegment.cs +++ b/src/Arius.Core/Shared/FileSystem/PathSegment.cs @@ -38,7 +38,7 @@ public static bool TryParse(string? value, out PathSegment segment) return false; } - if (value.Contains('/') || value.Contains('\\') || value.Any(char.IsControl)) + if (value.Contains('/') || value.Contains('\\') || ContainsControlCharacter(value)) { segment = default; return false; @@ -72,4 +72,18 @@ public PathSegment RemoveSuffix(string suffix, StringComparison comparisonType) } public override string ToString() => Value; + + /// + /// Allocation-free replacement for value.Any(char.IsControl), which allocates a CharEnumerator + /// on every call. Parse runs for every blob path and every filetree entry, so it is hot. + /// foreach over a string is compiled to indexer access, so this allocates nothing. + /// + private static bool ContainsControlCharacter(string value) + { + foreach (var c in value) + if (char.IsControl(c)) + return true; + + return false; + } } diff --git a/src/Arius.Core/Shared/FileSystem/RelativePath.cs b/src/Arius.Core/Shared/FileSystem/RelativePath.cs index 59fc4d875..bbfca1834 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativePath.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativePath.cs @@ -95,7 +95,7 @@ public static bool TryParse(string? value, out RelativePath path) return false; } - if (value.Contains('\\') || value.Contains("//", StringComparison.Ordinal) || value.Any(char.IsControl)) + if (value.Contains('\\') || value.Contains("//", StringComparison.Ordinal) || ContainsControlCharacter(value)) { path = default; return false; @@ -164,4 +164,18 @@ public RelativePath RemoveSuffix(string suffix, StringComparison comparisonType) => path.Value.Length == 0 ? new RelativePath(segment.ToString()) : new RelativePath($"{path.Value}/{segment}"); public override string ToString() => Value; + + /// + /// Allocation-free replacement for value.Any(char.IsControl), which allocates a CharEnumerator + /// on every call. Parse runs for every blob path and every filetree entry, so it is hot. + /// foreach over a string is compiled to indexer access, so this allocates nothing. + /// + private static bool ContainsControlCharacter(string value) + { + foreach (var c in value) + if (char.IsControl(c)) + return true; + + return false; + } } diff --git a/src/Arius.Core/Shared/Storage/BlobConstants.cs b/src/Arius.Core/Shared/Storage/BlobConstants.cs index 4a5f27941..e3b7c7fe0 100644 --- a/src/Arius.Core/Shared/Storage/BlobConstants.cs +++ b/src/Arius.Core/Shared/Storage/BlobConstants.cs @@ -77,25 +77,25 @@ public static class BlobPaths // NOTE: These methods require a strong domain type (unless there is none). For string convenience overloads used in Test suites, see Arius.Tests.Shared.BlobPathsExtensions. /// Content-addressable chunks: large files and thin pointers. - public static RelativePath ChunksPrefix => RelativePath.Root / PathSegment.Parse("chunks"); + public static readonly RelativePath ChunksPrefix = RelativePath.Root / PathSegment.Parse("chunks"); /// Temporary Hot-tier copies for in-progress rehydration. - public static RelativePath ChunksRehydratedPrefix => RelativePath.Root / PathSegment.Parse("chunks-rehydrated"); + public static readonly RelativePath ChunksRehydratedPrefix = RelativePath.Root / PathSegment.Parse("chunks-rehydrated"); /// /// Chunk metadata sidecars for chunks whose own metadata cannot be written — Archive-tier blobs migrated from v5, where Azure forbids Set Blob Metadata. /// NOTE: if chunk pruning/GC is ever added, deleting a chunk must also delete its metadata sidecar. /// - public static RelativePath V5LegacySideCarPrefix => RelativePath.Root / PathSegment.Parse("chunks-v5legacy-metadata"); + public static readonly RelativePath V5LegacySideCarPrefix = RelativePath.Root / PathSegment.Parse("chunks-v5legacy-metadata"); /// Merkle tree blobs (one per directory). - public static RelativePath FileTreesPrefix => RelativePath.Root / PathSegment.Parse("filetrees"); + public static readonly RelativePath FileTreesPrefix = RelativePath.Root / PathSegment.Parse("filetrees"); /// Snapshot manifests. - public static RelativePath SnapshotsPrefix => RelativePath.Root / PathSegment.Parse("snapshots"); + public static readonly RelativePath SnapshotsPrefix = RelativePath.Root / PathSegment.Parse("snapshots"); /// Chunk index shards. - public static RelativePath ChunkIndexPrefix => RelativePath.Root / PathSegment.Parse("chunk-index"); + public static readonly RelativePath ChunkIndexPrefix = RelativePath.Root / PathSegment.Parse("chunk-index"); public static RelativePath ChunkPath(ChunkHash hash) => ChunksPrefix / PathSegment.Parse(hash.ToString()); public static RelativePath ThinChunkPath(ContentHash hash) => ChunksPrefix / PathSegment.Parse(hash.ToString()); From a9bdefa71978fec293c0856a4df9b4eb931ee0a2 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:49:34 +0200 Subject: [PATCH 12/46] perf(filetree): keep staging append handles open, and fix a boxing stripe hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems in FileTreeStagingWriter.AppendLineAsync, which runs once per archived file plus once per new directory edge. The stripe hash boxed on every append: _lockStripes[(uint)StringComparer.Ordinal.GetHashCode(path) % ...] RelativePath is a struct, so this bound to IEqualityComparer.GetHashCode(object) — boxing the struct, then falling through to obj.GetHashCode() because it is not a string. The StringComparer was doing nothing at all. Now path.GetHashCode(). This changes which stripe a path maps to, which is harmless: stripes only partition a lock and carry no persisted meaning. And the append opened, wrote, and closed the node file every time — via File.AppendAllTextAsync, plus a Directory.CreateDirectory before each one. Staging node paths are flat (Root / directoryId), so that was the *same* directory being re-created thousands of times per run. Give each stripe one open append handle. The stripe's gate already serializes every write to a node, so it also guards that stripe's handle — no second lock, and no race between a write and a re-target. A stripe re-points its handle when a different node arrives; the archive walk is depth-first, so consecutive files land in the same directory and hit the same node, which is the common case. Handles are bounded by StripeCount (256), which matters on the small NAS hardware Arius targets. The line is written as pooled UTF-8 bytes plus a '\n' rather than `line + "\n"`, so there is no per-line string concat, and no BOM — byte-identical to what File.AppendAllTextAsync produced. Two deliberate choices: - Flush per line. Staging is disposable scratch (ADR-0006) so buffering would be durable enough, but flushing keeps failure semantics exact: a write error still surfaces from this call, which is what the claim-release in AppendDirectoryEntriesAsync depends on to avoid orphaning a subtree. The win is dropping the open/create-directory/close, not the write. - FileShare.Read on the handle, matching what File.AppendAllTextAsync used. FileShare.None was the first attempt and it broke 56 tests: FileTreeBuilder reads staged nodes, and holding them exclusively for the length of an archive would be a real behaviour change, not just a test inconvenience. Because the flush is per line and sharing is unchanged, Dispose stays synchronous — no IAsyncDisposable churn across the ~10 construction sites. Verified: dotnet test src/Arius.Core.Tests — 662 passed, 1 skipped dotnet test src/Arius.Integration.Tests — 82 passed, 4 skipped (Azurite) dotnet test src/Arius.E2E.Tests — the Azurite workflow passes; the 4 real-Azure tests fail on DNS (ariuscibec.blob.core.windows.net is NXDOMAIN and no ARIUS_AZURE_* vars are set locally), i.e. environmental, not caused by this change. Co-Authored-By: Claude Opus 5 (1M context) --- .../Shared/FileSystem/RelativeFileSystem.cs | 15 ++++ .../Shared/FileTree/FileTreeStagingWriter.cs | 71 ++++++++++++++++--- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs index ac13532ea..c2fa37e79 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs @@ -252,6 +252,21 @@ public async Task WriteAllTextAsync(RelativePath path, string content, Cancellat /// /// Appends text to a file within the rooted directory, creating the file (and parent directories) if needed. /// + /// + /// Opens a file for appending, creating it if needed, and leaves the handle open for the caller to + /// reuse across writes. opens, writes, and closes per call — fine for + /// one-off appends, wasteful when the same file is appended to repeatedly. + /// + public Stream OpenAppend(RelativePath path) + { + var fullPath = root.Resolve(path); + CreateDirectory(path.Parent ?? RelativePath.Root); + // FileShare.Read matches what File.AppendAllTextAsync used, so a reader can still open the file + // while the handle is held — FileTreeBuilder reads staged nodes, and holding them exclusively for + // the length of an archive would be a behaviour change, not just a test inconvenience. + return new FileStream(fullPath, FileMode.Append, FileAccess.Write, FileShare.Read, 65536, useAsync: true); + } + public async Task AppendAllTextAsync(RelativePath path, string content, CancellationToken cancellationToken) { var fullPath = root.Resolve(path); diff --git a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs index a06a09409..376e4d62b 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs @@ -1,4 +1,6 @@ +using System.Buffers; using System.Collections.Concurrent; +using System.Text; namespace Arius.Core.Shared.FileTree; @@ -7,9 +9,28 @@ internal sealed class FileTreeStagingWriter : IDisposable { private const int StripeCount = 256; // Note: we used to have a lock for every staging file, but that was unbounded. Now we have bounded memory by striping the locks - private readonly SemaphoreSlim[] _lockStripes; + private readonly Stripe[] _lockStripes; private readonly RelativeFileSystem _stagingFileSystem; - private bool _disposed; + private bool _disposed; + + /// + /// One lock plus one open append handle. Every write to a node goes through its stripe's gate, so the + /// gate also guards that stripe's handle — which is what makes keeping it open safe without a second + /// lock or a race between a write and an eviction. Handles are bounded by , + /// which matters on the small NAS hardware Arius targets. + /// + private sealed class Stripe : IDisposable + { + public readonly SemaphoreSlim Gate = new(1, 1); + public RelativePath? OpenPath; + public Stream? Handle; + + public void Dispose() + { + Handle?.Dispose(); + Gate.Dispose(); + } + } // A directory id is globally unique to its full path, so its parent→child edge line is identical // no matter which descendant file triggers it. Emit each edge once: without this, a deep tree @@ -23,7 +44,7 @@ public FileTreeStagingWriter(LocalDirectory stagingRoot) { _stagingFileSystem = new RelativeFileSystem(stagingRoot); _lockStripes = Enumerable.Range(0, StripeCount) - .Select(_ => new SemaphoreSlim(1, 1)) + .Select(_ => new Stripe()) .ToArray(); } @@ -96,18 +117,50 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati private async Task AppendLineAsync(RelativePath path, string line, CancellationToken cancellationToken) { - var nodeLock = _lockStripes[(uint)StringComparer.Ordinal.GetHashCode(path) % (uint)_lockStripes.Length]; - await nodeLock.WaitAsync(cancellationToken); + // path.GetHashCode(), not StringComparer.Ordinal.GetHashCode(path): RelativePath is a struct, so + // the latter bound to IEqualityComparer.GetHashCode(object) — boxing on every append and then + // falling through to path.GetHashCode() anyway, so the comparer was doing nothing. + var stripe = _lockStripes[(uint)path.GetHashCode() % (uint)_lockStripes.Length]; + await stripe.Gate.WaitAsync(cancellationToken); try { + // Reuse this stripe's handle when it is already pointed at the node. The archive walk is + // depth-first, so consecutive files land in the same directory and hit the same node, making + // this the common case. Re-targeting closes the previous handle first. + if (stripe.OpenPath != path) + { + stripe.Handle?.Dispose(); + stripe.Handle = _stagingFileSystem.OpenAppend(path); + stripe.OpenPath = path; + } + // Fixed '\n' (not Environment.NewLine): staged lines are re-serialized by FileTreeSerializer // before hashing, but keep the staging format platform-independent and consistent with it. - await _stagingFileSystem.AppendAllTextAsync(path, line + "\n", cancellationToken); + // Written as UTF-8 bytes with no BOM, matching what File.AppendAllTextAsync produced. + var buffer = ArrayPool.Shared.Rent(Encoding.UTF8.GetMaxByteCount(line.Length) + 1); + try + { + var count = Encoding.UTF8.GetBytes(line, buffer); + buffer[count++] = (byte)'\n'; + + await stripe.Handle!.WriteAsync(buffer.AsMemory(0, count), cancellationToken); + + // Flush per line rather than buffering. Staging is disposable scratch, so buffering would + // be durable enough, but flushing keeps the failure semantics exact: a write error surfaces + // from this call, which is what the caller's claim-release in AppendDirectoryEntriesAsync + // depends on. The win here is dropping the per-line open/create-directory/close, not the + // write itself. + await stripe.Handle.FlushAsync(cancellationToken); + } + finally + { + ArrayPool.Shared.Return(buffer); + } } finally { - nodeLock.Release(); + stripe.Gate.Release(); } } @@ -118,7 +171,7 @@ public void Dispose() _disposed = true; - foreach (var nodeLock in _lockStripes) - nodeLock.Dispose(); + foreach (var stripe in _lockStripes) + stripe.Dispose(); } } From 10947e7bbb38e1161669cdb9aebe90af17384759 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:51:27 +0200 Subject: [PATCH 13/46] perf(cli): stop the archive progress state growing with the run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems in the CLI's archive progress bookkeeping. There was no FileDedupedEvent handler at all — the Mediator source generator has been warning "found message without any registered handler" for it — so a deduped file stayed in ProgressState.TrackedFiles in state Hashed for the entire run. That dictionary grew unboundedly with the deduped file count, and BuildDisplay snapshots it (ConcurrentDictionary.Values, which copies under every bucket lock) about ten times a second. Redraw cost therefore scaled with total files archived rather than with files in flight — worst on exactly the runs where dedup is doing its job. Add FileDedupedHandler to remove those rows. That is also why the .Values.Where(...) in BuildDisplay is left as it is: with the leak fixed, TrackedFiles only holds files actually being hashed or uploaded, which is bounded by the worker counts, so the snapshot is now cheap. TarEntryAddedHandler ran, once per small file, a Values snapshot plus Where(Accumulating).OrderByDescending(BundleNumber).FirstOrDefault() just to find the bundle currently accumulating. TarBundleSealingHandler did the same. Hold the reference on ProgressState instead: set it when a bundle starts, clear it when one seals. Only the single-threaded TarBuilder stage raises those lifecycle events, so the writes are ordered; the reference is published via Volatile for the display thread. Rendered output is unchanged — this is bookkeeping only. Still outstanding and deliberately not touched here: ProgressState.ContentHashToPath is never trimmed and allocates a ConcurrentBag per distinct content hash. Trimming it safely needs a decision about which event marks a hash definitively finished (a tar entry's content hash and its parent tar's chunk hash are different keys), which is a bigger question than this commit. Verified: dotnet test src/Arius.Cli.Tests — 164 passed dotnet build src/Arius.slnx — succeeded; the FileDedupedEvent "no registered handler" warning is gone (RoutingCompleteEvent and FinalizingSnapshotEvent still warn, pre-existing and out of scope) Co-Authored-By: Claude Opus 5 (1M context) --- .../Archive/ArchiveProgressHandlers.cs | 43 +++++++++++++------ src/Arius.Cli/ProgressState.cs | 14 ++++++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs index 1b5b16325..a5e12c433 100644 --- a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs +++ b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs @@ -91,6 +91,7 @@ public ValueTask Handle(TarBundleStartedEvent notification, CancellationToken ca var bundleNumber = state.NextBundleNumber(); var tar = new TrackedTar(bundleNumber, state.TarTargetSize); state.TrackedTars.TryAdd(bundleNumber, tar); + state.AccumulatingTar = tar; return ValueTask.CompletedTask; } } @@ -110,12 +111,8 @@ public ValueTask Handle(TarEntryAddedEvent notification, CancellationToken cance foreach (var path in paths) state.RemoveFile(path); - var tar = state.TrackedTars.Values - .Where(t => t.State == TarState.Accumulating) - .OrderByDescending(t => t.BundleNumber) - .FirstOrDefault(); - - if (tar != null) + var tar = state.AccumulatingTar; + if (tar is { State: TarState.Accumulating }) { var addedBytes = notification.CurrentTarSize - tar.AccumulatedBytes; tar.AddEntry(addedBytes > 0 ? addedBytes : 0); @@ -134,12 +131,8 @@ public sealed class TarBundleSealingHandler(ProgressState state) : INotification { public ValueTask Handle(TarBundleSealingEvent notification, CancellationToken cancellationToken) { - var tar = state.TrackedTars.Values - .Where(t => t.State == TarState.Accumulating || t.State == TarState.Sealing) - .OrderByDescending(t => t.BundleNumber) - .FirstOrDefault(); - - if (tar != null) + var tar = state.AccumulatingTar; + if (tar is { State: TarState.Accumulating or TarState.Sealing }) { tar.TarHash = notification.TarHash; // Use the sealed tar's archive byte size (headers + padding included), not the sum of file @@ -149,6 +142,32 @@ public ValueTask Handle(TarBundleSealingEvent notification, CancellationToken ca tar.State = TarState.Sealing; } + // The bundle is no longer accumulating; the next TarBundleStartedEvent supplies the next one. + state.AccumulatingTar = null; + + return ValueTask.CompletedTask; + } +} + +// ── FileDedupedHandler ──────────────────────────────────────────────────────── + +/// +/// Removes the rows for a file that deduplicated away. +/// +/// Without this there was no FileDedupedEvent handler at all — the Mediator source generator warns +/// "found message without any registered handler" for it — so a deduped file stayed in +/// in state Hashed for the whole run. That dictionary grew +/// unboundedly with the deduped file count *and* is snapshotted by the display roughly ten times a +/// second, making the redraw cost scale with total files archived rather than files in flight. +/// +public sealed class FileDedupedHandler(ProgressState state) : INotificationHandler +{ + public ValueTask Handle(FileDedupedEvent notification, CancellationToken cancellationToken) + { + if (state.ContentHashToPath.TryGetValue(notification.ContentHash, out var paths)) + foreach (var path in paths) + state.RemoveFile(path); + return ValueTask.CompletedTask; } } diff --git a/src/Arius.Cli/ProgressState.cs b/src/Arius.Cli/ProgressState.cs index c4973275d..3d182eadc 100644 --- a/src/Arius.Cli/ProgressState.cs +++ b/src/Arius.Cli/ProgressState.cs @@ -322,6 +322,20 @@ public void SkipFileDuringHashing(RelativePath relativePath) /// TAR bundles currently tracked, keyed by bundle number. public ConcurrentDictionary TrackedTars { get; } = new(); + /// + /// The bundle currently accumulating entries, or null between bundles. Held directly because + /// TarEntryAddedHandler needs it once per small file, and deriving it from + /// meant a Values snapshot plus a Where/OrderByDescending scan per file. + /// Only the single-threaded TarBuilder stage raises the bundle lifecycle events, so writes are + /// ordered; the reference is published via for the display thread. + /// + public TrackedTar? AccumulatingTar + { + get => Volatile.Read(ref _accumulatingTar); + set => Volatile.Write(ref _accumulatingTar, value); + } + private TrackedTar? _accumulatingTar; + /// Monotonically increasing bundle counter; call to allocate a new ID. private long _bundleCounter; From 59d7fa7db78196335b3d03b495a42b8548beacc7 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 10:56:32 +0200 Subject: [PATCH 14/46] perf(benchmarks): record the full-scale archive run for this branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RepresentativeScaleDivisor=1 run of ArchiveStepBenchmarks after the allocation work, with the divisor constant reverted to 8 afterwards as usual. Allocated: 478.22 MB -> 407.74 MB, -14.7%. That comparison is against a base run of ee4e9f3e (the branch point) executed on this same machine, Docker daemon, and session — not against the committed 2026-05-01 row. That row reports 33.16 s where this host runs the same commit in ~4 s, so it is not a like-for-like reference, and an earlier attempt to compare against it suggested an implausible 9x speedup. Only Allocated is treated as a real signal here. With InvocationCount=1, WarmupCount=0 and 3 iterations, the base run reported Mean 4.089 s with Error +/-30.995 s (one iteration at 6.05 s against two near 3.1 s), so wall-clock and the Gen0/1/2 collection counts are inside the noise at this sample size. Per component, the attributable wins are in AllocationBenchmarks, recorded in the individual commits. Note this figure includes the Azurite client, the TestContainers fixture, and synthetic-data materialization, so a 14.7% end-to-end reduction sits on top of a large fixed overhead that none of these changes touch. Co-Authored-By: Claude Opus 5 (1M context) --- src/Arius.Benchmarks/benchmark-tail.md | 1 + ....ArchiveStepBenchmarks-20260908-105224.log | 0 .../20260908T085224.165Z/benchmark-output.log | 159 ++++++++++++++++++ ...rks.ArchiveStepBenchmarks-report-github.md | 15 ++ ...enchmarks.ArchiveStepBenchmarks-report.csv | 2 + ...nchmarks.ArchiveStepBenchmarks-report.html | 32 ++++ 6 files changed, 209 insertions(+) create mode 100644 src/Arius.Benchmarks/raw/20260908T085224.165Z/Arius.Benchmarks.ArchiveStepBenchmarks-20260908-105224.log create mode 100644 src/Arius.Benchmarks/raw/20260908T085224.165Z/benchmark-output.log create mode 100644 src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md create mode 100644 src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv create mode 100644 src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html diff --git a/src/Arius.Benchmarks/benchmark-tail.md b/src/Arius.Benchmarks/benchmark-tail.md index f9f2f62c4..18df09dcf 100644 --- a/src/Arius.Benchmarks/benchmark-tail.md +++ b/src/Arius.Benchmarks/benchmark-tail.md @@ -12,3 +12,4 @@ | woutbook6 | 2026-05-01T15:23:58.2515390+00:00 | 1eedb04c4fe77ae90f7faa384c2c7b7dedbe0600 | 1 | 3 | 1.453 m | 0.1986 m | 0.0109 m | 434000.0000 | 54000.0000 | 4000.0000 | 3.52 GB | 506427.0000 | 15.0000 | src/Arius.Benchmarks/raw/20260501T152358.251Z | | woutbook6 | 2026-05-01T16:44:07.0942170+00:00 | 1eedb04c4fe77ae90f7faa384c2c7b7dedbe0600 | 1 | 3 | 33.16 s | 1.135 s | 0.062 s | 56000.0000 | 5000.0000 | | 444.88 MB | 39835.0000 | 2.0000 | src/Arius.Benchmarks/raw/20260501T164407.094Z | | woutbook6 | 2026-05-01T19:11:23.3076330+00:00 | 468d2479d64cef776b02c14f98ed9b667ecd2b46 | 8 | 3 | 7.352 s | 2.350 s | 0.1288 s | 14000.0000 | 3000.0000 | | 115.94 MB | 7893.0000 | 1.0000 | src/Arius.Benchmarks/raw/20260501T191123.307Z | +| woutbook6 | 2026-09-08T08:52:24.1656440+00:00 | 10947e7bbb38e1161669cdb9aebe90af17384759 | 1 | 3 | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 11000.0000 | 2000.0000 | 407.74 MB | 28623.0000 | 429.0000 | src/Arius.Benchmarks/raw/20260908T085224.165Z | # After the allocation-optimization branch; controlled base run on the same machine/session was 478.22 MB (ee4e9f3e). Time/GC counts on this host are too noisy to compare (3 iterations, no warmup, Error +/-31 s on the base). diff --git a/src/Arius.Benchmarks/raw/20260908T085224.165Z/Arius.Benchmarks.ArchiveStepBenchmarks-20260908-105224.log b/src/Arius.Benchmarks/raw/20260908T085224.165Z/Arius.Benchmarks.ArchiveStepBenchmarks-20260908-105224.log new file mode 100644 index 000000000..e69de29bb diff --git a/src/Arius.Benchmarks/raw/20260908T085224.165Z/benchmark-output.log b/src/Arius.Benchmarks/raw/20260908T085224.165Z/benchmark-output.log new file mode 100644 index 000000000..93d0807cc --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260908T085224.165Z/benchmark-output.log @@ -0,0 +1,159 @@ +// Validating benchmarks: +// ***** BenchmarkRunner: Start ***** +// ***** Found 1 benchmark(s) in total ***** +// ***** Building 1 exe(s) in Parallel: Start ***** +// start dotnet restore --nodeReuse:false /p:UseSharedCompilation=false /p:Deterministic=true /p:Optimize=true /p:ArtifactsPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/" /p:OutDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:OutputPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:PublishDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/publish/" in /Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1 +// command took 1.09 sec and exited with 0 +// start dotnet build -c Release --no-restore --nodeReuse:false /p:UseSharedCompilation=false /p:Deterministic=true /p:Optimize=true /p:ArtifactsPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/" /p:OutDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:OutputPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:PublishDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/publish/" --output "/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" in /Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1 +// command took 11.54 sec and exited with 0 +// ***** Done, took 00:00:12 (12.68 sec) ***** +// Found 1 benchmarks: +// ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) + +// ************************** +// Benchmark: ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) +// *** Execute *** +// Launch: 1 / 1 +// Execute: dotnet Arius.Benchmarks-Job-HILDPN-1.dll --anonymousPipes 126 127 --benchmarkName Arius.Benchmarks.ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite --job "InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0" --benchmarkId 0 in /Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0 +// Failed to set up high priority (Permission denied). In order to run benchmarks with high priority, make sure you have the right permissions. +// BeforeAnythingElse + +// Benchmark Process Environment Information: +// BenchmarkDotNet v0.15.8 +// Runtime=.NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a +// GC=Concurrent Workstation +// HardwareIntrinsics=ArmBase+AdvSimd,AES,CRC32,DP,RDM,SHA1,SHA256 VectorSize=128 +// Job: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) + +[testcontainers.org 00:00:00.08] Connected to Docker: + Host: unix:///var/run/docker.sock + Server Version: 29.3.1 + Kernel Version: 6.12.76-linuxkit + API Version: 1.54 + Operating System: Docker Desktop + Total Memory: 7.65 GB + Labels: + com.docker.desktop.address=unix:///Users/wouter/Library/Containers/com.docker.docker/Data/docker-cli.sock +[testcontainers.org 00:00:00.19] Docker container db7721aecdbe created +[testcontainers.org 00:00:00.21] Start Docker container db7721aecdbe +[testcontainers.org 00:00:00.30] Wait for Docker container db7721aecdbe to complete readiness checks +[testcontainers.org 00:00:01.33] Docker container db7721aecdbe ready +[testcontainers.org 00:00:01.41] Docker container a79377d769d1 created +[testcontainers.org 00:00:01.42] Start Docker container a79377d769d1 +[testcontainers.org 00:00:01.51] Wait for Docker container a79377d769d1 to complete readiness checks +[testcontainers.org 00:00:02.58] Docker container a79377d769d1 ready +OverheadJitting 1: 1 op, 61167.00 ns, 61.1670 us/op +WorkloadJitting 1: 1 op, 3657928917.00 ns, 3.6579 s/op + +OverheadWarmup 1: 1 op, 417.00 ns, 417.0000 ns/op +OverheadWarmup 2: 1 op, 84.00 ns, 84.0000 ns/op +OverheadWarmup 3: 1 op, 0.00 ns, 0.0000 ns/op +OverheadWarmup 4: 1 op, 0.00 ns, 0.0000 ns/op +OverheadWarmup 5: 1 op, 41.00 ns, 41.0000 ns/op +OverheadWarmup 6: 1 op, 0.00 ns, 0.0000 ns/op +OverheadWarmup 7: 1 op, 0.00 ns, 0.0000 ns/op + +OverheadActual 1: 1 op, 167.00 ns, 167.0000 ns/op +OverheadActual 2: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 3: 1 op, 209.00 ns, 209.0000 ns/op +OverheadActual 4: 1 op, 42.00 ns, 42.0000 ns/op +OverheadActual 5: 1 op, 250.00 ns, 250.0000 ns/op +OverheadActual 6: 1 op, 250.00 ns, 250.0000 ns/op +OverheadActual 7: 1 op, 42.00 ns, 42.0000 ns/op +OverheadActual 8: 1 op, 41.00 ns, 41.0000 ns/op +OverheadActual 9: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 10: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 11: 1 op, 42.00 ns, 42.0000 ns/op +OverheadActual 12: 1 op, 42.00 ns, 42.0000 ns/op +OverheadActual 13: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 14: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 15: 1 op, 208.00 ns, 208.0000 ns/op +OverheadActual 16: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 17: 1 op, 84.00 ns, 84.0000 ns/op +OverheadActual 18: 1 op, 41.00 ns, 41.0000 ns/op +OverheadActual 19: 1 op, 416.00 ns, 416.0000 ns/op +OverheadActual 20: 1 op, 83.00 ns, 83.0000 ns/op + + +// BeforeActualRun +WorkloadActual 1: 1 op, 3805428292.00 ns, 3.8054 s/op +WorkloadActual 2: 1 op, 4008702500.00 ns, 4.0087 s/op +WorkloadActual 3: 1 op, 3610687125.00 ns, 3.6107 s/op + +// AfterActualRun +WorkloadResult 1: 1 op, 3805428250.00 ns, 3.8054 s/op +WorkloadResult 2: 1 op, 4008702458.00 ns, 4.0087 s/op +WorkloadResult 3: 1 op, 3610687083.00 ns, 3.6107 s/op +// GC: 33 11 2 427543360 1 +// Threading: 28623 429 1 +// Exceptions: 4 + +[testcontainers.org 00:00:25.04] Delete Docker container a79377d769d1 +// AfterAll +// Benchmark Process 75165 has exited with code 0. + +Mean = 3.808 s, StdErr = 0.115 s (3.02%), N = 3, StdDev = 0.199 s +Min = 3.611 s, Q1 = 3.708 s, Median = 3.805 s, Q3 = 3.907 s, Max = 4.009 s +IQR = 0.199 s, LowerFence = 3.410 s, UpperFence = 4.206 s +ConfidenceInterval = [0.177 s; 7.439 s] (CI 99.9%), Margin = 3.631 s (95.34% of Mean) +Skewness = 0.01, Kurtosis = 0.67, MValue = 2 + +// ** Remained 0 (0,0%) benchmark(s) to run. Estimated finish 2026-09-08 10:53 (0h 0m from now) ** +// ***** BenchmarkRunner: Finish ***** + +// * Export * + src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv + src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md + src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html + +// * Detailed results * +ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) +Runtime = .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a; GC = Concurrent Workstation +Mean = 3.808 s, StdErr = 0.115 s (3.02%), N = 3, StdDev = 0.199 s +Min = 3.611 s, Q1 = 3.708 s, Median = 3.805 s, Q3 = 3.907 s, Max = 4.009 s +IQR = 0.199 s, LowerFence = 3.410 s, UpperFence = 4.206 s +ConfidenceInterval = [0.177 s; 7.439 s] (CI 99.9%), Margin = 3.631 s (95.34% of Mean) +Skewness = 0.01, Kurtosis = 0.67, MValue = 2 +-------------------- Histogram -------------------- +[3.527 s ; 3.889 s) | @@ +[3.889 s ; 4.190 s) | @ +--------------------------------------------------- + +// * Summary * + +BenchmarkDotNet v0.15.8, macOS Tahoe 26.6.2 (25G83) [Darwin 25.6.0] +Apple M4, 1 CPU, 10 logical and 10 physical cores +.NET SDK 10.0.201 + [Host] : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + Job-HILDPN : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + +InvocationCount=1 IterationCount=3 LaunchCount=1 +UnrollFactor=1 WarmupCount=0 + +| Method | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Gen2 | Allocated | +|--------------------------------------- |--------:|--------:|---------:|-----------:|---------------------:|-----------------:|-----------:|----------:|----------:| +| Archive_Step_V1_Representative_Azurite | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 28623.0000 | 429.0000 | 11000.0000 | 2000.0000 | 407.74 MB | + +// * Legends * + Mean : Arithmetic mean of all measurements + Error : Half of 99.9% confidence interval + StdDev : Standard deviation of all measurements + Gen0 : GC Generation 0 collects per 1000 operations + Completed Work Items : The number of work items that have been processed in ThreadPool (per single operation) + Lock Contentions : The number of times there was contention upon trying to take a Monitor's lock (per single operation) + Gen1 : GC Generation 1 collects per 1000 operations + Gen2 : GC Generation 2 collects per 1000 operations + Allocated : Allocated memory per single operation (managed only, inclusive, 1KB = 1024B) + 1 s : 1 Second (1 sec) + +// * Diagnostic Output - MemoryDiagnoser * + +// * Diagnostic Output - ThreadingDiagnoser * + + +// ***** BenchmarkRunner: End ***** +Run time: 00:00:25 (25.83 sec), executed benchmarks: 1 + +Global total time: 00:00:38 (38.78 sec), executed benchmarks: 1 +// * Artifacts cleanup * +Artifacts cleanup is finished diff --git a/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md b/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md new file mode 100644 index 000000000..26e495fc8 --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md @@ -0,0 +1,15 @@ +``` + +BenchmarkDotNet v0.15.8, macOS Tahoe 26.6.2 (25G83) [Darwin 25.6.0] +Apple M4, 1 CPU, 10 logical and 10 physical cores +.NET SDK 10.0.201 + [Host] : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + Job-HILDPN : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + +InvocationCount=1 IterationCount=3 LaunchCount=1 +UnrollFactor=1 WarmupCount=0 + +``` +| Method | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Gen2 | Allocated | +|--------------------------------------- |--------:|--------:|---------:|-----------:|---------------------:|-----------------:|-----------:|----------:|----------:| +| Archive_Step_V1_Representative_Azurite | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 28623.0000 | 429.0000 | 11000.0000 | 2000.0000 | 407.74 MB | diff --git a/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv b/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv new file mode 100644 index 000000000..7127f63e1 --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv @@ -0,0 +1,2 @@ +Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,Gen0,Completed Work Items,Lock Contentions,Gen1,Gen2,Allocated +Archive_Step_V1_Representative_Azurite,Job-HILDPN,False,Default,Default,Default,Default,Default,Default,0000000000,Empty,RyuJit,Default,Arm64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 10.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,1,3,Default,1,Default,Default,Default,Default,Default,Default,1,0,3.808 s,3.631 s,0.1990 s,33000.0000,28623.0000,429.0000,11000.0000,2000.0000,407.74 MB diff --git a/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html b/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html new file mode 100644 index 000000000..9221acfaf --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260908T085224.165Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html @@ -0,0 +1,32 @@ + + + + +Arius.Benchmarks.ArchiveStepBenchmarks-20260908-105237 + + + + +

+BenchmarkDotNet v0.15.8, macOS Tahoe 26.6.2 (25G83) [Darwin 25.6.0]
+Apple M4, 1 CPU, 10 logical and 10 physical cores
+.NET SDK 10.0.201
+  [Host]     : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a
+  Job-HILDPN : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a
+
+
InvocationCount=1  IterationCount=3  LaunchCount=1  
+UnrollFactor=1  WarmupCount=0  
+
+ + + + + +
Method MeanErrorStdDevGen0Completed Work ItemsLock ContentionsGen1Gen2Allocated
Archive_Step_V1_Representative_Azurite3.808 s3.631 s0.1990 s33000.000028623.0000429.000011000.00002000.0000407.74 MB
+ + From 6bb3aed30ddeba7dc684b77b29875c9f4c755b05 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 12:29:29 +0200 Subject: [PATCH 15/46] chore: update docstrings --- docs/design/core/shared/streaming.md | 4 +- src/Arius.Benchmarks/AllocationBenchmarks.cs | 44 +++++-------------- .../Archive/ArchiveProgressHandlers.cs | 10 +---- src/Arius.Cli/ProgressState.cs | 6 +-- .../ChunkStorageServiceUploadTests.cs | 4 +- .../Shared/Streaming/ProgressStreamTests.cs | 4 +- .../Features/ArchiveCommand/Models.cs | 5 +-- .../Features/ArchiveCommand/TarBuilder.cs | 13 +----- .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 28 +++--------- .../Shared/ChunkIndex/ChunkIndexService.cs | 5 +-- .../ChunkStorage/ChunkStorageService.cs | 8 +--- .../Shared/FileSystem/PathSegment.cs | 4 +- .../Shared/FileSystem/RelativeFileSystem.cs | 11 +---- .../Shared/FileSystem/RelativePath.cs | 4 +- .../Shared/FileTree/FileTreeService.cs | 5 +-- .../Shared/FileTree/FileTreeStagingWriter.cs | 39 +++------------- .../Shared/HashCache/SparseFingerprint.cs | 8 +--- .../Shared/HashCache/SparseSamplingStream.cs | 3 +- src/Arius.Core/Shared/Hashes/HashCodec.cs | 7 +-- .../Shared/Streaming/ProgressStream.cs | 22 +++------- 20 files changed, 52 insertions(+), 182 deletions(-) diff --git a/docs/design/core/shared/streaming.md b/docs/design/core/shared/streaming.md index 6d047d1b4..d7c3ec146 100644 --- a/docs/design/core/shared/streaming.md +++ b/docs/design/core/shared/streaming.md @@ -21,7 +21,7 @@ flowchart LR tee -. "tee for inline verify" .-> ver[RoundTripVerifier] ``` -- **`ProgressStream(inner, IProgress)`** wraps the *source* at the top of the chain so progress tracks logical bytes read off disk, not compressed bytes on the wire. `Length` is delegated to the inner `FileStream`, so the consumer knows the total up front and can compute a percentage. Reports are **coalesced to at most one per 500 ms**: the first read reports immediately (so a progress bar moves at once), later reads report only once the interval has elapsed, and the true total is always emitted at EOF and again on dispose. A zero-length source still reports nothing. The download path reuses it the same way, wrapping the blob's read stream. +- **`ProgressStream(inner, IProgress)`** wraps the *source* at the top of the chain so progress tracks logical bytes read off disk, not compressed bytes on the wire. `Length` is delegated to the inner `FileStream`, so the consumer knows the total up front and can compute a percentage. Reports are **coalesced to at most one per 500 ms**: the first read reports immediately, later reads wait for the interval, and the true total is emitted at EOF or disposal. Empty sources report nothing. The download path reuses it the same way, wrapping the blob's read stream. - **`CountingStream(inner)`** sits at the *bottom* of the chain, directly above `OpenWriteAsync`, and increments `BytesWritten` on every write. It is read *after* the chain is disposed to capture the final compressed-and-encrypted blob size, which `UploadChunkAsync` then writes into blob metadata (`chunk-size`). The note in `UploadChunkAsync` is load-bearing here: the encryption stream is disposed *explicitly* before reading `BytesWritten`, because GCM flushes its final auth tag on dispose — reading the count earlier would undercount by the tag bytes. @@ -40,5 +40,5 @@ Splitting "report read progress" and "count written bytes" into separate one-lin ## Open seams / future -- `ProgressStream` throttles to one report per 500 ms at the source, because consumers wrap the callback in `Progress`, which posts a thread-pool work item per report — one per ~64-80 KiB of every archived and restored byte before this. `UploadChunkAsync`'s `CallbackProgress` still de-dupes non-increasing reports on top of it. The interval is a fixed constant, not a per-consumer setting. +- `ProgressStream` throttles reports at the source to one per 500 ms. `UploadChunkAsync`'s `CallbackProgress` still de-dupes non-increasing reports on top of it. The interval is a fixed constant, not a per-consumer setting. - Any future upload layer (e.g. a second integrity tee) slots into the same push chain in `UploadChunkAsync` between source and `OpenWriteAsync`; these two wrappers stay unchanged as the progress/size endpoints. diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index eaade8ec5..734655dd3 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -12,26 +12,16 @@ namespace Arius.Benchmarks; /// -/// In-process allocation micro-benchmarks for the byte-pushing components of Arius.Core. -/// -/// These exist because is end-to-end against Azurite, where fixture -/// and SDK overhead dominate and no single optimization is separable. Each benchmark here isolates one -/// component so a change's effect on Allocated is directly attributable. -/// -/// Needs no Azurite and no Docker. Run with --class micro. +/// In-process allocation benchmarks for selected Arius.Core components. +/// Run with --class micro; no Azurite or Docker is required. /// [MemoryDiagnoser] public class AllocationBenchmarks { private const int EntryCount = 1_000; - // ── Hash codec ─────────────────────────────────────────────────────────────── - /// - /// A digest whose hex form actually contains a-f nibbles. This matters: a digest of all-zero - /// bytes hexes to "000...0", and then returns the same instance via its - /// no-change fast path — hiding the second allocation that ToLowerHex really makes on realistic input. - /// Deliberately not , which pins a leading 0x00 for the range-query fixtures. + /// Digest containing hexadecimal letters, ensuring the lowercase conversion path is exercised. /// private readonly byte[] _digest = CreateHighNibbleDigest(); @@ -41,7 +31,7 @@ public class AllocationBenchmarks [Benchmark(Description = "HashCodec.ToLowerHex")] public string HashCodec_ToLowerHex() => HashCodec.ToLowerHex(_digest); - /// The common case: parsing a value Arius itself wrote, already canonical lowercase. + /// Parses an already canonical lowercase value. [Benchmark(Description = "HashCodec.NormalizeHex (already canonical)")] public string HashCodec_NormalizeHex_Canonical() => HashCodec.NormalizeHex(CanonicalHex); @@ -55,10 +45,7 @@ public class AllocationBenchmarks private byte[] _readBuffer = null!; - /// - /// The dominant real-world shape: a sub-1 MiB file, where Regions returns a single region - /// equal to the whole file, so the sampler buffers the entire file while it is hashed. - /// + /// Exercises the single-region sampler path for a small file. [Benchmark(Description = "SparseFingerprint.Sampler small file (200 KB)")] public byte[] SparseFingerprint_Sampler_SmallFile() { @@ -75,7 +62,7 @@ public byte[] SparseFingerprint_Sampler_SmallFile() return sampler.Finish(); } - /// Worst case for the capture buffers: 64 regions x 256 KiB = 16 MiB per in-flight file. + /// Exercises the maximum sampled-region buffer size. [Benchmark(Description = "SparseFingerprint.Sampler large file (64 GB logical)")] public byte[] SparseFingerprint_Sampler_LargeFile() { @@ -102,10 +89,7 @@ public byte[] SparseFingerprint_Sampler_LargeFile() private byte[] _tarEntryPayload = null!; private IEncryptionService _encryption = null!; - /// - /// Accumulates and seals one full 64 MB bundle. The per-entry wrappers are - /// ~100 bytes each and negligible against the bundle buffer this is measuring. - /// + /// Builds and seals one 64 MB TAR bundle. [Benchmark(Description = "TarBuilder seal one 64 MB bundle")] public async Task TarBuilder_Seal_64MB() { @@ -141,10 +125,7 @@ public int ChunkIndexLocalStore_ReadRangeEntries() return count; } - /// - /// The per-hash lookup shape that ChunkIndexService.LookupAsync runs 256 times per "batch", - /// twice over (pending-flush probe then entry probe). - /// + /// Measures the per-hash lookup shape for a 256-hash deduplication batch. [Benchmark(Description = "ChunkIndexLocalStore.FindEntry x256 (one dedup batch)")] public int ChunkIndexLocalStore_FindEntry_256() { @@ -156,9 +137,7 @@ public int ChunkIndexLocalStore_FindEntry_256() return found; } - /// - /// The batched replacement for the above: one IN (...) query for the whole 256-hash dedup batch. - /// + /// Measures the batched lookup for a 256-hash deduplication batch. [Benchmark(Description = "ChunkIndexLocalStore.FindEntries x1 (one dedup batch)")] public int ChunkIndexLocalStore_FindEntries_Batch() => _store.FindEntries(_lookupHashes).Count; @@ -202,8 +181,7 @@ public void Cleanup() // ── Deterministic fixtures ─────────────────────────────────────────────────── /// - /// A distinct 32-byte digest per index, always with a leading 0x00 so every generated hash falls - /// under the "00" prefix that ranges over. + /// Creates distinct digests under the "00" range prefix used by the range benchmark. /// private static byte[] CreateDigest(int seed) { @@ -214,7 +192,7 @@ private static byte[] CreateDigest(int seed) private static ContentHash CreateContentHash(int seed) => ContentHash.FromDigest(CreateDigest(seed)); - /// A deterministic 32-byte digest whose every byte has a high nibble in the a-f range. + /// Creates a digest containing hexadecimal letters. private static byte[] CreateHighNibbleDigest() { var digest = new byte[32]; diff --git a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs index a5e12c433..2d46e6c11 100644 --- a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs +++ b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs @@ -149,16 +149,8 @@ public ValueTask Handle(TarBundleSealingEvent notification, CancellationToken ca } } -// ── FileDedupedHandler ──────────────────────────────────────────────────────── - /// -/// Removes the rows for a file that deduplicated away. -/// -/// Without this there was no FileDedupedEvent handler at all — the Mediator source generator warns -/// "found message without any registered handler" for it — so a deduped file stayed in -/// in state Hashed for the whole run. That dictionary grew -/// unboundedly with the deduped file count *and* is snapshotted by the display roughly ten times a -/// second, making the redraw cost scale with total files archived rather than files in flight. +/// Removes tracked files when their content is deduplicated. /// public sealed class FileDedupedHandler(ProgressState state) : INotificationHandler { diff --git a/src/Arius.Cli/ProgressState.cs b/src/Arius.Cli/ProgressState.cs index 3d182eadc..0380309fc 100644 --- a/src/Arius.Cli/ProgressState.cs +++ b/src/Arius.Cli/ProgressState.cs @@ -323,11 +323,7 @@ public void SkipFileDuringHashing(RelativePath relativePath) public ConcurrentDictionary TrackedTars { get; } = new(); /// - /// The bundle currently accumulating entries, or null between bundles. Held directly because - /// TarEntryAddedHandler needs it once per small file, and deriving it from - /// meant a Values snapshot plus a Where/OrderByDescending scan per file. - /// Only the single-threaded TarBuilder stage raises the bundle lifecycle events, so writes are - /// ordered; the reference is published via for the display thread. + /// The bundle currently receiving entries, or null between bundles. /// public TrackedTar? AccumulatingTar { diff --git a/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs b/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs index c5640c6d0..6a7e48dbd 100644 --- a/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs +++ b/src/Arius.Core.Tests/Shared/ChunkStorage/ChunkStorageServiceUploadTests.cs @@ -224,9 +224,7 @@ public async Task UploadLargeAsync_RetryAfterMetadataConflict_ReportsSingleProgr result.AlreadyExisted.ShouldBeFalse(); blobs.DeletedBlobNames.ShouldContain(blobName); - // The point of this test is that the retry does not restart or duplicate the progress sequence. - // ProgressStream coalesces reports, so assert that property rather than an exact byte sequence: - // strictly increasing (no repeat, no reset) and finishing at the true total. + // Verify that retry progress is monotonic and ends at the true total; intermediate reports are throttled. reports.ShouldNotBeEmpty(); reports.ShouldBeInOrder(); reports.Distinct().Count().ShouldBe(reports.Count); diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index 8ae21dc06..a6c3d6d8c 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -17,9 +17,7 @@ public void Read_CoalescesReports_ButStillReportsTheTotal() var buf = new byte[256]; while (ps.Read(buf, 0, buf.Length) > 0) { } - // Reports are throttled to one per 500 ms (plus the first read and the EOF total), so four - // in-memory reads no longer produce four reports. What must hold is that progress starts - // promptly, never goes backwards, and ends at the true total. + // Reports are throttled, so assert prompt, monotonic progress ending at the true total rather than an exact sequence. reports.ShouldNotBeEmpty(); reports.Count.ShouldBeLessThanOrEqualTo(4); reports.ShouldBeInOrder(); diff --git a/src/Arius.Core/Features/ArchiveCommand/Models.cs b/src/Arius.Core/Features/ArchiveCommand/Models.cs index 8553baa6c..0f9718d42 100644 --- a/src/Arius.Core/Features/ArchiveCommand/Models.cs +++ b/src/Arius.Core/Features/ArchiveCommand/Models.cs @@ -29,10 +29,7 @@ internal sealed record BinaryFile public required RelativePath Path { get; init; } /// - /// Size in bytes, captured once by when the entry is discovered. - /// Carried here for the same reason carries the timestamps: so no later - /// stage re-stats the file. Enumerate, hash, and dedup all need the size, and each calling - /// GetFileSize meant four stat syscalls (and four path resolutions) per file. + /// Size in bytes captured during enumeration and reused by later archive stages. /// public required long FileSize { get; init; } } diff --git a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs index 120a30d09..7fcf2d976 100644 --- a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs +++ b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs @@ -32,18 +32,12 @@ internal sealed class TarBuilder : IAsyncDisposable private long _currentSize; /// - /// Headroom over the target size for tar framing. A bundle seals on the summed *file* sizes, but the - /// stream also holds two 512-byte header blocks plus an optional PAX extended block per entry, so the - /// serialized tar always runs over the target. Without headroom the buffer grows once more, doubling a - /// 64 MB allocation into a 128 MB one. + /// Headroom for TAR framing beyond the summed file sizes. /// private const int FramingHeadroomDivisor = 4; /// - /// Initial capacity for a bundle's backing buffer. is a and - /// takes an , so it is clamped; the clamp only engages for a - /// TarTargetSize far beyond any practical bundle. A bundle of very many very small files can still - /// exceed even the headroom and take one growth step — that is accepted, not worked around. + /// Initial backing-buffer capacity for a bundle, clamped to the limit. /// private int InitialCapacity => (int)Math.Min(_targetSize + _targetSize / FramingHeadroomDivisor, int.MaxValue / 2); @@ -77,9 +71,6 @@ public TarBuilder( TarWriter writer; if (_tarWriter is null) { - // Pre-size the bundle buffer. Growing from zero capacity allocates and abandons every - // intermediate array (256 B, 512 B, ... 32 MB, 64 MB, 128 MB) to reach one 64 MB bundle — - // measured at 276 MB of transient garbage per bundle, nearly all of it on the LOH. _tarStream = new MemoryStream(InitialCapacity); writer = _tarWriter = new TarWriter(_tarStream, leaveOpen: true); await (_onBundleStarted?.Invoke() ?? ValueTask.CompletedTask); diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index 69e55e1a0..220601364 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -97,17 +97,11 @@ public IReadOnlyDictionary FindPendingFlushEntries(IRea => FindEntriesCore(contentHashes, pendingFlushOnly: true); /// - /// Looks up a set of hashes with a single IN (...) query instead of one query per hash. - /// A dedup batch is 256 hashes and each per-hash lookup previously cost its own pooled connection, its - /// own PRAGMA synchronous round-trip, and its own command — so a "batched" lookup was issuing - /// 512 statements. + /// Looks up a set of hashes with one IN (...) query. /// /// - /// The parameter count is padded to a fixed bucket so the command text repeats across calls (batches - /// are almost always full, with one ragged tail), letting SQLite reuse the prepared statement instead - /// of compiling fresh SQL per distinct batch size. Padding slots repeat the first hash, which is - /// harmless: this is a set membership test, so duplicates cannot add rows. - /// SQLITE_MAX_VARIABLE_NUMBER is 32766 on modern SQLite, well above the largest bucket. + /// Parameters are padded to fixed buckets so the generated command text can be reused. Unused slots + /// repeat the first hash, which is harmless for set membership. /// private IReadOnlyDictionary FindEntriesCore(IReadOnlyCollection contentHashes, bool pendingFlushOnly) { @@ -141,7 +135,6 @@ private IReadOnlyDictionary FindEntriesCore(IReadOnlyCo WriteDigest(hex, digests[slot++]); } - // Pad the unused slots with a repeat of the first hash. for (; slot < slots; slot++) WriteDigest(first, digests[slot]); @@ -978,10 +971,7 @@ ON CONFLICT(content_hash) DO UPDATE SET } /// - /// Binds one entry onto a command reused across a batch. and - /// are the command's already-bound BLOB buffers, overwritten in place: - /// each ExecuteNonQuery completes before the next row rewrites them, so a batch of 256 rows binds - /// 2 arrays rather than 512. + /// Binds one row using reusable digest buffers; each execution completes before the buffers are overwritten. /// private static void BindEntry(SqliteCommand command, ShardEntry entry, byte[] contentDigest, byte[] chunkDigest) { @@ -1026,11 +1016,6 @@ ON CONFLICT(prefix) DO UPDATE SET return command.ExecuteNonQuery(); } - // Reads the hash columns with (byte[])reader.GetValue(...), which allocates one byte[32] per hash per - // row. That is deliberate: reading into a reusable buffer via SqliteDataReader.GetBytes was measured - // 5.6x WORSE on ReadRangeEntries (641 KB -> 3620 KB per 1000 rows), because the provider routes - // GetBytes through a SqliteBlob. The cast itself is free — byte[] is a reference type, so nothing is - // boxed. The remaining per-row cost here is dominated by the two hash *strings*, not the arrays. private static ShardEntry ReadEntry(SqliteDataReader reader) => new( ContentHash.FromDigest((byte[])reader.GetValue(0)), @@ -1040,13 +1025,12 @@ private static ShardEntry ReadEntry(SqliteDataReader reader) ShardEntry.DeserializeTier(reader.GetInt32(4))); /// - /// Writes the 32 digest bytes of a canonical-hex hash into . - /// The hash types guarantee exactly 64 canonical hex characters by construction, so this cannot fail. + /// Writes a canonical hexadecimal hash into a 32-byte digest buffer. /// private static void WriteDigest(string hex, byte[] destination) => Convert.FromHexString(hex, destination, out _, out _); - /// Rents a reusable 32-byte digest buffer for one command or one read loop. + /// Creates a reusable 32-byte digest buffer. private static byte[] CreateDigestBuffer() => new byte[HashCodec.Sha256ByteLength]; } diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs index 997f000af..e2762ec32 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs @@ -104,8 +104,7 @@ public async Task> LookupAsync(IEnu if (hashes.Length == 0) return result; - // One set query for the whole batch rather than one per hash. Dirty (pending-flush) rows still win - // over remote-backed rows, which is why this probe runs before validation. + // Probe pending-flush entries in one batch before remote validation. var pendingFlush = _localStore.FindPendingFlushEntries(hashes); var validationWork = new List<(PathSegment Root, List Hashes)>(); @@ -146,7 +145,7 @@ await Parallel.ForEachAsync( await EnsureCoverageForHashesAsync(item.Root, item.Hashes, latestSnapshotName, ct); }); - // Construct the result from the validated shards, one set query per root rather than one per hash. + // Populate the result from validated shards using batched lookups. foreach (var item in validationWork) { foreach (var (contentHash, entry) in _localStore.FindEntries(item.Hashes)) diff --git a/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs b/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs index 4459a5408..d6a07bd94 100644 --- a/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs +++ b/src/Arius.Core/Shared/ChunkStorage/ChunkStorageService.cs @@ -307,11 +307,7 @@ private sealed class ChunkDownloadStream(Stream inner) : Stream public override void Flush() => inner.Flush(); public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); - // The span/memory overrides are load-bearing, not tidiness. Without them Stream's base - // implementations route ReadAsync(Memory) back through ReadAsync(byte[], ...) and on to - // BeginEndReadAsync, which blocks a thread-pool thread on the synchronous Read for every buffer of - // every restored byte — even though every stream underneath (AesGcmDecryptingStream, - // AutoDetectDecompressionStream, PrefixedStream, ProgressStream) implements the async path properly. + // Override the async members so restore uses the inner stream's asynchronous read and copy paths. public override int Read(Span buffer) => inner.Read(buffer); public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => inner.ReadAsync(buffer, offset, count, cancellationToken); public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => inner.ReadAsync(buffer, cancellationToken); @@ -348,4 +344,4 @@ private void DisposeResources() inner.Dispose(); } } -} \ No newline at end of file +} diff --git a/src/Arius.Core/Shared/FileSystem/PathSegment.cs b/src/Arius.Core/Shared/FileSystem/PathSegment.cs index 9ed801bbd..c791f55dd 100644 --- a/src/Arius.Core/Shared/FileSystem/PathSegment.cs +++ b/src/Arius.Core/Shared/FileSystem/PathSegment.cs @@ -74,9 +74,7 @@ public PathSegment RemoveSuffix(string suffix, StringComparison comparisonType) public override string ToString() => Value; /// - /// Allocation-free replacement for value.Any(char.IsControl), which allocates a CharEnumerator - /// on every call. Parse runs for every blob path and every filetree entry, so it is hot. - /// foreach over a string is compiled to indexer access, so this allocates nothing. + /// Allocation-free replacement for value.Any(char.IsControl) /// private static bool ContainsControlCharacter(string value) { diff --git a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs index c2fa37e79..c414423aa 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs @@ -250,20 +250,13 @@ public async Task WriteAllTextAsync(RelativePath path, string content, Cancellat } /// - /// Appends text to a file within the rooted directory, creating the file (and parent directories) if needed. - /// - /// - /// Opens a file for appending, creating it if needed, and leaves the handle open for the caller to - /// reuse across writes. opens, writes, and closes per call — fine for - /// one-off appends, wasteful when the same file is appended to repeatedly. + /// Opens a file for appending, creating it if necessary, and leaves the handle open for reuse. /// public Stream OpenAppend(RelativePath path) { var fullPath = root.Resolve(path); CreateDirectory(path.Parent ?? RelativePath.Root); - // FileShare.Read matches what File.AppendAllTextAsync used, so a reader can still open the file - // while the handle is held — FileTreeBuilder reads staged nodes, and holding them exclusively for - // the length of an archive would be a behaviour change, not just a test inconvenience. + // Preserve read sharing so staged nodes remain readable while the append handle is open. return new FileStream(fullPath, FileMode.Append, FileAccess.Write, FileShare.Read, 65536, useAsync: true); } diff --git a/src/Arius.Core/Shared/FileSystem/RelativePath.cs b/src/Arius.Core/Shared/FileSystem/RelativePath.cs index bbfca1834..02c05c717 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativePath.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativePath.cs @@ -166,9 +166,7 @@ public RelativePath RemoveSuffix(string suffix, StringComparison comparisonType) public override string ToString() => Value; /// - /// Allocation-free replacement for value.Any(char.IsControl), which allocates a CharEnumerator - /// on every call. Parse runs for every blob path and every filetree entry, so it is hot. - /// foreach over a string is compiled to indexer access, so this allocates nothing. + /// Allocation-free replacement for value.Any(char.IsControl) /// private static bool ContainsControlCharacter(string value) { diff --git a/src/Arius.Core/Shared/FileTree/FileTreeService.cs b/src/Arius.Core/Shared/FileTree/FileTreeService.cs index d12ca552d..817ed76ca 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeService.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeService.cs @@ -244,10 +244,7 @@ private async Task SerializeStorageAsync(ReadOnlyMemory plaintext, await compressionStream.WriteAsync(plaintext, cancellationToken); } - // ToArray() and not ToArraySegment(): disposing the encryption/compression chain above also closes - // `ms`, and ToArray() is the only MemoryStream buffer accessor that stays valid after close - // (Length and TryGetBuffer both throw ObjectDisposedException). Avoiding this copy would need a - // leaveOpen on IEncryptionService.WrapForEncryption, or a non-closing stream shim. + // The codec chain closes ms; ToArray remains valid after close, unlike buffer-based accessors. return ms.ToArray(); } diff --git a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs index 376e4d62b..85ccaab71 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs @@ -14,10 +14,7 @@ internal sealed class FileTreeStagingWriter : IDisposable private bool _disposed; /// - /// One lock plus one open append handle. Every write to a node goes through its stripe's gate, so the - /// gate also guards that stripe's handle — which is what makes keeping it open safe without a second - /// lock or a race between a write and an eviction. Handles are bounded by , - /// which matters on the small NAS hardware Arius targets. + /// One gate and one reusable append handle per stripe; the gate protects both. /// private sealed class Stripe : IDisposable { @@ -32,12 +29,7 @@ public void Dispose() } } - // A directory id is globally unique to its full path, so its parent→child edge line is identical - // no matter which descendant file triggers it. Emit each edge once: without this, a deep tree - // re-appends every ancestor edge for every file (the root node once per file), which both dominates - // the staging I/O and funnels all writers onto the root node's single stripe lock. The reader - // (FileTreeBuilder.ReadNodeEntriesAsync) already collapses duplicate directory entries, so writing - // each once is behaviourally identical. Bounded by directory count and released with the writer. + // Emit each directory edge once; the reader treats duplicate edges as equivalent. private readonly ConcurrentDictionary _emittedDirectories = new(); public FileTreeStagingWriter(LocalDirectory stagingRoot) @@ -81,9 +73,6 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati { var currentPath = RelativePath.Root; - // Materialize once: RelativePath.Segments re-splits and re-parses the path on every - // enumeration, so Take(Segments.Count() - 1) would parse the whole path twice on this hot - // staging path. Iterate the segments by index instead, skipping the trailing file segment. var segments = filePath.Segments.ToArray(); for (var i = 0; i < segments.Length - 1; i++) @@ -93,8 +82,7 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati currentPath = currentPath / segment; var directoryId = FileTreePaths.GetStagingDirectoryId(currentPath); - // Claim the edge (but keep descending) so concurrent writers don't double-emit it. - // TryAdd is atomic: exactly one writer wins the claim and writes the edge. + // Claim each edge once; failed writes release the claim below. if (!_emittedDirectories.TryAdd(directoryId, true)) continue; @@ -106,9 +94,7 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati } catch { - // The claim must reflect a committed edge: if the append fails (I/O error or - // cancellation), release it so a later writer can re-emit. Otherwise the parent→child - // edge is permanently skipped and its subtree orphaned. + // Allow a later writer to retry an edge whose append failed. _emittedDirectories.TryRemove(directoryId, out _); throw; } @@ -117,17 +103,12 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati private async Task AppendLineAsync(RelativePath path, string line, CancellationToken cancellationToken) { - // path.GetHashCode(), not StringComparer.Ordinal.GetHashCode(path): RelativePath is a struct, so - // the latter bound to IEqualityComparer.GetHashCode(object) — boxing on every append and then - // falling through to path.GetHashCode() anyway, so the comparer was doing nothing. var stripe = _lockStripes[(uint)path.GetHashCode() % (uint)_lockStripes.Length]; await stripe.Gate.WaitAsync(cancellationToken); try { - // Reuse this stripe's handle when it is already pointed at the node. The archive walk is - // depth-first, so consecutive files land in the same directory and hit the same node, making - // this the common case. Re-targeting closes the previous handle first. + // Reuse the open handle for this node; close it when retargeting. if (stripe.OpenPath != path) { stripe.Handle?.Dispose(); @@ -135,9 +116,7 @@ private async Task AppendLineAsync(RelativePath path, string line, CancellationT stripe.OpenPath = path; } - // Fixed '\n' (not Environment.NewLine): staged lines are re-serialized by FileTreeSerializer - // before hashing, but keep the staging format platform-independent and consistent with it. - // Written as UTF-8 bytes with no BOM, matching what File.AppendAllTextAsync produced. + // Keep the staging format platform-independent and consistent with FileTreeSerializer. var buffer = ArrayPool.Shared.Rent(Encoding.UTF8.GetMaxByteCount(line.Length) + 1); try { @@ -146,11 +125,7 @@ private async Task AppendLineAsync(RelativePath path, string line, CancellationT await stripe.Handle!.WriteAsync(buffer.AsMemory(0, count), cancellationToken); - // Flush per line rather than buffering. Staging is disposable scratch, so buffering would - // be durable enough, but flushing keeps the failure semantics exact: a write error surfaces - // from this call, which is what the caller's claim-release in AppendDirectoryEntriesAsync - // depends on. The win here is dropping the per-line open/create-directory/close, not the - // write itself. + // Surface append failures before releasing the directory-edge claim. await stripe.Handle.FlushAsync(cancellationToken); } finally diff --git a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs index 3afa8ba8a..ed645b079 100644 --- a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs +++ b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs @@ -95,9 +95,7 @@ public sealed class Sampler : IDisposable private readonly IReadOnlyList<(long Off, int Len)> _regions; /// - /// The regions laid out back-to-back in one pooled buffer, so can hash the whole - /// span in one call — byte-identical to hashing each region in order, which is what the framing - /// contract with requires. + /// Stores sampled regions contiguously so can hash them in one pass. /// private readonly byte[] _buffer; private readonly int[] _bufferOffsets; @@ -121,9 +119,7 @@ public Sampler(long size) _capturedLength = total; _buffer = ArrayPool.Shared.Rent(total); - // A rented buffer arrives dirty. A region that is never offered to Capture — a file that shrank - // mid-read, or a read that stopped early — must contribute zeros, exactly as the previous - // per-region `new byte[]` did, or the fingerprint stops being a function of the content. + // Clear uncaptured regions because pooled buffers are not initialized. _buffer.AsSpan(0, _capturedLength).Clear(); } diff --git a/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs b/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs index 23dea8831..897841988 100644 --- a/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs +++ b/src/Arius.Core/Shared/HashCache/SparseSamplingStream.cs @@ -65,8 +65,7 @@ protected override void Dispose(bool disposing) { if (disposing) { - // Returns the sampler's pooled capture buffer. Fingerprint() must therefore be called before - // this stream is disposed — which is the existing call order in ArchiveCommandHandler. + // Fingerprint must be called before disposal releases the sampler buffer. _sampler.Dispose(); _inner.Dispose(); } diff --git a/src/Arius.Core/Shared/Hashes/HashCodec.cs b/src/Arius.Core/Shared/Hashes/HashCodec.cs index 486008245..e1434b244 100644 --- a/src/Arius.Core/Shared/Hashes/HashCodec.cs +++ b/src/Arius.Core/Shared/Hashes/HashCodec.cs @@ -36,9 +36,7 @@ public static string NormalizeHex(string value) } } - // The dominant case is re-parsing a value Arius itself wrote (SQLite, snapshot JSON, blob names, - // pointer files), which is already canonical lowercase. Returning the input then avoids allocating - // a second identical string. Validation above has still run over every character. + // Preserve canonical input to avoid allocating a duplicate string. return alreadyCanonical ? value : new string(chars); } @@ -47,9 +45,6 @@ public static string ToLowerHex(ReadOnlySpan digest) if (digest.Length != Sha256ByteLength) throw new ArgumentException($"Expected {Sha256ByteLength}-byte SHA-256 digest.", nameof(digest)); - // Not Convert.ToHexString(...).ToLowerInvariant(): that allocates the uppercase string and then a - // second lowercased copy. This is the funnel every ContentHash/ChunkHash/FileTreeHash construction - // passes through. return Convert.ToHexStringLower(digest); } } diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index d65e29dce..ec5cc2f40 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -3,15 +3,9 @@ namespace Arius.Core.Shared.Streaming; /// -/// Read-mode stream wrapper that reports cumulative bytes read via . -/// Delegates all reads to the inner stream and does not buffer any data. -/// -/// The first read reports immediately; after that reports are coalesced to at most one per -/// . Consumers wrap the callback in -/// , which posts a thread-pool work item per report, so reporting after every -/// read queued one work item per buffer — roughly one per 64-80 KiB of every archived and restored byte. -/// The true total is always emitted once the source reaches EOF (a read returning 0), and again on -/// dispose for a stream abandoned before EOF, so a consumer never ends up short of the real figure. +/// Read-mode stream wrapper that reports cumulative bytes read without buffering. +/// The first read reports immediately; later reports are limited to one per +/// . The final total is emitted at EOF or disposal. /// public sealed class ProgressStream : Stream { @@ -39,15 +33,12 @@ public ProgressStream(Stream inner, IProgress progress) } /// - /// Reports the running total, but at most once per . - /// Uses rather than wall-clock time so it is monotonic. + /// Reports the running total at most once per using monotonic time. /// private void ReportThrottled() { var now = Stopwatch.GetTimestamp(); - // The first read always reports, so a consumer sees work start immediately rather than after a - // blank interval — and so a stream consumed in a single read still reports before EOF. if (_hasReported && Stopwatch.GetElapsedTime(_lastReportTimestamp, now) < ReportInterval) return; @@ -58,12 +49,11 @@ private void ReportThrottled() } /// - /// Emits the running total if the throttle has held anything back. Called at EOF and on dispose, so a - /// consumer is never left short of the real figure by up to one interval's worth of bytes. + /// Reports any total withheld by throttling at EOF or disposal. /// private void ReportFinal() { - // A source that yielded nothing reports nothing — an empty stream must not emit a spurious 0. + // Empty sources do not emit a spurious zero. if (_bytesRead == 0) return; From 98d6f8de6a1b80f8b1e3ed14a37249caf8a5c93b Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:05:22 +0200 Subject: [PATCH 16/46] Revert "perf(archive): stat each file once instead of four times" This reverts commit 0ae0798cca2a0949e646ffae2f0f3dbefdcb62bf. --- .../Features/ArchiveCommand/ArchiveCommandHandler.cs | 8 ++++---- .../Features/ArchiveCommand/LocalFileEnumerator.cs | 1 - src/Arius.Core/Features/ArchiveCommand/Models.cs | 5 ----- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs b/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs index 5cca55704..f9c36df9c 100644 --- a/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs @@ -313,7 +313,7 @@ public async ValueTask Handle(ArchiveCommand command, Cancellatio await foreach (var pair in pairs) { count++; - var fileSize = pair.Binary?.FileSize ?? 0L; + var fileSize = pair.Binary is null ? 0L : fs.GetFileSize(pair.RelativePath); totalBytes += fileSize; await _mediator.Publish(new FileScannedEvent(pair.RelativePath, fileSize), cancellationToken); await filePairChannel.Writer.WriteAsync(pair, cancellationToken); @@ -342,7 +342,7 @@ await Parallel.ForEachAsync( { try { - var fileSize = pair.Binary?.FileSize ?? 0L; + var fileSize = pair.Binary is null ? 0L : fs.GetFileSize(pair.RelativePath); await _mediator.Publish(new FileHashingEvent(pair.RelativePath, fileSize), ct); @@ -491,14 +491,14 @@ async ValueTask FullHashAndRecordAsync(RelativePath relativePath, l _logger.LogInformation("[dedup] {Path} -> hit ({Hash})", hashed.FilePair.RelativePath, hashed.ContentHash.Short8); await fileTreeEntryChannel.Writer.WriteAsync(hashed, cancellationToken); Interlocked.Increment(ref filesDeduped); - var size = hashed.FilePair.Binary!.FileSize; + var size = fs.GetFileSize(hashed.FilePair.RelativePath); Interlocked.Add(ref originalSize, size); await _mediator.Publish(new FileDedupedEvent(hashed.ContentHash, size), cancellationToken); } else { // Needs upload → mark in-flight, route by size - var fileSize = hashed.FilePair.Binary!.FileSize; + var fileSize = fs.GetFileSize(hashed.FilePair.RelativePath); inFlightHashes.TryAdd(hashed.ContentHash, fileSize); Interlocked.Add(ref originalSize, fileSize); Interlocked.Add(ref incrementalSize, fileSize); diff --git a/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs b/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs index 48611b359..0867e1d9b 100644 --- a/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs +++ b/src/Arius.Core/Features/ArchiveCommand/LocalFileEnumerator.cs @@ -114,7 +114,6 @@ private async IAsyncEnumerable EnumerateDirectoryAsync(RelativeFileSys Binary = new BinaryFile { Path = relativePath, - FileSize = fileSystem.GetFileSize(relativePath) }, Pointer = hasPointer ? new PointerFile diff --git a/src/Arius.Core/Features/ArchiveCommand/Models.cs b/src/Arius.Core/Features/ArchiveCommand/Models.cs index 0f9718d42..0862a5d3e 100644 --- a/src/Arius.Core/Features/ArchiveCommand/Models.cs +++ b/src/Arius.Core/Features/ArchiveCommand/Models.cs @@ -27,11 +27,6 @@ internal sealed record FilePair internal sealed record BinaryFile { public required RelativePath Path { get; init; } - - /// - /// Size in bytes captured during enumeration and reused by later archive stages. - /// - public required long FileSize { get; init; } } /// From 18b41a1446e676fcaba228278741d3b87b932a0f Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:07:34 +0200 Subject: [PATCH 17/46] test(benchmarks): exercise large sampler captures --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index 734655dd3..7c470cc81 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -67,6 +67,10 @@ public byte[] SparseFingerprint_Sampler_SmallFile() public byte[] SparseFingerprint_Sampler_LargeFile() { using var sampler = new SparseFingerprint.Sampler(LargeFileSize); + + foreach (var (offset, length) in SparseFingerprint.Regions(LargeFileSize)) + sampler.Capture(offset, _readBuffer.AsSpan(0, length)); + return sampler.Finish(); } @@ -146,7 +150,7 @@ public int ChunkIndexLocalStore_FindEntry_256() [GlobalSetup] public void Setup() { - _readBuffer = new byte[81920]; + _readBuffer = new byte[256 * 1024]; _tarEntryPayload = new byte[TarEntrySize]; Random.Shared.NextBytes(_readBuffer); Random.Shared.NextBytes(_tarEntryPayload); From 6e2e6b43f02d0370fd19651725d3b788158aa21a Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:07:54 +0200 Subject: [PATCH 18/46] docs(benchmarks): fix benchmark table notes column --- src/Arius.Benchmarks/benchmark-tail.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Arius.Benchmarks/benchmark-tail.md b/src/Arius.Benchmarks/benchmark-tail.md index 18df09dcf..aa901ab5e 100644 --- a/src/Arius.Benchmarks/benchmark-tail.md +++ b/src/Arius.Benchmarks/benchmark-tail.md @@ -1,5 +1,5 @@ -| ComputerName | DateTimeUtc | Git Head | RepresentativeScaleDivisor | Iterations | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | Completed Work Items | Lock Contentions | RawOutputPath | -| ------------- | --------------------------------- | ---------------------------------------- | -------------------------- | ---------- | -------- | --------- | -------- | ----------- | ----------- | ---------- | --------- | -------------------- | ---------------- | --------------------------------------------- | +| ComputerName | DateTimeUtc | Git Head | RepresentativeScaleDivisor | Iterations | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | Completed Work Items | Lock Contentions | RawOutputPath | Notes | +| ------------- | --------------------------------- | ---------------------------------------- | -------------------------- | ---------- | -------- | --------- | -------- | ----------- | ----------- | ---------- | --------- | -------------------- | ---------------- | --------------------------------------------- | ----- | | woutbook6 | 2026-04-28T04:05:05.9286260+00:00 | 391911db0ae4c2bd3f7871be4e1bd38295770521 | 8 | 3 | 25.75 s | 3.873 s | 0.212 s | 120000.0000 | 27000.0000 | 15000.0000 | 1.08 GB | 74874.0000 | - | src/Arius.Benchmarks/raw/20260428T040505.928Z | # Baseline woutbook run | | runnervmeorf1 | 2026-04-28T04:26:53.0615861+00:00 | 4eb6d183d1984113f94c69d2e1e5832aecefdc0f | 8 | 3 | 39.12 s | 5.755 s | 0.315 s | 66000.0000 | 17000.0000 | 13000.0000 | 1.07 GB | 72894.0000 | 2.0000 | src/Arius.Benchmarks/raw/20260428T042653.061Z | # Baseline GH runner | | woutbook6 | 2026-04-28T04:45:04.1518940+00:00 | 0eccc408f7eb421a76563d69b13d4de9b8a86c0a | 8 | 3 | 25.68 s | 2.462 s | 0.135 s | 87000.0000 | 23000.0000 | 16000.0000 | 840.47 MB | 72540.0000 | 2.0000 | src/Arius.Benchmarks/raw/20260428T044504.151Z | # Run on Woutbook after refactor to hashes | @@ -12,4 +12,4 @@ | woutbook6 | 2026-05-01T15:23:58.2515390+00:00 | 1eedb04c4fe77ae90f7faa384c2c7b7dedbe0600 | 1 | 3 | 1.453 m | 0.1986 m | 0.0109 m | 434000.0000 | 54000.0000 | 4000.0000 | 3.52 GB | 506427.0000 | 15.0000 | src/Arius.Benchmarks/raw/20260501T152358.251Z | | woutbook6 | 2026-05-01T16:44:07.0942170+00:00 | 1eedb04c4fe77ae90f7faa384c2c7b7dedbe0600 | 1 | 3 | 33.16 s | 1.135 s | 0.062 s | 56000.0000 | 5000.0000 | | 444.88 MB | 39835.0000 | 2.0000 | src/Arius.Benchmarks/raw/20260501T164407.094Z | | woutbook6 | 2026-05-01T19:11:23.3076330+00:00 | 468d2479d64cef776b02c14f98ed9b667ecd2b46 | 8 | 3 | 7.352 s | 2.350 s | 0.1288 s | 14000.0000 | 3000.0000 | | 115.94 MB | 7893.0000 | 1.0000 | src/Arius.Benchmarks/raw/20260501T191123.307Z | -| woutbook6 | 2026-09-08T08:52:24.1656440+00:00 | 10947e7bbb38e1161669cdb9aebe90af17384759 | 1 | 3 | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 11000.0000 | 2000.0000 | 407.74 MB | 28623.0000 | 429.0000 | src/Arius.Benchmarks/raw/20260908T085224.165Z | # After the allocation-optimization branch; controlled base run on the same machine/session was 478.22 MB (ee4e9f3e). Time/GC counts on this host are too noisy to compare (3 iterations, no warmup, Error +/-31 s on the base). +| woutbook6 | 2026-09-08T08:52:24.1656440+00:00 | 10947e7bbb38e1161669cdb9aebe90af17384759 | 1 | 3 | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 11000.0000 | 2000.0000 | 407.74 MB | 28623.0000 | 429.0000 | src/Arius.Benchmarks/raw/20260908T085224.165Z | # After the allocation-optimization branch; controlled base run on the same machine/session was 478.22 MB (ee4e9f3e). Time/GC counts on this host are too noisy to compare (3 iterations, no warmup, Error +/-31 s on the base). | From 90c88455c85df4eb205f6f086267995e7b3e44c8 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:08:02 +0200 Subject: [PATCH 19/46] fix(benchmarks): reject archive filters --- src/Arius.Benchmarks/BenchmarkRunOptions.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Arius.Benchmarks/BenchmarkRunOptions.cs b/src/Arius.Benchmarks/BenchmarkRunOptions.cs index 82d62cbb4..83ff3ef0d 100644 --- a/src/Arius.Benchmarks/BenchmarkRunOptions.cs +++ b/src/Arius.Benchmarks/BenchmarkRunOptions.cs @@ -55,6 +55,9 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) } } + if (filter is not null && benchmarkClass is not BenchmarkClass.Micro) + throw new ArgumentException("--filter is only supported with --class micro."); + return new( repositoryRoot, Path.GetFullPath(rawOutputRoot), From 039cdee89300b2b534955fcf693c51d473083f3d Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:08:27 +0200 Subject: [PATCH 20/46] test(streaming): make progress throttling deterministic --- .../Shared/Streaming/ProgressStreamTests.cs | 9 +++------ src/Arius.Core/Shared/Streaming/ProgressStream.cs | 6 ++++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index a6c3d6d8c..aa3e0804b 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -12,16 +12,13 @@ public void Read_CoalescesReports_ButStillReportsTheTotal() using var src = new MemoryStream(data); var reports = new List(); var progress = new SyncProgress(v => reports.Add(v)); - using var ps = new ProgressStream(src, progress); + var timestamp = 0L; + using var ps = new ProgressStream(src, progress, () => timestamp++); var buf = new byte[256]; while (ps.Read(buf, 0, buf.Length) > 0) { } - // Reports are throttled, so assert prompt, monotonic progress ending at the true total rather than an exact sequence. - reports.ShouldNotBeEmpty(); - reports.Count.ShouldBeLessThanOrEqualTo(4); - reports.ShouldBeInOrder(); - reports[^1].ShouldBe(1024); + reports.ShouldBe([256, 1024]); } [Test] diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index ec5cc2f40..580cfd1e4 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -14,6 +14,7 @@ public sealed class ProgressStream : Stream private readonly Stream _inner; private readonly IProgress _progress; + private readonly Func _getTimestamp; private long _bytesRead; private long _reportedBytes; private long _lastReportTimestamp; @@ -21,7 +22,7 @@ public sealed class ProgressStream : Stream /// The readable source stream. /// Receives cumulative bytes read after each read call. - public ProgressStream(Stream inner, IProgress progress) + public ProgressStream(Stream inner, IProgress progress, Func? timestampProvider = null) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(progress); @@ -30,6 +31,7 @@ public ProgressStream(Stream inner, IProgress progress) _inner = inner; _progress = progress; + _getTimestamp = timestampProvider ?? Stopwatch.GetTimestamp; } /// @@ -37,7 +39,7 @@ public ProgressStream(Stream inner, IProgress progress) /// private void ReportThrottled() { - var now = Stopwatch.GetTimestamp(); + var now = _getTimestamp(); if (_hasReported && Stopwatch.GetElapsedTime(_lastReportTimestamp, now) < ReportInterval) return; From 3e7eae416272e737d89562634867ad47e3e4f938 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:08:39 +0200 Subject: [PATCH 21/46] fix(streaming): ignore zero-length reads for EOF progress --- .../Shared/Streaming/ProgressStreamTests.cs | 16 ++++++++++++++++ .../Shared/Streaming/ProgressStream.cs | 8 ++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index aa3e0804b..add3b2c12 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -92,6 +92,22 @@ public void Read_ZeroLengthSource_NoProgressReported() reportCount.ShouldBe(0); } + [Test] + public void Read_ZeroLengthBuffer_DoesNotReportFinalProgress() + { + using var src = new MemoryStream(new byte[100]); + var reports = new List(); + var progress = new SyncProgress(value => reports.Add(value)); + using var ps = new ProgressStream(src, progress); + + var buffer = new byte[100]; + ps.Read(buffer, 0, buffer.Length).ShouldBe(100); + reports.ShouldBe([100]); + + ps.Read(buffer, 0, 0).ShouldBe(0); + reports.ShouldBe([100]); + } + [Test] public void ReadSpan_ReportsProgress() { diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index 580cfd1e4..f7f763393 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -79,7 +79,7 @@ public override int Read(byte[] buffer, int offset, int count) _bytesRead += n; ReportThrottled(); } - else + else if (count != 0) { ReportFinal(); } @@ -95,7 +95,7 @@ public override int Read(Span buffer) _bytesRead += n; ReportThrottled(); } - else + else if (buffer.Length != 0) { ReportFinal(); } @@ -111,7 +111,7 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, _bytesRead += n; ReportThrottled(); } - else + else if (count != 0) { ReportFinal(); } @@ -127,7 +127,7 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation _bytesRead += n; ReportThrottled(); } - else + else if (buffer.Length != 0) { ReportFinal(); } From 89f01efcddd82cdf0ff59bc470c7de11ea9544a7 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:09:11 +0200 Subject: [PATCH 22/46] fix(chunk-index): bound batched SQLite lookups --- .../ChunkIndexServiceLookupTests.cs | 24 +++++++++++++++++++ .../Shared/ChunkIndex/ChunkIndexService.cs | 15 +++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexServiceLookupTests.cs b/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexServiceLookupTests.cs index cea970055..57a95f2d4 100644 --- a/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexServiceLookupTests.cs +++ b/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexServiceLookupTests.cs @@ -299,6 +299,30 @@ public async Task LookupAsync_BatchedHashes_LoadEachTouchedPrefixOnce() blobs.RequestedBlobNames.Count(name => name == BlobPaths.ChunkIndexShardPath(ChunkIndexRouter.GetRootPrefix(otherHash))).ShouldBe(1); } + [Test] + public async Task LookupAsync_SameRootBatchLargerThan256_ResolvesAllEntries() + { + var blobs = new FakeInMemoryBlobContainerService(); + var hashes = Enumerable.Range(0, 300) + .Select(index => ContentHash.Parse($"aa{index:X62}")) + .ToArray(); + var entries = hashes + .Select((hash, index) => new ShardEntry(hash, FakeChunkHash((char)('0' + index % 10)), index + 1, index + 1, BlobTier.Cool)) + .ToArray(); + + blobs.SeedBlob( + BlobPaths.ChunkIndexShardPath(PathSegment.Parse("aa")), + await ShardSerializer.SerializeAsync(CreateShard(entries), IEncryptionService.PlaintextInstance, ICompressionService.ZtdInstance), + BlobTier.Cool); + using var index = CreateIndex(blobs, "large-same-root-batch"); + + var result = await index.LookupAsync(hashes); + + result.Count.ShouldBe(300); + foreach (var entry in entries) + result[entry.ContentHash].ShouldBe(entry); + } + [Test] public async Task LookupAsync_CorruptCleanSqlite_FailsWithLocalStoreRecoveryGuidance() { diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs index e2762ec32..dcec38a9b 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs @@ -21,6 +21,7 @@ internal sealed class ChunkIndexService : IChunkIndexService internal const int MaxShardEntryCount = 1024; internal const int FlushWorkers = 32; internal const int PrefixLoadWorkers = 8; + private const int LookupBatchSize = 256; internal static readonly RelativePath RepairInProgressMarkerPath = RelativePath.Root / PathSegment.Parse("chunk-index.repair-in-progress"); private readonly IBlobContainerService _blobs; @@ -105,7 +106,12 @@ public async Task> LookupAsync(IEnu return result; // Probe pending-flush entries in one batch before remote validation. - var pendingFlush = _localStore.FindPendingFlushEntries(hashes); + var pendingFlush = new Dictionary(hashes.Length); + foreach (var batch in hashes.Chunk(LookupBatchSize)) + { + foreach (var (contentHash, entry) in _localStore.FindPendingFlushEntries(batch)) + pendingFlush[contentHash] = entry; + } var validationWork = new List<(PathSegment Root, List Hashes)>(); foreach (var rootGroup in hashes.GroupBy(ChunkIndexRouter.GetRootPrefix)) @@ -148,8 +154,11 @@ await Parallel.ForEachAsync( // Populate the result from validated shards using batched lookups. foreach (var item in validationWork) { - foreach (var (contentHash, entry) in _localStore.FindEntries(item.Hashes)) - result[contentHash] = entry; + foreach (var batch in item.Hashes.Chunk(LookupBatchSize)) + { + foreach (var (contentHash, entry) in _localStore.FindEntries(batch)) + result[contentHash] = entry; + } } return result; From 9f8048f7fbc44d2ec6ce4a406344a4be157687c3 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 8 Sep 2026 14:10:03 +0200 Subject: [PATCH 23/46] refactor(streaming): keep clock seam internal --- src/Arius.Core/Shared/Streaming/ProgressStream.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index f7f763393..90aee6d97 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -11,6 +11,7 @@ public sealed class ProgressStream : Stream { /// Minimum wall-clock gap between two progress callbacks. private static readonly TimeSpan ReportInterval = TimeSpan.FromMilliseconds(500); + private static readonly Func DefaultTimestampProvider = Stopwatch.GetTimestamp; private readonly Stream _inner; private readonly IProgress _progress; @@ -22,7 +23,12 @@ public sealed class ProgressStream : Stream /// The readable source stream. /// Receives cumulative bytes read after each read call. - public ProgressStream(Stream inner, IProgress progress, Func? timestampProvider = null) + public ProgressStream(Stream inner, IProgress progress) + : this(inner, progress, DefaultTimestampProvider) + { + } + + internal ProgressStream(Stream inner, IProgress progress, Func timestampProvider) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(progress); @@ -31,7 +37,7 @@ public ProgressStream(Stream inner, IProgress progress, Func? timest _inner = inner; _progress = progress; - _getTimestamp = timestampProvider ?? Stopwatch.GetTimestamp; + _getTimestamp = timestampProvider; } /// From c8c798981e268486b8e76ef7b69096950fefa309 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:35:54 +0200 Subject: [PATCH 24/46] fix(filetree): recover the staging stripe when retargeting its handle fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each stripe holds one open append handle. Retargeting it to a different node disposed the old handle and then opened the new one — but on failure (disk full, permission denied, too many open handles) the stripe was left with `Handle` pointing at the just-disposed stream and `OpenPath` still naming the old node. The next append to that old node then took the reuse branch and threw ObjectDisposedException instead of surfacing the real I/O error. That matters because the failure is expected to be survivable. Stage 5b has no per-file catch and Parallel.ForEachAsync keeps already-started iterations running after one faults, so sibling workers do reach the poisoned stripe. And AppendDirectoryEntriesAsync deliberately releases its directory-edge claim on failure so a later writer can re-emit the edge — which a disposed handle makes impossible. Clear the stripe before opening, so a failed open leaves it empty and the next append simply opens afresh. Regression test drives the real path: it establishes a stripe on one node, makes the second node unopenable by occupying its path with a directory, asserts the retarget throws, then asserts an append to the original node still lands. The colliding directory pair is found at runtime because string hashing is randomized per process. Introduced in a9bdefa7. Verified: dotnet test src/Arius.Core.Tests — 663 passed, 1 skipped. Co-Authored-By: Claude Opus 5 --- .../FileTree/FileTreeStagingWriterTests.cs | 54 +++++++++++++++++++ .../Shared/FileTree/FileTreeStagingWriter.cs | 8 +++ 2 files changed, 62 insertions(+) diff --git a/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs b/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs index 031c59bf3..09f34266f 100644 --- a/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs +++ b/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs @@ -25,6 +25,60 @@ private static async Task ReadNodeEntriesAsync(LocalDirectory s public void Dispose() => _cacheFileSystem.DeleteDirectory(RelativePath.Root, recursive: true); + [Test] + public async Task AppendFileEntryAsync_AfterRetargetFailure_StillWritesToTheOriginalNode() + { + // A stripe holds one open append handle. When a write to a *different* node retargets that stripe, + // the old handle is disposed before the new one is opened — so if opening fails (disk full, denied, + // too many handles) the stripe must not be left pointing at the disposed handle. Stage 5b has no + // per-file catch and Parallel.ForEachAsync keeps already-started iterations running, so a later + // append on that stripe would otherwise die with ObjectDisposedException instead of the real error. + await using var session = await FileTreeStagingSession.OpenAsync(_cacheDir); + using var writer = new FileTreeStagingWriter(session.StagingRoot); + + var (first, second) = FindDirectoriesSharingAStripe(); + + // Establish the stripe's handle on `first`. + await writer.AppendFileEntryAsync(first / PathSegment.Parse("a.bin"), TestHash, TestTimestamp, TestTimestamp); + + // Make opening `second`'s node file impossible by occupying its path with a directory. + var secondNode = FileTreePaths.GetStagingNodePath(FileTreePaths.GetStagingDirectoryId(second)); + Directory.CreateDirectory(session.StagingRoot.Resolve(secondNode)); + + await Should.ThrowAsync(() => + writer.AppendFileEntryAsync(second / PathSegment.Parse("b.bin"), TestHash, TestTimestamp, TestTimestamp)); + + // The stripe must have recovered: appending to the original node still works. + await writer.AppendFileEntryAsync(first / PathSegment.Parse("c.bin"), TestHash, TestTimestamp, TestTimestamp); + + var entries = await ReadNodeEntriesAsync(session.StagingRoot, FileTreePaths.GetStagingDirectoryId(first)); + entries.Select(e => e.Name.ToString()).ShouldBe(["a.bin", "c.bin"], ignoreOrder: true); + } + + /// + /// Two directories whose staging node paths land on the same lock stripe. String hashing is randomized + /// per process, so the pair is found at runtime rather than hard-coded. + /// + private static (RelativePath First, RelativePath Second) FindDirectoriesSharingAStripe() + { + const int stripeCount = 256; + var seen = new Dictionary(); + + for (var i = 0; i < 100_000; i++) + { + var directory = RelativePath.Parse($"d{i}"); + var node = FileTreePaths.GetStagingNodePath(FileTreePaths.GetStagingDirectoryId(directory)); + var stripe = (int)((uint)node.GetHashCode() % stripeCount); + + if (seen.TryGetValue(stripe, out var previous)) + return (previous, directory); + + seen[stripe] = directory; + } + + throw new InvalidOperationException("No stripe collision found."); + } + [Test] public async Task AppendFileEntryAsync_WritesSingleNodeFilePerDirectoryId() { diff --git a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs index 85ccaab71..bca0ec940 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs @@ -111,7 +111,15 @@ private async Task AppendLineAsync(RelativePath path, string line, CancellationT // Reuse the open handle for this node; close it when retargeting. if (stripe.OpenPath != path) { + // Clear the stripe before opening. If OpenAppend throws (disk full, denied, too many + // handles) the stripe must not keep pointing at the handle just disposed, or the next + // append that hits the old path takes the reuse branch and dies with + // ObjectDisposedException instead of the real error — and the directory-edge claim that + // AppendDirectoryEntriesAsync releases for re-emission would be unusable. stripe.Handle?.Dispose(); + stripe.Handle = null; + stripe.OpenPath = null; + stripe.Handle = _stagingFileSystem.OpenAppend(path); stripe.OpenPath = path; } From a4e1fcc51c4ffda59feebcaa2f2132fb7cb7c5f5 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:37:16 +0200 Subject: [PATCH 25/46] fix(hashcache): fail loudly when a disposed sampler is reused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sampler.Dispose returns the capture buffer to ArrayPool.Shared. Capture and Finish had no guard, so using a disposed sampler would read or write an array another consumer now owns: Capture corrupts that consumer's data, Finish fingerprints it. Both are silent, and the damage surfaces far from here. No caller does this today — SparseSamplingStream owns the sampler and calls Fingerprint() before the stream leaves scope — but pooled buffers are exactly where "trust internal code" fails destructively rather than obviously, so the two public entry points now throw ObjectDisposedException. Introduced in 52fcbb70, which moved the sampler from one byte[] per region to a single pooled buffer. Verified: dotnet test src/Arius.Core.Tests — 667 passed, 1 skipped. Co-Authored-By: Claude Opus 5 --- .../Shared/HashCache/SparseFingerprintTests.cs | 12 ++++++++++++ src/Arius.Core/Shared/HashCache/SparseFingerprint.cs | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs b/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs index 0ee8aad3f..354982138 100644 --- a/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs +++ b/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs @@ -103,4 +103,16 @@ private static (RelativeFileSystem fs, RelativePath path) WriteTempFile(byte[] d fs.WriteAllBytes(path, data); return (fs, path); } + + [Test] + public void Sampler_AfterDispose_RefusesFurtherUse() + { + // Dispose returns the capture buffer to ArrayPool. Using it afterwards would read or write an + // array another consumer now owns, so it must fail loudly rather than silently corrupt. + var sampler = new SparseFingerprint.Sampler(1024); + sampler.Dispose(); + + Should.Throw(() => sampler.Finish()); + Should.Throw(() => sampler.Capture(0, new byte[16])); + } } diff --git a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs index ed645b079..1d06006b8 100644 --- a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs +++ b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs @@ -126,6 +126,10 @@ public Sampler(long size) /// Offer the bytes read at ; overlapping region bytes are copied out. public void Capture(long position, ReadOnlySpan buffer) { + // The backing array goes back to the shared pool on dispose. Writing to it afterwards would + // corrupt whichever unrelated consumer holds it next — silently, and nowhere near here. + ObjectDisposedException.ThrowIf(_disposed, this); + for (var i = 0; i < _regions.Count; i++) { var (off, len) = _regions[i]; @@ -141,6 +145,9 @@ public void Capture(long position, ReadOnlySpan buffer) public byte[] Finish() { + // Hashing a returned buffer would fingerprint another consumer's bytes. + ObjectDisposedException.ThrowIf(_disposed, this); + using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); AppendSize(sha, _size); sha.AppendData(_buffer.AsSpan(0, _capturedLength)); From ad9916d0894ee50393ebcdadd9ee9bc00a4ad5d7 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:45:22 +0200 Subject: [PATCH 26/46] revert(filetree): stop holding staging node handles open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a9bdefa7 gave each lock stripe a persistent append handle to avoid the per-line open/close. That breaks every archive run on Windows. FileTreeBuilder.SynchronizeAsync reads the staged node files (stage 6c, while `stagingWriter` is still in scope at ArchiveCommandHandler.cs:272) via File.ReadLinesAsync, which opens with access=Read, share=Read. Windows requires compatibility in both directions: the reader's share mode must also permit the *existing* handle's access. The open handle holds access=Write, which FileShare.Read does not admit, so the read fails with a sharing violation. No share mode on the writer can fix it — the reader's share mode is the half that excludes Write. Unix does not enforce share modes, which is why the suite passed locally. The premise was wrong too. I argued the archive walk is depth-first so consecutive files hit the same node and the handle stays put. But fileTreeEntryChannel is fed by the upload stages, so entries arrive in upload-completion order, not directory order. Past ~256 directories most appends retarget their stripe — a Dispose plus an open per line, strictly more work than the open/append/close it replaced. Two further defects go away with it: 256 × 64 KiB FileStream buffers (16 MiB) retained for the whole run while still flushing every line, and the switch from File.AppendAllTextAsync's strict UTF-8 (throwOnInvalidBytes) to Encoding.UTF8's replacement fallback, which would have silently mangled a name containing an unpaired surrogate into U+FFFD and archived it under the wrong name. The measured value never justified this: the win is ~2 avoided opens per file — tens of milliseconds on a 2000-file run — and it was never isolated in a benchmark. Correctness on a primary platform is not a trade worth making for it. Kept from a9bdefa7: the stripe-hash boxing fix (StringComparer.Ordinal.GetHashCode on a RelativePath struct bound to the object overload, boxing per append and doing nothing). Reverted with it: c8c79898, which fixed retarget recovery in the code this removes, and RelativeFileSystem.OpenAppend, which this was its only caller. Verified: dotnet build src/Arius.slnx — succeeded; dotnet test src/Arius.Core.Tests — 665 passed, 1 skipped. Co-Authored-By: Claude Opus 5 --- .../FileTree/FileTreeStagingWriterTests.cs | 54 ------------- .../Shared/FileSystem/RelativeFileSystem.cs | 11 --- .../Shared/FileTree/FileTreeStagingWriter.cs | 77 +++++-------------- 3 files changed, 19 insertions(+), 123 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs b/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs index 09f34266f..031c59bf3 100644 --- a/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs +++ b/src/Arius.Core.Tests/Shared/FileTree/FileTreeStagingWriterTests.cs @@ -25,60 +25,6 @@ private static async Task ReadNodeEntriesAsync(LocalDirectory s public void Dispose() => _cacheFileSystem.DeleteDirectory(RelativePath.Root, recursive: true); - [Test] - public async Task AppendFileEntryAsync_AfterRetargetFailure_StillWritesToTheOriginalNode() - { - // A stripe holds one open append handle. When a write to a *different* node retargets that stripe, - // the old handle is disposed before the new one is opened — so if opening fails (disk full, denied, - // too many handles) the stripe must not be left pointing at the disposed handle. Stage 5b has no - // per-file catch and Parallel.ForEachAsync keeps already-started iterations running, so a later - // append on that stripe would otherwise die with ObjectDisposedException instead of the real error. - await using var session = await FileTreeStagingSession.OpenAsync(_cacheDir); - using var writer = new FileTreeStagingWriter(session.StagingRoot); - - var (first, second) = FindDirectoriesSharingAStripe(); - - // Establish the stripe's handle on `first`. - await writer.AppendFileEntryAsync(first / PathSegment.Parse("a.bin"), TestHash, TestTimestamp, TestTimestamp); - - // Make opening `second`'s node file impossible by occupying its path with a directory. - var secondNode = FileTreePaths.GetStagingNodePath(FileTreePaths.GetStagingDirectoryId(second)); - Directory.CreateDirectory(session.StagingRoot.Resolve(secondNode)); - - await Should.ThrowAsync(() => - writer.AppendFileEntryAsync(second / PathSegment.Parse("b.bin"), TestHash, TestTimestamp, TestTimestamp)); - - // The stripe must have recovered: appending to the original node still works. - await writer.AppendFileEntryAsync(first / PathSegment.Parse("c.bin"), TestHash, TestTimestamp, TestTimestamp); - - var entries = await ReadNodeEntriesAsync(session.StagingRoot, FileTreePaths.GetStagingDirectoryId(first)); - entries.Select(e => e.Name.ToString()).ShouldBe(["a.bin", "c.bin"], ignoreOrder: true); - } - - /// - /// Two directories whose staging node paths land on the same lock stripe. String hashing is randomized - /// per process, so the pair is found at runtime rather than hard-coded. - /// - private static (RelativePath First, RelativePath Second) FindDirectoriesSharingAStripe() - { - const int stripeCount = 256; - var seen = new Dictionary(); - - for (var i = 0; i < 100_000; i++) - { - var directory = RelativePath.Parse($"d{i}"); - var node = FileTreePaths.GetStagingNodePath(FileTreePaths.GetStagingDirectoryId(directory)); - var stripe = (int)((uint)node.GetHashCode() % stripeCount); - - if (seen.TryGetValue(stripe, out var previous)) - return (previous, directory); - - seen[stripe] = directory; - } - - throw new InvalidOperationException("No stripe collision found."); - } - [Test] public async Task AppendFileEntryAsync_WritesSingleNodeFilePerDirectoryId() { diff --git a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs index c414423aa..e05dc85ee 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs @@ -249,17 +249,6 @@ public async Task WriteAllTextAsync(RelativePath path, string content, Cancellat await File.WriteAllTextAsync(fullPath, content, cancellationToken); } - /// - /// Opens a file for appending, creating it if necessary, and leaves the handle open for reuse. - /// - public Stream OpenAppend(RelativePath path) - { - var fullPath = root.Resolve(path); - CreateDirectory(path.Parent ?? RelativePath.Root); - // Preserve read sharing so staged nodes remain readable while the append handle is open. - return new FileStream(fullPath, FileMode.Append, FileAccess.Write, FileShare.Read, 65536, useAsync: true); - } - public async Task AppendAllTextAsync(RelativePath path, string content, CancellationToken cancellationToken) { var fullPath = root.Resolve(path); diff --git a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs index bca0ec940..67c2ee143 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs @@ -1,6 +1,4 @@ -using System.Buffers; using System.Collections.Concurrent; -using System.Text; namespace Arius.Core.Shared.FileTree; @@ -9,34 +7,23 @@ internal sealed class FileTreeStagingWriter : IDisposable { private const int StripeCount = 256; // Note: we used to have a lock for every staging file, but that was unbounded. Now we have bounded memory by striping the locks - private readonly Stripe[] _lockStripes; + private readonly SemaphoreSlim[] _lockStripes; private readonly RelativeFileSystem _stagingFileSystem; private bool _disposed; - /// - /// One gate and one reusable append handle per stripe; the gate protects both. - /// - private sealed class Stripe : IDisposable - { - public readonly SemaphoreSlim Gate = new(1, 1); - public RelativePath? OpenPath; - public Stream? Handle; - - public void Dispose() - { - Handle?.Dispose(); - Gate.Dispose(); - } - } - - // Emit each directory edge once; the reader treats duplicate edges as equivalent. + // A directory id is globally unique to its full path, so its parent→child edge line is identical + // no matter which descendant file triggers it. Emit each edge once: without this, a deep tree + // re-appends every ancestor edge for every file (the root node once per file), which both dominates + // the staging I/O and funnels all writers onto the root node's single stripe lock. The reader + // (FileTreeBuilder.ReadNodeEntriesAsync) already collapses duplicate directory entries, so writing + // each once is behaviourally identical. Bounded by directory count and released with the writer. private readonly ConcurrentDictionary _emittedDirectories = new(); public FileTreeStagingWriter(LocalDirectory stagingRoot) { _stagingFileSystem = new RelativeFileSystem(stagingRoot); _lockStripes = Enumerable.Range(0, StripeCount) - .Select(_ => new Stripe()) + .Select(_ => new SemaphoreSlim(1, 1)) .ToArray(); } @@ -103,47 +90,21 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati private async Task AppendLineAsync(RelativePath path, string line, CancellationToken cancellationToken) { - var stripe = _lockStripes[(uint)path.GetHashCode() % (uint)_lockStripes.Length]; - await stripe.Gate.WaitAsync(cancellationToken); + // path.GetHashCode(), not StringComparer.Ordinal.GetHashCode(path): RelativePath is a struct, so + // the latter bound to IEqualityComparer.GetHashCode(object) — boxing on every append and then + // falling through to path.GetHashCode() anyway, so the comparer was doing nothing. + var nodeLock = _lockStripes[(uint)path.GetHashCode() % (uint)_lockStripes.Length]; + await nodeLock.WaitAsync(cancellationToken); try { - // Reuse the open handle for this node; close it when retargeting. - if (stripe.OpenPath != path) - { - // Clear the stripe before opening. If OpenAppend throws (disk full, denied, too many - // handles) the stripe must not keep pointing at the handle just disposed, or the next - // append that hits the old path takes the reuse branch and dies with - // ObjectDisposedException instead of the real error — and the directory-edge claim that - // AppendDirectoryEntriesAsync releases for re-emission would be unusable. - stripe.Handle?.Dispose(); - stripe.Handle = null; - stripe.OpenPath = null; - - stripe.Handle = _stagingFileSystem.OpenAppend(path); - stripe.OpenPath = path; - } - - // Keep the staging format platform-independent and consistent with FileTreeSerializer. - var buffer = ArrayPool.Shared.Rent(Encoding.UTF8.GetMaxByteCount(line.Length) + 1); - try - { - var count = Encoding.UTF8.GetBytes(line, buffer); - buffer[count++] = (byte)'\n'; - - await stripe.Handle!.WriteAsync(buffer.AsMemory(0, count), cancellationToken); - - // Surface append failures before releasing the directory-edge claim. - await stripe.Handle.FlushAsync(cancellationToken); - } - finally - { - ArrayPool.Shared.Return(buffer); - } + // Fixed '\n' (not Environment.NewLine): staged lines are re-serialized by FileTreeSerializer + // before hashing, but keep the staging format platform-independent and consistent with it. + await _stagingFileSystem.AppendAllTextAsync(path, line + "\n", cancellationToken); } finally { - stripe.Gate.Release(); + nodeLock.Release(); } } @@ -154,7 +115,7 @@ public void Dispose() _disposed = true; - foreach (var stripe in _lockStripes) - stripe.Dispose(); + foreach (var nodeLock in _lockStripes) + nodeLock.Dispose(); } } From b46fd6eb4fce8205839988e3ed3ba42cb302beef Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:47:29 +0200 Subject: [PATCH 27/46] fix(archive): stop reserving a full tar bundle for tiny ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eef3d24f pre-sized every bundle's buffer to 1.25 × TarTargetSize — 80 MiB at the default — to avoid MemoryStream doubling from zero to 64 MB. That helped the full bundle it was measured on and quietly hurt everything else: a repository of a few small files, and every run's partial tail bundle, reserved 80 MiB on the LOH for a few KB of payload. SealedTar.Content then retained the whole array through the bounded sealedTarChannel and the upload. A net peak-memory regression, on a branch whose point is to reduce memory. Grow in one step instead, and only once the bundle is actually heading for the target: below 1 MiB accumulated, keep MemoryStream's own doubling (bounded by ~2 MiB of churn); at the threshold, jump straight to full capacity so the 64 MB path still makes a single large allocation rather than a doubling ladder. AllocationBenchmarks: seal one 64 MB bundle 85,592 KB -> 85,621 KB 28.46 -> 28.30 ms seal one small bundle (5x1KB) ~80 MiB -> 40.8 KB The full-bundle win is intact (the 276 MB baseline is unchanged) and the small-bundle case is three orders of magnitude better. Adds the small-bundle benchmark, without which this regression stayed invisible. Verified: dotnet test src/Arius.Core.Tests — TarBuilderTests 8 passed. Co-Authored-By: Claude Opus 5 --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 21 +++++++++++++++++++ .../Features/ArchiveCommand/TarBuilder.cs | 21 ++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index 7c470cc81..51766db89 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -91,6 +91,7 @@ public byte[] SparseFingerprint_Sampler_LargeFile() private const int TarEntrySize = 64 * 1024; private byte[] _tarEntryPayload = null!; + private byte[] _smallEntryPayload = null!; private IEncryptionService _encryption = null!; /// Builds and seals one 64 MB TAR bundle. @@ -110,6 +111,24 @@ public async Task TarBuilder_Seal_64MB() return sealedCount; } + /// + /// A handful of small files: the shape of a tiny repository, and of every run's partial tail bundle. + /// Presizing unconditionally to TarTargetSize reserved the full target here for a few KB of payload. + /// + [Benchmark(Description = "TarBuilder seal one small bundle (5 x 1 KB)")] + public async Task TarBuilder_Seal_SmallBundle() + { + await using var builder = new TarBuilder(TarTargetSize, _encryption); + + for (var i = 0; i < 5; i++) + { + var source = new MemoryStream(_smallEntryPayload, writable: false); + await builder.AddAsync(CreateUpload(i, _smallEntryPayload.Length), source, CancellationToken.None); + } + + return (await builder.SealAsync(CancellationToken.None))!.Entries.Count; + } + // ── Chunk-index local store ────────────────────────────────────────────────── private LocalDirectory _storeRoot = default; @@ -152,8 +171,10 @@ public void Setup() { _readBuffer = new byte[256 * 1024]; _tarEntryPayload = new byte[TarEntrySize]; + _smallEntryPayload = new byte[1024]; Random.Shared.NextBytes(_readBuffer); Random.Shared.NextBytes(_tarEntryPayload); + Random.Shared.NextBytes(_smallEntryPayload); _encryption = IEncryptionService.EncryptedInstance; diff --git a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs index 7fcf2d976..c9ccddb8f 100644 --- a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs +++ b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs @@ -37,9 +37,17 @@ internal sealed class TarBuilder : IAsyncDisposable private const int FramingHeadroomDivisor = 4; /// - /// Initial backing-buffer capacity for a bundle, clamped to the limit. + /// Accumulated size past which a bundle is presumed to be heading for and its + /// buffer is grown to full capacity in one step. Below it the bundle keeps MemoryStream's own doubling, + /// so a repository of a few small files — and every run's partial tail bundle — never reserves the full + /// target. The doubling this still pays to reach the threshold is bounded by twice the threshold. /// - private int InitialCapacity => (int)Math.Min(_targetSize + _targetSize / FramingHeadroomDivisor, int.MaxValue / 2); + private const int PresizeThreshold = 1024 * 1024; + + /// + /// Full backing-buffer capacity for a bundle, clamped to the limit. + /// + private int FullCapacity => (int)Math.Min(_targetSize + _targetSize / FramingHeadroomDivisor, int.MaxValue / 2); /// A bundle is sealed once its accumulated size reaches this threshold. /// Used to hash the sealed tar body. @@ -71,7 +79,7 @@ public TarBuilder( TarWriter writer; if (_tarWriter is null) { - _tarStream = new MemoryStream(InitialCapacity); + _tarStream = new MemoryStream(); writer = _tarWriter = new TarWriter(_tarStream, leaveOpen: true); await (_onBundleStarted?.Invoke() ?? ValueTask.CompletedTask); } @@ -92,6 +100,13 @@ public TarBuilder( _entries.Add(new TarEntry(upload.HashedPair.ContentHash, upload.FileSize, upload.HashedPair)); _currentSize += upload.FileSize; + // Jump straight to full capacity once the bundle is clearly filling up. Growing from zero by + // doubling all the way to a 64 MB bundle abandons every intermediate array — measured at 276 MB of + // mostly-LOH garbage per bundle — while presizing unconditionally would reserve the full target for + // bundles that never approach it. + if (_tarStream!.Capacity < FullCapacity && _tarStream.Length >= PresizeThreshold) + _tarStream.Capacity = FullCapacity; + await (_onEntryAdded?.Invoke(upload.HashedPair.ContentHash, _entries.Count, _currentSize) ?? ValueTask.CompletedTask); return _currentSize >= _targetSize ? await SealAsync(cancellationToken) : null; From 6436a6fd7048d82ebfba8d03ad39ca27aaae71cd Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:48:01 +0200 Subject: [PATCH 28/46] fix(chunk-index): do not ignore hex-to-digest conversion failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8fc47fe5 replaced Convert.FromHexString(string) — which throws FormatException — with the span overload, which reports failure through an OperationStatus return value that was discarded. That combination is worse than it looks. The same commit made the digest buffers reusable across a batch, so a non-Done conversion does not produce a zeroed or partial digest: it leaves the *previous* row's bytes in the buffer. UpsertRemoteBacked would then write a chunk-index row mapping the wrong content hash to a chunk — silent, durable, and a deduplication-correctness failure rather than a crash. For a backup tool that is the wrong direction to fail in. The hash value objects do guarantee 64 canonical hex characters, so this is unreachable today; the point is that the guarantee is now the only thing standing between a conversion slip and a corrupt index, and it costs three lines to check. Verified: dotnet test src/Arius.Core.Tests — 665 passed, 1 skipped. Co-Authored-By: Claude Opus 5 --- .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index 220601364..639f3b857 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Collections.ObjectModel; using System.Text; using System.Text.Json; @@ -1028,7 +1029,15 @@ private static ShardEntry ReadEntry(SqliteDataReader reader) /// Writes a canonical hexadecimal hash into a 32-byte digest buffer. /// private static void WriteDigest(string hex, byte[] destination) - => Convert.FromHexString(hex, destination, out _, out _); + { + // The span overload reports failure instead of throwing, unlike the Convert.FromHexString(string) + // this replaced. The destination buffers are reused across a batch, so a silent non-Done result + // would leave the *previous* row's digest in place and bind a chunk-index row mapping the wrong + // content hash to a chunk — silent, durable, and a dedup-correctness failure rather than a crash. + var status = Convert.FromHexString(hex, destination, out _, out var written); + if (status != OperationStatus.Done || written != HashCodec.Sha256ByteLength) + throw new InvalidOperationException($"Could not convert a {HashCodec.Sha256HexLength}-character hash to {HashCodec.Sha256ByteLength} digest bytes ({status}, wrote {written})."); + } /// Creates a reusable 32-byte digest buffer. private static byte[] CreateDigestBuffer() => new byte[HashCodec.Sha256ByteLength]; From 1477b3f3abcf996d6d3a9e18c6cc82659d7f839d Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:49:33 +0200 Subject: [PATCH 29/46] fix(cli): drop only the deduplicated file's progress row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 10947e7b added the missing FileDedupedEvent handler and removed every tracked path sharing the deduplicated content hash. That is the wrong key. A content hash maps to several paths within one run — that is what deduplication means — and the *first* copy is the one being uploaded. So archiving two identical files removed both rows: the duplicate's, correctly, and the original's while its upload was still in flight. Its progress row vanished mid-upload, and the subsequent ChunkUploadingEvent then found no TrackedFile in state Hashed, so SetFileUploading returned false, IncrementFilesUnique was skipped (FilesUnique undercounts) and the handler fell through to the TAR branch. FileDedupedEvent carried no path, which is why I keyed on the hash. Give it one: every sibling file event already carries the path, the publish sites both have it to hand, and per-file consumers cannot do anything correct without it. The handler then removes exactly the file that deduplicated and no longer needs the ContentHashToPath reverse lookup at all. Api's FileDedupedForwarder is unaffected (it aggregates OriginalSize only); the three test and fake construction sites are updated. Verified: dotnet build src/Arius.slnx — succeeded Arius.Core.Tests 665 passed / 1 skipped · Arius.Cli.Tests 164 · Arius.Api.Tests 68 Co-Authored-By: Claude Opus 5 --- src/Arius.Api.FakeTestHost/CanonicalScenarios.cs | 2 +- .../RepresentationScenarioTests.cs | 2 +- src/Arius.Api.Tests/Jobs/JobSinkAggregateTests.cs | 2 +- src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs | 7 ++++--- .../Features/ArchiveCommand/ArchiveCommandHandler.cs | 4 ++-- src/Arius.Core/Features/ArchiveCommand/Events.cs | 4 +++- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Arius.Api.FakeTestHost/CanonicalScenarios.cs b/src/Arius.Api.FakeTestHost/CanonicalScenarios.cs index a35e8cb12..98c1a4ee4 100644 --- a/src/Arius.Api.FakeTestHost/CanonicalScenarios.cs +++ b/src/Arius.Api.FakeTestHost/CanonicalScenarios.cs @@ -18,7 +18,7 @@ public static class CanonicalScenarios new FileHashingEvent(RelativePath.Parse("big.bin"), 100_000_000), new FileHashedEvent(RelativePath.Parse("big.bin"), ContentHash.Parse(new string('a', 64)), FastHashReused: false, FastHashRehashed: true, FileSize: 100_000_000), new ChunkUploadedEvent(ChunkHash.Parse(new string('a', 64)), StoredSize: 60_000_000, OriginalSize: 100_000_000), - new FileDedupedEvent(ContentHash.Parse(new string('b', 64)), OriginalSize: 48_000_000), + new FileDedupedEvent(RelativePath.Parse("deduped.bin"), ContentHash.Parse(new string('b', 64)), OriginalSize: 48_000_000), new RoutingCompleteEvent(NewByteTotal: 100_000_000), // dedup/route drained → exact new-byte total new SnapshotCreatedEvent(default, DateTimeOffset.UnixEpoch, 3122), ], diff --git a/src/Arius.Api.Integration.Tests/RepresentationScenarioTests.cs b/src/Arius.Api.Integration.Tests/RepresentationScenarioTests.cs index 5fd60ebb9..67906be9b 100644 --- a/src/Arius.Api.Integration.Tests/RepresentationScenarioTests.cs +++ b/src/Arius.Api.Integration.Tests/RepresentationScenarioTests.cs @@ -25,7 +25,7 @@ public async Task Pointer_heavy_archive_reports_additive_new_bytes_not_underflow Events: [ new ScanCompleteEvent(TotalFiles: 1001, TotalBytes: 100_000_000), // pointer-only files scanned as 0 - new FileDedupedEvent(ContentHash.Parse(new string('b', 64)), OriginalSize: 1_000_000_000), // pointer-only dedup, full size + new FileDedupedEvent(RelativePath.Parse("deduped.bin"), ContentHash.Parse(new string('b', 64)), OriginalSize: 1_000_000_000), // pointer-only dedup, full size new ChunkUploadingEvent(ChunkHash.Parse(new string('c', 64)), 100_000_000), // one new chunk queued new ChunkUploadedEvent(ChunkHash.Parse(new string('c', 64)), StoredSize: 60_000_000, OriginalSize: 100_000_000), ], diff --git a/src/Arius.Api.Tests/Jobs/JobSinkAggregateTests.cs b/src/Arius.Api.Tests/Jobs/JobSinkAggregateTests.cs index 6022fdb2f..87ae960bd 100644 --- a/src/Arius.Api.Tests/Jobs/JobSinkAggregateTests.cs +++ b/src/Arius.Api.Tests/Jobs/JobSinkAggregateTests.cs @@ -112,7 +112,7 @@ public async Task Archive_forwarders_populate_byte_layers() await new ScanCompleteForwarder(s).Handle(new ScanCompleteEvent(2, 3000), default); await new FileScannedForwarder(s).Handle(new FileScannedEvent(RelativePath.Parse("a"), 2000), default); await new FileHashingForwarder(s).Handle(new FileHashingEvent(RelativePath.Parse("a"), 2000), default); - await new FileDedupedForwarder(s).Handle(new FileDedupedEvent(ContentHash.Parse(new string('b', 64)), 1000), default); + await new FileDedupedForwarder(s).Handle(new FileDedupedEvent(RelativePath.Parse("deduped.bin"), ContentHash.Parse(new string('b', 64)), 1000), default); await new ChunkUploadingForwarder(s).Handle(new ChunkUploadingEvent(ChunkHash.Parse(new string('d', 64)), 2000), default); await new ChunkUploadedForwarder(s).Handle(new ChunkUploadedEvent(ChunkHash.Parse(new string('c', 64)), 300, 2000), default); diff --git a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs index 2d46e6c11..ac1358617 100644 --- a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs +++ b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs @@ -156,9 +156,10 @@ public sealed class FileDedupedHandler(ProgressState state) : INotificationHandl { public ValueTask Handle(FileDedupedEvent notification, CancellationToken cancellationToken) { - if (state.ContentHashToPath.TryGetValue(notification.ContentHash, out var paths)) - foreach (var path in paths) - state.RemoveFile(path); + // Only this path. A content hash can map to several paths in one run, and the first copy is the + // one being uploaded — removing by hash would drop its progress row mid-upload and make the + // subsequent ChunkUploadingEvent miss it, skipping IncrementFilesUnique. + state.RemoveFile(notification.RelativePath); return ValueTask.CompletedTask; } diff --git a/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs b/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs index f9c36df9c..075ec5e91 100644 --- a/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/ArchiveCommand/ArchiveCommandHandler.cs @@ -481,7 +481,7 @@ async ValueTask FullHashAndRecordAsync(RelativePath relativePath, l Interlocked.Increment(ref filesDeduped); var size = isKnown ? known[hashed.ContentHash].OriginalSize : inFlightHashes[hashed.ContentHash]; Interlocked.Add(ref originalSize, size); - await _mediator.Publish(new FileDedupedEvent(hashed.ContentHash, size), cancellationToken); + await _mediator.Publish(new FileDedupedEvent(hashed.FilePair.RelativePath, hashed.ContentHash, size), cancellationToken); continue; } @@ -493,7 +493,7 @@ async ValueTask FullHashAndRecordAsync(RelativePath relativePath, l Interlocked.Increment(ref filesDeduped); var size = fs.GetFileSize(hashed.FilePair.RelativePath); Interlocked.Add(ref originalSize, size); - await _mediator.Publish(new FileDedupedEvent(hashed.ContentHash, size), cancellationToken); + await _mediator.Publish(new FileDedupedEvent(hashed.FilePair.RelativePath, hashed.ContentHash, size), cancellationToken); } else { diff --git a/src/Arius.Core/Features/ArchiveCommand/Events.cs b/src/Arius.Core/Features/ArchiveCommand/Events.cs index 505589023..8603f973a 100644 --- a/src/Arius.Core/Features/ArchiveCommand/Events.cs +++ b/src/Arius.Core/Features/ArchiveCommand/Events.cs @@ -88,9 +88,11 @@ public sealed record ChunkUploadedEvent(ChunkHash ChunkHash, long StoredSize, lo /// A file's contents are already present in the remote. /// Contrast , which fires for content that is uploaded. /// +/// The deduplicated file. Consumers tracking per-file state need this: a content +/// hash can map to several paths in one run, and the other copies may still be uploading. /// Content hash of the deduplicated file (already present in the repository). /// Uncompressed size in bytes of the file whose content was not re-uploaded. -public sealed record FileDedupedEvent(ContentHash ContentHash, long OriginalSize) : INotification; +public sealed record FileDedupedEvent(RelativePath RelativePath, ContentHash ContentHash, long OriginalSize) : INotification; /// A tar bundle is being sealed. /// Number of entries in the sealed tar. From d0f6e3e15cc65e2047cc722c1ae2313b0b0cfc6d Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:50:56 +0200 Subject: [PATCH 30/46] fix(streaming): release the inner stream even if the final progress report throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8f367c6f added a flush-on-dispose so a throttled stream still reports its true total, but put ReportFinal() before _inner.Dispose() with nothing between them. The progress callback is caller-supplied — a CLI or Api sink — so it can throw during teardown, for instance reporting into a progress task a cancellation has already completed. When it does, Dispose abandons the inner SparseSamplingStream and the FileStream handle underneath it, and the callback's exception replaces whatever was unwinding the using block. Wrap the report in try/finally so disposal is unconditional. The regression test has to let the first report succeed — the first read always reports by design — then read again inside the throttle window so the total is left unreported, and only then fail the flush on dispose. Verified: dotnet test src/Arius.Core.Tests — 666 passed, 1 skipped. Co-Authored-By: Claude Opus 5 --- .../Shared/Streaming/ProgressStreamTests.cs | 22 +++++++++++++++++++ .../Shared/Streaming/ProgressStream.cs | 14 ++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index add3b2c12..f89672910 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -124,4 +124,26 @@ public void ReadSpan_ReportsProgress() n.ShouldBe(100); lastReport.ShouldBe(100); } + + [Test] + public void Dispose_WhenFinalProgressReportThrows_StillDisposesTheInnerStream() + { + var src = new MemoryStream(new byte[64]); + + // The first read always reports; let that succeed, then fail the flush-on-dispose report. + var reports = 0; + var ps = new ProgressStream(src, new SyncProgress(_ => + { + if (reports++ > 0) + throw new InvalidOperationException("sink is gone"); + })); + + ps.Read(new byte[16], 0, 16); // reports (first read) + ps.Read(new byte[16], 0, 16); // throttled, so the total is left unreported + + Should.Throw(() => ps.Dispose()); + + // The inner stream must be released even though the caller's progress sink failed. + src.CanRead.ShouldBeFalse(); + } } diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index 90aee6d97..6ba8b5c09 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -147,8 +147,18 @@ protected override void Dispose(bool disposing) { if (disposing) { - ReportFinal(); - _inner.Dispose(); + // The progress callback is caller-supplied (a CLI/Api sink), so it can throw during teardown — + // e.g. reporting into a progress task already completed by a cancellation. Releasing the inner + // stream must not depend on it, or a failing callback leaks the file handle and replaces + // whatever exception was unwinding the using block. + try + { + ReportFinal(); + } + finally + { + _inner.Dispose(); + } } base.Dispose(disposing); From 529f1a16e38a3e71eb4bd87dad80e943c0bd2f21 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 06:52:57 +0200 Subject: [PATCH 31/46] fix(chunk-index): bound the batched lookup in the store, drop the slot padding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems with the IN (...) lookup a3e957a6 introduced. The parameter padding was justified by a claim that is simply false. I wrote that rounding the count to 1/4/16/64/256 makes "the generated SQL — and its prepared statement — repeat", but FindEntriesCore builds a new SqliteCommand and new command text on every call, and Microsoft.Data.Sqlite does not cache prepared statements across command objects. Nothing was ever reused, so the buckets only padded every lookup with redundant parameters, digest buffers and binds — a 3-hash lookup bound 16. Removed; the query now uses exactly the hashes it has. The bucket fallback also left the store unbounded (`_ => count`). 89f01efc fixed the real call paths by chunking in ChunkIndexService, but FindEntries and FindPendingFlushEntries are public on the store, so the invariant lived only in the caller and the next one to skip it gets SQLite's "too many SQL variables" — which CreateLocalStoreException reports as a corrupt cache the operator should delete, an actively misleading diagnosis. Page inside the store instead, so the bound holds wherever it is called from. The service-level chunking stays; it is now belt and braces rather than the only guard. Regression test looks up 40,001 hashes directly against the store and asserts the one seeded entry comes back. AllocationBenchmarks (one 256-hash dedup batch) is unchanged by the paging: FindEntries 205.48 KB / 380.9 us against FindEntry x256 at 610 KB / 1017.7 us. Verified: dotnet test src/Arius.Core.Tests — 667 passed, 1 skipped. Co-Authored-By: Claude Opus 5 --- .../ChunkIndex/ChunkIndexLocalStoreTests.cs | 30 +++++++ .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 79 +++++++++---------- 2 files changed, 68 insertions(+), 41 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs b/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs index 428d3899f..19cfd12e0 100644 --- a/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs +++ b/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs @@ -23,6 +23,36 @@ public void Initialize_CreatesSchemaVersionAndWalMode() journalMode.ExecuteScalar().ShouldBe("wal"); } + [Test] + public void FindEntries_WithMoreHashesThanSqliteVariableLimit_ReturnsMatches() + { + // The store's own API must stay usable at any width: `ls` looks up one hash per file in a + // directory, so a wide directory produces a wide set. Binding one parameter per hash fails past + // SQLITE_MAX_VARIABLE_NUMBER (32766) with "too many SQL variables", which CreateLocalStoreException + // then reports as a corrupt cache the operator should delete. + var repositoryKey = $"acct-local-store-wide-{Guid.NewGuid():N}"; + var root = RepositoryLocalStatePaths.GetChunkIndexCacheRoot(repositoryKey, repositoryKey); + var store = new ChunkIndexLocalStore(root); + + var stored = new ShardEntry(FakeContentHash('a'), FakeChunkHash('b'), 10, 5, BlobTier.Cool); + store.UpsertPendingFlush(stored); + + var hashes = new List(40_001) { stored.ContentHash }; + hashes.AddRange(Enumerable.Range(0, 40_000).Select(WideLookupHash)); + + var found = store.FindEntries(hashes); + + found.Count.ShouldBe(1); + found[stored.ContentHash].ShouldBe(stored); + } + + private static ContentHash WideLookupHash(int seed) + { + var digest = new byte[32]; + BitConverter.TryWriteBytes(digest.AsSpan(1), seed); + return ContentHash.FromDigest(digest); + } + [Test] public void UpsertPendingFlush_AndLookup_RoundTripsEntry() { diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index 639f3b857..c77496ba1 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -104,6 +104,16 @@ public IReadOnlyDictionary FindPendingFlushEntries(IRea /// Parameters are padded to fixed buckets so the generated command text can be reused. Unused slots /// repeat the first hash, which is harmless for set membership. /// + /// + /// Looks up a set of hashes with a single IN (...) query per page instead of one query per hash. + /// + /// + /// Paged because the set is not bounded by the caller: the archive dedup stage batches 256, but + /// ListQueryHandler looks up one hash per file in a directory and ChunkHydrationStatusQuery one per + /// selected file, so a wide directory would otherwise exceed SQLite's SQLITE_MAX_VARIABLE_NUMBER + /// (32766). That surfaces through CreateLocalStoreException as "corrupt local store, delete the cache" + /// — an actively misleading diagnosis for a query that is simply too wide. + /// private IReadOnlyDictionary FindEntriesCore(IReadOnlyCollection contentHashes, bool pendingFlushOnly) { if (contentHashes.Count == 0) @@ -111,42 +121,13 @@ private IReadOnlyDictionary FindEntriesCore(IReadOnlyCo try { - var slots = LookupBucketSize(contentHashes.Count); var results = new Dictionary(contentHashes.Count); using var connection = OpenConnection(); - using var command = connection.CreateCommand(); - command.CommandText = BuildFindEntriesSql(slots, pendingFlushOnly); - - var digests = new byte[slots][]; - for (var i = 0; i < slots; i++) - { - digests[i] = CreateDigestBuffer(); - command.Parameters.Add($"$h{i}", SqliteType.Blob).Value = digests[i]; - } - - var slot = 0; - var first = string.Empty; - foreach (var contentHash in contentHashes) - { - var hex = contentHash.ToString(); - if (slot == 0) - first = hex; - - WriteDigest(hex, digests[slot++]); - } - - for (; slot < slots; slot++) - WriteDigest(first, digests[slot]); - - using var reader = command.ExecuteReader(); - while (reader.Read()) - { - var entry = ReadEntry(reader); - results[entry.ContentHash] = entry; - } + foreach (var page in contentHashes.Chunk(MaxLookupPageSize)) + ReadEntryPage(connection, page, pendingFlushOnly, results); - _logger.LogDebug("[chunk-index-local] FindEntries: requested={Requested} slots={Slots} pendingFlushOnly={PendingFlushOnly} found={Found}", contentHashes.Count, slots, pendingFlushOnly, results.Count); + _logger.LogDebug("[chunk-index-local] FindEntries: requested={Requested} pendingFlushOnly={PendingFlushOnly} found={Found}", contentHashes.Count, pendingFlushOnly, results.Count); return results; } catch (SqliteException ex) @@ -155,16 +136,32 @@ private IReadOnlyDictionary FindEntriesCore(IReadOnlyCo } } - /// Fixed parameter-count buckets, so the generated SQL — and its prepared statement — repeats. - private static int LookupBucketSize(int count) => count switch + /// Reads one page of hashes with a single IN (...) query, merging into . + private static void ReadEntryPage(SqliteConnection connection, ContentHash[] page, bool pendingFlushOnly, Dictionary results) { - <= 1 => 1, - <= 4 => 4, - <= 16 => 16, - <= 64 => 64, - <= 256 => 256, - _ => count, - }; + using var command = connection.CreateCommand(); + command.CommandText = BuildFindEntriesSql(page.Length, pendingFlushOnly); + + for (var i = 0; i < page.Length; i++) + { + var digest = CreateDigestBuffer(); + WriteDigest(page[i].ToString(), digest); + command.Parameters.Add($"$h{i}", SqliteType.Blob).Value = digest; + } + + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var entry = ReadEntry(reader); + results[entry.ContentHash] = entry; + } + } + + /// + /// Largest number of hashes bound into one IN (...) query, keeping the parameter count well + /// inside SQLite's limit regardless of how wide a caller's lookup is. + /// + private const int MaxLookupPageSize = 256; private static string BuildFindEntriesSql(int slots, bool pendingFlushOnly) { From 9a6326cea953ad905152b8302d7efe0acd255830 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:36:35 +0200 Subject: [PATCH 32/46] fix(archive): size the tar buffer from observed framing, trim tail bundles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The presize from b46fd6eb jumped to a fixed 1.25 × TarTargetSize once a bundle passed 1 MB. Two problems with that: - Tar framing is ~1.5 KB per entry, so a full bundle of small files outgrows a 25% headroom (4 KB files: ~37%, 1 KB files: ~150%). MemoryStream then doubled past it and retained more than master did: 160 MB vs 128 MB for 4 KB files, 320 MB vs 256 MB for 1 KB files. - A tail bundle that passed 1 MB (e.g. a 3 MB incremental) reserved the full 80 MB and held it through the upload, where master held ~4 MB. Grow once to the projected full size instead — tar bytes per file byte observed at the threshold, applied to the target plus 1/8 headroom — and when a bundle seals at less than half its buffer, trim it to its length so the tail does not retain the presized array. The transient allocation for a >1 MB tail remains. AllocationBenchmarks (64 KB entries): seal one 64 MB bundle 85,621 KB -> 79,157 KB seal one small bundle (5x1KB) 40.8 KB -> 40.8 KB Verified: dotnet test src/Arius.Core.Tests — 669 passed, 1 skipped. Co-Authored-By: Claude Opus 5.5 --- .../ArchiveCommand/TarBuilderTests.cs | 35 ++++++++++++++++++ .../Features/ArchiveCommand/TarBuilder.cs | 37 +++++++++---------- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/Arius.Core.Tests/Features/ArchiveCommand/TarBuilderTests.cs b/src/Arius.Core.Tests/Features/ArchiveCommand/TarBuilderTests.cs index 80ed525b3..6df8aaccd 100644 --- a/src/Arius.Core.Tests/Features/ArchiveCommand/TarBuilderTests.cs +++ b/src/Arius.Core.Tests/Features/ArchiveCommand/TarBuilderTests.cs @@ -103,6 +103,34 @@ public async Task SealAsync_TarHash_MatchesHashOfBundleContent() bundle.TarHash.ShouldBe(ChunkHashOf(bundle.Content, IEncryptionService.PlaintextInstance)); } + // ── Buffer sizing ───────────────────────────────────────────────────────── + + [Test] + public async Task AddAsync_FullBundleOfTinyFiles_BufferFitsTheirTarFraming() + { + // 4 KB files carry ~37% tar framing (headers + padding), far above a fixed headroom over the target. + await using var builder = new TarBuilder(targetSize: 8 * 1024 * 1024, IEncryptionService.PlaintextInstance); + + SealedTar? bundle = null; + for (var i = 0; bundle is null; i++) + bundle = await builder.AddAsync(Upload($"f{i}", UniqueHash(i), 4 * 1024), new MemoryStream(new byte[4 * 1024]), CancellationToken.None); + + bundle.Content.Array!.Length.ShouldBeLessThan((int)(bundle.Content.Count * 1.25)); + } + + [Test] + public async Task SealAsync_PartialBundle_DoesNotHoldTheFullTargetBuffer() + { + await using var builder = new TarBuilder(targetSize: 64 * 1024 * 1024, IEncryptionService.PlaintextInstance); + + for (var i = 0; i < 768; i++) // 3 MB of 4 KB files: past any presize threshold, far short of the target + await builder.AddAsync(Upload($"f{i}", UniqueHash(i), 4 * 1024), new MemoryStream(new byte[4 * 1024]), CancellationToken.None); + + var bundle = await builder.SealAsync(CancellationToken.None); + + bundle!.Content.Array!.Length.ShouldBeLessThanOrEqualTo(bundle.Content.Count * 2); + } + // ── Lifecycle callbacks ─────────────────────────────────────────────────── [Test] @@ -176,6 +204,13 @@ private static (byte[] Content, ContentHash Hash) Content(byte fill, int count) return (bytes, IEncryptionService.PlaintextInstance.ComputeHash(bytes)); } + private static ContentHash UniqueHash(int seed) + { + var digest = new byte[32]; + BitConverter.TryWriteBytes(digest, seed); + return ContentHash.FromDigest(digest); + } + /// Reads a sealed tar bundle back into a map of entry-name → entry-bytes. private static async Task> ReadBundleAsync(SealedTar bundle) { diff --git a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs index c9ccddb8f..30a7ddfe6 100644 --- a/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs +++ b/src/Arius.Core/Features/ArchiveCommand/TarBuilder.cs @@ -32,22 +32,13 @@ internal sealed class TarBuilder : IAsyncDisposable private long _currentSize; /// - /// Headroom for TAR framing beyond the summed file sizes. - /// - private const int FramingHeadroomDivisor = 4; - - /// - /// Accumulated size past which a bundle is presumed to be heading for and its - /// buffer is grown to full capacity in one step. Below it the bundle keeps MemoryStream's own doubling, - /// so a repository of a few small files — and every run's partial tail bundle — never reserves the full - /// target. The doubling this still pays to reach the threshold is bounded by twice the threshold. + /// Tar length past which the buffer grows in one step to the bundle's projected full size. Below it, + /// MemoryStream keeps doubling, so a bundle of a few small files never reserves the full target. /// private const int PresizeThreshold = 1024 * 1024; - /// - /// Full backing-buffer capacity for a bundle, clamped to the limit. - /// - private int FullCapacity => (int)Math.Min(_targetSize + _targetSize / FramingHeadroomDivisor, int.MaxValue / 2); + /// Headroom over the projected size, for the entry that overshoots the target. + private const int ProjectionHeadroomDivisor = 8; /// A bundle is sealed once its accumulated size reaches this threshold. /// Used to hash the sealed tar body. @@ -89,6 +80,7 @@ public TarBuilder( } // Write the entry named by content-hash (not original path). + var lengthBefore = _tarStream!.Length; var tarEntry = new PaxTarEntry(TarEntryType.RegularFile, upload.HashedPair.ContentHash.ToString()); await using (source) { @@ -100,12 +92,14 @@ public TarBuilder( _entries.Add(new TarEntry(upload.HashedPair.ContentHash, upload.FileSize, upload.HashedPair)); _currentSize += upload.FileSize; - // Jump straight to full capacity once the bundle is clearly filling up. Growing from zero by - // doubling all the way to a 64 MB bundle abandons every intermediate array — measured at 276 MB of - // mostly-LOH garbage per bundle — while presizing unconditionally would reserve the full target for - // bundles that never approach it. - if (_tarStream!.Capacity < FullCapacity && _tarStream.Length >= PresizeThreshold) - _tarStream.Capacity = FullCapacity; + // Rather than doubling all the way to the target, grow once, sized by the tar bytes written per file + // byte so far: per-entry framing makes a bundle of tiny files far longer than its summed file sizes. + if (lengthBefore < PresizeThreshold && _tarStream.Length >= PresizeThreshold) + { + var projected = _tarStream.Length * (_targetSize + _targetSize / ProjectionHeadroomDivisor) / _currentSize; + if (projected > _tarStream.Capacity) + _tarStream.Capacity = (int)Math.Min(projected, int.MaxValue / 2); + } await (_onEntryAdded?.Invoke(upload.HashedPair.ContentHash, _entries.Count, _currentSize) ?? ValueTask.CompletedTask); @@ -126,6 +120,11 @@ public TarBuilder( _tarWriter = null; var body = _tarStream!; // set together with _tarWriter in AddAsync, so non-null whenever the writer was + + // A bundle sealed well short of its presized buffer (the run's tail) must not hold it through the upload. + if (body.Capacity > 2 * body.Length) + body.Capacity = (int)body.Length; + body.Position = 0; var tarHash = ChunkHash.Parse(await _encryption.ComputeHashAsync(body, cancellationToken)); From ac197a49b2723dd0a510202cd1aea0e79f5f9322 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:37:25 +0200 Subject: [PATCH 33/46] revert(streaming): drop the throwing-sink guard in ProgressStream.Dispose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts d0f6e3e1. The try/finally did not do what its comment claimed: a throwing ReportFinal still escaped Dispose and replaced the exception that was unwinding the using block. It also guarded a path production never takes: every sink handed to ProgressStream is Progress, whose Report posts the callback and never throws synchronously, and the upload path's CallbackProgress stream is never disposed. The regression test goes with it; it relied on two reads landing inside the real 500 ms throttle window, so it was timing-dependent on a loaded runner. Verified: dotnet test src/Arius.Core.Tests --treenode-filter ProgressStreamTests — 7 passed. Co-Authored-By: Claude Opus 5.5 --- .../Shared/Streaming/ProgressStreamTests.cs | 22 ------------------- .../Shared/Streaming/ProgressStream.cs | 14 ++---------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index f89672910..add3b2c12 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -124,26 +124,4 @@ public void ReadSpan_ReportsProgress() n.ShouldBe(100); lastReport.ShouldBe(100); } - - [Test] - public void Dispose_WhenFinalProgressReportThrows_StillDisposesTheInnerStream() - { - var src = new MemoryStream(new byte[64]); - - // The first read always reports; let that succeed, then fail the flush-on-dispose report. - var reports = 0; - var ps = new ProgressStream(src, new SyncProgress(_ => - { - if (reports++ > 0) - throw new InvalidOperationException("sink is gone"); - })); - - ps.Read(new byte[16], 0, 16); // reports (first read) - ps.Read(new byte[16], 0, 16); // throttled, so the total is left unreported - - Should.Throw(() => ps.Dispose()); - - // The inner stream must be released even though the caller's progress sink failed. - src.CanRead.ShouldBeFalse(); - } } diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index 6ba8b5c09..90aee6d97 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -147,18 +147,8 @@ protected override void Dispose(bool disposing) { if (disposing) { - // The progress callback is caller-supplied (a CLI/Api sink), so it can throw during teardown — - // e.g. reporting into a progress task already completed by a cancellation. Releasing the inner - // stream must not depend on it, or a failing callback leaks the file handle and replaces - // whatever exception was unwinding the using block. - try - { - ReportFinal(); - } - finally - { - _inner.Dispose(); - } + ReportFinal(); + _inner.Dispose(); } base.Dispose(disposing); From 68f0dae05dbda2bf6d0474249c8306e419e0be43 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:38:12 +0200 Subject: [PATCH 34/46] test(cli): cover deduplicating a copy of a file that is still uploading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1477b3f3 changed FileDedupedHandler to drop only the deduplicated path's row, without a test. Removing rows by content hash instead drops the original's row mid-upload, so the following ChunkUploadingEvent finds nothing and FilesUnique undercounts. The new test fails against that behaviour and passes now. Verified: dotnet test src/Arius.Cli.Tests — 165 passed. Co-Authored-By: Claude Opus 5.5 --- .../Archive/NotificationHandlerTests.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs b/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs index 47a2caf5d..36c0eb581 100644 --- a/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs +++ b/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs @@ -145,6 +145,29 @@ public async Task FileSkippedHandler_DuringUpload_RemovesTrackedFileWithoutIncre state.TrackedFiles.ContainsKey(path).ShouldBeFalse(); } + [Test] + public async Task FileDedupedHandler_DuplicateOfFileBeingUploaded_KeepsTheUploadingRow() + { + var state = new ProgressState(); + var hashingH = new FileHashingHandler(state); + var hashedH = new FileHashedHandler(state); + var dedupedH = new FileDedupedHandler(state); + var uploadingH = new ChunkUploadingHandler(state); + var original = RelativePath.Parse("original.bin"); + var duplicate = RelativePath.Parse("copy/original.bin"); + + await hashingH.Handle(new FileHashingEvent(original, 5000), CancellationToken.None); + await hashingH.Handle(new FileHashingEvent(duplicate, 5000), CancellationToken.None); + await hashedH.Handle(new FileHashedEvent(original, FakeContentHash('7'), false, false), CancellationToken.None); + await hashedH.Handle(new FileHashedEvent(duplicate, FakeContentHash('7'), false, false), CancellationToken.None); + await dedupedH.Handle(new FileDedupedEvent(duplicate, FakeContentHash('7'), 5000), CancellationToken.None); + await uploadingH.Handle(new ChunkUploadingEvent(FakeChunkHash('7'), 5000), CancellationToken.None); + + state.TrackedFiles.ContainsKey(duplicate).ShouldBeFalse(); + state.TrackedFiles[original].State.ShouldBe(FileState.Uploading); + state.FilesUnique.ShouldBe(1L); + } + [Test] public async Task TarBundleStartedHandler_CreatesTrackedTar() { From cedb4b632386e8dfc65cbdd84b355ed1437f03f9 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:39:26 +0200 Subject: [PATCH 35/46] fix(cli): stop the content-hash reverse lookup growing with the run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 10947e7b stopped TrackedFiles growing, but ContentHashToPath still gained an entry (with a ConcurrentBag) for every hashed file and never lost one — held for the whole run, so a 1M-file archive kept 1M of them. Remove a content hash's entry when its rows are done: when its chunk is uploaded or it is added to a tar bundle (both already removed every row for the hash), and when a deduplicated copy was the last tracked path for it. A copy deduplicated while the original is uploading leaves the entry in place, since the upload's events still need it. Files skipped after hashing still leave an entry behind; they are rare and bounded by the failure count. Verified: dotnet test src/Arius.Cli.Tests — 166 passed. Co-Authored-By: Claude Opus 5.5 --- .../Archive/NotificationHandlerTests.cs | 18 +++++++++++++++ .../Archive/ArchiveProgressHandlers.cs | 13 +++-------- src/Arius.Cli/ProgressState.cs | 22 ++++++++++++++++++- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs b/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs index 36c0eb581..59f149ddd 100644 --- a/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs +++ b/src/Arius.Cli.Tests/Commands/Archive/NotificationHandlerTests.cs @@ -168,6 +168,22 @@ public async Task FileDedupedHandler_DuplicateOfFileBeingUploaded_KeepsTheUpload state.FilesUnique.ShouldBe(1L); } + [Test] + public async Task FileDedupedHandler_LastTrackedCopy_ForgetsTheContentHash() + { + var state = new ProgressState(); + var hashingH = new FileHashingHandler(state); + var hashedH = new FileHashedHandler(state); + var dedupedH = new FileDedupedHandler(state); + var path = RelativePath.Parse("known.bin"); + + await hashingH.Handle(new FileHashingEvent(path, 5000), CancellationToken.None); + await hashedH.Handle(new FileHashedEvent(path, FakeContentHash('6'), false, false), CancellationToken.None); + await dedupedH.Handle(new FileDedupedEvent(path, FakeContentHash('6'), 5000), CancellationToken.None); + + state.ContentHashToPath.ContainsKey(FakeContentHash('6')).ShouldBeFalse(); + } + [Test] public async Task TarBundleStartedHandler_CreatesTrackedTar() { @@ -213,6 +229,7 @@ public async Task TarEntryAddedHandler_RemovesTrackedFileAndUpdatesTrackedTar() await tarEntryH.Handle(new TarEntryAddedEvent(FakeContentHash('b'), 1, 500), CancellationToken.None); state.TrackedFiles.ContainsKey(path).ShouldBeFalse(); + state.ContentHashToPath.ContainsKey(FakeContentHash('b')).ShouldBeFalse(); state.TrackedTars[1].FileCount.ShouldBe(1); state.TrackedTars[1].AccumulatedBytes.ShouldBe(500L); } @@ -304,6 +321,7 @@ public async Task ChunkUploadedHandler_RemovesFileAndIncrementsChunksUploaded() await uploadedH.Handle(new ChunkUploadedEvent(FakeChunkHash('9'), 4000, 5000), CancellationToken.None); state.TrackedFiles.ContainsKey(path).ShouldBeFalse(); + state.ContentHashToPath.ContainsKey(FakeContentHash('9')).ShouldBeFalse(); state.ChunksUploaded.ShouldBe(1L); state.BytesUploaded.ShouldBe(4000L); } diff --git a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs index ac1358617..dd356e0a0 100644 --- a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs +++ b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs @@ -107,9 +107,7 @@ public sealed class TarEntryAddedHandler(ProgressState state) : INotificationHan { public ValueTask Handle(TarEntryAddedEvent notification, CancellationToken cancellationToken) { - if (state.ContentHashToPath.TryGetValue(notification.ContentHash, out var paths)) - foreach (var path in paths) - state.RemoveFile(path); + state.RemoveContentHash(notification.ContentHash); var tar = state.AccumulatingTar; if (tar is { State: TarState.Accumulating }) @@ -156,10 +154,7 @@ public sealed class FileDedupedHandler(ProgressState state) : INotificationHandl { public ValueTask Handle(FileDedupedEvent notification, CancellationToken cancellationToken) { - // Only this path. A content hash can map to several paths in one run, and the first copy is the - // one being uploaded — removing by hash would drop its progress row mid-upload and make the - // subsequent ChunkUploadingEvent miss it, skipping IncrementFilesUnique. - state.RemoveFile(notification.RelativePath); + state.RemoveDeduplicatedFile(notification.RelativePath, notification.ContentHash); return ValueTask.CompletedTask; } @@ -204,9 +199,7 @@ public sealed class ChunkUploadedHandler(ProgressState state) : INotificationHan { public ValueTask Handle(ChunkUploadedEvent notification, CancellationToken cancellationToken) { - if (state.ContentHashToPath.TryGetValue(ContentHash.Parse(notification.ChunkHash), out var paths)) - foreach (var path in paths) - state.RemoveFile(path); + state.RemoveContentHash(ContentHash.Parse(notification.ChunkHash)); state.IncrementChunksUploaded(notification.StoredSize); return ValueTask.CompletedTask; } diff --git a/src/Arius.Cli/ProgressState.cs b/src/Arius.Cli/ProgressState.cs index 0380309fc..f2e053b84 100644 --- a/src/Arius.Cli/ProgressState.cs +++ b/src/Arius.Cli/ProgressState.cs @@ -253,7 +253,8 @@ public sealed class ProgressState /// /// Reverse lookup: ContentHash → one or more RelativePaths. /// One-to-many because two files can legitimately hash to the same content in one run. - /// Populated when FileHashedEvent fires; used by downstream content-hash-keyed events. + /// Populated when FileHashedEvent fires; used by downstream content-hash-keyed events, and + /// removed once the content's rows are done. /// public ConcurrentDictionary> ContentHashToPath { get; } = new(); @@ -307,6 +308,25 @@ public bool SetFileUploading(ContentHash contentHash) public void RemoveFile(RelativePath relativePath) => TrackedFiles.TryRemove(relativePath, out _); + /// Removes the reverse lookup for and the rows of all its paths. + public void RemoveContentHash(ContentHash contentHash) + { + if (ContentHashToPath.TryRemove(contentHash, out var paths)) + foreach (var path in paths) + RemoveFile(path); + } + + /// + /// Removes the row of one deduplicated path, and the reverse lookup once none of its paths is tracked. + /// Other paths keep theirs: the first copy of the content may still be uploading. + /// + public void RemoveDeduplicatedFile(RelativePath relativePath, ContentHash contentHash) + { + RemoveFile(relativePath); + if (ContentHashToPath.TryGetValue(contentHash, out var paths) && !paths.Any(TrackedFiles.ContainsKey)) + ContentHashToPath.TryRemove(contentHash, out _); + } + /// /// Removes the tracked row for a skipped file. Only files still in /// count toward hashing completion; later-stage skips still clear the row but must not double-count. From cf24575f59bc5bf441d3a56b13a9faf05cd59904 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:40:07 +0200 Subject: [PATCH 36/46] refactor(chunk-index): let the local store own lookup paging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 89f01efc split LookupAsync's hashes into 256-hash batches, and 529f1a16 then made ChunkIndexLocalStore page every batched lookup into 256-hash queries itself. The service-side split is now redundant: it paged twice, copied the hashes again, opened a connection per batch instead of per lookup, and presized an extra dictionary. Pass the whole set to the store. Verified: dotnet test src/Arius.Core.Tests — 668 passed, 1 skipped (includes LookupAsync_SameRootBatchLargerThan256_ResolvesAllEntries and FindEntries_WithMoreHashesThanSqliteVariableLimit_ReturnsMatches). Co-Authored-By: Claude Opus 5.5 --- .../Shared/ChunkIndex/ChunkIndexService.cs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs index dcec38a9b..e2762ec32 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexService.cs @@ -21,7 +21,6 @@ internal sealed class ChunkIndexService : IChunkIndexService internal const int MaxShardEntryCount = 1024; internal const int FlushWorkers = 32; internal const int PrefixLoadWorkers = 8; - private const int LookupBatchSize = 256; internal static readonly RelativePath RepairInProgressMarkerPath = RelativePath.Root / PathSegment.Parse("chunk-index.repair-in-progress"); private readonly IBlobContainerService _blobs; @@ -106,12 +105,7 @@ public async Task> LookupAsync(IEnu return result; // Probe pending-flush entries in one batch before remote validation. - var pendingFlush = new Dictionary(hashes.Length); - foreach (var batch in hashes.Chunk(LookupBatchSize)) - { - foreach (var (contentHash, entry) in _localStore.FindPendingFlushEntries(batch)) - pendingFlush[contentHash] = entry; - } + var pendingFlush = _localStore.FindPendingFlushEntries(hashes); var validationWork = new List<(PathSegment Root, List Hashes)>(); foreach (var rootGroup in hashes.GroupBy(ChunkIndexRouter.GetRootPrefix)) @@ -154,11 +148,8 @@ await Parallel.ForEachAsync( // Populate the result from validated shards using batched lookups. foreach (var item in validationWork) { - foreach (var batch in item.Hashes.Chunk(LookupBatchSize)) - { - foreach (var (contentHash, entry) in _localStore.FindEntries(batch)) - result[contentHash] = entry; - } + foreach (var (contentHash, entry) in _localStore.FindEntries(item.Hashes)) + result[contentHash] = entry; } return result; From 53c3bee5faedc60932dfa70c4df827662f42bf5c Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:40:59 +0200 Subject: [PATCH 37/46] refactor: drop two guards for states that cannot occur MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ChunkIndexLocalStore.WriteDigest: ContentHash and ChunkHash hold exactly 64 canonical hex characters by construction (NormalizeHex / ToLowerHex), so the span conversion into a 32-byte buffer always returns Done. The status check from 6436a6fd guarded nothing; the doc comment now states the invariant. - SparseFingerprint.Sampler.Capture: its only caller, SparseSamplingStream, reads from the inner stream first, and that read already throws once the stream is disposed. Finish keeps its guard: Fingerprint() after dispose is a reachable misuse and would persist a fingerprint of pooled bytes. Verified: dotnet test src/Arius.Core.Tests — 668 passed, 1 skipped. Co-Authored-By: Claude Opus 5.5 --- .../Shared/HashCache/SparseFingerprintTests.cs | 7 +++---- .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 14 +++----------- .../Shared/HashCache/SparseFingerprint.cs | 4 ---- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs b/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs index 354982138..d6c5841c9 100644 --- a/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs +++ b/src/Arius.Core.Tests/Shared/HashCache/SparseFingerprintTests.cs @@ -105,14 +105,13 @@ private static (RelativeFileSystem fs, RelativePath path) WriteTempFile(byte[] d } [Test] - public void Sampler_AfterDispose_RefusesFurtherUse() + public void Sampler_FinishAfterDispose_Throws() { - // Dispose returns the capture buffer to ArrayPool. Using it afterwards would read or write an - // array another consumer now owns, so it must fail loudly rather than silently corrupt. + // Dispose returns the capture buffer to ArrayPool; fingerprinting it afterwards would hash bytes + // another consumer now owns. var sampler = new SparseFingerprint.Sampler(1024); sampler.Dispose(); Should.Throw(() => sampler.Finish()); - Should.Throw(() => sampler.Capture(0, new byte[16])); } } diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index c77496ba1..0b99d6ca3 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -1,4 +1,3 @@ -using System.Buffers; using System.Collections.ObjectModel; using System.Text; using System.Text.Json; @@ -1023,18 +1022,11 @@ private static ShardEntry ReadEntry(SqliteDataReader reader) ShardEntry.DeserializeTier(reader.GetInt32(4))); /// - /// Writes a canonical hexadecimal hash into a 32-byte digest buffer. + /// Writes a canonical hexadecimal hash into a 32-byte digest buffer. The hash types guarantee exactly + /// 64 canonical hex characters, so the conversion cannot fail. /// private static void WriteDigest(string hex, byte[] destination) - { - // The span overload reports failure instead of throwing, unlike the Convert.FromHexString(string) - // this replaced. The destination buffers are reused across a batch, so a silent non-Done result - // would leave the *previous* row's digest in place and bind a chunk-index row mapping the wrong - // content hash to a chunk — silent, durable, and a dedup-correctness failure rather than a crash. - var status = Convert.FromHexString(hex, destination, out _, out var written); - if (status != OperationStatus.Done || written != HashCodec.Sha256ByteLength) - throw new InvalidOperationException($"Could not convert a {HashCodec.Sha256HexLength}-character hash to {HashCodec.Sha256ByteLength} digest bytes ({status}, wrote {written})."); - } + => Convert.FromHexString(hex, destination, out _, out _); /// Creates a reusable 32-byte digest buffer. private static byte[] CreateDigestBuffer() => new byte[HashCodec.Sha256ByteLength]; diff --git a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs index 1d06006b8..5bd4d55e0 100644 --- a/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs +++ b/src/Arius.Core/Shared/HashCache/SparseFingerprint.cs @@ -126,10 +126,6 @@ public Sampler(long size) /// Offer the bytes read at ; overlapping region bytes are copied out. public void Capture(long position, ReadOnlySpan buffer) { - // The backing array goes back to the shared pool on dispose. Writing to it afterwards would - // corrupt whichever unrelated consumer holds it next — silently, and nowhere near here. - ObjectDisposedException.ThrowIf(_disposed, this); - for (var i = 0; i < _regions.Count; i++) { var (off, len) = _regions[i]; From 17440094f0988c73588ee9e5d1a9df89374338ae Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:42:13 +0200 Subject: [PATCH 38/46] refactor(streaming): trim ProgressStream's throttling machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use TimeProvider for the clock seam instead of a custom Func plus a static default; tests pass a FrozenTimeProvider. - Drop _hasReported: the first report is always non-zero, so it equals _reportedBytes > 0, and ReportFinal's empty-source check folds into _reportedBytes == _bytesRead. - Route the four Read overloads through one OnRead helper instead of four copies of the same accounting block. - The progress parameter doc no longer promises a report after every read. No behaviour change. Verified: dotnet test src/Arius.Core.Tests — 668 passed, 1 skipped. Co-Authored-By: Claude Opus 5.5 --- .../Shared/Streaming/FrozenTimeProvider.cs | 7 ++ .../Shared/Streaming/ProgressStreamTests.cs | 3 +- .../Shared/Streaming/ProgressStream.cs | 109 +++++------------- 3 files changed, 39 insertions(+), 80 deletions(-) create mode 100644 src/Arius.Core.Tests/Shared/Streaming/FrozenTimeProvider.cs diff --git a/src/Arius.Core.Tests/Shared/Streaming/FrozenTimeProvider.cs b/src/Arius.Core.Tests/Shared/Streaming/FrozenTimeProvider.cs new file mode 100644 index 000000000..bdf8a775e --- /dev/null +++ b/src/Arius.Core.Tests/Shared/Streaming/FrozenTimeProvider.cs @@ -0,0 +1,7 @@ +namespace Arius.Core.Tests.Shared.Streaming; + +/// A clock that never advances, so every throttle window stays open. +internal sealed class FrozenTimeProvider : TimeProvider +{ + public override long GetTimestamp() => 0; +} diff --git a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs index add3b2c12..6b6e127b4 100644 --- a/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs +++ b/src/Arius.Core.Tests/Shared/Streaming/ProgressStreamTests.cs @@ -12,8 +12,7 @@ public void Read_CoalescesReports_ButStillReportsTheTotal() using var src = new MemoryStream(data); var reports = new List(); var progress = new SyncProgress(v => reports.Add(v)); - var timestamp = 0L; - using var ps = new ProgressStream(src, progress, () => timestamp++); + using var ps = new ProgressStream(src, progress, new FrozenTimeProvider()); var buf = new byte[256]; while (ps.Read(buf, 0, buf.Length) > 0) { } diff --git a/src/Arius.Core/Shared/Streaming/ProgressStream.cs b/src/Arius.Core/Shared/Streaming/ProgressStream.cs index 90aee6d97..91fff7d8c 100644 --- a/src/Arius.Core/Shared/Streaming/ProgressStream.cs +++ b/src/Arius.Core/Shared/Streaming/ProgressStream.cs @@ -1,5 +1,3 @@ -using System.Diagnostics; - namespace Arius.Core.Shared.Streaming; /// @@ -11,33 +9,47 @@ public sealed class ProgressStream : Stream { /// Minimum wall-clock gap between two progress callbacks. private static readonly TimeSpan ReportInterval = TimeSpan.FromMilliseconds(500); - private static readonly Func DefaultTimestampProvider = Stopwatch.GetTimestamp; private readonly Stream _inner; private readonly IProgress _progress; - private readonly Func _getTimestamp; + private readonly TimeProvider _timeProvider; private long _bytesRead; private long _reportedBytes; private long _lastReportTimestamp; - private bool _hasReported; /// The readable source stream. - /// Receives cumulative bytes read after each read call. + /// Receives the cumulative bytes read, throttled as described on the type. public ProgressStream(Stream inner, IProgress progress) - : this(inner, progress, DefaultTimestampProvider) + : this(inner, progress, TimeProvider.System) { } - internal ProgressStream(Stream inner, IProgress progress, Func timestampProvider) + internal ProgressStream(Stream inner, IProgress progress, TimeProvider timeProvider) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(progress); if (!inner.CanRead) throw new ArgumentException("Inner stream must be readable.", nameof(inner)); - _inner = inner; - _progress = progress; - _getTimestamp = timestampProvider; + _inner = inner; + _progress = progress; + _timeProvider = timeProvider; + } + + /// Accounts for one inner read of bytes out of . + private int OnRead(int read, int requested) + { + if (read > 0) + { + _bytesRead += read; + ReportThrottled(); + } + else if (requested != 0) // a zero-length read is not EOF + { + ReportFinal(); + } + + return read; } /// @@ -45,12 +57,11 @@ internal ProgressStream(Stream inner, IProgress progress, Func times /// private void ReportThrottled() { - var now = _getTimestamp(); + var now = _timeProvider.GetTimestamp(); - if (_hasReported && Stopwatch.GetElapsedTime(_lastReportTimestamp, now) < ReportInterval) + if (_reportedBytes > 0 && _timeProvider.GetElapsedTime(_lastReportTimestamp, now) < ReportInterval) return; - _hasReported = true; _lastReportTimestamp = now; _reportedBytes = _bytesRead; _progress.Report(_bytesRead); @@ -61,14 +72,10 @@ private void ReportThrottled() /// private void ReportFinal() { - // Empty sources do not emit a spurious zero. - if (_bytesRead == 0) - return; - - if (_hasReported && _reportedBytes == _bytesRead) + // Also keeps an empty source from emitting a spurious zero. + if (_reportedBytes == _bytesRead) return; - _hasReported = true; _reportedBytes = _bytesRead; _progress.Report(_bytesRead); } @@ -77,69 +84,15 @@ private void ReportFinal() public override bool CanWrite => false; public override bool CanSeek => false; - public override int Read(byte[] buffer, int offset, int count) - { - var n = _inner.Read(buffer, offset, count); - if (n > 0) - { - _bytesRead += n; - ReportThrottled(); - } - else if (count != 0) - { - ReportFinal(); - } + public override int Read(byte[] buffer, int offset, int count) => OnRead(_inner.Read(buffer, offset, count), count); - return n; - } - - public override int Read(Span buffer) - { - var n = _inner.Read(buffer); - if (n > 0) - { - _bytesRead += n; - ReportThrottled(); - } - else if (buffer.Length != 0) - { - ReportFinal(); - } - - return n; - } + public override int Read(Span buffer) => OnRead(_inner.Read(buffer), buffer.Length); public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken ct) - { - var n = await _inner.ReadAsync(buffer, offset, count, ct); - if (n > 0) - { - _bytesRead += n; - ReportThrottled(); - } - else if (count != 0) - { - ReportFinal(); - } - - return n; - } + => OnRead(await _inner.ReadAsync(buffer, offset, count, ct), count); public override async ValueTask ReadAsync(Memory buffer, CancellationToken ct = default) - { - var n = await _inner.ReadAsync(buffer, ct); - if (n > 0) - { - _bytesRead += n; - ReportThrottled(); - } - else if (buffer.Length != 0) - { - ReportFinal(); - } - - return n; - } + => OnRead(await _inner.ReadAsync(buffer, ct), buffer.Length); public override void Flush() => _inner.Flush(); From b2de8ad56777e617375ff8908f1761054f942b03 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:43:05 +0200 Subject: [PATCH 39/46] refactor(filesystem): leave control-character checks to PathSegment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RelativePath.TryParse validates every segment with PathSegment.TryParse, which already rejects control characters, so its own whole-string scan (and the ContainsControlCharacter copy 63acf128 added for it) checked each character twice. Adds a test pinning that a control character anywhere still fails Parse. Verified: dotnet test src/Arius.Core.Tests — 670 passed, 1 skipped. Co-Authored-By: Claude Opus 5.5 --- .../Shared/FileSystem/RelativePathTests.cs | 8 ++++++++ src/Arius.Core/Shared/FileSystem/RelativePath.cs | 14 +------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/Arius.Core.Tests/Shared/FileSystem/RelativePathTests.cs b/src/Arius.Core.Tests/Shared/FileSystem/RelativePathTests.cs index ab9a75621..898af7818 100644 --- a/src/Arius.Core.Tests/Shared/FileSystem/RelativePathTests.cs +++ b/src/Arius.Core.Tests/Shared/FileSystem/RelativePathTests.cs @@ -28,6 +28,14 @@ public void Parse_DotSegment_Throws() Should.Throw(() => RelativePath.Parse("photos/../pic.jpg")); } + [Test] + [Arguments("photos/pi\u0001c.jpg")] + [Arguments("pho\ntos/pic.jpg")] + public void Parse_ControlCharacter_Throws(string value) + { + Should.Throw(() => RelativePath.Parse(value)); + } + [Test] public void FromPlatformRelativePath_NormalizesDirectorySeparators() { diff --git a/src/Arius.Core/Shared/FileSystem/RelativePath.cs b/src/Arius.Core/Shared/FileSystem/RelativePath.cs index 02c05c717..ca2a8d083 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativePath.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativePath.cs @@ -95,7 +95,7 @@ public static bool TryParse(string? value, out RelativePath path) return false; } - if (value.Contains('\\') || value.Contains("//", StringComparison.Ordinal) || ContainsControlCharacter(value)) + if (value.Contains('\\') || value.Contains("//", StringComparison.Ordinal)) { path = default; return false; @@ -164,16 +164,4 @@ public RelativePath RemoveSuffix(string suffix, StringComparison comparisonType) => path.Value.Length == 0 ? new RelativePath(segment.ToString()) : new RelativePath($"{path.Value}/{segment}"); public override string ToString() => Value; - - /// - /// Allocation-free replacement for value.Any(char.IsControl) - /// - private static bool ContainsControlCharacter(string value) - { - foreach (var c in value) - if (char.IsControl(c)) - return true; - - return false; - } } From 435b3ff398d83d7c168dc5b8371ebf227a2c43d8 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:44:08 +0200 Subject: [PATCH 40/46] refactor: make single-use helpers local methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the repo convention, a helper used by one method lives inside it: - ChunkIndexLocalStore: ReadEntryPage is inlined into FindEntriesCore's page loop, and BuildFindEntriesSql becomes its local BuildSql. - PathSegment: ContainsControlCharacter moves into TryParse. No behaviour change. Verified: dotnet test src/Arius.Core.Tests — 670 passed, 1 skipped. Co-Authored-By: Claude Opus 5.5 --- .../Shared/ChunkIndex/ChunkIndexLocalStore.cs | 66 +++++++++---------- .../Shared/FileSystem/PathSegment.cs | 22 +++---- 2 files changed, 41 insertions(+), 47 deletions(-) diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index 0b99d6ca3..e6b837037 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -124,7 +124,24 @@ private IReadOnlyDictionary FindEntriesCore(IReadOnlyCo using var connection = OpenConnection(); foreach (var page in contentHashes.Chunk(MaxLookupPageSize)) - ReadEntryPage(connection, page, pendingFlushOnly, results); + { + using var command = connection.CreateCommand(); + command.CommandText = BuildSql(page.Length, pendingFlushOnly); + + for (var i = 0; i < page.Length; i++) + { + var digest = CreateDigestBuffer(); + WriteDigest(page[i].ToString(), digest); + command.Parameters.Add($"$h{i}", SqliteType.Blob).Value = digest; + } + + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var entry = ReadEntry(reader); + results[entry.ContentHash] = entry; + } + } _logger.LogDebug("[chunk-index-local] FindEntries: requested={Requested} pendingFlushOnly={PendingFlushOnly} found={Found}", contentHashes.Count, pendingFlushOnly, results.Count); return results; @@ -133,26 +150,22 @@ private IReadOnlyDictionary FindEntriesCore(IReadOnlyCo { throw CreateLocalStoreException(ex); } - } - - /// Reads one page of hashes with a single IN (...) query, merging into . - private static void ReadEntryPage(SqliteConnection connection, ContentHash[] page, bool pendingFlushOnly, Dictionary results) - { - using var command = connection.CreateCommand(); - command.CommandText = BuildFindEntriesSql(page.Length, pendingFlushOnly); - for (var i = 0; i < page.Length; i++) + static string BuildSql(int slots, bool pendingFlushOnly) { - var digest = CreateDigestBuffer(); - WriteDigest(page[i].ToString(), digest); - command.Parameters.Add($"$h{i}", SqliteType.Blob).Value = digest; - } + var sql = new StringBuilder("SELECT content_hash, chunk_hash, original_size, chunk_size, storage_tier_hint FROM chunk_index_entries WHERE content_hash IN ("); + for (var i = 0; i < slots; i++) + { + if (i > 0) + sql.Append(", "); + sql.Append("$h").Append(i); + } - using var reader = command.ExecuteReader(); - while (reader.Read()) - { - var entry = ReadEntry(reader); - results[entry.ContentHash] = entry; + sql.Append(')'); + if (pendingFlushOnly) + sql.Append(" AND pending_flush = 1"); + + return sql.Append(';').ToString(); } } @@ -162,23 +175,6 @@ private static void ReadEntryPage(SqliteConnection connection, ContentHash[] pag /// private const int MaxLookupPageSize = 256; - private static string BuildFindEntriesSql(int slots, bool pendingFlushOnly) - { - var sql = new StringBuilder("SELECT content_hash, chunk_hash, original_size, chunk_size, storage_tier_hint FROM chunk_index_entries WHERE content_hash IN ("); - for (var i = 0; i < slots; i++) - { - if (i > 0) - sql.Append(", "); - sql.Append("$h").Append(i); - } - - sql.Append(')'); - if (pendingFlushOnly) - sql.Append(" AND pending_flush = 1"); - - return sql.Append(';').ToString(); - } - private ShardEntry? FindEntryCore(ContentHash contentHash, bool pendingFlushOnly) { try diff --git a/src/Arius.Core/Shared/FileSystem/PathSegment.cs b/src/Arius.Core/Shared/FileSystem/PathSegment.cs index c791f55dd..4a34833c7 100644 --- a/src/Arius.Core/Shared/FileSystem/PathSegment.cs +++ b/src/Arius.Core/Shared/FileSystem/PathSegment.cs @@ -46,6 +46,16 @@ public static bool TryParse(string? value, out PathSegment segment) segment = new PathSegment(value); return true; + + // Allocation-free replacement for value.Any(char.IsControl). + static bool ContainsControlCharacter(string value) + { + foreach (var c in value) + if (char.IsControl(c)) + return true; + + return false; + } } public bool Contains(string value, StringComparison comparisonType) => @@ -72,16 +82,4 @@ public PathSegment RemoveSuffix(string suffix, StringComparison comparisonType) } public override string ToString() => Value; - - /// - /// Allocation-free replacement for value.Any(char.IsControl) - /// - private static bool ContainsControlCharacter(string value) - { - foreach (var c in value) - if (char.IsControl(c)) - return true; - - return false; - } } From 60900080328b3c170bfe4003cb4442e574d731b1 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:44:38 +0200 Subject: [PATCH 41/46] refactor(cli): drop always-true tar state checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AccumulatingTar is set when a bundle starts and cleared when it seals, and a tracked tar only leaves Accumulating in the sealing handler, so whenever the reference is non-null its state is Accumulating. The State patterns in the entry-added and sealing handlers could never fail; keep only the null check. Verified: dotnet test src/Arius.Cli.Tests — 166 passed. Co-Authored-By: Claude Opus 5.5 --- src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs index dd356e0a0..3a315cf64 100644 --- a/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs +++ b/src/Arius.Cli/Commands/Archive/ArchiveProgressHandlers.cs @@ -109,8 +109,7 @@ public ValueTask Handle(TarEntryAddedEvent notification, CancellationToken cance { state.RemoveContentHash(notification.ContentHash); - var tar = state.AccumulatingTar; - if (tar is { State: TarState.Accumulating }) + if (state.AccumulatingTar is { } tar) { var addedBytes = notification.CurrentTarSize - tar.AccumulatedBytes; tar.AddEntry(addedBytes > 0 ? addedBytes : 0); @@ -129,8 +128,7 @@ public sealed class TarBundleSealingHandler(ProgressState state) : INotification { public ValueTask Handle(TarBundleSealingEvent notification, CancellationToken cancellationToken) { - var tar = state.AccumulatingTar; - if (tar is { State: TarState.Accumulating or TarState.Sealing }) + if (state.AccumulatingTar is { } tar) { tar.TarHash = notification.TarHash; // Use the sealed tar's archive byte size (headers + padding included), not the sum of file From ca816152a2ee9afb726169c78140c5e7800ee578 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:46:26 +0200 Subject: [PATCH 42/46] refactor(benchmarks): hand micro runs to BenchmarkSwitcher The --class/--filter options (a BenchmarkClass enum, two record fields, a class parser, a filter-vs-class check and help text) rebuilt what BenchmarkDotNet's own switcher already parses. `micro` as the first argument now hands the rest to BenchmarkSwitcher, so every BenchmarkDotNet option works, e.g. `micro --filter '*TarBuilder*'`. BenchmarkRunOptions is back to master's shape plus one help line. Micro runs also no longer write into the git-tracked raw/ folder; their output goes to BenchmarkDotNet.Artifacts, which is ignored. Verified: `micro --list flat` lists the AllocationBenchmarks cases, and `micro --filter '*HashCodec_ToLowerHex*' --job dry` runs one. Co-Authored-By: Claude Opus 5.5 --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 2 +- src/Arius.Benchmarks/BenchmarkRunOptions.cs | 42 ++------------------ src/Arius.Benchmarks/Program.cs | 33 +++++---------- 3 files changed, 14 insertions(+), 63 deletions(-) diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index 51766db89..b24506168 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -13,7 +13,7 @@ namespace Arius.Benchmarks; /// /// In-process allocation benchmarks for selected Arius.Core components. -/// Run with --class micro; no Azurite or Docker is required. +/// Run with micro (e.g. micro --filter '*TarBuilder*'); no Azurite or Docker is required. /// [MemoryDiagnoser] public class AllocationBenchmarks diff --git a/src/Arius.Benchmarks/BenchmarkRunOptions.cs b/src/Arius.Benchmarks/BenchmarkRunOptions.cs index 83ff3ef0d..c7cd98c47 100644 --- a/src/Arius.Benchmarks/BenchmarkRunOptions.cs +++ b/src/Arius.Benchmarks/BenchmarkRunOptions.cs @@ -1,21 +1,9 @@ namespace Arius.Benchmarks; -/// Which benchmark class to run. -internal enum BenchmarkClass -{ - /// The end-to-end archive step against Azurite. Appends to the benchmark tail log. - Archive, - - /// In-process allocation micro-benchmarks. No Docker; does not touch the tail log. - Micro, -} - internal sealed record BenchmarkRunOptions( string RepositoryRoot, string RawOutputRoot, - string TailLogPath, - BenchmarkClass Class, - string? Filter) + string TailLogPath) { public const int Iterations = 3; @@ -27,8 +15,6 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) var rawOutputRoot = defaultRawOutputRoot; var tailLogPath = Path.Combine(defaultBenchmarkRoot, "benchmark-tail.md"); - var benchmarkClass = BenchmarkClass.Archive; - string? filter = null; for (var i = 0; i < args.Count; i++) { @@ -40,12 +26,6 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) case "--tail-log": tailLogPath = RequireValue(args, ref i, "--tail-log"); break; - case "--class": - benchmarkClass = ParseClass(RequireValue(args, ref i, "--class")); - break; - case "--filter": - filter = RequireValue(args, ref i, "--filter"); - break; case "--help" or "-h": PrintHelp(); Environment.Exit(0); @@ -55,24 +35,12 @@ public static BenchmarkRunOptions Parse(IReadOnlyList args) } } - if (filter is not null && benchmarkClass is not BenchmarkClass.Micro) - throw new ArgumentException("--filter is only supported with --class micro."); - return new( repositoryRoot, Path.GetFullPath(rawOutputRoot), - Path.GetFullPath(tailLogPath), - benchmarkClass, - filter); + Path.GetFullPath(tailLogPath)); } - static BenchmarkClass ParseClass(string value) => value.ToLowerInvariant() switch - { - "archive" => BenchmarkClass.Archive, - "micro" => BenchmarkClass.Micro, - _ => throw new ArgumentException($"Unknown benchmark class '{value}'. Expected 'archive' or 'micro'."), - }; - static string RequireValue(IReadOnlyList args, ref int index, string optionName) { if (index + 1 >= args.Count) @@ -103,12 +71,10 @@ static string FindRepositoryRoot() static void PrintHelp() { Console.WriteLine("Runs the canonical representative workflow benchmark on Azurite."); + Console.WriteLine("Pass 'micro [BenchmarkDotNet options]' instead for the in-process allocation benchmarks."); Console.WriteLine(); Console.WriteLine("Options:"); - Console.WriteLine(" --class 'archive' (default, end-to-end on Azurite; needs Docker)"); - Console.WriteLine(" or 'micro' (in-process allocation benchmarks)."); - Console.WriteLine(" --filter Run only matching benchmarks, e.g. '*TarBuilder*' (micro only)."); Console.WriteLine(" --raw-output Folder where per-run raw BenchmarkDotNet output is saved."); - Console.WriteLine(" --tail-log Markdown benchmark tail log to append to (archive only)."); + Console.WriteLine(" --tail-log Markdown benchmark tail log to append to."); } } diff --git a/src/Arius.Benchmarks/Program.cs b/src/Arius.Benchmarks/Program.cs index c0adef7cf..0f9b70be5 100644 --- a/src/Arius.Benchmarks/Program.cs +++ b/src/Arius.Benchmarks/Program.cs @@ -1,11 +1,18 @@ using Arius.Benchmarks; using Arius.E2E.Tests.Datasets; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; +// In-process allocation micro-benchmarks run on BenchmarkDotNet's default job and take its own options, +// e.g. `micro --filter '*TarBuilder*'`. They need no Docker and do not append to the tail log. +if (args is ["micro", .. var microArgs]) +{ + BenchmarkSwitcher.FromTypes([typeof(AllocationBenchmarks)]).Run(microArgs); + return; +} + var options = BenchmarkRunOptions.Parse(args); var runStartedAt = DateTimeOffset.UtcNow; var runId = runStartedAt.ToString("yyyyMMddTHHmmss.fffZ"); @@ -13,28 +20,6 @@ Directory.CreateDirectory(rawOutputDirectory); Directory.CreateDirectory(Path.GetDirectoryName(options.TailLogPath)!); -var logger = new StreamLogger(Path.Combine(rawOutputDirectory, "benchmark-output.log"), append: false); - -if (options.Class is BenchmarkClass.Micro) -{ - // Micro-benchmarks need BenchmarkDotNet's normal warmup/invocation defaults to produce meaningful - // per-operation numbers, so they deliberately do NOT use the single-invocation job below. They also - // do not append to the tail log: its schema (RepresentativeScaleDivisor, ...) is archive-specific. - var microConfig = ManualConfig - .Create(DefaultConfig.Instance) - .AddLogger(logger) - .WithArtifactsPath(rawOutputDirectory); - - // --filter lets a single optimization be re-measured on its own benchmark rather than re-running - // the whole suite after every change. - if (options.Filter is { } filter) - microConfig = microConfig.AddFilter(new GlobFilter([filter])); - - BenchmarkRunner.Run(microConfig); - - return; -} - // The archive step runs for tens of seconds and mutates real fixtures per iteration, so it is measured // with one invocation per iteration and no warmup. var config = ManualConfig @@ -45,7 +30,7 @@ .WithIterationCount(BenchmarkRunOptions.Iterations) .WithInvocationCount(1) .WithUnrollFactor(1)) - .AddLogger(logger) + .AddLogger(new StreamLogger(Path.Combine(rawOutputDirectory, "benchmark-output.log"), append: false)) .WithArtifactsPath(rawOutputDirectory); var summary = BenchmarkRunner.Run(config); From e6d14b77426b539c5b710bc048decfb5eacfff44 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:47:20 +0200 Subject: [PATCH 43/46] docs: trim comments that retell the branch's history Comments added by this branch's fix commits repeated their commit messages: why the old stripe hash boxed, which callers used to hit the SQLite variable limit and how the error surfaced, what an earlier presize reserved. The history stays in git; the comments keep only what the code still needs said. The other flagged comments (TarBuilder presize, ProgressStream.Dispose, FileDedupedHandler, WriteDigest) were already rewritten or removed with their code in the preceding commits. Co-Authored-By: Claude Opus 5.5 --- src/Arius.Benchmarks/AllocationBenchmarks.cs | 5 +---- .../Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs | 5 +---- src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs | 7 ++----- src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs | 3 --- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/Arius.Benchmarks/AllocationBenchmarks.cs b/src/Arius.Benchmarks/AllocationBenchmarks.cs index b24506168..03dfaa78d 100644 --- a/src/Arius.Benchmarks/AllocationBenchmarks.cs +++ b/src/Arius.Benchmarks/AllocationBenchmarks.cs @@ -111,10 +111,7 @@ public async Task TarBuilder_Seal_64MB() return sealedCount; } - /// - /// A handful of small files: the shape of a tiny repository, and of every run's partial tail bundle. - /// Presizing unconditionally to TarTargetSize reserved the full target here for a few KB of payload. - /// + /// A handful of small files: the shape of a tiny repository or a small tail bundle. [Benchmark(Description = "TarBuilder seal one small bundle (5 x 1 KB)")] public async Task TarBuilder_Seal_SmallBundle() { diff --git a/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs b/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs index 19cfd12e0..4249943bb 100644 --- a/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs +++ b/src/Arius.Core.Tests/Shared/ChunkIndex/ChunkIndexLocalStoreTests.cs @@ -26,10 +26,7 @@ public void Initialize_CreatesSchemaVersionAndWalMode() [Test] public void FindEntries_WithMoreHashesThanSqliteVariableLimit_ReturnsMatches() { - // The store's own API must stay usable at any width: `ls` looks up one hash per file in a - // directory, so a wide directory produces a wide set. Binding one parameter per hash fails past - // SQLITE_MAX_VARIABLE_NUMBER (32766) with "too many SQL variables", which CreateLocalStoreException - // then reports as a corrupt cache the operator should delete. + // Wider than SQLITE_MAX_VARIABLE_NUMBER (32766), as a large directory listing can be. var repositoryKey = $"acct-local-store-wide-{Guid.NewGuid():N}"; var root = RepositoryLocalStatePaths.GetChunkIndexCacheRoot(repositoryKey, repositoryKey); var store = new ChunkIndexLocalStore(root); diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index e6b837037..f6218f476 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -107,11 +107,8 @@ public IReadOnlyDictionary FindPendingFlushEntries(IRea /// Looks up a set of hashes with a single IN (...) query per page instead of one query per hash. /// /// - /// Paged because the set is not bounded by the caller: the archive dedup stage batches 256, but - /// ListQueryHandler looks up one hash per file in a directory and ChunkHydrationStatusQuery one per - /// selected file, so a wide directory would otherwise exceed SQLite's SQLITE_MAX_VARIABLE_NUMBER - /// (32766). That surfaces through CreateLocalStoreException as "corrupt local store, delete the cache" - /// — an actively misleading diagnosis for a query that is simply too wide. + /// Paged because callers are unbounded (ls looks up one hash per file of a directory), and one + /// parameter per hash would otherwise exceed SQLite's variable limit. /// private IReadOnlyDictionary FindEntriesCore(IReadOnlyCollection contentHashes, bool pendingFlushOnly) { diff --git a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs index 67c2ee143..48468ceda 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs @@ -90,9 +90,6 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati private async Task AppendLineAsync(RelativePath path, string line, CancellationToken cancellationToken) { - // path.GetHashCode(), not StringComparer.Ordinal.GetHashCode(path): RelativePath is a struct, so - // the latter bound to IEqualityComparer.GetHashCode(object) — boxing on every append and then - // falling through to path.GetHashCode() anyway, so the comparer was doing nothing. var nodeLock = _lockStripes[(uint)path.GetHashCode() % (uint)_lockStripes.Length]; await nodeLock.WaitAsync(cancellationToken); From 5a8d0fd6214fa785e267028cc84cafe5ae4691ea Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:47:31 +0200 Subject: [PATCH 44/46] docs: fix stale descriptions of the batched lookup and streaming wrappers - FindEntriesCore carried two summary/remarks pairs; the first still described padding parameters to fixed buckets, which 529f1a16 removed. - memory-boundedness.md still said the streaming decorators hold "only a long counter"; ProgressStream now keeps a few for throttling (streaming.md was already updated). Co-Authored-By: Claude Opus 5.5 --- docs/design/cross-cutting/memory-boundedness.md | 2 +- src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/design/cross-cutting/memory-boundedness.md b/docs/design/cross-cutting/memory-boundedness.md index ae2965d08..5e2011983 100644 --- a/docs/design/cross-cutting/memory-boundedness.md +++ b/docs/design/cross-cutting/memory-boundedness.md @@ -46,7 +46,7 @@ Earlier the [chunk index](../../glossary.md#chunk-index) kept an in-memory LRU o ### 4. Streaming up/download — bound the single-file byte axis -A single multi-GB file must not be buffered. `ChunkStorageService.UploadChunkAsync` assembles a push chain (source → `ProgressStream` → zstd → encryption → `CountingStream` → `OpenWriteAsync`) with no `MemoryStream` and no intermediate temp file (the tar bundle being the one streamed-in exception). The `Streaming` decorators hold only a `long` counter, so the chain moves a file of any size with a working set of roughly one buffer. This is the byte-scale complement to the file-count techniques above — see [streaming](../core/shared/streaming.md). +A single multi-GB file must not be buffered. `ChunkStorageService.UploadChunkAsync` assembles a push chain (source → `ProgressStream` → zstd → encryption → `CountingStream` → `OpenWriteAsync`) with no `MemoryStream` and no intermediate temp file (the tar bundle being the one streamed-in exception). The `Streaming` decorators hold only a few counters, so the chain moves a file of any size with a working set of roughly one buffer. This is the byte-scale complement to the file-count techniques above — see [streaming](../core/shared/streaming.md). ## Key invariants diff --git a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs index f6218f476..cbed3509f 100644 --- a/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs +++ b/src/Arius.Core/Shared/ChunkIndex/ChunkIndexLocalStore.cs @@ -96,13 +96,6 @@ public IReadOnlyDictionary FindEntries(IReadOnlyCollect public IReadOnlyDictionary FindPendingFlushEntries(IReadOnlyCollection contentHashes) => FindEntriesCore(contentHashes, pendingFlushOnly: true); - /// - /// Looks up a set of hashes with one IN (...) query. - /// - /// - /// Parameters are padded to fixed buckets so the generated command text can be reused. Unused slots - /// repeat the first hash, which is harmless for set membership. - /// /// /// Looks up a set of hashes with a single IN (...) query per page instead of one query per hash. /// From b4e5a01171380b121bb12619d193e67dba48446f Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:48:08 +0200 Subject: [PATCH 45/46] docs(filetree): restore the staging writer's invariant comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docstring cleanup (6bb3aed3) and the later handle revert (ad9916d0) left FileTreeStagingWriter with shortened versions of master's comments. The shortened versions lost the reasons the code needs: a failed append must release its edge claim, or that parent→child edge is skipped for good and its subtree orphaned; the claim is what stops concurrent writers double-emitting; Segments is materialized once because each enumeration re-parses the path. Restore master's comments and field alignment. Also restore the AppendAllTextAsync summary the cleanup deleted from RelativeFileSystem. The file now differs from master only in the stripe hash fix. Verified: dotnet test src/Arius.Core.Tests — 670 passed, 1 skipped. Co-Authored-By: Claude Opus 5.5 --- .../Shared/FileSystem/RelativeFileSystem.cs | 3 +++ .../Shared/FileTree/FileTreeStagingWriter.cs | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs index e05dc85ee..ac13532ea 100644 --- a/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs +++ b/src/Arius.Core/Shared/FileSystem/RelativeFileSystem.cs @@ -249,6 +249,9 @@ public async Task WriteAllTextAsync(RelativePath path, string content, Cancellat await File.WriteAllTextAsync(fullPath, content, cancellationToken); } + /// + /// Appends text to a file within the rooted directory, creating the file (and parent directories) if needed. + /// public async Task AppendAllTextAsync(RelativePath path, string content, CancellationToken cancellationToken) { var fullPath = root.Resolve(path); diff --git a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs index 48468ceda..4f7a3dba5 100644 --- a/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs +++ b/src/Arius.Core/Shared/FileTree/FileTreeStagingWriter.cs @@ -7,9 +7,9 @@ internal sealed class FileTreeStagingWriter : IDisposable { private const int StripeCount = 256; // Note: we used to have a lock for every staging file, but that was unbounded. Now we have bounded memory by striping the locks - private readonly SemaphoreSlim[] _lockStripes; + private readonly SemaphoreSlim[] _lockStripes; private readonly RelativeFileSystem _stagingFileSystem; - private bool _disposed; + private bool _disposed; // A directory id is globally unique to its full path, so its parent→child edge line is identical // no matter which descendant file triggers it. Emit each edge once: without this, a deep tree @@ -60,6 +60,9 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati { var currentPath = RelativePath.Root; + // Materialize once: RelativePath.Segments re-splits and re-parses the path on every + // enumeration, so Take(Segments.Count() - 1) would parse the whole path twice on this hot + // staging path. Iterate the segments by index instead, skipping the trailing file segment. var segments = filePath.Segments.ToArray(); for (var i = 0; i < segments.Length - 1; i++) @@ -69,7 +72,8 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati currentPath = currentPath / segment; var directoryId = FileTreePaths.GetStagingDirectoryId(currentPath); - // Claim each edge once; failed writes release the claim below. + // Claim the edge (but keep descending) so concurrent writers don't double-emit it. + // TryAdd is atomic: exactly one writer wins the claim and writes the edge. if (!_emittedDirectories.TryAdd(directoryId, true)) continue; @@ -81,7 +85,9 @@ private async Task AppendDirectoryEntriesAsync(RelativePath filePath, Cancellati } catch { - // Allow a later writer to retry an edge whose append failed. + // The claim must reflect a committed edge: if the append fails (I/O error or + // cancellation), release it so a later writer can re-emit. Otherwise the parent→child + // edge is permanently skipped and its subtree orphaned. _emittedDirectories.TryRemove(directoryId, out _); throw; } From b85a350e6191ad04d5d893316993bc62482261b0 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 29 Sep 2026 13:50:43 +0200 Subject: [PATCH 46/46] perf(benchmarks): record the full-scale archive run at the branch head The 407.74 MB row from 59d7fa7d was measured at 10947e7b, which still included two optimizations the branch later reverted (0ae0798c, a9bdefa7) and predates the later tar-buffer, lookup and progress-state fixes, so it did not describe the code being merged. That row is now labelled as mid-branch. Re-ran ArchiveStepBenchmarks at RepresentativeScaleDivisor=1 on the same machine, with the constant reverted to 8 afterwards as usual: Allocated: 478.22 MB (base ee4e9f3e) -> 392.1 MB, -18.0% As before, only Allocated is a signal; Mean 3.445 s has Error +/-7.2 s at three iterations with no warmup. Co-Authored-By: Claude Opus 5.5 --- src/Arius.Benchmarks/benchmark-tail.md | 3 +- ....ArchiveStepBenchmarks-20260929-134941.log | 0 .../20260929T114940.996Z/benchmark-output.log | 159 ++++++++++++++++++ ...rks.ArchiveStepBenchmarks-report-github.md | 15 ++ ...enchmarks.ArchiveStepBenchmarks-report.csv | 2 + ...nchmarks.ArchiveStepBenchmarks-report.html | 32 ++++ 6 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 src/Arius.Benchmarks/raw/20260929T114940.996Z/Arius.Benchmarks.ArchiveStepBenchmarks-20260929-134941.log create mode 100644 src/Arius.Benchmarks/raw/20260929T114940.996Z/benchmark-output.log create mode 100644 src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md create mode 100644 src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv create mode 100644 src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html diff --git a/src/Arius.Benchmarks/benchmark-tail.md b/src/Arius.Benchmarks/benchmark-tail.md index aa901ab5e..f7631579a 100644 --- a/src/Arius.Benchmarks/benchmark-tail.md +++ b/src/Arius.Benchmarks/benchmark-tail.md @@ -12,4 +12,5 @@ | woutbook6 | 2026-05-01T15:23:58.2515390+00:00 | 1eedb04c4fe77ae90f7faa384c2c7b7dedbe0600 | 1 | 3 | 1.453 m | 0.1986 m | 0.0109 m | 434000.0000 | 54000.0000 | 4000.0000 | 3.52 GB | 506427.0000 | 15.0000 | src/Arius.Benchmarks/raw/20260501T152358.251Z | | woutbook6 | 2026-05-01T16:44:07.0942170+00:00 | 1eedb04c4fe77ae90f7faa384c2c7b7dedbe0600 | 1 | 3 | 33.16 s | 1.135 s | 0.062 s | 56000.0000 | 5000.0000 | | 444.88 MB | 39835.0000 | 2.0000 | src/Arius.Benchmarks/raw/20260501T164407.094Z | | woutbook6 | 2026-05-01T19:11:23.3076330+00:00 | 468d2479d64cef776b02c14f98ed9b667ecd2b46 | 8 | 3 | 7.352 s | 2.350 s | 0.1288 s | 14000.0000 | 3000.0000 | | 115.94 MB | 7893.0000 | 1.0000 | src/Arius.Benchmarks/raw/20260501T191123.307Z | -| woutbook6 | 2026-09-08T08:52:24.1656440+00:00 | 10947e7bbb38e1161669cdb9aebe90af17384759 | 1 | 3 | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 11000.0000 | 2000.0000 | 407.74 MB | 28623.0000 | 429.0000 | src/Arius.Benchmarks/raw/20260908T085224.165Z | # After the allocation-optimization branch; controlled base run on the same machine/session was 478.22 MB (ee4e9f3e). Time/GC counts on this host are too noisy to compare (3 iterations, no warmup, Error +/-31 s on the base). | +| woutbook6 | 2026-09-08T08:52:24.1656440+00:00 | 10947e7bbb38e1161669cdb9aebe90af17384759 | 1 | 3 | 3.808 s | 3.631 s | 0.1990 s | 33000.0000 | 11000.0000 | 2000.0000 | 407.74 MB | 28623.0000 | 429.0000 | src/Arius.Benchmarks/raw/20260908T085224.165Z | # Mid-branch (10947e7b): includes 0ae0798c and a9bdefa7, both reverted later, so it is not the merged code; superseded by the next row. Controlled base run on the same machine/session was 478.22 MB (ee4e9f3e). Time/GC counts on this host are too noisy to compare (3 iterations, no warmup, Error +/-31 s on the base). | +| woutbook6 | 2026-09-29T11:49:40.9965940+00:00 | b4e5a01171380b121bb12619d193e67dba48446f | 1 | 3 | 3.445 s | 7.194 s | 0.3943 s | 29000.0000 | 9000.0000 | 1000.0000 | 392.1 MB | 28159.0000 | 491.0000 | src/Arius.Benchmarks/raw/20260929T114940.996Z | # Branch head after the review fixes: 392.1 MB vs 478.22 MB at the base (ee4e9f3e), -18%. Allocated only; time/GC counts are noise at 3 iterations. | diff --git a/src/Arius.Benchmarks/raw/20260929T114940.996Z/Arius.Benchmarks.ArchiveStepBenchmarks-20260929-134941.log b/src/Arius.Benchmarks/raw/20260929T114940.996Z/Arius.Benchmarks.ArchiveStepBenchmarks-20260929-134941.log new file mode 100644 index 000000000..e69de29bb diff --git a/src/Arius.Benchmarks/raw/20260929T114940.996Z/benchmark-output.log b/src/Arius.Benchmarks/raw/20260929T114940.996Z/benchmark-output.log new file mode 100644 index 000000000..e9ed7c079 --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260929T114940.996Z/benchmark-output.log @@ -0,0 +1,159 @@ +// Validating benchmarks: +// ***** BenchmarkRunner: Start ***** +// ***** Found 1 benchmark(s) in total ***** +// ***** Building 1 exe(s) in Parallel: Start ***** +// start dotnet restore --nodeReuse:false /p:UseSharedCompilation=false /p:Deterministic=true /p:Optimize=true /p:ArtifactsPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/" /p:OutDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:OutputPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:PublishDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/publish/" in /Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1 +// command took 0.88 sec and exited with 0 +// start dotnet build -c Release --no-restore --nodeReuse:false /p:UseSharedCompilation=false /p:Deterministic=true /p:Optimize=true /p:ArtifactsPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/" /p:OutDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:OutputPath="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" /p:PublishDir="/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/publish/" --output "/Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0/" in /Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1 +// command took 10.44 sec and exited with 0 +// ***** Done, took 00:00:11 (11.45 sec) ***** +// Found 1 benchmarks: +// ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) + +// ************************** +// Benchmark: ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) +// *** Execute *** +// Launch: 1 / 1 +// Execute: dotnet Arius.Benchmarks-Job-HILDPN-1.dll --anonymousPipes 126 127 --benchmarkName Arius.Benchmarks.ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite --job "InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0" --benchmarkId 0 in /Users/wouter/.superset/worktrees/288a93b0-804e-460f-999d-8c3f427fff33/memory-opt/src/Arius.Benchmarks/bin/Release/net10.0/Arius.Benchmarks-Job-HILDPN-1/bin/Release/net10.0 +// Failed to set up high priority (Permission denied). In order to run benchmarks with high priority, make sure you have the right permissions. +// BeforeAnythingElse + +// Benchmark Process Environment Information: +// BenchmarkDotNet v0.15.8 +// Runtime=.NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a +// GC=Concurrent Workstation +// HardwareIntrinsics=ArmBase+AdvSimd,AES,CRC32,DP,RDM,SHA1,SHA256 VectorSize=128 +// Job: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) + +[testcontainers.org 00:00:00.07] Connected to Docker: + Host: unix:///var/run/docker.sock + Server Version: 29.3.1 + Kernel Version: 6.12.76-linuxkit + API Version: 1.54 + Operating System: Docker Desktop + Total Memory: 7.65 GB + Labels: + com.docker.desktop.address=unix:///Users/wouter/Library/Containers/com.docker.docker/Data/docker-cli.sock +[testcontainers.org 00:00:00.19] Docker container 02e615d7dc78 created +[testcontainers.org 00:00:00.21] Start Docker container 02e615d7dc78 +[testcontainers.org 00:00:00.30] Wait for Docker container 02e615d7dc78 to complete readiness checks +[testcontainers.org 00:00:01.33] Docker container 02e615d7dc78 ready +[testcontainers.org 00:00:01.40] Docker container 4574fa6d9a39 created +[testcontainers.org 00:00:01.41] Start Docker container 4574fa6d9a39 +[testcontainers.org 00:00:01.49] Wait for Docker container 4574fa6d9a39 to complete readiness checks +[testcontainers.org 00:00:02.54] Docker container 4574fa6d9a39 ready +OverheadJitting 1: 1 op, 54708.00 ns, 54.7080 us/op +WorkloadJitting 1: 1 op, 3172194458.00 ns, 3.1722 s/op + +OverheadWarmup 1: 1 op, 333.00 ns, 333.0000 ns/op +OverheadWarmup 2: 1 op, 42.00 ns, 42.0000 ns/op +OverheadWarmup 3: 1 op, 125.00 ns, 125.0000 ns/op +OverheadWarmup 4: 1 op, 83.00 ns, 83.0000 ns/op +OverheadWarmup 5: 1 op, 41.00 ns, 41.0000 ns/op +OverheadWarmup 6: 1 op, 42.00 ns, 42.0000 ns/op +OverheadWarmup 7: 1 op, 42.00 ns, 42.0000 ns/op + +OverheadActual 1: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 2: 1 op, 167.00 ns, 167.0000 ns/op +OverheadActual 3: 1 op, 41.00 ns, 41.0000 ns/op +OverheadActual 4: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 5: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 6: 1 op, 41.00 ns, 41.0000 ns/op +OverheadActual 7: 1 op, 42.00 ns, 42.0000 ns/op +OverheadActual 8: 1 op, 167.00 ns, 167.0000 ns/op +OverheadActual 9: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 10: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 11: 1 op, 84.00 ns, 84.0000 ns/op +OverheadActual 12: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 13: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 14: 1 op, 125.00 ns, 125.0000 ns/op +OverheadActual 15: 1 op, 84.00 ns, 84.0000 ns/op +OverheadActual 16: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 17: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 18: 1 op, 0.00 ns, 0.0000 ns/op +OverheadActual 19: 1 op, 83.00 ns, 83.0000 ns/op +OverheadActual 20: 1 op, 84.00 ns, 84.0000 ns/op + + +// BeforeActualRun +WorkloadActual 1: 1 op, 3713232958.00 ns, 3.7132 s/op +WorkloadActual 2: 1 op, 2992224084.00 ns, 2.9922 s/op +WorkloadActual 3: 1 op, 3629489209.00 ns, 3.6295 s/op + +// AfterActualRun +WorkloadResult 1: 1 op, 3713232875.00 ns, 3.7132 s/op +WorkloadResult 2: 1 op, 2992224001.00 ns, 2.9922 s/op +WorkloadResult 3: 1 op, 3629489126.00 ns, 3.6295 s/op +// GC: 29 9 1 411149920 1 +// Threading: 28159 491 1 +// Exceptions: 4 + +[testcontainers.org 00:00:22.89] Delete Docker container 4574fa6d9a39 +// AfterAll +// Benchmark Process 49638 has exited with code 0. + +Mean = 3.445 s, StdErr = 0.228 s (6.61%), N = 3, StdDev = 0.394 s +Min = 2.992 s, Q1 = 3.311 s, Median = 3.629 s, Q3 = 3.671 s, Max = 3.713 s +IQR = 0.361 s, LowerFence = 2.770 s, UpperFence = 4.212 s +ConfidenceInterval = [-3.749 s; 10.639 s] (CI 99.9%), Margin = 7.194 s (208.83% of Mean) +Skewness = -0.37, Kurtosis = 0.67, MValue = 2 + +// ** Remained 0 (0,0%) benchmark(s) to run. Estimated finish 2026-09-29 13:50 (0h 0m from now) ** +// ***** BenchmarkRunner: Finish ***** + +// * Export * + raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv + raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md + raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html + +// * Detailed results * +ArchiveStepBenchmarks.Archive_Step_V1_Representative_Azurite: Job-HILDPN(InvocationCount=1, IterationCount=3, LaunchCount=1, UnrollFactor=1, WarmupCount=0) +Runtime = .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a; GC = Concurrent Workstation +Mean = 3.445 s, StdErr = 0.228 s (6.61%), N = 3, StdDev = 0.394 s +Min = 2.992 s, Q1 = 3.311 s, Median = 3.629 s, Q3 = 3.671 s, Max = 3.713 s +IQR = 0.361 s, LowerFence = 2.770 s, UpperFence = 4.212 s +ConfidenceInterval = [-3.749 s; 10.639 s] (CI 99.9%), Margin = 7.194 s (208.83% of Mean) +Skewness = -0.37, Kurtosis = 0.67, MValue = 2 +-------------------- Histogram -------------------- +[2.633 s ; 3.313 s) | @ +[3.313 s ; 4.072 s) | @@ +--------------------------------------------------- + +// * Summary * + +BenchmarkDotNet v0.15.8, macOS Tahoe 26.6.2 (25G83) [Darwin 25.6.0] +Apple M4, 1 CPU, 10 logical and 10 physical cores +.NET SDK 10.0.201 + [Host] : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + Job-HILDPN : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + +InvocationCount=1 IterationCount=3 LaunchCount=1 +UnrollFactor=1 WarmupCount=0 + +| Method | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Gen2 | Allocated | +|--------------------------------------- |--------:|--------:|---------:|-----------:|---------------------:|-----------------:|----------:|----------:|----------:| +| Archive_Step_V1_Representative_Azurite | 3.445 s | 7.194 s | 0.3943 s | 29000.0000 | 28159.0000 | 491.0000 | 9000.0000 | 1000.0000 | 392.1 MB | + +// * Legends * + Mean : Arithmetic mean of all measurements + Error : Half of 99.9% confidence interval + StdDev : Standard deviation of all measurements + Gen0 : GC Generation 0 collects per 1000 operations + Completed Work Items : The number of work items that have been processed in ThreadPool (per single operation) + Lock Contentions : The number of times there was contention upon trying to take a Monitor's lock (per single operation) + Gen1 : GC Generation 1 collects per 1000 operations + Gen2 : GC Generation 2 collects per 1000 operations + Allocated : Allocated memory per single operation (managed only, inclusive, 1KB = 1024B) + 1 s : 1 Second (1 sec) + +// * Diagnostic Output - MemoryDiagnoser * + +// * Diagnostic Output - ThreadingDiagnoser * + + +// ***** BenchmarkRunner: End ***** +Run time: 00:00:23 (23.57 sec), executed benchmarks: 1 + +Global total time: 00:00:35 (35.21 sec), executed benchmarks: 1 +// * Artifacts cleanup * +Artifacts cleanup is finished diff --git a/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md b/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md new file mode 100644 index 000000000..d35a75319 --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report-github.md @@ -0,0 +1,15 @@ +``` + +BenchmarkDotNet v0.15.8, macOS Tahoe 26.6.2 (25G83) [Darwin 25.6.0] +Apple M4, 1 CPU, 10 logical and 10 physical cores +.NET SDK 10.0.201 + [Host] : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + Job-HILDPN : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a + +InvocationCount=1 IterationCount=3 LaunchCount=1 +UnrollFactor=1 WarmupCount=0 + +``` +| Method | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Gen2 | Allocated | +|--------------------------------------- |--------:|--------:|---------:|-----------:|---------------------:|-----------------:|----------:|----------:|----------:| +| Archive_Step_V1_Representative_Azurite | 3.445 s | 7.194 s | 0.3943 s | 29000.0000 | 28159.0000 | 491.0000 | 9000.0000 | 1000.0000 | 392.1 MB | diff --git a/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv b/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv new file mode 100644 index 000000000..6f408da36 --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.csv @@ -0,0 +1,2 @@ +Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,Gen0,Completed Work Items,Lock Contentions,Gen1,Gen2,Allocated +Archive_Step_V1_Representative_Azurite,Job-HILDPN,False,Default,Default,Default,Default,Default,Default,0000000000,Empty,RyuJit,Default,Arm64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 10.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,1,3,Default,1,Default,Default,Default,Default,Default,Default,1,0,3.445 s,7.194 s,0.3943 s,29000.0000,28159.0000,491.0000,9000.0000,1000.0000,392.1 MB diff --git a/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html b/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html new file mode 100644 index 000000000..a925c1e2d --- /dev/null +++ b/src/Arius.Benchmarks/raw/20260929T114940.996Z/results/Arius.Benchmarks.ArchiveStepBenchmarks-report.html @@ -0,0 +1,32 @@ + + + + +Arius.Benchmarks.ArchiveStepBenchmarks-20260929-134952 + + + + +

+BenchmarkDotNet v0.15.8, macOS Tahoe 26.6.2 (25G83) [Darwin 25.6.0]
+Apple M4, 1 CPU, 10 logical and 10 physical cores
+.NET SDK 10.0.201
+  [Host]     : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a
+  Job-HILDPN : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a
+
+
InvocationCount=1  IterationCount=3  LaunchCount=1  
+UnrollFactor=1  WarmupCount=0  
+
+ + + + + +
Method MeanErrorStdDevGen0Completed Work ItemsLock ContentionsGen1Gen2Allocated
Archive_Step_V1_Representative_Azurite3.445 s7.194 s0.3943 s29000.000028159.0000491.00009000.00001000.0000392.1 MB
+ +