From 27a2af3efe010d06fbd7df2695f6056f7d7ff563 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 3 Oct 2026 23:55:55 -0700 Subject: [PATCH 1/2] perf(read): defer proven blob leaves during pack planning --- crab/src/git/remote_helper.rs | 132 ++++++++++++++++++++++++++++ crates/crab-read/README.md | 8 ++ crates/crab-read/src/upload_pack.rs | 57 ++++++++++-- 3 files changed, 189 insertions(+), 8 deletions(-) diff --git a/crab/src/git/remote_helper.rs b/crab/src/git/remote_helper.rs index aeb56c891..04f710e11 100644 --- a/crab/src/git/remote_helper.rs +++ b/crab/src/git/remote_helper.rs @@ -5324,6 +5324,138 @@ mod tests { (source, store, router, commits, tag) } + #[tokio::test] + async fn capsule_shallow_planning_does_not_inflate_proven_blob_leaves() { + let (source, store, router, _, _) = capsule_history_fixture().await; + let content = vec![b'x'; 2 * 1024 * 1024]; + std::fs::write(source.path().join("large.txt"), &content).expect("write large blob"); + run_git(source.path(), &["add", "large.txt"]); + run_git(source.path(), &["commit", "-q", "-m", "large blob"]); + let tip = gix_hash::ObjectId::from_hex( + run_git(source.path(), &["rev-parse", "HEAD"]).trim_ascii(), + ) + .expect("tip oid"); + let blob = gix_hash::ObjectId::from_hex( + run_git(source.path(), &["rev-parse", "HEAD:large.txt"]).trim_ascii(), + ) + .expect("blob oid"); + let cancellation = tokio_util::sync::CancellationToken::new(); + { + let git_dir = source.path().join(".git"); + let _guard = GitEnvCwdGuard::set(source.path(), &git_dir, source.path()); + let (result, _) = crate::git::capsule_push::run( + &PushConfig { + git_dir: Some(git_dir), + ..Default::default() + }, + &[PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &router, + None, + &[], + None, + None, + None, + &cancellation, + ) + .await + .expect("publish large blob"); + assert!(result.all_ok()); + } + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let view = crab_read::capsule_protocol::open_view( + &layout, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 32 * 1024 * 1024, + max_frontier_bytes: 64 * 1024 * 1024, + }, + ) + .await + .expect("open capsule view"); + let options = crab_remote_git::RepositoryOptions::new( + crab_remote_git::ObjectLimits::default(), + crab_remote_git::OperationLimits { + max_inflated_bytes: 64 * 1024, + ..Default::default() + }, + ) + .expect("bounded options"); + let repository = view + .git_repository_from_store( + layout, + crab_remote_git::RepositoryIdentity::new("memory", router.repo_prefix(), 1) + .expect("repository identity"), + Arc::new(crab_remote_git::RemoteGitRuntime::default()), + options, + 64 * 1024 * 1024, + &cancellation, + ) + .await + .expect("open bounded capsule reader"); + let visibility = view.git_visibility_index().expect("visibility proof"); + let refs = ["refs/heads/main".to_owned()]; + let plan = crab_read::plan_upload_pack( + &repository, + &visibility, + &refs, + &crab_read::UploadPackRequest { + wants: vec![tip], + deepen: Some(1), + ..Default::default() + }, + &cancellation, + ) + .await + .expect("shallow planning must fit without reconstructing blob leaves"); + assert!(plan.object_ids.contains(&blob)); + assert_eq!(plan.shallow, vec![tip]); + let pack = repository + .generate_pack(&plan.object_ids, &cancellation) + .await + .expect("generate bounded compressed response"); + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let pack_dir = target.path().join(".git/objects/pack"); + std::fs::copy(pack.path(), pack_dir.join("pack-response.pack")) + .expect("retain response pack"); + run_git( + target.path(), + &["index-pack", ".git/objects/pack/pack-response.pack"], + ); + assert_eq!( + run_git(target.path(), &["cat-file", "blob", &blob.to_string()]), + content + ); + let error = crab_read::plan_upload_pack( + &repository, + &visibility, + &refs, + &crab_read::UploadPackRequest { + wants: vec![blob], + deepen: Some(1), + ..Default::default() + }, + &cancellation, + ) + .await + .expect_err("explicit blob roots must still enforce the inflation budget"); + assert!(matches!( + error, + crab_read::ReadError::RemoteGit(crab_remote_git::Error::LimitExceeded { + limit: "inflated bytes", + .. + }) + )); + } + #[tokio::test] async fn classic_capsule_fetch_releases_one_reader_slot_on_success_and_failure() { for rejected in [false, true] { diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 3fd77eb2a..da23dfa5a 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -229,6 +229,14 @@ index bytes. Larger frontiers retain bounded suffix and range reads; stable checkpoint pack bodies stay cold in either case. This is a read strategy, not a new format or an authorization shortcut. +Upload-pack graph selection defers content reads for non-root blob leaves only +after visibility admission and pinned locator proof of their resolved kind. +Size-independent filters already admit these leaves without inspecting bytes; +they still consume the operation's logical-object budget. Explicit blob roots, +missing kind proof, and size-dependent filters retain bounded reads. Deferral +changes selection work only: packed-entry verification, response limits, and +native Git installation checks remain mandatory. + `CapsuleRepositoryView::git_snapshot` captures the same canonical pack inventory and Git identity used by both capsule Git readers. It performs no storage I/O and does not publish a v1 manifest. Its synthetic ETag covers the root and every diff --git a/crates/crab-read/src/upload_pack.rs b/crates/crab-read/src/upload_pack.rs index 1f579f8de..5fb59eeb3 100644 --- a/crates/crab-read/src/upload_pack.rs +++ b/crates/crab-read/src/upload_pack.rs @@ -7,9 +7,9 @@ use bstr::ByteSlice; use crab_metadata::git_object_locator::{GitObjectMetadata, GitObjectOrdinal}; use crab_metadata::git_visibility::GitVisibilityIndex; use crab_remote_git::{ - CorruptionStage, Error as RemoteGitError, GitCatalogVisibilityIndex, ObjectLimits, - OperationContext, OperationKind, OperationLimits, RemoteGitObject, RemoteGitRepository, - RepositoryOptions, RepositoryRef, RepositoryStateError, Revision, + BudgetDimension, CorruptionStage, Error as RemoteGitError, GitCatalogVisibilityIndex, + ObjectLimits, OperationContext, OperationKind, OperationLimits, RemoteGitObject, + RemoteGitRepository, RepositoryOptions, RepositoryRef, RepositoryStateError, Revision, }; use gix_hash::ObjectId; use tokio_util::sync::CancellationToken; @@ -1158,23 +1158,65 @@ async fn plan_with_operation( continue; } - let batch_oids = + let mut batch_oids = admit_batch_source(operation, visibility, visible_ref_names, &batch).await?; let needs_blob_metadata = filter_requires_blob_size(&request.filter); + let mut deferred_blobs = HashSet::new(); + if !needs_blob_metadata { + let known_blobs = batch + .iter() + .filter(|item| { + item.known_kind == Some(gix_object::Kind::Blob) && !roots.contains(&item.oid) + }) + .map(|item| item.oid) + .collect::>(); + let metadata = operation.pinned_object_metadata(&known_blobs).await?; + if metadata.len() != known_blobs.len() { + return Err(RemoteGitError::InternalInvariant { + invariant: "pinned blob metadata changed request cardinality", + }); + } + // Authorized, proven blobs are leaves; size-independent filters already admitted + // them above. Reconstructing their bodies here duplicates the response producer's + // work. Missing proof and explicit roots retain bounded content reads. + deferred_blobs.extend(known_blobs.into_iter().zip(metadata).filter_map( + |(oid, metadata)| { + (metadata.kind.map(gix_kind) == Some(gix_object::Kind::Blob)).then_some(oid) + }, + )); + let deferred_count = batch + .iter() + .filter(|item| deferred_blobs.contains(&item.oid)) + .count(); + operation + .charge(BudgetDimension::LogicalObjects, deferred_count as u64) + .await?; + batch_oids.retain(|oid| !deferred_blobs.contains(oid)); + } let batch_objects = if needs_blob_metadata { Vec::new() } else { let objects = operation.read_objects(&batch_oids).await?; - if objects.len() != batch.len() { + if objects.len() != batch_oids.len() { return Err(RemoteGitError::InternalInvariant { invariant: "batched upload-pack reads changed request order or cardinality", }); } objects }; + let mut batch_objects = batch_objects.into_iter(); - for (index, item) in batch.into_iter().enumerate() { + for item in batch { + if deferred_blobs.contains(&item.oid) { + if !excluded_commits.contains(&item.oid) + && !common_haves.contains(&item.oid) + && selected.insert(item.oid) + { + object_ids.push(item.oid); + } + continue; + } let object = if needs_blob_metadata && item.known_kind == Some(gix_object::Kind::Blob) && !roots.contains(&item.oid) @@ -1194,8 +1236,7 @@ async fn plan_with_operation( operation.read_object(item.oid).await? } else { batch_objects - .get(index) - .cloned() + .next() .ok_or(RemoteGitError::InternalInvariant { invariant: "batched upload-pack read is missing an object", })? From 06920cdce38ca20e1e0c999e822496db2206559d Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 00:05:22 -0700 Subject: [PATCH 2/2] fix(container): refresh unavailable Bookworm PCRE2 pin --- crates/crab-http-server/deploy/Dockerfile | 2 +- crates/crab-http-server/examples/compose_kv_service/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/crab-http-server/deploy/Dockerfile b/crates/crab-http-server/deploy/Dockerfile index 2ac57affa..dffd024d8 100644 --- a/crates/crab-http-server/deploy/Dockerfile +++ b/crates/crab-http-server/deploy/Dockerfile @@ -24,7 +24,7 @@ FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe6 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ git \ - libpcre2-8-0=10.42-1+deb12u1 \ + libpcre2-8-0=10.42-1+deb12u2 \ && rm -rf /var/lib/apt/lists/* \ && printf 'crab:x:10001:\n' >> /etc/group \ && printf 'crab:x:10001:10001:Crab server:/var/lib/crab:/bin/false\n' >> /etc/passwd \ diff --git a/crates/crab-http-server/examples/compose_kv_service/Dockerfile b/crates/crab-http-server/examples/compose_kv_service/Dockerfile index 9aac03af1..58f9e836a 100644 --- a/crates/crab-http-server/examples/compose_kv_service/Dockerfile +++ b/crates/crab-http-server/examples/compose_kv_service/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 RUN apt-get update \ - && apt-get install --yes --no-install-recommends ca-certificates curl libpcre2-8-0=10.42-1+deb12u1 \ + && apt-get install --yes --no-install-recommends ca-certificates curl libpcre2-8-0=10.42-1+deb12u2 \ && rm -rf /var/lib/apt/lists/* \ && printf 'crab:x:10001:\n' >> /etc/group \ && printf 'crab:x:10001:10001:Crab Cell example:/var/lib/crab:/bin/false\n' >> /etc/passwd \