From 3ba149d937c944d20efd9e9091f3098825084a86 Mon Sep 17 00:00:00 2001 From: ztzhou Date: Mon, 16 Nov 2026 00:48:44 +0800 Subject: [PATCH] feat(git): serve cached refs during background refresh Opt in with --max-stale-seconds: serve cached refs within the age bound and refresh in the background, so clients do not wait on a TTL-expired upstream fetch. Age is persisted in proxy-refresh.json so a restart does not reset it. Refs: rolandjitsu/git-cache-proxy#66 Assisted-by: Codex:gpt-6.1-terra Assisted-by: Claude:claude-opus-4-8 --- README.md | 5 +- chart/README.md | 4 +- chart/templates/deployment.yaml | 4 + chart/values.yaml | 8 +- docs/architecture.md | 25 + docs/configuration.md | 29 +- docs/security.md | 4 + src/config.rs | 38 +- src/git.rs | 2081 ++++++++++++++++++++++++++++++- src/main.rs | 10 +- src/server.rs | 79 +- tests/e2e.rs | 127 +- tests/helpers.rs | 30 + tests/http.rs | 205 +++ tests/lfs.rs | 53 + 15 files changed, 2615 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 002471d..cf4d332 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ Every flag has an environment-variable equivalent; the full reference is in to upstream. - `--serve-token` / `--auth-provider`: how clients authenticate to the proxy. - `--cache-max-mb`: cap on-disk cache size (LRU eviction; `0` = unlimited). +- `--max-stale-seconds`: opt in to serving cached refs while a background fetch + refreshes them (see [Stale serving](./docs/configuration.md#stale-serving)). ## Metrics @@ -129,8 +131,7 @@ Pin a version and try it against your own setup before relying on it. Not yet implemented: -- A background/scheduled refresh (today every `info/refs` triggers an on-demand, - TTL-coalesced fetch). +- Scheduled refresh. Background refresh is triggered by requests. **Non-goal: a git-free image.** The `git` binary is a permanent runtime dependency: the proxy serves clients by running `git upload-pack`, and no diff --git a/chart/README.md b/chart/README.md index 2347051..5209559 100644 --- a/chart/README.md +++ b/chart/README.md @@ -114,8 +114,10 @@ pointing at the Service. | `clientAuth.signingKey.existingSecret` | `""` | Secret with the HMAC key signing LFS download grants; empty = a random key per pod | | `clientAuth.signingKey.key` | `signing-key` | Key in that Secret | | `config.fetchTtlSeconds` | `10` | Skip upstream fetch if refreshed within this window (`0` = always) | +| `config.maxStaleSeconds` | `0` | Maximum age of cached refs served while refreshing (`0` = disabled) | +| `config.maxBackgroundFetches` | `4` | Maximum concurrent background refreshes (at least `1`) | | `config.cacheMaxMb` | `0` | Cap on on-disk cache, MiB; evicts LRU idle mirrors (`0` = unlimited) | -| `config.maxConcurrentRequests` | `64` | Max concurrent in-flight requests (`0` = unlimited) | +| `config.maxConcurrentRequests` | `64` | Request capacity; stale mode isolates local/upstream work and rejects overload (see [Stale serving](../docs/configuration.md#stale-serving)) | | `config.maxDecodedBodyMb` | `512` | Cap on a decoded upload-pack request body, MiB | | `config.logLevel` | `info` | Log filter directive | | `config.logFormat` | `text` | `text` or `json` | diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 389d1f3..4830397 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -65,6 +65,10 @@ spec: value: /var/cache/git-cache-proxy - name: GITCACHEPROXY_FETCH_TTL_SECONDS value: {{ .Values.config.fetchTtlSeconds | quote }} + - name: GITCACHEPROXY_MAX_STALE_SECONDS + value: {{ .Values.config.maxStaleSeconds | quote }} + - name: GITCACHEPROXY_MAX_BACKGROUND_FETCHES + value: {{ .Values.config.maxBackgroundFetches | quote }} - name: GITCACHEPROXY_CACHE_MAX_MB value: {{ .Values.config.cacheMaxMb | quote }} - name: GITCACHEPROXY_MAX_CONCURRENT_REQUESTS diff --git a/chart/values.yaml b/chart/values.yaml index e741396..6f76a2b 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -81,12 +81,16 @@ config: # Skip the upstream fetch on info/refs if the mirror was refreshed within this # many seconds; coalesces bursts of clones for one repo. 0 = always fetch. fetchTtlSeconds: 10 + # Serve cached refs for at most this age while refreshing. 0 disables it. + maxStaleSeconds: 0 + # Maximum simultaneous background refreshes, at least 1. + maxBackgroundFetches: 4 # Cap on total on-disk mirror cache, in MiB; evicts least-recently-used idle # mirrors when exceeded. 0 = unlimited (no eviction). Set this whenever the # cache volume is bounded. cacheMaxMb: 0 - # Max concurrent in-flight requests; excess queue. Bounds the upstream - # clone/fetch a burst can trigger. 0 = unlimited. + # Request capacity: strict mode queues; stale mode isolates local and + # upstream work and rejects overload. See docs/configuration.md (Stale serving) for zero-limit behavior. maxConcurrentRequests: 64 # Cap on a decoded upload-pack request body, in MiB (bounds memory, defuses a # gzip bomb). Caps only the negotiation request, never the streamed packfile. diff --git a/docs/architecture.md b/docs/architecture.md index 322d30b..10164b7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -114,6 +114,31 @@ linked. Cached objects share the `--cache-max-mb` budget and LRU eviction with t git mirrors. No configuration is needed: LFS is served on the same endpoints the git client already routes through the proxy. +### Stale serving + +With `--max-stale-seconds` set, the TTL-expired path changes from "fetch, then +serve" to "serve, and fetch in the background" (see +[Stale serving](./configuration.md#stale-serving) for the knobs). + +- A request checks the mirror's age. Within the bound it serves from the mirror + and, if the TTL has passed, starts one coalesced background fetch. Past the + bound it waits for a clone or fetch, which must finish within the bound. +- Age is the start of the last successful full-ref refresh, persisted in + `proxy-refresh.json` so a restart does not reset it. The record is trusted only + when it matches the repository, configured upstream and directory identity; + directory mtime is not evidence of success. A failed refresh or a SHA-only + backfill never renews age. +- Body decoding, freshness inspection, mandatory upstream work and local delivery + admit through separate lanes, so slow upstream work cannot take every local + slot. A full lane returns 503 rather than queueing another expensive owner. +- An upload that waited for a slot rechecks identity and age right before Git + starts. A stream already started may finish after the refs expire. +- Fetch-triggered automatic maintenance is disabled so a refresh does not delete + objects just advertised. Active requests are protected from eviction until their + Git process exits, but nothing pins an object between an advertisement and the + following upload-pack, so eviction or external maintenance in that gap can + remove it. A missing SHA is fetched on demand when upstream allows it. + ### Cache management The cache is self-managing. Mirrors are created on first request, and with diff --git a/docs/configuration.md b/docs/configuration.md index e730c3a..95516d0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,7 +20,9 @@ Every flag has an environment-variable equivalent. | `--auth-max-concurrent` | `GITCACHEPROXY_AUTH_MAX_CONCURRENT` | `32` | Max token-validation requests in flight; a check waits up to `--auth-timeout-seconds` for a slot, then fails with 503 (`0` = unlimited) | | `--auth-allow-http` | `GITCACHEPROXY_AUTH_ALLOW_HTTP` | `false` | Provider modes: accept a plain `http://` validation endpoint, which sends client credentials in cleartext | | `--fetch-ttl-seconds` | `GITCACHEPROXY_FETCH_TTL_SECONDS` | `10` | Skip upstream fetch if the last one started within this window (`0` = always fetch) | -| `--max-concurrent-requests` | `GITCACHEPROXY_MAX_CONCURRENT_REQUESTS` | `64` | Max concurrent in-flight requests; excess queue (`0` = unlimited) | +| `--max-stale-seconds` | `GITCACHEPROXY_MAX_STALE_SECONDS` | `0` | Maximum age of cached refs served during background refresh (`0` = disabled) | +| `--max-background-fetches` | `GITCACHEPROXY_MAX_BACKGROUND_FETCHES` | `4` | Maximum concurrent background refreshes (at least `1`) | +| `--max-concurrent-requests` | `GITCACHEPROXY_MAX_CONCURRENT_REQUESTS` | `64` | Request capacity: strict mode queues; stale mode isolates local and upstream work and rejects overload (see [Stale serving](#stale-serving) for `0`) | | `--max-concurrent-upload-packs` | `GITCACHEPROXY_MAX_CONCURRENT_UPLOAD_PACKS` | `0` | Max concurrent `git upload-pack` processes, held until each exits; bounds `pack-objects` memory. Clones waiting for a slot hold a `--max-concurrent-requests` slot (`0` = unlimited) | | `--max-decoded-body-mb` | `GITCACHEPROXY_MAX_DECODED_BODY_MB` | `512` | Cap on a decoded upload-pack request body, in MiB (bounds memory / gzip bombs) | | `--cache-max-mb` | `GITCACHEPROXY_CACHE_MAX_MB` | `0` | Cap on total on-disk mirror cache, in MiB; evicts least-recently-used idle mirrors when exceeded (`0` = unlimited, no eviction) | @@ -43,3 +45,28 @@ their recorded `origin` URL is not rewritten. Normal fetch TTL rules still apply Old cached objects and local want-by-SHA pins remain and can still be served. Changing the fetch URL is not a security-domain migration. When switching trust domains, use a fresh or separate cache root, including for LFS objects. + +## Stale serving + +By default a request after the fetch TTL waits for an upstream fetch. With a +positive `--max-stale-seconds`, an existing mirror serves its cached refs +instead, while at most one background fetch per repository updates it. A cold +mirror still needs a complete clone first. + +- **Age** is measured from the start of the last successful clone or fetch. Past + `--max-stale-seconds` a request waits for a refresh, or fails if that fails. +- **Restarts** keep the age: a successful full-ref refresh records its start time + in `proxy-refresh.json` inside the mirror. A missing, invalid, oversized or + future-dated record, or one that does not match the repository, upstream and + mirror directory, forces a refresh. This assumes a correct wall clock across + downtime and one proxy process per cache root. +- **Retries**: background attempts are limited globally by + `--max-background-fetches` and retried no sooner than the fetch TTL, or one + second, after the previous attempt completes. +- **Overload**: `--max-concurrent-requests` bounds body decoding, freshness + inspection and local serving in separate lanes. Mandatory upstream work has at + most `max(1, N - 1)` slots for limit `N`; with `N = 0` only that lane stays + bounded, by `--max-background-fetches`. A full lane returns 503 instead of + queueing. The strict default keeps its queueing behavior. + +See [Architecture](./architecture.md#stale-serving) for how a request is handled. diff --git a/docs/security.md b/docs/security.md index f4b1a4d..8f660a3 100644 --- a/docs/security.md +++ b/docs/security.md @@ -187,6 +187,10 @@ explicit before you expose it: Each junk credential still costs one call, and failed logins count against the proxy's egress IP at the git host: rate-limit unauthenticated clients at the ingress. +- **Stale serving exposes unreachable objects.** With `--max-stale-seconds` set, + a client that can read a mirror can request any object already in it by SHA, + including one no longer reachable from an advertised ref (for example after a + force-push or branch deletion upstream), until it is garbage collected or the mirror is evicted. - **Read-only.** Only `git-upload-pack` (clone/fetch) is served; `git-receive-pack` (push) is refused and upstream is only ever pulled from, never written. diff --git a/src/config.rs b/src/config.rs index f104922..75b13c8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -215,12 +215,19 @@ pub struct Config { #[arg(long, env = "GITCACHEPROXY_FETCH_TTL_SECONDS", default_value_t = 10)] pub fetch_ttl_seconds: u64, - /// Maximum number of requests handled concurrently, shared across all - /// connections. Excess requests queue until a slot frees. This bounds the - /// concurrent upstream clone/fetch work a burst can trigger (the slot is held - /// for the handler, then released before the packfile streams, so it caps - /// setup rather than in-flight streaming; see `max_concurrent_upload_packs`). - /// `0` = unlimited. Complementary to any ingress-level rate limiting. + /// Maximum age of cached refs served while refreshing; 0 disables this mode. + #[arg(long, env = "GITCACHEPROXY_MAX_STALE_SECONDS", default_value_t = 0)] + pub max_stale_seconds: u64, + + /// Maximum number of concurrent background refreshes. + #[arg(long, env = "GITCACHEPROXY_MAX_BACKGROUND_FETCHES", default_value_t = 4, value_parser = positive_usize)] + pub max_background_fetches: usize, + + /// Request capacity shared across connections. Strict mode queues excess + /// handlers. Stale mode bounds decoding, inspection and local serving + /// independently and rejects overload; mandatory upstream work has at most + /// max(1, capacity - 1) slots. Zero disables local limits; stale upstream + /// preparation still uses max_background_fetches as its bound. #[arg( long, env = "GITCACHEPROXY_MAX_CONCURRENT_REQUESTS", @@ -231,7 +238,8 @@ pub struct Config { /// Maximum number of `git upload-pack` processes at once, to bound /// `pack-objects` memory on large repos. A slot frees when the process exits, /// so it covers the streamed pack. Excess clones queue while holding a - /// `max_concurrent_requests` slot, so a full cap can also block `info/refs`. + /// `max_concurrent_requests` slot. In stale mode it stays held until Git is + /// reaped; a full cap can also return overload for `info/refs`. /// `0` = unlimited. #[arg( long, @@ -302,6 +310,14 @@ pub struct Config { pub log_format: LogFormat, } +fn positive_usize(value: &str) -> Result { + let value: usize = value.parse().map_err(|_| "expected a positive integer")?; + if value == 0 || value > tokio::sync::Semaphore::MAX_PERMITS { + return Err("value is outside the semaphore range".into()); + } + Ok(value) +} + #[cfg(test)] mod tests { use super::*; @@ -350,4 +366,12 @@ mod tests { .to_string(); assert!(err.contains("--upstream-auth-header"), "{err}"); } + + #[test] + fn background_fetch_limit_must_be_positive_and_fit_a_semaphore() { + assert_eq!(positive_usize("1").unwrap(), 1); + assert!(positive_usize("0").is_err()); + assert!(positive_usize("invalid").is_err()); + assert!(positive_usize(&usize::MAX.to_string()).is_err()); + } } diff --git a/src/git.rs b/src/git.rs index c9a1c8c..e66c498 100644 --- a/src/git.rs +++ b/src/git.rs @@ -12,8 +12,8 @@ //! //! It is strictly read-only: only `git-upload-pack` (clone/fetch) is served, and //! upstream is only ever *pulled* from - nothing is pushed or replicated -//! proactively. A miss transparently pulls from upstream, so the cache is never -//! stale for the ref the client actually asked for. +//! proactively. A miss transparently pulls from upstream; optional stale serving +//! can return older refs while a refresh runs. //! //! Want-by-SHA: a client can `want` a commit that lives only under an unadvertised //! upstream ref (e.g. a GitHub PR merge commit), which `clone --mirror` never @@ -24,15 +24,16 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use std::pin::Pin; use std::process::Stdio; -use std::sync::Arc; +use std::sync::{Arc, Mutex as StdMutex}; use std::task::Poll; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; use bytes::Bytes; +use sha2::{Digest, Sha256}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt, ReadBuf}; use tokio::process::{Child, ChildStdout, Command}; -use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore, oneshot}; +use tokio::sync::{Mutex, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore, oneshot}; use tokio::time::Instant; use tokio_util::io::ReaderStream; @@ -49,6 +50,20 @@ const UPSTREAM_RETRY_DELAYS: [Duration; 3] = [ ]; const MAX_UPSTREAM_RETRIES: usize = UPSTREAM_RETRY_DELAYS.len(); const MAX_UPSTREAM_STDERR_BYTES: usize = 64 * 1024; +const FRESHNESS_FILE: &str = "proxy-refresh.json"; +const MAX_FRESHNESS_BYTES: u64 = 4096; + +/// A bounded stale-serving admission lane has no capacity. +#[derive(Debug)] +pub struct CacheBusy; + +impl std::fmt::Display for CacheBusy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("cache request capacity exhausted") + } +} + +impl std::error::Error for CacheBusy {} /// What `ensure_fresh` did - for metrics. #[derive(Debug, Clone, Copy)] @@ -104,16 +119,29 @@ pub struct GitConfig { pub pack_window_memory: Option, } -/// Per-repo serialization point. `fetch_lock` is a `Mutex`, not an `RwLock`, on -/// purpose: it does double duty as (1) the mutual-exclusion point that collapses a -/// burst of concurrent clones for one repo into a single upstream pull, and (2) the -/// guard for the start `Instant` of the last upstream pull. Both uses need *exclusive* access -/// while a fetch runs - there is no read-only fast path to share - so an `RwLock` -/// would only add overhead and never grant a concurrent reader. +/// Serialize mutations independently of readers and protect the mirror from eviction. struct RepoSlot { - fetch_lock: Mutex>, + fetch_lock: Arc>>, + lifecycle: Arc>, + refresh: Mutex, +} + +#[derive(Default)] +struct RefreshState { + last_success: Option, + next_attempt: Option, + in_flight: bool, + record: Option, + identity: Option, +} + +/// Holds a request's mirror guard and setup permit. +pub struct RequestLease { + guard: OwnedRwLockReadGuard<()>, + permit: Option, } +#[derive(Clone)] pub struct GitCache { cfg: GitConfig, metrics: Arc, @@ -121,13 +149,20 @@ pub struct GitCache { /// because every access is a get-or-insert (`entry`), which needs a write lock /// anyway, and the critical section is a single O(1) map operation - far too /// short for reader/writer separation to pay off. - slots: Mutex>>, + slots: Arc>>>, /// Cache-size index for LRU eviction, or `None` when no cap is configured. /// When present it is `touch`ed on every request and `record`ed after each /// clone/fetch; when absent the eviction machinery is entirely inert. index: Option>, /// `None` when `max_upload_packs` is `0` (unlimited). upload_packs: Option>, + max_stale: Duration, + background: Arc, + request_setups: Option>, + bodies: Option>, + preflights: Option>, + upstream: Arc, + tasks: Arc>>>, } impl GitCache { @@ -141,10 +176,155 @@ impl GitCache { Self { cfg, metrics, - slots: Mutex::new(HashMap::new()), + slots: Arc::new(Mutex::new(HashMap::new())), index, upload_packs, + max_stale: Duration::ZERO, + background: Arc::new(Semaphore::new(1)), + request_setups: None, + bodies: None, + preflights: None, + upstream: Arc::new(Semaphore::new(1)), + tasks: Arc::new(StdMutex::new(Vec::new())), + } + } + + /// Enable bounded request-triggered background refreshes. + pub fn with_stale(mut self, max_stale: Duration, max_background: usize) -> Self { + self.max_stale = max_stale; + self.background = Arc::new(Semaphore::new(max_background)); + if self.request_setups.is_none() { + self.upstream = Arc::new(Semaphore::new(max_background)); + } + self + } + + /// Bound request setup tasks, including those whose clients disconnected. + pub fn with_request_limit(mut self, limit: usize) -> Self { + self.request_setups = (limit > 0).then(|| Arc::new(Semaphore::new(limit))); + self.bodies = (limit > 0).then(|| Arc::new(Semaphore::new(limit))); + self.preflights = (limit > 0).then(|| Arc::new(Semaphore::new(limit))); + let upstream_limit = if limit == 0 { + self.background.available_permits().max(1) + } else { + limit.saturating_sub(1).max(1) + }; + self.upstream = Arc::new(Semaphore::new(upstream_limit)); + self + } + + /// Wait for admitted background refreshes after the HTTP server drains. + pub async fn shutdown(&self) { + loop { + let tasks = std::mem::take(&mut *self.tasks.lock().unwrap()); + if tasks.is_empty() { + break; + } + for task in tasks { + let _ = task.await; + } + } + } + + /// Prepare a request and keep the mirror protected if its client disconnects. + pub async fn prepare( + &self, + repo: RepoRef, + want_fetch: bool, + body: Bytes, + cred: Option<&str>, + ) -> Result<(CacheOutcome, RequestLease)> { + let permit = if self.stale_enabled() { + None + } else { + self.request_permit().await? + }; + let preflight = self.preflight_permit()?; + let cache = self.clone(); + let cred = cred.map(str::to_owned); + self.spawn_tracked(async move { + let guard = if cache.stale_enabled() { + cache + .slot(&repo.name) + .await + .lifecycle + .clone() + .try_read_owned() + .map_err(|_| CacheBusy)? + } else { + cache.read_guard(&repo).await + }; + let outcome = cache + .ensure_fresh_inner(&repo, want_fetch, &body, cred.as_deref(), preflight) + .await?; + let permit = if cache.stale_enabled() { + cache + .request_setups + .as_ref() + .map(|s| s.clone().try_acquire_owned()) + .transpose() + .map_err(|_| CacheBusy)? + } else { + permit + }; + Ok((outcome, RequestLease { guard, permit })) + }) + .await + .context("request preparation task")? + } + + /// Whether request-triggered stale serving is enabled. + pub fn stale_enabled(&self) -> bool { + !self.max_stale.is_zero() + } + + /// Bound request decoding independently of slow upstream preparation. + pub fn body_permit(&self) -> Result> { + Ok(self + .bodies + .as_ref() + .map(|s| s.clone().try_acquire_owned()) + .transpose() + .map_err(|_| CacheBusy)?) + } + + fn preflight_permit(&self) -> Result> { + if !self.stale_enabled() { + return Ok(None); } + Ok(self + .preflights + .as_ref() + .map(|s| s.clone().try_acquire_owned()) + .transpose() + .map_err(|_| CacheBusy)?) + } + + async fn request_permit(&self) -> Result> { + let permit = match &self.request_setups { + Some(limit) => Some( + limit + .clone() + .acquire_owned() + .await + .context("request setup cap")?, + ), + None => None, + }; + Ok(permit) + } + + fn spawn_tracked( + &self, + task: impl std::future::Future + Send + 'static, + ) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + let mut tasks = self.tasks.lock().unwrap(); + tasks.retain(|task| !task.is_finished()); + tasks.push(tokio::spawn(async move { + let _ = tx.send(task.await); + })); + rx } async fn slot(&self, name: &str) -> Arc { @@ -154,12 +334,24 @@ impl GitCache { .entry(name.to_string()) .or_insert_with(|| { Arc::new(RepoSlot { - fetch_lock: Mutex::new(None), + fetch_lock: Arc::new(Mutex::new(None)), + lifecycle: Arc::new(RwLock::new(())), + refresh: Mutex::new(RefreshState::default()), }) }) .clone() } + /// Protect a mirror from eviction for the duration of a request. + pub async fn read_guard(&self, repo: &RepoRef) -> OwnedRwLockReadGuard<()> { + self.slot(&repo.name) + .await + .lifecycle + .clone() + .read_owned() + .await + } + /// Flag a mirror as changed after a clone/fetch so the background evictor /// (re)measures it and rebalances the cache. O(1) bookkeeping only - the size /// walk and any eviction run off this request path. No-op when eviction is @@ -175,10 +367,10 @@ impl GitCache { /// that populates the mirror. The rename is atomic and fast; the (possibly slow) /// removal runs after the lock is released. `ensure_fresh` keys off /// `HEAD.exists()`, so the next request for an evicted repo transparently - /// re-clones. An `upload-pack` already streaming from the old directory keeps - /// its open file descriptors and drains cleanly (POSIX unlink semantics). + /// re-clones. Active HTTP requests hold a read guard until Git exits. pub async fn evict(&self, name: &str, cache_dir: &Path) -> Result<()> { let slot = self.slot(name).await; + let lifecycle = slot.lifecycle.write().await; let guard = slot.fetch_lock.lock().await; if !cache_dir.join("HEAD").exists() { return Ok(()); // already gone (raced a prior eviction or manual removal) @@ -194,6 +386,7 @@ impl GitCache { .await .with_context(|| format!("rename mirror for eviction: {name}"))?; drop(guard); // the mirror is gone from its path; free the slot before the slow delete + drop(lifecycle); let _ = tokio::fs::remove_dir_all(&trash).await; Ok(()) } @@ -209,6 +402,18 @@ impl GitCache { want_fetch: bool, body: &[u8], cred: Option<&str>, + ) -> Result { + self.ensure_fresh_inner(repo, want_fetch, body, cred, self.preflight_permit()?) + .await + } + + async fn ensure_fresh_inner( + &self, + repo: &RepoRef, + want_fetch: bool, + body: &[u8], + cred: Option<&str>, + preflight: Option, ) -> Result { let slot = self.slot(&repo.name).await; // Mark the repo used on every request - a served cache hit counts as much as @@ -216,6 +421,11 @@ impl GitCache { if let Some(idx) = &self.index { idx.touch(&repo.name); } + if !self.max_stale.is_zero() { + return self + .ensure_fresh_stale(repo, &slot, body, cred, preflight) + .await; + } let mut last = slot.fetch_lock.lock().await; // Stamp the start: upstream is read then, so a ref pushed mid-fetch is @@ -246,12 +456,271 @@ impl GitCache { Ok(outcome) } + async fn ensure_fresh_stale( + &self, + repo: &RepoRef, + slot: &Arc, + body: &[u8], + cred: Option<&str>, + preflight: Option, + ) -> Result { + self.restore_freshness(repo, slot).await; + let exists = repo.cache_dir.join("HEAD").exists(); + let mut state = slot.refresh.lock().await; + if self.within_max_age(&state, exists) { + self.spawn_background_refresh(repo, slot, cred, &mut state); + drop(state); + return self.serve_cached(repo, slot, body, cred, preflight).await; + } + drop(state); + self.refresh_blocking(repo, slot, body, cred, preflight) + .await + } + + fn within_max_age(&self, state: &RefreshState, exists: bool) -> bool { + exists + && state + .last_success + .is_some_and(|t| t.elapsed() <= self.max_stale) + } + + /// Start one detached refresh once the TTL has passed. Best effort: a busy + /// lane skips it, so serving cached refs never waits on this. + fn spawn_background_refresh( + &self, + repo: &RepoRef, + slot: &Arc, + cred: Option<&str>, + state: &mut RefreshState, + ) { + let stale = state + .last_success + .is_some_and(|t| self.cfg.fetch_ttl.is_zero() || t.elapsed() >= self.cfg.fetch_ttl); + let ready = state.next_attempt.is_none_or(|t| Instant::now() >= t); + if !stale || !ready || state.in_flight { + return; + } + let Ok(permit) = self.background.clone().try_acquire_owned() else { + return; + }; + let Ok(lifecycle) = slot.lifecycle.clone().try_read_owned() else { + return; + }; + let Ok(fetch_guard) = slot.fetch_lock.clone().try_lock_owned() else { + return; + }; + state.in_flight = true; + let cache = self.clone(); + let repo = repo.clone(); + let slot = slot.clone(); + let cred = cred.map(str::to_owned); + let mut tasks = self.tasks.lock().unwrap(); + tasks.retain(|task| !task.is_finished()); + tasks.push(tokio::spawn(async move { + let _permit = permit; + let _fetch_guard = fetch_guard; + let _lifecycle = lifecycle; + cache + .background_refresh(&repo, &slot, cred.as_deref()) + .await; + })); + } + + async fn background_refresh(&self, repo: &RepoRef, slot: &RepoSlot, cred: Option<&str>) { + let started = Instant::now(); + let result = if repo.cache_dir.join("HEAD").exists() { + self.fetch(repo, cred).await + } else { + Err(anyhow::anyhow!("mirror was evicted before refresh")) + }; + self.restore_freshness(repo, slot).await; + let mut state = slot.refresh.lock().await; + if result.is_ok() { + state.last_success = Some(started); + } else { + tracing::warn!(repo = %repo.name, error = ?result, "background refresh failed"); + } + state.next_attempt = Some(Instant::now() + self.cooldown()); + state.in_flight = false; + } + + /// Serve the mirror as it is: only fetch SHAs a client asked for, then + /// confirm the age bound still holds. + async fn serve_cached( + &self, + repo: &RepoRef, + slot: &RepoSlot, + body: &[u8], + cred: Option<&str>, + preflight: Option, + ) -> Result { + self.ensure_wants_stale(repo, slot, body, cred, preflight) + .await?; + self.check_age(slot).await?; + Ok(CacheOutcome::Cached) + } + + /// The mirror is missing or past the age bound: a clone or fetch must finish + /// before the request is served. + async fn refresh_blocking( + &self, + repo: &RepoRef, + slot: &RepoSlot, + body: &[u8], + cred: Option<&str>, + preflight: Option, + ) -> Result { + let _upstream = self + .upstream + .clone() + .try_acquire_owned() + .map_err(|_| CacheBusy)?; + drop(preflight); + let mut last = slot.fetch_lock.try_lock().map_err(|_| CacheBusy)?; + self.restore_freshness(repo, slot).await; + let mut state = slot.refresh.lock().await; + let exists = repo.cache_dir.join("HEAD").exists(); + if self.within_max_age(&state, exists) { + drop(state); + if let Err(e) = self.ensure_wanted_oids(repo, body, cred).await { + tracing::warn!(repo = %repo.name, error = %e, "ensure wanted oids failed"); + } + drop(last); + self.check_age(slot).await?; + return Ok(CacheOutcome::Cached); + } + if state.next_attempt.is_some_and(|t| Instant::now() < t) { + bail!("upstream refresh is in cooldown"); + } + drop(state); + let started = Instant::now(); + let result = if exists { + self.fetch(repo, cred).await + } else { + self.clone_mirror(repo, cred).await + }; + self.restore_freshness(repo, slot).await; + state = slot.refresh.lock().await; + state.next_attempt = Some(Instant::now() + self.cooldown()); + if result.is_ok() { + state.last_success = Some(started); + *last = Some(started); + } + drop(state); + result?; + if started.elapsed() > self.max_stale { + bail!("refreshed mirror exceeds maximum stale age"); + } + if let Err(e) = self.ensure_wanted_oids(repo, body, cred).await { + tracing::warn!(repo = %repo.name, error = %e, "ensure wanted oids failed"); + } + drop(last); + self.check_age(slot).await?; + Ok(if exists { + CacheOutcome::Fetched + } else { + CacheOutcome::Cloned + }) + } + + async fn restore_freshness(&self, repo: &RepoRef, slot: &RepoSlot) { + let record = read_freshness(repo).await.ok(); + let identity = format!( + "{}:{}", + repository_identity(repo), + directory_identity(&repo.cache_dir) + .await + .unwrap_or_default() + ); + let mut state = slot.refresh.lock().await; + if state.identity.as_ref() != Some(&identity) { + state.last_success = None; + state.record = None; + state.next_attempt = None; + state.identity = Some(identity); + } + let text = record.as_ref().map(|(text, _)| text.clone()); + if state.record != text || text.is_none() { + state.last_success = + record.and_then(|(_, started)| Instant::now().checked_sub(started)); + state.record = text; + } + } + + async fn check_age(&self, slot: &RepoSlot) -> Result<()> { + if slot + .refresh + .lock() + .await + .last_success + .is_none_or(|t| t.elapsed() > self.max_stale) + { + bail!("mirror exceeds maximum stale age"); + } + Ok(()) + } + + fn cooldown(&self) -> Duration { + self.cfg.fetch_ttl.max(Duration::from_secs(1)) + } + + async fn ensure_wants_stale( + &self, + repo: &RepoRef, + slot: &RepoSlot, + body: &[u8], + cred: Option<&str>, + preflight: Option, + ) -> Result<()> { + let wants = parse_wants(body); + if wants.is_empty() { + return Ok(()); + } + match self.missing_oids(repo, &wants).await { + Ok(missing) if missing.is_empty() => return Ok(()), + _ => {} + } + let _permit = self + .upstream + .clone() + .try_acquire_owned() + .map_err(|_| CacheBusy)?; + drop(preflight); + let _lock = slot.fetch_lock.try_lock().map_err(|_| CacheBusy)?; + if let Err(e) = self.ensure_wanted_oids(repo, body, cred).await { + tracing::warn!(repo = %repo.name, error = %e, "ensure wanted oids failed"); + } + Ok(()) + } + + async fn check_serving_age(&self, repo: &RepoRef) -> Result<()> { + if self.stale_enabled() { + let slot = self.slot(&repo.name).await; + self.restore_freshness(repo, &slot).await; + self.check_age(&slot).await?; + } + Ok(()) + } + /// `git upload-pack --advertise-refs`, wrapped with the smart-HTTP service /// header - i.e. the `GET .../info/refs?service=git-upload-pack` body. pub async fn advertise_refs( &self, repo: &RepoRef, git_protocol: Option<&str>, + ) -> Result { + let permit = self.request_permit().await?; + let guard = self.read_guard(repo).await; + self.advertise_refs_guarded(repo, git_protocol, RequestLease { guard, permit }) + .await + } + + /// Advertise while retaining the request's eviction guard until Git exits. + pub async fn advertise_refs_guarded( + &self, + repo: &RepoRef, + git_protocol: Option<&str>, + lease: RequestLease, ) -> Result { let mut cmd = self.local_cmd(git_protocol); cmd.arg("upload-pack") @@ -259,9 +728,16 @@ impl GitCache { .arg("--advertise-refs") .arg(&repo.cache_dir); let started = Instant::now(); - let out = cmd - .output() + let cache = self.clone(); + let owned_repo = repo.clone(); + let out = self + .spawn_tracked(async move { + let _lease = lease; + cache.check_serving_age(&owned_repo).await?; + cmd.output().await.context("spawn git advertisement") + }) .await + .context("advertise-refs task")? .context("spawn git upload-pack --advertise-refs")?; if !out.status.success() { bail!( @@ -291,6 +767,36 @@ impl GitCache { git_protocol: Option<&str>, body: Bytes, ) -> Result>> { + let guard = self.read_guard(repo).await; + self.upload_pack_rpc_guarded( + repo, + git_protocol, + body, + RequestLease { + guard, + permit: None, + }, + ) + .await + } + + /// Start an upload-pack while retaining the request's eviction guard. + pub async fn upload_pack_rpc_guarded( + &self, + repo: &RepoRef, + git_protocol: Option<&str>, + body: Bytes, + lease: RequestLease, + ) -> Result>> { + let RequestLease { + guard, + permit: mut setup_permit, + } = lease; + let _strict_setup = if self.stale_enabled() { + None + } else { + setup_permit.take() + }; let permit = match &self.upload_packs { Some(s) => Some(s.clone().acquire_owned().await.context("upload-pack cap")?), None => None, @@ -309,6 +815,7 @@ impl GitCache { // nothing while it counts and compresses and so never gets SIGPIPE. #[cfg(unix)] cmd.process_group(0); + self.check_serving_age(repo).await?; let mut child = cmd.spawn().context("spawn git upload-pack")?; let mut stdin = child.stdin.take().context("upload-pack: no stdin")?; let stdout = child.stdout.take().context("upload-pack: no stdout")?; @@ -317,12 +824,16 @@ impl GitCache { // wait for exit without buffering the whole pack. Spawned before the stdin // write, so even a request dropped mid-write still kills and reaps the child. let (served_tx, served_rx) = oneshot::channel(); - tokio::spawn(reap_upload_pack( + drop(self.spawn_tracked(reap_upload_pack( child, served_rx, permit, repo.name.clone(), - )); + Some(RequestLease { + guard, + permit: setup_permit, + }), + ))); stdin .write_all(&body) @@ -362,6 +873,7 @@ impl GitCache { // sync. The client then negotiates whatever ref it wants via upload-pack, so // the mirror can serve any branch/tag/sha the origin has - never HEAD-only. let started = Instant::now(); + let observed = self.stale_enabled().then(unix_millis).transpose()?; let mut cmd = self.fetch_cmd(cred); cmd.arg("clone") .arg("--mirror") @@ -377,6 +889,9 @@ impl GitCache { return Err(error); } let elapsed = started.elapsed().as_secs_f64(); + if let Some(observed) = observed { + write_freshness(repo, &tmp, observed).await?; + } tokio::fs::rename(&tmp, &repo.cache_dir) .await .context("rename mirror into place")?; @@ -392,10 +907,18 @@ impl GitCache { tracing::debug!(repo = %repo.name, "fetching updates from upstream"); // A reused mirror can still record an older upstream in origin. let started = Instant::now(); + let observed = self.stale_enabled().then(unix_millis).transpose()?; + let directory = if self.stale_enabled() { + Some(directory_identity(&repo.cache_dir).await?) + } else { + None + }; let mut cmd = self.fetch_cmd(cred); - cmd.current_dir(&repo.cache_dir) - .arg("fetch") - .arg("--prune") + cmd.current_dir(&repo.cache_dir).arg("fetch"); + if !self.max_stale.is_zero() { + cmd.arg("--atomic"); + } + cmd.arg("--prune") .arg("--quiet") .arg(&repo.upstream_url) // Mirror refspec, minus the local-only pin namespace: `--prune` would @@ -407,6 +930,13 @@ impl GitCache { .record_upstream(UpstreamOp::Fetch, Status::Error, "-"); return Err(error); } + if let Some(observed) = observed { + anyhow::ensure!( + Some(directory_identity(&repo.cache_dir).await?) == directory, + "mirror replaced during refresh" + ); + write_freshness(repo, &repo.cache_dir, observed).await?; + } self.metrics .record_upstream(UpstreamOp::Fetch, Status::Ok, &repo.name); self.metrics.observe_upstream( @@ -608,6 +1138,11 @@ impl GitCache { let mut c = Command::new(&self.cfg.git_binary); c.env("GIT_TERMINAL_PROMPT", "0"); // fail instead of hanging on a prompt c.env("LC_ALL", "C"); // Retry classification relies on Git's English diagnostics. + if !self.max_stale.is_zero() { + // Keep objects advertised before a refresh available to clients. + c.arg("-c").arg("gc.auto=0"); + c.arg("-c").arg("maintenance.auto=false"); + } for (k, v) in git_config_env( &self.cfg.big_file_threshold, cred.or(self.cfg.upstream_auth_header.as_deref()), @@ -632,6 +1167,10 @@ impl GitCache { fn local_cmd(&self, git_protocol: Option<&str>) -> Command { let mut c = Command::new(&self.cfg.git_binary); c.arg("-c").arg("uploadpack.allowFilter=true"); + if !self.max_stale.is_zero() { + // A client may request an object whose advertised ref was removed. + c.arg("-c").arg("uploadpack.allowAnySHA1InWant=true"); + } // Hide the pin namespace from the advertisement so it never leaks into a // client's ref list; a hidden ref is still honored as a `want` tip. c.arg("-c") @@ -649,6 +1188,88 @@ impl GitCache { } } +fn unix_millis() -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("clock predates Unix epoch")? + .as_millis() + .try_into()?) +} + +fn repository_identity(repo: &RepoRef) -> String { + let upstream = match reqwest::Url::parse(&repo.upstream_url) { + Ok(mut url) => { + let _ = url.set_username(""); + let _ = url.set_password(None); + url.set_query(None); + url.set_fragment(None); + url.to_string() + } + Err(_) => repo.upstream_url.clone(), + }; + Sha256::digest(format!("{}\n{upstream}", repo.name)) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +async fn directory_identity(path: &Path) -> Result { + let metadata = tokio::fs::metadata(path).await?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + Ok(format!("{}:{}", metadata.dev(), metadata.ino())) + } + #[cfg(not(unix))] + { + Ok(format!("{:?}", metadata.created()?)) + } +} + +async fn write_freshness(repo: &RepoRef, directory: &Path, observed: u64) -> Result<()> { + let record = serde_json::json!({ + "version": 1, + "directory": directory_identity(directory).await?, + "repository": repository_identity(repo), + "observed_ms": observed, + }); + let temporary = directory.join("proxy-refresh.tmp"); + let mut file = tokio::fs::File::create(&temporary).await?; + file.write_all(record.to_string().as_bytes()).await?; + file.sync_all().await?; + drop(file); + tokio::fs::rename(temporary, directory.join(FRESHNESS_FILE)).await?; + #[cfg(unix)] + tokio::fs::File::open(directory).await?.sync_all().await?; + Ok(()) +} + +async fn read_freshness(repo: &RepoRef) -> Result<(String, Duration)> { + let file = tokio::fs::File::open(repo.cache_dir.join(FRESHNESS_FILE)).await?; + let mut text = String::new(); + file.take(MAX_FRESHNESS_BYTES + 1) + .read_to_string(&mut text) + .await?; + anyhow::ensure!( + text.len() <= MAX_FRESHNESS_BYTES as usize, + "oversized freshness record" + ); + let record: serde_json::Value = serde_json::from_str(&text)?; + anyhow::ensure!( + record["version"] == 1 + && record["directory"] == directory_identity(&repo.cache_dir).await? + && record["repository"] == repository_identity(repo), + "freshness identity mismatch" + ); + let observed = record["observed_ms"] + .as_u64() + .context("invalid freshness time")?; + let age = unix_millis()? + .checked_sub(observed) + .context("future freshness time")?; + Ok((text, Duration::from_millis(age))) +} + // The caller holds the repository lock throughout retries. Never retry local // upload-pack: once its response starts streaming it cannot be replayed safely. async fn run_upstream( @@ -836,6 +1457,7 @@ async fn reap_upload_pack( served: oneshot::Receiver<()>, _permit: Option, repo: String, + _lease: Option, ) { let status = tokio::select! { biased; @@ -1014,20 +1636,64 @@ mod tests { use super::*; #[tokio::test] - async fn missing_oids_drains_more_than_a_pipe_buffer() { + async fn request_setup_retains_permit_in_lease() { + for stale in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let mirror = dir.path().join("mirror.git"); + std::fs::create_dir(&mirror).unwrap(); + std::fs::write(mirror.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + let cache = GitCache::new( + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ) + .with_stale(Duration::from_secs(if stale { 60 } else { 0 }), 1) + .with_request_limit(1); + let repo = RepoRef { + name: "repo.git".into(), + upstream_url: String::new(), + cache_dir: mirror, + }; + let slot = cache.slot(&repo.name).await; + *slot.fetch_lock.lock().await = Some(Instant::now()); + if stale { + write_freshness(&repo, &repo.cache_dir, unix_millis().unwrap()) + .await + .unwrap(); + } + slot.refresh.lock().await.last_success = Some(Instant::now()); + let (_, lease) = cache.prepare(repo, true, Bytes::new(), None).await.unwrap(); + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 0 + ); + drop(lease); + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 1 + ); + cache.shutdown().await; + } + } + + #[tokio::test(start_paused = true)] + async fn request_setup_waits_for_a_permit_before_spawning() { let dir = tempfile::tempdir().unwrap(); - let status = std::process::Command::new("git") - .args(["init", "--bare", "--quiet"]) - .arg(dir.path()) - .status() - .unwrap(); - assert!(status.success()); let cache = GitCache::new( GitConfig { - git_binary: "git".into(), + git_binary: "false".into(), upstream_auth_header: None, big_file_threshold: "8m".into(), - fetch_ttl: Duration::from_secs(60), + fetch_ttl: Duration::from_secs(10), max_wants: 100, max_upload_packs: 0, pack_threads: None, @@ -1035,31 +1701,618 @@ mod tests { }, Arc::new(Metrics::new()), None, - ); + ) + .with_request_limit(1); + let permit = cache + .request_setups + .as_ref() + .unwrap() + .acquire() + .await + .unwrap(); let repo = RepoRef { - name: "test/repo.git".into(), + name: "repo.git".into(), upstream_url: String::new(), - cache_dir: dir.path().to_path_buf(), + cache_dir: dir.path().join("mirror.git"), }; - let oids = (0..20_000).map(|n| format!("{n:040x}")).collect(); - let missing = - tokio::time::timeout(Duration::from_secs(10), cache.missing_oids(&repo, &oids)) + let setup = cache.prepare(repo, true, Bytes::new(), None); + tokio::pin!(setup); + assert!( + tokio::time::timeout(Duration::from_secs(1), &mut setup) .await - .expect("cat-file must not deadlock") - .unwrap(); - assert_eq!(missing.len(), oids.len()); + .is_err() + ); + assert!(cache.tasks.lock().unwrap().is_empty()); + drop(permit); + assert!(setup.await.is_err()); + cache.shutdown().await; } - #[test] - fn upstream_retry_classifies_transient_diagnostics() { - for message in [ - "TLS connect error: error:0A000126:SSL routines::unexpected eof while reading", - "gnutls_handshake() failed: The TLS connection was non-properly terminated.", - "GnuTLS recv error (-110): The TLS connection was non-properly terminated.", - "OpenSSL SSL_connect: SSL_ERROR_SYSCALL in connection to github.com:443", - "Recv failure: Connection reset by peer", - "Failed to connect to github.com port 443: Connection refused", - "Operation timed out after 30000 milliseconds", + #[cfg(unix)] + #[tokio::test(start_paused = true)] + async fn cancelled_advertisement_holds_mirror_until_child_exits() { + let dir = tempfile::tempdir().unwrap(); + let mirror = dir.path().join("mirror.git"); + std::fs::create_dir(&mirror).unwrap(); + std::fs::write(mirror.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + let marker = dir.path().join("started"); + let gate = dir.path().join("gate"); + for fifo in [&marker, &gate] { + assert!( + std::process::Command::new("mkfifo") + .arg(fifo) + .status() + .unwrap() + .success() + ); + } + let wrapper = dir.path().join("git-wrapper"); + write_script( + &wrapper, + &format!( + "#!/bin/sh\necho started > '{}'\nread gate < '{}'\n", + marker.display(), + gate.display() + ), + ); + let cache = GitCache::new( + GitConfig { + git_binary: wrapper.to_string_lossy().into_owned(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ) + .with_request_limit(1); + let repo = RepoRef { + name: "repo.git".into(), + upstream_url: String::new(), + cache_dir: mirror.clone(), + }; + let started = tokio::spawn(async move { tokio::fs::read_to_string(marker).await.unwrap() }); + let request = tokio::spawn({ + let cache = cache.clone(); + let repo = repo.clone(); + async move { cache.advertise_refs(&repo, None).await } + }); + assert_eq!(started.await.unwrap().trim(), "started"); + request.abort(); + let _ = request.await; + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 0 + ); + let eviction = cache.evict(&repo.name, &mirror); + tokio::pin!(eviction); + assert!( + tokio::time::timeout(Duration::from_secs(1), &mut eviction) + .await + .is_err() + ); + assert!(mirror.exists()); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + eviction.await.unwrap(); + cache.shutdown().await; + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 1 + ); + assert!(!mirror.exists()); + } + + #[cfg(unix)] + #[tokio::test(start_paused = true)] + async fn failed_refresh_cools_down_and_expired_mirror_fails() { + let dir = tempfile::tempdir().unwrap(); + let mirror = dir.path().join("mirror.git"); + std::fs::create_dir(&mirror).unwrap(); + std::fs::write(mirror.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + let log = dir.path().join("attempts"); + let wrapper = dir.path().join("git-wrapper"); + write_script( + &wrapper, + &format!("#!/bin/sh\necho fetch >> '{}'\nexit 1\n", log.display()), + ); + let cache = GitCache::new( + GitConfig { + git_binary: wrapper.to_string_lossy().into_owned(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ) + .with_stale(Duration::from_secs(12), 1); + let repo = RepoRef { + name: "repo.git".into(), + upstream_url: String::new(), + cache_dir: mirror, + }; + write_freshness(&repo, &repo.cache_dir, unix_millis().unwrap()) + .await + .unwrap(); + cache + .restore_freshness(&repo, cache.slot(&repo.name).await.as_ref()) + .await; + tokio::time::advance(Duration::from_secs(10)).await; + assert!(matches!( + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(), + CacheOutcome::Cached + )); + cache.shutdown().await; + for _ in 0..3 { + assert!(matches!( + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(), + CacheOutcome::Cached + )); + } + tokio::time::advance(Duration::from_secs(3)).await; + assert!(cache.ensure_fresh(&repo, true, &[], None).await.is_err()); + assert_eq!(std::fs::read_to_string(&log).unwrap().lines().count(), 1); + tokio::time::advance(Duration::from_secs(7)).await; + assert!(cache.ensure_fresh(&repo, true, &[], None).await.is_err()); + assert_eq!(std::fs::read_to_string(&log).unwrap().lines().count(), 2); + tokio::fs::remove_file(repo.cache_dir.join(FRESHNESS_FILE)) + .await + .unwrap(); + let restarted = GitCache::new(cache.cfg.clone(), Arc::new(Metrics::new()), None) + .with_stale(Duration::from_secs(12), 1); + assert!( + restarted + .ensure_fresh(&repo, false, &[], None) + .await + .is_err() + ); + assert_eq!(std::fs::read_to_string(&log).unwrap().lines().count(), 3); + } + + #[cfg(unix)] + #[tokio::test(start_paused = true)] + async fn stale_reads_share_a_background_fetch() { + let dir = tempfile::tempdir().unwrap(); + let origin = dir.path().join("origin.git"); + let work = dir.path().join("work"); + std::fs::create_dir(&work).unwrap(); + assert!( + std::process::Command::new("git") + .args(["init", "--quiet", "-b", "main"]) + .arg(&work) + .status() + .unwrap() + .success() + ); + assert!( + std::process::Command::new("git") + .current_dir(&work) + .args([ + "-c", + "user.name=test", + "-c", + "user.email=test@example.invalid", + "commit", + "--quiet", + "--allow-empty", + "-m", + "initial" + ]) + .status() + .unwrap() + .success() + ); + let head = std::process::Command::new("git") + .current_dir(&work) + .args(["rev-parse", "HEAD"]) + .output() + .unwrap(); + let head = String::from_utf8(head.stdout).unwrap(); + assert!( + std::process::Command::new("git") + .args(["clone", "--quiet", "--mirror"]) + .arg(&work) + .arg(&origin) + .status() + .unwrap() + .success() + ); + let marker = dir.path().join("started"); + let gate = dir.path().join("gate"); + for fifo in [&marker, &gate] { + assert!( + std::process::Command::new("mkfifo") + .arg(fifo) + .status() + .unwrap() + .success() + ); + } + let wrapper = dir.path().join("git-wrapper"); + let script = format!( + "#!/bin/sh\nfor arg do\n if [ \"$arg\" = fetch ]; then\n echo started > '{}'\n read gate < '{}'\n break\n fi\ndone\nexec git \"$@\"\n", + marker.display(), + gate.display() + ); + write_script(&wrapper, &script); + let cache = GitCache::new( + GitConfig { + git_binary: wrapper.to_string_lossy().into_owned(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ) + .with_stale(Duration::from_secs(100), 1); + let repo = RepoRef { + name: "repo.git".into(), + upstream_url: origin.to_string_lossy().into_owned(), + cache_dir: dir.path().join("mirror.git"), + }; + assert!(matches!( + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(), + CacheOutcome::Cloned + )); + let other = RepoRef { + name: "other.git".into(), + upstream_url: origin.to_string_lossy().into_owned(), + cache_dir: dir.path().join("other.git"), + }; + assert!(matches!( + cache.ensure_fresh(&other, true, &[], None).await.unwrap(), + CacheOutcome::Cloned + )); + tokio::time::advance(Duration::from_secs(11)).await; + let started = tokio::spawn({ + let marker = marker.clone(); + async move { tokio::fs::read_to_string(marker).await.unwrap() } + }); + let (ready_tx, ready_rx) = oneshot::channel(); + let trigger = tokio::spawn({ + let cache = cache.clone(); + let repo = repo.clone(); + async move { + let (outcome, _guard) = + cache.prepare(repo, true, Bytes::new(), None).await.unwrap(); + assert!(matches!(outcome, CacheOutcome::Cached)); + let _ = ready_tx.send(()); + std::future::pending::<()>().await; + } + }); + ready_rx.await.unwrap(); + assert_eq!(started.await.unwrap().trim(), "started"); + trigger.abort(); + let _ = trigger.await; + tokio::time::advance(Duration::from_secs(20)).await; + assert!(matches!( + cache.ensure_fresh(&other, true, &[], None).await.unwrap(), + CacheOutcome::Cached + )); + assert!(!cache.slot(&other.name).await.refresh.lock().await.in_flight); + assert!(matches!( + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(), + CacheOutcome::Cached + )); + assert!(!cache.advertise_refs(&repo, None).await.unwrap().is_empty()); + let mut request = pkt_line(&format!("want {}\n", head.trim())); + request.extend_from_slice(b"0000"); + request.extend(pkt_line("done\n")); + let request = Bytes::from(request); + let (_, guard) = cache + .prepare(repo.clone(), false, request.clone(), None) + .await + .unwrap(); + let stream = cache + .upload_pack_rpc_guarded(&repo, None, request, guard) + .await + .unwrap(); + let pack = axum::body::to_bytes(axum::body::Body::from_stream(stream), usize::MAX) + .await + .unwrap(); + assert!(pack.windows(4).any(|bytes| bytes == b"PACK")); + let tree = std::process::Command::new("git") + .current_dir(&work) + .args(["rev-parse", "HEAD^{tree}"]) + .output() + .unwrap(); + let tree = String::from_utf8(tree.stdout).unwrap(); + let replacement = std::process::Command::new("git") + .current_dir(&work) + .args([ + "-c", + "user.name=test", + "-c", + "user.email=test@example.invalid", + "commit-tree", + tree.trim(), + "-m", + "replacement", + ]) + .output() + .unwrap(); + assert!(replacement.status.success()); + let replacement = String::from_utf8(replacement.stdout).unwrap(); + assert!( + std::process::Command::new("git") + .current_dir(&work) + .args([ + "push", + "--quiet", + "--force", + origin.to_str().unwrap(), + &format!("{}:refs/heads/main", replacement.trim()) + ]) + .status() + .unwrap() + .success() + ); + tokio::time::advance(Duration::from_secs(70)).await; + let expired = cache.prepare(repo.clone(), true, Bytes::new(), None).await; + assert!(expired.err().unwrap().is::()); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(&gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + cache.shutdown().await; + let (outcome, guard) = cache + .prepare(repo.clone(), true, Bytes::new(), None) + .await + .unwrap(); + assert!(matches!(outcome, CacheOutcome::Cached)); + drop(guard); + assert!(!cache.slot(&repo.name).await.refresh.lock().await.in_flight); + let current = std::process::Command::new("git") + .arg("--git-dir") + .arg(&repo.cache_dir) + .args(["rev-parse", "refs/heads/main"]) + .output() + .unwrap(); + assert_eq!( + String::from_utf8(current.stdout).unwrap().trim(), + replacement.trim() + ); + assert!(matches!( + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(), + CacheOutcome::Cached + )); + assert!(!cache.slot(&repo.name).await.refresh.lock().await.in_flight); + let mut request = pkt_line(&format!("want {}\n", head.trim())); + request.extend_from_slice(b"0000"); + request.extend(pkt_line("done\n")); + let stream = cache + .upload_pack_rpc(&repo, None, Bytes::from(request)) + .await + .unwrap(); + let pack = axum::body::to_bytes(axum::body::Body::from_stream(stream), usize::MAX) + .await + .unwrap(); + assert!(pack.windows(4).any(|bytes| bytes == b"PACK")); + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&origin) + .args(["update-ref", "-d", "refs/heads/main"]) + .status() + .unwrap() + .success() + ); + let started = tokio::spawn(async move { tokio::fs::read_to_string(marker).await.unwrap() }); + let fetch = tokio::spawn({ + let cache = cache.clone(); + let repo = repo.clone(); + async move { cache.fetch(&repo, None).await.unwrap() } + }); + assert_eq!(started.await.unwrap().trim(), "started"); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(&gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + fetch.await.unwrap(); + assert!( + std::process::Command::new("git") + .arg("--git-dir") + .arg(&repo.cache_dir) + .args(["cat-file", "-e", head.trim()]) + .status() + .unwrap() + .success() + ); + let mut request = pkt_line(&format!("want {}\n", head.trim())); + request.extend_from_slice(b"0000"); + request.extend(pkt_line("done\n")); + let stream = cache + .upload_pack_rpc(&repo, None, Bytes::from(request)) + .await + .unwrap(); + let pack = axum::body::to_bytes(axum::body::Body::from_stream(stream), usize::MAX) + .await + .unwrap(); + assert!(pack.windows(4).any(|bytes| bytes == b"PACK")); + let missing = "f".repeat(40); + let mut request = pkt_line(&format!("want {missing}\n")); + request.extend_from_slice(b"0000"); + request.extend(pkt_line("done\n")); + let stream = cache + .upload_pack_rpc(&repo, None, Bytes::from(request)) + .await + .unwrap(); + let rejected = axum::body::to_bytes(axum::body::Body::from_stream(stream), usize::MAX) + .await + .unwrap(); + assert!(!rejected.windows(4).any(|bytes| bytes == b"PACK")); + } + + #[tokio::test(start_paused = true)] + async fn eviction_waits_for_active_reader() { + let dir = tempfile::tempdir().unwrap(); + let mirror = dir.path().join("repo.git"); + tokio::fs::create_dir(&mirror).await.unwrap(); + tokio::fs::write(mirror.join("HEAD"), "ref: refs/heads/main\n") + .await + .unwrap(); + let cache = GitCache::new( + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ); + let repo = RepoRef { + name: "repo.git".into(), + upstream_url: String::new(), + cache_dir: mirror.clone(), + }; + let guard = cache.read_guard(&repo).await; + let eviction = cache.evict(&repo.name, &mirror); + tokio::pin!(eviction); + assert!( + tokio::time::timeout(Duration::from_secs(1), &mut eviction) + .await + .is_err() + ); + assert!(mirror.join("HEAD").exists()); + drop(guard); + eviction.await.unwrap(); + assert!(!mirror.exists()); + } + + #[tokio::test] + async fn upstream_commands_disable_automatic_maintenance() { + let cache = GitCache::new( + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ); + let args: Vec<_> = cache + .local_cmd(None) + .as_std() + .get_args() + .map(|arg| arg.to_owned()) + .collect(); + assert!( + !args + .iter() + .any(|arg| arg == "uploadpack.allowAnySHA1InWant=true") + ); + let args: Vec<_> = cache + .fetch_cmd(None) + .as_std() + .get_args() + .map(|arg| arg.to_owned()) + .collect(); + assert!(!args.iter().any(|arg| arg == "gc.auto=0")); + let cache = cache.with_stale(Duration::from_secs(60), 1); + let args: Vec<_> = cache + .local_cmd(None) + .as_std() + .get_args() + .map(|arg| arg.to_owned()) + .collect(); + assert!( + args.iter() + .any(|arg| arg == "uploadpack.allowAnySHA1InWant=true") + ); + for (key, expected) in [("gc.auto", "0"), ("maintenance.auto", "false")] { + let output = cache + .fetch_cmd(None) + .args(["config", "--get", key]) + .output() + .await + .unwrap(); + assert!(output.status.success()); + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), expected); + } + } + + #[tokio::test] + async fn missing_oids_drains_more_than_a_pipe_buffer() { + let dir = tempfile::tempdir().unwrap(); + let status = std::process::Command::new("git") + .args(["init", "--bare", "--quiet"]) + .arg(dir.path()) + .status() + .unwrap(); + assert!(status.success()); + let cache = GitCache::new( + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ); + let repo = RepoRef { + name: "test/repo.git".into(), + upstream_url: String::new(), + cache_dir: dir.path().to_path_buf(), + }; + let oids = (0..20_000).map(|n| format!("{n:040x}")).collect(); + let missing = + tokio::time::timeout(Duration::from_secs(10), cache.missing_oids(&repo, &oids)) + .await + .expect("cat-file must not deadlock") + .unwrap(); + assert_eq!(missing.len(), oids.len()); + } + + #[test] + fn upstream_retry_classifies_transient_diagnostics() { + for message in [ + "TLS connect error: error:0A000126:SSL routines::unexpected eof while reading", + "gnutls_handshake() failed: The TLS connection was non-properly terminated.", + "GnuTLS recv error (-110): The TLS connection was non-properly terminated.", + "OpenSSL SSL_connect: SSL_ERROR_SYSCALL in connection to github.com:443", + "Recv failure: Connection reset by peer", + "Failed to connect to github.com port 443: Connection refused", + "Operation timed out after 30000 milliseconds", "Empty reply from server", "Send failure: Broken pipe", "Temporary failure in name resolution", @@ -1280,7 +2533,7 @@ mod tests { drop(tx); tokio::time::timeout( Duration::from_secs(10), - reap_upload_pack(child, rx, None, "repo.git".into()), + reap_upload_pack(child, rx, None, "repo.git".into(), None), ) .await .expect("reaper should kill the leader and return"); @@ -1301,7 +2554,7 @@ mod tests { tx.send(()).unwrap(); let reaped = tokio::time::timeout( Duration::from_millis(200), - reap_upload_pack(child, rx, None, "repo.git".into()), + reap_upload_pack(child, rx, None, "repo.git".into(), None), ) .await; assert!(reaped.is_err(), "EOF must not kill upload-pack"); @@ -1315,6 +2568,726 @@ mod tests { assert!(kill_group(&child).is_err()); } + #[tokio::test] + async fn freshness_survives_restart_and_rejects_invalid_or_replaced_mirrors() { + let (dir, repo, cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let marker = repo.cache_dir.join(FRESHNESS_FILE); + let original = tokio::fs::read_to_string(&marker).await.unwrap(); + let mut offline = cache.cfg.clone(); + offline.git_binary = "false".into(); + let restarted = || { + GitCache::new(offline.clone(), Arc::new(Metrics::new()), None) + .with_stale(Duration::from_secs(60), 1) + }; + assert!(matches!( + restarted() + .ensure_fresh(&repo, true, &[], None) + .await + .unwrap(), + CacheOutcome::Cached + )); + let mut other = repo.clone(); + other.upstream_url.push_str("-different"); + assert!( + restarted() + .ensure_fresh(&other, true, &[], None) + .await + .is_err() + ); + other = repo.clone(); + other.name.push_str("-different"); + assert!( + restarted() + .ensure_fresh(&other, true, &[], None) + .await + .is_err() + ); + for (key, value) in [ + ("version", serde_json::json!(2)), + ("directory", serde_json::json!("replacement")), + ("repository", serde_json::json!("other")), + ( + "observed_ms", + serde_json::json!(unix_millis().unwrap() + 60_000), + ), + ("observed_ms", serde_json::json!("invalid")), + ( + "observed_ms", + serde_json::json!(unix_millis().unwrap() - 61_000), + ), + ] { + let mut record: serde_json::Value = serde_json::from_str(&original).unwrap(); + record[key] = value; + tokio::fs::write(&marker, record.to_string()).await.unwrap(); + assert!( + restarted() + .ensure_fresh(&repo, false, &[], None) + .await + .is_err(), + "{key}" + ); + } + for invalid in ["{".to_owned(), "x".repeat(MAX_FRESHNESS_BYTES as usize + 1)] { + tokio::fs::write(&marker, invalid).await.unwrap(); + assert!( + restarted() + .ensure_fresh(&repo, true, &[], None) + .await + .is_err() + ); + } + tokio::fs::remove_file(&marker).await.unwrap(); + tokio::fs::write(repo.cache_dir.join("unrelated"), "changed directory mtime") + .await + .unwrap(); + assert!( + restarted() + .ensure_fresh(&repo, true, &[], None) + .await + .is_err() + ); + tokio::fs::rename(&repo.cache_dir, dir.path().join("old.git")) + .await + .unwrap(); + fixture_git( + dir.path(), + &[ + "clone", + "--mirror", + &repo.upstream_url, + repo.cache_dir.to_str().unwrap(), + ], + ); + tokio::fs::write(&marker, original).await.unwrap(); + assert!( + restarted() + .ensure_fresh(&repo, true, &[], None) + .await + .is_err() + ); + } + + #[tokio::test] + async fn full_refs_renew_freshness_but_sha_and_failed_metadata_do_not() { + let (dir, repo, cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let marker = repo.cache_dir.join(FRESHNESS_FILE); + let original = tokio::fs::read_to_string(&marker).await.unwrap(); + let origin = Path::new(&repo.upstream_url); + let tree = fixture_git(origin, &["rev-parse", "HEAD^{tree}"]); + let oid = fixture_git( + origin, + &[ + "commit-tree", + tree.trim(), + "-p", + "HEAD", + "-m", + "unadvertised", + ], + ); + let body = pkt_line(&format!("want {}\n", oid.trim())); + cache.ensure_wanted_oids(&repo, &body, None).await.unwrap(); + assert_eq!(tokio::fs::read_to_string(&marker).await.unwrap(), original); + assert_eq!( + fixture_git( + &repo.cache_dir, + &["rev-parse", &format!("{PROXY_WANTS_NS}/{}", oid.trim())] + ) + .trim(), + oid.trim() + ); + fixture_git(origin, &["update-ref", "refs/heads/second", oid.trim()]); + fixture_git(origin, &["update-ref", "refs/tags/new", oid.trim()]); + fixture_git(origin, &["update-ref", "-d", "refs/heads/main"]); + tokio::fs::create_dir(repo.cache_dir.join("proxy-refresh.tmp")) + .await + .unwrap(); + assert!(cache.fetch(&repo, None).await.is_err()); + assert_eq!(tokio::fs::read_to_string(&marker).await.unwrap(), original); + tokio::fs::remove_dir(repo.cache_dir.join("proxy-refresh.tmp")) + .await + .unwrap(); + cache.fetch(&repo, None).await.unwrap(); + assert_eq!( + fixture_git(&repo.cache_dir, &["rev-parse", "refs/heads/second"]).trim(), + oid.trim() + ); + assert_eq!( + fixture_git(&repo.cache_dir, &["rev-parse", "refs/tags/new"]).trim(), + oid.trim() + ); + assert!( + !std::process::Command::new("git") + .current_dir(&repo.cache_dir) + .args(["show-ref", "--verify", "--quiet", "refs/heads/main"]) + .status() + .unwrap() + .success() + ); + let record = read_freshness(&repo).await.unwrap(); + assert!(record.1 < Duration::from_secs(60)); + let _ = dir; + } + + #[cfg(unix)] + #[tokio::test(start_paused = true)] + async fn slow_successful_cold_clone_cannot_exceed_stale_bound() { + tokio::time::resume(); + let (dir, repo, mut cache) = freshness_fixture(); + let ready = dir.path().join("ready"); + let gate = dir.path().join("gate"); + for path in [&ready, &gate] { + assert!( + std::process::Command::new("mkfifo") + .arg(path) + .status() + .unwrap() + .success() + ); + } + cache.cfg.git_binary = fixture_script( + dir.path(), + &format!( + "#!/bin/sh\ngit \"$@\" || exit $?\necho ready > '{}'\nread gate < '{}'\n", + ready.display(), + gate.display() + ), + ); + let owner = tokio::spawn({ + let cache = cache.clone(); + let repo = repo.clone(); + async move { cache.ensure_fresh(&repo, true, &[], None).await } + }); + tokio::fs::read_to_string(ready).await.unwrap(); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + assert!( + owner + .await + .unwrap() + .unwrap_err() + .to_string() + .contains("maximum stale age") + ); + assert!(repo.cache_dir.join(FRESHNESS_FILE).exists()); + cache.shutdown().await; + } + + #[test] + fn repository_identity_redacts_transport_secrets() { + let mut repo = RepoRef { + name: "repo.git".into(), + upstream_url: "https://user:secret@example.invalid/repo.git?token=secret#fragment" + .into(), + cache_dir: "/unused".into(), + }; + let identity = repository_identity(&repo); + repo.upstream_url = "https://example.invalid/repo.git".into(); + assert_eq!(identity, repository_identity(&repo)); + assert!(!identity.contains("secret")); + } + + #[tokio::test] + async fn refresh_uses_configured_upstream_after_identity_changes() { + let (dir, mut repo, cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let origin = dir.path().join("new-origin.git"); + fixture_git( + dir.path(), + &[ + "clone", + "--mirror", + &repo.upstream_url, + origin.to_str().unwrap(), + ], + ); + let tree = fixture_git(&origin, &["rev-parse", "HEAD^{tree}"]); + let head = fixture_git( + &origin, + &[ + "commit-tree", + tree.trim(), + "-p", + "HEAD", + "-m", + "new upstream", + ], + ); + fixture_git(&origin, &["update-ref", "refs/heads/main", head.trim()]); + repo.upstream_url = origin.to_string_lossy().into_owned(); + assert!(matches!( + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(), + CacheOutcome::Fetched + )); + assert_eq!( + fixture_git(&repo.cache_dir, &["rev-parse", "HEAD"]).trim(), + head.trim() + ); + assert!(read_freshness(&repo).await.is_ok()); + let next = fixture_git( + &origin, + &[ + "commit-tree", + tree.trim(), + "-p", + head.trim(), + "-m", + "sha upstream", + ], + ); + cache + .ensure_wanted_oids(&repo, &pkt_line(&format!("want {}\n", next.trim())), None) + .await + .unwrap(); + assert_eq!( + fixture_git( + &repo.cache_dir, + &["rev-parse", &format!("{PROXY_WANTS_NS}/{}", next.trim())] + ) + .trim(), + next.trim() + ); + } + + #[tokio::test] + async fn upload_admission_rejects_unknown_or_changed_identity_after_prepare() { + for invalid in ["missing", "directory", "repository"] { + let (_dir, repo, cache) = freshness_fixture(); + let cache = cache.with_request_limit(1); + let (_, lease) = cache + .prepare(repo.clone(), true, Bytes::new(), None) + .await + .unwrap(); + let marker = repo.cache_dir.join(FRESHNESS_FILE); + if invalid == "missing" { + tokio::fs::remove_file(marker).await.unwrap(); + } else { + let mut record: serde_json::Value = + serde_json::from_slice(&tokio::fs::read(&marker).await.unwrap()).unwrap(); + record[invalid] = serde_json::json!("changed"); + tokio::fs::write(marker, record.to_string()).await.unwrap(); + } + let result = cache + .upload_pack_rpc_guarded(&repo, None, Bytes::new(), lease) + .await; + assert_eq!( + result.err().unwrap().to_string(), + "mirror exceeds maximum stale age" + ); + assert_eq!(cache.upload_packs.as_ref().unwrap().available_permits(), 1); + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 1 + ); + assert!(cache.slot(&repo.name).await.lifecycle.try_write().is_ok()); + cache.shutdown().await; + } + } + + #[tokio::test] + async fn queued_upload_rechecks_age_before_spawning_and_releases_leases() { + let (_dir, repo, cache) = freshness_fixture(); + let mut cache = cache.with_request_limit(1); + let (_, lease) = cache + .prepare(repo.clone(), true, Bytes::new(), None) + .await + .unwrap(); + cache.cfg.git_binary = "missing-upload-pack-binary".into(); + let blocked = cache + .upload_packs + .as_ref() + .unwrap() + .clone() + .acquire_owned() + .await + .unwrap(); + let mut upload = Box::pin(cache.upload_pack_rpc_guarded(&repo, None, Bytes::new(), lease)); + assert!(matches!( + std::future::poll_fn(|cx| Poll::Ready(upload.as_mut().poll(cx))).await, + Poll::Pending + )); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + drop(blocked); + assert_eq!( + upload.await.err().unwrap().to_string(), + "mirror exceeds maximum stale age" + ); + assert_eq!(cache.upload_packs.as_ref().unwrap().available_permits(), 1); + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 1 + ); + assert!(cache.slot(&repo.name).await.lifecycle.try_write().is_ok()); + } + + #[tokio::test] + async fn admitted_pack_can_finish_after_its_refs_expire() { + let (_dir, repo, cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let oid = fixture_git(&repo.cache_dir, &["rev-parse", "HEAD"]); + let mut request = pkt_line(&format!("want {}\n", oid.trim())); + request.extend_from_slice(b"0000"); + request.extend(pkt_line("done\n")); + let stream = cache + .upload_pack_rpc(&repo, None, Bytes::from(request)) + .await + .unwrap(); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let pack = axum::body::to_bytes(axum::body::Body::from_stream(stream), usize::MAX) + .await + .unwrap(); + assert!(pack.windows(4).any(|bytes| bytes == b"PACK")); + cache.shutdown().await; + } + + #[cfg(unix)] + #[tokio::test] + async fn slow_upstream_does_not_take_local_capacity_or_lose_client_credentials() { + let (dir, repo, mut cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let ready = dir.path().join("ready"); + let gate = dir.path().join("gate"); + let header = dir.path().join("header"); + for fifo in [&ready, &gate] { + assert!( + std::process::Command::new("mkfifo") + .arg(fifo) + .status() + .unwrap() + .success() + ); + } + cache.cfg.git_binary = fixture_script( + dir.path(), + &format!( + "#!/bin/sh\nfor arg do\n if [ \"$arg\" = clone ]; then\n printf '%s' \"$GIT_CONFIG_VALUE_2\" > '{}'\n echo ready > '{}'\n read gate < '{}'\n fi\ndone\nexec git \"$@\"\n", + header.display(), + ready.display(), + gate.display() + ), + ); + let cache = cache.with_request_limit(1); + let mut cold = repo.clone(); + cold.name = "cold.git".into(); + cold.cache_dir = dir.path().join("cold.git"); + let request = tokio::spawn({ + let cache = cache.clone(); + let cold = cold.clone(); + async move { + cache + .prepare( + cold, + true, + Bytes::new(), + Some("Authorization: Bearer fixture-client"), + ) + .await + } + }); + assert_eq!( + tokio::fs::read_to_string(ready).await.unwrap().trim(), + "ready" + ); + request.abort(); + assert!(request.await.err().unwrap().is_cancelled()); + let busy = cache + .prepare(cold.clone(), true, Bytes::new(), None) + .await + .err() + .unwrap(); + assert!(busy.is::()); + let (_, lease) = cache + .prepare(repo.clone(), true, Bytes::new(), None) + .await + .unwrap(); + assert!( + !cache + .advertise_refs_guarded(&repo, None, lease) + .await + .unwrap() + .is_empty() + ); + let metrics = Arc::new(Metrics::new()); + let cache_root = dir.path().join("cache"); + let upstream_base = dir.path().join("upstream").to_string_lossy().into_owned(); + let state = crate::server::AppState { + cache: Arc::new(cache.clone()), + lfs: Arc::new(crate::lfs::Lfs::new( + crate::lfs::LfsConfig { + upstream_base: upstream_base.clone(), + cache_root: cache_root.clone(), + upstream_auth_header: None, + }, + None, + )), + upstream_base, + cache_root, + auth: Arc::new(crate::auth::Auth::anonymous()), + upstream_credential: crate::config::UpstreamCredential::Static, + max_decoded_body: 1024, + max_concurrent: 1, + metrics, + }; + use tower::ServiceExt; + let blocked_body = cache.body_permit().unwrap(); + let app = crate::server::router(state); + let healthy = app + .clone() + .oneshot( + axum::http::Request::get("/repo.git/info/refs?service=git-upload-pack") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(healthy.status(), axum::http::StatusCode::OK); + let overloaded = app + .oneshot( + axum::http::Request::post("/repo.git/git-upload-pack") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + overloaded.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE + ); + drop(blocked_body); + assert_eq!(cache.upstream.available_permits(), 0); + assert_eq!( + tokio::fs::read_to_string(header).await.unwrap(), + "Authorization: Bearer fixture-client" + ); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + cache.shutdown().await; + assert_eq!(cache.upstream.available_permits(), 1); + assert!(cold.cache_dir.join("HEAD").exists()); + assert!(cache.slot(&cold.name).await.lifecycle.try_write().is_ok()); + } + + #[tokio::test] + async fn strict_mode_does_not_write_freshness_records() { + let (_dir, repo, cache) = freshness_fixture(); + let cache = cache.with_stale(Duration::ZERO, 1); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + assert!(!repo.cache_dir.join(FRESHNESS_FILE).exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn background_refresh_keeps_the_triggering_client_credential() { + let (dir, repo, mut cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + cache.cfg.fetch_ttl = Duration::ZERO; + cache + .slot(&repo.name) + .await + .refresh + .lock() + .await + .next_attempt = None; + let header = dir.path().join("header"); + cache.cfg.git_binary = fixture_script( + dir.path(), + &format!( + "#!/bin/sh\nprintf '%s' \"$GIT_CONFIG_VALUE_2\" > '{}'\nexec git \"$@\"\n", + header.display() + ), + ); + let credential = String::from("Authorization: Bearer background-fixture"); + assert!(matches!( + cache + .ensure_fresh(&repo, true, &[], Some(&credential)) + .await + .unwrap(), + CacheOutcome::Cached + )); + drop(credential); + cache.shutdown().await; + assert_eq!( + tokio::fs::read_to_string(header).await.unwrap(), + "Authorization: Bearer background-fixture" + ); + let record = tokio::fs::read_to_string(repo.cache_dir.join(FRESHNESS_FILE)) + .await + .unwrap(); + assert!(!record.contains("background-fixture")); + } + + #[cfg(unix)] + #[tokio::test] + async fn stale_preparation_preserves_typed_upstream_auth_failure() { + let (dir, repo, mut cache) = freshness_fixture(); + cache.cfg.git_binary = fixture_script( + dir.path(), + "#!/bin/sh\necho 'fatal: Authentication failed' >&2\nexit 1\n", + ); + let error = cache + .prepare( + repo, + true, + Bytes::new(), + Some("Authorization: Bearer rejected"), + ) + .await + .err() + .unwrap(); + assert!(is_auth_failure(&error)); + assert!(!error.is::()); + cache.shutdown().await; + } + + #[cfg(unix)] + #[tokio::test] + async fn stale_upload_holds_local_capacity_until_cancelled_child_is_reaped() { + let (dir, repo, mut cache) = freshness_fixture(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + cache.cfg.git_binary = fixture_script(dir.path(), "#!/bin/sh\nexec sleep 300\n"); + let cache = cache.with_request_limit(1); + let (_, lease) = cache + .prepare(repo.clone(), false, Bytes::new(), None) + .await + .unwrap(); + let stream = cache + .upload_pack_rpc_guarded(&repo, None, Bytes::new(), lease) + .await + .unwrap(); + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 0 + ); + assert!(cache.slot(&repo.name).await.lifecycle.try_write().is_err()); + drop(stream); + tokio::time::timeout(Duration::from_secs(5), cache.shutdown()) + .await + .unwrap(); + assert_eq!( + cache.request_setups.as_ref().unwrap().available_permits(), + 1 + ); + assert_eq!(cache.upload_packs.as_ref().unwrap().available_permits(), 1); + assert!(cache.slot(&repo.name).await.lifecycle.try_write().is_ok()); + } + + fn freshness_fixture() -> (tempfile::TempDir, RepoRef, GitCache) { + let dir = tempfile::tempdir().unwrap(); + let upstream = dir.path().join("upstream"); + std::fs::create_dir(&upstream).unwrap(); + let origin = upstream.join("repo.git"); + fixture_git( + dir.path(), + &[ + "init", + "--bare", + "-q", + "-b", + "main", + origin.to_str().unwrap(), + ], + ); + let tree = fixture_git(&origin, &["hash-object", "-t", "tree", "--stdin", "-w"]); + let head = fixture_git(&origin, &["commit-tree", tree.trim(), "-m", "initial"]); + fixture_git(&origin, &["update-ref", "refs/heads/main", head.trim()]); + let repo = RepoRef { + name: "repo.git".into(), + upstream_url: origin.to_string_lossy().into_owned(), + cache_dir: dir.path().join("cache/repo.git"), + }; + let cache = GitCache::new( + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 10, + max_upload_packs: 1, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ) + .with_stale(Duration::from_secs(60), 1); + (dir, repo, cache) + } + + fn fixture_git(directory: &Path, args: &[&str]) -> String { + let output = std::process::Command::new("git") + .current_dir(directory) + .args(args) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + #[cfg(unix)] + fn fixture_script(directory: &Path, script: &str) -> String { + let path = directory.join("git-wrapper"); + write_script(&path, script); + path.to_string_lossy().into_owned() + } + + /// Write an executable script that can be spawned right away. While this + /// file is open for writing, a fork on another test thread inherits the + /// descriptor until it execs, and spawning the script fails with ETXTBSY. + /// Probe until the file is no longer busy; a script is guarded to exit + /// early under the probe. + fn write_script(path: &Path, script: &str) { + use std::os::unix::fs::PermissionsExt; + let (shebang, rest) = script.split_once('\n').unwrap(); + std::fs::write( + path, + format!("{shebang}\n[ -n \"$SCRIPT_PROBE\" ] && exit 0\n{rest}"), + ) + .unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + loop { + match std::process::Command::new(path) + .env("SCRIPT_PROBE", "1") + .status() + { + Err(e) if e.kind() == std::io::ErrorKind::ExecutableFileBusy => { + std::thread::sleep(Duration::from_millis(1)); + } + result => { + result.unwrap(); + return; + } + } + } + } + /// Collect an iterator of oids into a `HashSet` for order-insensitive assertions /// against `parse_wants`. fn want_set>(oids: I) -> HashSet { diff --git a/src/main.rs b/src/main.rs index e0ec343..b93f1af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -89,7 +89,14 @@ async fn main() -> Result<()> { ) }); - let cache = Arc::new(git::GitCache::new(git_cfg, metrics.clone(), index.clone())); + let cache = Arc::new( + git::GitCache::new(git_cfg, metrics.clone(), index.clone()) + .with_stale( + Duration::from_secs(cfg.max_stale_seconds), + cfg.max_background_fetches, + ) + .with_request_limit(cfg.max_concurrent_requests), + ); let upstream_base = cfg.upstream.trim_end_matches('/').to_string(); @@ -172,6 +179,7 @@ async fn main() -> Result<()> { let _ = shutdown_tx.send(true); let _ = handle.await; } + cache.shutdown().await; tracing::info!("shutdown complete"); Ok(()) diff --git a/src/server.rs b/src/server.rs index 7707f91..8c4ade7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -22,7 +22,7 @@ use tower::limit::GlobalConcurrencyLimitLayer; use crate::auth; use crate::config::UpstreamCredential; -use crate::git::{GitCache, is_auth_failure}; +use crate::git::{CacheBusy, GitCache, is_auth_failure}; use crate::lfs::{Lfs, Outcome}; use crate::metrics::{LfsResult, Metrics, RequestKind, Status}; use crate::repo; @@ -72,9 +72,11 @@ pub fn router(state: AppState) -> Router { // fallback as the merged fallback, while the observability routes above are // matched first and bypass it. The gate runs outside the limit, so a slow or // flooded token-validation endpoint cannot hold the permits serving needs; - // validation is bounded by its own `--auth-max-concurrent` instead. + // validation is bounded by its own `--auth-max-concurrent` instead. Stale + // mode isolates admission inside the cache so upstream work cannot consume + // the capacity needed to serve mature mirrors. let mut git = Router::new().fallback(handle_git); - if max_concurrent != 0 { + if max_concurrent != 0 && !state.cache.stale_enabled() { git = git.layer(GlobalConcurrencyLimitLayer::new(max_concurrent)); } let git = git @@ -266,6 +268,16 @@ async fn handle_git( Extension(route): Extension, req: Request, ) -> Response { + let admission = if st.cache.stale_enabled() + && matches!(route, Route::UploadPack { .. } | Route::LfsBatch { .. }) + { + match st.cache.body_permit() { + Ok(permit) => permit, + Err(_) => return err(StatusCode::SERVICE_UNAVAILABLE, "request admission full"), + } + } else { + None + }; let (parts, body) = req.into_parts(); let git_protocol = parts .headers @@ -288,6 +300,7 @@ async fn handle_git( ) } Route::InfoRefs { repo, smart: true } => { + drop(admission); info_refs(st, &repo, git_protocol.as_deref(), cred, &parts.headers).await } Route::UploadPack { repo } => { @@ -308,6 +321,7 @@ async fn handle_git( Ok(b) => b, Err(_) => return err(StatusCode::BAD_REQUEST, "failed to decode request body"), }; + drop(admission); upload_pack( st, &repo, @@ -324,6 +338,7 @@ async fn handle_git( Ok(b) => b, Err(_) => return err(StatusCode::BAD_REQUEST, "failed to read request body"), }; + drop(admission); lfs_batch(st, &repo, &parts.headers, body, cred).await } Route::LfsObject { repo, oid, size } => lfs_object(st, repo, oid, size, cred).await, @@ -489,9 +504,19 @@ async fn info_refs( // where the clone/fetch/TTL decision is made, so it is the per-operation // local-vs-upstream signal (upload-pack runs with `want_fetch=false` and is // almost always `Cached`, which would skew the ratio). - match st.cache.ensure_fresh(&repo, true, &[], cred).await { - Ok(outcome) => st.metrics.record_cache_outcome(outcome), + let guard = match st + .cache + .prepare(repo.clone(), true, Bytes::new(), cred) + .await + { + Ok((outcome, guard)) => { + st.metrics.record_cache_outcome(outcome); + guard + } Err(e) => { + if e.is::() { + return err(StatusCode::SERVICE_UNAVAILABLE, "request admission full"); + } // In `client` mode the credential is the client's own, so an upstream // rejection is the client's 401, not a 502; drop the stale verdict so // the next request re-probes (a token revoked within the TTL window). @@ -506,9 +531,13 @@ async fn info_refs( tracing::warn!(repo = %name, error = %e, "ensure_fresh failed"); return err(StatusCode::BAD_GATEWAY, "upstream fetch failed"); } - } + }; - match st.cache.advertise_refs(&repo, git_protocol).await { + match st + .cache + .advertise_refs_guarded(&repo, git_protocol, guard) + .await + { Ok(body) => { st.metrics .record_request(RequestKind::InfoRefs, Status::Ok, name); @@ -550,20 +579,34 @@ async fn upload_pack( // The preceding info/refs already refreshed; here just ensure the mirror is // present (a client could POST against a not-yet-cloned repo) and that any // want-by-SHA the body asks for is fetched and pinned before serving. - if let Err(e) = st.cache.ensure_fresh(&repo, false, &body, cred).await { - if st.upstream_credential == UpstreamCredential::Client && is_auth_failure(&e) { - st.auth.invalidate(headers, name); + let guard = match st + .cache + .prepare(repo.clone(), false, body.clone(), cred) + .await + { + Ok((_, guard)) => guard, + Err(e) => { + if e.is::() { + return err(StatusCode::SERVICE_UNAVAILABLE, "request admission full"); + } + if st.upstream_credential == UpstreamCredential::Client && is_auth_failure(&e) { + st.auth.invalidate(headers, name); + st.metrics + .record_request(RequestKind::UploadPack, Status::Unauthorized, "-"); + return unauthorized(&st.auth, "upstream rejected the credential"); + } st.metrics - .record_request(RequestKind::UploadPack, Status::Unauthorized, "-"); - return unauthorized(&st.auth, "upstream rejected the credential"); + .record_request(RequestKind::UploadPack, Status::UpstreamError, "-"); + tracing::warn!(repo = %name, error = %e, "ensure mirror exists failed"); + return err(StatusCode::BAD_GATEWAY, "upstream unavailable"); } - st.metrics - .record_request(RequestKind::UploadPack, Status::UpstreamError, "-"); - tracing::warn!(repo = %name, error = %e, "ensure mirror exists failed"); - return err(StatusCode::BAD_GATEWAY, "upstream unavailable"); - } + }; - match st.cache.upload_pack_rpc(&repo, git_protocol, body).await { + match st + .cache + .upload_pack_rpc_guarded(&repo, git_protocol, body, guard) + .await + { Ok(stream) => { st.metrics .record_request(RequestKind::UploadPack, Status::Ok, name); diff --git a/tests/e2e.rs b/tests/e2e.rs index 731bed3..9aa5f9b 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -505,6 +505,15 @@ async fn want_by_sha_fetch_failure_is_recorded_and_degrades_gracefully() { /// the reserved namespace never accumulates beyond the cap. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn want_by_sha_pins_are_capped_per_mirror() { + check_want_by_sha_pins_are_capped(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stale_mode_want_by_sha_pins_are_capped_per_mirror() { + check_want_by_sha_pins_are_capped(true).await; +} + +async fn check_want_by_sha_pins_are_capped(stale: bool) { let work = tempfile::tempdir().unwrap(); git(work.path(), &["init", "-q", "-b", "main", "."]); std::fs::write(work.path().join("README.md"), "base\n").unwrap(); @@ -568,7 +577,10 @@ async fn want_by_sha_pins_are_capped_per_mirror() { pack_window_memory: None, }; let state = AppState { - cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), + cache: Arc::new( + GitCache::new(cfg, metrics.clone(), None) + .with_stale(Duration::from_secs(if stale { 60 } else { 0 }), 1), + ), lfs: Arc::new(Lfs::new( LfsConfig { upstream_base: format!("file://{}", up.path().display()), @@ -1628,6 +1640,119 @@ async fn upload_pack_runs_with_pack_options() { assert!(rpc.contains("-c pack.windowMemory=64m"), "{rpc}"); } +#[cfg(unix)] +#[tokio::test(start_paused = true)] +async fn clone_completes_while_background_fetch_is_blocked() { + let work = tempfile::tempdir().unwrap(); + git(work.path(), &["init", "-q", "-b", "main", "."]); + std::fs::write(work.path().join("README.md"), "cached content\n").unwrap(); + git(work.path(), &["add", "."]); + git(work.path(), &["commit", "-q", "-m", "initial"]); + let up = tempfile::tempdir().unwrap(); + git( + work.path(), + &[ + "clone", + "-q", + "--mirror", + ".", + up.path().join("repo.git").to_str().unwrap(), + ], + ); + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("started"); + let gate = dir.path().join("gate"); + for fifo in [&marker, &gate] { + assert!(Command::new("mkfifo").arg(fifo).status().unwrap().success()); + } + let wrapper = dir.path().join("git-wrapper"); + helpers::write_script( + &wrapper, + &format!( + "#!/bin/sh\nfor arg do\n if [ \"$arg\" = fetch ]; then\n echo started > '{}'\n read gate < '{}'\n break\n fi\ndone\nexec git \"$@\"\n", + marker.display(), + gate.display() + ), + ); + let root = dir.path().join("cache"); + let upstream = format!("file://{}", up.path().display()); + let metrics = Arc::new(Metrics::new()); + let cache = Arc::new( + GitCache::new( + GitConfig { + git_binary: wrapper.to_string_lossy().into_owned(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(0), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + metrics.clone(), + None, + ) + .with_stale(Duration::from_secs(60), 1), + ); + let repo = resolve("repo.git", &upstream, &root).unwrap(); + cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let state = AppState { + cache: cache.clone(), + lfs: Arc::new(Lfs::new( + LfsConfig { + upstream_base: upstream.clone(), + cache_root: root.clone(), + upstream_auth_header: None, + }, + None, + )), + upstream_base: upstream, + cache_root: root, + auth: Arc::new(Auth::anonymous()), + upstream_credential: UpstreamCredential::Static, + max_decoded_body: 512 * 1024 * 1024, + max_concurrent: 8, + metrics, + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router(state)).await.unwrap() }); + tokio::time::advance(Duration::from_secs(2)).await; + let started = tokio::spawn(async move { tokio::fs::read_to_string(marker).await.unwrap() }); + let checkout = dir.path().join("checkout"); + let clone = tokio::spawn({ + let checkout = checkout.clone(); + async move { + tokio::process::Command::new("git") + .args(["clone", "--quiet", &format!("http://{addr}/repo.git")]) + .arg(checkout) + .output() + .await + .unwrap() + } + }); + assert_eq!(started.await.unwrap().trim(), "started"); + let output = clone.await.unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + std::fs::read_to_string(checkout.join("README.md")).unwrap(), + "cached content\n" + ); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + cache.shutdown().await; + server.abort(); +} + // --- Test helpers --------------------------------------------------------- /// Run a git command in `cwd` with a hermetic identity/config, asserting success. diff --git a/tests/helpers.rs b/tests/helpers.rs index 4d451ac..08f9893 100644 --- a/tests/helpers.rs +++ b/tests/helpers.rs @@ -21,6 +21,36 @@ use git_cache_proxy::auth::AuthConfig; use git_cache_proxy::config::AuthProvider; use tokio::sync::{Mutex, Notify}; +/// Write an executable script that can be spawned right away. While this file is +/// open for writing, a fork on another test thread inherits the descriptor until +/// it execs, and spawning the script fails with ETXTBSY. Probe until the file is +/// no longer busy; the script is guarded to exit early under the probe. +#[cfg(unix)] +pub fn write_script(path: &std::path::Path, script: &str) { + use std::os::unix::fs::PermissionsExt; + let (shebang, rest) = script.split_once('\n').unwrap(); + std::fs::write( + path, + format!("{shebang}\n[ -n \"$SCRIPT_PROBE\" ] && exit 0\n{rest}"), + ) + .unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + loop { + match std::process::Command::new(path) + .env("SCRIPT_PROBE", "1") + .status() + { + Err(e) if e.kind() == std::io::ErrorKind::ExecutableFileBusy => { + std::thread::sleep(Duration::from_millis(1)); + } + result => { + result.unwrap(); + return; + } + } + } +} + /// The signing key the test replicas share, long enough for HMAC-SHA256. pub const SIGNING_KEY: &str = "0123456789abcdef0123456789abcdef"; diff --git a/tests/http.rs b/tests/http.rs index 783da58..9d984a1 100644 --- a/tests/http.rs +++ b/tests/http.rs @@ -18,6 +18,8 @@ use git_cache_proxy::metrics::Metrics; use git_cache_proxy::server::{AppState, router}; use tower::ServiceExt; // for `oneshot` +mod helpers; + #[tokio::test] async fn health_endpoints_return_ok() { assert_eq!( @@ -328,6 +330,78 @@ async fn metrics_endpoint_renders_recorded_series() { ); } +#[cfg(unix)] +#[tokio::test] +async fn local_git_failures_return_internal_server_error() { + let dir = tempfile::tempdir().unwrap(); + let wrapper = dir.path().join("git"); + helpers::write_script( + &wrapper, + "#!/bin/sh\nfor arg do\n if [ \"$arg\" = fetch ]; then exit 0; fi\ndone\nexit 1\n", + ); + let mut st = state(anon_auth()); + st.cache_root = dir.path().join("cache"); + let mirror = st.cache_root.join("repo.git"); + std::fs::create_dir_all(&mirror).unwrap(); + std::fs::write(mirror.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + st.cache = Arc::new(GitCache::new( + GitConfig { + git_binary: wrapper.to_string_lossy().into_owned(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + st.metrics.clone(), + None, + )); + let repo = + git_cache_proxy::repo::resolve("repo.git", &st.upstream_base, &st.cache_root).unwrap(); + // Establish freshness so the HTTP requests exercise only serving failures. + st.cache.ensure_fresh(&repo, true, &[], None).await.unwrap(); + let response = get(st.clone(), "/repo.git/info/refs?service=git-upload-pack").await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(body_string(response).await, "advertise-refs failed\n"); + + assert!(st.cache.advertise_refs(&repo, None).await.is_err()); + std::fs::remove_file(&wrapper).unwrap(); + assert!(st.cache.advertise_refs(&repo, None).await.is_err()); + assert!( + st.cache + .upload_pack_rpc(&repo, None, axum::body::Bytes::new()) + .await + .is_err() + ); + let response = router(st.clone()) + .oneshot( + Request::post("/repo.git/git-upload-pack") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(body_string(response).await, "upload-pack failed\n"); + st.cache.shutdown().await; +} + +#[test] +fn auth_debug_redacts_shared_credentials() { + let auth = Auth::shared("private-serve-token"); + let debug = format!("{auth:?}"); + assert!(debug.contains("Auth")); + assert!(!debug.contains("private-serve-token")); +} + +#[test] +fn default_metrics_exports_registered_counters() { + let metrics = Metrics::default(); + assert!(metrics.gather().contains("gitcacheproxy_")); +} + // --- Test helpers --------------------------------------------------------- /// An anonymous `Auth`: no client credential is checked. @@ -389,3 +463,134 @@ async fn body_string(resp: axum::http::Response) -> String { .unwrap(); String::from_utf8_lossy(&bytes).into_owned() } + +#[cfg(unix)] +#[tokio::test(start_paused = true)] +async fn cancelled_advertisements_keep_request_cap_until_exit() { + check_cancelled_advertisements(false).await; + check_cancelled_advertisements(true).await; +} + +#[cfg(unix)] +async fn check_cancelled_advertisements(stale: bool) { + use tokio::io::AsyncWriteExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("cache"); + let mirror = root.join("repo.git"); + std::fs::create_dir_all(&mirror).unwrap(); + std::fs::write(mirror.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + let marker = dir.path().join("started"); + let gate = dir.path().join("gate"); + let log = dir.path().join("log"); + for fifo in [&marker, &gate] { + assert!( + std::process::Command::new("mkfifo") + .arg(fifo) + .status() + .unwrap() + .success() + ); + } + let wrapper = dir.path().join("git"); + helpers::write_script( + &wrapper, + &format!( + "#!/bin/sh\nfor arg do\n if [ \"$arg\" = --advertise-refs ]; then\n echo started >> '{}'\n echo started > '{}'\n read gate < '{}'\n fi\ndone\nexit 0\n", + log.display(), + marker.display(), + gate.display() + ), + ); + let metrics = Arc::new(Metrics::new()); + let cfg = GitConfig { + git_binary: wrapper.to_string_lossy().into_owned(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }; + let cache = Arc::new( + GitCache::new(cfg, metrics.clone(), None) + .with_stale(Duration::from_secs(if stale { 60 } else { 0 }), 1) + .with_request_limit(1), + ); + let mut st = state(anon_auth()); + st.cache = cache.clone(); + st.cache_root = root; + st.metrics = metrics; + st.max_concurrent = 1; + let app = router(st); + + for attempt in 1..=3 { + if stale && attempt > 1 { + let rejected = app + .clone() + .oneshot( + Request::get("/repo.git/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(rejected.status(), StatusCode::SERVICE_UNAVAILABLE); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(&gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + cache.shutdown().await; + } + let started = tokio::spawn({ + let marker = marker.clone(); + async move { tokio::fs::read_to_string(marker).await.unwrap() } + }); + let request = tokio::spawn({ + let app = app.clone(); + async move { + app.oneshot( + Request::get("/repo.git/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(), + ) + .await + } + }); + if !stale && attempt > 1 { + tokio::time::advance(Duration::from_secs(1)).await; + assert!(!request.is_finished()); + assert_eq!( + std::fs::read_to_string(&log).unwrap().lines().count(), + attempt - 1 + ); + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(&gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + } + assert_eq!(started.await.unwrap().trim(), "started"); + assert_eq!( + std::fs::read_to_string(&log).unwrap().lines().count(), + attempt + ); + request.abort(); + let _ = request.await; + } + let mut writer = tokio::fs::OpenOptions::new() + .write(true) + .open(&gate) + .await + .unwrap(); + writer.write_all(b"go\n").await.unwrap(); + drop(writer); + cache.evict("repo.git", &mirror).await.unwrap(); + cache.shutdown().await; + assert!(!mirror.exists()); +} diff --git a/tests/lfs.rs b/tests/lfs.rs index a03ec48..6e276ef 100644 --- a/tests/lfs.rs +++ b/tests/lfs.rs @@ -292,6 +292,59 @@ async fn batch_upstream_failure_returns_bad_gateway() { )); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn object_download_does_not_take_request_body_admission() { + // Stale mode bounds request bodies; an object download has none, so it must + // not be refused while that lane is full. + let (addr, _batches) = spawn_upstream(Mode::Ok).await; + let cache = tempfile::tempdir().unwrap(); + let metrics = Arc::new(Metrics::new()); + let mut state = proxy_state(addr, cache.path(), metrics.clone()); + let git = GitCache::new( + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + metrics, + None, + ) + .with_stale(Duration::from_secs(60), 1) + .with_request_limit(1); + let _full = git.body_permit().unwrap(); + state.cache = Arc::new(git); + let oid = oid_of(PAYLOAD); + + let resp = router(state.clone()) + .oneshot( + Request::get(format!( + "/{REPO}/info/lfs/objects/{oid}?size={}", + PAYLOAD.len() + )) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let resp = router(state) + .oneshot( + Request::post(format!("/{REPO}/info/lfs/objects/batch")) + .header(header::HOST, "proxy.test:8080") + .body(Body::from(batch_body(&oid, PAYLOAD.len()))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); +} + // --- test helpers -------------------------------------------------------- const REPO: &str = "group/repo.git";