From a463b51d64fa02b67501da57e31069d3eca30d7e Mon Sep 17 00:00:00 2001 From: ztzhou Date: Sun, 15 Nov 2026 23:39:31 +0800 Subject: [PATCH] feat(git): refresh recently used mirrors in the background Assisted-by: OpenAI:Codex --- README.md | 7 +- docs/architecture.md | 37 +- docs/configuration.md | 39 ++ src/config.rs | 57 +- src/evict.rs | 182 +++++- src/git.rs | 1282 ++++++++++++++++++++++++++++++++++++++++- src/main.rs | 10 + tests/refresh.rs | 114 ++++ 8 files changed, 1707 insertions(+), 21 deletions(-) create mode 100644 tests/refresh.rs diff --git a/README.md b/README.md index fdbe124..7b49795 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,9 @@ API proxied; objects stored content-addressed and integrity-checked). Early, single-maintainer software: no independent review or wide deployment yet. Pin a version and try it against your own setup before relying on it. -Not yet implemented: - -- A background/scheduled refresh (today every `info/refs` triggers an on-demand, - TTL-coalesced fetch). +Optional background refresh of recently used mirrors is available with +`--refresh-interval-seconds` (default `0`, disabled; minimum enabled value +`3600`). See [Configuration](docs/configuration.md#periodic-refresh). **Non-goal: a git-free image.** The `git` binary is a permanent runtime dependency: the proxy serves clients by running `git upload-pack`, and no diff --git a/docs/architecture.md b/docs/architecture.md index 48311e9..675cb29 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,8 +13,9 @@ works internally, and a throughput benchmark. For usage see the replicating every repo, and it can lag the exact commit CI just pushed. This proxy is **lazy and pull-only**: it caches on first request, and on every -request runs an incremental `git fetch` before serving - so a client always gets -the ref it asked for, and nothing is ever pushed or replicated to the origin. +request checks freshness before serving, using a TTL-coalesced incremental +`git fetch`. Optional background refresh updates recently used existing mirrors; +it never discovers or clones repositories. Nothing is pushed to the origin. That pull-only property is what makes it viable against a sensitive origin. ## How it compares @@ -34,8 +35,9 @@ That pull-only property is what makes it viable against a sensitive origin. mirror can lag its refresh interval (`lag`); `--reference` is only as fresh as its local seed (`seed`). 4. Only new objects cross the WAN per request; the bulk history is served from the local mirror. -5. Never writes to or proactively replicates from the origin - it only ever pulls what a - client requested. +5. Never writes to the origin. With background refresh disabled (the default), + upstream pulls serve client requests only. Optional refresh pulls recently used + existing mirrors without discovering or cloning repositories. 6. Works against any unmodified Git smart-HTTP origin. GitLab Geo needs GitLab on both ends. ## How it works @@ -70,9 +72,10 @@ Retries stay inside the existing per-repository lock. Each failed clone's stagin directory is removed before another attempt; an existing fetch mirror is kept. Only success updates the freshness timestamp. After retries are exhausted the request still fails with 502; stale refs are not served as a fallback. The retry -budget limits attempts and adds at most seven seconds of backoff; it does not -impose a new transfer timeout on large repositories. LFS and local upload-pack -operations are outside this retry policy. +budget limits attempts and adds at most seven seconds of backoff. Each upstream +Git operation also has one `--upstream-timeout-seconds` budget (default `3600`) +across all attempts and backoff, including background refresh. LFS and local +upload-pack operations are outside this retry policy and operation budget. If a client `want`s a commit by SHA that the mirror never captured - typically a GitHub pull-request merge commit, which lives only under the unadvertised @@ -105,6 +108,26 @@ headers are sent terminate the response body. On Unix targets with remaining group members are stopped before the leader is reaped. Other targets wait for request completion and stdout EOF before observing exit. +### Background refresh + +Optional background refresh keeps recently used existing mirrors warm without +repository discovery or cold clones. See [Periodic refresh configuration](./configuration.md#periodic-refresh) +for flags, defaults, scheduling, and failure behavior. + +Refresh shares the per-repository fetch lock and TTL with requests and eviction. +Client recency controls eligibility so periodic work cannot keep a mirror active +forever. Background fetches do not promote the eviction LRU; successful fetches +mark tracked entries for deferred size measurement and capacity enforcement by +the existing evictor. Entries already selected for eviction are not recreated, +and an eviction cancels a refresh fetch running for that mirror instead of +waiting for it. + +Round deadlines stop admission while running fetches drain, bounding schedule +overlap without discarding work already in progress. Shutdown instead cancels +upstream operations and waits for child cleanup. Refresh uses only the static +server credential or anonymous access, so scheduling cannot retain or replay +transient client credentials. + ### git-LFS LFS objects use a different HTTP API from the git protocol, so they are cached diff --git a/docs/configuration.md b/docs/configuration.md index e730c3a..62a90e2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -19,7 +19,11 @@ Every flag has an environment-variable equivalent. | `--auth-ttl-seconds` | `GITCACHEPROXY_AUTH_TTL_SECONDS` | `0` | Reuse a validation verdict per `(credential, repo)` for this many seconds, so the requests of one clone, or repeated submodule fetches, validate once (`0` = validate every request) | | `--auth-max-concurrent` | `GITCACHEPROXY_AUTH_MAX_CONCURRENT` | `32` | Max token-validation requests in flight; a check waits up to `--auth-timeout-seconds` for a slot, then fails with 503 (`0` = unlimited) | | `--auth-allow-http` | `GITCACHEPROXY_AUTH_ALLOW_HTTP` | `false` | Provider modes: accept a plain `http://` validation endpoint, which sends client credentials in cleartext | +| `--upstream-timeout-seconds` | `GITCACHEPROXY_UPSTREAM_TIMEOUT_SECONDS` | `3600` | Total budget for each upstream Git operation, including attempts and retry backoff; at least 1 | | `--fetch-ttl-seconds` | `GITCACHEPROXY_FETCH_TTL_SECONDS` | `10` | Skip upstream fetch if the last one started within this window (`0` = always fetch) | +| `--refresh-interval-seconds` | `GITCACHEPROXY_REFRESH_INTERVAL_SECONDS` | `0` | Background refresh round spacing; 0 disables it, otherwise at least 3600 | +| `--refresh-active-window-seconds` | `GITCACHEPROXY_REFRESH_ACTIVE_WINDOW_SECONDS` | `604800` | Only refresh mirrors requested within this window; at least 1 when enabled | +| `--max-concurrent-refreshes` | `GITCACHEPROXY_MAX_CONCURRENT_REFRESHES` | `4` | Maximum background fetches in flight; at least 1 when enabled | | `--max-concurrent-requests` | `GITCACHEPROXY_MAX_CONCURRENT_REQUESTS` | `64` | Max concurrent in-flight requests; excess queue (`0` = unlimited) | | `--max-concurrent-upload-packs` | `GITCACHEPROXY_MAX_CONCURRENT_UPLOAD_PACKS` | `0` | Max concurrent `git upload-pack` processes, held until each exits; bounds `pack-objects` memory. Clones waiting for a slot hold a `--max-concurrent-requests` slot (`0` = unlimited) | | `--max-decoded-body-mb` | `GITCACHEPROXY_MAX_DECODED_BODY_MB` | `512` | Cap on a decoded upload-pack request body, in MiB (bounds memory / gzip bombs) | @@ -33,6 +37,41 @@ Every flag has an environment-variable equivalent. Endpoints: `/healthz`, `/readyz`, `/metrics` (see [Metrics](./metrics.md)). +## Periodic refresh + +Refresh is disabled by default. When enabled, the first round starts after one +interval. Each round selects recently requested mirrors, most recent first. +Usage is kept only in memory and is reset on restart. Refresh never clones a +missing mirror, records client use, or promotes a mirror in the eviction LRU. +Successful background fetches schedule deferred size measurement and capacity +enforcement without changing LRU order. A TTL hit does not schedule measurement. +The active-window and concurrency settings have no effect when disabled. +The interval and its maximum backoff must fit in a monotonic deadline. +Disabled refresh neither records refresh usage nor locks refresh state on +requests. Library enabling counts only requests admitted after enabling. +Usage is recorded after successful cache preparation (ensure_fresh returning +Ok), including TTL hits, not after the full HTTP response or pack stream. +Failed, cancelled, and auth-rejected preparation leaves prior successful use +unchanged. Concurrent completions cannot move the timestamp backwards. + +Rounds start every interval, measured from the previous round's start. At the +round deadline no new repository starts; running fetches finish. The next +round waits until both that deadline and all previous fetches have completed. +A deadline skip is not a fetch failure. Refresh shares the request fetch lock +and TTL. A request arriving during refresh waits for that fetch. Shutdown +cancels upstream operations and reaps child processes; library users must await +both request owners and the refresh handle. + +Refresh uses only the static upstream auth header, or anonymous access when +none is configured. It never stores or reuses client credentials. An upstream +auth rejection blocks further refresh attempts for that mirror until a fetch +using the same server credential succeeds, or the process restarts. A client +credential success clears ordinary failure backoff but preserves this mark. +The first ordinary failure is eligible in the next scheduled round. Repeated +failures wait interval * 2^(failures - 1) after failure completion, capped at +32 intervals. Any successful upstream fetch clears +ordinary failure and backoff state. Request serving ignores refresh backoff. + ## Reusing a cache after changing upstream Mirrors are keyed by repo path under `--cache-root`, not by upstream URL. diff --git a/src/config.rs b/src/config.rs index 9df1a3f..b6f30da 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,6 +3,9 @@ use std::fmt; use std::path::PathBuf; +use std::time::Duration; + +use tokio::time::Instant; use anyhow::{Result, bail}; use clap::{Parser, ValueEnum}; @@ -221,6 +224,30 @@ pub struct Config { #[arg(long, env = "GITCACHEPROXY_FETCH_TTL_SECONDS", default_value_t = 10)] pub fetch_ttl_seconds: u64, + /// Pause between refresh round starts; 0 disables background refresh. + #[arg( + long, + env = "GITCACHEPROXY_REFRESH_INTERVAL_SECONDS", + default_value_t = 0 + )] + pub refresh_interval_seconds: u64, + + /// Refresh only mirrors requested within this many seconds. + #[arg( + long, + env = "GITCACHEPROXY_REFRESH_ACTIVE_WINDOW_SECONDS", + default_value_t = 604800 + )] + pub refresh_active_window_seconds: u64, + + /// Maximum background fetches in flight when refresh is enabled. + #[arg( + long, + env = "GITCACHEPROXY_MAX_CONCURRENT_REFRESHES", + default_value_t = 4 + )] + pub max_concurrent_refreshes: usize, + /// Maximum number of requests handled concurrently, shared across all /// connections. Excess requests queue until a slot frees. This bounds the /// concurrent upstream clone/fetch work a burst can trigger (the slot is held @@ -310,11 +337,39 @@ pub struct Config { fn parse_upstream_timeout(value: &str) -> Result { let seconds = value.parse::().map_err(|e| e.to_string())?; - crate::git::validate_upstream_timeout(std::time::Duration::from_secs(seconds)) + crate::git::validate_upstream_timeout(Duration::from_secs(seconds)) .map_err(|e| e.to_string())?; Ok(seconds) } +impl Config { + /// Validate refresh options before starting background work. + pub fn validate_refresh(&self) -> Result<()> { + if self.refresh_interval_seconds != 0 { + if self.refresh_interval_seconds < 3600 { + bail!("--refresh-interval-seconds must be 0 or at least 3600"); + } + let interval = Duration::from_secs(self.refresh_interval_seconds); + if interval + .checked_mul(32) + .and_then(|delay| Instant::now().checked_add(delay)) + .is_none() + { + bail!( + "--refresh-interval-seconds and its maximum backoff must fit in a monotonic deadline" + ); + } + if self.refresh_active_window_seconds == 0 { + bail!("--refresh-active-window-seconds must be at least 1 when refresh is enabled"); + } + if self.max_concurrent_refreshes == 0 { + bail!("--max-concurrent-refreshes must be at least 1"); + } + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/evict.rs b/src/evict.rs index 239de85..a444b70 100644 --- a/src/evict.rs +++ b/src/evict.rs @@ -5,7 +5,8 @@ //! mirror's size and access order in memory. The request path only ever does O(1) //! bookkeeping against it - never disk IO: //! - `touch` on every request (a served cache hit counts as use), -//! - `mark_changed` after a clone/fetch, flagging the mirror for (re)measurement. +//! - `mark_changed` after a request clone/fetch, promoting and remeasuring, +//! - `mark_dirty` after a background fetch, remeasuring without promotion. //! //! All disk work - measuring a changed mirror's size, and evicting mirrors - runs //! in the background [`run`] task, off the critical path, so a client's clone/fetch @@ -135,6 +136,19 @@ impl CacheIndex { self.work.notify_one(); } + /// Schedule deferred measurement without promoting or recreating an entry. + /// A refresh can finish after eviction has already removed its index entry. + pub fn mark_dirty(&self, name: &str) { + { + let mut inner = self.lock(); + if !inner.cache.contains(name) { + return; + } + inner.dirty.insert(name.to_string()); + } + self.work.notify_one(); + } + /// On-disk path of a tracked mirror. pub fn cache_dir(&self, name: &str) -> PathBuf { self.cache_root.join(name) @@ -461,6 +475,142 @@ mod tests { use crate::git::{GitCache, GitConfig}; + #[cfg(unix)] + #[tokio::test] + async fn refresh_growth_updates_totals_and_metrics_without_lru_promotion() { + let tmp = tempfile::tempdir().unwrap(); + let (cache, idx, repo) = refresh_growth_fixture( + tmp.path(), + u64::MAX, + "printf '%01024d' 0 > objects/pack.data\n", + ); + let handle = cache + .start_refresh(Duration::from_secs(3600), Duration::from_secs(604800), 1) + .unwrap(); + cache.ensure_fresh(&repo, false, b"", None).await.unwrap(); + idx.touch("b.git"); + let before = idx.totals().0; + cache + .refresh_repo( + &repo, + Duration::from_secs(3600), + Duration::from_secs(604800), + tokio::time::Instant::now() + Duration::from_secs(3600), + ) + .await; + tokio::time::timeout(Duration::from_secs(5), idx.work.notified()) + .await + .unwrap(); + assert_eq!(idx.lock().cache.peek_lru().unwrap().0, "a.git"); + maintain(&cache, &idx).await; + let actual = measure(&repo.cache_dir).0 + measure(&tmp.path().join("b.git")).0; + assert!(actual > before); + assert_eq!(idx.totals(), (actual, 2)); + assert!( + idx.metrics + .gather() + .contains(&format!("gitcacheproxy_cache_bytes {actual}\n")) + ); + assert_eq!(idx.lock().cache.peek_lru().unwrap().0, "a.git"); + cache.ensure_fresh(&repo, true, b"", None).await.unwrap(); + assert!( + idx.take_dirty().is_empty(), + "a TTL hit must not schedule measurement" + ); + cache.shutdown(); + handle.await.unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_only_growth_enforces_the_cap_and_evicts_the_same_lru() { + let tmp = tempfile::tempdir().unwrap(); + let (cache, idx, repo) = + refresh_growth_fixture(tmp.path(), 256, "printf '%01024d' 0 > objects/pack.data\n"); + let handle = cache + .start_refresh(Duration::from_secs(3600), Duration::from_secs(604800), 1) + .unwrap(); + cache.ensure_fresh(&repo, false, b"", None).await.unwrap(); + idx.touch("b.git"); + cache + .refresh_repo( + &repo, + Duration::from_secs(3600), + Duration::from_secs(604800), + tokio::time::Instant::now() + Duration::from_secs(3600), + ) + .await; + assert_eq!(idx.lock().cache.peek_lru().unwrap().0, "a.git"); + maintain(&cache, &idx).await; + assert!(!repo.cache_dir.exists()); + let remaining = measure(&tmp.path().join("b.git")).0; + assert_eq!(idx.totals(), (remaining, 1)); + let metrics = idx.metrics.gather(); + assert!(metrics.contains(&format!("gitcacheproxy_cache_bytes {remaining}\n"))); + assert!(metrics.contains("gitcacheproxy_evictions_total 1\n")); + assert!(remaining <= 256); + cache.shutdown(); + handle.await.unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn eviction_after_victim_selection_preempts_refresh_without_resurrecting_entry() { + use tokio::io::AsyncReadExt; + let tmp = tempfile::tempdir().unwrap(); + let (cache, idx, repo) = + refresh_growth_fixture(tmp.path(), 256, "printf x > started\nexec sleep 30\n"); + assert!( + std::process::Command::new("mkfifo") + .arg(repo.cache_dir.join("started")) + .status() + .unwrap() + .success() + ); + let handle = cache + .start_refresh(Duration::from_secs(3600), Duration::from_secs(604800), 1) + .unwrap(); + cache.ensure_fresh(&repo, false, b"", None).await.unwrap(); + idx.touch("b.git"); + let refresh = cache.refresh_repo( + &repo, + Duration::from_secs(3600), + Duration::from_secs(604800), + tokio::time::Instant::now() + Duration::from_secs(3600), + ); + tokio::pin!(refresh); + let select_victim = async { + tokio::fs::File::open(repo.cache_dir.join("started")) + .await + .unwrap() + .read_exact(&mut [0]) + .await + .unwrap(); + idx.set_size("a.git", 1024); + let victims = idx.take_victims(); + assert_eq!(victims[0].0, "a.git"); + // The eviction cancels the running refresh instead of waiting it out. + tokio::time::timeout( + Duration::from_secs(5), + cache.evict(&repo.name, &repo.cache_dir), + ) + .await + .unwrap() + .unwrap(); + }; + tokio::join!(refresh, select_victim); + assert!(!repo.cache_dir.exists()); + assert!(!idx.lock().cache.contains("a.git")); + assert!(idx.take_dirty().is_empty()); + let before = idx.totals(); + idx.set_size("a.git", 9999); + idx.mark_dirty("a.git"); + assert_eq!(idx.totals(), before); + assert!(idx.take_dirty().is_empty()); + cache.shutdown(); + handle.await.unwrap(); + } + #[test] fn size_accounting_tracks_the_total() { let tmp = tempfile::tempdir().unwrap(); @@ -720,6 +870,36 @@ mod tests { assert_eq!(idx.totals(), (0, 1)); } + #[cfg(unix)] + fn refresh_growth_fixture( + root: &Path, + cap: u64, + script: &str, + ) -> (GitCache, Arc, crate::repo::RepoRef) { + use std::os::unix::fs::PermissionsExt; + make_mirror(&root.join("a.git"), 10, None); + make_mirror(&root.join("b.git"), 10, None); + let binary = root.join("git-wrapper"); + // Avoid writable descriptors inherited by parallel tests' forked children. + assert!( + std::process::Command::new("sh") + .args(["-c", "printf '%s' \"$2\" > \"$1\"", "sh"]) + .arg(&binary) + .arg(format!("#!/bin/sh\n{script}")) + .status() + .unwrap() + .success() + ); + std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)).unwrap(); + let metrics = Arc::new(Metrics::new()); + let idx = CacheIndex::new(root.to_path_buf(), cap, metrics.clone()); + let mut cfg = dummy_cfg(); + cfg.git_binary = binary.to_str().unwrap().into(); + let cache = GitCache::new(cfg, metrics, Some(idx.clone())); + let repo = crate::repo::resolve("a.git", "https://example.com", root).unwrap(); + (cache, idx, repo) + } + fn dummy_cfg() -> GitConfig { // Eviction never shells out to git, so the binary is irrelevant. GitConfig { diff --git a/src/git.rs b/src/git.rs index c7c30c2..7d73069 100644 --- a/src/git.rs +++ b/src/git.rs @@ -26,6 +26,7 @@ use std::path::Path; use std::pin::Pin; use std::process::Stdio; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::task::Poll; use std::time::Duration; @@ -120,11 +121,35 @@ pub struct GitConfig { /// would only add overhead and never grant a concurrent reader. struct RepoSlot { fetch_lock: Mutex>, + refresh: Mutex, + /// Bumped by `evict` before it waits for `fetch_lock`, so a background + /// refresh holding the lock can yield instead of fetching data that is + /// about to be deleted. + evict_requests: watch::Sender, +} + +#[derive(Default)] +struct RefreshState { + used: Option<(RepoRef, Instant)>, + failures: u32, + retry_after: Option, + auth_blocked: bool, +} + +impl RefreshState { + fn succeeded(&mut self, server_credential: bool) { + self.failures = 0; + self.retry_after = None; + if server_credential { + self.auth_blocked = false; + } + } } #[derive(Clone)] pub struct GitCache { shutdown: CancellationToken, + refresh_enabled: Arc, owners: TaskTracker, upstream_timeout: Duration, cfg: GitConfig, @@ -135,8 +160,7 @@ pub struct GitCache { /// short for reader/writer separation to pay off. slots: Arc>>>, /// Cache-size index for LRU eviction, or `None` when no cap is configured. - /// When present it is `touch`ed on every request and `record`ed after each - /// clone/fetch; when absent the eviction machinery is entirely inert. + /// Requests promote entries; background refresh only schedules remeasurement. index: Option>, /// `None` when `max_upload_packs` is `0` (unlimited). upload_packs: Option>, @@ -155,6 +179,7 @@ impl GitCache { metrics, slots: Arc::new(Mutex::new(HashMap::new())), shutdown: CancellationToken::new(), + refresh_enabled: Arc::new(AtomicBool::new(false)), owners: TaskTracker::new(), upstream_timeout: Duration::from_secs(3600), index, @@ -176,7 +201,8 @@ impl GitCache { self.owners.close(); } - /// Wait for upstream owners to reap children and release repository locks. + /// Wait for request upstream owners to reap children and release repository locks. + /// Also await the handle returned by start_refresh to finish refresh cleanup. pub async fn wait_for_shutdown(&self) { self.owners.wait().await; } @@ -189,11 +215,177 @@ impl GitCache { .or_insert_with(|| { Arc::new(RepoSlot { fetch_lock: Mutex::new(None), + refresh: Mutex::new(RefreshState::default()), + evict_requests: watch::channel(0).0, }) }) .clone() } + async fn record_fetch_success(&self, repo: &RepoRef, cred: Option<&str>) { + if !self.refresh_enabled.load(Ordering::Acquire) { + return; + } + self.slot(&repo.name) + .await + .refresh + .lock() + .await + .succeeded(cred.is_none() || cred == self.cfg.upstream_auth_header.as_deref()); + } + + async fn record_client_use(&self, repo: &RepoRef, finished: Instant) { + let slot = self.slot(&repo.name).await; + let mut state = slot.refresh.lock().await; + if !self.shutdown.is_cancelled() + && state + .used + .as_ref() + .is_none_or(|(_, previous)| *previous <= finished) + { + state.used = Some((repo.clone(), finished)); + } + } + + /// Start optional refresh work. Disabled refresh creates no task. + /// Callers must validate the interval and concurrency with Config first. + /// Only requests admitted after enabling contribute successful cache use. + pub fn start_refresh( + &self, + interval: Duration, + active_window: Duration, + concurrency: usize, + ) -> Option> { + if interval.is_zero() { + return None; + } + assert!(concurrency > 0); + self.refresh_enabled.store(true, Ordering::Release); + let cache = self.clone(); + Some(tokio::spawn(async move { + refresh_loop(interval, &cache.shutdown, |deadline| { + let cache = &cache; + async move { + let candidates = cache.refresh_candidates(active_window).await; + refresh_round( + candidates.into_iter().map(|(repo, _)| repo), + concurrency, + deadline, + &cache.shutdown, + |repo| { + let worker = cache.clone(); + async move { + worker + .refresh_repo(&repo, interval, active_window, deadline) + .await; + } + }, + ) + .await; + } + }) + .await; + })) + } + + async fn refresh_candidates(&self, active_window: Duration) -> Vec<(RepoRef, Instant)> { + let mut candidates = Vec::new(); + for slot in self.slots.lock().await.values() { + let state = slot.refresh.lock().await; + if let Some((repo, used)) = &state.used + && used.elapsed() <= active_window + { + candidates.push((repo.clone(), *used)); + } + } + candidates.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.name.cmp(&b.0.name))); + candidates + } + + /// Refresh an existing mirror without recording client use or LRU access. + pub(crate) async fn refresh_repo( + &self, + repo: &RepoRef, + interval: Duration, + active_window: Duration, + deadline: Instant, + ) { + let slot = self.slot(&repo.name).await; + // Subscribe before taking the lock so an eviction queued behind us is seen. + let mut evict_requested = slot.evict_requests.subscribe(); + let mut last = tokio::select! { + biased; + _ = self.shutdown.cancelled() => return, + _ = tokio::time::sleep_until(deadline) => return, + guard = slot.fetch_lock.lock() => guard, + }; + { + let state = slot.refresh.lock().await; + if Instant::now() >= deadline + || self.shutdown.is_cancelled() + || state.auth_blocked + || state.retry_after.is_some_and(|t| t > Instant::now()) + || state + .used + .as_ref() + .is_none_or(|(_, used)| used.elapsed() > active_window) + || !repo.cache_dir.join("HEAD").exists() + || last.is_some_and(|t| { + !self.cfg.fetch_ttl.is_zero() && t.elapsed() < self.cfg.fetch_ttl + }) + { + return; + } + } + let started = Instant::now(); + let mut worker = self.clone(); + worker.shutdown = self.shutdown.child_token(); + let cancel = worker.shutdown.clone(); + let fetch = worker.fetch(repo, None); + tokio::pin!(fetch); + let mut preempted = false; + // Cancel rather than drop the fetch so its child is still killed and reaped. + let result = tokio::select! { + result = &mut fetch => result, + _ = evict_requested.changed() => { + preempted = true; + cancel.cancel(); + fetch.await + } + }; + if preempted { + return; + } + match result { + Ok(()) => { + *last = Some(started); + if let Some(index) = &self.index { + index.mark_dirty(&repo.name); + } + } + Err(error) => { + if self.shutdown.is_cancelled() { + return; + } + tracing::warn!(repo = %repo.name, error = %error, "background refresh failed"); + let mut state = slot.refresh.lock().await; + state.auth_blocked |= is_auth_failure(&error); + state.failures = state.failures.saturating_add(1); + let multiplier = 1u32 << state.failures.saturating_sub(1).min(5); + state.retry_after = if state.failures == 1 { + None + } else { + // Startup validation reserves space for the maximum backoff. + Some( + Instant::now() + .checked_add(interval.saturating_mul(multiplier)) + .expect("validated refresh interval fits maximum backoff"), + ) + }; + } + } + } + /// Flag a mirror as changed after a clone/fetch so the background evictor /// (re)measures it and rebalances the cache. O(1) bookkeeping only - the size /// walk and any eviction run off this request path. No-op when eviction is @@ -213,6 +405,7 @@ impl GitCache { /// its open file descriptors and drains cleanly (POSIX unlink semantics). pub async fn evict(&self, name: &str, cache_dir: &Path) -> Result<()> { let slot = self.slot(name).await; + slot.evict_requests.send_modify(|n| *n = n.wrapping_add(1)); let guard = slot.fetch_lock.lock().await; if !cache_dir.join("HEAD").exists() { return Ok(()); // already gone (raced a prior eviction or manual removal) @@ -237,6 +430,8 @@ impl GitCache { /// the same repo are serialized; the first does the work, the rest see it already /// fresh. `body` is the client's upload-pack request (empty for `info/refs`), /// scanned for want-by-SHA misses - see `ensure_wanted_oids`. + /// Refresh use is recorded after successful preparation, including TTL hits, + /// independently of later HTTP response or pack-stream success. pub async fn ensure_fresh( &self, repo: &RepoRef, @@ -244,20 +439,26 @@ impl GitCache { body: &[u8], cred: Option<&str>, ) -> Result { + let track_use = self.refresh_enabled.load(Ordering::Acquire); let mut owner = self.clone(); owner.shutdown = self.shutdown.child_token(); let _cancel_on_drop = owner.shutdown.clone().drop_guard(); - let repo = repo.clone(); + let owner_repo = repo.clone(); let wants = parse_wants(body); let cred = cred.map(str::to_owned); - self.owners + let outcome = self + .owners .spawn(async move { owner - .ensure_fresh_owned(&repo, want_fetch, &wants, cred.as_deref()) + .ensure_fresh_owned(&owner_repo, want_fetch, &wants, cred.as_deref()) .await }) .await - .context("join upstream owner")? + .context("join upstream owner")??; + if track_use { + self.record_client_use(repo, Instant::now()).await; + } + Ok(outcome) } async fn ensure_fresh_owned( @@ -293,6 +494,7 @@ impl GitCache { } { self.fetch(repo, cred).await?; + self.mark_changed(repo); *last = Some(started); CacheOutcome::Fetched } else { @@ -480,6 +682,7 @@ impl GitCache { self.metrics .observe_upstream(UpstreamOp::Clone, &repo.name, elapsed); self.mark_changed(repo); + self.record_fetch_success(repo, cred).await; Ok(()) } @@ -518,7 +721,7 @@ impl GitCache { &repo.name, started.elapsed().as_secs_f64(), ); - self.mark_changed(repo); + self.record_fetch_success(repo, cred).await; Ok(()) } @@ -591,6 +794,7 @@ impl GitCache { started.elapsed().as_secs_f64(), ); self.mark_changed(repo); + self.record_fetch_success(repo, cred).await; // Best-effort: a failed prune is not worth failing the request over. if let Err(e) = self.prune_wants(repo).await { tracing::warn!(repo = %repo.name, error = %e, "pruning want-by-sha pins failed"); @@ -1332,9 +1536,1005 @@ fn parse_wants(body: &[u8]) -> HashSet { oids } +async fn refresh_loop(interval: Duration, shutdown: &CancellationToken, round: F) +where + F: Fn(Instant) -> Fut, + Fut: std::future::Future, +{ + let Some(mut next) = Instant::now().checked_add(interval) else { + tracing::warn!("refresh interval exceeds the monotonic deadline range"); + return; + }; + loop { + tokio::select! { + biased; + _ = shutdown.cancelled() => break, + _ = tokio::time::sleep_until(next) => {} + } + let Some(deadline) = Instant::now().checked_add(interval) else { + tracing::warn!("refresh interval exceeds the monotonic deadline range"); + break; + }; + round(deadline).await; + next = deadline; + } +} + +async fn refresh_round( + candidates: impl IntoIterator, + concurrency: usize, + deadline: Instant, + shutdown: &CancellationToken, + fetch: F, +) where + F: Fn(RepoRef) -> Fut, + Fut: std::future::Future + Send + 'static, +{ + let mut jobs = tokio::task::JoinSet::new(); + for repo in candidates { + if jobs.len() == concurrency + && let Some(Err(error)) = jobs.join_next().await + { + tracing::warn!(%error, "background refresh job failed"); + } + if Instant::now() >= deadline || shutdown.is_cancelled() { + break; + } + jobs.spawn(fetch(repo)); + } + while let Some(result) = jobs.join_next().await { + if let Err(error) = result { + tracing::warn!(%error, "background refresh job failed"); + } + } +} + #[cfg(test)] mod tests { use super::*; + use std::sync::Mutex as StdMutex; + + #[tokio::test(start_paused = true)] + async fn refresh_usage_tracks_enabled_success_and_ttl_hits_only() { + let root = tempfile::tempdir().unwrap(); + let repo = refresh_test_mirror(root.path(), "repo.git"); + let mut cfg = test_config(); + cfg.git_binary = "true".into(); + let cache = GitCache::new(cfg, Arc::new(Metrics::new()), None); + let slot = cache.slot(&repo.name).await; + let guard = slot.fetch_lock.lock().await; + let request = cache.ensure_fresh(&repo, false, b"", None); + tokio::pin!(request); + std::future::poll_fn(|cx| { + assert!(request.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + let handle = cache + .start_refresh(Duration::from_secs(3600), Duration::from_secs(604800), 1) + .unwrap(); + drop(guard); + request.await.unwrap(); + assert!( + slot.refresh.lock().await.used.is_none(), + "enabling must not count an earlier request" + ); + cache + .clone() + .ensure_fresh(&repo, false, b"", None) + .await + .unwrap(); + let first = slot.refresh.lock().await.used.as_ref().unwrap().1; + *slot.fetch_lock.lock().await = Some(Instant::now()); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(matches!( + cache.ensure_fresh(&repo, true, b"", None).await.unwrap(), + CacheOutcome::Cached + )); + let latest = slot.refresh.lock().await.used.as_ref().unwrap().1; + assert!(latest > first); + cache + .refresh_repo( + &repo, + Duration::from_secs(3600), + Duration::from_secs(604800), + Instant::now() + Duration::from_secs(3600), + ) + .await; + assert_eq!(slot.refresh.lock().await.used.as_ref().unwrap().1, latest); + cache.shutdown(); + handle.await.unwrap(); + } + + #[tokio::test(start_paused = true)] + async fn refresh_failed_preparation_does_not_add_or_replace_usage() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let slot = cache.slot(&repo.name).await; + let previous = slot.refresh.lock().await.used.as_ref().unwrap().1; + tokio::time::advance(Duration::from_secs(2)).await; + assert!(cache.ensure_fresh(&repo, true, b"", None).await.is_err()); + assert_eq!(slot.refresh.lock().await.used.as_ref().unwrap().1, previous); + let unseen = refresh_test_mirror(root.path(), "unseen.git"); + assert!(cache.ensure_fresh(&unseen, true, b"", None).await.is_err()); + assert!( + cache + .slot(&unseen.name) + .await + .refresh + .lock() + .await + .used + .is_none() + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_auth_failed_preparation_preserves_successful_usage() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_fixture( + root.path(), + "echo 'fatal: Authentication failed' >&2\nexit 1\n", + ) + .await; + let slot = cache.slot(&repo.name).await; + let previous = slot.refresh.lock().await.used.as_ref().unwrap().1; + let error = cache + .ensure_fresh(&repo, true, b"", None) + .await + .unwrap_err(); + assert!(is_auth_failure(&error)); + assert_eq!(slot.refresh.lock().await.used.as_ref().unwrap().1, previous); + let unseen = refresh_test_mirror(root.path(), "unseen.git"); + assert!(is_auth_failure( + &cache + .ensure_fresh(&unseen, true, b"", None) + .await + .unwrap_err() + )); + assert!( + cache + .slot(&unseen.name) + .await + .refresh + .lock() + .await + .used + .is_none() + ); + } + + #[tokio::test(start_paused = true)] + async fn refresh_cancelled_preparation_preserves_usage() { + for previously_used in [false, true] { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let slot = cache.slot(&repo.name).await; + if !previously_used { + slot.refresh.lock().await.used = None; + } + let previous = slot + .refresh + .lock() + .await + .used + .as_ref() + .map(|(_, used)| *used); + let guard = slot.fetch_lock.lock().await; + let worker = cache.clone(); + let mirror = repo.clone(); + let request = + tokio::spawn(async move { worker.ensure_fresh(&mirror, false, b"", None).await }); + tokio::time::sleep(Duration::from_millis(1)).await; + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + cache.shutdown(); + drop(guard); + cache.wait_for_shutdown().await; + assert_eq!( + slot.refresh + .lock() + .await + .used + .as_ref() + .map(|(_, used)| *used), + previous + ); + } + } + + #[tokio::test(start_paused = true)] + async fn refresh_usage_cannot_move_backwards() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let older = Instant::now(); + tokio::time::advance(Duration::from_secs(2)).await; + let newer = Instant::now(); + tokio::join!( + cache.record_client_use(&repo, newer), + cache.record_client_use(&repo, older) + ); + assert_eq!( + cache + .slot(&repo.name) + .await + .refresh + .lock() + .await + .used + .as_ref() + .unwrap() + .1, + newer + ); + } + + #[tokio::test(start_paused = true)] + async fn refresh_loop_rejects_an_overflowing_deadline_without_panicking() { + let round = |_| async { panic!("test refresh round") }; + let valid_round = tokio::spawn(async move { + refresh_loop(Duration::from_secs(1), &CancellationToken::new(), round).await; + }); + assert!(valid_round.await.unwrap_err().is_panic()); + let now = Instant::now(); + let (mut valid, mut invalid) = (0, u64::MAX); + while invalid - valid > 1 { + let seconds = valid + (invalid - valid) / 2; + if now.checked_add(Duration::from_secs(seconds)).is_some() { + valid = seconds; + } else { + invalid = seconds; + } + } + let interval = Duration::from_secs(invalid); + assert!(now.checked_add(Duration::from_secs(valid)).is_some()); + assert!(now.checked_add(interval).is_none()); + refresh_loop(interval, &CancellationToken::new(), round).await; + let interval = Duration::from_secs(1); + let token = CancellationToken::new(); + let refresh = refresh_loop(interval, &token, round); + tokio::pin!(refresh); + std::future::poll_fn(|cx| { + assert!(refresh.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::advance(Duration::from_secs(valid)).await; + assert!(Instant::now().checked_add(interval).is_none()); + refresh.await; + } + + #[tokio::test(start_paused = true)] + async fn start_refresh_attempts_request_recorded_repos_in_recency_order() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let stale = refresh_test_mirror(root.path(), "stale.git"); + cache.ensure_fresh(&stale, false, b"", None).await.unwrap(); + let cold = refresh_test_mirror(root.path(), "cold.git"); + tokio::time::advance(Duration::from_secs(200)).await; + cache.ensure_fresh(&repo, false, b"", None).await.unwrap(); + tokio::time::advance(Duration::from_secs(1)).await; + let newest_z = refresh_test_mirror(root.path(), "z.git"); + let newest_a = refresh_test_mirror(root.path(), "a.git"); + for mirror in [&newest_z, &newest_a] { + cache.ensure_fresh(mirror, false, b"", None).await.unwrap(); + } + let output = RefreshLog::default(); + let writer = output.clone(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + let handle = cache + .start_refresh(Duration::from_secs(10), Duration::from_secs(100), 1) + .unwrap(); + tokio::time::sleep(Duration::from_millis(10001)).await; + cache.shutdown(); + handle.await.unwrap(); + let log = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); + let attempts: Vec<_> = log + .lines() + .filter(|line| line.contains("background refresh failed")) + .collect(); + assert_eq!(attempts.len(), 3, "{log}"); + for (line, name) in attempts.iter().zip(["a.git", "z.git", "repo.git"]) { + assert!(line.contains(&format!("repo={name} ")), "{line}"); + assert_eq!(cache.slot(name).await.refresh.lock().await.failures, 1); + } + assert_eq!( + cache.slot(&stale.name).await.refresh.lock().await.failures, + 0 + ); + assert!(!cache.slots.lock().await.contains_key(&cold.name)); + } + + #[tokio::test(start_paused = true)] + async fn refresh_job_panics_are_logged_and_other_jobs_finish() { + let root = tempfile::tempdir().unwrap(); + let output = RefreshLog::default(); + let writer = output.clone(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + for names in [["a.git", "b.git"], ["b.git", "a.git"]] { + let candidates = names.map(|name| refresh_test_mirror(root.path(), name)); + let completed = Arc::new(Mutex::new(Vec::new())); + refresh_round( + candidates, + 1, + Instant::now() + Duration::from_secs(10), + &CancellationToken::new(), + |repo| { + let completed = completed.clone(); + async move { + if repo.name == "a.git" { + panic!("test refresh panic"); + } + completed.lock().await.push(repo.name); + } + }, + ) + .await; + assert_eq!(*completed.lock().await, ["b.git"]); + } + std::io::Write::flush(&mut output.clone()).unwrap(); + let log = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); + assert!(log.contains("WARN")); + assert_eq!(log.matches("background refresh job failed").count(), 2); + } + + #[tokio::test(start_paused = true)] + async fn refresh_candidates_use_client_recency_and_window_boundary() { + let cache = GitCache::new(test_config(), Arc::new(Metrics::new()), None); + let root = tempfile::tempdir().unwrap(); + for name in ["old.git", "recent.git", "newest.git"] { + let repo = crate::repo::resolve(name, "https://example.com", root.path()).unwrap(); + cache.slot(name).await.refresh.lock().await.used = Some((repo, Instant::now())); + tokio::time::advance(Duration::from_secs(10)).await; + } + let candidates = cache.refresh_candidates(Duration::from_secs(20)).await; + assert_eq!( + candidates + .iter() + .map(|(repo, _)| repo.name.as_str()) + .collect::>(), + ["newest.git", "recent.git"] + ); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!( + cache + .refresh_candidates(Duration::from_secs(20)) + .await + .len(), + 1 + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_want_fetch_success_clears_backoff_with_credential_rules() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_fixture(root.path(), + "if [ \"$1\" = cat-file ]; then while read -r oid; do printf '%s missing\\n' \"$oid\"; done; fi\n").await; + let slot = cache.slot(&repo.name).await; + for cred in [Some("Authorization: client"), None] { + { + let mut state = slot.refresh.lock().await; + state.failures = 3; + state.retry_after = Some(Instant::now() + Duration::from_secs(100)); + state.auth_blocked = true; + } + let body = pkt_line(&format!("want {}\n", "a".repeat(40))); + cache.ensure_fresh(&repo, false, &body, cred).await.unwrap(); + let state = slot.refresh.lock().await; + assert_eq!(state.failures, 0); + assert!(state.retry_after.is_none()); + assert_eq!(state.auth_blocked, cred.is_some()); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_shutdown_reaps_running_fetch_and_releases_lock() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = + refresh_fixture(root.path(), "printf x > started\nexec sleep 30\n").await; + assert!( + std::process::Command::new("mkfifo") + .arg(repo.cache_dir.join("started")) + .status() + .unwrap() + .success() + ); + let worker = cache.clone(); + let mirror = repo.clone(); + let job = tokio::spawn(async move { + worker + .refresh_repo( + &mirror, + Duration::from_secs(10), + Duration::from_secs(100), + Instant::now() + Duration::from_secs(30), + ) + .await; + }); + tokio::fs::File::open(repo.cache_dir.join("started")) + .await + .unwrap() + .read_exact(&mut [0]) + .await + .unwrap(); + cache.shutdown(); + tokio::time::timeout(Duration::from_secs(5), job) + .await + .unwrap() + .unwrap(); + let slot = cache.slot(&repo.name).await; + assert!(slot.fetch_lock.try_lock().is_ok()); + assert_eq!(slot.refresh.lock().await.failures, 0); + } + + #[cfg(unix)] + #[tokio::test] + async fn eviction_preempts_a_running_refresh_fetch() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = + refresh_fixture(root.path(), "printf x > started\nexec sleep 30\n").await; + assert!( + std::process::Command::new("mkfifo") + .arg(repo.cache_dir.join("started")) + .status() + .unwrap() + .success() + ); + let worker = cache.clone(); + let mirror = repo.clone(); + let job = tokio::spawn(async move { + worker + .refresh_repo( + &mirror, + Duration::from_secs(10), + Duration::from_secs(100), + Instant::now() + Duration::from_secs(30), + ) + .await; + }); + tokio::fs::File::open(repo.cache_dir.join("started")) + .await + .unwrap() + .read_exact(&mut [0]) + .await + .unwrap(); + tokio::time::timeout( + Duration::from_secs(5), + cache.evict(&repo.name, &repo.cache_dir), + ) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), job) + .await + .unwrap() + .unwrap(); + assert!(!repo.cache_dir.join("HEAD").exists()); + let slot = cache.slot(&repo.name).await; + let state = slot.refresh.lock().await; + assert_eq!(state.failures, 0); + assert!(!state.auth_blocked); + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_uses_static_auth_and_client_waits_for_refresh() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_fixture(root.path(), + "printf '%s' \"$GIT_CONFIG_VALUE_2\" > auth\nprintf '%s\\n' \"$@\" > args\nprintf x >> fetches\nprintf x > started\ncat release >/dev/null\n").await; + for name in ["started", "release"] { + assert!( + std::process::Command::new("mkfifo") + .arg(repo.cache_dir.join(name)) + .status() + .unwrap() + .success() + ); + } + let worker = cache.clone(); + let mirror = repo.clone(); + let job = tokio::spawn(async move { + worker + .refresh_repo( + &mirror, + Duration::from_secs(10), + Duration::from_secs(100), + Instant::now() + Duration::from_secs(30), + ) + .await; + }); + let mut signal = tokio::fs::File::open(repo.cache_dir.join("started")) + .await + .unwrap(); + signal.read_exact(&mut [0]).await.unwrap(); + assert!(cache.slot(&repo.name).await.fetch_lock.try_lock().is_err()); + let request = cache.ensure_fresh(&repo, true, b"", Some("Authorization: client")); + tokio::pin!(request); + std::future::poll_fn(|cx| { + assert!(request.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::pause(); + tokio::time::sleep(Duration::from_millis(1)).await; + std::future::poll_fn(|cx| { + assert!(request.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::resume(); + assert_eq!( + tokio::fs::read(repo.cache_dir.join("fetches")) + .await + .unwrap(), + b"x" + ); + assert_eq!( + tokio::fs::read_to_string(repo.cache_dir.join("auth")) + .await + .unwrap(), + "Authorization: server" + ); + let args = tokio::fs::read_to_string(repo.cache_dir.join("args")) + .await + .unwrap(); + assert!(args.lines().any(|arg| arg == repo.upstream_url)); + assert!(!args.lines().any(|arg| arg == "origin")); + tokio::fs::write(repo.cache_dir.join("release"), b"done") + .await + .unwrap(); + job.await.unwrap(); + assert!(matches!(request.await.unwrap(), CacheOutcome::Cached)); + assert_eq!( + tokio::fs::read(repo.cache_dir.join("fetches")) + .await + .unwrap(), + b"x" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_failure_backoff_auth_rejection_and_request_recovery() { + let root = tempfile::tempdir().unwrap(); + let (mut cache, repo) = refresh_fixture(root.path(), "exit 1\n").await; + let interval = Duration::from_secs(10); + let slot = cache.slot(&repo.name).await; + for failures in 1..=8 { + slot.refresh.lock().await.retry_after = None; + let before = Instant::now(); + cache + .refresh_repo(&repo, interval, Duration::from_secs(100), before + interval) + .await; + let state = slot.refresh.lock().await; + assert_eq!(state.failures, failures); + let delay = interval * (1 << (failures - 1).min(5)); + if failures == 1 { + assert!(state.retry_after.is_none()); + } else { + let retry_after = state.retry_after.unwrap(); + assert!(retry_after >= before + delay); + if failures >= 6 { + assert!(retry_after <= Instant::now() + interval * 32); + } + } + assert!(!state.auth_blocked); + } + cache.cfg.git_binary = "true".into(); + cache + .ensure_fresh(&repo, true, b"", Some("Authorization: client")) + .await + .unwrap(); + assert_eq!(slot.refresh.lock().await.failures, 0); + cache.cfg.git_binary = root.path().join("git-wrapper").to_str().unwrap().into(); + write_test_script( + Path::new(&cache.cfg.git_binary), + "#!/bin/sh\necho 'fatal: Authentication failed' >&2\nexit 1\n", + ); + *slot.fetch_lock.lock().await = None; + cache + .refresh_repo( + &repo, + interval, + Duration::from_secs(100), + Instant::now() + interval, + ) + .await; + assert!(slot.refresh.lock().await.auth_blocked); + assert_eq!(slot.refresh.lock().await.failures, 1); + cache + .refresh_repo( + &repo, + interval, + Duration::from_secs(100), + Instant::now() + interval, + ) + .await; + assert_eq!(slot.refresh.lock().await.failures, 1); + cache.cfg.git_binary = "true".into(); + cache + .ensure_fresh(&repo, true, b"", Some("Authorization: client")) + .await + .unwrap(); + assert!(slot.refresh.lock().await.auth_blocked); + *slot.fetch_lock.lock().await = None; + cache.ensure_fresh(&repo, true, b"", None).await.unwrap(); + assert!(!slot.refresh.lock().await.auth_blocked); + } + + #[tokio::test(start_paused = true)] + async fn refresh_backoff_stops_at_thirty_two_intervals() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let slot = cache.slot(&repo.name).await; + let interval = Duration::from_secs(10); + for failures in 1..=8 { + slot.refresh.lock().await.retry_after = None; + cache + .refresh_repo( + &repo, + interval, + Duration::from_secs(100), + Instant::now() + interval, + ) + .await; + let state = slot.refresh.lock().await; + assert_eq!(state.failures, failures); + let expected = if failures == 1 { + None + } else { + Some(Instant::now() + interval * (1 << (failures - 1).min(5))) + }; + assert_eq!(state.retry_after, expected); + } + } + + #[tokio::test(start_paused = true)] + async fn refresh_first_failure_retries_in_the_next_round() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let interval = Duration::from_secs(10); + let token = CancellationToken::new(); + let slot = cache.slot(&repo.name).await; + let started = Instant::now(); + tokio::time::timeout( + Duration::from_secs(30), + refresh_loop(interval, &token, |deadline| { + let (cache, repo, slot, token) = (&cache, &repo, &slot, &token); + async move { + tokio::time::sleep(Duration::from_secs(3)).await; + cache + .refresh_repo(repo, interval, Duration::from_secs(100), deadline) + .await; + if slot.refresh.lock().await.failures == 2 { + token.cancel(); + } + } + }), + ) + .await + .expect("first failure must retry in round two"); + assert_eq!(started.elapsed(), Duration::from_secs(23)); + assert_eq!(slot.refresh.lock().await.failures, 2); + } + + #[tokio::test(start_paused = true)] + async fn refresh_second_failure_waits_two_intervals() { + let root = tempfile::tempdir().unwrap(); + let (cache, repo) = refresh_spawn_failure_fixture(root.path()).await; + let interval = Duration::from_secs(10); + let slot = cache.slot(&repo.name).await; + for _ in 0..2 { + cache + .refresh_repo( + &repo, + interval, + Duration::from_secs(100), + Instant::now() + interval, + ) + .await; + } + let retry_after = slot.refresh.lock().await.retry_after.unwrap(); + assert_eq!(retry_after, Instant::now() + interval * 2); + tokio::time::advance(interval).await; + cache + .refresh_repo( + &repo, + interval, + Duration::from_secs(100), + Instant::now() + interval, + ) + .await; + assert_eq!(slot.refresh.lock().await.failures, 2); + tokio::time::advance(interval).await; + cache + .refresh_repo( + &repo, + interval, + Duration::from_secs(100), + Instant::now() + interval, + ) + .await; + assert_eq!(slot.refresh.lock().await.failures, 3); + } + + #[tokio::test(start_paused = true)] + async fn refresh_schedule_spaces_short_rounds_and_waits_for_overruns() { + for length in [3, 15] { + let token = CancellationToken::new(); + let starts = Mutex::new(Vec::new()); + let start = Instant::now(); + refresh_loop(Duration::from_secs(10), &token, |_| async { + starts.lock().await.push(Instant::now()); + tokio::time::sleep(Duration::from_secs(length)).await; + if starts.lock().await.len() == 3 { + token.cancel(); + } + }) + .await; + let starts = starts.lock().await; + assert_eq!(starts[0] - start, Duration::from_secs(10)); + assert_eq!(starts[1] - starts[0], Duration::from_secs(length.max(10))); + assert_eq!(starts[2] - starts[1], Duration::from_secs(length.max(10))); + } + } + + #[tokio::test] + async fn refresh_disabled_creates_no_task_or_request_state_access() { + let root = tempfile::tempdir().unwrap(); + let repo = refresh_test_mirror(root.path(), "repo.git"); + let mut cfg = test_config(); + cfg.git_binary = "true".into(); + let cache = GitCache::new(cfg, Arc::new(Metrics::new()), None); + let slot = cache.slot(&repo.name).await; + let guard = slot.refresh.lock().await; + assert!( + cache + .start_refresh(Duration::ZERO, Duration::ZERO, 0) + .is_none() + ); + tokio::time::timeout( + Duration::from_secs(1), + cache.ensure_fresh(&repo, true, b"", None), + ) + .await + .unwrap() + .unwrap(); + assert!(guard.used.is_none()); + assert_eq!(guard.failures, 0); + assert!(cache.owners.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn refresh_round_deadline_drains_running_jobs_without_starting_waiters() { + let root = tempfile::tempdir().unwrap(); + let candidates = ["new.git", "middle.git", "old.git"] + .map(|name| crate::repo::resolve(name, "https://example.com", root.path()).unwrap()); + let events = Arc::new(Mutex::new(Vec::new())); + let start = Instant::now(); + let token = CancellationToken::new(); + refresh_round( + candidates, + 2, + start + Duration::from_secs(10), + &token, + |repo| { + let events = events.clone(); + async move { + events.lock().await.push((repo.name.clone(), false)); + tokio::time::sleep(Duration::from_secs(15)).await; + events.lock().await.push((repo.name, true)); + } + }, + ) + .await; + assert_eq!(start.elapsed(), Duration::from_secs(15)); + let events = events.lock().await; + assert_eq!( + &events[..2], + &[("new.git".into(), false), ("middle.git".into(), false)] + ); + let mut completed = events[2..].to_vec(); + completed.sort(); + assert_eq!( + completed, + [("middle.git".into(), true), ("new.git".into(), true)] + ); + } + + #[tokio::test(start_paused = true)] + async fn refresh_round_short_jobs_start_in_order_with_bounded_concurrency() { + let root = tempfile::tempdir().unwrap(); + let candidates = ["a.git", "b.git", "c.git"] + .map(|name| crate::repo::resolve(name, "https://example.com", root.path()).unwrap()); + let events = Arc::new(Mutex::new(Vec::new())); + let start = Instant::now(); + refresh_round( + candidates, + 1, + start + Duration::from_secs(10), + &CancellationToken::new(), + |repo| { + let events = events.clone(); + async move { + events.lock().await.push((repo.name, Instant::now())); + tokio::time::sleep(Duration::from_secs(1)).await; + } + }, + ) + .await; + assert_eq!(start.elapsed(), Duration::from_secs(3)); + let events = events.lock().await; + for (i, (name, when)) in events.iter().enumerate() { + assert_eq!(name, &["a.git", "b.git", "c.git"][i]); + assert_eq!(*when - start, Duration::from_secs(i as u64)); + } + } + + #[tokio::test(start_paused = true)] + async fn refresh_lock_deadline_and_shutdown_leave_failure_state_unchanged() { + let cache = GitCache::new(test_config(), Arc::new(Metrics::new()), None); + let root = tempfile::tempdir().unwrap(); + let repo = crate::repo::resolve("repo.git", "https://example.com", root.path()).unwrap(); + let slot = cache.slot(&repo.name).await; + slot.refresh.lock().await.failures = 3; + let guard = slot.fetch_lock.lock().await; + let start = Instant::now(); + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(100), + start + Duration::from_secs(10), + ) + .await; + assert_eq!(start.elapsed(), Duration::from_secs(10)); + assert_eq!(slot.refresh.lock().await.failures, 3); + cache.shutdown(); + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(100), + Instant::now() + Duration::from_secs(10), + ) + .await; + assert_eq!(slot.refresh.lock().await.failures, 3); + drop(guard); + assert!(slot.fetch_lock.try_lock().is_ok()); + } + + #[tokio::test] + async fn refresh_success_shares_ttl_and_preserves_client_usage() { + let root = tempfile::tempdir().unwrap(); + let repo = crate::repo::resolve("repo.git", "https://example.com", root.path()).unwrap(); + std::fs::create_dir_all(&repo.cache_dir).unwrap(); + std::fs::write(repo.cache_dir.join("HEAD"), "ref: refs/heads/main").unwrap(); + let mut cfg = test_config(); + cfg.git_binary = "true".into(); + cfg.fetch_ttl = Duration::from_secs(60); + let cache = GitCache::new(cfg, Arc::new(Metrics::new()), None); + cache.refresh_enabled.store(true, Ordering::Release); + let slot = cache.slot(&repo.name).await; + let used = Instant::now(); + slot.refresh.lock().await.used = Some((repo.clone(), used)); + slot.refresh.lock().await.failures = 3; + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(100), + Instant::now() + Duration::from_secs(10), + ) + .await; + assert!(slot.fetch_lock.lock().await.is_some()); + assert_eq!(slot.refresh.lock().await.used.as_ref().unwrap().1, used); + assert_eq!(slot.refresh.lock().await.failures, 0); + assert!(matches!( + cache.ensure_fresh(&repo, true, b"", None).await.unwrap(), + CacheOutcome::Cached + )); + } + + #[tokio::test(start_paused = true)] + async fn refresh_skips_inactive_missing_auth_blocked_and_backoff_repos() { + let cache = GitCache::new(test_config(), Arc::new(Metrics::new()), None); + let root = tempfile::tempdir().unwrap(); + let repo = crate::repo::resolve("repo.git", "https://example.com", root.path()).unwrap(); + let slot = cache.slot(&repo.name).await; + let used = Instant::now(); + slot.refresh.lock().await.used = Some((repo.clone(), used)); + tokio::time::advance(Duration::from_secs(11)).await; + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(10), + Instant::now() + Duration::from_secs(10), + ) + .await; + assert!(!repo.cache_dir.exists()); + assert_eq!(slot.refresh.lock().await.failures, 0); + slot.refresh.lock().await.used = Some((repo.clone(), Instant::now())); + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(10), + Instant::now() + Duration::from_secs(10), + ) + .await; + assert!(!repo.cache_dir.exists()); + std::fs::create_dir_all(&repo.cache_dir).unwrap(); + std::fs::write(repo.cache_dir.join("HEAD"), "ref: refs/heads/main").unwrap(); + slot.refresh.lock().await.auth_blocked = true; + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(10), + Instant::now() + Duration::from_secs(10), + ) + .await; + assert_eq!(slot.refresh.lock().await.failures, 0); + slot.refresh.lock().await.auth_blocked = false; + slot.refresh.lock().await.retry_after = Some(Instant::now() + Duration::from_secs(20)); + cache + .refresh_repo( + &repo, + Duration::from_secs(10), + Duration::from_secs(10), + Instant::now() + Duration::from_secs(10), + ) + .await; + assert_eq!(slot.refresh.lock().await.failures, 0); + } + + #[tokio::test] + async fn refresh_auth_mark_requires_server_credential_success() { + let mut cfg = test_config(); + cfg.upstream_auth_header = Some("Authorization: server".into()); + let cache = GitCache::new(cfg, Arc::new(Metrics::new()), None); + cache.refresh_enabled.store(true, Ordering::Release); + let root = tempfile::tempdir().unwrap(); + let repo = crate::repo::resolve("repo.git", "https://example.com", root.path()).unwrap(); + let slot = cache.slot(&repo.name).await; + { + let mut state = slot.refresh.lock().await; + state.auth_blocked = true; + state.failures = 4; + state.retry_after = Some(Instant::now() + Duration::from_secs(100)); + } + cache + .record_fetch_success(&repo, Some("Authorization: client")) + .await; + { + let state = slot.refresh.lock().await; + assert!(state.auth_blocked); + assert_eq!(state.failures, 0); + assert!(state.retry_after.is_none()); + } + cache.record_fetch_success(&repo, None).await; + assert!(!slot.refresh.lock().await.auth_blocked); + slot.refresh.lock().await.auth_blocked = true; + cache + .record_fetch_success(&repo, Some("Authorization: server")) + .await; + assert!(!slot.refresh.lock().await.auth_blocked); + } #[tokio::test] async fn missing_oids_drains_more_than_a_pipe_buffer() { @@ -3039,4 +4239,70 @@ mod tests { None, ) } + + #[derive(Clone, Default)] + struct RefreshLog(Arc>>); + + impl std::io::Write for RefreshLog { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + fn refresh_test_mirror(root: &Path, name: &str) -> RepoRef { + let repo = crate::repo::resolve(name, "https://example.com", root).unwrap(); + std::fs::create_dir_all(&repo.cache_dir).unwrap(); + std::fs::write(repo.cache_dir.join("HEAD"), "ref: refs/heads/main").unwrap(); + repo + } + + async fn refresh_spawn_failure_fixture(root: &Path) -> (GitCache, RepoRef) { + let mut cfg = test_config(); + cfg.git_binary = root.join("missing-git").to_str().unwrap().into(); + let cache = GitCache::new(cfg, Arc::new(Metrics::new()), None); + let repo = crate::repo::resolve("repo.git", "https://example.com", root).unwrap(); + std::fs::create_dir_all(&repo.cache_dir).unwrap(); + std::fs::write(repo.cache_dir.join("HEAD"), "ref: refs/heads/main").unwrap(); + cache.refresh_enabled.store(true, Ordering::Release); + cache.ensure_fresh(&repo, false, b"", None).await.unwrap(); + (cache, repo) + } + + #[cfg(unix)] + async fn refresh_fixture(root: &Path, script: &str) -> (GitCache, RepoRef) { + use std::os::unix::fs::PermissionsExt; + let binary = root.join("git-wrapper"); + write_test_script(&binary, &format!("#!/bin/sh\n{script}")); + std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut cfg = test_config(); + cfg.git_binary = binary.to_str().unwrap().into(); + cfg.upstream_auth_header = Some("Authorization: server".into()); + cfg.fetch_ttl = Duration::from_secs(60); + let cache = GitCache::new(cfg, Arc::new(Metrics::new()), None); + let repo = crate::repo::resolve("repo.git", "https://example.com", root).unwrap(); + std::fs::create_dir_all(&repo.cache_dir).unwrap(); + std::fs::write(repo.cache_dir.join("HEAD"), "ref: refs/heads/main").unwrap(); + cache.refresh_enabled.store(true, Ordering::Release); + cache.ensure_fresh(&repo, false, b"", None).await.unwrap(); + (cache, repo) + } + + #[cfg(unix)] + fn write_test_script(path: &Path, script: &str) { + // Avoid writable descriptors inherited by parallel tests' forked children. + assert!( + std::process::Command::new("sh") + .args(["-c", "printf '%s' \"$2\" > \"$1\"", "sh"]) + .arg(path) + .arg(script) + .status() + .unwrap() + .success() + ); + } } diff --git a/src/main.rs b/src/main.rs index a762ff3..227e21e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,7 @@ use git_cache_proxy::{auth, evict, git, lfs, metrics, repo, server}; #[tokio::main(flavor = "multi_thread")] async fn main() -> Result<()> { let cfg = Config::parse(); + cfg.validate_refresh()?; // Hold the non-blocking writer's guard for the whole process; its drop flushes // any buffered log lines on exit. let _log_guard = init_tracing(&cfg.log, cfg.log_format); @@ -164,6 +165,12 @@ async fn main() -> Result<()> { cfg.cache_root.display(), ); + let refresher = cache.start_refresh( + Duration::from_secs(cfg.refresh_interval_seconds), + Duration::from_secs(cfg.refresh_active_window_seconds), + cfg.max_concurrent_refreshes, + ); + let shutdown_cache = cache.clone(); axum::serve(listener, server::router(state)) .with_graceful_shutdown(async move { @@ -174,6 +181,9 @@ async fn main() -> Result<()> { .context("http server")?; cache.wait_for_shutdown().await; + if let Some(handle) = refresher { + handle.await.context("join refresh task")?; + } // The server has drained; stop the evictor and wait for it to finish any // in-progress eviction before exiting. diff --git a/tests/refresh.rs b/tests/refresh.rs new file mode 100644 index 0000000..ef9a187 --- /dev/null +++ b/tests/refresh.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Public refresh configuration and startup behavior. + +use std::sync::Arc; +use std::time::Duration; + +use clap::Parser; +use git_cache_proxy::config::Config; +use git_cache_proxy::git::{GitCache, GitConfig}; +use git_cache_proxy::metrics::Metrics; + +#[tokio::test(start_paused = true)] +async fn refresh_default_disabled_does_not_start_a_task() { + let cfg = Config::try_parse_from(["proxy", "--upstream", "https://example.com"]).unwrap(); + assert_eq!(cfg.refresh_interval_seconds, 0); + assert_eq!(cfg.refresh_active_window_seconds, 604800); + assert_eq!(cfg.max_concurrent_refreshes, 4); + cfg.validate_refresh().unwrap(); + let cache = GitCache::new( + GitConfig { + git_binary: "missing-git-must-not-run".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }, + Arc::new(Metrics::new()), + None, + ); + assert!( + cache + .start_refresh( + Duration::from_secs(cfg.refresh_interval_seconds), + Duration::from_secs(cfg.refresh_active_window_seconds), + cfg.max_concurrent_refreshes, + ) + .is_none() + ); + tokio::time::advance(Duration::from_secs(604800)).await; + cache.shutdown(); + cache.wait_for_shutdown().await; +} + +#[test] +fn refresh_validation_ignores_inactive_options_when_disabled() { + let mut cfg = Config::try_parse_from([ + "proxy", + "--upstream", + "https://example.com", + "--max-concurrent-refreshes", + "0", + ]) + .unwrap(); + assert!(cfg.validate_refresh().is_ok()); + cfg.refresh_interval_seconds = 3599; + assert!(cfg.validate_refresh().is_err()); + cfg.refresh_interval_seconds = 3600; + assert!(cfg.validate_refresh().is_err()); + cfg.max_concurrent_refreshes = 1; + assert!(cfg.validate_refresh().is_ok()); +} + +#[test] +fn refresh_validation_rejects_overflow_and_zero_active_window() { + let mut cfg = Config::try_parse_from(["proxy", "--upstream", "https://example.com"]).unwrap(); + cfg.refresh_interval_seconds = u64::MAX; + assert!( + cfg.validate_refresh() + .unwrap_err() + .to_string() + .contains("monotonic deadline") + ); + cfg.refresh_interval_seconds = 3600; + cfg.refresh_active_window_seconds = 0; + assert!( + cfg.validate_refresh() + .unwrap_err() + .to_string() + .contains("--refresh-active-window-seconds") + ); + cfg.refresh_interval_seconds = 0; + assert!(cfg.validate_refresh().is_ok()); +} + +#[test] +fn refresh_invalid_settings_fail_before_creating_cache() { + for (interval, concurrency, diagnostic) in [ + ("3599", "4", "--refresh-interval-seconds"), + ("18446744073709551615", "4", "monotonic deadline"), + ("3600", "0", "--max-concurrent-refreshes"), + ] { + let root = tempfile::tempdir().unwrap(); + let cache = root.path().join("not-created"); + let output = std::process::Command::new(env!("CARGO_BIN_EXE_git-cache-proxy")) + .args([ + "--upstream", + "https://example.com", + "--refresh-interval-seconds", + interval, + "--max-concurrent-refreshes", + concurrency, + "--cache-root", + ]) + .arg(&cache) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains(diagnostic)); + assert!(!cache.exists()); + } +}