diff --git a/Cargo.lock b/Cargo.lock index 86c5afb..3f2e1b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,6 +410,17 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "futures-sink" version = "0.3.33" @@ -429,6 +440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "slab", @@ -1612,6 +1624,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "libc", "pin-project-lite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 884d69f..0741241 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ tokio = { version = "1", features = [ "sync", "time", ] } -tokio-util = { version = "0.7", features = ["io"] } +tokio-util = { version = "0.7", features = ["io", "rt"] } tower = { version = "0.5", features = ["limit"] } tracing = "0.1" tracing-appender = "0.2" diff --git a/README.md b/README.md index 002471d..fdbe124 100644 --- a/README.md +++ b/README.md @@ -144,3 +144,20 @@ Contributions welcome. ## License [Apache-2.0](./LICENSE). Self-contained - no dependencies outside crates.io. + +### Upstream operation lifetime + +`--upstream-timeout-seconds` (`GITCACHEPROXY_UPSTREAM_TIMEOUT_SECONDS`, default +`3600`) bounds each clone, full-ref fetch and SHA backfill across attempts and +retry delays. The value must be positive and fit in a monotonic deadline. +Library users can set the same budget with `GitCache::with_upstream_timeout`. +The budget starts when the operation starts, after the repository lock is acquired. + +SIGTERM and Ctrl-C reject new upstream work and cancel active commands, retry +backoff and repository lock waits before HTTP drain. Upstream owners kill process +groups on Unix and wait for the direct child before releasing repository locks. +Dropped requests also cancel their upstream owner. Library hosts call +`GitCache::shutdown` before draining HTTP and `wait_for_shutdown` after draining. +Active local upload-pack responses continue to drain. This does not impose a hard +shutdown bound on active uploads or blocking filesystem work. On non-Unix targets, +cancellation terminates only the direct child. diff --git a/chart/README.md b/chart/README.md index 2347051..ef73c5d 100644 --- a/chart/README.md +++ b/chart/README.md @@ -113,6 +113,7 @@ pointing at the Service. | `clientAuth.allowHttp` | `false` | Accept a plain http validation endpoint (credentials in cleartext) | | `clientAuth.signingKey.existingSecret` | `""` | Secret with the HMAC key signing LFS download grants; empty = a random key per pod | | `clientAuth.signingKey.key` | `signing-key` | Key in that Secret | +| `config.upstreamTimeoutSeconds` | `3600` | Positive total seconds per upstream operation, including retries | | `config.fetchTtlSeconds` | `10` | Skip upstream fetch if refreshed within this window (`0` = always) | | `config.cacheMaxMb` | `0` | Cap on on-disk cache, MiB; evicts LRU idle mirrors (`0` = unlimited) | | `config.maxConcurrentRequests` | `64` | Max concurrent in-flight requests (`0` = unlimited) | diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 389d1f3..1c9f87e 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -1,3 +1,6 @@ +{{- if or (not (regexMatch "^[1-9][0-9]*$" (toString .Values.config.upstreamTimeoutSeconds))) (le (int64 .Values.config.upstreamTimeoutSeconds) 0) }} +{{- fail "config.upstreamTimeoutSeconds must be a positive integer" }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -63,6 +66,8 @@ spec: value: {{ .Values.upstream | quote }} - name: GITCACHEPROXY_CACHE_ROOT value: /var/cache/git-cache-proxy + - name: GITCACHEPROXY_UPSTREAM_TIMEOUT_SECONDS + value: {{ .Values.config.upstreamTimeoutSeconds | quote }} - name: GITCACHEPROXY_FETCH_TTL_SECONDS value: {{ .Values.config.fetchTtlSeconds | quote }} - name: GITCACHEPROXY_CACHE_MAX_MB diff --git a/chart/values.yaml b/chart/values.yaml index e741396..a9b896b 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -81,6 +81,8 @@ config: # Skip the upstream fetch on info/refs if the mirror was refreshed within this # many seconds; coalesces bursts of clones for one repo. 0 = always fetch. fetchTtlSeconds: 10 + # Total upstream command budget, including retries (positive seconds). + upstreamTimeoutSeconds: 3600 # Cap on total on-disk mirror cache, in MiB; evicts least-recently-used idle # mirrors when exceeded. 0 = unlimited (no eviction). Set this whenever the # cache volume is bounded. diff --git a/src/config.rs b/src/config.rs index f104922..9df1a3f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -209,6 +209,12 @@ pub struct Config { #[arg(long, env = "GITCACHEPROXY_AUTH_TTL_SECONDS", default_value_t = 0)] pub auth_ttl_seconds: u64, + /// Total seconds per upstream clone, full-ref fetch or SHA backfill, + /// including all attempts and retry delays. Must be positive. + #[arg(long, env = "GITCACHEPROXY_UPSTREAM_TIMEOUT_SECONDS", default_value = "3600", + value_parser = parse_upstream_timeout)] + pub upstream_timeout_seconds: u64, + /// Skip the upstream fetch on `info/refs` if the last fetch started within /// this many seconds. Coalesces bursts of clones for the same repo. `0` = /// always fetch (freshest, more upstream load). @@ -302,10 +308,25 @@ pub struct Config { pub log_format: LogFormat, } +fn parse_upstream_timeout(value: &str) -> Result { + let seconds = value.parse::().map_err(|e| e.to_string())?; + crate::git::validate_upstream_timeout(std::time::Duration::from_secs(seconds)) + .map_err(|e| e.to_string())?; + Ok(seconds) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn upstream_timeout_parser_matches_library_validation() { + assert_eq!(parse_upstream_timeout("3600").unwrap(), 3600); + for value in ["0", "-1", "18446744073709551615", "18446744073709551616"] { + assert!(parse_upstream_timeout(value).is_err(), "{value}"); + } + } + #[test] fn static_mode_never_conflicts() { for provider in [ diff --git a/src/git.rs b/src/git.rs index c9a1c8c..fd3975f 100644 --- a/src/git.rs +++ b/src/git.rs @@ -35,6 +35,8 @@ use tokio::process::{Child, ChildStdout, Command}; use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore, oneshot}; use tokio::time::Instant; use tokio_util::io::ReaderStream; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; use crate::metrics::{Metrics, ServeKind, Status, UpstreamOp}; use crate::repo::RepoRef; @@ -114,14 +116,18 @@ struct RepoSlot { fetch_lock: Mutex>, } +#[derive(Clone)] pub struct GitCache { + shutdown: CancellationToken, + owners: TaskTracker, + upstream_timeout: Duration, cfg: GitConfig, metrics: Arc, /// Map of repo name -> its serialization slot. A plain `Mutex` (not `RwLock`) /// because every access is a get-or-insert (`entry`), which needs a write lock /// anyway, and the critical section is a single O(1) map operation - far too /// short for reader/writer separation to pay off. - slots: Mutex>>, + slots: Arc>>>, /// Cache-size index for LRU eviction, or `None` when no cap is configured. /// When present it is `touch`ed on every request and `record`ed after each /// clone/fetch; when absent the eviction machinery is entirely inert. @@ -141,12 +147,34 @@ impl GitCache { Self { cfg, metrics, - slots: Mutex::new(HashMap::new()), + slots: Arc::new(Mutex::new(HashMap::new())), + shutdown: CancellationToken::new(), + owners: TaskTracker::new(), + upstream_timeout: Duration::from_secs(3600), index, upload_packs, } } + /// Set the total budget for each upstream operation, including retries. + pub fn with_upstream_timeout(mut self, timeout: Duration) -> Result { + validate_upstream_timeout(timeout)?; + self.upstream_timeout = timeout; + Ok(self) + } + + /// Reject new upstream work and cancel active operations before HTTP drain. + /// Local upload-pack processes continue to drain. + pub fn shutdown(&self) { + self.shutdown.cancel(); + self.owners.close(); + } + + /// Wait for upstream owners to reap children and release repository locks. + pub async fn wait_for_shutdown(&self) { + self.owners.wait().await; + } + async fn slot(&self, name: &str) -> Arc { self.slots .lock() @@ -209,6 +237,29 @@ impl GitCache { want_fetch: bool, body: &[u8], cred: Option<&str>, + ) -> Result { + let mut owner = self.clone(); + owner.shutdown = self.shutdown.child_token(); + let _cancel_on_drop = owner.shutdown.clone().drop_guard(); + let repo = repo.clone(); + let wants = parse_wants(body); + let cred = cred.map(str::to_owned); + self.owners + .spawn(async move { + owner + .ensure_fresh_owned(&repo, want_fetch, &wants, cred.as_deref()) + .await + }) + .await + .context("join upstream owner")? + } + + async fn ensure_fresh_owned( + &self, + repo: &RepoRef, + want_fetch: bool, + wants: &HashSet, + cred: Option<&str>, ) -> Result { let slot = self.slot(&repo.name).await; // Mark the repo used on every request - a served cache hit counts as much as @@ -216,7 +267,11 @@ impl GitCache { if let Some(idx) = &self.index { idx.touch(&repo.name); } - let mut last = slot.fetch_lock.lock().await; + let mut last = tokio::select! { + biased; + _ = self.shutdown.cancelled() => bail!("upstream operation cancelled"), + guard = slot.fetch_lock.lock() => guard, + }; // Stamp the start: upstream is read then, so a ref pushed mid-fetch is // missing, and a caller queued behind a slow fetch must see it as stale. @@ -240,7 +295,10 @@ impl GitCache { // Best-effort: on failure, fall through and let upload-pack surface the // normal "not our ref" error for any want it still cannot satisfy. - if let Err(e) = self.ensure_wanted_oids(repo, body, cred).await { + if let Err(e) = self.ensure_wanted_oids(repo, wants, cred).await { + if is_auth_failure(&e) || self.shutdown.is_cancelled() || e.is::() { + return Err(e); + } tracing::warn!(repo = %repo.name, error = %e, "ensure wanted oids failed"); } Ok(outcome) @@ -368,7 +426,16 @@ impl GitCache { .arg("--quiet") .arg(&repo.upstream_url) .arg(&tmp); - if let Err(error) = run_upstream(&mut cmd, "clone", &repo.name, Some(&tmp)).await { + if let Err(error) = run_upstream( + &mut cmd, + "clone", + &repo.name, + Some(&tmp), + self.upstream_timeout, + &self.shutdown, + ) + .await + { // `-` not the repo name: a failed clone must not mint a per-repo series // for an arbitrary client-supplied path (see `metrics`). The failing // repo is still named in the returned error, which the caller logs. @@ -402,7 +469,16 @@ impl GitCache { // otherwise drop those refs as "not on upstream" (see `prune_wants`). .arg("+refs/*:refs/*") .arg(format!("^{PROXY_WANTS_NS}/*")); - if let Err(error) = run_upstream(&mut cmd, "fetch", &repo.name, None).await { + if let Err(error) = run_upstream( + &mut cmd, + "fetch", + &repo.name, + None, + self.upstream_timeout, + &self.shutdown, + ) + .await + { self.metrics .record_upstream(UpstreamOp::Fetch, Status::Error, "-"); return Err(error); @@ -426,14 +502,13 @@ impl GitCache { async fn ensure_wanted_oids( &self, repo: &RepoRef, - body: &[u8], + wants: &HashSet, cred: Option<&str>, ) -> Result<()> { - let wants = parse_wants(body); if wants.is_empty() { return Ok(()); } - let mut missing = self.missing_oids(repo, &wants).await?; + let mut missing = self.missing_oids(repo, wants).await?; if missing.is_empty() { return Ok(()); } @@ -466,18 +541,19 @@ impl GitCache { // already exist), making the object a `want` tip safe from `git gc`. cmd.arg(format!("+{oid}:{PROXY_WANTS_NS}/{oid}")); } - let status = cmd - .status() - .await - .context("spawn git fetch (want-by-sha)")?; - if !status.success() { - // Upstream would not serve one of these SHAs (e.g. arbitrary-SHA fetches - // are disabled). Leave it: upload-pack will reject the want with "not - // our ref". + if let Err(error) = run_upstream( + &mut cmd, + "fetch (want-by-sha)", + &repo.name, + None, + self.upstream_timeout, + &self.shutdown, + ) + .await + { self.metrics .record_upstream(UpstreamOp::WantFetch, Status::Error, "-"); - tracing::warn!(repo = %repo.name, "want-by-sha fetch from upstream failed"); - return Ok(()); + return Err(error); } self.metrics .record_upstream(UpstreamOp::WantFetch, Status::Ok, &repo.name); @@ -649,6 +725,25 @@ impl GitCache { } } +/// Validate the budget for both library configuration and CLI parsing. +pub fn validate_upstream_timeout(timeout: Duration) -> Result<()> { + if timeout.is_zero() || Instant::now().checked_add(timeout).is_none() { + bail!("upstream timeout must be positive and fit in a monotonic deadline"); + } + Ok(()) +} + +#[derive(Debug)] +struct UpstreamStopped; + +impl std::fmt::Display for UpstreamStopped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("upstream operation cancelled or total budget exhausted") + } +} + +impl std::error::Error for UpstreamStopped {} + // The caller holds the repository lock throughout retries. Never retry local // upload-pack: once its response starts streaming it cannot be replayed safely. async fn run_upstream( @@ -656,34 +751,60 @@ async fn run_upstream( operation: &str, repo: &str, staging: Option<&Path>, + budget: Duration, + cancellation: &CancellationToken, ) -> Result<()> { + let deadline = Instant::now() + budget; cmd.stdout(Stdio::null()) .stderr(Stdio::piped()) .kill_on_drop(true); + #[cfg(unix)] + cmd.process_group(0); for attempt in 0..=MAX_UPSTREAM_RETRIES { + if cancellation.is_cancelled() || Instant::now() >= deadline { + return Err(UpstreamStopped.into()); + } if let Some(path) = staging { clean_staging(path).await?; } let mut child = cmd.spawn().context("spawn upstream git")?; let mut stderr = child.stderr.take().context("upstream git: no stderr")?; - // Drain even after the cap so a noisy child cannot block on its pipe. - // Only retain the tail; credentials and raw remote output are never logged. - let mut tail = Vec::new(); - let mut chunk = [0_u8; 4096]; - loop { - let count = stderr - .read(&mut chunk) - .await - .context("read upstream git stderr")?; - if count == 0 { - break; + let operation_result = async { + // Drain all output but retain only a bounded diagnostic tail. + let mut tail = Vec::new(); + let mut chunk = [0_u8; 4096]; + loop { + let count = stderr + .read(&mut chunk) + .await + .context("read upstream git stderr")?; + if count == 0 { + break; + } + tail.extend_from_slice(&chunk[..count]); + if tail.len() > MAX_UPSTREAM_STDERR_BYTES { + tail.drain(..tail.len() - MAX_UPSTREAM_STDERR_BYTES); + } } - tail.extend_from_slice(&chunk[..count]); - if tail.len() > MAX_UPSTREAM_STDERR_BYTES { - tail.drain(..tail.len() - MAX_UPSTREAM_STDERR_BYTES); + let status = child.wait().await.context("wait for upstream git")?; + Ok::<_, anyhow::Error>((status, tail)) + }; + let result = tokio::select! { + biased; + _ = cancellation.cancelled() => Err(UpstreamStopped.into()), + _ = tokio::time::sleep_until(deadline) => Err(UpstreamStopped.into()), + result = operation_result => result, + }; + let (status, tail) = match result { + Ok(result) => result, + Err(error) => { + stop_upstream(&mut child).await; + if let Some(path) = staging { + clean_staging(path).await?; + } + return Err(error); } - } - let status = child.wait().await.context("wait for upstream git")?; + }; if status.success() { return Ok(()); } @@ -704,7 +825,7 @@ async fn run_upstream( reason, "retrying upstream git" ); - tokio::time::sleep(*delay).await; + wait_upstream_retry(*delay, deadline, cancellation).await?; continue; } // An upstream credential rejection is carried out typed, so a `client`-mode @@ -723,6 +844,26 @@ async fn run_upstream( unreachable!("final attempt returns an error or succeeds") } +async fn stop_upstream(child: &mut Child) { + if kill_group(child).is_err() { + let _ = child.start_kill(); + } + let _ = child.wait().await; +} + +async fn wait_upstream_retry( + delay: Duration, + deadline: Instant, + cancellation: &CancellationToken, +) -> Result<()> { + tokio::select! { + biased; + _ = cancellation.cancelled() => Err(UpstreamStopped.into()), + _ = tokio::time::sleep_until(deadline) => Err(UpstreamStopped.into()), + _ = tokio::time::sleep(delay) => Ok(()), + } +} + async fn clean_staging(path: &Path) -> Result<()> { match tokio::fs::remove_dir_all(path).await { Ok(()) => Ok(()), @@ -1050,6 +1191,297 @@ mod tests { assert_eq!(missing.len(), oids.len()); } + #[tokio::test(start_paused = true)] + async fn cancelled_upstream_does_not_spawn_a_command() { + let dir = tempfile::tempdir().unwrap(); + let mut cmd = Command::new(dir.path().join("missing-git")); + let token = CancellationToken::new(); + token.cancel(); + let error = run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::from_secs(60), + &token, + ) + .await + .unwrap_err(); + assert!( + error.is::(), + "cancellation must precede spawn: {error}" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn upstream_cleanup_reaps_leader_when_process_group_is_missing() { + use std::os::unix::process::ExitStatusExt; + let mut child = Command::new("sleep").arg("30").spawn().unwrap(); + tokio::time::timeout(Duration::from_secs(5), stop_upstream(&mut child)) + .await + .expect("fallback must terminate and reap the direct child"); + assert!( + child.id().is_none(), + "child must be reaped before cleanup returns" + ); + assert_eq!(child.wait().await.unwrap().signal(), Some(9)); + } + + #[tokio::test(start_paused = true)] + async fn retry_wait_obeys_shared_deadline_and_cancellation() { + let token = CancellationToken::new(); + let started = Instant::now(); + let deadline = started + Duration::from_secs(3); + wait_upstream_retry(Duration::from_secs(2), deadline, &token) + .await + .unwrap(); + assert!( + wait_upstream_retry(Duration::from_secs(2), deadline, &token) + .await + .is_err() + ); + assert_eq!(started.elapsed(), Duration::from_secs(3)); + let started = Instant::now(); + let deadline = started + Duration::from_secs(20); + let cancel = async { + tokio::time::sleep(Duration::from_secs(1)).await; + token.cancel(); + }; + let (result, ()) = tokio::join!( + wait_upstream_retry(Duration::from_secs(10), deadline, &token), + cancel + ); + assert!(result.is_err()); + assert_eq!(started.elapsed(), Duration::from_secs(1)); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_cancels_repository_waiters_and_rejects_admission() { + let cache = GitCache::new(test_config(), Arc::new(Metrics::new()), None); + let root = tempfile::tempdir().unwrap(); + let repo = crate::repo::resolve("repo.git", "https://example.com", root.path()).unwrap(); + let slot = cache.slot(&repo.name).await; + let guard = slot.fetch_lock.lock().await; + let wants = HashSet::new(); + let waiter = cache.ensure_fresh_owned(&repo, true, &wants, None); + let shutdown = async { + cache.shutdown(); + }; + let (result, ()) = tokio::join!(waiter, shutdown); + assert!(result.is_err()); + cache.wait_for_shutdown().await; + drop(guard); + assert!(slot.fetch_lock.try_lock().is_ok()); + assert!(cache.ensure_fresh(&repo, true, b"", None).await.is_err()); + cache.wait_for_shutdown().await; + assert!(cache.owners.is_empty()); + } + + #[cfg(unix)] + #[tokio::test] + async fn upstream_shutdown_preserves_active_upload_and_releases_its_permit() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let release = root.path().join("release"); + assert!( + std::process::Command::new("mkfifo") + .arg(&release) + .status() + .unwrap() + .success() + ); + let binary = root.path().join("git-wrapper"); + // Keep the writable descriptor out of parallel test processes' forked children. + assert!( + std::process::Command::new("sh") + .args(["-c", "printf '%s' \"$2\" > \"$1\"", "sh"]) + .arg(&binary) + .arg(format!( + "#!/bin/sh\ncat >/dev/null\ncat '{}' >/dev/null\nprintf pack\n", + release.display() + )) + .status() + .unwrap() + .success() + ); + std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut config = test_config(); + config.git_binary = binary.to_str().unwrap().into(); + let cache = GitCache::new(config, Arc::new(Metrics::new()), None); + let repo = crate::repo::resolve("repo.git", "https://example.com", root.path()).unwrap(); + let stream = cache + .upload_pack_rpc(&repo, None, Bytes::new()) + .await + .unwrap(); + assert_eq!(cache.upload_packs.as_ref().unwrap().available_permits(), 0); + cache.shutdown(); + cache.wait_for_shutdown().await; + tokio::fs::write(&release, b"").await.unwrap(); + let mut body = Vec::new(); + tokio::time::timeout( + Duration::from_secs(5), + tokio_util::io::StreamReader::new(stream).read_to_end(&mut body), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(body, b"pack"); + let permit = tokio::time::timeout( + Duration::from_secs(5), + cache.upload_packs.as_ref().unwrap().acquire(), + ) + .await + .unwrap() + .unwrap(); + drop(permit); + assert_eq!(cache.upload_packs.as_ref().unwrap().available_permits(), 1); + } + + #[test] + fn upstream_budget_rejects_zero_and_overflow() { + assert!(validate_upstream_timeout(Duration::ZERO).is_err()); + assert!(validate_upstream_timeout(Duration::MAX).is_err()); + assert!(validate_upstream_timeout(Duration::from_secs(3600)).is_ok()); + } + + #[cfg(unix)] + #[tokio::test] + async fn upstream_budget_stops_stalled_and_noisy_commands() { + for script in ["sleep 30", "while :; do echo private-diagnostic >&2; done"] { + let token = CancellationToken::new(); + let mut cmd = Command::new("sh"); + cmd.args(["-c", script]); + let error = run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::from_millis(100), + &token, + ) + .await + .unwrap_err(); + assert!(error.is::()); + assert!(!error.to_string().contains("private-diagnostic")); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn upstream_cancellation_reaps_child_and_stops_descendants() { + let root = tempfile::tempdir().unwrap(); + let pidfile = root.path().join("pids"); + assert!( + std::process::Command::new("mkfifo") + .arg(&pidfile) + .status() + .unwrap() + .success() + ); + let token = CancellationToken::new(); + let mut cmd = Command::new("sh"); + cmd.arg("-c") + .arg("sleep 30 & echo \"$$ $!\" > \"$1\"; wait") + .arg("test") + .arg(&pidfile); + let cancel = async { + let pids = tokio::fs::read_to_string(&pidfile).await.unwrap(); + token.cancel(); + pids + }; + let (result, pids) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!( + run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::from_secs(30), + &token + ), + cancel + ) + }) + .await + .unwrap(); + assert!(result.unwrap_err().is::()); + let mut pids = pids.split_whitespace(); + let leader = pids.next().unwrap(); + assert!( + process_state(leader).await.is_empty(), + "leader must be reaped" + ); + for pid in pids { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let state = process_state(pid).await; + if state.is_empty() || state.starts_with('Z') { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("descendant must stop after group kill"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn upstream_budget_is_shared_across_retry_backoff() { + let token = CancellationToken::new(); + let mut cmd = Command::new("sh"); + cmd.args([ + "-c", + "echo 'Recv failure: Connection reset by peer' >&2; exit 128", + ]); + let started = Instant::now(); + let error = run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::from_millis(100), + &token, + ) + .await + .unwrap_err(); + assert!(error.is::()); + assert!(started.elapsed() < Duration::from_secs(1)); + } + + #[cfg(unix)] + #[tokio::test] + async fn upstream_success_within_budget_and_typed_auth_error() { + let token = CancellationToken::new(); + let mut cmd = Command::new("sh"); + cmd.args(["-c", "sleep 0.1"]); + run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::from_secs(2), + &token, + ) + .await + .unwrap(); + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo 'fatal: Authentication failed' >&2; exit 128"]); + let error = run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::from_secs(2), + &token, + ) + .await + .unwrap_err(); + assert!(is_auth_failure(&error)); + } + #[test] fn upstream_retry_classifies_transient_diagnostics() { for message in [ @@ -1329,4 +1761,27 @@ mod tests { served: Some(served), } } + fn test_config() -> GitConfig { + GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(60), + max_wants: 0, + max_upload_packs: 1, + pack_threads: None, + pack_window_memory: None, + } + } + + /// `ps` state of `pid`: empty once reaped, `Z` while a zombie awaits reaping. + /// `kill -0` reports zombies as alive, and /proc is Linux-only. + async fn process_state(pid: &str) -> String { + let out = Command::new("ps") + .args(["-o", "stat=", "-p", pid]) + .output() + .await + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_owned() + } } diff --git a/src/main.rs b/src/main.rs index e0ec343..a762ff3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -89,7 +89,10 @@ async fn main() -> Result<()> { ) }); - let cache = Arc::new(git::GitCache::new(git_cfg, metrics.clone(), index.clone())); + let cache = Arc::new( + git::GitCache::new(git_cfg, metrics.clone(), index.clone()) + .with_upstream_timeout(Duration::from_secs(cfg.upstream_timeout_seconds))?, + ); let upstream_base = cfg.upstream.trim_end_matches('/').to_string(); @@ -161,11 +164,17 @@ async fn main() -> Result<()> { cfg.cache_root.display(), ); + let shutdown_cache = cache.clone(); axum::serve(listener, server::router(state)) - .with_graceful_shutdown(shutdown_signal()) + .with_graceful_shutdown(async move { + shutdown_signal().await; + shutdown_cache.shutdown(); + }) .await .context("http server")?; + cache.wait_for_shutdown().await; + // The server has drained; stop the evictor and wait for it to finish any // in-progress eviction before exiting. if let Some((shutdown_tx, handle)) = evictor { diff --git a/tests/lifecycle.rs b/tests/lifecycle.rs new file mode 100644 index 0000000..7a9499e --- /dev/null +++ b/tests/lifecycle.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Exercise real signals against the binary and a stalled upstream process group. +#![cfg(unix)] + +use std::os::unix::fs::PermissionsExt; +use std::process::Stdio; +use std::time::Duration; + +use rustix::process::{Pid, Signal, kill_process}; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; + +#[tokio::test] +async fn signals_cancel_upstream_before_http_drain() { + for signal in [Signal::TERM, Signal::INT] { + let root = tempfile::tempdir().unwrap(); + let ready = root.path().join("ready"); + assert!( + std::process::Command::new("mkfifo") + .arg(&ready) + .status() + .unwrap() + .success() + ); + let wrapper = root.path().join("git-wrapper"); + std::fs::write( + &wrapper, + "#!/bin/sh\nsleep 30 &\necho \"$$ $!\" > \"$READY_FIFO\"\nwait\n", + ) + .unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + let mut server = Command::new(env!("CARGO_BIN_EXE_git-cache-proxy")) + .args([ + "--upstream", + "https://example.com", + "--bind", + &address.to_string(), + "--upstream-timeout-seconds", + "10", + ]) + .arg("--cache-root") + .arg(root.path().join("cache")) + .arg("--git-binary") + .arg(&wrapper) + .env("READY_FIFO", &ready) + .env_remove("RUST_LOG") + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut lines = BufReader::new(server.stdout.take().unwrap()).lines(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let line = lines + .next_line() + .await + .unwrap() + .expect("server exited before binding"); + if line.contains("listening on") { + break; + } + } + }) + .await + .expect("server startup"); + let request = tokio::spawn(reqwest::get(format!( + "http://{address}/repo.git/info/refs?service=git-upload-pack" + ))); + let pids = tokio::time::timeout(Duration::from_secs(5), tokio::fs::read_to_string(&ready)) + .await + .expect("upstream started") + .unwrap(); + kill_process(Pid::from_raw(server.id().unwrap() as i32).unwrap(), signal).unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), server.wait()) + .await + .expect("shutdown must cancel upstream before drain") + .unwrap(); + assert!(status.success(), "signal {signal:?}: {status}"); + let response = request.await.unwrap().unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::BAD_GATEWAY); + let mut pids = pids.split_whitespace(); + let leader = pids.next().unwrap(); + assert!( + process_state(leader).await.is_empty(), + "leader must be reaped" + ); + let descendant = pids.next().unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let state = process_state(descendant).await; + if state.is_empty() || state.starts_with('Z') { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("descendant must stop after group kill"); + } +} + +/// `ps` state of `pid`: empty once reaped, `Z` while a zombie awaits reaping. +/// `kill -0` reports zombies as alive, and /proc is Linux-only. +async fn process_state(pid: &str) -> String { + let out = Command::new("ps") + .args(["-o", "stat=", "-p", pid]) + .output() + .await + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_owned() +} diff --git a/tests/retry.rs b/tests/retry.rs index 97591d3..b04d57b 100644 --- a/tests/retry.rs +++ b/tests/retry.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 -//! Real local Git operations with deterministic injected upstream failures. +//! Real local Git operations with injected upstream failures. +//! Keep real time while child processes run; a paused clock can expire the total +//! operation deadline before the operating system schedules a child. #![cfg(unix)] use std::os::unix::fs::PermissionsExt; @@ -12,7 +14,7 @@ use git_cache_proxy::git::{CacheOutcome, GitCache, GitConfig}; use git_cache_proxy::metrics::Metrics; use git_cache_proxy::repo::{RepoRef, resolve}; -#[tokio::test(start_paused = true)] +#[tokio::test] async fn clone_retries_tls_eof_and_removes_partial_staging() { let f = Fixture::new(); f.fail( @@ -41,7 +43,7 @@ async fn clone_retries_tls_eof_and_removes_partial_staging() { assert_eq!(2, f.attempts("clone")); } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn fetch_retries_http_503_and_keeps_the_mirror() { let f = Fixture::new(); f.seed_mirror(); @@ -59,7 +61,7 @@ async fn fetch_retries_http_503_and_keeps_the_mirror() { assert!(f.repo.cache_dir.join("HEAD").exists()); } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn retries_are_bounded_and_failure_does_not_refresh_ttl() { for op in ["clone", "fetch"] { let f = Fixture::new(); @@ -83,7 +85,7 @@ async fn retries_are_bounded_and_failure_does_not_refresh_ttl() { } } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn permanent_errors_are_not_retried() { for op in ["clone", "fetch"] { for message in [ @@ -114,7 +116,7 @@ async fn permanent_errors_are_not_retried() { } } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn concurrent_clients_share_one_retry_sequence() { let f = Fixture::new(); f.fail( @@ -144,7 +146,7 @@ async fn concurrent_clients_share_one_retry_sequence() { assert_eq!(2, f.attempts("clone")); } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn waiter_behind_a_slow_fetch_refetches_once_the_ttl_has_passed() { // TTL is measured from the start of a fetch, the moment it reads upstream. // Three retries take 7s, so a caller queued behind them sees stale refs. @@ -160,7 +162,7 @@ async fn waiter_behind_a_slow_fetch_refetches_once_the_ttl_has_passed() { assert_eq!(5, f.attempts("fetch")); } -#[tokio::test(start_paused = true)] +#[tokio::test] async fn noisy_stderr_is_drained_and_not_exposed_in_errors() { let f = Fixture::new(); let message = format!( @@ -179,6 +181,67 @@ async fn noisy_stderr_is_drained_and_not_exposed_in_errors() { assert!(error.contains("TLS connection interrupted")); } +#[tokio::test] +async fn sha_backfill_propagates_auth_but_preserves_best_effort_failure() { + for (message, auth) in [ + ("fatal: Authentication failed", true), + ("fatal: not our ref", false), + ] { + let f = Fixture::new(); + f.seed_mirror(); + f.fail("fetch", 10, message); + let result = f + .cache + .ensure_fresh( + &f.repo, + false, + b"0032want 1111111111111111111111111111111111111111\n0000", + None, + ) + .await; + if auth { + assert!(git_cache_proxy::git::is_auth_failure(&result.unwrap_err())); + } else { + assert!(matches!(result.unwrap(), CacheOutcome::Cached)); + } + assert_eq!(f.attempts("fetch"), 1); + } +} + +#[tokio::test] +async fn dropped_request_cancels_owner_and_releases_repository_lock() { + let f = Fixture::new(); + std::fs::write(f.root.path().join("stall-clone"), "").unwrap(); + let ready = f.root.path().join("ready-clone"); + assert!( + Command::new("mkfifo") + .arg(&ready) + .status() + .unwrap() + .success() + ); + let cache = f.cache.clone(); + let repo = f.repo.clone(); + let request = tokio::spawn(async move { cache.ensure_fresh(&repo, true, b"", None).await }); + tokio::time::timeout(Duration::from_secs(5), tokio::fs::read_to_string(ready)) + .await + .unwrap() + .unwrap(); + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + std::fs::remove_file(f.root.path().join("stall-clone")).unwrap(); + let result = tokio::time::timeout( + Duration::from_secs(5), + f.cache.ensure_fresh(&f.repo, true, b"", None), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!(result, CacheOutcome::Cloned)); + f.cache.shutdown(); + f.cache.wait_for_shutdown().await; +} + struct Fixture { root: tempfile::TempDir, repo: RepoRef, @@ -202,7 +265,11 @@ impl Fixture { set -eu base=$(dirname "$0") op=$1 -test "$LC_ALL" = C +if test -e "$base/stall-$op"; then + echo ready > "$base/ready-$op" + sleep 30 +fi +case "$op" in clone|fetch) test "$LC_ALL" = C ;; esac count=0 test ! -f "$base/count-$op" || count=$(cat "$base/count-$op") count=$((count + 1))