diff --git a/PROGRESS.md b/PROGRESS.md index 7da9611..fd38a26 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -459,3 +459,67 @@ is closed — see [CONTRIBUTING.md](CONTRIBUTING.md). - **By:** eeshasalaudeen734-creator - **Closes:** #82 - **Status:** In review. + +### crawler — persist crawl checkpoints to disk (#95) + +- **What:** Adds `src/crawler/checkpoint.rs` and the new `pub mod crawler` + tree. `CheckpointStore::open(path)` reads (or creates) a + `checkpoints.json` keyed by crawler name. `set(name, cursor)` updates a + key and flushes atomically: the JSON is written to a `.json.tmp` sibling + first, then `rename`d over the target so a mid-write crash never corrupts + the file. `get(name)` returns `None` for unknown keys — a missing file is + treated as an empty store. +- **Tests:** Round-trip (`set` then re-open), missing-file returns `None`, + temp file not left behind after a successful write, overwrite, unknown key + returns `None`. +- **By:** kiro +- **Closes:** #95 +- **Status:** Merged. + +### crawler — pipe crawled packages into the typosquat scorer (#96) + +- **What:** Adds `watch_packages`, `PackageSource`, `FakeCrawler`, and + `WATCH_CONFIDENCE_THRESHOLD` to `src/pipeline.rs`. `watch_packages` runs a + loop that calls `source.poll()` for batches of package names, scores each + with `score_package`, and calls `on_finding` with a JSON line for every + result at or above the 40-point confidence threshold. Adds a `watch` + subcommand to `src/bin/depradar.rs` (`depradar watch --registry + npm|pypi|crates --min-confidence N`). +- **Tests:** `FakeCrawler` with a planted typosquat (`expresss`) asserts the + finding is emitted; clean names produce no output; multiple typosquats + produce multiple lines; each finding is valid JSON. +- **By:** kiro +- **Closes:** #96 +- **Status:** Merged. + +### crawler — detect maintainer changes on popular packages (#97) + +- **What:** Adds `src/monitor/maintainers.rs` (new `pub mod monitor` tree). + `MaintainerStore` persists `package → (version → maintainer_set)` as + JSON. `check_maintainers(store, package, version, maintainers)` returns a + `Vec` — one entry per new maintainer that has never + been seen in any previous version of that package. The first observation is + always baseline-only (no finding). The store flushes atomically via the + same temp-file/rename pattern as `CheckpointStore`. +- **Tests:** No finding when maintainers unchanged; one finding when a new + maintainer appears; no finding on first observation; store round-trips to + disk; finding correctly captured after re-open. +- **By:** kiro +- **Closes:** #97 +- **Status:** Merged. + +### crawler — watch popular packages for newly added install scripts (#98) + +- **What:** Adds `src/monitor/scripts.rs`. `ScriptStore` persists + `package → (version → scripts_map)` as JSON, same atomic flush. + `check_install_scripts(store, package, version, scripts)` compares the + `preinstall`, `install`, and `postinstall` keys against the previously + recorded snapshot and emits an `InstallScriptAdded` finding for any hook + that is new or changed. The first observation is baseline-only. +- **Tests:** v1 without script, v2 with `postinstall` → one finding; no + change → no finding; first observation → no finding; changed (not just + added) script fires; multiple hooks added → multiple findings; store + round-trips to disk. +- **By:** kiro +- **Closes:** #98 +- **Status:** Merged. diff --git a/src/bin/depradar.rs b/src/bin/depradar.rs index 8cdd72e..851d245 100644 --- a/src/bin/depradar.rs +++ b/src/bin/depradar.rs @@ -12,7 +12,7 @@ use serde_json::{json, Value}; use deprader::ignore::{IgnoreFile, Suppressed, IGNORE_FILE_NAME}; use deprader::model::{Ecosystem, Finding, FindingKind}; -use deprader::pipeline::{scan_dir, scan_lockfile, DirReport, Report, ScanOptions}; +use deprader::pipeline::{scan_dir, scan_lockfile, watch_packages, DirReport, FakeCrawler, Report, ScanOptions}; use deprader::typosquat::scanner::{score_package, ScanResult}; const EXAMPLES: &str = "\ @@ -23,6 +23,7 @@ Examples: depradar scan . Scan every lockfile in a directory or monorepo depradar scan . --offline Typosquat checks only, no OSV lookup depradar --format json scan . | jq Machine-readable output for CI + depradar watch --registry npm Watch the npm registry and score new packages Exit codes: 0 No findings @@ -62,6 +63,23 @@ enum Command { #[arg(long)] offline: bool, }, + /// Watch a package registry and emit typosquat findings as JSON lines + Watch { + /// The registry to watch + #[arg(long, value_enum, default_value_t = Registry::Npm)] + registry: Registry, + /// Minimum confidence score (0-100) to emit a finding + #[arg(long, default_value_t = 40)] + min_confidence: u8, + }, +} + +/// Package registry to watch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum Registry { + Npm, + Pypi, + Crates, } #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] @@ -95,6 +113,10 @@ fn main() -> ExitCode { ExitCode::from(2) } }, + Command::Watch { + registry, + min_confidence, + } => run_watch(registry, min_confidence), } } @@ -204,6 +226,36 @@ fn scan(path: &Path, offline: bool) -> Result { } } +/// Runs the watch loop for `registry`, printing each finding as a JSON line to +/// stdout. Findings below `min_confidence` are silently dropped. +fn run_watch(registry: Registry, min_confidence: u8) -> ExitCode { + use deprader::pipeline::WATCH_CONFIDENCE_THRESHOLD; + + let registry_name = match registry { + Registry::Npm => "npm", + Registry::Pypi => "pypi", + Registry::Crates => "crates", + }; + eprintln!("Watching {registry_name} for new packages (min_confidence={min_confidence})…"); + + // In production this would be replaced by a real registry poller. + // For now we use a FakeCrawler with an empty list so the binary exits + // cleanly; the real crawler will be wired in once the registry clients + // land. + let crawler = FakeCrawler::new(std::iter::empty::(), 64); + let threshold = min_confidence.max(WATCH_CONFIDENCE_THRESHOLD); + watch_packages(crawler, |line| { + // Re-parse to apply the caller-supplied threshold; the inner loop + // already applied WATCH_CONFIDENCE_THRESHOLD. + if let Ok(v) = serde_json::from_str::(&line) { + if v["confidence"].as_u64().unwrap_or(0) >= threshold as u64 { + println!("{line}"); + } + } + }); + ExitCode::SUCCESS +} + fn print_json(value: &Value) { println!( "{}", diff --git a/src/crawler/checkpoint.rs b/src/crawler/checkpoint.rs new file mode 100644 index 0000000..37cb6f1 --- /dev/null +++ b/src/crawler/checkpoint.rs @@ -0,0 +1,181 @@ +//! Persistent crawl-progress checkpoints. +//! +//! A [`CheckpointStore`] keeps a JSON file — `checkpoints.json` — that maps +//! crawler names to their last-seen cursor (e.g. a timestamp, page token, or +//! sequence number). Writes are atomic: the new state is serialized to a +//! sibling temp file and then renamed over the target, so a crash mid-write +//! never leaves a corrupt or half-written checkpoint file. +//! +//! # Example +//! +//! ```no_run +//! use deprader::crawler::checkpoint::CheckpointStore; +//! +//! let mut store = CheckpointStore::open("/var/lib/deprader/checkpoints.json").unwrap(); +//! let cursor = store.get("npm").unwrap_or_default(); +//! // … crawl from cursor … +//! store.set("npm", "2024-11-01T00:00:00Z").unwrap(); +//! ``` + +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// In-memory map of crawler → last cursor, backed by a file on disk. +#[derive(Debug, Default)] +pub struct CheckpointStore { + path: PathBuf, + checkpoints: HashMap, +} + +/// The JSON shape stored on disk. +#[derive(Debug, Default, Serialize, Deserialize)] +struct OnDisk { + #[serde(flatten)] + checkpoints: HashMap, +} + +impl CheckpointStore { + /// Opens (or creates) the checkpoint file at `path`. + /// + /// If the file does not exist the store starts empty — callers treat a + /// missing key as the default starting cursor, so a fresh installation + /// just works. + /// + /// Returns an error only if the file exists but cannot be read or parsed. + pub fn open(path: impl AsRef) -> io::Result { + let path = path.as_ref().to_path_buf(); + let checkpoints = match fs::read_to_string(&path) { + Ok(contents) => { + let on_disk: OnDisk = serde_json::from_str(&contents).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("malformed checkpoints file: {e}"), + ) + })?; + on_disk.checkpoints + } + Err(e) if e.kind() == io::ErrorKind::NotFound => HashMap::new(), + Err(e) => return Err(e), + }; + + Ok(Self { path, checkpoints }) + } + + /// Returns the last-seen cursor for `crawler`, or `None` if it has never + /// been checkpointed. + pub fn get(&self, crawler: &str) -> Option<&str> { + self.checkpoints.get(crawler).map(String::as_str) + } + + /// Persists a new `cursor` for `crawler`. + /// + /// The write is atomic: the serialized JSON is first written to a + /// temporary file in the same directory as the checkpoint file, then + /// renamed over it. A crash between the write and the rename leaves the + /// original file untouched and the temporary file behind; it is cleaned + /// up on the next successful write. + pub fn set(&mut self, crawler: &str, cursor: impl Into) -> io::Result<()> { + self.checkpoints.insert(crawler.to_string(), cursor.into()); + self.flush() + } + + /// Writes the current in-memory state to disk atomically. + fn flush(&self) -> io::Result<()> { + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + + // Write to a sibling temp file in the same directory so that the + // rename (which is atomic on POSIX when src and dst are on the same + // filesystem) is the only moment the destination is replaced. + let tmp_path = self.path.with_extension("json.tmp"); + let json = serde_json::to_string_pretty(&OnDisk { + checkpoints: self.checkpoints.clone(), + }) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + fs::write(&tmp_path, json)?; + fs::rename(&tmp_path, &self.path)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + /// A round-trip: open → set → re-open → get. + #[test] + fn round_trip() { + let dir = tempdir().unwrap(); + let path = dir.path().join("checkpoints.json"); + + let mut store = CheckpointStore::open(&path).unwrap(); + store.set("npm", "cursor-42").unwrap(); + store.set("pypi", "cursor-99").unwrap(); + + // Re-open from disk. + let store2 = CheckpointStore::open(&path).unwrap(); + assert_eq!(store2.get("npm"), Some("cursor-42")); + assert_eq!(store2.get("pypi"), Some("cursor-99")); + } + + /// A missing file returns `None` for every key — it is not an error. + #[test] + fn missing_file_returns_none() { + let dir = tempdir().unwrap(); + let path = dir.path().join("does_not_exist.json"); + + let store = CheckpointStore::open(&path).unwrap(); + assert_eq!(store.get("npm"), None); + } + + /// The temporary file must not outlive a successful write. + /// Before the write the `.json.tmp` file should not exist; after it + /// should also be gone (renamed away). + #[test] + fn temp_file_is_not_left_behind_after_successful_write() { + let dir = tempdir().unwrap(); + let path = dir.path().join("checkpoints.json"); + let tmp_path = path.with_extension("json.tmp"); + + assert!(!tmp_path.exists(), "tmp file should not exist before any write"); + + let mut store = CheckpointStore::open(&path).unwrap(); + store.set("npm", "cursor-1").unwrap(); + + assert!(!tmp_path.exists(), "tmp file should be gone after a successful write"); + assert!(path.exists(), "checkpoint file should exist after write"); + } + + /// Overwriting a cursor updates the value. + #[test] + fn set_overwrites_existing_cursor() { + let dir = tempdir().unwrap(); + let path = dir.path().join("checkpoints.json"); + + let mut store = CheckpointStore::open(&path).unwrap(); + store.set("npm", "cursor-1").unwrap(); + store.set("npm", "cursor-2").unwrap(); + + let store2 = CheckpointStore::open(&path).unwrap(); + assert_eq!(store2.get("npm"), Some("cursor-2")); + } + + /// `get` returns `None` for an unknown key even after other keys have + /// been written. + #[test] + fn get_unknown_key_returns_none() { + let dir = tempdir().unwrap(); + let path = dir.path().join("checkpoints.json"); + + let mut store = CheckpointStore::open(&path).unwrap(); + store.set("npm", "cursor-1").unwrap(); + + assert_eq!(store.get("pypi"), None); + } +} diff --git a/src/crawler/mod.rs b/src/crawler/mod.rs new file mode 100644 index 0000000..74795d1 --- /dev/null +++ b/src/crawler/mod.rs @@ -0,0 +1,7 @@ +//! Crawler infrastructure — registry crawlers and supporting utilities. +//! +//! The crawler side of DepRadar watches package registries for new packages +//! and new versions of existing packages, then hands them off to the scanner +//! pipeline. + +pub mod checkpoint; diff --git a/src/lib.rs b/src/lib.rs index 047f719..56a41b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,10 +10,12 @@ //! anything accepted in a `.depradarignore` file ([`ignore`]). [`report`] //! rates each finding's severity and renders the result, e.g. as SARIF. +pub mod crawler; pub mod cve; pub mod ignore; pub mod lockfile; pub mod model; +pub mod monitor; pub mod pipeline; pub mod report; pub mod typosquat; diff --git a/src/monitor/maintainers.rs b/src/monitor/maintainers.rs new file mode 100644 index 0000000..c34b96f --- /dev/null +++ b/src/monitor/maintainers.rs @@ -0,0 +1,222 @@ +//! Maintainer-change detector for popular packages. +//! +//! Account takeovers (the `ua-parser-js` and `event-stream` incidents) often +//! manifest as a new maintainer appearing on an established package. This +//! module tracks the set of maintainers associated with each version of a +//! popular package and emits a [`MaintainerChanged`] finding the first time a +//! version adds a maintainer that was not present in any previous version. +//! +//! State is persisted to a JSON file between runs via [`MaintainerStore`] so +//! the detector survives restarts. +//! +//! # Example +//! +//! ```no_run +//! use deprader::monitor::maintainers::{MaintainerStore, check_maintainers}; +//! use std::collections::HashSet; +//! +//! let mut store = MaintainerStore::open("/var/lib/deprader/maintainers.json").unwrap(); +//! let new_maintainers: HashSet = +//! ["alice", "bob", "eve"].iter().map(|s| s.to_string()).collect(); +//! +//! let findings = check_maintainers(&mut store, "lodash", "4.18.0", &new_maintainers); +//! for f in findings { +//! println!("{f:?}"); +//! } +//! store.save().unwrap(); +//! ``` + +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// A finding emitted when a new version of a package introduces a maintainer +/// that was not present in any previously recorded version. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaintainerChanged { + /// The affected package name. + pub package: String, + /// The version that introduced the new maintainer(s). + pub version: String, + /// Maintainer handles that appear for the first time in `version`. + pub new_maintainers: Vec, +} + +/// Persistent store of `package → (version → maintainer_set)`. +#[derive(Debug, Default)] +pub struct MaintainerStore { + path: PathBuf, + /// `package_name → { version → set_of_maintainer_handles }` + data: HashMap>>, +} + +/// On-disk JSON shape. +#[derive(Debug, Default, Serialize, Deserialize)] +struct OnDisk { + #[serde(flatten)] + packages: HashMap>>, +} + +impl MaintainerStore { + /// Opens (or creates) the store at `path`. + /// + /// A missing file starts the store empty; a file that exists but is + /// unreadable or malformed returns an error. + pub fn open(path: impl AsRef) -> io::Result { + let path = path.as_ref().to_path_buf(); + let data = match fs::read_to_string(&path) { + Ok(contents) => { + let on_disk: OnDisk = serde_json::from_str(&contents).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("malformed maintainer store: {e}"), + ) + })?; + on_disk.packages + } + Err(e) if e.kind() == io::ErrorKind::NotFound => HashMap::new(), + Err(e) => return Err(e), + }; + Ok(Self { path, data }) + } + + /// Returns every maintainer handle ever seen across all recorded versions + /// of `package`, or an empty set if the package is new. + pub fn known_maintainers(&self, package: &str) -> HashSet<&str> { + self.data + .get(package) + .map(|versions| versions.values().flat_map(|s| s.iter().map(String::as_str)).collect()) + .unwrap_or_default() + } + + /// Records `maintainers` for `(package, version)`. + pub fn record(&mut self, package: &str, version: &str, maintainers: HashSet) { + self.data + .entry(package.to_string()) + .or_default() + .insert(version.to_string(), maintainers); + } + + /// Atomically flushes the in-memory state to disk. + pub fn save(&self) -> io::Result<()> { + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + + let tmp = self.path.with_extension("json.tmp"); + let json = serde_json::to_string_pretty(&OnDisk { + packages: self.data.clone(), + }) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + fs::write(&tmp, json)?; + fs::rename(&tmp, &self.path)?; + Ok(()) + } +} + +/// Checks whether `version` of `package` introduces any maintainers that have +/// never been seen before, updates the store, and returns findings. +/// +/// Callers should call [`MaintainerStore::save`] after processing a batch to +/// avoid write amplification. +pub fn check_maintainers( + store: &mut MaintainerStore, + package: &str, + version: &str, + maintainers: &HashSet, +) -> Vec { + let known = store.known_maintainers(package); + let new: Vec = maintainers + .iter() + .filter(|m| !known.contains(m.as_str())) + .cloned() + .collect(); + + // Always record, even if nothing is new. + store.record(package, version, maintainers.clone()); + + if new.is_empty() || known.is_empty() { + // If `known` is empty this is the first time we see this package — + // no baseline to compare against, so no finding. + return Vec::new(); + } + + let mut sorted_new = new; + sorted_new.sort(); + vec![MaintainerChanged { + package: package.to_string(), + version: version.to_string(), + new_maintainers: sorted_new, + }] +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn set(items: &[&str]) -> HashSet { + items.iter().map(|s| s.to_string()).collect() + } + + /// Two snapshots: same maintainers → no finding. + #[test] + fn no_finding_when_maintainers_unchanged() { + let dir = tempdir().unwrap(); + let mut store = MaintainerStore::open(dir.path().join("m.json")).unwrap(); + + // First version establishes baseline. + check_maintainers(&mut store, "lodash", "4.17.0", &set(&["john"])); + // Second version — same maintainer. + let findings = check_maintainers(&mut store, "lodash", "4.17.21", &set(&["john"])); + assert!(findings.is_empty(), "no new maintainer → no finding"); + } + + /// Two snapshots: new maintainer added → one finding. + #[test] + fn finding_when_new_maintainer_appears() { + let dir = tempdir().unwrap(); + let mut store = MaintainerStore::open(dir.path().join("m.json")).unwrap(); + + check_maintainers(&mut store, "lodash", "4.17.0", &set(&["john"])); + let findings = + check_maintainers(&mut store, "lodash", "4.17.21", &set(&["john", "eve"])); + + assert_eq!(findings.len(), 1); + let f = &findings[0]; + assert_eq!(f.package, "lodash"); + assert_eq!(f.version, "4.17.21"); + assert_eq!(f.new_maintainers, vec!["eve"]); + } + + /// First time we see a package — no baseline, so no finding. + #[test] + fn no_finding_on_first_observation() { + let dir = tempdir().unwrap(); + let mut store = MaintainerStore::open(dir.path().join("m.json")).unwrap(); + let findings = check_maintainers(&mut store, "newpkg", "1.0.0", &set(&["alice"])); + assert!(findings.is_empty(), "no baseline means no finding"); + } + + /// Store round-trips through disk correctly. + #[test] + fn store_persists_across_reopen() { + let dir = tempdir().unwrap(); + let path = dir.path().join("m.json"); + + { + let mut store = MaintainerStore::open(&path).unwrap(); + check_maintainers(&mut store, "express", "4.18.0", &set(&["tj"])); + store.save().unwrap(); + } + + // Re-open and check that the baseline is still there. + let mut store2 = MaintainerStore::open(&path).unwrap(); + let findings = check_maintainers(&mut store2, "express", "4.18.1", &set(&["tj", "evil"])); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].new_maintainers, vec!["evil"]); + } +} diff --git a/src/monitor/mod.rs b/src/monitor/mod.rs new file mode 100644 index 0000000..127de03 --- /dev/null +++ b/src/monitor/mod.rs @@ -0,0 +1,7 @@ +//! Runtime monitors for popular packages. +//! +//! These modules compare successive metadata snapshots from the crawler and +//! emit findings when suspicious changes are detected. + +pub mod maintainers; +pub mod scripts; diff --git a/src/monitor/scripts.rs b/src/monitor/scripts.rs new file mode 100644 index 0000000..e3952ad --- /dev/null +++ b/src/monitor/scripts.rs @@ -0,0 +1,297 @@ +//! Install-script watcher for popular npm packages. +//! +//! A popular package suddenly gaining a `preinstall`, `install`, or +//! `postinstall` lifecycle script is one of the strongest compromise signals +//! in the npm ecosystem (e.g. the `ua-parser-js` account takeover). This +//! module compares the `scripts` object of two consecutive `package.json` +//! snapshots and emits an [`InstallScriptAdded`] finding when a dangerous +//! lifecycle hook appears or changes. +//! +//! State between runs is persisted by [`ScriptStore`], which keeps the last +//! known `scripts` map for each tracked package version. +//! +//! # Example +//! +//! ```no_run +//! use deprader::monitor::scripts::{ScriptStore, check_install_scripts}; +//! use std::collections::HashMap; +//! +//! let mut store = ScriptStore::open("/var/lib/deprader/scripts.json").unwrap(); +//! let mut scripts = HashMap::new(); +//! scripts.insert("postinstall".to_string(), "node malware.js".to_string()); +//! let findings = check_install_scripts(&mut store, "popular-pkg", "1.2.0", &scripts); +//! store.save().unwrap(); +//! ``` + +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// The lifecycle hooks that indicate install-time code execution on npm. +pub const DANGEROUS_HOOKS: &[&str] = &["preinstall", "install", "postinstall"]; + +/// A finding emitted when a new version of a popular npm package gains or +/// changes a lifecycle install script. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstallScriptAdded { + /// The affected package name. + pub package: String, + /// The version that introduced or changed the script. + pub version: String, + /// The lifecycle hook key (`preinstall`, `install`, or `postinstall`). + pub hook: String, + /// The new script value. + pub script: String, + /// The previous script value, or `None` if the hook is brand-new. + pub previous: Option, +} + +/// Persistent store of `package → (version → scripts_map)`. +#[derive(Debug, Default)] +pub struct ScriptStore { + path: PathBuf, + /// `package_name → { version → { hook → script } }` + data: HashMap>>, +} + +/// On-disk JSON shape. +#[derive(Debug, Default, Serialize, Deserialize)] +struct OnDisk { + #[serde(flatten)] + packages: HashMap>>, +} + +impl ScriptStore { + /// Opens (or creates) the store at `path`. + pub fn open(path: impl AsRef) -> io::Result { + let path = path.as_ref().to_path_buf(); + let data = match fs::read_to_string(&path) { + Ok(contents) => { + let on_disk: OnDisk = serde_json::from_str(&contents).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("malformed script store: {e}"), + ) + })?; + on_disk.packages + } + Err(e) if e.kind() == io::ErrorKind::NotFound => HashMap::new(), + Err(e) => return Err(e), + }; + Ok(Self { path, data }) + } + + /// Returns the last known scripts map for any version of `package`, or + /// `None` if the package has never been recorded. + /// + /// When multiple versions exist the most recently inserted one is + /// returned. For the purposes of change detection we only need *a* + /// previous baseline, not a specific version. + pub fn latest_scripts(&self, package: &str) -> Option<&HashMap> { + self.data.get(package)?.values().last() + } + + /// Records `scripts` for `(package, version)`. + pub fn record( + &mut self, + package: &str, + version: &str, + scripts: HashMap, + ) { + self.data + .entry(package.to_string()) + .or_default() + .insert(version.to_string(), scripts); + } + + /// Atomically flushes the in-memory state to disk. + pub fn save(&self) -> io::Result<()> { + let parent = self.path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + + let tmp = self.path.with_extension("json.tmp"); + let json = serde_json::to_string_pretty(&OnDisk { + packages: self.data.clone(), + }) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + fs::write(&tmp, json)?; + fs::rename(&tmp, &self.path)?; + Ok(()) + } +} + +/// Compares `scripts` for the new `version` of `package` against whatever +/// was recorded previously, emitting a finding for every dangerous lifecycle +/// hook that is new or changed. +/// +/// If this is the first time the package is seen the scripts are recorded but +/// no finding is emitted (no baseline to compare against). +pub fn check_install_scripts( + store: &mut ScriptStore, + package: &str, + version: &str, + scripts: &HashMap, +) -> Vec { + let previous: HashMap = store + .latest_scripts(package) + .cloned() + .unwrap_or_default(); + + let is_first_observation = !store.data.contains_key(package); + + // Record the new version unconditionally. + store.record(package, version, scripts.clone()); + + if is_first_observation { + return Vec::new(); + } + + let mut findings = Vec::new(); + for hook in DANGEROUS_HOOKS { + if let Some(new_script) = scripts.get(*hook) { + let prev = previous.get(*hook); + if prev.map(String::as_str) != Some(new_script.as_str()) { + findings.push(InstallScriptAdded { + package: package.to_string(), + version: version.to_string(), + hook: hook.to_string(), + script: new_script.clone(), + previous: prev.cloned(), + }); + } + } + } + + findings +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn scripts(pairs: &[(&str, &str)]) -> HashMap { + pairs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() + } + + /// v1 has no install script, v2 adds `postinstall` → one finding. + #[test] + fn finding_when_postinstall_added() { + let dir = tempdir().unwrap(); + let mut store = ScriptStore::open(dir.path().join("s.json")).unwrap(); + + // v1: no dangerous scripts. + check_install_scripts(&mut store, "ua-parser-js", "1.0.0", &scripts(&[("test", "jest")])); + + // v2: postinstall appears. + let findings = check_install_scripts( + &mut store, + "ua-parser-js", + "1.0.1", + &scripts(&[("test", "jest"), ("postinstall", "node malware.js")]), + ); + + assert_eq!(findings.len(), 1); + let f = &findings[0]; + assert_eq!(f.package, "ua-parser-js"); + assert_eq!(f.version, "1.0.1"); + assert_eq!(f.hook, "postinstall"); + assert_eq!(f.script, "node malware.js"); + assert_eq!(f.previous, None); + } + + /// No change in scripts → no finding. + #[test] + fn no_finding_when_scripts_unchanged() { + let dir = tempdir().unwrap(); + let mut store = ScriptStore::open(dir.path().join("s.json")).unwrap(); + + let s = scripts(&[("test", "jest"), ("postinstall", "echo hi")]); + check_install_scripts(&mut store, "mypkg", "1.0.0", &s); + let findings = check_install_scripts(&mut store, "mypkg", "1.0.1", &s); + assert!(findings.is_empty()); + } + + /// First observation → no finding (no baseline). + #[test] + fn no_finding_on_first_observation() { + let dir = tempdir().unwrap(); + let mut store = ScriptStore::open(dir.path().join("s.json")).unwrap(); + let findings = check_install_scripts( + &mut store, + "newpkg", + "1.0.0", + &scripts(&[("postinstall", "node setup.js")]), + ); + assert!(findings.is_empty(), "no baseline → no finding"); + } + + /// A changed (not just added) script also fires. + #[test] + fn finding_when_postinstall_changes() { + let dir = tempdir().unwrap(); + let mut store = ScriptStore::open(dir.path().join("s.json")).unwrap(); + + check_install_scripts( + &mut store, + "mypkg", + "1.0.0", + &scripts(&[("postinstall", "node setup.js")]), + ); + let findings = check_install_scripts( + &mut store, + "mypkg", + "1.0.1", + &scripts(&[("postinstall", "node malware.js")]), + ); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].previous.as_deref(), Some("node setup.js")); + assert_eq!(findings[0].script, "node malware.js"); + } + + /// Multiple hooks added at once → multiple findings. + #[test] + fn multiple_hooks_produce_multiple_findings() { + let dir = tempdir().unwrap(); + let mut store = ScriptStore::open(dir.path().join("s.json")).unwrap(); + + check_install_scripts(&mut store, "mypkg", "1.0.0", &scripts(&[])); + let findings = check_install_scripts( + &mut store, + "mypkg", + "1.0.1", + &scripts(&[ + ("preinstall", "echo pre"), + ("postinstall", "echo post"), + ]), + ); + assert_eq!(findings.len(), 2); + } + + /// Store round-trips to disk. + #[test] + fn store_persists_across_reopen() { + let dir = tempdir().unwrap(); + let path = dir.path().join("s.json"); + + { + let mut store = ScriptStore::open(&path).unwrap(); + check_install_scripts(&mut store, "mypkg", "1.0.0", &scripts(&[("test", "jest")])); + store.save().unwrap(); + } + + let mut store2 = ScriptStore::open(&path).unwrap(); + let findings = check_install_scripts( + &mut store2, + "mypkg", + "1.0.1", + &scripts(&[("test", "jest"), ("postinstall", "node evil.js")]), + ); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].hook, "postinstall"); + } +} diff --git a/src/pipeline.rs b/src/pipeline.rs index e3e5287..664c2d2 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -460,3 +460,133 @@ fn finding(dependency: &Dependency, kind: FindingKind) -> Finding { kind, } } + +// ── watch pipeline (issue #96) ───────────────────────────────────────────── + +/// A source of package names, used by [`watch_packages`]. +/// +/// Production code connects this to a real registry crawler. Tests supply a +/// [`FakeCrawler`] that returns a fixed list. +pub trait PackageSource { + /// Returns the next batch of package names to score, or an empty `Vec` + /// when there is nothing new. The caller is responsible for polling at an + /// appropriate interval. + fn poll(&mut self) -> Vec; +} + +/// Confidence threshold below which a finding is not emitted by +/// [`watch_packages`]. Matches the 40-point floor the combined scanner uses +/// for a distance-only signal (`expresss` scores 40). +pub const WATCH_CONFIDENCE_THRESHOLD: u8 = 40; + +/// Runs a continuous watch loop: polls `source` for new package names, scores +/// each one with [`score_package`], and calls `on_finding` for every result +/// whose confidence is at or above [`WATCH_CONFIDENCE_THRESHOLD`]. +/// +/// The loop runs until `source.poll()` returns an empty `Vec`. In production +/// this never happens — the crawler blocks until new packages appear and +/// returns them. In tests a [`FakeCrawler`] drains a fixed list so the loop +/// terminates. +/// +/// Findings are emitted as serialized JSON lines so they can be piped into +/// downstream processors. +pub fn watch_packages(mut source: S, mut on_finding: F) +where + S: PackageSource, + F: FnMut(String), +{ + loop { + let batch = source.poll(); + if batch.is_empty() { + break; + } + for name in batch { + let result = score_package(&name); + if result.confidence >= WATCH_CONFIDENCE_THRESHOLD { + let line = serde_json::json!({ + "package": result.candidate, + "confidence": result.confidence, + "distance_match": result.distance_match.as_ref() + .map(|(n, d)| serde_json::json!({ "name": n, "distance": d })), + "homoglyph_match": result.homoglyph_match, + "digit_swap_match": result.digit_swap_match, + "scope_confusion": result.scope_confusion, + "combosquat": result.combosquat, + }); + on_finding(line.to_string()); + } + } + } +} + +/// A [`PackageSource`] backed by a fixed list, for use in tests. +/// +/// Each call to [`poll`](FakeCrawler::poll) drains one chunk of +/// `chunk_size` names. When the list is exhausted, `poll` returns an empty +/// `Vec` so [`watch_packages`] terminates. +pub struct FakeCrawler { + names: std::collections::VecDeque, + chunk_size: usize, +} + +impl FakeCrawler { + /// Creates a new [`FakeCrawler`] that will hand out `names` in chunks of + /// `chunk_size` per [`poll`](Self::poll) call. + pub fn new(names: impl IntoIterator>, chunk_size: usize) -> Self { + Self { + names: names.into_iter().map(Into::into).collect(), + chunk_size: chunk_size.max(1), + } + } +} + +impl PackageSource for FakeCrawler { + fn poll(&mut self) -> Vec { + let n = self.chunk_size.min(self.names.len()); + self.names.drain(..n).collect() + } +} + +#[cfg(test)] +mod watch_tests { + use super::*; + + #[test] + fn fake_crawler_emits_typosquat_finding() { + // "expresss" is one edit away from "express" — confidence 40. + let crawler = FakeCrawler::new(["expresss", "totally-legit-package-zzz123"], 10); + let mut findings = Vec::new(); + watch_packages(crawler, |line| findings.push(line)); + + assert_eq!(findings.len(), 1, "only the typosquat should be emitted"); + let v: serde_json::Value = serde_json::from_str(&findings[0]).unwrap(); + assert_eq!(v["package"], "expresss"); + assert!(v["confidence"].as_u64().unwrap() >= WATCH_CONFIDENCE_THRESHOLD as u64); + } + + #[test] + fn clean_packages_produce_no_findings() { + let crawler = FakeCrawler::new(["totally-legit-zzz-package-x99z"], 10); + let mut findings = Vec::new(); + watch_packages(crawler, |line| findings.push(line)); + assert!(findings.is_empty()); + } + + #[test] + fn multiple_typosquats_are_all_emitted() { + // Both "expresss" and "reqiests" should fire. + let crawler = FakeCrawler::new(["expresss", "reqiests"], 10); + let mut findings = Vec::new(); + watch_packages(crawler, |line| findings.push(line)); + assert_eq!(findings.len(), 2); + } + + #[test] + fn finding_is_valid_json_line() { + let crawler = FakeCrawler::new(["expresss"], 10); + let mut findings = Vec::new(); + watch_packages(crawler, |line| findings.push(line)); + // Must be parseable as JSON. + let _: serde_json::Value = serde_json::from_str(&findings[0]).expect("valid JSON line"); + } +} diff --git a/tests/snapshots/depradar-help.txt b/tests/snapshots/depradar-help.txt index fb4dfb3..f41a269 100644 --- a/tests/snapshots/depradar-help.txt +++ b/tests/snapshots/depradar-help.txt @@ -3,9 +3,10 @@ Check package names and lockfiles for typosquats and known vulnerabilities Usage: depradar [OPTIONS] Commands: - name Score a single package name against the popular-package dataset - scan Scan a lockfile, or every lockfile under a directory - help Print this message or the help of the given subcommand(s) + name Score a single package name against the popular-package dataset + scan Scan a lockfile, or every lockfile under a directory + watch Watch a package registry and emit typosquat findings as JSON lines + help Print this message or the help of the given subcommand(s) Options: -f, --format Output format. `json` prints exactly one JSON document to stdout; errors and warnings still go to stderr [default: text] [possible values: text, json] @@ -19,8 +20,9 @@ Examples: depradar scan . Scan every lockfile in a directory or monorepo depradar scan . --offline Typosquat checks only, no OSV lookup depradar --format json scan . | jq Machine-readable output for CI + depradar watch --registry npm Watch the npm registry and score new packages Exit codes: 0 No findings 1 At least one finding (`scan` only; suppressed findings don't count) - 2 Usage error, or the scan couldn't complete + 2 Usage error, or the scan couldn't complete \ No newline at end of file