From b13f09eeb7a158ab967186571fd7e5dd97af5e57 Mon Sep 17 00:00:00 2001 From: piekstra Date: Wed, 7 Oct 2026 09:49:57 -0400 Subject: [PATCH 1/2] feat: affected flows, evidence bundles and webhook notifications `lest affected --base ` lists the flows a change touches (affects globs, the flow file itself, and callers of affected flows). `lest bundle` zips a run's report and artifacts with a manifest of checksums. Projects can notify webhooks after a run, as a JSON summary or a Slack-compatible message. --- Cargo.lock | 97 ++++++++++++++++++ README.md | 7 +- crates/lest-core/Cargo.toml | 1 + crates/lest-core/src/affected.rs | 158 +++++++++++++++++++++++++++++ crates/lest-core/src/bundle.rs | 74 ++++++++++++++ crates/lest-core/src/event.rs | 3 + crates/lest-core/src/lib.rs | 3 + crates/lest-core/src/notify.rs | 132 ++++++++++++++++++++++++ crates/lest-core/src/runner/mod.rs | 18 ++++ crates/lest-core/tests/runner.rs | 58 +++++++++++ crates/lest/src/args.rs | 23 +++++ crates/lest/src/main.rs | 28 +++++ crates/lest/src/render.rs | 4 + docs/ci.md | 92 +++++++++++++++++ 14 files changed, 696 insertions(+), 2 deletions(-) create mode 100644 crates/lest-core/src/affected.rs create mode 100644 crates/lest-core/src/bundle.rs create mode 100644 crates/lest-core/src/notify.rs create mode 100644 docs/ci.md diff --git a/Cargo.lock b/Cargo.lock index 2700c07..ffba1a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -92,6 +98,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -346,6 +361,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.1.7" @@ -362,6 +392,17 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "derive_arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "digest" version = "0.10.7" @@ -438,6 +479,16 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -936,6 +987,7 @@ dependencies = [ "tokio-util", "walkdir", "which", + "zip", ] [[package]] @@ -998,6 +1050,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.4" @@ -1626,6 +1688,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.4" @@ -2416,8 +2484,37 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/README.md b/README.md index a042424..d17b96c 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,10 @@ lest runs list - **Cleanup:** steps register commands that undo them; they run after the flow, on failure and on cancel, and can be replayed later with `lest cleanup `. -- **Reports:** a JSON report per run (`lest schema report` describes it) and - JUnit XML for CI. +- **Reports:** a JSON report per run (`lest schema report` describes it), + JUnit XML, and `lest bundle` evidence zips with checksums. +- **CI:** `lest affected --base origin/main` lists the flows a change touches; + webhook notifications (JSON or Slack-compatible) after each run. ## Examples @@ -93,6 +95,7 @@ lest run everything - [Flow format](docs/flow-format.md) - [Browser steps](docs/browser.md) +- [Running in CI](docs/ci.md) - [Design](docs/design.md): why Lest works the way it does - [Development](docs/development.md) diff --git a/crates/lest-core/Cargo.toml b/crates/lest-core/Cargo.toml index 5f89e0e..fe164d4 100644 --- a/crates/lest-core/Cargo.toml +++ b/crates/lest-core/Cargo.toml @@ -29,6 +29,7 @@ walkdir.workspace = true which.workspace = true reqwest.workspace = true rand.workspace = true +zip = { version = "2", default-features = false, features = ["deflate"] } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/lest-core/src/affected.rs b/crates/lest-core/src/affected.rs new file mode 100644 index 0000000..3ede45f --- /dev/null +++ b/crates/lest-core/src/affected.rs @@ -0,0 +1,158 @@ +//! Change-based selection: which flows does a set of changed files touch? +//! +//! A flow is affected when a changed path matches one of its `affects:` +//! globs, when its own file changed, or when a flow it calls is affected. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::process::Command; + +use globset::{Glob, GlobSetBuilder}; +use serde::Serialize; + +use crate::catalog::Catalog; +use crate::spec::Step; + +#[derive(Debug, Clone, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Affected { + pub flow_id: String, + /// Why: the matching paths, `flow file changed`, or `calls `. + pub reasons: Vec, +} + +/// Flows affected by `changed` paths (relative to the project root). +pub fn affected(catalog: &Catalog, changed: &[String]) -> Result, String> { + let mut direct: BTreeMap> = BTreeMap::new(); + for lf in &catalog.flows { + let mut reasons = Vec::new(); + if changed.iter().any(|c| c == &lf.rel_path) { + reasons.push("flow file changed".to_string()); + } + if !lf.flow.affects.is_empty() { + let mut b = GlobSetBuilder::new(); + for g in &lf.flow.affects { + b.add(Glob::new(g).map_err(|e| format!("{}: affects '{g}': {e}", lf.rel_path))?); + } + let set = b.build().map_err(|e| e.to_string())?; + reasons.extend(changed.iter().filter(|c| set.is_match(c.as_str())).cloned()); + } + if !reasons.is_empty() { + direct.insert(lf.flow.id.clone(), reasons); + } + } + // Callers of affected flows are affected too, transitively. + fn calls(steps: &[Step], out: &mut BTreeSet) { + for s in steps { + if let Some(f) = &s.flow { + out.insert(f.clone()); + } + calls(s.children(), out); + } + } + let callees: BTreeMap> = catalog + .flows + .iter() + .map(|lf| { + let mut c = BTreeSet::new(); + calls(&lf.flow.steps, &mut c); + calls(&lf.flow.finally, &mut c); + (lf.flow.id.clone(), c) + }) + .collect(); + let mut result: BTreeMap> = direct.clone(); + loop { + let mut added = false; + for (caller, cs) in &callees { + for callee in cs { + if result.contains_key(callee) + && !result.get(caller).is_some_and(|r| r.contains(&format!("calls {callee}"))) + { + result.entry(caller.clone()).or_default().push(format!("calls {callee}")); + added = true; + } + } + } + if !added { + break; + } + } + Ok(result.into_iter().map(|(flow_id, reasons)| Affected { flow_id, reasons }).collect()) +} + +fn git(dir: &Path, args: &[&str]) -> Result { + let out = Command::new("git").args(args).current_dir(dir).output().map_err(|e| format!("git: {e}"))?; + if out.status.success() { + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) + } else { + Err(format!("git {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim())) + } +} + +/// Files changed since `base` (committed on this branch, staged, unstaged +/// and untracked), relative to the project root. +pub fn changed_since(project_root: &Path, base: &str) -> Result, String> { + let top = git(project_root, &["rev-parse", "--show-toplevel"])?; + let top = std::fs::canonicalize(top.trim()).map_err(|e| e.to_string())?; + let root = std::fs::canonicalize(project_root).map_err(|e| e.to_string())?; + let merge_base = git(project_root, &["merge-base", base, "HEAD"])?; + let mut files = BTreeSet::new(); + for out in [ + git(project_root, &["diff", "--name-only", merge_base.trim()])?, + git(project_root, &["ls-files", "--others", "--exclude-standard", "--full-name"])?, + ] { + for line in out.lines().filter(|l| !l.is_empty()) { + files.insert(line.to_string()); + } + } + // Git paths are relative to the repository; flows use the project root. + let prefix = root.strip_prefix(&top).map_err(|_| "the project is outside the git repository".to_string())?; + let prefix = prefix.to_string_lossy().replace('\\', "/"); + Ok(files + .into_iter() + .filter_map( + |f| { + if prefix.is_empty() { Some(f) } else { f.strip_prefix(&format!("{prefix}/")).map(str::to_string) } + }, + ) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::{LoadedFlow, parse_flow}; + + fn catalog(flows: &[(&str, &str)]) -> Catalog { + let mut c = Catalog::default(); + for (path, text) in flows { + c.flows.push(LoadedFlow { + path: format!("/p/{path}").into(), + rel_path: path.to_string(), + flow: parse_flow(text).unwrap(), + sha256: String::new(), + }); + } + c + } + + #[test] + fn matches_globs_own_files_and_callers() { + let c = catalog(&[ + ( + "flows/login.lest.yaml", + "apiVersion: lest/v1\nid: login\nname: L\naffects: ['src/auth/**']\nsteps: [{id: a, run: 'true'}]\n", + ), + ("flows/suite.lest.yaml", "apiVersion: lest/v1\nid: suite\nname: S\nsteps: [{id: l, flow: login}]\n"), + ( + "flows/other.lest.yaml", + "apiVersion: lest/v1\nid: other\nname: O\naffects: ['docs/**']\nsteps: [{id: a, run: 'true'}]\n", + ), + ]); + let a = affected(&c, &["src/auth/session.ts".into(), "flows/other.lest.yaml".into()]).unwrap(); + let by: BTreeMap<_, _> = a.iter().map(|x| (x.flow_id.as_str(), x.reasons.clone())).collect(); + assert_eq!(by["login"], vec!["src/auth/session.ts"]); + assert_eq!(by["suite"], vec!["calls login"]); + assert_eq!(by["other"], vec!["flow file changed"]); + } +} diff --git a/crates/lest-core/src/bundle.rs b/crates/lest-core/src/bundle.rs new file mode 100644 index 0000000..4863bdb --- /dev/null +++ b/crates/lest-core/src/bundle.rs @@ -0,0 +1,74 @@ +//! Evidence bundles: a run's report and artifacts in one zip, with a +//! manifest listing every file and its SHA-256, for attaching to a ticket, +//! a release or an audit. + +use std::io::{Read, Write}; +use std::path::Path; + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::report::RunReport; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub flow_id: String, + pub flow_name: String, + pub run_id: String, + pub result: String, + pub environment: Option, + pub started_at: String, + pub duration_ms: u64, + pub lest_version: String, + pub files: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ManifestFile { + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +/// Zips the run directory into `out`. Returns the manifest. +pub fn bundle(run_dir: &Path, report: &RunReport, out: &Path) -> Result { + let mut files: Vec = walkdir::WalkDir::new(run_dir) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| e.into_path()) + .collect(); + files.sort(); + let file = std::fs::File::create(out).map_err(|e| format!("cannot create {}: {e}", out.display()))?; + let mut zip = zip::ZipWriter::new(file); + let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let mut manifest = Manifest { + flow_id: report.flow_id.clone(), + flow_name: report.flow_name.clone(), + run_id: report.run_id.clone(), + result: report.result.as_str().to_string(), + environment: report.environment.clone(), + started_at: report.started_at.clone(), + duration_ms: report.duration_ms, + lest_version: env!("CARGO_PKG_VERSION").to_string(), + files: Vec::new(), + }; + for path in files { + let rel = crate::catalog::rel_path(run_dir, &path); + let mut bytes = Vec::new(); + std::fs::File::open(&path).and_then(|mut f| f.read_to_end(&mut bytes)).map_err(|e| format!("{rel}: {e}"))?; + zip.start_file(&rel, opts).map_err(|e| e.to_string())?; + zip.write_all(&bytes).map_err(|e| e.to_string())?; + manifest.files.push(ManifestFile { + sha256: hex::encode(Sha256::digest(&bytes)), + bytes: bytes.len() as u64, + path: rel, + }); + } + let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?; + zip.start_file("manifest.json", opts).map_err(|e| e.to_string())?; + zip.write_all(&json).map_err(|e| e.to_string())?; + zip.finish().map_err(|e| e.to_string())?; + Ok(manifest) +} diff --git a/crates/lest-core/src/event.rs b/crates/lest-core/src/event.rs index 69e842b..3575e0c 100644 --- a/crates/lest-core/src/event.rs +++ b/crates/lest-core/src/event.rs @@ -77,6 +77,9 @@ pub enum EventBody { /// Post-production of a demo recording. #[serde(rename_all = "camelCase")] DemoProgress { message: String }, + /// A notification was sent after the run (or failed to send). + #[serde(rename_all = "camelCase")] + Notified { ok: bool, message: String }, #[serde(rename_all = "camelCase")] RunFinished { result: RunResult, duration_ms: u64, error: Option, report_path: String }, } diff --git a/crates/lest-core/src/lib.rs b/crates/lest-core/src/lib.rs index 956f8e9..e79e494 100644 --- a/crates/lest-core/src/lib.rs +++ b/crates/lest-core/src/lib.rs @@ -1,10 +1,13 @@ //! Lest core: the flow format, validation, the runner and run reports. +pub mod affected; pub mod browser; +pub mod bundle; pub mod catalog; pub mod event; pub mod expr; pub mod headline; +pub mod notify; pub mod paths; pub mod process; pub mod project; diff --git a/crates/lest-core/src/notify.rs b/crates/lest-core/src/notify.rs new file mode 100644 index 0000000..ce4c41e --- /dev/null +++ b/crates/lest-core/src/notify.rs @@ -0,0 +1,132 @@ +//! Webhook notifications after a run: a JSON summary, or a Slack-compatible +//! incoming-webhook message. + +use std::time::Duration; + +use serde_json::{Value as Json, json}; + +use crate::expr::{self, Scope}; +use crate::project::{NotifyFormat, NotifyTarget, NotifyWhen, Project}; +use crate::report::{RunReport, RunResult}; +use crate::secrets::{Keyring, Resolver}; + +fn wanted(t: &NotifyTarget, result: RunResult) -> bool { + match t.on.unwrap_or(NotifyWhen::Failed) { + NotifyWhen::Always => true, + NotifyWhen::Passed => result == RunResult::Passed, + NotifyWhen::Failed => result != RunResult::Passed && result != RunResult::Cancelled, + } +} + +/// The JSON summary posted for `format: json`. +pub fn summary(report: &RunReport) -> Json { + let failure = report.first_failure().map(|s| { + json!({ + "step": s.id, + "name": s.name, + "headline": s.headline, + }) + }); + json!({ + "flowId": report.flow_id, + "flowName": report.flow_name, + "runId": report.run_id, + "result": report.result.as_str(), + "environment": report.environment, + "startedAt": report.started_at, + "durationMs": report.duration_ms, + "error": report.error, + "failure": failure, + "warnings": report.warnings, + }) +} + +/// A Slack incoming-webhook message. +pub fn slack_message(report: &RunReport) -> Json { + let icon = match report.result { + RunResult::Passed => ":white_check_mark:", + RunResult::Failed | RunResult::Errored => ":x:", + RunResult::Cancelled => ":black_square_for_stop:", + }; + let env = report.environment.as_deref().map(|e| format!(" ({e})")).unwrap_or_default(); + let mut text = format!( + "{icon} *{}*{env} {} in {:.1}s", + report.flow_name, + report.result.as_str(), + report.duration_ms as f64 / 1000.0 + ); + if let Some(f) = report.first_failure() { + text.push_str(&format!("\n`{}`: {}", f.id, f.headline.as_deref().unwrap_or("failed"))); + } else if let Some(e) = &report.error { + text.push_str(&format!("\n{e}")); + } + for w in &report.warnings { + text.push_str(&format!("\n:warning: {w}")); + } + text.push_str(&format!("\nrun `{}` · rerun: `lest run {}`", report.run_id, report.flow_id)); + json!({ "text": text }) +} + +/// Sends every configured notification for this result. Returns one line +/// per target: what was sent, or why it failed. +pub async fn send(project: &Project, keyring: &dyn Keyring, report: &RunReport) -> Vec> { + let mut out = Vec::new(); + for (i, t) in project.config.notify.iter().enumerate() { + if !wanted(t, report.result) { + continue; + } + let label = format!("notify[{i}]"); + let resolver = Resolver::new(project.secrets_config(), keyring); + let secrets = match resolver.resolve_all(&t.secrets) { + Ok(s) => s, + Err(e) => { + out.push(Err(format!("{label}: {e}"))); + continue; + } + }; + let mut scope = Scope::new(); + scope.set("secrets", Json::Object(secrets.into_iter().map(|(k, v)| (k, Json::String(v))).collect())); + let url = match expr::interpolate(&t.webhook, &scope) { + Ok(u) => u, + Err(e) => { + out.push(Err(format!("{label}: webhook: {e}"))); + continue; + } + }; + let body = match t.format.unwrap_or(NotifyFormat::Json) { + NotifyFormat::Json => summary(report), + NotifyFormat::Slack => slack_message(report), + }; + let client = match reqwest::Client::builder().timeout(Duration::from_secs(10)).build() { + Ok(c) => c, + Err(e) => { + out.push(Err(format!("{label}: {e}"))); + continue; + } + }; + // The URL may embed a secret: never echo it. + out.push(match client.post(&url).json(&body).send().await { + Ok(r) if r.status().is_success() => Ok(format!("{label}: sent ({})", r.status().as_u16())), + Ok(r) => Err(format!("{label}: the webhook answered {}", r.status().as_u16())), + Err(e) => Err(format!( + "{label}: {}", + if e.is_timeout() { "timed out".to_string() } else { "could not reach the webhook".to_string() } + )), + }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_is_failures_only() { + let t = NotifyTarget { webhook: "x".into(), on: None, format: None, secrets: vec![] }; + assert!(wanted(&t, RunResult::Failed)); + assert!(wanted(&t, RunResult::Errored)); + assert!(!wanted(&t, RunResult::Passed)); + assert!(!wanted(&t, RunResult::Cancelled)); + } +} diff --git a/crates/lest-core/src/runner/mod.rs b/crates/lest-core/src/runner/mod.rs index 141ab49..9456f73 100644 --- a/crates/lest-core/src/runner/mod.rs +++ b/crates/lest-core/src/runner/mod.rs @@ -399,6 +399,24 @@ pub async fn execute_with_id( run_id: String, ) -> RunReport { let emitter = Emitter::new(&run_id, tx); + let report = run_flow(engine, req, &emitter, cancel).await; + // Notifications go out after the report is written and the run has + // finished, so a slow webhook never delays either. + if !engine.project.config.notify.is_empty() { + for outcome in crate::notify::send(&engine.project, engine.keyring.as_ref(), &report).await { + let (ok, message) = match outcome { + Ok(m) => (true, m), + Err(m) => (false, m), + }; + emitter.emit(EventBody::Notified { ok, message }); + } + } + report +} + +async fn run_flow(engine: &Engine, req: RunRequest, emitter: &Emitter, cancel: RunCancel) -> RunReport { + let emitter = emitter.clone(); + let run_id = emitter.run_id.clone(); let started = Instant::now(); let started_at = now_rfc3339(); let started_at_ms = now_ms(); diff --git a/crates/lest-core/tests/runner.rs b/crates/lest-core/tests/runner.rs index 19aecba..3f454c0 100644 --- a/crates/lest-core/tests/runner.rs +++ b/crates/lest-core/tests/runner.rs @@ -649,3 +649,61 @@ steps: let log = std::fs::read_to_string(dir.join("services/service-stubborn.log")).unwrap(); assert!(log.contains("token is [redacted:token]"), "{log}"); } + +#[tokio::test] +async fn failed_runs_notify_webhooks_with_a_summary() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let (got_tx, mut got_rx) = mpsc::unbounded_channel::(); + tokio::spawn(async move { + loop { + let (mut sock, _) = listener.accept().await.unwrap(); + let got_tx = got_tx.clone(); + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = vec![0u8; 16384]; + let mut req = String::new(); + loop { + let n = sock.read(&mut buf).await.unwrap(); + req.push_str(&String::from_utf8_lossy(&buf[..n])); + if n == 0 || (req.contains("\r\n\r\n") && req.trim_end().ends_with('}')) { + break; + } + } + let _ = got_tx.send(req); + sock.write_all(b"HTTP/1.1 204 No Content\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + }); + } + }); + let f = Fixture::new(&[( + "a.lest.yaml", + r#" +apiVersion: lest/v1 +id: notify +name: Notify me +steps: + - { id: boom, run: 'echo "error: broken" >&2; exit 1' } +"#, + )]); + std::fs::write( + f.path().join("lest.yaml"), + format!( + "flows: [flows]\nsecrets:\n backends: [env: {{}}]\nnotify:\n - webhook: \"http://127.0.0.1:{port}/hook/${{{{ secrets.hook_token }}}}\"\n secrets: [hook_token]\n" + ), + ) + .unwrap(); + // SAFETY: unique variable name; no other test reads it. + unsafe { std::env::set_var("LEST_SECRET_HOOK_TOKEN", "t0ken-xyz") }; + let (r, events) = run(&f.engine(&[]), "notify", &[]).await; + assert_eq!(r.result, RunResult::Failed); + let req = got_rx.recv().await.unwrap(); + assert!(req.starts_with("POST /hook/t0ken-xyz "), "{req}"); + assert!(req.contains("\"result\":\"failed\""), "{req}"); + assert!(req.contains("exited 1: error: broken"), "{req}"); + let notified: Vec<_> = events.iter().filter(|e| matches!(e.body, EventBody::Notified { ok: true, .. })).collect(); + assert_eq!(notified.len(), 1); + // The message never contains the secret URL. + assert!(!format!("{:?}", notified[0].body).contains("t0ken")); +} diff --git a/crates/lest/src/args.rs b/crates/lest/src/args.rs index b79a7f7..a10ca0c 100644 --- a/crates/lest/src/args.rs +++ b/crates/lest/src/args.rs @@ -34,6 +34,10 @@ pub enum Command { }, /// Run the cleanups a past run recorded but did not run. Cleanup(CleanupArgs), + /// List the flows affected by changes since a git ref. + Affected(AffectedArgs), + /// Zip a run's report and artifacts, with a manifest of checksums. + Bundle(BundleArgs), /// Create a lest.yaml and an example flow. Init(InitArgs), /// Check that tools, browsers and settings are ready. @@ -149,6 +153,25 @@ pub struct CleanupArgs { pub force: bool, } +#[derive(Debug, Args)] +pub struct AffectedArgs { + /// Compare against this ref (its merge base with HEAD). + #[arg(long, default_value = "origin/main")] + pub base: String, + /// Print only flow ids, one per line (for `xargs -n1 lest run`). + #[arg(long)] + pub id: bool, +} + +#[derive(Debug, Args)] +pub struct BundleArgs { + /// Run id or unique prefix. + pub run: String, + /// Where to write the zip (default: lest--.zip). + #[arg(short = 'o', long, value_name = "PATH")] + pub output: Option, +} + #[derive(Debug, Args)] pub struct InitArgs { /// Project name. diff --git a/crates/lest/src/main.rs b/crates/lest/src/main.rs index 7ca54be..d71a8a0 100644 --- a/crates/lest/src/main.rs +++ b/crates/lest/src/main.rs @@ -101,6 +101,8 @@ async fn dispatch(cli: Cli, style: Style) -> Result { Command::Validate(a) => cmd_validate(&ctx, a, style), Command::Runs { command } => cmd_runs(&ctx, command, style), Command::Cleanup(a) => cmd_cleanup(&ctx, a, style).await, + Command::Affected(a) => cmd_affected(&ctx, a), + Command::Bundle(a) => cmd_bundle(&ctx, a), Command::Init(a) => cmd_init(&ctx, a), Command::Doctor => cmd_doctor(&ctx, style).await, Command::Secrets { command } => cmd_secrets(&ctx, command), @@ -617,6 +619,32 @@ fn regex_redacted() -> regex::Regex { regex::Regex::new(r"\[redacted:([A-Za-z_][A-Za-z0-9_]*)\]").expect("re") } +fn cmd_affected(ctx: &Ctx, a: AffectedArgs) -> Result { + let catalog = ctx.catalog(); + let changed = lest_core::affected::changed_since(&ctx.project.root, &a.base).map_err(|e| fail(exit::ERRORED, e))?; + let affected = lest_core::affected::affected(&catalog, &changed).map_err(|e| fail(exit::USAGE, e))?; + if a.id { + for f in &affected { + println!("{}", f.flow_id); + } + } else { + let rows: Vec> = affected.iter().map(|f| vec![f.flow_id.clone(), f.reasons.join(", ")]).collect(); + print!("{}", render::table(&["FLOW", "WHY"], &rows)); + eprintln!("{} changed file(s), {} affected flow(s)", changed.len(), affected.len()); + } + Ok(0) +} + +fn cmd_bundle(ctx: &Ctx, a: BundleArgs) -> Result { + let store = ctx.store(); + let report = store.find(&a.run).map_err(|e| fail(exit::NOT_FOUND, e.to_string()))?; + let out = a.output.unwrap_or_else(|| PathBuf::from(format!("lest-{}-{}.zip", report.flow_id, report.run_id))); + let manifest = lest_core::bundle::bundle(&store.run_dir(&report.flow_id, &report.run_id), &report, &out) + .map_err(|e| fail(exit::ERRORED, e))?; + eprintln!("wrote {} ({} files)", out.display(), manifest.files.len()); + Ok(0) +} + const EXAMPLE_FLOW: &str = r#"apiVersion: lest/v1 id: hello name: Hello, Lest diff --git a/crates/lest/src/render.rs b/crates/lest/src/render.rs index 3655d01..221b3b9 100644 --- a/crates/lest/src/render.rs +++ b/crates/lest/src/render.rs @@ -155,6 +155,10 @@ impl Progress { eprintln!(" {} {tool} needs a sign-in: {command}", s.yellow("!")); } EventBody::DemoProgress { message } => eprintln!(" {}", s.dim(message)), + EventBody::Notified { ok, message } => { + let glyph = if *ok { s.green("✓") } else { s.yellow("!") }; + eprintln!(" {glyph} {}", s.dim(message)); + } _ => {} } } diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000..483ab56 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,92 @@ +# Running Lest in CI + +Everything the UI does is available from the CLI, so a CI job runs flows the +same way you do locally. + +## A GitHub Actions job + +```yaml +jobs: + flows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: { fetch-depth: 0 } # `lest affected` needs history + - uses: actions/setup-node@v5 + with: { node-version: 22 } + - run: npm ci && npx playwright install --with-deps chromium + - run: sudo apt-get install -y ffmpeg # only for demo recordings + - run: cargo install --git https://github.com/open-cli-collective/lest --locked lest + - name: Run the flows this change affects + env: + LEST_SECRET_API_TOKEN: ${{ secrets.API_TOKEN }} + run: | + for flow in $(lest affected --base origin/main --id); do + lest run "$flow" --non-interactive --junit "junit-$flow.xml" || failed=1 + done + exit ${failed:-0} +``` + +## Choosing what to run + +`lest affected --base ` lists the flows a change touches: + +- a changed file matches one of the flow's `affects:` globs, +- the flow's own file changed, or +- the flow calls an affected flow (suites run when any member is affected). + +Changes are counted from the merge base of `` and `HEAD`, plus staged, +unstaged and untracked files. Paths are relative to the project root, so a +project in a subdirectory of the repository works. + +```console +$ lest affected --base origin/main +FLOW | WHY +dashboard | app/server.mjs, calls sign-in +plants-api | app/server.mjs +sign-in | app/server.mjs +``` + +`--id` prints one id per line for scripts. + +## Results + +- **Exit codes:** 0 passed, 1 failed, 2 invalid flow or usage, 3 could not + run (a tool missing or signed out, a missing secret), 4 not found, 130 + cancelled. +- **JUnit:** `--junit ` writes one test case per step, for CI test + views. +- **JSON:** `-o ` writes the full report (`lest schema report` + describes it). +- **Evidence:** `lest bundle -o evidence.zip` zips the report and every + artifact with a `manifest.json` of SHA-256 checksums, ready to attach to a + ticket, release or audit. Secret values are already redacted in everything + it contains. + +## Secrets + +In CI, give secrets as environment variables: `${{ secrets.api_token }}` in a +flow reads `LEST_SECRET_API_TOKEN` with the default backends. Resolved values +are redacted from all output, and env-backend variables are removed from the +environment of every step, so a step only sees the secrets its flow passes +through `env:`. + +`--non-interactive` makes sure no tool sign-in waits for input. + +## Notifications + +Send a summary to a webhook after each run: + +```yaml +# lest.yaml +notify: + - webhook: "https://hooks.example.com/services/${{ secrets.hook_path }}" + secrets: [hook_path] + on: failed # failed (default), passed, or always + format: slack # json (default) or slack +``` + +`json` posts the run summary (flow, run id, result, environment, duration, +the failing step and its headline, warnings). `slack` posts an +incoming-webhook message. Notifications go out after the report is written, +time out after 10 seconds, and never print the URL, which may embed a secret. From 70cae3eac57a829bab293831a3d49f075d398d2a Mon Sep 17 00:00:00 2001 From: piekstra Date: Wed, 7 Oct 2026 10:19:33 -0400 Subject: [PATCH 2/2] fix: address review findings in affected, bundle and notifications --- crates/lest-core/src/affected.rs | 144 ++++++++++++++++++++++++------- crates/lest-core/src/bundle.rs | 105 ++++++++++++++++++---- crates/lest-core/src/event.rs | 3 +- crates/lest-core/src/notify.rs | 67 +++++++++----- crates/lest/src/main.rs | 12 +++ docs/ci.md | 39 ++++++--- 6 files changed, 282 insertions(+), 88 deletions(-) diff --git a/crates/lest-core/src/affected.rs b/crates/lest-core/src/affected.rs index 3ede45f..1ef8323 100644 --- a/crates/lest-core/src/affected.rs +++ b/crates/lest-core/src/affected.rs @@ -1,16 +1,18 @@ //! Change-based selection: which flows does a set of changed files touch? //! //! A flow is affected when a changed path matches one of its `affects:` -//! globs, when its own file changed, or when a flow it calls is affected. +//! globs, when its own file or a browser script it runs changed, when the +//! project's `lest.yaml` changed, or when a flow it calls is affected. use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::process::Command; -use globset::{Glob, GlobSetBuilder}; +use globset::{GlobBuilder, GlobSetBuilder}; use serde::Serialize; use crate::catalog::Catalog; +use crate::project::PROJECT_FILE; use crate::spec::Step; #[derive(Debug, Clone, Serialize, PartialEq)] @@ -21,22 +23,69 @@ pub struct Affected { pub reasons: Vec, } +fn scripts(steps: &[Step], out: &mut Vec) { + for s in steps { + if let Some(script) = s.browser.as_ref().and_then(|b| b.script.clone()) { + out.push(script); + } + scripts(s.children(), out); + } +} + +/// Joins a path relative to a flow's directory onto the flow's project +/// path, resolving `..`. +fn project_relative(flow_rel: &str, script: &str) -> String { + let mut parts: Vec<&str> = flow_rel.split('/').collect(); + parts.pop(); + for seg in script.split('/') { + match seg { + "" | "." => {} + ".." => { + parts.pop(); + } + s => parts.push(s), + } + } + parts.join("/") +} + /// Flows affected by `changed` paths (relative to the project root). +/// +/// In `affects:` globs, `*` stays within one path segment and `**` crosses +/// directories. pub fn affected(catalog: &Catalog, changed: &[String]) -> Result, String> { + let config_changed = changed.iter().any(|c| c == PROJECT_FILE); let mut direct: BTreeMap> = BTreeMap::new(); for lf in &catalog.flows { let mut reasons = Vec::new(); + if config_changed { + reasons.push(format!("{PROJECT_FILE} changed")); + } if changed.iter().any(|c| c == &lf.rel_path) { reasons.push("flow file changed".to_string()); } + let mut s = Vec::new(); + scripts(&lf.flow.steps, &mut s); + scripts(&lf.flow.finally, &mut s); + for script in s { + let path = project_relative(&lf.rel_path, &script); + if changed.contains(&path) { + reasons.push(path); + } + } if !lf.flow.affects.is_empty() { let mut b = GlobSetBuilder::new(); for g in &lf.flow.affects { - b.add(Glob::new(g).map_err(|e| format!("{}: affects '{g}': {e}", lf.rel_path))?); + let glob = GlobBuilder::new(g) + .literal_separator(true) + .build() + .map_err(|e| format!("{}: affects '{g}': {e}", lf.rel_path))?; + b.add(glob); } let set = b.build().map_err(|e| e.to_string())?; reasons.extend(changed.iter().filter(|c| set.is_match(c.as_str())).cloned()); } + reasons.dedup(); if !reasons.is_empty() { direct.insert(lf.flow.id.clone(), reasons); } @@ -60,15 +109,14 @@ pub fn affected(catalog: &Catalog, changed: &[String]) -> Result, (lf.flow.id.clone(), c) }) .collect(); - let mut result: BTreeMap> = direct.clone(); + let mut result = direct; loop { let mut added = false; for (caller, cs) in &callees { for callee in cs { - if result.contains_key(callee) - && !result.get(caller).is_some_and(|r| r.contains(&format!("calls {callee}"))) - { - result.entry(caller.clone()).or_default().push(format!("calls {callee}")); + let reason = format!("calls {callee}"); + if result.contains_key(callee) && !result.get(caller).is_some_and(|r| r.contains(&reason)) { + result.entry(caller.clone()).or_default().push(reason); added = true; } } @@ -80,40 +128,42 @@ pub fn affected(catalog: &Catalog, changed: &[String]) -> Result, Ok(result.into_iter().map(|(flow_id, reasons)| Affected { flow_id, reasons }).collect()) } -fn git(dir: &Path, args: &[&str]) -> Result { +fn git(dir: &Path, args: &[&str]) -> Result, String> { let out = Command::new("git").args(args).current_dir(dir).output().map_err(|e| format!("git: {e}"))?; if out.status.success() { - Ok(String::from_utf8_lossy(&out.stdout).into_owned()) + Ok(out.stdout) } else { Err(format!("git {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim())) } } +/// NUL-separated git output, so paths with any characters come through. +fn nul_paths(out: &[u8]) -> impl Iterator + '_ { + out.split(|b| *b == 0).filter(|p| !p.is_empty()).map(|p| String::from_utf8_lossy(p).into_owned()) +} + /// Files changed since `base` (committed on this branch, staged, unstaged -/// and untracked), relative to the project root. +/// and untracked), relative to the project root. A rename counts as both +/// its old and its new path. pub fn changed_since(project_root: &Path, base: &str) -> Result, String> { - let top = git(project_root, &["rev-parse", "--show-toplevel"])?; - let top = std::fs::canonicalize(top.trim()).map_err(|e| e.to_string())?; + let top = String::from_utf8_lossy(&git(project_root, &["rev-parse", "--show-toplevel"])?).trim().to_string(); + let top = std::fs::canonicalize(&top).map_err(|e| e.to_string())?; let root = std::fs::canonicalize(project_root).map_err(|e| e.to_string())?; - let merge_base = git(project_root, &["merge-base", base, "HEAD"])?; + let merge_base = String::from_utf8_lossy(&git(project_root, &["merge-base", base, "HEAD"])?).trim().to_string(); let mut files = BTreeSet::new(); - for out in [ - git(project_root, &["diff", "--name-only", merge_base.trim()])?, - git(project_root, &["ls-files", "--others", "--exclude-standard", "--full-name"])?, - ] { - for line in out.lines().filter(|l| !l.is_empty()) { - files.insert(line.to_string()); - } - } + // --no-renames: both paths of a rename; --no-relative: repository paths + // whatever diff.relative is set to. + let diff = git(project_root, &["diff", "--name-only", "--no-renames", "--no-relative", "-z", &merge_base])?; + files.extend(nul_paths(&diff)); + let untracked = git(&top, &["ls-files", "--others", "--exclude-standard", "-z"])?; + files.extend(nul_paths(&untracked)); // Git paths are relative to the repository; flows use the project root. let prefix = root.strip_prefix(&top).map_err(|_| "the project is outside the git repository".to_string())?; let prefix = prefix.to_string_lossy().replace('\\', "/"); Ok(files .into_iter() .filter_map( - |f| { - if prefix.is_empty() { Some(f) } else { f.strip_prefix(&format!("{prefix}/")).map(str::to_string) } - }, + |f| if prefix.is_empty() { Some(f) } else { f.strip_prefix(&format!("{prefix}/")).map(str::to_string) }, ) .collect()) } @@ -136,23 +186,51 @@ mod tests { c } + fn by(a: &[Affected]) -> BTreeMap<&str, Vec> { + a.iter().map(|x| (x.flow_id.as_str(), x.reasons.clone())).collect() + } + #[test] - fn matches_globs_own_files_and_callers() { + fn matches_globs_own_files_scripts_and_callers() { let c = catalog(&[ ( "flows/login.lest.yaml", - "apiVersion: lest/v1\nid: login\nname: L\naffects: ['src/auth/**']\nsteps: [{id: a, run: 'true'}]\n", + "apiVersion: lest/v1\nid: login\nname: L\naffects: ['src/auth/**']\nsteps: [{id: a, browser: {script: ../scripts/login.mjs}}]\n", ), ("flows/suite.lest.yaml", "apiVersion: lest/v1\nid: suite\nname: S\nsteps: [{id: l, flow: login}]\n"), ( "flows/other.lest.yaml", - "apiVersion: lest/v1\nid: other\nname: O\naffects: ['docs/**']\nsteps: [{id: a, run: 'true'}]\n", + "apiVersion: lest/v1\nid: other\nname: O\naffects: ['docs/*.md']\nsteps: [{id: a, run: 'true'}]\n", ), ]); - let a = affected(&c, &["src/auth/session.ts".into(), "flows/other.lest.yaml".into()]).unwrap(); - let by: BTreeMap<_, _> = a.iter().map(|x| (x.flow_id.as_str(), x.reasons.clone())).collect(); - assert_eq!(by["login"], vec!["src/auth/session.ts"]); - assert_eq!(by["suite"], vec!["calls login"]); - assert_eq!(by["other"], vec!["flow file changed"]); + let a = affected( + &c, + &[ + "src/auth/session.ts".into(), + "flows/other.lest.yaml".into(), + "scripts/login.mjs".into(), + "docs/deep/x.md".into(), + ], + ) + .unwrap(); + let m = by(&a); + assert_eq!(m["login"], vec!["scripts/login.mjs", "src/auth/session.ts"]); + assert_eq!(m["suite"], vec!["calls login"]); + // `*` does not cross `/`, so docs/deep/x.md is not a match. + assert_eq!(m["other"], vec!["flow file changed"]); + } + + #[test] + fn a_project_config_change_affects_every_flow() { + let c = + catalog(&[("flows/a.lest.yaml", "apiVersion: lest/v1\nid: a\nname: A\nsteps: [{id: x, run: 'true'}]\n")]); + let a = affected(&c, &["lest.yaml".into()]).unwrap(); + assert_eq!(by(&a)["a"], vec!["lest.yaml changed"]); + } + + #[test] + fn script_paths_resolve_against_the_flow_directory() { + assert_eq!(project_relative("flows/web/a.lest.yaml", "../../scripts/x.mjs"), "scripts/x.mjs"); + assert_eq!(project_relative("a.lest.yaml", "x.mjs"), "x.mjs"); } } diff --git a/crates/lest-core/src/bundle.rs b/crates/lest-core/src/bundle.rs index 4863bdb..d543521 100644 --- a/crates/lest-core/src/bundle.rs +++ b/crates/lest-core/src/bundle.rs @@ -1,6 +1,10 @@ -//! Evidence bundles: a run's report and artifacts in one zip, with a -//! manifest listing every file and its SHA-256, for attaching to a ticket, -//! a release or an audit. +//! Evidence bundles: a run's report and the artifacts it lists, in one zip +//! with a manifest of every file's SHA-256, for attaching to a ticket, a +//! release or an audit. +//! +//! Only `report.json` and the files the report names go in: those are what +//! Lest redacted. Anything else a step wrote into the run directory stays +//! out. use std::io::{Read, Write}; use std::path::Path; @@ -22,6 +26,9 @@ pub struct Manifest { pub duration_ms: u64, pub lest_version: String, pub files: Vec, + /// Listed files that could not be included, and why. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub missing: Vec, } #[derive(Debug, Serialize)] @@ -31,18 +38,60 @@ pub struct ManifestFile { pub sha256: String, } -/// Zips the run directory into `out`. Returns the manifest. +/// The run-relative paths of every file the report names. +fn listed_files(report: &RunReport) -> Vec { + let mut out = vec!["report.json".to_string()]; + for s in report.all_steps() { + out.extend(s.artifacts.iter().map(|a| a.path.clone())); + } + out.extend(report.services.iter().filter_map(|s| s.log.clone())); + if let Some(d) = &report.demo { + out.extend( + [d.video.clone(), d.raw_video.clone(), d.chapters_vtt.clone(), d.beat_sheet.clone()].into_iter().flatten(), + ); + out.extend(d.chapters.iter().filter_map(|c| c.still.clone())); + } + out.sort(); + out.dedup(); + out +} + +/// Hashes everything read through it. +struct Hashing { + inner: R, + hash: Sha256, + bytes: u64, +} + +impl Read for Hashing { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.hash.update(&buf[..n]); + self.bytes += n as u64; + Ok(n) + } +} + +/// Zips the run into `out` (written to a temporary file, then renamed). +/// Returns the manifest. pub fn bundle(run_dir: &Path, report: &RunReport, out: &Path) -> Result { - let mut files: Vec = walkdir::WalkDir::new(run_dir) - .into_iter() - .filter_map(Result::ok) - .filter(|e| e.file_type().is_file()) - .map(|e| e.into_path()) - .collect(); - files.sort(); + let tmp = out.with_extension(format!("zip.tmp-{}", std::process::id())); + let result = write_zip(run_dir, report, &tmp); + match result { + Ok(m) => { + std::fs::rename(&tmp, out).map_err(|e| format!("cannot write {}: {e}", out.display()))?; + Ok(m) + } + Err(e) => { + let _ = std::fs::remove_file(&tmp); + Err(e) + } + } +} + +fn write_zip(run_dir: &Path, report: &RunReport, out: &Path) -> Result { let file = std::fs::File::create(out).map_err(|e| format!("cannot create {}: {e}", out.display()))?; let mut zip = zip::ZipWriter::new(file); - let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); let mut manifest = Manifest { flow_id: report.flow_id.clone(), flow_name: report.flow_name.clone(), @@ -53,20 +102,38 @@ pub fn bundle(run_dir: &Path, report: &RunReport, out: &Path) -> Result p, + Ok(_) => { + manifest.missing.push(format!("{rel}: outside the run directory")); + continue; + } + Err(e) => { + manifest.missing.push(format!("{rel}: {e}")); + continue; + } + }; + let f = std::fs::File::open(&path).map_err(|e| format!("{rel}: {e}"))?; + let size = f.metadata().map(|m| m.len()).unwrap_or(0); + let opts = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .large_file(size >= u32::MAX as u64); zip.start_file(&rel, opts).map_err(|e| e.to_string())?; - zip.write_all(&bytes).map_err(|e| e.to_string())?; + // Streamed: large videos are never held in memory. + let mut reader = Hashing { inner: f, hash: Sha256::new(), bytes: 0 }; + std::io::copy(&mut reader, &mut zip).map_err(|e| format!("{rel}: {e}"))?; manifest.files.push(ManifestFile { - sha256: hex::encode(Sha256::digest(&bytes)), - bytes: bytes.len() as u64, path: rel, + bytes: reader.bytes, + sha256: hex::encode(reader.hash.finalize()), }); } let json = serde_json::to_vec_pretty(&manifest).map_err(|e| e.to_string())?; + let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); zip.start_file("manifest.json", opts).map_err(|e| e.to_string())?; zip.write_all(&json).map_err(|e| e.to_string())?; zip.finish().map_err(|e| e.to_string())?; diff --git a/crates/lest-core/src/event.rs b/crates/lest-core/src/event.rs index 3575e0c..1471f50 100644 --- a/crates/lest-core/src/event.rs +++ b/crates/lest-core/src/event.rs @@ -77,7 +77,8 @@ pub enum EventBody { /// Post-production of a demo recording. #[serde(rename_all = "camelCase")] DemoProgress { message: String }, - /// A notification was sent after the run (or failed to send). + /// A notification was sent after the run (or failed to send). These + /// are the only events that follow `run_finished`. #[serde(rename_all = "camelCase")] Notified { ok: bool, message: String }, #[serde(rename_all = "camelCase")] diff --git a/crates/lest-core/src/notify.rs b/crates/lest-core/src/notify.rs index ce4c41e..be6ac6a 100644 --- a/crates/lest-core/src/notify.rs +++ b/crates/lest-core/src/notify.rs @@ -41,6 +41,12 @@ pub fn summary(report: &RunReport) -> Json { }) } +/// Slack's three control characters, so a flow name or error text cannot +/// mention a channel or forge a link. +fn slack_escape(s: &str) -> String { + s.replace('&', "&").replace('<', "<").replace('>', ">") +} + /// A Slack incoming-webhook message. pub fn slack_message(report: &RunReport) -> Json { let icon = match report.result { @@ -48,28 +54,34 @@ pub fn slack_message(report: &RunReport) -> Json { RunResult::Failed | RunResult::Errored => ":x:", RunResult::Cancelled => ":black_square_for_stop:", }; - let env = report.environment.as_deref().map(|e| format!(" ({e})")).unwrap_or_default(); + let env = report.environment.as_deref().map(|e| format!(" ({})", slack_escape(e))).unwrap_or_default(); let mut text = format!( "{icon} *{}*{env} {} in {:.1}s", - report.flow_name, + slack_escape(&report.flow_name), report.result.as_str(), report.duration_ms as f64 / 1000.0 ); if let Some(f) = report.first_failure() { - text.push_str(&format!("\n`{}`: {}", f.id, f.headline.as_deref().unwrap_or("failed"))); + text.push_str(&format!( + "\n`{}`: {}", + slack_escape(&f.id), + slack_escape(f.headline.as_deref().unwrap_or("failed")) + )); } else if let Some(e) = &report.error { - text.push_str(&format!("\n{e}")); + text.push_str(&format!("\n{}", slack_escape(e))); } for w in &report.warnings { - text.push_str(&format!("\n:warning: {w}")); + text.push_str(&format!("\n:warning: {}", slack_escape(w))); } text.push_str(&format!("\nrun `{}` · rerun: `lest run {}`", report.run_id, report.flow_id)); json!({ "text": text }) } -/// Sends every configured notification for this result. Returns one line -/// per target: what was sent, or why it failed. +/// Sends every configured notification for this result, all at once, so +/// the slowest webhook bounds the wait at 10 seconds. Returns one line per +/// target: what was sent, or why it failed. pub async fn send(project: &Project, keyring: &dyn Keyring, report: &RunReport) -> Vec> { + let mut prepared = Vec::new(); let mut out = Vec::new(); for (i, t) in project.config.notify.iter().enumerate() { if !wanted(t, report.result) { @@ -97,23 +109,27 @@ pub async fn send(project: &Project, keyring: &dyn Keyring, report: &RunReport) NotifyFormat::Json => summary(report), NotifyFormat::Slack => slack_message(report), }; - let client = match reqwest::Client::builder().timeout(Duration::from_secs(10)).build() { - Ok(c) => c, - Err(e) => { - out.push(Err(format!("{label}: {e}"))); - continue; - } - }; - // The URL may embed a secret: never echo it. - out.push(match client.post(&url).json(&body).send().await { - Ok(r) if r.status().is_success() => Ok(format!("{label}: sent ({})", r.status().as_u16())), - Ok(r) => Err(format!("{label}: the webhook answered {}", r.status().as_u16())), - Err(e) => Err(format!( - "{label}: {}", - if e.is_timeout() { "timed out".to_string() } else { "could not reach the webhook".to_string() } - )), - }); + prepared.push((label, url, body)); } + let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(10)).build() else { + out.push(Err("notify: cannot create an HTTP client".into())); + return out; + }; + let sends = prepared.into_iter().map(|(label, url, body)| { + let client = client.clone(); + async move { + // The URL may embed a secret: never echo it. + match client.post(&url).json(&body).send().await { + Ok(r) if r.status().is_success() => Ok(format!("{label}: sent ({})", r.status().as_u16())), + Ok(r) => Err(format!("{label}: the webhook answered {}", r.status().as_u16())), + Err(e) => Err(format!( + "{label}: {}", + if e.is_timeout() { "timed out".to_string() } else { "could not reach the webhook".to_string() } + )), + } + } + }); + out.extend(futures::future::join_all(sends).await); out } @@ -121,6 +137,11 @@ pub async fn send(project: &Project, keyring: &dyn Keyring, report: &RunReport) mod tests { use super::*; + #[test] + fn slack_text_cannot_ping_or_link() { + assert_eq!(slack_escape(" a & "), "<!channel> a & <https://x|y>"); + } + #[test] fn default_is_failures_only() { let t = NotifyTarget { webhook: "x".into(), on: None, format: None, secrets: vec![] }; diff --git a/crates/lest/src/main.rs b/crates/lest/src/main.rs index 7e20c4c..e46a670 100644 --- a/crates/lest/src/main.rs +++ b/crates/lest/src/main.rs @@ -622,6 +622,18 @@ fn regex_redacted() -> regex::Regex { fn cmd_affected(ctx: &Ctx, a: AffectedArgs) -> Result { let catalog = ctx.catalog(); + // A flow that does not parse cannot be selected; say so rather than + // quietly leaving it (and its callers) out. + let broken: Vec<_> = catalog.diagnostics.iter().filter(|d| d.severity == Severity::Error).collect(); + if !broken.is_empty() { + for d in &broken { + eprintln!("{d}"); + } + return Err(fail( + exit::USAGE, + format!("{} validation error(s); fix them before selecting flows", broken.len()), + )); + } let changed = lest_core::affected::changed_since(&ctx.project.root, &a.base).map_err(|e| fail(exit::ERRORED, e))?; let affected = lest_core::affected::affected(&catalog, &changed).map_err(|e| fail(exit::USAGE, e))?; if a.id { diff --git a/docs/ci.md b/docs/ci.md index 483ab56..d5f6797 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -21,23 +21,35 @@ jobs: env: LEST_SECRET_API_TOKEN: ${{ secrets.API_TOKEN }} run: | - for flow in $(lest affected --base origin/main --id); do + set -e + # On its own line, so a failure (a missing base, a broken flow) + # fails the job instead of selecting nothing. + flows="$(lest affected --base origin/main --id)" + set +e + failed=0 + for flow in $flows; do lest run "$flow" --non-interactive --junit "junit-$flow.xml" || failed=1 done - exit ${failed:-0} + exit $failed ``` ## Choosing what to run `lest affected --base ` lists the flows a change touches: -- a changed file matches one of the flow's `affects:` globs, -- the flow's own file changed, or +- a changed file matches one of the flow's `affects:` globs (`*` stays + within a directory, `**` crosses directories), +- the flow's own file, or a browser `script:` it runs, changed, +- `lest.yaml` changed (every flow is affected), or - the flow calls an affected flow (suites run when any member is affected). Changes are counted from the merge base of `` and `HEAD`, plus staged, -unstaged and untracked files. Paths are relative to the project root, so a -project in a subdirectory of the repository works. +unstaged and untracked files; a rename counts as both paths. Paths are +relative to the project root, so a project in a subdirectory of the +repository works. On the base branch itself the merge base is `HEAD`, so +only uncommitted changes count: run everything there instead (for example +`lest run everything`). If any flow fails to validate, `lest affected` exits +2 rather than leaving it out. ```console $ lest affected --base origin/main @@ -58,10 +70,12 @@ sign-in | app/server.mjs views. - **JSON:** `-o ` writes the full report (`lest schema report` describes it). -- **Evidence:** `lest bundle -o evidence.zip` zips the report and every - artifact with a `manifest.json` of SHA-256 checksums, ready to attach to a - ticket, release or audit. Secret values are already redacted in everything - it contains. +- **Evidence:** `lest bundle -o evidence.zip` zips the report and the + files it lists (step artifacts, service logs, the demo cut) with a + `manifest.json` of SHA-256 checksums, ready to attach to a ticket, release + or audit. Anything else a step wrote into the run directory stays out. Text + artifacts and the report are redacted; binary artifacts (images, videos) + are included as they are, so keep secrets off screen. ## Secrets @@ -88,5 +102,6 @@ notify: `json` posts the run summary (flow, run id, result, environment, duration, the failing step and its headline, warnings). `slack` posts an -incoming-webhook message. Notifications go out after the report is written, -time out after 10 seconds, and never print the URL, which may embed a secret. +incoming-webhook message, with flow names and errors escaped. Notifications +go out after the report is written, all at once, so the slowest one bounds +the wait at 10 seconds; they never print the URL, which may embed a secret.