From ca60236f8a4b3c3ba185000ee2ef10268a094901 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:25:23 -0800 Subject: [PATCH 01/11] chore(quest): claim quest/m0/ffi-test-builds.md Quest-Claim: 6e30551a24d6ef07 From a8a6c66e4b10858887761a7c6262c713bbd25459 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:39:53 -0800 Subject: [PATCH 02/11] perf(ffi): a warm test run rebuilds nothing unchanged The exporter writes only files whose content changed, prunes only what it no longer writes, and keeps the lockfile's mtime when it resolves as before. The fixtures cache each language's compile behind a content-keyed stamp, build each glue feature in its own target directory, and write their scratch packages only on change. Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 3 + ffi/src/cli.rs | 17 +- ffi/src/driver.rs | 4 + ffi/src/glue.rs | 96 ++++- ffi/tests/fixtures.rs | 691 +++++++++++++++++++++++------------ ffi/tests/fixtures/shapes.rs | 175 +++++---- ffi/tests/fixtures/step.rs | 358 ++++++++++++++++++ 7 files changed, 1034 insertions(+), 310 deletions(-) create mode 100644 ffi/tests/fixtures/step.rs diff --git a/ffi/README.md b/ffi/README.md index ed040187d..22fdf92af 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -48,6 +48,9 @@ extension the crate builds with `--features python`, `swift/.swift` hold Swift, over the C header as a clang module, `go/` holds it in Go, a package per crate over the C ABI through cgo, and `dart/.dart` holds it in Dart, over the C ABI through `dart:ffi` and what the crate builds with `--features dart`. +An export again rewrites only the files whose content changed, and removes from +`include//` and `go/` only what it no longer writes, so a build of an unchanged +export stays fresh. ```toml # ffi.toml, beside the package's Cargo.toml unless --manifest-path says otherwise. diff --git a/ffi/src/cli.rs b/ffi/src/cli.rs index c8da50144..ec01eb7f0 100644 --- a/ffi/src/cli.rs +++ b/ffi/src/cli.rs @@ -146,13 +146,28 @@ fn try_run(args: &[String]) -> Result { /// check did not resolve for the same dependent, which the glue was not generated against. fn lock(checked: &Checked, out: &Path, picks: &Picks<'_>) -> Result<(), String> { let lockfile = checked.metadata.workspace_root.join("Cargo.lock"); - std::fs::copy(&lockfile, out.join("Cargo.lock")) + let written = out.join("Cargo.lock"); + let earlier = std::fs::read(&written) + .ok() + .zip(std::fs::metadata(&written).and_then(|meta| meta.modified()).ok()); + std::fs::copy(&lockfile, &written) .map_err(|err| format!("copying {} into {}: {err}", lockfile.display(), out.display()))?; let output = run_cargo( &out.join("Cargo.toml"), &["metadata", "--format-version", "1"], &picks.resolving(), )?; + // Resolved as before, the lockfile keeps its mtime, as every unchanged file of the + // export does: cargo reruns a build script when any file of its package is newer. + if let Some((bytes, mtime)) = earlier + && std::fs::read(&written).is_ok_and(|now| now == bytes) + { + std::fs::File::options() + .write(true) + .open(&written) + .and_then(|file| file.set_modified(mtime)) + .map_err(|err| format!("writing {}: {err}", written.display()))?; + } let generated: Metadata = serde_json::from_slice(&output).map_err(|err| format!("reading cargo metadata: {err}"))?; // The check's resolution holds more than its build: cargo metadata enables what the diff --git a/ffi/src/driver.rs b/ffi/src/driver.rs index 3deb4d693..53deb2db6 100644 --- a/ffi/src/driver.rs +++ b/ffi/src/driver.rs @@ -529,7 +529,11 @@ fn at(tcx: TyCtxt<'_>, span: Span) -> String { format!("{name}:{line}:{col}") } +/// Write `text` to `path` unless it already holds it, so its mtime moves only on a change. pub fn write(path: &Path, text: &str) -> Result<(), String> { + if std::fs::read(path).is_ok_and(|old| old == text.as_bytes()) { + return Ok(()); + } if let Some(dir) = path.parent() { std::fs::create_dir_all(dir).map_err(|err| format!("creating {}: {err}", dir.display()))?; } diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index c307d3368..4695c38be 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -5,6 +5,7 @@ //! never frees, and lowered into memory the caller frees. A handle is borrowed as the //! receiver or parameter is, or moved out of when taken by value. +use std::collections::HashSet; use std::fmt::Write; use std::path::{Path, PathBuf}; @@ -121,7 +122,8 @@ pub struct Crate<'a> { pub struct Output { pub names: crate::lock::Names, files: Vec<(PathBuf, String)>, - /// Directories an earlier export wrote, which may name crates or packages this one has not. + /// Directories the export owns, where an earlier export may have written files for + /// crates or packages this one has not. stale: Vec, } @@ -130,7 +132,7 @@ impl Output { self.files.push((path.to_path_buf(), text.to_string())); } - /// Refuse a baseline below a directory this export removes before writing. + /// Refuse a baseline below a directory the export owns, which removes what it did not write. pub fn check_baseline(&self, baseline: &Path) -> Result<(), String> { let baseline = std::fs::canonicalize(baseline).map_err(|err| format!("reading {}: {err}", baseline.display()))?; @@ -146,18 +148,46 @@ impl Output { Ok(()) } - /// Write a validated export, removing stale C++ headers and Go packages from an earlier one. + /// Write a validated export, removing the C++ headers and Go files an earlier one wrote + /// that this one does not. A file whose content is unchanged keeps its mtime, so a build + /// of the export that cargo or a consumer fingerprints stays fresh. pub fn write(self) -> Result { + let files: HashSet<&Path> = self.files.iter().map(|(path, _)| path.as_path()).collect(); for dir in self.stale.iter().filter(|dir| dir.exists()) { - std::fs::remove_dir_all(dir).map_err(|err| format!("removing {}: {err}", dir.display()))?; + prune(dir, &files)?; } - for (path, text) in self.files { - crate::driver::write(&path, &text)?; + for (path, text) in &self.files { + crate::driver::write(path, text)?; } Ok(self.names) } } +/// Remove what `dir` holds that is not one of `keep`, and each directory that leaves empty. +fn prune(dir: &Path, keep: &HashSet<&Path>) -> Result<(), String> { + let entries = std::fs::read_dir(dir).map_err(|err| format!("reading {}: {err}", dir.display()))?; + for entry in entries { + let entry = entry.map_err(|err| format!("reading {}: {err}", dir.display()))?; + let path = entry.path(); + let kind = entry + .file_type() + .map_err(|err| format!("reading {}: {err}", path.display()))?; + if kind.is_dir() { + prune(&path, keep)?; + } else if !keep.contains(path.as_path()) { + std::fs::remove_file(&path).map_err(|err| format!("removing {}: {err}", path.display()))?; + } + } + let empty = std::fs::read_dir(dir) + .map_err(|err| format!("reading {}: {err}", dir.display()))? + .next() + .is_none(); + if empty { + std::fs::remove_dir(dir).map_err(|err| format!("removing {}: {err}", dir.display()))?; + } + Ok(()) +} + impl Crate<'_> { /// The facade's package name. fn packages(&self) -> String { @@ -1912,3 +1942,57 @@ fn go_module_resolves_missing_parent() { assert!(generated.check_baseline(&root.join("baseline-link.json")).is_err()); } } + +/// A second write of the same export keeps every file's mtime, and drops only the files in +/// the directories it owns that it no longer writes, with any directory that leaves empty. +#[test] +fn rewrite_keeps_unchanged_files() { + let root = std::env::current_dir() + .unwrap() + .join(".tmp") + .join(format!("rewrite-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let go = root.join("go"); + let output = |files: &[(&str, &str)]| Output { + names: Default::default(), + files: files + .iter() + .map(|(path, text)| (root.join(path), text.to_string())) + .collect(), + stale: vec![go.clone()], + }; + let first = [ + ("Cargo.toml", "a"), + ("go/rt/rt.go", "rt"), + ("go/old/old.go", "old"), + ("go/old/deep/old.go", "old"), + ]; + output(&first).write().unwrap(); + std::fs::write(root.join("go/rt/stray.go"), "stray").unwrap(); + std::fs::write(root.join("kept.txt"), "kept").unwrap(); + // Set in the past, so a rewrite would move it. + let past = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1 << 30); + for path in ["Cargo.toml", "go/rt/rt.go"] { + let file = std::fs::File::options().write(true).open(root.join(path)).unwrap(); + file.set_modified(past).unwrap(); + } + let mtime = |path: &str| std::fs::metadata(root.join(path)).unwrap().modified().unwrap(); + output(&[("Cargo.toml", "a"), ("go/rt/rt.go", "rt"), ("go/new.go", "new")]) + .write() + .unwrap(); + assert_eq!([mtime("Cargo.toml"), mtime("go/rt/rt.go")], [past, past]); + assert_eq!(std::fs::read_to_string(root.join("go/new.go")).unwrap(), "new"); + assert!(!root.join("go/old").exists(), "an emptied directory stays"); + assert!( + !root.join("go/rt/stray.go").exists(), + "a file the export does not write stays" + ); + assert!( + root.join("kept.txt").exists(), + "a file outside the export's directories went" + ); + output(&[("Cargo.toml", "b")]).write().unwrap(); + assert_eq!(std::fs::read_to_string(root.join("Cargo.toml")).unwrap(), "b"); + assert!(!go.exists(), "the export's emptied directory stays"); + std::fs::remove_dir_all(&root).unwrap(); +} diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 751095fe0..baccd3f4f 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -8,6 +8,8 @@ #[path = "fixtures/shapes.rs"] mod shapes; +#[path = "fixtures/step.rs"] +mod step; use std::env::consts::EXE_SUFFIX; use std::ffi::OsString; @@ -15,6 +17,8 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::OnceLock; +use step::Step; + fn manifest_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } @@ -38,19 +42,33 @@ fn host() -> &'static str { /// host lacks and a variant of an enum. const WASM: &str = "wasm32-unknown-unknown"; +/// Cargo, on a `PATH` without what changes each run, as a Cargo shim's session does: build +/// scripts that read `PATH`, such as pyo3's, would otherwise run again each time, and +/// rebuild the glue. fn cargo() -> Command { - Command::new(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())) + let mut cmd = Command::new(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())); + cmd.env("PATH", step::path()); + cmd } fn run(cmd: &mut Command) -> Output { - let output = cmd.output().unwrap_or_else(|err| panic!("running {cmd:?}: {err}")); + let output = output(cmd); + check(cmd, &output); + output +} + +fn output(cmd: &mut Command) -> Output { + cmd.output().unwrap_or_else(|err| panic!("running {cmd:?}: {err}")) +} + +/// Asserts `cmd` succeeded, printing what it printed if not. +fn check(cmd: &Command, output: &Output) { assert!( output.status.success(), "{cmd:?} failed:\n{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - output } /// Export the fixture `name` into `out` for `targets` into one binding, else the host, @@ -114,7 +132,7 @@ fn api() -> &'static Api { String::from_utf8_lossy(&output.stderr) ); let target = scratch().join("api").join("glue"); - let build = |feature: &str| { + let build = |target: &Path, feature: &str| { run(cargo() .arg("build") .arg("--quiet") @@ -122,21 +140,23 @@ fn api() -> &'static Api { .arg("--manifest-path") .arg(out.join("Cargo.toml")) .arg("--target-dir") - .arg(&target) + .arg(target) .args(["--features", feature]) // Glue the exporter writes builds without warnings. .env("RUSTFLAGS", "-D warnings --print=native-static-libs")) }; // The CPython extension's symbols would leave the C programs' links undefined, so - // it builds apart, before the library they link, and moves beside the Python. - build("python"); - std::fs::rename( - target.join("debug").join(PYTHON_LIB), - out.join("python").join(PYTHON_LIB), - ) - .unwrap(); + // it builds apart, before the library they link, and is copied beside the Python. + // Each build has its own target directory, since cargo keeps one fingerprint of a + // package across its features, and each build would make the other's stale. + let python = scratch().join("api").join("glue-python"); + build(&python, "python"); + step::copy( + &python.join("debug").join(PYTHON_LIB), + &out.join("python").join(PYTHON_LIB), + ); // The shared library Kotlin and Dart load. - let built = build("kotlin,dart"); + let built = build(&target, "kotlin,dart"); let native = String::from_utf8_lossy(&built.stderr) .lines() .find_map(|line| line.strip_prefix("note: native-static-libs: ")) @@ -174,14 +194,23 @@ fn api_c() { let api = api(); let exe = scratch().join("api").join("api-test"); let cc = std::env::var_os("CC").unwrap_or_else(|| "cc".into()); - run(Command::new(cc) - .args(["-std=c11", "-pedantic", "-Wall", "-Wextra", "-Werror", "-I"]) - .arg(api.out.join("include")) - .arg(manifest_dir().join("tests").join("api.c")) - .arg(api.staticlib()) - .args(&api.native) - .arg("-o") - .arg(&exe)); + let src = manifest_dir().join("tests").join("api.c"); + let include = api.out.join("include"); + Step { + stamp: &step::stamp(&exe), + reads: &[&src, &include, &api.staticlib()], + writes: &[&exe], + } + .run( + Command::new(cc) + .args(["-std=c11", "-pedantic", "-Wall", "-Wextra", "-Werror", "-I"]) + .arg(&include) + .arg(&src) + .arg(api.staticlib()) + .args(&api.native) + .arg("-o") + .arg(&exe), + ); let c = run(&mut Command::new(&exe)); assert_eq!( String::from_utf8_lossy(&c.stdout), @@ -342,13 +371,13 @@ fn api_cpp() { let api = api(); for (standard, plain) in [(17, true), (20, false), (23, true)] { let exe = scratch().join("api").join(format!("api-c++{standard}{EXE_SUFFIX}")); - run(&mut cxx( + cxx( &api.link(), standard, plain, &manifest_dir().join("tests").join("api.cpp"), &exe, - )); + ); let cpp = run(&mut Command::new(&exe)); assert_eq!( String::from_utf8_lossy(&cpp.stdout), @@ -402,20 +431,19 @@ fn api_cpp() { assert!(headers.len() > 2, "the api fixture has a header per crate: {headers:?}"); for header in headers { let src = scratch().join("api").join(format!("alone-{header}.cpp")); - std::fs::write(&src, format!("#include \"api/{header}\"\nint main() {{}}\n")).unwrap(); + step::write(&src, format!("#include \"api/{header}\"\nint main() {{}}\n")); let exe = scratch().join("api").join(format!("alone-{header}{EXE_SUFFIX}")); - run(&mut cxx(&api.link(), 17, true, &src, &exe)); + cxx(&api.link(), 17, true, &src, &exe); } // A C++ program whose header names another contract than its library's aborts at load. let mismatched = scratch().join("api").join("contract.cpp"); - std::fs::write( + step::write( &mismatched, "#include \"api.h\"\n#undef API_CONTRACT\n#define API_CONTRACT ((uint64_t)0)\n#include \"api.hpp\"\nint main() {}\n", - ) - .unwrap(); + ); let contract = scratch().join("api").join(format!("contract{EXE_SUFFIX}")); - run(&mut cxx(&api.link(), 17, false, &mismatched, &contract)); + cxx(&api.link(), 17, false, &mismatched, &contract); let output = Command::new(&contract).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not abort"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -428,19 +456,36 @@ fn api_cpp() { /// Clang's ownership checks: the whole script builds clean under them, and each case of /// `tests/api_strict.cpp` fails with its diagnostic. fn cpp_strict(api: &Api) { + let include = api.out.join("include"); + // What each case printed, kept apart from the others'; its object is not. let clang = |src: &Path, case: u8| { let mut cmd = Command::new("clang++"); cmd.args(["-std=c++17", "-Wconsumed", "-Wdangling", "-Werror", "-c", "-I"]) - .arg(api.out.join("include")) + .arg(&include) .arg(format!("-DCASE={case}")) .arg(src) .arg("-o") .arg(scratch().join("api").join("strict.o")); - cmd + let stamp = scratch() + .join("api") + .join("strict") + .join(format!("{}-{case}.stamp", src.file_stem().unwrap().to_string_lossy())); + let step = Step { + stamp: &stamp, + reads: &[src, &include], + writes: &[], + }; + step.output(&mut cmd) }; let tests = manifest_dir().join("tests"); - run(&mut clang(&tests.join("api.cpp"), 0)); - run(&mut clang(&tests.join("api_strict.cpp"), 0)); + for src in ["api.cpp", "api_strict.cpp"] { + let output = clang(&tests.join(src), 0); + assert!( + output.status.success(), + "{src} failed Clang's ownership checks:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + } for (case, message) in [ ( 1, @@ -462,7 +507,7 @@ fn cpp_strict(api: &Api) { (6, "no matching member function for call to 'emplace'"), (7, "temporary whose address is used as value of local variable 'name'"), ] { - let output = clang(&tests.join("api_strict.cpp"), case).output().unwrap(); + let output = clang(&tests.join("api_strict.cpp"), case); assert!(!output.status.success(), "case {case} built"); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains(message), "case {case} printed {stderr}"); @@ -487,10 +532,19 @@ impl Api { } } -/// A build of the C++ `src` against the `link`ed library into `exe`, at C++`standard`, with +/// Build the C++ `src` against the `link`ed library into `exe`, at C++`standard`, with /// exceptions and RTTI off if `plain`, and warnings as errors: by `CXX` or the platform's /// compiler, given MSVC's flags where the target is MSVC's. -fn cxx(link: &Link, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command { +fn cxx(link: &Link, standard: u8, plain: bool, src: &Path, exe: &Path) { + let step = Step { + stamp: &step::stamp(exe), + reads: &[src, &link.include, &link.staticlib], + writes: &[exe], + }; + step.run(&mut cxx_command(link, standard, plain, src, exe)); +} + +fn cxx_command(link: &Link, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command { let include = &link.include; if cfg!(target_env = "msvc") { // cl from a developer environment. Its C++23 is `c++latest`; its objects go beside `exe`. @@ -541,34 +595,40 @@ fn cxx(link: &Link, standard: u8, plain: bool, src: &Path, exe: &Path) -> Comman fn api_kotlin() { let api = api(); let classes = scratch().join("api").join("kotlin"); - let _ = std::fs::remove_dir_all(&classes); - run(&mut kotlinc(&api.out, &classes)); - let mut classpath = classes.clone().into_os_string(); - classpath.push(":"); - classpath.push(coroutines()); - let java = |args: &[&str]| { + let jar = PathBuf::from(coroutines()); + let step = Step { + stamp: &step::stamp(&classes), + reads: &[ + &api.out.join("kotlin").join("api.kt"), + &manifest_dir().join("tests").join("api.kt"), + &jar, + ], + writes: &[&classes], + }; + step.run(&mut kotlinc(&api.out, &classes)); + let java = |classpath: &[&Path], args: &[&str]| { let mut cmd = Command::new("kotlin"); cmd.arg(format!("-Djava.library.path={}", api.lib.display())) .arg("-cp") - .arg(&classpath) + .arg(std::env::join_paths(classpath.iter().copied().chain([jar.as_path()])).unwrap()) .arg("ApiKt") .args(args); cmd }; - kotlin(&api.transcript, &java); + kotlin(&api.transcript, &|args| java(&[&classes], args)); // The same classes, with another contract than the library's in place of theirs, - // refuse to load it. - let native_class = classes.join("api").join("Native.class"); - let bytes = std::fs::read(&native_class).unwrap(); + // refuse to load it: a Native.class of its own, ahead on the classpath, leaves the + // compiled classes as they were. + let bytes = std::fs::read(classes.join("api").join("Native.class")).unwrap(); let contract = kotlin_contract(&api.out.join("kotlin").join("api.kt")).to_be_bytes(); let at: Vec = (0..bytes.len() - 8).filter(|&i| bytes[i..i + 8] == contract).collect(); assert_eq!(at.len(), 1, "Native.class holds the contract once"); let mut mismatched = bytes.clone(); mismatched[at[0]] ^= 0xff; - std::fs::write(&native_class, &mismatched).unwrap(); - let output = java(&[]).output().unwrap(); - std::fs::write(&native_class, &bytes).unwrap(); + let shadow = scratch().join("api").join("kotlin-mismatched"); + step::write(&shadow.join("api").join("Native.class"), &mismatched); + let output = java(&[&shadow, &classes], &[]).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); assert!( @@ -612,8 +672,9 @@ fn api_android() { .arg(dir.join("lib")) .args(["build", "--quiet", "--locked", "--manifest-path"]) .arg(out.join("Cargo.toml")) + // Apart from the host's glue, whose units share the export's names. .arg("--target-dir") - .arg(scratch().join("api").join("glue")) + .arg(dir.join("glue")) .args(["--features", "kotlin"]) .env("RUSTFLAGS", "-D warnings")); @@ -808,9 +869,7 @@ fn api_python() { // The same Python, with another contract than the library's in place of its own, // refuses to load it. let mismatched = scratch().join("api").join("python-mismatched"); - let _ = std::fs::remove_dir_all(&mismatched); - std::fs::create_dir_all(&mismatched).unwrap(); - std::fs::copy(dir.join(PYTHON_LIB), mismatched.join(PYTHON_LIB)).unwrap(); + step::copy(&dir.join(PYTHON_LIB), &mismatched.join(PYTHON_LIB)); let source = std::fs::read_to_string(dir.join("api.py")).unwrap(); let at = source .find("_n.contract() != 0x") @@ -821,7 +880,7 @@ fn api_python() { _ => "0", }; let source = format!("{}{flipped}{}", &source[..at], &source[at + 1..]); - std::fs::write(mismatched.join("api.py"), source).unwrap(); + step::write(&mismatched.join("api.py"), source); let output = python(&mismatched, &[]).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -837,20 +896,27 @@ fn api_python() { #[test] fn api_py_types() { let dir = scratch().join("api").join("python-types"); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); let module = dir.join("api.py"); let caller = dir.join("api_types.py"); - std::fs::copy(api().out.join("python").join("api.py"), &module).unwrap(); - std::fs::copy(manifest_dir().join("tests").join("api_types.py"), &caller).unwrap(); - run(Command::new("mypy") - .args(["--strict", "--python-version", "3.11", "--cache-dir"]) - .arg(scratch().join("mypy")) - .args([&module, &caller])); + step::copy(&api().out.join("python").join("api.py"), &module); + step::copy(&manifest_dir().join("tests").join("api_types.py"), &caller); // Rooted in `dir`, so `import api` finds the module rather than `tests/api.py`. let config = dir.join("pyrightconfig.json"); - std::fs::copy(manifest_dir().join("tests").join("pyright.json"), &config).unwrap(); - run(Command::new("pyright").arg("-p").arg(&config).args([&module, &caller])); + step::copy(&manifest_dir().join("tests").join("pyright.json"), &config); + let mut mypy = Command::new("mypy"); + mypy.args(["--strict", "--python-version", "3.11", "--cache-dir"]) + .arg(scratch().join("mypy")) + .args([&module, &caller]); + let mut pyright = Command::new("pyright"); + pyright.arg("-p").arg(&config).args([&module, &caller]); + for (checker, mut cmd) in [("mypy", mypy), ("pyright", pyright)] { + let step = Step { + stamp: &dir.join(format!("{checker}.stamp")), + reads: &[&module, &caller, &config], + writes: &[], + }; + step.run(&mut cmd); + } } /// The contract a Kotlin file checks its library against. @@ -872,34 +938,56 @@ fn kotlin_contract(kotlin: &Path) -> u64 { fn api_swift() { let api = api(); let dir = scratch().join("api").join("swift"); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); let module = api.out.join("swift"); + let include = api.out.join("include"); let source = module.join("api.swift"); let collections = swift_collections(); - run(swiftc(&module, 13) - .args(["-swift-version", "5", "-typecheck"]) - .arg(&source)); - run(swiftc(&module, 13) - .args(["-swift-version", "6", "-parse-as-library", "-module-name", "Api"]) - .args(["-emit-module", "-emit-module-path"]) - .arg(dir.join("Api.swiftmodule")) - .args(["-emit-library", "-static", "-o"]) - .arg(dir.join("libApi.a")) - .arg(&source)); + let built = collections_dir(); + let step = Step { + stamp: &dir.join("typecheck.stamp"), + reads: &[&module, &include, &built], + writes: &[], + }; + step.run( + swiftc(&module, 13) + .args(["-swift-version", "5", "-typecheck"]) + .arg(&source), + ); + let (swiftmodule, lib) = (dir.join("Api.swiftmodule"), dir.join("libApi.a")); + let step = Step { + stamp: &dir.join("Api.stamp"), + reads: &[&module, &include, &built], + writes: &[&swiftmodule, &lib], + }; + step.run( + swiftc(&module, 13) + .args(["-swift-version", "6", "-parse-as-library", "-module-name", "Api"]) + .args(["-emit-module", "-emit-module-path"]) + .arg(&swiftmodule) + .args(["-emit-library", "-static", "-o"]) + .arg(&lib) + .arg(&source), + ); // The test's executor preference is macOS 15's. let exe = dir.join("api-test"); let test = manifest_dir().join("tests").join("api.swift"); - run(swiftc(&module, 15) - .args(["-parse-as-library", "-I"]) - .arg(&dir) - .arg(&test) - .arg(dir.join("libApi.a")) - .args(&collections) - .arg(api.staticlib()) - .args(&api.native) - .arg("-o") - .arg(&exe)); + let step = Step { + stamp: &step::stamp(&exe), + reads: &[&test, &module, &include, &built, &swiftmodule, &lib, &api.staticlib()], + writes: &[&exe], + }; + step.run( + swiftc(&module, 15) + .args(["-parse-as-library", "-I"]) + .arg(&dir) + .arg(&test) + .arg(&lib) + .args(&collections) + .arg(api.staticlib()) + .args(&api.native) + .arg("-o") + .arg(&exe), + ); let swift = run(&mut Command::new(&exe)); assert_eq!( String::from_utf8_lossy(&swift.stdout), @@ -919,15 +1007,20 @@ fn api_swift() { ), ("LENT_KEPT", "implicit conversion to 'Tally?' is consuming"), ] { - let output = swiftc(&module, 15) - .args(["-parse-as-library", "-emit-sil", "-o"]) - .arg(dir.join("broken.sil")) - .args(["-I"]) - .arg(&dir) - .arg(format!("-D{case}")) - .arg(&test) - .output() - .unwrap(); + let step = Step { + stamp: &dir.join(format!("{case}.stamp")), + reads: &[&test, &module, &include, &built, &swiftmodule], + writes: &[], + }; + let output = step.output( + swiftc(&module, 15) + .args(["-parse-as-library", "-emit-sil", "-o"]) + .arg(dir.join("broken.sil")) + .args(["-I"]) + .arg(&dir) + .arg(format!("-D{case}")) + .arg(&test), + ); assert!(!output.status.success(), "Swift {case} compiled"); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains(message), "Swift {case} printed {stderr}"); @@ -959,19 +1052,26 @@ fn api_swift() { let digit = if &generated[at..at + 1] == "0" { "1" } else { "0" }; mismatched.replace_range(at..at + 1, digit); let contract = dir.join("contract"); - std::fs::create_dir_all(&contract).unwrap(); - std::fs::write(contract.join("api.swift"), mismatched).unwrap(); - std::fs::write(contract.join("main.swift"), "_ = dropped()\n").unwrap(); + let (swift, main) = (contract.join("api.swift"), contract.join("main.swift")); + step::write(&swift, mismatched); + step::write(&main, "_ = dropped()\n"); let exe = contract.join("contract"); - run(swiftc(&module, 13) - .args(["-module-name", "Api"]) - .arg(contract.join("api.swift")) - .arg(contract.join("main.swift")) - .args(&collections) - .arg(api.staticlib()) - .args(&api.native) - .arg("-o") - .arg(&exe)); + let step = Step { + stamp: &step::stamp(&exe), + reads: &[&swift, &main, &module, &include, &built, &api.staticlib()], + writes: &[&exe], + }; + step.run( + swiftc(&module, 13) + .args(["-module-name", "Api"]) + .arg(&swift) + .arg(&main) + .args(&collections) + .arg(api.staticlib()) + .args(&api.native) + .arg("-o") + .arg(&exe), + ); let output = Command::new(&exe).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -988,10 +1088,13 @@ fn api_swift() { fn api_dart() { let api = api(); let dir = scratch().join("api").join("dart"); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); let generated = api.out.join("dart"); - run(Command::new("dart").args(["analyze", "--fatal-infos"]).arg(&generated)); + let step = Step { + stamp: &dir.join("analyze.stamp"), + reads: &[&generated], + writes: &[], + }; + step.run(Command::new("dart").args(["analyze", "--fatal-infos"]).arg(&generated)); let lib = api.lib.join(PYTHON_LIB); let exe = dart_compile( "api", @@ -1044,16 +1147,16 @@ fn api_dart() { let mut mismatched = source.clone(); let digit = if &source[at..at + 1] == "0" { "1" } else { "0" }; mismatched.replace_range(at..at + 1, digit); + // The package apart from the program's directory, which its build writes into. let contract = dir.join("contract"); - std::fs::create_dir_all(&contract).unwrap(); - std::fs::write(contract.join("api.dart"), mismatched).unwrap(); + let package = contract.join("package"); + step::write(&package.join("api.dart"), mismatched); let main = contract.join("main.dart"); - std::fs::write( + step::write( &main, "import 'package:api/api.dart' as api;\n\nvoid main() => api.dropped();\n", - ) - .unwrap(); - let exe = dart_compile("api", &contract, &contract, &lib, &main); + ); + let exe = dart_compile("api", &contract, &package, &lib, &main); let output = dart_run(&exe, &[]).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -1075,13 +1178,12 @@ fn dart_compile(name: &str, dir: &Path, package: &Path, lib: &Path, main: &Path) .to_path_buf(); let config = dir.join("package_config.json"); let root = url(package); - std::fs::write( + step::write( &config, format!( "{{\"configVersion\": 2, \"packages\": [{{\"name\": \"{name}\", \"rootUri\": \"{root}/\", \"packageUri\": \"./\"}}]}}\n" ), - ) - .unwrap(); + ); let os = match std::env::consts::OS { "macos" => "macos", "windows" => "windows", @@ -1092,16 +1194,21 @@ fn dart_compile(name: &str, dir: &Path, package: &Path, lib: &Path, main: &Path) _ => "x64", }; let assets = dir.join("native_assets.yaml"); - std::fs::write( + step::write( &assets, format!( "format-version: [1, 0, 0]\nnative-assets:\n {os}_{arch}:\n \"package:{name}/{name}.dart\": [absolute, {:?}]\n", lib.display().to_string() ), - ) - .unwrap(); + ); let dill = dir.join(format!("{}.dill", main.file_stem().unwrap().to_string_lossy())); - run( + // The snapshot names the library by path, so a change to it rebuilds nothing. + let step = Step { + stamp: &step::stamp(&dill), + reads: &[main, package, &config, &assets, &sdk.join("version")], + writes: &[&dill], + }; + step.run( Command::new(sdk.join("bin").join(format!("dartaotruntime{EXE_SUFFIX}"))) .arg(sdk.join("bin/snapshots/gen_kernel_aot.dart.snapshot")) .arg("--platform") @@ -1189,28 +1296,24 @@ fn swiftc(module: &Path, macos: u8) -> Command { #[test] fn api_go() { let api = api(); - // The export's Go, the module of a package per crate that it declares, beside the header - // as the export lays them out, and the test's module, which requires it. - let module = scratch().join("api").join("go"); - let _ = std::fs::remove_dir_all(&module); + let dir = scratch().join("api").join("go"); + let module = dir.join("module"); + go_layout( + api, + &module, + &std::fs::read_to_string(manifest_dir().join("tests").join("api.go")).unwrap(), + ); let export = module.join("export"); - copy_dir(&api.out.join("go"), &export); - std::fs::create_dir_all(module.join("include")).unwrap(); - std::fs::copy( - api.out.join("include").join("api.h"), - module.join("include").join("api.h"), - ) - .unwrap(); - std::fs::copy(manifest_dir().join("tests").join("api.go"), module.join("main.go")).unwrap(); - std::fs::write( - module.join("go.mod"), - "module apitest\n\ngo 1.24\n\nrequire api v0.0.0\n\nreplace api => ./export\n", - ) - .unwrap(); - run(&mut go(&export, api, &["vet", "./..."])); - let exe = module.join(format!("api-test{EXE_SUFFIX}")); - run(&mut go(&module, api, &["vet", "./..."])); - run(go(&module, api, &["build", "-o"]).arg(&exe)); + for (name, root) in [("vet-export", &export), ("vet", &module)] { + let step = Step { + stamp: &dir.join(format!("{name}.stamp")), + reads: &[&module], + writes: &[], + }; + step.run(&mut go(root, api, &["vet", "./..."])); + } + let exe = dir.join(format!("api-test{EXE_SUFFIX}")); + go_build(api, &module, &exe); let output = run(&mut Command::new(&exe)); assert_eq!( String::from_utf8_lossy(&output.stdout), @@ -1245,8 +1348,14 @@ fn api_go() { } // The same packages, with another contract than the library's in place of their own, - // refuse it when they load. - let runtime = export.join("rt").join("rt.go"); + // refuse it when they load: in a module of their own, which leaves the test's as it was. + let contract = dir.join("contract"); + go_layout( + api, + &contract, + "package main\n\nimport \"api/api\"\n\nfunc main() {\n\tapi.Dropped()\n}\n", + ); + let runtime = contract.join("export").join("rt").join("rt.go"); let generated = std::fs::read_to_string(&runtime).unwrap(); let at = generated .find("api_contract() != 0x") @@ -1255,13 +1364,9 @@ fn api_go() { let mut mismatched = generated.clone(); let digit = if &generated[at..at + 1] == "0" { "1" } else { "0" }; mismatched.replace_range(at..at + 1, digit); - std::fs::write(&runtime, mismatched).unwrap(); - std::fs::write( - module.join("main.go"), - "package main\n\nimport \"api/api\"\n\nfunc main() {\n\tapi.Dropped()\n}\n", - ) - .unwrap(); - run(go(&module, api, &["build", "-o"]).arg(&exe)); + step::write(&runtime, mismatched); + let exe = dir.join(format!("contract-test{EXE_SUFFIX}")); + go_build(api, &contract, &exe); let output = Command::new(&exe).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -1271,6 +1376,31 @@ fn api_go() { ); } +/// Lay out `module` as a Go module whose main package is `main`, requiring the export's +/// module of a package per crate, beside the header as the export lays them out. +fn go_layout(api: &Api, module: &Path, main: &str) { + step::mirror(&api.out.join("go"), &module.join("export")); + step::copy( + &api.out.join("include").join("api.h"), + &module.join("include").join("api.h"), + ); + step::write(&module.join("main.go"), main); + step::write( + &module.join("go.mod"), + "module apitest\n\ngo 1.24\n\nrequire api v0.0.0\n\nreplace api => ./export\n", + ); +} + +/// Build the Go `module` into `exe` against the api fixture's static library. +fn go_build(api: &Api, module: &Path, exe: &Path) { + let step = Step { + stamp: &step::stamp(exe), + reads: &[module, &api.staticlib()], + writes: &[exe], + }; + step.run(go(module, api, &["build", "-o"]).arg(exe)); +} + /// Under a `go.mod`, the generated packages import each other by its module, as the go /// command reads it from a block among comments, and the export writes no `go.mod` of its own. #[test] @@ -1303,7 +1433,7 @@ fn go_module() { #[test] fn go_reserved_packages() { for name in ["rt", "func", "main"] { - let package = package(&format!("go_reserved_{name}")); + let package = package(&format!("go_reserved_{name}"), &[]); let manifest = format!( "[package]\nname = \"go_reserved_{name}\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[lib]\nname = \"{name}\"\n\n[workspace]\n" ); @@ -1333,7 +1463,7 @@ fn go_reserved_packages() { /// `pkg2d`, which Go vets. #[test] fn go_package_names() { - let package = package("go_package_names"); + let package = package("go_package_names", &[]); let dep = |lib: &str| { ( format!( @@ -1399,21 +1529,6 @@ fn vet(go: &Path) { .env("GOTOOLCHAIN", "local")); } -/// A copy of the directory `from` at `to`. -fn copy_dir(from: &Path, to: &Path) { - std::fs::create_dir_all(to).unwrap(); - for entry in std::fs::read_dir(from).unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - match entry.file_type().unwrap().is_dir() { - true => copy_dir(&path, &to.join(entry.file_name())), - false => { - std::fs::copy(&path, to.join(entry.file_name())).unwrap(); - } - } - } -} - /// The go command in `module`, linking cgo against the api fixture's static library. fn go(module: &Path, api: &Api, args: &[&str]) -> Command { let mut ldflags = api.staticlib().into_os_string(); @@ -1432,6 +1547,99 @@ fn go(module: &Path, api: &Api, args: &[&str]) -> Command { cmd } +/// Exporting an unchanged crate again moves no file's mtime, the lockfile's included, so a +/// build of the export stays fresh, and drops a file it no longer writes from a directory +/// it owns. +#[test] +fn reexport() { + let package = package("reexport", &[]); + let out = package.join("out"); + let export = || { + run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(package.join("ffi.toml")) + .arg("--out") + .arg(&out) + .env("CARGO_TARGET_DIR", package.join("build"))) + }; + export(); + fn files(dir: &Path) -> Vec { + let mut found = Vec::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + match path.is_dir() { + true => found.extend(files(&path)), + false => found.push(path), + } + } + found.sort(); + found + } + let written = files(&out); + assert!(written.contains(&out.join("Cargo.lock")), "{written:?}"); + // In the past, so a rewrite would move it. + let past = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1 << 30); + for file in &written { + let file = std::fs::File::options().write(true).open(file).unwrap(); + file.set_modified(past).unwrap(); + } + let stray = out.join("go").join("stray").join("stray.go"); + step::write(&stray, "package stray\n"); + export(); + assert_eq!(files(&out), written, "the second export wrote other files"); + for file in &written { + let mtime = std::fs::metadata(file).unwrap().modified().unwrap(); + assert_eq!(mtime, past, "the second export rewrote {}", file.display()); + } + assert!(!stray.parent().unwrap().exists(), "the export kept what it no longer writes"); +} + +/// A step replays while what it reads and what it wrote are as it left them, and runs again +/// once either changes; a failure a test expects replays too. +#[test] +fn step_reruns_on_change() { + let dir = scratch().join("step"); + let _ = std::fs::remove_dir_all(&dir); + let (input, repo) = (dir.join("input"), dir.join("repo")); + step::write(&input, "one"); + let stamp = dir.join("init.stamp"); + let step = Step { + stamp: &stamp, + reads: &[&input], + writes: &[&repo], + }; + let head = repo.join(".git").join("HEAD"); + let past = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1 << 30); + // Whether the step ran since `head` was last set in the past. + let ran = |run: &dyn Fn()| { + let file = std::fs::File::options().write(true).open(&head).unwrap(); + file.set_modified(past).unwrap(); + run(); + std::fs::metadata(&head).unwrap().modified().unwrap() != past + }; + let init = || { + step.run(Command::new("git").args(["init", "--quiet"]).arg(&repo)); + }; + init(); + assert!(!ran(&init), "an unchanged step ran again"); + step::write(&input, "two"); + assert!(ran(&init), "a step whose input changed replayed"); + step::write(&repo.join("stray"), ""); + assert!(ran(&init), "a step whose output changed replayed"); + assert!(!repo.join("stray").exists(), "a step's run kept what an earlier one left"); + + let stamp = dir.join("failed.stamp"); + let failed = Step { + stamp: &stamp, + reads: &[&input], + writes: &[], + }; + let first = failed.output(Command::new("git").arg("--no-such-flag")); + assert!(!first.status.success() && stamp.exists(), "an expected failure was not kept"); + let again = failed.output(Command::new("git").arg("--no-such-flag")); + assert_eq!((again.status, again.stderr), (first.status, first.stderr)); +} + /// A relative `CARGO_TARGET_DIR` is the caller's, wherever cargo runs. #[test] fn relative_target() { @@ -1837,14 +2045,22 @@ fn feature_of_dependency() { ); } -/// A one-function package named `name` in its own workspace, written into scratch. -fn package(name: &str) -> PathBuf { +/// A one-function package named `name` in its own workspace, written into scratch, with +/// `files` in place of its own or besides: each written once, so a second run moves no +/// mtime and cargo finds the package fresh. +fn package(name: &str, files: &[(&str, &str)]) -> PathBuf { let package = scratch().join(name); - std::fs::create_dir_all(package.join("src")).unwrap(); let manifest = format!("[package]\nname = \"{name}\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n"); - std::fs::write(package.join("Cargo.toml"), manifest).unwrap(); - std::fs::write(package.join("src/lib.rs"), "pub fn one() -> u32 {\n\t1\n}\n").unwrap(); - std::fs::write(package.join("ffi.toml"), "").unwrap(); + let defaults = [ + ("Cargo.toml", manifest.as_str()), + ("src/lib.rs", "pub fn one() -> u32 {\n\t1\n}\n"), + ("ffi.toml", ""), + ]; + let defaults = defaults + .into_iter() + .filter(|(path, _)| files.iter().all(|(file, _)| file != path)); + write_all(&package, &defaults.collect::>()); + write_all(&package, files); package } @@ -1853,7 +2069,7 @@ fn package(name: &str) -> PathBuf { /// and starts with `@`, which inside an argfile rustc reads as itself, not another file. #[test] fn argfile() { - let package = package("argfile"); + let package = package("argfile", &[]); let value = format!("@{}=/", "a".repeat(1 << 21)); write_all( &package, @@ -1888,7 +2104,7 @@ fn census(config: &Path, cargo: &[&str]) -> Command { /// check then accepts, nor passes. #[test] fn locked() { - let package = package("locked"); + let package = package("locked", &[]); let _ = std::fs::remove_file(package.join("Cargo.lock")); let output = census(&package.join("ffi.toml"), &["--locked"]).output().unwrap(); assert!(!output.status.success(), "a census without a lockfile passed --locked"); @@ -1900,7 +2116,7 @@ fn locked() { #[cfg(unix)] #[test] fn symlinked() { - let package = package("symlinked"); + let package = package("symlinked", &[]); let link = scratch().join("symlinked-link"); if !link.exists() { std::os::unix::fs::symlink(&package, &link).unwrap(); @@ -1918,11 +2134,25 @@ fn symlinked() { /// their sources remapped. #[test] fn lookalike() { - let package = package("lookalike"); + let package = package( + "lookalike", + &[ + ( + "Cargo.toml", + "[package]\nname = \"lookalike\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[lib]\nname = \"shared\"\n\n\ + [dependencies]\nshared = { package = \"lookalike-exp\", path = \"exp\" }\n\ + bytes = { package = \"lookalike-other\", path = \"other\" }\n\n\ + [workspace]\nexclude = [\"exp\", \"other\"]\n", + ), + ( + "src/lib.rs", + "pub mod exp {\n\tpub use shared::Size;\n}\n\npub mod other {\n\tpub use bytes::Size;\n}\n\n\ + pub fn total(a: shared::Size, b: bytes::Size) -> u32 {\n\ta.width + b.width\n}\n", + ), + ], + ); let write = |path: &str, text: &str| { - let path = package.join(path); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(path, text).unwrap(); + step::write(&package.join(path), text); }; // Both libraries are crate `shared`. for name in ["exp", "other"] { @@ -1937,18 +2167,6 @@ fn lookalike() { "pub struct Size {\n\tpub width: u32,\n}\n\nimpl Size {\n\tpub fn new(width: u32) -> Self {\n\t\tSelf { width }\n\t}\n}\n", ); } - write( - "Cargo.toml", - "[package]\nname = \"lookalike\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[lib]\nname = \"shared\"\n\n\ - [dependencies]\nshared = { package = \"lookalike-exp\", path = \"exp\" }\n\ - bytes = { package = \"lookalike-other\", path = \"other\" }\n\n\ - [workspace]\nexclude = [\"exp\", \"other\"]\n", - ); - write( - "src/lib.rs", - "pub mod exp {\n\tpub use shared::Size;\n}\n\npub mod other {\n\tpub use bytes::Size;\n}\n\n\ - pub fn total(a: shared::Size, b: bytes::Size) -> u32 {\n\ta.width + b.width\n}\n", - ); let out = package.join("out"); let remap = format!("--remap-path-prefix={}=/src", package.display()); let output = Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) @@ -1978,7 +2196,7 @@ fn lookalike() { #[test] fn standard_names() { for name in ["core", "alloc", "std"] { - let package = package(&format!("standard_{name}")); + let package = package(&format!("standard_{name}"), &[]); let manifest = format!( "[package]\nname = \"standard_{name}\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[lib]\nname = \"{name}\"\n\n[workspace]\n" ); @@ -2011,9 +2229,8 @@ fn standard_names() { /// through a dependency of its own, apart from the glue's dependency on the facade. #[test] fn foreign_lib_name() { - let package = package("foreign_lib_name"); - write_all( - &package, + let package = package( + "foreign_lib_name", &[ ( "Cargo.toml", @@ -2033,9 +2250,8 @@ fn foreign_lib_name() { /// would spell them: `serde_json::Value` is `Value`, apart from `Json`. #[test] fn foreign_role_names() { - let package = package("foreign_role_names"); - write_all( - &package, + let package = package( + "foreign_role_names", &[ ( "Cargo.toml", @@ -2059,7 +2275,7 @@ fn foreign_role_names() { /// leaves out. #[test] fn configured_out() { - let package = package("configured_out"); + let package = package("configured_out", &[]); let dep = manifest_dir().join("fixtures/dep"); let manifest = format!( "[package]\nname = \"configured_out\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n\ @@ -2109,9 +2325,8 @@ fn configured_out() { /// The glue declares the facade's `rust-version`, raised to the 1.85 its edition needs. #[test] fn rust_version() { - let package = package("rust_version"); - write_all( - &package, + let package = package( + "rust_version", &[( "Cargo.toml", "[package]\nname = \"rust_version\"\nversion = \"0.0.0\"\nedition = \"2021\"\nrust-version = \"1.70\"\n\n[workspace]\n", @@ -2132,7 +2347,7 @@ fn rust_version() { /// cargo's resolution holds but neither the check nor the glue builds, moves nothing. #[test] fn dev_feature() { - let package = package("dev_feature"); + let package = package("dev_feature", &[]); let dep = |name: &str, rest: &str| { format!("[package]\nname = \"{name}\"\nversion = \"1.0.0\"\nedition = \"2024\"\n{rest}") }; @@ -2166,9 +2381,8 @@ fn dev_feature() { /// the nightly the export checks on: a stable rustc exports the stable item alone. #[test] fn channel() { - let package = package("channel"); - write_all( - &package, + let package = package( + "channel", &[ ( "build.rs", @@ -2242,7 +2456,6 @@ fn stable_rustc() -> PathBuf { /// 32-bit `usize`, which Kotlin's 64-bit `ULong` crosses into checked. #[test] fn target() { - let package = package("targeted"); let shared = "pub fn halve(n: usize) -> usize {\n\tn / 2\n}\n\n\ #[cfg(target_arch = \"wasm32\")]\npub fn wasm() -> u32 {\n\t32\n}\n\n\ #[cfg(not(target_arch = \"wasm32\"))]\npub fn desktop() -> u32 {\n\t64\n}\n"; @@ -2250,7 +2463,8 @@ fn target() { let differing = "pub const MAX: usize = usize::MAX;\n\n\ #[cfg(target_arch = \"wasm32\")]\npub fn width() -> u32 {\n\t32\n}\n\n\ #[cfg(not(target_arch = \"wasm32\"))]\npub fn width() -> u64 {\n\t64\n}\n"; - write_all(&package, &[("src/lib.rs", &format!("{shared}\n{differing}"))]); + // Each run starts from these, and drops `differing` partway, which rebuilds the glue. + let package = package("targeted", &[("src/lib.rs", &format!("{shared}\n{differing}"))]); let exporter = |args: &[&str]| { let mut cmd = Command::new(env!("CARGO_BIN_EXE_rust2-ffi")); cmd.arg("--config") @@ -2271,7 +2485,11 @@ fn target() { "{header}" ); assert!(header.contains(&u64::MAX.to_string()), "{header}"); - build_offline(&host_out, &package.join("glue")); + // Each export builds in a target directory of its own: cargo names a package's units + // by its path in its workspace, which each export's is the root of, so the exports' + // units would share one, and reuse another's build script where the mtimes allow. + let target = |out: &Path| package.join("glue").join(out.file_name().unwrap()); + build_offline(&host_out, &target(&host_out)); let (wasm, header) = export("wasm", &["--target", WASM]); assert!( header.contains("targeted_wasm(") && !header.contains("targeted_desktop("), @@ -2281,7 +2499,7 @@ fn target() { header.contains(&u32::MAX.to_string()) && !header.contains(&u64::MAX.to_string()), "{header}" ); - let glue = |out: &Path, target: &[&str]| { + let glue = |out: &Path, args: &[&str]| { let mut cmd = cargo(); cmd.args([ "check", @@ -2293,8 +2511,8 @@ fn target() { ]) .arg(out.join("Cargo.toml")) .arg("--target-dir") - .arg(package.join("glue")) - .args(target) + .arg(target(out)) + .args(args) .env("RUSTFLAGS", "-D warnings"); cmd }; @@ -2379,12 +2597,11 @@ fn target() { } } -/// Writes each `(path, text)` into `dir`. +/// Writes each `(path, text)` into `dir`, leaving a file that already holds its text alone, +/// so cargo finds a package written again fresh. fn write_all(dir: &Path, files: &[(&str, &str)]) { for (path, text) in files { - let path = dir.join(path); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(path, text).unwrap(); + step::write(&dir.join(path), text); } } @@ -2429,7 +2646,7 @@ fn build_offline(out: &Path, target: &Path) { /// warnings in every language's module too, writing no import it leaves unused. #[test] fn plain() { - let package = package("plain"); + let package = package("plain", &[]); export_and_build(&package, &[]); run(cargo() .args([ @@ -2451,9 +2668,8 @@ fn plain() { /// name. So is a dependency of no platform's, which cargo resolves all the same. #[test] fn patched() { - let package = package("patched"); - write_all( - &package, + let package = package( + "patched", &[ ( "dep/Cargo.toml", @@ -2496,6 +2712,13 @@ const COMMIT: [&str; 8] = [ "commit", ]; +/// A commit's dates, fixed so a repository made again from the same files has the same +/// commits, whose checkout cargo then reuses. +const GIT_DATES: [(&str, &str); 2] = [ + ("GIT_AUTHOR_DATE", "2000-01-01T00:00:00Z"), + ("GIT_COMMITTER_DATE", "2000-01-01T00:00:00Z"), +]; + /// The URL of a new git repository in `dir`, whose branch `main` holds `files`. fn repository(dir: &Path, files: &[(&str, &str)]) -> String { let _ = std::fs::remove_dir_all(dir); @@ -2505,7 +2728,7 @@ fn repository(dir: &Path, files: &[(&str, &str)]) -> String { &["add", "."], &COMMIT, ] { - run(Command::new("git").args(args).current_dir(dir)); + run(Command::new("git").args(args).envs(GIT_DATES).current_dir(dir)); } format!("file://{}", dir.display()) } @@ -2514,15 +2737,15 @@ fn repository(dir: &Path, files: &[(&str, &str)]) -> String { /// picking one by version, is carried with the version it resolved to. #[test] fn git_patched() { - let package = package("git_patched"); + let dir = scratch().join("git_patched"); let manifest = |version| format!("[package]\nname = \"rust2-ffi-git-dep\"\nversion = \"{version}\"\nedition = \"2024\"\n"); let stale = repository( - &package.join("stale"), + &dir.join("stale"), &[("Cargo.toml", &manifest("1.0.0")), ("src/lib.rs", "")], ); let fresh = repository( - &package.join("fresh"), + &dir.join("fresh"), &[ ("one/Cargo.toml", &manifest("1.0.0")), ("one/src/lib.rs", ""), @@ -2536,7 +2759,7 @@ fn git_patched() { [patch.\"{stale}\"]\nrust2-ffi-git-dep = {{ git = \"{fresh}\", version = \"1\" }}\n\n\ [workspace]\nexclude = [\"stale\", \"fresh\"]\n" ); - write_all(&package, &[("Cargo.toml", &text)]); + let package = package("git_patched", &[("Cargo.toml", &text)]); export_and_build(&package, &[]); } @@ -2546,7 +2769,7 @@ fn git_patched() { /// replacement from the source it replaces is an error, since no patch can carry it. #[test] fn replaced() { - let package = package("replaced"); + let package = scratch().join("replaced"); // A git repository the dependency names, whose default branch lacks the field the // package reads, named for the package so the spec names it by its URL. let manifest = "[package]\nname = \"rust2-ffi-replaced-dep\"\nversion = \"1.0.0\"\nedition = \"2024\"\n"; @@ -2562,7 +2785,7 @@ fn replaced() { .args(["checkout", "--quiet", "-b", "fixed"]) .current_dir(&git)); write_all(&git, &[("src/lib.rs", "pub struct Size {\n\tpub width: u32,\n}\n")]); - run(Command::new("git").args(COMMIT).current_dir(&git)); + run(Command::new("git").args(COMMIT).envs(GIT_DATES).current_dir(&git)); let files = |replacement: &str| { write_all( &package, @@ -2608,9 +2831,7 @@ fn replaced() { fn config_patched() { let root = scratch().join("config-patched"); let write = |path: &str, text: &str| { - let path = root.join(path); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(path, text).unwrap(); + step::write(&root.join(path), text); }; let dep = |dir: &str, name: &str, item: &str| { write( diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index a04c48eed..704aa7126 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -10,6 +10,7 @@ use std::fmt::Write; use std::path::{Path, PathBuf}; use std::process::Command; +use super::step::{self, Step}; use super::{cargo, manifest_dir, run, scratch}; /// The integer types of the IR's `Int`. @@ -563,7 +564,6 @@ struct Crate { /// exports, and run its script natively. fn emit(name: &str, shapes: &[Shape]) -> Crate { let dir = scratch().join(name); - let _ = std::fs::remove_dir_all(dir.join("src")); let mut emit = Emit { items: String::new(), defined: 0, @@ -599,7 +599,7 @@ fn emit(name: &str, shapes: &[Shape]) -> Crate { &[("Cargo.toml", &manifest), ("src/lib.rs", &lib), ("ffi.toml", "")], ); // The api fixture's versions, resolved offline. - std::fs::copy(manifest_dir().join("Cargo.lock"), dir.join("Cargo.lock")).unwrap(); + super::step::copy(&manifest_dir().join("Cargo.lock"), &dir.join("Cargo.lock")); let refused = refused(&dir); let exported = |item: Item, n: usize| !refused.iter().any(|path| *path == item.name(n)); @@ -609,7 +609,7 @@ fn emit(name: &str, shapes: &[Shape]) -> Crate { .iter() .map(|path| format!(" \"{path}\",\n")) .collect::(); - std::fs::write(dir.join("ffi.toml"), format!("exclude = [\n{config}]\n")).unwrap(); + super::step::write(&dir.join("ffi.toml"), format!("exclude = [\n{config}]\n")); let mut calls = Vec::new(); for n in 0..shapes.len() { @@ -648,9 +648,10 @@ fn emit(name: &str, shapes: &[Shape]) -> Crate { } script.push_str("}\n"); super::write_all(&dir, &[("examples/transcript.rs", &script)]); + // Apart from the exporter's build, which cleans the crate each time. let transcript = run(cargo() .args(["run", "--quiet", "--example", "transcript"]) - .env("CARGO_TARGET_DIR", dir.join("build")) + .env("CARGO_TARGET_DIR", dir.join("native")) .current_dir(&dir)); Crate { dir, @@ -743,7 +744,6 @@ fn shared(name: &str) -> String { fn build(name: &str, shapes: &[Shape]) -> Built { let krate = emit(name, shapes); let out = krate.dir.join("out"); - let _ = std::fs::remove_dir_all(&out); run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) .arg("--config") .arg(krate.dir.join("ffi.toml")) @@ -751,20 +751,22 @@ fn build(name: &str, shapes: &[Shape]) -> Built { .arg(&out) .env("CARGO_TARGET_DIR", krate.dir.join("build"))); let target = krate.dir.join("glue"); - let build = |feature: &str| { + let build = |target: &Path, feature: &str| { run(cargo() .args(["build", "--quiet", "--locked", "--manifest-path"]) .arg(out.join("Cargo.toml")) .arg("--target-dir") - .arg(&target) + .arg(target) .args(["--features", feature]) .env("RUSTFLAGS", "-D warnings --print=native-static-libs")) }; - // As the api fixture's: the CPython extension builds apart and moves beside the Python. - build("python"); + // As the api fixture's: the CPython extension builds apart, in a target directory of its + // own, and is copied beside the Python. + let python = krate.dir.join("glue-python"); + build(&python, "python"); let lib = shared(name); - std::fs::rename(target.join("debug").join(&lib), out.join("python").join(&lib)).unwrap(); - let built = build("kotlin,dart"); + super::step::copy(&python.join("debug").join(&lib), &out.join("python").join(&lib)); + let built = build(&target, "kotlin,dart"); let native = String::from_utf8_lossy(&built.stderr) .lines() .find_map(|line| line.strip_prefix("note: native-static-libs: ")) @@ -795,10 +797,7 @@ impl Built { /// A directory of its own for `language`'s driver. fn dir(&self, language: &str) -> PathBuf { - let dir = self.krate.dir.join("drivers").join(language); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - dir + self.krate.dir.join("drivers").join(language) } /// The exported shapes `language` gets wrong today, by [`KNOWN`], each entry of which must @@ -1433,41 +1432,58 @@ fn round_trip(built: &Built, language: &str) { let output = match language { "c" => { let main = dir.join("main.c"); - std::fs::write(&main, c(built, calls)).unwrap(); + step::write(&main, c(built, calls)); let cc = std::env::var_os("CC").unwrap_or_else(|| "cc".into()); - run(Command::new(cc) - .args(["-std=c23", "-pedantic", "-Wall", "-Wextra", "-Werror", "-I"]) - .arg(built.out.join("include")) - .arg(&main) - .arg(built.staticlib()) - .args(&built.native) - .arg("-o") - .arg(&exe)); + let include = built.out.join("include"); + let step = Step { + stamp: &step::stamp(&exe), + reads: &[&main, &include, &built.staticlib()], + writes: &[&exe], + }; + step.run( + Command::new(cc) + .args(["-std=c23", "-pedantic", "-Wall", "-Wextra", "-Werror", "-I"]) + .arg(&include) + .arg(&main) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe), + ); run(&mut Command::new(&exe)) } "cpp" => { let main = dir.join("main.cpp"); - std::fs::write(&main, cpp(built, calls)).unwrap(); + step::write(&main, cpp(built, calls)); // The api fixture's build, so MSVC's runner builds it with cl. let link = super::Link { include: built.out.join("include"), staticlib: built.staticlib(), native: &built.native, }; - run(&mut super::cxx(&link, 17, false, &main, &exe)); + super::cxx(&link, 17, false, &main, &exe); run(&mut Command::new(&exe)) } "kotlin" => { let main = dir.join("main.kt"); - std::fs::write(&main, kotlin(built, calls)).unwrap(); + step::write(&main, kotlin(built, calls)); let classes = dir.join("classes"); - run(Command::new("kotlinc") - .args(["-Werror", "-cp"]) - .arg(super::coroutines()) - .arg(built.out.join("kotlin").join(format!("{name}.kt"))) - .arg(&main) - .arg("-d") - .arg(&classes)); + let generated = built.out.join("kotlin").join(format!("{name}.kt")); + let jar = PathBuf::from(super::coroutines()); + let step = Step { + stamp: &step::stamp(&classes), + reads: &[&main, &generated, &jar], + writes: &[&classes], + }; + step.run( + Command::new("kotlinc") + .args(["-Werror", "-cp"]) + .arg(&jar) + .arg(&generated) + .arg(&main) + .arg("-d") + .arg(&classes), + ); let mut classpath = classes.into_os_string(); classpath.push(if cfg!(windows) { ";" } else { ":" }); classpath.push(super::coroutines()); @@ -1479,7 +1495,7 @@ fn round_trip(built: &Built, language: &str) { } "python" => { let main = dir.join("main.py"); - std::fs::write(&main, python(built, calls)).unwrap(); + step::write(&main, python(built, calls)); run( Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())) .args(["-P", "-X", "dev", "-W", "error"]) @@ -1489,55 +1505,78 @@ fn round_trip(built: &Built, language: &str) { } "swift" => { let module = built.out.join("swift"); - std::fs::copy(module.join(format!("{name}.swift")), dir.join(format!("{name}.swift"))).unwrap(); - std::fs::write(dir.join("main.swift"), swift(calls)).unwrap(); - run(super::swiftc(&module, 15) - .args(["-swift-version", "6", "-module-name", "Shapes"]) - .arg(dir.join(format!("{name}.swift"))) - .arg(dir.join("main.swift")) - .args(super::swift_collections()) - .arg(built.staticlib()) - .args(&built.native) - .arg("-o") - .arg(&exe)); + let (generated, main) = (dir.join(format!("{name}.swift")), dir.join("main.swift")); + step::copy(&module.join(format!("{name}.swift")), &generated); + step::write(&main, swift(calls)); + let collections = super::swift_collections(); + let include = built.out.join("include"); + let step = Step { + stamp: &step::stamp(&exe), + reads: &[ + &generated, + &main, + &module, + &include, + &super::collections_dir(), + &built.staticlib(), + ], + writes: &[&exe], + }; + step.run( + super::swiftc(&module, 15) + .args(["-swift-version", "6", "-module-name", "Shapes"]) + .arg(&generated) + .arg(&main) + .args(collections) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe), + ); run(&mut Command::new(&exe)) } "go" => { - let export = dir.join("export"); - super::copy_dir(&built.out.join("go"), &export); - std::fs::create_dir_all(dir.join("include")).unwrap(); + // The module apart from the program, which its build would otherwise read. + let module = dir.join("module"); + let export = module.join("export"); + step::mirror(&built.out.join("go"), &export); let header = format!("{name}.h"); - std::fs::copy( - built.out.join("include").join(&header), - dir.join("include").join(&header), - ) - .unwrap(); + step::copy( + &built.out.join("include").join(&header), + &module.join("include").join(&header), + ); let source = std::fs::read_to_string(export.join(&name).join(format!("{name}.go"))).unwrap(); let rt = std::fs::read_to_string(export.join("rt").join("types.go")).unwrap_or_default(); - std::fs::write(dir.join("main.go"), go(built, calls, &source, &rt)).unwrap(); - std::fs::write( - dir.join("go.mod"), + step::write(&module.join("main.go"), go(built, calls, &source, &rt)); + step::write( + &module.join("go.mod"), format!("module shapestest\n\ngo 1.24\n\nrequire {name} v0.0.0\n\nreplace {name} => ./export\n"), - ) - .unwrap(); + ); let mut ldflags = built.staticlib().into_os_string(); for lib in &built.native { ldflags.push(" "); ldflags.push(lib); } - run(Command::new("go") - .args(["build", "-o"]) - .arg(&exe) - .current_dir(&dir) - .env("CGO_ENABLED", "1") - .env("CGO_LDFLAGS", ldflags) - .env("GOFLAGS", "-buildvcs=false") - .env("GOTOOLCHAIN", "local")); + let step = Step { + stamp: &step::stamp(&exe), + reads: &[&module, &built.staticlib()], + writes: &[&exe], + }; + step.run( + Command::new("go") + .args(["build", "-o"]) + .arg(&exe) + .current_dir(&module) + .env("CGO_ENABLED", "1") + .env("CGO_LDFLAGS", ldflags) + .env("GOFLAGS", "-buildvcs=false") + .env("GOTOOLCHAIN", "local"), + ); run(&mut Command::new(&exe)) } "dart" => { let main = dir.join("main.dart"); - std::fs::write(&main, dart(built, calls)).unwrap(); + step::write(&main, dart(built, calls)); let lib = built.lib.join(shared(&name)); let dill = super::dart_compile(&name, &dir, &built.out.join("dart"), &lib, &main); run(&mut super::dart_run(&dill, &[])) diff --git a/ffi/tests/fixtures/step.rs b/ffi/tests/fixtures/step.rs new file mode 100644 index 000000000..9dbeee356 --- /dev/null +++ b/ffi/tests/fixtures/step.rs @@ -0,0 +1,358 @@ +//! Build steps the tests run, each replayed from a stamp while what it reads is unchanged, +//! and file writes that leave a file alone when it already holds what they would write. +//! +//! The target directory's `tmp`, where the steps write, persists between runs, so a second +//! run of the tests compiles and links nothing it did not change. Keys are on content, not +//! mtimes: the export rewrites nothing unchanged, but a copy or a checkout still moves them. + +use std::collections::HashMap; +use std::ffi::OsString; +use std::hash::{DefaultHasher, Hasher}; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus, Output}; +use std::sync::{Arc, Mutex, OnceLock}; + +/// A build step: its stamp holds the key it last ran with, what it printed, and a digest of +/// what it wrote, so a run with the same key replays that output instead of running it. +pub struct Step<'a> { + /// Where the key and the output are kept between runs. + pub stamp: &'a Path, + /// The files and directories the step reads besides its program, by content. + pub reads: &'a [&'a Path], + /// The files and directories it writes, which a replay needs as it left them; each is + /// removed before the step runs, so nothing an earlier run wrote outlives it. + pub writes: &'a [&'a Path], +} + +impl Step<'_> { + /// The output of `cmd`, which must succeed. + pub fn run(&self, cmd: &mut Command) -> Output { + let output = self.execute(cmd, false); + super::check(cmd, &output); + output + } + + /// The output of `cmd`, whether it succeeds or fails, as a compile a test expects to + /// fail does. + pub fn output(&self, cmd: &mut Command) -> Output { + self.execute(cmd, true) + } + + fn execute(&self, cmd: &mut Command, failure: bool) -> Output { + let key = key(cmd, self.reads); + if let Some(output) = self.replay(&key) { + return output; + } + // No stamp may outlive a run that stops partway. + remove(self.stamp); + for path in self.writes { + remove(path); + } + let output = super::output(cmd); + // A failed build is kept only where the test expects one: any other is run again, + // and one a signal ended may not be what its inputs make. + if (output.status.success() || failure) && output.status.code().is_some() { + self.record(&key, &output); + } + output + } + + fn replay(&self, key: &str) -> Option { + let stamp = std::fs::read(self.stamp).ok()?; + let mut fields = Fields(&stamp); + if fields.next()? != key.as_bytes() || fields.next()? != digest(self.writes).to_le_bytes() { + return None; + } + let status = status(i64::from_le_bytes(fields.next()?.try_into().ok()?)); + let stdout = fields.next()?.to_vec(); + let stderr = fields.next()?.to_vec(); + Some(Output { status, stdout, stderr }) + } + + fn record(&self, key: &str, output: &Output) { + let code = raw(output.status); + let mut stamp = Vec::new(); + for field in [ + key.as_bytes(), + &digest(self.writes).to_le_bytes(), + &code.to_le_bytes(), + &output.stdout, + &output.stderr, + ] { + stamp.extend_from_slice(&(field.len() as u64).to_le_bytes()); + stamp.extend_from_slice(field); + } + // Whole or not at all: a stamp cut short would fail to parse, but say so plainly. + let partial = self.stamp.with_extension("partial"); + std::fs::create_dir_all(self.stamp.parent().unwrap()).unwrap(); + std::fs::write(&partial, stamp).unwrap(); + std::fs::rename(&partial, self.stamp).unwrap(); + } +} + +/// The length-prefixed fields of a stamp. +struct Fields<'a>(&'a [u8]); + +impl<'a> Iterator for Fields<'a> { + type Item = &'a [u8]; + + fn next(&mut self) -> Option<&'a [u8]> { + let (len, rest) = self.0.split_first_chunk::<8>()?; + let len = usize::try_from(u64::from_le_bytes(*len)).ok()?; + let field = rest.get(..len)?; + self.0 = &rest[len..]; + Some(field) + } +} + +#[cfg(unix)] +fn raw(status: ExitStatus) -> i64 { + use std::os::unix::process::ExitStatusExt; + status.into_raw().into() +} + +#[cfg(unix)] +fn status(raw: i64) -> ExitStatus { + use std::os::unix::process::ExitStatusExt; + ExitStatus::from_raw(raw as i32) +} + +#[cfg(windows)] +fn raw(status: ExitStatus) -> i64 { + status.code().unwrap().into() +} + +#[cfg(windows)] +fn status(raw: i64) -> ExitStatus { + use std::os::windows::process::ExitStatusExt; + ExitStatus::from_raw(raw as u32) +} + +/// What a step's output depends on besides its own files: its program, by path and version, +/// its arguments, directory, and environment, and the content of each file it `reads`. +fn key(cmd: &Command, reads: &[&Path]) -> String { + let program = program(cmd); + let mut key = format!("program {program:?}\nversion {:?}\n", version(&program)); + for arg in cmd.get_args() { + key += &format!("arg {arg:?}\n"); + } + key += &format!("dir {:?}\n", cmd.get_current_dir()); + let set: Vec<_> = cmd.get_envs().collect(); + for (name, value) in inherited() { + if !set.iter().any(|(set, _)| set == name) { + key += &format!("env {name:?}={value:?}\n"); + } + } + for (name, value) in set { + key += &format!("env {name:?}={value:?}\n"); + } + for path in reads { + key += &format!("read {path:?} {:016x}\n", digest(&[path])); + } + key +} + +/// The program `cmd` runs, resolved through `PATH` as the command would and through links, +/// so another toolchain on `PATH`, as another dev shell puts there, is another key. +fn program(cmd: &Command) -> PathBuf { + let program = Path::new(cmd.get_program()); + let path = match program.components().count() { + 1 => super::which(&program.to_string_lossy()), + _ => program.to_path_buf(), + }; + std::fs::canonicalize(&path).unwrap_or_else(|err| panic!("resolving {}: {err}", path.display())) +} + +/// What `program` says its version is, asked once each run: a toolchain updated in place, +/// or picked by `xcode-select`, keeps its path. +fn version(program: &Path) -> String { + static VERSIONS: OnceLock>>>> = OnceLock::new(); + let once = VERSIONS + .get_or_init(Default::default) + .lock() + .unwrap() + .entry(program.to_path_buf()) + .or_default() + .clone(); + once.get_or_init(|| { + let flag: &[&str] = match program.file_stem().and_then(|stem| stem.to_str()) { + Some("go") => &["version"], + Some("kotlinc") => &["-version"], + // MSVC's compiler prints its banner bare. + Some("cl") => &[], + _ => &["--version"], + }; + let output = Command::new(program) + .args(flag) + .output() + .unwrap_or_else(|err| panic!("running {}: {err}", program.display())); + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }) + .clone() +} + +/// The inherited variables a compiler, linker, or JVM reads, by prefix, such as the Nix C +/// wrapper's flags, the SDK an Apple or MSVC toolchain finds, and Go's; others, such as a +/// shell's temporary directory, would make each run's key new. So would a directory on +/// `PATH` in that temporary directory, such as a Cargo shim's session, which is left out. +fn inherited() -> &'static [(OsString, OsString)] { + const PREFIXES: &[&str] = &[ + "PATH", + "NIX_", + "CC", + "CXX", + "CFLAGS", + "CPPFLAGS", + "CXXFLAGS", + "LD", + "AR", + "CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "LIBRARY_PATH", + "INCLUDE", + "LIB", + "SDKROOT", + "DEVELOPER_DIR", + "MACOSX_", + "JAVA", + "KOTLIN", + "GO", + "CGO_", + "DART", + "SWIFT", + "VC", + "WINDOWSSDK", + "UCRT", + ]; + // A Nix shell's own build directory, which each shell makes anew. + const VOLATILE: &[&str] = &["NIX_BUILD_TOP"]; + static ENV: OnceLock> = OnceLock::new(); + ENV.get_or_init(|| { + let mut env: Vec<(OsString, OsString)> = std::env::vars_os() + .filter(|(name, _)| { + let upper = name.to_string_lossy().to_uppercase(); + PREFIXES.iter().any(|prefix| upper.starts_with(prefix)) && !VOLATILE.contains(&upper.as_str()) + }) + .map( + |(name, value)| match name.to_string_lossy().eq_ignore_ascii_case("PATH") { + true => (name, path()), + false => (name, value), + }, + ) + .collect(); + env.sort(); + env + }) +} + +/// `PATH` without the directories in the temporary one, where a shell, or a Cargo shim's +/// session, puts what it makes anew each run. +pub fn path() -> OsString { + let temp = std::env::temp_dir(); + let path = std::env::var_os("PATH").unwrap_or_default(); + let kept = std::env::split_paths(&path).filter(|dir| !dir.starts_with(&temp)); + std::env::join_paths(kept).unwrap() +} + +/// A digest of the content of `paths`, each a file or a directory of them, by relative +/// path; a missing one digests as missing. +fn digest(paths: &[&Path]) -> u64 { + let mut hasher = DefaultHasher::new(); + for path in paths { + walk(path, Path::new(""), &mut hasher); + } + hasher.finish() +} + +fn walk(path: &Path, rel: &Path, hasher: &mut DefaultHasher) { + hasher.write(rel.as_os_str().as_encoded_bytes()); + hasher.write_u8(0); + match std::fs::metadata(path) { + Err(err) if err.kind() == std::io::ErrorKind::NotFound => hasher.write_u8(0), + Err(err) => panic!("reading {}: {err}", path.display()), + Ok(meta) if meta.is_dir() => { + hasher.write_u8(1); + let mut entries: Vec<_> = std::fs::read_dir(path) + .unwrap_or_else(|err| panic!("reading {}: {err}", path.display())) + .map(|entry| entry.unwrap().file_name()) + .collect(); + entries.sort(); + for name in entries { + walk(&path.join(&name), &rel.join(&name), hasher); + } + } + Ok(_) => { + hasher.write_u8(2); + let bytes = std::fs::read(path).unwrap_or_else(|err| panic!("reading {}: {err}", path.display())); + hasher.write_usize(bytes.len()); + hasher.write(&bytes); + } + } +} + +fn remove(path: &Path) { + let removed = match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_dir() => std::fs::remove_dir_all(path), + Ok(_) => std::fs::remove_file(path), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return, + Err(err) => Err(err), + }; + removed.unwrap_or_else(|err| panic!("removing {}: {err}", path.display())); +} + +/// Write `text` to `path` unless it already holds it, creating its directory. +pub fn write(path: &Path, text: impl AsRef<[u8]>) { + let text = text.as_ref(); + if std::fs::read(path).is_ok_and(|old| old == text) { + return; + } + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap_or_else(|err| panic!("writing {}: {err}", path.display())); +} + +/// Copy the file `from` to `to` unless `to` already holds what it holds. +pub fn copy(from: &Path, to: &Path) { + let bytes = std::fs::read(from).unwrap_or_else(|err| panic!("reading {}: {err}", from.display())); + write(to, bytes); +} + +/// Make the directory `to` a copy of `from`, rewriting only what differs and removing what +/// `from` lacks. +pub fn mirror(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + let mut names = Vec::new(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + if target.is_file() { + remove(&target); + } + mirror(&entry.path(), &target); + } else { + if target.is_dir() { + remove(&target); + } + copy(&entry.path(), &target); + } + names.push(entry.file_name()); + } + for entry in std::fs::read_dir(to).unwrap() { + let entry = entry.unwrap(); + if !names.contains(&entry.file_name()) { + remove(&entry.path()); + } + } +} + +/// The stamp of a step that writes `output`: beside it, named after all of it. +pub fn stamp(output: &Path) -> PathBuf { + let mut name = output.as_os_str().to_owned(); + name.push(".stamp"); + name.into() +} From 5aa887043056934ba0970ab676239ff9874a0a37 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:40:25 -0800 Subject: [PATCH 03/11] quest: finish ffi-test-builds, add dart-exit Co-Authored-By: Claude Opus 5.5 --- quest/m0/dart-exit.md | 23 +++++++++++ quest/m0/ffi-test-builds.md | 77 ------------------------------------- 2 files changed, 23 insertions(+), 77 deletions(-) create mode 100644 quest/m0/dart-exit.md delete mode 100644 quest/m0/ffi-test-builds.md diff --git a/quest/m0/dart-exit.md b/quest/m0/dart-exit.md new file mode 100644 index 000000000..5beea4f3b --- /dev/null +++ b/quest/m0/dart-exit.md @@ -0,0 +1,23 @@ +# [S] The Dart api script exits when its transcript ends + +## Goal + +`tests/api.dart` run with no case, as `api_dart` runs it, exits once its `main` returns. +Today it prints its whole transcript in about 0.5 s, then the Dart VM stays alive about 61 s +more before it exits 0: alone on an idle host, and every run. That wait is the critical path +of a warm `just ffi test`, about 61 of its 70 s, and every agent's run pays it. Each other +case (`futures`, `owned`, `closed`, `bytes`, `bounds`) exits in about 0.4 s. + +## Plan + +Found on 2026-11-13 by measuring the warm run after +[FFI test programs build once per input change](https://github.com/kixel-ai/rust2/pull/780). + +The lead: the generated Dart's port keeps the isolate alive while any wake token is held +(`_hold` and `listen` in the runtime `dart.rs` writes). Something the transcript path does +leaves a token held until a Rust-side wake about a minute later, such as a future dropped or +cancelled before it finished whose token is never forgotten. Recommended: bisect the +script's sections to the one that holds it, then fix the root cause. If the generated +binding leaks the hold, every consumer's isolate outlives its work the same way. That fix +is in `dart.rs`, with an `api_dart` case that would hang without it. If the hold is the +test's own, fix the test. diff --git a/quest/m0/ffi-test-builds.md b/quest/m0/ffi-test-builds.md deleted file mode 100644 index 60d66b559..000000000 --- a/quest/m0/ffi-test-builds.md +++ /dev/null @@ -1,77 +0,0 @@ -# [M] FFI test programs build once per input change - -## Goal - -A second `just test` of `ffi` with nothing changed exports, builds, compiles, and links -nothing new for [`ffi/tests/fixtures.rs`](/ffi/tests/fixtures.rs) and its -[shapes](/ffi/tests/fixtures/shapes.rs): the glue crate's libraries and each language's -test program (C, C++, Kotlin, Python's extension, Swift, Go, Dart) are reused, and still run -and assert as before. A change to anything a step reads, such as a test source, the -exporter, the generated binding, the static library, or the compiler, rebuilds that step -and only the steps downstream of it. The PR reports the bytes written and the wall time -of a warm second run before and after. - -## Plan - -Found by the 2026-11-13 write audit of the `nixos` host -([kixel-ai/dotfiles#135](https://github.com/kixel-ai/dotfiles/pull/135)): a 15-minute trace -charged 2.8 GB of writes to `target/tmp/api`, across agents' `just test`, rust2-ci, and a -rust2 land fixer. Agents write about 250 GB a day on that host, and this is the part a -warm run should not write at all. No single PR caused it: the fixtures have rebuilt -everything since they were added. - -Decided with the maintainer on 2026-11-13: ✅ add this quest, under the host's write -cuts; milestone ✅ m0 (rejected: m1), since every agent's `just test` pays these writes and -the wait. - -What is already true, from reading the code on 2026-11-13: - -- `CARGO_TARGET_TMPDIR` is Cargo's persistent `target/tmp`, not a fresh directory, so the - outputs survive between runs; the steps just never check them. -- Every export rewrites every generated file (`ffi/src/driver.rs`'s `write`) and deletes - the C++ and Go directories first (`ffi/src/glue.rs`), so each file's mtime moves even - when its content doesn't. Cargo fingerprints path sources by mtime, so the glue crate - likely recompiles and relinks both feature builds each run. `api()` also moves the - Python extension out of Cargo's `target/debug`. -- Each language's compile and link (`cc`, `c++` per standard, `kotlinc`, `swiftc`, `go - build`, `dart compile`) runs unconditionally. - -Measure first: trace one warm second run, with bytes written and wall time per step: the -export, each glue build, and each language's compile. Fix what dominates. The -recommendations below are a starting point, not settled. - -- Recommended: the exporter writes a file only when its content differs, and removes only - the files an export no longer produces rather than whole directories. Then Cargo's own - fingerprint skips the glue build, and a consumer that re-exports before each build - gains the same. That changes rust2-ffi's behavior, not its API. -- Recommended: each test program's step is skipped when a hash of what it reads matches - a stamp beside its output: its sources, the generated files it includes, the library it - links, the compiler's path and version, and its arguments. Keep the outputs in - `target/tmp`, which already persists, and is cleared with the rest of `target`. Rejected - for now: each language's own incremental build (Gradle, SwiftPM, Go's build cache - alone), since that adds a build system per language for what one stamp covers; Go's - cache may already make its compile cheap, which the measurement shows. -- An mtime check is not enough on its own: the exporter's rewrite moves mtimes, and a - stale program that passes would hide a real break, so key on content. -- A test that edits a cached output must edit a copy: `api_kotlin`'s contract-mismatch - case rewrites `Native.class` in place, so a run that stops before restoring it would - leave every later run reusing the broken class. -- A death test runs the same cached program; caching its build changes nothing about - what it asserts. - -Expect to merge in `fixtures.rs` and the test programs with -[kixel-ai/rust2#767](https://github.com/kixel-ai/rust2/pull/767)'s non-dumpable -`broken(case)` entry and with [failed checks raise](/quest/m1/ffi-export/checks-raise.md) -([kixel-ai/rust2#730](https://github.com/kixel-ai/rust2/pull/730)), which rewrites many of -the same cases. A content key includes their edits to the test sources, so neither makes -a cached program stale. - -Out of scope: warming the queue's or a land fixer's `target`, which kixel-ai/dotfiles -owns; a cold run, which builds everything once as today. - -Public API: none; the exporter may stop rewriting unchanged files. - -## Related - -- [The moq export gates exporter merges](/quest/m0/ffi-moq-gate.md) - adds moq's export and C test to the gate, which the same skip should cover -- [Failed checks raise](/quest/m1/ffi-export/checks-raise.md) - rewrites the same test programs and fixture cases From 5152a71287a8da9ad06f3dabc3e8ee60f6e53089 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:43:22 -0800 Subject: [PATCH 04/11] style(ffi): format the new fixture tests Co-Authored-By: Claude Opus 5.5 --- ffi/tests/fixtures.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index baccd3f4f..7155792e2 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -1591,7 +1591,10 @@ fn reexport() { let mtime = std::fs::metadata(file).unwrap().modified().unwrap(); assert_eq!(mtime, past, "the second export rewrote {}", file.display()); } - assert!(!stray.parent().unwrap().exists(), "the export kept what it no longer writes"); + assert!( + !stray.parent().unwrap().exists(), + "the export kept what it no longer writes" + ); } /// A step replays while what it reads and what it wrote are as it left them, and runs again @@ -1626,7 +1629,10 @@ fn step_reruns_on_change() { assert!(ran(&init), "a step whose input changed replayed"); step::write(&repo.join("stray"), ""); assert!(ran(&init), "a step whose output changed replayed"); - assert!(!repo.join("stray").exists(), "a step's run kept what an earlier one left"); + assert!( + !repo.join("stray").exists(), + "a step's run kept what an earlier one left" + ); let stamp = dir.join("failed.stamp"); let failed = Step { @@ -1635,7 +1641,10 @@ fn step_reruns_on_change() { writes: &[], }; let first = failed.output(Command::new("git").arg("--no-such-flag")); - assert!(!first.status.success() && stamp.exists(), "an expected failure was not kept"); + assert!( + !first.status.success() && stamp.exists(), + "an expected failure was not kept" + ); let again = failed.output(Command::new("git").arg("--no-such-flag")); assert_eq!((again.status, again.stderr), (first.status, first.stderr)); } From 705613d775e4405bce52fee92f4b800250c0238b Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:06:46 -0800 Subject: [PATCH 05/11] perf(ffi): the glue's build script reruns only when it changes Without rerun-if-changed, cargo reruns it when any file of the export is newer, such as the Python extension a build copies beside its Python. Also adapt main's new fixtures to the cached scratch packages. Co-Authored-By: Claude Opus 5.5 --- ffi/src/glue.rs | 3 +++ ffi/tests/fixtures.rs | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 0b4aa6b3e..ab3bdbc6f 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -28,6 +28,9 @@ fn build(targets: &[String]) -> String { format!( "// Generated by rust2-ffi. Do not edit.\n\n\ fn main() {{\n\ + \t// What it prints depends on this script alone, and on what cargo passes it, not on\n\ + \t// the other files beside it, such as the Python extension a build copies there.\n\ + \tprintln!(\"cargo::rerun-if-changed=build.rs\");\n\ \tlet target = std::env::var(\"TARGET\").expect(\"cargo sets TARGET\");\n\ \tprintln!(\"cargo::rustc-check-cfg=cfg({CFG}, values({}))\");\n\ \tif ![{list}].contains(&target.as_str()) {{\n\ diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 84f5a47b9..ff0d66538 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -1526,10 +1526,13 @@ fn go_package_names() { /// handle, refused bare. #[test] fn unreached_types() { - let package = package("unreached_types"); - write_all( - &package, + let package = package( + "unreached_types", &[ + ( + "ffi.toml", + "exclude = [\"seal\", \"bare\", \"opaque\", \"closed\", \"own\", \"Buffer::first\"]\n", + ), ( "Cargo.toml", "[package]\nname = \"unreached_types\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[lib]\nname = \"pkg\"\n\n\ @@ -1599,11 +1602,6 @@ fn unreached_types() { ], ); // The export builds, naming `kio`'s types through its own dependency on `kio`. - std::fs::write( - package.join("ffi.toml"), - "exclude = [\"seal\", \"bare\", \"opaque\", \"closed\", \"own\", \"Buffer::first\"]\n", - ) - .unwrap(); let out = package.join("out"); run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) .arg("--config") @@ -1628,11 +1626,12 @@ fn unreached_types() { glue.contains("::net_reached::Link") && !glue.contains("Fault") && !glue.contains("net_reached::Handle"), "{glue}" ); + // Apart from the export's own build, which cleans the facade each time. run(cargo() .arg("build") .arg("--manifest-path") .arg(out.join("Cargo.toml")) - .env("CARGO_TARGET_DIR", package.join("build"))); + .env("CARGO_TARGET_DIR", package.join("glue"))); // Staged on another host, with the checkout elsewhere, the merged export depends on the // exporting host's `kio`, not the staging host's, though only the staged target reaches it. let staged = package.join("wasm.json"); @@ -1644,7 +1643,7 @@ fn unreached_types() { let moved = scratch().join("unreached_types-moved"); let _ = std::fs::remove_dir_all(&moved); for dir in ["src", "kio", "net", "conv"] { - copy_dir(&package.join(dir), &moved.join(dir)); + step::mirror(&package.join(dir), &moved.join(dir)); } for file in ["Cargo.toml", "ffi.toml"] { std::fs::copy(package.join(file), moved.join(file)).unwrap(); @@ -3119,7 +3118,7 @@ fn config_patched() { /// package, or `*`, is an error at its line. #[test] fn profiled() { - let package = package("profiled"); + let package = package("profiled", &[]); let files = |spec: &str| { write_all( &package, From 50d6d9e7a43dc0834ee6f0eee218860bb3405c3c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:12:26 -0800 Subject: [PATCH 06/11] docs(ffi): build each export in a target directory of its own Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ffi/README.md b/ffi/README.md index cafd0c64b..29e28f863 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -51,7 +51,10 @@ over the C ABI through cgo, and `dart/.dart` holds it in Dart, over the `dart:ffi` and what the crate builds with `--features dart`. An export again rewrites only the files whose content changed, and removes from `include//` and `go/` only what it no longer writes, so a build of an unchanged -export stays fresh. +export stays fresh. Build each export in a target directory of its own, as its own workspace +does by default: cargo names the crate's units by its path in that workspace, so two exports +of one facade sharing a target directory would share a build script, and one could reuse the +other's. ```toml # ffi.toml, beside the package's Cargo.toml unless --manifest-path says otherwise. From e3f1e96034abbabb99ac79c5946dc27fcd666d2b Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:20:58 -0800 Subject: [PATCH 07/11] fix(ffi): prune no symlinked root, find a suffixed program, probe its version by its name Co-Authored-By: Claude Opus 5.5 --- ffi/src/glue.rs | 23 ++++++++++++++-- ffi/tests/fixtures/step.rs | 56 +++++++++++++++++++++++++++----------- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index ab3bdbc6f..d006af63c 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -165,8 +165,16 @@ impl Output { /// of the export that cargo or a consumer fingerprints stays fresh. pub fn write(self) -> Result { let files: HashSet<&Path> = self.files.iter().map(|(path, _)| path.as_path()).collect(); - for dir in self.stale.iter().filter(|dir| dir.exists()) { - prune(dir, &files)?; + for dir in &self.stale { + match std::fs::symlink_metadata(dir) { + // A link, as the export never writes, goes itself: pruning would follow it. + Ok(meta) if meta.is_symlink() => { + std::fs::remove_file(dir).map_err(|err| format!("removing {}: {err}", dir.display()))? + } + Ok(_) => prune(dir, &files)?, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(format!("reading {}: {err}", dir.display())), + } } for (path, text) in &self.files { crate::driver::write(path, text)?; @@ -2034,5 +2042,16 @@ fn rewrite_keeps_unchanged_files() { output(&[("Cargo.toml", "b")]).write().unwrap(); assert_eq!(std::fs::read_to_string(root.join("Cargo.toml")).unwrap(), "b"); assert!(!go.exists(), "the export's emptied directory stays"); + // A directory it owns that is a link goes as the link, leaving what it points to. + #[cfg(unix)] + { + let elsewhere = root.join("elsewhere"); + std::fs::create_dir_all(&elsewhere).unwrap(); + std::fs::write(elsewhere.join("kept.go"), "kept").unwrap(); + std::os::unix::fs::symlink(&elsewhere, &go).unwrap(); + output(&[("go/rt/rt.go", "rt")]).write().unwrap(); + assert!(elsewhere.join("kept.go").exists(), "pruning followed a link"); + assert!(!go.is_symlink() && go.join("rt/rt.go").exists()); + } std::fs::remove_dir_all(&root).unwrap(); } diff --git a/ffi/tests/fixtures/step.rs b/ffi/tests/fixtures/step.rs index 9dbeee356..c01b8c346 100644 --- a/ffi/tests/fixtures/step.rs +++ b/ffi/tests/fixtures/step.rs @@ -131,8 +131,8 @@ fn status(raw: i64) -> ExitStatus { /// What a step's output depends on besides its own files: its program, by path and version, /// its arguments, directory, and environment, and the content of each file it `reads`. fn key(cmd: &Command, reads: &[&Path]) -> String { - let program = program(cmd); - let mut key = format!("program {program:?}\nversion {:?}\n", version(&program)); + let (found, program) = program(cmd); + let mut key = format!("program {program:?}\nversion {:?}\n", version(&found)); for arg in cmd.get_args() { key += &format!("arg {arg:?}\n"); } @@ -152,40 +152,64 @@ fn key(cmd: &Command, reads: &[&Path]) -> String { key } -/// The program `cmd` runs, resolved through `PATH` as the command would and through links, -/// so another toolchain on `PATH`, as another dev shell puts there, is another key. -fn program(cmd: &Command) -> PathBuf { +/// The program `cmd` runs, as found on `PATH` the way the command would, and that found +/// path resolved through links, so another toolchain on `PATH`, as another dev shell puts +/// there, is another key. +fn program(cmd: &Command) -> (PathBuf, PathBuf) { let program = Path::new(cmd.get_program()); - let path = match program.components().count() { - 1 => super::which(&program.to_string_lossy()), + let found = match program.components().count() { + 1 => find(program), _ => program.to_path_buf(), }; - std::fs::canonicalize(&path).unwrap_or_else(|err| panic!("resolving {}: {err}", path.display())) + let resolved = std::fs::canonicalize(&found).unwrap_or_else(|err| panic!("resolving {}: {err}", found.display())); + (found, resolved) } -/// What `program` says its version is, asked once each run: a toolchain updated in place, -/// or picked by `xcode-select`, keeps its path. -fn version(program: &Path) -> String { +/// The file `name` names in a directory on `PATH`, as is or, where executables take a +/// suffix and it has none, with it. +fn find(name: &Path) -> PathBuf { + let suffixed = match name.extension() { + None => name.with_extension(std::env::consts::EXE_EXTENSION), + Some(_) => name.to_path_buf(), + }; + let path = std::env::var_os("PATH").expect("PATH is set"); + std::env::split_paths(&path) + .flat_map(|dir| [dir.join(name), dir.join(&suffixed)]) + .find(|candidate| candidate.is_file()) + .unwrap_or_else(|| panic!("{} is not on PATH", name.display())) +} + +/// What the program at `found` says its version is, asked once each run by the name the +/// command runs it by, since a driver such as `swiftc` reads it: a toolchain updated in +/// place, or picked by `xcode-select`, keeps its path. +fn version(found: &Path) -> String { static VERSIONS: OnceLock>>>> = OnceLock::new(); let once = VERSIONS .get_or_init(Default::default) .lock() .unwrap() - .entry(program.to_path_buf()) + .entry(found.to_path_buf()) .or_default() .clone(); once.get_or_init(|| { - let flag: &[&str] = match program.file_stem().and_then(|stem| stem.to_str()) { + let stem = found.file_stem().and_then(|stem| stem.to_str()); + let flag: &[&str] = match stem { Some("go") => &["version"], Some("kotlinc") => &["-version"], - // MSVC's compiler prints its banner bare. + // MSVC's compiler prints its banner bare, and its status says nothing of it. Some("cl") => &[], _ => &["--version"], }; - let output = Command::new(program) + let output = Command::new(found) .args(flag) .output() - .unwrap_or_else(|err| panic!("running {}: {err}", program.display())); + .unwrap_or_else(|err| panic!("running {}: {err}", found.display())); + assert!( + output.status.success() || stem == Some("cl"), + "{} {flag:?} failed, so its version cannot key a step:\n{}", + found.display(), + String::from_utf8_lossy(&output.stderr) + ); format!( "{}{}", String::from_utf8_lossy(&output.stdout), From 1053bb3b2a7b7b8e9e95c711065211ab57e1e1cb Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:38:52 -0800 Subject: [PATCH 08/11] quest: record the maintainer keeping dart-exit Co-Authored-By: Claude Opus 5.5 --- quest/m0/dart-exit.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quest/m0/dart-exit.md b/quest/m0/dart-exit.md index 5beea4f3b..e9c6c4a7f 100644 --- a/quest/m0/dart-exit.md +++ b/quest/m0/dart-exit.md @@ -10,6 +10,8 @@ case (`futures`, `owned`, `closed`, `bytes`, `bounds`) exits in about 0.4 s. ## Plan +- Maintainer decision (2026-11-13): ✅ keep this follow-up (rejected: drop). + Found on 2026-11-13 by measuring the warm run after [FFI test programs build once per input change](https://github.com/kixel-ai/rust2/pull/780). From ae95dcd3714febe80ef98dedb8eda63af182dd2c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:43:43 -0800 Subject: [PATCH 09/11] fix(ffi): replace a changed file's entry, and cache no failed strict baseline A link at a generated path is replaced rather than written through, and cpp_strict's must-build case 0 runs through Step::run so a failure there is not replayed. Co-Authored-By: Claude Opus 5.5 --- ffi/src/driver.rs | 8 ++++++++ ffi/src/glue.rs | 15 +++++++++++++++ ffi/tests/fixtures.rs | 13 ++++++------- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/ffi/src/driver.rs b/ffi/src/driver.rs index d4bbad248..32ff44ef7 100644 --- a/ffi/src/driver.rs +++ b/ffi/src/driver.rs @@ -545,6 +545,8 @@ fn at(tcx: TyCtxt<'_>, span: Span) -> String { } /// Write `text` to `path` unless it already holds it, so its mtime moves only on a change. +/// A changed file replaces the entry rather than writing through it, so a link at `path` +/// goes and leaves what it points to. pub fn write(path: &Path, text: &str) -> Result<(), String> { if std::fs::read(path).is_ok_and(|old| old == text.as_bytes()) { return Ok(()); @@ -552,6 +554,12 @@ pub fn write(path: &Path, text: &str) -> Result<(), String> { if let Some(dir) = path.parent() { std::fs::create_dir_all(dir).map_err(|err| format!("creating {}: {err}", dir.display()))?; } + match std::fs::remove_file(path) { + Err(err) if err.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("removing {}: {err}", path.display())); + } + _ => {} + } std::fs::write(path, text).map_err(|err| format!("writing {}: {err}", path.display())) } diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index d006af63c..ee69ea131 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -2052,6 +2052,21 @@ fn rewrite_keeps_unchanged_files() { output(&[("go/rt/rt.go", "rt")]).write().unwrap(); assert!(elsewhere.join("kept.go").exists(), "pruning followed a link"); assert!(!go.is_symlink() && go.join("rt/rt.go").exists()); + // So does a changed file that is a link. + std::fs::remove_file(go.join("rt/rt.go")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("kept.go"), go.join("rt/rt.go")).unwrap(); + output(&[("go/rt/rt.go", "rt")]).write().unwrap(); + assert_eq!(std::fs::read_to_string(elsewhere.join("kept.go")).unwrap(), "kept"); + assert!(!go.join("rt/rt.go").is_symlink()); + assert_eq!(std::fs::read_to_string(go.join("rt/rt.go")).unwrap(), "rt"); } + // A hard link to a changed file keeps its content. + let linked = root.join("linked.txt"); + std::fs::write(&linked, "kept").unwrap(); + std::fs::remove_file(root.join("Cargo.toml")).unwrap(); + std::fs::hard_link(&linked, root.join("Cargo.toml")).unwrap(); + output(&[("Cargo.toml", "c")]).write().unwrap(); + assert_eq!(std::fs::read_to_string(&linked).unwrap(), "kept"); + assert_eq!(std::fs::read_to_string(root.join("Cargo.toml")).unwrap(), "c"); std::fs::remove_dir_all(&root).unwrap(); } diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index ff0d66538..5569bbbb9 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -475,16 +475,15 @@ fn cpp_strict(api: &Api) { reads: &[src, &include], writes: &[], }; - step.output(&mut cmd) + // Case 0 must build, so only the failing cases keep a failure. + match case { + 0 => step.run(&mut cmd), + _ => step.output(&mut cmd), + } }; let tests = manifest_dir().join("tests"); for src in ["api.cpp", "api_strict.cpp"] { - let output = clang(&tests.join(src), 0); - assert!( - output.status.success(), - "{src} failed Clang's ownership checks:\n{}", - String::from_utf8_lossy(&output.stderr) - ); + clang(&tests.join(src), 0); } for (case, message) in [ ( From 70c7cde905d90875859d04a089a22f79e6d0302c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:40:55 -0800 Subject: [PATCH 10/11] fix(ffi): exports apart in a shared target directory, atomic writes, and a warm moq gate - The glue's version is 0.0.0+ of its Rust sources and manifest, so two exports of one facade never share a cargo unit, whatever target directory they build in. The `target` fixture builds its exports in one directory again, which on a warm run failed with wasm32's export built for the host. - driver::write writes a changed file beside its path and renames it over it. - A failed step is replayed only where a failure is expected. - just ffi standin replaces a file only when it changes, and moq-c skips its link by a content stamp, so a warm moq gate rebuilds nothing. - Fix #750's Kotlin returns call for java's classpath argument. Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 6 ++---- ffi/justfile | 31 ++++++++++++++++++++++++---- ffi/src/driver.rs | 28 ++++++++++++++----------- ffi/src/glue.rs | 42 ++++++++++++++++++++++++++++++-------- ffi/tests/fixtures.rs | 14 ++++++------- ffi/tests/fixtures/step.rs | 5 ++++- 6 files changed, 89 insertions(+), 37 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index 38cdaab74..f87cbe306 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -51,10 +51,8 @@ over the C ABI through cgo, and `dart/.dart` holds it in Dart, over the `dart:ffi` and what the crate builds with `--features dart`. An export again rewrites only the files whose content changed, and removes from `include//` and `go/` only what it no longer writes, so a build of an unchanged -export stays fresh. Build each export in a target directory of its own, as its own workspace -does by default: cargo names the crate's units by its path in that workspace, so two exports -of one facade sharing a target directory would share a build script, and one could reuse the -other's. +export stays fresh. The crate's version, `0.0.0+`, hashes its Rust sources and +manifest, so exports that differ build apart in a target directory they share. ```toml # ffi.toml, beside the package's Cargo.toml unless --manifest-path says otherwise. diff --git a/ffi/justfile b/ffi/justfile index 00e6de6aa..62632a3dd 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -286,7 +286,19 @@ moq-c checkout="": (moq-write checkout) cargo rustc --locked --quiet --manifest-path "$out/Cargo.toml" --lib --crate-type staticlib --features kotlin,dart 2>&1 | tee target/moq-c-build.log sed -n 's/^note: native-static-libs: //p' target/moq-c-build.log > target/moq-c-native read -ra native < target/moq-c-native - cc -std=c11 -pedantic -Wall -Wextra -Werror -I "$out/include" moq/moq.c "$out/target/debug/libmoq.a" "${native[@]}" -o target/moq-c-test + link=(cc -std=c11 -pedantic -Wall -Wextra -Werror -I "$out/include" moq/moq.c "$out/target/debug/libmoq.a" "${native[@]}" -o target/moq-c-test) + # The link reruns only when what it reads, the compiler, or the command changed, or its + # program did: the stamp holds a hash of each, by content, since an export keeps mtimes. + key() { + { command -v cc; cc --version; printf '%s\n' "${link[@]}"; find "$out/include" -type f | LC_ALL=C sort | xargs sha256sum; sha256sum moq/moq.c "$out/target/debug/libmoq.a"; } | sha256sum + sha256sum < target/moq-c-test + } + stamp=target/moq-c-test.stamp + if ! [ -f target/moq-c-test ] || ! [ -f "$stamp" ] || [ "$(key)" != "$(cat "$stamp")" ]; then + rm -f "$stamp" + "${link[@]}" + key > "$stamp" + fi target/moq-c-test # Export moq's crates' C ABI into target/export/moq, unbuilt. @@ -335,6 +347,15 @@ standin checkout="": # moq-ffi's dependencies, moq-tokio with its default features as moq-ffi has them. deps=(hang kio moq-mux moq-net moq-pattern moq-tokio) mkdir -p "$dir/src" + # Replace a file only when its content changes, so an unchanged stand-in, and the + # export over it, stay fresh. + update() { + if cmp -s "$1" "$2"; then + rm "$1" + else + mv "$1" "$2" + fi + } { printf '# Generated by just ffi standin, from %s.\n' "$src" printf '[package]\nname = "moq-ffi"\nversion = "0.0.0"\nedition = "2024"\npublish = false\n\n' @@ -347,14 +368,15 @@ standin checkout="": # moq's profiles, which the export carries as it would moq-ffi's own; the dev ones # tune test-only crates the stand-in does not resolve. awk '/^\[/ { keep = /^\[profile\./ && !/^\[profile\.dev[].]/ } keep' "$src/Cargo.toml" - } > "$dir/Cargo.toml" + } > "$dir/Cargo.toml.new" + update "$dir/Cargo.toml.new" "$dir/Cargo.toml" # What moq-ffi ships, re-exported: moq-net and moq-pattern at the root, beside the # waiter a driver's poll registers and the clock it polls at, and each other crate in a # module of its own, since several name an item alike, such as `Error`. moq-tokio's # session side only: the client, server, and origin a binding drives, with the configs # and the bind they take. Its QUIC driver, CLI, worker groups, and listener internals # stay out. - cat > "$dir/src/lib.rs" <<'EOF' + cat > "$dir/src/lib.rs.new" <<'EOF' pub use kio::{Park, Waiter, WaiterList, clock, coop, log}; pub use moq_net::*; pub use moq_pattern::*; @@ -400,8 +422,9 @@ standin checkout="": } } EOF + update "$dir/src/lib.rs.new" "$dir/src/lib.rs" # moq's lockfile pins what moq's own builds use. - cp "$src/Cargo.lock" "$dir/Cargo.lock" + cmp -s "$src/Cargo.lock" "$dir/Cargo.lock" || cp "$src/Cargo.lock" "$dir/Cargo.lock" # Check out the pinned moq revision in target/moq, apart from `just ts`'s so neither recipe touches the other's checkout, copying the objects its clone already has. [private] diff --git a/ffi/src/driver.rs b/ffi/src/driver.rs index 32ff44ef7..a9d9ff29f 100644 --- a/ffi/src/driver.rs +++ b/ffi/src/driver.rs @@ -545,22 +545,26 @@ fn at(tcx: TyCtxt<'_>, span: Span) -> String { } /// Write `text` to `path` unless it already holds it, so its mtime moves only on a change. -/// A changed file replaces the entry rather than writing through it, so a link at `path` -/// goes and leaves what it points to. +/// A changed file is written beside `path` and renamed over it, so a reader sees the old +/// content or the new, never neither or part, and a link at `path` goes and leaves what it +/// points to. pub fn write(path: &Path, text: &str) -> Result<(), String> { if std::fs::read(path).is_ok_and(|old| old == text.as_bytes()) { return Ok(()); } - if let Some(dir) = path.parent() { - std::fs::create_dir_all(dir).map_err(|err| format!("creating {}: {err}", dir.display()))?; - } - match std::fs::remove_file(path) { - Err(err) if err.kind() != std::io::ErrorKind::NotFound => { - return Err(format!("removing {}: {err}", path.display())); - } - _ => {} - } - std::fs::write(path, text).map_err(|err| format!("writing {}: {err}", path.display())) + let (Some(dir), Some(name)) = (path.parent(), path.file_name()) else { + return Err(format!("writing {}: not a file path", path.display())); + }; + std::fs::create_dir_all(dir).map_err(|err| format!("creating {}: {err}", dir.display()))?; + let mut temp = std::ffi::OsString::from("."); + temp.push(name); + temp.push(".rust2-ffi"); + let temp = dir.join(temp); + std::fs::write(&temp, text).map_err(|err| format!("writing {}: {err}", temp.display()))?; + std::fs::rename(&temp, path).map_err(|err| { + let _ = std::fs::remove_file(&temp); + format!("writing {}: {err}", path.display()) + }) } /// Whether the struct, enum, or union `adt` declares a field or variant `name` under a diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index ee69ea131..2da298f21 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -7,8 +7,11 @@ use std::collections::HashSet; use std::fmt::Write; +use std::hash::Hasher; use std::path::{Path, PathBuf}; +use rustc_data_structures::stable_hash::StableHasher; + use crate::abi::{self, Abi, CFG, Ty}; use crate::config::Package; use crate::ir::{ @@ -285,11 +288,12 @@ impl Crate<'_> { } _ => String::new(), }; - let manifest = format!( - "# Generated by rust2-ffi: the C ABI of the pub API of {packages}.\n\ + let manifest = |version: &str| { + format!( + "# Generated by rust2-ffi: the C ABI of the pub API of {packages}.\n\ [package]\n\ name = \"{package}-ffi\"\n\ - version = \"0.0.0\"\n\ + version = \"{version}\"\n\ edition = \"2024\"\n\ {rust_version}\ publish = false\n\ @@ -319,11 +323,12 @@ impl Crate<'_> { {targeted}\ [workspace]\n\ {patch}", - patch = self.patch, - package = facade.name, - packages = self.packages(), - lib = self.abi.prefix, - ); + patch = self.patch, + package = facade.name, + packages = self.packages(), + lib = self.abi.prefix, + ) + }; let (header, contract) = crate::c::header(self.abi, &self.packages()); let cpp = crate::cpp::headers(self.abi, &self.packages())?; let kotlin = crate::kotlin::generate(self.abi, &self.packages(), contract)?; @@ -358,7 +363,7 @@ impl Crate<'_> { out: String::new(), }; generator.lib(); - output.file(&out.join("Cargo.toml"), &manifest); + output.file(&out.join("Cargo.toml"), &manifest("0.0.0")); output.file(&out.join("src/lib.rs"), &generator.out); output.file(&out.join("build.rs"), &build(&self.abi.api.targets)); output.file(&out.join("src/rt.rs"), RUNTIME); @@ -400,6 +405,25 @@ impl Crate<'_> { for (path, text) in go.files { output.file(&go_dir.join(path), &text); } + // Cargo names a path package's units by its name, version, and path in its + // workspace, the same for every export of one facade, and judges them fresh by + // mtimes, which an unchanged file keeps. So two exports in one target directory + // would share units, and one could build with the other's. A version whose build + // metadata hashes what cargo compiles separates them; an unchanged export keeps it. + let mut hasher = StableHasher::new(); + for (path, text) in &output.files { + if path.extension().is_some_and(|e| e == "rs" || e == "toml") { + hasher.write_u64(text.len() as u64); + hasher.write(text.as_bytes()); + } + } + let version = format!("0.0.0+{:016x}", Hasher::finish(&hasher)); + let cargo_toml = out.join("Cargo.toml"); + for (path, text) in &mut output.files { + if *path == cargo_toml { + *text = manifest(&version); + } + } Ok(output) } } diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 5992161f2..61e3c5b18 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -625,7 +625,7 @@ fn api_kotlin() { .arg("--example") .arg("returns") .current_dir(manifest_dir())); - let kotlin_returns = run(&mut java(&["returns"])); + let kotlin_returns = run(&mut java(&[&classes], &["returns"])); assert_eq!( kotlin_returns.stdout, native_returns.stdout, "Kotlin composite returns differ from native Rust" @@ -2720,11 +2720,11 @@ fn target() { "{header}" ); assert!(header.contains(&u64::MAX.to_string()), "{header}"); - // Each export builds in a target directory of its own: cargo names a package's units - // by its path in its workspace, which each export's is the root of, so the exports' - // units would share one, and reuse another's build script where the mtimes allow. - let target = |out: &Path| package.join("glue").join(out.file_name().unwrap()); - build_offline(&host_out, &target(&host_out)); + // The exports share a target directory, where each export's units are its own though + // they are the same package at the same path in its workspace: on a second run, with + // every file's mtime kept, `wasm` would otherwise reuse `merged`'s build script. + let target = package.join("glue"); + build_offline(&host_out, &target); let (wasm, header) = export("wasm", &["--target", WASM]); assert!( header.contains("targeted_wasm(") && !header.contains("targeted_desktop("), @@ -2746,7 +2746,7 @@ fn target() { ]) .arg(out.join("Cargo.toml")) .arg("--target-dir") - .arg(target(out)) + .arg(&target) .args(args) .env("RUSTFLAGS", "-D warnings"); cmd diff --git a/ffi/tests/fixtures/step.rs b/ffi/tests/fixtures/step.rs index c01b8c346..5245d7f3a 100644 --- a/ffi/tests/fixtures/step.rs +++ b/ffi/tests/fixtures/step.rs @@ -40,7 +40,10 @@ impl Step<'_> { fn execute(&self, cmd: &mut Command, failure: bool) -> Output { let key = key(cmd, self.reads); - if let Some(output) = self.replay(&key) { + // A failure recorded where one was expected is run again where none is. + if let Some(output) = self.replay(&key) + && (output.status.success() || failure) + { return output; } // No stamp may outlive a run that stops partway. From 2e1fc198998e7b2741a797cb40d7aa646faf9ad8 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:03:07 -0800 Subject: [PATCH 11/11] fix(ffi): write a changed file through no link left at its temporary path Co-Authored-By: Claude Opus 5.5 --- ffi/src/driver.rs | 14 +++++++++++++- ffi/src/glue.rs | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ffi/src/driver.rs b/ffi/src/driver.rs index a9d9ff29f..6b9c0dbf6 100644 --- a/ffi/src/driver.rs +++ b/ffi/src/driver.rs @@ -560,7 +560,19 @@ pub fn write(path: &Path, text: &str) -> Result<(), String> { temp.push(name); temp.push(".rust2-ffi"); let temp = dir.join(temp); - std::fs::write(&temp, text).map_err(|err| format!("writing {}: {err}", temp.display()))?; + // A link left at the temporary path goes first, so the write cannot follow it. + match std::fs::remove_file(&temp) { + Err(err) if err.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("removing {}: {err}", temp.display())); + } + _ => {} + } + std::fs::File::options() + .write(true) + .create_new(true) + .open(&temp) + .and_then(|mut file| std::io::Write::write_all(&mut file, text.as_bytes())) + .map_err(|err| format!("writing {}: {err}", temp.display()))?; std::fs::rename(&temp, path).map_err(|err| { let _ = std::fs::remove_file(&temp); format!("writing {}: {err}", path.display()) diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 2da298f21..8f7b97fc9 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -2083,6 +2083,12 @@ fn rewrite_keeps_unchanged_files() { assert_eq!(std::fs::read_to_string(elsewhere.join("kept.go")).unwrap(), "kept"); assert!(!go.join("rt/rt.go").is_symlink()); assert_eq!(std::fs::read_to_string(go.join("rt/rt.go")).unwrap(), "rt"); + // And a link left where a changed file is first written. + std::os::unix::fs::symlink(elsewhere.join("kept.go"), root.join(".Cargo.toml.rust2-ffi")).unwrap(); + output(&[("Cargo.toml", "linked")]).write().unwrap(); + assert_eq!(std::fs::read_to_string(elsewhere.join("kept.go")).unwrap(), "kept"); + assert!(!root.join("Cargo.toml").is_symlink()); + assert_eq!(std::fs::read_to_string(root.join("Cargo.toml")).unwrap(), "linked"); } // A hard link to a changed file keeps its content. let linked = root.join("linked.txt");