diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2437e90f1..9420ccf55 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -100,6 +100,10 @@ jobs: - name: Export run: nix develop --command just ffi moq + # Every sampled type shape round-tripped in every language; the gate runs a subset. + - name: Shapes + run: nix develop --command just ffi shapes + # What exports, and every refusal, ranked. - name: Census run: nix develop --command just ffi census diff --git a/ffi/README.md b/ffi/README.md index 1722d0748..2d3ca2364 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -950,6 +950,7 @@ just ffi moq # export moq's crates at the pinned revision, build them, and just ffi moq-cpp # the same export, running only moq/moq.cpp just ffi moq-swift # the same export, running only moq/moq.swift just ffi census # census each into target/census +just ffi shapes # round-trip every sampled type shape in every language but KNOWN's, as the nightly does nix develop .#android --command just ffi android # the Kotlin test, shrunk by R8, on the Android device adb reaches nix develop .#bench --command just ffi bench # the binding benchmark: bench/ffi/README.md's cases from Rust, and through the export from C, C++, Kotlin, Python, Swift, Go, and Dart nix develop .#sanitize --command just ffi sanitize # api's C and C++ scripts under ASan and TSan, and its C ABI under Miri @@ -994,6 +995,17 @@ each runtime check, which must panic with its message; a runtime package from an build must fail its contract check. The `target` test exports a package for the host, for wasm32, and for both into one binding, which builds for each, and staged with `--api` and `--with`; a build for a target not exported fails, and items that differ between the two are refused, naming both. +The shape tests (`tests/fixtures/shapes.rs`) sample value types from the IR's `Type`, each +leaf alone and in each wrapper, then nested to depth 3 from a fixed seed, and emit a crate +of `make_N(seed)`, `echo_N(v)`, and `digest_N(&v)` per shape, with the shape in a closure's +parameter and return and in a future besides. Its census decides what exports, so a shape +refused by design is left out rather than failed. Each language's driver is a generated +flat list of `digest(echo(make(seed)))` calls whose printed digests must match the crate's +own, run natively. The gate runs a subset; `just ffi shapes` runs every sampled shape, as +the nightly does. `KNOWN` lists what a binding still gets wrong, each with the quest that +fixes it; the driver skips those shapes, so the fix deletes its entry, and the nightly fails +on an entry that matches no exported shape. A new IR `Type` variant fails `ir_covered` +until the sampler covers it. The `reject` fixture's expected errors, each with its notes, are `fixtures/reject/expected.txt`. The `lock` test exports the `lock` fixture for wasm32, staged, and the host, then again against that export's `api.json`: its `added` feature only adds and must pass, and its `next` diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index e948743c4..036f09eab 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -2264,6 +2264,14 @@ pub fn prefixed(payload: &[u8], prefix: Option<&[u8]>) -> Vec { [prefix.unwrap_or_default(), payload].concat() } +/// The sum of `a`, `b`, and `c`, if any: a list owned, then lent as a slice and as itself. +pub fn sum_lists(a: Vec, b: Option<&[i32]>, c: Option<&Vec>) -> i32 { + a.iter() + .chain(b.unwrap_or_default()) + .chain(c.into_iter().flatten()) + .sum() +} + /// `point`'s coordinates summed, or 0 for none: an optional value, lent shared. pub fn coordinates(point: Option<&Point>) -> i32 { point.map_or(0, |point| point.x + point.y) diff --git a/ffi/justfile b/ffi/justfile index c7483d5a4..89becba4f 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -31,6 +31,10 @@ check: test *args: cargo test --locked {{ args }} +# Round-trip every sampled type shape in every language, as the nightly does; the gate runs a subset. +shapes: + cargo test --locked --test fixtures shapes::nightly -- --ignored + # Shrink the api fixture's Kotlin with R8 and its generated rules, and run its test on the Android device adb reaches, in the Android shell (nix develop .#android) with ANDROID_HOME and ANDROID_NDK_HOME set. android: cargo test --locked --test fixtures -- --ignored --exact api_android diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 7f39cf902..a8f4d4fe9 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -828,15 +828,16 @@ pub fn lent_bytes(buf: rt::Buf) -> R { } /// A path's bytes, from a `str`, `bytes`, or `os.PathLike`, as `os.fsencode` encodes it, -/// so a name a Unix path holds that is not UTF-8 crosses as it came. +/// so a name a Unix path holds that is not UTF-8 crosses as it came, and so does a NUL, +/// which a Rust path holds and `PyUnicode_FSConverter` refuses. pub fn path_in(obj: *mut PyObject, s: &mut Scope) -> R { - let mut encoded: *mut PyObject = null_mut(); - // SAFETY: the GIL is held, and `obj` is live; the converter writes a new `bytes`. + // SAFETY: the GIL is held, and `obj` is live; each call returns a new reference. unsafe { - if PyUnicode_FSConverter(obj, (&raw mut encoded).cast()) == 0 { - return Err(Raised); - } - let encoded = s.keep(Obj::new(encoded)?); + let path = s.keep(Obj::new(PyOS_FSPath(obj))?); + let encoded = match PyBytes_Check(path) != 0 { + true => path, + false => s.keep(Obj::new(PyUnicode_EncodeFSDefault(path))?), + }; let mut ptr: *mut c_char = null_mut(); let mut len = 0; if PyBytes_AsStringAndSize(encoded, &mut ptr, &mut len) != 0 { diff --git a/ffi/runtime/rt.hpp b/ffi/runtime/rt.hpp index a07e362da..3b39e0440 100644 --- a/ffi/runtime/rt.hpp +++ b/ffi/runtime/rt.hpp @@ -778,17 +778,22 @@ class arena { // `items` as the C array `C`, each lowered by `lower`: never null, so an empty one is // told apart from a missing one. Each item is mutable where `items` is, so a handle // moved in can be owned alone. + // The array is no `std::vector`, whose `bool` form holds no array, and `items` may be + // one, whose items are proxies rather than references. template C list(R &items, F lower) { using E = ::std::remove_const_t<::std::remove_pointer_t>; - auto array = ::std::make_shared<::std::vector>(); - array->reserve(items.size()); - for (auto &item : items) { - array->push_back(lower(item)); - } static const E empty{}; - const E *ptr = array->empty() ? &empty : array->data(); - ::std::size_t len = array->size(); + ::std::size_t len = items.size(); + if (len == 0) { + return C{&empty, 0, nullptr}; + } + ::std::shared_ptr array(new E[len]); + ::std::size_t i = 0; + for (auto &&item : items) { + array[i++] = lower(item); + } + const E *ptr = array.get(); held_.push_back(::std::move(array)); return C{ptr, len, nullptr}; } diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index a3be6599d..0cc625ec6 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -72,6 +72,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result { natives: BTreeMap, /// The conversions, polls, and trampolines, by name. helpers: BTreeMap, + /// Each writer's name, by the type it writes: types of one C struct, such as a `Vec` of + /// `SocketAddrV4` and one of `SocketAddr`, check what they write apart. + writers: HashMap, /// The types, then the top-level functions and constants. types: String, top: String, @@ -895,7 +899,7 @@ impl<'a> Gen<'a> { Type::Slice | Type::Vec | Type::Bytes => format!("a_.bytes({dst}, {e});"), Type::List(..) => { let c = self.c_name(inner); - let w = self.writer(&c, inner); + let w = self.writer(&c, inner.base()); format!("{w}({e}, {dst}, a_);") } _ => { @@ -929,10 +933,17 @@ impl<'a> Gen<'a> { /// The function writing a Dart value of type `ty` into the C struct `c`. fn writer(&mut self, c: &str, ty: &Type) -> String { - let name = format!("_w_{c}"); - if self.helpers.contains_key(&name) { - return name; + if let Some(name) = self.writers.get(ty) { + return name.clone(); + } + let mut name = format!("_w_{c}"); + for k in 2.. { + if !self.helpers.contains_key(&name) { + break; + } + name = format!("_w_{c}_{k}"); } + self.writers.insert(ty.clone(), name.clone()); self.helpers.insert(name.clone(), String::new()); let class = self.struct_class(&Ty::Struct(c.to_string())); let dt = self.dt(ty); @@ -1848,10 +1859,17 @@ impl<'a> Gen<'a> { /// The Dart statements closing each object the value `e` of type `ty` holds, once a /// call moved them out. fn close(&mut self, ty: &Type, e: &str) -> String { + // A local named apart from `e`, which may be a parameter named `v`, and from the + // local an outer level declared, which Dart would read before it is declared. + let v: String = e + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect::() + + "_"; match ty { Type::Option(inner) => { - let close = self.close(inner, "v"); - format!("{{ final v = {e}; if (v != null) {{ {close} }} }}") + let close = self.close(inner, &v); + format!("{{ final {v} = {e}; if ({v} != null) {{ {close} }} }}") } Type::Box(inner) => self.close(inner, e), Type::Tuple(elements) => { @@ -1862,8 +1880,8 @@ impl<'a> Gen<'a> { closes.join(" ") } Type::List(_, inner) => { - let close = self.close(inner, "v"); - format!("for (final v in {e}) {{ {close} }}") + let close = self.close(inner, &v); + format!("for (final {v} in {e}) {{ {close} }}") } _ => format!("{e}.close();"), } diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index e9611b59c..b9fb81e73 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -1015,7 +1015,12 @@ impl Gen<'_> { .fields .iter() .zip(&bindings) - .map(|(f, b)| format!("{}: {b}", f.name)) + // A field named as its binding is, `f0`, takes the shorthand, which + // the lint would ask for. + .map(|(f, b)| match f.name == *b { + true => b.clone(), + false => format!("{}: {b}", f.name), + }) .collect(); format!("{path} {{ {} }}", fields.join(", ")) } @@ -1367,6 +1372,18 @@ impl Gen<'_> { format!("let {local} = {};", self.lift(ty, c)), format!("{local}.as_deref()"), ), + // A lifted `Vec` lends itself or, by coercion to the parameter's type, its slice, + // since a borrowed `Vec` and a slice share their type. + (Type::List(Collection::Vec, _), _) => { + let (binding, borrow) = match m { + Mutability::Mut => ("let mut", "&mut"), + Mutability::Not => ("let", "&"), + }; + ( + format!("{binding} {local} = {};", self.lift(ty, c)), + format!("match {borrow} {local} {{ Some(v) => Some(v), None => None }}"), + ) + } _ => ( format!("let {local} = {};", self.lift(ty, c)), format!("{local}.as_ref()"), diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 40e8335fd..0626134f5 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -110,6 +110,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result { thrown: HashMap, /// Each module-level Python name, with what claimed it. names: BTreeMap, + /// The suffix of the glue's `r_` and `w_` of each option, tuple, and list type by + /// its C struct and what the struct holds: types of one struct, such as a `Vec` of + /// `SocketAddrV4` and one of `SocketAddr`, convert apart. + converters: HashMap<(String, Vec), String>, errors: Vec, /// The targets of the item being written, whose native functions check theirs. on: On, @@ -498,7 +503,7 @@ impl<'a> Gen<'a> { Type::U128 => format!("rt::u128_in({o})"), Type::Duration => format!("rt::duration_in({o})"), Type::SocketAddr(family) => format!("rt::addr_in({o}, {})", family_of(*family)), - Type::List(..) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + Type::List(..) => format!("r_{}({o}, s, {m})", self.converter(ty)), Type::Option(inner) => match inner.base() { Type::Object(i) => { let m = match abi.api.objects[*i].copy || matches!(**inner, Type::Ref(..)) { @@ -515,7 +520,7 @@ impl<'a> Gen<'a> { let some = self.lift(inner, o, m); format!("(if rt::is_none({o}) {{ Ok(crate::rt::List::NONE) }} else {{ {some} }})") } - _ => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + _ => format!("r_{}({o}, s, {m})", self.converter(ty)), }, Type::Box(inner) => self.lift(inner, o, m), Type::Ref(_, inner) => self.lift(inner, o, "false"), @@ -551,7 +556,7 @@ impl<'a> Gen<'a> { Type::Future => format!("rt::future_in({o}, s)"), Type::Range(int) => format!("rt::range_in::<{}>({o})", int.rust()), Type::Bound(_) | Type::Bounds(_) => format!("r_{}({o}, s)", abi.range(ty)), - Type::Tuple(_) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + Type::Tuple(_) => format!("r_{}({o}, s, {m})", self.converter(ty)), Type::Unit | Type::Closure(..) => unreachable!("{ty:?} is no value Python passes"), } } @@ -575,7 +580,7 @@ impl<'a> Gen<'a> { Type::U128 => format!("rt::u128_out({v})"), Type::Duration => format!("rt::duration_out({v})"), Type::SocketAddr(_) => format!("rt::addr_out({v})"), - Type::List(..) => format!("w_{}({v})", self.c_struct(ty)), + Type::List(..) => format!("w_{}({v})", self.converter(ty)), Type::Option(inner) => match inner.base() { Type::Object(i) => format!( "rt::object_opt({v}, {}, crate::{}_free)", @@ -586,7 +591,7 @@ impl<'a> Gen<'a> { let some = self.lower(inner, "v"); format!("{{ let v = {v}; if v.is_none() {{ rt::none() }} else {{ {some} }} }}") } - _ => format!("w_{}({v})", self.c_struct(ty)), + _ => format!("w_{}({v})", self.converter(ty)), }, Type::Box(inner) | Type::Ref(_, inner) => self.lower(inner, v), Type::Record(_) => format!("w_{}({v})", self.c_struct(ty)), @@ -605,7 +610,7 @@ impl<'a> Gen<'a> { format!("{{ let r = {v}; rt::range_out({start}, {end}) }}") } Type::Bound(_) | Type::Bounds(_) => format!("w_{}({v})", abi.range(ty)), - Type::Tuple(_) => format!("w_{}({v})", self.c_struct(ty)), + Type::Tuple(_) => format!("w_{}({v})", self.converter(ty)), Type::Waker => format!("rt::waker_out({v})"), Type::Callback(..) | Type::Closure(..) | Type::Future | Type::Reader => { unreachable!("{ty:?} is no value Rust returns") @@ -621,6 +626,51 @@ impl<'a> Gen<'a> { } } + /// The suffix of the glue's `r_` and `w_` converting the option, tuple, or list `ty`: + /// its C struct's name for the type the struct was made for, which [`Self::write`] + /// writes, else a name of its own, written here once. + fn converter(&mut self, ty: &Type) -> String { + let abi = self.abi; + let c = self.c_struct(ty); + let (held, made): (Vec, Vec) = match strip_refs(ty) { + Type::Option(inner) => (vec![*inner], vec![abi.options[&c].clone()]), + Type::List(_, elem) => (vec![*elem], vec![abi.lists[&c].clone()]), + Type::Tuple(elements) => (elements, abi.tuples[&c].clone()), + other => unreachable!("{other:?} has no converter"), + }; + let made: Vec = made.iter().map(strip_refs).collect(); + if held.iter().map(strip_refs).eq(made.iter().cloned()) { + return c; + } + let key = (c.clone(), held.clone()); + if let Some(name) = self.converters.get(&key) { + return name.clone(); + } + // A numbered name must miss every converter's: each type's own, and the others numbered. + let taken = |name: &str| { + abi.options.contains_key(name) + || abi.lists.contains_key(name) + || abi.tuples.contains_key(name) + || abi.ranges.contains_key(name) + || abi.arrays.contains_key(name) + || (0..abi.api.records.len()).any(|i| abi.record(i) == name) + || (0..abi.api.enums.len()).any(|i| abi.enumeration(i) == name) + || (0..abi.api.objects.len()).any(|i| abi.object(i) == name) + || self.converters.values().any(|n| n == name) + }; + let name = (self.converters.len() + 2..) + .map(|k| format!("{c}_{k}")) + .find(|name| !taken(name)) + .expect("some number is free"); + self.converters.insert(key, name.clone()); + match strip_refs(ty) { + Type::Option(_) => self.option(&name, &c, &held[0]), + Type::List(..) => self.list(&name, &c, &held[0]), + _ => self.tuple(&name, &c, &held), + } + name + } + fn write(&mut self) { let abi = self.abi; let api = abi.api; @@ -636,10 +686,10 @@ impl<'a> Gen<'a> { self.range(c, ty); } for (c, inner) in &abi.options.clone() { - self.option(c, inner); + self.option(c, c, inner); } for (c, elements) in &abi.tuples.clone() { - self.tuple(c, elements); + self.tuple(c, c, elements); } for (c, n) in &abi.arrays { let name = Abi::array_class(*n); @@ -650,7 +700,7 @@ impl<'a> Gen<'a> { ); } for (c, elem) in &abi.lists.clone() { - self.list(c, elem); + self.list(c, c, elem); } for constant in &api.constants { let name = self.bare(&abi.name(&constant.path)).to_uppercase(); @@ -756,19 +806,19 @@ impl<'a> Gen<'a> { ); } - /// The glue of an option struct. - fn option(&mut self, c: &str, inner: &Type) { + /// The glue of an option struct `c` holding `inner`, as `r_` and `w_`. + fn option(&mut self, name: &str, c: &str, inner: &Type) { let read = self.lift(inner, "o", "m"); let write = self.lower(inner, "v.value"); let _ = write!( self.glue, - "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\tmatch rt::is_none(o) {{\n\t\ttrue => Ok(crate::{c} {{ some: false, value: unsafe {{ std::mem::zeroed() }} }}),\n\t\tfalse => Ok(crate::{c} {{ some: true, value: {}? }}),\n\t}}\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\tmatch v.some {{\n\t\ttrue => {write},\n\t\tfalse => rt::none(),\n\t}}\n}}\n\n", + "fn r_{name}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\tmatch rt::is_none(o) {{\n\t\ttrue => Ok(crate::{c} {{ some: false, value: unsafe {{ std::mem::zeroed() }} }}),\n\t\tfalse => Ok(crate::{c} {{ some: true, value: {}? }}),\n\t}}\n}}\n\nfn w_{name}(v: crate::{c}) -> R {{\n\tmatch v.some {{\n\t\ttrue => {write},\n\t\tfalse => rt::none(),\n\t}}\n}}\n\n", read ); } - /// The glue of a tuple struct, element by element. - fn tuple(&mut self, c: &str, elements: &[Type]) { + /// The glue of a tuple struct `c`, element by element, as `r_` and `w_`. + fn tuple(&mut self, name: &str, c: &str, elements: &[Type]) { let n = elements.len(); let mut reads = Vec::new(); let mut fields = Vec::new(); @@ -781,20 +831,20 @@ impl<'a> Gen<'a> { } let _ = write!( self.glue, - "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\t{}\n\tOk(crate::{c} {{ {} }})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::tuple(vec![{}])\n}}\n\n", + "fn r_{name}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\t{}\n\tOk(crate::{c} {{ {} }})\n}}\n\nfn w_{name}(v: crate::{c}) -> R {{\n\trt::tuple(vec![{}])\n}}\n\n", reads.join("\n\t"), fields.join(", "), writes.join(", ") ); } - /// The glue of a `Vec`'s array. - fn list(&mut self, c: &str, elem: &Type) { + /// The glue of a `Vec`'s array `c`, as `r_` and `w_`. + fn list(&mut self, name: &str, c: &str, elem: &Type) { let read = self.lift(elem, "o", "m"); let write = self.lower(elem, "e"); let _ = write!( self.glue, - "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\trt::list_in(o, s, |o, s| {read})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::list_out(v, |e| {write})\n}}\n\n" + "fn r_{name}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\trt::list_in(o, s, |o, s| {read})\n}}\n\nfn w_{name}(v: crate::{c}) -> R {{\n\trt::list_out(v, |e| {write})\n}}\n\n" ); } diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 0e44d2872..f43b7a877 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -3,7 +3,11 @@ //! and, shrunk by R8, on Android, from Python by `tests/api.py`, from Swift by //! `tests/api.swift`, from Go by `tests/api.go`, and from Dart by `tests/api.dart`, whose //! transcripts must match the fixture's own script run natively. -//! The reject fixture's export must fail at each item it cannot export. +//! The reject fixture's export must fail at each item it cannot export, and generated +//! type shapes round-trip in every language ([`shapes`]). + +#[path = "fixtures/shapes.rs"] +mod shapes; use std::env::consts::EXE_SUFFIX; use std::ffi::OsString; @@ -339,7 +343,7 @@ fn api_cpp() { for (standard, plain) in [(17, true), (20, false), (23, true)] { let exe = scratch().join("api").join(format!("api-c++{standard}{EXE_SUFFIX}")); run(&mut cxx( - api, + &api.link(), standard, plain, &manifest_dir().join("tests").join("api.cpp"), @@ -400,7 +404,7 @@ fn api_cpp() { let src = scratch().join("api").join(format!("alone-{header}.cpp")); std::fs::write(&src, format!("#include \"api/{header}\"\nint main() {{}}\n")).unwrap(); let exe = scratch().join("api").join(format!("alone-{header}{EXE_SUFFIX}")); - run(&mut cxx(api, 17, true, &src, &exe)); + run(&mut cxx(&api.link(), 17, true, &src, &exe)); } // A C++ program whose header names another contract than its library's aborts at load. @@ -411,7 +415,7 @@ fn api_cpp() { ) .unwrap(); let contract = scratch().join("api").join(format!("contract{EXE_SUFFIX}")); - run(&mut cxx(api, 17, false, &mismatched, &contract)); + run(&mut cxx(&api.link(), 17, false, &mismatched, &contract)); let output = Command::new(&contract).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not abort"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -465,11 +469,29 @@ fn cpp_strict(api: &Api) { } } -/// A build of the C++ `src` against the api library into `exe`, at C++`standard`, with +/// What a program links of an export: its headers, its static library, and what that +/// links besides. +struct Link<'a> { + include: PathBuf, + staticlib: PathBuf, + native: &'a [String], +} + +impl Api { + fn link(&self) -> Link<'_> { + Link { + include: self.out.join("include"), + staticlib: self.staticlib(), + native: &self.native, + } + } +} + +/// A build of the C++ `src` against the `link`ed library into `exe`, at C++`standard`, with /// exceptions and RTTI off if `plain`, and warnings as errors: by `CXX` or the platform's /// compiler, given MSVC's flags where the target is MSVC's. -fn cxx(api: &Api, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command { - let include = api.out.join("include"); +fn cxx(link: &Link, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command { + let include = &link.include; if cfg!(target_env = "msvc") { // cl from a developer environment. Its C++23 is `c++latest`; its objects go beside `exe`. let mut cmd = Command::new(std::env::var_os("CXX").unwrap_or_else(|| "cl".into())); @@ -494,7 +516,7 @@ fn cxx(api: &Api, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command let mut out = OsString::from("-Fe"); out.push(exe); cmd.arg("-I").arg(include).arg(src).arg(obj).arg(out); - cmd.arg("-link").arg(api.staticlib()).args(&api.native); + cmd.arg("-link").arg(&link.staticlib).args(link.native); return cmd; } let mut cmd = Command::new(std::env::var_os("CXX").unwrap_or_else(|| "c++".into())); @@ -506,8 +528,8 @@ fn cxx(api: &Api, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command cmd.args(["-pedantic", "-Wall", "-Wextra", "-Wshadow", "-Werror", "-I"]) .arg(include) .arg(src) - .arg(api.staticlib()) - .args(&api.native) + .arg(&link.staticlib) + .args(link.native) .arg("-o") .arg(exe); cmd @@ -971,7 +993,13 @@ fn api_dart() { let generated = api.out.join("dart"); run(Command::new("dart").args(["analyze", "--fatal-infos"]).arg(&generated)); let lib = api.lib.join(PYTHON_LIB); - let exe = dart_compile(&dir, &generated, &lib, &manifest_dir().join("tests").join("api.dart")); + let exe = dart_compile( + "api", + &dir, + &generated, + &lib, + &manifest_dir().join("tests").join("api.dart"), + ); let dart = run(&mut dart_run(&exe, &[])); assert_eq!( String::from_utf8_lossy(&dart.stdout), @@ -1025,7 +1053,7 @@ fn api_dart() { "import 'package:api/api.dart' as api;\n\nvoid main() => api.dropped();\n", ) .unwrap(); - let exe = dart_compile(&contract, &contract, &lib, &main); + let exe = dart_compile("api", &contract, &contract, &lib, &main); let output = dart_run(&exe, &[]).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -1036,9 +1064,9 @@ fn api_dart() { } /// `main` compiled into a kernel snapshot in `dir`, importing the generated Dart in -/// `package` as `package:api/api.dart`, whose native asset is the shared library `lib`: as -/// a package's build hook would bundle it, which `dart compile` has no flag for. -fn dart_compile(dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf { +/// `package` as `package:/.dart`, whose native asset is the shared library +/// `lib`: as a package's build hook would bundle it, which `dart compile` has no flag for. +fn dart_compile(name: &str, dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf { let sdk = std::fs::canonicalize(which("dart")) .unwrap() .parent() @@ -1050,7 +1078,7 @@ fn dart_compile(dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf std::fs::write( &config, format!( - "{{\"configVersion\": 2, \"packages\": [{{\"name\": \"api\", \"rootUri\": \"{root}/\", \"packageUri\": \"./\"}}]}}\n" + "{{\"configVersion\": 2, \"packages\": [{{\"name\": \"{name}\", \"rootUri\": \"{root}/\", \"packageUri\": \"./\"}}]}}\n" ), ) .unwrap(); @@ -1067,7 +1095,7 @@ fn dart_compile(dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf std::fs::write( &assets, format!( - "format-version: [1, 0, 0]\nnative-assets:\n {os}_{arch}:\n \"package:api/api.dart\": [absolute, {:?}]\n", + "format-version: [1, 0, 0]\nnative-assets:\n {os}_{arch}:\n \"package:{name}/{name}.dart\": [absolute, {:?}]\n", lib.display().to_string() ), ) @@ -1116,11 +1144,15 @@ fn url(dir: &Path) -> String { /// The static libraries of swift-collections' modules the generated Swift imports, which /// `just ffi swift-collections` builds beside their modules, where [`swiftc`] finds them. +/// Built once, since two tests running the recipe at once race on its checkout and build. fn swift_collections() -> Vec { - run(Command::new("just") - .arg("--justfile") - .arg(manifest_dir().join("justfile")) - .arg("swift-collections")); + static BUILT: OnceLock<()> = OnceLock::new(); + BUILT.get_or_init(|| { + run(Command::new("just") + .arg("--justfile") + .arg(manifest_dir().join("justfile")) + .arg("swift-collections")); + }); let build = collections_dir(); ["BasicContainers", "InternalCollectionsUtilities", "SpanPreview"] .iter() diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs new file mode 100644 index 000000000..a04c48eed --- /dev/null +++ b/ffi/tests/fixtures/shapes.rs @@ -0,0 +1,1588 @@ +//! Generated type shapes, round-tripped through every language: a seeded sampler builds +//! value types from what the exporter's IR holds, nested to a bounded depth, and emits a +//! crate of `make_N(seed)`, `echo_N(v)`, and `digest_N(&v)` per shape, with the shape in a +//! closure's parameter and return and a future's output besides. The census of that crate +//! says which items export, so a shape the exporter refuses by design fails the export, +//! not the test; each language's driver is a generated flat list of calls whose printed +//! digests must match the crate's own, run natively. + +use std::fmt::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use super::{cargo, manifest_dir, run, scratch}; + +/// The integer types of the IR's `Int`. +const INTS: [&str; 10] = ["u8", "u16", "u32", "u64", "usize", "i8", "i16", "i32", "i64", "isize"]; + +/// A value type the sampler builds, after the IR's `Type`. +#[derive(Clone, Debug)] +enum Shape { + Bool, + Int(&'static str), + NonZero(&'static str), + U128, + F32, + F64, + Char, + String, + /// `Vec` + Bytes, + /// `bytes::Bytes` + Buf, + Duration, + Array(usize), + PathBuf, + /// `SocketAddr`, `SocketAddrV4`, or `SocketAddrV6`. + SocketAddr(&'static str), + Range(&'static str), + Bound(&'static str), + /// `serde_json::Value` + Json, + /// `serde_json::Map` + JsonMap, + /// `url::Url` + Url, + Option(Box), + Vec(Box), + Set(Box), + Map(Box, Box), + Box(Box), + Tuple(Vec), + /// A struct of `pub` fields: a record. + Record(Vec), + /// An enum of a unit variant, a tuple variant, and a struct variant. + Enum(Box), + /// A struct of a private field: an object. + Object(Box), +} + +/// Each variant of the IR's `Type` (`src/ir.rs`), and how the shapes cover it: the +/// [`Shape`] that builds it, or why none does. A new variant fails [`ir_covered`] until it +/// is named here. +const COVERED: &[(&str, &str)] = &[ + ("Unit", "none: no value, so a function returning it returns nothing"), + ("Bool", "Bool"), + ("Int", "Int, of each width"), + ("NonZero", "NonZero, of each width"), + ("U128", "U128"), + ("F32", "F32"), + ("F64", "F64"), + ("Char", "Char"), + ("Str", "String, which digest_N and peek_N's closure borrow as &str"), + ("String", "String"), + ("Slice", "Bytes, which digest_N and peek_N's closure borrow as &[u8]"), + ("Vec", "Bytes"), + ("Bytes", "Buf"), + ("Duration", "Duration"), + ("Array", "Array"), + ("Path", "PathBuf, which digest_N and peek_N's closure borrow as &Path"), + ("PathBuf", "PathBuf"), + ("SocketAddr", "SocketAddr, of each family"), + ("List", "Vec, Set, Map, and JsonMap"), + ("Waker", "none: a waker holds no value"), + ("Future", "none: a foreign future is of (), which holds no value"), + ("Reader", "none: a foreign reader is lent for a call and holds no value"), + ("Option", "Option"), + ("Box", "Box"), + ("Ref", "every shape, which digest_N and peek_N's closure borrow"), + ("Record", "Record"), + ("Enum", "Enum"), + ("Object", "Object, and Json and Url, which the export maps to objects"), + ("Callback", "none: a foreign trait implementation holds no value"), + ("Closure", "the positions lent_N, peek_N, and made_N"), + ("Range", "Range"), + ("Bound", "Bound"), + ( + "Bounds", + "none: the form of an impl RangeBounds parameter, which no value has", + ), + ("Tuple", "Tuple"), + ("Const", "none: a const argument, which no value has"), +]; + +/// Every leaf, once for each integer type and socket address family. +fn leaves() -> Vec { + let mut leaves = vec![Shape::Bool]; + leaves.extend(INTS.map(Shape::Int)); + leaves.extend(INTS.map(Shape::NonZero)); + leaves.extend([ + Shape::U128, + Shape::F32, + Shape::F64, + Shape::Char, + Shape::String, + Shape::Bytes, + Shape::Buf, + Shape::Duration, + Shape::Array(1), + Shape::Array(16), + Shape::PathBuf, + Shape::SocketAddr("SocketAddr"), + Shape::SocketAddr("SocketAddrV4"), + Shape::SocketAddr("SocketAddrV6"), + Shape::Range("u32"), + Shape::Range("i64"), + Shape::Bound("u8"), + Shape::Bound("isize"), + Shape::Json, + Shape::JsonMap, + Shape::Url, + ]); + leaves +} + +/// splitmix64: a fixed seed samples the same shapes and values on every host. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} + +/// A shape of at most `depth` levels: a leaf, or a wrapper of shapes one level shallower. +fn sample(rng: &mut Rng, leaves: &[Shape], depth: u32) -> Shape { + if depth == 0 || rng.below(3) == 0 { + return leaves[rng.below(leaves.len())].clone(); + } + let wrapper = rng.below(10); + let mut inner = || Box::new(sample(rng, leaves, depth - 1)); + match wrapper { + 0 => Shape::Option(inner()), + 1 => Shape::Vec(inner()), + 2 => Shape::Box(inner()), + 3 => Shape::Enum(inner()), + 4 => Shape::Object(inner()), + 5 => { + let element = ordered(rng, leaves); + Shape::Set(Box::new(element)) + } + 6 => { + let key = ordered(rng, leaves); + Shape::Map(Box::new(key), Box::new(sample(rng, leaves, depth - 1))) + } + 7 => { + let n = 1 + rng.below(3); + Shape::Tuple((0..n).map(|_| sample(rng, leaves, depth - 1)).collect()) + } + _ => { + let n = 1 + rng.below(3); + Shape::Record((0..n).map(|_| sample(rng, leaves, depth - 1)).collect()) + } + } +} + +/// A leaf that is `Ord`, to key a set or a map. +fn ordered(rng: &mut Rng, leaves: &[Shape]) -> Shape { + loop { + let leaf = &leaves[rng.below(leaves.len())]; + if is_ord(leaf) { + return leaf.clone(); + } + } +} + +fn is_ord(shape: &Shape) -> bool { + !matches!( + shape, + Shape::F32 | Shape::F64 | Shape::Json | Shape::JsonMap | Shape::Bound(_) | Shape::Range(_) + ) +} + +/// The shapes a run covers: each leaf alone, each leaf in each wrapper, then `extra` +/// sampled to depth 3 from a fixed seed. +fn shapes(extra: usize) -> Vec { + let leaves = leaves(); + let mut shapes = leaves.clone(); + let mut rng = Rng(0x5eed); + for leaf in &leaves { + let inner = || Box::new(leaf.clone()); + shapes.extend([ + Shape::Option(inner()), + Shape::Vec(inner()), + Shape::Box(inner()), + Shape::Tuple(vec![leaf.clone(), Shape::Bool]), + Shape::Record(vec![leaf.clone()]), + Shape::Enum(inner()), + Shape::Object(inner()), + ]); + if is_ord(leaf) { + shapes.push(Shape::Set(inner())); + shapes.push(Shape::Map(inner(), Box::new(Shape::String))); + } + } + shapes.extend((0..extra).map(|_| sample(&mut rng, &leaves, 3))); + shapes +} + +/// The gate's subset: each leaf alone, and each wrapper around a buffer, a +/// record field, and an object. +fn gate() -> Vec { + let mut shapes = leaves(); + for inner in [Shape::String, Shape::Int("i16"), Shape::Object(Box::new(Shape::U128))] { + let boxed = || Box::new(inner.clone()); + shapes.extend([ + Shape::Option(boxed()), + Shape::Vec(boxed()), + Shape::Box(boxed()), + Shape::Tuple(vec![inner.clone(), Shape::F64]), + Shape::Record(vec![inner.clone(), Shape::Char]), + Shape::Enum(boxed()), + ]); + } + shapes.push(Shape::Map(Box::new(Shape::String), Box::new(Shape::Bytes))); + shapes.push(Shape::Set(Box::new(Shape::Int("u64")))); + shapes +} + +/// The Rust source of the shapes' crate. +struct Emit { + /// Each record, enum, and object the shapes define. + items: String, + /// How many it has defined, which names the next. + defined: usize, +} + +impl Emit { + /// The Rust type of `shape`, and an expression making one from `r`, a `&mut Rng`. + fn shape(&mut self, shape: &Shape) -> (String, String) { + match shape { + Shape::Bool => ("bool".into(), "r.next() % 2 == 0".into()), + Shape::Int(int) => (int.to_string(), format!("r.int() as {int}")), + Shape::NonZero(int) => ( + format!("std::num::NonZero<{int}>"), + format!("std::num::NonZero::new(r.int() as {int}).unwrap_or(std::num::NonZero::<{int}>::MAX)"), + ), + Shape::U128 => ("u128".into(), "(r.int() as u128) << 64 | r.int() as u128".into()), + Shape::F32 => ("f32".into(), "r.f32()".into()), + Shape::F64 => ("f64".into(), "r.f64()".into()), + Shape::Char => ("char".into(), "r.char()".into()), + Shape::String => ("String".into(), "r.string()".into()), + Shape::Bytes => ("Vec".into(), "r.bytes()".into()), + Shape::Buf => ("bytes::Bytes".into(), "bytes::Bytes::from(r.bytes())".into()), + Shape::Duration => ("std::time::Duration".into(), "r.duration()".into()), + Shape::Array(n) => (format!("[u8; {n}]"), "std::array::from_fn(|_| r.next() as u8)".into()), + Shape::PathBuf => ( + "std::path::PathBuf".into(), + "std::path::PathBuf::from(r.string())".into(), + ), + Shape::SocketAddr(family) => (format!("std::net::{family}"), format!("r.{}()", family.to_lowercase())), + Shape::Range(int) => ( + format!("std::ops::Range<{int}>"), + format!("(r.int() as {int})..(r.int() as {int})"), + ), + Shape::Bound(int) => ( + format!("std::ops::Bound<{int}>"), + format!( + "match r.below(3) {{ 0 => std::ops::Bound::Included(r.int() as {int}), 1 => std::ops::Bound::Excluded(r.int() as {int}), _ => std::ops::Bound::Unbounded }}" + ), + ), + Shape::Json => ("serde_json::Value".into(), "r.json(2)".into()), + Shape::JsonMap => ( + "serde_json::Map".into(), + "(0..r.below(3)).map(|_| (r.string(), r.json(1))).collect()".into(), + ), + Shape::Url => ("url::Url".into(), "r.url()".into()), + Shape::Option(inner) => { + let (ty, make) = self.shape(inner); + ( + format!("Option<{ty}>"), + format!("match r.below(4) {{ 0 => None, _ => Some({make}) }}"), + ) + } + Shape::Vec(inner) => { + let (ty, make) = self.shape(inner); + ( + format!("Vec<{ty}>"), + format!("(0..r.below(4)).map(|_| {make}).collect()"), + ) + } + Shape::Set(inner) => { + let (ty, make) = self.shape(inner); + ( + format!("std::collections::BTreeSet<{ty}>"), + format!("(0..r.below(4)).map(|_| {make}).collect()"), + ) + } + Shape::Map(key, value) => { + let (key, make_key) = self.shape(key); + let (value, make_value) = self.shape(value); + ( + format!("std::collections::BTreeMap<{key}, {value}>"), + format!("(0..r.below(4)).map(|_| ({make_key}, {make_value})).collect()"), + ) + } + Shape::Box(inner) => { + let (ty, make) = self.shape(inner); + (format!("Box<{ty}>"), format!("Box::new({make})")) + } + Shape::Tuple(elements) => { + let (types, makes): (Vec<_>, Vec<_>) = elements.iter().map(|e| self.shape(e)).unzip(); + (format!("({},)", types.join(", ")), format!("({},)", makes.join(", "))) + } + Shape::Record(fields) => { + let (types, makes): (Vec<_>, Vec<_>) = fields.iter().map(|f| self.shape(f)).unzip(); + let name = self.name("Record"); + let _ = writeln!(self.items, "#[derive(Debug)]\npub struct {name} {{"); + for (i, ty) in types.iter().enumerate() { + let _ = writeln!(self.items, "\tpub f{i}: {ty},"); + } + let _ = writeln!(self.items, "}}\n"); + let fields: Vec = makes.iter().enumerate().map(|(i, m)| format!("f{i}: {m}")).collect(); + (name.clone(), format!("{name} {{ {} }}", fields.join(", "))) + } + Shape::Enum(inner) => { + let (ty, make) = self.shape(inner); + let name = self.name("Enum"); + let _ = writeln!( + self.items, + "#[derive(Debug)]\npub enum {name} {{\n\tUnit,\n\tTuple({ty}),\n\tStruct {{ value: {ty} }},\n}}\n" + ); + ( + name.clone(), + format!( + "match r.below(3) {{ 0 => {name}::Unit, 1 => {name}::Tuple({make}), _ => {name}::Struct {{ value: {make} }} }}" + ), + ) + } + Shape::Object(inner) => { + let (ty, make) = self.shape(inner); + let name = self.name("Object"); + let _ = writeln!(self.items, "#[derive(Debug)]\npub struct {name}({ty});\n"); + (name.clone(), format!("{name}({make})")) + } + } + } + + fn name(&mut self, kind: &str) -> String { + self.defined += 1; + format!("{kind}{}", self.defined) + } +} + +/// The borrowed form `digest_N` takes of a value of `ty`. +fn borrowed(shape: &Shape, ty: &str) -> String { + match shape { + Shape::String => "&str".into(), + Shape::Bytes => "&[u8]".into(), + Shape::PathBuf => "&std::path::Path".into(), + Shape::Vec(_) => format!("&[{}]", &ty[4..ty.len() - 1]), + _ => format!("&{ty}"), + } +} + +/// What `make_N` draws values from, and `digest`, which every `digest_N` calls. +const RUNTIME: &str = r#" +/// splitmix64, with edge values drawn often. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng(seed) + } + + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + + /// Bits for an integer of any width, each edge of every width among them. + fn int(&mut self) -> u64 { + match self.below(8) { + 0 => 0, + 1 => u64::MAX, + // The sign bit of a width, and the most a signed one holds. + 2 => 1 << ((8 << self.below(4)) - 1), + 3 => (1 << ((8 << self.below(4)) - 1)) - 1, + _ => self.next(), + } + } + + fn f32(&mut self) -> f32 { + let f = f32::from_bits(self.int() as u32); + if f.is_nan() { -0.0 } else { f } + } + + fn f64(&mut self) -> f64 { + let f = f64::from_bits(self.int()); + if f.is_nan() { f64::INFINITY } else { f } + } + + fn char(&mut self) -> char { + char::from_u32(self.next() as u32 % 0x11_0000).unwrap_or('\u{10ffff}') + } + + fn string(&mut self) -> String { + const PARTS: [&str; 6] = ["a", "é", "😀", "\0", "Z", "語"]; + (0..self.below(5)).map(|_| PARTS[self.below(6) as usize]).collect() + } + + fn bytes(&mut self) -> Vec { + (0..self.below(6)).map(|_| self.next() as u8).collect() + } + + /// Up to 2^33 seconds, past 136 years: Go's `time.Duration` holds 292. + fn duration(&mut self) -> std::time::Duration { + std::time::Duration::new(self.next() >> 31, (self.next() % 1_000_000_000) as u32) + } + + fn socketaddrv4(&mut self) -> std::net::SocketAddrV4 { + std::net::SocketAddrV4::new((self.next() as u32).into(), self.next() as u16) + } + + fn socketaddrv6(&mut self) -> std::net::SocketAddrV6 { + let ip = ((self.next() as u128) << 64 | self.next() as u128).into(); + std::net::SocketAddrV6::new(ip, self.next() as u16, self.next() as u32, self.next() as u32) + } + + fn socketaddr(&mut self) -> std::net::SocketAddr { + match self.below(2) { + 0 => self.socketaddrv4().into(), + _ => self.socketaddrv6().into(), + } + } + + fn json(&mut self, depth: u32) -> serde_json::Value { + match self.below(if depth == 0 { 4 } else { 6 }) { + 0 => serde_json::Value::Null, + 1 => (self.next() % 2 == 0).into(), + 2 => (self.int() as i64).into(), + 3 => self.string().into(), + 4 => (0..self.below(3)).map(|_| self.json(depth - 1)).collect(), + _ => (0..self.below(3)).map(|_| (self.string(), self.json(depth - 1))).collect(), + } + } + + fn url(&mut self) -> url::Url { + let text = format!("https://h{}.test:{}/p/{}?q={}", self.below(100), self.next() as u16, self.below(10), self.next()); + url::Url::parse(&text).unwrap() + } +} + +/// FNV-1a of the value's `Debug`, which prints every bit of each shape's value. +fn digest(value: &dyn std::fmt::Debug) -> u64 { + format!("{value:?}").bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| (h ^ b as u64).wrapping_mul(0x100_0000_01b3)) +} +"#; + +/// The functions each shape `N` exports, which the census may refuse one by one. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Item { + Make, + Echo, + Digest, + /// `lent_N(seed, f)`: calls `f` with the made value, by value. + Lent, + /// `peek_N(seed, f)`: calls `f` with the made value, borrowed as `digest_N` takes it. + Peek, + /// `made_N(seed, f)`: digests what `f` returns for the seed. + Made, + /// `later_N(v)`: a future of `v`. + Later, +} + +impl Item { + fn name(self, n: usize) -> String { + let item = match self { + Item::Make => "make", + Item::Echo => "echo", + Item::Digest => "digest", + Item::Lent => "lent", + Item::Peek => "peek", + Item::Made => "made", + Item::Later => "later", + }; + format!("{item}_{n}") + } +} + +/// A line of the script each driver runs, printing `label value`. +#[derive(Clone, Copy, Debug)] +enum Call { + /// `digest_N(echo_N(make_N(seed)))` + Echo(usize, u64), + /// `lent_N(seed, digest_N)` + Lent(usize, u64), + /// `peek_N(seed, digest_N)` + Peek(usize, u64), + /// `made_N(seed, make_N)` + Made(usize, u64), + /// `digest_N(await later_N(make_N(seed)))` + Later(usize, u64), +} + +impl Call { + /// The `N` of its shape. + fn shape(self) -> usize { + let (Call::Echo(n, _) | Call::Lent(n, _) | Call::Peek(n, _) | Call::Made(n, _) | Call::Later(n, _)) = self; + n + } + + fn label(self) -> String { + match self { + Call::Echo(n, seed) => format!("echo {n} {seed}"), + Call::Lent(n, seed) => format!("lent {n} {seed}"), + Call::Peek(n, seed) => format!("peek {n} {seed}"), + Call::Made(n, seed) => format!("made {n} {seed}"), + Call::Later(n, seed) => format!("later {n} {seed}"), + } + } +} + +/// The seeds each shape is made from. +const SEEDS: [u64; 3] = [0, 1, 0xdead_beef]; + +/// The shapes' crate, as the census left it: which shapes export, and the script. +struct Crate { + dir: PathBuf, + shapes: Vec, + /// Each shape's Rust type, by `N`. + types: Vec, + calls: Vec, + /// The digests the script prints natively. + transcript: String, +} + +/// Write the crate of `shapes` named `name` in the scratch directory, ask the exporter what +/// exports, and run its script natively. +fn emit(name: &str, shapes: &[Shape]) -> Crate { + let dir = scratch().join(name); + let _ = std::fs::remove_dir_all(dir.join("src")); + let mut emit = Emit { + items: String::new(), + defined: 0, + }; + let mut functions = String::new(); + let mut types = Vec::new(); + for (n, shape) in shapes.iter().enumerate() { + let (ty, make) = emit.shape(shape); + let lent = borrowed(shape, &ty); + let _ = write!( + functions, + "pub fn make_{n}(seed: u64) -> {ty} {{\n\tlet r = &mut Rng::new(seed);\n\t{make}\n}}\n\n\ + pub fn echo_{n}(v: {ty}) -> {ty} {{\n\tv\n}}\n\n\ + pub fn digest_{n}(v: {lent}) -> u64 {{\n\tdigest(&v)\n}}\n\n\ + pub fn lent_{n}(seed: u64, f: impl FnOnce({ty}) -> u64) -> u64 {{\n\tf(make_{n}(seed))\n}}\n\n\ + pub fn peek_{n}(seed: u64, f: impl FnOnce({lent}) -> u64) -> u64 {{\n\tf(&make_{n}(seed))\n}}\n\n\ + pub fn made_{n}(seed: u64, f: impl FnOnce(u64) -> {ty}) -> u64 {{\n\tdigest(&f(seed))\n}}\n\n\ + pub async fn later_{n}(v: {ty}) -> {ty} {{\n\tv\n}}\n\n" + ); + types.push(ty); + } + let lib = format!( + "//! Generated by `tests/fixtures/shapes.rs`: each shape's `make`, `echo`, and `digest`.\n\ + #![allow(clippy::all, dead_code, reason = \"generated\")]\n{RUNTIME}\n{}{functions}", + emit.items + ); + let manifest = format!( + "[package]\nname = \"{name}\"\nversion = \"0.0.0\"\nedition = \"2024\"\npublish = false\n\n\ + [dependencies]\nbytes = \"1\"\nserde_json = \"1\"\nurl = \"2\"\n\n[workspace]\n" + ); + super::write_all( + &dir, + &[("Cargo.toml", &manifest), ("src/lib.rs", &lib), ("ffi.toml", "")], + ); + // The api fixture's versions, resolved offline. + std::fs::copy(manifest_dir().join("Cargo.lock"), dir.join("Cargo.lock")).unwrap(); + + let refused = refused(&dir); + let exported = |item: Item, n: usize| !refused.iter().any(|path| *path == item.name(n)); + let mut excludes: Vec<&String> = refused.iter().collect(); + excludes.sort(); + let config = excludes + .iter() + .map(|path| format!(" \"{path}\",\n")) + .collect::(); + std::fs::write(dir.join("ffi.toml"), format!("exclude = [\n{config}]\n")).unwrap(); + + let mut calls = Vec::new(); + for n in 0..shapes.len() { + if ![Item::Make, Item::Digest].iter().all(|item| exported(*item, n)) { + continue; + } + for seed in SEEDS { + if exported(Item::Echo, n) { + calls.push(Call::Echo(n, seed)); + } + if exported(Item::Lent, n) { + calls.push(Call::Lent(n, seed)); + } + if exported(Item::Peek, n) { + calls.push(Call::Peek(n, seed)); + } + if exported(Item::Made, n) { + calls.push(Call::Made(n, seed)); + } + if exported(Item::Later, n) { + calls.push(Call::Later(n, seed)); + } + } + } + + let mut script = format!("//! The shapes' script, run natively.\n\nuse {name}::*;\n{NATIVE}"); + for call in &calls { + let value = match *call { + Call::Echo(n, seed) => format!("digest_{n}(&echo_{n}(make_{n}({seed})))"), + Call::Lent(n, seed) => format!("lent_{n}({seed}, |v| digest_{n}(&v))"), + Call::Peek(n, seed) => format!("peek_{n}({seed}, digest_{n})"), + Call::Made(n, seed) => format!("made_{n}({seed}, make_{n})"), + Call::Later(n, seed) => format!("digest_{n}(&later(later_{n}(make_{n}({seed}))))"), + }; + let _ = writeln!(script, "\tprintln!(\"{} {{}}\", {value});", call.label()); + } + script.push_str("}\n"); + super::write_all(&dir, &[("examples/transcript.rs", &script)]); + let transcript = run(cargo() + .args(["run", "--quiet", "--example", "transcript"]) + .env("CARGO_TARGET_DIR", dir.join("build")) + .current_dir(&dir)); + Crate { + dir, + shapes: shapes.to_vec(), + types, + calls, + transcript: String::from_utf8_lossy(&transcript.stdout).into_owned(), + } +} + +/// The native script's head: `later` polls a future that is ready at once. +const NATIVE: &str = " +fn later(f: impl Future) -> T { + match std::pin::pin!(f).poll(&mut std::task::Context::from_waker(std::task::Waker::noop())) { + std::task::Poll::Ready(v) => v, + std::task::Poll::Pending => unreachable!(), + } +} + +fn main() { +"; + +/// The paths the census of the crate in `dir` refuses: its config that exports what maps. +fn refused(dir: &Path) -> Vec { + let census = dir.join("census"); + run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(dir.join("ffi.toml")) + .arg("--census") + .arg(&census) + .env("CARGO_TARGET_DIR", dir.join("build"))); + let name = dir.file_name().unwrap().to_string_lossy(); + let report = std::fs::read_to_string(census.join(format!("{name}.md"))).unwrap(); + let Some((_, config)) = report.split_once("## The config that exports what maps") else { + return Vec::new(); + }; + config + .lines() + .skip_while(|line| *line != "exclude = [") + .skip(1) + .take_while(|line| *line != "]") + .map(|line| line.trim().trim_end_matches(',').trim_matches('"').to_string()) + .collect() +} + +/// Each variant of the IR's `Type`, from its source. +fn ir_variants() -> Vec { + let ir = std::fs::read_to_string(manifest_dir().join("src").join("ir.rs")).unwrap(); + let body = ir.split_once("pub enum Type {").expect("ir.rs declares Type").1; + let body = &body[..body.find("\n}").unwrap()]; + body.lines() + .map(str::trim) + .filter(|line| !line.starts_with("//") && !line.is_empty()) + .map(|line| line.split(|c: char| !c.is_alphanumeric()).next().unwrap().to_string()) + .collect() +} + +/// Every variant of the IR's `Type` is sampled or named as unsampled, once. +#[test] +fn ir_covered() { + let variants = ir_variants(); + let covered: Vec<&str> = COVERED.iter().map(|(name, _)| *name).collect(); + assert_eq!( + variants, covered, + "COVERED in tests/fixtures/shapes.rs names each variant of the IR's Type, in order" + ); +} + +/// The shapes' crate, exported and built once for each language's test. +struct Built { + krate: Crate, + /// The export. + out: PathBuf, + /// Where cargo built the static library and the shared one Kotlin and Dart load. + lib: PathBuf, + /// What a program linking the static library links besides, as rustc prints it. + native: Vec, +} + +/// The shared library of the crate `name`, which Python, Kotlin, and Dart load. +fn shared(name: &str) -> String { + match (cfg!(windows), cfg!(target_os = "macos")) { + (true, _) => format!("{name}.dll"), + (_, true) => format!("lib{name}.dylib"), + _ => format!("lib{name}.so"), + } +} + +/// Emit, census, export, and build the crate of `shapes` named `name`. +fn build(name: &str, shapes: &[Shape]) -> Built { + let krate = emit(name, shapes); + let out = krate.dir.join("out"); + let _ = std::fs::remove_dir_all(&out); + run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(krate.dir.join("ffi.toml")) + .arg("--out") + .arg(&out) + .env("CARGO_TARGET_DIR", krate.dir.join("build"))); + let target = krate.dir.join("glue"); + let build = |feature: &str| { + run(cargo() + .args(["build", "--quiet", "--locked", "--manifest-path"]) + .arg(out.join("Cargo.toml")) + .arg("--target-dir") + .arg(&target) + .args(["--features", feature]) + .env("RUSTFLAGS", "-D warnings --print=native-static-libs")) + }; + // As the api fixture's: the CPython extension builds apart and moves beside the Python. + build("python"); + let lib = shared(name); + std::fs::rename(target.join("debug").join(&lib), out.join("python").join(&lib)).unwrap(); + let built = build("kotlin,dart"); + let native = String::from_utf8_lossy(&built.stderr) + .lines() + .find_map(|line| line.strip_prefix("note: native-static-libs: ")) + .expect("rustc prints the static library's native libraries") + .split_whitespace() + .map(str::to_string) + .collect(); + Built { + krate, + out, + lib: target.join("debug"), + native, + } +} + +impl Built { + fn name(&self) -> String { + self.krate.dir.file_name().unwrap().to_string_lossy().into_owned() + } + + fn staticlib(&self) -> PathBuf { + let name = self.name(); + self.lib.join(match cfg!(target_env = "msvc") { + true => format!("{name}.lib"), + false => format!("lib{name}.a"), + }) + } + + /// A directory of its own for `language`'s driver. + fn dir(&self, language: &str) -> PathBuf { + let dir = self.krate.dir.join("drivers").join(language); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// The exported shapes `language` gets wrong today, by [`KNOWN`], each entry of which must + /// match one. + fn known(&self, language: &str) -> Vec { + let mut known = Vec::new(); + for Known { what, skips, quest, .. } in KNOWN.iter().filter(|k| k.language == language) { + let matched: Vec = (0..self.krate.shapes.len()) + .filter(|&n| skips(&self.krate.shapes[n]) && self.krate.calls.iter().any(|call| call.shape() == n)) + .collect(); + // The gate's subset may hold none of an entry's shapes; the nightly's holds each. + assert!( + !matched.is_empty() || self.name() != "sampled", + "KNOWN's {language} entry for {what} ({quest}) matches no shape: remove it" + ); + known.extend(matched); + } + known + } + + /// Asserts `language`'s driver printed the native transcript but for the shapes it gets + /// wrong today, naming each line that differs by its shape's Rust type. + fn check(&self, language: &str, known: &[usize], output: &std::process::Output) { + let got = String::from_utf8_lossy(&output.stdout); + let mut wrong = String::new(); + let expected: Vec<&str> = (self.krate.transcript.lines().zip(&self.krate.calls)) + .filter(|(_, call)| !known.contains(&call.shape())) + .map(|(line, _)| line) + .collect(); + let lines: Vec<&str> = got.lines().collect(); + for (i, want) in expected.iter().enumerate() { + let line = lines.get(i).copied().unwrap_or(""); + if line != *want { + let n: usize = want.split(' ').nth(1).unwrap().parse().unwrap(); + let _ = writeln!(wrong, " {want} ({}), got {line}", self.krate.types[n]); + } + } + assert!( + wrong.is_empty() && lines.len() == expected.len(), + "{language}'s round trips differ from native Rust's in {} of {} lines, {} printed:\n{wrong}{}", + wrong.lines().count(), + expected.len(), + lines.len(), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +/// Whether `shape` or a shape within it meets `f`. +fn within(shape: &Shape, f: &dyn Fn(&Shape) -> bool) -> bool { + f(shape) + || match shape { + Shape::Option(inner) + | Shape::Vec(inner) + | Shape::Set(inner) + | Shape::Box(inner) + | Shape::Enum(inner) + | Shape::Object(inner) => within(inner, f), + Shape::Map(key, value) => within(key, f) || within(value, f), + Shape::Tuple(elements) | Shape::Record(elements) => elements.iter().any(|e| within(e, f)), + _ => false, + } +} + +fn duration(shape: &Shape) -> bool { + within(shape, &|s| matches!(s, Shape::Duration)) +} + +/// A `Vec` or `Bytes`: a `Vec` of `u8` ints is one, as Rust types it alike. +fn is_bytes(shape: &Shape) -> bool { + match shape { + Shape::Bytes | Shape::Buf => true, + Shape::Vec(inner) => matches!(**inner, Shape::Int("u8")), + _ => false, + } +} + +fn bytes(shape: &Shape) -> bool { + within(shape, &is_bytes) +} + +fn ipv6(shape: &Shape) -> bool { + within(shape, &|s| { + matches!(s, Shape::SocketAddr("SocketAddr" | "SocketAddrV6")) + }) +} + +/// A collection whose items are bytes or tuples holding them, which C++ takes as views. +fn viewed_items(shape: &Shape) -> bool { + let buf = |s: &Shape| matches!(s, Shape::String | Shape::PathBuf) || is_bytes(s); + let item = |s: &Shape| buf(s) || matches!(s, Shape::Tuple(elements) if elements.iter().any(buf)); + within(shape, &|s| match s { + Shape::Vec(inner) | Shape::Set(inner) => item(inner), + Shape::Map(key, value) => buf(key) || buf(value), + _ => false, + }) +} + +/// A tuple of one element, which Kotlin returns as its element but takes whole. +fn single(shape: &Shape) -> bool { + matches!(shape, Shape::Tuple(elements) if elements.len() == 1) +} + +/// A round trip a binding gets wrong today. +struct Known { + language: &'static str, + what: &'static str, + /// Whether a shape holds what it gets wrong, which the language's driver skips. + skips: fn(&Shape) -> bool, + /// The quest that fixes it. + quest: &'static str, +} + +/// The round trips a binding gets wrong today, each the quest's to fix: the language, what +/// it gets wrong, the shapes it skips for it, and the quest. The nightly fails on an entry +/// matching no exported shape, but its shapes stay skipped after a fix, so the quest that +/// fixes one deletes its entry. +const KNOWN: &[Known] = &[ + Known { + language: "python", + what: "a Duration's nanoseconds", + skips: duration, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "dart", + what: "a Duration's nanoseconds", + skips: duration, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "kotlin", + what: "a Duration's nanoseconds past 146 years", + skips: duration, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "kotlin", + what: "an IPv6 flow label", + skips: ipv6, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "kotlin", + what: "returned bytes passed back", + skips: bytes, + quest: "/quest/m1/ffi-export/owned-bufs.md", + }, + Known { + language: "kotlin", + what: "a returned tuple of one passed back", + skips: single, + quest: "/quest/m1/ffi-export/round-trips.md", + }, + Known { + language: "go", + what: "an IPv6 flow label", + skips: ipv6, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "go", + what: "returned bytes passed back", + skips: bytes, + quest: "/quest/m1/ffi-export/owned-bufs.md", + }, + Known { + language: "cpp", + what: "a returned collection of bytes passed back", + skips: viewed_items, + quest: "/quest/m1/ffi-export/round-trips.md", + }, +]; + +/// A function the C header declares: each parameter's type and name. +struct Proto { + params: Vec<(String, String)>, +} + +/// The prototype of `function` in the C header. +fn prototype(header: &str, function: &str) -> Proto { + let declared = format!("{function}("); + let at = header + .match_indices(&declared) + .map(|(at, _)| at) + .find(|&at| header[..at].ends_with([' ', '*'])) + .unwrap_or_else(|| panic!("the header declares {function}")); + let rest = &header[at + declared.len()..]; + let list = &rest[..rest.find(");").unwrap()]; + // Split at the commas outside a function pointer's parentheses. + let (mut params, mut depth, mut from) = (Vec::new(), 0, 0); + for (i, c) in list.char_indices() { + match c { + '(' => depth += 1, + ')' => depth -= 1, + ',' if depth == 0 => { + params.push(&list[from..i]); + from = i + 1; + } + _ => {} + } + } + params.push(&list[from..]); + let params = params + .into_iter() + .map(|param| { + let param = param.trim(); + match param.find("(*") { + // A function pointer: `R (*f)(void *data, P)`, named `f`. + Some(at) => ( + param.to_string(), + param[at + 2..at + 2 + param[at + 2..].find(')').unwrap()].to_string(), + ), + None => { + let split = param.rfind([' ', '*']).unwrap() + 1; + (param[..split].trim().to_string(), param[split..].to_string()) + } + } + }) + .collect(); + Proto { params } +} + +impl Proto { + /// The type of the parameter named `name`. + fn param(&self, name: &str) -> &str { + &self.params.iter().find(|(_, n)| n == name).unwrap().0 + } + + /// Statements calling it with `args` that declare `var` of type `ty`: its return, or + /// what it writes through its out pointers, a returned tuple's gathered into the struct + /// `ty`. A poll's call must return `ready`. + fn call(&self, function: &str, args: &[String], ty: &str, var: &str, ready: Option<&str>) -> String { + let args = args.join(", "); + let outs = &self.params[self.params.len() - self.outs()..]; + if outs.is_empty() { + return format!("\t\t{ty} {var} = {function}({args});\n"); + } + let mut out = String::new(); + let mut pointers = Vec::new(); + let mut values = Vec::new(); + for (k, (pointer, _)) in outs.iter().enumerate() { + let _ = writeln!(out, "\t\t{} {var}_{k};", pointer.strip_suffix('*').unwrap().trim()); + pointers.push(format!("&{var}_{k}")); + values.push(format!("{var}_{k}")); + } + let call = format!("{function}({args}, {})", pointers.join(", ")); + let _ = match ready { + Some(ready) => writeln!(out, "\t\tif ({call} != {ready}) abort();"), + None => writeln!(out, "\t\t{call};"), + }; + // `out` is the value; `out0` on are a returned tuple's elements, of one or more. + let _ = match outs[0].1 == "out" { + true => writeln!(out, "\t\t{ty} {var} = {var}_0;"), + false => writeln!(out, "\t\t{ty} {var} = ({ty}){{{}}};", values.join(", ")), + }; + out + } + + /// How many out pointers it writes its output through: `out`, or `out0` on for a tuple. + fn outs(&self) -> usize { + let out = |name: &str| name == "out" || name.strip_prefix("out").is_some_and(|k| k.parse::().is_ok()); + self.params.iter().filter(|(_, name)| out(name)).count() + } + + /// What a function pointer parameter `(*name)` returns and takes after its data. + fn pointer(&self, name: &str) -> (String, String) { + let ty = self.param(name); + let (ret, rest) = ty.split_once(&format!(" (*{name})(")).unwrap(); + let param = rest.strip_prefix("void *data, ").unwrap().strip_suffix(')').unwrap(); + (ret.to_string(), param.to_string()) + } +} + +/// The C driver: each call's value in a block of its own, typed by the header's prototypes. +fn c(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let header = std::fs::read_to_string(built.out.join("include").join(format!("{name}.h"))).unwrap(); + let proto = |item: Item, n: usize| prototype(&header, &format!("{name}_{}", item.name(n))); + let mut closures = String::new(); + let mut body = String::new(); + let mut defined = std::collections::BTreeSet::new(); + for call in calls { + let label = call.label(); + let (Call::Echo(n, seed) + | Call::Lent(n, seed) + | Call::Peek(n, seed) + | Call::Made(n, seed) + | Call::Later(n, seed)) = *call; + let make = proto(Item::Make, n); + let digest = proto(Item::Digest, n); + let digested = digest.param("v").to_string(); + let fname = |item: Item| format!("{name}_{}", item.name(n)); + let mut block = String::new(); + match *call { + Call::Echo(..) => { + let echo = proto(Item::Echo, n); + block += &make.call(&fname(Item::Make), &[format!("{seed}u")], echo.param("v"), "a", None); + block += &echo.call(&fname(Item::Echo), &["a".into()], &digested, "b", None); + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", + fname(Item::Digest) + ); + } + Call::Lent(..) | Call::Peek(..) => { + let item = if matches!(call, Call::Lent(..)) { + Item::Lent + } else { + Item::Peek + }; + if defined.insert(item.name(n)) { + let (_, param) = proto(item, n).pointer("f"); + let _ = writeln!( + closures, + "static uint64_t {}(void *data, {param} v) {{\n\t(void)data;\n\treturn {}(v);\n}}\n", + item.name(n), + fname(Item::Digest) + ); + } + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}({seed}u, {}, NULL));", + fname(item), + item.name(n) + ); + } + Call::Made(..) => { + if defined.insert(Item::Made.name(n)) { + let (ret, _) = proto(Item::Made, n).pointer("f"); + let made = make.call(&fname(Item::Make), &["seed".into()], &ret, "v", None); + // A buffer C hands Rust is one buf_copy made. + let value = match ret == format!("{name}_buf") { + true => format!("{name}_buf_copy(v)"), + false => "v".into(), + }; + let _ = writeln!( + closures, + "static {ret} {}(void *data, uint64_t seed) {{\n\t(void)data;\n\t{{\n{made}\t\treturn {value};\n\t}}\n}}\n", + Item::Made.name(n) + ); + } + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}({seed}u, {}, NULL));", + fname(Item::Made), + Item::Made.name(n) + ); + } + Call::Later(..) => { + let later = proto(Item::Later, n); + let future = format!("{}_future", fname(Item::Later)); + let poll = prototype(&header, &format!("{}_poll", fname(Item::Later))); + block += &make.call(&fname(Item::Make), &[format!("{seed}u")], later.param("v"), "a", None); + let _ = writeln!(block, "\t\t{future} *f = {}(a);", fname(Item::Later)); + // Ready at once: an async fn that awaits nothing. + let ready = format!("{}_POLL_READY", name.to_uppercase()); + let polled = poll.call( + &format!("{}_poll", fname(Item::Later)), + &["f".into(), format!("({name}_waker){{0}}")], + &digested, + "b", + Some(&ready), + ); + block += &polled; + let _ = writeln!(block, "\t\t{future}_free(f);"); + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", + fname(Item::Digest) + ); + } + } + let _ = writeln!(body, "\t{{\n{block}\t}}"); + } + format!( + "#include \n#include \n#include \n\n#include \"{name}.h\"\n\n{closures}int main(void) {{\n{body}\treturn 0;\n}}\n" + ) +} + +/// The C++ driver, over the header's own types and futures. +fn cpp(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("{name}::digest_{n}({name}::echo_{n}({name}::make_{n}({seed}u)))"), + Call::Lent(n, seed) => { + format!("{name}::lent_{n}({seed}u, [](auto v) {{ return {name}::digest_{n}(std::move(v)); }})") + } + Call::Peek(n, seed) => { + format!("{name}::peek_{n}({seed}u, [](auto &&v) {{ return {name}::digest_{n}(v); }})") + } + Call::Made(n, seed) => { + format!("{name}::made_{n}({seed}u, [](std::uint64_t s) {{ return {name}::make_{n}(s); }})") + } + Call::Later(n, seed) => format!("{name}::digest_{n}({name}::later_{n}({name}::make_{n}({seed}u)).get())"), + }; + let _ = writeln!(body, "\tstd::cout << \"{} \" << {value} << '\\n';", call.label()); + } + format!( + "#include \n#include \n#include \n\n#include \"{name}.hpp\"\n\nint main() {{\n{body}}}\n" + ) +} + +/// Each shape's calls in a function of its own, so no function grows past a compiler's +/// limit: `(n, lines)`. +fn by_shape(calls: &[Call], line: impl Fn(Call) -> String) -> Vec<(usize, String)> { + let mut shapes: Vec<(usize, String)> = Vec::new(); + for call in calls { + let (Call::Echo(n, _) | Call::Lent(n, _) | Call::Peek(n, _) | Call::Made(n, _) | Call::Later(n, _)) = *call; + if shapes.last().is_none_or(|(last, _)| *last != n) { + shapes.push((n, String::new())); + } + let _ = writeln!(shapes.last_mut().unwrap().1, "{}", line(*call)); + } + shapes +} + +/// The Kotlin driver, on the JVM with kotlinx-coroutines. +fn kotlin(built: &Built, calls: &[Call]) -> String { + let shapes = by_shape(calls, |call| { + let value = match call { + Call::Echo(n, seed) => format!("digest{n}(echo{n}(make{n}({seed}uL)))"), + Call::Lent(n, seed) => format!("lent{n}({seed}uL) {{ digest{n}(it) }}"), + Call::Peek(n, seed) => format!("peek{n}({seed}uL) {{ digest{n}(it) }}"), + Call::Made(n, seed) => format!("made{n}({seed}uL) {{ make{n}(it) }}"), + Call::Later(n, seed) => format!("digest{n}(later{n}(make{n}({seed}uL)))"), + }; + format!("\tprintln(\"{} ${{{value}}}\")", call.label()) + }); + let mut out = format!("import {}.*\nimport kotlinx.coroutines.runBlocking\n\n", built.name()); + for (n, lines) in &shapes { + let _ = writeln!(out, "suspend fun shape{n}() {{\n{lines}}}\n"); + } + out.push_str("fun main() = runBlocking {\n"); + for (n, _) in &shapes { + let _ = writeln!(out, "\tshape{n}()"); + } + out.push_str("}\n"); + out +} + +/// The Python driver, awaiting futures on asyncio. +fn python(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("s.digest_{n}(s.echo_{n}(s.make_{n}({seed})))"), + Call::Lent(n, seed) => format!("s.lent_{n}({seed}, s.digest_{n})"), + Call::Peek(n, seed) => format!("s.peek_{n}({seed}, s.digest_{n})"), + Call::Made(n, seed) => format!("s.made_{n}({seed}, s.make_{n})"), + Call::Later(n, seed) => format!("s.digest_{n}(await s.later_{n}(s.make_{n}({seed})))"), + }; + let _ = writeln!(body, "\tprint(f\"{} {{{value}}}\")", call.label()); + } + format!("import asyncio\n\nimport {name} as s\n\n\nasync def main() -> None:\n{body}\n\nasyncio.run(main())\n") +} + +/// The Swift driver, in the generated module's `main.swift`: each shape's calls in a +/// function of its own, since type-checking them as one body of top-level code takes +/// tens of gigabytes. +fn swift(calls: &[Call]) -> String { + let shapes = by_shape(calls, |call| { + let value = match call { + Call::Echo(n, seed) => format!("digest{n}(echo{n}(make{n}({seed})))"), + Call::Lent(n, seed) => format!("lent{n}({seed}) {{ digest{n}($0) }}"), + Call::Peek(n, seed) => format!("peek{n}({seed}) {{ digest{n}($0) }}"), + Call::Made(n, seed) => format!("made{n}({seed}) {{ make{n}($0) }}"), + Call::Later(n, seed) => format!("digest{n}(try await later{n}(make{n}({seed})))"), + }; + format!("\tprint(\"{} \\({value})\")", call.label()) + }); + let mut out = String::new(); + for (n, lines) in &shapes { + let _ = writeln!(out, "func shape{n}() async throws {{\n{lines}}}\n"); + } + for (n, _) in &shapes { + let _ = writeln!(out, "try await shape{n}()"); + } + out +} + +/// The Go driver: `one` takes the value of a call returning it alone or with an error, +/// `tupleK` gathers a returned tuple's values into the `rt.TupleK` Go takes, and `lift` +/// makes a closure of a function of either form. +fn go(built: &Built, calls: &[Call], source: &str, rt: &str) -> String { + let mut body = String::new(); + // How the generated `func` returns: its values, and whether an error follows them. + let results = |function: &str| -> (usize, bool) { + let at = source + .find(&format!("\nfunc {function}(")) + .unwrap_or_else(|| panic!("the Go declares {function}")); + let line = source[at + 1..].lines().next().unwrap(); + let mut depth = 0; + let mut close = 0; + for (i, c) in line.char_indices().skip(line.find('(').unwrap()) { + match c { + '(' | '[' => depth += 1, + ')' | ']' => depth -= 1, + _ => {} + } + if depth == 0 { + close = i; + break; + } + } + let result = line[close + 1..].trim().trim_end_matches('{').trim(); + let Some(list) = result.strip_prefix('(').and_then(|r| r.strip_suffix(')')) else { + return (1, false); + }; + let (mut types, mut depth) = (vec![String::new()], 0); + for c in list.chars() { + match c { + '(' | '[' => depth += 1, + ')' | ']' => depth -= 1, + ',' if depth == 0 => { + types.push(String::new()); + continue; + } + _ => {} + } + types.last_mut().unwrap().push(c); + } + let error = types.last().unwrap().trim() == "error"; + (types.len() - usize::from(error), error) + }; + // The value of a call of `function`, in the form a parameter takes it. + let value = |function: &str, call: String| match results(function) { + (1, _) => format!("one({call})"), + (k, false) => format!("tuple{k}({call})"), + (k, true) => format!("tuple{k}e({call})"), + }; + // A gatherer for each arity of tuple the runtime declares, which is each the API uses. + let mut tuples = String::new(); + for k in 2..=9 { + if !rt.contains(&format!("type Tuple{k}[")) { + continue; + } + let params: Vec = (0..k).map(|i| format!("T{i}")).collect(); + let types = params.join(", "); + let args: Vec = (0..k).map(|i| format!("v{i} T{i}")).collect(); + let fields: Vec = (0..k).map(|i| format!("V{i}: v{i}")).collect(); + let values: Vec = (0..k).map(|i| format!("v{i}")).collect(); + let _ = write!( + tuples, + "func tuple{k}[{types} any]({args}) rt.Tuple{k}[{types}] {{\n\treturn rt.Tuple{k}[{types}]{{{fields}}}\n}}\n\n\ + func tuple{k}e[{types} any]({args}, err error) rt.Tuple{k}[{types}] {{\n\treturn one(tuple{k}({values}), err)\n}}\n\n", + args = args.join(", "), + fields = fields.join(", "), + values = values.join(", "), + ); + } + for call in calls { + let n = call.shape(); + let printed = match *call { + Call::Echo(n, seed) => { + let made = value(&format!("Make{n}"), format!("s.Make{n}({seed})")); + let echoed = value(&format!("Echo{n}"), format!("s.Echo{n}({made})")); + format!("one(s.Digest{n}({echoed}))") + } + Call::Lent(_, seed) | Call::Peek(_, seed) => { + let item = if matches!(call, Call::Lent(..)) { "Lent" } else { "Peek" }; + let lift = if results(&format!("Digest{n}")).1 { + "liftErr" + } else { + "lift" + }; + format!("s.{item}{n}({seed}, {lift}(s.Digest{n}))") + } + Call::Made(_, seed) => { + let lift = if results(&format!("Make{n}")).1 { + "liftErr" + } else { + "lift" + }; + format!("s.Made{n}({seed}, {lift}(s.Make{n}))") + } + Call::Later(_, seed) => { + let made = value(&format!("Make{n}"), format!("s.Make{n}({seed})")); + let later = value(&format!("Later{n}"), format!("s.Later{n}(ctx, {made})")); + format!("one(s.Digest{n}({later}))") + } + }; + let _ = writeln!(body, "\tfmt.Println(\"{}\", {printed})", call.label()); + } + format!( + "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\ts \"{name}/{name}\"\n{import})\n\n\ + func one[T any](v T, err ...error) T {{\n\tif len(err) > 0 && err[0] != nil {{\n\t\tpanic(err[0])\n\t}}\n\treturn v\n}}\n\n\ + {tuples}func lift[A, B any](f func(A) B) func(A) B {{\n\treturn f\n}}\n\n\ + func liftErr[A, B any](f func(A) (B, error)) func(A) B {{\n\treturn func(a A) B {{ return one(f(a)) }}\n}}\n\n\ + func main() {{\n\tctx := context.Background()\n\t_ = ctx\n{body}}}\n", + name = built.name(), + import = match tuples.is_empty() { + true => String::new(), + false => format!("\t\"{}/rt\"\n", built.name()), + }, + ) +} + +/// The Dart driver; Dart's `int` is Rust's `u64` in two's complement, printed unsigned. +fn dart(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("s.digest{n}(s.echo{n}(s.make{n}({seed})))"), + Call::Lent(n, seed) => format!("s.lent{n}({seed}, s.digest{n})"), + Call::Peek(n, seed) => format!("s.peek{n}({seed}, s.digest{n})"), + Call::Made(n, seed) => format!("s.made{n}({seed}, s.make{n})"), + Call::Later(n, seed) => format!("s.digest{n}(await s.later{n}(s.make{n}({seed})))"), + }; + let _ = writeln!(body, " print('{} ${{u({value})}}');", call.label()); + } + format!( + "import 'package:{name}/{name}.dart' as s;\n\nString u(int v) => BigInt.from(v).toUnsigned(64).toString();\n\n\ + Future main() async {{\n{body}}}\n" + ) +} + +/// Build and run `language`'s driver of `built`, checking it against the native transcript. +fn round_trip(built: &Built, language: &str) { + let name = built.name(); + let dir = built.dir(language); + let exe = dir.join(format!("shapes{}", std::env::consts::EXE_SUFFIX)); + let known = built.known(language); + let calls: Vec = (built.krate.calls.iter().copied()) + .filter(|call| !known.contains(&call.shape())) + .collect(); + let calls = &calls[..]; + let output = match language { + "c" => { + let main = dir.join("main.c"); + std::fs::write(&main, c(built, calls)).unwrap(); + let cc = std::env::var_os("CC").unwrap_or_else(|| "cc".into()); + run(Command::new(cc) + .args(["-std=c23", "-pedantic", "-Wall", "-Wextra", "-Werror", "-I"]) + .arg(built.out.join("include")) + .arg(&main) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe)); + run(&mut Command::new(&exe)) + } + "cpp" => { + let main = dir.join("main.cpp"); + std::fs::write(&main, cpp(built, calls)).unwrap(); + // The api fixture's build, so MSVC's runner builds it with cl. + let link = super::Link { + include: built.out.join("include"), + staticlib: built.staticlib(), + native: &built.native, + }; + run(&mut super::cxx(&link, 17, false, &main, &exe)); + run(&mut Command::new(&exe)) + } + "kotlin" => { + let main = dir.join("main.kt"); + std::fs::write(&main, kotlin(built, calls)).unwrap(); + let classes = dir.join("classes"); + run(Command::new("kotlinc") + .args(["-Werror", "-cp"]) + .arg(super::coroutines()) + .arg(built.out.join("kotlin").join(format!("{name}.kt"))) + .arg(&main) + .arg("-d") + .arg(&classes)); + let mut classpath = classes.into_os_string(); + classpath.push(if cfg!(windows) { ";" } else { ":" }); + classpath.push(super::coroutines()); + run(Command::new("kotlin") + .arg(format!("-Djava.library.path={}", built.lib.display())) + .arg("-cp") + .arg(&classpath) + .arg("MainKt")) + } + "python" => { + let main = dir.join("main.py"); + std::fs::write(&main, python(built, calls)).unwrap(); + run( + Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())) + .args(["-P", "-X", "dev", "-W", "error"]) + .arg(&main) + .env("PYTHONPATH", built.out.join("python")), + ) + } + "swift" => { + let module = built.out.join("swift"); + std::fs::copy(module.join(format!("{name}.swift")), dir.join(format!("{name}.swift"))).unwrap(); + std::fs::write(dir.join("main.swift"), swift(calls)).unwrap(); + run(super::swiftc(&module, 15) + .args(["-swift-version", "6", "-module-name", "Shapes"]) + .arg(dir.join(format!("{name}.swift"))) + .arg(dir.join("main.swift")) + .args(super::swift_collections()) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe)); + run(&mut Command::new(&exe)) + } + "go" => { + let export = dir.join("export"); + super::copy_dir(&built.out.join("go"), &export); + std::fs::create_dir_all(dir.join("include")).unwrap(); + let header = format!("{name}.h"); + std::fs::copy( + built.out.join("include").join(&header), + dir.join("include").join(&header), + ) + .unwrap(); + let source = std::fs::read_to_string(export.join(&name).join(format!("{name}.go"))).unwrap(); + let rt = std::fs::read_to_string(export.join("rt").join("types.go")).unwrap_or_default(); + std::fs::write(dir.join("main.go"), go(built, calls, &source, &rt)).unwrap(); + std::fs::write( + dir.join("go.mod"), + format!("module shapestest\n\ngo 1.24\n\nrequire {name} v0.0.0\n\nreplace {name} => ./export\n"), + ) + .unwrap(); + let mut ldflags = built.staticlib().into_os_string(); + for lib in &built.native { + ldflags.push(" "); + ldflags.push(lib); + } + run(Command::new("go") + .args(["build", "-o"]) + .arg(&exe) + .current_dir(&dir) + .env("CGO_ENABLED", "1") + .env("CGO_LDFLAGS", ldflags) + .env("GOFLAGS", "-buildvcs=false") + .env("GOTOOLCHAIN", "local")); + run(&mut Command::new(&exe)) + } + "dart" => { + let main = dir.join("main.dart"); + std::fs::write(&main, dart(built, calls)).unwrap(); + let lib = built.lib.join(shared(&name)); + let dill = super::dart_compile(&name, &dir, &built.out.join("dart"), &lib, &main); + run(&mut super::dart_run(&dill, &[])) + } + _ => unreachable!("no driver for {language}"), + }; + built.check(language, &known, &output); +} + +/// The gate's subset of shapes, built once. +fn gated() -> &'static Built { + static BUILT: std::sync::OnceLock = std::sync::OnceLock::new(); + BUILT.get_or_init(|| build("shapes", &gate())) +} + +/// Every sampled shape, built once: the nightly's. +fn sampled() -> &'static Built { + static BUILT: std::sync::OnceLock = std::sync::OnceLock::new(); + BUILT.get_or_init(|| build("sampled", &shapes(SAMPLES))) +} + +/// How many shapes the nightly samples beyond each leaf in each wrapper. +const SAMPLES: usize = 64; + +macro_rules! languages { + ($($language:ident),*) => { + mod gate { + $( + #[test] + fn $language() { + super::round_trip(super::gated(), stringify!($language)); + } + )* + } + + mod nightly { + $( + #[test] + #[ignore = "the nightly's: every sampled shape, just ffi shapes"] + fn $language() { + super::round_trip(super::sampled(), stringify!($language)); + } + )* + } + }; +} + +languages!(c, cpp, kotlin, python, swift, go, dart); diff --git a/quest/m1/ffi-export/lossless.md b/quest/m1/ffi-export/lossless.md index 259292d0d..826f9cfa9 100644 --- a/quest/m1/ffi-export/lossless.md +++ b/quest/m1/ffi-export/lossless.md @@ -7,7 +7,8 @@ Rust type exactly: a value passed in and returned is the same value, and no conv truncates, clamps, panics, traps, or aborts. A native type stands in only where it is exact; otherwise the runtime's own type crosses, with checked conversions to and from the idiomatic one that fail with the language's error. Proven in every language's `api` test -by the edge values of each type crossing both ways. +by the edge values of each type crossing both ways, and by deleting this quest's `KNOWN` +entries in [the shape tests](/ffi/tests/fixtures/shapes.rs). ## Plan @@ -22,7 +23,8 @@ What the audit found, each a place native Rust would have gone on: clamp (`ffi/runtime/python.rs`, `rt.dart`); Go's `time.Duration` panics on a negative one or past 292 years (`ffi_cgo.go`); Swift traps on a negative `Swift.Duration` or one past `Duration::MAX` (`rt.swift`); Kotlin's maps past `i64` seconds to `INFINITE` - and back to `i64::MAX` seconds (`rt.kt`). Seconds and nanoseconds, as std holds them, + and back to `i64::MAX` seconds (`rt.kt`), and past about 146 years, where + `kotlin.time.Duration` holds milliseconds, drops what is below one. Seconds and nanoseconds, as std holds them, cross exactly. moq's switch also asks that Go stop panicking on a negative one and that a duration field default to zero ([moq's quest](https://github.com/moq-dev/moq/blob/main/quest/m3/ffi-exporter.md)). diff --git a/quest/m1/ffi-export/owned-bufs.md b/quest/m1/ffi-export/owned-bufs.md index 093b4715f..c86d5992d 100644 --- a/quest/m1/ffi-export/owned-bufs.md +++ b/quest/m1/ffi-export/owned-bufs.md @@ -31,7 +31,9 @@ the call and never frees", so its owner is never read. header fails at load. Prove it in every language's `api` test with a payload read and written back, checked for -identity of the storage where the language can see it, and re-measure the bench's write +identity of the storage where the language can see it, delete its Kotlin and Go `KNOWN` +entries in [the shape tests](/ffi/tests/fixtures/shapes.rs), which a returned payload +passed back now compiles in, and re-measure the bench's write and throughput rows. Public API: the C ABI's meaning of an input buffer's owner (breaking), `_buf_clone`, diff --git a/quest/m1/ffi-export/round-trips.md b/quest/m1/ffi-export/round-trips.md new file mode 100644 index 000000000..9112b66bf --- /dev/null +++ b/quest/m1/ffi-export/round-trips.md @@ -0,0 +1,26 @@ +# [S] Every binding takes back what it returns + +## Goal + +A value a [rust2-ffi](/ffi/README.md) binding returns passes back into a parameter of the +same Rust type as it is, with no conversion by the caller. Two bindings break this today, +each a `KNOWN` entry in [the shape tests](/ffi/tests/fixtures/shapes.rs) that this quest +deletes: + +- C++ returns a collection of bytes, such as `Vec` or `BTreeMap>`, + as a `std::vector` of `buf`s, but takes it as a `const std::vector` of `view`s, which a + vector of `buf`s does not convert to. +- Kotlin returns a tuple of one, `(T,)`, as its `T`, but takes it as `Tuple1`. + +## Plan + +Found by the shape tests' nightly sample on 2026-11-13. + +- C++: take a collection whose items hold bytes as any range whose items convert to a + `view`, as `view` already takes any contiguous range of bytes, so a returned vector of + `buf`s, and the caller's vector of `std::string`s, pass as they are. +- Kotlin: return a tuple of one as `Tuple1`, as it takes one, rather than unwrapping it. + Swift has no tuple of one, so its unwrapping stays. + +Public API: C++'s parameter types for collections of bytes, which take more than before; +Kotlin's return type of a function returning a tuple of one (breaking). diff --git a/quest/m1/ffi-speed/shape-tests.md b/quest/m1/ffi-speed/shape-tests.md deleted file mode 100644 index 4f448a675..000000000 --- a/quest/m1/ffi-speed/shape-tests.md +++ /dev/null @@ -1,27 +0,0 @@ -# [M] Generated type shapes round-trip in every language - -## Goal - -A generator samples type shapes from [rust2-ffi](/ffi/README.md)'s own IR, in every -position a type can take (parameter, return, field, `Option`, `Vec`, closure, future), and -each language's test checks `digest(echo(make(seed)))` against the value native Rust -computes. A shape a generator mishandles, or one that reaches one of its `unreachable!` -arms, fails the test in that language. - -## Plan - -Found by the 2026-11-12 audit. Every language already matches native Rust's transcript on -the api fixture (`ffi/tests/fixtures.rs`), but the fixture covers each rule once and each -script is ported by hand, while the generators hold 92 `unreachable!` sites keyed on type -shapes and a new rule touches 19 to 38 files. - -- Sample `ir::Type` trees to a bounded depth with a fixed seed, and emit a fixture crate - of `make_N(seed)`, `echo_N(v)`, and `digest_N(&v)` per shape. -- Each language's driver is a generated flat list of `digest(echo(make(s))) == K`, with - `K` computed natively, so no driver holds type-specific code. -- The sampled set runs nightly; a small fixed subset joins the gate, and the PR reports - what it adds to `just ffi test`. -- A shape a generator refuses by design must fail the export, not the test: the sampler - asks the exporter which shapes export. - -Public API: none.