From b11aff42cc44611c59d882ce68a8f9f508e803f4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 22:25:06 -0800 Subject: [PATCH 1/9] chore(quest): claim quest/m1/ffi-speed/shape-tests.md Quest-Claim: ba779c33baa6b18a From b6433791a07ff1f4d24311bb0636eb9a4546a2dd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 03:15:26 -0800 Subject: [PATCH 2/9] wip(ffi): shape sampler --- ffi/tests/fixtures.rs | 5 +- ffi/tests/fixtures/shapes.rs | 699 +++++++++++++++++++++++++++++++++++ 2 files changed, 703 insertions(+), 1 deletion(-) create mode 100644 ffi/tests/fixtures/shapes.rs diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 2385d229c..73cd52582 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -3,7 +3,10 @@ //! and, shrunk by R8, on Android, from Python by `tests/api.py`, from Swift by //! `tests/api.swift`, from Go by `tests/api.go`, and from Dart by `tests/api.dart`, whose //! transcripts must match the fixture's own script run natively. -//! The reject fixture's export must fail at each item it cannot export. +//! The reject fixture's export must fail at each item it cannot export, and generated +//! type shapes round-trip in every language ([`shapes`]). + +mod shapes; use std::env::consts::EXE_SUFFIX; use std::ffi::OsString; diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs new file mode 100644 index 000000000..796124ed1 --- /dev/null +++ b/ffi/tests/fixtures/shapes.rs @@ -0,0 +1,699 @@ +//! Generated type shapes, round-tripped through every language: a seeded sampler builds +//! value types from what the exporter's IR holds, nested to a bounded depth, and emits a +//! crate of `make_N(seed)`, `echo_N(v)`, and `digest_N(&v)` per shape, with the shape in a +//! closure's parameter and return and a future's output besides. The census of that crate +//! says which items export, so a shape the exporter refuses by design fails the export, +//! not the test; each language's driver is a generated flat list of calls whose printed +//! digests must match the crate's own, run natively. + +use std::fmt::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use super::{cargo, manifest_dir, run, scratch}; + +/// The integer types of the IR's `Int`. +const INTS: [&str; 10] = ["u8", "u16", "u32", "u64", "usize", "i8", "i16", "i32", "i64", "isize"]; + +/// A value type the sampler builds, after the IR's `Type`. +#[derive(Clone, Debug)] +enum Shape { + Bool, + Int(&'static str), + NonZero(&'static str), + U128, + F32, + F64, + Char, + String, + /// `Vec` + Bytes, + /// `bytes::Bytes` + Buf, + Duration, + Array(usize), + PathBuf, + /// `SocketAddr`, `SocketAddrV4`, or `SocketAddrV6`. + SocketAddr(&'static str), + Range(&'static str), + Bound(&'static str), + /// `serde_json::Value` + Json, + /// `serde_json::Map` + JsonMap, + /// `url::Url` + Url, + Option(Box), + Vec(Box), + Set(Box), + Map(Box, Box), + Box(Box), + Tuple(Vec), + /// A struct of `pub` fields: a record. + Record(Vec), + /// An enum of a unit variant, a tuple variant, and a struct variant. + Enum(Box), + /// A struct of a private field: an object. + Object(Box), +} + +/// Each variant of the IR's `Type` (`src/ir.rs`), and how the shapes cover it: the +/// [`Shape`] that builds it, or why none does. A new variant fails [`ir_covered`] until it +/// is named here. +const COVERED: &[(&str, &str)] = &[ + ("Unit", "none: no value, so a function returning it returns nothing"), + ("Bool", "Bool"), + ("Int", "Int, of each width"), + ("NonZero", "NonZero, of each width"), + ("U128", "U128"), + ("F32", "F32"), + ("F64", "F64"), + ("Char", "Char"), + ("Str", "String, which digest_N and peek_N's closure borrow as &str"), + ("String", "String"), + ("Slice", "Bytes, which digest_N and peek_N's closure borrow as &[u8]"), + ("Vec", "Bytes"), + ("Bytes", "Buf"), + ("Duration", "Duration"), + ("Array", "Array"), + ("Path", "PathBuf, which digest_N and peek_N's closure borrow as &Path"), + ("PathBuf", "PathBuf"), + ("SocketAddr", "SocketAddr, of each family"), + ("List", "Vec, Set, Map, and JsonMap"), + ("Waker", "none: a waker holds no value"), + ("Future", "none: a foreign future is of (), which holds no value"), + ("Reader", "none: a foreign reader is lent for a call and holds no value"), + ("Option", "Option"), + ("Box", "Box"), + ("Ref", "every shape, which digest_N and peek_N's closure borrow"), + ("Record", "Record"), + ("Enum", "Enum"), + ("Object", "Object, and Json and Url, which the export maps to objects"), + ("Callback", "none: a foreign trait implementation holds no value"), + ("Closure", "the positions lent_N, peek_N, and made_N"), + ("Range", "Range"), + ("Bound", "Bound"), + ("Bounds", "none: the form of an impl RangeBounds parameter, which no value has"), + ("Tuple", "Tuple"), + ("Const", "none: a const argument, which no value has"), +]; + +/// Every leaf, once for each integer type and socket address family. +fn leaves() -> Vec { + let mut leaves = vec![Shape::Bool]; + leaves.extend(INTS.map(Shape::Int)); + leaves.extend(INTS.map(Shape::NonZero)); + leaves.extend([ + Shape::U128, + Shape::F32, + Shape::F64, + Shape::Char, + Shape::String, + Shape::Bytes, + Shape::Buf, + Shape::Duration, + Shape::Array(1), + Shape::Array(16), + Shape::PathBuf, + Shape::SocketAddr("SocketAddr"), + Shape::SocketAddr("SocketAddrV4"), + Shape::SocketAddr("SocketAddrV6"), + Shape::Range("u32"), + Shape::Range("i64"), + Shape::Bound("u8"), + Shape::Bound("isize"), + Shape::Json, + Shape::JsonMap, + Shape::Url, + ]); + leaves +} + +/// splitmix64: a fixed seed samples the same shapes and values on every host. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} + +/// A shape of at most `depth` levels: a leaf, or a wrapper of shapes one level shallower. +fn sample(rng: &mut Rng, leaves: &[Shape], depth: u32) -> Shape { + if depth == 0 || rng.below(3) == 0 { + return leaves[rng.below(leaves.len())].clone(); + } + let mut inner = || Box::new(sample(rng, leaves, depth - 1)); + match rng.below(10) { + 0 => Shape::Option(inner()), + 1 => Shape::Vec(inner()), + 2 => Shape::Box(inner()), + 3 => Shape::Enum(inner()), + 4 => Shape::Object(inner()), + 5 => { + let element = ordered(rng, leaves); + Shape::Set(Box::new(element)) + } + 6 => { + let key = ordered(rng, leaves); + Shape::Map(Box::new(key), Box::new(sample(rng, leaves, depth - 1))) + } + 7 => { + let n = 1 + rng.below(3); + Shape::Tuple((0..n).map(|_| sample(rng, leaves, depth - 1)).collect()) + } + _ => { + let n = 1 + rng.below(3); + Shape::Record((0..n).map(|_| sample(rng, leaves, depth - 1)).collect()) + } + } +} + +/// A leaf that is `Ord`, to key a set or a map. +fn ordered(rng: &mut Rng, leaves: &[Shape]) -> Shape { + loop { + let leaf = &leaves[rng.below(leaves.len())]; + if is_ord(leaf) { + return leaf.clone(); + } + } +} + +fn is_ord(shape: &Shape) -> bool { + !matches!( + shape, + Shape::F32 | Shape::F64 | Shape::Json | Shape::JsonMap | Shape::Bound(_) | Shape::Range(_) + ) +} + +/// The shapes a run covers: each leaf alone, each leaf in each wrapper, then `extra` +/// sampled to depth 3 from a fixed seed. +pub fn shapes(extra: usize) -> Vec { + let leaves = leaves(); + let mut shapes = leaves.clone(); + let mut rng = Rng(0x5eed); + for leaf in &leaves { + let inner = || Box::new(leaf.clone()); + shapes.extend([ + Shape::Option(inner()), + Shape::Vec(inner()), + Shape::Box(inner()), + Shape::Tuple(vec![leaf.clone(), Shape::Bool]), + Shape::Record(vec![leaf.clone()]), + Shape::Enum(inner()), + Shape::Object(inner()), + ]); + if is_ord(leaf) { + shapes.push(Shape::Set(inner())); + shapes.push(Shape::Map(inner(), Box::new(Shape::String))); + } + } + shapes.extend((0..extra).map(|_| sample(&mut rng, &leaves, 3))); + shapes +} + +/// The gate's subset: each leaf alone, and each wrapper around a buffer, a +/// record field, and an object. +pub fn gate() -> Vec { + let mut shapes = leaves(); + for inner in [Shape::String, Shape::Int("i16"), Shape::Object(Box::new(Shape::U128))] { + let boxed = || Box::new(inner.clone()); + shapes.extend([ + Shape::Option(boxed()), + Shape::Vec(boxed()), + Shape::Box(boxed()), + Shape::Tuple(vec![inner.clone(), Shape::F64]), + Shape::Record(vec![inner.clone(), Shape::Char]), + Shape::Enum(boxed()), + ]); + } + shapes.push(Shape::Map(Box::new(Shape::String), Box::new(Shape::Bytes))); + shapes.push(Shape::Set(Box::new(Shape::Int("u64")))); + shapes +} + +/// The Rust source of the shapes' crate. +struct Emit { + /// Each record, enum, and object the shapes define. + items: String, + /// How many it has defined, which names the next. + defined: usize, +} + +impl Emit { + /// The Rust type of `shape`, and an expression making one from `r`, a `&mut Rng`. + fn shape(&mut self, shape: &Shape) -> (String, String) { + match shape { + Shape::Bool => ("bool".into(), "r.next() % 2 == 0".into()), + Shape::Int(int) => (int.to_string(), format!("r.int() as {int}")), + Shape::NonZero(int) => ( + format!("std::num::NonZero<{int}>"), + format!("std::num::NonZero::new(r.int() as {int}).unwrap_or(std::num::NonZero::<{int}>::MAX)"), + ), + Shape::U128 => ("u128".into(), "(r.int() as u128) << 64 | r.int() as u128".into()), + Shape::F32 => ("f32".into(), "r.f32()".into()), + Shape::F64 => ("f64".into(), "r.f64()".into()), + Shape::Char => ("char".into(), "r.char()".into()), + Shape::String => ("String".into(), "r.string()".into()), + Shape::Bytes => ("Vec".into(), "r.bytes()".into()), + Shape::Buf => ("bytes::Bytes".into(), "bytes::Bytes::from(r.bytes())".into()), + Shape::Duration => ("std::time::Duration".into(), "r.duration()".into()), + Shape::Array(n) => (format!("[u8; {n}]"), "std::array::from_fn(|_| r.next() as u8)".into()), + Shape::PathBuf => ("std::path::PathBuf".into(), "std::path::PathBuf::from(r.string())".into()), + Shape::SocketAddr(family) => (format!("std::net::{family}"), format!("r.{}()", family.to_lowercase())), + Shape::Range(int) => ( + format!("std::ops::Range<{int}>"), + format!("(r.int() as {int})..(r.int() as {int})"), + ), + Shape::Bound(int) => ( + format!("std::ops::Bound<{int}>"), + format!( + "match r.below(3) {{ 0 => std::ops::Bound::Included(r.int() as {int}), 1 => std::ops::Bound::Excluded(r.int() as {int}), _ => std::ops::Bound::Unbounded }}" + ), + ), + Shape::Json => ("serde_json::Value".into(), "r.json(2)".into()), + Shape::JsonMap => ( + "serde_json::Map".into(), + "(0..r.below(3)).map(|_| (r.string(), r.json(1))).collect()".into(), + ), + Shape::Url => ("url::Url".into(), "r.url()".into()), + Shape::Option(inner) => { + let (ty, make) = self.shape(inner); + (format!("Option<{ty}>"), format!("match r.below(4) {{ 0 => None, _ => Some({make}) }}")) + } + Shape::Vec(inner) => { + let (ty, make) = self.shape(inner); + (format!("Vec<{ty}>"), format!("(0..r.below(4)).map(|_| {make}).collect()")) + } + Shape::Set(inner) => { + let (ty, make) = self.shape(inner); + ( + format!("std::collections::BTreeSet<{ty}>"), + format!("(0..r.below(4)).map(|_| {make}).collect()"), + ) + } + Shape::Map(key, value) => { + let (key, make_key) = self.shape(key); + let (value, make_value) = self.shape(value); + ( + format!("std::collections::BTreeMap<{key}, {value}>"), + format!("(0..r.below(4)).map(|_| ({make_key}, {make_value})).collect()"), + ) + } + Shape::Box(inner) => { + let (ty, make) = self.shape(inner); + (format!("Box<{ty}>"), format!("Box::new({make})")) + } + Shape::Tuple(elements) => { + let (types, makes): (Vec<_>, Vec<_>) = elements.iter().map(|e| self.shape(e)).unzip(); + (format!("({},)", types.join(", ")), format!("({},)", makes.join(", "))) + } + Shape::Record(fields) => { + let (types, makes): (Vec<_>, Vec<_>) = fields.iter().map(|f| self.shape(f)).unzip(); + let name = self.name("Record"); + let _ = writeln!(self.items, "#[derive(Debug)]\npub struct {name} {{"); + for (i, ty) in types.iter().enumerate() { + let _ = writeln!(self.items, "\tpub f{i}: {ty},"); + } + let _ = writeln!(self.items, "}}\n"); + let fields: Vec = makes.iter().enumerate().map(|(i, m)| format!("f{i}: {m}")).collect(); + (name.clone(), format!("{name} {{ {} }}", fields.join(", "))) + } + Shape::Enum(inner) => { + let (ty, make) = self.shape(inner); + let name = self.name("Enum"); + let _ = writeln!( + self.items, + "#[derive(Debug)]\npub enum {name} {{\n\tUnit,\n\tTuple({ty}),\n\tStruct {{ value: {ty} }},\n}}\n" + ); + ( + name.clone(), + format!( + "match r.below(3) {{ 0 => {name}::Unit, 1 => {name}::Tuple({make}), _ => {name}::Struct {{ value: {make} }} }}" + ), + ) + } + Shape::Object(inner) => { + let (ty, make) = self.shape(inner); + let name = self.name("Object"); + let _ = writeln!(self.items, "#[derive(Debug)]\npub struct {name}({ty});\n"); + (name.clone(), format!("{name}({make})")) + } + } + } + + fn name(&mut self, kind: &str) -> String { + self.defined += 1; + format!("{kind}{}", self.defined) + } +} + +/// The borrowed form `digest_N` takes of a value of `ty`. +fn borrowed(shape: &Shape, ty: &str) -> String { + match shape { + Shape::String => "&str".into(), + Shape::Bytes => "&[u8]".into(), + Shape::PathBuf => "&std::path::Path".into(), + Shape::Vec(_) => format!("&[{}]", &ty[4..ty.len() - 1]), + _ => format!("&{ty}"), + } +} + +/// What `make_N` draws values from, and `digest`, which every `digest_N` calls. +const RUNTIME: &str = r#" +/// splitmix64, with edge values drawn often. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng(seed) + } + + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + + /// Bits for an integer of any width, each edge of every width among them. + fn int(&mut self) -> u64 { + match self.below(8) { + 0 => 0, + 1 => u64::MAX, + // The sign bit of a width, and the most a signed one holds. + 2 => 1 << ((8 << self.below(4)) - 1), + 3 => (1 << ((8 << self.below(4)) - 1)) - 1, + _ => self.next(), + } + } + + fn f32(&mut self) -> f32 { + let f = f32::from_bits(self.int() as u32); + if f.is_nan() { -0.0 } else { f } + } + + fn f64(&mut self) -> f64 { + let f = f64::from_bits(self.int()); + if f.is_nan() { f64::INFINITY } else { f } + } + + fn char(&mut self) -> char { + char::from_u32(self.next() as u32 % 0x11_0000).unwrap_or('\u{10ffff}') + } + + fn string(&mut self) -> String { + const PARTS: [&str; 6] = ["a", "é", "😀", "\0", "Z", "語"]; + (0..self.below(5)).map(|_| PARTS[self.below(6) as usize]).collect() + } + + fn bytes(&mut self) -> Vec { + (0..self.below(6)).map(|_| self.next() as u8).collect() + } + + /// Up to 2^33 seconds, past 136 years: Go's `time.Duration` holds 292. + fn duration(&mut self) -> std::time::Duration { + std::time::Duration::new(self.next() >> 31, (self.next() % 1_000_000_000) as u32) + } + + fn socketaddrv4(&mut self) -> std::net::SocketAddrV4 { + std::net::SocketAddrV4::new((self.next() as u32).into(), self.next() as u16) + } + + fn socketaddrv6(&mut self) -> std::net::SocketAddrV6 { + let ip = ((self.next() as u128) << 64 | self.next() as u128).into(); + std::net::SocketAddrV6::new(ip, self.next() as u16, self.next() as u32, self.next() as u32) + } + + fn socketaddr(&mut self) -> std::net::SocketAddr { + match self.below(2) { + 0 => self.socketaddrv4().into(), + _ => self.socketaddrv6().into(), + } + } + + fn json(&mut self, depth: u32) -> serde_json::Value { + match self.below(if depth == 0 { 4 } else { 6 }) { + 0 => serde_json::Value::Null, + 1 => (self.next() % 2 == 0).into(), + 2 => (self.int() as i64).into(), + 3 => self.string().into(), + 4 => (0..self.below(3)).map(|_| self.json(depth - 1)).collect(), + _ => (0..self.below(3)).map(|_| (self.string(), self.json(depth - 1))).collect(), + } + } + + fn url(&mut self) -> url::Url { + let text = format!("https://h{}.test:{}/p/{}?q={}", self.below(100), self.next() as u16, self.below(10), self.next()); + url::Url::parse(&text).unwrap() + } +} + +/// FNV-1a of the value's `Debug`, which prints every bit of each shape's value. +fn digest(value: &dyn std::fmt::Debug) -> u64 { + format!("{value:?}").bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| (h ^ b as u64).wrapping_mul(0x100_0000_01b3)) +} +"#; + +/// The functions each shape `N` exports, which the census may refuse one by one. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Item { + Make, + Echo, + Digest, + /// `lent_N(seed, f)`: calls `f` with the made value, by value. + Lent, + /// `peek_N(seed, f)`: calls `f` with the made value, borrowed as `digest_N` takes it. + Peek, + /// `made_N(seed, f)`: digests what `f` returns for the seed. + Made, + /// `later_N(v)`: a future of `v`. + Later, +} + +impl Item { + pub fn name(self, n: usize) -> String { + let item = match self { + Item::Make => "make", + Item::Echo => "echo", + Item::Digest => "digest", + Item::Lent => "lent", + Item::Peek => "peek", + Item::Made => "made", + Item::Later => "later", + }; + format!("{item}_{n}") + } +} + +/// A line of the script each driver runs, printing `label value`. +#[derive(Clone, Copy, Debug)] +pub enum Call { + /// `digest_N(echo_N(make_N(seed)))` + Echo(usize, u64), + /// `lent_N(seed, digest_N)` + Lent(usize, u64), + /// `peek_N(seed, digest_N)` + Peek(usize, u64), + /// `made_N(seed, make_N)` + Made(usize, u64), + /// `digest_N(await later_N(make_N(seed)))` + Later(usize, u64), +} + +impl Call { + pub fn label(self) -> String { + match self { + Call::Echo(n, seed) => format!("echo {n} {seed}"), + Call::Lent(n, seed) => format!("lent {n} {seed}"), + Call::Peek(n, seed) => format!("peek {n} {seed}"), + Call::Made(n, seed) => format!("made {n} {seed}"), + Call::Later(n, seed) => format!("later {n} {seed}"), + } + } +} + +/// The seeds each shape is made from. +const SEEDS: [u64; 3] = [0, 1, 0xdead_beef]; + +/// The shapes' crate, as the census left it: which shapes export, and the script. +pub struct Crate { + pub dir: PathBuf, + /// Each shape's Rust type, by `N`. + pub types: Vec, + pub calls: Vec, + /// The digests the script prints natively. + pub transcript: String, +} + +/// Write the crate of `shapes` named `name` in the scratch directory, ask the exporter what +/// exports, and run its script natively. +pub fn emit(name: &str, shapes: &[Shape]) -> Crate { + let dir = scratch().join(name); + let _ = std::fs::remove_dir_all(dir.join("src")); + let mut emit = Emit { + items: String::new(), + defined: 0, + }; + let mut functions = String::new(); + let mut types = Vec::new(); + for (n, shape) in shapes.iter().enumerate() { + let (ty, make) = emit.shape(shape); + let lent = borrowed(shape, &ty); + let _ = write!( + functions, + "pub fn make_{n}(seed: u64) -> {ty} {{\n\tlet r = &mut Rng::new(seed);\n\t{make}\n}}\n\n\ + pub fn echo_{n}(v: {ty}) -> {ty} {{\n\tv\n}}\n\n\ + pub fn digest_{n}(v: {lent}) -> u64 {{\n\tdigest(&v)\n}}\n\n\ + pub fn lent_{n}(seed: u64, f: impl FnOnce({ty}) -> u64) -> u64 {{\n\tf(make_{n}(seed))\n}}\n\n\ + pub fn peek_{n}(seed: u64, f: impl FnOnce({lent}) -> u64) -> u64 {{\n\tf(&make_{n}(seed))\n}}\n\n\ + pub fn made_{n}(seed: u64, f: impl FnOnce(u64) -> {ty}) -> u64 {{\n\tdigest(&f(seed))\n}}\n\n\ + pub async fn later_{n}(v: {ty}) -> {ty} {{\n\tv\n}}\n\n" + ); + types.push(ty); + } + let lib = format!( + "//! Generated by `tests/fixtures/shapes.rs`: each shape's `make`, `echo`, and `digest`.\n\ + #![allow(clippy::all, reason = \"generated\")]\n{RUNTIME}\n{}{functions}", + emit.items + ); + let manifest = format!( + "[package]\nname = \"{name}\"\nversion = \"0.0.0\"\nedition = \"2024\"\npublish = false\n\n\ + [dependencies]\nbytes = \"1\"\nserde_json = \"1\"\nurl = \"2\"\n\n[workspace]\n" + ); + super::write_all(&dir, &[("Cargo.toml", &manifest), ("src/lib.rs", &lib), ("ffi.toml", "")]); + // The api fixture's versions, resolved offline. + std::fs::copy(manifest_dir().join("Cargo.lock"), dir.join("Cargo.lock")).unwrap(); + + let refused = refused(&dir); + let exported = |item: Item, n: usize| !refused.iter().any(|path| *path == item.name(n)); + let mut excludes: Vec<&String> = refused.iter().collect(); + excludes.sort(); + let config = excludes.iter().map(|path| format!(" \"{path}\",\n")).collect::(); + std::fs::write(dir.join("ffi.toml"), format!("exclude = [\n{config}]\n")).unwrap(); + + let mut calls = Vec::new(); + for n in 0..shapes.len() { + if ![Item::Make, Item::Digest].iter().all(|item| exported(*item, n)) { + continue; + } + for seed in SEEDS { + if exported(Item::Echo, n) { + calls.push(Call::Echo(n, seed)); + } + if exported(Item::Lent, n) { + calls.push(Call::Lent(n, seed)); + } + if exported(Item::Peek, n) { + calls.push(Call::Peek(n, seed)); + } + if exported(Item::Made, n) { + calls.push(Call::Made(n, seed)); + } + if exported(Item::Later, n) { + calls.push(Call::Later(n, seed)); + } + } + } + + let mut script = format!("//! The shapes' script, run natively.\n\nuse {name}::*;\n{NATIVE}"); + for call in &calls { + let value = match *call { + Call::Echo(n, seed) => format!("digest_{n}(&echo_{n}(make_{n}({seed})))"), + Call::Lent(n, seed) => format!("lent_{n}({seed}, |v| digest_{n}(&v))"), + Call::Peek(n, seed) => format!("peek_{n}({seed}, digest_{n})"), + Call::Made(n, seed) => format!("made_{n}({seed}, make_{n})"), + Call::Later(n, seed) => format!("digest_{n}(&later(later_{n}(make_{n}({seed}))))"), + }; + let _ = writeln!(script, "\tprintln!(\"{} {{}}\", {value});", call.label()); + } + script.push_str("}\n"); + super::write_all(&dir, &[("examples/transcript.rs", &script)]); + let transcript = run(cargo() + .args(["run", "--quiet", "--example", "transcript"]) + .env("CARGO_TARGET_DIR", dir.join("build")) + .current_dir(&dir)); + Crate { + dir, + types, + calls, + transcript: String::from_utf8_lossy(&transcript.stdout).into_owned(), + } +} + +/// The native script's head: `later` polls a future that is ready at once. +const NATIVE: &str = " +fn later(f: impl Future) -> T { + match std::pin::pin!(f).poll(&mut std::task::Context::from_waker(std::task::Waker::noop())) { + std::task::Poll::Ready(v) => v, + std::task::Poll::Pending => unreachable!(), + } +} + +fn main() { +"; + +/// The paths the census of the crate in `dir` refuses: its config that exports what maps. +fn refused(dir: &Path) -> Vec { + let census = dir.join("census"); + run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(dir.join("ffi.toml")) + .arg("--census") + .arg(&census) + .env("CARGO_TARGET_DIR", dir.join("build"))); + let name = dir.file_name().unwrap().to_string_lossy(); + let report = std::fs::read_to_string(census.join(format!("{name}.md"))).unwrap(); + let Some((_, config)) = report.split_once("## The config that exports what maps") else { + return Vec::new(); + }; + config + .lines() + .skip_while(|line| *line != "exclude = [") + .skip(1) + .take_while(|line| *line != "]") + .map(|line| line.trim().trim_end_matches(',').trim_matches('"').to_string()) + .collect() +} + +/// Each variant of the IR's `Type`, from its source. +fn ir_variants() -> Vec { + let ir = std::fs::read_to_string(manifest_dir().join("src").join("ir.rs")).unwrap(); + let body = ir.split_once("pub enum Type {").expect("ir.rs declares Type").1; + let body = &body[..body.find("\n}").unwrap()]; + body.lines() + .map(str::trim) + .filter(|line| !line.starts_with("//") && !line.is_empty()) + .map(|line| line.split(|c: char| !c.is_alphanumeric()).next().unwrap().to_string()) + .collect() +} + +/// Every variant of the IR's `Type` is sampled or named as unsampled, once. +#[test] +fn ir_covered() { + let variants = ir_variants(); + let covered: Vec<&str> = COVERED.iter().map(|(name, _)| *name).collect(); + assert_eq!( + variants, covered, + "COVERED in tests/fixtures/shapes.rs names each variant of the IR's Type, in order" + ); +} + +#[test] +fn probe() { + let krate = emit("shapes", &gate()); + eprintln!("{}", krate.transcript.lines().count()); +} From 813441c402b47b05e1fc9eab3b60f1641832ece2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 04:30:46 -0800 Subject: [PATCH 3/9] wip(ffi): shape round trips in every language --- ffi/src/dart.rs | 11 +- ffi/src/glue.rs | 7 +- ffi/tests/fixtures.rs | 15 +- ffi/tests/fixtures/shapes.rs | 763 ++++++++++++++++++++++++++++++++++- 4 files changed, 778 insertions(+), 18 deletions(-) diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index 0318d9dc4..3f22ca78c 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -1848,10 +1848,13 @@ impl<'a> Gen<'a> { /// The Dart statements closing each object the value `e` of type `ty` holds, once a /// call moved them out. fn close(&mut self, ty: &Type, e: &str) -> String { + // A local named apart from `e`, which may be a parameter named `v`, and from the + // local an outer level declared, which Dart would read before it is declared. + let v: String = e.chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }).collect::() + "_"; match ty { Type::Option(inner) => { - let close = self.close(inner, "v"); - format!("{{ final v = {e}; if (v != null) {{ {close} }} }}") + let close = self.close(inner, &v); + format!("{{ final {v} = {e}; if ({v} != null) {{ {close} }} }}") } Type::Box(inner) => self.close(inner, e), Type::Tuple(elements) => { @@ -1862,8 +1865,8 @@ impl<'a> Gen<'a> { closes.join(" ") } Type::List(_, inner) => { - let close = self.close(inner, "v"); - format!("for (final v in {e}) {{ {close} }}") + let close = self.close(inner, &v); + format!("for (final {v} in {e}) {{ {close} }}") } _ => format!("{e}.close();"), } diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 22667f7ac..abbdc935a 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -1007,7 +1007,12 @@ impl Gen<'_> { .fields .iter() .zip(&bindings) - .map(|(f, b)| format!("{}: {b}", f.name)) + // A field named as its binding is, `f0`, takes the shorthand, which + // the lint would ask for. + .map(|(f, b)| match f.name == *b { + true => b.clone(), + false => format!("{}: {b}", f.name), + }) .collect(); format!("{path} {{ {} }}", fields.join(", ")) } diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index e50144fc1..466d3d59c 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -6,6 +6,7 @@ //! The reject fixture's export must fail at each item it cannot export, and generated //! type shapes round-trip in every language ([`shapes`]). +#[path = "fixtures/shapes.rs"] mod shapes; use std::env::consts::EXE_SUFFIX; @@ -858,7 +859,7 @@ fn api_dart() { let generated = api.out.join("dart"); run(Command::new("dart").args(["analyze", "--fatal-infos"]).arg(&generated)); let lib = api.lib.join(PYTHON_LIB); - let exe = dart_compile(&dir, &generated, &lib, &manifest_dir().join("tests").join("api.dart")); + let exe = dart_compile("api", &dir, &generated, &lib, &manifest_dir().join("tests").join("api.dart")); let dart = run(&mut dart_run(&exe, &[])); assert_eq!( String::from_utf8_lossy(&dart.stdout), @@ -912,7 +913,7 @@ fn api_dart() { "import 'package:api/api.dart' as api;\n\nvoid main() => api.dropped();\n", ) .unwrap(); - let exe = dart_compile(&contract, &contract, &lib, &main); + let exe = dart_compile("api", &contract, &contract, &lib, &main); let output = dart_run(&exe, &[]).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not fail"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -923,9 +924,9 @@ fn api_dart() { } /// `main` compiled into a kernel snapshot in `dir`, importing the generated Dart in -/// `package` as `package:api/api.dart`, whose native asset is the shared library `lib`: as -/// a package's build hook would bundle it, which `dart compile` has no flag for. -fn dart_compile(dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf { +/// `package` as `package:/.dart`, whose native asset is the shared library +/// `lib`: as a package's build hook would bundle it, which `dart compile` has no flag for. +fn dart_compile(name: &str, dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf { let sdk = std::fs::canonicalize(which("dart")) .unwrap() .parent() @@ -937,7 +938,7 @@ fn dart_compile(dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf std::fs::write( &config, format!( - "{{\"configVersion\": 2, \"packages\": [{{\"name\": \"api\", \"rootUri\": \"{root}/\", \"packageUri\": \"./\"}}]}}\n" + "{{\"configVersion\": 2, \"packages\": [{{\"name\": \"{name}\", \"rootUri\": \"{root}/\", \"packageUri\": \"./\"}}]}}\n" ), ) .unwrap(); @@ -954,7 +955,7 @@ fn dart_compile(dir: &Path, package: &Path, lib: &Path, main: &Path) -> PathBuf std::fs::write( &assets, format!( - "format-version: [1, 0, 0]\nnative-assets:\n {os}_{arch}:\n \"package:api/api.dart\": [absolute, {:?}]\n", + "format-version: [1, 0, 0]\nnative-assets:\n {os}_{arch}:\n \"package:{name}/{name}.dart\": [absolute, {:?}]\n", lib.display().to_string() ), ) diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 796124ed1..f4bf5cd63 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -151,8 +151,9 @@ fn sample(rng: &mut Rng, leaves: &[Shape], depth: u32) -> Shape { if depth == 0 || rng.below(3) == 0 { return leaves[rng.below(leaves.len())].clone(); } + let wrapper = rng.below(10); let mut inner = || Box::new(sample(rng, leaves, depth - 1)); - match rng.below(10) { + match wrapper { 0 => Shape::Option(inner()), 1 => Shape::Vec(inner()), 2 => Shape::Box(inner()), @@ -515,6 +516,12 @@ pub enum Call { } impl Call { + /// The `N` of its shape. + pub fn shape(self) -> usize { + let (Call::Echo(n, _) | Call::Lent(n, _) | Call::Peek(n, _) | Call::Made(n, _) | Call::Later(n, _)) = self; + n + } + pub fn label(self) -> String { match self { Call::Echo(n, seed) => format!("echo {n} {seed}"), @@ -532,6 +539,7 @@ const SEEDS: [u64; 3] = [0, 1, 0xdead_beef]; /// The shapes' crate, as the census left it: which shapes export, and the script. pub struct Crate { pub dir: PathBuf, + pub shapes: Vec, /// Each shape's Rust type, by `N`. pub types: Vec, pub calls: Vec, @@ -567,7 +575,7 @@ pub fn emit(name: &str, shapes: &[Shape]) -> Crate { } let lib = format!( "//! Generated by `tests/fixtures/shapes.rs`: each shape's `make`, `echo`, and `digest`.\n\ - #![allow(clippy::all, reason = \"generated\")]\n{RUNTIME}\n{}{functions}", + #![allow(clippy::all, dead_code, reason = \"generated\")]\n{RUNTIME}\n{}{functions}", emit.items ); let manifest = format!( @@ -628,6 +636,7 @@ pub fn emit(name: &str, shapes: &[Shape]) -> Crate { .current_dir(&dir)); Crate { dir, + shapes: shapes.to_vec(), types, calls, transcript: String::from_utf8_lossy(&transcript.stdout).into_owned(), @@ -692,8 +701,750 @@ fn ir_covered() { ); } -#[test] -fn probe() { - let krate = emit("shapes", &gate()); - eprintln!("{}", krate.transcript.lines().count()); + +/// The shapes' crate, exported and built once for each language's test. +pub struct Built { + pub krate: Crate, + /// The export. + pub out: PathBuf, + /// Where cargo built the static library and the shared one Kotlin and Dart load. + pub lib: PathBuf, + /// What a program linking the static library links besides, as rustc prints it. + pub native: Vec, +} + +/// The shared library of the crate `name`, which Python, Kotlin, and Dart load. +fn shared(name: &str) -> String { + match (cfg!(windows), cfg!(target_os = "macos")) { + (true, _) => format!("{name}.dll"), + (_, true) => format!("lib{name}.dylib"), + _ => format!("lib{name}.so"), + } +} + +/// Emit, census, export, and build the crate of `shapes` named `name`. +pub fn build(name: &str, shapes: &[Shape]) -> Built { + let krate = emit(name, shapes); + let out = krate.dir.join("out"); + let _ = std::fs::remove_dir_all(&out); + run(Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(krate.dir.join("ffi.toml")) + .arg("--out") + .arg(&out) + .env("CARGO_TARGET_DIR", krate.dir.join("build"))); + let target = krate.dir.join("glue"); + let build = |feature: &str| { + run(cargo() + .args(["build", "--quiet", "--locked", "--manifest-path"]) + .arg(out.join("Cargo.toml")) + .arg("--target-dir") + .arg(&target) + .args(["--features", feature]) + .env("RUSTFLAGS", "-D warnings --print=native-static-libs")) + }; + // As the api fixture's: the CPython extension builds apart and moves beside the Python. + build("python"); + let lib = shared(name); + std::fs::rename(target.join("debug").join(&lib), out.join("python").join(&lib)).unwrap(); + let built = build("kotlin,dart"); + let native = String::from_utf8_lossy(&built.stderr) + .lines() + .find_map(|line| line.strip_prefix("note: native-static-libs: ")) + .expect("rustc prints the static library's native libraries") + .split_whitespace() + .map(str::to_string) + .collect(); + Built { + krate, + out, + lib: target.join("debug"), + native, + } +} + +impl Built { + fn name(&self) -> String { + self.krate.dir.file_name().unwrap().to_string_lossy().into_owned() + } + + fn staticlib(&self) -> PathBuf { + let name = self.name(); + self.lib.join(match cfg!(target_env = "msvc") { + true => format!("{name}.lib"), + false => format!("lib{name}.a"), + }) + } + + /// A directory of its own for `language`'s driver. + fn dir(&self, language: &str) -> PathBuf { + let dir = self.krate.dir.join("drivers").join(language); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// The shapes `language` gets wrong today, by [`KNOWN`], each entry of which must match + /// one. + fn known(&self, language: &str) -> Vec { + let mut known = Vec::new(); + for (_, what, skips, quest) in KNOWN.iter().filter(|(l, ..)| *l == language) { + let matched: Vec = (0..self.krate.shapes.len()).filter(|&n| skips(&self.krate.shapes[n])).collect(); + assert!( + !matched.is_empty(), + "KNOWN's {language} entry for {what} ({quest}) matches no shape: remove it" + ); + known.extend(matched); + } + known + } + + /// Asserts `language`'s driver printed the native transcript but for the shapes it gets + /// wrong today, naming each line that differs by its shape's Rust type. + fn check(&self, language: &str, known: &[usize], output: &std::process::Output) { + let got = String::from_utf8_lossy(&output.stdout); + let mut wrong = String::new(); + let expected: Vec<&str> = (self.krate.transcript.lines().zip(&self.krate.calls)) + .filter(|(_, call)| !known.contains(&call.shape())) + .map(|(line, _)| line) + .collect(); + let lines: Vec<&str> = got.lines().collect(); + for (i, want) in expected.iter().enumerate() { + let line = lines.get(i).copied().unwrap_or(""); + if line != *want { + let n: usize = want.split(' ').nth(1).unwrap().parse().unwrap(); + let _ = writeln!(wrong, " {want} ({}), got {line}", self.krate.types[n]); + } + } + assert!( + wrong.is_empty() && lines.len() == expected.len(), + "{language}'s round trips differ from native Rust's in {} of {} lines, {} printed:\n{wrong}{}", + wrong.lines().count(), + expected.len(), + lines.len(), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +/// Whether `shape` or a shape within it meets `f`. +fn within(shape: &Shape, f: &dyn Fn(&Shape) -> bool) -> bool { + f(shape) + || match shape { + Shape::Option(inner) + | Shape::Vec(inner) + | Shape::Set(inner) + | Shape::Box(inner) + | Shape::Enum(inner) + | Shape::Object(inner) => within(inner, f), + Shape::Map(key, value) => within(key, f) || within(value, f), + Shape::Tuple(elements) | Shape::Record(elements) => elements.iter().any(|e| within(e, f)), + _ => false, + } +} + +fn duration(shape: &Shape) -> bool { + within(shape, &|s| matches!(s, Shape::Duration)) +} + +fn bytes(shape: &Shape) -> bool { + within(shape, &|s| matches!(s, Shape::Bytes | Shape::Buf)) +} + +fn ipv6(shape: &Shape) -> bool { + within(shape, &|s| matches!(s, Shape::SocketAddr("SocketAddr" | "SocketAddrV6"))) +} + +/// A collection whose items are bytes or tuples holding them, which C++ takes as views. +fn viewed_items(shape: &Shape) -> bool { + let buf = |s: &Shape| matches!(s, Shape::String | Shape::Bytes | Shape::Buf | Shape::PathBuf); + let item = |s: &Shape| buf(s) || matches!(s, Shape::Tuple(elements) if elements.iter().any(buf)); + within(shape, &|s| match s { + Shape::Vec(inner) | Shape::Set(inner) => item(inner), + Shape::Map(key, value) => buf(key) || buf(value), + _ => false, + }) +} + +/// The round trips a binding gets wrong today, each the quest's to fix: the language, what +/// it gets wrong, the shapes it skips for it, and the quest. An entry matching no shape +/// fails, so the fix deletes it. +const KNOWN: &[(&str, &str, fn(&Shape) -> bool, &str)] = &[ + ("python", "a Duration's nanoseconds", duration, "/quest/m1/ffi-export/lossless.md"), + ("dart", "a Duration's nanoseconds", duration, "/quest/m1/ffi-export/lossless.md"), + ("kotlin", "a Duration's nanoseconds past 146 years", duration, "/quest/m1/ffi-export/lossless.md"), + ("kotlin", "an IPv6 flow label", ipv6, "/quest/m1/ffi-export/lossless.md"), + ("kotlin", "returned bytes passed back", bytes, "/quest/m1/ffi-export/owned-bufs.md"), + ("go", "an IPv6 flow label", ipv6, "/quest/m1/ffi-export/lossless.md"), + ("go", "returned bytes passed back", bytes, "/quest/m1/ffi-export/owned-bufs.md"), + ("cpp", "a returned collection of bytes passed back", viewed_items, "/quest/m1/ffi-export/cpp-views.md"), +]; + +/// A function the C header declares: its return type and each parameter's type and name. +struct Proto { + ret: String, + params: Vec<(String, String)>, +} + +/// The prototype of `function` in the C header. +fn prototype(header: &str, function: &str) -> Proto { + let declared = format!("{function}("); + let at = header + .match_indices(&declared) + .map(|(at, _)| at) + .find(|&at| header[..at].ends_with([' ', '*'])) + .unwrap_or_else(|| panic!("the header declares {function}")); + let start = header[..at].rfind('\n').map_or(0, |i| i + 1); + let ret = header[start..at].trim().to_string(); + let rest = &header[at + declared.len()..]; + let list = &rest[..rest.find(");").unwrap()]; + // Split at the commas outside a function pointer's parentheses. + let (mut params, mut depth, mut from) = (Vec::new(), 0, 0); + for (i, c) in list.char_indices() { + match c { + '(' => depth += 1, + ')' => depth -= 1, + ',' if depth == 0 => { + params.push(&list[from..i]); + from = i + 1; + } + _ => {} + } + } + params.push(&list[from..]); + let params = params + .into_iter() + .map(|param| { + let param = param.trim(); + match param.find("(*") { + // A function pointer: `R (*f)(void *data, P)`, named `f`. + Some(at) => (param.to_string(), param[at + 2..at + 2 + param[at + 2..].find(')').unwrap()].to_string()), + None => { + let split = param.rfind(|c: char| c == ' ' || c == '*').unwrap() + 1; + (param[..split].trim().to_string(), param[split..].to_string()) + } + } + }) + .collect(); + Proto { ret, params } +} + +impl Proto { + /// The type of the parameter named `name`. + fn param(&self, name: &str) -> &str { + &self.params.iter().find(|(_, n)| n == name).unwrap().0 + } + + /// Statements calling it with `args` that declare `var` of type `ty`: its return, or + /// what it writes through its out pointers, a returned tuple's gathered into the struct + /// `ty`. A poll's call must return `ready`. + fn call(&self, function: &str, args: &[String], ty: &str, var: &str, ready: Option<&str>) -> String { + let args = args.join(", "); + let outs = &self.params[self.params.len() - self.outs()..]; + if outs.is_empty() { + return format!("\t\t{ty} {var} = {function}({args});\n"); + } + let mut out = String::new(); + let mut pointers = Vec::new(); + let mut values = Vec::new(); + for (k, (pointer, _)) in outs.iter().enumerate() { + let _ = writeln!(out, "\t\t{} {var}_{k};", pointer.strip_suffix('*').unwrap().trim()); + pointers.push(format!("&{var}_{k}")); + values.push(format!("{var}_{k}")); + } + let call = format!("{function}({args}, {})", pointers.join(", ")); + let _ = match ready { + Some(ready) => writeln!(out, "\t\tif ({call} != {ready}) abort();"), + None => writeln!(out, "\t\t{call};"), + }; + let _ = match values.len() { + 1 => writeln!(out, "\t\t{ty} {var} = {var}_0;"), + _ => writeln!(out, "\t\t{ty} {var} = ({ty}){{{}}};", values.join(", ")), + }; + out + } + + /// How many out pointers it writes its output through: `out`, or `out0` on for a tuple. + fn outs(&self) -> usize { + let out = |name: &str| name == "out" || name.strip_prefix("out").is_some_and(|k| k.parse::().is_ok()); + self.params.iter().filter(|(_, name)| out(name)).count() + } + + /// What a function pointer parameter `(*name)` returns and takes after its data. + fn pointer(&self, name: &str) -> (String, String) { + let ty = self.param(name); + let (ret, rest) = ty.split_once(&format!(" (*{name})(")).unwrap(); + let param = rest.strip_prefix("void *data, ").unwrap().strip_suffix(')').unwrap(); + (ret.to_string(), param.to_string()) + } +} + +/// The C driver: each call's value in a block of its own, typed by the header's prototypes. +fn c(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let header = std::fs::read_to_string(built.out.join("include").join(format!("{name}.h"))).unwrap(); + let proto = |item: Item, n: usize| prototype(&header, &format!("{name}_{}", item.name(n))); + let mut closures = String::new(); + let mut body = String::new(); + let mut defined = std::collections::BTreeSet::new(); + for call in calls { + let label = call.label(); + let (Call::Echo(n, seed) | Call::Lent(n, seed) | Call::Peek(n, seed) | Call::Made(n, seed) | Call::Later(n, seed)) = *call; + let make = proto(Item::Make, n); + let digest = proto(Item::Digest, n); + let digested = digest.param("v").to_string(); + let fname = |item: Item| format!("{name}_{}", item.name(n)); + let mut block = String::new(); + match *call { + Call::Echo(..) => { + let echo = proto(Item::Echo, n); + block += &make.call(&fname(Item::Make), &[format!("{seed}u")], echo.param("v"), "a", None); + block += &echo.call(&fname(Item::Echo), &["a".into()], &digested, "b", None); + let _ = writeln!(block, "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", fname(Item::Digest)); + } + Call::Lent(..) | Call::Peek(..) => { + let item = if matches!(call, Call::Lent(..)) { Item::Lent } else { Item::Peek }; + if defined.insert(item.name(n)) { + let (_, param) = proto(item, n).pointer("f"); + let _ = writeln!( + closures, + "static uint64_t {}(void *data, {param} v) {{\n\t(void)data;\n\treturn {}(v);\n}}\n", + item.name(n), + fname(Item::Digest) + ); + } + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}({seed}u, {}, NULL));", + fname(item), + item.name(n) + ); + } + Call::Made(..) => { + if defined.insert(Item::Made.name(n)) { + let (ret, _) = proto(Item::Made, n).pointer("f"); + let made = make.call(&fname(Item::Make), &["seed".into()], &ret, "v", None); + // A buffer C hands Rust is one buf_copy made. + let value = match ret == format!("{name}_buf") { + true => format!("{name}_buf_copy(v)"), + false => "v".into(), + }; + let _ = writeln!( + closures, + "static {ret} {}(void *data, uint64_t seed) {{\n\t(void)data;\n\t{{\n{made}\t\treturn {value};\n\t}}\n}}\n", + Item::Made.name(n) + ); + } + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}({seed}u, {}, NULL));", + fname(Item::Made), + Item::Made.name(n) + ); + } + Call::Later(..) => { + let later = proto(Item::Later, n); + let future = format!("{}_future", fname(Item::Later)); + let poll = prototype(&header, &format!("{}_poll", fname(Item::Later))); + block += &make.call(&fname(Item::Make), &[format!("{seed}u")], later.param("v"), "a", None); + let _ = writeln!(block, "\t\t{future} *f = {}(a);", fname(Item::Later)); + // Ready at once: an async fn that awaits nothing. + let ready = format!("{}_POLL_READY", name.to_uppercase()); + let polled = poll.call( + &format!("{}_poll", fname(Item::Later)), + &["f".into(), format!("({name}_waker){{0}}")], + &digested, + "b", + Some(&ready), + ); + block += &polled; + let _ = writeln!(block, "\t\t{future}_free(f);"); + let _ = writeln!(block, "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", fname(Item::Digest)); + } + } + let _ = writeln!(body, "\t{{\n{block}\t}}"); + } + format!( + "#include \n#include \n#include \n\n#include \"{name}.h\"\n\n{closures}int main(void) {{\n{body}\treturn 0;\n}}\n" + ) } + +/// The C++ driver, over the header's own types and futures. +fn cpp(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("{name}::digest_{n}({name}::echo_{n}({name}::make_{n}({seed}u)))"), + Call::Lent(n, seed) => { + format!("{name}::lent_{n}({seed}u, [](auto v) {{ return {name}::digest_{n}(std::move(v)); }})") + } + Call::Peek(n, seed) => format!("{name}::peek_{n}({seed}u, [](auto &&v) {{ return {name}::digest_{n}(v); }})"), + Call::Made(n, seed) => format!("{name}::made_{n}({seed}u, [](std::uint64_t s) {{ return {name}::make_{n}(s); }})"), + Call::Later(n, seed) => format!("{name}::digest_{n}({name}::later_{n}({name}::make_{n}({seed}u)).get())"), + }; + let _ = writeln!(body, "\tstd::cout << \"{} \" << {value} << '\\n';", call.label()); + } + format!("#include \n#include \n#include \n\n#include \"{name}.hpp\"\n\nint main() {{\n{body}}}\n") +} + +/// Each shape's calls in a function of its own, so no function grows past a compiler's +/// limit: `(n, lines)`. +fn by_shape(calls: &[Call], line: impl Fn(Call) -> String) -> Vec<(usize, String)> { + let mut shapes: Vec<(usize, String)> = Vec::new(); + for call in calls { + let (Call::Echo(n, _) | Call::Lent(n, _) | Call::Peek(n, _) | Call::Made(n, _) | Call::Later(n, _)) = *call; + if shapes.last().is_none_or(|(last, _)| *last != n) { + shapes.push((n, String::new())); + } + let _ = writeln!(shapes.last_mut().unwrap().1, "{}", line(*call)); + } + shapes +} + +/// The Kotlin driver, on the JVM with kotlinx-coroutines. +fn kotlin(built: &Built, calls: &[Call]) -> String { + let shapes = by_shape(calls, |call| { + let value = match call { + Call::Echo(n, seed) => format!("digest{n}(echo{n}(make{n}({seed}uL)))"), + Call::Lent(n, seed) => format!("lent{n}({seed}uL) {{ digest{n}(it) }}"), + Call::Peek(n, seed) => format!("peek{n}({seed}uL) {{ digest{n}(it) }}"), + Call::Made(n, seed) => format!("made{n}({seed}uL) {{ make{n}(it) }}"), + Call::Later(n, seed) => format!("digest{n}(later{n}(make{n}({seed}uL)))"), + }; + format!("\tprintln(\"{} ${{{value}}}\")", call.label()) + }); + let mut out = format!("import {}.*\nimport kotlinx.coroutines.runBlocking\n\n", built.name()); + for (n, lines) in &shapes { + let _ = writeln!(out, "suspend fun shape{n}() {{\n{lines}}}\n"); + } + out.push_str("fun main() = runBlocking {\n"); + for (n, _) in &shapes { + let _ = writeln!(out, "\tshape{n}()"); + } + out.push_str("}\n"); + out +} + +/// The Python driver, awaiting futures on asyncio. +fn python(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("s.digest_{n}(s.echo_{n}(s.make_{n}({seed})))"), + Call::Lent(n, seed) => format!("s.lent_{n}({seed}, s.digest_{n})"), + Call::Peek(n, seed) => format!("s.peek_{n}({seed}, s.digest_{n})"), + Call::Made(n, seed) => format!("s.made_{n}({seed}, s.make_{n})"), + Call::Later(n, seed) => format!("s.digest_{n}(await s.later_{n}(s.make_{n}({seed})))"), + }; + let _ = writeln!(body, "\tprint(f\"{} {{{value}}}\")", call.label()); + } + format!("import asyncio\n\nimport {name} as s\n\n\nasync def main() -> None:\n{body}\n\nasyncio.run(main())\n") +} + +/// The Swift driver, top-level code in the generated module's `main.swift`. +fn swift(built: &Built, calls: &[Call]) -> String { + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("digest{n}(echo{n}(make{n}({seed})))"), + Call::Lent(n, seed) => format!("lent{n}({seed}) {{ digest{n}($0) }}"), + Call::Peek(n, seed) => format!("peek{n}({seed}) {{ digest{n}($0) }}"), + Call::Made(n, seed) => format!("made{n}({seed}) {{ make{n}($0) }}"), + Call::Later(n, seed) => format!("digest{n}(try await later{n}(make{n}({seed})))"), + }; + let _ = writeln!(body, "print(\"{} \\({value})\")", call.label()); + } + body +} + +/// The Go driver: `one` takes the value of a call returning it alone or with an error, +/// `tupleK` gathers a returned tuple's values into the `rt.TupleK` Go takes, and `lift` +/// makes a closure of a function of either form. +fn go(built: &Built, calls: &[Call], source: &str, rt: &str) -> String { + let mut body = String::new(); + // How the generated `func` returns: its values, and whether an error follows them. + let results = |function: &str| -> (usize, bool) { + let at = source.find(&format!("\nfunc {function}(")).unwrap_or_else(|| panic!("the Go declares {function}")); + let line = source[at + 1..].lines().next().unwrap(); + let mut depth = 0; + let mut close = 0; + for (i, c) in line.char_indices().skip(line.find('(').unwrap()) { + match c { + '(' | '[' => depth += 1, + ')' | ']' => depth -= 1, + _ => {} + } + if depth == 0 { + close = i; + break; + } + } + let result = line[close + 1..].trim().trim_end_matches('{').trim(); + let Some(list) = result.strip_prefix('(').and_then(|r| r.strip_suffix(')')) else { + return (1, false); + }; + let (mut types, mut depth) = (vec![String::new()], 0); + for c in list.chars() { + match c { + '(' | '[' => depth += 1, + ')' | ']' => depth -= 1, + ',' if depth == 0 => { + types.push(String::new()); + continue; + } + _ => {} + } + types.last_mut().unwrap().push(c); + } + let error = types.last().unwrap().trim() == "error"; + (types.len() - usize::from(error), error) + }; + // The value of a call of `function`, in the form a parameter takes it. + let value = |function: &str, call: String| match results(function) { + (1, _) => format!("one({call})"), + (k, false) => format!("tuple{k}({call})"), + (k, true) => format!("tuple{k}e({call})"), + }; + // A gatherer for each arity of tuple the runtime declares, which is each the API uses. + let mut tuples = String::new(); + for k in 2..=9 { + if !rt.contains(&format!("type Tuple{k}[")) { + continue; + } + let params: Vec = (0..k).map(|i| format!("T{i}")).collect(); + let types = params.join(", "); + let args: Vec = (0..k).map(|i| format!("v{i} T{i}")).collect(); + let fields: Vec = (0..k).map(|i| format!("V{i}: v{i}")).collect(); + let values: Vec = (0..k).map(|i| format!("v{i}")).collect(); + let _ = write!( + tuples, + "func tuple{k}[{types} any]({args}) rt.Tuple{k}[{types}] {{\n\treturn rt.Tuple{k}[{types}]{{{fields}}}\n}}\n\n\ + func tuple{k}e[{types} any]({args}, err error) rt.Tuple{k}[{types}] {{\n\treturn one(tuple{k}({values}), err)\n}}\n\n", + args = args.join(", "), + fields = fields.join(", "), + values = values.join(", "), + ); + } + for call in calls { + let n = call.shape(); + let printed = match *call { + Call::Echo(n, seed) => { + let made = value(&format!("Make{n}"), format!("s.Make{n}({seed})")); + let echoed = value(&format!("Echo{n}"), format!("s.Echo{n}({made})")); + format!("one(s.Digest{n}({echoed}))") + } + Call::Lent(_, seed) | Call::Peek(_, seed) => { + let item = if matches!(call, Call::Lent(..)) { "Lent" } else { "Peek" }; + let lift = if results(&format!("Digest{n}")).1 { "liftErr" } else { "lift" }; + format!("s.{item}{n}({seed}, {lift}(s.Digest{n}))") + } + Call::Made(_, seed) => { + let lift = if results(&format!("Make{n}")).1 { "liftErr" } else { "lift" }; + format!("s.Made{n}({seed}, {lift}(s.Make{n}))") + } + Call::Later(_, seed) => { + let made = value(&format!("Make{n}"), format!("s.Make{n}({seed})")); + let later = value(&format!("Later{n}"), format!("s.Later{n}(ctx, {made})")); + format!("one(s.Digest{n}({later}))") + } + }; + let _ = writeln!(body, "\tfmt.Println(\"{}\", {printed})", call.label()); + } + format!( + "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\ts \"{name}/{name}\"\n{import})\n\n\ + func one[T any](v T, err ...error) T {{\n\tif len(err) > 0 && err[0] != nil {{\n\t\tpanic(err[0])\n\t}}\n\treturn v\n}}\n\n\ + {tuples}func lift[A, B any](f func(A) B) func(A) B {{\n\treturn f\n}}\n\n\ + func liftErr[A, B any](f func(A) (B, error)) func(A) B {{\n\treturn func(a A) B {{ return one(f(a)) }}\n}}\n\n\ + func main() {{\n\tctx := context.Background()\n\t_ = ctx\n{body}}}\n", + name = built.name(), + import = match tuples.is_empty() { + true => String::new(), + false => format!("\t\"{}/rt\"\n", built.name()), + }, + ) +} + +/// The Dart driver; Dart's `int` is Rust's `u64` in two's complement, printed unsigned. +fn dart(built: &Built, calls: &[Call]) -> String { + let name = built.name(); + let mut body = String::new(); + for call in calls { + let value = match *call { + Call::Echo(n, seed) => format!("s.digest{n}(s.echo{n}(s.make{n}({seed})))"), + Call::Lent(n, seed) => format!("s.lent{n}({seed}, s.digest{n})"), + Call::Peek(n, seed) => format!("s.peek{n}({seed}, s.digest{n})"), + Call::Made(n, seed) => format!("s.made{n}({seed}, s.make{n})"), + Call::Later(n, seed) => format!("s.digest{n}(await s.later{n}(s.make{n}({seed})))"), + }; + let _ = writeln!(body, " print('{} ${{u({value})}}');", call.label()); + } + format!( + "import 'package:{name}/{name}.dart' as s;\n\nString u(int v) => BigInt.from(v).toUnsigned(64).toString();\n\n\ + Future main() async {{\n{body}}}\n" + ) +} + +/// Build and run `language`'s driver of `built`, checking it against the native transcript. +fn round_trip(built: &Built, language: &str) { + let name = built.name(); + let dir = built.dir(language); + let exe = dir.join(format!("shapes{}", std::env::consts::EXE_SUFFIX)); + let known = built.known(language); + let calls: Vec = (built.krate.calls.iter().copied()) + .filter(|call| !known.contains(&call.shape())) + .collect(); + let calls = &calls[..]; + let output = match language { + "c" => { + let main = dir.join("main.c"); + std::fs::write(&main, c(built, calls)).unwrap(); + let cc = std::env::var_os("CC").unwrap_or_else(|| "cc".into()); + run(Command::new(cc) + .args(["-std=c23", "-pedantic", "-Wall", "-Wextra", "-Werror", "-I"]) + .arg(built.out.join("include")) + .arg(&main) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe)); + run(&mut Command::new(&exe)) + } + "cpp" => { + let main = dir.join("main.cpp"); + std::fs::write(&main, cpp(built, calls)).unwrap(); + let cxx = std::env::var_os("CXX").unwrap_or_else(|| "c++".into()); + run(Command::new(cxx) + .args(["-std=c++17", "-Wall", "-Wextra", "-Werror", "-I"]) + .arg(built.out.join("include")) + .arg(&main) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe)); + run(&mut Command::new(&exe)) + } + "kotlin" => { + let main = dir.join("main.kt"); + std::fs::write(&main, kotlin(built, calls)).unwrap(); + let classes = dir.join("classes"); + run(Command::new("kotlinc") + .args(["-Werror", "-cp"]) + .arg(super::coroutines()) + .arg(built.out.join("kotlin").join(format!("{name}.kt"))) + .arg(&main) + .arg("-d") + .arg(&classes)); + let mut classpath = classes.into_os_string(); + classpath.push(if cfg!(windows) { ";" } else { ":" }); + classpath.push(super::coroutines()); + run(Command::new("kotlin") + .arg(format!("-Djava.library.path={}", built.lib.display())) + .arg("-cp") + .arg(&classpath) + .arg("MainKt")) + } + "python" => { + let main = dir.join("main.py"); + std::fs::write(&main, python(built, calls)).unwrap(); + run(Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())) + .args(["-P", "-X", "dev", "-W", "error"]) + .arg(&main) + .env("PYTHONPATH", built.out.join("python"))) + } + "swift" => { + let module = built.out.join("swift"); + std::fs::copy(module.join(format!("{name}.swift")), dir.join(format!("{name}.swift"))).unwrap(); + std::fs::write(dir.join("main.swift"), swift(built, calls)).unwrap(); + run(super::swiftc(&module, 15) + .args(["-swift-version", "6", "-module-name", "Shapes"]) + .arg(dir.join(format!("{name}.swift"))) + .arg(dir.join("main.swift")) + .args(super::swift_collections()) + .arg(built.staticlib()) + .args(&built.native) + .arg("-o") + .arg(&exe)); + run(&mut Command::new(&exe)) + } + "go" => { + let export = dir.join("export"); + super::copy_dir(&built.out.join("go"), &export); + std::fs::create_dir_all(dir.join("include")).unwrap(); + let header = format!("{name}.h"); + std::fs::copy(built.out.join("include").join(&header), dir.join("include").join(&header)).unwrap(); + let source = std::fs::read_to_string(export.join(&name).join(format!("{name}.go"))).unwrap(); + let rt = std::fs::read_to_string(export.join("rt").join("types.go")).unwrap_or_default(); + std::fs::write(dir.join("main.go"), go(built, calls, &source, &rt)).unwrap(); + std::fs::write( + dir.join("go.mod"), + format!("module shapestest\n\ngo 1.24\n\nrequire {name} v0.0.0\n\nreplace {name} => ./export\n"), + ) + .unwrap(); + let mut ldflags = built.staticlib().into_os_string(); + for lib in &built.native { + ldflags.push(" "); + ldflags.push(lib); + } + run(Command::new("go") + .args(["build", "-o"]) + .arg(&exe) + .current_dir(&dir) + .env("CGO_ENABLED", "1") + .env("CGO_LDFLAGS", ldflags) + .env("GOFLAGS", "-buildvcs=false") + .env("GOTOOLCHAIN", "local")); + run(&mut Command::new(&exe)) + } + "dart" => { + let main = dir.join("main.dart"); + std::fs::write(&main, dart(built, calls)).unwrap(); + let lib = built.lib.join(shared(&name)); + let dill = super::dart_compile(&name, &dir, &built.out.join("dart"), &lib, &main); + run(&mut super::dart_run(&dill, &[])) + } + _ => unreachable!("no driver for {language}"), + }; + built.check(language, &known, &output); +} + +/// The gate's subset of shapes, built once. +fn gated() -> &'static Built { + static BUILT: std::sync::OnceLock = std::sync::OnceLock::new(); + BUILT.get_or_init(|| build("shapes", &gate())) +} + +/// Every sampled shape, built once: the nightly's. +fn sampled() -> &'static Built { + static BUILT: std::sync::OnceLock = std::sync::OnceLock::new(); + BUILT.get_or_init(|| build("shapes_all", &shapes(SAMPLES))) +} + +/// How many shapes the nightly samples beyond each leaf in each wrapper. +const SAMPLES: usize = 64; + +macro_rules! languages { + ($($language:ident),*) => { + mod gate { + $( + #[test] + fn $language() { + super::round_trip(super::gated(), stringify!($language)); + } + )* + } + + mod nightly { + $( + #[test] + #[ignore = "the nightly's: every sampled shape, just ffi shapes"] + fn $language() { + super::round_trip(super::sampled(), stringify!($language)); + } + )* + } + }; +} + +languages!(c, cpp, kotlin, python, swift, go, dart); From 668b7a14f3c58f21c7ce5234a6065823b315d252 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 05:03:08 -0800 Subject: [PATCH 4/9] wip(ffi): nightly sample, Python and Dart converters by Rust type, C++ bool lists --- .github/workflows/nightly.yml | 4 ++ ffi/justfile | 4 ++ ffi/runtime/python.rs | 15 +++-- ffi/runtime/rt.hpp | 19 ++++-- ffi/src/dart.rs | 17 ++++- ffi/src/python.rs | 71 +++++++++++++++----- ffi/tests/fixtures/shapes.rs | 104 +++++++++++++++++------------ quest/m1/ffi-export/lossless.md | 6 +- quest/m1/ffi-export/owned-bufs.md | 4 +- quest/m1/ffi-export/round-trips.md | 26 ++++++++ 10 files changed, 191 insertions(+), 79 deletions(-) create mode 100644 quest/m1/ffi-export/round-trips.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8d79a734f..d6c417838 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -100,6 +100,10 @@ jobs: - name: Export run: nix develop --command just ffi moq + # Every sampled type shape round-tripped in every language; the gate runs a subset. + - name: Shapes + run: nix develop --command just ffi shapes + # What exports, and every refusal, ranked. - name: Census run: nix develop --command just ffi census diff --git a/ffi/justfile b/ffi/justfile index fc4c72ddd..ad54696f2 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -31,6 +31,10 @@ check: test *args: cargo test --locked {{ args }} +# Round-trip every sampled type shape in every language, as the nightly does; the gate runs a subset. +shapes: + cargo test --locked --test fixtures shapes::nightly -- --ignored + # Shrink the api fixture's Kotlin with R8 and its generated rules, and run its test on the Android device adb reaches, in the Android shell (nix develop .#android) with ANDROID_HOME and ANDROID_NDK_HOME set. android: cargo test --locked --test fixtures -- --ignored --exact api_android diff --git a/ffi/runtime/python.rs b/ffi/runtime/python.rs index 7f39cf902..a8f4d4fe9 100644 --- a/ffi/runtime/python.rs +++ b/ffi/runtime/python.rs @@ -828,15 +828,16 @@ pub fn lent_bytes(buf: rt::Buf) -> R { } /// A path's bytes, from a `str`, `bytes`, or `os.PathLike`, as `os.fsencode` encodes it, -/// so a name a Unix path holds that is not UTF-8 crosses as it came. +/// so a name a Unix path holds that is not UTF-8 crosses as it came, and so does a NUL, +/// which a Rust path holds and `PyUnicode_FSConverter` refuses. pub fn path_in(obj: *mut PyObject, s: &mut Scope) -> R { - let mut encoded: *mut PyObject = null_mut(); - // SAFETY: the GIL is held, and `obj` is live; the converter writes a new `bytes`. + // SAFETY: the GIL is held, and `obj` is live; each call returns a new reference. unsafe { - if PyUnicode_FSConverter(obj, (&raw mut encoded).cast()) == 0 { - return Err(Raised); - } - let encoded = s.keep(Obj::new(encoded)?); + let path = s.keep(Obj::new(PyOS_FSPath(obj))?); + let encoded = match PyBytes_Check(path) != 0 { + true => path, + false => s.keep(Obj::new(PyUnicode_EncodeFSDefault(path))?), + }; let mut ptr: *mut c_char = null_mut(); let mut len = 0; if PyBytes_AsStringAndSize(encoded, &mut ptr, &mut len) != 0 { diff --git a/ffi/runtime/rt.hpp b/ffi/runtime/rt.hpp index a07e362da..3b39e0440 100644 --- a/ffi/runtime/rt.hpp +++ b/ffi/runtime/rt.hpp @@ -778,17 +778,22 @@ class arena { // `items` as the C array `C`, each lowered by `lower`: never null, so an empty one is // told apart from a missing one. Each item is mutable where `items` is, so a handle // moved in can be owned alone. + // The array is no `std::vector`, whose `bool` form holds no array, and `items` may be + // one, whose items are proxies rather than references. template C list(R &items, F lower) { using E = ::std::remove_const_t<::std::remove_pointer_t>; - auto array = ::std::make_shared<::std::vector>(); - array->reserve(items.size()); - for (auto &item : items) { - array->push_back(lower(item)); - } static const E empty{}; - const E *ptr = array->empty() ? &empty : array->data(); - ::std::size_t len = array->size(); + ::std::size_t len = items.size(); + if (len == 0) { + return C{&empty, 0, nullptr}; + } + ::std::shared_ptr array(new E[len]); + ::std::size_t i = 0; + for (auto &&item : items) { + array[i++] = lower(item); + } + const E *ptr = array.get(); held_.push_back(::std::move(array)); return C{ptr, len, nullptr}; } diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index 3f22ca78c..8cb5f4063 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -72,6 +72,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result { natives: BTreeMap, /// The conversions, polls, and trampolines, by name. helpers: BTreeMap, + /// Each writer's name, by the type it writes: types of one C struct, such as a `Vec` of + /// `SocketAddrV4` and one of `SocketAddr`, check what they write apart. + writers: HashMap, /// The types, then the top-level functions and constants. types: String, top: String, @@ -929,10 +933,17 @@ impl<'a> Gen<'a> { /// The function writing a Dart value of type `ty` into the C struct `c`. fn writer(&mut self, c: &str, ty: &Type) -> String { - let name = format!("_w_{c}"); - if self.helpers.contains_key(&name) { - return name; + if let Some(name) = self.writers.get(ty) { + return name.clone(); + } + let mut name = format!("_w_{c}"); + for k in 2.. { + if !self.helpers.contains_key(&name) { + break; + } + name = format!("_w_{c}_{k}"); } + self.writers.insert(ty.clone(), name.clone()); self.helpers.insert(name.clone(), String::new()); let class = self.struct_class(&Ty::Struct(c.to_string())); let dt = self.dt(ty); diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 183115cfe..ade2e249f 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -110,6 +110,7 @@ pub fn generate(abi: &Abi<'_>, packages: &str, contract: u64) -> Result { thrown: HashMap, /// Each module-level Python name, with what claimed it. names: BTreeMap, + /// The suffix of the glue's `r_` and `w_` of each option, tuple, and list type by + /// its C struct and what the struct holds: types of one struct, such as a `Vec` of + /// `SocketAddrV4` and one of `SocketAddr`, convert apart. + converters: HashMap<(String, Vec), String>, errors: Vec, /// The targets of the item being written, whose native functions check theirs. on: On, @@ -498,7 +503,7 @@ impl<'a> Gen<'a> { Type::U128 => format!("rt::u128_in({o})"), Type::Duration => format!("rt::duration_in({o})"), Type::SocketAddr(family) => format!("rt::addr_in({o}, {})", family_of(*family)), - Type::List(..) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + Type::List(..) => format!("r_{}({o}, s, {m})", self.converter(ty)), Type::Option(inner) => match inner.base() { Type::Object(i) => { let m = match abi.api.objects[*i].copy || matches!(**inner, Type::Ref(..)) { @@ -515,7 +520,7 @@ impl<'a> Gen<'a> { let some = self.lift(inner, o, m); format!("(if rt::is_none({o}) {{ Ok(crate::rt::List::NONE) }} else {{ {some} }})") } - _ => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + _ => format!("r_{}({o}, s, {m})", self.converter(ty)), }, Type::Box(inner) => self.lift(inner, o, m), Type::Ref(_, inner) => self.lift(inner, o, "false"), @@ -551,7 +556,7 @@ impl<'a> Gen<'a> { Type::Future => format!("rt::future_in({o}, s)"), Type::Range(int) => format!("rt::range_in::<{}>({o})", int.rust()), Type::Bound(_) | Type::Bounds(_) => format!("r_{}({o}, s)", abi.range(ty)), - Type::Tuple(_) => format!("r_{}({o}, s, {m})", self.c_struct(ty)), + Type::Tuple(_) => format!("r_{}({o}, s, {m})", self.converter(ty)), Type::Unit | Type::Closure(..) => unreachable!("{ty:?} is no value Python passes"), } } @@ -575,7 +580,7 @@ impl<'a> Gen<'a> { Type::U128 => format!("rt::u128_out({v})"), Type::Duration => format!("rt::duration_out({v})"), Type::SocketAddr(_) => format!("rt::addr_out({v})"), - Type::List(..) => format!("w_{}({v})", self.c_struct(ty)), + Type::List(..) => format!("w_{}({v})", self.converter(ty)), Type::Option(inner) => match inner.base() { Type::Object(i) => format!( "rt::object_opt({v}, {}, crate::{}_free)", @@ -586,7 +591,7 @@ impl<'a> Gen<'a> { let some = self.lower(inner, "v"); format!("{{ let v = {v}; if v.is_none() {{ rt::none() }} else {{ {some} }} }}") } - _ => format!("w_{}({v})", self.c_struct(ty)), + _ => format!("w_{}({v})", self.converter(ty)), }, Type::Box(inner) | Type::Ref(_, inner) => self.lower(inner, v), Type::Record(_) => format!("w_{}({v})", self.c_struct(ty)), @@ -605,7 +610,7 @@ impl<'a> Gen<'a> { format!("{{ let r = {v}; rt::range_out({start}, {end}) }}") } Type::Bound(_) | Type::Bounds(_) => format!("w_{}({v})", abi.range(ty)), - Type::Tuple(_) => format!("w_{}({v})", self.c_struct(ty)), + Type::Tuple(_) => format!("w_{}({v})", self.converter(ty)), Type::Waker => format!("rt::waker_out({v})"), Type::Callback(..) | Type::Closure(..) | Type::Future | Type::Reader => { unreachable!("{ty:?} is no value Rust returns") @@ -621,6 +626,36 @@ impl<'a> Gen<'a> { } } + /// The suffix of the glue's `r_` and `w_` converting the option, tuple, or list `ty`: + /// its C struct's name for the type the struct was made for, which [`Self::write`] + /// writes, else a name of its own, written here once. + fn converter(&mut self, ty: &Type) -> String { + let abi = self.abi; + let c = self.c_struct(ty); + let (held, made): (Vec, Vec) = match strip_refs(ty) { + Type::Option(inner) => (vec![*inner], vec![abi.options[&c].clone()]), + Type::List(_, elem) => (vec![*elem], vec![abi.lists[&c].clone()]), + Type::Tuple(elements) => (elements, abi.tuples[&c].clone()), + other => unreachable!("{other:?} has no converter"), + }; + let made: Vec = made.iter().map(strip_refs).collect(); + if held.iter().map(strip_refs).eq(made.iter().cloned()) { + return c; + } + let key = (c.clone(), held.clone()); + if let Some(name) = self.converters.get(&key) { + return name.clone(); + } + let name = format!("{c}_{}", self.converters.len() + 2); + self.converters.insert(key, name.clone()); + match strip_refs(ty) { + Type::Option(_) => self.option(&name, &c, &held[0]), + Type::List(..) => self.list(&name, &c, &held[0]), + _ => self.tuple(&name, &c, &held), + } + name + } + fn write(&mut self) { let abi = self.abi; let api = abi.api; @@ -636,10 +671,10 @@ impl<'a> Gen<'a> { self.range(c, ty); } for (c, inner) in &abi.options.clone() { - self.option(c, inner); + self.option(c, c, inner); } for (c, elements) in &abi.tuples.clone() { - self.tuple(c, elements); + self.tuple(c, c, elements); } for (c, n) in &abi.arrays { let name = Abi::array_class(*n); @@ -650,7 +685,7 @@ impl<'a> Gen<'a> { ); } for (c, elem) in &abi.lists.clone() { - self.list(c, elem); + self.list(c, c, elem); } for constant in &api.constants { let name = self.bare(&abi.name(&constant.path)).to_uppercase(); @@ -756,19 +791,19 @@ impl<'a> Gen<'a> { ); } - /// The glue of an option struct. - fn option(&mut self, c: &str, inner: &Type) { + /// The glue of an option struct `c` holding `inner`, as `r_` and `w_`. + fn option(&mut self, name: &str, c: &str, inner: &Type) { let read = self.lift(inner, "o", "m"); let write = self.lower(inner, "v.value"); let _ = write!( self.glue, - "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\tmatch rt::is_none(o) {{\n\t\ttrue => Ok(crate::{c} {{ some: false, value: unsafe {{ std::mem::zeroed() }} }}),\n\t\tfalse => Ok(crate::{c} {{ some: true, value: {}? }}),\n\t}}\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\tmatch v.some {{\n\t\ttrue => {write},\n\t\tfalse => rt::none(),\n\t}}\n}}\n\n", + "fn r_{name}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\tmatch rt::is_none(o) {{\n\t\ttrue => Ok(crate::{c} {{ some: false, value: unsafe {{ std::mem::zeroed() }} }}),\n\t\tfalse => Ok(crate::{c} {{ some: true, value: {}? }}),\n\t}}\n}}\n\nfn w_{name}(v: crate::{c}) -> R {{\n\tmatch v.some {{\n\t\ttrue => {write},\n\t\tfalse => rt::none(),\n\t}}\n}}\n\n", read ); } - /// The glue of a tuple struct, element by element. - fn tuple(&mut self, c: &str, elements: &[Type]) { + /// The glue of a tuple struct `c`, element by element, as `r_` and `w_`. + fn tuple(&mut self, name: &str, c: &str, elements: &[Type]) { let n = elements.len(); let mut reads = Vec::new(); let mut fields = Vec::new(); @@ -781,20 +816,20 @@ impl<'a> Gen<'a> { } let _ = write!( self.glue, - "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\t{}\n\tOk(crate::{c} {{ {} }})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::tuple(vec![{}])\n}}\n\n", + "fn r_{name}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\t{}\n\tOk(crate::{c} {{ {} }})\n}}\n\nfn w_{name}(v: crate::{c}) -> R {{\n\trt::tuple(vec![{}])\n}}\n\n", reads.join("\n\t"), fields.join(", "), writes.join(", ") ); } - /// The glue of a `Vec`'s array. - fn list(&mut self, c: &str, elem: &Type) { + /// The glue of a `Vec`'s array `c`, as `r_` and `w_`. + fn list(&mut self, name: &str, c: &str, elem: &Type) { let read = self.lift(elem, "o", "m"); let write = self.lower(elem, "e"); let _ = write!( self.glue, - "fn r_{c}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\trt::list_in(o, s, |o, s| {read})\n}}\n\nfn w_{c}(v: crate::{c}) -> R {{\n\trt::list_out(v, |e| {write})\n}}\n\n" + "fn r_{name}(o: *mut PyObject, s: &mut rt::Scope, m: bool) -> R {{\n\trt::list_in(o, s, |o, s| {read})\n}}\n\nfn w_{name}(v: crate::{c}) -> R {{\n\trt::list_out(v, |e| {write})\n}}\n\n" ); } diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index f4bf5cd63..555f54be0 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -197,7 +197,7 @@ fn is_ord(shape: &Shape) -> bool { /// The shapes a run covers: each leaf alone, each leaf in each wrapper, then `extra` /// sampled to depth 3 from a fixed seed. -pub fn shapes(extra: usize) -> Vec { +fn shapes(extra: usize) -> Vec { let leaves = leaves(); let mut shapes = leaves.clone(); let mut rng = Rng(0x5eed); @@ -223,7 +223,7 @@ pub fn shapes(extra: usize) -> Vec { /// The gate's subset: each leaf alone, and each wrapper around a buffer, a /// record field, and an object. -pub fn gate() -> Vec { +fn gate() -> Vec { let mut shapes = leaves(); for inner in [Shape::String, Shape::Int("i16"), Shape::Object(Box::new(Shape::U128))] { let boxed = || Box::new(inner.clone()); @@ -471,7 +471,7 @@ fn digest(value: &dyn std::fmt::Debug) -> u64 { /// The functions each shape `N` exports, which the census may refuse one by one. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum Item { +enum Item { Make, Echo, Digest, @@ -486,7 +486,7 @@ pub enum Item { } impl Item { - pub fn name(self, n: usize) -> String { + fn name(self, n: usize) -> String { let item = match self { Item::Make => "make", Item::Echo => "echo", @@ -502,7 +502,7 @@ impl Item { /// A line of the script each driver runs, printing `label value`. #[derive(Clone, Copy, Debug)] -pub enum Call { +enum Call { /// `digest_N(echo_N(make_N(seed)))` Echo(usize, u64), /// `lent_N(seed, digest_N)` @@ -517,12 +517,12 @@ pub enum Call { impl Call { /// The `N` of its shape. - pub fn shape(self) -> usize { + fn shape(self) -> usize { let (Call::Echo(n, _) | Call::Lent(n, _) | Call::Peek(n, _) | Call::Made(n, _) | Call::Later(n, _)) = self; n } - pub fn label(self) -> String { + fn label(self) -> String { match self { Call::Echo(n, seed) => format!("echo {n} {seed}"), Call::Lent(n, seed) => format!("lent {n} {seed}"), @@ -537,19 +537,19 @@ impl Call { const SEEDS: [u64; 3] = [0, 1, 0xdead_beef]; /// The shapes' crate, as the census left it: which shapes export, and the script. -pub struct Crate { - pub dir: PathBuf, - pub shapes: Vec, +struct Crate { + dir: PathBuf, + shapes: Vec, /// Each shape's Rust type, by `N`. - pub types: Vec, - pub calls: Vec, + types: Vec, + calls: Vec, /// The digests the script prints natively. - pub transcript: String, + transcript: String, } /// Write the crate of `shapes` named `name` in the scratch directory, ask the exporter what /// exports, and run its script natively. -pub fn emit(name: &str, shapes: &[Shape]) -> Crate { +fn emit(name: &str, shapes: &[Shape]) -> Crate { let dir = scratch().join(name); let _ = std::fs::remove_dir_all(dir.join("src")); let mut emit = Emit { @@ -703,14 +703,14 @@ fn ir_covered() { /// The shapes' crate, exported and built once for each language's test. -pub struct Built { - pub krate: Crate, +struct Built { + krate: Crate, /// The export. - pub out: PathBuf, + out: PathBuf, /// Where cargo built the static library and the shared one Kotlin and Dart load. - pub lib: PathBuf, + lib: PathBuf, /// What a program linking the static library links besides, as rustc prints it. - pub native: Vec, + native: Vec, } /// The shared library of the crate `name`, which Python, Kotlin, and Dart load. @@ -723,7 +723,7 @@ fn shared(name: &str) -> String { } /// Emit, census, export, and build the crate of `shapes` named `name`. -pub fn build(name: &str, shapes: &[Shape]) -> Built { +fn build(name: &str, shapes: &[Shape]) -> Built { let krate = emit(name, shapes); let out = krate.dir.join("out"); let _ = std::fs::remove_dir_all(&out); @@ -790,8 +790,9 @@ impl Built { let mut known = Vec::new(); for (_, what, skips, quest) in KNOWN.iter().filter(|(l, ..)| *l == language) { let matched: Vec = (0..self.krate.shapes.len()).filter(|&n| skips(&self.krate.shapes[n])).collect(); + // The gate's subset may hold none of an entry's shapes; the nightly's holds each. assert!( - !matched.is_empty(), + !matched.is_empty() || self.name() != "sampled", "KNOWN's {language} entry for {what} ({quest}) matches no shape: remove it" ); known.extend(matched); @@ -847,8 +848,17 @@ fn duration(shape: &Shape) -> bool { within(shape, &|s| matches!(s, Shape::Duration)) } +/// A `Vec` or `Bytes`: a `Vec` of `u8` ints is one, as Rust types it alike. +fn is_bytes(shape: &Shape) -> bool { + match shape { + Shape::Bytes | Shape::Buf => true, + Shape::Vec(inner) => matches!(**inner, Shape::Int("u8")), + _ => false, + } +} + fn bytes(shape: &Shape) -> bool { - within(shape, &|s| matches!(s, Shape::Bytes | Shape::Buf)) + within(shape, &is_bytes) } fn ipv6(shape: &Shape) -> bool { @@ -857,7 +867,7 @@ fn ipv6(shape: &Shape) -> bool { /// A collection whose items are bytes or tuples holding them, which C++ takes as views. fn viewed_items(shape: &Shape) -> bool { - let buf = |s: &Shape| matches!(s, Shape::String | Shape::Bytes | Shape::Buf | Shape::PathBuf); + let buf = |s: &Shape| matches!(s, Shape::String | Shape::PathBuf) || is_bytes(s); let item = |s: &Shape| buf(s) || matches!(s, Shape::Tuple(elements) if elements.iter().any(buf)); within(shape, &|s| match s { Shape::Vec(inner) | Shape::Set(inner) => item(inner), @@ -866,6 +876,11 @@ fn viewed_items(shape: &Shape) -> bool { }) } +/// A tuple of one element, which Kotlin returns as its element but takes whole. +fn single(shape: &Shape) -> bool { + matches!(shape, Shape::Tuple(elements) if elements.len() == 1) +} + /// The round trips a binding gets wrong today, each the quest's to fix: the language, what /// it gets wrong, the shapes it skips for it, and the quest. An entry matching no shape /// fails, so the fix deletes it. @@ -877,12 +892,12 @@ const KNOWN: &[(&str, &str, fn(&Shape) -> bool, &str)] = &[ ("kotlin", "returned bytes passed back", bytes, "/quest/m1/ffi-export/owned-bufs.md"), ("go", "an IPv6 flow label", ipv6, "/quest/m1/ffi-export/lossless.md"), ("go", "returned bytes passed back", bytes, "/quest/m1/ffi-export/owned-bufs.md"), - ("cpp", "a returned collection of bytes passed back", viewed_items, "/quest/m1/ffi-export/cpp-views.md"), + ("kotlin", "a returned tuple of one passed back", single, "/quest/m1/ffi-export/round-trips.md"), + ("cpp", "a returned collection of bytes passed back", viewed_items, "/quest/m1/ffi-export/round-trips.md"), ]; -/// A function the C header declares: its return type and each parameter's type and name. +/// A function the C header declares: each parameter's type and name. struct Proto { - ret: String, params: Vec<(String, String)>, } @@ -894,8 +909,6 @@ fn prototype(header: &str, function: &str) -> Proto { .map(|(at, _)| at) .find(|&at| header[..at].ends_with([' ', '*'])) .unwrap_or_else(|| panic!("the header declares {function}")); - let start = header[..at].rfind('\n').map_or(0, |i| i + 1); - let ret = header[start..at].trim().to_string(); let rest = &header[at + declared.len()..]; let list = &rest[..rest.find(");").unwrap()]; // Split at the commas outside a function pointer's parentheses. @@ -926,7 +939,7 @@ fn prototype(header: &str, function: &str) -> Proto { } }) .collect(); - Proto { ret, params } + Proto { params } } impl Proto { @@ -957,9 +970,10 @@ impl Proto { Some(ready) => writeln!(out, "\t\tif ({call} != {ready}) abort();"), None => writeln!(out, "\t\t{call};"), }; - let _ = match values.len() { - 1 => writeln!(out, "\t\t{ty} {var} = {var}_0;"), - _ => writeln!(out, "\t\t{ty} {var} = ({ty}){{{}}};", values.join(", ")), + // `out` is the value; `out0` on are a returned tuple's elements, of one or more. + let _ = match outs[0].1 == "out" { + true => writeln!(out, "\t\t{ty} {var} = {var}_0;"), + false => writeln!(out, "\t\t{ty} {var} = ({ty}){{{}}};", values.join(", ")), }; out } @@ -1143,20 +1157,28 @@ fn python(built: &Built, calls: &[Call]) -> String { format!("import asyncio\n\nimport {name} as s\n\n\nasync def main() -> None:\n{body}\n\nasyncio.run(main())\n") } -/// The Swift driver, top-level code in the generated module's `main.swift`. -fn swift(built: &Built, calls: &[Call]) -> String { - let mut body = String::new(); - for call in calls { - let value = match *call { +/// The Swift driver, in the generated module's `main.swift`: each shape's calls in a +/// function of its own, since type-checking them as one body of top-level code takes +/// tens of gigabytes. +fn swift(calls: &[Call]) -> String { + let shapes = by_shape(calls, |call| { + let value = match call { Call::Echo(n, seed) => format!("digest{n}(echo{n}(make{n}({seed})))"), Call::Lent(n, seed) => format!("lent{n}({seed}) {{ digest{n}($0) }}"), Call::Peek(n, seed) => format!("peek{n}({seed}) {{ digest{n}($0) }}"), Call::Made(n, seed) => format!("made{n}({seed}) {{ make{n}($0) }}"), Call::Later(n, seed) => format!("digest{n}(try await later{n}(make{n}({seed})))"), }; - let _ = writeln!(body, "print(\"{} \\({value})\")", call.label()); + format!("\tprint(\"{} \\({value})\")", call.label()) + }); + let mut out = String::new(); + for (n, lines) in &shapes { + let _ = writeln!(out, "func shape{n}() async throws {{\n{lines}}}\n"); + } + for (n, _) in &shapes { + let _ = writeln!(out, "try await shape{n}()"); } - body + out } /// The Go driver: `one` takes the value of a call returning it alone or with an error, @@ -1356,7 +1378,7 @@ fn round_trip(built: &Built, language: &str) { "swift" => { let module = built.out.join("swift"); std::fs::copy(module.join(format!("{name}.swift")), dir.join(format!("{name}.swift"))).unwrap(); - std::fs::write(dir.join("main.swift"), swift(built, calls)).unwrap(); + std::fs::write(dir.join("main.swift"), swift(calls)).unwrap(); run(super::swiftc(&module, 15) .args(["-swift-version", "6", "-module-name", "Shapes"]) .arg(dir.join(format!("{name}.swift"))) @@ -1418,7 +1440,7 @@ fn gated() -> &'static Built { /// Every sampled shape, built once: the nightly's. fn sampled() -> &'static Built { static BUILT: std::sync::OnceLock = std::sync::OnceLock::new(); - BUILT.get_or_init(|| build("shapes_all", &shapes(SAMPLES))) + BUILT.get_or_init(|| build("sampled", &shapes(SAMPLES))) } /// How many shapes the nightly samples beyond each leaf in each wrapper. diff --git a/quest/m1/ffi-export/lossless.md b/quest/m1/ffi-export/lossless.md index 259292d0d..826f9cfa9 100644 --- a/quest/m1/ffi-export/lossless.md +++ b/quest/m1/ffi-export/lossless.md @@ -7,7 +7,8 @@ Rust type exactly: a value passed in and returned is the same value, and no conv truncates, clamps, panics, traps, or aborts. A native type stands in only where it is exact; otherwise the runtime's own type crosses, with checked conversions to and from the idiomatic one that fail with the language's error. Proven in every language's `api` test -by the edge values of each type crossing both ways. +by the edge values of each type crossing both ways, and by deleting this quest's `KNOWN` +entries in [the shape tests](/ffi/tests/fixtures/shapes.rs). ## Plan @@ -22,7 +23,8 @@ What the audit found, each a place native Rust would have gone on: clamp (`ffi/runtime/python.rs`, `rt.dart`); Go's `time.Duration` panics on a negative one or past 292 years (`ffi_cgo.go`); Swift traps on a negative `Swift.Duration` or one past `Duration::MAX` (`rt.swift`); Kotlin's maps past `i64` seconds to `INFINITE` - and back to `i64::MAX` seconds (`rt.kt`). Seconds and nanoseconds, as std holds them, + and back to `i64::MAX` seconds (`rt.kt`), and past about 146 years, where + `kotlin.time.Duration` holds milliseconds, drops what is below one. Seconds and nanoseconds, as std holds them, cross exactly. moq's switch also asks that Go stop panicking on a negative one and that a duration field default to zero ([moq's quest](https://github.com/moq-dev/moq/blob/main/quest/m3/ffi-exporter.md)). diff --git a/quest/m1/ffi-export/owned-bufs.md b/quest/m1/ffi-export/owned-bufs.md index 093b4715f..c86d5992d 100644 --- a/quest/m1/ffi-export/owned-bufs.md +++ b/quest/m1/ffi-export/owned-bufs.md @@ -31,7 +31,9 @@ the call and never frees", so its owner is never read. header fails at load. Prove it in every language's `api` test with a payload read and written back, checked for -identity of the storage where the language can see it, and re-measure the bench's write +identity of the storage where the language can see it, delete its Kotlin and Go `KNOWN` +entries in [the shape tests](/ffi/tests/fixtures/shapes.rs), which a returned payload +passed back now compiles in, and re-measure the bench's write and throughput rows. Public API: the C ABI's meaning of an input buffer's owner (breaking), `_buf_clone`, diff --git a/quest/m1/ffi-export/round-trips.md b/quest/m1/ffi-export/round-trips.md new file mode 100644 index 000000000..9112b66bf --- /dev/null +++ b/quest/m1/ffi-export/round-trips.md @@ -0,0 +1,26 @@ +# [S] Every binding takes back what it returns + +## Goal + +A value a [rust2-ffi](/ffi/README.md) binding returns passes back into a parameter of the +same Rust type as it is, with no conversion by the caller. Two bindings break this today, +each a `KNOWN` entry in [the shape tests](/ffi/tests/fixtures/shapes.rs) that this quest +deletes: + +- C++ returns a collection of bytes, such as `Vec` or `BTreeMap>`, + as a `std::vector` of `buf`s, but takes it as a `const std::vector` of `view`s, which a + vector of `buf`s does not convert to. +- Kotlin returns a tuple of one, `(T,)`, as its `T`, but takes it as `Tuple1`. + +## Plan + +Found by the shape tests' nightly sample on 2026-11-13. + +- C++: take a collection whose items hold bytes as any range whose items convert to a + `view`, as `view` already takes any contiguous range of bytes, so a returned vector of + `buf`s, and the caller's vector of `std::string`s, pass as they are. +- Kotlin: return a tuple of one as `Tuple1`, as it takes one, rather than unwrapping it. + Swift has no tuple of one, so its unwrapping stays. + +Public API: C++'s parameter types for collections of bytes, which take more than before; +Kotlin's return type of a function returning a tuple of one (breaking). From b3420a8afec76f19b8d64e3980868b8381a3293e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 05:10:55 -0800 Subject: [PATCH 5/9] feat(ffi): generated type shapes round-trip in every language --- ffi/README.md | 11 ++ ffi/src/dart.rs | 6 +- ffi/tests/fixtures.rs | 8 +- ffi/tests/fixtures/shapes.rs | 191 ++++++++++++++++++++++++------ quest/m1/ffi-speed/shape-tests.md | 31 ----- 5 files changed, 177 insertions(+), 70 deletions(-) delete mode 100644 quest/m1/ffi-speed/shape-tests.md diff --git a/ffi/README.md b/ffi/README.md index 07563cb96..5c8929f04 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -932,6 +932,7 @@ just ffi moq # export moq's crates at the pinned revision, build them, and just ffi moq-cpp # the same export, running only moq/moq.cpp just ffi moq-swift # the same export, running only moq/moq.swift just ffi census # census each into target/census +just ffi shapes # round-trip every sampled type shape in every language, as the nightly does nix develop .#android --command just ffi android # the Kotlin test, shrunk by R8, on the Android device adb reaches nix develop .#bench --command just ffi bench # the binding benchmark: bench/ffi/README.md's cases from Rust, and through the export from C, C++, Kotlin, Python, Swift, Go, and Dart ``` @@ -975,6 +976,16 @@ each runtime check, which must panic with its message; a runtime package from an build must fail its contract check. The `target` test exports a package for the host, for wasm32, and for both into one binding, which builds for each, and staged with `--api` and `--with`; a build for a target not exported fails, and items that differ between the two are refused, naming both. +The shape tests (`tests/fixtures/shapes.rs`) sample value types from the IR's `Type`, each +leaf alone and in each wrapper, then nested to depth 3 from a fixed seed, and emit a crate +of `make_N(seed)`, `echo_N(v)`, and `digest_N(&v)` per shape, with the shape in a closure's +parameter and return and in a future besides. Its census decides what exports, so a shape +refused by design is left out rather than failed. Each language's driver is a generated +flat list of `digest(echo(make(seed)))` calls whose printed digests must match the crate's +own, run natively. The gate runs a subset; `just ffi shapes` runs every sampled shape, as +the nightly does. `KNOWN` lists what a binding still gets wrong, each with the quest that +fixes it, and fails once an entry matches no sampled shape, so the fix deletes it; a new +IR `Type` variant fails `ir_covered` until the sampler covers it. The `reject` fixture's expected errors, each with its notes, are `fixtures/reject/expected.txt`. The `lock` test exports the `lock` fixture for wasm32, staged, and the host, then again against that export's `api.json`: its `added` feature only adds and must pass, and its `next` diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index 8cb5f4063..b5a7b5ef7 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -1861,7 +1861,11 @@ impl<'a> Gen<'a> { fn close(&mut self, ty: &Type, e: &str) -> String { // A local named apart from `e`, which may be a parameter named `v`, and from the // local an outer level declared, which Dart would read before it is declared. - let v: String = e.chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }).collect::() + "_"; + let v: String = e + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect::() + + "_"; match ty { Type::Option(inner) => { let close = self.close(inner, &v); diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 466d3d59c..3a7216554 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -859,7 +859,13 @@ fn api_dart() { let generated = api.out.join("dart"); run(Command::new("dart").args(["analyze", "--fatal-infos"]).arg(&generated)); let lib = api.lib.join(PYTHON_LIB); - let exe = dart_compile("api", &dir, &generated, &lib, &manifest_dir().join("tests").join("api.dart")); + let exe = dart_compile( + "api", + &dir, + &generated, + &lib, + &manifest_dir().join("tests").join("api.dart"), + ); let dart = run(&mut dart_run(&exe, &[])); assert_eq!( String::from_utf8_lossy(&dart.stdout), diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 555f54be0..1e788f66f 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -93,7 +93,10 @@ const COVERED: &[(&str, &str)] = &[ ("Closure", "the positions lent_N, peek_N, and made_N"), ("Range", "Range"), ("Bound", "Bound"), - ("Bounds", "none: the form of an impl RangeBounds parameter, which no value has"), + ( + "Bounds", + "none: the form of an impl RangeBounds parameter, which no value has", + ), ("Tuple", "Tuple"), ("Const", "none: a const argument, which no value has"), ]; @@ -268,7 +271,10 @@ impl Emit { Shape::Buf => ("bytes::Bytes".into(), "bytes::Bytes::from(r.bytes())".into()), Shape::Duration => ("std::time::Duration".into(), "r.duration()".into()), Shape::Array(n) => (format!("[u8; {n}]"), "std::array::from_fn(|_| r.next() as u8)".into()), - Shape::PathBuf => ("std::path::PathBuf".into(), "std::path::PathBuf::from(r.string())".into()), + Shape::PathBuf => ( + "std::path::PathBuf".into(), + "std::path::PathBuf::from(r.string())".into(), + ), Shape::SocketAddr(family) => (format!("std::net::{family}"), format!("r.{}()", family.to_lowercase())), Shape::Range(int) => ( format!("std::ops::Range<{int}>"), @@ -288,11 +294,17 @@ impl Emit { Shape::Url => ("url::Url".into(), "r.url()".into()), Shape::Option(inner) => { let (ty, make) = self.shape(inner); - (format!("Option<{ty}>"), format!("match r.below(4) {{ 0 => None, _ => Some({make}) }}")) + ( + format!("Option<{ty}>"), + format!("match r.below(4) {{ 0 => None, _ => Some({make}) }}"), + ) } Shape::Vec(inner) => { let (ty, make) = self.shape(inner); - (format!("Vec<{ty}>"), format!("(0..r.below(4)).map(|_| {make}).collect()")) + ( + format!("Vec<{ty}>"), + format!("(0..r.below(4)).map(|_| {make}).collect()"), + ) } Shape::Set(inner) => { let (ty, make) = self.shape(inner); @@ -582,7 +594,10 @@ fn emit(name: &str, shapes: &[Shape]) -> Crate { "[package]\nname = \"{name}\"\nversion = \"0.0.0\"\nedition = \"2024\"\npublish = false\n\n\ [dependencies]\nbytes = \"1\"\nserde_json = \"1\"\nurl = \"2\"\n\n[workspace]\n" ); - super::write_all(&dir, &[("Cargo.toml", &manifest), ("src/lib.rs", &lib), ("ffi.toml", "")]); + super::write_all( + &dir, + &[("Cargo.toml", &manifest), ("src/lib.rs", &lib), ("ffi.toml", "")], + ); // The api fixture's versions, resolved offline. std::fs::copy(manifest_dir().join("Cargo.lock"), dir.join("Cargo.lock")).unwrap(); @@ -590,7 +605,10 @@ fn emit(name: &str, shapes: &[Shape]) -> Crate { let exported = |item: Item, n: usize| !refused.iter().any(|path| *path == item.name(n)); let mut excludes: Vec<&String> = refused.iter().collect(); excludes.sort(); - let config = excludes.iter().map(|path| format!(" \"{path}\",\n")).collect::(); + let config = excludes + .iter() + .map(|path| format!(" \"{path}\",\n")) + .collect::(); std::fs::write(dir.join("ffi.toml"), format!("exclude = [\n{config}]\n")).unwrap(); let mut calls = Vec::new(); @@ -701,7 +719,6 @@ fn ir_covered() { ); } - /// The shapes' crate, exported and built once for each language's test. struct Built { krate: Crate, @@ -788,8 +805,10 @@ impl Built { /// one. fn known(&self, language: &str) -> Vec { let mut known = Vec::new(); - for (_, what, skips, quest) in KNOWN.iter().filter(|(l, ..)| *l == language) { - let matched: Vec = (0..self.krate.shapes.len()).filter(|&n| skips(&self.krate.shapes[n])).collect(); + for Known { what, skips, quest, .. } in KNOWN.iter().filter(|k| k.language == language) { + let matched: Vec = (0..self.krate.shapes.len()) + .filter(|&n| skips(&self.krate.shapes[n])) + .collect(); // The gate's subset may hold none of an entry's shapes; the nightly's holds each. assert!( !matched.is_empty() || self.name() != "sampled", @@ -862,7 +881,9 @@ fn bytes(shape: &Shape) -> bool { } fn ipv6(shape: &Shape) -> bool { - within(shape, &|s| matches!(s, Shape::SocketAddr("SocketAddr" | "SocketAddrV6"))) + within(shape, &|s| { + matches!(s, Shape::SocketAddr("SocketAddr" | "SocketAddrV6")) + }) } /// A collection whose items are bytes or tuples holding them, which C++ takes as views. @@ -884,16 +905,71 @@ fn single(shape: &Shape) -> bool { /// The round trips a binding gets wrong today, each the quest's to fix: the language, what /// it gets wrong, the shapes it skips for it, and the quest. An entry matching no shape /// fails, so the fix deletes it. -const KNOWN: &[(&str, &str, fn(&Shape) -> bool, &str)] = &[ - ("python", "a Duration's nanoseconds", duration, "/quest/m1/ffi-export/lossless.md"), - ("dart", "a Duration's nanoseconds", duration, "/quest/m1/ffi-export/lossless.md"), - ("kotlin", "a Duration's nanoseconds past 146 years", duration, "/quest/m1/ffi-export/lossless.md"), - ("kotlin", "an IPv6 flow label", ipv6, "/quest/m1/ffi-export/lossless.md"), - ("kotlin", "returned bytes passed back", bytes, "/quest/m1/ffi-export/owned-bufs.md"), - ("go", "an IPv6 flow label", ipv6, "/quest/m1/ffi-export/lossless.md"), - ("go", "returned bytes passed back", bytes, "/quest/m1/ffi-export/owned-bufs.md"), - ("kotlin", "a returned tuple of one passed back", single, "/quest/m1/ffi-export/round-trips.md"), - ("cpp", "a returned collection of bytes passed back", viewed_items, "/quest/m1/ffi-export/round-trips.md"), +/// A round trip a binding gets wrong today. +struct Known { + language: &'static str, + what: &'static str, + /// Whether a shape holds what it gets wrong, which the language's driver skips. + skips: fn(&Shape) -> bool, + /// The quest that fixes it. + quest: &'static str, +} + +const KNOWN: &[Known] = &[ + Known { + language: "python", + what: "a Duration's nanoseconds", + skips: duration, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "dart", + what: "a Duration's nanoseconds", + skips: duration, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "kotlin", + what: "a Duration's nanoseconds past 146 years", + skips: duration, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "kotlin", + what: "an IPv6 flow label", + skips: ipv6, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "kotlin", + what: "returned bytes passed back", + skips: bytes, + quest: "/quest/m1/ffi-export/owned-bufs.md", + }, + Known { + language: "kotlin", + what: "a returned tuple of one passed back", + skips: single, + quest: "/quest/m1/ffi-export/round-trips.md", + }, + Known { + language: "go", + what: "an IPv6 flow label", + skips: ipv6, + quest: "/quest/m1/ffi-export/lossless.md", + }, + Known { + language: "go", + what: "returned bytes passed back", + skips: bytes, + quest: "/quest/m1/ffi-export/owned-bufs.md", + }, + Known { + language: "cpp", + what: "a returned collection of bytes passed back", + skips: viewed_items, + quest: "/quest/m1/ffi-export/round-trips.md", + }, ]; /// A function the C header declares: each parameter's type and name. @@ -931,9 +1007,12 @@ fn prototype(header: &str, function: &str) -> Proto { let param = param.trim(); match param.find("(*") { // A function pointer: `R (*f)(void *data, P)`, named `f`. - Some(at) => (param.to_string(), param[at + 2..at + 2 + param[at + 2..].find(')').unwrap()].to_string()), + Some(at) => ( + param.to_string(), + param[at + 2..at + 2 + param[at + 2..].find(')').unwrap()].to_string(), + ), None => { - let split = param.rfind(|c: char| c == ' ' || c == '*').unwrap() + 1; + let split = param.rfind([' ', '*']).unwrap() + 1; (param[..split].trim().to_string(), param[split..].to_string()) } } @@ -1003,7 +1082,11 @@ fn c(built: &Built, calls: &[Call]) -> String { let mut defined = std::collections::BTreeSet::new(); for call in calls { let label = call.label(); - let (Call::Echo(n, seed) | Call::Lent(n, seed) | Call::Peek(n, seed) | Call::Made(n, seed) | Call::Later(n, seed)) = *call; + let (Call::Echo(n, seed) + | Call::Lent(n, seed) + | Call::Peek(n, seed) + | Call::Made(n, seed) + | Call::Later(n, seed)) = *call; let make = proto(Item::Make, n); let digest = proto(Item::Digest, n); let digested = digest.param("v").to_string(); @@ -1014,10 +1097,18 @@ fn c(built: &Built, calls: &[Call]) -> String { let echo = proto(Item::Echo, n); block += &make.call(&fname(Item::Make), &[format!("{seed}u")], echo.param("v"), "a", None); block += &echo.call(&fname(Item::Echo), &["a".into()], &digested, "b", None); - let _ = writeln!(block, "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", fname(Item::Digest)); + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", + fname(Item::Digest) + ); } Call::Lent(..) | Call::Peek(..) => { - let item = if matches!(call, Call::Lent(..)) { Item::Lent } else { Item::Peek }; + let item = if matches!(call, Call::Lent(..)) { + Item::Lent + } else { + Item::Peek + }; if defined.insert(item.name(n)) { let (_, param) = proto(item, n).pointer("f"); let _ = writeln!( @@ -1073,7 +1164,11 @@ fn c(built: &Built, calls: &[Call]) -> String { ); block += &polled; let _ = writeln!(block, "\t\t{future}_free(f);"); - let _ = writeln!(block, "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", fname(Item::Digest)); + let _ = writeln!( + block, + "\t\tprintf(\"{label} %\" PRIu64 \"\\n\", {}(b));", + fname(Item::Digest) + ); } } let _ = writeln!(body, "\t{{\n{block}\t}}"); @@ -1093,13 +1188,19 @@ fn cpp(built: &Built, calls: &[Call]) -> String { Call::Lent(n, seed) => { format!("{name}::lent_{n}({seed}u, [](auto v) {{ return {name}::digest_{n}(std::move(v)); }})") } - Call::Peek(n, seed) => format!("{name}::peek_{n}({seed}u, [](auto &&v) {{ return {name}::digest_{n}(v); }})"), - Call::Made(n, seed) => format!("{name}::made_{n}({seed}u, [](std::uint64_t s) {{ return {name}::make_{n}(s); }})"), + Call::Peek(n, seed) => { + format!("{name}::peek_{n}({seed}u, [](auto &&v) {{ return {name}::digest_{n}(v); }})") + } + Call::Made(n, seed) => { + format!("{name}::made_{n}({seed}u, [](std::uint64_t s) {{ return {name}::make_{n}(s); }})") + } Call::Later(n, seed) => format!("{name}::digest_{n}({name}::later_{n}({name}::make_{n}({seed}u)).get())"), }; let _ = writeln!(body, "\tstd::cout << \"{} \" << {value} << '\\n';", call.label()); } - format!("#include \n#include \n#include \n\n#include \"{name}.hpp\"\n\nint main() {{\n{body}}}\n") + format!( + "#include \n#include \n#include \n\n#include \"{name}.hpp\"\n\nint main() {{\n{body}}}\n" + ) } /// Each shape's calls in a function of its own, so no function grows past a compiler's @@ -1188,7 +1289,9 @@ fn go(built: &Built, calls: &[Call], source: &str, rt: &str) -> String { let mut body = String::new(); // How the generated `func` returns: its values, and whether an error follows them. let results = |function: &str| -> (usize, bool) { - let at = source.find(&format!("\nfunc {function}(")).unwrap_or_else(|| panic!("the Go declares {function}")); + let at = source + .find(&format!("\nfunc {function}(")) + .unwrap_or_else(|| panic!("the Go declares {function}")); let line = source[at + 1..].lines().next().unwrap(); let mut depth = 0; let mut close = 0; @@ -1259,11 +1362,19 @@ fn go(built: &Built, calls: &[Call], source: &str, rt: &str) -> String { } Call::Lent(_, seed) | Call::Peek(_, seed) => { let item = if matches!(call, Call::Lent(..)) { "Lent" } else { "Peek" }; - let lift = if results(&format!("Digest{n}")).1 { "liftErr" } else { "lift" }; + let lift = if results(&format!("Digest{n}")).1 { + "liftErr" + } else { + "lift" + }; format!("s.{item}{n}({seed}, {lift}(s.Digest{n}))") } Call::Made(_, seed) => { - let lift = if results(&format!("Make{n}")).1 { "liftErr" } else { "lift" }; + let lift = if results(&format!("Make{n}")).1 { + "liftErr" + } else { + "lift" + }; format!("s.Made{n}({seed}, {lift}(s.Make{n}))") } Call::Later(_, seed) => { @@ -1370,10 +1481,12 @@ fn round_trip(built: &Built, language: &str) { "python" => { let main = dir.join("main.py"); std::fs::write(&main, python(built, calls)).unwrap(); - run(Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())) - .args(["-P", "-X", "dev", "-W", "error"]) - .arg(&main) - .env("PYTHONPATH", built.out.join("python"))) + run( + Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())) + .args(["-P", "-X", "dev", "-W", "error"]) + .arg(&main) + .env("PYTHONPATH", built.out.join("python")), + ) } "swift" => { let module = built.out.join("swift"); @@ -1395,7 +1508,11 @@ fn round_trip(built: &Built, language: &str) { super::copy_dir(&built.out.join("go"), &export); std::fs::create_dir_all(dir.join("include")).unwrap(); let header = format!("{name}.h"); - std::fs::copy(built.out.join("include").join(&header), dir.join("include").join(&header)).unwrap(); + std::fs::copy( + built.out.join("include").join(&header), + dir.join("include").join(&header), + ) + .unwrap(); let source = std::fs::read_to_string(export.join(&name).join(format!("{name}.go"))).unwrap(); let rt = std::fs::read_to_string(export.join("rt").join("types.go")).unwrap_or_default(); std::fs::write(dir.join("main.go"), go(built, calls, &source, &rt)).unwrap(); diff --git a/quest/m1/ffi-speed/shape-tests.md b/quest/m1/ffi-speed/shape-tests.md deleted file mode 100644 index 9f86f11c4..000000000 --- a/quest/m1/ffi-speed/shape-tests.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] Generated type shapes round-trip in every language - -## Goal - -A generator samples type shapes from [rust2-ffi](/ffi/README.md)'s own IR, in every -position a type can take (parameter, return, field, `Option`, `Vec`, closure, future), and -each language's test checks `digest(echo(make(seed)))` against the value native Rust -computes. A shape a generator mishandles, or one that reaches one of its `unreachable!` -arms, fails the test in that language. - -## Plan - -Found by the 2026-11-12 audit. Every language already matches native Rust's transcript on -the api fixture (`ffi/tests/fixtures.rs`), but the fixture covers each rule once and each -script is ported by hand, while the generators hold 92 `unreachable!` sites keyed on type -shapes and a new rule touches 19 to 38 files. - -- Sample `ir::Type` trees to a bounded depth with a fixed seed, and emit a fixture crate - of `make_N(seed)`, `echo_N(v)`, and `digest_N(&v)` per shape. -- Each language's driver is a generated flat list of `digest(echo(make(s))) == K`, with - `K` computed natively, so no driver holds type-specific code. -- The sampled set runs nightly; a small fixed subset joins the gate, and the PR reports - what it adds to `just ffi test`. -- A shape a generator refuses by design must fail the export, not the test: the sampler - asks the exporter which shapes export. - -Public API: none. - -## Related - -- [The C ABI under sanitizers](/quest/m1/ffi-speed/sanitizers.md) - the second proof, of memory and threads, on the same glue From 9ec550439f2f4307542607e01f9da30792269f7d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 05:19:13 -0800 Subject: [PATCH 6/9] fix(ffi): the C++ shape driver builds as the api test's does, with cl on MSVC --- ffi/tests/fixtures.rs | 34 ++++++++++++++++++++++++++-------- ffi/tests/fixtures/shapes.rs | 16 +++++++--------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 3a7216554..4f4c2e038 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -243,7 +243,7 @@ fn api_cpp() { for (standard, plain) in [(17, true), (20, false), (23, true)] { let exe = scratch().join("api").join(format!("api-c++{standard}{EXE_SUFFIX}")); run(&mut cxx( - api, + &api.link(), standard, plain, &manifest_dir().join("tests").join("api.cpp"), @@ -299,7 +299,7 @@ fn api_cpp() { ) .unwrap(); let contract = scratch().join("api").join(format!("contract{EXE_SUFFIX}")); - run(&mut cxx(api, 17, false, &mismatched, &contract)); + run(&mut cxx(&api.link(), 17, false, &mismatched, &contract)); let output = Command::new(&contract).output().unwrap(); assert!(!output.status.success(), "a mismatched contract did not abort"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -353,11 +353,29 @@ fn cpp_strict(api: &Api) { } } -/// A build of the C++ `src` against the api library into `exe`, at C++`standard`, with +/// What a program links of an export: its headers, its static library, and what that +/// links besides. +struct Link<'a> { + include: PathBuf, + staticlib: PathBuf, + native: &'a [String], +} + +impl Api { + fn link(&self) -> Link<'_> { + Link { + include: self.out.join("include"), + staticlib: self.staticlib(), + native: &self.native, + } + } +} + +/// A build of the C++ `src` against the `link`ed library into `exe`, at C++`standard`, with /// exceptions and RTTI off if `plain`, and warnings as errors: by `CXX` or the platform's /// compiler, given MSVC's flags where the target is MSVC's. -fn cxx(api: &Api, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command { - let include = api.out.join("include"); +fn cxx(link: &Link, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command { + let include = &link.include; if cfg!(target_env = "msvc") { // cl from a developer environment. Its C++23 is `c++latest`; its objects go beside `exe`. let mut cmd = Command::new(std::env::var_os("CXX").unwrap_or_else(|| "cl".into())); @@ -382,7 +400,7 @@ fn cxx(api: &Api, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command let mut out = OsString::from("-Fe"); out.push(exe); cmd.arg("-I").arg(include).arg(src).arg(obj).arg(out); - cmd.arg("-link").arg(api.staticlib()).args(&api.native); + cmd.arg("-link").arg(&link.staticlib).args(link.native); return cmd; } let mut cmd = Command::new(std::env::var_os("CXX").unwrap_or_else(|| "c++".into())); @@ -394,8 +412,8 @@ fn cxx(api: &Api, standard: u8, plain: bool, src: &Path, exe: &Path) -> Command cmd.args(["-pedantic", "-Wall", "-Wextra", "-Wshadow", "-Werror", "-I"]) .arg(include) .arg(src) - .arg(api.staticlib()) - .args(&api.native) + .arg(&link.staticlib) + .args(link.native) .arg("-o") .arg(exe); cmd diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 1e788f66f..9f74abcbf 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -1447,15 +1447,13 @@ fn round_trip(built: &Built, language: &str) { "cpp" => { let main = dir.join("main.cpp"); std::fs::write(&main, cpp(built, calls)).unwrap(); - let cxx = std::env::var_os("CXX").unwrap_or_else(|| "c++".into()); - run(Command::new(cxx) - .args(["-std=c++17", "-Wall", "-Wextra", "-Werror", "-I"]) - .arg(built.out.join("include")) - .arg(&main) - .arg(built.staticlib()) - .args(&built.native) - .arg("-o") - .arg(&exe)); + // The api fixture's build, so MSVC's runner builds it with cl. + let link = super::Link { + include: built.out.join("include"), + staticlib: built.staticlib(), + native: &built.native, + }; + run(&mut super::cxx(&link, 17, false, &main, &exe)); run(&mut Command::new(&exe)) } "kotlin" => { From bf3e98f274cd4cfa3ef2c0e373443679848c6947 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:26:23 -0800 Subject: [PATCH 7/9] fix(ffi): KNOWN counts only exported shapes and says the fix deletes its entry; swift-collections builds once per run Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 7 ++++--- ffi/tests/fixtures.rs | 12 ++++++++---- ffi/tests/fixtures/shapes.rs | 13 +++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index 5c8929f04..88865237c 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -932,7 +932,7 @@ just ffi moq # export moq's crates at the pinned revision, build them, and just ffi moq-cpp # the same export, running only moq/moq.cpp just ffi moq-swift # the same export, running only moq/moq.swift just ffi census # census each into target/census -just ffi shapes # round-trip every sampled type shape in every language, as the nightly does +just ffi shapes # round-trip every sampled type shape in every language but KNOWN's, as the nightly does nix develop .#android --command just ffi android # the Kotlin test, shrunk by R8, on the Android device adb reaches nix develop .#bench --command just ffi bench # the binding benchmark: bench/ffi/README.md's cases from Rust, and through the export from C, C++, Kotlin, Python, Swift, Go, and Dart ``` @@ -984,8 +984,9 @@ refused by design is left out rather than failed. Each language's driver is a ge flat list of `digest(echo(make(seed)))` calls whose printed digests must match the crate's own, run natively. The gate runs a subset; `just ffi shapes` runs every sampled shape, as the nightly does. `KNOWN` lists what a binding still gets wrong, each with the quest that -fixes it, and fails once an entry matches no sampled shape, so the fix deletes it; a new -IR `Type` variant fails `ir_covered` until the sampler covers it. +fixes it; the driver skips those shapes, so the fix deletes its entry, and the nightly fails +on an entry that matches no exported shape. A new IR `Type` variant fails `ir_covered` +until the sampler covers it. The `reject` fixture's expected errors, each with its notes, are `fixtures/reject/expected.txt`. The `lock` test exports the `lock` fixture for wasm32, staged, and the host, then again against that export's `api.json`: its `added` feature only adds and must pass, and its `next` diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 4f4c2e038..1ab382655 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -1028,11 +1028,15 @@ fn url(dir: &Path) -> String { /// The static libraries of swift-collections' modules the generated Swift imports, which /// `just ffi swift-collections` builds beside their modules, where [`swiftc`] finds them. +/// Built once, since two tests running the recipe at once race on its checkout and build. fn swift_collections() -> Vec { - run(Command::new("just") - .arg("--justfile") - .arg(manifest_dir().join("justfile")) - .arg("swift-collections")); + static BUILT: OnceLock<()> = OnceLock::new(); + BUILT.get_or_init(|| { + run(Command::new("just") + .arg("--justfile") + .arg(manifest_dir().join("justfile")) + .arg("swift-collections")); + }); let build = collections_dir(); ["BasicContainers", "InternalCollectionsUtilities", "SpanPreview"] .iter() diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 9f74abcbf..a04c48eed 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -801,13 +801,13 @@ impl Built { dir } - /// The shapes `language` gets wrong today, by [`KNOWN`], each entry of which must match - /// one. + /// The exported shapes `language` gets wrong today, by [`KNOWN`], each entry of which must + /// match one. fn known(&self, language: &str) -> Vec { let mut known = Vec::new(); for Known { what, skips, quest, .. } in KNOWN.iter().filter(|k| k.language == language) { let matched: Vec = (0..self.krate.shapes.len()) - .filter(|&n| skips(&self.krate.shapes[n])) + .filter(|&n| skips(&self.krate.shapes[n]) && self.krate.calls.iter().any(|call| call.shape() == n)) .collect(); // The gate's subset may hold none of an entry's shapes; the nightly's holds each. assert!( @@ -902,9 +902,6 @@ fn single(shape: &Shape) -> bool { matches!(shape, Shape::Tuple(elements) if elements.len() == 1) } -/// The round trips a binding gets wrong today, each the quest's to fix: the language, what -/// it gets wrong, the shapes it skips for it, and the quest. An entry matching no shape -/// fails, so the fix deletes it. /// A round trip a binding gets wrong today. struct Known { language: &'static str, @@ -915,6 +912,10 @@ struct Known { quest: &'static str, } +/// The round trips a binding gets wrong today, each the quest's to fix: the language, what +/// it gets wrong, the shapes it skips for it, and the quest. The nightly fails on an entry +/// matching no exported shape, but its shapes stay skipped after a fix, so the quest that +/// fixes one deletes its entry. const KNOWN: &[Known] = &[ Known { language: "python", From e319a986b064838f743f0f1043bede204963b35a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:03:52 -0800 Subject: [PATCH 8/9] fix(ffi): Dart writes an optional borrowed list, Python's numbered converters miss every default name, glue lends Option<&[T]> Co-Authored-By: Claude Opus 5.5 --- ffi/fixtures/api/src/lib.rs | 5 +++++ ffi/src/dart.rs | 2 +- ffi/src/glue.rs | 11 +++++++++++ ffi/src/python.rs | 13 ++++++++++++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index 5faf446d3..3f210a884 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -2186,6 +2186,11 @@ pub fn prefixed(payload: &[u8], prefix: Option<&[u8]>) -> Vec { [prefix.unwrap_or_default(), payload].concat() } +/// `a`'s sum plus `b`'s, if any: a list lent shared and optional after the same list owned. +pub fn sum_lists(a: Vec, b: Option<&[i32]>) -> i32 { + a.iter().chain(b.unwrap_or_default()).sum() +} + /// `point`'s coordinates summed, or 0 for none: an optional value, lent shared. pub fn coordinates(point: Option<&Point>) -> i32 { point.map_or(0, |point| point.x + point.y) diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index b5a7b5ef7..2a4a6ed47 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -899,7 +899,7 @@ impl<'a> Gen<'a> { Type::Slice | Type::Vec | Type::Bytes => format!("a_.bytes({dst}, {e});"), Type::List(..) => { let c = self.c_name(inner); - let w = self.writer(&c, inner); + let w = self.writer(&c, inner.base()); format!("{w}({e}, {dst}, a_);") } _ => { diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index abbdc935a..4b673372d 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -1364,6 +1364,17 @@ impl Gen<'_> { format!("let {local} = {};", self.lift(ty, c)), format!("{local}.as_deref()"), ), + // A lifted `Vec` lends the slice it derefs to. + (Type::List(Collection::Vec, _), _) => match m { + Mutability::Mut => ( + format!("let mut {local} = {};", self.lift(ty, c)), + format!("{local}.as_deref_mut()"), + ), + Mutability::Not => ( + format!("let {local} = {};", self.lift(ty, c)), + format!("{local}.as_deref()"), + ), + }, _ => ( format!("let {local} = {};", self.lift(ty, c)), format!("{local}.as_ref()"), diff --git a/ffi/src/python.rs b/ffi/src/python.rs index ade2e249f..7c4739453 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -646,7 +646,18 @@ impl<'a> Gen<'a> { if let Some(name) = self.converters.get(&key) { return name.clone(); } - let name = format!("{c}_{}", self.converters.len() + 2); + // A numbered name must miss every converter's: each struct's own, and the others numbered. + let taken = |name: &str| { + abi.options.contains_key(name) + || abi.lists.contains_key(name) + || abi.tuples.contains_key(name) + || abi.ranges.contains_key(name) + || self.converters.values().any(|n| n == name) + }; + let name = (self.converters.len() + 2..) + .map(|k| format!("{c}_{k}")) + .find(|name| !taken(name)) + .expect("some number is free"); self.converters.insert(key, name.clone()); match strip_refs(ty) { Type::Option(_) => self.option(&name, &c, &held[0]), From 1fc638f42cd2718290faa7b8574c4d3c450cae89 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:18:14 -0800 Subject: [PATCH 9/9] fix(ffi): glue lends an optional borrowed Vec or slice by coercion; Python's numbered converters miss records', enums', objects', and arrays' names Co-Authored-By: Claude Opus 5.5 --- ffi/fixtures/api/src/lib.rs | 9 ++++++--- ffi/src/glue.rs | 23 ++++++++++++----------- ffi/src/python.rs | 6 +++++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/ffi/fixtures/api/src/lib.rs b/ffi/fixtures/api/src/lib.rs index 3f210a884..dcc765ab1 100644 --- a/ffi/fixtures/api/src/lib.rs +++ b/ffi/fixtures/api/src/lib.rs @@ -2186,9 +2186,12 @@ pub fn prefixed(payload: &[u8], prefix: Option<&[u8]>) -> Vec { [prefix.unwrap_or_default(), payload].concat() } -/// `a`'s sum plus `b`'s, if any: a list lent shared and optional after the same list owned. -pub fn sum_lists(a: Vec, b: Option<&[i32]>) -> i32 { - a.iter().chain(b.unwrap_or_default()).sum() +/// The sum of `a`, `b`, and `c`, if any: a list owned, then lent as a slice and as itself. +pub fn sum_lists(a: Vec, b: Option<&[i32]>, c: Option<&Vec>) -> i32 { + a.iter() + .chain(b.unwrap_or_default()) + .chain(c.into_iter().flatten()) + .sum() } /// `point`'s coordinates summed, or 0 for none: an optional value, lent shared. diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 4b673372d..3701ba6ee 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -1364,17 +1364,18 @@ impl Gen<'_> { format!("let {local} = {};", self.lift(ty, c)), format!("{local}.as_deref()"), ), - // A lifted `Vec` lends the slice it derefs to. - (Type::List(Collection::Vec, _), _) => match m { - Mutability::Mut => ( - format!("let mut {local} = {};", self.lift(ty, c)), - format!("{local}.as_deref_mut()"), - ), - Mutability::Not => ( - format!("let {local} = {};", self.lift(ty, c)), - format!("{local}.as_deref()"), - ), - }, + // A lifted `Vec` lends itself or, by coercion to the parameter's type, its slice, + // since a borrowed `Vec` and a slice share their type. + (Type::List(Collection::Vec, _), _) => { + let (binding, borrow) = match m { + Mutability::Mut => ("let mut", "&mut"), + Mutability::Not => ("let", "&"), + }; + ( + format!("{binding} {local} = {};", self.lift(ty, c)), + format!("match {borrow} {local} {{ Some(v) => Some(v), None => None }}"), + ) + } _ => ( format!("let {local} = {};", self.lift(ty, c)), format!("{local}.as_ref()"), diff --git a/ffi/src/python.rs b/ffi/src/python.rs index 7c4739453..188ab2700 100644 --- a/ffi/src/python.rs +++ b/ffi/src/python.rs @@ -646,12 +646,16 @@ impl<'a> Gen<'a> { if let Some(name) = self.converters.get(&key) { return name.clone(); } - // A numbered name must miss every converter's: each struct's own, and the others numbered. + // A numbered name must miss every converter's: each type's own, and the others numbered. let taken = |name: &str| { abi.options.contains_key(name) || abi.lists.contains_key(name) || abi.tuples.contains_key(name) || abi.ranges.contains_key(name) + || abi.arrays.contains_key(name) + || (0..abi.api.records.len()).any(|i| abi.record(i) == name) + || (0..abi.api.enums.len()).any(|i| abi.enumeration(i) == name) + || (0..abi.api.objects.len()).any(|i| abi.object(i) == name) || self.converters.values().any(|n| n == name) }; let name = (self.converters.len() + 2..)