diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8004e1b83..7809ad81f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -320,7 +320,8 @@ jobs: if: runner.os == 'macOS' run: just ffi test swift && just ffi moq-swift - # The Kotlin on ART, shrunk by R8 as an app shrinks it. x86_64, as moq's Android build: + # The Kotlin on ART, shrunk by R8 as an app shrinks it, and the library's abort messages + # and 16 KB alignment. x86_64, as moq's Android build: # this image ships the Android SDK and NDK, and its KVM runs an x86_64 emulator at speed. ffi-android: name: Export to Kotlin on Android @@ -357,6 +358,18 @@ jobs: arch: x86_64 script: nix develop .#android --command just ffi android + # An image whose kernel pages by 16 KB, as Google Play requires of 64-bit libraries from + # 2027-02-01: the library must load there, and its segments must align to that. + - name: API 35, 16 KB pages + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + with: + api-level: 35 + target: google_apis_ps16k + arch: x86_64 + script: | + test "$($ANDROID_HOME/platform-tools/adb shell getconf PAGE_SIZE | tr -d '\r')" = 16384 + nix develop .#android --command just ffi android + ffi-bench: name: Bench the bindings runs-on: ubuntu-24.04-arm diff --git a/ffi/README.md b/ffi/README.md index b27d0bea3..c522cb489 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -418,6 +418,11 @@ where it was while its file is unchanged. built with the header. A binding checks the two match before any other call, so a header and a library from different builds refuse to mix. - **Panics** abort, as an `extern "C"` function's do. +- **Android**: the build script links the library with 16 KB pages, which Google Play + requires of a 64-bit library from 2027-02-01 and an NDK before r28 does not do itself. A + panic or a failed check that aborts goes to logcat as `rust2-ffi` and to bionic's abort + message, which shows in the tombstone, `ApplicationExitInfo`, and Play vitals, since an + app's stderr goes nowhere. - **Names** are the prefix, then the item's path from the facade's root, each segment in Rust's case, joined by `_`: `track::Producer::append_group` is `moq_track_Producer_append_group`, and moq-net's `session::Stats` and `Session::stats` @@ -664,7 +669,7 @@ the rest cross encoded. which is never negative or infinite in Rust and drops what is below a millisecond past about 146 years. A path is a `PathBuf` of the OS's bytes, any on Unix and WTF-8 on Windows, made from them or from a `String`, and converted to one or to a - `java.nio.file.Path` by `toText()` and `toPath()`, which throw where it is not UTF-8; + `java.nio.file.Path` by `toText()` and `toPath()` (Android's `java.nio.file` starts at API 26), which throw where it is not UTF-8; its `toString()` shows what is not as U+FFFD, as Rust's `display` does. An `IpAddr` is a sealed `IpAddr` of an `Ipv4Addr`, a value class of its bits, or an `Ipv6Addr`, each kept of its family, so an IPv4-mapped address stays IPv6, and a `SocketAddr` a @@ -1480,8 +1485,11 @@ pushed out of one, and that each object of many threads' returns, closes, and dr freed once, and breaks each runtime check, which must throw or abort with its message. `just ffi android` runs the same on ART: the Kotlin and `tests/api.kt`, shrunk by R8 into an app's dex with only the generated rules keeping what JNI looks up, against the library built by the NDK for the -device's ABI, which ANDROID_HOME and ANDROID_NDK_HOME locate. Nightly runs it on an -x86_64 emulator at API 24, the oldest moq's Android library supports, and at API 35. +device's ABI, which ANDROID_HOME and ANDROID_NDK_HOME locate. It also reads the library's +segment alignment with `readelf`, which must be 16 KB, and checks that a failed check and a +panic reach logcat and bionic's abort message, which the tombstone shows. Nightly runs it on an +x86_64 emulator at API 24, the oldest moq's Android library supports, at API 35, and at +API 35 on an image whose kernel pages by 16 KB. `tests/api.py` runs the script through the Python, in CPython's development mode with warnings as errors, checks its futures the same way, what frees an object or bytes nobody closed, and the values Python's types bound, and breaks each runtime check, which must raise or diff --git a/ffi/runtime/rt.kt b/ffi/runtime/rt.kt index 97745ef79..25a61d558 100644 --- a/ffi/runtime/rt.kt +++ b/ffi/runtime/rt.kt @@ -257,7 +257,8 @@ class __Writer { /** Release the handles encoded, once the call that reads them returns. */ fun release() { held?.forEachIndexed { i, h -> h.release(slots[i]) } - java.lang.ref.Reference.reachabilityFence(kept) + // A fence on each, which keeps it reachable up to here. + kept?.forEach { if (it != null) __Keep.fence(it) } } /** Run [block] with this writer, releasing its handles once it returns. */ @@ -839,8 +840,7 @@ fun interface __Close { /** * Rust's buffer, which [owner] holds, freed once the GC collects this. Each read passes it * to the JNI function reading the buffer, as JNI keeps an argument reachable until the call - * returns: a fence on every JVM and Android level, where `Reference.reachabilityFence` - * needs API 28. + * returns, on every JVM and Android level. */ class __Lent(internal val owner: Long) { init { diff --git a/ffi/runtime/rt.rs b/ffi/runtime/rt.rs index 41649852d..21ee82a24 100644 --- a/ffi/runtime/rt.rs +++ b/ffi/runtime/rt.rs @@ -16,6 +16,8 @@ use std::sync::Arc; use std::sync::atomic::{self, AtomicBool, AtomicUsize, Ordering}; use std::task::Context; +#[cfg(target_os = "android")] +mod android; mod borrow; use borrow::{Borrow, EXCLUSIVE, RESERVED, TAKEN, Table}; mod thread; @@ -40,6 +42,8 @@ pub fn hash(value: &T) -> u64 { #[cold] pub fn fail(what: &str, msg: &str) -> ! { eprintln!("{what}: {msg}"); + #[cfg(target_os = "android")] + android::report(&format!("{what}: {msg}")); std::process::abort() } diff --git a/ffi/runtime/rt/android.rs b/ffi/runtime/rt/android.rs new file mode 100644 index 000000000..88c77a5c9 --- /dev/null +++ b/ffi/runtime/rt/android.rs @@ -0,0 +1,73 @@ +//! What an abort leaves on Android: stderr goes nowhere there, so a message that only +//! printed there vanishes. It goes to logcat and to bionic's abort message instead, which +//! the tombstone, `ApplicationExitInfo`, and Play vitals show. + +use std::cell::RefCell; +use std::ffi::{CString, c_char, c_int}; +use std::panic::PanicHookInfo; + +#[link(name = "log")] +unsafe extern "C" { + fn __android_log_write(priority: c_int, tag: *const c_char, text: *const c_char) -> c_int; +} + +unsafe extern "C" { + fn android_set_abort_message(message: *const c_char); +} + +/// `ANDROID_LOG_FATAL`. +const FATAL: c_int = 7; + +/// What a panic that reaches the C ABI prints second, after its own message, as the abort +/// it becomes. +const CANNOT_UNWIND: &str = "panic in a function that cannot unwind"; + +/// Write `text` to logcat and set it as the abort message, which the abort that follows +/// shows in its tombstone. Bionic keeps the first abort message and ignores each later one, +/// and liblog sets a fatal line as one too, so the first call in a process fixes what its +/// tombstone shows; logcat shows every call. +pub fn report(text: &str) { + // A NUL would end either early. + let text = CString::new(text.replace('\0', "\u{fffd}")).expect("no NUL is left"); + // SAFETY: both copy the NUL-terminated strings they are given. + unsafe { + __android_log_write(FATAL, c"rust2-ffi".as_ptr(), text.as_ptr()); + android_set_abort_message(text.as_ptr()); + } +} + +thread_local! { + /// The message of the last panic on this thread. + static LAST: RefCell = const { RefCell::new(String::new()) }; +} + +/// Report each panic, as the standard library's panic runtime does only in a build that +/// aborts on one, which this library is not. A panic that crosses the C ABI aborts after a +/// second panic whose message does not name its cause, so that one reports both. On a thread +/// with no Rust frame to unwind to, such as ART's, it aborts at once with no second panic, so +/// the hook cannot tell a panic that aborts from one a caller catches: it reports each, and +/// bionic keeps the first, which a caught panic that precedes an abort takes. +fn hook(info: &PanicHookInfo<'_>) { + let text = info.to_string(); + LAST.with_borrow_mut(|last| { + if text.ends_with(CANNOT_UNWIND) && !last.is_empty() { + report(&format!("{last}\n{text}")); + } else { + report(&text); + *last = text; + } + }); +} + +/// Run as the library loads, so a panic before any call reports too. +extern "C" fn install() { + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + hook(info); + previous(info); + })); +} + +#[used] +#[unsafe(link_section = ".init_array")] +static INSTALL: extern "C" fn() = install; diff --git a/ffi/runtime/types.kt b/ffi/runtime/types.kt index 9cb2778a2..67e2c0435 100644 --- a/ffi/runtime/types.kt +++ b/ffi/runtime/types.kt @@ -223,7 +223,7 @@ class PathBuf private constructor(bytes: ByteArray, given: Boolean) { throw kotlin.IllegalArgumentException("$this is not UTF-8", e) } - /** This as a `java.nio.file.Path`, throwing `IllegalArgumentException` where its bytes are not UTF-8. */ + /** This as a `java.nio.file.Path`, throwing `IllegalArgumentException` where its bytes are not UTF-8. Android has `java.nio.file` from API 26. */ fun toPath(): java.nio.file.Path = java.nio.file.Paths.get(toText()) override fun equals(other: Any?): Boolean = other is PathBuf && bytes.contentEquals(other.bytes) diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 154de21c0..55685dcec 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -21,14 +21,16 @@ use crate::ir::{ }; const RUNTIME: &str = include_str!("../runtime/rt.rs"); +const ANDROID: &str = include_str!("../runtime/rt/android.rs"); const BORROW: &str = include_str!("../runtime/rt/borrow.rs"); const THREAD: &str = include_str!("../runtime/rt/thread.rs"); const FUTURE: &str = include_str!("../runtime/rt/future.rs"); /// The crate's build script, which refuses to build for a target but `targets`, on which an /// item a target compiles would be missing, or one it leaves out left over, and tells the -/// crate which it builds for, as the `cfg` [`CFG`]. On macOS, a CPython extension leaves -/// Python's symbols for the interpreter that loads it to resolve. +/// crate which it builds for, as the `cfg` [`CFG`]. On Android, it aligns the library to +/// 16 KB pages. On macOS, a CPython extension leaves Python's symbols for the interpreter +/// that loads it to resolve. fn build(targets: &[String]) -> String { let list: Vec = targets.iter().map(|t| format!("{t:?}")).collect(); let list = list.join(", "); @@ -45,6 +47,11 @@ fn build(targets: &[String]) -> String { \t\tpanic!(\"exported for {names}, not {{target}}: export again with --target {{target}}\");\n\ \t}}\n\ \tprintln!(\"cargo::rustc-cfg={CFG}=\\\"{{target}}\\\"\");\n\ + \t// rustc leaves Android's page size to the NDK, whose r27 and older align to 4 KB, and\n\ + \t// Google Play rejects a 64-bit library that is not aligned to 16 KB.\n\ + \tif std::env::var(\"CARGO_CFG_TARGET_OS\").is_ok_and(|os| os == \"android\") {{\n\ + \t\tprintln!(\"cargo::rustc-link-arg-cdylib=-Wl,-z,max-page-size=16384\");\n\ + \t}}\n\ \t#[cfg(feature = \"python\")]\n\ \tpyo3_build_config::add_extension_module_link_args();\n\ }}\n", @@ -414,6 +421,7 @@ impl Crate<'_> { output.file(&out.join("src/lib.rs"), &generator.out); output.file(&out.join("build.rs"), &build(&self.abi.api.targets)); output.file(&out.join("src/rt.rs"), RUNTIME); + output.file(&out.join("src/rt/android.rs"), ANDROID); output.file(&out.join("src/rt/borrow.rs"), BORROW); output.file(&out.join("src/rt/thread.rs"), THREAD); output.file(&out.join("src/rt/future.rs"), FUTURE); diff --git a/ffi/tests/api.kt b/ffi/tests/api.kt index 10879ae71..578e5193f 100644 --- a/ffi/tests/api.kt +++ b/ffi/tests/api.kt @@ -988,7 +988,11 @@ fun lossless() { check(stray.toByteArray().contentEquals(byteArrayOf(-1)) && stray == PathBuf(byteArrayOf(-1)), "a path keeps bytes no String holds") check(rejects { stray.toText() } && rejects { stray.toPath() } && stray.toString() == "\ufffd", "a path that is not UTF-8 has no text") check(rejects { PathBuf("\ud800") }, "a lone surrogate has no path") - check(PathBuf("a/é").toText() == "a/é" && PathBuf("a/é").toPath() == java.nio.file.Paths.get("a/é"), "a UTF-8 path converts") + check(PathBuf("a/é").toText() == "a/é", "a UTF-8 path converts") + // `java.nio.file` is on Android from API 26. + if (runCatching { Class.forName("java.nio.file.Paths") }.isSuccess) { + check(PathBuf("a/é").toPath() == java.nio.file.Paths.get("a/é"), "a UTF-8 path converts to a Path") + } check(_wtf8(byteArrayOf(0xed.toByte(), 0xa0.toByte(), 0x80.toByte())) && !_wtf8(byteArrayOf(0xed.toByte(), 0xa0.toByte(), 0x80.toByte(), 0xed.toByte(), 0xb0.toByte(), 0x80.toByte())), "a lone surrogate is WTF-8, a pair encoded apart is not") check(!_wtf8(byteArrayOf(-1)) && !_wtf8(byteArrayOf(0xc0.toByte(), 0x80.toByte())) && _wtf8("a/é😀".encodeToByteArray()), "WTF-8 is UTF-8 otherwise") @@ -1291,6 +1295,8 @@ fun broken(which: String) { Node.new(2u, tail) tail.sum() } + // A Rust panic cannot unwind into the JVM, so it aborts. + "panicked" -> Clash.Unit.reading(0u) "outlived" -> Meters.new().let { meters -> val first = meters.first() meters.clear() diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 1efc53339..12a842a0e 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -925,6 +925,26 @@ fn api_android() { .args(["--features", "kotlin"]) .env("RUSTFLAGS", "-D warnings")); + // Every segment aligns to the 16 KB pages Google Play requires of a 64-bit library from + // 2027-02-01, whichever NDK built it: r27 and older align to 4 KB unless told otherwise. + let lib = dir.join("lib").join(&abi).join("libapi.so"); + let segments = run(Command::new("readelf").args(["--wide", "--program-headers"]).arg(&lib)); + let segments = String::from_utf8_lossy(&segments.stdout); + let aligns: Vec = segments + .lines() + .filter(|line| line.trim_start().starts_with("LOAD")) + .map(|line| { + let align = line.split_whitespace().last().unwrap(); + u64::from_str_radix(align.trim_start_matches("0x"), 16).unwrap() + }) + .collect(); + assert!(!aligns.is_empty(), "readelf listed no LOAD segment:\n{segments}"); + assert!( + aligns.iter().all(|&align| align >= 16384), + "{} is not aligned to 16 KB pages:\n{segments}", + lib.display() + ); + // The app: the script's entry point, kept as an app keeps its own, and everything it // reaches, which R8 shrinks and renames, keeping only what the rules keep: the // generated ones, and those kotlinx-coroutines' jar holds, which an app's build reads @@ -988,6 +1008,61 @@ fn api_android() { cmd }; kotlin(&api.transcript, &art); + + // An abort or a panic must reach logcat and bionic's abort message, which the tombstone + // shows: an app's stderr goes nowhere. The tombstone is logged as the crash dump + // finishes, after the process died, and one from an earlier run may still arrive, so the + // case prints its process id, and the read takes the abort message that follows the + // tombstone of that process. + for (case, logged, tombstone) in [ + ( + "callback threw", + "api::Log::log: a call into Kotlin threw", + "Abort message: 'api::Log::log: a call into Kotlin threw'", + ), + ( + "panicked", + "called `Option::unwrap()` on a `None` value", + "Abort message: 'panicked at", + ), + ] { + run(Command::new(&adb).args(["logcat", "-c"])); + let mut cmd = Command::new(&adb); + cmd.arg("shell").arg(format!( + "echo $$; ANDROID_DATA={DEVICE} exec dalvikvm -Djava.library.path={DEVICE} -cp {DEVICE}/classes.dex ApiKt '{case}'" + )); + let output = self::output(&mut cmd); + failed(&output, case); + let stdout = String::from_utf8_lossy(&output.stdout); + let pid = stdout.lines().next().unwrap_or_default().trim().to_string(); + assert!( + pid.parse::().is_ok(), + "{case} printed no process id first: {stdout}" + ); + let ours = format!("pid: {pid},"); + let log = run(Command::new(&adb).args(["logcat", "-d", "-s", "rust2-ffi:F"])); + let log = String::from_utf8_lossy(&log.stdout); + assert!(log.contains(logged), "{case} logged {log}"); + // Read the dump as it arrives, bounded by the device-side timeout. + let mut reader = Command::new(&adb) + .args(["shell", "timeout", "120", "logcat", "-s", "DEBUG:F"]) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let mut found = false; + let mut dump = String::new(); + for line in std::io::BufRead::lines(std::io::BufReader::new(reader.stdout.take().unwrap())) { + let line = line.unwrap(); + found |= line.contains(&ours); + if found && line.contains("Abort message") { + dump = line; + break; + } + } + let _ = reader.kill(); + let _ = reader.wait(); + assert!(dump.contains(tombstone), "{case}'s tombstone showed {dump:?}"); + } } /// The gate's Kotlin protocol scripts, fewer than C's, as each costs the JVM more. @@ -1087,16 +1162,14 @@ fn kotlin(native: &str, vm: &dyn Fn(&[&str]) -> Command) { ("lent kept", "Counter is closed"), ( "lent moved", - "Native$__Refused: api_Counter_into_label: a handle a call lent is borrowed exclusively or moved out", + "api_Counter_into_label: a handle a call lent is borrowed exclusively or moved out", ), + ("panicked", "called `Option::unwrap()` on a `None` value"), ("callback threw", "api::Log::log: a call into Kotlin threw"), ("closure threw", "api_made: a call into Kotlin threw"), ("closure failed", "api_drive: a call into Kotlin threw"), ("reader threw", "SeekableByteChannel.read: a call into Kotlin threw"), - ( - "outlived", - "Native$__Refused: api_Meter_count: a handle lent from another outlived it", - ), + ("outlived", "api_Meter_count: a handle lent from another outlived it"), ] { // Under a piped core pattern the handler still runs at a limit of 0, so the // status reports a core though none is written. diff --git a/quest/m1/ffi-android.md b/quest/m1/ffi-android.md deleted file mode 100644 index a56796964..000000000 --- a/quest/m1/ffi-android.md +++ /dev/null @@ -1,26 +0,0 @@ -# [S] Android: 16 KB pages and abort messages - -## Goal - -- The generated crate's build script links Android libraries with - `-Wl,-z,max-page-size=16384`, a platform boundary needing no config, and `just ffi - android` checks the built `.so`'s alignment with `readelf` and runs on a 16 KB page - emulator image beside today's. -- A failed run-time check or panic that aborts on Android sets bionic's abort message - (`android_set_abort_message`) and writes it to logcat, so it shows in the tombstone, - logcat, `ApplicationExitInfo`, and Play vitals instead of vanishing with stderr. - -## Plan - -Found by the 2026-11-18 language audit. rustc does not set `max-page-size` for Android, so -alignment depends on the consumer's NDK: r28 and later align to 16 KB, r27 does not -([page sizes](https://developer.android.com/guide/practices/page-sizes)). From 2027-02-01, -Google Play rejects updates targeting API 35 and later whose 64-bit libraries are not -16 KB aligned. `rt::fail` is `eprintln!` then `abort()` (`ffi/runtime/rt.rs:33-36`), and -Android sends stderr nowhere. - -Public API: none. - -## Hosts - -- linux - the Android emulator needs KVM diff --git a/quest/m1/ffi-kotlin-callback-lent.md b/quest/m1/ffi-kotlin-callback-lent.md new file mode 100644 index 000000000..b09c489e1 --- /dev/null +++ b/quest/m1/ffi-kotlin-callback-lent.md @@ -0,0 +1,9 @@ +# [S] Keep a Kotlin callback's lent return alive until Rust reads it + +## Goal + +A Kotlin closure or callback that returns Rust's lent bytes (a `Bytes` over a `__Lent`, which Rust shares without a copy) keeps the owner reachable until Rust has read the encoded return, so the GC cannot run `__bytes_free` between the return and Rust's read. A differential test returns a lent `Bytes` over 1 KiB from a callback while forcing collections, and Rust reads it intact. + +## Plan + +Found by CodeRabbit on PR #876, which did not cause it: `__Writer.use { w -> write; w.finish() }` (ffi/src/kotlin.rs:962) releases the writer, and so fences `kept`, before the expression returns the encoding, and Rust reads the lent bytes after that. The `__Lent` KDoc covers only an argument of a JNI call, where the native frame keeps it alive. Hold the owner past the read, for example by returning it with the encoding and fencing after the Rust call returns, or by copying a lent return at this boundary; settle which with a test that fails first. The older `Reference.reachabilityFence` in `release` had the same window on HotSpot, so the API 24 monitor fallback only extends it. diff --git a/quest/m1/ffi-packaging/maven.md b/quest/m1/ffi-packaging/maven.md index 0e100deb1..c296465a0 100644 --- a/quest/m1/ffi-packaging/maven.md +++ b/quest/m1/ffi-packaging/maven.md @@ -16,8 +16,7 @@ The artifactId and version come from the crate; the groupId is the caller's, sin `repository` can only suggest a Maven namespace it cannot prove the publisher owns (`ffi/release/README.md`'s "Other registries" table). -Public API: the Maven coordinates. - -## Required +The generated crate's build script already aligns each Android library to 16 KB pages, which Play +requires from 2027-02-01. -- [16 KB pages](/quest/m1/ffi-android.md) - the libraries the AAR ships are aligned for Play +Public API: the Maven coordinates.