From c93f68dd767a87891ec69b941cdd730e02c0c8c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 02:07:28 -0800 Subject: [PATCH 01/15] chore(quest): claim quest/m1/ffi-packaging/pubdev.md Quest-Claim: 50dc11b82e717266 From 0ecdf6c72b1d1c760e2c2f954b91fd581a6e1d2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 02:41:26 -0800 Subject: [PATCH 02/15] ffi!: a pub.dev package with a build hook The export writes dart/ as a Dart package: pubspec.yaml, lib/.dart, and hook/build.dart, which builds the crate beside it with Cargo. The builder builds each Dart target's library, Android's through the NDK, writes the pub.dev package whose hook/release.json names each by size and SHA-256 on the release, and checks it in clean Dart apps and a Flutter app on an Android emulator through a hot restart. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/bindings-test.yml | 4 + .github/workflows/bindings.yml | 210 +++++++++++++- bench/ffi/report.py | 2 +- ffi/README.md | 20 +- ffi/justfile | 4 +- ffi/release/README.md | 65 +++-- ffi/release/package.py | 433 +++++++++++++++++++++++++++- ffi/release/release.yml | 47 ++- ffi/release/test_package.py | 96 +++++- ffi/runtime/hook.dart | 193 +++++++++++++ ffi/src/dart.rs | 33 ++- ffi/src/glue.rs | 15 +- ffi/surface/surface.py | 2 +- ffi/tests/fixtures.rs | 6 +- ffi/tests/fixtures/corpus.rs | 2 +- quest/m1/ffi-packaging/pubdev.md | 18 -- 16 files changed, 1081 insertions(+), 69 deletions(-) create mode 100644 ffi/runtime/hook.dart delete mode 100644 quest/m1/ffi-packaging/pubdev.md diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index ef438281e..48d662d9f 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -30,6 +30,8 @@ jobs: go-module: example.com/api # No swift-check: api.swift compiles against the fixture's own test export, of the host # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. + # Nor a dart-check: api.dart runs the script tests/fixtures/transcript generates, which + # no check input carries, so the Dart consumers resolve the library alone. moq: name: moq stand-in @@ -44,3 +46,5 @@ jobs: scope: kixel-ai # Under the export's own module, `moq`, which moq.go imports. go-check: ffi/moq/moq.go + # The export's package, `moq`, which moq.dart imports. + dart-check: ffi/moq/moq.dart diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 706bf9e32..d6128f694 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -1,7 +1,7 @@ name: Binding packages # The shared, read-only builder of a crate's rust2-ffi Python wheels, Swift package, C archives, -# Go module, and web package, which a release workflow ffi/release/package.py's `init` writes +# Go module, pub.dev package, and web package, which a release workflow ffi/release/package.py's `init` writes # calls at a pinned rust2 commit. # It publishes nothing and holds no credentials: the caller's own jobs publish what it # uploads, once its consumers pass. See ffi/release/README.md. @@ -64,6 +64,10 @@ on: description: The main.go of a program, importing the module's packages, run against it default: '' type: string + dart-check: + description: The main.dart of a program, importing the package as package:/.dart, run by Dart and Flutter apps on it + default: '' + type: string outputs: version: description: The package's Cargo version @@ -85,6 +89,8 @@ env: CARGO_TERM_COLOR: always MANIFEST: ${{ inputs.manifest-path }} FEATURES: ${{ inputs.features }} + # The NDK the Android libraries build with: Flutter's default. + NDK: 28.2.13676358 jobs: facts: @@ -179,11 +185,15 @@ jobs: targets: aarch64-unknown-linux-gnu - host: apple os: macos-26 - targets: aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim + targets: aarch64-apple-darwin x86_64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim # MSVC's library for the wheel, MinGW's for Go, which links through MinGW. - host: windows os: windows-latest targets: x86_64-pc-windows-msvc x86_64-pc-windows-gnu + # The Dart package's Android libraries, through the NDK. + - host: android + os: ubuntu-24.04 + targets: aarch64-linux-android armv7-linux-androideabi x86_64-linux-android runs-on: ${{ matrix.os }} timeout-minutes: 60 defaults: @@ -211,6 +221,11 @@ jobs: if: needs.facts.outputs.fixture == 'true' working-directory: source run: just ffi standin + - name: Android NDK + if: matrix.host == 'android' + run: | + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install "ndk;$NDK" > /dev/null + echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/$NDK" >> "$GITHUB_ENV" - name: Exporter working-directory: rust2/ffi env: @@ -255,9 +270,16 @@ jobs: - host: apple-ios-sim os: macos-26 targets: aarch64-apple-ios-sim + # The Dart package's Intel Mac library, which a Flutter app's universal build needs. + - host: macos-x64 + os: macos-26 + targets: x86_64-apple-darwin - host: windows os: windows-latest targets: x86_64-pc-windows-msvc x86_64-pc-windows-gnu + - host: android + os: ubuntu-24.04 + targets: aarch64-linux-android armv7-linux-androideabi x86_64-linux-android runs-on: ${{ matrix.os }} timeout-minutes: 90 defaults: @@ -295,6 +317,11 @@ jobs: macOS) python -m pip install delocate==0.13.0 ;; Windows) python -m pip install delvewheel==1.13.2 ;; esac + - name: Android NDK + if: matrix.host == 'android' + run: | + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install "ndk;$NDK" > /dev/null + echo "ANDROID_NDK_HOME=$ANDROID_HOME/ndk/$NDK" >> "$GITHUB_ENV" - name: Exporter working-directory: rust2/ffi env: @@ -351,6 +378,7 @@ jobs: SWIFT_CHECK: ${{ inputs.swift-check }} C_CHECK: ${{ inputs.c-check }} GO_CHECK: ${{ inputs.go-check }} + DART_CHECK: ${{ inputs.dart-check }} run: | mkdir -p dist/checks if [[ -n "$PYTHON_CHECK" ]]; then cp "source/$PYTHON_CHECK" dist/checks/check.py; fi @@ -358,6 +386,19 @@ jobs: # Its extension says whether it is C or C++. if [[ -n "$C_CHECK" ]]; then mkdir dist/checks/c && cp "source/$C_CHECK" dist/checks/c/; fi if [[ -n "$GO_CHECK" ]]; then cp "source/$GO_CHECK" dist/checks/main.go; fi + if [[ -n "$DART_CHECK" ]]; then cp "source/$DART_CHECK" dist/checks/main.dart; fi + # The export's own package, as a checkout depends on it: its hook builds the crate. + - uses: dart-lang/setup-dart@2286a0c042b44434fe26d292bbce8d34f7ec662b # v1.8.2 + if: matrix.host == 'linux-x64' + with: + sdk: stable + - name: Dart from source + if: matrix.host == 'linux-x64' + run: | + args=() + if [[ -f dist/checks/main.dart ]]; then args+=(--check dist/checks/main.dart); fi + if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi + python3 -I rust2/ffi/release/package.py consume-source --manifest-path "source/$MANIFEST" --export export "${args[@]}" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} @@ -802,3 +843,168 @@ jobs: if [[ -f dist/checks/main.go ]]; then args+=(--check dist/checks/main.go); fi if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi python3 -I rust2/ffi/release/package.py consume-go --proxy proxy "${args[@]}" + + # The pub.dev package: the export's, its hook/release.json naming each target's library on + # the release by size and SHA-256, and what pub asks of a package beside, which pub checks + # as publishing would, and the analyzer with the lints pub.dev scores it by. + pub: + needs: [facts, build, agree, changes] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: | + ffi/release + ffi/tests/dart_lints.yaml + sparse-checkout-cone-mode: false + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: needs.facts.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: needs.facts.outputs.fixture == 'true' + working-directory: source + run: just ffi standin + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: ${{ needs.facts.outputs.name }}-build-* + path: builds + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-release + path: release + - uses: dart-lang/setup-dart@2286a0c042b44434fe26d292bbce8d34f7ec662b # v1.8.2 + with: + sdk: stable + - name: Package + env: + NAME: ${{ needs.facts.outputs.name }} + URL: https://github.com/${{ github.repository }}/releases/download/${{ needs.facts.outputs.tag }}/ + run: | + mkdir -p pub/libs + cp builds/*/dart/* pub/libs/ + cd source + python3 -I ../rust2/ffi/release/package.py pub --manifest-path "$MANIFEST" --export "../builds/$NAME-build-linux-x64/export" --libs ../pub/libs --url "$URL" --changes ../release/changes.md --out ../pub/package + # Each in a copy, as resolving writes into the package. + - name: Check + run: | + cp -R pub/package publish + (cd publish && dart pub publish --dry-run) + cp -R pub/package lint + cp rust2/ffi/tests/dart_lints.yaml lint/analysis_options.yaml + (cd lint && dart pub get && dart analyze --fatal-infos .) + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ needs.facts.outputs.name }}-pub + path: pub + if-no-files-found: error + + # New Dart apps depending on the package, its libraries served from the loopback as the + # release hosts them: one its hook refuses a library of other bytes, and one run by `dart + # run` and from `dart build cli`'s bundle, on the newest Dart and on 3.10, the oldest the + # package takes. + dart-consumer: + needs: [facts, pub] + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + os: ubuntu-24.04 + sdk: stable + - host: linux-x64 + os: ubuntu-24.04 + sdk: 3.10.0 + - host: linux-arm64 + os: ubuntu-24.04-arm + sdk: stable + - host: apple-darwin + os: macos-26 + sdk: stable + - host: windows + os: windows-latest + sdk: stable + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: ffi/release + persist-credentials: false + - uses: dart-lang/setup-dart@2286a0c042b44434fe26d292bbce8d34f7ec662b # v1.8.2 + with: + sdk: ${{ matrix.sdk }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-pub + path: pub + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} + path: dist + - name: Run + run: | + args=() + if [[ -f dist/checks/main.dart ]]; then args+=(--check dist/checks/main.dart); fi + if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi + python3 -I rust2/ffi/release/package.py consume-dart --package pub/package --libs pub/libs "${args[@]}" + + # A new Flutter app depending on the package, its libraries served from the loopback, + # run on an x86_64 Android emulator, which this runner's KVM runs at speed, then again + # after a hot restart, which keeps the library and Rust's statics but not the isolate. + flutter-consumer: + needs: [facts, pub] + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: ffi/release + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-pub + path: pub + # The check and native Rust's output, as Linux on x86_64 printed it. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-linux-x64 + path: dist + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2.23.0 + with: + channel: stable + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Arguments + run: | + args="--package pub/package --libs pub/libs --platform android --device emulator-5554" + if [[ -f dist/checks/main.dart ]]; then args+=" --check dist/checks/main.dart"; fi + if [[ -f dist/native.txt ]]; then args+=" --native dist/native.txt"; fi + echo "ARGS=$args" >> "$GITHUB_ENV" + # Flutter's oldest Android, the floor the hook checks the app's against. + - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + with: + api-level: 24 + arch: x86_64 + script: python3 -I rust2/ffi/release/package.py consume-flutter $ARGS diff --git a/bench/ffi/report.py b/bench/ffi/report.py index c34f7e309..d948de6f5 100644 --- a/bench/ffi/report.py +++ b/bench/ffi/report.py @@ -84,7 +84,7 @@ ("Python", ["python/moq.py"], ["src/python"]), ("Swift", ["swift"], []), ("Go", ["go"], []), - ("Dart", ["dart/moq.dart"], ["src/dart"]), + ("Dart", ["dart/lib/moq.dart"], ["src/dart"]), ] # The TypeScript, under the wasm32 stand-in's export, and the glue its module builds with. diff --git a/ffi/README.md b/ffi/README.md index c263b56c5..49bd2dd26 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -49,8 +49,8 @@ Beside the C header, `include/.hpp` holds the same API in C++, over a he those functions look up by name. `python/.py` holds it in Python, over a CPython extension the crate builds with `--features python`, `swift/.swift` holds it in Swift, over the C header as a clang module, `go/` holds it in Go, a package per crate -over the C ABI through cgo, and `dart/.dart` holds it in Dart, over the C ABI through -`dart:ffi` and what the crate builds with `--features dart`. When `wasm32-wasip1` is among +over the C ABI through cgo, and `dart/` holds it as a Dart package, `lib/.dart` over +the C ABI through `dart:ffi` and what the crate builds with `--features dart`. When `wasm32-wasip1` is among the targets, `web/` holds it in TypeScript for the web, ES modules over the `.wasm` the crate builds with `--target wasm32-wasip1 --features web`. An export again rewrites only the files whose content changed, and removes from @@ -1237,10 +1237,18 @@ other than one in its own form, `module ` and `go 1.24`. ## Dart -`.dart` is the library `package:/.dart` over the C ABI through -`dart:ffi`, tested on Dart 3.13. Each C function is an `@Native` external, which finds -the shared library the crate builds with `--features dart` as the package's native asset: -the library's own URI, mapped to the library by a build hook or by `--native-assets`. +`dart/` is the package ``, whose `lib/.dart` is the library +`package:/.dart` over the C ABI through `dart:ffi`, tested on Dart 3.13 and, +as its `pubspec.yaml` says, taken by Dart 3.10 and later, whose build hooks are stable. Each +C function is an `@Native` external, which finds the shared library the crate builds with +`--features dart` as the package's native asset: the library's own URI, which the package's +`hook/build.dart` maps to the library when an app depending on it builds or runs. In an +export, the hook builds the crate beside the package, `../Cargo.toml`, with Cargo, on the +target the app builds for, for the Android NDK of an app's Flutter build, and reruns when a +source the build read changes, Cargo's dep-info says which. A release's adds +`hook/release.json`, from which it downloads the library instead +([Binding releases](release/README.md)). The prefix must name a Dart package, lowercase +letters, digits, and `_` and no reserved word, or the export fails. - **Names** are the C names without the prefix: a type in Pascal case and a member in camel case, so `moq_track_Producer` is `TrackProducer`, and its method diff --git a/ffi/justfile b/ffi/justfile index 59c13263c..ad0f5bb0a 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -426,13 +426,13 @@ dart-program dir main: aarch64) arch=arm64 ;; *) arch=x64 ;; esac - printf '{"configVersion": 2, "packages": [{"name": "moq", "rootUri": "file://%s/dart/", "packageUri": "./"}]}\n' "$out" > "$dir/.dart_tool/package_config.json" + printf '{"configVersion": 2, "packages": [{"name": "moq", "rootUri": "file://%s/dart/", "packageUri": "lib/"}]}\n' "$out" > "$dir/.dart_tool/package_config.json" printf 'format-version: [1, 0, 0]\nnative-assets:\n %s_%s:\n "package:moq/moq.dart": [absolute, "%s"]\n' "{{ os() }}" "$arch" "$out/target/release/$lib" > "$dir/native_assets.yaml" dart analyze --fatal-infos "$dir" # pub.dev scores the library with the lints in tests/dart_lints.yaml, which the analyzer reads from the directory it analyzes. rm -rf "$dir-lint" mkdir "$dir-lint" - cp "$out"/dart/*.dart "$dir-lint" + cp "$out"/dart/lib/*.dart "$dir-lint" cp tests/dart_lints.yaml "$dir-lint/analysis_options.yaml" dart analyze --fatal-infos "$dir-lint" sdk=$(dirname "$(dirname "$(realpath "$(command -v dart)")")") diff --git a/ffi/release/README.md b/ffi/release/README.md index b67ddf78f..b4e746fe2 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -1,8 +1,8 @@ # Binding releases A crate's [rust2-ffi](/ffi/README.md) export, released as Python wheels, a Swift package, -C and C++ archives, a Go module, and a web package on npm and JSR. Two halves, split where -Cargo's knowledge ends: +C and C++ archives, a Go module, a Dart package on pub.dev, and a web package on npm and JSR. +Two halves, split where Cargo's knowledge ends: - **The builder**, [`bindings.yml`](/.github/workflows/bindings.yml) over `package.py`, shared and read-only. It derives everything from Cargo metadata and the export: package @@ -10,9 +10,10 @@ Cargo's knowledge ends: exporter gives the crate's library. It builds, repairs, and checks the packages in clean consumers, and publishes nothing. - **The release workflow**, the caller's own file, which `package.py init` writes once. - It holds what Cargo cannot establish: who may publish to PyPI, npm, and JSR, the scope - the web package is named under, where the XCFramework is hosted, which repository - SwiftPM resolves the package from, the Go module's path, and which tag releases. + It holds what Cargo cannot establish: who may publish to PyPI, pub.dev, npm, and JSR, the + scope the web package is named under, where the XCFramework and the Dart libraries are + hosted, which repository SwiftPM resolves the package from, the Go module's path, and + which tag releases. ## Adopting it @@ -36,6 +37,11 @@ outside the repository: resolves a `github.com` path by git alone, so its consumers set `GOPROXY` to the proxy. No job publishes it: a git mirror holds no library over GitHub's 100 MiB, and a proxy is the caller's to run. +- for the Dart package, publish its first version by hand, the `-pub` artifact's + `package/` after its libraries are on the release, as pub.dev configures automated + publishing only for a package it holds; then, on its admin page, enable publishing from + GitHub Actions for this repository, with the tag pattern `-v{{version}}` and the + `pub` environment required, and make that environment one only release tags may deploy to; - for the web package, publish its first version by hand, as npm configures a trusted publisher only for a package it holds, then register the workflow and the `npm` environment as its trusted publisher; and create the package in the JSR scope and link @@ -55,8 +61,10 @@ A Cargo prerelease, such as `1.0.0-rc.1`, is a GitHub prerelease too. The builder takes the crate's `Cargo.toml`, the `go-module` path, which without it is the export's own `` that no proxy serves, and optional inputs the caller's tests own: `features` to ship, an `example` whose stdout is native Rust's transcript, and the -`python-check`, `swift-check`, `c-check`, and `go-check` programs that must print it through -the installed packages, and `reproduce`, which builds everything twice. Without a check, a consumer only imports the module, or links a +`python-check`, `swift-check`, `c-check`, `go-check`, and `dart-check` programs that must +print it through the installed packages, and `reproduce`, which builds everything twice. +A `dart-check` prints through `print`, which a Flutter app on Android passes to its log, +not through `stdout`. Without a check, a consumer only imports the module, or links a function of each C and C++ header. A `c-check` is C or C++ by its extension. The web package takes `web-manifest-path`, the package exported for `wasm32-wasip1`, often the crate's own, which no Cargo fact says builds for the web, and `scope`; and an optional `web-check`, a @@ -80,7 +88,8 @@ crate apart from it takes `web-features` and `web-example` instead. missing ([Targets](/ffi/README.md#targets)). A job checks that every host generated the same binding. 4. Builds each target with the crate's own toolchain, as rustup resolves it from the - crate's directory; only the export runs on rust2's nightly. + crate's directory; only the export runs on rust2's nightly. Android's targets build, and + stage their APIs, on Linux through the NDK Flutter defaults to, r28. 5. Python: a `cp311-abi3` wheel per target, the oldest CPython the generated extension's stable ABI loads in, holding the module as `/__init__.py` beside its library, with PEP 639 license metadata and a complete hashed RECORD. Each is repaired by its platform's @@ -141,10 +150,25 @@ crate apart from it takes `web-features` and `web-example` instead. limit, each platform's library moves to a module of its own, whose one package's `#cgo` line links it and which the runtime imports under that platform's build constraint, so `go get` downloads only the platform it builds for. -9. The release's `api.json`, and `changes.md`, the items removed, changed, and added since +9. Dart: the export's `dart/`, a package `package:/.dart` names, whose + `hook/build.dart` bundles its library as the native asset its `@Native` functions find + ([Dart](/ffi/README.md#dart)). Each target's library, the C ABI with its Dart glue, + `--dart-.so`, `.dylib`, or `.dll`: Linux x64 and arm64 (against + glibc 2.28), macOS arm64 and x64 (at 10.15), iOS arm64 and its arm64 simulator (at 13.0), + Windows x64, and Android arm64, armv7, and x86_64 (at API 24), Flutter's floors. The + package adds `hook/release.json`, naming each at + `https://github.com//releases/download/-v/` by size and SHA-256, + and the oldest system it loads on, where an app could target an older one; the hook + downloads its target's into the build's shared cache, refusing other bytes, and refuses + an app targeting an older system. Its pubspec takes Cargo's description, repository, + and homepage; `LICENSE` holds each license file's text, or else Cargo's SPDX expression, + as pub reads no other name; `README.md` is the crate's readme, or else a line naming + the crate; and `CHANGELOG.md` is `changes.md` under the version. `dart pub publish + --dry-run` must find no warning, and the analyzer no issue with the lints pub.dev scores by. +10. The release's `api.json`, and `changes.md`, the items removed, changed, and added since the previous release's, by kind and Rust path; the caller's release publishes the first as an asset and the second as its notes. -10. Consumers, with no checkout, Rust, or source path: each wheel installed with +11. Consumers, with no checkout, Rust, or source path: each wheel installed with `--no-index --no-deps` into a fresh environment on its oldest CPython, run with `-I` from an empty directory; and the Swift package built by its own scheme for macOS, iOS, and the iOS Simulator, then consumed by a new executable package that runs the check; @@ -152,9 +176,17 @@ crate apart from it takes `web-features` and `web-example` instead. through its config package, and on Linux and macOS by pkg-config, linking each library, where a shared consumer must record the versioned name; and the Go module, unpacked from its zip into a new module that requires it, vetted, built with the newest Go and no - `CGO_LDFLAGS`, and run on Linux x64 and arm64, macOS, and Windows. + `CGO_LDFLAGS`, and run on Linux x64 and arm64, macOS, and Windows; and the Dart + package, its libraries served from the loopback, in a new Dart app by `dart run` and + from `dart build cli`'s bundle on Linux x64 and arm64, macOS, and Windows, on the newest + Dart and on Linux on 3.10, the oldest the package takes, after a hook served a library + of other bytes refused it; and in a new Flutter app on an Android emulator at API 24, + run in debug, then again after a hot restart, which keeps the library and Rust's + statics but not the isolate. With an `example`, each must print native Rust's stdout byte for byte. -11. Web: `web-manifest-path` exported for `wasm32-wasip1` alone, whose `web/` holds the + Beside them, on Linux x64, a new Dart app on the export's own package, whose hook builds + the crate with Cargo, as in a checkout. +12. Web: `web-manifest-path` exported for `wasm32-wasip1` alone, whose `web/` holds the TypeScript, its JavaScript and declarations, and a `package.json` ([TypeScript (web)](/ffi/README.md#typescript-web)), and its module built at `opt-level = "s"` unless the crate's release profile sets a level, as a page downloads @@ -172,7 +204,7 @@ rust2 runs it on every manual Check and nightly through [`bindings-test.yml`](/.github/workflows/bindings-test.yml): on the api fixture, whose wheels, C++ consumers, and web package must print its native transcript, and on moq's stand-in facade at the exporter's moq pin, through `moq/moq.py`, `moq/moq.swift`, -`moq/moq.cpp`, and `moq/moq.go`, with its wasm32 stand-in as the web package. +`moq/moq.cpp`, `moq/moq.go`, and `moq/moq.dart`, with its wasm32 stand-in as the web package. ## Reproducible builds @@ -190,8 +222,8 @@ holding exactly the files it would upload, and anyone with the commit can check builds also stamp no archive time (`ZERO_AR_DATE`), and Windows links with `/Brepro`. - **Checked.** The Swift job builds the XCFramework twice and fails unless both ZIPs match. With `reproduce: true`, every build host builds a second time, from a new export at the same - path, and `package.py same` fails unless each wheel, static library, and C archive, the Go - module's included, matches byte for byte, and the web job packs its npm tarball twice; rust2 turns it on + path, and `package.py same` fails unless each wheel, static library, C archive, and Dart + library, the Go module's included, matches byte for byte, and the web job packs its npm tarball twice; rust2 turns it on for its api fixture. It doubles the build, so a large crate may leave it to a nightly. - **Not yet.** The exporter writes the crate's absolute path into the generated crate's version, which cargo hashes into every symbol, so a rebuild must keep the crate at the @@ -206,7 +238,7 @@ the crates the binding links (normal dependencies, platform-neutral, with each r crate's checksum from `Cargo.lock`), which every wheel also carries in `.dist-info/sboms/` (PEP 770) and the build hosts must agree on. The caller's `attest` job runs on a tag, before anything is published, and gives every wheel, the XCFramework, the Go -module's `.zip` and `.mod`, each C archive, and the npm tarball a build provenance +module's `.zip` and `.mod`, each C archive, each Dart library, and the npm tarball a build provenance attestation and an SBOM attestation (the tarball's from its own `sbom.cdx.json`), which `gh attestation verify --repo /` checks. Only that job holds `id-token` and `attestations` write; the builder stays read-only. PyPI's trusted publishing uploads its own PEP 740 @@ -222,7 +254,6 @@ binding. | Destination | Builder work | What only the caller can say | | --- | --- | --- | | Maven Central | An AAR with classes.jar, JNI libraries per Android ABI and the generated R8 rules; POM, sources and javadoc jars | The verified groupId namespace, publisher authorization, and signing key | -| pub.dev | The generated Dart library with a native-assets build hook and per-platform archives, pubspec, and license | Publisher authorization and the archives' download host | | NuGet | Needs a C# generator first: the managed assembly and RID-specific native assets | Package ID ownership, credentials, and target frameworks | Cargo's `repository` can suggest a Go module path or a Maven namespace but cannot prove diff --git a/ffi/release/package.py b/ffi/release/package.py index 47b472d4a..cf91f8e60 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -59,6 +59,22 @@ "aarch64-apple-ios": "ios_arm64", "x86_64-pc-windows-gnu": "windows_amd64", } +# Every target the Dart package ships a library for, by the oldest system it loads on, which +# its hook checks an app's against: Flutter's own floors, Android API 24, iOS 13, and macOS +# 10.15. Every arm64 Mac runs macOS 11, Rust's floor there, and Linux's and Windows' floors +# are the wheels'. +DART_TARGETS = { + "x86_64-unknown-linux-gnu": None, + "aarch64-unknown-linux-gnu": None, + "aarch64-apple-darwin": None, + "x86_64-apple-darwin": "10.15", + "aarch64-apple-ios": "13.0", + "aarch64-apple-ios-sim": "13.0", + "x86_64-pc-windows-msvc": None, + "aarch64-linux-android": "24", + "armv7-linux-androideabi": "24", + "x86_64-linux-android": "24", +} # What the go command refuses to extract: a module zip, or its files, over 500 MiB, and a # go.mod over 16 MiB (golang.org/x/mod/zip's MaxZipFile and MaxGoMod). GO_ZIP_MAX = 500 << 20 @@ -142,6 +158,7 @@ def facts(manifest): "license_file": package.get("license_file"), "repository": package.get("repository"), "homepage": package.get("homepage"), + "readme": package.get("readme"), "lib": library(package), "dir": str(path.parent), "workspace": meta["workspace_root"], @@ -341,7 +358,8 @@ def digest(export, lib): export = Path(export) headers = sorted(p for p in (export / "include").rglob("*") if p.is_file()) paths = [export / "python" / f"{lib}.py", *headers, *sorted((export / "swift").iterdir()), - *sorted(p for p in (export / "go").rglob("*") if p.is_file())] + *sorted(p for p in (export / "go").rglob("*") if p.is_file()), + *sorted(p for p in (export / "dart").rglob("*") if p.is_file())] for path in paths: hasher.update(path.relative_to(export).as_posix().encode() + b"\0") # Exact bytes: each wheel ships its own host's module. @@ -1120,11 +1138,16 @@ def consume_python(check, native, module): def exporter(manifest, features, *args): - """Run rust2-ffi from this checkout on the package.""" + """Run rust2-ffi from this checkout on the package, an Android target's build scripts + given the NDK's compilers.""" manifest = Path(manifest).resolve() cargo = ["--", "--features", features] if features.strip() else [] + env = dict(os.environ) + for target in (str(a) for a in args): + if target in DART_TARGETS and "android" in target: + env.update(android_toolchain(target, DART_TARGETS[target])) run("cargo", "run", "--locked", "--release", "--quiet", "--", "--manifest-path", manifest, - *args, *cargo, cwd=ROOT / "ffi") + *args, *cargo, cwd=ROOT / "ffi", env=env) def version_key(version): @@ -1507,10 +1530,10 @@ def members(path): def same(first, second): - """Fail unless two builds' wheels, static libraries, C archives, and npm tarball, the Go module's included, are the same bytes, naming what differs.""" + """Fail unless two builds' wheels, static libraries, C archives, Dart libraries, and npm tarball, the Go module's included, are the same bytes, naming what differs.""" found = {} for root in (Path(first), Path(second)): - found[root] = {p.relative_to(root).as_posix(): p for pattern in ("wheels/*.whl", "static/*/*.a", "go/*.a", "c/*.tar.gz", "c/*.zip", "*.tgz") for p in root.glob(pattern)} + found[root] = {p.relative_to(root).as_posix(): p for pattern in ("wheels/*.whl", "static/*/*.a", "go/*.a", "c/*.tar.gz", "c/*.zip", "dart/*", "*.tgz") for p in root.glob(pattern)} names = sorted(found[Path(first)]) if not names or names != sorted(found[Path(second)]): raise SystemExit(f"the builds hold different packages: {names} against {sorted(found[Path(second)])}") @@ -1609,7 +1632,7 @@ def go_library(args, export_dir, lib, target, library, native, out): def build(args): - """This host's wheels, C archives, Apple and Go static libraries, the export's sources, and native output.""" + """This host's wheels, C archives, Apple and Go static libraries, Dart libraries, the export's sources, and native output.""" info = facts(args.manifest_path) lib, export_dir, out = info["lib"], Path(args.export).resolve(), Path(args.out).resolve() crate = export_dir / "Cargo.toml" @@ -1617,7 +1640,7 @@ def build(args): info["sbom"] = sbom(info, crate) (out / "sbom.cdx.json").write_bytes(info["sbom"]) for target in args.targets: - if target not in WHEEL_TARGETS and target not in APPLE_TARGETS and target not in GO_TARGETS: + if not any(target in t for t in (WHEEL_TARGETS, APPLE_TARGETS, GO_TARGETS, DART_TARGETS)): raise SystemExit(f"no package ships {target}") run("rustup", "target", "add", target, cwd=Path(args.manifest_path).resolve().parent) if target in WHEEL_TARGETS: @@ -1649,9 +1672,11 @@ def build(args): elif target in GO_TARGETS: library, native = staticlib(args.manifest_path, crate, lib, target, env=reproducible(info, export_dir, target)) go_library(args, export_dir, lib, target, library, native, out) + if target in DART_TARGETS: + dart_library(info, args.manifest_path, export_dir, target, out) # The generated sources, for the Swift package, the Go module, and the hosts' agreement. # Text this writes is LF on every host, as the jobs reading it compare bytes. - for part in ("include", "swift", "python", "go"): + for part in ("include", "swift", "python", "go", "dart"): shutil.copytree(export_dir / part, out / "export" / part, dirs_exist_ok=True) shutil.copy(crate, out / "export") shutil.copy(export_dir / "api.json", out / "export") @@ -1794,6 +1819,367 @@ def consume_go(proxy, check, native): compare(Path(native).read_bytes(), result.stdout, "Go module") +def android_toolchain(target, api): + """The NDK's clang for `target` at `api`, as Cargo links and cc builds C with it.""" + ndk = os.environ.get("ANDROID_NDK_HOME") + if not ndk: + raise SystemExit(f"building {target} needs the Android NDK: set ANDROID_NDK_HOME") + host = {"linux": "linux-x86_64", "darwin": "darwin-x86_64", "win32": "windows-x86_64"}[sys.platform] + bin_dir = Path(ndk) / "toolchains" / "llvm" / "prebuilt" / host / "bin" + # The NDK spells 32-bit Arm's compiler armv7a, where Rust's target is armv7. + triple = "armv7a-linux-androideabi" if target.startswith("armv7") else target + clang = bin_dir / f"{triple}{api}-clang" + if not clang.is_file(): + raise SystemExit(f"the NDK at {ndk} has no {clang.name}") + key = target.replace("-", "_") + return {f"CARGO_TARGET_{key.upper()}_LINKER": str(clang), f"CC_{key}": str(clang), f"CXX_{key}": f"{clang}++", + f"AR_{key}": str(bin_dir / "llvm-ar")} + + +def dart_file(info, target): + """The release asset of `target`'s Dart library.""" + return f"{info['name']}-{info['version']}-dart-{target}{Path(library_name(info['lib'], target)).suffix}" + + +def dart_library(info, manifest, export_dir, target, out): + """`target`'s library for the Dart package, the C ABI with its Dart glue, built for the + oldest system its hook lets an app target.""" + lib, floor = info["lib"], DART_TARGETS[target] + name = library_name(lib, target) + env = reproducible(info, export_dir, target, name) + if target.endswith("apple-darwin"): + env["MACOSX_DEPLOYMENT_TARGET"] = floor or MACOS_WHEEL + elif "apple-ios" in target: + env["IPHONEOS_DEPLOYMENT_TARGET"] = floor + elif "android" in target: + env.update(android_toolchain(target, floor)) + # Linux's against the glibc floor, as the wheel's. + glibc = target.endswith("linux-gnu") + command = ["cargo-zigbuild", "rustc"] if glibc else ["cargo", "rustc"] + built = run(*command, "--color", "never", "--locked", "--release", "--manifest-path", export_dir / "Cargo.toml", + "--target", f"{target}.{GLIBC}" if glibc else target, "--target-dir", export_dir / "target-dart", + "--lib", "--crate-type", "cdylib", "--features", "dart", cwd=Path(manifest).resolve().parent, + env={**os.environ, **env}, capture_output=True, encoding="utf-8") + sys.stderr.write(built.stderr) + dest = out / "dart" / dart_file(info, target) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(export_dir / "target-dart" / target / "release" / name, dest) + + +def readme(info): + """The crate's readme, as Cargo finds it, or else a line saying what the package is.""" + if info["readme"]: + return (Path(info["dir"]) / info["readme"]).read_text(encoding="utf-8") + said = f"{' '.join(info['description'].split())}\n\n" if info["description"] else "" + return f"# {info['lib']}\n\n{said}The Dart binding of the Rust crate {info['name']}, generated by rust2-ffi.\n" + + +def pub_package(info, export, libs, url, changes, out): + """The pub.dev package: the export's, its pubspec given Cargo's facts, with the license, + readme, and changelog pub asks for, and hook/release.json naming each target's library + in `libs`, hosted at `url`, by size and SHA-256.""" + out = Path(out) + if out.exists(): + shutil.rmtree(out) + shutil.copytree(Path(export) / "dart", out) + pubspec = (out / "pubspec.yaml").read_text(encoding="utf-8") + version = re.search(r"^version: (.*)$", pubspec, re.M) + if version is None or version.group(1) != info["version"]: + raise SystemExit(f"the export's pubspec.yaml is not the crate's version {info['version']}") + # Cargo's facts beside what the export wrote, each a JSON string, which YAML reads alike. + said = [f"{key}: {json.dumps(info[key])}" for key in ("repository", "homepage") if info[key]] + if info["description"]: + pubspec = re.sub(r"^description: .*$", lambda _: f"description: {json.dumps(' '.join(info['description'].split()))}", + pubspec, count=1, flags=re.M) + pubspec = pubspec.replace(version.group(0), "\n".join([version.group(0), *said]), 1) + (out / "pubspec.yaml").write_text(pubspec, newline="\n", encoding="utf-8") + libraries = {} + names = {dart_file(info, target): target for target in DART_TARGETS} + found = {path.name for path in Path(libs).iterdir()} + if found != set(names): + raise SystemExit(f"{libs} lacks {sorted(set(names) - found)} and holds {sorted(found - set(names))} beside the Dart libraries") + for name, target in names.items(): + data = (Path(libs) / name).read_bytes() + libraries[target] = {"file": name, "size": len(data), "sha256": hashlib.sha256(data).hexdigest()} + if DART_TARGETS[target]: + libraries[target]["minimum"] = DART_TARGETS[target] + if not url.endswith("/"): + raise SystemExit(f"{url} is no directory: end it with /") + release = {"url": url, "libraries": dict(sorted(libraries.items()))} + (out / "hook" / "release.json").write_text(json.dumps(release, indent=2) + "\n", newline="\n", encoding="utf-8") + # pub takes a license from a file named LICENSE alone, so one holds each of the crate's, + # or else Cargo's SPDX expression. + texts = [path.read_text(encoding="utf-8").strip() for path in licenses(info)] + if not texts and info["license"]: + texts = [f"SPDX-License-Identifier: {info['license']}"] + if not texts: + raise SystemExit("pub.dev needs a license: set Cargo's license, or put a license file beside the manifest") + (out / "LICENSE").write_text("\n\n".join(texts) + "\n", newline="\n", encoding="utf-8") + (out / "README.md").write_text(readme(info), newline="\n", encoding="utf-8") + # This release's changes, under the version pub looks for. + notes = Path(changes).read_text(encoding="utf-8") + (out / "CHANGELOG.md").write_text(f"## {info['version']}\n\n" + re.sub(r"^(#+) ", r"#\1 ", notes, flags=re.M), + newline="\n", encoding="utf-8") + + +def pub_name(package): + """The name of the package in `package`, as its pubspec says.""" + found = re.search(r"^name: (\S+)$", (Path(package) / "pubspec.yaml").read_text(encoding="utf-8"), re.M) + if found is None: + raise SystemExit(f"{package}/pubspec.yaml names no package") + return found.group(1) + + +class Served: + """A directory served over HTTP on the loopback interface, as a release's host would.""" + + def __init__(self, root): + import functools + import http.server + import threading + class Quiet(http.server.SimpleHTTPRequestHandler): + def log_message(self, *_): + pass + + handler = functools.partial(Quiet, directory=str(root)) + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.url = f"http://127.0.0.1:{self.server.server_address[1]}/" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.server.shutdown() + + +def local_package(package, dest, url): + """The package, its libraries named at `url` instead of the release's host.""" + shutil.copytree(package, dest) + path = Path(dest) / "hook" / "release.json" + release = json.loads(path.read_text(encoding="utf-8")) + release["url"] = url + path.write_text(json.dumps(release, indent=2) + "\n", encoding="utf-8") + return Path(dest) + + +def dart_main(lib, check): + """The consumer's program: the caller's check, or else one that resolves the library's contract.""" + if check: + return Path(check).read_text(encoding="utf-8") + return (f"import 'dart:ffi';\n\n" + f"@Native(symbol: '{lib}_contract', assetId: 'package:{lib}/{lib}.dart')\n" + f"external void _contract();\n\n" + f"void main() {{\n if (Native.addressOf>(_contract) == nullptr) throw StateError('no {lib}');\n}}\n") + + +def dart_app(work, name, package, lib): + """A new app depending on the package by path, as on a hosted one.""" + app = Path(work) / name + (app / "bin").mkdir(parents=True) + (app / "pubspec.yaml").write_text( + f"name: {name}\npublish_to: none\n\nenvironment:\n sdk: ^3.10.0\n\n" + f"dependencies:\n {lib}:\n path: {json.dumps(Path(package).resolve().as_posix())}\n", encoding="utf-8") + run("dart", "pub", "get", cwd=app, capture_output=True) + return app + + +def consume_dart(package, libs, lib, check, native): + """New Dart apps depending on the package, its libraries served from the loopback as the + release hosts them: one served a library of other bytes, which its hook refuses, and one + that runs the check by `dart run` and from the bundle `dart build cli` makes.""" + with tempfile.TemporaryDirectory() as raw: + work = Path(raw) + good, bad = work / "served", work / "tampered" + shutil.copytree(libs, good) + shutil.copytree(libs, bad) + for path in bad.iterdir(): + data = bytearray(path.read_bytes()) + data[len(data) // 2] ^= 1 + path.write_bytes(bytes(data)) + with Served(bad) as served: + app = dart_app(work, "tampered", local_package(package, work / "bad-package", served.url), lib) + (app / "bin" / "main.dart").write_text(dart_main(lib, None), encoding="utf-8") + result = subprocess.run(["dart", "run", "bin/main.dart"], cwd=app, capture_output=True, encoding="utf-8") + if result.returncode == 0 or "SHA-256" not in result.stdout + result.stderr: + sys.stderr.write(result.stdout + result.stderr) + raise SystemExit("the hook took a library of other bytes than the release names") + with Served(good) as served: + app = dart_app(work, "consumer", local_package(package, work / "package", served.url), lib) + (app / "bin" / "main.dart").write_text(dart_main(lib, check), encoding="utf-8") + run("dart", "build", "cli", "--target", "bin/main.dart", "--output", work / "build", cwd=app, + capture_output=True) + exe = work / "build" / "bundle" / "bin" / ("main.exe" if sys.platform == "win32" else "main") + run_dart(["dart", "run", "bin/main.dart"], app, check, native, "dart run") + run_dart([exe], work, check, native, "dart build cli") + + +def run_dart(command, cwd, check, native, what, env=None): + """Run a Dart consumer, and compare what it prints with native Rust.""" + result = subprocess.run([str(c) for c in command], cwd=cwd, capture_output=True, env={**os.environ, **(env or {})}) + sys.stderr.buffer.write(result.stderr) + if result.returncode: + raise SystemExit(f"{what} failed on the Dart package with status {result.returncode}") + # Dart 3.10's `dart run` says it runs the hooks on stdout, before the program does. + stdout = re.sub(rb"^(Running build hooks\.\.\.)+", b"", result.stdout) + if native and check: + compare(Path(native).read_bytes(), stdout, f"Dart package, {what}") + print(f"{what}: ok") + + +def consume_source(manifest, export, check, native): + """A new Dart app depending on the export's own package, whose hook builds the exported + crate with Cargo, as in a checkout, on the crate's own toolchain.""" + package = Path(export).resolve() / "dart" + lib = pub_name(package) + toolchain = output("rustup", "show", "active-toolchain", cwd=Path(manifest).resolve().parent).split()[0] + with tempfile.TemporaryDirectory() as raw: + app = dart_app(raw, "consumer", package, lib) + (app / "bin" / "main.dart").write_text(dart_main(lib, check), encoding="utf-8") + # The hook's environment keeps RUSTUP_ variables. + run_dart(["dart", "run", "bin/main.dart"], app, check, native, "dart run from source", + env={"RUSTUP_TOOLCHAIN": toolchain}) + + +# The Flutter consumer's entry, which runs the check, collecting what it prints, then reports +# it base64-encoded in short lines, as Android's log splits long ones, and how it ended, as an +# app has no exit status. +FLUTTER_MAIN = """import 'dart:async'; +import 'dart:convert'; +import 'dart:io' as io; +import 'dart:math'; + +import 'check.dart' as check; + +void main() { + final out = StringBuffer(); + void report(String end) { + final data = base64.encode(utf8.encode(out.toString())); + Zone.root.print('rust2: start'); + for (var i = 0; i < data.length; i += 800) { + Zone.root.print('rust2: ${data.substring(i, min(i + 800, data.length))}'); + } + Zone.root.print('rust2: $end'); + } + + runZonedGuarded( + () async { + final Function run = check.main; + final result = run is Function(List) ? run([]) : run(); + if (result is Future) await result; + report('exit ${io.exitCode}'); + }, + (error, stack) => report('threw $error $stack'), + zoneSpecification: ZoneSpecification(print: (_, _, _, line) => out.writeln(line)), + ); +} +""" + + +class Flutter: + """`flutter run --machine` on an app: its events, from a thread reading them, and its commands.""" + + def __init__(self, app, device): + import queue + import threading + self.process = subprocess.Popen(["flutter", "run", "--machine", "--debug", "-d", device], cwd=app, text=True, + encoding="utf-8", stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.events, self.app, self.started, self.ids = queue.Queue(), None, False, 0 + threading.Thread(target=self.read, daemon=True).start() + + def read(self): + for line in self.process.stdout: + line = line.strip() + if line.startswith("[{"): + for event in json.loads(line): + self.events.put(event) + elif line: + # A desktop app's own output, which flutter passes through as is. + self.events.put({"event": "app.log", "params": {"log": line}}) + self.events.put(None) + + def next(self): + """The next event, failing once flutter exits or a command fails.""" + event = self.events.get() + if event is None: + raise SystemExit("flutter run exited before the check did") + if "error" in event: + raise SystemExit(f"flutter run refused a command: {event['error']}") + params = event.get("params", {}) + if event.get("event") == "app.start": + self.app = params["appId"] + elif event.get("event") == "app.started": + self.started = True + return event + + def command(self, method, **params): + """Send a command once the app has started, as flutter takes none before.""" + while not self.started: + self.next() + self.ids += 1 + self.process.stdin.write(json.dumps([{"id": self.ids, "method": method, "params": {"appId": self.app, **params}}]) + "\n") + self.process.stdin.flush() + + def output(self): + """What one run of the check printed, failing unless it exits 0.""" + chunks = None + while True: + event = self.next() + if event.get("event") != "app.log": + continue + line = event["params"]["log"] + print(line, file=sys.stderr) + # After whatever prefix the device's log adds. + at = line.find("rust2: ") + if at < 0: + continue + said = line[at + len("rust2: "):] + if said == "start": + chunks = [] + elif chunks is None: + continue + elif said.startswith("threw "): + raise SystemExit(f"the check threw in the Flutter app: {said}") + elif said.startswith("exit "): + if said != "exit 0": + raise SystemExit(f"the check failed in the Flutter app: {said}") + return base64.b64decode("".join(chunks)) + else: + chunks.append(said) + + +def consume_flutter(package, libs, lib, check, native, platform, device): + """A new Flutter app depending on the package, its libraries served from the loopback as + the release hosts them, run on `device` in debug, then again after a hot restart, which + keeps the library and Rust's statics but not the isolate: each run prints the check's + output.""" + with tempfile.TemporaryDirectory() as raw, Served(libs) as served: + work = Path(raw) + app = work / "consumer" + run("flutter", "create", "--platforms", platform, "--empty", "--project-name", "consumer", "--org", "dev.rust2", + app, capture_output=True) + run("flutter", "pub", "add", lib, "--path", local_package(package, work / "package", served.url), cwd=app, + capture_output=True) + (app / "lib" / "check.dart").write_text(dart_main(lib, check), encoding="utf-8") + (app / "lib" / "main.dart").write_text(FLUTTER_MAIN, encoding="utf-8") + flutter = Flutter(app, device) + try: + first = flutter.output() + flutter.command("app.restart", fullRestart=True, pause=False) + second = flutter.output() + flutter.command("app.stop") + flutter.process.stdin.close() + except BaseException: + flutter.process.kill() + raise + if flutter.process.wait(): + raise SystemExit(f"flutter run failed with status {flutter.process.returncode}") + for output, what in ((first, "Flutter app"), (second, "Flutter app after a hot restart")): + if native and check: + compare(Path(native).read_bytes(), output, what) + print(f"{what}: ok") + + def package_name(info, scope): """The web package's name on npm and JSR, `@/`, refusing what JSR cannot spell.""" name = f"@{scope}/{info['name']}" @@ -2111,6 +2497,28 @@ def main(): p.add_argument("--archive", required=True) p.add_argument("--check") p.add_argument("--native") + p = sub.add_parser("pub", help="write the pub.dev package, naming each target's Dart library by its hash") + p.add_argument("--manifest-path", required=True) + p.add_argument("--export", required=True) + p.add_argument("--libs", required=True, help="a directory of every Dart target's library, as build names them") + p.add_argument("--url", required=True, help="where the release hosts the libraries, ending in /") + p.add_argument("--changes", required=True, help="the release's changes.md") + p.add_argument("--out", required=True) + for name, helptext in (("consume-dart", "run a check program, or a load, in new Dart apps on the package"), + ("consume-flutter", "run a check program, or a load, in a new Flutter app, then after a hot restart")): + p = sub.add_parser(name, help=helptext) + p.add_argument("--package", required=True, help="the directory `pub` wrote") + p.add_argument("--libs", required=True, help="the libraries, served as the release hosts them") + p.add_argument("--check") + p.add_argument("--native") + if name == "consume-flutter": + p.add_argument("--platform", required=True, help="the platform `flutter create` makes the app for") + p.add_argument("--device", required=True, help="the device `flutter run` runs it on") + p = sub.add_parser("consume-source", help="run a check program, or a load, in a new Dart app on the export's own package") + p.add_argument("--manifest-path", required=True) + p.add_argument("--export", required=True) + p.add_argument("--check") + p.add_argument("--native") p = sub.add_parser("compare", help="compare a consumer's output with native Rust's") p.add_argument("native") p.add_argument("actual") @@ -2171,6 +2579,15 @@ def main(): consume_python(args.check, args.native, args.module) elif args.command == "consume-c": consume_c(args.archive, args.check, args.native) + elif args.command == "pub": + pub_package(facts(args.manifest_path), args.export, args.libs, args.url, args.changes, args.out) + elif args.command == "consume-dart": + consume_dart(args.package, args.libs, pub_name(args.package), args.check, args.native) + elif args.command == "consume-source": + consume_source(args.manifest_path, args.export, args.check, args.native) + elif args.command == "consume-flutter": + consume_flutter(args.package, args.libs, pub_name(args.package), args.check, args.native, args.platform, + args.device) elif args.command == "compare": compare(Path(args.native).read_bytes(), Path(args.actual).read_bytes(), "installed Swift package") elif args.command == "init": diff --git a/ffi/release/release.yml b/ffi/release/release.yml index c0e758086..30d34d0cc 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -1,12 +1,12 @@ name: Release @NAME@ bindings # Written by rust2's `ffi/release/package.py init`; yours to edit. rust2's builder, pinned -# below, exports @NAME@'s pub API, builds its Python wheels, Swift package, C archives, and -# web package, and checks them in clean consumers, all read-only. The jobs after it publish +# below, exports @NAME@'s pub API, builds its Python wheels, Swift package, C archives, Go +# module, pub.dev package, and web package, and checks them in clean consumers, all read-only. The jobs after it publish # only a tag's build, once `attest` has signed it: # # - Attestations: a build provenance attestation and an SBOM attestation for every wheel, the -# XCFramework, the Go module's .zip and .mod, each C archive, and the npm tarball, from +# XCFramework, the Go module's .zip and .mod, each C archive, each Dart library, and the npm tarball, from # this repository's GitHub, checked with `gh attestation verify`. # - PyPI: the project @NAME@, through the trusted publisher this repository registers there # for this workflow and the `pypi` environment. @@ -15,6 +15,9 @@ name: Release @NAME@ bindings # The mirror's token comes from a GitHub App installed on it, whose ID and key live in # the `swift-mirror` environment. # - C and C++: --.tar.gz, or .zip on Windows, on the same release. +# - pub.dev: the package its pubspec names, through pub.dev's automated publishing from this +# repository's @NAME@-v{{version}} tags in the `pub` environment, once the same release +# hosts --dart-., each library its build hook downloads. # - npm and JSR: the web package, when the build makes one, through the trusted publisher # npm registers for this workflow and the `npm` environment, and JSR's link to this # repository, each with provenance. @@ -122,6 +125,7 @@ jobs: artifacts/go/**/@v/*.zip artifacts/go/**/@v/*.mod artifacts/c/* + artifacts/dart/* artifacts/web/*.tgz - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: @@ -131,6 +135,7 @@ jobs: artifacts/go/**/@v/*.zip artifacts/go/**/@v/*.mod artifacts/c/* + artifacts/dart/* sbom-path: artifacts/sbom.cdx.json # The npm tarball's own, from the web export's dependencies. - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 @@ -186,7 +191,8 @@ jobs: with: packages-dir: dist/wheels/ - # The XCFramework the mirror's Package.swift names and each target's C archive, uploaded + # The XCFramework the mirror's Package.swift names, each target's C archive, and each Dart + # library the pub.dev package's hook downloads, uploaded # once: a consumer that resolved a checksum must never see other bytes at that URL. The # release stays a draft until its assets are up, as an immutable release takes none once # published. The release's API, api.json, is an asset too: the next release's build checks @@ -222,7 +228,7 @@ jobs: gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag --draft --prerelease="$PRERELEASE" --title "$TAG" --notes-file release/changes.md fi shopt -s nullglob - for path in "$XCFRAMEWORK" release/api.json dist/c/*; do + for path in "$XCFRAMEWORK" release/api.json dist/c/* dist/dart/*; do asset=$(basename "$path") if gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --pattern "$asset" --dir existing; then cmp "existing/$asset" "$path" || { echo "::error::$TAG already hosts another $asset"; exit 1; } @@ -303,6 +309,37 @@ jobs: git tag "$VERSION" git push --quiet --atomic origin HEAD "refs/tags/$VERSION" + # The package, once the release hosts every library its hook downloads. pub.dev takes it + # from a tag push alone, by the OIDC token setup-dart makes. A rerun publishes only a + # version pub.dev lacks, and fails on one it hosts with other files. + pub: + needs: [build, tag, attest, github-release] + runs-on: ubuntu-24.04 + environment: pub + permissions: + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-pub + path: pub + - uses: dart-lang/setup-dart@2286a0c042b44434fe26d292bbce8d34f7ec662b # v1.8.2 + with: + sdk: stable + - name: Publish + working-directory: pub/package + run: | + name=$(sed -n 's/^name: //p' pubspec.yaml) + version=$(sed -n 's/^version: //p' pubspec.yaml) + if url=$(curl -sf "https://pub.dev/api/packages/$name/versions/$version" | jq -er .archive_url); then + mkdir ../hosted + curl -sfL "$url" | tar -xz -C ../hosted + diff -r . ../hosted || { echo "::error::pub.dev hosts another $name $version"; exit 1; } + echo "pub.dev already hosts $name $version" + exit 0 + fi + dart pub publish --force + # A rerun publishes only a version npm lacks, and fails on one it hosts with other bytes. npm: needs: [build, tag, attest] diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 3edf0732a..1c75b4349 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -23,7 +23,7 @@ def info(root, **overrides): base = { "name": "moq-ffi", "version": "0.4.12", "description": "Media over QUIC,\n bindings", "license": "MIT OR Apache-2.0", "license_file": None, "repository": "https://github.com/moq-dev/moq", - "homepage": None, "lib": "moq", "dir": str(root), "workspace": str(root), "examples": [], + "homepage": None, "readme": None, "lib": "moq", "dir": str(root), "workspace": str(root), "examples": [], "epoch": 1_700_000_000, "sbom": b"{}\n", } return {**base, **overrides} @@ -734,6 +734,98 @@ def test_archives_are_the_same_bytes_whenever_and_wherever_packed(self): self.assertEqual(made[2], made[3]) +def dart_export(root, version="0.4.12"): + """The Dart package an export writes, as `dart::pubspec` lays out its pubspec.""" + dart = root / "dart" + (dart / "lib").mkdir(parents=True) + (dart / "hook").mkdir() + (dart / "lib/moq.dart").write_text("library;\n") + (dart / "hook/build.dart").write_text("void main() {}\n") + (dart / "pubspec.yaml").write_text( + "# Generated by rust2-ffi.\nname: moq\ndescription: The pub API of moq-ffi.\n" + f"version: {version}\n\nenvironment:\n sdk: ^3.10.0\n") + return root + + +def dart_libs(root, facts, skip=()): + """A library of each Dart target, as `build` names it, its bytes its target.""" + root.mkdir() + for target in package.DART_TARGETS: + if target not in skip: + (root / package.dart_file(facts, target)).write_text(target) + return root + + +class PubPackage(unittest.TestCase): + URL = "https://github.com/moq-dev/moq/releases/download/moq-ffi-v0.4.12/" + + def make(self, facts, libs=None, changes="## API\n\nAdded `moq::Track`.\n", url=URL): + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + export = dart_export(tmp / "export") + (tmp / "changes.md").write_text(changes) + out = tmp / "pub" + package.pub_package(facts, export, libs or dart_libs(tmp / "libs", facts), url, tmp / "changes.md", out) + return out + + def test_names_each_library_by_size_hash_and_floor(self): + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + out = self.make(info(tmp)) + release = json.loads((out / "hook/release.json").read_text()) + self.assertEqual(release["url"], self.URL) + self.assertEqual(set(release["libraries"]), set(package.DART_TARGETS)) + android = release["libraries"]["aarch64-linux-android"] + self.assertEqual(android, {"file": "moq-ffi-0.4.12-dart-aarch64-linux-android.so", "size": 21, + "sha256": package.hashlib.sha256(b"aarch64-linux-android").hexdigest(), + "minimum": "24"}) + self.assertEqual(release["libraries"]["aarch64-apple-ios-sim"]["file"], + "moq-ffi-0.4.12-dart-aarch64-apple-ios-sim.dylib") + self.assertEqual(release["libraries"]["x86_64-pc-windows-msvc"]["file"], "moq-ffi-0.4.12-dart-x86_64-pc-windows-msvc.dll") + # Every arm64 Mac runs macOS 11, so no app is too old for its library. + self.assertNotIn("minimum", release["libraries"]["aarch64-apple-darwin"]) + self.assertEqual(release["libraries"]["x86_64-apple-darwin"]["minimum"], "10.15") + self.assertTrue((out / "lib/moq.dart").is_file() and (out / "hook/build.dart").is_file()) + + def test_states_cargos_facts(self): + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + pubspec = (self.make(info(tmp, homepage="https://moq.dev")) / "pubspec.yaml").read_text() + self.assertIn('\ndescription: "Media over QUIC, bindings"\nversion: 0.4.12\n' + 'repository: "https://github.com/moq-dev/moq"\nhomepage: "https://moq.dev"\n\nenvironment:', pubspec) + # Without Cargo's, the export's own description stays. + pubspec = (self.make(info(tmp, description=None, repository=None)) / "pubspec.yaml").read_text() + self.assertIn("\ndescription: The pub API of moq-ffi.\nversion: 0.4.12\n\n", pubspec) + + def test_license_readme_and_changelog(self): + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + (tmp / "LICENSE-MIT").write_text("MIT text\n") + (tmp / "LICENSE-APACHE").write_text("Apache text\n") + (tmp / "README.md").write_text("# moq\n") + out = self.make(info(tmp, readme="README.md")) + self.assertEqual((out / "LICENSE").read_text(), "Apache text\n\nMIT text\n") + self.assertEqual((out / "README.md").read_text(), "# moq\n") + self.assertEqual((out / "CHANGELOG.md").read_text(), "## 0.4.12\n\n### API\n\nAdded `moq::Track`.\n") + bare = Path(self.enterContext(tempfile.TemporaryDirectory())) + out = self.make(info(bare)) + self.assertEqual((out / "LICENSE").read_text(), "SPDX-License-Identifier: MIT OR Apache-2.0\n") + self.assertEqual((out / "README.md").read_text(), + "# moq\n\nMedia over QUIC, bindings\n\nThe Dart binding of the Rust crate moq-ffi, generated by rust2-ffi.\n") + with self.assertRaises(SystemExit): + self.make(info(bare, license=None)) + + def test_refuses_a_missing_or_stray_library_and_a_bad_url(self): + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + facts = info(tmp) + with self.assertRaises(SystemExit): + self.make(facts, libs=dart_libs(tmp / "some", facts, skip={"x86_64-linux-android"})) + stray = dart_libs(tmp / "stray", facts) + (stray / "moq-ffi-0.4.11-dart-x86_64-linux-android.so").write_text("old") + with self.assertRaises(SystemExit): + self.make(facts, libs=stray) + with self.assertRaises(SystemExit): + self.make(facts, url=self.URL.rstrip("/")) + with self.assertRaises(SystemExit): + self.make(info(tmp, version="0.4.13")) + + class Template(unittest.TestCase): def test_fills_every_identity(self): text = package.template(info("/src"), "rs/moq-ffi/Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi", "moq.dev/moq-ffi") @@ -743,6 +835,8 @@ def test_fills_every_identity(self): self.assertIn("owner: moq-dev\n", text) self.assertIn("repositories: moq-swift-ffi\n", text) self.assertIn("go-module: moq.dev/moq-ffi\n", text) + self.assertIn("name: moq-ffi-pub\n", text) + self.assertIn('"$XCFRAMEWORK" release/api.json dist/c/* dist/dart/*', text) self.assertNotRegex(text, r"@[A-Z_]+@") def test_refuses_a_movable_pin_or_bad_mirror(self): diff --git a/ffi/runtime/hook.dart b/ffi/runtime/hook.dart new file mode 100644 index 000000000..bc72c238b --- /dev/null +++ b/ffi/runtime/hook.dart @@ -0,0 +1,193 @@ +// The build hook of a Dart package rust2-ffi exports, written by it. It bundles the +// package's Rust library as the native asset its `@Native` functions find. A release's +// hook/release.json names each target's library by URL, size, and SHA-256, which this +// downloads; an export holds none, so this builds the crate rust2-ffi exported the package +// beside, ../Cargo.toml, with Cargo. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:code_assets/code_assets.dart'; +import 'package:crypto/crypto.dart'; +import 'package:hooks/hooks.dart'; + +Future main(List args) async { + await build(args, (input, output) async { + if (!input.config.buildCodeAssets) return; + final code = input.config.code; + final package = input.packageName; + // Dart links no static library yet, so a build that takes only one has no use for this. + if (code.linkModePreference == LinkModePreference.static) { + throw UnsupportedError('$package ships a dynamic library, which a static-only build cannot take'); + } + final target = _target(code); + final release = File.fromUri(input.packageRoot.resolve('hook/release.json')); + final File library; + if (release.existsSync()) { + library = await _download(input, code, release, target); + output.dependencies.add(release.uri); + } else { + final (built, inputs) = await _build(input, code, target); + library = built; + output.dependencies.addAll(inputs); + } + // A copy, as Cargo may rebuild its own and the shared cache serves other builds. + final bundled = input.outputDirectory.resolve(code.targetOS.dylibFileName(package)); + await library.copy(bundled.toFilePath()); + output.assets.code.add( + CodeAsset(package: package, name: '$package.dart', linkMode: DynamicLoadingBundled(), file: bundled), + ); + }); +} + +/// The Rust target of what the build is for. +String _target(CodeConfig code) { + final simulator = code.targetOS == OS.iOS && code.iOS.targetSdk == IOSSdk.iPhoneSimulator; + return switch ((code.targetOS, code.targetArchitecture)) { + (OS.android, Architecture.arm64) => 'aarch64-linux-android', + (OS.android, Architecture.arm) => 'armv7-linux-androideabi', + (OS.android, Architecture.x64) => 'x86_64-linux-android', + (OS.iOS, Architecture.arm64) => simulator ? 'aarch64-apple-ios-sim' : 'aarch64-apple-ios', + (OS.iOS, Architecture.x64) when simulator => 'x86_64-apple-ios', + (OS.macOS, Architecture.arm64) => 'aarch64-apple-darwin', + (OS.macOS, Architecture.x64) => 'x86_64-apple-darwin', + (OS.linux, Architecture.arm64) => 'aarch64-unknown-linux-gnu', + (OS.linux, Architecture.x64) => 'x86_64-unknown-linux-gnu', + (OS.windows, Architecture.arm64) => 'aarch64-pc-windows-msvc', + (OS.windows, Architecture.x64) => 'x86_64-pc-windows-msvc', + (final os, final arch) => throw UnsupportedError('no Rust target builds for $os on $arch'), + }; +} + +/// The release's library for `target`, from the shared cache when it holds the same bytes, +/// else downloaded and checked against the size and hash the release names. +Future _download(BuildInput input, CodeConfig code, File release, String target) async { + final manifest = jsonDecode(await release.readAsString()) as Map; + final libraries = manifest['libraries']! as Map; + final entry = libraries[target] as Map?; + if (entry == null) { + throw UnsupportedError('${input.packageName} ships no library for $target, only ${libraries.keys.join(', ')}'); + } + _floor(input.packageName, code, entry['minimum'] as String?); + final name = entry['file']! as String; + final size = entry['size']! as int; + final hash = entry['sha256']! as String; + final cached = File.fromUri(input.outputDirectoryShared.resolve(name)); + if (cached.existsSync() && sha256.convert(await cached.readAsBytes()).toString() == hash) return cached; + final url = Uri.parse(manifest['url']! as String).resolve(name); + final client = HttpClient()..findProxy = HttpClient.findProxyFromEnvironment; + try { + final response = await (await client.getUrl(url)).close(); + if (response.statusCode != HttpStatus.ok) { + throw HttpException('$url answered ${response.statusCode}', uri: url); + } + // Never more than the release names, whatever the host sends. + final bytes = BytesBuilder(copy: false); + await for (final chunk in response) { + bytes.add(chunk); + if (bytes.length > size) throw StateError('$url is longer than the $size bytes the release names'); + } + final data = bytes.takeBytes(); + final actual = sha256.convert(data).toString(); + if (data.length != size || actual != hash) { + throw StateError('$url is ${data.length} bytes of SHA-256 $actual, not the $size of $hash the release names'); + } + // Renamed into place, so no build sees part of it. + await cached.parent.create(recursive: true); + final part = File('${cached.path}.$pid.part'); + await part.writeAsBytes(data, flush: true); + await part.rename(cached.path); + return cached; + } finally { + client.close(force: true); + } +} + +/// Refuse an app older than the oldest system the released library loads on. +void _floor(String package, CodeConfig code, String? minimum) { + if (minimum == null) return; + final parts = minimum.split('.').map(int.parse).toList(); + if (code.targetOS == OS.android) { + if (code.android.targetNdkApi < parts[0]) { + throw UnsupportedError( + "$package's Android library needs API $minimum, above the app's ${code.android.targetNdkApi}: raise its minSdk", + ); + } + return; + } + final floor = AppleVersion(parts[0], minor: parts.length > 1 ? parts[1] : 0); + final (system, version) = switch (code.targetOS) { + OS.iOS => ('iOS', code.iOS.deploymentVersion), + OS.macOS => ('macOS', code.macOS.deploymentVersion), + final os => throw UnsupportedError('$package names a minimum $minimum for $os'), + }; + if (version < floor) { + throw UnsupportedError( + "$package's $system library needs $system $minimum, above the app's ${version.major}.${version.minor}: raise its deployment target", + ); + } +} + +/// The library Cargo builds of the exported crate for `target`, and the files it read. +Future<(File, List)> _build(BuildInput input, CodeConfig code, String target) async { + final manifest = File.fromUri(input.packageRoot.resolve('../Cargo.toml')); + if (!manifest.existsSync()) { + throw StateError( + '${input.packageName} holds neither hook/release.json, which its release writes, ' + 'nor ../Cargo.toml, the crate rust2-ffi exported it with', + ); + } + final environment = {}; + final key = target.replaceAll('-', '_'); + switch (code.targetOS) { + case OS.android: + // The NDK's clang for the app's API level, which Flutter passes beside its own. + final compiler = code.cCompiler; + if (compiler == null) throw StateError('building $target needs the Android NDK, which this build names none of'); + final triple = target == 'armv7-linux-androideabi' ? 'armv7a-linux-androideabi' : target; + final cmd = Platform.isWindows ? '.cmd' : ''; + final clang = compiler.compiler.resolve('$triple${code.android.targetNdkApi}-clang'); + environment['CARGO_TARGET_${key.toUpperCase()}_LINKER'] = '${clang.toFilePath()}$cmd'; + environment['CC_$key'] = '${clang.toFilePath()}$cmd'; + environment['CXX_$key'] = '${clang.toFilePath()}++$cmd'; + environment['AR_$key'] = compiler.archiver.toFilePath(); + case OS.iOS: + final version = code.iOS.deploymentVersion; + environment['IPHONEOS_DEPLOYMENT_TARGET'] = '${version.major}.${version.minor}'; + case OS.macOS: + final version = code.macOS.deploymentVersion; + environment['MACOSX_DEPLOYMENT_TARGET'] = '${version.major}.${version.minor}'; + } + final args = [ + 'build', '--release', '--lib', '--features', 'dart', '--target', target, // + '--manifest-path', manifest.path, '--message-format=json-render-diagnostics', + ]; + // From the crate's directory, whose toolchain rustup picks. + final result = await Process.run('cargo', args, workingDirectory: manifest.parent.path, environment: environment); + if (result.exitCode != 0) { + throw ProcessException('cargo', args, '${result.stderr}', result.exitCode); + } + final name = code.targetOS.dylibFileName(input.packageName); + final built = { + for (final line in LineSplitter.split(result.stdout as String)) + if (jsonDecode(line) case {'reason': 'compiler-artifact', 'filenames': final List files}) + for (final file in files.cast()) + if (File(file).uri.pathSegments.last == name) file, + }; + if (built.length != 1) throw StateError('cargo built ${built.isEmpty ? 'no' : built.length} $name for $target'); + final library = File(built.single); + // Cargo's dep-info beside it lists the sources of every path package the build read. + final info = File('${library.path.substring(0, library.path.length - name.length)}${name.split('.').first}.d'); + final inputs = [manifest.uri, for (final path in _depInfo(await info.readAsString())) File(path).uri]; + return (library, inputs); +} + +/// The files a Cargo dep-info file names its artifact reading: after each line's +/// `: `, paths split on spaces, a space in one escaped as `\ `. +List _depInfo(String text) => [ + for (final line in LineSplitter.split(text)) + if (line.indexOf(': ') case final at when at >= 0) + for (final match in RegExp(r'(?:\\ |[^ ])+').allMatches(line.substring(at + 2))) + match[0]!.replaceAll(r'\ ', ' '), +]; diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index 825a412fd..95a8160f6 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -1,4 +1,4 @@ -//! Dart over the C ABI through `dart:ffi`: `dart/.dart`, which binds each C function +//! Dart over the C ABI through `dart:ffi`: `dart/lib/.dart`, which binds each C function //! with `@Native`, so it finds the library as the package's native asset, and the Rust the //! crate's `dart` feature builds for what `dart:ffi` lacks. //! @@ -36,6 +36,37 @@ const RUNTIME: &str = include_str!("../runtime/rt.dart"); /// The Rust runtime the crate's `dart` feature builds. pub(crate) const GLUE_RUNTIME: &str = include_str!("../runtime/dart.rs"); +/// The package's build hook, `hook/build.dart`, which bundles its library. +pub(crate) const HOOK: &str = include_str!("../runtime/hook.dart"); + +/// The package's `pubspec.yaml`: `lib`, the library's prefix, as its name, at the facade's +/// version, depending on what the hook imports; or why pub takes no package of that name. +pub(crate) fn pubspec(facade: &crate::config::Package, lib: &str) -> Result { + let valid = lib.starts_with(|c: char| c.is_ascii_lowercase() || c == '_') + && lib + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); + if !valid || RESERVED.contains(&lib) || BUILT_IN.contains(&lib) { + return Err(format!( + "the library `{lib}` names no Dart package, whose name is lowercase letters, digits, and `_`, \ + and no reserved word: rename the crate's `[lib] name`" + )); + } + // The SDK floor is the one whose build hooks are stable, 3.10. + Ok(format!( + "# Generated by rust2-ffi: the Dart binding of the pub API of {name}.\n\ + name: {lib}\n\ + description: The pub API of {name} for Dart and Flutter, generated by rust2-ffi.\n\ + version: {version}\n\ + \n\ + environment:\n sdk: ^3.10.0\n\ + \n\ + dependencies:\n code_assets: ^2.2.0\n crypto: ^3.0.7\n hooks: ^2.2.0\n", + name = facade.name, + version = facade.version, + )) +} + /// Bytes Rust returns up to this long are copied into Dart's memory, and longer ones lent /// to it: a lent list's finalizer, and the external size it charges the garbage collector, /// cost more than a copy up to about this long. diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 09cebdd19..036600f39 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -257,7 +257,7 @@ impl Crate<'_> { /// `kotlin/.kt`, whose JNI functions the crate's `kotlin` feature builds, /// `python/.py`, whose CPython extension its `python` feature builds, /// `swift/.swift` with the clang module it imports the header through, and - /// `go/.go`, and `dart/.dart`, whose glue its `dart` feature builds. + /// `go/.go`, and the Dart package `dart/`, whose glue its `dart` feature builds. pub(crate) fn generate(&self, out: &Path) -> Result { // Cargo unifies in what the facade enables in its dependencies. let facade = self.facade; @@ -367,7 +367,7 @@ impl Crate<'_> { kotlin = [\"dep:jni-sys\"]\n\ # The CPython extension python/{lib}.py loads.\n\ python = [\"dep:pyo3-ffi\", \"dep:pyo3-build-config\", \"dep:smallvec\"]\n\ - # What dart/{lib}.dart calls that `dart:ffi` lacks.\n\ + # What dart/lib/{lib}.dart calls that `dart:ffi` lacks.\n\ dart = []\n\ # What web/index.ts calls that the C ABI lacks, built for wasm32-wasip1.\n\ web = []\n\ @@ -463,7 +463,16 @@ impl Crate<'_> { ); output.file(&out.join("src/dart/mod.rs"), &dart.glue); output.file(&out.join("src/dart/runtime.rs"), crate::dart::GLUE_RUNTIME); - output.file(&out.join("dart").join(format!("{}.dart", self.abi.prefix)), &dart.dart); + let package = out.join("dart"); + output.file( + &package.join("pubspec.yaml"), + &crate::dart::pubspec(facade, &self.abi.prefix)?, + ); + output.file( + &package.join("lib").join(format!("{}.dart", self.abi.prefix)), + &dart.dart, + ); + output.file(&package.join("hook").join("build.dart"), crate::dart::HOOK); if let Some(web) = web { output.file(&out.join("src/web/mod.rs"), &web.glue); output.file(&out.join("src/web/runtime.rs"), crate::web::GLUE_RUNTIME); diff --git a/ffi/surface/surface.py b/ffi/surface/surface.py index 9ff28e270..75dba9610 100644 --- a/ffi/surface/surface.py +++ b/ffi/surface/surface.py @@ -865,7 +865,7 @@ def files(export): for lang, glob, name, outline, label in [ ("kotlin", "kotlin/*.kt", "kotlin.kt", lambda t: indented(t, "kotlin"), "// {}"), ("swift", "swift/*.swift", "swift.swift", lambda t: indented(t, "swift"), "// {}"), - ("dart", "dart/*.dart", "dart.dart", lambda t: braced(t, "dart"), "// {}"), + ("dart", "dart/lib/*.dart", "dart.dart", lambda t: braced(t, "dart"), "// {}"), ("python", "python/*.py", "python.pyi", python_stub, "# {}"), ]: sources = sorted(export.glob(glob)) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 3749dc3e0..382f48698 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -2637,7 +2637,7 @@ fn dart_small_api_lints() { ); step::write( &lint.join("edge.dart"), - std::fs::read_to_string(out.join("dart").join("edge.dart")).unwrap(), + std::fs::read_to_string(out.join("dart/lib/edge.dart")).unwrap(), ); let step = Step { stamp: &dir.join("analyze.stamp"), @@ -2655,7 +2655,7 @@ fn api_dart() { let api = api(); let scripts = generated("dart", "transcript.dart", &transcript::dart::emit(transcript())); let dir = scratch().join("api").join("dart"); - let generated = api.out.join("dart"); + let generated = api.out.join("dart/lib"); // The analyzer reads its options from the analyzed directory, so analyze a copy with them. let lint = dir.join("lint"); step::write( @@ -2800,7 +2800,7 @@ fn protocol_dart(api: &Api, dir: &Path) -> PathBuf { dart_compile( "api", dir, - &api.out.join("dart"), + &api.out.join("dart/lib"), &api.lib.join(PYTHON_LIB), &manifest_dir().join("tests").join("protocol.dart"), &[], diff --git a/ffi/tests/fixtures/corpus.rs b/ffi/tests/fixtures/corpus.rs index d11b20359..cc9e3ed6f 100644 --- a/ffi/tests/fixtures/corpus.rs +++ b/ffi/tests/fixtures/corpus.rs @@ -462,7 +462,7 @@ fn languages(lib: &str, out: &Path, dir: &Path, built: &Built) -> Result<(), (&' format!("import 'package:{lib}/{lib}.dart';\n\nvoid main() {{}}\n"), ); let shared = built.lib.join(super::shared(lib)); - let (mut kernel, _) = super::dart_kernel(lib, &dart, &out.join("dart"), &shared, &main, &[]); + let (mut kernel, _) = super::dart_kernel(lib, &dart, &out.join("dart/lib"), &shared, &main, &[]); checked("Dart", &mut kernel)?; Ok(()) } diff --git a/quest/m1/ffi-packaging/pubdev.md b/quest/m1/ffi-packaging/pubdev.md deleted file mode 100644 index 0a47ce1c2..000000000 --- a/quest/m1/ffi-packaging/pubdev.md +++ /dev/null @@ -1,18 +0,0 @@ -# [M] A pub.dev package with a build hook - -## Goal - -The export writes a Dart package, `pubspec.yaml` and `hook/build.dart`, and the builder -releases it to pub.dev with checksummed per-target libraries the hook downloads, building -from source inside a checkout. A clean Dart app and a Flutter app on the Android emulator -consume it and print the transcript; the Flutter run includes a hot restart, since Rust's -statics outlive the restarted isolate. - -## Plan - -Found by the 2026-11-18 language audit: moq already publishes `moq_ffi` with a -hand-written UniFFI hook (`dart/moq_ffi/hook/build.dart` in moq-dev/moq) that downloads -checksummed libraries and accepts a `user_defines` override. Build hooks are stable since -Dart 3.10 and Flutter 3.38 ([announcement](https://blog.dart.dev/announcing-dart-3-10-ea8b952b6088)). - -Public API: the generated package layout. From 55907bb32fbff6593facd8db2efb2f67330b8ab6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 02:53:15 -0800 Subject: [PATCH 03/15] ffi: the shapes test reads the Dart from lib/ Co-Authored-By: Claude Opus 5.5 --- ffi/tests/fixtures/shapes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 4587e6a04..6a24683f0 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -2548,7 +2548,7 @@ func failed[A, B any](f func(A) B, err error) B { /// a `try350` exception its `on` clause names go uncaught, which alone it catches. fn dart(built: &Built, calls: &[Call]) -> String { let name = built.name(); - let source = std::fs::read_to_string(built.out.join("dart").join(format!("{name}.dart"))).unwrap(); + let source = std::fs::read_to_string(built.out.join("dart/lib").join(format!("{name}.dart"))).unwrap(); let mut sinks = String::new(); // Each sink's class, defined by its first call the driver runs. let mut echoes = std::collections::BTreeSet::new(); @@ -2930,7 +2930,7 @@ fn round_trip(built: &Built, language: &str) { let main = dir.join("main.dart"); step::write(&main, dart(built, calls)); let lib = built.lib.join(super::shared(&name)); - let dill = super::dart_compile(&name, &dir, &built.out.join("dart"), &lib, &main, &[]); + let dill = super::dart_compile(&name, &dir, &built.out.join("dart/lib"), &lib, &main, &[]); run(&mut super::dart_run(&dill, &[])) } "web" => { From 3d87bc9ff36917a0f4dbbb5830cab6ce16c1cfcb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 02:54:19 -0800 Subject: [PATCH 04/15] ffi: the fixtures state the repository pub.dev asks a package for Co-Authored-By: Claude Opus 5.5 --- ffi/fixtures/api/Cargo.toml | 1 + ffi/justfile | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ffi/fixtures/api/Cargo.toml b/ffi/fixtures/api/Cargo.toml index b7aa14f45..e1939d6de 100644 --- a/ffi/fixtures/api/Cargo.toml +++ b/ffi/fixtures/api/Cargo.toml @@ -4,6 +4,7 @@ description = "Each construct rust2-ffi exports, driven from C by tests/api.c" version = "0.0.0" edition = "2024" license = "MIT OR Apache-2.0" +repository = "https://github.com/kixel-ai/rust2" publish = false # Named apart from the package: the export's prefix, which the export must follow. diff --git a/ffi/justfile b/ffi/justfile index ad0f5bb0a..0163abcbf 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -585,8 +585,9 @@ standin checkout="" web="": fi ffi=$(cd "$src" && cargo metadata --format-version 1 --no-deps | jq -c '.packages[] | select(.name == "moq-ffi")') features=$(jq -c '[.dependencies[] | select(.kind == null) | {key: .name, value: .features}] | from_entries' <<< "$ffi") - # moq-ffi's license, which a package of the stand-in states as moq-ffi's would. + # moq-ffi's license and repository, which a package of the stand-in states as moq-ffi's would. license=$(jq -r '.license // empty' <<< "$ffi") + repository=$(jq -r '.repository // empty' <<< "$ffi") mkdir -p "$dir/src" # Replace a file only when its content changes, so an unchanged stand-in, and the # export over it, stay fresh. @@ -601,6 +602,7 @@ standin checkout="" web="": printf '# Generated by just ffi standin, from %s.\n' "$src" printf '[package]\nname = "moq-ffi"\nversion = "0.0.0"\nedition = "2024"\npublish = false\n' if [ -n "$license" ]; then printf 'license = "%s"\n' "$license"; fi + if [ -n "$repository" ]; then printf 'repository = "%s"\n' "$repository"; fi printf '\n' # The prefix of every C name. printf '[lib]\nname = "moq"\n\n[dependencies]\n' From 02dca418815bddc48f82ea773e96fa9abf59438d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 03:06:21 -0800 Subject: [PATCH 05/15] ffi: the source hook builds on the facade's toolchain and reruns on its manifests and lock Co-Authored-By: Claude Opus 5.5 --- .github/workflows/bindings.yml | 2 +- ffi/release/package.py | 16 +++++--------- ffi/runtime/hook.dart | 40 ++++++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index d6128f694..d229f8ff3 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -398,7 +398,7 @@ jobs: args=() if [[ -f dist/checks/main.dart ]]; then args+=(--check dist/checks/main.dart); fi if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi - python3 -I rust2/ffi/release/package.py consume-source --manifest-path "source/$MANIFEST" --export export "${args[@]}" + python3 -I rust2/ffi/release/package.py consume-source --export export "${args[@]}" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} diff --git a/ffi/release/package.py b/ffi/release/package.py index cf91f8e60..9dafdd384 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -2014,9 +2014,9 @@ def consume_dart(package, libs, lib, check, native): run_dart([exe], work, check, native, "dart build cli") -def run_dart(command, cwd, check, native, what, env=None): +def run_dart(command, cwd, check, native, what): """Run a Dart consumer, and compare what it prints with native Rust.""" - result = subprocess.run([str(c) for c in command], cwd=cwd, capture_output=True, env={**os.environ, **(env or {})}) + result = subprocess.run([str(c) for c in command], cwd=cwd, capture_output=True) sys.stderr.buffer.write(result.stderr) if result.returncode: raise SystemExit(f"{what} failed on the Dart package with status {result.returncode}") @@ -2027,18 +2027,15 @@ def run_dart(command, cwd, check, native, what, env=None): print(f"{what}: ok") -def consume_source(manifest, export, check, native): +def consume_source(export, check, native): """A new Dart app depending on the export's own package, whose hook builds the exported - crate with Cargo, as in a checkout, on the crate's own toolchain.""" + crate with Cargo, as in a checkout.""" package = Path(export).resolve() / "dart" lib = pub_name(package) - toolchain = output("rustup", "show", "active-toolchain", cwd=Path(manifest).resolve().parent).split()[0] with tempfile.TemporaryDirectory() as raw: app = dart_app(raw, "consumer", package, lib) (app / "bin" / "main.dart").write_text(dart_main(lib, check), encoding="utf-8") - # The hook's environment keeps RUSTUP_ variables. - run_dart(["dart", "run", "bin/main.dart"], app, check, native, "dart run from source", - env={"RUSTUP_TOOLCHAIN": toolchain}) + run_dart(["dart", "run", "bin/main.dart"], app, check, native, "dart run from source") # The Flutter consumer's entry, which runs the check, collecting what it prints, then reports @@ -2515,7 +2512,6 @@ def main(): p.add_argument("--platform", required=True, help="the platform `flutter create` makes the app for") p.add_argument("--device", required=True, help="the device `flutter run` runs it on") p = sub.add_parser("consume-source", help="run a check program, or a load, in a new Dart app on the export's own package") - p.add_argument("--manifest-path", required=True) p.add_argument("--export", required=True) p.add_argument("--check") p.add_argument("--native") @@ -2584,7 +2580,7 @@ def main(): elif args.command == "consume-dart": consume_dart(args.package, args.libs, pub_name(args.package), args.check, args.native) elif args.command == "consume-source": - consume_source(args.manifest_path, args.export, args.check, args.native) + consume_source(args.export, args.check, args.native) elif args.command == "consume-flutter": consume_flutter(args.package, args.libs, pub_name(args.package), args.check, args.native, args.platform, args.device) diff --git a/ffi/runtime/hook.dart b/ffi/runtime/hook.dart index bc72c238b..1c231e282 100644 --- a/ffi/runtime/hook.dart +++ b/ffi/runtime/hook.dart @@ -159,18 +159,27 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe final version = code.macOS.deploymentVersion; environment['MACOSX_DEPLOYMENT_TARGET'] = '${version.major}.${version.minor}'; } - final args = [ + // The crate the export depends on as `_export`, from whose directory rustup picks + // its toolchain, as its own build would, and each path package's manifest and the lock, + // which change the build as its sources do. + final metadata = await _cargo(['metadata', '--format-version', '1', '--manifest-path', manifest.path, // + '--filter-platform', target], manifest.parent.path, environment); + final resolved = jsonDecode(metadata) as Map; + final packages = (resolved['packages']! as List).cast>(); + final root = (resolved['resolve']! as Map)['root']; + final facade = [ + for (final package in packages) + if (package['id'] == root) + for (final dependency in (package['dependencies']! as List).cast>()) + if (dependency['rename'] == '${input.packageName}_export') dependency['path']! as String, + ].single; + final output = await _cargo([ 'build', '--release', '--lib', '--features', 'dart', '--target', target, // '--manifest-path', manifest.path, '--message-format=json-render-diagnostics', - ]; - // From the crate's directory, whose toolchain rustup picks. - final result = await Process.run('cargo', args, workingDirectory: manifest.parent.path, environment: environment); - if (result.exitCode != 0) { - throw ProcessException('cargo', args, '${result.stderr}', result.exitCode); - } + ], facade, environment); final name = code.targetOS.dylibFileName(input.packageName); final built = { - for (final line in LineSplitter.split(result.stdout as String)) + for (final line in LineSplitter.split(output)) if (jsonDecode(line) case {'reason': 'compiler-artifact', 'filenames': final List files}) for (final file in files.cast()) if (File(file).uri.pathSegments.last == name) file, @@ -179,10 +188,23 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe final library = File(built.single); // Cargo's dep-info beside it lists the sources of every path package the build read. final info = File('${library.path.substring(0, library.path.length - name.length)}${name.split('.').first}.d'); - final inputs = [manifest.uri, for (final path in _depInfo(await info.readAsString())) File(path).uri]; + final lock = File('${resolved['workspace_root']}${Platform.pathSeparator}Cargo.lock'); + final inputs = [ + lock.uri, + for (final package in packages) + if (package['source'] == null) File(package['manifest_path']! as String).uri, + for (final path in _depInfo(await info.readAsString())) File(path).uri, + ]; return (library, inputs); } +/// What Cargo prints, run from `directory`, failing with what it says on stderr. +Future _cargo(List args, String directory, Map environment) async { + final result = await Process.run('cargo', args, workingDirectory: directory, environment: environment); + if (result.exitCode != 0) throw ProcessException('cargo', args, '${result.stderr}', result.exitCode); + return result.stdout as String; +} + /// The files a Cargo dep-info file names its artifact reading: after each line's /// `: `, paths split on spaces, a space in one escaped as `\ `. List _depInfo(String text) => [ From 743cb7a2d640b39172894080b22c4d53f5707c55 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 03:12:21 -0800 Subject: [PATCH 06/15] ffi: the source hook finds the facade before any Cargo call Co-Authored-By: Claude Opus 5.5 --- ffi/runtime/hook.dart | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ffi/runtime/hook.dart b/ffi/runtime/hook.dart index 1c231e282..32080c801 100644 --- a/ffi/runtime/hook.dart +++ b/ffi/runtime/hook.dart @@ -159,20 +159,18 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe final version = code.macOS.deploymentVersion; environment['MACOSX_DEPLOYMENT_TARGET'] = '${version.major}.${version.minor}'; } - // The crate the export depends on as `_export`, from whose directory rustup picks - // its toolchain, as its own build would, and each path package's manifest and the lock, - // which change the build as its sources do. + // The crate the export depends on as `_export`, as the exporter writes its line, + // its path a Rust string: from its directory rustup picks the toolchain for each Cargo + // call, as its own build would. + final dependency = RegExp('^${input.packageName}_export = \\{ package = "(?:[^"\\\\]|\\\\.)*", path = ("(?:[^"\\\\]|\\\\.)*")', multiLine: true) + .firstMatch(await manifest.readAsString()); + if (dependency == null) throw StateError('${manifest.path} depends on no ${input.packageName}_export'); + final facade = jsonDecode(dependency[1]!) as String; + // Each path package's manifest and the lock, which change the build as its sources do. final metadata = await _cargo(['metadata', '--format-version', '1', '--manifest-path', manifest.path, // - '--filter-platform', target], manifest.parent.path, environment); + '--filter-platform', target], facade, environment); final resolved = jsonDecode(metadata) as Map; final packages = (resolved['packages']! as List).cast>(); - final root = (resolved['resolve']! as Map)['root']; - final facade = [ - for (final package in packages) - if (package['id'] == root) - for (final dependency in (package['dependencies']! as List).cast>()) - if (dependency['rename'] == '${input.packageName}_export') dependency['path']! as String, - ].single; final output = await _cargo([ 'build', '--release', '--lib', '--features', 'dart', '--target', target, // '--manifest-path', manifest.path, '--message-format=json-render-diagnostics', From ed5a3c7335256a602e4ad72ea9063569a8e7ff26 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 04:33:12 -0800 Subject: [PATCH 07/15] ffi: the Dart consumer's smoke program calls the contract, which Dart 3.14's AOT resolves Co-Authored-By: Claude Opus 5.5 --- ffi/release/package.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index 9dafdd384..4e5708fe6 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -1964,13 +1964,15 @@ def local_package(package, dest, url): def dart_main(lib, check): - """The consumer's program: the caller's check, or else one that resolves the library's contract.""" + """The consumer's program: the caller's check, or else one that calls the library's contract. + It calls rather than takes the address: Dart 3.14's AOT resolves no `Native.addressOf` of a + function naming its `assetId`, as this one must from outside the package.""" if check: return Path(check).read_text(encoding="utf-8") return (f"import 'dart:ffi';\n\n" - f"@Native(symbol: '{lib}_contract', assetId: 'package:{lib}/{lib}.dart')\n" - f"external void _contract();\n\n" - f"void main() {{\n if (Native.addressOf>(_contract) == nullptr) throw StateError('no {lib}');\n}}\n") + f"@Native(symbol: '{lib}_contract', assetId: 'package:{lib}/{lib}.dart', isLeaf: true)\n" + f"external int _contract();\n\n" + f"void main() {{\n _contract();\n}}\n") def dart_app(work, name, package, lib): From 9f73cd09c3881fc04aa77cbf728c6a39c97e23b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 05:31:47 -0800 Subject: [PATCH 08/15] ffi: an Apple library leaves header room for the install name Dart bundles it at Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 2 ++ ffi/src/glue.rs | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ffi/README.md b/ffi/README.md index 49bd2dd26..e28cac836 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -448,6 +448,8 @@ Linux's exports, as the merge queue runs on, since a header names its targets. panic or a failed check that aborts goes to logcat as `rust2-ffi` and to bionic's abort message, which shows in the tombstone, `ApplicationExitInfo`, and Play vitals, since an app's stderr goes nowhere. +- **Apple**: the build script links the library with room in its header for the longest + install name, as Dart and Flutter rename it to the path they bundle it at. - **Names** are the prefix, then the item's path from the facade's root, each segment in Rust's case, joined by `_`: `track::Producer::append_group` is `moq_track_Producer_append_group`, and moq-net's `session::Stats` and `Session::stats` diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 036600f39..a346be714 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -29,7 +29,8 @@ const FUTURE: &str = include_str!("../runtime/rt/future.rs"); /// The crate's build script, which refuses to build for a target but `targets`, on which an /// item a target compiles would be missing, or one it leaves out left over, and tells the /// crate which it builds for, as the `cfg` [`CFG`]. On Android, it aligns the library to -/// 16 KB pages. On macOS, a CPython extension leaves Python's symbols for the interpreter +/// 16 KB pages. On Apple systems, it leaves room in the library's header for the longest +/// install name a bundler may give it. On macOS, a CPython extension leaves Python's symbols for the interpreter /// that loads it to resolve. fn build(targets: &[String]) -> String { let list: Vec = targets.iter().map(|t| format!("{t:?}")).collect(); @@ -52,6 +53,11 @@ fn build(targets: &[String]) -> String { \tif std::env::var(\"CARGO_CFG_TARGET_OS\").is_ok_and(|os| os == \"android\") {{\n\ \t\tprintln!(\"cargo::rustc-link-arg-cdylib=-Wl,-z,max-page-size=16384\");\n\ \t}}\n\ + \t// Dart and Flutter rename an Apple library to the path they bundle it at, which\n\ + \t// install_name_tool fits only in the room its load commands leave.\n\ + \tif std::env::var(\"CARGO_CFG_TARGET_VENDOR\").is_ok_and(|vendor| vendor == \"apple\") {{\n\ + \t\tprintln!(\"cargo::rustc-link-arg-cdylib=-Wl,-headerpad_max_install_names\");\n\ + \t}}\n\ \t#[cfg(feature = \"python\")]\n\ \tpyo3_build_config::add_extension_module_link_args();\n\ }}\n", From 8587de1222819938addd4bea73c32fefc9e742b5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 06:31:12 -0800 Subject: [PATCH 09/15] ffi: the Go module links the macOS x64 and Android libraries the export now names Co-Authored-By: Claude Opus 5.5 --- ffi/release/README.md | 4 +++- ffi/release/package.py | 13 +++++++++++-- ffi/release/test_package.py | 3 ++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index b4e746fe2..03c92743e 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -139,7 +139,9 @@ crate apart from it takes `web-features` and `web-example` instead. version's `/vN` suffix checked: the export's `go/`, whose runtime's `#cgo` lines link `rt/lib__.a` for `linux/amd64` and `linux/arm64` (built against glibc 2.28), `darwin/arm64` and `ios/arm64` (the Swift package's macOS and iOS device - libraries; no consumer here runs iOS), and `windows/amd64` (an + libraries; no consumer here runs iOS), `darwin/amd64` (at the Swift package's macOS + floor), `android/arm64` and `android/amd64` (at the Dart package's API 24; no consumer + here runs Go on Android), and `windows/amd64` (an `x86_64-pc-windows-gnu` library, for Go's MinGW linker), and the license files. Each library must need exactly the system libraries the export's line names, as rustc prints them building it, and loses its debug info and the LLVM bitcode Go refuses. The module's diff --git a/ffi/release/package.py b/ffi/release/package.py index 4e5708fe6..d5904ba04 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -55,9 +55,12 @@ GO_TARGETS = { "x86_64-unknown-linux-gnu": "linux_amd64", "aarch64-unknown-linux-gnu": "linux_arm64", + "x86_64-apple-darwin": "darwin_amd64", "aarch64-apple-darwin": "darwin_arm64", "aarch64-apple-ios": "ios_arm64", "x86_64-pc-windows-gnu": "windows_amd64", + "x86_64-linux-android": "android_amd64", + "aarch64-linux-android": "android_arm64", } # Every target the Dart package ships a library for, by the oldest system it loads on, which # its hook checks an app's against: Flutter's own floors, Android API 24, iOS 13, and macOS @@ -1568,7 +1571,7 @@ def staticlib(manifest, crate, lib, target, env=None): # Its own target directory, since cargo does not rebuild for a changed deployment target. target_dir = crate.parent / ("target-swift" if target in APPLE_TARGETS else "target-go") # cargo-zigbuild links Linux against the glibc floor's symbols, whichever glibc the host has. - linux = "linux" in target + linux = target.endswith("linux-gnu") # Uncolored, so the note below reads as written whatever the caller's terminal. built = run(*(["cargo-zigbuild"] if linux else ["cargo"]), "rustc", "--color", "never", "--locked", "--release", "--manifest-path", crate, "--target", f"{target}.{GLIBC}" if linux else target, @@ -1670,7 +1673,13 @@ def build(args): if target in GO_TARGETS: go_library(args, export_dir, lib, target, library, native, out) elif target in GO_TARGETS: - library, native = staticlib(args.manifest_path, crate, lib, target, env=reproducible(info, export_dir, target)) + env = reproducible(info, export_dir, target) + # Each at its sibling's floor: the Swift package's macOS, and the Dart package's Android API. + if "apple-darwin" in target: + env["MACOSX_DEPLOYMENT_TARGET"] = SWIFT_PLATFORMS["macos"] + elif "android" in target: + env.update(android_toolchain(target, DART_TARGETS[target])) + library, native = staticlib(args.manifest_path, crate, lib, target, env=env) go_library(args, export_dir, lib, target, library, native, out) if target in DART_TARGETS: dart_library(info, args.manifest_path, export_dir, target, out) diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 1c75b4349..7075b13d0 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -572,7 +572,8 @@ def test_module_as_a_proxy_serves_it(self): self.assertEqual((versions / "list").read_text(), "v0.4.12\n") with zipfile.ZipFile(versions / "v0.4.12.zip") as z: self.assertEqual(sorted(z.namelist()), [f"moq.dev/moq-ffi@v0.4.12/{n}" for n in [ - "LICENSE-MIT", "go.mod", "rt/libmoq_darwin_arm64.a", "rt/libmoq_ios_arm64.a", + "LICENSE-MIT", "go.mod", "rt/libmoq_android_amd64.a", "rt/libmoq_android_arm64.a", + "rt/libmoq_darwin_amd64.a", "rt/libmoq_darwin_arm64.a", "rt/libmoq_ios_arm64.a", "rt/libmoq_linux_amd64.a", "rt/libmoq_linux_arm64.a", "rt/libmoq_windows_amd64.a", "rt/rt.go"]]) # The module ships exactly the libraries the runtime links. (root / "libs/libmoq_linux_arm64.a").unlink() From 9010da03812afc12fbd679e0526c76774cdc27f3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 07:52:39 -0800 Subject: [PATCH 10/15] ffi(go): Android's #cgo line ends with the glue's log, as rustc lists it Co-Authored-By: Claude Opus 5.5 --- ffi/src/go.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ffi/src/go.rs b/ffi/src/go.rs index a66b8e664..ede290567 100644 --- a/ffi/src/go.rs +++ b/ffi/src/go.rs @@ -262,6 +262,11 @@ fn link(abi: &Abi<'_>) -> Result<(String, String), String> { let not = not.map(|os| format!(",!{os}")).unwrap_or_default(); let mut flags = vec![format!("${{SRCDIR}}/lib{p}_{goos}_{goarch}.a")]; flags.extend(libs.iter().cloned()); + // Then the glue's own, which rustc lists after its dependencies', dropping one that + // repeats the last: on Android, the log its abort writes to (`runtime/rt/android.rs`). + if goos == "android" && flags.last().map(String::as_str) != Some("-llog") { + flags.push("-llog".to_string()); + } let _ = writeln!(lines, "#cgo {goos}{not},{goarch} LDFLAGS: {}", flags.join(" ")); let not = not.replace(",!", " && !"); constraints.push(format!("{goos}{not} && {goarch}")); @@ -3820,7 +3825,9 @@ fn links_each_target_go_builds_for() { let targets = [ "aarch64-apple-darwin", "aarch64-apple-ios", + "aarch64-linux-android", "wasm32-wasip1", + "x86_64-linux-android", "x86_64-pc-windows-gnu", "x86_64-pc-windows-msvc", ]; @@ -3837,18 +3844,24 @@ fn links_each_target_go_builds_for() { links: (targets.iter()) .filter(|t| **t != "wasm32-wasip1") .map(|t| (t.to_string(), vec![format!("-l{}", &t[..3])])) + .chain([("x86_64-linux-android".to_string(), vec!["-ldl".into(), "-llog".into()])]) .collect(), }; let (lines, platform) = link(&Abi::new(&api, "lock")).unwrap(); // iOS also satisfies `darwin`; MSVC's library is none Go links; wasm32 runs no cgo. + // Android's ends with the glue's log, which rustc does not repeat. assert_eq!( lines, "#cgo darwin,!ios,arm64 LDFLAGS: ${SRCDIR}/liblock_darwin_arm64.a -laar\n\ #cgo ios,arm64 LDFLAGS: ${SRCDIR}/liblock_ios_arm64.a -laar\n\ + #cgo android,amd64 LDFLAGS: ${SRCDIR}/liblock_android_amd64.a -ldl -llog\n\ + #cgo android,arm64 LDFLAGS: ${SRCDIR}/liblock_android_arm64.a -laar -llog\n\ #cgo windows,amd64 LDFLAGS: ${SRCDIR}/liblock_windows_amd64.a -lx86\n" ); assert!( - platform.contains("//go:build !(darwin && !ios && arm64 || ios && arm64 || windows && amd64)\n"), + platform.contains( + "//go:build !(darwin && !ios && arm64 || ios && arm64 || android && amd64 || android && arm64 || windows && amd64)\n" + ), "{platform}" ); let mut stale = api.clone(); From e538a08986f71d634a4096279ed8c50c0badf52c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 07:52:39 -0800 Subject: [PATCH 11/15] ffi: the sdist and unreached_types read the relative paths the export writes #909's sdist keyed each local package by its absolute path and #917 made the generated manifest spell them relative to the export, so the sdist refused every export on a checkout without a symlink, and unreached_types expected the absolute spelling. Co-Authored-By: Claude Opus 5.5 --- ffi/release/package.py | 9 ++++++++- ffi/tests/fixtures.rs | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index bb84b00cc..f59ef6d9a 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -442,7 +442,14 @@ def moves(at): for root in roots: at = place(root.parent) files[f"{at}/Cargo.toml"] = (relocate(workspace_manifest(root), moves(at)).encode(), False) - generated = relocate((export / "Cargo.toml").read_text(encoding="utf-8"), moves("binding"), strict=True) + # The exporter writes each path relative to the export where one reads right, and else absolute. + spelled = moves("binding") + for directory, there in placed.items(): + try: + spelled[Path(os.path.relpath(directory, export)).as_posix()] = posixpath.relpath(there, "binding") + except ValueError: + pass # On another Windows drive, which no relative path reaches. + generated = relocate((export / "Cargo.toml").read_text(encoding="utf-8"), spelled, strict=True) files["binding/Cargo.toml"] = (generated.encode(), False) for path in [export / "Cargo.lock", export / "build.rs", export / "python" / f"{info['lib']}.py", *sorted(p for p in (export / "src").rglob("*") if p.is_file())]: diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index 3616ebffe..49c014df6 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -3458,8 +3458,10 @@ fn unreached_types() { .arg(&staged) .env("CARGO_TARGET_DIR", moved.join("build"))); let manifest = std::fs::read_to_string(moved.join("out/Cargo.toml")).unwrap(); + // Relative to the export, or absolute where the scratch path holds a symlink. + let kio = format!("path = {:?}", moved.join("kio").display().to_string()); assert!( - manifest.contains(&format!("path = {:?}", moved.join("kio").display().to_string())) + (manifest.contains("path = \"../kio\"") || manifest.contains(&kio)) && !manifest.contains(&format!("{:?}", package.join("kio").display().to_string())), "{manifest}" ); From 781c14c92867c99fd67c16151454398a45c71900 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 21 Nov 2026 10:23:21 -0800 Subject: [PATCH 12/15] ffi: a Dart package takes the names pub takes, and the hook builds each floor apart pub refuses its own reservedWords, not Dart's identifier lists, and a package cannot depend on itself; Cargo rebuilds for no change in a deployment target or API level; Windows' NDK wraps clang in .cmd files. Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 10 ++++--- ffi/release/package.py | 6 ++-- ffi/runtime/hook.dart | 7 +++++ ffi/src/dart.rs | 64 +++++++++++++++++++++++++++++++++++------- 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index ac5f4c530..1a38a61d3 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -1376,10 +1376,12 @@ C function is an `@Native` external, which finds the shared library the crate bu `hook/build.dart` maps to the library when an app depending on it builds or runs. In an export, the hook builds the crate beside the package, `../Cargo.toml`, with Cargo, on the target the app builds for, for the Android NDK of an app's Flutter build, and reruns when a -source the build read changes, Cargo's dep-info says which. A release's adds -`hook/release.json`, from which it downloads the library instead -([Binding releases](release/README.md)). The prefix must name a Dart package, lowercase -letters, digits, and `_` and no reserved word, or the export fails. +source the build read changes, Cargo's dep-info says which. It builds each Android API +level and Apple deployment target in a target directory of its own, as Cargo rebuilds for +no change in them. A release's adds `hook/release.json`, from which it downloads the +library instead ([Binding releases](release/README.md)). The prefix must name a Dart +package, lowercase letters, digits, and `_`, no word pub reserves, and none of the +packages the hook depends on (`code_assets`, `crypto`, `hooks`), or the export fails. - **Names** are the C names without the prefix: a type in Pascal case and a member in camel case, so `moq_track_Producer` is `TrackProducer`, and its method diff --git a/ffi/release/package.py b/ffi/release/package.py index 0f78e4e0d..4e6f84470 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -2159,11 +2159,13 @@ def android_toolchain(target, api): bin_dir = Path(ndk) / "toolchains" / "llvm" / "prebuilt" / host / "bin" # The NDK spells 32-bit Arm's compiler armv7a, where Rust's target is armv7. triple = "armv7a-linux-androideabi" if target.startswith("armv7") else target - clang = bin_dir / f"{triple}{api}-clang" + # Windows' NDK wraps each in a batch file. + cmd = ".cmd" if sys.platform == "win32" else "" + clang, clangxx = (bin_dir / f"{triple}{api}-{name}{cmd}" for name in ("clang", "clang++")) if not clang.is_file(): raise SystemExit(f"the NDK at {ndk} has no {clang.name}") key = target.replace("-", "_") - return {f"CARGO_TARGET_{key.upper()}_LINKER": str(clang), f"CC_{key}": str(clang), f"CXX_{key}": f"{clang}++", + return {f"CARGO_TARGET_{key.upper()}_LINKER": str(clang), f"CC_{key}": str(clang), f"CXX_{key}": str(clangxx), f"AR_{key}": str(bin_dir / "llvm-ar")} diff --git a/ffi/runtime/hook.dart b/ffi/runtime/hook.dart index 701489882..f3c24a1bc 100644 --- a/ffi/runtime/hook.dart +++ b/ffi/runtime/hook.dart @@ -140,6 +140,9 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe } final environment = {}; final key = target.replaceAll('-', '_'); + // The system the app targets, which Cargo rebuilds for no change in, so each builds in a + // target directory of its own. + String? floor; switch (code.targetOS) { case OS.android: // The NDK's clang for the app's API level, which Flutter passes beside its own. @@ -152,12 +155,15 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe environment['CC_$key'] = '${clang.toFilePath()}$cmd'; environment['CXX_$key'] = '${clang.toFilePath()}++$cmd'; environment['AR_$key'] = compiler.archiver.toFilePath(); + floor = 'android-${code.android.targetNdkApi}'; case OS.iOS: final version = code.iOS.deploymentVersion; environment['IPHONEOS_DEPLOYMENT_TARGET'] = '${version.major}.${version.minor}'; + floor = 'ios-${version.major}.${version.minor}'; case OS.macOS: final version = code.macOS.deploymentVersion; environment['MACOSX_DEPLOYMENT_TARGET'] = '${version.major}.${version.minor}'; + floor = 'macos-${version.major}.${version.minor}'; } // The crate the export depends on as `_export`, as the exporter writes its line, // its path a Rust string, relative to the export or absolute: from its directory rustup @@ -176,6 +182,7 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe final output = await _cargo([ 'build', '--release', '--lib', '--features', 'dart', '--target', target, // '--manifest-path', manifest.path, '--message-format=json-render-diagnostics', + if (floor != null) ...['--target-dir', '${resolved['target_directory']}${Platform.pathSeparator}$floor'], ], facade, environment); final name = code.targetOS.dylibFileName(input.packageName); final built = { diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index 0a3ad427a..eded4ecb3 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -42,16 +42,7 @@ pub(crate) const HOOK: &str = include_str!("../runtime/hook.dart"); /// The package's `pubspec.yaml`: `lib`, the library's prefix, as its name, at the facade's /// version, depending on what the hook imports; or why pub takes no package of that name. pub(crate) fn pubspec(facade: &crate::config::Package, lib: &str) -> Result { - let valid = lib.starts_with(|c: char| c.is_ascii_lowercase() || c == '_') - && lib - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); - if !valid || RESERVED.contains(&lib) || BUILT_IN.contains(&lib) { - return Err(format!( - "the library `{lib}` names no Dart package, whose name is lowercase letters, digits, and `_`, \ - and no reserved word: rename the crate's `[lib] name`" - )); - } + package_name(lib)?; // The SDK floor is the one whose build hooks are stable, 3.10. Ok(format!( "# Generated by rust2-ffi: the Dart binding of the pub API of {name}.\n\ @@ -67,6 +58,37 @@ pub(crate) fn pubspec(facade: &crate::config::Package, lib: &str) -> Result Result<(), String> { + let valid = lib.starts_with(|c: char| c.is_ascii_lowercase() || c == '_') + && lib + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); + if !valid || PUB_RESERVED.split(' ').any(|word| word == lib) { + return Err(format!( + "the library `{lib}` names no Dart package, whose name is lowercase letters, digits, and `_`, \ + and no word pub reserves: rename the crate's `[lib] name`" + )); + } + if HOOK_PACKAGES.contains(&lib) { + return Err(format!( + "the library `{lib}` names a package the Dart package's build hook depends on: rename the crate's `[lib] name`" + )); + } + Ok(()) +} + +/// The words pub refuses as a package's name: its `reservedWords`, Dart's reserved words but +/// those a hosted package already took (https://github.com/dart-lang/pub/blob/master/lib/src/utils.dart). +const PUB_RESERVED: &str = "abstract as assert await break case catch class const continue covariant default deferred \ + do else enum export extends external false final finally for hide if implements import in inline interface is late \ + library mixin new null of on operator part required rethrow return sealed set super switch this throw true try type \ + var void while with yield"; + +/// The packages the generated pubspec depends on, which it cannot also be. +const HOOK_PACKAGES: &[&str] = &["code_assets", "crypto", "hooks"]; + /// Bytes Rust returns up to this long are copied into Dart's memory, and longer ones lent /// to it: a lent list's finalizer, and the external size it charges the garbage collector, /// cost more than a copy up to about this long. @@ -3811,6 +3833,28 @@ fn indent(code: &str) -> String { mod tests { use super::*; + /// A package takes the names pub takes, Dart identifiers it reserves for code included, + /// and no other. + #[test] + fn package_names_are_pubs() { + for name in ["moq_ffi", "_x", "num", "get", "dynamic", "int"] { + assert_eq!(package_name(name), Ok(()), "{name}"); + } + for name in [ + "abstract", + "sealed", + "on", + "yield", + "Moq", + "1x", + "hooks", + "crypto", + "code_assets", + ] { + assert!(package_name(name).is_err(), "{name}"); + } + } + /// Every type the runtime names unqualified is its own or [`TAKEN`], so no exported /// item shadows it. #[test] From 5862c03e8c9b051f129510426fe7ddf4b47c1f26 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 21 Nov 2026 10:24:01 -0800 Subject: [PATCH 13/15] quest: the api fixture's Dart, Go, and Swift packages print its transcript Co-Authored-By: Claude Opus 5.5 --- quest/m1/ffi-packaging/native-transcripts.md | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 quest/m1/ffi-packaging/native-transcripts.md diff --git a/quest/m1/ffi-packaging/native-transcripts.md b/quest/m1/ffi-packaging/native-transcripts.md new file mode 100644 index 000000000..167980e73 --- /dev/null +++ b/quest/m1/ffi-packaging/native-transcripts.md @@ -0,0 +1,23 @@ +# [S] The api fixture's Dart, Go, and Swift packages print its transcript + +## Goal + +The builder's Dart, Flutter, Go, and Swift consumers of the api fixture run its checks, +`tests/api.dart`, `api.go`, and `api.swift`, through the released packages and print +native Rust's transcript byte for byte, as its Python, C++, and web consumers do since +[#930](https://github.com/kixel-ai/rust2/pull/930). Today they only load the library. + +## Plan + +Found by [#915](https://github.com/kixel-ai/rust2/pull/915), which adds the Dart package +and its `dart-check` input: #930 left `api.dart` to it, but each of the three checks names +wasm32-only items (`targets()` in each, `api.wasmOnly()` and the like). The builder's api +fixture export holds no wasm32 API, so Dart, Go, and Swift, which resolve names when they +compile, refuse the check; Python and C++ reach those names only at run time. + +Recommended: move each check's wasm32 cases into a file of their own that the exporter's +tests compile beside the check, so the check compiles against any export, and give +`dart-check`, `go-check`, and `swift-check` #930's directory form, which +`just ffi transcripts` fills with each check and its generated script. +Rejected: exporting the builder's api fixture for wasm32 too, which would put a target no +native package ships into every package's API. From 68557b0d2785b8e010fc075a1cbd29a04df2a26f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 21 Nov 2026 10:40:45 -0800 Subject: [PATCH 14/15] ffi: the source hook reruns on the Cargo files above each path package, and pub's names exclude the hook's graph Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 6 ++++-- ffi/runtime/hook.dart | 20 ++++++++++++++++++-- ffi/src/dart.rs | 25 +++++++++++++++++++++---- quest/m1/ffi-packaging/dart-privacy.md | 23 +++++++++++++++++++++++ 4 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 quest/m1/ffi-packaging/dart-privacy.md diff --git a/ffi/README.md b/ffi/README.md index 1a38a61d3..437a409c6 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -1376,12 +1376,14 @@ C function is an `@Native` external, which finds the shared library the crate bu `hook/build.dart` maps to the library when an app depending on it builds or runs. In an export, the hook builds the crate beside the package, `../Cargo.toml`, with Cargo, on the target the app builds for, for the Android NDK of an app's Flutter build, and reruns when a -source the build read changes, Cargo's dep-info says which. It builds each Android API +source the build read changes, Cargo's dep-info says which, or a Cargo manifest or config at +or above a path package it builds. It builds each Android API level and Apple deployment target in a target directory of its own, as Cargo rebuilds for no change in them. A release's adds `hook/release.json`, from which it downloads the library instead ([Binding releases](release/README.md)). The prefix must name a Dart package, lowercase letters, digits, and `_`, no word pub reserves, and none of the -packages the hook depends on (`code_assets`, `crypto`, `hooks`), or the export fails. +packages the hook resolves with (`hooks`, `code_assets`, `crypto`, and theirs), or the export +fails. - **Names** are the C names without the prefix: a type in Pascal case and a member in camel case, so `moq_track_Producer` is `TrackProducer`, and its method diff --git a/ffi/runtime/hook.dart b/ffi/runtime/hook.dart index f3c24a1bc..859d42b50 100644 --- a/ffi/runtime/hook.dart +++ b/ffi/runtime/hook.dart @@ -198,8 +198,10 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe final lock = File('${resolved['workspace_root']}${Platform.pathSeparator}Cargo.lock'); final inputs = [ lock.uri, - for (final package in packages) - if (package['source'] == null) File(package['manifest_path']! as String).uri, + ...{ + for (final package in packages) + if (package['source'] == null) ..._above(File(package['manifest_path']! as String).parent), + }.map((path) => File(path).uri), for (final path in _depInfo(await info.readAsString())) File(path).uri, ]; return (library, inputs); @@ -220,3 +222,17 @@ List _depInfo(String text) => [ for (final match in RegExp(r'(?:\\ |[^ ])+').allMatches(line.substring(at + 2))) match[0]!.replaceAll(r'\ ', ' '), ]; + +/// The Cargo files at and above a path package's `directory`, which change its build and +/// which no dep-info names: each `Cargo.toml`, among which Cargo finds the workspace it +/// inherits from, and each `.cargo/config.toml` or `.cargo/config`. +Iterable _above(Directory directory) sync* { + final sep = Platform.pathSeparator; + for (var dir = directory.absolute; ; dir = dir.parent) { + for (final name in ['Cargo.toml', '.cargo${sep}config.toml', '.cargo${sep}config']) { + final file = File('${dir.path}$sep$name'); + if (file.existsSync()) yield file.path; + } + if (dir.parent.path == dir.path) return; + } +} diff --git a/ffi/src/dart.rs b/ffi/src/dart.rs index eded4ecb3..2fc7cde7c 100644 --- a/ffi/src/dart.rs +++ b/ffi/src/dart.rs @@ -59,7 +59,7 @@ pub(crate) fn pubspec(facade: &crate::config::Package, lib: &str) -> Result Result<(), String> { let valid = lib.starts_with(|c: char| c.is_ascii_lowercase() || c == '_') && lib @@ -73,7 +73,7 @@ fn package_name(lib: &str) -> Result<(), String> { } if HOOK_PACKAGES.contains(&lib) { return Err(format!( - "the library `{lib}` names a package the Dart package's build hook depends on: rename the crate's `[lib] name`" + "the library `{lib}` names a package the Dart package's build hook resolves with: rename the crate's `[lib] name`" )); } Ok(()) @@ -86,8 +86,24 @@ const PUB_RESERVED: &str = "abstract as assert await break case catch class cons library mixin new null of on operator part required rethrow return sealed set super switch this throw true try type \ var void while with yield"; -/// The packages the generated pubspec depends on, which it cannot also be. -const HOOK_PACKAGES: &[&str] = &["code_assets", "crypto", "hooks"]; +/// The packages the hook resolves with, which pub takes no package of the same name beside: +/// those the generated pubspec depends on and theirs, as Dart 3.13's pub resolves them. One +/// they add later fails at `dart pub get` instead. +const HOOK_PACKAGES: &[&str] = &[ + "code_assets", + "collection", + "crypto", + "hooks", + "logging", + "meta", + "path", + "pub_semver", + "record_use", + "source_span", + "string_scanner", + "term_glyph", + "yaml", +]; /// Bytes Rust returns up to this long are copied into Dart's memory, and longer ones lent /// to it: a lent list's finalizer, and the external size it charges the garbage collector, @@ -3850,6 +3866,7 @@ mod tests { "hooks", "crypto", "code_assets", + "logging", ] { assert!(package_name(name).is_err(), "{name}"); } diff --git a/quest/m1/ffi-packaging/dart-privacy.md b/quest/m1/ffi-packaging/dart-privacy.md new file mode 100644 index 000000000..6f76ae1f0 --- /dev/null +++ b/quest/m1/ffi-packaging/dart-privacy.md @@ -0,0 +1,23 @@ +# [S] Apple privacy declarations for the Dart package's libraries + +## Goal + +A Flutter app on the App Store declares every required-reason API the Dart package's +Apple libraries call, as the Swift package's `PrivacyInfo.xcprivacy` does since +[#912](https://github.com/kixel-ai/rust2/pull/912), and a library calling one whose reason +only the crate's author knows fails the release. + +## Plan + +Found by [#915](https://github.com/kixel-ai/rust2/pull/915)'s review: its Apple Dart +libraries reach `fstat` and `mach_absolute_time` through std, as the Swift package's +slices do, but ship no manifest. A code asset is a bare dylib that Flutter wraps in a +framework it generates, so the package has no place to put one Flutter would bundle +([llamadart-native#98](https://github.com/leehack/llamadart-native/pull/98) hits the same +wall). + +Recommended: scan each Apple Dart library's imports with #912's `REQUIRED_REASON_APIS`, +failing as it does on a category with no universal reason, and ship the manifest in the +package with a README line telling the app to merge it into its own, until dart-lang/native +lets a code asset carry a resource; then bundle it. Rejected: refusing every Apple library +that calls one, which would refuse std's `fstat` everywhere. From cea13be31bf4730fe68824d68d63338fccffca08 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 21 Nov 2026 10:49:34 -0800 Subject: [PATCH 15/15] ffi: the source hook tracks each path package's workspace root wherever it lies, and the configs Cargo reads Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 4 ++-- ffi/runtime/hook.dart | 36 ++++++++++++++++++++++++------------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index 437a409c6..2c66f43ee 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -1376,8 +1376,8 @@ C function is an `@Native` external, which finds the shared library the crate bu `hook/build.dart` maps to the library when an app depending on it builds or runs. In an export, the hook builds the crate beside the package, `../Cargo.toml`, with Cargo, on the target the app builds for, for the Android NDK of an app's Flutter build, and reruns when a -source the build read changes, Cargo's dep-info says which, or a Cargo manifest or config at -or above a path package it builds. It builds each Android API +source the build read changes, Cargo's dep-info says which, or a path package's manifest, +its workspace's, or a Cargo config the build reads. It builds each Android API level and Apple deployment target in a target directory of its own, as Cargo rebuilds for no change in them. A release's adds `hook/release.json`, from which it downloads the library instead ([Binding releases](release/README.md)). The prefix must name a Dart diff --git a/ffi/runtime/hook.dart b/ffi/runtime/hook.dart index 859d42b50..f637024c4 100644 --- a/ffi/runtime/hook.dart +++ b/ffi/runtime/hook.dart @@ -196,12 +196,19 @@ Future<(File, List)> _build(BuildInput input, CodeConfig code, String targe // Cargo's dep-info beside it lists the sources of every path package the build read. final info = File('${library.path.substring(0, library.path.length - name.length)}${name.split('.').first}.d'); final lock = File('${resolved['workspace_root']}${Platform.pathSeparator}Cargo.lock'); + final local = [ + for (final package in packages) + if (package['source'] == null) package['manifest_path']! as String, + ]; + // Each one's workspace root, whose manifest it may inherit from, which need not lie above it. + final roots = await Future.wait([ + for (final path in local) + _cargo(['locate-project', '--workspace', '--message-format', 'plain', '--manifest-path', path], facade, environment) + .then((root) => root.trim()), + ]); final inputs = [ lock.uri, - ...{ - for (final package in packages) - if (package['source'] == null) ..._above(File(package['manifest_path']! as String).parent), - }.map((path) => File(path).uri), + ...{...local, ...roots, ..._configs(facade)}.map((path) => File(path).uri), for (final path in _depInfo(await info.readAsString())) File(path).uri, ]; return (library, inputs); @@ -223,16 +230,21 @@ List _depInfo(String text) => [ match[0]!.replaceAll(r'\ ', ' '), ]; -/// The Cargo files at and above a path package's `directory`, which change its build and -/// which no dep-info names: each `Cargo.toml`, among which Cargo finds the workspace it -/// inherits from, and each `.cargo/config.toml` or `.cargo/config`. -Iterable _above(Directory directory) sync* { +/// The config files Cargo reads when it runs from `directory`: `.cargo/config.toml` or +/// `.cargo/config` at and above it, and in Cargo's home. +Iterable _configs(String directory) sync* { final sep = Platform.pathSeparator; - for (var dir = directory.absolute; ; dir = dir.parent) { - for (final name in ['Cargo.toml', '.cargo${sep}config.toml', '.cargo${sep}config']) { - final file = File('${dir.path}$sep$name'); + final home = Platform.environment['CARGO_HOME'] ?? + '${Platform.environment[Platform.isWindows ? 'USERPROFILE' : 'HOME']}$sep.cargo'; + final dirs = [home]; + for (var dir = Directory(directory).absolute; ; dir = dir.parent) { + dirs.add('${dir.path}$sep.cargo'); + if (dir.parent.path == dir.path) break; + } + for (final dir in dirs) { + for (final name in ['config.toml', 'config']) { + final file = File('$dir$sep$name'); if (file.existsSync()) yield file.path; } - if (dir.parent.path == dir.path) return; } }