From 0fe9eba40024e7ecf518820231f1d8d07eec4102 Mon Sep 17 00:00:00 2001 From: spacebear Date: Wed, 28 Oct 2026 14:55:07 +0000 Subject: [PATCH 1/9] Align Dart's Android libraries to 16 KB on NDK r27 native_toolchain_rust links Android libraries with the NDK the consumer's Flutter project selects, and accepts r27, the first NDK with the API 35 compiler wrappers it calls. r27 still aligns 64-bit libraries to 4 KB, which Google Play no longer accepts for apps that target Android 15 or later. NDK r28 and later align to 16 KB by default. Pass -z max-page-size=16384 to the linker for the 64-bit Android targets. Linking payjoin-ffi with NDK r27 changes its LOAD alignment from 0x1000 to 0x4000, and with NDK r29 the library is byte-identical with or without the flag, so no version check is needed. 32-bit ARM has no 16 KB requirement and is left alone. --- payjoin-ffi/CHANGELOG.md | 2 ++ payjoin-ffi/dart/hook/build.dart | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/payjoin-ffi/CHANGELOG.md b/payjoin-ffi/CHANGELOG.md index 9c92fdf69..34616225d 100644 --- a/payjoin-ffi/CHANGELOG.md +++ b/payjoin-ffi/CHANGELOG.md @@ -1,6 +1,8 @@ ## Unreleased - Python: require Python 3.11+ and replace httpx with httpx2. Callers catching HTTP errors from fetch_ohttp_keys should use httpx2.HTTPError instead of httpx.HTTPError. +- Dart: Android builds with NDK r27 now align 64-bit libraries to + 16 KB, as Google Play requires. Later NDKs already did. ## [0.25.1] diff --git a/payjoin-ffi/dart/hook/build.dart b/payjoin-ffi/dart/hook/build.dart index aa11d6df5..04a0129b1 100644 --- a/payjoin-ffi/dart/hook/build.dart +++ b/payjoin-ffi/dart/hook/build.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:code_assets/code_assets.dart'; import 'package:hooks/hooks.dart'; import 'package:native_toolchain_rust/native_toolchain_rust.dart'; @@ -22,6 +23,7 @@ void main(List args) async { final rustupHome = nonEmpty(environment['RUSTUP_HOME']) ?? (home == null ? null : '${strip(home)}$separator.rustup'); + final code = input.config.buildCodeAssets ? input.config.code : null; // Encoded rather than RUSTFLAGS, which Cargo splits on whitespace: the // output directory lives under the consumer's project, whose path may // contain spaces. hooks_runner never passes RUSTFLAGS to a build hook, @@ -37,6 +39,12 @@ void main(List args) async { if (rustupHome != null) (rustupHome, '/rustup'), ]) '--remap-path-prefix=${strip(from)}=$to', + // native_toolchain_rust accepts NDK r27, which aligns 64-bit libraries + // to 4 KB, and Google Play requires 16 KB. Later NDKs already align to + // 16 KB, and the flag leaves their output unchanged. + if (code?.targetOS == OS.android && + code?.targetArchitecture != Architecture.arm) + '-Clink-arg=-Wl,-z,max-page-size=16384', ].join('\x1f'); // The path overlay doubles as the development marker. `.pubignore` From 7bfe538cbf62838b873a407e450509d3f497b010 Mon Sep 17 00:00:00 2001 From: spacebear Date: Thu, 29 Oct 2026 22:30:50 +0000 Subject: [PATCH 2/9] Build release bindings with Rust 1.85.1 Dart's published crate builds with Rust 1.85.1, which its rust-toolchain.toml pins, but the jobs that build the rest of each release still used the 1.85.0 MSRV toolchain. pack-npm compiles the published wasm and build-wheel builds the published library and bindings, in the javascript and python devshells. pack-nuget generates its bindings, and build-nuget-native the NuGet libraries, with 1.85.0 from the runner. Move those two devshells, pack-nuget and build-nuget-native to 1.85.1, named once in the flake as release-version, so the NuGet libraries and their bindings build with the same toolchain. The devshells' test jobs now build with 1.85.1 too. rust.yml's 1.85.0 leg keeps the MSRV for the core crates, and the C# and Kotlin test jobs keep it for payjoin-ffi. payjoin-ffi now declares rust-version 1.85 like the other crates, so Cargo names the floor to anyone building below it. The dart devshell stays on 1.85.0. Dart publishes generated Dart source and the Rust crate, which consumers compile with the package's own 1.85.1 toolchain file. Its build hook runs rustup's 1.85.1, which sets LD_LIBRARY_PATH to that toolchain's libraries while cargo still finds the devshell's rustc first in PATH. A nix 1.85.1 rustc then loads rustup's librustc_driver of the same name and fails to find libz. The C# Windows test job installed 1.85.0 but selected `1.85`, which rustup resolves to the newest 1.85 release, 1.85.1, so it did not test the MSRV like the Linux and macOS jobs. It now names 1.85.0. --- .github/workflows/csharp.yml | 16 ++++++++-------- flake.nix | 14 ++++++++------ payjoin-ffi/Cargo.toml | 1 + 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 0d8118188..b71ce9e20 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -70,7 +70,7 @@ jobs: if: github.event_name != 'pull_request' runs-on: windows-latest env: - RUSTUP_TOOLCHAIN: 1.85 + RUSTUP_TOOLCHAIN: 1.85.0 defaults: run: working-directory: payjoin-ffi/csharp @@ -119,7 +119,7 @@ jobs: # Must match the dtolnay/rust-toolchain version exactly: the cross-target # std components are installed for that toolchain, and a partial version # here selects a different rustup toolchain entry without them (E0463). - RUSTUP_TOOLCHAIN: 1.85.0 + RUSTUP_TOOLCHAIN: 1.85.1 defaults: run: working-directory: payjoin-ffi/csharp @@ -147,10 +147,10 @@ jobs: with: persist-credentials: false # unneeded: no git auth beyond checkout - - name: Install Rust 1.85.0 + - name: Install Rust 1.85.1 uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 with: - toolchain: 1.85.0 + toolchain: 1.85.1 targets: ${{ matrix.target }} - name: Use cache @@ -204,8 +204,8 @@ jobs: needs: build-nuget-native env: # Exact version so the packaged bindings build with the toolchain the - # action installs, not an implicitly-installed latest 1.85.x. - RUSTUP_TOOLCHAIN: 1.85.0 + # action installs, the release toolchain. + RUSTUP_TOOLCHAIN: 1.85.1 defaults: run: working-directory: payjoin-ffi/csharp @@ -215,10 +215,10 @@ jobs: with: persist-credentials: false # unneeded: no git auth beyond checkout - - name: Install Rust 1.85.0 + - name: Install Rust 1.85.1 uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 with: - toolchain: 1.85.0 + toolchain: 1.85.1 - name: Use cache uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 diff --git a/flake.nix b/flake.nix index 160ab703e..461e37b37 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,8 @@ }; msrv-version = "1.85.0"; + # Rust version used to build release artifacts. + release-version = "1.85.1"; nginxWithStream = pkgs.nginxMainline.overrideAttrs (oldAttrs: { configureFlags = oldAttrs.configureFlags ++ [ @@ -319,11 +321,11 @@ builtins.removeAttrs pythonWorkspace.deps.all [ "payjoin" ] ); - # Rust toolchain for the javascript dev shell: msrv pinned to match - # payjoin-ffi/javascript build requirements, with wasm32 target added + # Rust toolchain for the javascript dev shell: the release version, as + # pack-npm builds the published wasm here, with wasm32 target added # so cargo can build per-arch artifacts under nix (rustup target add # is a no-op against a nix-provided toolchain). - jsRustToolchain = pkgs.rust-bin.stable.${msrv-version}.default.override { + jsRustToolchain = pkgs.rust-bin.stable.${release-version}.default.override { extensions = [ "rust-src" "rustfmt" @@ -411,11 +413,11 @@ BITCOIND_SKIP_DOWNLOAD = 1; }; - # Rust toolchain for the python dev shell: msrv pinned to match - # payjoin-ffi/python build requirements, with per-arch targets added + # Rust toolchain for the python dev shell: the release version, as + # build-wheel generates the published bindings here, with per-arch targets added # so cargo can build artifacts under nix for payjoin-ffi/python/scripts/generate_bindings.sh # (rustup target add is a no-op against a nix-provided toolchain). - pythonRustToolchain = pkgs.rust-bin.stable.${msrv-version}.default.override { + pythonRustToolchain = pkgs.rust-bin.stable.${release-version}.default.override { extensions = [ "rust-src" "rustfmt" diff --git a/payjoin-ffi/Cargo.toml b/payjoin-ffi/Cargo.toml index 0fb4197f7..87e1b0dfa 100644 --- a/payjoin-ffi/Cargo.toml +++ b/payjoin-ffi/Cargo.toml @@ -2,6 +2,7 @@ name = "payjoin-ffi" version = "0.25.1" edition = "2024" +rust-version = "1.85" license = "MIT OR Apache-2.0" exclude = ["tests"] From b7346b9877d85481080d8dfa7aa12594a462edf2 Mon Sep 17 00:00:00 2001 From: spacebear Date: Wed, 28 Oct 2026 14:55:07 +0000 Subject: [PATCH 3/9] Build Kotlin's production bindings from the lock The Kotlin workflow compiles production bindings after its tests by calling generate_bindings.sh directly, and only test.sh put Cargo-recent.lock in place. CI still built that step from the lock, but only because a fresh checkout has no Cargo.lock for test.sh to restore on exit. Calling the script on its own, or after test.sh restored a different lockfile, resolves the dependency graph afresh. Have generate_bindings.sh use the maintained lockfile itself, as the C# script does. When test.sh has already put it in place, the call does nothing. --- payjoin-ffi/kotlin/scripts/generate_bindings.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/payjoin-ffi/kotlin/scripts/generate_bindings.sh b/payjoin-ffi/kotlin/scripts/generate_bindings.sh index cb611cb2d..44262bb8f 100755 --- a/payjoin-ffi/kotlin/scripts/generate_bindings.sh +++ b/payjoin-ffi/kotlin/scripts/generate_bindings.sh @@ -16,6 +16,14 @@ else fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Build against the maintained lockfile instead of resolving the dependency +# graph fresh on every run. use_lockfile copies Cargo-recent.lock into place +# and restores the previous state when this script exits. +cd "$SCRIPT_DIR/../../.." +source contrib/lockfile.sh +use_lockfile Cargo-recent.lock + cd "$SCRIPT_DIR/../.." echo "Generating payjoin Kotlin..." From 939aae66f570a2ab4d91324ed7ae032225620ec9 Mon Sep 17 00:00:00 2001 From: spacebear Date: Wed, 28 Oct 2026 14:54:52 +0000 Subject: [PATCH 4/9] Add the native build contract Each binding builds its native libraries in its own job, on its own runner, with whatever toolchain that job sets up, so no shipped file can be rebuilt bit for bit. Add one script that builds every shipped target on Linux with one pinned toolchain: contrib/native/build.sh --- contrib/native/README.md | 36 +++++++++ contrib/native/apple-cc.sh | 21 ++++++ contrib/native/apple-stubs.sh | 31 ++++++++ contrib/native/build.sh | 133 +++++++++++++++++++++++++++++++++ contrib/native/vs-channel.json | 17 +++++ flake.nix | 61 +++++++++++++++ 6 files changed, 299 insertions(+) create mode 100644 contrib/native/README.md create mode 100755 contrib/native/apple-cc.sh create mode 100755 contrib/native/apple-stubs.sh create mode 100755 contrib/native/build.sh create mode 100644 contrib/native/vs-channel.json diff --git a/contrib/native/README.md b/contrib/native/README.md new file mode 100644 index 000000000..31501a209 --- /dev/null +++ b/contrib/native/README.md @@ -0,0 +1,36 @@ +# Native build contract + +The payjoin-ffi native libraries are built on Linux, by one script, in one +devshell. The language bindings move onto these libraries one at a time. + +```sh +nix develop .#native -c contrib/native/build.sh +``` + +The Android targets need the NDK, which only `.#native-android` provides. + +The library lands in `//`. + +| Invariant | Rule | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Toolchain | Rust 1.85.1. CI keeps testing the 1.85.0 MSRV. | +| Lockfile | `Cargo-recent.lock`, built `--locked` | +| Features | None. Bindings are generated separately, so the library is the same for every language. | +| Paths | The repository is remapped to `/payjoin`, the target dir to `/payjoin/target`, the cargo home to `/cargo`, the rustup home to `/rustup`, the NDK to `/ndk` and the Windows SDK to `/xwin`, for rustc and the C compiler | +| Profile | Cargo's release defaults | +| Linux | `cargo zigbuild` against glibc 2.17 | +| macOS | zig through `apple-cc.sh` against the stub SDK, deployment target 10.15 on x86_64 and 11.0 on arm64, Flutter's floor and the first macOS on Apple silicon. `universal2-apple-darwin` is the two slices combined with `llvm-lipo`. | +| iOS | zig through `apple-cc.sh` against the stub SDK, minimum iOS 13.0. Device and arm64 simulator only. | +| Windows | `xwin` fetches MSVC's CRT 14.44.17.14 and Windows SDK 10.0.26100 from the Visual Studio 17.14.37824.6 manifest pinned in `vs-channel.json`. LLVM 21's clang-cl, llvm-lib and lld-link build against them, linked with `/Brepro` and without a PDB. | +| Android | NDK 29.0.14206865 at API 21, 16 KB aligned on 64-bit by the NDK's default | + +The Apple dylibs are linked with an `@rpath/libpayjoin_ffi.dylib` install +name, `-headerpad_max_install_names` for Flutter's `install_name_tool`, and +`-S`, which keeps object paths and mtimes out of the stabs. +`apple-stubs.sh` generates the stub SDK from the macOS `libSystem.tbd` and +Darwin headers that zig ships, so no Apple SDK file is used or committed. + +`x86_64-apple-ios`, the Intel simulator, is not built. Rust 1.85.1's std +for it targets iOS 10.0, so its objects carry `LC_VERSION_MIN_IPHONEOS`, +which zig 0.16 reads as a device platform and rejects for a simulator +link. diff --git a/contrib/native/apple-cc.sh b/contrib/native/apple-cc.sh new file mode 100755 index 000000000..0d6176e20 --- /dev/null +++ b/contrib/native/apple-cc.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Compiler and linker for the Apple targets: zig for $APPLE_CC_TARGET +# against the stub SDK in $SDKROOT. The target flags rustc and cc-rs pass +# are dropped in favor of those. +set -euo pipefail + +args=() +while (($#)); do + case $1 in + -target | -arch | -isysroot) shift ;; + --target=* | -mmacosx-version-min=* | -mios-version-min=* | -mios-simulator-version-min=* | -miphoneos-version-min=*) ;; + *) args+=("$1") ;; + esac + shift +done + +# -headerpad_max_install_names leaves room for Flutter's install_name_tool, +# and -S keeps zig from writing object paths and mtimes into stabs. +exec zig cc -target "$APPLE_CC_TARGET" --sysroot "$SDKROOT" \ + -isystem "$SDKROOT/usr/include" -L/usr/lib \ + -Wl,-headerpad_max_install_names -Wl,-S "${args[@]}" diff --git a/contrib/native/apple-stubs.sh b/contrib/native/apple-stubs.sh new file mode 100755 index 000000000..6f7705385 --- /dev/null +++ b/contrib/native/apple-stubs.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Generate a stub Apple SDK in $1 from the macOS libSystem.tbd and Darwin +# headers that zig ships, so the iOS and macOS targets link on Linux +# without Apple's SDK. Every target list is widened to all four targets; +# the macOS verification job checks the imports against Xcode's real stubs. +set -euo pipefail + +sdk=$1 +zig_lib=$(zig env | sed -n 's/^ *\.lib_dir = "\(.*\)",$/\1/p') +apple_targets="[ arm64-ios, arm64-ios-simulator, arm64-macos, x86_64-macos ]" + +rm -rf "$sdk" +mkdir -p "$sdk/usr/include" "$sdk/usr/lib" +# The nix store copy is read-only, and the next run must remove it. +cp -R --no-preserve=mode "$zig_lib/libc/include/any-darwin-any/." "$sdk/usr/include/" +sed -zE "s/targets:( *)\[[^]]*\]/targets:\1$apple_targets/g" \ + "$zig_lib/libc/darwin/libSystem.tbd" >"$sdk/usr/lib/libSystem.tbd" +# std links libiconv on Apple targets, and zig ships no stub for it. +cat >"$sdk/usr/lib/libiconv.tbd" </. +set -euo pipefail + +target=$1 +out_dir=$(realpath -m "$2") +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +target_dir=$(realpath -m "${CARGO_TARGET_DIR:-$repo/target}") +target_env=${target//-/_} +CARGO_HOME=$(realpath -m "${CARGO_HOME:-$HOME/.cargo}") +export CARGO_HOME +# Two builds of the same zig version share a cache key, so keep the cache +# with the build rather than in $HOME. +export ZIG_GLOBAL_CACHE_DIR=$target_dir/zig-cache + +cd "$repo" +source contrib/lockfile.sh +use_lockfile Cargo-recent.lock + +# rustc and the C compiler see the same prefixes, and the last match wins. +prefixes=( + "$repo=/payjoin" + "$target_dir=/payjoin/target" + "$CARGO_HOME=/cargo" + "${RUSTUP_HOME:-$HOME/.rustup}=/rustup" +) +rustflags=() +cflags=() +cargo=(cargo build) +cargo_target=$target +cflag= + +case $target in + *-unknown-linux-gnu) + lib=libpayjoin_ffi.so + cargo=(cargo zigbuild) + cargo_target=$target.2.17 + ;; + *-pc-windows-msvc) + lib=payjoin_ffi.dll + arch=${target%%-*} + # MSVC's CRT and the Windows SDK from the Visual Studio manifest in + # vs-channel.json, splatted once per set of pins. + sdk=10.0.26100 + crt=14.44.17.14 + manifest=$(sha256sum contrib/native/vs-channel.json | cut -c1-16) + xwin_dir=${XDG_CACHE_HOME:-$HOME/.cache}/payjoin-xwin/$manifest-$sdk-$crt + if [[ ! -d $xwin_dir ]]; then + # Concurrent first builds splat their own copies, and the first + # one done is kept. + mkdir -p "$(dirname "$xwin_dir")" + tmp=$(mktemp -d "$xwin_dir.XXXXXX") + xwin --accept-license --cache-dir "$tmp/download" \ + --manifest contrib/native/vs-channel.json \ + --sdk-version "$sdk" --crt-version "$crt" \ + --arch x86_64,aarch64 splat --output "$tmp/splat" + mv -T "$tmp/splat" "$xwin_dir" 2>/dev/null || true + rm -rf "$tmp" + fi + export "CARGO_TARGET_${target_env^^}_LINKER=lld-link" \ + "CC_$target_env=clang-cl" "AR_$target_env=llvm-lib" + # lld-link otherwise stamps the build time, and the PDB rustc asks + # for gives the DLL an id that depends on the build paths. + rustflags+=(-Clink-arg=/Brepro -Clink-arg=/DEBUG:NONE) + for dir in crt/lib sdk/lib/um sdk/lib/ucrt; do + rustflags+=("-Lnative=$xwin_dir/$dir/$arch") + done + # clang-cl takes clang's own options behind /clang:. + cflag=/clang: + cflags+=("--target=$target") + for dir in crt/include sdk/include/ucrt sdk/include/um sdk/include/shared; do + cflags+=(/imsvc "$xwin_dir/$dir") + done + prefixes+=("$xwin_dir=/xwin") + ;; + *-linux-android*) + lib=libpayjoin_ffi.so + # API 21 is Rust's floor. NDK 29 aligns 64-bit libraries to 16 KB. + ndk_bin=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin + clang=$ndk_bin/${target/armv7/armv7a}21-clang + export "CARGO_TARGET_${target_env^^}_LINKER=$clang" "CC_$target_env=$clang" \ + "AR_$target_env=$ndk_bin/llvm-ar" + prefixes+=("$(realpath "$ANDROID_NDK_HOME")=/ndk") + ;; + *-apple-darwin) + # Flutter's floor, and the first macOS on Apple silicon. + macos=10.15 + if [[ $target == aarch64-* ]]; then + macos=11.0 + fi + export APPLE_CC_TARGET=${target%%-*}-macos.$macos + export MACOSX_DEPLOYMENT_TARGET=$macos + ;;& + aarch64-apple-ios | aarch64-apple-ios-sim) + export APPLE_CC_TARGET=aarch64-ios.13.0 + if [[ $target == *-sim ]]; then + APPLE_CC_TARGET+=-simulator + fi + export IPHONEOS_DEPLOYMENT_TARGET=13.0 + ;;& + *-apple-darwin | aarch64-apple-ios | aarch64-apple-ios-sim) + lib=libpayjoin_ffi.dylib + export SDKROOT=$target_dir/apple-stubs + bash contrib/native/apple-stubs.sh "$SDKROOT" + export "CARGO_TARGET_${target_env^^}_LINKER=$repo/contrib/native/apple-cc.sh" + export "CC_$target_env=$repo/contrib/native/apple-cc.sh" + export "AR_$target_env=$(command -v zig) ar" + rustflags+=("-Clink-arg=-Wl,-install_name,@rpath/$lib") + ;; + *) + echo "unsupported target: $target" >&2 + exit 1 + ;; +esac + +for prefix in "${prefixes[@]}"; do + rustflags+=("--remap-path-prefix=$prefix") + cflags+=("$cflag-ffile-prefix-map=$prefix") +done +# Quoted, because cc-rs would otherwise split a path with spaces. +export CC_SHELL_ESCAPED_FLAGS=1 +export "CFLAGS_$target_env=$(printf '%q ' "${cflags[@]}")" +CARGO_ENCODED_RUSTFLAGS=$( + IFS=$'\x1f' + echo "${rustflags[*]}" +) +export CARGO_ENCODED_RUSTFLAGS + +"${cargo[@]}" --locked --release --package payjoin-ffi --lib \ + --target "$cargo_target" --target-dir "$target_dir" +mkdir -p "$out_dir/$target" +cp "$target_dir/$target/release/$lib" "$out_dir/$target/" diff --git a/contrib/native/vs-channel.json b/contrib/native/vs-channel.json new file mode 100644 index 000000000..2ecc4fea0 --- /dev/null +++ b/contrib/native/vs-channel.json @@ -0,0 +1,17 @@ +{ + "channelItems": [ + { + "id": "Microsoft.VisualStudio.Manifests.VisualStudio", + "version": "17.14.37824.6", + "type": "Manifest", + "payloads": [ + { + "fileName": "VisualStudio.vsman", + "sha256": "5bd3b0b2ae509af2e9559b852f78545df0bf2c081dd850caed76f1dc134c511e", + "size": 30443792, + "url": "https://download.visualstudio.microsoft.com/download/pr/4bc08c27-bd31-470c-9a30-3b62707fbf0f/5bd3b0b2ae509af2e9559b852f78545df0bf2c081dd850caed76f1dc134c511e/VisualStudio.vsman" + } + ] + } + ] +} diff --git a/flake.nix b/flake.nix index 461e37b37..febfb306f 100644 --- a/flake.nix +++ b/flake.nix @@ -474,6 +474,65 @@ ''; }; + # Release builds of the payjoin-ffi native library through + # contrib/native/build.sh. The tools come from unstable: the release + # branch only has zig 0.15, and 0.17 regressed the iOS Mach-O linker. + unstablePkgs = nixpkgs-unstable.legacyPackages.${system}; + + nativeRustToolchain = pkgs.rust-bin.stable.${release-version}.minimal.override { + targets = [ + "aarch64-apple-ios" + "aarch64-apple-ios-sim" + "aarch64-unknown-linux-gnu" + "x86_64-unknown-linux-gnu" + "aarch64-apple-darwin" + "x86_64-apple-darwin" + "aarch64-pc-windows-msvc" + "x86_64-pc-windows-msvc" + "aarch64-linux-android" + "armv7-linux-androideabi" + "x86_64-linux-android" + ]; + }; + + # The NDK and the SDK parts it pulls in are unfree, so they come from + # an instance used for nothing else. + androidPkgs = import nixpkgs-unstable { + inherit system; + config = { + android_sdk.accept_license = true; + allowUnfree = true; + }; + }; + + androidNdk = + (androidPkgs.androidenv.composeAndroidPackages { + includeNDK = true; + ndkVersions = [ "29.0.14206865" ]; + }).ndk-bundle; + + nativeDevShell = pkgs.mkShell { + name = "native"; + packages = with unstablePkgs; [ + nativeRustToolchain + zig_0_16 + # nixpkgs wraps cargo-zigbuild with its default zig, 0.17. + (cargo-zigbuild.override { zig = zig_0_16; }) + xwin + # The Windows targets compile C with clang-cl, archive with + # llvm-lib and link with lld-link. + llvmPackages_21.clang-unwrapped + llvmPackages_21.lld + llvmPackages_21.llvm + ]; + }; + + # Only the Android builds fetch the NDK. + nativeAndroidDevShell = nativeDevShell.overrideAttrs { + name = "native-android"; + ANDROID_NDK_HOME = "${androidNdk}/libexec/android-sdk/ndk-bundle"; + }; + simpleCheck = args: pkgs.stdenvNoCC.mkDerivation ( @@ -527,6 +586,8 @@ csharp = csharpDevShell; dart = dartDevShell; kotlin = kotlinDevShell; + native = nativeDevShell; + native-android = nativeAndroidDevShell; }; formatter = treefmtEval.config.build.wrapper; checks = From e327c13ec69cee0e98c377c1fbb582b51499e86a Mon Sep 17 00:00:00 2001 From: spacebear Date: Wed, 28 Oct 2026 14:54:52 +0000 Subject: [PATCH 5/9] Check in CI that the native libraries reproduce Nothing yet checks that build.sh produces the same bytes twice. Build each target twice, on separate runners. The second build runs from another checkout path, home and cargo home, all with a space in them, so any path that leaks into a library breaks the match, including one from a cache the toolchain keeps under the home directory. A compare job fails unless the second build matches the first one's hashes. Once they match, the compare job combines the two macOS slices with llvm-lipo into universal2-apple-darwin, the file Python ships. lipo output depends only on its inputs, so the derived file needs no second build. Every checked library is uploaded as one `payjoin-native` artifact, laid out by triple, for the bindings to package. The workflow runs in the daily bindings run and not on pull requests: two builds of eleven targets is 22 jobs, too many for every PR, so the scheduled run carries this check. --- .github/workflows/cron-daily-bindings.yml | 2 + .github/workflows/native.yml | 113 ++++++++++++++++++++++ contrib/native/README.md | 5 +- 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/native.yml diff --git a/.github/workflows/cron-daily-bindings.yml b/.github/workflows/cron-daily-bindings.yml index 5da0b4e0e..ac95ea388 100644 --- a/.github/workflows/cron-daily-bindings.yml +++ b/.github/workflows/cron-daily-bindings.yml @@ -23,3 +23,5 @@ jobs: uses: $/.github/workflows/dart.yml kotlin: uses: $/.github/workflows/kotlin.yml + native: + uses: $/.github/workflows/native.yml diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml new file mode 100644 index 000000000..4f849d388 --- /dev/null +++ b/.github/workflows/native.yml @@ -0,0 +1,113 @@ +name: Native libraries + +# Builds the payjoin-ffi native library for each target twice, on separate +# runners from different paths, and fails unless both builds are +# byte-identical. + +on: + workflow_call: + +permissions: + contents: read + +jobs: + build: + name: Build ${{ matrix.target }} (${{ matrix.build }}) + runs-on: ubuntu-26.04 + timeout-minutes: 60 + strategy: + matrix: + target: + - aarch64-apple-ios + - aarch64-apple-ios-sim + - aarch64-unknown-linux-gnu + - x86_64-unknown-linux-gnu + - aarch64-apple-darwin + - x86_64-apple-darwin + - aarch64-pc-windows-msvc + - x86_64-pc-windows-msvc + - aarch64-linux-android + - armv7-linux-androideabi + - x86_64-linux-android + build: [first, second] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Set up nix + uses: $/.github/actions/setup-nix + - name: Build + env: + TARGET: ${{ matrix.target }} + BUILD: ${{ matrix.build }} + run: | + set -euo pipefail + # The second build runs from other paths, with a space in them, so + # any path that leaks into the library breaks the match. Its own + # home moves the caches the toolchain keeps there, such as zig's. + repo=$GITHUB_WORKSPACE + if [[ $BUILD == second ]]; then + repo="$RUNNER_TEMP/second build/payjoin" + mkdir -p "$repo" + cp -R . "$repo" + export HOME="$RUNNER_TEMP/second build/home" CARGO_HOME="$RUNNER_TEMP/second build/cargo" + fi + shell=native + if [[ $TARGET == *-android* ]]; then + shell=native-android + fi + nix develop ".#$shell" -c "$repo/contrib/native/build.sh" "$TARGET" "$RUNNER_TEMP/out" + - name: Upload the library + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: native-${{ matrix.build }}-${{ matrix.target }} + path: ${{ runner.temp }}/out + if-no-files-found: error + + compare: + name: Compare the two builds + needs: build + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Set up nix + uses: $/.github/actions/setup-nix + - name: Download the first build + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-first-* + merge-multiple: true + path: ${{ runner.temp }}/first + - name: Download the second build + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-second-* + merge-multiple: true + path: ${{ runner.temp }}/second + - name: Compare + working-directory: ${{ runner.temp }} + run: | + set -euo pipefail + (cd first && sha256sum -- */*) | tee SHA256SUMS + (cd second && sha256sum -c ../SHA256SUMS) + # Both slices just matched, so the file derived from them needs no + # second build of its own. + - name: Derive the universal2 macOS library + run: | + set -euo pipefail + dir="$RUNNER_TEMP/first" + mkdir -p "$dir/universal2-apple-darwin" + nix shell --inputs-from . nixpkgs-unstable#llvmPackages_21.llvm -c llvm-lipo -create \ + "$dir/aarch64-apple-darwin/libpayjoin_ffi.dylib" \ + "$dir/x86_64-apple-darwin/libpayjoin_ffi.dylib" \ + -output "$dir/universal2-apple-darwin/libpayjoin_ffi.dylib" + - name: Upload the libraries + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/first + if-no-files-found: error diff --git a/contrib/native/README.md b/contrib/native/README.md index 31501a209..5bcf18e3d 100644 --- a/contrib/native/README.md +++ b/contrib/native/README.md @@ -9,7 +9,10 @@ nix develop .#native -c contrib/native/build.sh The Android targets need the NDK, which only `.#native-android` provides. -The library lands in `//`. +The library lands in `//`. The +[native workflow](../../.github/workflows/native.yml) builds every target +twice, on separate runners from different paths, and fails unless both +builds are byte-identical. | Invariant | Rule | | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | From 3b95956ddd52799630b6b9d03501a6ad79297186 Mon Sep 17 00:00:00 2001 From: spacebear Date: Wed, 28 Oct 2026 14:54:52 +0000 Subject: [PATCH 6/9] Verify the Apple libraries on a macOS runner The Linux builds link against stubs derived from zig's macOS libSystem, not Apple's SDKs, so a symbol missing on the platform would link and then fail to load. Nothing has yet loaded these libraries on an Apple system. Add a macOS job that verifies the libraries and never produces any: * Xcode's linker resolves every import of each iOS and macOS library against the real SDK of its platform. * Each library, the universal2 one included, is renamed with install_name_tool and signed with codesign, as Flutter does when it wraps a dylib in a framework. * A small Flutter app runs a smoke test in the arm64 simulator. A stand-in build hook makes it bundle the Linux-built library, so the test goes through Flutter's own embed path with those exact bytes. The smoke test does not reuse the Dart unit tests, because those call helpers from the `_test-utils` feature, which the shipped library is built without. --- .github/workflows/native.yml | 76 ++++++++++++++++++- contrib/native/README.md | 3 +- .../integration_test/payjoin_test.dart | 60 +++++++++++++++ .../contrib/ios-simulator/prebuilt_hook.dart | 19 +++++ .../dart/contrib/ios-simulator/pubspec.yaml | 23 ++++++ 5 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart create mode 100644 payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart create mode 100644 payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 4f849d388..4c46af493 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -2,7 +2,8 @@ name: Native libraries # Builds the payjoin-ffi native library for each target twice, on separate # runners from different paths, and fails unless both builds are -# byte-identical. +# byte-identical. The libraries only ever come from Linux. The macOS job +# checks the Apple ones against Xcode and produces nothing. on: workflow_call: @@ -111,3 +112,76 @@ jobs: name: payjoin-native path: ${{ runner.temp }}/first if-no-files-found: error + + verify-apple: + name: Verify the Apple libraries on macOS + needs: compare + runs-on: macos-latest + timeout-minutes: 60 + env: + RUSTUP_TOOLCHAIN: 1.85.0 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Download the libraries + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/native + - name: Check the imports against Xcode's SDKs + working-directory: ${{ runner.temp }}/native + run: | + set -euo pipefail + # Apple's linker resolves every symbol a library imports against + # the SDK of the platform it runs on, and fails on any it lacks. + check() { + local triple=$1 target=$2 sdk=$3 + nm -u -j "$triple/libpayjoin_ffi.dylib" | sed 's/^/-Wl,-u,/' >"$triple.args" + xcrun --sdk "$sdk" clang -target "$target" -dynamiclib -x c /dev/null \ + -liconv @"$triple.args" -o "$RUNNER_TEMP/$triple-imports.dylib" + } + check aarch64-apple-ios arm64-apple-ios13.0 iphoneos + check aarch64-apple-ios-sim arm64-apple-ios13.0-simulator iphonesimulator + check aarch64-apple-darwin arm64-apple-macos11.0 macosx + check x86_64-apple-darwin x86_64-apple-macos10.15 macosx + - name: Rename and sign each library the way Flutter embeds it + run: | + set -euo pipefail + lipo "$RUNNER_TEMP/native/universal2-apple-darwin/libpayjoin_ffi.dylib" -verify_arch arm64 x86_64 + for lib in "$RUNNER_TEMP"/native/*/libpayjoin_ffi.dylib; do + cp "$lib" "$RUNNER_TEMP/payjoin_ffi" + install_name_tool -id @rpath/payjoin_ffi.framework/payjoin_ffi "$RUNNER_TEMP/payjoin_ffi" + codesign --force --sign - "$RUNNER_TEMP/payjoin_ffi" + codesign --verify --verbose "$RUNNER_TEMP/payjoin_ffi" + done + - name: "Install Rust 1.85.0" + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 + with: + toolchain: 1.85.0 + - name: Set up Flutter + uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2.23.0 + with: + channel: stable + flutter-version: 3.44.9 + - name: Generate the Dart bindings + run: | + set -euo pipefail + cp Cargo-recent.lock Cargo.lock + cd payjoin-ffi/dart + # The library carries no features, so neither may the bindings. + PAYJOIN_FFI_FEATURES='' bash scripts/generate_bindings.sh + - name: Run a smoke test in the iOS simulator + working-directory: payjoin-ffi/dart + run: | + set -euo pipefail + # Bundle the libraries built on Linux instead of building one. + cp contrib/ios-simulator/prebuilt_hook.dart hook/build.dart + cp -R "$RUNNER_TEMP/native" prebuilt + cd contrib/ios-simulator + flutter create --platforms ios . + udid=$(xcrun simctl list devices available --json | + jq -r '[.devices[][] | select(.name | startswith("iPhone"))][0].udid') + xcrun simctl boot "$udid" + flutter test integration_test -d "$udid" diff --git a/contrib/native/README.md b/contrib/native/README.md index 5bcf18e3d..e94e46f4f 100644 --- a/contrib/native/README.md +++ b/contrib/native/README.md @@ -12,7 +12,8 @@ The Android targets need the NDK, which only `.#native-android` provides. The library lands in `//`. The [native workflow](../../.github/workflows/native.yml) builds every target twice, on separate runners from different paths, and fails unless both -builds are byte-identical. +builds are byte-identical. A macOS runner checks the Apple libraries +against Xcode and never produces any. | Invariant | Rule | | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart b/payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart new file mode 100644 index 000000000..a8eceeccc --- /dev/null +++ b/payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart @@ -0,0 +1,60 @@ +import 'dart:typed_data'; + +import 'package:convert/convert.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:payjoin/payjoin.dart' as payjoin; + +// The bundled library is built without the test utilities, so the unit tests +// that call them cannot run here. +class _ReceiverPersister implements payjoin.JsonReceiverSessionPersister { + final List events = []; + + @override + void save(String event) => events.add(event); + + @override + List load() => events; + + @override + void close() {} +} + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + test('parses a payjoin URI', () { + final uri = payjoin.Uri.parse( + uri: 'bitcoin:12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX?amount=1&pj=https://example.com', + ); + expect(uri.address(), '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + }); + + test('rejects userinfo in the payjoin URL', () { + expect( + () => payjoin.Uri.parse( + uri: 'bitcoin:12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX?amount=1&pj=https://example@evil.com', + ), + throwsA(isA()), + ); + }); + + test('persists a new receiver session', () { + final persister = _ReceiverPersister(); + payjoin.ReceiverBuilder( + address: 'tb1q6d3a2w975yny0asuvd9a67ner4nks58ff0q8g4', + directory: 'https://example.com', + ohttpKeys: payjoin.OhttpKeys.decode( + bytes: Uint8List.fromList( + hex.decode( + '01001604ba48c49c3d4a92a3ad00ecc63a024da10ced02180c73ec12d8a7ad2cc91bb483824fe2bee8d28bfe2eb2fc6453bc4d31cd851e8a6540e86c5382af588d370957000400010003', + ), + ), + ), + ).build().save(persister: persister); + expect( + payjoin.replayReceiverEventLog(persister: persister).state(), + isA(), + ); + }); +} diff --git a/payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart b/payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart new file mode 100644 index 000000000..e2476a74b --- /dev/null +++ b/payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart @@ -0,0 +1,19 @@ +import 'package:code_assets/code_assets.dart'; +import 'package:hooks/hooks.dart'; + +// Replaces the package's build hook in the iOS simulator check, so the app +// bundles the library cross-built on Linux instead of building its own. +void main(List args) async { + await build(args, (input, output) async { + output.assets.code.add( + CodeAsset( + package: input.packageName, + name: 'uniffi:payjoin_ffi', + linkMode: DynamicLoadingBundled(), + file: input.packageRoot.resolve( + 'prebuilt/aarch64-apple-ios-sim/libpayjoin_ffi.dylib', + ), + ), + ); + }); +} diff --git a/payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml b/payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml new file mode 100644 index 000000000..79515bb64 --- /dev/null +++ b/payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml @@ -0,0 +1,23 @@ +name: payjoin_ios_simulator +description: >- + A Flutter app that runs a payjoin smoke test in the iOS simulator against + the library cross-built on Linux. +publish_to: none +version: 0.0.0 + +environment: + sdk: ^3.10.0 + +dependencies: + flutter: + sdk: flutter + # CI swaps in prebuilt_hook.dart as the package's build hook. + payjoin: + path: ../.. + +dev_dependencies: + convert: ^3.1.0 + flutter_test: + sdk: flutter + integration_test: + sdk: flutter From 9813467bc889cde1e01d157a78a61a8a836b5b40 Mon Sep 17 00:00:00 2001 From: spacebear Date: Wed, 28 Oct 2026 20:06:38 +0000 Subject: [PATCH 7/9] Attach native libraries to the bindings release The release shipped native libraries only inside the wheels and the NuGet package, and nothing in it came from the native workflow's checked double build. The release workflow now calls the native workflow once, and every publish job waits for it, so a library that does not reproduce or fails the Apple checks stops the release. The GitHub release job names each library payjoin_ffi--., attaches all 12, lists them in SHA256SUMS and attests them, so a consumer can check a library against the release without unpacking a package. VERIFYING.md explains how to check one, or rebuild it from the Linux devshell, iOS included, and compare. --- .github/workflows/bindings-release.yml | 41 ++++++++++++---- payjoin-ffi/CHANGELOG.md | 3 ++ payjoin-ffi/RELEASING.md | 7 +-- payjoin-ffi/VERIFYING.md | 67 ++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 payjoin-ffi/VERIFYING.md diff --git a/.github/workflows/bindings-release.yml b/.github/workflows/bindings-release.yml index 5b9605d81..209960232 100644 --- a/.github/workflows/bindings-release.yml +++ b/.github/workflows/bindings-release.yml @@ -1,10 +1,11 @@ name: Release language bindings # Pushing a signed `payjoin-ffi-` tag verifies the tag and the -# shared binding version, runs every language's build, package and smoke -# checks at that commit, then publishes Python, JavaScript, C#, and Dart -# through the `release` environment (required reviewer) with keyless OIDC -# and cuts one GitHub release holding every package. +# shared binding version, builds the native libraries once, runs every +# language's build, package and smoke checks at that commit, then +# publishes Python, JavaScript, C#, and Dart through the `release` +# environment (required reviewer) with keyless OIDC and cuts one GitHub +# release holding every package and native library. # # Each registry has its own publish job. All four wait on every language # build, so they reach the environment gate together and one approval @@ -84,10 +85,13 @@ jobs: kotlin: needs: verify-tag uses: $/.github/workflows/kotlin.yml + native: + needs: verify-tag + uses: $/.github/workflows/native.yml publish-python: name: Publish to PyPI - needs: [python, javascript, csharp, dart, kotlin] + needs: [python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -121,7 +125,7 @@ jobs: publish-javascript: name: Publish to npmjs.com - needs: [verify-tag, python, javascript, csharp, dart, kotlin] + needs: [verify-tag, python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -165,7 +169,7 @@ jobs: publish-csharp: name: Publish to nuget.org - needs: [python, javascript, csharp, dart, kotlin] + needs: [python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -216,7 +220,7 @@ jobs: publish-dart: name: Publish to pub.dev - needs: [python, javascript, csharp, dart, kotlin] + needs: [python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -254,6 +258,8 @@ jobs: runs-on: ubuntu-26.04 permissions: contents: write # create the Release for this tag and upload assets + id-token: write # OIDC: used by actions/attest + attestations: write # actions/attest writes the attestation env: VERSION: ${{ needs.verify-tag.outputs.version }} steps: @@ -267,6 +273,15 @@ jobs: pattern: payjoin-* merge-multiple: true path: dist + - name: Name the native libraries for the release + # payjoin-native lands as one directory per target. + run: | + set -euo pipefail + for lib in dist/*/*; do + target=$(basename "$(dirname "$lib")") + mv "$lib" "dist/payjoin_ffi-$VERSION-$target.${lib##*.}" + rmdir "$(dirname "$lib")" + done - name: Check the asset set is complete run: | set -euo pipefail @@ -274,11 +289,17 @@ jobs: wheels=(dist/*.whl) npm=(dist/*.tgz) nuget=(dist/*.nupkg) - if [[ ${#wheels[@]} != 2 || ${#npm[@]} != 1 || ${#nuget[@]} != 1 || ! -f dist/payjoin-dart.tar.gz ]]; then - echo "::error::expected two wheels, one npm tarball, one NuGet package, and the Dart archive in dist/" + native=(dist/payjoin_ffi-*) + if [[ ${#wheels[@]} != 2 || ${#npm[@]} != 1 || ${#nuget[@]} != 1 || ! -f dist/payjoin-dart.tar.gz || ${#native[@]} != 12 ]]; then + echo "::error::expected two wheels, one npm tarball, one NuGet package, the Dart archive and 12 native libraries in dist/" ls dist exit 1 fi + - name: Attest build provenance of the native libraries + # A consumer runs: gh attestation verify -R payjoin/rust-payjoin + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: dist/payjoin_ffi-* - name: Generate SHA256SUMS # The NuGet package's native libraries are listed individually so # each can be verified on its own. A maintainer's GPG signature over diff --git a/payjoin-ffi/CHANGELOG.md b/payjoin-ffi/CHANGELOG.md index 34616225d..98a508f3f 100644 --- a/payjoin-ffi/CHANGELOG.md +++ b/payjoin-ffi/CHANGELOG.md @@ -3,6 +3,9 @@ - Python: require Python 3.11+ and replace httpx with httpx2. Callers catching HTTP errors from fetch_ohttp_keys should use httpx2.HTTPError instead of httpx.HTTPError. - Dart: Android builds with NDK r27 now align 64-bit libraries to 16 KB, as Google Play requires. Later NDKs already did. +- The GitHub release now attaches every native library as + `payjoin_ffi--.`, listed in `SHA256SUMS` and + attested. `VERIFYING.md` explains how to check one or rebuild it. ## [0.25.1] diff --git a/payjoin-ffi/RELEASING.md b/payjoin-ffi/RELEASING.md index c68e27310..8739beedc 100644 --- a/payjoin-ffi/RELEASING.md +++ b/payjoin-ffi/RELEASING.md @@ -83,9 +83,10 @@ environment, and allow `payjoin-ffi-*` in the repository's tag rules. environment. Approve the pending deployment once; it covers all four jobs. Each registry is published with OIDC from the tested artifacts. 5. The run attaches the wheels, npm tarball, NuGet package, staged Dart - package, and one `SHA256SUMS` covering them all, including the native - libraries inside the NuGet package, to the GitHub release. Verify an - install from each registry and check provenance with + package, every native library as `payjoin_ffi--.`, + and one `SHA256SUMS` covering them all, including the native libraries + inside the NuGet package, to the GitHub release. Verify an install from + each registry and check provenance with `gh attestation verify FILE -R payjoin/rust-payjoin`. Optionally sign `SHA256SUMS` locally and upload `SHA256SUMS.asc`; no signing key reaches a runner. diff --git a/payjoin-ffi/VERIFYING.md b/payjoin-ffi/VERIFYING.md new file mode 100644 index 000000000..ae263ea96 --- /dev/null +++ b/payjoin-ffi/VERIFYING.md @@ -0,0 +1,67 @@ +# Verifying the native libraries + +Every bindings release attaches each native library it ships to its +[GitHub release](https://github.com/payjoin/rust-payjoin/releases) as +`payjoin_ffi--.`, listed in `SHA256SUMS` and +attested by the release workflow. + +CI builds each library twice, on separate runners from different paths, +and the release only goes ahead when both builds match. You can rebuild +any of them yourself, the Apple ones included, on Linux. + +## Check a download + +From a directory holding `SHA256SUMS` and the files you downloaded: + +```shell +sha256sum --check --ignore-missing SHA256SUMS +gh attestation verify payjoin_ffi--. -R payjoin/rust-payjoin +``` + +A maintainer may also upload `SHA256SUMS.asc`, a signature over +`SHA256SUMS` made with a key in +[`contrib/release/keys/`](../contrib/release/keys/). + +## Rebuild a library + +You need an x86_64 Linux machine with [nix](https://nixos.org/download/) +and flakes enabled. The build does not depend on where the checkout or +your home directory is. Build from a clean target directory, as in a +fresh clone: Cargo does not rebuild dependencies when the deployment +target changes, so a reused one can mix objects built for another target +and give a false mismatch. + +```shell +git clone https://github.com/payjoin/rust-payjoin.git +cd rust-payjoin +git checkout payjoin-ffi- +nix develop .#native -c contrib/native/build.sh /tmp/out +sha256sum /tmp/out//* +``` + +Compare the hash with the line for `payjoin_ffi--.` +in `SHA256SUMS`. The Android targets need the NDK, which only the +`.#native-android` devshell provides, so use it in place of `.#native` +for them. + +| Target | File | +| -------------------------------------------------------------------------- | ---------------------- | +| `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu` | `libpayjoin_ffi.so` | +| `aarch64-linux-android`, `armv7-linux-androideabi`, `x86_64-linux-android` | `libpayjoin_ffi.so` | +| `aarch64-apple-darwin`, `x86_64-apple-darwin` | `libpayjoin_ffi.dylib` | +| `aarch64-apple-ios`, `aarch64-apple-ios-sim` | `libpayjoin_ffi.dylib` | +| `x86_64-pc-windows-msvc`, `aarch64-pc-windows-msvc` | `payjoin_ffi.dll` | + +`universal2-apple-darwin` is not built on its own. Build both macOS +targets, then combine them: + +```shell +nix develop .#native -c llvm-lipo -create \ + /tmp/out/aarch64-apple-darwin/libpayjoin_ffi.dylib \ + /tmp/out/x86_64-apple-darwin/libpayjoin_ffi.dylib \ + -output /tmp/out/libpayjoin_ffi.dylib +sha256sum /tmp/out/libpayjoin_ffi.dylib +``` + +The [native build contract](../contrib/native/README.md) lists the +toolchain, flags and pinned inputs behind these builds. From 46074ac4b232c927920db9f0e39e84490e9c9e4a Mon Sep 17 00:00:00 2001 From: spacebear Date: Thu, 29 Oct 2026 22:31:42 +0000 Subject: [PATCH 8/9] Package Python and C# from the native workflow The NuGet native assets were cross-built with their own toolchain: cargo-zigbuild and cargo-xwin from pip, clang-cl and llvm-lib from the runner image, Rust 1.85.0, and the `csharp` feature, which only the bindings generator needs but which changes the library's metadata hash. The wheel's libraries were built by build-wheel.sh itself: the Linux one with the python devshell's host cargo, so its glibc floor was whatever the runner's toolchain linked against, and the universal2 macOS one with cargo-zigbuild, which links with `-g` and needs a toolchain of its own. None of it followed the native build contract, so the shipped files could not be rebuilt bit for bit, and they were not the ones the native workflow had checked. Both now download the payjoin-native artifact of the same run: the wheels take the Linux and universal2 files, and the NuGet package takes one file per RID, which removes the six build-nuget-native jobs, the pip pins, the clang-cl symlinks and the cross branch of build_nuget_native.sh. The release and the daily run make python and csharp wait for native. The per-RID smoke tests still install the packed nupkg on each platform. build-wheel.sh now takes the library to bundle as its argument. It generates the bindings from a development build of the same featureless library, because uniffi reads its metadata from the static symbol table, which the zig-linked Linux library lacks. The Linux library needs at most glibc 2.17, so auditwheel tags the wheel manylinux_2_17, and the macOS wheel is tagged macosx_10_15_universal2, the contract's floor, as the changelog notes. The python devshell no longer cross-links anything, so it drops cargo-zigbuild and the macOS targets on Linux. Since the GitHub release lists each library on its own, the wheel build and the pack job compare every library inside the package with the native file and fail on any difference. They run before publishing, which the GitHub release job does not. With that check in place, SHA256SUMS no longer lists the NuGet members a second time, and generate-sha256sums.sh loses its archive option, which nothing else uses. --- .github/workflows/bindings-release.yml | 13 +- .github/workflows/cron-daily-bindings.yml | 2 + .github/workflows/csharp.yml | 126 ++++-------------- .github/workflows/python.yml | 19 ++- contrib/release/generate-sha256sums.sh | 41 +----- flake.nix | 25 ++-- payjoin-ffi/CHANGELOG.md | 3 + payjoin-ffi/RELEASING.md | 6 +- payjoin-ffi/VERIFYING.md | 3 +- payjoin-ffi/csharp/CONTRIBUTING.md | 10 +- payjoin-ffi/csharp/RELEASING.md | 4 +- .../csharp/scripts/build_nuget_native.sh | 79 +---------- .../csharp/scripts/cross-requirements.txt | 19 --- payjoin-ffi/python/RELEASING.md | 11 +- payjoin-ffi/python/contrib/build-wheel.sh | 67 +++++----- 15 files changed, 122 insertions(+), 306 deletions(-) delete mode 100644 payjoin-ffi/csharp/scripts/cross-requirements.txt diff --git a/.github/workflows/bindings-release.yml b/.github/workflows/bindings-release.yml index 209960232..f08db1863 100644 --- a/.github/workflows/bindings-release.yml +++ b/.github/workflows/bindings-release.yml @@ -71,13 +71,13 @@ jobs: } python: - needs: verify-tag + needs: [verify-tag, native] uses: $/.github/workflows/python.yml javascript: needs: verify-tag uses: $/.github/workflows/javascript.yml csharp: - needs: verify-tag + needs: [verify-tag, native] uses: $/.github/workflows/csharp.yml dart: needs: verify-tag @@ -301,13 +301,8 @@ jobs: with: subject-path: dist/payjoin_ffi-* - name: Generate SHA256SUMS - # The NuGet package's native libraries are listed individually so - # each can be verified on its own. A maintainer's GPG signature over - # SHA256SUMS is made locally and uploaded as SHA256SUMS.asc; no - # signing key reaches a runner. - env: - ARCHIVE_GLOBS: "*.nupkg" - MEMBER_GLOBS: runtimes/*/native/* + # A maintainer's GPG signature over SHA256SUMS is made locally and + # uploaded as SHA256SUMS.asc; no signing key reaches a runner. run: | ./contrib/release/generate-sha256sums.sh dist cat dist/SHA256SUMS diff --git a/.github/workflows/cron-daily-bindings.yml b/.github/workflows/cron-daily-bindings.yml index ac95ea388..c68f0c07c 100644 --- a/.github/workflows/cron-daily-bindings.yml +++ b/.github/workflows/cron-daily-bindings.yml @@ -14,10 +14,12 @@ permissions: jobs: python: + needs: native uses: $/.github/workflows/python.yml javascript: uses: $/.github/workflows/javascript.yml csharp: + needs: native uses: $/.github/workflows/csharp.yml dart: uses: $/.github/workflows/dart.yml diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index b71ce9e20..0945bba76 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -106,106 +106,24 @@ jobs: - name: Run tests run: dotnet test Payjoin.Tests.csproj --logger "console;verbosity=minimal" - build-nuget-native: - # Every RID cross-compiles from Linux: cargo-zigbuild links the Linux targets - # against a pinned glibc floor and the macOS targets against zig's bundled - # libSystem stubs; cargo-xwin links MSVC-ABI Windows targets against the - # Microsoft CRT/SDK. The per-RID smoke jobs verify each produced asset on - # real hardware. - name: "Build NuGet native asset" - if: github.event_name != 'pull_request' - runs-on: ubuntu-26.04 - env: - # Must match the dtolnay/rust-toolchain version exactly: the cross-target - # std components are installed for that toolchain, and a partial version - # here selects a different rustup toolchain entry without them (E0463). - RUSTUP_TOOLCHAIN: 1.85.1 - defaults: - run: - working-directory: payjoin-ffi/csharp - strategy: - # Let every RID report its own result; a failure in one target should not - # cancel the signal from the others. - fail-fast: false - matrix: - include: - - rid: linux-arm64 - target: aarch64-unknown-linux-gnu - - rid: linux-x64 - target: x86_64-unknown-linux-gnu - - rid: osx-arm64 - target: aarch64-apple-darwin - - rid: osx-x64 - target: x86_64-apple-darwin - - rid: win-arm64 - target: aarch64-pc-windows-msvc - - rid: win-x64 - target: x86_64-pc-windows-msvc - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # unneeded: no git auth beyond checkout - - - name: Install Rust 1.85.1 - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 - with: - toolchain: 1.85.1 - targets: ${{ matrix.target }} - - - name: Use cache - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - - - name: Install cross toolchain - # Version- and hash-pinned: these tools produce the shipped binaries. - run: pip install --require-hashes -r scripts/cross-requirements.txt - - - name: Install MSVC toolchain frontends (clang-cl, llvm-lib) - if: startsWith(matrix.rid, 'win-') - # cargo-xwin compiles C dependencies with clang-cl and archives them with - # llvm-lib. The runner image ships LLVM with versioned binary names only, - # so expose the unversioned names cc-rs looks for. - run: | - for tool in clang-cl llvm-lib; do - if command -v "$tool" >/dev/null; then - continue - fi - latest=$(ls -1 /usr/bin/"$tool"-* 2>/dev/null | sort -V | tail -n 1) - if [ -z "$latest" ]; then - sudo apt-get update -q - sudo apt-get install -y -q llvm clang - latest=$(ls -1 /usr/bin/"$tool"-* 2>/dev/null | sort -V | tail -n 1) - fi - if [ -z "$latest" ]; then - echo "$tool not found on the runner" >&2 - exit 1 - fi - sudo ln -s "$latest" /usr/local/bin/"$tool" - done - command -v clang-cl llvm-lib - - - name: Build release native asset - run: | - PAYJOIN_FFI_CROSS=1 PAYJOIN_FFI_RID=${{ matrix.rid }} bash ./scripts/build_nuget_native.sh - - - name: Upload NuGet native asset - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ matrix.rid }} - path: payjoin-ffi/csharp/artifacts/runtimes/${{ matrix.rid }}/native/* - if-no-files-found: error - pack-nuget: + # Every RID's native asset comes from the native workflow's checked + # double build in the same run. The per-RID smoke jobs verify each one + # on real hardware. name: "Pack C# NuGet package" if: github.event_name != 'pull_request' runs-on: ubuntu-26.04 - needs: build-nuget-native env: # Exact version so the packaged bindings build with the toolchain the - # action installs, the release toolchain. + # action installs, the release toolchain of the native build contract. RUSTUP_TOOLCHAIN: 1.85.1 + RIDS: >- + linux-arm64=aarch64-unknown-linux-gnu + linux-x64=x86_64-unknown-linux-gnu + osx-arm64=aarch64-apple-darwin + osx-x64=x86_64-apple-darwin + win-arm64=aarch64-pc-windows-msvc + win-x64=x86_64-pc-windows-msvc defaults: run: working-directory: payjoin-ffi/csharp @@ -230,16 +148,19 @@ jobs: with: dotnet-version: "10.0.x" - - name: Download native assets + - name: Download the native libraries uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - path: payjoin-ffi/csharp/artifacts/downloaded + name: payjoin-native + path: ${{ runner.temp }}/native - name: Assemble RID assets run: | - for rid in linux-arm64 linux-x64 osx-arm64 osx-x64 win-arm64 win-x64; do + set -euo pipefail + for pair in $RIDS; do + rid=${pair%%=*} mkdir -p "artifacts/runtimes/$rid/native" - cp "artifacts/downloaded/$rid"/* "artifacts/runtimes/$rid/native/" + cp "$RUNNER_TEMP/native/${pair#*=}"/* "artifacts/runtimes/$rid/native/" done - name: Generate production bindings @@ -249,6 +170,17 @@ jobs: - name: Pack NuGet package run: dotnet pack Payjoin.csproj --configuration Release --output artifacts/packages + # The release lists each library on its own, so the package must carry + # them unchanged. + - name: Check the packed libraries + run: | + set -euo pipefail + for pair in $RIDS; do + for lib in "$RUNNER_TEMP/native/${pair#*=}"/*; do + unzip -p artifacts/packages/*.nupkg "runtimes/${pair%%=*}/native/$(basename "$lib")" | cmp - "$lib" + done + done + - name: Upload NuGet package uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c75efb35b..ebe3bb698 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -68,15 +68,20 @@ jobs: build-wheel: # The wheel bundles a prebuilt native library, so each supported # platform packs its own wheel: x86_64 on Linux (retagged manylinux by - # auditwheel) and a fat universal2 dylib for macOS. Both wheels build - # on a Linux host. The smoke jobs below verify each wheel on real hardware. + # auditwheel) and a fat universal2 dylib for macOS. Both libraries come + # from the native workflow's checked double build in the same run. The + # smoke jobs below verify each wheel on real hardware. name: "Build wheel" if: github.event_name != 'pull_request' runs-on: ubuntu-26.04 strategy: fail-fast: false matrix: - platform: [linux-x64, macos-universal2] + include: + - platform: linux-x64 + library: x86_64-unknown-linux-gnu/libpayjoin_ffi.so + - platform: macos-universal2 + library: universal2-apple-darwin/libpayjoin_ffi.dylib steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -88,10 +93,16 @@ jobs: lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Set up nix uses: $/.github/actions/setup-nix + - name: Download the native libraries + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/native - name: Build the wheel - run: nix develop .#python --command bash ./payjoin-ffi/python/contrib/build-wheel.sh env: PAYJOIN_WHEEL_PLATFORM: ${{ matrix.platform }} + LIBRARY: ${{ runner.temp }}/native/${{ matrix.library }} + run: nix develop .#python --command bash ./payjoin-ffi/python/contrib/build-wheel.sh "$LIBRARY" - name: Upload wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/contrib/release/generate-sha256sums.sh b/contrib/release/generate-sha256sums.sh index 725fd57e9..6721e8a84 100755 --- a/contrib/release/generate-sha256sums.sh +++ b/contrib/release/generate-sha256sums.sh @@ -1,20 +1,10 @@ #!/usr/bin/env bash # -# Write SHA256SUMS over every file in a release asset directory. With -# ARCHIVE_GLOBS and MEMBER_GLOBS set, also open each asset matching -# ARCHIVE_GLOBS and append the hash of each member matching MEMBER_GLOBS, -# under its in-archive path, so files nested inside a package can be -# verified individually. Both variables are whitespace-separated glob -# lists and must be set together. Archives must be zip format, which -# every current package with nested libraries is (.nupkg, .whl). +# Write SHA256SUMS over every file in a release asset directory. set -euo pipefail [ "$#" -eq 1 ] || { - echo "usage: [ARCHIVE_GLOBS=... MEMBER_GLOBS=...] generate-sha256sums.sh " >&2 - exit 1 -} -die() { - echo "generate-sha256sums: $*" >&2 + echo "usage: generate-sha256sums.sh " >&2 exit 1 } @@ -22,30 +12,3 @@ cd "$1" # shellcheck disable=SC2094 # find excludes SHA256SUMS, so it is never read find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 -r sha256sum >SHA256SUMS - -archive_globs="${ARCHIVE_GLOBS:-}" -member_globs="${MEMBER_GLOBS:-}" -if [ -z "$archive_globs" ] && [ -z "$member_globs" ]; then - exit 0 -fi -{ [ -n "$archive_globs" ] && [ -n "$member_globs" ]; } || - die "ARCHIVE_GLOBS and MEMBER_GLOBS must be set together" - -read -ra members <<<"$member_globs" -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT -shopt -s nullglob -archives=() -# shellcheck disable=SC2086 # unquoted so the shell expands the caller's globs -for archive in $archive_globs; do - archives+=("$archive") -done -[ "${#archives[@]}" -gt 0 ] || die "no files match ARCHIVE_GLOBS ($archive_globs)" - -# unzip matches the member globs itself and fails when an archive -# contains no match, or is not a zip. -for archive in "${archives[@]}"; do - unzip -q "$archive" -d "$tmp" "${members[@]}" -done - -(cd "$tmp" && find . -type f -printf '%P\0' | sort -z | xargs -0 -r sha256sum) >>SHA256SUMS diff --git a/flake.nix b/flake.nix index febfb306f..bbf361811 100644 --- a/flake.nix +++ b/flake.nix @@ -423,16 +423,15 @@ "rustfmt" "llvm-tools-preview" ]; - targets = [ - # On Darwin the native test builds lipo both arches; on Linux - # cargo-zigbuild cross-links the release wheel's universal2 - # dylib from these same targets. - "aarch64-apple-darwin" - "x86_64-apple-darwin" - ] - ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ - "x86_64-unknown-linux-gnu" - ]; + targets = + # On Darwin the native test builds lipo both arches. + pkgs.lib.optionals pkgs.stdenv.isDarwin [ + "aarch64-apple-darwin" + "x86_64-apple-darwin" + ] + ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + "x86_64-unknown-linux-gnu" + ]; }; pythonDevShell = pkgs.mkShell { @@ -454,10 +453,8 @@ # Applies the manylinux platform tag the release wheel satisfies; # PyPI rejects raw linux_x86_64 wheels. auditwheel - # Cross-links the macOS release wheel against zig's bundled - # Apple SDK stubs, so the dylib records system install names - # instead of nix store paths. - cargo-zigbuild + # build-wheel.sh compares the packed library with its input. + unzip ]; env = { diff --git a/payjoin-ffi/CHANGELOG.md b/payjoin-ffi/CHANGELOG.md index 98a508f3f..83e34ca3c 100644 --- a/payjoin-ffi/CHANGELOG.md +++ b/payjoin-ffi/CHANGELOG.md @@ -1,6 +1,9 @@ ## Unreleased - Python: require Python 3.11+ and replace httpx with httpx2. Callers catching HTTP errors from fetch_ohttp_keys should use httpx2.HTTPError instead of httpx.HTTPError. +- Python: the Linux wheel's library now needs at most glibc 2.17, and + the wheel is tagged `manylinux_2_17`. The macOS wheel now supports + macOS 10.15 and later, down from 13. - Dart: Android builds with NDK r27 now align 64-bit libraries to 16 KB, as Google Play requires. Later NDKs already did. - The GitHub release now attaches every native library as diff --git a/payjoin-ffi/RELEASING.md b/payjoin-ffi/RELEASING.md index 8739beedc..b46afb11b 100644 --- a/payjoin-ffi/RELEASING.md +++ b/payjoin-ffi/RELEASING.md @@ -84,9 +84,9 @@ environment, and allow `payjoin-ffi-*` in the repository's tag rules. jobs. Each registry is published with OIDC from the tested artifacts. 5. The run attaches the wheels, npm tarball, NuGet package, staged Dart package, every native library as `payjoin_ffi--.`, - and one `SHA256SUMS` covering them all, including the native libraries - inside the NuGet package, to the GitHub release. Verify an install from - each registry and check provenance with + and one `SHA256SUMS` covering them all to the GitHub release. The + wheels and the NuGet package carry those same native libraries, byte for + byte. Verify an install from each registry and check provenance with `gh attestation verify FILE -R payjoin/rust-payjoin`. Optionally sign `SHA256SUMS` locally and upload `SHA256SUMS.asc`; no signing key reaches a runner. diff --git a/payjoin-ffi/VERIFYING.md b/payjoin-ffi/VERIFYING.md index ae263ea96..8c75a5181 100644 --- a/payjoin-ffi/VERIFYING.md +++ b/payjoin-ffi/VERIFYING.md @@ -3,7 +3,8 @@ Every bindings release attaches each native library it ships to its [GitHub release](https://github.com/payjoin/rust-payjoin/releases) as `payjoin_ffi--.`, listed in `SHA256SUMS` and -attested by the release workflow. +attested by the release workflow. The Python wheels and the NuGet package +carry those same files, byte for byte. CI builds each library twice, on separate runners from different paths, and the release only goes ahead when both builds match. You can rebuild diff --git a/payjoin-ffi/csharp/CONTRIBUTING.md b/payjoin-ffi/csharp/CONTRIBUTING.md index 513ecc61e..4517ee4de 100644 --- a/payjoin-ffi/csharp/CONTRIBUTING.md +++ b/payjoin-ffi/csharp/CONTRIBUTING.md @@ -52,13 +52,13 @@ the C# bindings): bash ./scripts/build_nuget_native.sh ``` -Any supported RID can also be cross-compiled from a Linux host, which is how CI -builds every native asset (`pip install -r scripts/cross-requirements.txt` for -the version- and hash-pinned toolchain CI uses, and `rustup target add` the -matching triple first): +CI packs every RID's native asset from the native workflow, which builds them +on Linux under the [native build contract](../../contrib/native/README.md). To +build one the same way, run from the repository root with the target triple for the RID, then copy +`//*` to `payjoin-ffi/csharp/artifacts/runtimes/{rid}/native/`: ```shell -PAYJOIN_FFI_CROSS=1 PAYJOIN_FFI_RID=osx-arm64 bash ./scripts/build_nuget_native.sh +nix develop .#native -c contrib/native/build.sh aarch64-apple-darwin ``` On Windows: diff --git a/payjoin-ffi/csharp/RELEASING.md b/payjoin-ffi/csharp/RELEASING.md index ada426152..28bd56da2 100644 --- a/payjoin-ffi/csharp/RELEASING.md +++ b/payjoin-ffi/csharp/RELEASING.md @@ -20,9 +20,9 @@ CI is the release path. On every bindings release and in the `Daily language bindings` run on `master`, the `Build and Test CSharp` workflow: -1. builds release-profile native assets for each supported RID +1. takes the native workflow's library for each supported RID (`linux-arm64`, `linux-x64`, `osx-arm64`, `osx-x64`, `win-arm64`, - `win-x64`), + `win-x64`), built twice on Linux under the native build contract, 2. generates production bindings (no `_test-utils`) and packs the `.nupkg` with all RID assets, 3. installs the package into a clean console app and runs a smoke test on diff --git a/payjoin-ffi/csharp/scripts/build_nuget_native.sh b/payjoin-ffi/csharp/scripts/build_nuget_native.sh index 6a9c94fe4..1d8433d2b 100755 --- a/payjoin-ffi/csharp/scripts/build_nuget_native.sh +++ b/payjoin-ffi/csharp/scripts/build_nuget_native.sh @@ -55,50 +55,6 @@ native_library_name() { esac } -# Rust target triple for cross-compiling a RID from a Linux host. Linux targets pin a -# glibc floor through zig so the produced .so loads on older distributions instead of -# inheriting the build host's glibc version. -rid_to_cross_target() { - case "$1" in - linux-x64) - echo "x86_64-unknown-linux-gnu.$PAYJOIN_FFI_GLIBC_FLOOR" - ;; - linux-arm64) - echo "aarch64-unknown-linux-gnu.$PAYJOIN_FFI_GLIBC_FLOOR" - ;; - osx-x64) - echo "x86_64-apple-darwin" - ;; - osx-arm64) - echo "aarch64-apple-darwin" - ;; - win-x64) - echo "x86_64-pc-windows-msvc" - ;; - win-arm64) - echo "aarch64-pc-windows-msvc" - ;; - *) - echo "Unsupported RID for cross build: $1" >&2 - exit 1 - ;; - esac -} - -# cargo-zigbuild links Linux (with a chosen glibc floor) and macOS targets (zig ships -# redistributable libSystem stubs); cargo-xwin links MSVC-ABI Windows targets against -# the Microsoft CRT/SDK it fetches. -rid_to_cross_tool() { - case "$1" in - linux-* | osx-*) - echo "zigbuild" - ;; - win-*) - echo "xwin" - ;; - esac -} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CSHARP_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" @@ -114,42 +70,11 @@ LIBNAME=$(native_library_name "$RID") export PAYJOIN_FFI_FEATURES="${PAYJOIN_FFI_FEATURES-}" export PAYJOIN_FFI_PROFILE="${PAYJOIN_FFI_PROFILE:-release}" -PAYJOIN_FFI_GLIBC_FLOOR=${PAYJOIN_FFI_GLIBC_FLOOR:-2.17} - -if [[ ${PAYJOIN_FFI_CROSS:-0} == 1 ]]; then - # Cross-compile the native library for $RID from a Linux host. Bindings are never - # generated here; the pack step generates the production bindings it packages. - TARGET=$(rid_to_cross_target "$RID") - TOOL=$(rid_to_cross_tool "$RID") - if ! command -v "cargo-$TOOL" >/dev/null; then - echo "cargo-$TOOL is required for cross builds: pip install cargo-zigbuild cargo-xwin" >&2 - exit 1 - fi - - CROSS_CARGO=(cargo "$TOOL") - if [[ $TOOL == xwin ]]; then - CROSS_CARGO+=(build) - fi - - GENERATOR_FEATURES="csharp${PAYJOIN_FFI_FEATURES:+,$PAYJOIN_FFI_FEATURES}" - (cd .. && "${CROSS_CARGO[@]}" --target "$TARGET" --features "$GENERATOR_FEATURES" --profile "$PAYJOIN_FFI_PROFILE" -p payjoin-ffi) - - # Cargo writes to the triple without any zig glibc suffix. - TARGET_DIR=${TARGET%."$PAYJOIN_FFI_GLIBC_FLOOR"} - if [[ $PAYJOIN_FFI_PROFILE == "dev" ]]; then - PROFILE_DIR=debug - else - PROFILE_DIR=$PAYJOIN_FFI_PROFILE - fi - BUILT_LIB="../../target/$TARGET_DIR/$PROFILE_DIR/$LIBNAME" -else - bash ./scripts/generate_bindings.sh --native-only - BUILT_LIB="lib/$LIBNAME" -fi +bash ./scripts/generate_bindings.sh --native-only ARTIFACT_DIR="artifacts/runtimes/$RID/native" rm -rf "$ARTIFACT_DIR" mkdir -p "$ARTIFACT_DIR" -cp "$BUILT_LIB" "$ARTIFACT_DIR/$LIBNAME" +cp "lib/$LIBNAME" "$ARTIFACT_DIR/$LIBNAME" echo "Wrote $ARTIFACT_DIR/$LIBNAME" diff --git a/payjoin-ffi/csharp/scripts/cross-requirements.txt b/payjoin-ffi/csharp/scripts/cross-requirements.txt deleted file mode 100644 index 9f3db226f..000000000 --- a/payjoin-ffi/csharp/scripts/cross-requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Pinned cross-compilation toolchain for the CI native-asset job (Linux runners). -# These tools produce the exact binaries shipped in the NuGet package, so they -# are pinned by version and hash: a floating install would let an upstream -# release or a compromised PyPI package silently change the shipped artifacts. -# Update deliberately, never implicitly. Hashes cover the x86_64 and aarch64 -# manylinux wheels. -ziglang==0.16.0 \ - --hash=sha256:9fcda73f62b851dd72a54b710ad40a209896db14cfb13649e62191243556342b \ - --hash=sha256:e27d409812b11e0fb89ed0200cf2e55b6464d43f9461553104e4a4f9a94a1fd5 -cargo-zigbuild==0.23.0 \ - --hash=sha256:3a2e021435a5e3c8b3a70bfc490580a3af6952185aa5ffb722b005d1fbe000c9 \ - --hash=sha256:d984af93da7a230606452ef5f6df47e6a081b12dcead8a3e8e5ff024728c94f0 -cargo-xwin==0.22.0 \ - --hash=sha256:eebe464a84371c98c9a3e2d24a87b6c0fdc575146f6670f49823a366158027bf \ - --hash=sha256:8a6d3f235590a1e583286884cbdf0b2af67164713f8a81d292f3312e0fa2355e -# Transitive dependency of cargo-xwin. -ninja==1.13.0 \ - --hash=sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa \ - --hash=sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630 diff --git a/payjoin-ffi/python/RELEASING.md b/payjoin-ffi/python/RELEASING.md index 46153efd4..514d72411 100644 --- a/payjoin-ffi/python/RELEASING.md +++ b/payjoin-ffi/python/RELEASING.md @@ -20,10 +20,13 @@ workflow builds release wheels with `_test-utils`) and smoke-installs them on every supported platform: - `manylinux` x86_64, tagged by auditwheel with the glibc floor the binary - actually satisfies; -- macOS `universal2` (a fat x86_64 + arm64 dylib), cross-compiled from the - Linux host with cargo-zigbuild, so the dylib links the Apple SDK stubs - zig bundles and records system install names rather than nix store paths. + actually satisfies, 2.17; +- macOS `universal2` (a fat x86_64 + arm64 dylib), combined from the two + macOS slices. + +Both libraries come from the native workflow, which builds each one twice +on Linux under the [native build contract](../../contrib/native/README.md), +and the wheel carries them byte for byte. The wheels are tagged `py3-none` because the generated bindings load the bundled library through `ctypes` and do not depend on a CPython ABI; any diff --git a/payjoin-ffi/python/contrib/build-wheel.sh b/payjoin-ffi/python/contrib/build-wheel.sh index 2f2ffd476..76916b8a9 100755 --- a/payjoin-ffi/python/contrib/build-wheel.sh +++ b/payjoin-ffi/python/contrib/build-wheel.sh @@ -1,21 +1,31 @@ #!/usr/bin/env bash set -euo pipefail -# Build the production wheel for one platform into dist/: release profile, -# no test utils. Select the platform with PAYJOIN_WHEEL_PLATFORM -# (linux-x64, the default, or macos-universal2). -# -# Both wheels build on a Linux host: a dylib linked inside the nix dev shell -# on a mac records nix store install names that exist on no user machine, -# while cargo-zigbuild links against zig's bundled Apple SDK stubs and records -# the system ones. +# Build the production wheel for one platform into dist/ around a native +# library built under the native build contract (contrib/native/build.sh): +# release profile, no test utils. Select the platform with +# PAYJOIN_WHEEL_PLATFORM (linux-x64, the default, or macos-universal2) and +# pass the library, libpayjoin_ffi.so or the universal2 +# libpayjoin_ffi.dylib, as the only argument. if [[ "$(uname -s)" != Linux ]]; then echo "error: release wheels build on a Linux host only" >&2 exit 1 fi +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi +LIBRARY=$(realpath "$1") PLATFORM=${PAYJOIN_WHEEL_PLATFORM:-linux-x64} +case "$PLATFORM" in + linux-x64 | macos-universal2) ;; + *) + echo "error: unsupported PAYJOIN_WHEEL_PLATFORM: $PLATFORM" >&2 + exit 1 + ;; +esac # Build against the maintained lockfile instead of resolving the dependency # graph fresh on every run. use_lockfile copies Cargo-recent.lock into place @@ -27,28 +37,18 @@ use_lockfile Cargo-recent.lock cd "$REPO_ROOT/payjoin-ffi/python" +# uniffi reads its metadata from the static symbol table, which the +# zig-linked Linux library does not have, so the bindings come from a +# development build of the same featureless library. echo "==> Generating production FFI bindings..." -PAYJOIN_FFI_FEATURES="" PAYJOIN_FFI_PROFILE=release bash ./scripts/generate_bindings.sh +(cd .. && cargo build --lib && + cargo run --bin uniffi-bindgen generate \ + --library ../target/debug/libpayjoin_ffi.so \ + --language python --out-dir python/src/payjoin/) -# generate_bindings.sh stages the host's library; make sure the wheel -# ships exactly one platform's binary. -case "$PLATFORM" in - linux-x64) - rm -f src/payjoin/libpayjoin_ffi.dylib - ;; - macos-universal2) - echo "==> Cross-compiling the universal2 macOS library..." - (cd "$REPO_ROOT/payjoin-ffi" && - cargo zigbuild --profile release --target universal2-apple-darwin) - rm -f src/payjoin/libpayjoin_ffi.so - cp "$REPO_ROOT/target/universal2-apple-darwin/release/libpayjoin_ffi.dylib" \ - src/payjoin/ - ;; - *) - echo "error: unsupported PAYJOIN_WHEEL_PLATFORM: $PLATFORM" >&2 - exit 1 - ;; -esac +# A development build may have staged the other platform's library. +rm -f src/payjoin/libpayjoin_ffi.so src/payjoin/libpayjoin_ffi.dylib +cp "$LIBRARY" src/payjoin/ echo "==> Building the wheel..." # Drop setuptools' build/ staging dir too: it survives across runs and @@ -72,11 +72,14 @@ if [[ $PLATFORM == linux-x64 ]]; then rm dist/*-linux_x86_64.whl else # setup.py tagged the wheel with the build machine's (Linux) platform. - # The macOS floor is zig's: its bundled SDK stubs currently support - # macOS >= 13, and that is the minos both slices record. Keep this tag - # in sync with the LC_BUILD_VERSION of the built dylib if zig moves. - wheel tags --platform-tag macosx_13_0_universal2 --remove dist/*.whl + # The native build contract's macOS deployment target is 10.15 for the + # x86_64 slice and 11.0, the first macOS on Apple silicon, for arm64. + wheel tags --platform-tag macosx_10_15_universal2 --remove dist/*.whl fi +# The release lists the library on its own, so the wheel must carry it +# unchanged. +unzip -p dist/*.whl "payjoin/$(basename "$LIBRARY")" | cmp - "$LIBRARY" + echo "==> Built wheel:" ls -l dist From 9c01ab79c49e474f832a091fdb318db0ecde2668 Mon Sep 17 00:00:00 2001 From: spacebear Date: Thu, 29 Oct 2026 18:01:53 +0000 Subject: [PATCH 9/9] Ship Dart's native libraries in the package Every Dart consumer compiled payjoin-ffi with their own Rust toolchain, so none of them got the library the release checks for reproducibility, and every app build needed Rust and the targets' toolchains. prepare-publish.sh now copies the native workflow's payjoin-native artifact into the staged package as prebuilt//, for the 11 targets the hook maps; the universal macOS library is left out. The release and the daily run make dart wait for native to get it. The 11 libraries add about 26 MiB to the compressed archive, well under pub.dev's 100 MiB limit. The build hook now has three modes, chosen by the `build` user define in the root package's pubspec: - prebuilt, the default, registers the packaged library for the target. The build needs no network, no GitHub and no Rust, so it works offline and is hermetic. pub checks the archive against the sha256 in the consumer's pubspec.lock, and the release attests the staged archive. A target without a library is a build error that says how to switch to source, never a silent source build. - source builds the shipped native/Cargo.lock with --locked, as consumers did until now. - workspace builds payjoin-ffi from the checkout with _test-utils, and is what the package's own pubspec sets. It replaces the presence of .cargo/config.local.toml as the development signal, so a copy of the package directory no longer becomes a development build by accident. Both kinds of library register as uniffi:payjoin_ffi. uniffi-dart's bindings bind every function with @Native(assetId:), so they resolve by that id and the file name does not matter. A prebuilt-linux job runs the consumer program against the staged package. Pull requests build no release libraries, so the Linux test job places the host library its tests built under prebuilt/ and runs the same check. The reproducibility job's consumer now selects source, which it exists to test. Because the libraries travel inside the package, publish-dart keeps its place beside the other publish jobs and the release still needs one approval. It now also attests the staged package archive the release attaches, as the other registries attest theirs. --- .github/workflows/bindings-release.yml | 12 ++- .github/workflows/cron-daily-bindings.yml | 1 + .github/workflows/dart.yml | 42 ++++++++++- payjoin-ffi/CHANGELOG.md | 5 ++ payjoin-ffi/RELEASING.md | 10 +-- payjoin-ffi/VERIFYING.md | 18 ++++- payjoin-ffi/dart/.gitignore | 4 + payjoin-ffi/dart/.pubignore | 4 +- payjoin-ffi/dart/CONTRIBUTING.md | 35 +++++---- payjoin-ffi/dart/README.md | 45 ++++++++++-- payjoin-ffi/dart/contrib/prepare-publish.sh | 36 ++++++--- payjoin-ffi/dart/contrib/test-prebuilt.sh | 17 +++++ payjoin-ffi/dart/hook/build.dart | 81 +++++++++++++++++++-- payjoin-ffi/dart/pubspec.yaml | 5 ++ 14 files changed, 265 insertions(+), 50 deletions(-) create mode 100755 payjoin-ffi/dart/contrib/test-prebuilt.sh diff --git a/.github/workflows/bindings-release.yml b/.github/workflows/bindings-release.yml index f08db1863..02d2a3177 100644 --- a/.github/workflows/bindings-release.yml +++ b/.github/workflows/bindings-release.yml @@ -64,7 +64,7 @@ jobs: set -euo pipefail source contrib/release/crates.sh core="$(nix --quiet develop .#release -c bash -c 'source contrib/release/crates.sh && manifest_version payjoin')" - # Dart consumers build the bindings from source against it. + # Dart's source build mode builds the bindings against it. crate_published payjoin "$core" || { echo "::error::payjoin $core is not on crates.io; publish it before the bindings" exit 1 @@ -80,7 +80,7 @@ jobs: needs: [verify-tag, native] uses: $/.github/workflows/csharp.yml dart: - needs: verify-tag + needs: [verify-tag, native] uses: $/.github/workflows/dart.yml kotlin: needs: verify-tag @@ -224,7 +224,8 @@ jobs: runs-on: ubuntu-26.04 environment: release permissions: - id-token: write # OIDC: pub.dev automated publishing + id-token: write # OIDC: pub.dev automated publishing and actions/attest + attestations: write # actions/attest writes the attestation contents: read steps: - name: Checkout repo @@ -238,6 +239,11 @@ jobs: with: name: payjoin-dart-pub-package path: dist + - name: Attest build provenance + # A consumer runs: gh attestation verify payjoin-dart.tar.gz -R payjoin/rust-payjoin + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: dist/payjoin-dart.tar.gz - name: Extract the staged package run: | mkdir -p "$RUNNER_TEMP/payjoin-pub" diff --git a/.github/workflows/cron-daily-bindings.yml b/.github/workflows/cron-daily-bindings.yml index c68f0c07c..01c1f7bbf 100644 --- a/.github/workflows/cron-daily-bindings.yml +++ b/.github/workflows/cron-daily-bindings.yml @@ -22,6 +22,7 @@ jobs: needs: native uses: $/.github/workflows/csharp.yml dart: + needs: native uses: $/.github/workflows/dart.yml kotlin: uses: $/.github/workflows/kotlin.yml diff --git a/.github/workflows/dart.yml b/.github/workflows/dart.yml index c10fdcf50..e5b8e9ee1 100644 --- a/.github/workflows/dart.yml +++ b/.github/workflows/dart.yml @@ -39,6 +39,14 @@ jobs: uses: $/.github/actions/setup-nix - name: "Build and test" run: nix develop .#dart -c bash ./payjoin-ffi/dart/contrib/test.sh + # Pull requests build no release libraries, so the host library the + # tests just built stands in for one. + - name: Load the library in prebuilt mode + run: | + set -euo pipefail + mkdir -p payjoin-ffi/dart/prebuilt/x86_64-unknown-linux-gnu + cp target/debug/libpayjoin_ffi.so payjoin-ffi/dart/prebuilt/x86_64-unknown-linux-gnu/ + nix develop .#dart -c payjoin-ffi/dart/contrib/test-prebuilt.sh payjoin-ffi/dart build-dart-and-test-macos: name: "Build and test dart (macos)" @@ -80,6 +88,11 @@ jobs: lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Set up nix uses: $/.github/actions/setup-nix + - name: Download the native libraries + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/native - name: Generate production bindings and dry-run publish run: | # A release tag stages payjoin and payjoin-mailroom against @@ -90,7 +103,7 @@ jobs: if [[ $GITHUB_REF != refs/tags/* ]]; then flags+=(--core-from-revision) fi - nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh "${flags[@]}" "$RUNNER_TEMP/payjoin-pub" + nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh "${flags[@]}" "$RUNNER_TEMP/native" "$RUNNER_TEMP/payjoin-pub" # The release workflow publishes this exact stage, so it is archived # rather than regenerated in the publish job. - name: Archive the staged package @@ -152,6 +165,7 @@ jobs: rustup toolchain install 1.85.1 --profile minimal cp -a "$RUNNER_TEMP/stage" "$root/payjoin" cp -a payjoin-ffi/dart/contrib/consumer "$root/app" + printf '\nhooks:\n user_defines:\n payjoin:\n build: source\n' >>"$root/app/pubspec.yaml" (cd "$root/app" && dart pub get && dart run bin/main.dart) done # Cargo leaves a second copy under release/deps, which is not @@ -161,3 +175,29 @@ jobs: -path '*/x86_64-unknown-linux-gnu/release/libpayjoin_ffi_wrapper.so' } cmp "$(library first)" "$(library "second/with space")" + + # The staged package in its default prebuilt mode, with the libraries + # prepare-publish.sh packed into it. + prebuilt-linux: + name: "Load the prebuilt Linux library" + if: github.event_name != 'pull_request' + needs: verify-pub-package + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Set up nix + uses: $/.github/actions/setup-nix + - name: Download the staged package + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-dart-pub-package + path: dist + - name: Run the consumer in prebuilt mode + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/stage" + tar -xzf dist/payjoin-dart.tar.gz -C "$RUNNER_TEMP/stage" + nix develop .#dart -c ./payjoin-ffi/dart/contrib/test-prebuilt.sh "$RUNNER_TEMP/stage" diff --git a/payjoin-ffi/CHANGELOG.md b/payjoin-ffi/CHANGELOG.md index 83e34ca3c..a62807157 100644 --- a/payjoin-ffi/CHANGELOG.md +++ b/payjoin-ffi/CHANGELOG.md @@ -9,6 +9,11 @@ - The GitHub release now attaches every native library as `payjoin_ffi--.`, listed in `SHA256SUMS` and attested. `VERIFYING.md` explains how to check one or rebuild it. +- Dart: the package now ships the release's prebuilt native libraries, + and the build hook uses them instead of compiling Rust, so apps no + longer need a Rust toolchain or network access at build time. Set + `build: source` under `hooks: user_defines: payjoin:` in your app's + `pubspec.yaml` to keep compiling it. ## [0.25.1] diff --git a/payjoin-ffi/RELEASING.md b/payjoin-ffi/RELEASING.md index b46afb11b..ada1ecece 100644 --- a/payjoin-ffi/RELEASING.md +++ b/payjoin-ffi/RELEASING.md @@ -85,11 +85,11 @@ environment, and allow `payjoin-ffi-*` in the repository's tag rules. 5. The run attaches the wheels, npm tarball, NuGet package, staged Dart package, every native library as `payjoin_ffi--.`, and one `SHA256SUMS` covering them all to the GitHub release. The - wheels and the NuGet package carry those same native libraries, byte for - byte. Verify an install from each registry and check provenance with - `gh attestation verify FILE -R payjoin/rust-payjoin`. Optionally sign - `SHA256SUMS` locally and upload `SHA256SUMS.asc`; no signing key reaches - a runner. + wheels, the NuGet package and the Dart package carry those same native + libraries, byte for byte. Verify an install from each registry and + check provenance with `gh attestation verify FILE -R payjoin/rust-payjoin`. + Optionally sign `SHA256SUMS` locally and upload `SHA256SUMS.asc`; no + signing key reaches a runner. ## If a publish job fails diff --git a/payjoin-ffi/VERIFYING.md b/payjoin-ffi/VERIFYING.md index 8c75a5181..6cdf751bb 100644 --- a/payjoin-ffi/VERIFYING.md +++ b/payjoin-ffi/VERIFYING.md @@ -3,8 +3,8 @@ Every bindings release attaches each native library it ships to its [GitHub release](https://github.com/payjoin/rust-payjoin/releases) as `payjoin_ffi--.`, listed in `SHA256SUMS` and -attested by the release workflow. The Python wheels and the NuGet package -carry those same files, byte for byte. +attested by the release workflow. The Python wheels, the NuGet package and +the Dart package carry those same files, byte for byte. CI builds each library twice, on separate runners from different paths, and the release only goes ahead when both builds match. You can rebuild @@ -23,6 +23,20 @@ A maintainer may also upload `SHA256SUMS.asc`, a signature over `SHA256SUMS` made with a key in [`contrib/release/keys/`](../contrib/release/keys/). +## Check the Dart package + +The Dart package ships each library as `prebuilt//`, and +`dart pub get` unpacks it into the pub cache. Compare one with the line +for `payjoin_ffi--.` in `SHA256SUMS`: + +```shell +sha256sum ~/.pub-cache/hosted/pub.dev/payjoin-/prebuilt//* +``` + +Use `$PUB_CACHE` in place of `~/.pub-cache` if you set it. The release +also attests the staged package, `payjoin-dart.tar.gz`, which it +attaches. + ## Rebuild a library You need an x86_64 Linux machine with [nix](https://nixos.org/download/) diff --git a/payjoin-ffi/dart/.gitignore b/payjoin-ffi/dart/.gitignore index 3aa61ef2a..c30c7ee13 100644 --- a/payjoin-ffi/dart/.gitignore +++ b/payjoin-ffi/dart/.gitignore @@ -12,3 +12,7 @@ doc/api/ # Auto-generated bindings lib/payjoin.dart + +# Release libraries, which contrib/prepare-publish.sh copies into the +# staged package +prebuilt/ diff --git a/payjoin-ffi/dart/.pubignore b/payjoin-ffi/dart/.pubignore index c8c635895..2a1168cce 100644 --- a/payjoin-ffi/dart/.pubignore +++ b/payjoin-ffi/dart/.pubignore @@ -9,10 +9,12 @@ scripts/ .DS_Store -# Native libraries are platform specific and built per consumer +# Native libraries are platform specific and built per consumer, except +# the release's own, which prepare-publish.sh copies into prebuilt/ *.dylib *.so *.dll +!/prebuilt/** # Cargo build output native/target/ diff --git a/payjoin-ffi/dart/CONTRIBUTING.md b/payjoin-ffi/dart/CONTRIBUTING.md index f4f093465..f3310e03a 100644 --- a/payjoin-ffi/dart/CONTRIBUTING.md +++ b/payjoin-ffi/dart/CONTRIBUTING.md @@ -27,16 +27,19 @@ dart test ### The native build hook -`hook/build.dart` builds the `native/` wrapper crate for the consumer's -target. It takes the presence of `.cargo/config.local.toml` as the sign -that it is running inside this repository rather than from a published -archive, because `.pubignore` withholds `.cargo/` from that archive. In -the repository the overlay redirects `payjoin-ffi` to the workspace and -the hook enables `_test-utils`. From an archive the hook builds the -shipped `native/Cargo.lock` with `--locked`. - -This cannot be an environment variable. `hooks_runner` spawns build hooks -with an allowlisted environment (`HOME`, `PATH` and a few toolchain +`hook/build.dart` has three modes, chosen by the `build` user define in +the root package's `pubspec.yaml` (see the [README](README.md)): + +- `prebuilt`, the default, uses the release's library for the target + from `prebuilt//`, which only a staged package has. +- `source` builds the `native/` wrapper crate from the shipped + `native/Cargo.lock` with `--locked`. +- `workspace`, which this package's own `pubspec.yaml` sets, builds it + through the `.cargo/config.local.toml` overlay, which redirects + `payjoin-ffi` to the workspace, and enables `_test-utils`. + +The mode cannot be an environment variable. `hooks_runner` spawns build +hooks with an allowlisted environment (`HOME`, `PATH` and a few toolchain prefixes), so a project specific variable never reaches the hook. The hook also passes `--remap-path-prefix` for the package, output, Cargo @@ -88,11 +91,17 @@ with a publish dry run with `--core-from-revision`, which every run but a release tag uses since it may wrap core changes not yet on crates.io. That option takes them from the same revision and writes an `UNPUBLISHED_CORE` - marker, which `contrib/publish.sh` refuses. Both the publish dry run - and publication use the staged package. For local checks: + marker, which `contrib/publish.sh` refuses. The script also copies + the native workflow's `payjoin-native` artifact of the same run, the + libraries the release attaches, into `prebuilt//` for every + target but `universal2-apple-darwin`. Both the publish dry run and + publication use the staged package. For local checks, with the + libraries built under the + [native build contract](../../contrib/native/README.md) in + `/tmp/native//`: ```shell - nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh /tmp/payjoin-pub + nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh /tmp/native /tmp/payjoin-pub ``` 2. Follow the [shared bindings release instructions](../RELEASING.md). diff --git a/payjoin-ffi/dart/README.md b/payjoin-ffi/dart/README.md index 50a5ba489..bdfddfb8d 100644 --- a/payjoin-ffi/dart/README.md +++ b/payjoin-ffi/dart/README.md @@ -22,20 +22,49 @@ For OHTTP relay helpers: import 'package:payjoin/http.dart' as payjoin_http; ``` -### Native build +### Native library -The package compiles its Rust library on your machine through a Dart -build hook, with the toolchain named in `native/rust-toolchain.toml`. +A Dart build hook provides the Rust library. Choose how in your app's +`pubspec.yaml`: + +```yaml +hooks: + user_defines: + payjoin: + build: prebuilt # the default, or source +``` + +#### Prebuilt (default) + +The package ships the library the release built for each target, and +the hook bundles the one for yours. The build needs no Rust toolchain +and no network access. Your `pubspec.lock` pins the sha256 of the +package archive, and pub checks it on every download. Each library is +built twice by CI and attested by the release, and +[`VERIFYING.md`](https://github.com/payjoin/rust-payjoin/blob/master/payjoin-ffi/VERIFYING.md) +explains how to compare one with the GitHub release or rebuild it +yourself. + +- **No fallback.** A target the package has no library for fails the + build with a message that says how to switch to `build: source`. It + never compiles instead. +- **Targets.** Android (arm64, armv7, x86_64) from API 21, iOS 13 + devices and arm64 simulators, macOS 10.15 on x86_64 and 11 on arm64, + Linux with glibc 2.17 (arm64, x86_64), and Windows (arm64, x86_64). + The Intel iOS simulator needs `build: source`. + +#### Source + +The hook compiles the library on your machine with the toolchain named +in `native/rust-toolchain.toml`. - **Hosted releases are locked.** A version from pub.dev ships the `native/Cargo.lock` it was released with, and the hook builds it with `--locked`. Every release compiles the same dependency versions, whenever and wherever you build it. -- **Git dependencies are development builds.** A `git:` dependency on - this repository checks out the monorepo layout, which the hook treats - as a development build: it builds `payjoin-ffi` from the checkout, - enables test-only helpers and resolves dependencies afresh, without a - lockfile. Use a hosted release in production. +- **Use a hosted release.** Only the package on pub.dev has the + generated bindings, the prebuilt libraries and the pinned lockfile, so + a `git:` dependency on this repository builds in neither mode. - **Cargo config `rustflags` do not apply.** The hook sets `CARGO_ENCODED_RUSTFLAGS` to strip build paths from the library, and Cargo then ignores `build.rustflags` and `target.*.rustflags` from any diff --git a/payjoin-ffi/dart/contrib/prepare-publish.sh b/payjoin-ffi/dart/contrib/prepare-publish.sh index a6511a411..a2c97a1df 100755 --- a/payjoin-ffi/dart/contrib/prepare-publish.sh +++ b/payjoin-ffi/dart/contrib/prepare-publish.sh @@ -5,18 +5,20 @@ if [[ ${1:-} == --core-from-revision ]]; then core_from_revision=true shift fi -[ "$#" -eq 1 ] || { - echo "usage: prepare-publish.sh [--core-from-revision] " >&2 +[ "$#" -eq 2 ] || { + echo "usage: prepare-publish.sh [--core-from-revision] " >&2 exit 1 } # Resolve before changing directories; the destination must not exist. -stage="$(realpath -m "$1")" +native="$(realpath "$1")" +stage="$(realpath -m "$2")" # Prepare the package for publishing to pub.dev and verify the archive with -# a dry run. The archive ships Dart source plus the native/ wrapper crate; -# consumers compile the Rust themselves via hook/build.dart, so no binaries -# are packed here. Publishing itself stays a separate step so this script -# can run anywhere, including on pull requests. +# a dry run. The archive ships Dart source, the native/ wrapper crate for +# source builds, and under prebuilt// the libraries in +# , the native workflow's payjoin-native artifact, which +# hook/build.dart uses by default. Publishing itself stays a separate step +# so this script can run anywhere. # Build against the maintained lockfile instead of resolving the dependency # graph fresh on every run. use_lockfile copies Cargo-recent.lock into place @@ -33,14 +35,28 @@ PAYJOIN_FFI_FEATURES="" bash ./scripts/generate_bindings.sh revision="$(git rev-parse HEAD)" mkdir "$stage" -# Include generated bindings while excluding development Cargo overlays, -# then pin the native crate to the checkout that generated them. -rsync -a --exclude=.git --exclude-from=.pubignore ./ "$stage/" +# Include generated bindings while excluding development Cargo overlays and +# any local prebuilt/, then pin the native crate to the checkout that +# generated them. +rsync -a --exclude=.git --exclude=/prebuilt/ --exclude-from=.pubignore ./ "$stage/" sed -i "s/rev = \"[0-9a-f]*\"/rev = \"$revision\"/" "$stage/native/Cargo.toml" grep -Fq "rev = \"$revision\"" "$stage/native/Cargo.toml" || { echo "Could not pin the staged native crate to $revision" >&2 exit 1 } +# Every target the hook maps, which is all of them but the universal macOS +# library. +for lib in "$native"/*/*; do + target="$(basename "$(dirname "$lib")")" + [[ $target == universal2-apple-darwin ]] && continue + mkdir -p "$stage/prebuilt/$target" + cp "$lib" "$stage/prebuilt/$target/" +done +count="$(find "$stage/prebuilt" -type f | wc -l)" +[ "$count" -eq 11 ] || { + echo "Expected 11 prebuilt libraries in $native, found $count" >&2 + exit 1 +} # payjoin-ffi and payjoin-test-utils come from the pinned revision, but the # release crates they require, payjoin and payjoin-mailroom, come from diff --git a/payjoin-ffi/dart/contrib/test-prebuilt.sh b/payjoin-ffi/dart/contrib/test-prebuilt.sh new file mode 100755 index 000000000..bf3506f6d --- /dev/null +++ b/payjoin-ffi/dart/contrib/test-prebuilt.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run the consumer program against in the default prebuilt +# mode, so the hook must find the library for this host under prebuilt/. +set -euo pipefail +[ "$#" -eq 1 ] || { + echo "usage: test-prebuilt.sh " >&2 + exit 1 +} +package="$(realpath "$1")" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +cp -a "$(dirname "${BASH_SOURCE[0]}")/consumer" "$work/app" +ln -s "$package" "$work/payjoin" +cd "$work/app" +dart pub get +dart run bin/main.dart diff --git a/payjoin-ffi/dart/hook/build.dart b/payjoin-ffi/dart/hook/build.dart index 04a0129b1..a58e2338b 100644 --- a/payjoin-ffi/dart/hook/build.dart +++ b/payjoin-ffi/dart/hook/build.dart @@ -4,8 +4,25 @@ import 'package:code_assets/code_assets.dart'; import 'package:hooks/hooks.dart'; import 'package:native_toolchain_rust/native_toolchain_rust.dart'; +const _assetName = 'uniffi:payjoin_ffi'; + void main(List args) async { await build(args, (input, output) async { + final mode = input.userDefines['build'] ?? 'prebuilt'; + if (mode == 'prebuilt') { + if (input.config.buildCodeAssets) { + _prebuilt(input, output); + } + return; + } + if (mode != 'source' && mode != 'workspace') { + throw BuildError( + message: + 'payjoin: unknown build mode "$mode". Set ' + 'hooks.user_defines.payjoin.build to prebuilt or source.', + ); + } + final environment = Platform.environment; final separator = Platform.pathSeparator; String? nonEmpty(String? value) => @@ -47,18 +64,16 @@ void main(List args) async { '-Clink-arg=-Wl,-z,max-page-size=16384', ].join('\x1f'); - // The path overlay doubles as the development marker. `.pubignore` - // withholds `.cargo/` from the archive, so the file only exists in a - // repository checkout. It cannot be an environment variable: hooks_runner - // spawns build hooks with an allowlisted environment, and a project - // specific name never survives that filter. + // The repository's own pubspec selects the workspace build, which + // builds payjoin-ffi from the checkout through this path overlay. + // `.pubignore` withholds `.cargo/` from the archive. final localCargoConfig = File.fromUri( input.packageRoot.resolve('.cargo/config.local.toml'), ); - final isWorkspaceBuild = localCargoConfig.existsSync(); + final isWorkspaceBuild = mode == 'workspace'; await RustBuilder( - assetName: 'uniffi:payjoin_ffi', + assetName: _assetName, features: isWorkspaceBuild ? const ['_test-utils'] : const [], extraCargoBuildArgs: [ // Cargo does not discover this file name on its own, and it @@ -82,3 +97,55 @@ void main(List args) async { ).run(input: input, output: output); }); } + +/// Registers the library the release built for this target, which +/// prepare-publish.sh copied into the package under prebuilt/. +void _prebuilt(BuildInput input, BuildOutputBuilder output) { + final code = input.config.code; + final target = switch ((code.targetOS, code.targetArchitecture)) { + (OS.android, Architecture.arm64) => 'aarch64-linux-android', + (OS.android, Architecture.arm) => 'armv7-linux-androideabi', + (OS.android, Architecture.x64) => 'x86_64-linux-android', + (OS.iOS, Architecture.arm64) => + code.iOS.targetSdk == IOSSdk.iPhoneSimulator + ? 'aarch64-apple-ios-sim' + : 'aarch64-apple-ios', + (OS.macOS, Architecture.arm64) => 'aarch64-apple-darwin', + (OS.macOS, Architecture.x64) => 'x86_64-apple-darwin', + (OS.linux, Architecture.arm64) => 'aarch64-unknown-linux-gnu', + (OS.linux, Architecture.x64) => 'x86_64-unknown-linux-gnu', + (OS.windows, Architecture.arm64) => 'aarch64-pc-windows-msvc', + (OS.windows, Architecture.x64) => 'x86_64-pc-windows-msvc', + (final os, final arch) => '$os on $arch', + }; + final file = File.fromUri( + input.packageRoot.resolve( + 'prebuilt/$target/${code.targetOS.dylibFileName('payjoin_ffi')}', + ), + ); + if (!file.existsSync()) { + _fail('this copy of the package has no library for $target'); + } + output.dependencies.add(file.uri); + + output.assets.code.add( + CodeAsset( + package: input.packageName, + name: _assetName, + linkMode: DynamicLoadingBundled(), + file: file.uri, + ), + ); +} + +Never _fail(String reason) => throw BuildError( + message: + 'payjoin: cannot use the prebuilt native library: $reason.\n' + 'To compile it from source with your own Rust toolchain instead, add ' + 'this to the pubspec.yaml of your app:\n' + '\n' + 'hooks:\n' + ' user_defines:\n' + ' payjoin:\n' + ' build: source\n', +); diff --git a/payjoin-ffi/dart/pubspec.yaml b/payjoin-ffi/dart/pubspec.yaml index 74bb50ec3..b4071de10 100644 --- a/payjoin-ffi/dart/pubspec.yaml +++ b/payjoin-ffi/dart/pubspec.yaml @@ -18,3 +18,8 @@ dev_dependencies: hooks: build: hook/build.dart + # Applies only where this package is the root, that is in the repository. + # Consumers choose their own mode; see README.md. + user_defines: + payjoin: + build: workspace