diff --git a/.github/workflows/bindings-release.yml b/.github/workflows/bindings-release.yml index 5b9605d81..02d2a3177 100644 --- a/.github/workflows/bindings-release.yml +++ b/.github/workflows/bindings-release.yml @@ -1,10 +1,11 @@ name: Release language bindings # Pushing a signed `payjoin-ffi-` tag verifies the tag and the -# shared binding version, runs every language's build, package and smoke -# checks at that commit, then publishes Python, JavaScript, C#, and Dart -# through the `release` environment (required reviewer) with keyless OIDC -# and cuts one GitHub release holding every package. +# shared binding version, builds the native libraries once, runs every +# language's build, package and smoke checks at that commit, then +# publishes Python, JavaScript, C#, and Dart through the `release` +# environment (required reviewer) with keyless OIDC and cuts one GitHub +# release holding every package and native library. # # Each registry has its own publish job. All four wait on every language # build, so they reach the environment gate together and one approval @@ -63,31 +64,34 @@ jobs: set -euo pipefail source contrib/release/crates.sh core="$(nix --quiet develop .#release -c bash -c 'source contrib/release/crates.sh && manifest_version payjoin')" - # Dart consumers build the bindings from source against it. + # Dart's source build mode builds the bindings against it. crate_published payjoin "$core" || { echo "::error::payjoin $core is not on crates.io; publish it before the bindings" exit 1 } python: - needs: verify-tag + needs: [verify-tag, native] uses: $/.github/workflows/python.yml javascript: needs: verify-tag uses: $/.github/workflows/javascript.yml csharp: - needs: verify-tag + needs: [verify-tag, native] uses: $/.github/workflows/csharp.yml dart: - needs: verify-tag + needs: [verify-tag, native] uses: $/.github/workflows/dart.yml kotlin: needs: verify-tag uses: $/.github/workflows/kotlin.yml + native: + needs: verify-tag + uses: $/.github/workflows/native.yml publish-python: name: Publish to PyPI - needs: [python, javascript, csharp, dart, kotlin] + needs: [python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -121,7 +125,7 @@ jobs: publish-javascript: name: Publish to npmjs.com - needs: [verify-tag, python, javascript, csharp, dart, kotlin] + needs: [verify-tag, python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -165,7 +169,7 @@ jobs: publish-csharp: name: Publish to nuget.org - needs: [python, javascript, csharp, dart, kotlin] + needs: [python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: @@ -216,11 +220,12 @@ jobs: publish-dart: name: Publish to pub.dev - needs: [python, javascript, csharp, dart, kotlin] + needs: [python, javascript, csharp, dart, kotlin, native] runs-on: ubuntu-26.04 environment: release permissions: - id-token: write # OIDC: pub.dev automated publishing + id-token: write # OIDC: pub.dev automated publishing and actions/attest + attestations: write # actions/attest writes the attestation contents: read steps: - name: Checkout repo @@ -234,6 +239,11 @@ jobs: with: name: payjoin-dart-pub-package path: dist + - name: Attest build provenance + # A consumer runs: gh attestation verify payjoin-dart.tar.gz -R payjoin/rust-payjoin + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: dist/payjoin-dart.tar.gz - name: Extract the staged package run: | mkdir -p "$RUNNER_TEMP/payjoin-pub" @@ -254,6 +264,8 @@ jobs: runs-on: ubuntu-26.04 permissions: contents: write # create the Release for this tag and upload assets + id-token: write # OIDC: used by actions/attest + attestations: write # actions/attest writes the attestation env: VERSION: ${{ needs.verify-tag.outputs.version }} steps: @@ -267,6 +279,15 @@ jobs: pattern: payjoin-* merge-multiple: true path: dist + - name: Name the native libraries for the release + # payjoin-native lands as one directory per target. + run: | + set -euo pipefail + for lib in dist/*/*; do + target=$(basename "$(dirname "$lib")") + mv "$lib" "dist/payjoin_ffi-$VERSION-$target.${lib##*.}" + rmdir "$(dirname "$lib")" + done - name: Check the asset set is complete run: | set -euo pipefail @@ -274,19 +295,20 @@ jobs: wheels=(dist/*.whl) npm=(dist/*.tgz) nuget=(dist/*.nupkg) - if [[ ${#wheels[@]} != 2 || ${#npm[@]} != 1 || ${#nuget[@]} != 1 || ! -f dist/payjoin-dart.tar.gz ]]; then - echo "::error::expected two wheels, one npm tarball, one NuGet package, and the Dart archive in dist/" + native=(dist/payjoin_ffi-*) + if [[ ${#wheels[@]} != 2 || ${#npm[@]} != 1 || ${#nuget[@]} != 1 || ! -f dist/payjoin-dart.tar.gz || ${#native[@]} != 12 ]]; then + echo "::error::expected two wheels, one npm tarball, one NuGet package, the Dart archive and 12 native libraries in dist/" ls dist exit 1 fi + - name: Attest build provenance of the native libraries + # A consumer runs: gh attestation verify -R payjoin/rust-payjoin + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: dist/payjoin_ffi-* - name: Generate SHA256SUMS - # The NuGet package's native libraries are listed individually so - # each can be verified on its own. A maintainer's GPG signature over - # SHA256SUMS is made locally and uploaded as SHA256SUMS.asc; no - # signing key reaches a runner. - env: - ARCHIVE_GLOBS: "*.nupkg" - MEMBER_GLOBS: runtimes/*/native/* + # A maintainer's GPG signature over SHA256SUMS is made locally and + # uploaded as SHA256SUMS.asc; no signing key reaches a runner. run: | ./contrib/release/generate-sha256sums.sh dist cat dist/SHA256SUMS diff --git a/.github/workflows/cron-daily-bindings.yml b/.github/workflows/cron-daily-bindings.yml index 5da0b4e0e..01c1f7bbf 100644 --- a/.github/workflows/cron-daily-bindings.yml +++ b/.github/workflows/cron-daily-bindings.yml @@ -14,12 +14,17 @@ permissions: jobs: python: + needs: native uses: $/.github/workflows/python.yml javascript: uses: $/.github/workflows/javascript.yml csharp: + needs: native uses: $/.github/workflows/csharp.yml dart: + needs: native uses: $/.github/workflows/dart.yml kotlin: uses: $/.github/workflows/kotlin.yml + native: + uses: $/.github/workflows/native.yml diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 0d8118188..0945bba76 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -70,7 +70,7 @@ jobs: if: github.event_name != 'pull_request' runs-on: windows-latest env: - RUSTUP_TOOLCHAIN: 1.85 + RUSTUP_TOOLCHAIN: 1.85.0 defaults: run: working-directory: payjoin-ffi/csharp @@ -106,106 +106,24 @@ jobs: - name: Run tests run: dotnet test Payjoin.Tests.csproj --logger "console;verbosity=minimal" - build-nuget-native: - # Every RID cross-compiles from Linux: cargo-zigbuild links the Linux targets - # against a pinned glibc floor and the macOS targets against zig's bundled - # libSystem stubs; cargo-xwin links MSVC-ABI Windows targets against the - # Microsoft CRT/SDK. The per-RID smoke jobs verify each produced asset on - # real hardware. - name: "Build NuGet native asset" - if: github.event_name != 'pull_request' - runs-on: ubuntu-26.04 - env: - # Must match the dtolnay/rust-toolchain version exactly: the cross-target - # std components are installed for that toolchain, and a partial version - # here selects a different rustup toolchain entry without them (E0463). - RUSTUP_TOOLCHAIN: 1.85.0 - defaults: - run: - working-directory: payjoin-ffi/csharp - strategy: - # Let every RID report its own result; a failure in one target should not - # cancel the signal from the others. - fail-fast: false - matrix: - include: - - rid: linux-arm64 - target: aarch64-unknown-linux-gnu - - rid: linux-x64 - target: x86_64-unknown-linux-gnu - - rid: osx-arm64 - target: aarch64-apple-darwin - - rid: osx-x64 - target: x86_64-apple-darwin - - rid: win-arm64 - target: aarch64-pc-windows-msvc - - rid: win-x64 - target: x86_64-pc-windows-msvc - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # unneeded: no git auth beyond checkout - - - name: Install Rust 1.85.0 - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 - with: - toolchain: 1.85.0 - targets: ${{ matrix.target }} - - - name: Use cache - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - - - name: Install cross toolchain - # Version- and hash-pinned: these tools produce the shipped binaries. - run: pip install --require-hashes -r scripts/cross-requirements.txt - - - name: Install MSVC toolchain frontends (clang-cl, llvm-lib) - if: startsWith(matrix.rid, 'win-') - # cargo-xwin compiles C dependencies with clang-cl and archives them with - # llvm-lib. The runner image ships LLVM with versioned binary names only, - # so expose the unversioned names cc-rs looks for. - run: | - for tool in clang-cl llvm-lib; do - if command -v "$tool" >/dev/null; then - continue - fi - latest=$(ls -1 /usr/bin/"$tool"-* 2>/dev/null | sort -V | tail -n 1) - if [ -z "$latest" ]; then - sudo apt-get update -q - sudo apt-get install -y -q llvm clang - latest=$(ls -1 /usr/bin/"$tool"-* 2>/dev/null | sort -V | tail -n 1) - fi - if [ -z "$latest" ]; then - echo "$tool not found on the runner" >&2 - exit 1 - fi - sudo ln -s "$latest" /usr/local/bin/"$tool" - done - command -v clang-cl llvm-lib - - - name: Build release native asset - run: | - PAYJOIN_FFI_CROSS=1 PAYJOIN_FFI_RID=${{ matrix.rid }} bash ./scripts/build_nuget_native.sh - - - name: Upload NuGet native asset - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ matrix.rid }} - path: payjoin-ffi/csharp/artifacts/runtimes/${{ matrix.rid }}/native/* - if-no-files-found: error - pack-nuget: + # Every RID's native asset comes from the native workflow's checked + # double build in the same run. The per-RID smoke jobs verify each one + # on real hardware. name: "Pack C# NuGet package" if: github.event_name != 'pull_request' runs-on: ubuntu-26.04 - needs: build-nuget-native env: # Exact version so the packaged bindings build with the toolchain the - # action installs, not an implicitly-installed latest 1.85.x. - RUSTUP_TOOLCHAIN: 1.85.0 + # action installs, the release toolchain of the native build contract. + RUSTUP_TOOLCHAIN: 1.85.1 + RIDS: >- + linux-arm64=aarch64-unknown-linux-gnu + linux-x64=x86_64-unknown-linux-gnu + osx-arm64=aarch64-apple-darwin + osx-x64=x86_64-apple-darwin + win-arm64=aarch64-pc-windows-msvc + win-x64=x86_64-pc-windows-msvc defaults: run: working-directory: payjoin-ffi/csharp @@ -215,10 +133,10 @@ jobs: with: persist-credentials: false # unneeded: no git auth beyond checkout - - name: Install Rust 1.85.0 + - name: Install Rust 1.85.1 uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 with: - toolchain: 1.85.0 + toolchain: 1.85.1 - name: Use cache uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 @@ -230,16 +148,19 @@ jobs: with: dotnet-version: "10.0.x" - - name: Download native assets + - name: Download the native libraries uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - path: payjoin-ffi/csharp/artifacts/downloaded + name: payjoin-native + path: ${{ runner.temp }}/native - name: Assemble RID assets run: | - for rid in linux-arm64 linux-x64 osx-arm64 osx-x64 win-arm64 win-x64; do + set -euo pipefail + for pair in $RIDS; do + rid=${pair%%=*} mkdir -p "artifacts/runtimes/$rid/native" - cp "artifacts/downloaded/$rid"/* "artifacts/runtimes/$rid/native/" + cp "$RUNNER_TEMP/native/${pair#*=}"/* "artifacts/runtimes/$rid/native/" done - name: Generate production bindings @@ -249,6 +170,17 @@ jobs: - name: Pack NuGet package run: dotnet pack Payjoin.csproj --configuration Release --output artifacts/packages + # The release lists each library on its own, so the package must carry + # them unchanged. + - name: Check the packed libraries + run: | + set -euo pipefail + for pair in $RIDS; do + for lib in "$RUNNER_TEMP/native/${pair#*=}"/*; do + unzip -p artifacts/packages/*.nupkg "runtimes/${pair%%=*}/native/$(basename "$lib")" | cmp - "$lib" + done + done + - name: Upload NuGet package uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/.github/workflows/dart.yml b/.github/workflows/dart.yml index c10fdcf50..e5b8e9ee1 100644 --- a/.github/workflows/dart.yml +++ b/.github/workflows/dart.yml @@ -39,6 +39,14 @@ jobs: uses: $/.github/actions/setup-nix - name: "Build and test" run: nix develop .#dart -c bash ./payjoin-ffi/dart/contrib/test.sh + # Pull requests build no release libraries, so the host library the + # tests just built stands in for one. + - name: Load the library in prebuilt mode + run: | + set -euo pipefail + mkdir -p payjoin-ffi/dart/prebuilt/x86_64-unknown-linux-gnu + cp target/debug/libpayjoin_ffi.so payjoin-ffi/dart/prebuilt/x86_64-unknown-linux-gnu/ + nix develop .#dart -c payjoin-ffi/dart/contrib/test-prebuilt.sh payjoin-ffi/dart build-dart-and-test-macos: name: "Build and test dart (macos)" @@ -80,6 +88,11 @@ jobs: lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Set up nix uses: $/.github/actions/setup-nix + - name: Download the native libraries + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/native - name: Generate production bindings and dry-run publish run: | # A release tag stages payjoin and payjoin-mailroom against @@ -90,7 +103,7 @@ jobs: if [[ $GITHUB_REF != refs/tags/* ]]; then flags+=(--core-from-revision) fi - nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh "${flags[@]}" "$RUNNER_TEMP/payjoin-pub" + nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh "${flags[@]}" "$RUNNER_TEMP/native" "$RUNNER_TEMP/payjoin-pub" # The release workflow publishes this exact stage, so it is archived # rather than regenerated in the publish job. - name: Archive the staged package @@ -152,6 +165,7 @@ jobs: rustup toolchain install 1.85.1 --profile minimal cp -a "$RUNNER_TEMP/stage" "$root/payjoin" cp -a payjoin-ffi/dart/contrib/consumer "$root/app" + printf '\nhooks:\n user_defines:\n payjoin:\n build: source\n' >>"$root/app/pubspec.yaml" (cd "$root/app" && dart pub get && dart run bin/main.dart) done # Cargo leaves a second copy under release/deps, which is not @@ -161,3 +175,29 @@ jobs: -path '*/x86_64-unknown-linux-gnu/release/libpayjoin_ffi_wrapper.so' } cmp "$(library first)" "$(library "second/with space")" + + # The staged package in its default prebuilt mode, with the libraries + # prepare-publish.sh packed into it. + prebuilt-linux: + name: "Load the prebuilt Linux library" + if: github.event_name != 'pull_request' + needs: verify-pub-package + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Set up nix + uses: $/.github/actions/setup-nix + - name: Download the staged package + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-dart-pub-package + path: dist + - name: Run the consumer in prebuilt mode + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/stage" + tar -xzf dist/payjoin-dart.tar.gz -C "$RUNNER_TEMP/stage" + nix develop .#dart -c ./payjoin-ffi/dart/contrib/test-prebuilt.sh "$RUNNER_TEMP/stage" diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml new file mode 100644 index 000000000..4c46af493 --- /dev/null +++ b/.github/workflows/native.yml @@ -0,0 +1,187 @@ +name: Native libraries + +# Builds the payjoin-ffi native library for each target twice, on separate +# runners from different paths, and fails unless both builds are +# byte-identical. The libraries only ever come from Linux. The macOS job +# checks the Apple ones against Xcode and produces nothing. + +on: + workflow_call: + +permissions: + contents: read + +jobs: + build: + name: Build ${{ matrix.target }} (${{ matrix.build }}) + runs-on: ubuntu-26.04 + timeout-minutes: 60 + strategy: + matrix: + target: + - aarch64-apple-ios + - aarch64-apple-ios-sim + - aarch64-unknown-linux-gnu + - x86_64-unknown-linux-gnu + - aarch64-apple-darwin + - x86_64-apple-darwin + - aarch64-pc-windows-msvc + - x86_64-pc-windows-msvc + - aarch64-linux-android + - armv7-linux-androideabi + - x86_64-linux-android + build: [first, second] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Set up nix + uses: $/.github/actions/setup-nix + - name: Build + env: + TARGET: ${{ matrix.target }} + BUILD: ${{ matrix.build }} + run: | + set -euo pipefail + # The second build runs from other paths, with a space in them, so + # any path that leaks into the library breaks the match. Its own + # home moves the caches the toolchain keeps there, such as zig's. + repo=$GITHUB_WORKSPACE + if [[ $BUILD == second ]]; then + repo="$RUNNER_TEMP/second build/payjoin" + mkdir -p "$repo" + cp -R . "$repo" + export HOME="$RUNNER_TEMP/second build/home" CARGO_HOME="$RUNNER_TEMP/second build/cargo" + fi + shell=native + if [[ $TARGET == *-android* ]]; then + shell=native-android + fi + nix develop ".#$shell" -c "$repo/contrib/native/build.sh" "$TARGET" "$RUNNER_TEMP/out" + - name: Upload the library + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: native-${{ matrix.build }}-${{ matrix.target }} + path: ${{ runner.temp }}/out + if-no-files-found: error + + compare: + name: Compare the two builds + needs: build + runs-on: ubuntu-26.04 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Set up nix + uses: $/.github/actions/setup-nix + - name: Download the first build + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-first-* + merge-multiple: true + path: ${{ runner.temp }}/first + - name: Download the second build + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: native-second-* + merge-multiple: true + path: ${{ runner.temp }}/second + - name: Compare + working-directory: ${{ runner.temp }} + run: | + set -euo pipefail + (cd first && sha256sum -- */*) | tee SHA256SUMS + (cd second && sha256sum -c ../SHA256SUMS) + # Both slices just matched, so the file derived from them needs no + # second build of its own. + - name: Derive the universal2 macOS library + run: | + set -euo pipefail + dir="$RUNNER_TEMP/first" + mkdir -p "$dir/universal2-apple-darwin" + nix shell --inputs-from . nixpkgs-unstable#llvmPackages_21.llvm -c llvm-lipo -create \ + "$dir/aarch64-apple-darwin/libpayjoin_ffi.dylib" \ + "$dir/x86_64-apple-darwin/libpayjoin_ffi.dylib" \ + -output "$dir/universal2-apple-darwin/libpayjoin_ffi.dylib" + - name: Upload the libraries + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/first + if-no-files-found: error + + verify-apple: + name: Verify the Apple libraries on macOS + needs: compare + runs-on: macos-latest + timeout-minutes: 60 + env: + RUSTUP_TOOLCHAIN: 1.85.0 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Download the libraries + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/native + - name: Check the imports against Xcode's SDKs + working-directory: ${{ runner.temp }}/native + run: | + set -euo pipefail + # Apple's linker resolves every symbol a library imports against + # the SDK of the platform it runs on, and fails on any it lacks. + check() { + local triple=$1 target=$2 sdk=$3 + nm -u -j "$triple/libpayjoin_ffi.dylib" | sed 's/^/-Wl,-u,/' >"$triple.args" + xcrun --sdk "$sdk" clang -target "$target" -dynamiclib -x c /dev/null \ + -liconv @"$triple.args" -o "$RUNNER_TEMP/$triple-imports.dylib" + } + check aarch64-apple-ios arm64-apple-ios13.0 iphoneos + check aarch64-apple-ios-sim arm64-apple-ios13.0-simulator iphonesimulator + check aarch64-apple-darwin arm64-apple-macos11.0 macosx + check x86_64-apple-darwin x86_64-apple-macos10.15 macosx + - name: Rename and sign each library the way Flutter embeds it + run: | + set -euo pipefail + lipo "$RUNNER_TEMP/native/universal2-apple-darwin/libpayjoin_ffi.dylib" -verify_arch arm64 x86_64 + for lib in "$RUNNER_TEMP"/native/*/libpayjoin_ffi.dylib; do + cp "$lib" "$RUNNER_TEMP/payjoin_ffi" + install_name_tool -id @rpath/payjoin_ffi.framework/payjoin_ffi "$RUNNER_TEMP/payjoin_ffi" + codesign --force --sign - "$RUNNER_TEMP/payjoin_ffi" + codesign --verify --verbose "$RUNNER_TEMP/payjoin_ffi" + done + - name: "Install Rust 1.85.0" + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 + with: + toolchain: 1.85.0 + - name: Set up Flutter + uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2.23.0 + with: + channel: stable + flutter-version: 3.44.9 + - name: Generate the Dart bindings + run: | + set -euo pipefail + cp Cargo-recent.lock Cargo.lock + cd payjoin-ffi/dart + # The library carries no features, so neither may the bindings. + PAYJOIN_FFI_FEATURES='' bash scripts/generate_bindings.sh + - name: Run a smoke test in the iOS simulator + working-directory: payjoin-ffi/dart + run: | + set -euo pipefail + # Bundle the libraries built on Linux instead of building one. + cp contrib/ios-simulator/prebuilt_hook.dart hook/build.dart + cp -R "$RUNNER_TEMP/native" prebuilt + cd contrib/ios-simulator + flutter create --platforms ios . + udid=$(xcrun simctl list devices available --json | + jq -r '[.devices[][] | select(.name | startswith("iPhone"))][0].udid') + xcrun simctl boot "$udid" + flutter test integration_test -d "$udid" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c75efb35b..ebe3bb698 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -68,15 +68,20 @@ jobs: build-wheel: # The wheel bundles a prebuilt native library, so each supported # platform packs its own wheel: x86_64 on Linux (retagged manylinux by - # auditwheel) and a fat universal2 dylib for macOS. Both wheels build - # on a Linux host. The smoke jobs below verify each wheel on real hardware. + # auditwheel) and a fat universal2 dylib for macOS. Both libraries come + # from the native workflow's checked double build in the same run. The + # smoke jobs below verify each wheel on real hardware. name: "Build wheel" if: github.event_name != 'pull_request' runs-on: ubuntu-26.04 strategy: fail-fast: false matrix: - platform: [linux-x64, macos-universal2] + include: + - platform: linux-x64 + library: x86_64-unknown-linux-gnu/libpayjoin_ffi.so + - platform: macos-universal2 + library: universal2-apple-darwin/libpayjoin_ffi.dylib steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -88,10 +93,16 @@ jobs: lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Set up nix uses: $/.github/actions/setup-nix + - name: Download the native libraries + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-native + path: ${{ runner.temp }}/native - name: Build the wheel - run: nix develop .#python --command bash ./payjoin-ffi/python/contrib/build-wheel.sh env: PAYJOIN_WHEEL_PLATFORM: ${{ matrix.platform }} + LIBRARY: ${{ runner.temp }}/native/${{ matrix.library }} + run: nix develop .#python --command bash ./payjoin-ffi/python/contrib/build-wheel.sh "$LIBRARY" - name: Upload wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/contrib/native/README.md b/contrib/native/README.md new file mode 100644 index 000000000..e94e46f4f --- /dev/null +++ b/contrib/native/README.md @@ -0,0 +1,40 @@ +# Native build contract + +The payjoin-ffi native libraries are built on Linux, by one script, in one +devshell. The language bindings move onto these libraries one at a time. + +```sh +nix develop .#native -c contrib/native/build.sh +``` + +The Android targets need the NDK, which only `.#native-android` provides. + +The library lands in `//`. The +[native workflow](../../.github/workflows/native.yml) builds every target +twice, on separate runners from different paths, and fails unless both +builds are byte-identical. A macOS runner checks the Apple libraries +against Xcode and never produces any. + +| Invariant | Rule | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Toolchain | Rust 1.85.1. CI keeps testing the 1.85.0 MSRV. | +| Lockfile | `Cargo-recent.lock`, built `--locked` | +| Features | None. Bindings are generated separately, so the library is the same for every language. | +| Paths | The repository is remapped to `/payjoin`, the target dir to `/payjoin/target`, the cargo home to `/cargo`, the rustup home to `/rustup`, the NDK to `/ndk` and the Windows SDK to `/xwin`, for rustc and the C compiler | +| Profile | Cargo's release defaults | +| Linux | `cargo zigbuild` against glibc 2.17 | +| macOS | zig through `apple-cc.sh` against the stub SDK, deployment target 10.15 on x86_64 and 11.0 on arm64, Flutter's floor and the first macOS on Apple silicon. `universal2-apple-darwin` is the two slices combined with `llvm-lipo`. | +| iOS | zig through `apple-cc.sh` against the stub SDK, minimum iOS 13.0. Device and arm64 simulator only. | +| Windows | `xwin` fetches MSVC's CRT 14.44.17.14 and Windows SDK 10.0.26100 from the Visual Studio 17.14.37824.6 manifest pinned in `vs-channel.json`. LLVM 21's clang-cl, llvm-lib and lld-link build against them, linked with `/Brepro` and without a PDB. | +| Android | NDK 29.0.14206865 at API 21, 16 KB aligned on 64-bit by the NDK's default | + +The Apple dylibs are linked with an `@rpath/libpayjoin_ffi.dylib` install +name, `-headerpad_max_install_names` for Flutter's `install_name_tool`, and +`-S`, which keeps object paths and mtimes out of the stabs. +`apple-stubs.sh` generates the stub SDK from the macOS `libSystem.tbd` and +Darwin headers that zig ships, so no Apple SDK file is used or committed. + +`x86_64-apple-ios`, the Intel simulator, is not built. Rust 1.85.1's std +for it targets iOS 10.0, so its objects carry `LC_VERSION_MIN_IPHONEOS`, +which zig 0.16 reads as a device platform and rejects for a simulator +link. diff --git a/contrib/native/apple-cc.sh b/contrib/native/apple-cc.sh new file mode 100755 index 000000000..0d6176e20 --- /dev/null +++ b/contrib/native/apple-cc.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Compiler and linker for the Apple targets: zig for $APPLE_CC_TARGET +# against the stub SDK in $SDKROOT. The target flags rustc and cc-rs pass +# are dropped in favor of those. +set -euo pipefail + +args=() +while (($#)); do + case $1 in + -target | -arch | -isysroot) shift ;; + --target=* | -mmacosx-version-min=* | -mios-version-min=* | -mios-simulator-version-min=* | -miphoneos-version-min=*) ;; + *) args+=("$1") ;; + esac + shift +done + +# -headerpad_max_install_names leaves room for Flutter's install_name_tool, +# and -S keeps zig from writing object paths and mtimes into stabs. +exec zig cc -target "$APPLE_CC_TARGET" --sysroot "$SDKROOT" \ + -isystem "$SDKROOT/usr/include" -L/usr/lib \ + -Wl,-headerpad_max_install_names -Wl,-S "${args[@]}" diff --git a/contrib/native/apple-stubs.sh b/contrib/native/apple-stubs.sh new file mode 100755 index 000000000..6f7705385 --- /dev/null +++ b/contrib/native/apple-stubs.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Generate a stub Apple SDK in $1 from the macOS libSystem.tbd and Darwin +# headers that zig ships, so the iOS and macOS targets link on Linux +# without Apple's SDK. Every target list is widened to all four targets; +# the macOS verification job checks the imports against Xcode's real stubs. +set -euo pipefail + +sdk=$1 +zig_lib=$(zig env | sed -n 's/^ *\.lib_dir = "\(.*\)",$/\1/p') +apple_targets="[ arm64-ios, arm64-ios-simulator, arm64-macos, x86_64-macos ]" + +rm -rf "$sdk" +mkdir -p "$sdk/usr/include" "$sdk/usr/lib" +# The nix store copy is read-only, and the next run must remove it. +cp -R --no-preserve=mode "$zig_lib/libc/include/any-darwin-any/." "$sdk/usr/include/" +sed -zE "s/targets:( *)\[[^]]*\]/targets:\1$apple_targets/g" \ + "$zig_lib/libc/darwin/libSystem.tbd" >"$sdk/usr/lib/libSystem.tbd" +# std links libiconv on Apple targets, and zig ships no stub for it. +cat >"$sdk/usr/lib/libiconv.tbd" </. +set -euo pipefail + +target=$1 +out_dir=$(realpath -m "$2") +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +target_dir=$(realpath -m "${CARGO_TARGET_DIR:-$repo/target}") +target_env=${target//-/_} +CARGO_HOME=$(realpath -m "${CARGO_HOME:-$HOME/.cargo}") +export CARGO_HOME +# Two builds of the same zig version share a cache key, so keep the cache +# with the build rather than in $HOME. +export ZIG_GLOBAL_CACHE_DIR=$target_dir/zig-cache + +cd "$repo" +source contrib/lockfile.sh +use_lockfile Cargo-recent.lock + +# rustc and the C compiler see the same prefixes, and the last match wins. +prefixes=( + "$repo=/payjoin" + "$target_dir=/payjoin/target" + "$CARGO_HOME=/cargo" + "${RUSTUP_HOME:-$HOME/.rustup}=/rustup" +) +rustflags=() +cflags=() +cargo=(cargo build) +cargo_target=$target +cflag= + +case $target in + *-unknown-linux-gnu) + lib=libpayjoin_ffi.so + cargo=(cargo zigbuild) + cargo_target=$target.2.17 + ;; + *-pc-windows-msvc) + lib=payjoin_ffi.dll + arch=${target%%-*} + # MSVC's CRT and the Windows SDK from the Visual Studio manifest in + # vs-channel.json, splatted once per set of pins. + sdk=10.0.26100 + crt=14.44.17.14 + manifest=$(sha256sum contrib/native/vs-channel.json | cut -c1-16) + xwin_dir=${XDG_CACHE_HOME:-$HOME/.cache}/payjoin-xwin/$manifest-$sdk-$crt + if [[ ! -d $xwin_dir ]]; then + # Concurrent first builds splat their own copies, and the first + # one done is kept. + mkdir -p "$(dirname "$xwin_dir")" + tmp=$(mktemp -d "$xwin_dir.XXXXXX") + xwin --accept-license --cache-dir "$tmp/download" \ + --manifest contrib/native/vs-channel.json \ + --sdk-version "$sdk" --crt-version "$crt" \ + --arch x86_64,aarch64 splat --output "$tmp/splat" + mv -T "$tmp/splat" "$xwin_dir" 2>/dev/null || true + rm -rf "$tmp" + fi + export "CARGO_TARGET_${target_env^^}_LINKER=lld-link" \ + "CC_$target_env=clang-cl" "AR_$target_env=llvm-lib" + # lld-link otherwise stamps the build time, and the PDB rustc asks + # for gives the DLL an id that depends on the build paths. + rustflags+=(-Clink-arg=/Brepro -Clink-arg=/DEBUG:NONE) + for dir in crt/lib sdk/lib/um sdk/lib/ucrt; do + rustflags+=("-Lnative=$xwin_dir/$dir/$arch") + done + # clang-cl takes clang's own options behind /clang:. + cflag=/clang: + cflags+=("--target=$target") + for dir in crt/include sdk/include/ucrt sdk/include/um sdk/include/shared; do + cflags+=(/imsvc "$xwin_dir/$dir") + done + prefixes+=("$xwin_dir=/xwin") + ;; + *-linux-android*) + lib=libpayjoin_ffi.so + # API 21 is Rust's floor. NDK 29 aligns 64-bit libraries to 16 KB. + ndk_bin=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin + clang=$ndk_bin/${target/armv7/armv7a}21-clang + export "CARGO_TARGET_${target_env^^}_LINKER=$clang" "CC_$target_env=$clang" \ + "AR_$target_env=$ndk_bin/llvm-ar" + prefixes+=("$(realpath "$ANDROID_NDK_HOME")=/ndk") + ;; + *-apple-darwin) + # Flutter's floor, and the first macOS on Apple silicon. + macos=10.15 + if [[ $target == aarch64-* ]]; then + macos=11.0 + fi + export APPLE_CC_TARGET=${target%%-*}-macos.$macos + export MACOSX_DEPLOYMENT_TARGET=$macos + ;;& + aarch64-apple-ios | aarch64-apple-ios-sim) + export APPLE_CC_TARGET=aarch64-ios.13.0 + if [[ $target == *-sim ]]; then + APPLE_CC_TARGET+=-simulator + fi + export IPHONEOS_DEPLOYMENT_TARGET=13.0 + ;;& + *-apple-darwin | aarch64-apple-ios | aarch64-apple-ios-sim) + lib=libpayjoin_ffi.dylib + export SDKROOT=$target_dir/apple-stubs + bash contrib/native/apple-stubs.sh "$SDKROOT" + export "CARGO_TARGET_${target_env^^}_LINKER=$repo/contrib/native/apple-cc.sh" + export "CC_$target_env=$repo/contrib/native/apple-cc.sh" + export "AR_$target_env=$(command -v zig) ar" + rustflags+=("-Clink-arg=-Wl,-install_name,@rpath/$lib") + ;; + *) + echo "unsupported target: $target" >&2 + exit 1 + ;; +esac + +for prefix in "${prefixes[@]}"; do + rustflags+=("--remap-path-prefix=$prefix") + cflags+=("$cflag-ffile-prefix-map=$prefix") +done +# Quoted, because cc-rs would otherwise split a path with spaces. +export CC_SHELL_ESCAPED_FLAGS=1 +export "CFLAGS_$target_env=$(printf '%q ' "${cflags[@]}")" +CARGO_ENCODED_RUSTFLAGS=$( + IFS=$'\x1f' + echo "${rustflags[*]}" +) +export CARGO_ENCODED_RUSTFLAGS + +"${cargo[@]}" --locked --release --package payjoin-ffi --lib \ + --target "$cargo_target" --target-dir "$target_dir" +mkdir -p "$out_dir/$target" +cp "$target_dir/$target/release/$lib" "$out_dir/$target/" diff --git a/contrib/native/vs-channel.json b/contrib/native/vs-channel.json new file mode 100644 index 000000000..2ecc4fea0 --- /dev/null +++ b/contrib/native/vs-channel.json @@ -0,0 +1,17 @@ +{ + "channelItems": [ + { + "id": "Microsoft.VisualStudio.Manifests.VisualStudio", + "version": "17.14.37824.6", + "type": "Manifest", + "payloads": [ + { + "fileName": "VisualStudio.vsman", + "sha256": "5bd3b0b2ae509af2e9559b852f78545df0bf2c081dd850caed76f1dc134c511e", + "size": 30443792, + "url": "https://download.visualstudio.microsoft.com/download/pr/4bc08c27-bd31-470c-9a30-3b62707fbf0f/5bd3b0b2ae509af2e9559b852f78545df0bf2c081dd850caed76f1dc134c511e/VisualStudio.vsman" + } + ] + } + ] +} diff --git a/contrib/release/generate-sha256sums.sh b/contrib/release/generate-sha256sums.sh index 725fd57e9..6721e8a84 100755 --- a/contrib/release/generate-sha256sums.sh +++ b/contrib/release/generate-sha256sums.sh @@ -1,20 +1,10 @@ #!/usr/bin/env bash # -# Write SHA256SUMS over every file in a release asset directory. With -# ARCHIVE_GLOBS and MEMBER_GLOBS set, also open each asset matching -# ARCHIVE_GLOBS and append the hash of each member matching MEMBER_GLOBS, -# under its in-archive path, so files nested inside a package can be -# verified individually. Both variables are whitespace-separated glob -# lists and must be set together. Archives must be zip format, which -# every current package with nested libraries is (.nupkg, .whl). +# Write SHA256SUMS over every file in a release asset directory. set -euo pipefail [ "$#" -eq 1 ] || { - echo "usage: [ARCHIVE_GLOBS=... MEMBER_GLOBS=...] generate-sha256sums.sh " >&2 - exit 1 -} -die() { - echo "generate-sha256sums: $*" >&2 + echo "usage: generate-sha256sums.sh " >&2 exit 1 } @@ -22,30 +12,3 @@ cd "$1" # shellcheck disable=SC2094 # find excludes SHA256SUMS, so it is never read find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 -r sha256sum >SHA256SUMS - -archive_globs="${ARCHIVE_GLOBS:-}" -member_globs="${MEMBER_GLOBS:-}" -if [ -z "$archive_globs" ] && [ -z "$member_globs" ]; then - exit 0 -fi -{ [ -n "$archive_globs" ] && [ -n "$member_globs" ]; } || - die "ARCHIVE_GLOBS and MEMBER_GLOBS must be set together" - -read -ra members <<<"$member_globs" -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT -shopt -s nullglob -archives=() -# shellcheck disable=SC2086 # unquoted so the shell expands the caller's globs -for archive in $archive_globs; do - archives+=("$archive") -done -[ "${#archives[@]}" -gt 0 ] || die "no files match ARCHIVE_GLOBS ($archive_globs)" - -# unzip matches the member globs itself and fails when an archive -# contains no match, or is not a zip. -for archive in "${archives[@]}"; do - unzip -q "$archive" -d "$tmp" "${members[@]}" -done - -(cd "$tmp" && find . -type f -printf '%P\0' | sort -z | xargs -0 -r sha256sum) >>SHA256SUMS diff --git a/flake.nix b/flake.nix index 160ab703e..bbf361811 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,8 @@ }; msrv-version = "1.85.0"; + # Rust version used to build release artifacts. + release-version = "1.85.1"; nginxWithStream = pkgs.nginxMainline.overrideAttrs (oldAttrs: { configureFlags = oldAttrs.configureFlags ++ [ @@ -319,11 +321,11 @@ builtins.removeAttrs pythonWorkspace.deps.all [ "payjoin" ] ); - # Rust toolchain for the javascript dev shell: msrv pinned to match - # payjoin-ffi/javascript build requirements, with wasm32 target added + # Rust toolchain for the javascript dev shell: the release version, as + # pack-npm builds the published wasm here, with wasm32 target added # so cargo can build per-arch artifacts under nix (rustup target add # is a no-op against a nix-provided toolchain). - jsRustToolchain = pkgs.rust-bin.stable.${msrv-version}.default.override { + jsRustToolchain = pkgs.rust-bin.stable.${release-version}.default.override { extensions = [ "rust-src" "rustfmt" @@ -411,26 +413,25 @@ BITCOIND_SKIP_DOWNLOAD = 1; }; - # Rust toolchain for the python dev shell: msrv pinned to match - # payjoin-ffi/python build requirements, with per-arch targets added + # Rust toolchain for the python dev shell: the release version, as + # build-wheel generates the published bindings here, with per-arch targets added # so cargo can build artifacts under nix for payjoin-ffi/python/scripts/generate_bindings.sh # (rustup target add is a no-op against a nix-provided toolchain). - pythonRustToolchain = pkgs.rust-bin.stable.${msrv-version}.default.override { + pythonRustToolchain = pkgs.rust-bin.stable.${release-version}.default.override { extensions = [ "rust-src" "rustfmt" "llvm-tools-preview" ]; - targets = [ - # On Darwin the native test builds lipo both arches; on Linux - # cargo-zigbuild cross-links the release wheel's universal2 - # dylib from these same targets. - "aarch64-apple-darwin" - "x86_64-apple-darwin" - ] - ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ - "x86_64-unknown-linux-gnu" - ]; + targets = + # On Darwin the native test builds lipo both arches. + pkgs.lib.optionals pkgs.stdenv.isDarwin [ + "aarch64-apple-darwin" + "x86_64-apple-darwin" + ] + ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + "x86_64-unknown-linux-gnu" + ]; }; pythonDevShell = pkgs.mkShell { @@ -452,10 +453,8 @@ # Applies the manylinux platform tag the release wheel satisfies; # PyPI rejects raw linux_x86_64 wheels. auditwheel - # Cross-links the macOS release wheel against zig's bundled - # Apple SDK stubs, so the dylib records system install names - # instead of nix store paths. - cargo-zigbuild + # build-wheel.sh compares the packed library with its input. + unzip ]; env = { @@ -472,6 +471,65 @@ ''; }; + # Release builds of the payjoin-ffi native library through + # contrib/native/build.sh. The tools come from unstable: the release + # branch only has zig 0.15, and 0.17 regressed the iOS Mach-O linker. + unstablePkgs = nixpkgs-unstable.legacyPackages.${system}; + + nativeRustToolchain = pkgs.rust-bin.stable.${release-version}.minimal.override { + targets = [ + "aarch64-apple-ios" + "aarch64-apple-ios-sim" + "aarch64-unknown-linux-gnu" + "x86_64-unknown-linux-gnu" + "aarch64-apple-darwin" + "x86_64-apple-darwin" + "aarch64-pc-windows-msvc" + "x86_64-pc-windows-msvc" + "aarch64-linux-android" + "armv7-linux-androideabi" + "x86_64-linux-android" + ]; + }; + + # The NDK and the SDK parts it pulls in are unfree, so they come from + # an instance used for nothing else. + androidPkgs = import nixpkgs-unstable { + inherit system; + config = { + android_sdk.accept_license = true; + allowUnfree = true; + }; + }; + + androidNdk = + (androidPkgs.androidenv.composeAndroidPackages { + includeNDK = true; + ndkVersions = [ "29.0.14206865" ]; + }).ndk-bundle; + + nativeDevShell = pkgs.mkShell { + name = "native"; + packages = with unstablePkgs; [ + nativeRustToolchain + zig_0_16 + # nixpkgs wraps cargo-zigbuild with its default zig, 0.17. + (cargo-zigbuild.override { zig = zig_0_16; }) + xwin + # The Windows targets compile C with clang-cl, archive with + # llvm-lib and link with lld-link. + llvmPackages_21.clang-unwrapped + llvmPackages_21.lld + llvmPackages_21.llvm + ]; + }; + + # Only the Android builds fetch the NDK. + nativeAndroidDevShell = nativeDevShell.overrideAttrs { + name = "native-android"; + ANDROID_NDK_HOME = "${androidNdk}/libexec/android-sdk/ndk-bundle"; + }; + simpleCheck = args: pkgs.stdenvNoCC.mkDerivation ( @@ -525,6 +583,8 @@ csharp = csharpDevShell; dart = dartDevShell; kotlin = kotlinDevShell; + native = nativeDevShell; + native-android = nativeAndroidDevShell; }; formatter = treefmtEval.config.build.wrapper; checks = diff --git a/payjoin-ffi/CHANGELOG.md b/payjoin-ffi/CHANGELOG.md index 9c92fdf69..a62807157 100644 --- a/payjoin-ffi/CHANGELOG.md +++ b/payjoin-ffi/CHANGELOG.md @@ -1,6 +1,19 @@ ## Unreleased - Python: require Python 3.11+ and replace httpx with httpx2. Callers catching HTTP errors from fetch_ohttp_keys should use httpx2.HTTPError instead of httpx.HTTPError. +- Python: the Linux wheel's library now needs at most glibc 2.17, and + the wheel is tagged `manylinux_2_17`. The macOS wheel now supports + macOS 10.15 and later, down from 13. +- Dart: Android builds with NDK r27 now align 64-bit libraries to + 16 KB, as Google Play requires. Later NDKs already did. +- The GitHub release now attaches every native library as + `payjoin_ffi--.`, listed in `SHA256SUMS` and + attested. `VERIFYING.md` explains how to check one or rebuild it. +- Dart: the package now ships the release's prebuilt native libraries, + and the build hook uses them instead of compiling Rust, so apps no + longer need a Rust toolchain or network access at build time. Set + `build: source` under `hooks: user_defines: payjoin:` in your app's + `pubspec.yaml` to keep compiling it. ## [0.25.1] diff --git a/payjoin-ffi/Cargo.toml b/payjoin-ffi/Cargo.toml index 0fb4197f7..87e1b0dfa 100644 --- a/payjoin-ffi/Cargo.toml +++ b/payjoin-ffi/Cargo.toml @@ -2,6 +2,7 @@ name = "payjoin-ffi" version = "0.25.1" edition = "2024" +rust-version = "1.85" license = "MIT OR Apache-2.0" exclude = ["tests"] diff --git a/payjoin-ffi/RELEASING.md b/payjoin-ffi/RELEASING.md index c68e27310..ada1ecece 100644 --- a/payjoin-ffi/RELEASING.md +++ b/payjoin-ffi/RELEASING.md @@ -83,12 +83,13 @@ environment, and allow `payjoin-ffi-*` in the repository's tag rules. environment. Approve the pending deployment once; it covers all four jobs. Each registry is published with OIDC from the tested artifacts. 5. The run attaches the wheels, npm tarball, NuGet package, staged Dart - package, and one `SHA256SUMS` covering them all, including the native - libraries inside the NuGet package, to the GitHub release. Verify an - install from each registry and check provenance with - `gh attestation verify FILE -R payjoin/rust-payjoin`. Optionally sign - `SHA256SUMS` locally and upload `SHA256SUMS.asc`; no signing key reaches - a runner. + package, every native library as `payjoin_ffi--.`, + and one `SHA256SUMS` covering them all to the GitHub release. The + wheels, the NuGet package and the Dart package carry those same native + libraries, byte for byte. Verify an install from each registry and + check provenance with `gh attestation verify FILE -R payjoin/rust-payjoin`. + Optionally sign `SHA256SUMS` locally and upload `SHA256SUMS.asc`; no + signing key reaches a runner. ## If a publish job fails diff --git a/payjoin-ffi/VERIFYING.md b/payjoin-ffi/VERIFYING.md new file mode 100644 index 000000000..6cdf751bb --- /dev/null +++ b/payjoin-ffi/VERIFYING.md @@ -0,0 +1,82 @@ +# Verifying the native libraries + +Every bindings release attaches each native library it ships to its +[GitHub release](https://github.com/payjoin/rust-payjoin/releases) as +`payjoin_ffi--.`, listed in `SHA256SUMS` and +attested by the release workflow. The Python wheels, the NuGet package and +the Dart package carry those same files, byte for byte. + +CI builds each library twice, on separate runners from different paths, +and the release only goes ahead when both builds match. You can rebuild +any of them yourself, the Apple ones included, on Linux. + +## Check a download + +From a directory holding `SHA256SUMS` and the files you downloaded: + +```shell +sha256sum --check --ignore-missing SHA256SUMS +gh attestation verify payjoin_ffi--. -R payjoin/rust-payjoin +``` + +A maintainer may also upload `SHA256SUMS.asc`, a signature over +`SHA256SUMS` made with a key in +[`contrib/release/keys/`](../contrib/release/keys/). + +## Check the Dart package + +The Dart package ships each library as `prebuilt//`, and +`dart pub get` unpacks it into the pub cache. Compare one with the line +for `payjoin_ffi--.` in `SHA256SUMS`: + +```shell +sha256sum ~/.pub-cache/hosted/pub.dev/payjoin-/prebuilt//* +``` + +Use `$PUB_CACHE` in place of `~/.pub-cache` if you set it. The release +also attests the staged package, `payjoin-dart.tar.gz`, which it +attaches. + +## Rebuild a library + +You need an x86_64 Linux machine with [nix](https://nixos.org/download/) +and flakes enabled. The build does not depend on where the checkout or +your home directory is. Build from a clean target directory, as in a +fresh clone: Cargo does not rebuild dependencies when the deployment +target changes, so a reused one can mix objects built for another target +and give a false mismatch. + +```shell +git clone https://github.com/payjoin/rust-payjoin.git +cd rust-payjoin +git checkout payjoin-ffi- +nix develop .#native -c contrib/native/build.sh /tmp/out +sha256sum /tmp/out//* +``` + +Compare the hash with the line for `payjoin_ffi--.` +in `SHA256SUMS`. The Android targets need the NDK, which only the +`.#native-android` devshell provides, so use it in place of `.#native` +for them. + +| Target | File | +| -------------------------------------------------------------------------- | ---------------------- | +| `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu` | `libpayjoin_ffi.so` | +| `aarch64-linux-android`, `armv7-linux-androideabi`, `x86_64-linux-android` | `libpayjoin_ffi.so` | +| `aarch64-apple-darwin`, `x86_64-apple-darwin` | `libpayjoin_ffi.dylib` | +| `aarch64-apple-ios`, `aarch64-apple-ios-sim` | `libpayjoin_ffi.dylib` | +| `x86_64-pc-windows-msvc`, `aarch64-pc-windows-msvc` | `payjoin_ffi.dll` | + +`universal2-apple-darwin` is not built on its own. Build both macOS +targets, then combine them: + +```shell +nix develop .#native -c llvm-lipo -create \ + /tmp/out/aarch64-apple-darwin/libpayjoin_ffi.dylib \ + /tmp/out/x86_64-apple-darwin/libpayjoin_ffi.dylib \ + -output /tmp/out/libpayjoin_ffi.dylib +sha256sum /tmp/out/libpayjoin_ffi.dylib +``` + +The [native build contract](../contrib/native/README.md) lists the +toolchain, flags and pinned inputs behind these builds. diff --git a/payjoin-ffi/csharp/CONTRIBUTING.md b/payjoin-ffi/csharp/CONTRIBUTING.md index 513ecc61e..4517ee4de 100644 --- a/payjoin-ffi/csharp/CONTRIBUTING.md +++ b/payjoin-ffi/csharp/CONTRIBUTING.md @@ -52,13 +52,13 @@ the C# bindings): bash ./scripts/build_nuget_native.sh ``` -Any supported RID can also be cross-compiled from a Linux host, which is how CI -builds every native asset (`pip install -r scripts/cross-requirements.txt` for -the version- and hash-pinned toolchain CI uses, and `rustup target add` the -matching triple first): +CI packs every RID's native asset from the native workflow, which builds them +on Linux under the [native build contract](../../contrib/native/README.md). To +build one the same way, run from the repository root with the target triple for the RID, then copy +`//*` to `payjoin-ffi/csharp/artifacts/runtimes/{rid}/native/`: ```shell -PAYJOIN_FFI_CROSS=1 PAYJOIN_FFI_RID=osx-arm64 bash ./scripts/build_nuget_native.sh +nix develop .#native -c contrib/native/build.sh aarch64-apple-darwin ``` On Windows: diff --git a/payjoin-ffi/csharp/RELEASING.md b/payjoin-ffi/csharp/RELEASING.md index ada426152..28bd56da2 100644 --- a/payjoin-ffi/csharp/RELEASING.md +++ b/payjoin-ffi/csharp/RELEASING.md @@ -20,9 +20,9 @@ CI is the release path. On every bindings release and in the `Daily language bindings` run on `master`, the `Build and Test CSharp` workflow: -1. builds release-profile native assets for each supported RID +1. takes the native workflow's library for each supported RID (`linux-arm64`, `linux-x64`, `osx-arm64`, `osx-x64`, `win-arm64`, - `win-x64`), + `win-x64`), built twice on Linux under the native build contract, 2. generates production bindings (no `_test-utils`) and packs the `.nupkg` with all RID assets, 3. installs the package into a clean console app and runs a smoke test on diff --git a/payjoin-ffi/csharp/scripts/build_nuget_native.sh b/payjoin-ffi/csharp/scripts/build_nuget_native.sh index 6a9c94fe4..1d8433d2b 100755 --- a/payjoin-ffi/csharp/scripts/build_nuget_native.sh +++ b/payjoin-ffi/csharp/scripts/build_nuget_native.sh @@ -55,50 +55,6 @@ native_library_name() { esac } -# Rust target triple for cross-compiling a RID from a Linux host. Linux targets pin a -# glibc floor through zig so the produced .so loads on older distributions instead of -# inheriting the build host's glibc version. -rid_to_cross_target() { - case "$1" in - linux-x64) - echo "x86_64-unknown-linux-gnu.$PAYJOIN_FFI_GLIBC_FLOOR" - ;; - linux-arm64) - echo "aarch64-unknown-linux-gnu.$PAYJOIN_FFI_GLIBC_FLOOR" - ;; - osx-x64) - echo "x86_64-apple-darwin" - ;; - osx-arm64) - echo "aarch64-apple-darwin" - ;; - win-x64) - echo "x86_64-pc-windows-msvc" - ;; - win-arm64) - echo "aarch64-pc-windows-msvc" - ;; - *) - echo "Unsupported RID for cross build: $1" >&2 - exit 1 - ;; - esac -} - -# cargo-zigbuild links Linux (with a chosen glibc floor) and macOS targets (zig ships -# redistributable libSystem stubs); cargo-xwin links MSVC-ABI Windows targets against -# the Microsoft CRT/SDK it fetches. -rid_to_cross_tool() { - case "$1" in - linux-* | osx-*) - echo "zigbuild" - ;; - win-*) - echo "xwin" - ;; - esac -} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CSHARP_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" @@ -114,42 +70,11 @@ LIBNAME=$(native_library_name "$RID") export PAYJOIN_FFI_FEATURES="${PAYJOIN_FFI_FEATURES-}" export PAYJOIN_FFI_PROFILE="${PAYJOIN_FFI_PROFILE:-release}" -PAYJOIN_FFI_GLIBC_FLOOR=${PAYJOIN_FFI_GLIBC_FLOOR:-2.17} - -if [[ ${PAYJOIN_FFI_CROSS:-0} == 1 ]]; then - # Cross-compile the native library for $RID from a Linux host. Bindings are never - # generated here; the pack step generates the production bindings it packages. - TARGET=$(rid_to_cross_target "$RID") - TOOL=$(rid_to_cross_tool "$RID") - if ! command -v "cargo-$TOOL" >/dev/null; then - echo "cargo-$TOOL is required for cross builds: pip install cargo-zigbuild cargo-xwin" >&2 - exit 1 - fi - - CROSS_CARGO=(cargo "$TOOL") - if [[ $TOOL == xwin ]]; then - CROSS_CARGO+=(build) - fi - - GENERATOR_FEATURES="csharp${PAYJOIN_FFI_FEATURES:+,$PAYJOIN_FFI_FEATURES}" - (cd .. && "${CROSS_CARGO[@]}" --target "$TARGET" --features "$GENERATOR_FEATURES" --profile "$PAYJOIN_FFI_PROFILE" -p payjoin-ffi) - - # Cargo writes to the triple without any zig glibc suffix. - TARGET_DIR=${TARGET%."$PAYJOIN_FFI_GLIBC_FLOOR"} - if [[ $PAYJOIN_FFI_PROFILE == "dev" ]]; then - PROFILE_DIR=debug - else - PROFILE_DIR=$PAYJOIN_FFI_PROFILE - fi - BUILT_LIB="../../target/$TARGET_DIR/$PROFILE_DIR/$LIBNAME" -else - bash ./scripts/generate_bindings.sh --native-only - BUILT_LIB="lib/$LIBNAME" -fi +bash ./scripts/generate_bindings.sh --native-only ARTIFACT_DIR="artifacts/runtimes/$RID/native" rm -rf "$ARTIFACT_DIR" mkdir -p "$ARTIFACT_DIR" -cp "$BUILT_LIB" "$ARTIFACT_DIR/$LIBNAME" +cp "lib/$LIBNAME" "$ARTIFACT_DIR/$LIBNAME" echo "Wrote $ARTIFACT_DIR/$LIBNAME" diff --git a/payjoin-ffi/csharp/scripts/cross-requirements.txt b/payjoin-ffi/csharp/scripts/cross-requirements.txt deleted file mode 100644 index 9f3db226f..000000000 --- a/payjoin-ffi/csharp/scripts/cross-requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Pinned cross-compilation toolchain for the CI native-asset job (Linux runners). -# These tools produce the exact binaries shipped in the NuGet package, so they -# are pinned by version and hash: a floating install would let an upstream -# release or a compromised PyPI package silently change the shipped artifacts. -# Update deliberately, never implicitly. Hashes cover the x86_64 and aarch64 -# manylinux wheels. -ziglang==0.16.0 \ - --hash=sha256:9fcda73f62b851dd72a54b710ad40a209896db14cfb13649e62191243556342b \ - --hash=sha256:e27d409812b11e0fb89ed0200cf2e55b6464d43f9461553104e4a4f9a94a1fd5 -cargo-zigbuild==0.23.0 \ - --hash=sha256:3a2e021435a5e3c8b3a70bfc490580a3af6952185aa5ffb722b005d1fbe000c9 \ - --hash=sha256:d984af93da7a230606452ef5f6df47e6a081b12dcead8a3e8e5ff024728c94f0 -cargo-xwin==0.22.0 \ - --hash=sha256:eebe464a84371c98c9a3e2d24a87b6c0fdc575146f6670f49823a366158027bf \ - --hash=sha256:8a6d3f235590a1e583286884cbdf0b2af67164713f8a81d292f3312e0fa2355e -# Transitive dependency of cargo-xwin. -ninja==1.13.0 \ - --hash=sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa \ - --hash=sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630 diff --git a/payjoin-ffi/dart/.gitignore b/payjoin-ffi/dart/.gitignore index 3aa61ef2a..c30c7ee13 100644 --- a/payjoin-ffi/dart/.gitignore +++ b/payjoin-ffi/dart/.gitignore @@ -12,3 +12,7 @@ doc/api/ # Auto-generated bindings lib/payjoin.dart + +# Release libraries, which contrib/prepare-publish.sh copies into the +# staged package +prebuilt/ diff --git a/payjoin-ffi/dart/.pubignore b/payjoin-ffi/dart/.pubignore index c8c635895..2a1168cce 100644 --- a/payjoin-ffi/dart/.pubignore +++ b/payjoin-ffi/dart/.pubignore @@ -9,10 +9,12 @@ scripts/ .DS_Store -# Native libraries are platform specific and built per consumer +# Native libraries are platform specific and built per consumer, except +# the release's own, which prepare-publish.sh copies into prebuilt/ *.dylib *.so *.dll +!/prebuilt/** # Cargo build output native/target/ diff --git a/payjoin-ffi/dart/CONTRIBUTING.md b/payjoin-ffi/dart/CONTRIBUTING.md index f4f093465..f3310e03a 100644 --- a/payjoin-ffi/dart/CONTRIBUTING.md +++ b/payjoin-ffi/dart/CONTRIBUTING.md @@ -27,16 +27,19 @@ dart test ### The native build hook -`hook/build.dart` builds the `native/` wrapper crate for the consumer's -target. It takes the presence of `.cargo/config.local.toml` as the sign -that it is running inside this repository rather than from a published -archive, because `.pubignore` withholds `.cargo/` from that archive. In -the repository the overlay redirects `payjoin-ffi` to the workspace and -the hook enables `_test-utils`. From an archive the hook builds the -shipped `native/Cargo.lock` with `--locked`. - -This cannot be an environment variable. `hooks_runner` spawns build hooks -with an allowlisted environment (`HOME`, `PATH` and a few toolchain +`hook/build.dart` has three modes, chosen by the `build` user define in +the root package's `pubspec.yaml` (see the [README](README.md)): + +- `prebuilt`, the default, uses the release's library for the target + from `prebuilt//`, which only a staged package has. +- `source` builds the `native/` wrapper crate from the shipped + `native/Cargo.lock` with `--locked`. +- `workspace`, which this package's own `pubspec.yaml` sets, builds it + through the `.cargo/config.local.toml` overlay, which redirects + `payjoin-ffi` to the workspace, and enables `_test-utils`. + +The mode cannot be an environment variable. `hooks_runner` spawns build +hooks with an allowlisted environment (`HOME`, `PATH` and a few toolchain prefixes), so a project specific variable never reaches the hook. The hook also passes `--remap-path-prefix` for the package, output, Cargo @@ -88,11 +91,17 @@ with a publish dry run with `--core-from-revision`, which every run but a release tag uses since it may wrap core changes not yet on crates.io. That option takes them from the same revision and writes an `UNPUBLISHED_CORE` - marker, which `contrib/publish.sh` refuses. Both the publish dry run - and publication use the staged package. For local checks: + marker, which `contrib/publish.sh` refuses. The script also copies + the native workflow's `payjoin-native` artifact of the same run, the + libraries the release attaches, into `prebuilt//` for every + target but `universal2-apple-darwin`. Both the publish dry run and + publication use the staged package. For local checks, with the + libraries built under the + [native build contract](../../contrib/native/README.md) in + `/tmp/native//`: ```shell - nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh /tmp/payjoin-pub + nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh /tmp/native /tmp/payjoin-pub ``` 2. Follow the [shared bindings release instructions](../RELEASING.md). diff --git a/payjoin-ffi/dart/README.md b/payjoin-ffi/dart/README.md index 50a5ba489..bdfddfb8d 100644 --- a/payjoin-ffi/dart/README.md +++ b/payjoin-ffi/dart/README.md @@ -22,20 +22,49 @@ For OHTTP relay helpers: import 'package:payjoin/http.dart' as payjoin_http; ``` -### Native build +### Native library -The package compiles its Rust library on your machine through a Dart -build hook, with the toolchain named in `native/rust-toolchain.toml`. +A Dart build hook provides the Rust library. Choose how in your app's +`pubspec.yaml`: + +```yaml +hooks: + user_defines: + payjoin: + build: prebuilt # the default, or source +``` + +#### Prebuilt (default) + +The package ships the library the release built for each target, and +the hook bundles the one for yours. The build needs no Rust toolchain +and no network access. Your `pubspec.lock` pins the sha256 of the +package archive, and pub checks it on every download. Each library is +built twice by CI and attested by the release, and +[`VERIFYING.md`](https://github.com/payjoin/rust-payjoin/blob/master/payjoin-ffi/VERIFYING.md) +explains how to compare one with the GitHub release or rebuild it +yourself. + +- **No fallback.** A target the package has no library for fails the + build with a message that says how to switch to `build: source`. It + never compiles instead. +- **Targets.** Android (arm64, armv7, x86_64) from API 21, iOS 13 + devices and arm64 simulators, macOS 10.15 on x86_64 and 11 on arm64, + Linux with glibc 2.17 (arm64, x86_64), and Windows (arm64, x86_64). + The Intel iOS simulator needs `build: source`. + +#### Source + +The hook compiles the library on your machine with the toolchain named +in `native/rust-toolchain.toml`. - **Hosted releases are locked.** A version from pub.dev ships the `native/Cargo.lock` it was released with, and the hook builds it with `--locked`. Every release compiles the same dependency versions, whenever and wherever you build it. -- **Git dependencies are development builds.** A `git:` dependency on - this repository checks out the monorepo layout, which the hook treats - as a development build: it builds `payjoin-ffi` from the checkout, - enables test-only helpers and resolves dependencies afresh, without a - lockfile. Use a hosted release in production. +- **Use a hosted release.** Only the package on pub.dev has the + generated bindings, the prebuilt libraries and the pinned lockfile, so + a `git:` dependency on this repository builds in neither mode. - **Cargo config `rustflags` do not apply.** The hook sets `CARGO_ENCODED_RUSTFLAGS` to strip build paths from the library, and Cargo then ignores `build.rustflags` and `target.*.rustflags` from any diff --git a/payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart b/payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart new file mode 100644 index 000000000..a8eceeccc --- /dev/null +++ b/payjoin-ffi/dart/contrib/ios-simulator/integration_test/payjoin_test.dart @@ -0,0 +1,60 @@ +import 'dart:typed_data'; + +import 'package:convert/convert.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:payjoin/payjoin.dart' as payjoin; + +// The bundled library is built without the test utilities, so the unit tests +// that call them cannot run here. +class _ReceiverPersister implements payjoin.JsonReceiverSessionPersister { + final List events = []; + + @override + void save(String event) => events.add(event); + + @override + List load() => events; + + @override + void close() {} +} + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + test('parses a payjoin URI', () { + final uri = payjoin.Uri.parse( + uri: 'bitcoin:12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX?amount=1&pj=https://example.com', + ); + expect(uri.address(), '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + }); + + test('rejects userinfo in the payjoin URL', () { + expect( + () => payjoin.Uri.parse( + uri: 'bitcoin:12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX?amount=1&pj=https://example@evil.com', + ), + throwsA(isA()), + ); + }); + + test('persists a new receiver session', () { + final persister = _ReceiverPersister(); + payjoin.ReceiverBuilder( + address: 'tb1q6d3a2w975yny0asuvd9a67ner4nks58ff0q8g4', + directory: 'https://example.com', + ohttpKeys: payjoin.OhttpKeys.decode( + bytes: Uint8List.fromList( + hex.decode( + '01001604ba48c49c3d4a92a3ad00ecc63a024da10ced02180c73ec12d8a7ad2cc91bb483824fe2bee8d28bfe2eb2fc6453bc4d31cd851e8a6540e86c5382af588d370957000400010003', + ), + ), + ), + ).build().save(persister: persister); + expect( + payjoin.replayReceiverEventLog(persister: persister).state(), + isA(), + ); + }); +} diff --git a/payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart b/payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart new file mode 100644 index 000000000..e2476a74b --- /dev/null +++ b/payjoin-ffi/dart/contrib/ios-simulator/prebuilt_hook.dart @@ -0,0 +1,19 @@ +import 'package:code_assets/code_assets.dart'; +import 'package:hooks/hooks.dart'; + +// Replaces the package's build hook in the iOS simulator check, so the app +// bundles the library cross-built on Linux instead of building its own. +void main(List args) async { + await build(args, (input, output) async { + output.assets.code.add( + CodeAsset( + package: input.packageName, + name: 'uniffi:payjoin_ffi', + linkMode: DynamicLoadingBundled(), + file: input.packageRoot.resolve( + 'prebuilt/aarch64-apple-ios-sim/libpayjoin_ffi.dylib', + ), + ), + ); + }); +} diff --git a/payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml b/payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml new file mode 100644 index 000000000..79515bb64 --- /dev/null +++ b/payjoin-ffi/dart/contrib/ios-simulator/pubspec.yaml @@ -0,0 +1,23 @@ +name: payjoin_ios_simulator +description: >- + A Flutter app that runs a payjoin smoke test in the iOS simulator against + the library cross-built on Linux. +publish_to: none +version: 0.0.0 + +environment: + sdk: ^3.10.0 + +dependencies: + flutter: + sdk: flutter + # CI swaps in prebuilt_hook.dart as the package's build hook. + payjoin: + path: ../.. + +dev_dependencies: + convert: ^3.1.0 + flutter_test: + sdk: flutter + integration_test: + sdk: flutter diff --git a/payjoin-ffi/dart/contrib/prepare-publish.sh b/payjoin-ffi/dart/contrib/prepare-publish.sh index a6511a411..a2c97a1df 100755 --- a/payjoin-ffi/dart/contrib/prepare-publish.sh +++ b/payjoin-ffi/dart/contrib/prepare-publish.sh @@ -5,18 +5,20 @@ if [[ ${1:-} == --core-from-revision ]]; then core_from_revision=true shift fi -[ "$#" -eq 1 ] || { - echo "usage: prepare-publish.sh [--core-from-revision] " >&2 +[ "$#" -eq 2 ] || { + echo "usage: prepare-publish.sh [--core-from-revision] " >&2 exit 1 } # Resolve before changing directories; the destination must not exist. -stage="$(realpath -m "$1")" +native="$(realpath "$1")" +stage="$(realpath -m "$2")" # Prepare the package for publishing to pub.dev and verify the archive with -# a dry run. The archive ships Dart source plus the native/ wrapper crate; -# consumers compile the Rust themselves via hook/build.dart, so no binaries -# are packed here. Publishing itself stays a separate step so this script -# can run anywhere, including on pull requests. +# a dry run. The archive ships Dart source, the native/ wrapper crate for +# source builds, and under prebuilt// the libraries in +# , the native workflow's payjoin-native artifact, which +# hook/build.dart uses by default. Publishing itself stays a separate step +# so this script can run anywhere. # Build against the maintained lockfile instead of resolving the dependency # graph fresh on every run. use_lockfile copies Cargo-recent.lock into place @@ -33,14 +35,28 @@ PAYJOIN_FFI_FEATURES="" bash ./scripts/generate_bindings.sh revision="$(git rev-parse HEAD)" mkdir "$stage" -# Include generated bindings while excluding development Cargo overlays, -# then pin the native crate to the checkout that generated them. -rsync -a --exclude=.git --exclude-from=.pubignore ./ "$stage/" +# Include generated bindings while excluding development Cargo overlays and +# any local prebuilt/, then pin the native crate to the checkout that +# generated them. +rsync -a --exclude=.git --exclude=/prebuilt/ --exclude-from=.pubignore ./ "$stage/" sed -i "s/rev = \"[0-9a-f]*\"/rev = \"$revision\"/" "$stage/native/Cargo.toml" grep -Fq "rev = \"$revision\"" "$stage/native/Cargo.toml" || { echo "Could not pin the staged native crate to $revision" >&2 exit 1 } +# Every target the hook maps, which is all of them but the universal macOS +# library. +for lib in "$native"/*/*; do + target="$(basename "$(dirname "$lib")")" + [[ $target == universal2-apple-darwin ]] && continue + mkdir -p "$stage/prebuilt/$target" + cp "$lib" "$stage/prebuilt/$target/" +done +count="$(find "$stage/prebuilt" -type f | wc -l)" +[ "$count" -eq 11 ] || { + echo "Expected 11 prebuilt libraries in $native, found $count" >&2 + exit 1 +} # payjoin-ffi and payjoin-test-utils come from the pinned revision, but the # release crates they require, payjoin and payjoin-mailroom, come from diff --git a/payjoin-ffi/dart/contrib/test-prebuilt.sh b/payjoin-ffi/dart/contrib/test-prebuilt.sh new file mode 100755 index 000000000..bf3506f6d --- /dev/null +++ b/payjoin-ffi/dart/contrib/test-prebuilt.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run the consumer program against in the default prebuilt +# mode, so the hook must find the library for this host under prebuilt/. +set -euo pipefail +[ "$#" -eq 1 ] || { + echo "usage: test-prebuilt.sh " >&2 + exit 1 +} +package="$(realpath "$1")" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +cp -a "$(dirname "${BASH_SOURCE[0]}")/consumer" "$work/app" +ln -s "$package" "$work/payjoin" +cd "$work/app" +dart pub get +dart run bin/main.dart diff --git a/payjoin-ffi/dart/hook/build.dart b/payjoin-ffi/dart/hook/build.dart index aa11d6df5..a58e2338b 100644 --- a/payjoin-ffi/dart/hook/build.dart +++ b/payjoin-ffi/dart/hook/build.dart @@ -1,10 +1,28 @@ import 'dart:io'; +import 'package:code_assets/code_assets.dart'; import 'package:hooks/hooks.dart'; import 'package:native_toolchain_rust/native_toolchain_rust.dart'; +const _assetName = 'uniffi:payjoin_ffi'; + void main(List args) async { await build(args, (input, output) async { + final mode = input.userDefines['build'] ?? 'prebuilt'; + if (mode == 'prebuilt') { + if (input.config.buildCodeAssets) { + _prebuilt(input, output); + } + return; + } + if (mode != 'source' && mode != 'workspace') { + throw BuildError( + message: + 'payjoin: unknown build mode "$mode". Set ' + 'hooks.user_defines.payjoin.build to prebuilt or source.', + ); + } + final environment = Platform.environment; final separator = Platform.pathSeparator; String? nonEmpty(String? value) => @@ -22,6 +40,7 @@ void main(List args) async { final rustupHome = nonEmpty(environment['RUSTUP_HOME']) ?? (home == null ? null : '${strip(home)}$separator.rustup'); + final code = input.config.buildCodeAssets ? input.config.code : null; // Encoded rather than RUSTFLAGS, which Cargo splits on whitespace: the // output directory lives under the consumer's project, whose path may // contain spaces. hooks_runner never passes RUSTFLAGS to a build hook, @@ -37,20 +56,24 @@ void main(List args) async { if (rustupHome != null) (rustupHome, '/rustup'), ]) '--remap-path-prefix=${strip(from)}=$to', + // native_toolchain_rust accepts NDK r27, which aligns 64-bit libraries + // to 4 KB, and Google Play requires 16 KB. Later NDKs already align to + // 16 KB, and the flag leaves their output unchanged. + if (code?.targetOS == OS.android && + code?.targetArchitecture != Architecture.arm) + '-Clink-arg=-Wl,-z,max-page-size=16384', ].join('\x1f'); - // The path overlay doubles as the development marker. `.pubignore` - // withholds `.cargo/` from the archive, so the file only exists in a - // repository checkout. It cannot be an environment variable: hooks_runner - // spawns build hooks with an allowlisted environment, and a project - // specific name never survives that filter. + // The repository's own pubspec selects the workspace build, which + // builds payjoin-ffi from the checkout through this path overlay. + // `.pubignore` withholds `.cargo/` from the archive. final localCargoConfig = File.fromUri( input.packageRoot.resolve('.cargo/config.local.toml'), ); - final isWorkspaceBuild = localCargoConfig.existsSync(); + final isWorkspaceBuild = mode == 'workspace'; await RustBuilder( - assetName: 'uniffi:payjoin_ffi', + assetName: _assetName, features: isWorkspaceBuild ? const ['_test-utils'] : const [], extraCargoBuildArgs: [ // Cargo does not discover this file name on its own, and it @@ -74,3 +97,55 @@ void main(List args) async { ).run(input: input, output: output); }); } + +/// Registers the library the release built for this target, which +/// prepare-publish.sh copied into the package under prebuilt/. +void _prebuilt(BuildInput input, BuildOutputBuilder output) { + final code = input.config.code; + final target = switch ((code.targetOS, code.targetArchitecture)) { + (OS.android, Architecture.arm64) => 'aarch64-linux-android', + (OS.android, Architecture.arm) => 'armv7-linux-androideabi', + (OS.android, Architecture.x64) => 'x86_64-linux-android', + (OS.iOS, Architecture.arm64) => + code.iOS.targetSdk == IOSSdk.iPhoneSimulator + ? 'aarch64-apple-ios-sim' + : 'aarch64-apple-ios', + (OS.macOS, Architecture.arm64) => 'aarch64-apple-darwin', + (OS.macOS, Architecture.x64) => 'x86_64-apple-darwin', + (OS.linux, Architecture.arm64) => 'aarch64-unknown-linux-gnu', + (OS.linux, Architecture.x64) => 'x86_64-unknown-linux-gnu', + (OS.windows, Architecture.arm64) => 'aarch64-pc-windows-msvc', + (OS.windows, Architecture.x64) => 'x86_64-pc-windows-msvc', + (final os, final arch) => '$os on $arch', + }; + final file = File.fromUri( + input.packageRoot.resolve( + 'prebuilt/$target/${code.targetOS.dylibFileName('payjoin_ffi')}', + ), + ); + if (!file.existsSync()) { + _fail('this copy of the package has no library for $target'); + } + output.dependencies.add(file.uri); + + output.assets.code.add( + CodeAsset( + package: input.packageName, + name: _assetName, + linkMode: DynamicLoadingBundled(), + file: file.uri, + ), + ); +} + +Never _fail(String reason) => throw BuildError( + message: + 'payjoin: cannot use the prebuilt native library: $reason.\n' + 'To compile it from source with your own Rust toolchain instead, add ' + 'this to the pubspec.yaml of your app:\n' + '\n' + 'hooks:\n' + ' user_defines:\n' + ' payjoin:\n' + ' build: source\n', +); diff --git a/payjoin-ffi/dart/pubspec.yaml b/payjoin-ffi/dart/pubspec.yaml index 74bb50ec3..b4071de10 100644 --- a/payjoin-ffi/dart/pubspec.yaml +++ b/payjoin-ffi/dart/pubspec.yaml @@ -18,3 +18,8 @@ dev_dependencies: hooks: build: hook/build.dart + # Applies only where this package is the root, that is in the repository. + # Consumers choose their own mode; see README.md. + user_defines: + payjoin: + build: workspace diff --git a/payjoin-ffi/kotlin/scripts/generate_bindings.sh b/payjoin-ffi/kotlin/scripts/generate_bindings.sh index cb611cb2d..44262bb8f 100755 --- a/payjoin-ffi/kotlin/scripts/generate_bindings.sh +++ b/payjoin-ffi/kotlin/scripts/generate_bindings.sh @@ -16,6 +16,14 @@ else fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Build against the maintained lockfile instead of resolving the dependency +# graph fresh on every run. use_lockfile copies Cargo-recent.lock into place +# and restores the previous state when this script exits. +cd "$SCRIPT_DIR/../../.." +source contrib/lockfile.sh +use_lockfile Cargo-recent.lock + cd "$SCRIPT_DIR/../.." echo "Generating payjoin Kotlin..." diff --git a/payjoin-ffi/python/RELEASING.md b/payjoin-ffi/python/RELEASING.md index 46153efd4..514d72411 100644 --- a/payjoin-ffi/python/RELEASING.md +++ b/payjoin-ffi/python/RELEASING.md @@ -20,10 +20,13 @@ workflow builds release wheels with `_test-utils`) and smoke-installs them on every supported platform: - `manylinux` x86_64, tagged by auditwheel with the glibc floor the binary - actually satisfies; -- macOS `universal2` (a fat x86_64 + arm64 dylib), cross-compiled from the - Linux host with cargo-zigbuild, so the dylib links the Apple SDK stubs - zig bundles and records system install names rather than nix store paths. + actually satisfies, 2.17; +- macOS `universal2` (a fat x86_64 + arm64 dylib), combined from the two + macOS slices. + +Both libraries come from the native workflow, which builds each one twice +on Linux under the [native build contract](../../contrib/native/README.md), +and the wheel carries them byte for byte. The wheels are tagged `py3-none` because the generated bindings load the bundled library through `ctypes` and do not depend on a CPython ABI; any diff --git a/payjoin-ffi/python/contrib/build-wheel.sh b/payjoin-ffi/python/contrib/build-wheel.sh index 2f2ffd476..76916b8a9 100755 --- a/payjoin-ffi/python/contrib/build-wheel.sh +++ b/payjoin-ffi/python/contrib/build-wheel.sh @@ -1,21 +1,31 @@ #!/usr/bin/env bash set -euo pipefail -# Build the production wheel for one platform into dist/: release profile, -# no test utils. Select the platform with PAYJOIN_WHEEL_PLATFORM -# (linux-x64, the default, or macos-universal2). -# -# Both wheels build on a Linux host: a dylib linked inside the nix dev shell -# on a mac records nix store install names that exist on no user machine, -# while cargo-zigbuild links against zig's bundled Apple SDK stubs and records -# the system ones. +# Build the production wheel for one platform into dist/ around a native +# library built under the native build contract (contrib/native/build.sh): +# release profile, no test utils. Select the platform with +# PAYJOIN_WHEEL_PLATFORM (linux-x64, the default, or macos-universal2) and +# pass the library, libpayjoin_ffi.so or the universal2 +# libpayjoin_ffi.dylib, as the only argument. if [[ "$(uname -s)" != Linux ]]; then echo "error: release wheels build on a Linux host only" >&2 exit 1 fi +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi +LIBRARY=$(realpath "$1") PLATFORM=${PAYJOIN_WHEEL_PLATFORM:-linux-x64} +case "$PLATFORM" in + linux-x64 | macos-universal2) ;; + *) + echo "error: unsupported PAYJOIN_WHEEL_PLATFORM: $PLATFORM" >&2 + exit 1 + ;; +esac # Build against the maintained lockfile instead of resolving the dependency # graph fresh on every run. use_lockfile copies Cargo-recent.lock into place @@ -27,28 +37,18 @@ use_lockfile Cargo-recent.lock cd "$REPO_ROOT/payjoin-ffi/python" +# uniffi reads its metadata from the static symbol table, which the +# zig-linked Linux library does not have, so the bindings come from a +# development build of the same featureless library. echo "==> Generating production FFI bindings..." -PAYJOIN_FFI_FEATURES="" PAYJOIN_FFI_PROFILE=release bash ./scripts/generate_bindings.sh +(cd .. && cargo build --lib && + cargo run --bin uniffi-bindgen generate \ + --library ../target/debug/libpayjoin_ffi.so \ + --language python --out-dir python/src/payjoin/) -# generate_bindings.sh stages the host's library; make sure the wheel -# ships exactly one platform's binary. -case "$PLATFORM" in - linux-x64) - rm -f src/payjoin/libpayjoin_ffi.dylib - ;; - macos-universal2) - echo "==> Cross-compiling the universal2 macOS library..." - (cd "$REPO_ROOT/payjoin-ffi" && - cargo zigbuild --profile release --target universal2-apple-darwin) - rm -f src/payjoin/libpayjoin_ffi.so - cp "$REPO_ROOT/target/universal2-apple-darwin/release/libpayjoin_ffi.dylib" \ - src/payjoin/ - ;; - *) - echo "error: unsupported PAYJOIN_WHEEL_PLATFORM: $PLATFORM" >&2 - exit 1 - ;; -esac +# A development build may have staged the other platform's library. +rm -f src/payjoin/libpayjoin_ffi.so src/payjoin/libpayjoin_ffi.dylib +cp "$LIBRARY" src/payjoin/ echo "==> Building the wheel..." # Drop setuptools' build/ staging dir too: it survives across runs and @@ -72,11 +72,14 @@ if [[ $PLATFORM == linux-x64 ]]; then rm dist/*-linux_x86_64.whl else # setup.py tagged the wheel with the build machine's (Linux) platform. - # The macOS floor is zig's: its bundled SDK stubs currently support - # macOS >= 13, and that is the minos both slices record. Keep this tag - # in sync with the LC_BUILD_VERSION of the built dylib if zig moves. - wheel tags --platform-tag macosx_13_0_universal2 --remove dist/*.whl + # The native build contract's macOS deployment target is 10.15 for the + # x86_64 slice and 11.0, the first macOS on Apple silicon, for arm64. + wheel tags --platform-tag macosx_10_15_universal2 --remove dist/*.whl fi +# The release lists the library on its own, so the wheel must carry it +# unchanged. +unzip -p dist/*.whl "payjoin/$(basename "$LIBRARY")" | cmp - "$LIBRARY" + echo "==> Built wheel:" ls -l dist