diff --git a/.github/workflows/dart.yml b/.github/workflows/dart.yml index 506bc4032..c7cbd3a65 100644 --- a/.github/workflows/dart.yml +++ b/.github/workflows/dart.yml @@ -66,3 +66,62 @@ jobs: name: payjoin-dart-pub-package path: ${{ runner.temp }}/payjoin-dart.tar.gz if-no-files-found: error + + # Build the staged package twice the way a consumer would and require + # byte-identical native libraries. The two builds differ in every path + # the hook remaps: package root, app directory, and the HOME that holds + # the Cargo, rustup and pub caches. + reproducible-linux: + name: "Reproduce the Linux native library" + needs: verify-pub-package + runs-on: ubuntu-26.04 + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # unneeded: no git auth beyond checkout + - name: Download the staged package + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: payjoin-dart-pub-package + path: dist + - name: Extract the staged package + run: | + mkdir -p "$RUNNER_TEMP/stage" + tar -xzf dist/payjoin-dart.tar.gz -C "$RUNNER_TEMP/stage" + - name: "Install rustup" + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1 + with: + toolchain: 1.85.1 + # Flutter 3.44.9 bundles the Dart SDK Bull Bitcoin builds with. + - name: Set up Flutter + uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2.23.0 + with: + channel: stable + flutter-version: 3.44.9 + - name: Build the consumer twice and compare the libraries + run: | + set -euo pipefail + # The space checks that the hook keeps such paths in one flag. + for build in first "second/with space"; do + root="$RUNNER_TEMP/linux/$build" + mkdir -p "$root/home" + # The runner image exports CARGO_HOME and RUSTUP_HOME, which + # hooks_runner withholds from the hook, so the hook uses the + # defaults under HOME. Point both there so the toolchain + # installed here is the one the hook finds. + export HOME="$root/home" CARGO_HOME="$root/home/.cargo" \ + RUSTUP_HOME="$root/home/.rustup" PUB_CACHE="$root/home/.pub-cache" + rustup toolchain install 1.85.1 --profile minimal + cp -a "$RUNNER_TEMP/stage" "$root/payjoin" + cp -a payjoin-ffi/dart/contrib/consumer "$root/app" + (cd "$root/app" && dart pub get && dart run bin/main.dart) + done + # Cargo leaves a second copy under release/deps, which is not + # the hook's output. + library() { + find "$RUNNER_TEMP/linux/$1/app/.dart_tool" -type f \ + -path '*/x86_64-unknown-linux-gnu/release/libpayjoin_ffi_wrapper.so' + } + cmp "$(library first)" "$(library "second/with space")" diff --git a/payjoin-ffi/CHANGELOG.md b/payjoin-ffi/CHANGELOG.md index 31f103cda..62fe69e10 100644 --- a/payjoin-ffi/CHANGELOG.md +++ b/payjoin-ffi/CHANGELOG.md @@ -1,5 +1,14 @@ ## Unreleased +- Dart: the native library no longer records the paths it was built + from, so two builds of one release from different directories produce + the same bytes. The build hook sets `CARGO_ENCODED_RUSTFLAGS`, so + `rustflags` from your Cargo config no longer apply to the payjoin + native build. +- Dart: the package ships the `native/Cargo.lock` it was released with + and the build hook builds it with `--locked`, so every build compiles + the dependency versions CI tested. + ## [0.25.0] - Publish the Python, JavaScript, C#, and Dart packages from one signed diff --git a/payjoin-ffi/Cargo.toml b/payjoin-ffi/Cargo.toml index c4c8df83d..bac9891f5 100644 --- a/payjoin-ffi/Cargo.toml +++ b/payjoin-ffi/Cargo.toml @@ -26,7 +26,7 @@ async-trait = "0.1.89" getrandom = "0.2.10" lazy_static = "1.5.0" payjoin = { version = "1.2.0", features = ["v1", "v2"] } -payjoin-test-utils = { version = "0.0.1", optional = true } +payjoin-test-utils = { path = "../payjoin-test-utils", optional = true } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" thiserror = "2.0.18" diff --git a/payjoin-ffi/dart/.cargo/config.local.toml b/payjoin-ffi/dart/.cargo/config.local.toml new file mode 100644 index 000000000..c81981073 --- /dev/null +++ b/payjoin-ffi/dart/.cargo/config.local.toml @@ -0,0 +1,13 @@ +# Local Cargo overlay for development in the monorepo. The name keeps it +# out of Cargo's automatic config discovery, so only hook/build.dart reads +# it, through an explicit --config. Its presence is also how the hook tells +# a repository checkout from a published archive, which .pubignore strips +# .cargo/ from. +[patch."https://github.com/payjoin/rust-payjoin.git"] +payjoin-ffi = { path = ".." } +payjoin = { path = "../../payjoin" } + +[patch.crates-io] +payjoin = { path = "../../payjoin" } +payjoin-mailroom = { path = "../../payjoin-mailroom" } +payjoin-test-utils = { path = "../../payjoin-test-utils" } diff --git a/payjoin-ffi/dart/.cargo/config.toml b/payjoin-ffi/dart/.cargo/config.toml deleted file mode 100644 index a0d02e6be..000000000 --- a/payjoin-ffi/dart/.cargo/config.toml +++ /dev/null @@ -1,10 +0,0 @@ -# Local Cargo.toml overlay for development in monorepo -# These patches are intentionally ignored in .pubignore for publishing to pub.dev -[patch."https://github.com/payjoin/rust-payjoin.git"] -payjoin-ffi = { path = ".." } -payjoin = { path = "../../payjoin" } - -[patch.crates-io] -payjoin = { path = "../../payjoin" } -payjoin-mailroom = { path = "../../payjoin-mailroom" } -payjoin-test-utils = { path = "../../payjoin-test-utils" } diff --git a/payjoin-ffi/dart/.gitignore b/payjoin-ffi/dart/.gitignore index 9d8352fea..3aa61ef2a 100644 --- a/payjoin-ffi/dart/.gitignore +++ b/payjoin-ffi/dart/.gitignore @@ -12,6 +12,3 @@ doc/api/ # Auto-generated bindings lib/payjoin.dart - -# Local Cargo overlay for monorepo dev -!.cargo/config.toml diff --git a/payjoin-ffi/dart/.pubignore b/payjoin-ffi/dart/.pubignore index d0ac83c8c..c8c635895 100644 --- a/payjoin-ffi/dart/.pubignore +++ b/payjoin-ffi/dart/.pubignore @@ -16,7 +16,6 @@ scripts/ # Cargo build output native/target/ -native/Cargo.lock # Local-only test helper re-export lib/test_utils.dart diff --git a/payjoin-ffi/dart/CONTRIBUTING.md b/payjoin-ffi/dart/CONTRIBUTING.md index f450935d9..2c84f7811 100644 --- a/payjoin-ffi/dart/CONTRIBUTING.md +++ b/payjoin-ffi/dart/CONTRIBUTING.md @@ -25,6 +25,35 @@ bash ./scripts/generate_bindings.sh dart test ``` +### The native build hook + +`hook/build.dart` builds the `native/` wrapper crate for the consumer's +target. It takes the presence of `.cargo/config.local.toml` as the sign +that it is running inside this repository rather than from a published +archive, because `.pubignore` withholds `.cargo/` from that archive. In +the repository the overlay redirects `payjoin-ffi` to the workspace and +the hook enables `_test-utils`. From an archive the hook builds the +shipped `native/Cargo.lock` with `--locked`. + +This cannot be an environment variable. `hooks_runner` spawns build hooks +with an allowlisted environment (`HOME`, `PATH` and a few toolchain +prefixes), so a project specific variable never reaches the hook. + +The hook also passes `--remap-path-prefix` for the package, output, Cargo +and Rustup directories, so the artifact does not record where it was +built. The prefixes go through `CARGO_ENCODED_RUSTFLAGS` rather than +`RUSTFLAGS`, since Cargo splits the latter on whitespace and a consumer's +project path may contain spaces. + +### Reproducible native builds + +CI builds the staged package twice the way a Dart consumer does, with +`contrib/consumer`, and requires byte-identical host libraries. Each +build gets its own copy of the stage, its own app directory and its own +`HOME`, and with it its own Cargo, rustup and pub homes. The +`reproducible-linux` job in `.github/workflows/dart.yml` lists the +steps. + ## Releasing Maintainer instructions for publishing to @@ -49,8 +78,16 @@ validates the archive with a publish dry run 1. The preparation script stages a separate package and pins its Rust dependency to the checkout used to generate production bindings. The development pin in `native/Cargo.toml` is replaced only in that staging - directory. Cargo overlays and local lockfiles are excluded. Both the - publish dry run and publication use the staged package. For local checks: + directory. Cargo overlays are excluded. The staged `native/Cargo.lock` + is seeded from `Cargo-recent.lock` and resolved against that revision, + so consumers build the versions CI tested. A dependency fix reaches + Dart consumers through `Cargo-recent.lock` and a new release. Staging + fetches the pinned commit from GitHub, so it must be pushed. When a + `payjoin` or `payjoin-mailroom` version that `payjoin-ffi` or + `payjoin-test-utils` requires is not on crates.io yet, staging takes + it from the same revision and writes an `UNPUBLISHED_CORE` marker, + which `contrib/publish.sh` refuses. Both the publish dry run and + publication use the staged package. For local checks: ```shell nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh /tmp/payjoin-pub diff --git a/payjoin-ffi/dart/README.md b/payjoin-ffi/dart/README.md index 399312bd5..50a5ba489 100644 --- a/payjoin-ffi/dart/README.md +++ b/payjoin-ffi/dart/README.md @@ -22,4 +22,23 @@ For OHTTP relay helpers: import 'package:payjoin/http.dart' as payjoin_http; ``` +### Native build + +The package compiles its Rust library on your machine through a Dart +build hook, with the toolchain named in `native/rust-toolchain.toml`. + +- **Hosted releases are locked.** A version from pub.dev ships the + `native/Cargo.lock` it was released with, and the hook builds it with + `--locked`. Every release compiles the same dependency versions, + whenever and wherever you build it. +- **Git dependencies are development builds.** A `git:` dependency on + this repository checks out the monorepo layout, which the hook treats + as a development build: it builds `payjoin-ffi` from the checkout, + enables test-only helpers and resolves dependencies afresh, without a + lockfile. Use a hosted release in production. +- **Cargo config `rustflags` do not apply.** The hook sets + `CARGO_ENCODED_RUSTFLAGS` to strip build paths from the library, and + Cargo then ignores `build.rustflags` and `target.*.rustflags` from any + Cargo config file for this build. + Made with [uniffi-dart](https://github.com/Uniffi-Dart/uniffi-dart) diff --git a/payjoin-ffi/dart/contrib/consumer/bin/main.dart b/payjoin-ffi/dart/contrib/consumer/bin/main.dart new file mode 100644 index 000000000..54f4fac5a --- /dev/null +++ b/payjoin-ffi/dart/contrib/consumer/bin/main.dart @@ -0,0 +1,10 @@ +import 'package:payjoin/payjoin.dart' as payjoin; + +void main() { + // Parsing goes through the native library, so this only runs once the + // payjoin build hook has produced the asset. + final uri = payjoin.Uri.parse( + uri: 'bitcoin:12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX?amount=1', + ); + print(uri.address()); +} diff --git a/payjoin-ffi/dart/contrib/consumer/pubspec.yaml b/payjoin-ffi/dart/contrib/consumer/pubspec.yaml new file mode 100644 index 000000000..67c13bf5f --- /dev/null +++ b/payjoin-ffi/dart/contrib/consumer/pubspec.yaml @@ -0,0 +1,14 @@ +name: payjoin_consumer +description: >- + A minimal Dart program that builds the staged payjoin package the way a + desktop or server consumer depending on it from pub.dev would. +publish_to: none +version: 0.0.0 + +environment: + sdk: ^3.10.0 + +dependencies: + # CI places a copy of the staged package next to the program. + payjoin: + path: ../payjoin diff --git a/payjoin-ffi/dart/contrib/prepare-publish.sh b/payjoin-ffi/dart/contrib/prepare-publish.sh index 5a57e6706..bdda75c38 100755 --- a/payjoin-ffi/dart/contrib/prepare-publish.sh +++ b/payjoin-ffi/dart/contrib/prepare-publish.sh @@ -28,14 +28,67 @@ PAYJOIN_FFI_FEATURES="" bash ./scripts/generate_bindings.sh revision="$(git rev-parse HEAD)" mkdir "$stage" -# Include generated bindings while excluding development Cargo overlays and -# lockfiles, then pin the native crate to the checkout that generated them. +# Include generated bindings while excluding development Cargo overlays, +# then pin the native crate to the checkout that generated them. rsync -a --exclude=.git --exclude-from=.pubignore ./ "$stage/" sed -i "s/rev = \"[0-9a-f]*\"/rev = \"$revision\"/" "$stage/native/Cargo.toml" grep -Fq "rev = \"$revision\"" "$stage/native/Cargo.toml" || { echo "Could not pin the staged native crate to $revision" >&2 exit 1 } + +# payjoin-ffi and payjoin-test-utils come from the pinned revision, but the +# release crates they require, payjoin and payjoin-mailroom, come from +# crates.io. Between a version bump of one of those and its crates.io +# release, the lock below could not resolve it. Take such a crate from the +# pinned revision instead so pull requests can still stage, and leave a +# marker that publish.sh refuses: a published package must build against +# crates its consumers can get from crates.io. +source "$REPO_ROOT/contrib/release/crates.sh" +# crates.io answers 404 for a version it does not have. Any other failure, +# such as a rate limit or an outage, must stop here rather than read as +# unpublished, or a release run would stage a package publish.sh refuses. +crates_io_status() { + curl -sL --retry 3 -o /dev/null -w '%{http_code}' \ + -H "User-Agent: rust-payjoin release tooling" \ + "https://crates.io/api/v1/crates/$1/$2" || true +} +siblings="$(sibling_deps payjoin-ffi && sibling_deps payjoin-test-utils)" +[[ $siblings == *payjoin$'\t'* ]] || { + echo "Could not read the payjoin version payjoin-ffi requires" >&2 + exit 1 +} +while IFS=$'\t' read -r dep req; do + version="${req#^}" + version="${version#=}" + status="$(crates_io_status "$dep" "$version")" + case $status in + 200) continue ;; + 404) ;; + *) + echo "Could not check $dep $version on crates.io (HTTP $status)" >&2 + exit 1 + ;; + esac + echo "==> $dep $version is not on crates.io; staging it from $revision" + cat >>"$stage/native/Cargo.toml" <>"$stage/UNPUBLISHED_CORE" +done < <(sort -u <<<"$siblings") + +# Ship the dependency graph this release was tested against; hook/build.dart +# builds it with --locked. Seeding from the maintained lockfile keeps every +# pin that still applies, so cargo resolves only what differs from the +# workspace, chiefly payjoin-ffi coming from the pinned revision. +echo "==> Locking the staged native crate..." +cp "$REPO_ROOT/Cargo-recent.lock" "$stage/native/Cargo.lock" +cargo update --workspace --manifest-path "$stage/native/Cargo.toml" cd "$stage" echo "==> Verifying the publish archive..." diff --git a/payjoin-ffi/dart/contrib/publish.sh b/payjoin-ffi/dart/contrib/publish.sh index 3a6f7c877..e7549638a 100755 --- a/payjoin-ffi/dart/contrib/publish.sh +++ b/payjoin-ffi/dart/contrib/publish.sh @@ -11,6 +11,13 @@ grep -Fq "rev = \"$revision\"" native/Cargo.toml || { echo "Staged native/Cargo.toml must pin the release checkout $revision" >&2 exit 1 } +# A stage built against an unreleased core is only fit for pull request +# checks; consumers must build the core they can fetch from crates.io. +if [[ -e UNPUBLISHED_CORE ]]; then + echo "Refusing to publish a stage built against an unpublished core:" >&2 + cat UNPUBLISHED_CORE >&2 + exit 1 +fi # Publish the package to pub.dev via automated publishing (OIDC). # `dart pub publish` has no non-interactive authentication mode: without a diff --git a/payjoin-ffi/dart/hook/build.dart b/payjoin-ffi/dart/hook/build.dart index 2fcccec9c..aa11d6df5 100644 --- a/payjoin-ffi/dart/hook/build.dart +++ b/payjoin-ffi/dart/hook/build.dart @@ -5,13 +5,72 @@ import 'package:native_toolchain_rust/native_toolchain_rust.dart'; void main(List args) async { await build(args, (input, output) async { + final environment = Platform.environment; + final separator = Platform.pathSeparator; + String? nonEmpty(String? value) => + value == null || value.isEmpty ? null : value; + // Directory URIs render with a trailing separator, and rustc substitutes + // the prefix literally: without this, /native/src/lib.rs + // would remap to /payjoin/packagenative/src/lib.rs. + String strip(String path) => path.length > 1 && path.endsWith(separator) + ? path.substring(0, path.length - separator.length) + : path; + final home = nonEmpty(environment['HOME'] ?? environment['USERPROFILE']); + final cargoHome = + nonEmpty(environment['CARGO_HOME']) ?? + (home == null ? null : '${strip(home)}$separator.cargo'); + final rustupHome = + nonEmpty(environment['RUSTUP_HOME']) ?? + (home == null ? null : '${strip(home)}$separator.rustup'); + // Encoded rather than RUSTFLAGS, which Cargo splits on whitespace: the + // output directory lives under the consumer's project, whose path may + // contain spaces. hooks_runner never passes RUSTFLAGS to a build hook, + // so only the encoded variable is inherited. + final rustFlags = [ + ...?environment['CARGO_ENCODED_RUSTFLAGS'] + ?.split('\x1f') + .where((flag) => flag.isNotEmpty), + for (final (from, to) in [ + (input.packageRoot.toFilePath(), '/payjoin/package'), + (input.outputDirectory.toFilePath(), '/payjoin/output'), + if (cargoHome != null) (cargoHome, '/cargo'), + if (rustupHome != null) (rustupHome, '/rustup'), + ]) + '--remap-path-prefix=${strip(from)}=$to', + ].join('\x1f'); + + // The path overlay doubles as the development marker. `.pubignore` + // withholds `.cargo/` from the archive, so the file only exists in a + // repository checkout. It cannot be an environment variable: hooks_runner + // spawns build hooks with an allowlisted environment, and a project + // specific name never survives that filter. final localCargoConfig = File.fromUri( - input.packageRoot.resolve('.cargo/config.toml'), + input.packageRoot.resolve('.cargo/config.local.toml'), ); + final isWorkspaceBuild = localCargoConfig.existsSync(); await RustBuilder( assetName: 'uniffi:payjoin_ffi', - features: localCargoConfig.existsSync() ? ['_test-utils'] : [], + features: isWorkspaceBuild ? const ['_test-utils'] : const [], + extraCargoBuildArgs: [ + // Cargo does not discover this file name on its own, and it + // resolves a relative --config path against its own working + // directory, so pass the absolute one. + if (isWorkspaceBuild) ...['--config', localCargoConfig.path], + // A published package ships the native/Cargo.lock its release was + // staged with, so consumers build that graph instead of resolving + // their own. The workspace overlay redirects `payjoin-ffi` to local + // paths, which changes the graph, so development builds run unlocked. + if (!isWorkspaceBuild) '--locked', + ], + extraCargoEnvironmentVariables: { + // Cargo takes rustflags from the first source that sets any, and + // this variable comes before every Cargo config file. It therefore + // replaces `build.rustflags`, `target..rustflags` and + // `target..rustflags` from the consumer's config, rather than + // adding to them. + 'CARGO_ENCODED_RUSTFLAGS': rustFlags, + }, ).run(input: input, output: output); }); }