Lock and remap Dart native builds for reproducibility

.github/workflows/dart.yml

@@ -66,3 +66,62 @@ jobs:
           name: payjoin-dart-pub-package
           path: ${{ runner.temp }}/payjoin-dart.tar.gz
           if-no-files-found: error
+
+  # Build the staged package twice the way a consumer would and require
+  # byte-identical native libraries. The two builds differ in every path
+  # the hook remaps: package root, app directory, and the HOME that holds
+  # the Cargo, rustup and pub caches.
+  reproducible-linux:
+    name: "Reproduce the Linux native library"
+    needs: verify-pub-package
+    runs-on: ubuntu-26.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false # unneeded: no git auth beyond checkout
+      - name: Download the staged package
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: payjoin-dart-pub-package
+          path: dist
+      - name: Extract the staged package
+        run: |
+          mkdir -p "$RUNNER_TEMP/stage"
+          tar -xzf dist/payjoin-dart.tar.gz -C "$RUNNER_TEMP/stage"
+      - name: "Install rustup"
+        uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
+        with:
+          toolchain: 1.85.1
+      # Flutter 3.44.9 bundles the Dart SDK Bull Bitcoin builds with.
+      - name: Set up Flutter
+        uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2.23.0
+        with:
+          channel: stable
+          flutter-version: 3.44.9
+      - name: Build the consumer twice and compare the libraries
+        run: |
+          set -euo pipefail
+          # The space checks that the hook keeps such paths in one flag.
+          for build in first "second/with space"; do
+            root="$RUNNER_TEMP/linux/$build"
+            mkdir -p "$root/home"
+            # The runner image exports CARGO_HOME and RUSTUP_HOME, which
+            # hooks_runner withholds from the hook, so the hook uses the
+            # defaults under HOME. Point both there so the toolchain
+            # installed here is the one the hook finds.
+            export HOME="$root/home" CARGO_HOME="$root/home/.cargo" \
+              RUSTUP_HOME="$root/home/.rustup" PUB_CACHE="$root/home/.pub-cache"
+            rustup toolchain install 1.85.1 --profile minimal
+            cp -a "$RUNNER_TEMP/stage" "$root/payjoin"
+            cp -a payjoin-ffi/dart/contrib/consumer "$root/app"
+            (cd "$root/app" && dart pub get && dart run bin/main.dart)
+          done
+          # Cargo leaves a second copy under release/deps, which is not
+          # the hook's output.
+          library() {
+            find "$RUNNER_TEMP/linux/$1/app/.dart_tool" -type f \
+              -path '*/x86_64-unknown-linux-gnu/release/libpayjoin_ffi_wrapper.so'
+          }
+          cmp "$(library first)" "$(library "second/with space")"

payjoin-ffi/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## Unreleased
 
+- Dart: the native library no longer records the paths it was built
+  from, so two builds of one release from different directories produce
+  the same bytes. The build hook sets `CARGO_ENCODED_RUSTFLAGS`, so
+  `rustflags` from your Cargo config no longer apply to the payjoin
+  native build.
+- Dart: the package ships the `native/Cargo.lock` it was released with
+  and the build hook builds it with `--locked`, so every build compiles
+  the dependency versions CI tested.
+
 ## [0.25.0]
 
 - Publish the Python, JavaScript, C#, and Dart packages from one signed

payjoin-ffi/Cargo.toml

@@ -26,7 +26,7 @@ async-trait = "0.1.89"
 getrandom = "0.2.10"
 lazy_static = "1.5.0"
 payjoin = { version = "1.2.0", features = ["v1", "v2"] }
-payjoin-test-utils = { version = "0.0.1", optional = true }
+payjoin-test-utils = { path = "../payjoin-test-utils", optional = true }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.149"
 thiserror = "2.0.18"

payjoin-ffi/dart/.cargo/config.local.toml

@@ -0,0 +1,13 @@
+# Local Cargo overlay for development in the monorepo. The name keeps it
+# out of Cargo's automatic config discovery, so only hook/build.dart reads
+# it, through an explicit --config. Its presence is also how the hook tells
+# a repository checkout from a published archive, which .pubignore strips
+# .cargo/ from.
+[patch."https://github.com/payjoin/rust-payjoin.git"]
+payjoin-ffi = { path = ".." }
+payjoin = { path = "../../payjoin" }
+
+[patch.crates-io]
+payjoin = { path = "../../payjoin" }
+payjoin-mailroom = { path = "../../payjoin-mailroom" }
+payjoin-test-utils = { path = "../../payjoin-test-utils" }

payjoin-ffi/dart/.gitignore

@@ -12,6 +12,3 @@ doc/api/
 
 # Auto-generated bindings
 lib/payjoin.dart
-
-# Local Cargo overlay for monorepo dev
-!.cargo/config.toml

payjoin-ffi/dart/.pubignore

@@ -16,7 +16,6 @@ scripts/
 
 # Cargo build output
 native/target/
-native/Cargo.lock
 
 # Local-only test helper re-export
 lib/test_utils.dart

payjoin-ffi/dart/CONTRIBUTING.md

@@ -25,6 +25,35 @@ bash ./scripts/generate_bindings.sh
 dart test
 ```
 
+### The native build hook
+
+`hook/build.dart` builds the `native/` wrapper crate for the consumer's
+target. It takes the presence of `.cargo/config.local.toml` as the sign
+that it is running inside this repository rather than from a published
+archive, because `.pubignore` withholds `.cargo/` from that archive. In
+the repository the overlay redirects `payjoin-ffi` to the workspace and
+the hook enables `_test-utils`. From an archive the hook builds the
+shipped `native/Cargo.lock` with `--locked`.
+
+This cannot be an environment variable. `hooks_runner` spawns build hooks
+with an allowlisted environment (`HOME`, `PATH` and a few toolchain
+prefixes), so a project specific variable never reaches the hook.
+
+The hook also passes `--remap-path-prefix` for the package, output, Cargo
+and Rustup directories, so the artifact does not record where it was
+built. The prefixes go through `CARGO_ENCODED_RUSTFLAGS` rather than
+`RUSTFLAGS`, since Cargo splits the latter on whitespace and a consumer's
+project path may contain spaces.
+
+### Reproducible native builds
+
+CI builds the staged package twice the way a Dart consumer does, with
+`contrib/consumer`, and requires byte-identical host libraries. Each
+build gets its own copy of the stage, its own app directory and its own
+`HOME`, and with it its own Cargo, rustup and pub homes. The
+`reproducible-linux` job in `.github/workflows/dart.yml` lists the
+steps.
+
 ## Releasing
 
 Maintainer instructions for publishing to
@@ -49,8 +78,16 @@ validates the archive with a publish dry run
 1. The preparation script stages a separate package and pins its Rust
    dependency to the checkout used to generate production bindings. The
    development pin in `native/Cargo.toml` is replaced only in that staging
-   directory. Cargo overlays and local lockfiles are excluded. Both the
-   publish dry run and publication use the staged package. For local checks:
+   directory. Cargo overlays are excluded. The staged `native/Cargo.lock`
+   is seeded from `Cargo-recent.lock` and resolved against that revision,
+   so consumers build the versions CI tested. A dependency fix reaches
+   Dart consumers through `Cargo-recent.lock` and a new release. Staging
+   fetches the pinned commit from GitHub, so it must be pushed. When a
+   `payjoin` or `payjoin-mailroom` version that `payjoin-ffi` or
+   `payjoin-test-utils` requires is not on crates.io yet, staging takes
+   it from the same revision and writes an `UNPUBLISHED_CORE` marker,
+   which `contrib/publish.sh` refuses. Both the publish dry run and
+   publication use the staged package. For local checks:
 
    ```shell
    nix develop .#dart -c ./payjoin-ffi/dart/contrib/prepare-publish.sh /tmp/payjoin-pub

payjoin-ffi/dart/README.md

@@ -22,4 +22,23 @@ For OHTTP relay helpers:
 import 'package:payjoin/http.dart' as payjoin_http;
 ```
 
+### Native build
+
+The package compiles its Rust library on your machine through a Dart
+build hook, with the toolchain named in `native/rust-toolchain.toml`.
+
+- **Hosted releases are locked.** A version from pub.dev ships the
+  `native/Cargo.lock` it was released with, and the hook builds it with
+  `--locked`. Every release compiles the same dependency versions,
+  whenever and wherever you build it.
+- **Git dependencies are development builds.** A `git:` dependency on
+  this repository checks out the monorepo layout, which the hook treats
+  as a development build: it builds `payjoin-ffi` from the checkout,
+  enables test-only helpers and resolves dependencies afresh, without a
+  lockfile. Use a hosted release in production.
+- **Cargo config `rustflags` do not apply.** The hook sets
+  `CARGO_ENCODED_RUSTFLAGS` to strip build paths from the library, and
+  Cargo then ignores `build.rustflags` and `target.*.rustflags` from any
+  Cargo config file for this build.
+
 Made with [uniffi-dart](https://github.com/Uniffi-Dart/uniffi-dart)

payjoin-ffi/dart/contrib/consumer/bin/main.dart

@@ -0,0 +1,10 @@
+import 'package:payjoin/payjoin.dart' as payjoin;
+
+void main() {
+  // Parsing goes through the native library, so this only runs once the
+  // payjoin build hook has produced the asset.
+  final uri = payjoin.Uri.parse(
+    uri: 'bitcoin:12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX?amount=1',
+  );
+  print(uri.address());
+}

payjoin-ffi/dart/contrib/consumer/pubspec.yaml

@@ -0,0 +1,14 @@
+name: payjoin_consumer
+description: >-
+  A minimal Dart program that builds the staged payjoin package the way a
+  desktop or server consumer depending on it from pub.dev would.
+publish_to: none
+version: 0.0.0
+
+environment:
+  sdk: ^3.10.0
+
+dependencies:
+  # CI places a copy of the staged package next to the program.
+  payjoin:
+    path: ../payjoin

payjoin-ffi/dart/contrib/prepare-publish.sh

@@ -28,14 +28,67 @@ PAYJOIN_FFI_FEATURES="" bash ./scripts/generate_bindings.sh
 
 revision="$(git rev-parse HEAD)"
 mkdir "$stage"
-# Include generated bindings while excluding development Cargo overlays and
-# lockfiles, then pin the native crate to the checkout that generated them.
+# Include generated bindings while excluding development Cargo overlays,
+# then pin the native crate to the checkout that generated them.
 rsync -a --exclude=.git --exclude-from=.pubignore ./ "$stage/"
 sed -i "s/rev = \"[0-9a-f]*\"/rev = \"$revision\"/" "$stage/native/Cargo.toml"
 grep -Fq "rev = \"$revision\"" "$stage/native/Cargo.toml" || {
     echo "Could not pin the staged native crate to $revision" >&2
     exit 1
 }
+
+# payjoin-ffi and payjoin-test-utils come from the pinned revision, but the
+# release crates they require, payjoin and payjoin-mailroom, come from
+# crates.io. Between a version bump of one of those and its crates.io
+# release, the lock below could not resolve it. Take such a crate from the
+# pinned revision instead so pull requests can still stage, and leave a
+# marker that publish.sh refuses: a published package must build against
+# crates its consumers can get from crates.io.
+source "$REPO_ROOT/contrib/release/crates.sh"
+# crates.io answers 404 for a version it does not have. Any other failure,
+# such as a rate limit or an outage, must stop here rather than read as
+# unpublished, or a release run would stage a package publish.sh refuses.
+crates_io_status() {
+    curl -sL --retry 3 -o /dev/null -w '%{http_code}' \
+        -H "User-Agent: rust-payjoin release tooling" \
+        "https://crates.io/api/v1/crates/$1/$2" || true
+}
+siblings="$(sibling_deps payjoin-ffi && sibling_deps payjoin-test-utils)"
+[[ $siblings == *payjoin$'\t'* ]] || {
+    echo "Could not read the payjoin version payjoin-ffi requires" >&2
+    exit 1
+}
+while IFS=$'\t' read -r dep req; do
+    version="${req#^}"
+    version="${version#=}"
+    status="$(crates_io_status "$dep" "$version")"
+    case $status in
+        200) continue ;;
+        404) ;;
+        *)
+            echo "Could not check $dep $version on crates.io (HTTP $status)" >&2
+            exit 1
+            ;;
+    esac
+    echo "==> $dep $version is not on crates.io; staging it from $revision"
+    cat >>"$stage/native/Cargo.toml" <<EOF
+
+# Added by prepare-publish.sh: $dep $version is not on crates.io yet.
+[patch.crates-io.$dep]
+git = "https://github.com/payjoin/rust-payjoin.git"
+rev = "$revision"
+EOF
+    printf '%s %s comes from %s, not crates.io\n' "$dep" "$version" \
+        "$revision" >>"$stage/UNPUBLISHED_CORE"
+done < <(sort -u <<<"$siblings")
+
+# Ship the dependency graph this release was tested against; hook/build.dart
+# builds it with --locked. Seeding from the maintained lockfile keeps every
+# pin that still applies, so cargo resolves only what differs from the
+# workspace, chiefly payjoin-ffi coming from the pinned revision.
+echo "==> Locking the staged native crate..."
+cp "$REPO_ROOT/Cargo-recent.lock" "$stage/native/Cargo.lock"
+cargo update --workspace --manifest-path "$stage/native/Cargo.toml"
 cd "$stage"
 
 echo "==> Verifying the publish archive..."

payjoin-ffi/dart/contrib/publish.sh

@@ -11,6 +11,13 @@ grep -Fq "rev = \"$revision\"" native/Cargo.toml || {
     echo "Staged native/Cargo.toml must pin the release checkout $revision" >&2
     exit 1
 }
+# A stage built against an unreleased core is only fit for pull request
+# checks; consumers must build the core they can fetch from crates.io.
+if [[ -e UNPUBLISHED_CORE ]]; then
+    echo "Refusing to publish a stage built against an unpublished core:" >&2
+    cat UNPUBLISHED_CORE >&2
+    exit 1
+fi
 
 # Publish the package to pub.dev via automated publishing (OIDC).
 # `dart pub publish` has no non-interactive authentication mode: without a

payjoin-ffi/dart/hook/build.dart

@@ -5,13 +5,72 @@ import 'package:native_toolchain_rust/native_toolchain_rust.dart';
 
 void main(List<String> args) async {
   await build(args, (input, output) async {
+    final environment = Platform.environment;
+    final separator = Platform.pathSeparator;
+    String? nonEmpty(String? value) =>
+        value == null || value.isEmpty ? null : value;
+    // Directory URIs render with a trailing separator, and rustc substitutes
+    // the prefix literally: without this, <packageRoot>/native/src/lib.rs
+    // would remap to /payjoin/packagenative/src/lib.rs.
+    String strip(String path) => path.length > 1 && path.endsWith(separator)
+        ? path.substring(0, path.length - separator.length)
+        : path;
+    final home = nonEmpty(environment['HOME'] ?? environment['USERPROFILE']);
+    final cargoHome =
+        nonEmpty(environment['CARGO_HOME']) ??
+        (home == null ? null : '${strip(home)}$separator.cargo');
+    final rustupHome =
+        nonEmpty(environment['RUSTUP_HOME']) ??
+        (home == null ? null : '${strip(home)}$separator.rustup');
+    // Encoded rather than RUSTFLAGS, which Cargo splits on whitespace: the
+    // output directory lives under the consumer's project, whose path may
+    // contain spaces. hooks_runner never passes RUSTFLAGS to a build hook,
+    // so only the encoded variable is inherited.
+    final rustFlags = [
+      ...?environment['CARGO_ENCODED_RUSTFLAGS']
+          ?.split('\x1f')
+          .where((flag) => flag.isNotEmpty),
+      for (final (from, to) in [
+        (input.packageRoot.toFilePath(), '/payjoin/package'),
+        (input.outputDirectory.toFilePath(), '/payjoin/output'),
+        if (cargoHome != null) (cargoHome, '/cargo'),
+        if (rustupHome != null) (rustupHome, '/rustup'),
+      ])
+        '--remap-path-prefix=${strip(from)}=$to',
+    ].join('\x1f');
+
+    // The path overlay doubles as the development marker. `.pubignore`
+    // withholds `.cargo/` from the archive, so the file only exists in a
+    // repository checkout. It cannot be an environment variable: hooks_runner
+    // spawns build hooks with an allowlisted environment, and a project
+    // specific name never survives that filter.
     final localCargoConfig = File.fromUri(
-      input.packageRoot.resolve('.cargo/config.toml'),
+      input.packageRoot.resolve('.cargo/config.local.toml'),
     );
+    final isWorkspaceBuild = localCargoConfig.existsSync();
 
     await RustBuilder(
       assetName: 'uniffi:payjoin_ffi',
-      features: localCargoConfig.existsSync() ? ['_test-utils'] : [],
+      features: isWorkspaceBuild ? const ['_test-utils'] : const [],
+      extraCargoBuildArgs: [
+        // Cargo does not discover this file name on its own, and it
+        // resolves a relative --config path against its own working
+        // directory, so pass the absolute one.
+        if (isWorkspaceBuild) ...['--config', localCargoConfig.path],
+        // A published package ships the native/Cargo.lock its release was
+        // staged with, so consumers build that graph instead of resolving
+        // their own. The workspace overlay redirects `payjoin-ffi` to local
+        // paths, which changes the graph, so development builds run unlocked.
+        if (!isWorkspaceBuild) '--locked',
+      ],
+      extraCargoEnvironmentVariables: {
+        // Cargo takes rustflags from the first source that sets any, and
+        // this variable comes before every Cargo config file. It therefore
+        // replaces `build.rustflags`, `target.<triple>.rustflags` and
+        // `target.<cfg>.rustflags` from the consumer's config, rather than
+        // adding to them.
+        'CARGO_ENCODED_RUSTFLAGS': rustFlags,
+      },
     ).run(input: input, output: output);
   });
 }