diff --git a/.cargo/config.toml b/.cargo/config.toml index c5f4371a6f6..ce50e6c5f7b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -11,6 +11,6 @@ moq = "run --bin moq --" # name carries no feature hash, so a build with other features replaces # `libmoq_ffi.so` under everyone reading it; such a build takes a target # directory of its own (py, `sh/size/collect.sh`, `build.sh --no-default-features`) -# or a `--target` triple no default build uses (dart's `testing` build). +# or dart's `target/dart` directory for its `testing` build. # Aliases resolve from the working directory, so run it inside the checkout. build-ffi = "build --locked --package moq-ffi --features openh264" diff --git a/.github/ffi.md b/.github/ffi.md index c028d950c49..9040799eed3 100644 --- a/.github/ffi.md +++ b/.github/ffi.md @@ -97,10 +97,14 @@ stripped for `moq-ffi`. The toolchain action installed stable Rust 1.99.0, but the repository's `rust-toolchain.toml` selects Rust 1.98.1 (`48a229cea`) for the actual Cargo builds on both paths. C++ invokes `cargo build --locked --package moq-ffi --release` through -CMake; the shared candidate adds `--target ` through +CMake; in this sampled revision, the shared candidate adds `--target ` through `rs/moq-ffi/build.sh`. C++ is always a native host build. Only the shared ARM Linux candidate crosses from an x86 Linux host. +Current native packaging omits `--target`, matching the C++ invocation. The +table and debug-path differences below describe the older sampled invocation; +only cross-builds still pass an explicit target. + | Target | C++ host | Shared candidate host | Native archive comparison | | --- | --- | --- | --- | | x86\_64-unknown-linux-gnu | Ubuntu 24.04, image 20261004.327.1 | Same image and architecture | 772 common objects byte-identical; one differently named Rust LTO object | diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index 9ec05d4068d..a178c017e75 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -102,8 +102,12 @@ jobs: if: needs.scope.outputs.c == 'true' || needs.scope.outputs.cpp == 'true' || needs.scope.outputs.obs == 'true' with: generator: cpp + # Later suites are independent once tool and generator setup succeeds. + - name: Setup complete + id: setup + run: echo "Platform tools ready" - name: Rust compile and tests - if: needs.scope.outputs.rs == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.rs == 'true') }} shell: bash env: CI_BASE: ${{ github.event_name != 'pull_request' && '--all' || '' }} @@ -111,15 +115,15 @@ jobs: just rs windows "$CI_BASE" just rs windows-test - name: C - if: needs.scope.outputs.c == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.c == 'true') }} shell: bash run: just c check - name: C++ - if: needs.scope.outputs.cpp == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.cpp == 'true') }} shell: bash run: just cpp check - name: OBS - if: needs.scope.outputs.obs == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.obs == 'true') }} shell: bash run: just obs package --target x86_64-pc-windows-msvc --output "$RUNNER_TEMP/obs" @@ -151,36 +155,40 @@ jobs: if: needs.scope.outputs.obs == 'true' with: generator: cpp + # Later suites are independent once tool and generator setup succeeds. + - name: Setup complete + id: setup + run: echo "Platform tools ready" - name: Rust compile and tests - if: needs.scope.outputs.rs == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.rs == 'true') }} env: CI_BASE: ${{ github.event_name != 'pull_request' && '--all' || '' }} run: | just rs macos "$CI_BASE" just rs macos-test - name: Swift simulator check - if: needs.scope.outputs.swift == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.swift == 'true') }} run: just swift simulator - name: Swift - if: needs.scope.outputs.swift == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.swift == 'true') }} run: just swift check # At a token budget, so the runner keeps working at this commit; rust2's # `just bench-ffi` measures it at its pinned moq revision. - name: Swift binding benchmark runner - if: inputs.benchmarks + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (inputs.benchmarks) }} env: MOQ_BENCH_SECONDS: "0.05" run: | just swift bench test -s target/bench/ffi/swift.json - name: OBS - if: needs.scope.outputs.obs == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.obs == 'true') }} run: just obs package --target aarch64-apple-darwin --output "$RUNNER_TEMP/obs" - name: Installer fixtures - if: needs.scope.outputs.moq_sh == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.moq_sh == 'true') }} run: just infra moq-sh test - name: Installer smoke - if: needs.scope.outputs.moq_sh == 'true' + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.moq_sh == 'true') }} env: GH_TOKEN: ${{ github.token }} run: just infra moq-sh smoke diff --git a/dart/moq_ffi/hook/build.dart b/dart/moq_ffi/hook/build.dart index 16de949eef6..70566c00e06 100644 --- a/dart/moq_ffi/hook/build.dart +++ b/dart/moq_ffi/hook/build.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:code_assets/code_assets.dart'; @@ -88,17 +89,38 @@ Future<_Resolved> _resolveLibrary( /// another recipe with different features. Cargo's own timestamp check makes /// the no-op case cheap. Future<_Resolved> _build(Uri workspace, String target, String library) async { - // The shared default build, an alias .cargo/config.toml defines for Cargo - // runs in the workspace; it ships the software H.264 fallback. A checkout build is a test build, carrying the hooks - // leak_test.dart reads; `--target` keeps it out of the default build's - // directory, since a cdylib's file name carries no feature hash. - final cargoArgs = ['build-ffi', '--features', 'testing']; + // Keep the leak-test allocator out of the shared cdylib. A separate output + // directory preserves Cargo's native compilation mode and dependency cache + // identities; passing the host as --target would split them. + final targetDir = workspace.resolve('target/dart/'); + final cargoArgs = [ + 'build-ffi', + '--message-format=json-render-diagnostics', + '--features', + 'testing', + '--target-dir', + targetDir.toFilePath(), + ]; + final rustc = await Process.run('rustc', [ + '-vV', + ], workingDirectory: workspace.toFilePath()); + if (rustc.exitCode != 0) { + throw StateError('rustc -vV failed: ${rustc.stderr}'); + } + final host = LineSplitter.split( + rustc.stdout as String, + ).firstWhere((line) => line.startsWith('host: ')).substring('host: '.length); + final native = target == host; + final android = target.contains('-android'); // Android needs the NDK's linker and sysroot, which only cargo-ndk sets up. // This mirrors rs/moq-ffi/build.sh. - final args = target.contains('-android') + final args = android ? ['ndk', '--target', target, '--platform', '24', '--', ...cargoArgs] - : [...cargoArgs, '--target', target]; + : [ + ...cargoArgs, + if (!native) ...['--target', target], + ]; // The iOS floor release-ffi.yml builds with, which also avoids its // ___chkstk_darwin linker errors. The hook's filtered environment drops it. @@ -116,7 +138,32 @@ Future<_Resolved> _build(Uri workspace, String target, String library) async { ); } - final built = workspace.resolve('target/$target/debug/$library'); + final scoped = targetDir.resolve('$target/debug/$library'); + final Uri built; + if (android) { + // cargo-ndk consumes Cargo's artifact messages and always sets --target. + built = scoped; + } else { + final artifacts = LineSplitter.split(result.stdout as String) + .map((line) => jsonDecode(line) as Map) + .where( + (message) => + message['reason'] == 'compiler-artifact' && + (message['target'] as Map)['name'] == 'moq_ffi', + ) + .expand((message) => (message['filenames'] as List).cast()) + .map((path) => File(path).uri) + .where((uri) => uri.pathSegments.last == library) + .toSet(); + if (artifacts.length != 1) { + throw StateError('expected one moq-ffi cdylib from Cargo: $artifacts'); + } + built = artifacts.single; + final unscoped = targetDir.resolve('debug/$library'); + if (built != scoped && !(native && built == unscoped)) { + throw StateError('Cargo artifact does not match target $target: $built'); + } + } if (!await File.fromUri(built).exists()) { throw StateError('cargo did not produce ${built.toFilePath()}'); } diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index d310db6a9e8..7c063c4e792 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -167,7 +167,7 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out - [Verify the R2 Rust cache](/quest/m0/ci/mbx-r2-verify.md) - confirm the hosted Nightly jobs reuse the previous Nightly, then measure restore time and expiry - [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - confirm releases use `nix.moq.dev`, check its bucket sizes, and retire Cachix -- [Unify dependency features](/quest/m0/ci/hakari.md) - align the FFI builds' features and record the hosted platform jobs' hits once the workspace-hack lands +- [Record hosted cargo-hakari evidence](/quest/m0/ci/hakari.md) - compare scheduled macOS and Windows cache hits and estimated compiler work saved before and after the workspace-hack - [Drop main from the publish Environment](/quest/m0/ci/publish-release-only.md) - the maintainer narrows `publish` to `release` and release tags once no job naming it runs as `main` - [Binary release publishers run on their tag](/quest/m0/ci/release-tag-publishers.md) - brew, Go and Swift releases stop running through `workflow_run` on main - [The bench comparison builds moq-gst's catalog bench on its base](/quest/m0/ci/bench-base-build.md) - A paired bench comparison that selects `rust/gst/moq-gst/catalog` against `origin/main` builds the base side and reports a comparison, instead of marking it unavailable with "base build failed" diff --git a/quest/m0/ci/hakari.md b/quest/m0/ci/hakari.md index 377f473477f..9ac40761ed3 100644 --- a/quest/m0/ci/hakari.md +++ b/quest/m0/ci/hakari.md @@ -1,31 +1,60 @@ -# [S] Unify dependency features with cargo-hakari +# [S] Record hosted cargo-hakari cache evidence ## Goal -The FFI builds compile each dependency with the same features as each other, -so mbx's cache hits across them, and Nightly's hosted platform jobs record -their hit rate and CPU time before and after the workspace-hack. +Nightly's hosted macOS and Windows jobs record their mbx cache-hit evidence and +estimated compiler work saved before and after the workspace-hack. ## Plan -Decided with the maintainer on 2026-10-25: use cargo-hakari, the standard tool -from guppy's authors, rather than only trimming CI's feature permutations. mbx -keys every compilation on its inputs, so a dependency built with a different -feature set in another job is a miss, however warm the cache. - -[moq-dev/moq#6985](https://github.com/moq-dev/moq/pull/6985) added -`rs/workspace-hack` as a path-only dev-dependency of every member outside -`.config/hakari.toml`'s excludes. Test, lint, and bench builds now unify; -library-only builds do not, since dev-dependencies stay out of them. That -leaves the FFI builds (`cargo build --release -p moq-ffi` in the bindings and -release workflows), which also split on their own flags: the Dart build passes -the host triple as `--target` and adds the `testing` feature. Align those -invocations rather than add the hack to them, unless the PR's decision on the -layout says otherwise. - -Nightly's macOS and Windows jobs read and write R2, so their before and after -can only come from scheduled Nightlies on `main` -(`gh run list --workflow nightly.yml --branch main --event schedule`): compare -the mbx summaries of the last run before the merge with the first two after. - -Public API: none; the workspace-hack crate is not published. +Decided 2026-11-04 (maintainer, /quest-unblock): ✅ accept the mbx cache-hit +evidence and estimated compiler savings as sufficient. Not chosen: add per-job +CPU instrumentation and measure CPU totals. The estimates are not measured CPU +savings. + +Decided with the maintainer on 2026-10-25: use cargo-hakari rather than only +trimming CI's feature permutations. +[moq-dev/moq#6985](https://github.com/moq-dev/moq/pull/6985) added the path-only +dev-dependency layout, unifying test, lint, and bench builds. Keep that layout; +library-only builds do not need the hack. + +The remaining FFI invocation alignment is implemented in +[moq-dev/moq#7666](https://github.com/moq-dev/moq/pull/7666): default builds use +`cargo build-ffi` with `openh264`, native builds omit `--target`, and only cross +builds name a target. Python and Dart host tests retain `testing` for the manual +clock and leak allocator, with separate target directories so they cannot +replace the default cdylib. Their third-party feature identities match the +default host build; the intentional `kio/manual` split affects workspace crates. + +Compare the mbx summaries of the last scheduled Nightly before the +[2026-11-02 merge](https://github.com/moq-dev/moq/commit/8e53bf32185fec578e460ff6ccbe41a80a039a6a) +with the first two after. Query only scheduled runs on `main`: +`gh run list --workflow nightly.yml --branch main --event schedule`. +The baseline is [2026-11-01](https://github.com/moq-dev/moq/actions/runs/41313466339). +The [2026-11-02 run](https://github.com/moq-dev/moq/actions/runs/41483654426) +has partial macOS summaries before a websocket-test timeout and no Windows +compilation (NASM installation returned HTTP 503). +[2026-11-03](https://github.com/moq-dev/moq/actions/runs/41656613559) completed +both platform jobs. [2026-11-04](https://github.com/moq-dev/moq/actions/runs/41852517544) +completed Windows, but cancelled macOS before any step ran. Windows therefore +has two complete after-runs; macOS has only one. Logs expose object-cache counts +and estimated compiler work saved, but no measured user/system CPU totals. +Do not call elapsed time CPU time, or attribute these earlier runs to the FFI +alignment that has not landed yet. + +The 2026-11-04 maintainer decision accepts hosted CI as the strict Nix gate; +a local namespace-probe skip is expected. The pkg-config and Gradle repairs +have landed, so neither remains an external prerequisite. + +Public API and wire: none. + +## Waiting + +- moq-dev/moq: inspect the next scheduled Nightly for a second complete macOS + after-run. As of 2026-11-04, the latest scheduled run is + [Nov 4](https://github.com/moq-dev/moq/actions/runs/41852517544); its + [macOS job](https://github.com/moq-dev/moq/actions/runs/41852517544/job/126656206251) + was cancelled with no steps. Run + `gh run list --repo moq-dev/moq --workflow nightly.yml --event schedule`, + inspect the next macOS job and record its mbx summaries alongside Nov 3. + This measurement wait outlives the implementation PR; never dispatch Nightly. diff --git a/rs/moq-ffi/build.sh b/rs/moq-ffi/build.sh index d8075080c40..08336d42ad7 100755 --- a/rs/moq-ffi/build.sh +++ b/rs/moq-ffi/build.sh @@ -15,9 +15,8 @@ RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" # Resolve cargo target directory (respects CARGO_TARGET_DIR, .cargo/config, etc.) -TARGET_BASE_DIR=$(cargo metadata --format-version 1 --manifest-path "$WORKSPACE_DIR/Cargo.toml" --no-deps 2>/dev/null | - sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p' || - echo "$WORKSPACE_DIR/target") +TARGET_BASE_DIR=$(cargo metadata --format-version 1 --manifest-path "$WORKSPACE_DIR/Cargo.toml" --no-deps | + jq -er .target_directory) # Defaults TARGET="" @@ -122,7 +121,7 @@ build_target() { # The shared default build (.cargo/config.toml): releases carry the OpenH264 # software fallback, the only encoder on API-24 Android and the Apple # simulators. It is a no-op under --no-default-features. - local cargo_args=(build-ffi --release --target-dir "$TARGET_BASE_DIR" --manifest-path "$WORKSPACE_DIR/Cargo.toml") + local cargo_args=(build-ffi --release --message-format=json-render-diagnostics --target-dir "$TARGET_BASE_DIR" --manifest-path "$WORKSPACE_DIR/Cargo.toml") if [[ "$NO_DEFAULT_FEATURES" == true ]]; then cargo_args+=(--no-default-features) fi @@ -136,31 +135,34 @@ build_target() { rustup target add "$target" fi + local messages if is_android "$target"; then - # Android targets use cargo-ndk + # cargo-ndk consumes Cargo's artifact messages. It always passes an + # explicit target, so its output directory is unambiguous. (cd "$WORKSPACE_DIR" && cargo ndk --target "$target" --platform 24 -- "${cargo_args[@]}") + BUILT_LIBRARY="$TARGET_BASE_DIR/$target/release/libmoq_ffi.so" + return else # Set up cross-compilation for Linux ARM64 - if [[ "$target" == "aarch64-unknown-linux-gnu" ]]; then + if [[ "$target" == "aarch64-unknown-linux-gnu" && "$target" != "$HOST_TARGET" ]]; then export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc fi - (cd "$WORKSPACE_DIR" && cargo "${cargo_args[@]}" --target "$target") - fi -} - -# Find the cdylib for binding generation -find_cdylib() { - local target="$1" - local target_dir="$TARGET_BASE_DIR/$target/release" - - if [[ "$target" == *"-apple-"* ]]; then - echo "$target_dir/libmoq_ffi.dylib" - elif [[ "$target" == *"-windows-"* ]]; then - echo "$target_dir/moq_ffi.dll" - else - echo "$target_dir/libmoq_ffi.so" + # An explicit host --target changes dependency cache identities too. + if [[ "$target" != "$HOST_TARGET" ]]; then + cargo_args+=(--target "$target") + fi + messages=$(cd "$WORKSPACE_DIR" && cargo "${cargo_args[@]}") fi + local library + case "$target" in + *-apple-*) library=libmoq_ffi.dylib ;; + *-windows-*) library=moq_ffi.dll ;; + *) library=libmoq_ffi.so ;; + esac + BUILT_LIBRARY=$(jq -ers --arg root "$TARGET_BASE_DIR" --arg target "$target" \ + --arg host "$HOST_TARGET" --arg profile release --arg library "$library" \ + -f "$WORKSPACE_DIR/sh/rs/ffi-artifact.jq" <<<"$messages") } # Generate language bindings into $OUTPUT_DIR/bindings//. Tarring is @@ -228,7 +230,7 @@ if [[ "$BINDINGS_ONLY" == true ]]; then cdylib="$LIBRARY" else build_target "$HOST_TARGET" - cdylib=$(find_cdylib "$HOST_TARGET") + cdylib="$BUILT_LIBRARY" fi if [[ ! -f "$cdylib" ]]; then echo "Error: cdylib not found at $cdylib" >&2 @@ -251,9 +253,9 @@ echo "Packaging $NAME..." rm -rf "$PACKAGE_DIR" mkdir -p "$PACKAGE_DIR/lib" +TARGET_DIR=$(dirname "$BUILT_LIBRARY") if [[ "$TARGET" == *"-windows-"* ]]; then - TARGET_DIR="$TARGET_BASE_DIR/$TARGET/release" cp "$TARGET_DIR/moq_ffi.dll" "$PACKAGE_DIR/lib/" cp "$TARGET_DIR/moq_ffi.dll.lib" "$PACKAGE_DIR/lib/" 2>/dev/null || true cp "$TARGET_DIR/moq_ffi.lib" "$PACKAGE_DIR/lib/" 2>/dev/null || true @@ -263,24 +265,20 @@ elif is_ios "$TARGET"; then # Assets hook bundles the cdylib (the SDK does not implement static # linking). rustc emits a PLATFORM_IOS dylib for device and simulator # alike, so this is a packaging choice rather than a platform limit. - TARGET_DIR="$TARGET_BASE_DIR/$TARGET/release" cp "$TARGET_DIR/libmoq_ffi.a" "$PACKAGE_DIR/lib/" cp "$TARGET_DIR/libmoq_ffi.dylib" "$PACKAGE_DIR/lib/" elif is_android "$TARGET"; then # Android: shared library only - TARGET_DIR="$TARGET_BASE_DIR/$TARGET/release" cp "$TARGET_DIR/libmoq_ffi.so" "$PACKAGE_DIR/lib/" elif [[ "$TARGET" == *"-apple-"* ]]; then - TARGET_DIR="$TARGET_BASE_DIR/$TARGET/release" cp "$TARGET_DIR/libmoq_ffi.a" "$PACKAGE_DIR/lib/" cp "$TARGET_DIR/libmoq_ffi.dylib" "$PACKAGE_DIR/lib/" else # Linux: ship both the cdylib (.so, consumed by JNA/Kotlin and # maturin/Python) and the staticlib (.a, linked by the Go module's cgo). - TARGET_DIR="$TARGET_BASE_DIR/$TARGET/release" cp "$TARGET_DIR/libmoq_ffi.so" "$PACKAGE_DIR/lib/" cp "$TARGET_DIR/libmoq_ffi.a" "$PACKAGE_DIR/lib/" fi @@ -313,7 +311,7 @@ fi # Generate bindings if we can run the library on this host cd "$WORKSPACE_DIR" if can_run_on_host "$TARGET"; then - cdylib=$(find_cdylib "$TARGET") + cdylib="$BUILT_LIBRARY" if [[ -f "$cdylib" ]]; then generate_bindings "$cdylib" diff --git a/sh/dart/bench.sh b/sh/dart/bench.sh index 5b68e096d1c..b671bc435a6 100755 --- a/sh/dart/bench.sh +++ b/sh/dart/bench.sh @@ -23,12 +23,16 @@ case "$host" in *-windows-*) library=moq_ffi.dll ;; *) library=libmoq_ffi.so ;; esac +target_dir=$(cargo metadata --format-version 1 --no-deps | jq -r .target_directory) +messages=$(cargo build-ffi --release --message-format=json-render-diagnostics) +library_path=$(jq -cs --arg root "$target_dir" --arg target "$host" --arg host "$host" \ + --arg profile release --arg library "$library" -f sh/rs/ffi-artifact.jq <<<"$messages") cat >>dart/moq/pubspec.yaml <> "{calls}"\n', + "cargo": f"#!/bin/sh\nif [ \"$1\" = metadata ]; then echo '{metadata}'; exit; fi\n" + f'printf "%s\\n" "$*" >> "{calls}"\n' + f'printf "%s" "${{CC:-unset}}" > "{compiler_env}"\n' + f"echo '{artifact}'\n", "rustc": '#!/bin/sh\necho "host: x86_64-unknown-linux-gnu"\n', "dart": "#!/bin/sh\nexit 0\n", - "uniffi_bindgen_dart": '#!/bin/sh\nwhile [ "$1" != --out-dir ]; do shift; done\n' + "uniffi_bindgen_dart": f'#!/bin/sh\ntest "$2" = "{library}" || exit 2\n' + 'while [ "$1" != --out-dir ]; do shift; done\n' 'mkdir -p "$2"\nfor name in moq.dart uniffi_runtime.dart; do ' 'echo "// ignore_for_file: unused_import" > "$2/$name"; done\n', }.items(): @@ -29,7 +47,11 @@ def test_generator_profiles(self): tool.write_text(source) tool.chmod(0o755) env = dict( - os.environ, PATH=f"{scratch}:{os.environ['PATH']}", MOQ_FFI_PROFILE=profile, BASH_ENV=os.devnull + os.environ, + PATH=f"{scratch}:{os.environ['PATH']}", + MOQ_FFI_PROFILE=profile, + BASH_ENV=os.devnull, + CC="fixture-compiler", ) result = subprocess.run( ["bash", "sh/dart/generate.sh", str(scratch / "generated")], @@ -39,15 +61,23 @@ def test_generator_profiles(self): capture_output=True, ) self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(compiler_env.read_text(), "fixture-compiler" if release else "unset") args = calls.read_text().split() self.assertEqual(args[0], "build-ffi") self.assertEqual("--release" in args, release) - self.assertEqual(args[1 : args.index("--target")], [*features, *(["--release"] if release else [])]) + self.assertNotIn("--target", args) + self.assertEqual( + args[1:], + (["--release"] if release else ["--features", "testing", "--target-dir", str(ROOT / "target/dart")]) + + ["--message-format=json-render-diagnostics"], + ) def test_failed_benchmark_restores_manifest(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / "sh/dart").mkdir(parents=True) + (root / "sh/rs").mkdir() + shutil.copy(ROOT / "sh/rs/ffi-artifact.jq", root / "sh/rs/ffi-artifact.jq") (root / "dart/moq").mkdir(parents=True) shutil.copy(ROOT / "sh/dart/bench.sh", root / "sh/dart/bench.sh") generate = root / "sh/dart/generate.sh" @@ -57,11 +87,21 @@ def test_failed_benchmark_restores_manifest(self): manifest.write_text("name: fixture\n") tools = root / "tools" tools.mkdir() + target = root / "external output" + metadata = json.dumps({"target_directory": str(target)}) + artifact = json.dumps( + { + "reason": "compiler-artifact", + "target": {"name": "moq_ffi"}, + "filenames": [str(target / "release/libmoq_ffi.so")], + } + ) for name, source in { + "cargo": f"#!/bin/sh\nif [ \"$1\" = metadata ]; then echo '{metadata}'; else echo '{artifact}'; fi\n", "git": f'#!/bin/sh\necho "{root}"\n', "rustc": '#!/bin/sh\necho "host: x86_64-unknown-linux-gnu"\n', "dart": '#!/bin/sh\nif [ "$1" = run ]; then ' - 'grep -q "release/libmoq_ffi.so" pubspec.yaml || exit 2; exit 7; fi\n', + f"grep -Fq '{target}/release/libmoq_ffi.so' pubspec.yaml || exit 2; exit 7; fi\n", }.items(): tool = tools / name tool.write_text(source) diff --git a/sh/dart/check.sh b/sh/dart/check.sh index b76df9323de..94a0fe72026 100755 --- a/sh/dart/check.sh +++ b/sh/dart/check.sh @@ -125,8 +125,7 @@ done # changes the hook's filter would otherwise silently double this check's Rust # build. Cargo rewrites the .d dep-info file even on a no-op, so only the # libraries count. -host=$(rustc -vV | sed -n 's/^host: //p') -if [[ -n "$(find "$DART_DIR/../target/$host/debug" -maxdepth 1 -name '*moq_ffi.*' ! -name '*.d' -newer "$generated/built")" ]]; then +if [[ -n "$(find "$DART_DIR/../target/dart" -mindepth 2 -maxdepth 3 -name '*moq_ffi.*' ! -name '*.d' -newer "$generated/built")" ]]; then drift=$( { diff <(tr '\0' '\n' <"$generated/generate.env") <(tr '\0' '\n' <"$generated/hook.env") || true; } | sed -n 's/^[<>] \([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' | sort -u | paste -sd ' ' - diff --git a/sh/dart/generate.sh b/sh/dart/generate.sh index a7c9a62a9ef..ad5a4c87932 100755 --- a/sh/dart/generate.sh +++ b/sh/dart/generate.sh @@ -18,7 +18,7 @@ command -v dart >/dev/null 2>&1 || { exit 1 } -# Build with the environment dart/moq_ffi/hook/build.dart gets, so the hook's +# Debug builds use the environment dart/moq_ffi/hook/build.dart gets, so the hook's # own Cargo run during `dart test` finds this build fresh. Dart filters a hook's # environment, and build scripts such as aws-lc-sys's rerun when a variable they # track (CC, PKG_CONFIG, ...) changes, rebuilding everything above them. This @@ -37,24 +37,31 @@ shopt -u nocasematch host=$(rustc -vV | sed -n 's/^host: //p') # Checkouts use a dev build; benchmark and release staging opt into release. profile=${MOQ_FFI_PROFILE:-debug} -# The shared default build (.cargo/config.toml). The debug one adds the hook's test -# hooks, which is safe only because `--target` gives it a directory no default build -# writes; the release build is benchmarked, so it leaves out the counting allocator. +# Keep the test hooks out of the shared cdylib without changing Cargo's host +# compilation mode. Release benchmarks use the shared default build. case "$profile" in - debug) profile_args=(--features testing) ;; - release) profile_args=(--release) ;; + debug) build=(env -i "${hook_env[@]}" cargo build-ffi --features testing --target-dir "$WORKSPACE_DIR/target/dart") ;; + release) build=(cargo build-ffi --release) ;; *) echo "error: unsupported MOQ_FFI_PROFILE: $profile" >&2 exit 1 ;; esac -(cd "$WORKSPACE_DIR" && env -i "${hook_env[@]}" cargo build-ffi "${profile_args[@]}" --target "$host") +messages=$(cd "$WORKSPACE_DIR" && "${build[@]}" --message-format=json-render-diagnostics) +if [[ "$profile" == debug ]]; then + target_dir=$(cd "$WORKSPACE_DIR" && env -i "${hook_env[@]}" \ + CARGO_TARGET_DIR="$WORKSPACE_DIR/target/dart" cargo metadata --format-version 1 --no-deps | jq -er .target_directory) +else + target_dir=$(cd "$WORKSPACE_DIR" && cargo metadata --format-version 1 --no-deps | jq -er .target_directory) +fi case "$host" in - *-apple-*) library="$WORKSPACE_DIR/target/$host/$profile/libmoq_ffi.dylib" ;; - *-windows-*) library="$WORKSPACE_DIR/target/$host/$profile/moq_ffi.dll" ;; - *) library="$WORKSPACE_DIR/target/$host/$profile/libmoq_ffi.so" ;; + *-apple-*) name=libmoq_ffi.dylib ;; + *-windows-*) name=moq_ffi.dll ;; + *) name=libmoq_ffi.so ;; esac +library=$(jq -ers --arg root "$target_dir" --arg target "$host" --arg host "$host" \ + --arg profile "$profile" --arg library "$name" -f "$WORKSPACE_DIR/sh/rs/ffi-artifact.jq" <<<"$messages") mkdir -p "$OUTPUT_DIR" rm -f "$OUTPUT_DIR/moq.dart" "$OUTPUT_DIR/uniffi_runtime.dart" diff --git a/sh/gh/ffi.test.py b/sh/gh/ffi.test.py index c16e99086d4..5521e741cb5 100644 --- a/sh/gh/ffi.test.py +++ b/sh/gh/ffi.test.py @@ -35,6 +35,8 @@ def setUp(self): GITHUB_REPOSITORY="moq-dev/moq", TEST_ROOT=str(self.root), ) + # The native case asserts that only a cross build sets this linker. + self.env.pop("CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER", None) def tool(self, name, body): path = self.bin / name @@ -178,6 +180,108 @@ def test_direct_and_legacy_triggers_need_no_api(self): self.assertEqual(self.run_helper("completed", "go"), "ready=true\n") self.assertFalse((self.root / "queried").exists()) + def test_native_and_cross_builds_stage_the_selected_artifacts(self): + # A native request must share host units, even when --target was + # explicitly passed to the packager. A cross request must keep its + # triple and linker. Minimal builds must never overwrite the default. + for host, target, minimal, configured in [ + ("x86_64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", False, False), + ("aarch64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", False, False), + ("x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", False, False), + ("x86_64-unknown-linux-gnu", "aarch64-linux-android", False, False), + ("x86_64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", True, False), + ("x86_64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", False, True), + ("x86_64-unknown-linux-gnu", "x86_64-unknown-linux-gnu", False, "wrong"), + ]: + with self.subTest(host=host, target=target, minimal=minimal): + payload = f"{host}:{target}:{minimal}".encode() + self.env["BUILD_ID"] = payload.decode() + self.env["CONFIGURED_TARGET"] = ( + "aarch64-unknown-linux-gnu" if configured == "wrong" else host if configured else "" + ) + if configured: + default_library = self.root / "target/release/libmoq_ffi.so" + default_library.parent.mkdir(parents=True, exist_ok=True) + default_library.write_bytes(b"stale default") + default_library = self.root / "target/release/libmoq_ffi.so" + if minimal: + default_library.parent.mkdir(parents=True, exist_ok=True) + default_bytes = b"default library sentinel" + default_library.write_bytes(default_bytes) + # Packaging the native host also generates bindings. Only the + # library lookup matters here; no generator needs to execute. + self.tool("rustc", f'print("host: {host}")\n') + self.tool("rustup", "pass\n") + self.tool( + "cargo", + """import json, os, pathlib, sys +root = pathlib.Path(os.environ["TEST_ROOT"]) +args = sys.argv[1:] +ndk = args[0] == "ndk" +if ndk: + target = args[args.index("--target") + 1] + args = args[args.index("--") + 1:] + ["--target", target] +if args[0] == "metadata": + print(json.dumps({"target_directory": str(root / "target")}, separators=(",", ":"))) +elif args[0] == "build-ffi": + target = args[args.index("--target") + 1] if "--target" in args else None + (root / "build").write_text(json.dumps({ + "target": target, "args": args, + "linker": os.environ.get("CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER"), + })) + directory = pathlib.Path(args[args.index("--target-dir") + 1]) + effective = target or os.environ.get("CONFIGURED_TARGET") + if effective: directory /= effective + directory /= "release" + directory.mkdir(parents=True, exist_ok=True) + for name in ["libmoq_ffi.so", "libmoq_ffi.a"]: + (directory / name).write_bytes(os.environ["BUILD_ID"].encode()) + if not ndk: + print(json.dumps({"reason": "compiler-artifact", "target": {"name": "moq_ffi"}, + "filenames": [str(directory / "libmoq_ffi.so"), str(directory / "libmoq_ffi.a")]})) +elif args[0] == "run": + assert pathlib.Path(args[args.index("--library") + 1]).read_bytes() == os.environ["BUILD_ID"].encode() +else: raise AssertionError(args) +""", + ) + for generator in ["uniffi-bindgen-go", "uniffi_bindgen_dart", "uniffi-bindgen-cpp"]: + self.tool(generator, "pass\n") + args = [ + "bash", + str(ROOT / "rs/moq-ffi/build.sh"), + "--target", + target, + "--version", + "1.2.3", + "--output", + str(self.root / "dist"), + ] + if minimal: + args.append("--no-default-features") + result = subprocess.run(args, env=self.env, cwd=self.root, capture_output=True, text=True) + if configured == "wrong": + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not match the requested target", result.stderr) + continue + self.assertEqual(result.returncode, 0, result.stderr) + built = json.loads((self.root / "build").read_text()) + self.assertEqual(built["target"], target if target != host else None) + self.assertEqual( + built["linker"], + "aarch64-linux-gnu-gcc" if target == "aarch64-unknown-linux-gnu" and target != host else None, + ) + self.assertEqual("--no-default-features" in built["args"], minimal) + self.assertEqual( + built["args"][built["args"].index("--target-dir") + 1], + str(self.root / "target" / "minimal" if minimal else self.root / "target"), + ) + staged = self.root / f"dist/moq-ffi-1.2.3-{target}/lib" + if not target.endswith("-android"): + self.assertEqual((staged / "libmoq_ffi.a").read_bytes(), payload) + self.assertEqual((staged / "libmoq_ffi.so").read_bytes(), payload) + if minimal: + self.assertEqual(default_library.read_bytes(), default_bytes) + def test_bindings_from_existing_library_never_build_moq_ffi(self): library = self.root / "libmoq_ffi.so" library.write_bytes(b"metadata") @@ -186,7 +290,7 @@ def test_bindings_from_existing_library_never_build_moq_ffi(self): """import json, os, pathlib, sys root = pathlib.Path(os.environ["TEST_ROOT"]) with (root / "calls").open("a") as log: log.write(json.dumps(sys.argv[1:]) + "\\n") -if sys.argv[1] == "metadata": print(json.dumps({"target_directory": str(root / "target")})) +if sys.argv[1] == "metadata": print(json.dumps({"target_directory": str(root / "target")}, separators=(",", ":"))) elif sys.argv[1] == "run": assert sys.argv[sys.argv.index("--library") + 1] == str(root / "libmoq_ffi.so") pathlib.Path(sys.argv[sys.argv.index("--out-dir") + 1]).mkdir(parents=True) diff --git a/sh/rs/ffi-artifact.jq b/sh/rs/ffi-artifact.jq new file mode 100644 index 00000000000..fdef619147f --- /dev/null +++ b/sh/rs/ffi-artifact.jq @@ -0,0 +1,10 @@ +# Select only this invocation's cdylib, rejecting an implicit different target. +def path: gsub("\\\\"; "/"); +($root | path) as $base | +[.[] | select(.reason == "compiler-artifact" and .target.name == "moq_ffi") | + .filenames[] | path | select(endswith("/" + $library))] | unique | +if length != 1 then error("expected one moq-ffi cdylib from Cargo") else .[0] end | +. as $file | ($file | rtrimstr("/" + $library)) as $directory | +if $directory == ($base + "/" + $target + "/" + $profile) or + ($target == $host and $directory == ($base + "/" + $profile)) +then $file else error("Cargo artifact does not match the requested target/profile: " + $file) end