diff --git a/.gitattributes b/.gitattributes index f99358e59..d4f40f3d0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ quest/** merge=quest driver/pub/tests/fixture/api/src/crlf.rs -text +# The exporter embeds its runtime in every binding it writes, so a Windows checkout must +# keep LF for the output, and its contract hash, to match every other host's. +ffi/runtime/** text eol=lf diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml new file mode 100644 index 000000000..545c7f8b3 --- /dev/null +++ b/.github/workflows/bindings-test.yml @@ -0,0 +1,30 @@ +name: Binding packages test + +# bindings.yml on the api fixture, whose Python consumers must print native Rust's transcript +# byte for byte, and on moq's stand-in facade at the exporter's moq pin, its first consumer. +# Read-only: nothing here publishes. +on: + workflow_call: + +permissions: + contents: read + +jobs: + api: + name: api fixture + uses: ./.github/workflows/bindings.yml + with: + manifest-path: ffi/fixtures/api/Cargo.toml + features: doubled + example: transcript + python-check: ffi/tests/api.py + # No swift-check: api.swift compiles against the fixture's own test export, of the host + # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. + + moq: + name: moq stand-in + uses: ./.github/workflows/bindings.yml + with: + manifest-path: ffi/target/moq-ffi/Cargo.toml + python-check: ffi/moq/moq.py + swift-check: ffi/moq/moq.swift diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml new file mode 100644 index 000000000..677c7ea4b --- /dev/null +++ b/.github/workflows/bindings.yml @@ -0,0 +1,421 @@ +name: Binding packages + +# The shared, read-only builder of a crate's rust2-ffi Python wheels and Swift package, which +# a release workflow ffi/release/package.py's `init` writes calls at a pinned rust2 commit. +# It publishes nothing and holds no credentials: the caller's own jobs publish what it +# uploads, once its consumers pass. See ffi/release/README.md. +on: + workflow_call: + inputs: + manifest-path: + description: The Cargo.toml of the package whose pub API the bindings export + required: true + type: string + features: + description: Cargo features the export and every build turn on, space-separated + default: '' + type: string + example: + description: An example of the package whose stdout every check program must print + default: '' + type: string + python-check: + description: A program run on each installed wheel from a clean directory + default: '' + type: string + swift-check: + description: The main.swift of an executable run on the installed Swift package + default: '' + type: string + outputs: + version: + description: The package's Cargo version + value: ${{ jobs.facts.outputs.version }} + tag: + description: The release tag the hosted XCFramework URL names, -v + value: ${{ jobs.facts.outputs.tag }} + module: + description: The Swift module, whose XCFramework is FFI.xcframework.zip + value: ${{ jobs.facts.outputs.module }} + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + MANIFEST: ${{ inputs.manifest-path }} + FEATURES: ${{ inputs.features }} + +jobs: + facts: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + name: ${{ steps.facts.outputs.name }} + version: ${{ steps.facts.outputs.version }} + tag: ${{ steps.facts.outputs.tag }} + module: ${{ steps.facts.outputs.module }} + fixture: ${{ steps.mode.outputs.fixture }} + rust2: ${{ steps.identity.outputs.workflow_repository }} + rust2-sha: ${{ steps.identity.outputs.workflow_sha }} + steps: + # Read new job identity fields through JSON until actionlint knows their types. + - id: identity + env: + JOB: ${{ toJSON(job) }} + run: | + for field in workflow_repository workflow_sha; do + value=$(jq -er --arg field "$field" '.[$field] | select(type == "string" and length > 0)' <<< "$JOB") + echo "$field=$value" >> "$GITHUB_OUTPUT" + done + # rust2's own test exports moq's stand-in facade, which `just ffi standin` writes from + # moq's pinned sources: no repository holds it. + - id: mode + env: + RUST2: ${{ steps.identity.outputs.workflow_repository }} + run: | + if [[ "$RUST2" == "$GITHUB_REPOSITORY" && "$MANIFEST" == ffi/target/moq-ffi/Cargo.toml ]]; then + echo fixture=true >> "$GITHUB_OUTPUT" + else + echo fixture=false >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ steps.identity.outputs.workflow_repository }} + ref: ${{ steps.identity.outputs.workflow_sha }} + path: rust2 + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: steps.mode.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: steps.mode.outputs.fixture == 'true' + working-directory: source + run: just ffi standin && cp ffi/moq/ffi.toml ffi/target/moq-ffi/ffi.toml + - id: facts + working-directory: source + run: python3 -I ../rust2/ffi/release/package.py facts --manifest-path "$MANIFEST" >> "$GITHUB_OUTPUT" + + # Each target's API, checked on the host whose C toolchain its build scripts need, for + # every build host's export to merge into one binding. + api: + needs: facts + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + os: ubuntu-24.04 + targets: x86_64-unknown-linux-gnu + - host: linux-arm64 + os: ubuntu-24.04-arm + targets: aarch64-unknown-linux-gnu + - host: apple + os: macos-26 + targets: aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim + - host: windows + os: windows-latest + targets: x86_64-pc-windows-msvc + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + defaults: + run: + shell: bash + steps: + # ffi/rust-toolchain.toml links to ts/'s. + - if: runner.os == 'Windows' + run: git config --global core.symlinks true + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: needs.facts.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: needs.facts.outputs.fixture == 'true' + working-directory: source + run: just ffi standin && cp ffi/moq/ffi.toml ffi/target/moq-ffi/ffi.toml + - name: Exporter + working-directory: rust2/ffi + env: + TARGETS: ${{ matrix.targets }} + run: | + read -ra targets <<< "$TARGETS" + rustup target add "${targets[@]}" + cargo build --locked --release + - name: Stage + env: + TARGETS: ${{ matrix.targets }} + run: | + read -ra targets <<< "$TARGETS" + python3 -I rust2/ffi/release/package.py api --manifest-path "source/$MANIFEST" --features "$FEATURES" --out api "${targets[@]}" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ needs.facts.outputs.name }}-api-${{ matrix.host }} + path: api + if-no-files-found: error + + # Each host exports the binding with every target's API, builds its own targets, and runs + # the native example the consumers compare with. + build: + needs: [facts, api] + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + os: ubuntu-24.04 + targets: x86_64-unknown-linux-gnu + - host: linux-arm64 + os: ubuntu-24.04-arm + targets: aarch64-unknown-linux-gnu + # One leg per Apple slice: a 3-core runner takes about 20 minutes for each of moq's. + - host: apple-darwin + os: macos-26 + targets: aarch64-apple-darwin + - host: apple-ios + os: macos-26 + targets: aarch64-apple-ios + - host: apple-ios-sim + os: macos-26 + targets: aarch64-apple-ios-sim + - host: windows + os: windows-latest + targets: x86_64-pc-windows-msvc + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + defaults: + run: + shell: bash + steps: + - if: runner.os == 'Windows' + run: git config --global core.symlinks true + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: needs.facts.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: needs.facts.outputs.fixture == 'true' + working-directory: source + run: just ffi standin && cp ffi/moq/ffi.toml ffi/target/moq-ffi/ffi.toml + # The tools that build and repair, apart from the CPython the consumers test on. + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.13' + - name: Packaging tools + run: | + case "$RUNNER_OS" in + Linux) python -m pip install auditwheel==6.8.2 patchelf==0.17.2.4 cargo-zigbuild==0.24.1 ziglang==0.17.1 ;; + macOS) python -m pip install delocate==0.13.0 ;; + Windows) python -m pip install delvewheel==1.13.2 ;; + esac + - name: Exporter + working-directory: rust2/ffi + env: + TARGETS: ${{ matrix.targets }} + run: | + read -ra targets <<< "$TARGETS" + rustup target add "${targets[@]}" + cargo build --locked --release + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: ${{ needs.facts.outputs.name }}-api-* + path: api + merge-multiple: true + - name: Export + env: + TARGETS: ${{ matrix.targets }} + run: | + read -ra targets <<< "$TARGETS" + python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --out export "${targets[@]}" + - name: Build + env: + TARGETS: ${{ matrix.targets }} + EXAMPLE: ${{ inputs.example }} + run: | + read -ra targets <<< "$TARGETS" + python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export export --out dist --example "$EXAMPLE" "${targets[@]}" + - name: Checks + env: + PYTHON_CHECK: ${{ inputs.python-check }} + SWIFT_CHECK: ${{ inputs.swift-check }} + run: | + mkdir -p dist/checks + if [[ -n "$PYTHON_CHECK" ]]; then cp "source/$PYTHON_CHECK" dist/checks/check.py; fi + if [[ -n "$SWIFT_CHECK" ]]; then cp "source/$SWIFT_CHECK" dist/checks/main.swift; fi + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} + path: dist + if-no-files-found: error + + # The hosts' exports must be one binding: a module and a library from different builds + # refuse to load, but two wheels of different bindings would each load. + agree: + needs: [facts, build] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: ${{ needs.facts.outputs.name }}-build-* + path: builds + - run: | + sort -u builds/*/digest.txt > digests + cat builds/*/digest.txt + if [[ $(wc -l < digests) != 1 ]]; then + echo '::error::The build hosts exported different bindings.' + exit 1 + fi + + swift: + needs: [facts, build] + runs-on: macos-26 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: needs.facts.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: needs.facts.outputs.fixture == 'true' + working-directory: source + run: just ffi standin && cp ffi/moq/ffi.toml ffi/target/moq-ffi/ffi.toml + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: ${{ needs.facts.outputs.name }}-build-apple-* + path: apple + merge-multiple: true + - name: Package + env: + URL: https://github.com/${{ github.repository }}/releases/download/${{ needs.facts.outputs.tag }}/${{ needs.facts.outputs.module }}FFI.xcframework.zip + run: | + args=() + for target in aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim; do + args+=(--lib "$target=$(echo "$PWD"/apple/static/"$target"/*.a)" --native "$target=$PWD/apple/static/$target/native.txt") + done + cd source + python3 -I ../rust2/ffi/release/package.py swift --manifest-path "$MANIFEST" --export ../apple/export "${args[@]}" --url "$URL" --out ../swift + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ needs.facts.outputs.name }}-swift + path: swift + if-no-files-found: error + + # A fresh environment per wheel, on the oldest CPython its stable ABI claims, with no + # checkout, Rust, or source path: what a user's `pip install` gets. + python: + needs: [facts, build, agree] + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + os: ubuntu-24.04 + - host: linux-arm64 + os: ubuntu-24.04-arm + - host: apple-darwin + os: macos-26 + - host: windows + os: windows-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: ffi/release + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} + path: dist + - name: Python floor + id: floor + run: | + shopt -s nullglob + wheels=(dist/wheels/*.whl) + [[ ${#wheels[@]} == 1 ]] || { echo "::error::expected one wheel, found ${wheels[*]}"; exit 1; } + tag=$(basename "${wheels[0]}" .whl | cut -d- -f3) + [[ "$tag" =~ ^cp3([0-9]+)$ ]] || { echo "::error::no CPython floor in ${wheels[0]}"; exit 1; } + echo "version=3.${BASH_REMATCH[1]}" >> "$GITHUB_OUTPUT" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ steps.floor.outputs.version }} + - name: Install and run + run: | + python -m venv "$RUNNER_TEMP/env" + if [[ "$RUNNER_OS" == Windows ]]; then py="$RUNNER_TEMP/env/Scripts/python.exe"; else py="$RUNNER_TEMP/env/bin/python"; fi + "$py" -m pip install --no-index --no-deps dist/wheels/*.whl + args=() + if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi + if [[ -f dist/checks/check.py ]]; then + "$py" -I rust2/ffi/release/package.py consume-python --check dist/checks/check.py "${args[@]}" + else + "$py" -I rust2/ffi/release/package.py consume-python --import "$(cat dist/lib.txt)" + fi + + # A new SwiftPM package whose only library is the staged package, as a user's would be, + # run on macOS and built for iOS devices and simulators. + swift-consumer: + needs: [facts, swift, agree] + runs-on: macos-26 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: ffi/release + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-swift + path: swift + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-apple-darwin + path: apple + - name: Consume + env: + MODULE: ${{ needs.facts.outputs.module }} + run: bash rust2/ffi/release/consume-swift.sh swift apple diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 45e2b033e..a9660d60c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -12,9 +12,9 @@ concurrency: cancel-in-progress: true jobs: - registry-spike: - name: Registry package consumers - uses: ./.github/workflows/registry.yml + bindings: + name: Binding packages + uses: ./.github/workflows/bindings-test.yml check: name: Check diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a34e8507f..f28208923 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -17,9 +17,9 @@ concurrency: cancel-in-progress: true jobs: - registry-spike: - name: Registry package consumers - uses: ./.github/workflows/registry.yml + bindings: + name: Binding packages + uses: ./.github/workflows/bindings-test.yml # The full check over every module, catching what a merge group's diff-scoped # `just ci` let into main. @@ -442,7 +442,7 @@ jobs: # schedule, which runs main, gets the write token: a dispatch can run any branch. report: name: Report - needs: [registry-spike, gate, ts, bench, ffi-api, ffi, ffi-web, ffi-sanitize, ffi-cpp, ffi-android, ffi-bench, census] + needs: [bindings, gate, ts, bench, ffi-api, ffi, ffi-web, ffi-sanitize, ffi-cpp, ffi-android, ffi-bench, census] if: ${{ !cancelled() && github.event_name == 'schedule' }} runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.github/workflows/registry.yml b/.github/workflows/registry.yml deleted file mode 100644 index fe007448f..000000000 --- a/.github/workflows/registry.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Registry packaging spike - -on: - workflow_call: - workflow_dispatch: - -permissions: - contents: read - -jobs: - package: - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-24.04-arm - private: '["self-hosted", "Linux", "ARM64"]' - - runner: macos-26 - private: '["self-hosted", "macOS", "ARM64"]' - runs-on: ${{ fromJSON(github.event.repository.private && matrix.private || format('["{0}"]', matrix.runner)) }} - steps: - # Fixed spike input: consumers need only a workflow call, no source or settings. - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: kixel-ai/rust2 - ref: ${{ github.repository == 'kixel-ai/rust2' && github.sha || '13710f7b7f1dcc649fa5cdabff348b497bc835ff' }} - persist-credentials: false - - uses: ./.github/actions/nix - - name: Build fixture packages - run: nix develop --command bench/registry/build.sh - - name: Include standalone verifiers - run: | - cp bench/registry/consume.py ffi/target/registry/verify.py - cp bench/registry/consume-swift.sh ffi/target/registry/verify-swift.sh - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: registry-${{ matrix.runner }} - if-no-files-found: error - path: | - ffi/target/registry/*.whl - ffi/target/registry/*.zip - ffi/target/registry/*.py - ffi/target/registry/*.swift - ffi/target/registry/*.sh - ffi/target/registry/native.txt - - consumer: - needs: package - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-24.04-arm - private: '["self-hosted", "Linux", "ARM64"]' - - runner: macos-26 - private: '["self-hosted", "macOS", "ARM64"]' - runs-on: ${{ fromJSON(github.event.repository.private && matrix.private || format('["{0}"]', matrix.runner)) }} - env: - SPIKE_ROOT: ${{ github.workspace }}/.tmp/registry-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }} - defaults: - run: - working-directory: ${{ env.SPIKE_ROOT }} - steps: - # No checkout, Rust toolchain, exporter, source paths or PYTHONPATH. - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: registry-${{ matrix.runner }} - path: ${{ env.SPIKE_ROOT }}/artifacts - - name: Install and run wheel - run: | - python3 -m venv env - env/bin/python -m pip install --no-index --no-deps artifacts/*.whl - mkdir clean - cd clean - ../env/bin/python -I ../artifacts/verify.py ../artifacts - - name: Install and run Swift package - if: runner.os == 'macOS' - run: | - cd artifacts - unzip Api.zip - bash verify-swift.sh . - - name: Remove consumer workspace - if: always() - working-directory: ${{ github.workspace }} - run: rm -rf "$SPIKE_ROOT" diff --git a/README.md b/README.md index 8a96a95dc..b2dab7c77 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Rust to other places. - **TypeScript**: an own rustc driver that translates a [documented Rust subset](ts/README.md) to TypeScript, without WASM. -- **C, then any language**: an [exporter](ffi/README.md) that turns a crate's `pub` API into a C ABI and header with no attributes, sharing the TypeScript driver's front end. +- **C, then any language**: an [exporter](ffi/README.md) that turns a crate's `pub` API into a C ABI and header with no attributes, sharing the TypeScript driver's front end. Its [release builder](ffi/release/README.md) turns an export into checked Python wheels and a Swift package, published by the crate's own workflow. - **Public-visibility audit**: [rust2-pub](driver/pub/README.md), on the drivers' front end, finds the `pub` items no other crate needs and validates narrowing them. - **Faster builds**: a reproducible [build-speed benchmark](bench/build/README.md) of upstream's levers on moq. diff --git a/bench/registry/README.md b/bench/registry/README.md deleted file mode 100644 index e14b5c400..000000000 --- a/bench/registry/README.md +++ /dev/null @@ -1,201 +0,0 @@ -# Registry packaging spike - -Accepted direction: generate release templates for moq. A single configless publisher -cannot infer registry ownership from Cargo metadata. Shared package builders are -still useful. This spike publishes nothing and changes no binding semantics. - -The maintainer [accepted the no-go on 2026-11-16](https://github.com/kixel-ai/rust2/pull/761#issuecomment-6930429582): -release templates with shared builders are the full version (rejected: keep -investigating a configless publisher; stop the spike with no full version). -The [follow-up quest](/quest/m2/release-templates.md) starts with moq's existing -Python and Swift releases and preserves their identities and platform policy. - -`build.sh` exports the existing api fixture (including its existing exclusion file), -builds a CPython stable-ABI library, writes a wheel, and on macOS wraps a separate -C static library in an XCFramework with a Swift source package. The Swift wrapper -stays source so the consumer compiles it with its compiler; the Rust library is -binary. The XCFramework currently has only the build host's macOS slice. This is -not an iOS or universal distribution claim. - -Run from the repository root through `quest gate nix develop --command -bench/registry/build.sh`. Output is in `ffi/target/registry`. Install the wheel in a -fresh Python environment, then run `consume.py ` with that -environment's Python from outside the checkout. The consumer compares stdout -byte for byte with the fixture's native Rust transcript, then exercises futures, -ownership, integer values and calls. The wheel contains the generated module as -`api/__init__.py` and its library beside it, with a complete hashed RECORD. - -On macOS, unpack `Api.zip` beside a fresh consumer directory and run -`consume-swift.sh ` from there. The script uses SwiftPM to -resolve the package, runs the same transcript against Rust, then futures and -ownership checks. Swift 6.2 or newer and macOS 15 are required for the fixture's -executor APIs. BasicContainers is pinned to the exporter's swift-collections -revision. No registry credentials, signing keys or publishing permissions are used. - -## Scope and evidence - -The fixture is intentional: this measures packaging an existing export, not finding -an arbitrary facade in a workspace. Name, version and description come from its -Cargo manifest; the library's distinct `api` name is respected. The fixture is -`publish = false`; this spike does not upload it anywhere. Existing binding -fixtures own semantic differential tests; these consumers check that packaging -preserves the generated API and runtime loading. - -The native wheel tag only claims the build host's platform. Linux packaging removes -RUNPATH and checks that the only dependencies are glibc, its loader and libgcc_s, -with no symbol newer than Ubuntu 24.04's glibc 2.39. Anything else fails packaging. -This is checked on each actual architecture; it is not a manylinux claim. macOS -uses a major_0 tag (the form pip accepts on macOS 11+) and the host architecture. A release needs -manylinux builds and auditwheel repair on Linux, delocate on macOS, and DLL -inspection on Windows, including transitive native dependencies. Stable ABI alone -does not establish OS portability. The generated extension requests abi3-py311, -so the wheel uses cp311-abi3 and Requires-Python >=3.11. - -Linux x86_64 execution passed on 2026-11-13 with CPython 3.14.10: the built -`api_fixture-0.0.0-cp311-abi3-linux_x86_64.whl` installed into a fresh uv-created -virtual environment with no index or dependencies. Running from a clean directory -with `-I` matched the native Rust transcript byte for byte and passed futures, -ownership, values and calls. A separate ZIP inspection checked RECORD coverage, -file sizes and SHA-256 hashes. The full `quest gate nix develop --command just ci` -passed, including the exporter's existing Python and Swift differential tests. - -The export includes host and wasm32-wasip1 API descriptions, as the existing fixture -suite does: its values test checks that wasm32-only calls are refused on the host. -Only the host native library is packaged. Initial export attempts exposed missing -CACHEDIR.TAG markers in imported local build artifacts; restoring those markers -allowed the unchanged exporter to run. No cache tools or checks were disabled. - -macOS 26.6.2 arm64 execution passed on kixtop on 2026-11-16, with Xcode 26.6 and -Apple Swift 6.3.3. In an isolated checkout, `nix develop --command quest gate -bench/registry/build.sh` built `Api.zip`. From a fresh artifact directory containing -only the ZIP, native transcript, Swift consumer and verifier, `unzip -q Api.zip` -and the verifier built a release SwiftPM consumer with Apple's compiler. It ran -through `nix develop --command quest gate env -i HOME=/Users/kixelated -PATH=/usr/bin:/bin:/usr/sbin:/sbin /bin/bash /verify-swift.sh -`, so the consumer inherited only the host home and Apple -toolchain search path. The transcript matched byte for byte, followed by -`futures ok` and `owned ok`. Both transcripts had SHA-256 -`729ffddcec8928c564d29fd1ee3c4fc34ccaecf18900d34be63c4e6fd0a50714`. -`plutil -p Api/ApiFFI.xcframework/Info.plist` and `lipo -info -Api/ApiFFI.xcframework/macos-arm64/libapi.a` independently confirmed one macOS arm64 -slice. - -The macOS builder selects the installed Xcode and its SDK explicitly: Nix's SDK -bundle lacks `libz.tbd` and cannot serve as Xcode's developer directory. The current -spike exports the same WASI target as the fixture suite, so target availability -agrees, and the Swift fixture avoids shadowing its `debug()` helper in Swift 6. -The original external-call pin, commit `901f7baa`, also passed Swift validation; that -older fixture needs an explicit `@Sendable` type on its duration-conversion helper. -After merging main to resolve the fixture conflict, the current source passed -again with the transcript hash above. Apple Swift still warns about a pointer -capture in the fixture's separate shared-call stress test; the consumer exercises -the transcript, futures and ownership, not that stress test. -The first GitHub reusable invocation, -[run 43968322384](https://github.com/kixel-ai/rust2/actions/runs/43968322384), -built both platforms and passed the clean Linux consumer. The macOS wheel -installed but could not import: Nix's linker wrapper embedded a Nix-store -libiconv dependency absent from the consumer runner. The builder now uses -Apple's linker, and macOS wheel packaging inspects `otool -L` and refuses any -dependency outside system libraries and frameworks. Its dylib ID is relative. -The next GitHub invocation passed both artifact consumers, as recorded below. -Do not delete moq's release workflows on this evidence. - -## GitHub invocation and artifact handoff - -On 2026-11-16, ran `gh workflow run check.yml --repo kixel-ai/rust2 --ref -quest/m2/registry`. The registered manual Check workflow called -`./.github/workflows/registry.yml` at -`13710f7b7f1dcc649fa5cdabff348b497bc835ff` in -[run 43971814703](https://github.com/kixel-ai/rust2/actions/runs/43971814703). -Direct dispatch of `registry.yml` returned 404 because that new workflow was not -yet registered on the default branch. The manual caller also runs the existing -full Check job; nightly's report now includes the reusable job's result. - -Both builders passed and uploaded `registry-ubuntu-24.04-arm` and -`registry-macos-26`. Separate jobs downloaded these artifacts without checkout, -Nix, Rust, exporter or source paths. They created a venv, installed the wheel -with `pip install --no-index --no-deps artifacts/*.whl`, and ran the copied -verifier with Python's `-I` from a clean directory. -[Linux arm64](https://github.com/kixel-ai/rust2/actions/runs/43971814703/job/133671565856) -and [macOS arm64](https://github.com/kixel-ai/rust2/actions/runs/43971814703/job/133671565931) -each printed `installed wheel: transcript matches Rust; futures, ownership, -values and calls pass`. The macOS 26.7.2 consumer then unpacked `Api.zip` and ran -`bash verify-swift.sh .`: SwiftPM's release build passed, its transcript matched -Rust byte for byte, then `futures ok` and `owned ok` passed. Both consumer jobs -removed their workspaces successfully. - -The Mac builder's independent `otool -L` check reported only the relative dylib -ID, `/usr/lib/libiconv.2.dylib` and `/usr/lib/libSystem.B.dylib`. The external-call -fixture pin now uses this validated revision, including the Apple linker fix. -Downloaded both uploaded artifacts with `gh run download 43971814703 --repo -kixel-ai/rust2 --dir .tmp/registry-verified-artifacts`: independent ZIP inspection -verified both wheels' complete RECORD coverage, sizes and SHA-256 hashes, and -the XCFramework plist described exactly one macOS arm64 slice. Both packaged -native transcripts had SHA-256 -`729ffddcec8928c564d29fd1ee3c4fc34ccaecf18900d34be63c4e6fd0a50714`. -This completes the fixture-packaging spike and establishes the artifact handoff, -not a configless publisher or moq release migration. - -## What each registry still needs - -| Destination | Package work | Information Cargo cannot establish | -| --- | --- | --- | -| PyPI | Wheel matrix, dependency repair, name/version normalization, license metadata, sdist policy | Ownership of the distribution name and trusted-publisher registration | -| SwiftPM | macOS/iOS device/simulator slices, merged exports for cfg differences, minimum OS versions, checksummed hosted zip and release Package.swift | Artifact hosting URL and repository/tag policy; code signing identity if signing is required | -| Maven Central | AAR with classes.jar, JNI libraries per Android ABI, generated R8 rules, POM, sources/javadoc artifacts and signatures | Verified groupId namespace, publisher authorization and signing key | -| npm | A Node binding backend (not currently generated), platform packages and JS loader, TypeScript declarations | Scope/name ownership and trusted-publisher registration | -| NuGet | A C# binding backend (not currently generated), managed assembly plus RID-specific native assets | Package ID ownership, credentials and desired managed target frameworks | -| pub.dev | Generated Dart library plus native asset hooks and platform archives, pubspec and license files | Publisher authorization and the distribution/download host | -| Go | Generated cgo packages, module-relative imports and native library delivery for supported targets | Canonical module path, repository/subdirectory and release tags (including major-version suffixes) | - -Cargo's repository URL can suggest a Go module path or Maven namespace. It cannot -prove either is the publisher's intended existing identity. Cargo package names -also need not equal existing PyPI names or npm scopes. A manifest's license and -repository fields are optional; absence must produce a packaging error where the -registry requires them. Authentication setup is an external platform boundary, -not something a workflow can manufacture from a crate. - -For moq, the existing release-py-ffi, release-swift-ffi and release-swift-lib -workflows additionally encode their current names, hosting and platform policy. -Replacing those workflows requires preserving those contracts and testing moq, -not just this fixture. The quest's go condition is therefore not established; -templates are the recommended no-go outcome even if both artifact consumers pass. - -## Prior art and primary sources - -- [BoltFFI](https://github.com/boltffi/boltffi): annotation-based generation and local packaging are adjacent to this problem; do not treat it as evidence of registry ownership automation. -- [maturin](https://www.maturin.rs/distribution.html): Python distribution tooling can own wheel repair/build concerns but does not solve the other registries. -- [Wheel format](https://packaging.python.org/en/latest/specifications/binary-distribution-format/): tags, metadata and RECORD. -- [Apple binary packages](https://developer.apple.com/documentation/Xcode/distributing-binary-frameworks-as-swift-packages): XCFramework binary targets and hosted artifact checksums. -- [Maven Central requirements](https://central.sonatype.org/publish/requirements/) and [namespace registration](https://central.sonatype.org/register/namespace/). -- [npm trusted publishing](https://docs.npmjs.com/trusted-publishers/) and [scopes](https://docs.npmjs.com/about-scopes/). -- [NuGet native assets](https://learn.microsoft.com/nuget/create-packages/native-files-in-net-packages). -- [Dart hooks](https://dart.dev/tools/hooks) and [publishing](https://dart.dev/tools/pub/publishing). -- [Go module publishing](https://go.dev/doc/modules/developing). - -The nightly workflow calls the same reusable workflow, outside the merge gate. -Consumer workspaces are unique per run attempt and removed in an always-run -cleanup step, including on self-hosted runners. -Builders replace their own generated output; Swift consumers create a fresh package -beside the artifact regardless of the invoking directory. Rust's reported native -link libraries become SwiftPM linker settings; unrecognized flags fail packaging. -The collections revision is read from ffi/justfile and carried inside the package. - -Registry requirements checked on 2026-11-13. The table is a design assessment, -not a claim that any of those unimplemented publishers was exercised. - -## Reusable workflow - -`.github/workflows/registry.yml` is callable with no inputs or secrets. It builds -this fixed fixture on Linux arm64 and macOS arm64 and passes only artifacts to -fresh consumer jobs. A caller needs just a job with -`uses: kixel-ai/rust2/.github/workflows/registry.yml@`. -External calls pin the fixture, exporter and packaging scripts together to an -immutable rust2 revision, independently of the caller's checkout. Calls within -rust2 test the triggering commit so nightly catches changes on main. It is a spike -runner, not a publisher of the caller's crate. Private callers select self-hosted -runners with OS/architecture labels. Public callers use hosted runners. - -These two targets overlap crabx's planned matrix. The pipeline quest has not -landed a shared matrix to reuse. Extending this to stock crates requires facade -discovery and packaging of moq's existing identity; neither is claimed here. diff --git a/bench/registry/build.sh b/bench/registry/build.sh deleted file mode 100755 index b040576ab..000000000 --- a/bench/registry/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Export once, build packages, and retain the native oracle with the consumer scripts. -set -euo pipefail -export PATH="${RUST2_CARGO_SHIM:+$RUST2_CARGO_SHIM:}${RUST2_TS_TOOLCHAIN:?run in nix develop}/bin:$PATH" -if [[ $(uname) == Darwin ]]; then - # Nix selects an SDK bundle; this build needs the installed Xcode and its full SDK. - export DEVELOPER_DIR - DEVELOPER_DIR=$(env -u DEVELOPER_DIR /usr/bin/xcode-select -p) - export SDKROOT - SDKROOT=$(env -u SDKROOT /usr/bin/xcrun --sdk macosx --show-sdk-path) - # Nix's linker wrapper injects store libraries unavailable to wheel consumers. - export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=/usr/bin/clang - export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=/usr/bin/clang -fi -cd "$(dirname "$0")/../.." -rm -rf ffi/target/registry -host=$(rustc -vV | sed -n 's/^host: //p') -just ffi run --target "$host" --target wasm32-wasip1 --config fixtures/api/ffi.toml --manifest-path fixtures/api/Cargo.toml --out target/registry-export -- --features doubled -cargo build --locked --release --manifest-path ffi/target/registry-export/Cargo.toml --features python -python3 bench/registry/package.py wheel -cargo run --quiet --locked --manifest-path ffi/Cargo.toml --package api-fixture --example transcript --features doubled >ffi/target/registry/native.txt -cp ffi/tests/api.py ffi/target/registry/consumer.py -if [[ $(uname) == Darwin ]]; then - # Build separately so Python symbols cannot enter the Swift library. - cargo build --locked --release --manifest-path ffi/target/registry-export/Cargo.toml --no-default-features - python3 bench/registry/package.py swift - cp ffi/tests/api.swift ffi/target/registry/consumer.swift -fi diff --git a/bench/registry/consume-swift.sh b/bench/registry/consume-swift.sh deleted file mode 100755 index 8083fabac..000000000 --- a/bench/registry/consume-swift.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Build a new executable package whose only library dependency is the artifact. -set -euo pipefail -artifacts=$(cd "$1" && pwd) -cd "$artifacts" -consumer=$(mktemp -d consumer.XXXXXXXX) -cd "$consumer" -mkdir -p Sources/Check -cp "$artifacts/consumer.swift" Sources/Check/check.swift -python3 - <<'PYTHON' -from pathlib import Path -import re -revision = Path('../Api/collections-revision.txt').read_text().strip() -assert re.fullmatch(r'[a-f0-9]{40}', revision) -Path('Package.swift').write_text('''// swift-tools-version: 6.2 -import PackageDescription -let package = Package( - name: "Check", platforms: [.macOS(.v15)], - dependencies: [ - .package(path: "../Api"), - .package(url: "https://github.com/apple/swift-collections", revision: "''' + revision + '''") - ], - targets: [.executableTarget(name: "Check", dependencies: [ - .product(name: "Api", package: "Api"), - .product(name: "BasicContainers", package: "swift-collections") - ])] -) -''') -PYTHON -swift build -c release -.build/release/Check >actual.txt -cmp "$artifacts/native.txt" actual.txt -.build/release/Check futures -.build/release/Check owned diff --git a/bench/registry/consume.py b/bench/registry/consume.py deleted file mode 100644 index fc14626cf..000000000 --- a/bench/registry/consume.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Run an installed wheel from an isolated consumer directory against native Rust.""" -from pathlib import Path -import difflib -import os -import subprocess -import sys - -artifacts = Path(sys.argv[1]).resolve() -python = Path(os.path.abspath(sys.executable)) -for case in [None, "futures", "owned", "values", "calls"]: - command = [str(python), "-I", "-X", "dev", "-W", "error", str(artifacts / "consumer.py")] - if case: - command.append(case) - output = subprocess.run(command, capture_output=True) - if output.returncode: - sys.stderr.buffer.write(output.stderr) - raise SystemExit(output.returncode) - if case is None: - native = (artifacts / "native.txt").read_bytes() - if output.stdout != native: - sys.stderr.buffer.writelines(difflib.diff_bytes( - difflib.unified_diff, native.splitlines(keepends=True), - output.stdout.splitlines(keepends=True), fromfile=b"native Rust", tofile=b"installed wheel")) - raise SystemExit("installed wheel differs from native Rust") -print("installed wheel: transcript matches Rust; futures, ownership, values and calls pass") diff --git a/bench/registry/package.py b/bench/registry/package.py deleted file mode 100644 index 6080034df..000000000 --- a/bench/registry/package.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Package the api fixture's export without changing the Rust or generated API.""" - -import base64 -import csv -import hashlib -import io -import json -import re -import shlex -import platform as host_platform -from pathlib import Path -import shutil -import subprocess -import sys -import sysconfig -import tomllib -import zipfile - -ROOT = Path(__file__).resolve().parents[2] -EXPORT = ROOT / "ffi/target/registry-export" -OUT = ROOT / "ffi/target/registry" - - -def run(*args, **kwargs): - """Fail the package build when any tool fails.""" - return subprocess.run(args, check=True, **kwargs) - - -def wheel(): - """Write a native wheel with hashes for every installed file.""" - manifest = tomllib.loads((ROOT / "ffi/fixtures/api/Cargo.toml").read_text()) - version = manifest["package"]["version"] - # Native tags deliberately make no manylinux or older macOS compatibility claim. - platform = sysconfig.get_platform().replace("-", "_").replace(".", "_") - if sys.platform == "darwin": - major, *_ = host_platform.mac_ver()[0].split(".") - platform = f"macosx_{major}_0_{host_platform.machine()}" - tag = f"cp311-abi3-{platform}" - info = f"api_fixture-{version}.dist-info" - library = {"linux": "libapi.so", "darwin": "libapi.dylib"}[sys.platform] - binary = EXPORT / "target/release" / library - if sys.platform == "linux": - binary_copy = OUT / library - shutil.copyfile(binary, binary_copy) - run("patchelf", "--remove-rpath", str(binary_copy)) - dynamic = run("readelf", "-d", str(binary_copy), capture_output=True, text=True).stdout - needed = set(re.findall(r"Shared library: \[(.*?)\]", dynamic)) - allowed = {"libgcc_s.so.1", "libc.so.6", "libm.so.6", "ld-linux-x86-64.so.2", "ld-linux-aarch64.so.1"} - if needed - allowed or "RPATH" in dynamic or "RUNPATH" in dynamic: - raise SystemExit(f"unpackaged native dependencies: {dynamic}") - versions = run("readelf", "--version-info", str(binary_copy), capture_output=True, text=True).stdout - glibc = [tuple(map(int, v.split("."))) for v in re.findall(r"GLIBC_([0-9.]+)", versions)] - if not glibc or max(glibc) > (2, 39): - raise SystemExit("wheel requires glibc newer than the Ubuntu 24.04 consumer") - binary = binary_copy - elif sys.platform == "darwin": - binary_copy = OUT / library - shutil.copyfile(binary, binary_copy) - run("/usr/bin/install_name_tool", "-id", f"@rpath/{library}", str(binary_copy)) - dynamic = run("/usr/bin/otool", "-L", str(binary_copy), capture_output=True, text=True).stdout - needed = {line.strip().split(" (", 1)[0] for line in dynamic.splitlines()[1:]} - external = {name for name in needed if name != f"@rpath/{library}" - and not name.startswith(("/usr/lib/", "/System/Library/Frameworks/"))} - if external: - raise SystemExit(f"unpackaged native dependencies: {dynamic}") - print(dynamic, end="") - binary = binary_copy - files = { - "api/__init__.py": (EXPORT / "python/api.py").read_bytes(), - f"api/{library}": binary.read_bytes(), - f"{info}/METADATA": ( - f"Metadata-Version: 2.1\nName: api-fixture\nVersion: {version}\n" - f"Summary: {manifest['package']['description']}\nRequires-Python: >=3.11\n\n" - ).encode(), - f"{info}/WHEEL": ( - f"Wheel-Version: 1.0\nGenerator: rust2-registry-spike\n" - f"Root-Is-Purelib: false\nTag: {tag}\n" - ).encode(), - } - record = io.StringIO(newline="") - writer = csv.writer(record) - for name, data in files.items(): - digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() - writer.writerow([name, f"sha256={digest}", len(data)]) - writer.writerow([f"{info}/RECORD", "", ""]) - files[f"{info}/RECORD"] = record.getvalue().encode() - with zipfile.ZipFile(OUT / f"api_fixture-{version}-{tag}.whl", "w", zipfile.ZIP_DEFLATED) as archive: - for name, data in files.items(): - archive.writestr(name, data) - - -def swift(): - """Wrap the host's C static library in an XCFramework and Swift source package.""" - if sys.platform != "darwin": - raise SystemExit("XCFramework packaging requires Apple's xcodebuild on macOS") - revision = re.search(r'swift_collections_rev := "([a-f0-9]{40})"', (ROOT / "ffi/justfile").read_text()) - if revision is None: - raise SystemExit("ffi/justfile must name an immutable swift_collections_rev") - collections = revision.group(1) - package = OUT / "Api" - if package.exists(): - shutil.rmtree(package) - package.mkdir() - headers = OUT / "headers" - if headers.exists(): - shutil.rmtree(headers) - headers.mkdir() - shutil.copy(EXPORT / "include/api.h", headers) - shutil.copy(EXPORT / "swift/api_atomics.h", headers) - modulemap = (EXPORT / "swift/module.modulemap").read_text().replace("../include/api.h", "api.h") - (headers / "module.modulemap").write_text(modulemap) - run("xcodebuild", "-create-xcframework", "-library", str(EXPORT / "target/release/libapi.a"), - "-headers", str(headers), "-output", str(package / "ApiFFI.xcframework")) - sources = package / "Sources/Api" - sources.mkdir(parents=True) - shutil.copy(EXPORT / "swift/api.swift", sources) - built = run("cargo", "rustc", "--locked", "--release", "--manifest-path", str(EXPORT / "Cargo.toml"), - "--lib", "--", "--print=native-static-libs", capture_output=True, text=True) - notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) - if len(notes) != 1: - raise SystemExit(f"rustc did not report native link libraries: {built.stderr}") - flags = iter(shlex.split(notes[0])) - settings = [] - for flag in flags: - if flag.startswith("-l"): - settings.append(f".linkedLibrary({json.dumps(flag[2:])})") - elif flag == "-framework": - settings.append(f".linkedFramework({json.dumps(next(flags))})") - else: - raise SystemExit(f"unsupported native linker flag: {flag}") - (package / "collections-revision.txt").write_text(collections + "\n") - (package / "Package.swift").write_text('''// swift-tools-version: 6.2 -import PackageDescription -let package = Package( - name: "Api", - platforms: [.macOS(.v15)], - products: [.library(name: "Api", targets: ["Api"])], - dependencies: [.package(url: "https://github.com/apple/swift-collections", revision: "''' + collections + '''")], - targets: [ - .binaryTarget(name: "ApiFFI", path: "ApiFFI.xcframework"), - .target(name: "Api", dependencies: ["ApiFFI", .product(name: "BasicContainers", package: "swift-collections")], linkerSettings: [''' + ", ".join(settings) + ''']) - ] -) -''') - # Consumers receive only the package, script and native reference transcript. - shutil.make_archive(str(OUT / "Api"), "zip", OUT, "Api") - - -if __name__ == "__main__": - OUT.mkdir(parents=True, exist_ok=True) - {"wheel": wheel, "swift": swift}[sys.argv[1]]() diff --git a/ffi/release/README.md b/ffi/release/README.md new file mode 100644 index 000000000..de013ca36 --- /dev/null +++ b/ffi/release/README.md @@ -0,0 +1,104 @@ +# Binding releases + +A crate's [rust2-ffi](/ffi/README.md) export, released as Python wheels and a Swift +package. Two halves, split where Cargo's knowledge ends: + +- **The builder**, [`bindings.yml`](/.github/workflows/bindings.yml) over `package.py`, + shared and read-only. It derives everything from Cargo metadata and the export: package + name, version, description, license and its files, URLs, and the module names the + exporter gives the crate's library. It builds, repairs, and checks the packages in clean + consumers, and publishes nothing. +- **The release workflow**, the caller's own file, which `package.py init` writes once. + It holds what Cargo cannot establish: who may publish to PyPI, where the XCFramework is + hosted, which repository SwiftPM resolves the package from, and which tag releases. + +## Adopting it + +From a rust2 checkout at the commit to pin, in the repository holding the crate: + +```sh +python3 /ffi/release/package.py init --manifest-path rs/moq-ffi/Cargo.toml --swift-mirror moq-dev/moq-swift-ffi +``` + +It writes `.github/workflows/release-.yml`, calling the builder at that commit, and +refuses to overwrite a different file. Then, outside the repository: + +- register the workflow and the `pypi` environment as the PyPI project's trusted publisher; +- create the mirror repository, install a GitHub App on it with contents write, and put its + `APP_CLIENT_ID` and `APP_PRIVATE_KEY` in a `swift-mirror` environment that only release + tags may deploy to. + +Releasing is pushing `-v`, the crate's version, which the release checks. +Called without a tag, such as from a nightly, it runs everything up to publication. +A release that failed while publishing is recovered with **Re-run failed jobs**, which keeps +the run's successful build and its artifacts: each publishing job then finishes what the +first attempt left, uploading only what is missing. A rebuild makes other bytes, as wheels +and the XCFramework archive carry build times, so **Re-run all jobs** fails wherever a +registry already holds the first attempt's files. +A Cargo prerelease, such as `1.0.0-rc.1`, is a GitHub prerelease too. + +The builder takes the crate's `Cargo.toml` and four optional inputs the caller's tests own: +`features` to ship, an `example` whose stdout is native Rust's transcript, and the +`python-check` and `swift-check` programs that must print it through the installed +packages. Without a check, a consumer only imports the module. + +## What the builder does + +1. Stages each target's API on the host whose C toolchain its build scripts need. +2. On each build host, exports the binding of every target, merging the others' staged + APIs, so an item only some targets have fails there at run time instead of being + missing ([Targets](/ffi/README.md#targets)). A job checks that every host generated the + same binding. +3. Builds each target with the crate's own toolchain, as rustup resolves it from the + crate's directory; only the export runs on rust2's nightly. +4. Python: a `cp311-abi3` wheel per target, the oldest CPython the generated extension's + stable ABI loads in, holding the module as `/__init__.py` beside its library, with + PEP 639 license metadata and a complete hashed RECORD. Each is repaired by its platform's + standard tool, which claims the portable tag or fails: auditwheel for + `manylinux_2_28` on x86_64 and aarch64 (built by cargo-zigbuild against glibc 2.28, as + [crabx](/crabx/README.md) builds), delocate for `macosx_11_0_arm64`, and delvewheel for + `win_amd64`. +5. Swift: static libraries for macOS arm64 at macOS 13, and iOS arm64 and its simulator + at iOS 16, the floors of the generated Swift's `Swift.Duration`, each checked against + every object's declared minimum OS; one XCFramework of the three, zipped with `ditto`, + its SHA-256 checked against `swift package compute-checksum`; and a `Package.swift` + naming it at `https://github.com//releases/download/-v/FFI.xcframework.zip`. + The package depends on swift-collections by the release tag at the exporter's pinned + revision, since a package resolved by version cannot depend on a revision. Rust's + native link libraries become linker settings, per platform where they differ; any + other flag fails the build. +6. Consumers, with no checkout, Rust, or source path: each wheel installed with + `--no-index --no-deps` into a fresh environment on its oldest CPython, run with `-I` + from an empty directory; and the Swift package built by its own scheme for macOS, iOS, + and the iOS Simulator, then consumed by a new executable package that runs the check. + With an `example`, each must print native Rust's stdout byte for byte. + +rust2 runs it on every manual Check and nightly through +[`bindings-test.yml`](/.github/workflows/bindings-test.yml): on the api fixture, whose +wheels must print its native transcript, and on moq's stand-in facade at the exporter's moq +pin, through `moq/moq.py` and `moq/moq.swift`. + +## Other registries + +The same boundary holds for each destination: the builder makes and checks the artifact +from Cargo facts, the caller's workflow names the identity and credential. A language whose +generator rust2-ffi lacks, such as Node or C#, is unsupported, not a package without its +binding. + +| Destination | Builder work | What only the caller can say | +| --- | --- | --- | +| Maven Central | An AAR with classes.jar, JNI libraries per Android ABI and the generated R8 rules; POM, sources and javadoc jars | The verified groupId namespace, publisher authorization, and signing key | +| pub.dev | The generated Dart library with a native-assets build hook and per-platform archives, pubspec, and license | Publisher authorization and the archives' download host | +| Go | The generated cgo packages with module-relative imports, and native libraries per target | The module path, repository subdirectory, and release tags, major-version suffixes included | +| npm | Needs a Node generator first: platform packages and a JS loader | Scope ownership and trusted-publisher registration | +| NuGet | Needs a C# generator first: the managed assembly and RID-specific native assets | Package ID ownership, credentials, and target frameworks | + +Cargo's `repository` can suggest a Go module path or a Maven namespace but cannot prove +either is the publisher's; a Cargo name need not be the existing PyPI or npm name. Where +a registry requires a field Cargo leaves optional, such as a license, its absence fails +the build. + +Prior art: [maturin](https://www.maturin.rs/distribution.html) and +[cibuildwheel](https://cibuildwheel.pypa.io) own Python wheel matrices, and +[BoltFFI](https://github.com/boltffi/boltffi) packages annotated crates locally; none +establishes registry ownership, which is why publication stays in the caller. diff --git a/ffi/release/consume-swift.sh b/ffi/release/consume-swift.sh new file mode 100644 index 000000000..a30e85195 --- /dev/null +++ b/ffi/release/consume-swift.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Consume a staged Swift package as a user would, before any upload: its own scheme built +# for macOS, iOS, and the iOS Simulator at the floors it declares, then a new executable +# package whose only library is it, run and compared with native Rust. +# usage: consume-swift.sh +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +swift=$(cd "$1" && pwd) +apple=$(cd "$2" && pwd) +module=$(python3 -I -c 'import json, sys; print(json.load(open(sys.argv[1]))["module"])' "$swift/swift.json") +work=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/swift-consumer.XXXXXXXX") +trap 'rm -rf "$work"' EXIT + +# The ZIP a release hosts, unpacked as SwiftPM would after checking its checksum. +[[ "$(shasum -a 256 "$swift/${module}FFI.xcframework.zip" | cut -d' ' -f1)" == "$(cat "$swift/checksum.txt")" ]] +cp -R "$swift/package" "$work/$module" +# SwiftPM takes a local binary target only from inside the package. +(cd "$work/$module" && unzip -q "$swift/${module}FFI.xcframework.zip") +python3 -I "$here/package.py" swift-local --state "$swift/swift.json" --package "$work/$module" --framework "$work/$module/${module}FFI.xcframework" + +( + cd "$work/$module" + for destination in 'generic/platform=macOS' 'generic/platform=iOS' 'generic/platform=iOS Simulator'; do + xcodebuild -quiet -scheme "$module" -destination "$destination" -derivedDataPath "$work/derived" \ + CODE_SIGNING_ALLOWED=NO build + done +) + +mkdir -p "$work/consumer/Sources/Check" +if [[ -f "$apple/checks/main.swift" ]]; then + cp "$apple/checks/main.swift" "$work/consumer/Sources/Check/Check.swift" +else + printf 'import %s\n\n@main\nstruct Check {\n\tstatic func main() {}\n}\n' "$module" >"$work/consumer/Sources/Check/Check.swift" +fi +collections=$(python3 -I -c 'import json, sys; print(json.load(open(sys.argv[1]))["collections"])' "$swift/swift.json") +# The check program may use newer APIs than the package needs, such as an executor's, and +# builds in Swift 5's language mode, as rust2's own tests build it; the package builds in 6. +cat >"$work/consumer/Package.swift" <"$work/actual.txt" +# Only a check program prints the transcript; the stub prints nothing. +if [[ -f "$apple/checks/main.swift" && -f "$apple/native.txt" ]]; then + python3 -I "$here/package.py" compare "$apple/native.txt" "$work/actual.txt" +fi +echo "installed Swift package: builds for macOS, iOS and the iOS Simulator, and its check passes" diff --git a/ffi/release/package.py b/ffi/release/package.py new file mode 100644 index 000000000..9bc9b55d6 --- /dev/null +++ b/ffi/release/package.py @@ -0,0 +1,658 @@ +"""Build and check the Python wheels and Swift package of a crate's rust2-ffi export. + +bindings.yml runs each step on the host it needs; release.yml is the caller `init` writes. +Every fact about the package comes from Cargo metadata or the export, so nothing here +names a crate: what Cargo cannot say, such as who may publish, stays in the caller. +""" + +import argparse +import base64 +import csv +import difflib +import hashlib +import io +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import zipfile + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +# The wheel's Linux floor: glibc 2.28 (RHEL 8, Debian 10), crabx's, and moq's manylinux_2_28. +GLIBC = "2.28" +# The macOS a wheel loads on: the oldest Rust's aarch64-apple-darwin target supports. +MACOS_WHEEL = "11.0" +# The Swift package's floors: what the generated Swift's `Swift.Duration` needs. +SWIFT_PLATFORMS = {"macos": "13.0", "ios": "16.0"} +# Each Apple slice the Swift package ships, and the platform its minimum OS is checked on. +APPLE_TARGETS = { + "aarch64-apple-darwin": "macos", + "aarch64-apple-ios": "ios", + "aarch64-apple-ios-sim": "ios", +} +# Every target a wheel is built for. +WHEEL_TARGETS = [ + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", +] +# Cargo metadata names a library target by its crate types; a proc-macro is no library to export. +LIBRARY_KINDS = {"lib", "rlib", "dylib", "cdylib", "staticlib"} +COLLECTIONS = "https://github.com/apple/swift-collections" +SEMVER = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-(alpha|beta|rc)\.(0|[1-9][0-9]*))?") + + +def run(*args, **kwargs): + """Run a tool, failing the build with its captured output when it fails.""" + try: + return subprocess.run([str(a) for a in args], check=True, **kwargs) + except subprocess.CalledProcessError as error: + for stream in (error.stdout, error.stderr): + if isinstance(stream, bytes): + sys.stderr.flush() + sys.stderr.buffer.write(stream) + elif stream: + sys.stderr.write(stream) + sys.stderr.flush() + raise SystemExit(f"{error.cmd[0]} failed with status {error.returncode}") from None + + +def output(*args, **kwargs): + """A tool's standard output, failing the build when it fails.""" + return run(*args, capture_output=True, encoding="utf-8", **kwargs).stdout + + +def library(package): + """The package's library target name, as Rust spells it, whatever crate types it declares.""" + libs = [t for t in package["targets"] if LIBRARY_KINDS & set(t["kind"])] + if len(libs) != 1: + raise SystemExit(f"{package['name']} has no library to export") + return libs[0]["name"].replace("-", "_") + + +def facts(manifest): + """The package at `manifest`, as Cargo metadata describes it.""" + meta = json.loads(output("cargo", "metadata", "--format-version", "1", "--no-deps", + "--manifest-path", manifest)) + path = Path(manifest).resolve() + package = next((p for p in meta["packages"] if Path(p["manifest_path"]).resolve() == path), None) + if package is None: + raise SystemExit(f"{manifest} is a virtual manifest: name the package to export") + return { + "name": package["name"], + "version": package["version"], + "description": package.get("description"), + "license": package.get("license"), + "license_file": package.get("license_file"), + "repository": package.get("repository"), + "homepage": package.get("homepage"), + "lib": library(package), + "dir": str(path.parent), + "workspace": meta["workspace_root"], + "examples": sorted(t["name"] for t in package["targets"] if "example" in t["kind"]), + } + + +def pep440(version): + """Cargo's version as PyPI spells it, refusing what has no exact spelling there.""" + match = SEMVER.fullmatch(version) + if match is None: + raise SystemExit(f"version {version} has no PEP 440 equivalent: use X.Y.Z or X.Y.Z-(alpha|beta|rc).N") + major, minor, patch, pre, number = match.groups() + suffix = {"alpha": "a", "beta": "b", "rc": "rc", None: ""}[pre] + return f"{major}.{minor}.{patch}{suffix}{number or ''}" + + +def pascal(name): + """The exporter's Swift module name for a prefix: `moq_ffi` is `MoqFfi`.""" + return "".join(part[:1].upper() + part[1:] for part in name.split("_") if part) + + +def python_floor(export): + """The oldest CPython the export's stable-ABI extension loads in, from its glue's pyo3 feature.""" + found = set(re.findall(r'"abi3-py3([0-9]+)"', (Path(export) / "Cargo.toml").read_text(encoding="utf-8"))) + if len(found) != 1: + raise SystemExit(f"{export}/Cargo.toml names no single abi3 floor: {sorted(found)}") + return int(found.pop()) + + +def licenses(info): + """The license files the package names, or else those beside its manifest or workspace.""" + if info["license_file"]: + return [Path(info["dir"]) / info["license_file"]] + for root in (Path(info["dir"]), Path(info["workspace"])): + found = sorted(p for p in root.iterdir() if p.is_file() and re.match(r"(LICEN[CS]E|COPYING)", p.name)) + if found: + return found + return [] + + +def metadata(info, floor): + """The wheel's core metadata, version 2.4.""" + lines = ["Metadata-Version: 2.4", f"Name: {info['name']}", f"Version: {pep440(info['version'])}"] + if info["description"]: + lines.append(f"Summary: {' '.join(info['description'].split())}") + if info["license"]: + # Cargo still takes the legacy MIT/Apache-2.0, which PyPI refuses only at upload. + if "/" in info["license"]: + raise SystemExit(f"license {info['license']!r} is no SPDX expression: write it with OR") + lines.append(f"License-Expression: {info['license']}") + lines += [f"License-File: {path.name}" for path in licenses(info)] + for label, url in (("Homepage", info["homepage"]), ("Repository", info["repository"])): + if url: + lines.append(f"Project-URL: {label}, {url}") + lines.append(f"Requires-Python: >=3.{floor}") + return "\n".join(lines) + "\n\n" + + +def record(files): + """RECORD: every file's hash and size, and RECORD's own row unhashed.""" + out = io.StringIO(newline="") + writer = csv.writer(out, lineterminator="\n") + for name, data in files.items(): + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + writer.writerow([name, f"sha256={digest}", len(data)]) + return out + + +def wheel_platform(target): + """The tag a wheel for `target` starts with, before repair narrows it.""" + tags = { + "x86_64-unknown-linux-gnu": "linux_x86_64", + "aarch64-unknown-linux-gnu": "linux_aarch64", + "aarch64-apple-darwin": f"macosx_{MACOS_WHEEL.replace('.', '_')}_arm64", + "x86_64-pc-windows-msvc": "win_amd64", + } + if target not in tags: + raise SystemExit(f"no wheel is built for {target}: the targets are {', '.join(tags)}") + return tags[target] + + +def library_name(lib, target): + """The file the generated module loads from beside itself.""" + if "windows" in target: + return f"{lib}.dll" + return f"lib{lib}.dylib" if "apple" in target else f"lib{lib}.so" + + +def write_wheel(info, export, library, target, out): + """An unrepaired wheel holding the generated module as a package beside its library.""" + lib = info["lib"] + floor = python_floor(export) + # The binary distribution format's escaped name. + dist = re.sub(r"[-_.]+", "_", info["name"]).lower() + version = pep440(info["version"]) + tag = f"cp3{floor}-abi3-{wheel_platform(target)}" + meta = f"{dist}-{version}.dist-info" + files = { + f"{lib}/__init__.py": (Path(export) / "python" / f"{lib}.py").read_bytes(), + f"{lib}/{library_name(lib, target)}": Path(library).read_bytes(), + } + for path in licenses(info): + files[f"{meta}/licenses/{path.name}"] = path.read_bytes() + files[f"{meta}/METADATA"] = metadata(info, floor).encode() + files[f"{meta}/WHEEL"] = ( + f"Wheel-Version: 1.0\nGenerator: rust2-ffi\nRoot-Is-Purelib: false\nTag: {tag}\n" + ).encode() + rows = record(files) + csv.writer(rows, lineterminator="\n").writerow([f"{meta}/RECORD", "", ""]) + files[f"{meta}/RECORD"] = rows.getvalue().encode() + path = Path(out) / f"{dist}-{version}-{tag}.whl" + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in files.items(): + archive.writestr(name, data) + return path + + +def repair(wheel, target, out): + """Hand the wheel to its platform's repair tool, which claims the portable tag or fails.""" + Path(out).mkdir(parents=True, exist_ok=True) + if "linux" in target: + arch = target.split("-")[0] + run("auditwheel", "repair", "--plat", f"manylinux_{GLIBC.replace('.', '_')}_{arch}", "-w", out, wheel) + elif "apple" in target: + run("delocate-wheel", "--require-archs", "arm64", "--require-target-macos-version", MACOS_WHEEL, + "-w", out, wheel) + else: + # The wheel's library is a .dll, which delvewheel analyzes only when asked. + run("delvewheel", "repair", "--analyze-existing", "--wheel-dir", out, wheel) + repaired = list(Path(out).glob("*.whl")) + if len(repaired) != 1: + raise SystemExit(f"repair wrote {len(repaired)} wheels into {out}") + check_wheel(repaired[0]) + return repaired[0] + + +def check_wheel(path): + """Fail unless RECORD covers every file with its true size and hash, as pip checks it.""" + with zipfile.ZipFile(path) as archive: + # Repair tools add directory entries, which RECORD never lists. + names = {n for n in archive.namelist() if not n.endswith("/")} + records = [n for n in names if n.endswith(".dist-info/RECORD")] + if len(records) != 1: + raise SystemExit(f"{path.name} has {len(records)} RECORD files") + rows = list(csv.reader(io.StringIO(archive.read(records[0]).decode()))) + listed = {row[0] for row in rows} + if listed != names: + raise SystemExit(f"{path.name}: RECORD and the archive differ: {sorted(listed ^ names)}") + for name, digest, size in rows: + if name == records[0]: + continue + data = archive.read(name) + want = "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + if digest != want or int(size) != len(data): + raise SystemExit(f"{path.name}: RECORD's row for {name} is wrong") + + +def digest(export, lib): + """A hash of the generated bindings, which every build host's export must agree on.""" + hasher = hashlib.sha256() + export = Path(export) + paths = [export / "python" / f"{lib}.py", export / "include" / f"{lib}.h", *sorted((export / "swift").iterdir())] + for path in paths: + hasher.update(path.relative_to(export).as_posix().encode() + b"\0") + # Exact bytes: each wheel ships its own host's module. + hasher.update(path.read_bytes() + b"\0") + return hasher.hexdigest() + + +def collections(): + """swift-collections' release whose `BasicContainers` the generated Swift was tested with. + + rust2 pins it by revision; a package resolved by version may depend only on a version, so + the tag at that revision stands in, and its absence is an error. + """ + revision = re.search(r'swift_collections_rev := "([0-9a-f]{40})"', (ROOT / "ffi/justfile").read_text(encoding="utf-8")) + if revision is None: + raise SystemExit("ffi/justfile pins no swift_collections_rev") + tags = output("git", "ls-remote", "--tags", COLLECTIONS) + for line in tags.splitlines(): + sha, ref = line.split("\t") + name = ref.removeprefix("refs/tags/").removesuffix("^{}") + if sha == revision.group(1) and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", name): + return name + raise SystemExit(f"swift-collections has no release at {revision.group(1)}") + + +def linker_settings(flags_by_platform): + """SwiftPM linker settings for the native libraries rustc reports on each platform.""" + seen = {} + for apple, flags in flags_by_platform.items(): + args = iter(shlex.split(flags)) + for flag in args: + if flag.startswith("-l") and len(flag) > 2: + setting = f".linkedLibrary({json.dumps(flag[2:])}" + elif flag == "-framework": + setting = f".linkedFramework({json.dumps(next(args))}" + else: + raise SystemExit(f"unsupported native linker flag on {apple}: {flag}") + seen.setdefault(setting, set()).add(apple) + swift = {"macos": ".macOS", "ios": ".iOS"} + out = [] + for setting, platforms in seen.items(): + if platforms == set(flags_by_platform): + out.append(setting + ")") + else: + out.append(setting + ", .when(platforms: [" + ", ".join(swift[p] for p in sorted(platforms)) + "]))") + return out + + +def package_swift(module, version, settings, binary): + """The package manifest: the Swift source over the XCFramework, either by URL or by path.""" + platforms = f'.macOS("{SWIFT_PLATFORMS["macos"]}"), .iOS("{SWIFT_PLATFORMS["ios"]}")' + links = "".join(f"\n {s}," for s in settings) + return f'''// swift-tools-version: 6.2 +// Generated by rust2-ffi's release builder. +import PackageDescription + +let package = Package( + name: "{module}", + platforms: [{platforms}], + products: [.library(name: "{module}", targets: ["{module}"])], + dependencies: [.package(url: "{COLLECTIONS}", exact: "{version}")], + targets: [ + {binary}, + .target( + name: "{module}", + dependencies: ["{module}FFI", .product(name: "BasicContainers", package: "swift-collections")], + linkerSettings: [{links} + ] + ), + ] +) +''' + + +def min_os(archive): + """The newest minimum OS any object in a static library declares.""" + versions = set() + for line in output("/usr/bin/otool", "-l", archive).splitlines(): + match = re.match(r"\s*minos ([0-9.]+)$", line) + if match: + versions.add(tuple(int(x) for x in match.group(1).split("."))) + if not versions: + raise SystemExit(f"{archive} declares no minimum OS") + return max(versions) + + +def swift(info, export, libs, natives, out): + """The XCFramework of every Apple slice, its checksummed ZIP, and the package around it.""" + if sys.platform != "darwin": + raise SystemExit("XCFramework packaging needs Apple's xcodebuild, on macOS") + if set(libs) != set(APPLE_TARGETS): + raise SystemExit(f"the Swift package ships {sorted(APPLE_TARGETS)}, given {sorted(libs)}") + lib, export, out = info["lib"], Path(export), Path(out) + module = pascal(lib) + for target, path in libs.items(): + floor = tuple(int(x) for x in SWIFT_PLATFORMS[APPLE_TARGETS[target]].split(".")) + if min_os(path) > floor: + raise SystemExit(f"{path} needs a newer OS than {SWIFT_PLATFORMS[APPLE_TARGETS[target]]}: build it with the deployment target set") + if out.exists(): + shutil.rmtree(out) + headers = out / "headers" + headers.mkdir(parents=True) + shutil.copy(export / "include" / f"{lib}.h", headers) + shutil.copy(export / "swift" / f"{lib}_atomics.h", headers) + modulemap = (export / "swift/module.modulemap").read_text(encoding="utf-8") + if f"../include/{lib}.h" not in modulemap: + raise SystemExit("the module map no longer names ../include: update the packager") + (headers / "module.modulemap").write_text(modulemap.replace(f"../include/{lib}.h", f"{lib}.h"), encoding="utf-8") + framework = out / f"{module}FFI.xcframework" + args = ["xcodebuild", "-create-xcframework"] + for target in APPLE_TARGETS: + args += ["-library", libs[target], "-headers", headers] + run(*args, "-output", framework) + shutil.rmtree(headers) + archive = out / f"{module}FFI.xcframework.zip" + run("/usr/bin/ditto", "-c", "-k", "--sequesterRsrc", "--keepParent", framework, archive) + checksum = hashlib.sha256(archive.read_bytes()).hexdigest() + # SwiftPM's own computation, as a consumer resolving the URL runs it. + swift_sum = output("swift", "package", "compute-checksum", archive).strip() + if swift_sum != checksum: + raise SystemExit(f"swift computes {swift_sum}, not {checksum}") + (out / "checksum.txt").write_text(checksum + "\n", encoding="utf-8") + package = out / "package" + sources = package / "Sources" / module + sources.mkdir(parents=True) + shutil.copy(export / "swift" / f"{lib}.swift", sources) + for path in licenses(info): + shutil.copy(path, package) + by_platform = {} + for target, apple in APPLE_TARGETS.items(): + by_platform.setdefault(apple, set()).add(Path(natives[target]).read_text(encoding="utf-8").strip()) + flags = {} + for apple, found in by_platform.items(): + if len(found) != 1: + raise SystemExit(f"{apple}'s slices link different native libraries: {sorted(found)}") + flags[apple] = found.pop() + state = {"collections": collections(), "settings": linker_settings(flags), "module": module} + (out / "swift.json").write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + shutil.rmtree(framework) + return state + + +def release_manifest(out, url): + """Package.swift naming the hosted ZIP at `url` by its checksum.""" + out = Path(out) + state = json.loads((out / "swift.json").read_text(encoding="utf-8")) + checksum = (out / "checksum.txt").read_text(encoding="utf-8").strip() + binary = f'.binaryTarget(name: "{state["module"]}FFI", url: {json.dumps(url)}, checksum: "{checksum}")' + text = package_swift(state["module"], state["collections"], state["settings"], binary) + (out / "package" / "Package.swift").write_text(text, encoding="utf-8") + + +def local_manifest(state, package, framework): + """Package.swift naming an unpacked XCFramework by path, for a consumer before any upload.""" + package = Path(package) + state = json.loads(Path(state).read_text(encoding="utf-8")) + binary = f'.binaryTarget(name: "{state["module"]}FFI", path: {json.dumps(os.path.relpath(framework, package))})' + (package / "Package.swift").write_text(package_swift(state["module"], state["collections"], state["settings"], binary), encoding="utf-8") + + +def compare(native, actual, what): + """Fail with a diff unless a consumer printed exactly what native Rust printed.""" + if native == actual: + return + sys.stderr.buffer.writelines(difflib.diff_bytes( + difflib.unified_diff, native.splitlines(keepends=True), actual.splitlines(keepends=True), + fromfile=b"native Rust", tofile=what.encode())) + raise SystemExit(f"{what} differs from native Rust") + + +def consume_python(check, native, module): + """Run a check program, or else an import, on the installed wheel from an empty directory.""" + program = [str(Path(check).resolve())] if check else ["-c", f"import {module}"] + with tempfile.TemporaryDirectory() as clean: + result = subprocess.run([sys.executable, "-I", "-X", "dev", "-W", "error", *program], + capture_output=True, cwd=clean) + sys.stderr.buffer.write(result.stderr) + if result.returncode: + raise SystemExit(f"{check} failed on the installed wheel with status {result.returncode}") + if native: + # Windows' CPython writes text-mode stdout with CRLF, where Rust's println! writes LF. + actual = result.stdout.replace(b"\r\n", b"\n") if sys.platform == "win32" else result.stdout + compare(Path(native).read_bytes(), actual, "installed wheel") + + +def exporter(manifest, features, *args): + """Run rust2-ffi from this checkout on the package, with its ffi.toml if it keeps one.""" + manifest = Path(manifest).resolve() + config = manifest.parent / "ffi.toml" + if not config.exists(): + # The exporter still requires a config file; an empty one leaves nothing out. + config = Path(tempfile.mkdtemp()) / "ffi.toml" + config.write_text("", encoding="utf-8") + cargo = ["--", "--features", features] if features.strip() else [] + run("cargo", "run", "--locked", "--release", "--quiet", "--", "--config", config, "--manifest-path", manifest, + *args, *cargo, cwd=ROOT / "ffi") + + +def stage_api(args): + """Each target's API, checked on this host, for the build hosts' exports to merge.""" + out = Path(args.out).resolve() + out.mkdir(parents=True, exist_ok=True) + for target in args.targets: + exporter(args.manifest_path, args.features, "--target", target, "--api", out / f"{target}.json") + + +def export(args): + """The binding of every target: this host's checked here, the others' staged elsewhere.""" + staged = sorted(Path(args.api).resolve().glob("*.json")) + others = [p for p in staged if p.stem not in args.targets] + if len(others) + len(args.targets) != len(staged): + raise SystemExit(f"{args.api} lacks a staged API for one of {args.targets}") + targets = [arg for target in args.targets for arg in ("--target", target)] + withs = [arg for path in others for arg in ("--with", path)] + exporter(args.manifest_path, args.features, *targets, *withs, "--out", Path(args.out).resolve()) + + +def cargo(manifest, *args, env=None): + """Cargo with the package's own toolchain, as rustup finds it from the package's directory.""" + return run("cargo", *args, cwd=Path(manifest).resolve().parent, env={**os.environ, **(env or {})}, + capture_output=True, encoding="utf-8") + + +def build(args): + """This host's wheels and Apple static libraries, the export's sources, and native output.""" + info = facts(args.manifest_path) + lib, export_dir, out = info["lib"], Path(args.export).resolve(), Path(args.out).resolve() + crate = export_dir / "Cargo.toml" + out.mkdir(parents=True, exist_ok=True) + for target in args.targets: + if target not in WHEEL_TARGETS and target not in APPLE_TARGETS: + raise SystemExit(f"no package ships {target}") + run("rustup", "target", "add", target, cwd=Path(args.manifest_path).resolve().parent) + if target in WHEEL_TARGETS: + env = {"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if "apple" in target else {} + linux = "linux" in target + # cargo-zigbuild links Linux against the glibc floor's symbols, whichever glibc the host has. + built = cargo(args.manifest_path, *(["zigbuild"] if linux else ["build"]), "--locked", "--release", + "--manifest-path", crate, "--target", f"{target}.{GLIBC}" if linux else target, + "--target-dir", export_dir / "target-python", "--features", "python", env=env) + sys.stderr.write(built.stderr) + library = export_dir / "target-python" / target / "release" / library_name(lib, target) + with tempfile.TemporaryDirectory() as raw: + repair(write_wheel(info, export_dir, library, target, raw), target, out / "wheels") + if target in APPLE_TARGETS: + apple = APPLE_TARGETS[target] + name = "MACOSX_DEPLOYMENT_TARGET" if apple == "macos" else "IPHONEOS_DEPLOYMENT_TARGET" + # The C library alone, apart from the Python extension, whose CPython symbols it + # would leave undefined; its own target directory, since cargo does not rebuild + # for a changed deployment target. + # Uncolored, so the note below reads as written whatever the caller's terminal. + built = cargo(args.manifest_path, "rustc", "--color", "never", "--locked", "--release", "--manifest-path", crate, + "--target", target, "--target-dir", export_dir / "target-swift", "--lib", + "--crate-type", "staticlib", "--", "--print=native-static-libs", + env={name: SWIFT_PLATFORMS[apple]}) + sys.stderr.write(built.stderr) + notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) + if len(notes) != 1: + raise SystemExit(f"rustc reported no native libraries for {target}") + static = out / "static" / target + static.mkdir(parents=True) + shutil.copy(export_dir / "target-swift" / target / "release" / f"lib{lib}.a", static) + (static / "native.txt").write_text(notes[0].strip() + "\n", encoding="utf-8") + # The generated sources, for the Swift package and the hosts' agreement. + # Text this writes is LF on every host, as the jobs reading it compare bytes. + for part in ("include", "swift", "python"): + shutil.copytree(export_dir / part, out / "export" / part, dirs_exist_ok=True) + shutil.copy(crate, out / "export") + (out / "digest.txt").write_text(digest(export_dir, lib) + "\n", newline="\n", encoding="utf-8") + (out / "lib.txt").write_text(lib + "\n", newline="\n", encoding="utf-8") + # Native output for a host whose wheel a consumer runs; an iOS slice runs nowhere here. + if args.example and any(target in WHEEL_TARGETS for target in args.targets): + if args.example not in info["examples"]: + raise SystemExit(f"{info['name']} has no example {args.example}: it has {info['examples']}") + features = ["--features", args.features] if args.features.strip() else [] + manifest = Path(args.manifest_path).resolve() + # Bytes, as the consumers' output is compared, with no newline translation. + native = run("cargo", "run", "--locked", "--release", "--quiet", "--manifest-path", manifest, + "--example", args.example, *features, cwd=manifest.parent, capture_output=True) + (out / "native.txt").write_bytes(native.stdout) + + +def template(info, manifest, rust2, mirror): + """The caller workflow, with every identity Cargo cannot establish spelled out in it.""" + if not re.fullmatch(r"[0-9a-f]{40}", rust2): + raise SystemExit("pin rust2 by a full commit SHA") + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", mirror): + raise SystemExit(f"the Swift mirror must be OWNER/REPO, not {mirror!r}") + values = { + "NAME": info["name"], + "MANIFEST": manifest, + "RUST2": rust2, + "MODULE": pascal(info["lib"]), + "MIRROR": mirror, + "MIRROR_OWNER": mirror.split("/")[0], + "MIRROR_REPO": mirror.split("/")[1], + } + text = (HERE / "release.yml").read_text(encoding="utf-8") + for key, value in values.items(): + text = text.replace(f"@{key}@", value) + leftover = sorted(set(re.findall(r"@[A-Z0-9_]+@", text))) + if leftover: + raise SystemExit(f"release.yml has unfilled placeholders: {leftover}") + return text + + +def init(args): + """Write .github/workflows/release-.yml into the repository holding the crate.""" + manifest = Path(args.manifest_path) + root = Path(output("git", "-C", manifest.resolve().parent, "rev-parse", "--show-toplevel").strip()) + info = facts(manifest) + rust2 = args.rust2 or output("git", "-C", HERE, "rev-parse", "HEAD").strip() + relative = manifest.resolve().relative_to(root).as_posix() + path = root / ".github" / "workflows" / f"release-{info['name']}.yml" + text = template(info, relative, rust2, args.swift_mirror) + if path.exists() and path.read_text(encoding="utf-8") != text: + raise SystemExit(f"{path} exists; delete it to write the release workflow") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, newline="\n", encoding="utf-8") + print(path) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("facts", help="print the package's Cargo facts as GitHub outputs") + p.add_argument("--manifest-path", required=True) + for name, helptext in (("api", "stage each target's API on this host"), + ("export", "export the binding of every staged target"), + ("build", "build this host's wheels and Apple libraries")): + p = sub.add_parser(name, help=helptext) + p.add_argument("--manifest-path", required=True) + p.add_argument("--features", default="") + p.add_argument("--out", required=True) + p.add_argument("targets", nargs="+") + if name == "export": + p.add_argument("--api", required=True) + if name == "build": + p.add_argument("--export", required=True) + p.add_argument("--example", default="") + p = sub.add_parser("digest", help="print the hash of the generated bindings") + p.add_argument("--manifest-path", required=True) + p.add_argument("--export", required=True) + p = sub.add_parser("swift", help="write the XCFramework ZIP and the package around it") + p.add_argument("--manifest-path", required=True) + p.add_argument("--export", required=True) + p.add_argument("--lib", action="append", default=[], metavar="TARGET=PATH") + p.add_argument("--native", action="append", default=[], metavar="TARGET=PATH") + p.add_argument("--url", required=True, help="where the release hosts the ZIP") + p.add_argument("--out", required=True) + p = sub.add_parser("swift-local", help="point an unpacked package at a local XCFramework") + p.add_argument("--state", required=True, help="the swift.json `swift` wrote") + p.add_argument("--package", required=True) + p.add_argument("--framework", required=True) + p = sub.add_parser("consume-python", help="run a check program, or an import, on the installed wheel") + group = p.add_mutually_exclusive_group(required=True) + group.add_argument("--check") + group.add_argument("--import", dest="module") + p.add_argument("--native") + p = sub.add_parser("compare", help="compare a consumer's output with native Rust's") + p.add_argument("native") + p.add_argument("actual") + p = sub.add_parser("init", help="write the release workflow into the crate's repository") + p.add_argument("--manifest-path", required=True) + p.add_argument("--swift-mirror", required=True, metavar="OWNER/REPO", + help="the repository SwiftPM resolves the package from, whose root holds its Package.swift") + p.add_argument("--rust2", help="the rust2 commit to pin; this checkout's by default") + args = parser.parse_args() + + if args.command == "facts": + info = facts(args.manifest_path) + lines = [f"{key}={info[key]}" for key in ("name", "version", "lib")] + lines += [f"module={pascal(info['lib'])}", f"tag={info['name']}-v{info['version']}", + f"examples={json.dumps(info['examples'])}"] + print("\n".join(lines)) + elif args.command == "api": + stage_api(args) + elif args.command == "export": + export(args) + elif args.command == "build": + build(args) + elif args.command == "digest": + print(digest(args.export, facts(args.manifest_path)["lib"])) + elif args.command == "swift": + info = facts(args.manifest_path) + pairs = lambda items: dict(item.split("=", 1) for item in items) # noqa: E731 + swift(info, args.export, pairs(args.lib), pairs(args.native), args.out) + release_manifest(args.out, args.url) + elif args.command == "swift-local": + local_manifest(args.state, args.package, args.framework) + elif args.command == "consume-python": + consume_python(args.check, args.native, args.module) + elif args.command == "compare": + compare(Path(args.native).read_bytes(), Path(args.actual).read_bytes(), "installed Swift package") + elif args.command == "init": + init(args) + + +if __name__ == "__main__": + main() diff --git a/ffi/release/release.yml b/ffi/release/release.yml new file mode 100644 index 000000000..2c8c23bbd --- /dev/null +++ b/ffi/release/release.yml @@ -0,0 +1,202 @@ +name: Release @NAME@ bindings + +# Written by rust2's `ffi/release/package.py init`; yours to edit. rust2's builder, pinned +# below, exports @NAME@'s pub API, builds its Python wheels and Swift package, and checks +# them in clean consumers, all read-only. The jobs after it publish only a tag's build: +# +# - PyPI: the project @NAME@, through the trusted publisher this repository registers there +# for this workflow and the `pypi` environment. +# - SwiftPM: @MODULE@FFI.xcframework.zip on this repository's @NAME@-v release, which +# the Package.swift pushed to @MIRROR@ names by URL and checksum, tagged . +# The mirror's token comes from a GitHub App installed on it, whose ID and key live in +# the `swift-mirror` environment. +# +# To release, push the tag @NAME@-v, naming the crate's version. Without a tag, as +# from a nightly's workflow_call, every job up to publication runs. If publishing fails, use +# "Re-run failed jobs": it reuses this run's build, and the publishing jobs upload only what +# is missing. A rebuild's files differ, so "Re-run all jobs" stops at any already published. +on: + push: + tags: + - "@NAME@-v*" + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-@NAME@-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + uses: kixel-ai/rust2/.github/workflows/bindings.yml@@RUST2@ + with: + manifest-path: @MANIFEST@ + # Optional: cargo features to ship, an example of the crate whose output every check + # program must print, and those programs: `python-check` and `swift-check`. + + tag: + needs: build + if: github.ref_type == 'tag' + runs-on: ubuntu-24.04 + steps: + - name: Tag names the crate's version + env: + TAG: ${{ github.ref_name }} + WANT: ${{ needs.build.outputs.tag }} + run: | + if [[ "$TAG" != "$WANT" ]]; then + echo "::error::$TAG is not the crate's version, $WANT: bump Cargo.toml before tagging." + exit 1 + fi + + pypi: + needs: [build, tag] + runs-on: ubuntu-24.04 + environment: + name: pypi + url: https://pypi.org/p/@NAME@ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: @NAME@-build-* + path: dist + merge-multiple: true + # A rerun uploads only the wheels PyPI lacks, and fails on one it hosts with other bytes. + - name: Skip the wheels PyPI hosts + id: missing + env: + PROJECT: '@NAME@' + shell: python3 -I {0} + run: | + import hashlib, json, os, pathlib, sys, urllib.error, urllib.request + url = f"https://pypi.org/simple/{os.environ['PROJECT']}/" + request = urllib.request.Request(url, headers={"Accept": "application/vnd.pypi.simple.v1+json"}) + try: + with urllib.request.urlopen(request) as response: + hosted = {f["filename"]: f["hashes"].get("sha256") for f in json.load(response)["files"]} + except urllib.error.HTTPError as error: + if error.code != 404: + raise + hosted = {} + missing = 0 + for wheel in sorted(pathlib.Path("dist/wheels").glob("*.whl")): + if wheel.name not in hosted: + missing += 1 + elif hosted[wheel.name] == hashlib.sha256(wheel.read_bytes()).hexdigest(): + print(f"PyPI already hosts {wheel.name}") + wheel.unlink() + else: + sys.exit(f"::error::PyPI hosts another {wheel.name}") + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + print(f"upload={'true' if missing else 'false'}", file=output) + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 + if: steps.missing.outputs.upload == 'true' + with: + packages-dir: dist/wheels/ + + # The XCFramework the mirror's Package.swift names, uploaded once: a consumer that + # resolved its checksum must never see other bytes at that URL. The release stays a draft + # until its asset is up, as an immutable release takes no asset once published. + github-release: + needs: [build, tag] + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-swift + path: swift + - name: Upload + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + ASSET: ${{ needs.build.outputs.module }}FFI.xcframework.zip + # Cargo's prerelease, such as 1.0.0-rc.1, is GitHub's too. + PRERELEASE: ${{ contains(needs.build.outputs.version, '-') }} + run: | + if ! gh release view "$TAG" --repo "$GITHUB_REPOSITORY" > /dev/null; then + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag --draft --prerelease="$PRERELEASE" --title "$TAG" --notes "" + fi + if gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --pattern "$ASSET" --dir existing; then + cmp "existing/$ASSET" "swift/$ASSET" || { echo "::error::$TAG already hosts another $ASSET"; exit 1; } + else + gh release upload "$TAG" --repo "$GITHUB_REPOSITORY" "swift/$ASSET" + fi + gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --draft=false + + # The hosted package, resolved by URL and checksum as a user's SwiftPM resolves it, + # before the mirror points anyone at it. + swift-resolve: + needs: [build, github-release] + runs-on: macos-26 + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-swift + path: swift + - name: Resolve + env: + MODULE: ${{ needs.build.outputs.module }} + run: | + mkdir -p consumer/Sources/Check + printf 'import %s\n' "$MODULE" > consumer/Sources/Check/main.swift + cat > consumer/Package.swift < /dev/null; then + # A rerun after the push: the tag must hold exactly this package. + git checkout --quiet "$VERSION" + diff -r --exclude=.git . ../swift/package || { echo "::error::$MIRROR's $VERSION holds another package"; exit 1; } + exit 0 + fi + git rm -r --quiet --ignore-unmatch . + cp -R ../swift/package/. . + git add --all + # A rerun whose branch already holds this package, but no tag, commits nothing. + git diff --cached --quiet || git commit --quiet -m "$VERSION" + git tag "$VERSION" + git push --quiet --atomic origin HEAD "refs/tags/$VERSION" diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py new file mode 100644 index 000000000..95af04a4c --- /dev/null +++ b/ffi/release/test_package.py @@ -0,0 +1,187 @@ +"""The builder's decisions that need no toolchain: names, versions, metadata, and manifests.""" + +import importlib.util +from pathlib import Path +import tempfile +import unittest +import zipfile + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("package", HERE / "package.py") +package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package) + + +def info(root, **overrides): + """Facts as `facts` reads them, for a package in `root`.""" + base = { + "name": "moq-ffi", "version": "0.4.12", "description": "Media over QUIC,\n bindings", + "license": "MIT OR Apache-2.0", "license_file": None, "repository": "https://github.com/moq-dev/moq", + "homepage": None, "lib": "moq", "dir": str(root), "workspace": str(root), "examples": [], + } + return {**base, **overrides} + + +def export(root, floor="311"): + """The parts of an export a wheel and a digest read.""" + (root / "python").mkdir(parents=True) + (root / "swift").mkdir() + (root / "include").mkdir() + (root / "python/moq.py").write_text("PENDING = 0\n") + (root / "include/moq.h").write_text("int moq_contract(void);\n") + (root / "swift/moq.swift").write_text("import MoqFFI\n") + (root / "Cargo.toml").write_text(f'pyo3-ffi = {{ features = ["abi3-py{floor}", "extension-module"] }}\n') + return root + + +class Versions(unittest.TestCase): + def test_release_and_prereleases(self): + self.assertEqual(package.pep440("0.4.12"), "0.4.12") + self.assertEqual(package.pep440("1.0.0-alpha.1"), "1.0.0a1") + self.assertEqual(package.pep440("1.0.0-beta.0"), "1.0.0b0") + self.assertEqual(package.pep440("2.1.0-rc.3"), "2.1.0rc3") + + def test_refuses_what_pypi_cannot_spell(self): + for version in ("1.0.0-pre", "1.0.0+build.1", "1.0.0-alpha", "01.0.0", "1.0"): + with self.assertRaises(SystemExit, msg=version): + package.pep440(version) + + +class Names(unittest.TestCase): + def test_swift_module_matches_the_exporter(self): + self.assertEqual(package.pascal("moq"), "Moq") + self.assertEqual(package.pascal("moq_ffi"), "MoqFfi") + self.assertEqual(package.pascal("api"), "Api") + + def test_library_of_any_crate_type(self): + for kinds in (["lib"], ["rlib"], ["dylib"], ["cdylib", "rlib"], ["staticlib"]): + targets = [{"kind": kinds, "name": "dep-fixture"}, {"kind": ["example"], "name": "transcript"}] + self.assertEqual(package.library({"name": "dep-fixture", "targets": targets}), "dep_fixture", kinds) + for targets in ([{"kind": ["proc-macro"], "name": "derive"}], [{"kind": ["bin"], "name": "tool"}]): + with self.assertRaises(SystemExit): + package.library({"name": "tool", "targets": targets}) + + def test_wheel_platforms(self): + self.assertEqual(package.wheel_platform("aarch64-apple-darwin"), "macosx_11_0_arm64") + self.assertEqual(package.wheel_platform("x86_64-pc-windows-msvc"), "win_amd64") + with self.assertRaises(SystemExit): + package.wheel_platform("armv7-unknown-linux-gnueabihf") + self.assertEqual(package.library_name("moq", "x86_64-pc-windows-msvc"), "moq.dll") + self.assertEqual(package.library_name("moq", "aarch64-apple-darwin"), "libmoq.dylib") + self.assertEqual(package.library_name("moq", "aarch64-unknown-linux-gnu"), "libmoq.so") + + +class Wheel(unittest.TestCase): + def test_metadata_and_record(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "LICENSE-MIT").write_text("MIT\n") + (root / "LICENSE-APACHE").write_text("Apache\n") + built = export(root / "export") + library = root / "libmoq.so" + library.write_bytes(b"\x7fELF") + out = root / "out" + out.mkdir() + wheel = package.write_wheel(info(root), built, library, "x86_64-unknown-linux-gnu", out) + self.assertEqual(wheel.name, "moq_ffi-0.4.12-cp311-abi3-linux_x86_64.whl") + package.check_wheel(wheel) + with zipfile.ZipFile(wheel) as archive: + names = set(archive.namelist()) + meta = archive.read("moq_ffi-0.4.12.dist-info/METADATA").decode() + self.assertIn("moq/__init__.py", names) + self.assertIn("moq/libmoq.so", names) + self.assertIn("moq_ffi-0.4.12.dist-info/licenses/LICENSE-MIT", names) + self.assertIn("Requires-Python: >=3.11\n", meta) + self.assertIn("Summary: Media over QUIC, bindings\n", meta) + self.assertIn("License-Expression: MIT OR Apache-2.0\n", meta) + self.assertIn("Project-URL: Repository, https://github.com/moq-dev/moq\n", meta) + + def test_refuses_a_legacy_license(self): + with self.assertRaises(SystemExit): + package.metadata(info("/src", license="MIT/Apache-2.0"), 11) + + def test_directory_entries_are_not_files(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + library = root / "libmoq.dylib" + library.write_bytes(b"\xcf\xfa\xed\xfe") + wheel = package.write_wheel(info(root), export(root / "export"), library, "aarch64-apple-darwin", root) + repaired = root / "repaired.whl" + # As delocate and auditwheel rewrite a wheel: each directory listed before its files. + with zipfile.ZipFile(wheel) as src, zipfile.ZipFile(repaired, "w") as dst: + dst.writestr("moq/", b"") + dst.writestr("moq_ffi-0.4.12.dist-info/", b"") + for name in src.namelist(): + dst.writestr(name, src.read(name)) + package.check_wheel(repaired) + + def test_a_tampered_record_fails(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + library = root / "moq.dll" + library.write_bytes(b"MZ") + wheel = package.write_wheel(info(root), export(root / "export"), library, "x86_64-pc-windows-msvc", root) + tampered = root / "tampered.whl" + with zipfile.ZipFile(wheel) as src, zipfile.ZipFile(tampered, "w") as dst: + for name in src.namelist(): + data = src.read(name) + dst.writestr(name, b"MZ!" if name.endswith("moq.dll") else data) + with self.assertRaises(SystemExit): + package.check_wheel(tampered) + + def test_floor_comes_from_the_glue(self): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(package.python_floor(export(Path(tmp), "312")), 12) + + +class Digest(unittest.TestCase): + def test_any_byte_splits_the_hosts(self): + with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: + first, second = export(Path(a)), export(Path(b)) + self.assertEqual(package.digest(first, "moq"), package.digest(second, "moq")) + # A Windows checkout once turned the embedded runtime CRLF. + (second / "python/moq.py").write_bytes(b"PENDING = 0\r\n") + self.assertNotEqual(package.digest(first, "moq"), package.digest(second, "moq")) + + +class Swift(unittest.TestCase): + def test_linker_settings_by_platform(self): + settings = package.linker_settings({ + "macos": "-framework Security -framework CoreFoundation -liconv -lSystem", + "ios": "-framework Security -liconv -lSystem", + }) + self.assertIn('.linkedFramework("Security")', settings) + self.assertIn('.linkedFramework("CoreFoundation", .when(platforms: [.macOS]))', settings) + self.assertIn('.linkedLibrary("iconv")', settings) + + def test_unknown_linker_flags_fail(self): + with self.assertRaises(SystemExit): + package.linker_settings({"macos": "-Wl,-dead_strip"}) + + def test_manifest(self): + text = package.package_swift("Moq", "1.7.5", ['.linkedLibrary("iconv")'], + '.binaryTarget(name: "MoqFFI", url: "https://x/MoqFFI.xcframework.zip", checksum: "00")') + self.assertIn('platforms: [.macOS("13.0"), .iOS("16.0")]', text) + self.assertIn('exact: "1.7.5"', text) + self.assertIn('dependencies: ["MoqFFI", .product(name: "BasicContainers", package: "swift-collections")]', text) + + +class Template(unittest.TestCase): + def test_fills_every_identity(self): + text = package.template(info("/src"), "rs/moq-ffi/Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi") + self.assertIn("uses: kixel-ai/rust2/.github/workflows/bindings.yml@" + "a" * 40, text) + self.assertIn('- "moq-ffi-v*"', text) + self.assertIn("url: https://pypi.org/p/moq-ffi", text) + self.assertIn("owner: moq-dev\n", text) + self.assertIn("repositories: moq-swift-ffi\n", text) + self.assertNotRegex(text, r"@[A-Z_]+@") + + def test_refuses_a_movable_pin_or_bad_mirror(self): + with self.assertRaises(SystemExit): + package.template(info("/src"), "Cargo.toml", "main", "moq-dev/moq-swift-ffi") + with self.assertRaises(SystemExit): + package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-swift-ffi") + + +if __name__ == "__main__": + unittest.main() diff --git a/quest/m2/README.md b/quest/m2/README.md index 3adfe64f5..62d242c7e 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -4,9 +4,7 @@ Research work with a named consumer that is not waiting on it yet: rust2-wasm, moq's browser route, the TypeScript translator that has to beat it, -[crabx](/quest/m2/crabx/README.md), which runs moq's CLIs without a Rust toolchain, and -[release templates](/quest/m2/release-templates.md), which would share moq's binding -package builders while preserving its registry and platform policy. +and [crabx](/quest/m2/crabx/README.md), which runs moq's CLIs without a Rust toolchain. ## Plan diff --git a/quest/m2/release-templates.md b/quest/m2/release-templates.md deleted file mode 100644 index d401226f2..000000000 --- a/quest/m2/release-templates.md +++ /dev/null @@ -1,52 +0,0 @@ -# [L] Release templates with shared binding package builders - -## Goal - -Generate release workflow templates backed by shared package builders, using moq's -existing Python and Swift binding releases as the first consumer. A repository can -adopt the generated workflows without changing its stock Rust crates or existing -registry identities, artifact URLs, supported targets, or minimum platform versions. -Clean consumer jobs install the built packages and compare their behavior with -native Rust before any publication. - -## Plan - -Decided with the maintainer on 2026-11-16 at the packaging spike's merge pick: -✅ keep this follow-up unchanged (rejected: drop it). - -Decided with the maintainer on 2026-11-16 in the -[packaging spike](https://github.com/kixel-ai/rust2/pull/761#issuecomment-6930429582): -✅ release templates with shared builders (rejected: keep investigating a -configless publisher; stop the spike with no full version). Registry ownership -and hosting policy are platform boundaries Cargo metadata cannot establish. - -- Start from the [spike report](/bench/registry/README.md), then inspect moq's - `release-py-ffi`, `release-swift-ffi`, and `release-swift-lib` workflows. Preserve - their current public contracts; the fixture spike did not validate moq packaging. -- Keep registry identities and publishing authorization visible in the generated - repository-owned workflow. Derive crate facts from Cargo metadata; do not add - exporter annotations or a separate packaging configuration file. -- Share artifact construction and verification, with immutable builder revisions. - Reuse crabx's platform matrix where the targets match. Keep publication separate - from read-only build and consumer jobs, behind the repository's release policy. -- Python: build the supported wheel matrix, inspect and repair native dependencies, - preserve stable-ABI claims, and validate installation outside the source checkout. -- Swift: package the macOS/iOS device/simulator slices moq actually ships, merge - target API differences, preserve deployment targets, and generate hosted ZIP - checksums and the release Package.swift. Validate a clean SwiftPM consumer. -- Exercise the generated workflows on moq's source and compare installed packages - with native Rust. Replace moq's handwritten Python and Swift packaging logic only - after those consumers pass; this quest does not authorize a production publish. -- Document how the same builder/template boundary extends to Maven, npm, NuGet, - pub.dev, and Go, using the report's constraints. Missing binding backends are - unsupported, never a silently incomplete package. Backend implementation and - other registry publishers are outside this Python/Swift migration. - -## Hosts - -- linux - portable Python wheel construction and clean consumers -- macos - Apple slices and SwiftPM consumers - -## Related - -- [crabx](/quest/m2/crabx/README.md) - reusable build matrices for Rust binaries diff --git a/sh/dispatch.sh b/sh/dispatch.sh index e65817470..516b7bd67 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -91,7 +91,7 @@ if [[ -n "$query" ]]; then exit fi -scripts=(bench/*/*.sh sh/*.sh) +scripts=(bench/*/*.sh ffi/release/*.sh sh/*.sh) # One function per module and phase; a module skips a phase it has none for. check_lint() { @@ -113,6 +113,7 @@ test_crabx() { just crabx test; } test_driver() { just driver test; } # moq's export at the pin, cut down to C, before the slower test build. test_ffi() { + python3 -I ffi/release/test_package.py just ffi moq-c just ffi test }