diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ef479ba6a..45e2b033e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -12,6 +12,10 @@ concurrency: cancel-in-progress: true jobs: + registry-spike: + name: Registry package consumers + uses: ./.github/workflows/registry.yml + check: name: Check runs-on: ubuntu-24.04-arm diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fc373e329..d16194342 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -17,6 +17,10 @@ concurrency: cancel-in-progress: true jobs: + registry-spike: + name: Registry package consumers + uses: ./.github/workflows/registry.yml + # The full check over every module, catching what a merge group's diff-scoped # `just ci` let into main. gate: @@ -420,7 +424,7 @@ jobs: # schedule, which runs main, gets the write token: a dispatch can run any branch. report: name: Report - needs: [gate, ts, bench, ffi-api, ffi, ffi-sanitize, ffi-cpp, ffi-android, ffi-bench, census] + needs: [registry-spike, gate, ts, bench, ffi-api, ffi, ffi-sanitize, ffi-cpp, ffi-android, ffi-bench, census] if: ${{ !cancelled() && github.event_name == 'schedule' }} runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.github/workflows/registry.yml b/.github/workflows/registry.yml new file mode 100644 index 000000000..fe007448f --- /dev/null +++ b/.github/workflows/registry.yml @@ -0,0 +1,87 @@ +name: Registry packaging spike + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + package: + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04-arm + private: '["self-hosted", "Linux", "ARM64"]' + - runner: macos-26 + private: '["self-hosted", "macOS", "ARM64"]' + runs-on: ${{ fromJSON(github.event.repository.private && matrix.private || format('["{0}"]', matrix.runner)) }} + steps: + # Fixed spike input: consumers need only a workflow call, no source or settings. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: kixel-ai/rust2 + ref: ${{ github.repository == 'kixel-ai/rust2' && github.sha || '13710f7b7f1dcc649fa5cdabff348b497bc835ff' }} + persist-credentials: false + - uses: ./.github/actions/nix + - name: Build fixture packages + run: nix develop --command bench/registry/build.sh + - name: Include standalone verifiers + run: | + cp bench/registry/consume.py ffi/target/registry/verify.py + cp bench/registry/consume-swift.sh ffi/target/registry/verify-swift.sh + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: registry-${{ matrix.runner }} + if-no-files-found: error + path: | + ffi/target/registry/*.whl + ffi/target/registry/*.zip + ffi/target/registry/*.py + ffi/target/registry/*.swift + ffi/target/registry/*.sh + ffi/target/registry/native.txt + + consumer: + needs: package + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04-arm + private: '["self-hosted", "Linux", "ARM64"]' + - runner: macos-26 + private: '["self-hosted", "macOS", "ARM64"]' + runs-on: ${{ fromJSON(github.event.repository.private && matrix.private || format('["{0}"]', matrix.runner)) }} + env: + SPIKE_ROOT: ${{ github.workspace }}/.tmp/registry-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runner }} + defaults: + run: + working-directory: ${{ env.SPIKE_ROOT }} + steps: + # No checkout, Rust toolchain, exporter, source paths or PYTHONPATH. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: registry-${{ matrix.runner }} + path: ${{ env.SPIKE_ROOT }}/artifacts + - name: Install and run wheel + run: | + python3 -m venv env + env/bin/python -m pip install --no-index --no-deps artifacts/*.whl + mkdir clean + cd clean + ../env/bin/python -I ../artifacts/verify.py ../artifacts + - name: Install and run Swift package + if: runner.os == 'macOS' + run: | + cd artifacts + unzip Api.zip + bash verify-swift.sh . + - name: Remove consumer workspace + if: always() + working-directory: ${{ github.workspace }} + run: rm -rf "$SPIKE_ROOT" diff --git a/bench/registry/README.md b/bench/registry/README.md new file mode 100644 index 000000000..e14b5c400 --- /dev/null +++ b/bench/registry/README.md @@ -0,0 +1,201 @@ +# Registry packaging spike + +Accepted direction: generate release templates for moq. A single configless publisher +cannot infer registry ownership from Cargo metadata. Shared package builders are +still useful. This spike publishes nothing and changes no binding semantics. + +The maintainer [accepted the no-go on 2026-11-16](https://github.com/kixel-ai/rust2/pull/761#issuecomment-6930429582): +release templates with shared builders are the full version (rejected: keep +investigating a configless publisher; stop the spike with no full version). +The [follow-up quest](/quest/m2/release-templates.md) starts with moq's existing +Python and Swift releases and preserves their identities and platform policy. + +`build.sh` exports the existing api fixture (including its existing exclusion file), +builds a CPython stable-ABI library, writes a wheel, and on macOS wraps a separate +C static library in an XCFramework with a Swift source package. The Swift wrapper +stays source so the consumer compiles it with its compiler; the Rust library is +binary. The XCFramework currently has only the build host's macOS slice. This is +not an iOS or universal distribution claim. + +Run from the repository root through `quest gate nix develop --command +bench/registry/build.sh`. Output is in `ffi/target/registry`. Install the wheel in a +fresh Python environment, then run `consume.py ` with that +environment's Python from outside the checkout. The consumer compares stdout +byte for byte with the fixture's native Rust transcript, then exercises futures, +ownership, integer values and calls. The wheel contains the generated module as +`api/__init__.py` and its library beside it, with a complete hashed RECORD. + +On macOS, unpack `Api.zip` beside a fresh consumer directory and run +`consume-swift.sh ` from there. The script uses SwiftPM to +resolve the package, runs the same transcript against Rust, then futures and +ownership checks. Swift 6.2 or newer and macOS 15 are required for the fixture's +executor APIs. BasicContainers is pinned to the exporter's swift-collections +revision. No registry credentials, signing keys or publishing permissions are used. + +## Scope and evidence + +The fixture is intentional: this measures packaging an existing export, not finding +an arbitrary facade in a workspace. Name, version and description come from its +Cargo manifest; the library's distinct `api` name is respected. The fixture is +`publish = false`; this spike does not upload it anywhere. Existing binding +fixtures own semantic differential tests; these consumers check that packaging +preserves the generated API and runtime loading. + +The native wheel tag only claims the build host's platform. Linux packaging removes +RUNPATH and checks that the only dependencies are glibc, its loader and libgcc_s, +with no symbol newer than Ubuntu 24.04's glibc 2.39. Anything else fails packaging. +This is checked on each actual architecture; it is not a manylinux claim. macOS +uses a major_0 tag (the form pip accepts on macOS 11+) and the host architecture. A release needs +manylinux builds and auditwheel repair on Linux, delocate on macOS, and DLL +inspection on Windows, including transitive native dependencies. Stable ABI alone +does not establish OS portability. The generated extension requests abi3-py311, +so the wheel uses cp311-abi3 and Requires-Python >=3.11. + +Linux x86_64 execution passed on 2026-11-13 with CPython 3.14.10: the built +`api_fixture-0.0.0-cp311-abi3-linux_x86_64.whl` installed into a fresh uv-created +virtual environment with no index or dependencies. Running from a clean directory +with `-I` matched the native Rust transcript byte for byte and passed futures, +ownership, values and calls. A separate ZIP inspection checked RECORD coverage, +file sizes and SHA-256 hashes. The full `quest gate nix develop --command just ci` +passed, including the exporter's existing Python and Swift differential tests. + +The export includes host and wasm32-wasip1 API descriptions, as the existing fixture +suite does: its values test checks that wasm32-only calls are refused on the host. +Only the host native library is packaged. Initial export attempts exposed missing +CACHEDIR.TAG markers in imported local build artifacts; restoring those markers +allowed the unchanged exporter to run. No cache tools or checks were disabled. + +macOS 26.6.2 arm64 execution passed on kixtop on 2026-11-16, with Xcode 26.6 and +Apple Swift 6.3.3. In an isolated checkout, `nix develop --command quest gate +bench/registry/build.sh` built `Api.zip`. From a fresh artifact directory containing +only the ZIP, native transcript, Swift consumer and verifier, `unzip -q Api.zip` +and the verifier built a release SwiftPM consumer with Apple's compiler. It ran +through `nix develop --command quest gate env -i HOME=/Users/kixelated +PATH=/usr/bin:/bin:/usr/sbin:/sbin /bin/bash /verify-swift.sh +`, so the consumer inherited only the host home and Apple +toolchain search path. The transcript matched byte for byte, followed by +`futures ok` and `owned ok`. Both transcripts had SHA-256 +`729ffddcec8928c564d29fd1ee3c4fc34ccaecf18900d34be63c4e6fd0a50714`. +`plutil -p Api/ApiFFI.xcframework/Info.plist` and `lipo -info +Api/ApiFFI.xcframework/macos-arm64/libapi.a` independently confirmed one macOS arm64 +slice. + +The macOS builder selects the installed Xcode and its SDK explicitly: Nix's SDK +bundle lacks `libz.tbd` and cannot serve as Xcode's developer directory. The current +spike exports the same WASI target as the fixture suite, so target availability +agrees, and the Swift fixture avoids shadowing its `debug()` helper in Swift 6. +The original external-call pin, commit `901f7baa`, also passed Swift validation; that +older fixture needs an explicit `@Sendable` type on its duration-conversion helper. +After merging main to resolve the fixture conflict, the current source passed +again with the transcript hash above. Apple Swift still warns about a pointer +capture in the fixture's separate shared-call stress test; the consumer exercises +the transcript, futures and ownership, not that stress test. +The first GitHub reusable invocation, +[run 43968322384](https://github.com/kixel-ai/rust2/actions/runs/43968322384), +built both platforms and passed the clean Linux consumer. The macOS wheel +installed but could not import: Nix's linker wrapper embedded a Nix-store +libiconv dependency absent from the consumer runner. The builder now uses +Apple's linker, and macOS wheel packaging inspects `otool -L` and refuses any +dependency outside system libraries and frameworks. Its dylib ID is relative. +The next GitHub invocation passed both artifact consumers, as recorded below. +Do not delete moq's release workflows on this evidence. + +## GitHub invocation and artifact handoff + +On 2026-11-16, ran `gh workflow run check.yml --repo kixel-ai/rust2 --ref +quest/m2/registry`. The registered manual Check workflow called +`./.github/workflows/registry.yml` at +`13710f7b7f1dcc649fa5cdabff348b497bc835ff` in +[run 43971814703](https://github.com/kixel-ai/rust2/actions/runs/43971814703). +Direct dispatch of `registry.yml` returned 404 because that new workflow was not +yet registered on the default branch. The manual caller also runs the existing +full Check job; nightly's report now includes the reusable job's result. + +Both builders passed and uploaded `registry-ubuntu-24.04-arm` and +`registry-macos-26`. Separate jobs downloaded these artifacts without checkout, +Nix, Rust, exporter or source paths. They created a venv, installed the wheel +with `pip install --no-index --no-deps artifacts/*.whl`, and ran the copied +verifier with Python's `-I` from a clean directory. +[Linux arm64](https://github.com/kixel-ai/rust2/actions/runs/43971814703/job/133671565856) +and [macOS arm64](https://github.com/kixel-ai/rust2/actions/runs/43971814703/job/133671565931) +each printed `installed wheel: transcript matches Rust; futures, ownership, +values and calls pass`. The macOS 26.7.2 consumer then unpacked `Api.zip` and ran +`bash verify-swift.sh .`: SwiftPM's release build passed, its transcript matched +Rust byte for byte, then `futures ok` and `owned ok` passed. Both consumer jobs +removed their workspaces successfully. + +The Mac builder's independent `otool -L` check reported only the relative dylib +ID, `/usr/lib/libiconv.2.dylib` and `/usr/lib/libSystem.B.dylib`. The external-call +fixture pin now uses this validated revision, including the Apple linker fix. +Downloaded both uploaded artifacts with `gh run download 43971814703 --repo +kixel-ai/rust2 --dir .tmp/registry-verified-artifacts`: independent ZIP inspection +verified both wheels' complete RECORD coverage, sizes and SHA-256 hashes, and +the XCFramework plist described exactly one macOS arm64 slice. Both packaged +native transcripts had SHA-256 +`729ffddcec8928c564d29fd1ee3c4fc34ccaecf18900d34be63c4e6fd0a50714`. +This completes the fixture-packaging spike and establishes the artifact handoff, +not a configless publisher or moq release migration. + +## What each registry still needs + +| Destination | Package work | Information Cargo cannot establish | +| --- | --- | --- | +| PyPI | Wheel matrix, dependency repair, name/version normalization, license metadata, sdist policy | Ownership of the distribution name and trusted-publisher registration | +| SwiftPM | macOS/iOS device/simulator slices, merged exports for cfg differences, minimum OS versions, checksummed hosted zip and release Package.swift | Artifact hosting URL and repository/tag policy; code signing identity if signing is required | +| Maven Central | AAR with classes.jar, JNI libraries per Android ABI, generated R8 rules, POM, sources/javadoc artifacts and signatures | Verified groupId namespace, publisher authorization and signing key | +| npm | A Node binding backend (not currently generated), platform packages and JS loader, TypeScript declarations | Scope/name ownership and trusted-publisher registration | +| NuGet | A C# binding backend (not currently generated), managed assembly plus RID-specific native assets | Package ID ownership, credentials and desired managed target frameworks | +| pub.dev | Generated Dart library plus native asset hooks and platform archives, pubspec and license files | Publisher authorization and the distribution/download host | +| Go | Generated cgo packages, module-relative imports and native library delivery for supported targets | Canonical module path, repository/subdirectory and release tags (including major-version suffixes) | + +Cargo's repository URL can suggest a Go module path or Maven namespace. It cannot +prove either is the publisher's intended existing identity. Cargo package names +also need not equal existing PyPI names or npm scopes. A manifest's license and +repository fields are optional; absence must produce a packaging error where the +registry requires them. Authentication setup is an external platform boundary, +not something a workflow can manufacture from a crate. + +For moq, the existing release-py-ffi, release-swift-ffi and release-swift-lib +workflows additionally encode their current names, hosting and platform policy. +Replacing those workflows requires preserving those contracts and testing moq, +not just this fixture. The quest's go condition is therefore not established; +templates are the recommended no-go outcome even if both artifact consumers pass. + +## Prior art and primary sources + +- [BoltFFI](https://github.com/boltffi/boltffi): annotation-based generation and local packaging are adjacent to this problem; do not treat it as evidence of registry ownership automation. +- [maturin](https://www.maturin.rs/distribution.html): Python distribution tooling can own wheel repair/build concerns but does not solve the other registries. +- [Wheel format](https://packaging.python.org/en/latest/specifications/binary-distribution-format/): tags, metadata and RECORD. +- [Apple binary packages](https://developer.apple.com/documentation/Xcode/distributing-binary-frameworks-as-swift-packages): XCFramework binary targets and hosted artifact checksums. +- [Maven Central requirements](https://central.sonatype.org/publish/requirements/) and [namespace registration](https://central.sonatype.org/register/namespace/). +- [npm trusted publishing](https://docs.npmjs.com/trusted-publishers/) and [scopes](https://docs.npmjs.com/about-scopes/). +- [NuGet native assets](https://learn.microsoft.com/nuget/create-packages/native-files-in-net-packages). +- [Dart hooks](https://dart.dev/tools/hooks) and [publishing](https://dart.dev/tools/pub/publishing). +- [Go module publishing](https://go.dev/doc/modules/developing). + +The nightly workflow calls the same reusable workflow, outside the merge gate. +Consumer workspaces are unique per run attempt and removed in an always-run +cleanup step, including on self-hosted runners. +Builders replace their own generated output; Swift consumers create a fresh package +beside the artifact regardless of the invoking directory. Rust's reported native +link libraries become SwiftPM linker settings; unrecognized flags fail packaging. +The collections revision is read from ffi/justfile and carried inside the package. + +Registry requirements checked on 2026-11-13. The table is a design assessment, +not a claim that any of those unimplemented publishers was exercised. + +## Reusable workflow + +`.github/workflows/registry.yml` is callable with no inputs or secrets. It builds +this fixed fixture on Linux arm64 and macOS arm64 and passes only artifacts to +fresh consumer jobs. A caller needs just a job with +`uses: kixel-ai/rust2/.github/workflows/registry.yml@`. +External calls pin the fixture, exporter and packaging scripts together to an +immutable rust2 revision, independently of the caller's checkout. Calls within +rust2 test the triggering commit so nightly catches changes on main. It is a spike +runner, not a publisher of the caller's crate. Private callers select self-hosted +runners with OS/architecture labels. Public callers use hosted runners. + +These two targets overlap crabx's planned matrix. The pipeline quest has not +landed a shared matrix to reuse. Extending this to stock crates requires facade +discovery and packaging of moq's existing identity; neither is claimed here. diff --git a/bench/registry/build.sh b/bench/registry/build.sh new file mode 100755 index 000000000..b040576ab --- /dev/null +++ b/bench/registry/build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Export once, build packages, and retain the native oracle with the consumer scripts. +set -euo pipefail +export PATH="${RUST2_CARGO_SHIM:+$RUST2_CARGO_SHIM:}${RUST2_TS_TOOLCHAIN:?run in nix develop}/bin:$PATH" +if [[ $(uname) == Darwin ]]; then + # Nix selects an SDK bundle; this build needs the installed Xcode and its full SDK. + export DEVELOPER_DIR + DEVELOPER_DIR=$(env -u DEVELOPER_DIR /usr/bin/xcode-select -p) + export SDKROOT + SDKROOT=$(env -u SDKROOT /usr/bin/xcrun --sdk macosx --show-sdk-path) + # Nix's linker wrapper injects store libraries unavailable to wheel consumers. + export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=/usr/bin/clang + export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=/usr/bin/clang +fi +cd "$(dirname "$0")/../.." +rm -rf ffi/target/registry +host=$(rustc -vV | sed -n 's/^host: //p') +just ffi run --target "$host" --target wasm32-wasip1 --config fixtures/api/ffi.toml --manifest-path fixtures/api/Cargo.toml --out target/registry-export -- --features doubled +cargo build --locked --release --manifest-path ffi/target/registry-export/Cargo.toml --features python +python3 bench/registry/package.py wheel +cargo run --quiet --locked --manifest-path ffi/Cargo.toml --package api-fixture --example transcript --features doubled >ffi/target/registry/native.txt +cp ffi/tests/api.py ffi/target/registry/consumer.py +if [[ $(uname) == Darwin ]]; then + # Build separately so Python symbols cannot enter the Swift library. + cargo build --locked --release --manifest-path ffi/target/registry-export/Cargo.toml --no-default-features + python3 bench/registry/package.py swift + cp ffi/tests/api.swift ffi/target/registry/consumer.swift +fi diff --git a/bench/registry/consume-swift.sh b/bench/registry/consume-swift.sh new file mode 100755 index 000000000..8083fabac --- /dev/null +++ b/bench/registry/consume-swift.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Build a new executable package whose only library dependency is the artifact. +set -euo pipefail +artifacts=$(cd "$1" && pwd) +cd "$artifacts" +consumer=$(mktemp -d consumer.XXXXXXXX) +cd "$consumer" +mkdir -p Sources/Check +cp "$artifacts/consumer.swift" Sources/Check/check.swift +python3 - <<'PYTHON' +from pathlib import Path +import re +revision = Path('../Api/collections-revision.txt').read_text().strip() +assert re.fullmatch(r'[a-f0-9]{40}', revision) +Path('Package.swift').write_text('''// swift-tools-version: 6.2 +import PackageDescription +let package = Package( + name: "Check", platforms: [.macOS(.v15)], + dependencies: [ + .package(path: "../Api"), + .package(url: "https://github.com/apple/swift-collections", revision: "''' + revision + '''") + ], + targets: [.executableTarget(name: "Check", dependencies: [ + .product(name: "Api", package: "Api"), + .product(name: "BasicContainers", package: "swift-collections") + ])] +) +''') +PYTHON +swift build -c release +.build/release/Check >actual.txt +cmp "$artifacts/native.txt" actual.txt +.build/release/Check futures +.build/release/Check owned diff --git a/bench/registry/consume.py b/bench/registry/consume.py new file mode 100644 index 000000000..fc14626cf --- /dev/null +++ b/bench/registry/consume.py @@ -0,0 +1,25 @@ +"""Run an installed wheel from an isolated consumer directory against native Rust.""" +from pathlib import Path +import difflib +import os +import subprocess +import sys + +artifacts = Path(sys.argv[1]).resolve() +python = Path(os.path.abspath(sys.executable)) +for case in [None, "futures", "owned", "values", "calls"]: + command = [str(python), "-I", "-X", "dev", "-W", "error", str(artifacts / "consumer.py")] + if case: + command.append(case) + output = subprocess.run(command, capture_output=True) + if output.returncode: + sys.stderr.buffer.write(output.stderr) + raise SystemExit(output.returncode) + if case is None: + native = (artifacts / "native.txt").read_bytes() + if output.stdout != native: + sys.stderr.buffer.writelines(difflib.diff_bytes( + difflib.unified_diff, native.splitlines(keepends=True), + output.stdout.splitlines(keepends=True), fromfile=b"native Rust", tofile=b"installed wheel")) + raise SystemExit("installed wheel differs from native Rust") +print("installed wheel: transcript matches Rust; futures, ownership, values and calls pass") diff --git a/bench/registry/package.py b/bench/registry/package.py new file mode 100644 index 000000000..6080034df --- /dev/null +++ b/bench/registry/package.py @@ -0,0 +1,151 @@ +"""Package the api fixture's export without changing the Rust or generated API.""" + +import base64 +import csv +import hashlib +import io +import json +import re +import shlex +import platform as host_platform +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig +import tomllib +import zipfile + +ROOT = Path(__file__).resolve().parents[2] +EXPORT = ROOT / "ffi/target/registry-export" +OUT = ROOT / "ffi/target/registry" + + +def run(*args, **kwargs): + """Fail the package build when any tool fails.""" + return subprocess.run(args, check=True, **kwargs) + + +def wheel(): + """Write a native wheel with hashes for every installed file.""" + manifest = tomllib.loads((ROOT / "ffi/fixtures/api/Cargo.toml").read_text()) + version = manifest["package"]["version"] + # Native tags deliberately make no manylinux or older macOS compatibility claim. + platform = sysconfig.get_platform().replace("-", "_").replace(".", "_") + if sys.platform == "darwin": + major, *_ = host_platform.mac_ver()[0].split(".") + platform = f"macosx_{major}_0_{host_platform.machine()}" + tag = f"cp311-abi3-{platform}" + info = f"api_fixture-{version}.dist-info" + library = {"linux": "libapi.so", "darwin": "libapi.dylib"}[sys.platform] + binary = EXPORT / "target/release" / library + if sys.platform == "linux": + binary_copy = OUT / library + shutil.copyfile(binary, binary_copy) + run("patchelf", "--remove-rpath", str(binary_copy)) + dynamic = run("readelf", "-d", str(binary_copy), capture_output=True, text=True).stdout + needed = set(re.findall(r"Shared library: \[(.*?)\]", dynamic)) + allowed = {"libgcc_s.so.1", "libc.so.6", "libm.so.6", "ld-linux-x86-64.so.2", "ld-linux-aarch64.so.1"} + if needed - allowed or "RPATH" in dynamic or "RUNPATH" in dynamic: + raise SystemExit(f"unpackaged native dependencies: {dynamic}") + versions = run("readelf", "--version-info", str(binary_copy), capture_output=True, text=True).stdout + glibc = [tuple(map(int, v.split("."))) for v in re.findall(r"GLIBC_([0-9.]+)", versions)] + if not glibc or max(glibc) > (2, 39): + raise SystemExit("wheel requires glibc newer than the Ubuntu 24.04 consumer") + binary = binary_copy + elif sys.platform == "darwin": + binary_copy = OUT / library + shutil.copyfile(binary, binary_copy) + run("/usr/bin/install_name_tool", "-id", f"@rpath/{library}", str(binary_copy)) + dynamic = run("/usr/bin/otool", "-L", str(binary_copy), capture_output=True, text=True).stdout + needed = {line.strip().split(" (", 1)[0] for line in dynamic.splitlines()[1:]} + external = {name for name in needed if name != f"@rpath/{library}" + and not name.startswith(("/usr/lib/", "/System/Library/Frameworks/"))} + if external: + raise SystemExit(f"unpackaged native dependencies: {dynamic}") + print(dynamic, end="") + binary = binary_copy + files = { + "api/__init__.py": (EXPORT / "python/api.py").read_bytes(), + f"api/{library}": binary.read_bytes(), + f"{info}/METADATA": ( + f"Metadata-Version: 2.1\nName: api-fixture\nVersion: {version}\n" + f"Summary: {manifest['package']['description']}\nRequires-Python: >=3.11\n\n" + ).encode(), + f"{info}/WHEEL": ( + f"Wheel-Version: 1.0\nGenerator: rust2-registry-spike\n" + f"Root-Is-Purelib: false\nTag: {tag}\n" + ).encode(), + } + record = io.StringIO(newline="") + writer = csv.writer(record) + for name, data in files.items(): + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + writer.writerow([name, f"sha256={digest}", len(data)]) + writer.writerow([f"{info}/RECORD", "", ""]) + files[f"{info}/RECORD"] = record.getvalue().encode() + with zipfile.ZipFile(OUT / f"api_fixture-{version}-{tag}.whl", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in files.items(): + archive.writestr(name, data) + + +def swift(): + """Wrap the host's C static library in an XCFramework and Swift source package.""" + if sys.platform != "darwin": + raise SystemExit("XCFramework packaging requires Apple's xcodebuild on macOS") + revision = re.search(r'swift_collections_rev := "([a-f0-9]{40})"', (ROOT / "ffi/justfile").read_text()) + if revision is None: + raise SystemExit("ffi/justfile must name an immutable swift_collections_rev") + collections = revision.group(1) + package = OUT / "Api" + if package.exists(): + shutil.rmtree(package) + package.mkdir() + headers = OUT / "headers" + if headers.exists(): + shutil.rmtree(headers) + headers.mkdir() + shutil.copy(EXPORT / "include/api.h", headers) + shutil.copy(EXPORT / "swift/api_atomics.h", headers) + modulemap = (EXPORT / "swift/module.modulemap").read_text().replace("../include/api.h", "api.h") + (headers / "module.modulemap").write_text(modulemap) + run("xcodebuild", "-create-xcframework", "-library", str(EXPORT / "target/release/libapi.a"), + "-headers", str(headers), "-output", str(package / "ApiFFI.xcframework")) + sources = package / "Sources/Api" + sources.mkdir(parents=True) + shutil.copy(EXPORT / "swift/api.swift", sources) + built = run("cargo", "rustc", "--locked", "--release", "--manifest-path", str(EXPORT / "Cargo.toml"), + "--lib", "--", "--print=native-static-libs", capture_output=True, text=True) + notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) + if len(notes) != 1: + raise SystemExit(f"rustc did not report native link libraries: {built.stderr}") + flags = iter(shlex.split(notes[0])) + settings = [] + for flag in flags: + if flag.startswith("-l"): + settings.append(f".linkedLibrary({json.dumps(flag[2:])})") + elif flag == "-framework": + settings.append(f".linkedFramework({json.dumps(next(flags))})") + else: + raise SystemExit(f"unsupported native linker flag: {flag}") + (package / "collections-revision.txt").write_text(collections + "\n") + (package / "Package.swift").write_text('''// swift-tools-version: 6.2 +import PackageDescription +let package = Package( + name: "Api", + platforms: [.macOS(.v15)], + products: [.library(name: "Api", targets: ["Api"])], + dependencies: [.package(url: "https://github.com/apple/swift-collections", revision: "''' + collections + '''")], + targets: [ + .binaryTarget(name: "ApiFFI", path: "ApiFFI.xcframework"), + .target(name: "Api", dependencies: ["ApiFFI", .product(name: "BasicContainers", package: "swift-collections")], linkerSettings: [''' + ", ".join(settings) + ''']) + ] +) +''') + # Consumers receive only the package, script and native reference transcript. + shutil.make_archive(str(OUT / "Api"), "zip", OUT, "Api") + + +if __name__ == "__main__": + OUT.mkdir(parents=True, exist_ok=True) + {"wheel": wheel, "swift": swift}[sys.argv[1]]() diff --git a/crabx/npm/crabx.js b/crabx/npm/crabx.js index b77814410..e7969ba56 100644 --- a/crabx/npm/crabx.js +++ b/crabx/npm/crabx.js @@ -67,7 +67,6 @@ function main() { // Runs crabx as a child and exits as it did. function spawnAndWait(file, args) { - const child = spawn(file, args, { stdio: "inherit" }); // A terminal's Ctrl-C and Ctrl-\ already reach the whole foreground group, crabx included, // so this process only waits for crabx to exit. A supervisor's signals reach only this one. const ignore = () => {}; @@ -78,6 +77,8 @@ function spawnAndWait(file, args) { ? { SIGINT: ignore } : { SIGINT: ignore, SIGQUIT: ignore, SIGTERM: forward, SIGHUP: forward }; for (const [signal, handler] of Object.entries(handlers)) process.on(signal, handler); + // The child can run before spawn returns, so catch supervisor signals before starting it. + const child = spawn(file, args, { stdio: "inherit" }); child.on("error", (error) => fail(`cannot run ${file}: ${error.message}`)); child.on("exit", (code, signal) => { if (signal) { diff --git a/crabx/test_release.py b/crabx/test_release.py index bb2b36037..3d4974ba4 100644 --- a/crabx/test_release.py +++ b/crabx/test_release.py @@ -238,6 +238,28 @@ def test_missing_platform_package_fails_loudly(self): self.assertEqual(result.returncode, 1) self.assertIn("is not installed; reinstall crabx without omitting optional dependencies", result.stderr) + def test_signal_during_spawn_reaches_the_binary(self): + self.binary("exit 7\n") + # Deliver a real signal while spawn returns its child, before the launcher can + # take its next step. The stand-in child exits only once it receives that signal. + self.old_node.write_text('''process.execve = undefined; +const { EventEmitter } = require("node:events"); +require("node:child_process").spawn = () => { + const child = new EventEmitter(); + child.kill = (signal) => { + process.stdout.write(`forwarded ${signal}\\n`); + child.emit("exit", 7, null); + return true; + }; + // Keep one event-loop turn alive so Node dispatches the pending OS signal. + setImmediate(() => {}); + process.kill(process.pid, "SIGTERM"); + return child; +}; +''') + result = self.launch("old node") + self.assertEqual((result.returncode, result.stdout, result.stderr), (7, "forwarded SIGTERM\n", "")) + @unittest.skipIf(sys.platform == "win32", "install.sh is for Unix") class Install(unittest.TestCase): diff --git a/ffi/tests/api.swift b/ffi/tests/api.swift index 55bdc3c18..26b6d75de 100644 --- a/ffi/tests/api.swift +++ b/ffi/tests/api.swift @@ -1054,8 +1054,8 @@ func script() { check(refused("NonZero(_:)") { _ = try NonZero(0) }, "a zero makes no NonZero") check(Set([try! NonZero(-1), try! NonZero(-1), try! NonZero(.min)]).count == 2, "a set of NonZero") check(try! NonZero(.min) < NonZero(1), "a NonZero compares as its integer") - let debug: any CustomDebugStringConvertible = nonZeros() - check(debug.debugDescription == describeNonZeros(nonZeros()), "a record of NonZero is Debug") + let debuggable: any CustomDebugStringConvertible = nonZeros() + check(debuggable.debugDescription == describeNonZeros(nonZeros()), "a record of NonZero is Debug") targets() } diff --git a/quest/m2/README.md b/quest/m2/README.md index 93a7d7b97..3adfe64f5 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -5,8 +5,8 @@ Research work with a named consumer that is not waiting on it yet: rust2-wasm, moq's browser route, the TypeScript translator that has to beat it, [crabx](/quest/m2/crabx/README.md), which runs moq's CLIs without a Rust toolchain, and -[registry publishing](/quest/m2/registry.md), which would replace moq's binding release -workflows. +[release templates](/quest/m2/release-templates.md), which would share moq's binding +package builders while preserving its registry and platform policy. ## Plan diff --git a/quest/m2/registry.md b/quest/m2/registry.md deleted file mode 100644 index 403018ff7..000000000 --- a/quest/m2/registry.md +++ /dev/null @@ -1,39 +0,0 @@ -# [M] Spike: publish bindings to every registry from one workflow - -## Goal - -A report, from a working spike: one configless reusable workflow turns -[rust2-ffi](/ffi/README.md)'s export of the api fixture into Python wheels and an -XCFramework with a `Package.swift`, and a clean consumer job installs each and runs the -fixture's script. It says what the full version would take for the other registries -(an AAR on Maven, npm, NuGet, pub.dev, and a Go module), and whether any needs settings -`Cargo.toml` cannot supply. - -## Plan - -Decided with the maintainer on 2026-11-12, in the quest audit: ✅ a spike, in m2, with moq -as its consumer. moq carries about 13 `release-*-ffi` and `release-*-lib` -[workflows](https://github.com/moq-dev/moq/tree/main/.github/workflows) to do this by -hand, and the audit's survey found no tool that publishes one Rust library to all of -these registries. - -- Prior art, and why it falls short: [BoltFFI](https://github.com/boltffi/boltffi)'s - `pack all` is closest (XCFramework, wheels, npm, NuGet) but needs annotations, builds - local artifacts only, and emits jniLibs rather than an AAR; - [maturin](https://www.maturin.rs/distribution.html) covers PyPI only; cargo-swift, - cargo-xcframework, cargo-ndk, and napi-rs each cover one ecosystem. -- It shares [crabx](/quest/m2/crabx/README.md)'s shape: a workflow a repository opts - into, building each target in its own CI. Reuse its target matrix where the targets - match. -- Go: both packages install and run with nothing in the consumer's repository but the - workflow call, and moq's Python and Swift release workflows would become deletable. - No-go: a registry needs per-project settings, such as signing identities or a - namespace, that no manifest carries; then the report recommends templates instead. -- The spike publishes nothing: it builds the packages and installs them from the job's - artifacts. - -Public API: none; the report proposes one. - -## Related - -- [crabx](/quest/m2/crabx/README.md) - the same opt-in workflow, for binaries diff --git a/quest/m2/release-templates.md b/quest/m2/release-templates.md new file mode 100644 index 000000000..d401226f2 --- /dev/null +++ b/quest/m2/release-templates.md @@ -0,0 +1,52 @@ +# [L] Release templates with shared binding package builders + +## Goal + +Generate release workflow templates backed by shared package builders, using moq's +existing Python and Swift binding releases as the first consumer. A repository can +adopt the generated workflows without changing its stock Rust crates or existing +registry identities, artifact URLs, supported targets, or minimum platform versions. +Clean consumer jobs install the built packages and compare their behavior with +native Rust before any publication. + +## Plan + +Decided with the maintainer on 2026-11-16 at the packaging spike's merge pick: +✅ keep this follow-up unchanged (rejected: drop it). + +Decided with the maintainer on 2026-11-16 in the +[packaging spike](https://github.com/kixel-ai/rust2/pull/761#issuecomment-6930429582): +✅ release templates with shared builders (rejected: keep investigating a +configless publisher; stop the spike with no full version). Registry ownership +and hosting policy are platform boundaries Cargo metadata cannot establish. + +- Start from the [spike report](/bench/registry/README.md), then inspect moq's + `release-py-ffi`, `release-swift-ffi`, and `release-swift-lib` workflows. Preserve + their current public contracts; the fixture spike did not validate moq packaging. +- Keep registry identities and publishing authorization visible in the generated + repository-owned workflow. Derive crate facts from Cargo metadata; do not add + exporter annotations or a separate packaging configuration file. +- Share artifact construction and verification, with immutable builder revisions. + Reuse crabx's platform matrix where the targets match. Keep publication separate + from read-only build and consumer jobs, behind the repository's release policy. +- Python: build the supported wheel matrix, inspect and repair native dependencies, + preserve stable-ABI claims, and validate installation outside the source checkout. +- Swift: package the macOS/iOS device/simulator slices moq actually ships, merge + target API differences, preserve deployment targets, and generate hosted ZIP + checksums and the release Package.swift. Validate a clean SwiftPM consumer. +- Exercise the generated workflows on moq's source and compare installed packages + with native Rust. Replace moq's handwritten Python and Swift packaging logic only + after those consumers pass; this quest does not authorize a production publish. +- Document how the same builder/template boundary extends to Maven, npm, NuGet, + pub.dev, and Go, using the report's constraints. Missing binding backends are + unsupported, never a silently incomplete package. Backend implementation and + other registry publishers are outside this Python/Swift migration. + +## Hosts + +- linux - portable Python wheel construction and clean consumers +- macos - Apple slices and SwiftPM consumers + +## Related + +- [crabx](/quest/m2/crabx/README.md) - reusable build matrices for Rust binaries