From 88d8278078309a427178a5516839d37161a1476f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:44:28 -0800 Subject: [PATCH 1/9] chore(quest): claim quest/m1/ffi-packaging/go.md Quest-Claim: f6bfb397959d973e From 2e4a79e4a4794a894dfa933925d6b678f31c25f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:38:31 -0800 Subject: [PATCH 2/9] Link the Go module's static libraries by #cgo lines, and release it from the builder Co-Authored-By: Claude Opus 5.5 --- .github/workflows/bindings-test.yml | 5 + .github/workflows/bindings.yml | 113 ++++++++++++- ffi/README.md | 25 ++- ffi/justfile | 20 ++- ffi/release/README.md | 45 +++-- ffi/release/package.py | 253 +++++++++++++++++++++++++--- ffi/release/release.yml | 7 +- ffi/release/test_package.py | 76 ++++++++- ffi/src/collect.rs | 50 ++++++ ffi/src/glue.rs | 22 ++- ffi/src/go.rs | 120 +++++++++++-- ffi/src/ir.rs | 4 + ffi/src/lock.rs | 1 + ffi/src/merge.rs | 3 + ffi/tests/fixtures.rs | 79 +++++++-- ffi/tests/fixtures/shapes.rs | 12 +- 16 files changed, 736 insertions(+), 99 deletions(-) diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index 545c7f8b3..291ad38e8 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -18,6 +18,9 @@ jobs: features: doubled example: transcript python-check: ffi/tests/api.py + # A module path a proxy could serve. No go-check, as no swift-check: api.go needs the + # wasm32 items this export lacks, so the consumer only loads the runtime. + go-module: example.com/api # No swift-check: api.swift compiles against the fixture's own test export, of the host # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. @@ -28,3 +31,5 @@ jobs: manifest-path: ffi/target/moq-ffi/Cargo.toml python-check: ffi/moq/moq.py swift-check: ffi/moq/moq.swift + # Under the export's own module, `moq`, which moq.go imports. + go-check: ffi/moq/moq.go diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 677c7ea4b..c00699d06 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -1,6 +1,7 @@ name: Binding packages -# The shared, read-only builder of a crate's rust2-ffi Python wheels and Swift package, which +# The shared, read-only builder of a crate's rust2-ffi Python wheels, Swift package, and Go +# module, which # a release workflow ffi/release/package.py's `init` writes calls at a pinned rust2 commit. # It publishes nothing and holds no credentials: the caller's own jobs publish what it # uploads, once its consumers pass. See ffi/release/README.md. @@ -27,6 +28,14 @@ on: description: The main.swift of an executable run on the installed Swift package default: '' type: string + go-module: + description: The Go module's path; without it, the export's own, which no proxy serves + default: '' + type: string + go-check: + description: The main.go of a program, importing the module's packages, run against it + default: '' + type: string outputs: version: description: The package's Cargo version @@ -118,9 +127,10 @@ jobs: - host: apple os: macos-26 targets: aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim + # MSVC's library for the wheel, MinGW's for Go, which links through MinGW. - host: windows os: windows-latest - targets: x86_64-pc-windows-msvc + targets: x86_64-pc-windows-msvc x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} timeout-minutes: 60 defaults: @@ -194,7 +204,7 @@ jobs: targets: aarch64-apple-ios-sim - host: windows os: windows-latest - targets: x86_64-pc-windows-msvc + targets: x86_64-pc-windows-msvc x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} timeout-minutes: 90 defaults: @@ -248,9 +258,10 @@ jobs: - name: Export env: TARGETS: ${{ matrix.targets }} + GO_MODULE: ${{ inputs.go-module }} run: | read -ra targets <<< "$TARGETS" - python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --out export "${targets[@]}" + python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --go-module "$GO_MODULE" --out export "${targets[@]}" - name: Build env: TARGETS: ${{ matrix.targets }} @@ -262,10 +273,12 @@ jobs: env: PYTHON_CHECK: ${{ inputs.python-check }} SWIFT_CHECK: ${{ inputs.swift-check }} + GO_CHECK: ${{ inputs.go-check }} run: | mkdir -p dist/checks if [[ -n "$PYTHON_CHECK" ]]; then cp "source/$PYTHON_CHECK" dist/checks/check.py; fi if [[ -n "$SWIFT_CHECK" ]]; then cp "source/$SWIFT_CHECK" dist/checks/main.swift; fi + if [[ -n "$GO_CHECK" ]]; then cp "source/$GO_CHECK" dist/checks/main.go; fi - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} @@ -419,3 +432,95 @@ jobs: env: MODULE: ${{ needs.facts.outputs.module }} run: bash rust2/ffi/release/consume-swift.sh swift apple + + # The module as a GOPROXY serves it: the export's packages, and each Go target's library + # beside the runtime that links it. + go: + needs: [facts, build, agree] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: needs.facts.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: needs.facts.outputs.fixture == 'true' + working-directory: source + run: just ffi standin && cp ffi/moq/ffi.toml ffi/target/moq-ffi/ffi.toml + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: ${{ needs.facts.outputs.name }}-build-* + path: builds + - name: Module + env: + NAME: ${{ needs.facts.outputs.name }} + run: | + mkdir libs + cp builds/*/go/*.a libs/ + cd source + python3 -I ../rust2/ffi/release/package.py go --manifest-path "$MANIFEST" --export "../builds/$NAME-build-linux-x64/export" --libs ../libs --out ../proxy >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ needs.facts.outputs.name }}-go + path: proxy + if-no-files-found: error + + # A new module whose only requirement is the built one, unpacked from its zip, as `go get` + # unpacks it: built and run on each platform with no checkout, Rust, or CGO_LDFLAGS. + go-consumer: + needs: [facts, go] + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + os: ubuntu-24.04 + - host: linux-arm64 + os: ubuntu-24.04-arm + - host: apple-darwin + os: macos-26 + - host: windows + os: windows-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: ffi/release + persist-credentials: false + # The newest Go, whose cgo checks what a #cgo line links most strictly. + - uses: actions/setup-go@d8cbb8045d7f43706b9177cceb36c1f91a6644d5 # v7.1.0 + with: + go-version: stable + cache: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-go + path: proxy + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} + path: dist + - name: Build and run + run: | + args=() + if [[ -f dist/checks/main.go ]]; then args+=(--check dist/checks/main.go); fi + if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi + python3 -I rust2/ffi/release/package.py consume-go --proxy proxy "${args[@]}" diff --git a/ffi/README.md b/ffi/README.md index fa002c4d9..613b871ab 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -970,8 +970,19 @@ value of a Rust type crosses as one Swift type that holds it exactly, and back u `go/` holds a Go package per crate whose items the facade exports, over the C header through cgo, for Go 1.24 or later, and the runtime they share, `go/rt`. Each includes -`.h` from the `include/` beside `go/`, as the export lays them out, and links the -library through `CGO_LDFLAGS`. A crate's package is its name without the prefix and its +`.h` from `go/rt`, so `go/` is a whole module, as a release's zip holds it. The +runtime links each exported target's static library from beside itself, as +`rt/lib__.a`, by a `#cgo , LDFLAGS` line naming it and the +system libraries it needs, as rustc's `--print native-static-libs` prints them, which the +check of each target reads from its crates' `#[link]` attributes and build scripts. So a +build needs no `CGO_LDFLAGS`, and on a platform with no library, `rt/platform.go` fails to +compile. Go links Windows through MinGW, so only `x86_64-pc-windows-gnu` and +`aarch64-pc-windows-gnullvm` are Windows targets it links, and iOS the device's library, as +Go builds the simulator's from the same `ios/arm64`; Linux and macOS lines exclude Android +and iOS, which Go also counts as `linux` and `darwin`. The export puts no library there: +copy it in, as [the release builder](release/README.md) does, without the LLVM bitcode +(`.llvmbc`, `__LLVM`) that Go refuses, from 1.26.11 and 1.27.4, in an archive a `#cgo` line +names. A crate's package is its name without the prefix and its `_`: moq-mux's is `mux`, and serde_json's `serdejson`, but keeps the prefix where its strip leaves no leading letter (`moq_2d` is `moq2d`). One that a Go keyword, `C`, `main`, the runtime, or an earlier crate's package takes gains a trailing `_`, as a crate `rt` is @@ -981,10 +992,12 @@ while Rust's modules may, so a package per module could not hold moq. An instanc generic type, or a function naming one, belongs to the last of its own and its arguments' crates in Cargo's order, which can name them all: moq's `mux::catalog::Producer` is in `mux`. The packages' import path is -the module of the `go.mod` above `--out`, as `go mod edit -json` reads it, with the path to -`go/`; or else ``, the module of the `go.mod` the export writes in `go/`, which a -caller requires and replaces with that directory. The export needs `go` on PATH whenever it -reads a go.mod. +the module of the module `go/` is in, as Go finds it: that of a `go.mod` in `go/` itself, +which the export keeps as it is, as a release's names its module path, or else of the +`go.mod` above `--out`, with the path to `go/`, each as `go mod edit -json` reads it; or else +``, the module of the `go.mod` the export writes in `go/`, which a caller requires and +replaces with that directory. The export needs `go` on PATH whenever it reads a go.mod +other than its own. - **Names** are the C names without the prefix and the segments outside the item's crate, such as the facade's module re-exporting it, in Pascal case: `moq_track_Producer` is diff --git a/ffi/justfile b/ffi/justfile index 4d1712aca..f230fd179 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -96,9 +96,9 @@ bench-build checkout="": (moq-export checkout) (moq-web checkout) ts-program cod cc -O2 -std=c11 -Wall -Wextra -Werror -I "$moqc/include" -I "$hooks" moq/bench/moq-c.c "$src/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-moq-c c++ -O2 -std=c++17 -Wall -Wextra -Werror -I "$out/include" -I "$hooks" moq/bench/export.cpp "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export-cpp {{ quote(just_executable()) }} go-module bench-go moq/bench/export.go - read -ra native < target/moq-native - # The include path goes in CGO_CPPFLAGS, so cgo keeps its default -O2 -g in CGO_CFLAGS. - export CGO_CPPFLAGS="-I$hooks" CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a $hooks/libcodspeed.a ${native[*]}" GOFLAGS=-buildvcs=false + # The include path goes in CGO_CPPFLAGS, so cgo keeps its default -O2 -g in CGO_CFLAGS; + # the runtime's #cgo line links moq. + export CGO_CPPFLAGS="-I$hooks" CGO_LDFLAGS="$hooks/libcodspeed.a" GOFLAGS=-buildvcs=false (cd target/bench-go && go build -o bench .) {{ quote(just_executable()) }} dart-program bench-dart moq/bench/export.dart rm -rf target/bench-kt @@ -257,9 +257,9 @@ moq checkout="": (moq-cpp checkout) mypy --strict --python-version 3.11 --cache-dir target/mypy "$out/python/moq.py" pyright -p tests/pyright.json "$out/python/moq.py" {{ quote(just_executable()) }} moq-swift-run - # The Go, through cgo, linking the library and what it needs besides. + # The Go, through cgo, whose runtime's #cgo line links the library and what it needs besides. {{ quote(just_executable()) }} go-module moq-go moq/moq.go - export CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a ${native[*]}" GOFLAGS=-buildvcs=false + export GOFLAGS=-buildvcs=false (cd target/moq-go/export && go vet ./...) (cd target/moq-go && go vet ./... && go run .) # The Dart, through the shared library the dart feature builds. @@ -341,7 +341,7 @@ swift-target macos: #!/usr/bin/env bash if [ "{{ os() }}" = macos ]; then echo "-target $(uname -m)-apple-macos{{ macos }}"; fi -# Lay out target/ as a Go module whose main package is
, requiring the export's module of a package per crate in export/, beside the header it includes, linking the library through cgo. +# Lay out target/ as a Go module whose main package is
, requiring the export's module of a package per crate in export/, with the library its runtime's #cgo line links, less the LLVM bitcode the go command refuses there. [private] go-module dir main: #!/usr/bin/env bash @@ -349,9 +349,13 @@ go-module dir main: out=target/export/moq dir=target/{{ dir }} rm -rf "$dir" - mkdir -p "$dir/include" + mkdir -p "$dir" cp -r "$out/go" "$dir/export" - cp "$out/include/moq.h" "$dir/include/" + lib=libmoq_$(go env GOOS)_$(go env GOARCH).a + grep -qF "LDFLAGS: \${SRCDIR}/$lib" "$dir/export/rt/rt.go" || { echo "the export's Go links no $lib" >&2; exit 1; } + "$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin/llvm-objcopy" \ + --remove-section=.llvmbc --remove-section=.llvmcmd --remove-section=__LLVM,__bitcode --remove-section=__LLVM,__cmdline \ + "$out/target/release/libmoq.a" "$dir/export/rt/$lib" cp {{ main }} "$dir/main.go" printf 'module moqtest\n\ngo 1.24\n\nrequire moq v0.0.0\n\nreplace moq => ./export\n' > "$dir/go.mod" diff --git a/ffi/release/README.md b/ffi/release/README.md index de013ca36..78cd94dcb 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -1,7 +1,7 @@ # Binding releases -A crate's [rust2-ffi](/ffi/README.md) export, released as Python wheels and a Swift -package. Two halves, split where Cargo's knowledge ends: +A crate's [rust2-ffi](/ffi/README.md) export, released as Python wheels, a Swift package, +and a Go module. Two halves, split where Cargo's knowledge ends: - **The builder**, [`bindings.yml`](/.github/workflows/bindings.yml) over `package.py`, shared and read-only. It derives everything from Cargo metadata and the export: package @@ -10,14 +10,15 @@ package. Two halves, split where Cargo's knowledge ends: consumers, and publishes nothing. - **The release workflow**, the caller's own file, which `package.py init` writes once. It holds what Cargo cannot establish: who may publish to PyPI, where the XCFramework is - hosted, which repository SwiftPM resolves the package from, and which tag releases. + hosted, which repository SwiftPM resolves the package from, the Go module's path, and + which tag releases. ## Adopting it From a rust2 checkout at the commit to pin, in the repository holding the crate: ```sh -python3 /ffi/release/package.py init --manifest-path rs/moq-ffi/Cargo.toml --swift-mirror moq-dev/moq-swift-ffi +python3 /ffi/release/package.py init --manifest-path rs/moq-ffi/Cargo.toml --swift-mirror moq-dev/moq-swift-ffi --go-module moq.dev/moq-ffi ``` It writes `.github/workflows/release-.yml`, calling the builder at that commit, and @@ -26,7 +27,10 @@ refuses to overwrite a different file. Then, outside the repository: - register the workflow and the `pypi` environment as the PyPI project's trusted publisher; - create the mirror repository, install a GitHub App on it with contents write, and put its `APP_CLIENT_ID` and `APP_PRIVATE_KEY` in a `swift-mirror` environment that only release - tags may deploy to. + tags may deploy to; +- serve the build's `-go` artifact, the module laid out as a GOPROXY serves it, from a + module proxy the module path's `go-import` tag names. No job publishes it: a git mirror + holds no library over GitHub's 100 MiB, and a proxy is the caller's to run. Releasing is pushing `-v`, the crate's version, which the release checks. Called without a tag, such as from a nightly, it runs everything up to publication. @@ -37,10 +41,11 @@ and the XCFramework archive carry build times, so **Re-run all jobs** fails wher registry already holds the first attempt's files. A Cargo prerelease, such as `1.0.0-rc.1`, is a GitHub prerelease too. -The builder takes the crate's `Cargo.toml` and four optional inputs the caller's tests own: +The builder takes the crate's `Cargo.toml`, the `go-module` path, which without it is the +export's own `` that no proxy serves, and optional inputs the caller's tests own: `features` to ship, an `example` whose stdout is native Rust's transcript, and the -`python-check` and `swift-check` programs that must print it through the installed -packages. Without a check, a consumer only imports the module. +`python-check`, `swift-check`, and `go-check` programs that must print it through the +installed packages. Without a check, a consumer only imports the module. ## What the builder does @@ -67,16 +72,31 @@ packages. Without a check, a consumer only imports the module. revision, since a package resolved by version cannot depend on a revision. Rust's native link libraries become linker settings, per platform where they differ; any other flag fails the build. -6. Consumers, with no checkout, Rust, or source path: each wheel installed with +6. Go: one module for every target, at the module path and `v`, the major + version's `/vN` suffix checked: the export's `go/`, whose runtime's `#cgo` lines link + `rt/lib__.a` for `linux/amd64` and `linux/arm64` (built against + glibc 2.28), `darwin/arm64` (the Swift package's macOS library), and `windows/amd64` (an + `x86_64-pc-windows-gnu` library, for Go's MinGW linker), and the license files. Each + library must need exactly the system libraries the export's line names, as rustc prints + them building it, and loses its debug info and the LLVM bitcode Go refuses. The module's + `.info`, `.mod`, and `.zip` are laid out as a GOPROXY serves them, under Go's 500 MiB + limit on the zip and on its files, whose sizes the run's summary reports. One module + holds every platform, so a release is one version of one module. Should it near the + limit, each platform's library moves to a module of its own, whose one package's `#cgo` + line links it and which the runtime imports under that platform's build constraint, so + `go get` downloads only the platform it builds for. +7. Consumers, with no checkout, Rust, or source path: each wheel installed with `--no-index --no-deps` into a fresh environment on its oldest CPython, run with `-I` from an empty directory; and the Swift package built by its own scheme for macOS, iOS, - and the iOS Simulator, then consumed by a new executable package that runs the check. - With an `example`, each must print native Rust's stdout byte for byte. + and the iOS Simulator, then consumed by a new executable package that runs the check; + and the Go module, unpacked from its zip into a new module that requires it, vetted, + built with the newest Go and no `CGO_LDFLAGS`, and run on Linux x64 and arm64, macOS, and + Windows. With an `example`, each must print native Rust's stdout byte for byte. rust2 runs it on every manual Check and nightly through [`bindings-test.yml`](/.github/workflows/bindings-test.yml): on the api fixture, whose wheels must print its native transcript, and on moq's stand-in facade at the exporter's moq -pin, through `moq/moq.py` and `moq/moq.swift`. +pin, through `moq/moq.py`, `moq/moq.swift`, and `moq/moq.go`. ## Other registries @@ -89,7 +109,6 @@ binding. | --- | --- | --- | | Maven Central | An AAR with classes.jar, JNI libraries per Android ABI and the generated R8 rules; POM, sources and javadoc jars | The verified groupId namespace, publisher authorization, and signing key | | pub.dev | The generated Dart library with a native-assets build hook and per-platform archives, pubspec, and license | Publisher authorization and the archives' download host | -| Go | The generated cgo packages with module-relative imports, and native libraries per target | The module path, repository subdirectory, and release tags, major-version suffixes included | | npm | Needs a Node generator first: platform packages and a JS loader | Scope ownership and trusted-publisher registration | | NuGet | Needs a C# generator first: the managed assembly and RID-specific native assets | Package ID ownership, credentials, and target frameworks | diff --git a/ffi/release/package.py b/ffi/release/package.py index 9bc9b55d6..d6e0a413b 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -1,4 +1,4 @@ -"""Build and check the Python wheels and Swift package of a crate's rust2-ffi export. +"""Build and check the Python wheels, Swift package, and Go module of a crate's rust2-ffi export. bindings.yml runs each step on the host it needs; release.yml is the caller `init` writes. Every fact about the package comes from Cargo metadata or the export, so nothing here @@ -43,6 +43,21 @@ "aarch64-apple-darwin", "x86_64-pc-windows-msvc", ] +# Every target the Go module links a static library of, by its GOOS_GOARCH. Go links +# Windows through MinGW, so a -gnu library. +GO_TARGETS = { + "x86_64-unknown-linux-gnu": "linux_amd64", + "aarch64-unknown-linux-gnu": "linux_arm64", + "aarch64-apple-darwin": "darwin_arm64", + "x86_64-pc-windows-gnu": "windows_amd64", +} +# What the go command refuses to extract: a module zip, or its files, over 500 MiB, and a +# go.mod over 16 MiB (golang.org/x/mod/zip's MaxZipFile and MaxGoMod). +GO_ZIP_MAX = 500 << 20 +GO_MOD_MAX = 16 << 20 +# The sections the go command refuses in an archive a #cgo directive names: LLVM's bitcode, +# which Rust's prebuilt compiler_builtins carries. +LTO_SECTIONS = [".llvmbc", ".llvmcmd", "__LLVM,__bitcode", "__LLVM,__cmdline"] # Cargo metadata names a library target by its crate types; a proc-macro is no library to export. LIBRARY_KINDS = {"lib", "rlib", "dylib", "cdylib", "staticlib"} COLLECTIONS = "https://github.com/apple/swift-collections" @@ -255,7 +270,8 @@ def digest(export, lib): """A hash of the generated bindings, which every build host's export must agree on.""" hasher = hashlib.sha256() export = Path(export) - paths = [export / "python" / f"{lib}.py", export / "include" / f"{lib}.h", *sorted((export / "swift").iterdir())] + paths = [export / "python" / f"{lib}.py", export / "include" / f"{lib}.h", *sorted((export / "swift").iterdir()), + *sorted(p for p in (export / "go").rglob("*") if p.is_file())] for path in paths: hasher.update(path.relative_to(export).as_posix().encode() + b"\0") # Exact bytes: each wheel ships its own host's module. @@ -470,7 +486,12 @@ def export(args): raise SystemExit(f"{args.api} lacks a staged API for one of {args.targets}") targets = [arg for target in args.targets for arg in ("--target", target)] withs = [arg for path in others for arg in ("--with", path)] - exporter(args.manifest_path, args.features, *targets, *withs, "--out", Path(args.out).resolve()) + out = Path(args.out).resolve() + if args.go_module: + # The export keeps a go.mod in go/, whose module its packages import each other by. + (out / "go").mkdir(parents=True, exist_ok=True) + (out / "go" / "go.mod").write_text(f"module {args.go_module}\n\ngo 1.24\n", newline="\n", encoding="utf-8") + exporter(args.manifest_path, args.features, *targets, *withs, "--out", out) def cargo(manifest, *args, env=None): @@ -479,14 +500,66 @@ def cargo(manifest, *args, env=None): capture_output=True, encoding="utf-8") +def staticlib(manifest, crate, lib, target, env=None): + """The C library alone, apart from the Python extension, whose CPython symbols it would + leave undefined, and the system libraries rustc says it needs beside it.""" + # Its own target directory, since cargo does not rebuild for a changed deployment target. + target_dir = crate.parent / ("target-swift" if target in APPLE_TARGETS else "target-go") + # cargo-zigbuild links Linux against the glibc floor's symbols, whichever glibc the host has. + linux = "linux" in target + # Uncolored, so the note below reads as written whatever the caller's terminal. + built = run(*(["cargo-zigbuild"] if linux else ["cargo"]), "rustc", "--color", "never", "--locked", "--release", + "--manifest-path", crate, "--target", f"{target}.{GLIBC}" if linux else target, + "--target-dir", target_dir, "--lib", "--crate-type", "staticlib", "--", "--print=native-static-libs", + cwd=Path(manifest).resolve().parent, env={**os.environ, **(env or {})}, + capture_output=True, encoding="utf-8") + sys.stderr.write(built.stderr) + notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) + if len(notes) != 1: + raise SystemExit(f"rustc reported no native libraries for {target}") + return target_dir / target / "release" / f"lib{lib}.a", notes[0].strip() + + +def go_links(export, lib, target): + """The system libraries the export's Go links beside `target`'s library, from its #cgo line.""" + platform = GO_TARGETS[target] + rt = (Path(export) / "go" / "rt" / "rt.go").read_text(encoding="utf-8") + match = re.search(rf"^#cgo \S+ LDFLAGS: \$\{{SRCDIR\}}/lib{lib}_{platform}\.a(.*)$", rt, re.M) + if match is None: + raise SystemExit(f"the export's Go links no library for {target}") + return match.group(1).strip() + + +def llvm_tool(manifest, name): + """One of LLVM's tools from the toolchain building the crate, which reads every target's objects.""" + cwd = Path(manifest).resolve().parent + run("rustup", "component", "add", "llvm-tools", cwd=cwd, capture_output=True) + sysroot = output("rustc", "--print", "sysroot", cwd=cwd).strip() + host = re.search(r"^host: (\S+)$", output("rustc", "-vV", cwd=cwd), re.M).group(1) + exe = ".exe" if sys.platform == "win32" else "" + return Path(sysroot) / "lib" / "rustlib" / host / "bin" / f"{name}{exe}" + + +def go_library(args, export_dir, lib, target, library, native, out): + """The Go module's copy of `target`'s library: checked against the export's #cgo line, + without debug info, which only bloats the module, or the bitcode the go command refuses.""" + want = go_links(export_dir, lib, target) + if native != want: + raise SystemExit(f"rustc links {target}'s library with `{native}`, but the export's Go with `{want}`") + objcopy = llvm_tool(args.manifest_path, "llvm-objcopy") + dest = out / "go" / f"lib{lib}_{GO_TARGETS[target]}.a" + dest.parent.mkdir(parents=True, exist_ok=True) + run(objcopy, "--strip-debug", *(f"--remove-section={s}" for s in LTO_SECTIONS), library, dest) + + def build(args): - """This host's wheels and Apple static libraries, the export's sources, and native output.""" + """This host's wheels, Apple and Go static libraries, the export's sources, and native output.""" info = facts(args.manifest_path) lib, export_dir, out = info["lib"], Path(args.export).resolve(), Path(args.out).resolve() crate = export_dir / "Cargo.toml" out.mkdir(parents=True, exist_ok=True) for target in args.targets: - if target not in WHEEL_TARGETS and target not in APPLE_TARGETS: + if target not in WHEEL_TARGETS and target not in APPLE_TARGETS and target not in GO_TARGETS: raise SystemExit(f"no package ships {target}") run("rustup", "target", "add", target, cwd=Path(args.manifest_path).resolve().parent) if target in WHEEL_TARGETS: @@ -503,30 +576,25 @@ def build(args): if target in APPLE_TARGETS: apple = APPLE_TARGETS[target] name = "MACOSX_DEPLOYMENT_TARGET" if apple == "macos" else "IPHONEOS_DEPLOYMENT_TARGET" - # The C library alone, apart from the Python extension, whose CPython symbols it - # would leave undefined; its own target directory, since cargo does not rebuild - # for a changed deployment target. - # Uncolored, so the note below reads as written whatever the caller's terminal. - built = cargo(args.manifest_path, "rustc", "--color", "never", "--locked", "--release", "--manifest-path", crate, - "--target", target, "--target-dir", export_dir / "target-swift", "--lib", - "--crate-type", "staticlib", "--", "--print=native-static-libs", - env={name: SWIFT_PLATFORMS[apple]}) - sys.stderr.write(built.stderr) - notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) - if len(notes) != 1: - raise SystemExit(f"rustc reported no native libraries for {target}") + library, native = staticlib(args.manifest_path, crate, lib, target, env={name: SWIFT_PLATFORMS[apple]}) static = out / "static" / target static.mkdir(parents=True) - shutil.copy(export_dir / "target-swift" / target / "release" / f"lib{lib}.a", static) - (static / "native.txt").write_text(notes[0].strip() + "\n", encoding="utf-8") - # The generated sources, for the Swift package and the hosts' agreement. + shutil.copy(library, static) + (static / "native.txt").write_text(native + "\n", encoding="utf-8") + # The Go module links the Swift package's macOS library, at its floor. + if target in GO_TARGETS: + go_library(args, export_dir, lib, target, library, native, out) + elif target in GO_TARGETS: + library, native = staticlib(args.manifest_path, crate, lib, target) + go_library(args, export_dir, lib, target, library, native, out) + # The generated sources, for the Swift package, the Go module, and the hosts' agreement. # Text this writes is LF on every host, as the jobs reading it compare bytes. - for part in ("include", "swift", "python"): + for part in ("include", "swift", "python", "go"): shutil.copytree(export_dir / part, out / "export" / part, dirs_exist_ok=True) shutil.copy(crate, out / "export") (out / "digest.txt").write_text(digest(export_dir, lib) + "\n", newline="\n", encoding="utf-8") (out / "lib.txt").write_text(lib + "\n", newline="\n", encoding="utf-8") - # Native output for a host whose wheel a consumer runs; an iOS slice runs nowhere here. + # Native output for a host whose wheel or Go module a consumer runs; an iOS slice runs nowhere here. if args.example and any(target in WHEEL_TARGETS for target in args.targets): if args.example not in info["examples"]: raise SystemExit(f"{info['name']} has no example {args.example}: it has {info['examples']}") @@ -538,13 +606,134 @@ def build(args): (out / "native.txt").write_bytes(native.stdout) -def template(info, manifest, rust2, mirror): +def go_version(version): + """Cargo's version as a Go module's: semver with a `v`, refusing build metadata, which Go has no spelling of.""" + if SEMVER.fullmatch(version) is None: + raise SystemExit(f"version {version} is no Go module version: use X.Y.Z or X.Y.Z-(alpha|beta|rc).N") + return f"v{version}" + + +def go_module_path(export): + """The module the export's go.mod declares.""" + text = (Path(export) / "go" / "go.mod").read_text(encoding="utf-8") + match = re.search(r"^module\s+(\S+)\s*$", text, re.M) + if match is None: + raise SystemExit("the export's go.mod declares no module") + return match.group(1) + + +def check_major(module, version): + """Fail unless the module path carries the suffix Go requires of its major version.""" + major = int(version[1:].split(".")[0]) + suffix = re.search(r"/v([0-9]+)$", module) + if major >= 2 and (suffix is None or int(suffix.group(1)) != major): + raise SystemExit(f"Go requires version {version} of {module} to be at a module path ending /v{major}") + if major < 2 and suffix is not None: + raise SystemExit(f"Go refuses version {version} of {module}, whose path names major version {suffix.group(1)}") + + +def escape(module): + """A module path as a proxy's URL spells it, each capital letter as `!` and its lower case.""" + return re.sub(r"[A-Z]", lambda m: "!" + m.group(0).lower(), module) + + +def go_files(info, export, libs): + """The module's files by path: the export's Go, every Go target's library from the + directory `libs` beside the runtime, and the license files, at the root where + pkg.go.dev looks for them.""" + root = Path(export) / "go" + files = {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()} + want = {f"lib{info['lib']}_{platform}.a" for platform in GO_TARGETS.values()} + found = {p.name for p in Path(libs).iterdir()} + if found != want: + raise SystemExit(f"the Go module links {sorted(want)}, given {sorted(found)}") + for name in sorted(want): + files[f"rt/{name}"] = (Path(libs) / name).read_bytes() + for path in licenses(info): + files[path.name] = path.read_bytes() + # Two files a case-insensitive file system would make one, which the go command refuses. + folded = {} + for name in files: + if folded.setdefault(name.lower(), name) != name: + raise SystemExit(f"the Go module holds both {folded[name.lower()]} and {name}") + return files + + +def go_module(info, export, libs, out): + """The module's .info, .mod, and .zip, laid out as a GOPROXY serves them, under `out`.""" + module, version = go_module_path(export), go_version(info["version"]) + check_major(module, version) + files = go_files(info, export, libs) + if len(files["go.mod"]) > GO_MOD_MAX: + raise SystemExit(f"go.mod is {len(files['go.mod'])} bytes, over Go's {GO_MOD_MAX}") + size = sum(len(data) for data in files.values()) + if size > GO_ZIP_MAX: + raise SystemExit(f"the Go module's files are {size >> 20} MiB, over Go's {GO_ZIP_MAX >> 20} MiB") + versions = Path(out) / escape(module) / "@v" + versions.mkdir(parents=True) + (versions / f"{version}.info").write_text(json.dumps({"Version": version}) + "\n", encoding="utf-8") + (versions / f"{version}.mod").write_bytes(files["go.mod"]) + archive = versions / f"{version}.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z: + for name, data in files.items(): + # A fixed time, so the zip is a function of its files. + entry = zipfile.ZipInfo(f"{module}@{version}/{name}", date_time=(1980, 1, 1, 0, 0, 0)) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + z.writestr(entry, data) + if archive.stat().st_size > GO_ZIP_MAX: + raise SystemExit(f"the Go module's zip is {archive.stat().st_size >> 20} MiB, over Go's {GO_ZIP_MAX >> 20} MiB") + rows = [f"| `{name}` | {len(data) / (1 << 20):.1f} MiB |" for name, data in files.items() if name.endswith(".a")] + return "\n".join([ + f"Go module `{module}` {version}: {size / (1 << 20):.1f} MiB of files, a " + f"{archive.stat().st_size / (1 << 20):.1f} MiB zip, of Go's {GO_ZIP_MAX >> 20} MiB limit on each.", + "", "| Library | Size |", "| --- | --- |", *rows, "", + ]) + + +def consume_go(proxy, check, native): + """Build and run a check program, or else an import of the runtime, against the module + unpacked from its zip, in a new module of its own: no checkout, Rust, or CGO_LDFLAGS.""" + zips = list(Path(proxy).rglob("*.zip")) + if len(zips) != 1: + raise SystemExit(f"expected one module zip in {proxy}, found {zips}") + with tempfile.TemporaryDirectory() as work, zipfile.ZipFile(zips[0]) as z: + work = Path(work) + prefix = z.namelist()[0].split("/", 1)[0] + module, version = prefix.rsplit("@", 1) + z.extractall(work / "mod") + consumer = work / "consumer" + consumer.mkdir() + (consumer / "go.mod").write_text( + f"module consumer\n\ngo 1.24\n\nrequire {module} {version}\n\nreplace {module} => ../mod/{prefix}\n", + encoding="utf-8") + main = Path(check).read_text(encoding="utf-8") if check else f'package main\n\nimport _ "{module}/rt"\n\nfunc main() {{}}\n' + (consumer / "main.go").write_text(main, encoding="utf-8") + env = {k: v for k, v in os.environ.items() if not k.startswith("CGO_") or k == "CGO_ENABLED"} + env.update(CGO_ENABLED="1", GOFLAGS="-mod=mod", GOPROXY="off", GOTOOLCHAIN="local", GOWORK="off") + run("go", "vet", f"{module}/...", ".", cwd=consumer, env=env) + exe = consumer / ("check.exe" if sys.platform == "win32" else "check") + run("go", "build", "-o", exe, ".", cwd=consumer, env=env) + result = subprocess.run([exe], capture_output=True, cwd=work) + sys.stderr.buffer.write(result.stderr) + if result.returncode: + raise SystemExit(f"{check or 'the import'} failed on the Go module with status {result.returncode}") + if native and check: + compare(Path(native).read_bytes(), result.stdout, "Go module") + + +def template(info, manifest, rust2, mirror, go_module): """The caller workflow, with every identity Cargo cannot establish spelled out in it.""" if not re.fullmatch(r"[0-9a-f]{40}", rust2): raise SystemExit("pin rust2 by a full commit SHA") if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", mirror): raise SystemExit(f"the Swift mirror must be OWNER/REPO, not {mirror!r}") + # A proxy serves only a path whose first element is a host name. + if not re.fullmatch(r"[a-z0-9-]+(\.[a-z0-9-]+)+(/[A-Za-z0-9._~-]+)*", go_module): + raise SystemExit(f"the Go module must be a path a proxy serves, such as example.com/name, not {go_module!r}") + check_major(go_module, go_version(info["version"])) values = { + "GO_MODULE": go_module, "NAME": info["name"], "MANIFEST": manifest, "RUST2": rust2, @@ -570,7 +759,7 @@ def init(args): rust2 = args.rust2 or output("git", "-C", HERE, "rev-parse", "HEAD").strip() relative = manifest.resolve().relative_to(root).as_posix() path = root / ".github" / "workflows" / f"release-{info['name']}.yml" - text = template(info, relative, rust2, args.swift_mirror) + text = template(info, relative, rust2, args.swift_mirror, args.go_module) if path.exists() and path.read_text(encoding="utf-8") != text: raise SystemExit(f"{path} exists; delete it to write the release workflow") path.parent.mkdir(parents=True, exist_ok=True) @@ -593,6 +782,7 @@ def main(): p.add_argument("targets", nargs="+") if name == "export": p.add_argument("--api", required=True) + p.add_argument("--go-module", default="", help="the Go module path; the export's own, the prefix, by default") if name == "build": p.add_argument("--export", required=True) p.add_argument("--example", default="") @@ -610,6 +800,15 @@ def main(): p.add_argument("--state", required=True, help="the swift.json `swift` wrote") p.add_argument("--package", required=True) p.add_argument("--framework", required=True) + p = sub.add_parser("go", help="write the Go module as a GOPROXY serves it") + p.add_argument("--manifest-path", required=True) + p.add_argument("--export", required=True) + p.add_argument("--libs", required=True, help="a directory of every Go target's library, as build names them") + p.add_argument("--out", required=True) + p = sub.add_parser("consume-go", help="run a check program, or an import, on the unpacked Go module") + p.add_argument("--proxy", required=True, help="the directory `go` wrote") + p.add_argument("--check") + p.add_argument("--native") p = sub.add_parser("consume-python", help="run a check program, or an import, on the installed wheel") group = p.add_mutually_exclusive_group(required=True) group.add_argument("--check") @@ -622,6 +821,8 @@ def main(): p.add_argument("--manifest-path", required=True) p.add_argument("--swift-mirror", required=True, metavar="OWNER/REPO", help="the repository SwiftPM resolves the package from, whose root holds its Package.swift") + p.add_argument("--go-module", required=True, metavar="PATH", + help="the Go module's path, which a proxy you run serves, /vN suffix included") p.add_argument("--rust2", help="the rust2 commit to pin; this checkout's by default") args = parser.parse_args() @@ -646,6 +847,10 @@ def main(): release_manifest(args.out, args.url) elif args.command == "swift-local": local_manifest(args.state, args.package, args.framework) + elif args.command == "go": + print(go_module(facts(args.manifest_path), args.export, args.libs, args.out)) + elif args.command == "consume-go": + consume_go(args.proxy, args.check, args.native) elif args.command == "consume-python": consume_python(args.check, args.native, args.module) elif args.command == "compare": diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 2c8c23bbd..bbc013827 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -11,6 +11,10 @@ name: Release @NAME@ bindings # The mirror's token comes from a GitHub App installed on it, whose ID and key live in # the `swift-mirror` environment. # +# The Go module @GO_MODULE@ is the build's @NAME@-go artifact, its .info, .mod, and .zip laid +# out as a GOPROXY serves them; no job here publishes it. Serve it from a module proxy that +# @GO_MODULE@'s go-import tag names: a git mirror holds no library over GitHub's 100 MiB. +# # To release, push the tag @NAME@-v, naming the crate's version. Without a tag, as # from a nightly's workflow_call, every job up to publication runs. If publishing fails, use # "Re-run failed jobs": it reuses this run's build, and the publishing jobs upload only what @@ -34,8 +38,9 @@ jobs: uses: kixel-ai/rust2/.github/workflows/bindings.yml@@RUST2@ with: manifest-path: @MANIFEST@ + go-module: @GO_MODULE@ # Optional: cargo features to ship, an example of the crate whose output every check - # program must print, and those programs: `python-check` and `swift-check`. + # program must print, and those programs: `python-check`, `swift-check`, and `go-check`. tag: needs: build diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 95af04a4c..b036fe780 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -1,6 +1,7 @@ """The builder's decisions that need no toolchain: names, versions, metadata, and manifests.""" import importlib.util +import json from pathlib import Path import tempfile import unittest @@ -31,9 +32,21 @@ def export(root, floor="311"): (root / "include/moq.h").write_text("int moq_contract(void);\n") (root / "swift/moq.swift").write_text("import MoqFFI\n") (root / "Cargo.toml").write_text(f'pyo3-ffi = {{ features = ["abi3-py{floor}", "extension-module"] }}\n') + (root / "go/rt").mkdir(parents=True) + (root / "go/go.mod").write_text("module moq.dev/moq-ffi\n\ngo 1.24\n") + (root / "go/rt/rt.go").write_text( + "package rt\n\n/*\n#cgo linux,!android,amd64 LDFLAGS: ${SRCDIR}/libmoq_linux_amd64.a -lgcc_s -lc\n*/\nimport \"C\"\n") return root +def go_libs(root): + """A directory of every Go target's library, as `build` names them.""" + (root / "libs").mkdir() + for platform in package.GO_TARGETS.values(): + (root / "libs" / f"libmoq_{platform}.a").write_bytes(b"!\n") + return root / "libs" + + class Versions(unittest.TestCase): def test_release_and_prereleases(self): self.assertEqual(package.pep440("0.4.12"), "0.4.12") @@ -142,6 +155,59 @@ def test_any_byte_splits_the_hosts(self): # A Windows checkout once turned the embedded runtime CRLF. (second / "python/moq.py").write_bytes(b"PENDING = 0\r\n") self.assertNotEqual(package.digest(first, "moq"), package.digest(second, "moq")) + (second / "python/moq.py").write_bytes(b"PENDING = 0\n") + (second / "go/rt/rt.go").write_text("package rt\n") + self.assertNotEqual(package.digest(first, "moq"), package.digest(second, "moq")) + + +class Go(unittest.TestCase): + def test_versions_and_major_suffixes(self): + self.assertEqual(package.go_version("1.0.0-rc.1"), "v1.0.0-rc.1") + with self.assertRaises(SystemExit): + package.go_version("1.0.0+build") + package.check_major("moq.dev/moq-ffi", "v1.2.3") + package.check_major("moq.dev/moq-ffi/v2", "v2.0.0") + for module, version in [("moq.dev/moq-ffi", "v2.0.0"), ("moq.dev/moq-ffi/v3", "v2.0.0"), ("moq.dev/x/v2", "v1.0.0")]: + with self.assertRaises(SystemExit): + package.check_major(module, version) + + def test_escapes_capitals(self): + self.assertEqual(package.escape("github.com/Azure/x"), "github.com/!azure/x") + + def test_links_from_the_cgo_line(self): + with tempfile.TemporaryDirectory() as root: + root = export(Path(root)) + self.assertEqual(package.go_links(root, "moq", "x86_64-unknown-linux-gnu"), "-lgcc_s -lc") + with self.assertRaises(SystemExit): + package.go_links(root, "moq", "aarch64-unknown-linux-gnu") + + def test_module_as_a_proxy_serves_it(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root) + built = export(root / "export") + (root / "LICENSE-MIT").write_text("MIT\n") + summary = package.go_module(info(root), built, go_libs(root), root / "proxy") + self.assertIn("`rt/libmoq_windows_amd64.a`", summary) + versions = root / "proxy/moq.dev/moq-ffi/@v" + self.assertEqual((versions / "v0.4.12.mod").read_text(), "module moq.dev/moq-ffi\n\ngo 1.24\n") + self.assertEqual(json.loads((versions / "v0.4.12.info").read_text()), {"Version": "v0.4.12"}) + with zipfile.ZipFile(versions / "v0.4.12.zip") as z: + self.assertEqual(sorted(z.namelist()), [f"moq.dev/moq-ffi@v0.4.12/{n}" for n in [ + "LICENSE-MIT", "go.mod", "rt/libmoq_darwin_arm64.a", "rt/libmoq_linux_amd64.a", + "rt/libmoq_linux_arm64.a", "rt/libmoq_windows_amd64.a", "rt/rt.go"]]) + # The module ships every Go target's library, and nothing else. + (root / "libs/libmoq_linux_arm64.a").unlink() + with self.assertRaises(SystemExit): + package.go_module(info(root), built, root / "libs", root / "other") + + def test_refuses_names_a_case_folds_together(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root) + built = export(root / "export") + (built / "go/RT.go").write_text("package rt\n") + (built / "go/rt.go").write_text("package rt\n") + with self.assertRaises(SystemExit): + package.go_files(info(root), built, go_libs(root)) class Swift(unittest.TestCase): @@ -168,19 +234,23 @@ def test_manifest(self): class Template(unittest.TestCase): def test_fills_every_identity(self): - text = package.template(info("/src"), "rs/moq-ffi/Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi") + text = package.template(info("/src"), "rs/moq-ffi/Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi", "moq.dev/moq-ffi") self.assertIn("uses: kixel-ai/rust2/.github/workflows/bindings.yml@" + "a" * 40, text) self.assertIn('- "moq-ffi-v*"', text) self.assertIn("url: https://pypi.org/p/moq-ffi", text) self.assertIn("owner: moq-dev\n", text) self.assertIn("repositories: moq-swift-ffi\n", text) + self.assertIn("go-module: moq.dev/moq-ffi\n", text) self.assertNotRegex(text, r"@[A-Z_]+@") def test_refuses_a_movable_pin_or_bad_mirror(self): with self.assertRaises(SystemExit): - package.template(info("/src"), "Cargo.toml", "main", "moq-dev/moq-swift-ffi") + package.template(info("/src"), "Cargo.toml", "main", "moq-dev/moq-swift-ffi", "moq.dev/moq-ffi") with self.assertRaises(SystemExit): - package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-swift-ffi") + package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-swift-ffi", "moq.dev/moq-ffi") + for module in ["moq", "moq.dev/moq-ffi/", "moq.dev/moq-ffi/v2"]: + with self.assertRaises(SystemExit): + package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi", module) if __name__ == "__main__": diff --git a/ffi/src/collect.rs b/ffi/src/collect.rs index b6713dc75..dd33beb6b 100644 --- a/ffi/src/collect.rs +++ b/ffi/src/collect.rs @@ -472,6 +472,7 @@ pub(crate) fn collect<'tcx>(tcx: TyCtxt<'tcx>, facade: &Facade<'_>) -> Collected crates: Vec::new(), reached: Vec::new(), targets: vec![target(tcx)], + links: [(target(tcx), links(tcx))].into(), }, }; c.walk(); @@ -6523,6 +6524,55 @@ fn target(tcx: TyCtxt<'_>) -> String { tcx.sess.opts.target_triple.tuple().to_string() } +/// The system libraries a static library of the facade links beside it on this target, as +/// rustc's `--print native-static-libs` prints them for one: what each linked crate's +/// `#[link]` and build script name that the archive does not bundle, the leaves last. +/// rustc lists only at a link, so this follows its `link_staticlib` over the checked crates. +fn links(tcx: TyCtxt<'_>) -> Vec { + use rustc_attr_parsing::eval_config_entry; + use rustc_structures::NativeLibKind; + let sess = tcx.sess; + // A dependent before what it depends on, `compiler_builtins` last; the facade, which the + // glue's build links as any other crate, depends on the rest. + let mut crates: Vec = (tcx.postorder_cnums(()).iter().rev().copied()) + .filter(|&krate| !tcx.crate_dep_kind(krate).macros_only()) + .collect(); + crates.sort_by_key(|&krate| tcx.is_compiler_builtins(krate)); + crates.insert(0, LOCAL_CRATE); + let mut out: Vec = Vec::new(); + let libs = crates.into_iter().flat_map(|krate| tcx.native_libraries(krate)); + for lib in libs.filter(|lib| (lib.cfg.as_ref()).is_none_or(|cfg| eval_config_entry(sess, cfg).as_bool())) { + let name = lib.name; + let verbatim = lib.verbatim.unwrap_or(false); + let arg = match lib.kind { + NativeLibKind::Static { + bundle: Some(false), .. + } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Unspecified => { + if sess.target.is_like_msvc { + let (prefix, suffix) = sess.staticlib_components(verbatim); + format!("{prefix}{name}{suffix}") + } else if sess.target.linker_flavor.is_gnu() { + format!("-l{}{name}", if verbatim { ":" } else { "" }) + } else { + format!("-l{name}") + } + } + NativeLibKind::Framework { .. } => format!("-framework {name}"), + NativeLibKind::Static { .. } + | NativeLibKind::LinkArg + | NativeLibKind::WasmImportModule + | NativeLibKind::RawDylib { .. } => continue, + }; + // rustc drops a library repeated right after itself, and only then. + if out.last() != Some(&arg) { + out.push(arg); + } + } + out +} + /// Whether every byte of `ty` is initialized, whatever its value and whatever layout a /// `rustc` picks at its size on this target: the bytes its fields always initialize fill /// it, so no field order leaves padding. The glue asserts the size, which is what lets diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 11896a261..c61995874 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -460,6 +460,8 @@ impl Crate<'_> { for (path, text) in go.files { output.file(&go_dir.join(path), &text); } + // The header the packages include, inside their module, as a module's zip holds it. + output.file(&go_dir.join("rt").join(format!("{}.h", self.abi.prefix)), &header); // Cargo names a path package's units by its name, version, and path in its // workspace, the same for every export of one facade, and judges them fresh by // mtimes, which an unchanged file keeps. So two exports in one target directory @@ -503,7 +505,7 @@ fn module_path(path: &Path) -> Result { .output() .map_err(|err| match err.kind() { std::io::ErrorKind::NotFound => format!( - "the go command is not on PATH: the export reads the module of {}, above --out, with `go mod edit -json`", + "the go command is not on PATH: the export reads the module of {} with `go mod edit -json`", path.display() ), _ => format!("running go mod edit -json {}: {err}", path.display()), @@ -544,10 +546,22 @@ fn resolve(path: &Path) -> Result { Ok(out) } -/// The import path of `/go`, from the `go.mod` of the module above it, or else the -/// module `prefix`, with the `go.mod` declaring it there. +/// The import path of `/go`, from the `go.mod` of the module it is in, as Go finds it: +/// its own, which the export keeps as it is, or one above it; or else the module `prefix`, +/// with the `go.mod` declaring it there. The `go.mod` to write in `go/` comes with it. fn go_module(out: &Path, prefix: &str) -> Result<(String, Option), String> { let out = resolve(out)?; + let default = format!("module {prefix}\n\ngo 1.24\n"); + let own = out.join("go").join("go.mod"); + if own.is_file() { + let text = std::fs::read_to_string(&own).map_err(|err| format!("reading {}: {err}", own.display()))?; + // The export's own needs no go command to read. + let module = match text == default { + true => prefix.to_string(), + false => module_path(&own)?, + }; + return Ok((module, Some(text))); + } for dir in out.ancestors() { let path = dir.join("go.mod"); if !path.is_file() { @@ -562,7 +576,7 @@ fn go_module(out: &Path, prefix: &str) -> Result<(String, Option), Strin .collect(); return Ok((format!("{module}/{}", rest.join("/")), None)); } - Ok((prefix.to_string(), Some(format!("module {prefix}\n\ngo 1.24\n")))) + Ok((prefix.to_string(), Some(default))) } struct Gen<'a> { diff --git a/ffi/src/go.rs b/ffi/src/go.rs index 7e3f68951..46bff3d93 100644 --- a/ffi/src/go.rs +++ b/ffi/src/go.rs @@ -1,12 +1,14 @@ //! Go over the C ABI through cgo: a package per crate whose items the facade exports, -//! `go//`, and the runtime they share, `go/rt/`, each including the C header from -//! `include/` beside `go/`. A crate's package is its name without the prefix and its `_` +//! `go//`, and the runtime they share, `go/rt/`, each including the C header the +//! runtime holds, so `go/` is a module of its own. The runtime links each exported target's +//! static library, `rt/lib__.a`, with the system libraries it needs, by +//! a `#cgo` line of that platform, and fails to compile on any other. A crate's package is its name without the prefix and its `_` //! (moq-mux's is `mux`), so callers write `mux.CodecOpusConfig_`. Cargo keeps crates from //! depending on each other in a cycle, so their packages import none in one either, as Go //! requires; Rust's modules may, and so cannot each be a package. An instance of a generic //! type, or a function naming one, belongs to the last of its own and its arguments' -//! crates, which can name them all. The import path of `go/` is the module of the `go.mod` -//! above it, or else the module `` of the `go.mod` the export writes there. +//! crates, which can name them all. The import path of `go/` is the module of its own +//! `go.mod` or one above it, or else the module `` of the `go.mod` the export writes there. //! //! Each name is its C name without the prefix and the segments outside its crate, in //! Pascal case, kept [`apart`](abi::apart) from another type's of the package that would @@ -186,7 +188,7 @@ pub(crate) fn generate(abi: &Abi<'_>, packages: &str, contract: u64, module: &st locked.insert(format!("crate {krate}"), name.to_string()); let doc = format!( "// Package {name} is the pub API of the crate {krate} in {packages}, over its C ABI\n\ - // through cgo. Link the library built with {p}.h, through CGO_LDFLAGS.\n", + // through cgo, whose runtime links the static library rt/lib{p}__.a.\n", p = abi.prefix ); files.push((format!("{name}/{name}.go"), g.file(&doc))); @@ -206,11 +208,63 @@ pub(crate) fn generate(abi: &Abi<'_>, packages: &str, contract: u64, module: &st params.join(", ") ); } - files.push(("rt/rt.go".to_string(), rt.runtime(packages, contract))); + let (links, platform) = link(abi)?; + files.push(("rt/rt.go".to_string(), rt.runtime(packages, contract, &links))); + files.push(("rt/platform.go".to_string(), platform)); files.push(("rt/types.go".to_string(), rt.file(""))); Ok(Output { files, names: locked }) } +/// Each target whose static library Go links: its `GOOS`, `GOARCH`, and the `GOOS` that +/// also satisfies its own as a build constraint, which is not it. Windows' is a `-gnu` +/// library, as Go links through MinGW; iOS's the device's, as Go tells no simulator apart. +const PLATFORMS: &[(&str, &str, &str, Option<&str>)] = &[ + ("x86_64-unknown-linux-gnu", "linux", "amd64", Some("android")), + ("aarch64-unknown-linux-gnu", "linux", "arm64", Some("android")), + ("x86_64-apple-darwin", "darwin", "amd64", Some("ios")), + ("aarch64-apple-darwin", "darwin", "arm64", Some("ios")), + ("aarch64-apple-ios", "ios", "arm64", None), + ("x86_64-linux-android", "android", "amd64", None), + ("aarch64-linux-android", "android", "arm64", None), + ("x86_64-pc-windows-gnu", "windows", "amd64", None), + ("aarch64-pc-windows-gnullvm", "windows", "arm64", None), +]; + +/// The `#cgo LDFLAGS` lines linking each exported target's library, `rt/lib__.a`, +/// and the system libraries it needs, and `rt/platform.go`, which fails to compile on any +/// other platform, so no build links without a library. +fn link(abi: &Abi<'_>) -> Result<(String, String), String> { + let p = &abi.prefix; + let mut lines = String::new(); + let mut constraints = Vec::new(); + for &(target, goos, goarch, not) in PLATFORMS { + if !abi.api.targets.iter().any(|t| t == target) { + continue; + } + let Some(libs) = abi.api.links.get(target) else { + return Err(format!( + "the staged API of {target} names no system libraries: stage it again with this rust2-ffi" + )); + }; + let not = not.map(|os| format!(",!{os}")).unwrap_or_default(); + let mut flags = vec![format!("${{SRCDIR}}/lib{p}_{goos}_{goarch}.a")]; + flags.extend(libs.iter().cloned()); + let _ = writeln!(lines, "#cgo {goos}{not},{goarch} LDFLAGS: {}", flags.join(" ")); + let not = not.replace(",!", " && !"); + constraints.push(format!("{goos}{not} && {goarch}")); + } + let only = match constraints.is_empty() { + true => String::new(), + false => format!("//go:build !({})\n\n", constraints.join(" || ")), + }; + let platform = format!( + "// Code generated by rust2-ffi. DO NOT EDIT.\n\n{only}package rt\n\n\ + // The export has no library for this platform, so nothing it declares would link.\n\ + var _ int = \"rust2-ffi exported no library of {p} for this GOOS and GOARCH\"\n" + ); + Ok((lines, platform)) +} + /// A call holding `object`'s handle as `h`, in the slot `s`, until it returns: through /// `Count`, inlined, then `Contend` once calls contend for it, running `failed` once closed. /// `slot` is the runtime's `Slot_`, as the package names it. @@ -546,7 +600,7 @@ impl<'a> Gen<'a> { let p = &self.abi.prefix; let _ = write!( out, - "/*\n#cgo CFLAGS: -I${{SRCDIR}}/../../include\n#include \n#include \"{p}.h\"\n\n" + "/*\n#cgo CFLAGS: -I${{SRCDIR}}/../rt\n#include \n#include \"{p}.h\"\n\n" ); for decl in &self.decls { out.push_str(decl); @@ -592,7 +646,7 @@ impl<'a> Gen<'a> { let mut out = format!( "// Code generated by rust2-ffi. DO NOT EDIT.\n\n\ package {name}\n\n\ - /*\n#cgo CFLAGS: -I${{SRCDIR}}/../../include\n#include \n#include \"{p}.h\"\n*/\nimport \"C\"\n\n\ + /*\n#cgo CFLAGS: -I${{SRCDIR}}/../rt\n#include \n#include \"{p}.h\"\n*/\nimport \"C\"\n\n\ import (\n\t\"context\"\n\t\"runtime\"\n\t\"runtime/cgo\"\n\t\"unicode/utf8\"\n\t\"unsafe\"\n\n\t\"{rt}\"\n)\n\n\ // The runtime's names for the export's C types and functions.\n\ type (\n\tcBuf = C.{p}_buf\n\tcWaker = C.{p}_waker\n\tcForeignFuture = C.{p}_foreign_future\n\tcForeignReader = C.{p}_foreign_reader\n\tcU128 = C.{p}_u128\n\tcDuration = C.{p}_duration\n\tcIpv4Addr = C.{p}_ipv4_addr\n\tcIpv6Addr = C.{p}_ipv6_addr\n\tcIpAddr = C.{p}_ip_addr\n\tcSocketAddrV4 = C.{p}_socket_addr_v4\n\tcSocketAddrV6 = C.{p}_socket_addr_v6\n\tcSocketAddr = C.{p}_socket_addr\n)\n\n\ @@ -606,7 +660,7 @@ impl<'a> Gen<'a> { } /// The runtime's package, which checks the library's contract once it loads. - fn runtime(&self, packages: &str, contract: u64) -> String { + fn runtime(&self, packages: &str, contract: u64, links: &str) -> String { let p = &self.abi.prefix; let upper = p.to_uppercase(); let go = format!("{p}Go"); @@ -615,7 +669,7 @@ impl<'a> Gen<'a> { // Package rt is what the Go packages of the pub API of {packages} share: the bytes\n\ // Rust returns, the error a closed object returns, and the types of no crate's.\n\ package rt\n\n\ - /*\n#cgo CFLAGS: -I${{SRCDIR}}/../../include\n#include \n#include \"{p}.h\"\n\n\ + /*\n#cgo CFLAGS: -I${{SRCDIR}}\n{links}#include \n#include \"{p}.h\"\n\n\ extern void {go}_call(void *data);\nextern void {go}_release(void *data);\nextern bool {go}_poll(void *data, {p}_waker waker);\n\ extern bool {go}_read(void *data, uint8_t *buf, size_t len, size_t *n, {p}_buf *err);\n\ extern bool {go}_seek(void *data, int whence, int64_t offset, uint64_t *position, {p}_buf *err);\n\n\ @@ -3425,6 +3479,7 @@ fn package_names_are_locked() { constants: vec![], crates: vec!["lock_foobar".into()], targets: vec!["host".into()], + links: Default::default(), reached: Vec::new(), }; let base = api.clone(); @@ -3443,3 +3498,48 @@ fn package_names_are_locked() { assert_eq!(breaks.len(), 1); assert!(breaks[0].what.contains("its Go name `foobar` is now `foobar_`")); } + +#[test] +fn links_each_target_go_builds_for() { + let targets = [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "wasm32-wasip1", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + ]; + let api = crate::ir::Api { + records: vec![], + enums: vec![], + objects: vec![], + callbacks: vec![], + functions: vec![], + constants: vec![], + crates: vec!["lock".into()], + targets: targets.map(String::from).to_vec(), + reached: Vec::new(), + links: (targets.iter()) + .filter(|t| **t != "wasm32-wasip1") + .map(|t| (t.to_string(), vec![format!("-l{}", &t[..3])])) + .collect(), + }; + let (lines, platform) = link(&Abi::new(&api, "lock")).unwrap(); + // iOS also satisfies `darwin`; MSVC's library is none Go links; wasm32 runs no cgo. + assert_eq!( + lines, + "#cgo darwin,!ios,arm64 LDFLAGS: ${SRCDIR}/liblock_darwin_arm64.a -laar\n\ + #cgo ios,arm64 LDFLAGS: ${SRCDIR}/liblock_ios_arm64.a -laar\n\ + #cgo windows,amd64 LDFLAGS: ${SRCDIR}/liblock_windows_amd64.a -lx86\n" + ); + assert!( + platform.contains("//go:build !(darwin && !ios && arm64 || ios && arm64 || windows && amd64)\n"), + "{platform}" + ); + let mut stale = api.clone(); + stale.links.clear(); + assert!( + link(&Abi::new(&stale, "lock")) + .unwrap_err() + .contains("aarch64-apple-darwin") + ); +} diff --git a/ffi/src/ir.rs b/ffi/src/ir.rs index aa822eda6..0c207e42d 100644 --- a/ffi/src/ir.rs +++ b/ffi/src/ir.rs @@ -23,6 +23,10 @@ pub(crate) struct Api { pub(crate) reached: Vec, /// The targets it serves, sorted: one for a single target's, several merged. pub(crate) targets: Vec, + /// By target, the system libraries a static library of the export links beside it, as + /// rustc's `--print native-static-libs` prints them. An API from before it holds none. + #[serde(default)] + pub(crate) links: std::collections::BTreeMap>, } /// A crate the glue depends on beside the facade, as the facade resolves it. diff --git a/ffi/src/lock.rs b/ffi/src/lock.rs index ea817a169..5c4dca59b 100644 --- a/ffi/src/lock.rs +++ b/ffi/src/lock.rs @@ -867,6 +867,7 @@ mod tests { constants: vec![], crates: vec!["lock_a".into(), "lock_b".into()], targets: vec!["host".into()], + links: Default::default(), reached: Vec::new(), } } diff --git a/ffi/src/merge.rs b/ffi/src/merge.rs index 4f084eb5f..57739ce9a 100644 --- a/ffi/src/merge.rs +++ b/ffi/src/merge.rs @@ -360,6 +360,7 @@ impl Merged { /// Add `api`'s items, of one target, to the merged API's. fn add(&mut self, mut api: Api) { + self.api.links.append(&mut api.links); let maps = Maps { records: Maps::of(&api.records, self.api.records.len(), |r| { (self.api.records.iter()).position(|m| m.shown == r.shown) @@ -821,6 +822,7 @@ mod tests { crates: ["a", "c"].map(String::from).to_vec(), reached: Vec::new(), targets: ["x", "y"].map(String::from).to_vec(), + links: Default::default(), }; let Err(cycle) = order_crates(&api) else { panic!("a cycle") @@ -853,6 +855,7 @@ mod tests { .into_iter() .collect(), targets: vec![target.into()], + links: Default::default(), }; let (_, differs) = super::merge(vec![api("x", true), api("y", false)]); assert_eq!(differs.len(), 1); diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index bbb48737d..b43defd9c 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -333,7 +333,7 @@ fn api_go_race() { &std::fs::read_to_string(manifest_dir().join("tests").join("api.go")).unwrap(), ); let exe = dir.join(format!("api-test{EXE_SUFFIX}")); - run(go(&module, api, &["build", "-race", "-o"]).arg(&exe)); + run(go(&module, &["build", "-race", "-o"]).arg(&exe)); // Every report fails the run. let env = [("GORACE", "halt_on_error=1")]; let output = run(Command::new(&exe).envs(env)); @@ -1794,7 +1794,7 @@ fn api_go() { reads: &[&module], writes: &[], }; - step.run(&mut go(root, api, &["vet", "./..."])); + step.run(&mut go(root, &["vet", "./..."])); } let exe = dir.join(format!("api-test{EXE_SUFFIX}")); go_build(api, &module, &exe); @@ -1858,13 +1858,10 @@ fn api_go() { } /// Lay out `module` as a Go module whose main package is `main`, requiring the export's -/// module of a package per crate, beside the header as the export lays them out. +/// module of a package per crate, with the static library its runtime links. fn go_layout(api: &Api, module: &Path, main: &str) { step::mirror(&api.out.join("go"), &module.join("export")); - step::copy( - &api.out.join("include").join("api.h"), - &module.join("include").join("api.h"), - ); + go_library(&api.staticlib(), &module.join("export")); step::write(&module.join("main.go"), main); step::write( &module.join("go.mod"), @@ -1879,7 +1876,7 @@ fn go_build(api: &Api, module: &Path, exe: &Path) { reads: &[module, &api.staticlib()], writes: &[exe], }; - step.run(go(module, api, &["build", "-o"]).arg(exe)); + step.run(go(module, &["build", "-o"]).arg(exe)); } /// Under a `go.mod`, the generated packages import each other by its module, as the go @@ -1907,6 +1904,23 @@ fn go_module() { ); let api = std::fs::read_to_string(out.join("go").join("api").join("api.go")).unwrap(); assert!(api.contains("\"example.com/app/sub/out/go/rt\""), "{api}"); + + // A go.mod in go/ itself makes it a module, as a release's is, which the export keeps + // as it is, above any other. + let own = module.join("own"); + let go_mod = "module example.com/own/v2 // a release\n\ngo 1.24\n"; + std::fs::create_dir_all(own.join("go")).unwrap(); + std::fs::write(own.join("go").join("go.mod"), go_mod).unwrap(); + let output = export("api", &own, &[], &[]); + assert!( + output.status.success(), + "exporting failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(std::fs::read_to_string(own.join("go").join("go.mod")).unwrap(), go_mod); + let api = std::fs::read_to_string(own.join("go").join("api").join("api.go")).unwrap(); + assert!(api.contains("\"example.com/own/v2/rt\""), "{api}"); + vet(&own.join("go")); } /// A crate named as the runtime, a Go keyword, or `main` is the Go package of that name and @@ -2214,18 +2228,53 @@ fn vet(go: &Path) { .env("GOTOOLCHAIN", "local")); } -/// The go command in `module`, linking cgo against the api fixture's static library. -fn go(module: &Path, api: &Api, args: &[&str]) -> Command { - let mut ldflags = api.staticlib().into_os_string(); - for lib in &api.native { - ldflags.push(" "); - ldflags.push(lib); +/// Put the static library `lib` in the Go module `export`'s runtime, as the file the +/// export's one `#cgo LDFLAGS` line links with what else it needs, without the LLVM bitcode +/// the go command refuses there. +fn go_library(lib: &Path, export: &Path) { + let rt = std::fs::read_to_string(export.join("rt").join("rt.go")).unwrap(); + let mut names = rt.lines().filter_map(|line| { + let flags = line.strip_prefix("#cgo ")?.split_once(" LDFLAGS: ${SRCDIR}/")?.1; + Some(flags.split(' ').next().unwrap().to_string()) + }); + let (Some(name), None) = (names.next(), names.next()) else { + panic!("the Go links no one library for the host: {rt}"); + }; + let stripped = lib.with_extension("go.a"); + let objcopy = llvm_objcopy(); + Step { + stamp: &step::stamp(&stripped), + reads: &[lib, objcopy], + writes: &[&stripped], } + .run( + Command::new(objcopy) + .args(["--remove-section=.llvmbc", "--remove-section=.llvmcmd"]) + .args(["--remove-section=__LLVM,__bitcode", "--remove-section=__LLVM,__cmdline"]) + .arg(lib) + .arg(&stripped), + ); + step::copy(&stripped, &export.join("rt").join(name)); +} + +/// The nightly's llvm-objcopy, which reads every target's objects. +fn llvm_objcopy() -> &'static Path { + static OBJCOPY: OnceLock = OnceLock::new(); + OBJCOPY.get_or_init(|| { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let sysroot = run(Command::new(rustc).args(["--print", "sysroot"])); + let sysroot = PathBuf::from(String::from_utf8_lossy(&sysroot.stdout).trim()); + let bin = sysroot.join("lib").join("rustlib").join(host()).join("bin"); + bin.join(format!("llvm-objcopy{EXE_SUFFIX}")) + }) +} + +/// The go command in `module`, whose runtime links the api fixture's static library. +fn go(module: &Path, args: &[&str]) -> Command { let mut cmd = Command::new("go"); cmd.args(args) .current_dir(module) .env("CGO_ENABLED", "1") - .env("CGO_LDFLAGS", ldflags) // The module is no checkout of its own, and must build with the Go on PATH. .env("GOFLAGS", "-buildvcs=false") .env("GOTOOLCHAIN", "local"); diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 20be278ef..c595bc9df 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -1536,11 +1536,7 @@ fn round_trip(built: &Built, language: &str) { let module = dir.join("module"); let export = module.join("export"); step::mirror(&built.out.join("go"), &export); - let header = format!("{name}.h"); - step::copy( - &built.out.join("include").join(&header), - &module.join("include").join(&header), - ); + super::go_library(&built.staticlib(), &export); let source = std::fs::read_to_string(export.join(&name).join(format!("{name}.go"))).unwrap(); let rt = std::fs::read_to_string(export.join("rt").join("types.go")).unwrap_or_default(); step::write(&module.join("main.go"), go(built, calls, &source, &rt)); @@ -1548,11 +1544,6 @@ fn round_trip(built: &Built, language: &str) { &module.join("go.mod"), format!("module shapestest\n\ngo 1.24\n\nrequire {name} v0.0.0\n\nreplace {name} => ./export\n"), ); - let mut ldflags = built.staticlib().into_os_string(); - for lib in &built.native { - ldflags.push(" "); - ldflags.push(lib); - } let step = Step { stamp: &step::stamp(&exe), reads: &[&module, &built.staticlib()], @@ -1564,7 +1555,6 @@ fn round_trip(built: &Built, language: &str) { .arg(&exe) .current_dir(&module) .env("CGO_ENABLED", "1") - .env("CGO_LDFLAGS", ldflags) .env("GOFLAGS", "-buildvcs=false") .env("GOTOOLCHAIN", "local"), ); From 21704667128a606f817708e9d200d636dbeb6822 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 01:59:22 -0800 Subject: [PATCH 3/9] Read a go.mod in the export's own form without the go command Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 2 +- ffi/src/glue.rs | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ffi/README.md b/ffi/README.md index 613b871ab..396d9c19a 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -997,7 +997,7 @@ which the export keeps as it is, as a release's names its module path, or else o `go.mod` above `--out`, with the path to `go/`, each as `go mod edit -json` reads it; or else ``, the module of the `go.mod` the export writes in `go/`, which a caller requires and replaces with that directory. The export needs `go` on PATH whenever it reads a go.mod -other than its own. +other than one in its own form, `module ` and `go 1.24`. - **Names** are the C names without the prefix and the segments outside the item's crate, such as the facade's module re-exporting it, in Pascal case: `moq_track_Producer` is diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index c61995874..800786caa 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -555,10 +555,18 @@ fn go_module(out: &Path, prefix: &str) -> Result<(String, Option), Strin let own = out.join("go").join("go.mod"); if own.is_file() { let text = std::fs::read_to_string(&own).map_err(|err| format!("reading {}: {err}", own.display()))?; - // The export's own needs no go command to read. - let module = match text == default { - true => prefix.to_string(), - false => module_path(&own)?, + // One in the form the export writes, as the release builder writes it too, needs no + // go command to read. + let plain = (text.strip_prefix("module ")) + .and_then(|rest| rest.strip_suffix("\n\ngo 1.24\n")) + .filter(|path| { + !path.is_empty() + && !path.contains("//") + && path.chars().all(|c| c.is_ascii_alphanumeric() || "._~/-".contains(c)) + }); + let module = match plain { + Some(path) => path.to_string(), + None => module_path(&own)?, }; return Ok((module, Some(text))); } From 6056f379bc6588b8c780c6c0889f87b5cfe23d37 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:08:41 -0800 Subject: [PATCH 4/9] Link winapi's system import libraries in the Go module's Windows library Co-Authored-By: Claude Opus 5.5 --- ffi/README.md | 5 ++++- ffi/release/package.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ffi/README.md b/ffi/README.md index 396d9c19a..74f785744 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -978,7 +978,10 @@ check of each target reads from its crates' `#[link]` attributes and build scrip build needs no `CGO_LDFLAGS`, and on a platform with no library, `rt/platform.go` fails to compile. Go links Windows through MinGW, so only `x86_64-pc-windows-gnu` and `aarch64-pc-windows-gnullvm` are Windows targets it links, and iOS the device's library, as -Go builds the simulator's from the same `ios/arm64`; Linux and macOS lines exclude Android +Go builds the simulator's from the same `ios/arm64`. A library a crate ships rather than the +system, as winapi's `winapi_` import libraries for MinGW, is not found where the Go +links: the release builder exports and builds with `WINAPI_NO_BUNDLED_LIBRARIES=1`, so +winapi links the system's; Linux and macOS lines exclude Android and iOS, which Go also counts as `linux` and `darwin`. The export puts no library there: copy it in, as [the release builder](release/README.md) does, without the LLVM bitcode (`.llvmbc`, `__LLVM`) that Go refuses, from 1.26.11 and 1.27.4, in an archive a `#cgo` line diff --git a/ffi/release/package.py b/ffi/release/package.py index d6e0a413b..fea45e9d2 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -825,6 +825,10 @@ def main(): help="the Go module's path, which a proxy you run serves, /vN suffix included") p.add_argument("--rust2", help="the rust2 commit to pin; this checkout's by default") args = parser.parse_args() + # winapi links MinGW import libraries of its own, from cargo's registry, where nothing + # linking the Go module's static library finds them: the system's serve every check and + # build, so the export's #cgo lines and rustc agree on them. + os.environ["WINAPI_NO_BUNDLED_LIBRARIES"] = "1" if args.command == "facts": info = facts(args.manifest_path) From 75a52bae90c11e4dfecc33e5e4ddc667b2104851 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:09:48 -0800 Subject: [PATCH 5/9] Read the module zip's root by its version, and finish the Go packaging quest Co-Authored-By: Claude Opus 5.5 --- ffi/release/package.py | 12 ++++++++++-- ffi/release/test_package.py | 5 +++++ quest/m1/ffi-packaging/go.md | 21 --------------------- 3 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 quest/m1/ffi-packaging/go.md diff --git a/ffi/release/package.py b/ffi/release/package.py index fea45e9d2..e3c376a87 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -691,6 +691,14 @@ def go_module(info, export, libs, out): ]) +def zip_root(name): + """The module and version a file of a module zip, `@/`, is under.""" + match = re.match(r"([^@]+)@(v[^/]+)/", name) + if match is None: + raise SystemExit(f"{name} is in no module's zip") + return match.groups() + + def consume_go(proxy, check, native): """Build and run a check program, or else an import of the runtime, against the module unpacked from its zip, in a new module of its own: no checkout, Rust, or CGO_LDFLAGS.""" @@ -699,8 +707,8 @@ def consume_go(proxy, check, native): raise SystemExit(f"expected one module zip in {proxy}, found {zips}") with tempfile.TemporaryDirectory() as work, zipfile.ZipFile(zips[0]) as z: work = Path(work) - prefix = z.namelist()[0].split("/", 1)[0] - module, version = prefix.rsplit("@", 1) + module, version = zip_root(z.namelist()[0]) + prefix = f"{module}@{version}" z.extractall(work / "mod") consumer = work / "consumer" consumer.mkdir() diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index b036fe780..ec1a86def 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -171,6 +171,11 @@ def test_versions_and_major_suffixes(self): with self.assertRaises(SystemExit): package.check_major(module, version) + def test_zip_root_of_a_nested_module(self): + self.assertEqual(package.zip_root("example.com/api/v2@v2.0.0-rc.1/rt/rt.go"), ("example.com/api/v2", "v2.0.0-rc.1")) + with self.assertRaises(SystemExit): + package.zip_root("rt/rt.go") + def test_escapes_capitals(self): self.assertEqual(package.escape("github.com/Azure/x"), "github.com/!azure/x") diff --git a/quest/m1/ffi-packaging/go.md b/quest/m1/ffi-packaging/go.md deleted file mode 100644 index 27b72ca89..000000000 --- a/quest/m1/ffi-packaging/go.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] A Go module that go get builds - -## Goal - -The generated Go package links its prebuilt static library with no `CGO_LDFLAGS`: the -export writes a `#cgo , LDFLAGS: ${SRCDIR}/...` line per target it exports, -with the system libraries rustc prints, and the builder releases the module with each -target's library, so `go get` and `go build` work for every target moq ships. CI builds -the Go test on Linux, macOS, and Windows (with a `-gnu` static library for Go's mingw -linker). - -## Plan - -Found by the 2026-11-18 language audit: the package comment says it links only through -`CGO_LDFLAGS` (`ffi/src/go.rs:188-189`); moq works around it with a hand-written -per-platform `cgo.go`, as [wasmtime-go](https://github.com/bytecodealliance/wasmtime-go/blob/main/ffi.go) -does. Go module zips are capped at 500 MiB, and the stand-in's `libmoq.a` is 59 to 104 MiB -stripped per target: report the module's size, and recommend a layout (one module, or one -per target) that stays under the cap with room to grow. - -Public API: the generated package's cgo directives, and the module layout. From ac4133da7364f6a792fb0e9c75dcbee628ae368a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:16:11 -0800 Subject: [PATCH 6/9] Serve the Go module's version list, and read a go.mod's module as Go does Co-Authored-By: Claude Opus 5.5 --- ffi/release/README.md | 3 ++- ffi/release/package.py | 11 ++++++++--- ffi/release/release.yml | 2 +- ffi/release/test_package.py | 11 +++++++++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index 78cd94dcb..76c86afbf 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -79,7 +79,8 @@ installed packages. Without a check, a consumer only imports the module. `x86_64-pc-windows-gnu` library, for Go's MinGW linker), and the license files. Each library must need exactly the system libraries the export's line names, as rustc prints them building it, and loses its debug info and the LLVM bitcode Go refuses. The module's - `.info`, `.mod`, and `.zip` are laid out as a GOPROXY serves them, under Go's 500 MiB + `.info`, `.mod`, and `.zip`, and a version `list` naming this release alone, which a proxy + serving earlier releases appends to its own, are laid out as a GOPROXY serves them, under Go's 500 MiB limit on the zip and on its files, whose sizes the run's summary reports. One module holds every platform, so a release is one version of one module. Should it near the limit, each platform's library moves to a module of its own, whose one package's `#cgo` diff --git a/ffi/release/package.py b/ffi/release/package.py index e3c376a87..b1f0c184d 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -616,10 +616,12 @@ def go_version(version): def go_module_path(export): """The module the export's go.mod declares.""" text = (Path(export) / "go" / "go.mod").read_text(encoding="utf-8") - match = re.search(r"^module\s+(\S+)\s*$", text, re.M) + # A path bare or quoted, then perhaps a comment, as the go command reads the directive. + match = re.search(r'^module\s+(?:"((?:[^"\\]|\\.)+)"|`([^`]+)`|([^\s"`/]+(?:/[^\s"`/]+)*))\s*(?://.*)?$', text, re.M) if match is None: - raise SystemExit("the export's go.mod declares no module") - return match.group(1) + raise SystemExit("the export's go.mod declares no module on one line") + quoted, raw, bare = match.groups() + return json.loads(f'"{quoted}"') if quoted else raw or bare def check_major(module, version): @@ -673,6 +675,9 @@ def go_module(info, export, libs, out): versions.mkdir(parents=True) (versions / f"{version}.info").write_text(json.dumps({"Version": version}) + "\n", encoding="utf-8") (versions / f"{version}.mod").write_bytes(files["go.mod"]) + # What `go get ` lists to pick a version: this release's alone, which a proxy + # serving earlier ones appends to its own list. + (versions / "list").write_text(version + "\n", encoding="utf-8") archive = versions / f"{version}.zip" with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z: for name, data in files.items(): diff --git a/ffi/release/release.yml b/ffi/release/release.yml index bbc013827..4bd3f1a73 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -11,7 +11,7 @@ name: Release @NAME@ bindings # The mirror's token comes from a GitHub App installed on it, whose ID and key live in # the `swift-mirror` environment. # -# The Go module @GO_MODULE@ is the build's @NAME@-go artifact, its .info, .mod, and .zip laid +# The Go module @GO_MODULE@ is the build's @NAME@-go artifact, its .info, .mod, .zip, and list laid # out as a GOPROXY serves them; no job here publishes it. Serve it from a module proxy that # @GO_MODULE@'s go-import tag names: a git mirror holds no library over GitHub's 100 MiB. # diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index ec1a86def..911c51508 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -176,6 +176,16 @@ def test_zip_root_of_a_nested_module(self): with self.assertRaises(SystemExit): package.zip_root("rt/rt.go") + def test_module_path_as_go_reads_it(self): + with tempfile.TemporaryDirectory() as root: + go = Path(root) / "go" + go.mkdir() + for text, want in [("module example.com/own/v2 // a release\n", "example.com/own/v2"), + ('module "example.com/q"\n', "example.com/q"), ("module `example.com/r`\n", "example.com/r"), + ("// module wrong\nmodule example.com/s\n\ngo 1.24\n", "example.com/s")]: + (go / "go.mod").write_text(text) + self.assertEqual(package.go_module_path(root), want) + def test_escapes_capitals(self): self.assertEqual(package.escape("github.com/Azure/x"), "github.com/!azure/x") @@ -196,6 +206,7 @@ def test_module_as_a_proxy_serves_it(self): versions = root / "proxy/moq.dev/moq-ffi/@v" self.assertEqual((versions / "v0.4.12.mod").read_text(), "module moq.dev/moq-ffi\n\ngo 1.24\n") self.assertEqual(json.loads((versions / "v0.4.12.info").read_text()), {"Version": "v0.4.12"}) + self.assertEqual((versions / "list").read_text(), "v0.4.12\n") with zipfile.ZipFile(versions / "v0.4.12.zip") as z: self.assertEqual(sorted(z.namelist()), [f"moq.dev/moq-ffi@v0.4.12/{n}" for n in [ "LICENSE-MIT", "go.mod", "rt/libmoq_darwin_arm64.a", "rt/libmoq_linux_amd64.a", From e9fdee66b11d3658701f49a9373a24d3394be14e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:17:35 -0800 Subject: [PATCH 7/9] Read the Go module's path with the go command's own parser Co-Authored-By: Claude Opus 5.5 --- .github/workflows/bindings.yml | 5 +++++ ffi/release/package.py | 14 ++++++-------- ffi/release/test_package.py | 5 +++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index c00699d06..67ee13861 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -462,6 +462,11 @@ jobs: with: pattern: ${{ needs.facts.outputs.name }}-build-* path: builds + # The go command reads the module's go.mod. + - uses: actions/setup-go@d8cbb8045d7f43706b9177cceb36c1f91a6644d5 # v7.1.0 + with: + go-version: stable + cache: false - name: Module env: NAME: ${{ needs.facts.outputs.name }} diff --git a/ffi/release/package.py b/ffi/release/package.py index b1f0c184d..4ae1f7219 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -614,14 +614,12 @@ def go_version(version): def go_module_path(export): - """The module the export's go.mod declares.""" - text = (Path(export) / "go" / "go.mod").read_text(encoding="utf-8") - # A path bare or quoted, then perhaps a comment, as the go command reads the directive. - match = re.search(r'^module\s+(?:"((?:[^"\\]|\\.)+)"|`([^`]+)`|([^\s"`/]+(?:/[^\s"`/]+)*))\s*(?://.*)?$', text, re.M) - if match is None: - raise SystemExit("the export's go.mod declares no module on one line") - quoted, raw, bare = match.groups() - return json.loads(f'"{quoted}"') if quoted else raw or bare + """The module the export's go.mod declares, as the go command reads it.""" + path = Path(export) / "go" / "go.mod" + module = json.loads(output("go", "mod", "edit", "-json", path, env={**os.environ, "GOTOOLCHAIN": "local"})).get("Module") + if not module: + raise SystemExit(f"{path} declares no module") + return module["Path"] def check_major(module, version): diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 911c51508..31fabe31b 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -181,8 +181,9 @@ def test_module_path_as_go_reads_it(self): go = Path(root) / "go" go.mkdir() for text, want in [("module example.com/own/v2 // a release\n", "example.com/own/v2"), - ('module "example.com/q"\n', "example.com/q"), ("module `example.com/r`\n", "example.com/r"), - ("// module wrong\nmodule example.com/s\n\ngo 1.24\n", "example.com/s")]: + ('module "example.com/q"\n', "example.com/q"), + ("// module wrong\nmodule example.com/s\n\ngo 1.24\n", "example.com/s"), + ('module ( // the app\n\t"example.com/app"\n)\n', "example.com/app")]: (go / "go.mod").write_text(text) self.assertEqual(package.go_module_path(root), want) From f3ea83e33dfb54f4519d77de3c0d293449217ec8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:29:00 -0800 Subject: [PATCH 8/9] Load std into a no_std facade's check, which the glue links all the same Co-Authored-By: Claude Opus 5.5 --- ffi/src/driver.rs | 22 ++++++++++++++++++++++ ffi/tests/fixtures.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/ffi/src/driver.rs b/ffi/src/driver.rs index d452f34d4..04eb7ccd9 100644 --- a/ffi/src/driver.rs +++ b/ffi/src/driver.rs @@ -150,6 +150,28 @@ impl Export { } impl Callbacks for Export { + /// Load std, unnamed, into a `no_std` facade too: the glue links it whatever the facade + /// does, so the check that lists the system libraries a static library needs must see it. + fn after_crate_root_parsing(&mut self, compiler: &Compiler, krate: &mut ast::Crate) -> Compilation { + let source = "extern crate std as _;"; + let item = rustc_parse::new_parser_from_source_str( + &compiler.sess.psess, + FileName::anon_source_code(source), + source.to_string(), + StripTokens::Nothing, + ) + .ok() + .and_then(|mut parser| { + parser + .parse_item(ForceCollect::No, AllowConstBlockItems::DoesNotMatter) + .ok() + }) + .flatten() + .expect("an extern crate parses"); + krate.items.insert(0, item); + Compilation::Continue + } + fn after_analysis<'tcx>(&mut self, _compiler: &Compiler, tcx: TyCtxt<'tcx>) -> Compilation { if tcx.dcx().has_errors().is_some() { return Compilation::Stop; diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index b43defd9c..dfd9d219b 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -2926,6 +2926,40 @@ fn package(name: &str, files: &[(&str, &str)]) -> PathBuf { package } +/// A `no_std` facade's API still names the system libraries std links, as the glue's +/// static library links std whatever the facade does: those rustc prints for the api +/// fixture's, whose crates link nothing of their own. +#[test] +fn no_std_links() { + let package = package( + "no_std_links", + &[( + "src/lib.rs", + "#![cfg_attr(not(test), no_std)]\n\npub fn one() -> u32 {\n\t1\n}\n", + )], + ); + let out = package.join("out"); + let output = Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(package.join("ffi.toml")) + .arg("--manifest-path") + .arg(package.join("Cargo.toml")) + .arg("--out") + .arg(&out) + .env("CARGO_TARGET_DIR", package.join("build")) + .output() + .unwrap(); + assert!( + output.status.success(), + "exporting failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + let exported: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(out.join("api.json")).unwrap()).unwrap(); + let links: Vec = serde_json::from_value(exported["api"]["links"][host()].clone()).unwrap(); + assert_eq!(links, api().native); +} + /// rustc's arguments in an argfile, as cargo passes them once they overflow a command line: /// on Windows, moq-ffi's many `-L`s do. One argument past every platform's limit forces it, /// and starts with `@`, which inside an argfile rustc reads as itself, not another file. From 9d3efa4a8a7596c011bc4451961bef1955f962a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 05:59:45 -0800 Subject: [PATCH 9/9] Ship the iOS library the Go module links, and check the module against its #cgo lines Co-Authored-By: Claude Opus 5.5 --- ffi/release/README.md | 9 ++++++--- ffi/release/package.py | 10 +++++++--- ffi/release/release.yml | 6 ++++-- ffi/release/test_package.py | 21 +++++++++++++++------ ffi/tests/fixtures.rs | 3 ++- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index 76c86afbf..e02dca74c 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -29,8 +29,10 @@ refuses to overwrite a different file. Then, outside the repository: `APP_CLIENT_ID` and `APP_PRIVATE_KEY` in a `swift-mirror` environment that only release tags may deploy to; - serve the build's `-go` artifact, the module laid out as a GOPROXY serves it, from a - module proxy the module path's `go-import` tag names. No job publishes it: a git mirror - holds no library over GitHub's 100 MiB, and a proxy is the caller's to run. + module proxy: on a host you control, a `go-import` tag of type `mod` names it, but Go + resolves a `github.com` path by git alone, so its consumers set `GOPROXY` to the proxy. No + job publishes it: a git mirror holds no library over GitHub's 100 MiB, and a proxy is the + caller's to run. Releasing is pushing `-v`, the crate's version, which the release checks. Called without a tag, such as from a nightly, it runs everything up to publication. @@ -75,7 +77,8 @@ installed packages. Without a check, a consumer only imports the module. 6. Go: one module for every target, at the module path and `v`, the major version's `/vN` suffix checked: the export's `go/`, whose runtime's `#cgo` lines link `rt/lib__.a` for `linux/amd64` and `linux/arm64` (built against - glibc 2.28), `darwin/arm64` (the Swift package's macOS library), and `windows/amd64` (an + glibc 2.28), `darwin/arm64` and `ios/arm64` (the Swift package's macOS and iOS device + libraries; no consumer here runs iOS), and `windows/amd64` (an `x86_64-pc-windows-gnu` library, for Go's MinGW linker), and the license files. Each library must need exactly the system libraries the export's line names, as rustc prints them building it, and loses its debug info and the LLVM bitcode Go refuses. The module's diff --git a/ffi/release/package.py b/ffi/release/package.py index 4ae1f7219..e6dcea60e 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -49,6 +49,7 @@ "x86_64-unknown-linux-gnu": "linux_amd64", "aarch64-unknown-linux-gnu": "linux_arm64", "aarch64-apple-darwin": "darwin_arm64", + "aarch64-apple-ios": "ios_arm64", "x86_64-pc-windows-gnu": "windows_amd64", } # What the go command refuses to extract: a module zip, or its files, over 500 MiB, and a @@ -638,12 +639,15 @@ def escape(module): def go_files(info, export, libs): - """The module's files by path: the export's Go, every Go target's library from the - directory `libs` beside the runtime, and the license files, at the root where + """The module's files by path: the export's Go, each library its runtime links from the + directory `libs` beside it, and the license files, at the root where pkg.go.dev looks for them.""" root = Path(export) / "go" files = {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()} - want = {f"lib{info['lib']}_{platform}.a" for platform in GO_TARGETS.values()} + # Exactly the libraries the runtime's #cgo lines name. + want = set(re.findall(r"^#cgo \S+ LDFLAGS: \$\{SRCDIR\}/(\S+\.a)", files["rt/rt.go"].decode(), re.M)) + if want != {f"lib{info['lib']}_{platform}.a" for platform in GO_TARGETS.values()}: + raise SystemExit(f"the export's Go links {sorted(want)}, not each of {sorted(GO_TARGETS)}") found = {p.name for p in Path(libs).iterdir()} if found != want: raise SystemExit(f"the Go module links {sorted(want)}, given {sorted(found)}") diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 4bd3f1a73..a3f341bf3 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -12,8 +12,10 @@ name: Release @NAME@ bindings # the `swift-mirror` environment. # # The Go module @GO_MODULE@ is the build's @NAME@-go artifact, its .info, .mod, .zip, and list laid -# out as a GOPROXY serves them; no job here publishes it. Serve it from a module proxy that -# @GO_MODULE@'s go-import tag names: a git mirror holds no library over GitHub's 100 MiB. +# out as a GOPROXY serves them; no job here publishes it. Serve it from a module proxy: on a +# host you control, a go-import tag of type `mod` names it, but Go resolves a github.com path +# by git alone, so its consumers set GOPROXY to the proxy. A git mirror holds no library +# over GitHub's 100 MiB. # # To release, push the tag @NAME@-v, naming the crate's version. Without a tag, as # from a nightly's workflow_call, every job up to publication runs. If publishing fails, use diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 31fabe31b..76df68b64 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -34,8 +34,9 @@ def export(root, floor="311"): (root / "Cargo.toml").write_text(f'pyo3-ffi = {{ features = ["abi3-py{floor}", "extension-module"] }}\n') (root / "go/rt").mkdir(parents=True) (root / "go/go.mod").write_text("module moq.dev/moq-ffi\n\ngo 1.24\n") - (root / "go/rt/rt.go").write_text( - "package rt\n\n/*\n#cgo linux,!android,amd64 LDFLAGS: ${SRCDIR}/libmoq_linux_amd64.a -lgcc_s -lc\n*/\nimport \"C\"\n") + lines = "".join(f"#cgo {p.replace('_', ',')} LDFLAGS: ${{SRCDIR}}/libmoq_{p}.a -lgcc_s -lc\n" + for p in package.GO_TARGETS.values()) + (root / "go/rt/rt.go").write_text(f"package rt\n\n/*\n{lines}*/\nimport \"C\"\n") return root @@ -194,8 +195,9 @@ def test_links_from_the_cgo_line(self): with tempfile.TemporaryDirectory() as root: root = export(Path(root)) self.assertEqual(package.go_links(root, "moq", "x86_64-unknown-linux-gnu"), "-lgcc_s -lc") + (root / "go/rt/rt.go").write_text("package rt\n") with self.assertRaises(SystemExit): - package.go_links(root, "moq", "aarch64-unknown-linux-gnu") + package.go_links(root, "moq", "x86_64-unknown-linux-gnu") def test_module_as_a_proxy_serves_it(self): with tempfile.TemporaryDirectory() as root: @@ -210,12 +212,17 @@ def test_module_as_a_proxy_serves_it(self): self.assertEqual((versions / "list").read_text(), "v0.4.12\n") with zipfile.ZipFile(versions / "v0.4.12.zip") as z: self.assertEqual(sorted(z.namelist()), [f"moq.dev/moq-ffi@v0.4.12/{n}" for n in [ - "LICENSE-MIT", "go.mod", "rt/libmoq_darwin_arm64.a", "rt/libmoq_linux_amd64.a", - "rt/libmoq_linux_arm64.a", "rt/libmoq_windows_amd64.a", "rt/rt.go"]]) - # The module ships every Go target's library, and nothing else. + "LICENSE-MIT", "go.mod", "rt/libmoq_darwin_arm64.a", "rt/libmoq_ios_arm64.a", + "rt/libmoq_linux_amd64.a", "rt/libmoq_linux_arm64.a", "rt/libmoq_windows_amd64.a", "rt/rt.go"]]) + # The module ships exactly the libraries the runtime links. (root / "libs/libmoq_linux_arm64.a").unlink() with self.assertRaises(SystemExit): package.go_module(info(root), built, root / "libs", root / "other") + (root / "libs/libmoq_linux_arm64.a").write_bytes(b"!\n") + rt = built / "go/rt/rt.go" + rt.write_text(rt.read_text().replace("#cgo ios,arm64", "// none")) + with self.assertRaises(SystemExit): + package.go_module(info(root), built, root / "libs", root / "third") def test_refuses_names_a_case_folds_together(self): with tempfile.TemporaryDirectory() as root: @@ -223,6 +230,8 @@ def test_refuses_names_a_case_folds_together(self): built = export(root / "export") (built / "go/RT.go").write_text("package rt\n") (built / "go/rt.go").write_text("package rt\n") + if len(list((built / "go").glob("[Rr][Tt].go"))) != 2: + self.skipTest("a case-insensitive file system holds one of the two names") with self.assertRaises(SystemExit): package.go_files(info(root), built, go_libs(root)) diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index dfd9d219b..32086635b 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -2240,7 +2240,8 @@ fn go_library(lib: &Path, export: &Path) { let (Some(name), None) = (names.next(), names.next()) else { panic!("the Go links no one library for the host: {rt}"); }; - let stripped = lib.with_extension("go.a"); + // Beside the module, so layouts that tests build at once strip apart. + let stripped = export.parent().unwrap().with_extension("a"); let objcopy = llvm_objcopy(); Step { stamp: &step::stamp(&stripped),