diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index 545c7f8b3..291ad38e8 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -18,6 +18,9 @@ jobs: features: doubled example: transcript python-check: ffi/tests/api.py + # A module path a proxy could serve. No go-check, as no swift-check: api.go needs the + # wasm32 items this export lacks, so the consumer only loads the runtime. + go-module: example.com/api # No swift-check: api.swift compiles against the fixture's own test export, of the host # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. @@ -28,3 +31,5 @@ jobs: manifest-path: ffi/target/moq-ffi/Cargo.toml python-check: ffi/moq/moq.py swift-check: ffi/moq/moq.swift + # Under the export's own module, `moq`, which moq.go imports. + go-check: ffi/moq/moq.go diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 677c7ea4b..67ee13861 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -1,6 +1,7 @@ name: Binding packages -# The shared, read-only builder of a crate's rust2-ffi Python wheels and Swift package, which +# The shared, read-only builder of a crate's rust2-ffi Python wheels, Swift package, and Go +# module, which # a release workflow ffi/release/package.py's `init` writes calls at a pinned rust2 commit. # It publishes nothing and holds no credentials: the caller's own jobs publish what it # uploads, once its consumers pass. See ffi/release/README.md. @@ -27,6 +28,14 @@ on: description: The main.swift of an executable run on the installed Swift package default: '' type: string + go-module: + description: The Go module's path; without it, the export's own, which no proxy serves + default: '' + type: string + go-check: + description: The main.go of a program, importing the module's packages, run against it + default: '' + type: string outputs: version: description: The package's Cargo version @@ -118,9 +127,10 @@ jobs: - host: apple os: macos-26 targets: aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim + # MSVC's library for the wheel, MinGW's for Go, which links through MinGW. - host: windows os: windows-latest - targets: x86_64-pc-windows-msvc + targets: x86_64-pc-windows-msvc x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} timeout-minutes: 60 defaults: @@ -194,7 +204,7 @@ jobs: targets: aarch64-apple-ios-sim - host: windows os: windows-latest - targets: x86_64-pc-windows-msvc + targets: x86_64-pc-windows-msvc x86_64-pc-windows-gnu runs-on: ${{ matrix.os }} timeout-minutes: 90 defaults: @@ -248,9 +258,10 @@ jobs: - name: Export env: TARGETS: ${{ matrix.targets }} + GO_MODULE: ${{ inputs.go-module }} run: | read -ra targets <<< "$TARGETS" - python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --out export "${targets[@]}" + python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --go-module "$GO_MODULE" --out export "${targets[@]}" - name: Build env: TARGETS: ${{ matrix.targets }} @@ -262,10 +273,12 @@ jobs: env: PYTHON_CHECK: ${{ inputs.python-check }} SWIFT_CHECK: ${{ inputs.swift-check }} + GO_CHECK: ${{ inputs.go-check }} run: | mkdir -p dist/checks if [[ -n "$PYTHON_CHECK" ]]; then cp "source/$PYTHON_CHECK" dist/checks/check.py; fi if [[ -n "$SWIFT_CHECK" ]]; then cp "source/$SWIFT_CHECK" dist/checks/main.swift; fi + if [[ -n "$GO_CHECK" ]]; then cp "source/$GO_CHECK" dist/checks/main.go; fi - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} @@ -419,3 +432,100 @@ jobs: env: MODULE: ${{ needs.facts.outputs.module }} run: bash rust2/ffi/release/consume-swift.sh swift apple + + # The module as a GOPROXY serves it: the export's packages, and each Go target's library + # beside the runtime that links it. + go: + needs: [facts, build, agree] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + persist-credentials: false + - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 + if: needs.facts.outputs.fixture == 'true' + with: + tool: just + - name: Stand-in + if: needs.facts.outputs.fixture == 'true' + working-directory: source + run: just ffi standin && cp ffi/moq/ffi.toml ffi/target/moq-ffi/ffi.toml + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: ${{ needs.facts.outputs.name }}-build-* + path: builds + # The go command reads the module's go.mod. + - uses: actions/setup-go@d8cbb8045d7f43706b9177cceb36c1f91a6644d5 # v7.1.0 + with: + go-version: stable + cache: false + - name: Module + env: + NAME: ${{ needs.facts.outputs.name }} + run: | + mkdir libs + cp builds/*/go/*.a libs/ + cd source + python3 -I ../rust2/ffi/release/package.py go --manifest-path "$MANIFEST" --export "../builds/$NAME-build-linux-x64/export" --libs ../libs --out ../proxy >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ needs.facts.outputs.name }}-go + path: proxy + if-no-files-found: error + + # A new module whose only requirement is the built one, unpacked from its zip, as `go get` + # unpacks it: built and run on each platform with no checkout, Rust, or CGO_LDFLAGS. + go-consumer: + needs: [facts, go] + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + os: ubuntu-24.04 + - host: linux-arm64 + os: ubuntu-24.04-arm + - host: apple-darwin + os: macos-26 + - host: windows + os: windows-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ needs.facts.outputs.rust2 }} + ref: ${{ needs.facts.outputs.rust2-sha }} + path: rust2 + sparse-checkout: ffi/release + persist-credentials: false + # The newest Go, whose cgo checks what a #cgo line links most strictly. + - uses: actions/setup-go@d8cbb8045d7f43706b9177cceb36c1f91a6644d5 # v7.1.0 + with: + go-version: stable + cache: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-go + path: proxy + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ${{ needs.facts.outputs.name }}-build-${{ matrix.host }} + path: dist + - name: Build and run + run: | + args=() + if [[ -f dist/checks/main.go ]]; then args+=(--check dist/checks/main.go); fi + if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi + python3 -I rust2/ffi/release/package.py consume-go --proxy proxy "${args[@]}" diff --git a/ffi/README.md b/ffi/README.md index fa002c4d9..74f785744 100644 --- a/ffi/README.md +++ b/ffi/README.md @@ -970,8 +970,22 @@ value of a Rust type crosses as one Swift type that holds it exactly, and back u `go/` holds a Go package per crate whose items the facade exports, over the C header through cgo, for Go 1.24 or later, and the runtime they share, `go/rt`. Each includes -`.h` from the `include/` beside `go/`, as the export lays them out, and links the -library through `CGO_LDFLAGS`. A crate's package is its name without the prefix and its +`.h` from `go/rt`, so `go/` is a whole module, as a release's zip holds it. The +runtime links each exported target's static library from beside itself, as +`rt/lib__.a`, by a `#cgo , LDFLAGS` line naming it and the +system libraries it needs, as rustc's `--print native-static-libs` prints them, which the +check of each target reads from its crates' `#[link]` attributes and build scripts. So a +build needs no `CGO_LDFLAGS`, and on a platform with no library, `rt/platform.go` fails to +compile. Go links Windows through MinGW, so only `x86_64-pc-windows-gnu` and +`aarch64-pc-windows-gnullvm` are Windows targets it links, and iOS the device's library, as +Go builds the simulator's from the same `ios/arm64`. A library a crate ships rather than the +system, as winapi's `winapi_` import libraries for MinGW, is not found where the Go +links: the release builder exports and builds with `WINAPI_NO_BUNDLED_LIBRARIES=1`, so +winapi links the system's; Linux and macOS lines exclude Android +and iOS, which Go also counts as `linux` and `darwin`. The export puts no library there: +copy it in, as [the release builder](release/README.md) does, without the LLVM bitcode +(`.llvmbc`, `__LLVM`) that Go refuses, from 1.26.11 and 1.27.4, in an archive a `#cgo` line +names. A crate's package is its name without the prefix and its `_`: moq-mux's is `mux`, and serde_json's `serdejson`, but keeps the prefix where its strip leaves no leading letter (`moq_2d` is `moq2d`). One that a Go keyword, `C`, `main`, the runtime, or an earlier crate's package takes gains a trailing `_`, as a crate `rt` is @@ -981,10 +995,12 @@ while Rust's modules may, so a package per module could not hold moq. An instanc generic type, or a function naming one, belongs to the last of its own and its arguments' crates in Cargo's order, which can name them all: moq's `mux::catalog::Producer` is in `mux`. The packages' import path is -the module of the `go.mod` above `--out`, as `go mod edit -json` reads it, with the path to -`go/`; or else ``, the module of the `go.mod` the export writes in `go/`, which a -caller requires and replaces with that directory. The export needs `go` on PATH whenever it -reads a go.mod. +the module of the module `go/` is in, as Go finds it: that of a `go.mod` in `go/` itself, +which the export keeps as it is, as a release's names its module path, or else of the +`go.mod` above `--out`, with the path to `go/`, each as `go mod edit -json` reads it; or else +``, the module of the `go.mod` the export writes in `go/`, which a caller requires and +replaces with that directory. The export needs `go` on PATH whenever it reads a go.mod +other than one in its own form, `module ` and `go 1.24`. - **Names** are the C names without the prefix and the segments outside the item's crate, such as the facade's module re-exporting it, in Pascal case: `moq_track_Producer` is diff --git a/ffi/justfile b/ffi/justfile index 4d1712aca..f230fd179 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -96,9 +96,9 @@ bench-build checkout="": (moq-export checkout) (moq-web checkout) ts-program cod cc -O2 -std=c11 -Wall -Wextra -Werror -I "$moqc/include" -I "$hooks" moq/bench/moq-c.c "$src/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-moq-c c++ -O2 -std=c++17 -Wall -Wextra -Werror -I "$out/include" -I "$hooks" moq/bench/export.cpp "$out/target/release/libmoq.a" "$hooks/libcodspeed.a" -lpthread -ldl -lm -o target/bench-export-cpp {{ quote(just_executable()) }} go-module bench-go moq/bench/export.go - read -ra native < target/moq-native - # The include path goes in CGO_CPPFLAGS, so cgo keeps its default -O2 -g in CGO_CFLAGS. - export CGO_CPPFLAGS="-I$hooks" CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a $hooks/libcodspeed.a ${native[*]}" GOFLAGS=-buildvcs=false + # The include path goes in CGO_CPPFLAGS, so cgo keeps its default -O2 -g in CGO_CFLAGS; + # the runtime's #cgo line links moq. + export CGO_CPPFLAGS="-I$hooks" CGO_LDFLAGS="$hooks/libcodspeed.a" GOFLAGS=-buildvcs=false (cd target/bench-go && go build -o bench .) {{ quote(just_executable()) }} dart-program bench-dart moq/bench/export.dart rm -rf target/bench-kt @@ -257,9 +257,9 @@ moq checkout="": (moq-cpp checkout) mypy --strict --python-version 3.11 --cache-dir target/mypy "$out/python/moq.py" pyright -p tests/pyright.json "$out/python/moq.py" {{ quote(just_executable()) }} moq-swift-run - # The Go, through cgo, linking the library and what it needs besides. + # The Go, through cgo, whose runtime's #cgo line links the library and what it needs besides. {{ quote(just_executable()) }} go-module moq-go moq/moq.go - export CGO_LDFLAGS="$PWD/$out/target/release/libmoq.a ${native[*]}" GOFLAGS=-buildvcs=false + export GOFLAGS=-buildvcs=false (cd target/moq-go/export && go vet ./...) (cd target/moq-go && go vet ./... && go run .) # The Dart, through the shared library the dart feature builds. @@ -341,7 +341,7 @@ swift-target macos: #!/usr/bin/env bash if [ "{{ os() }}" = macos ]; then echo "-target $(uname -m)-apple-macos{{ macos }}"; fi -# Lay out target/ as a Go module whose main package is
, requiring the export's module of a package per crate in export/, beside the header it includes, linking the library through cgo. +# Lay out target/ as a Go module whose main package is
, requiring the export's module of a package per crate in export/, with the library its runtime's #cgo line links, less the LLVM bitcode the go command refuses there. [private] go-module dir main: #!/usr/bin/env bash @@ -349,9 +349,13 @@ go-module dir main: out=target/export/moq dir=target/{{ dir }} rm -rf "$dir" - mkdir -p "$dir/include" + mkdir -p "$dir" cp -r "$out/go" "$dir/export" - cp "$out/include/moq.h" "$dir/include/" + lib=libmoq_$(go env GOOS)_$(go env GOARCH).a + grep -qF "LDFLAGS: \${SRCDIR}/$lib" "$dir/export/rt/rt.go" || { echo "the export's Go links no $lib" >&2; exit 1; } + "$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin/llvm-objcopy" \ + --remove-section=.llvmbc --remove-section=.llvmcmd --remove-section=__LLVM,__bitcode --remove-section=__LLVM,__cmdline \ + "$out/target/release/libmoq.a" "$dir/export/rt/$lib" cp {{ main }} "$dir/main.go" printf 'module moqtest\n\ngo 1.24\n\nrequire moq v0.0.0\n\nreplace moq => ./export\n' > "$dir/go.mod" diff --git a/ffi/release/README.md b/ffi/release/README.md index de013ca36..e02dca74c 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -1,7 +1,7 @@ # Binding releases -A crate's [rust2-ffi](/ffi/README.md) export, released as Python wheels and a Swift -package. Two halves, split where Cargo's knowledge ends: +A crate's [rust2-ffi](/ffi/README.md) export, released as Python wheels, a Swift package, +and a Go module. Two halves, split where Cargo's knowledge ends: - **The builder**, [`bindings.yml`](/.github/workflows/bindings.yml) over `package.py`, shared and read-only. It derives everything from Cargo metadata and the export: package @@ -10,14 +10,15 @@ package. Two halves, split where Cargo's knowledge ends: consumers, and publishes nothing. - **The release workflow**, the caller's own file, which `package.py init` writes once. It holds what Cargo cannot establish: who may publish to PyPI, where the XCFramework is - hosted, which repository SwiftPM resolves the package from, and which tag releases. + hosted, which repository SwiftPM resolves the package from, the Go module's path, and + which tag releases. ## Adopting it From a rust2 checkout at the commit to pin, in the repository holding the crate: ```sh -python3 /ffi/release/package.py init --manifest-path rs/moq-ffi/Cargo.toml --swift-mirror moq-dev/moq-swift-ffi +python3 /ffi/release/package.py init --manifest-path rs/moq-ffi/Cargo.toml --swift-mirror moq-dev/moq-swift-ffi --go-module moq.dev/moq-ffi ``` It writes `.github/workflows/release-.yml`, calling the builder at that commit, and @@ -26,7 +27,12 @@ refuses to overwrite a different file. Then, outside the repository: - register the workflow and the `pypi` environment as the PyPI project's trusted publisher; - create the mirror repository, install a GitHub App on it with contents write, and put its `APP_CLIENT_ID` and `APP_PRIVATE_KEY` in a `swift-mirror` environment that only release - tags may deploy to. + tags may deploy to; +- serve the build's `-go` artifact, the module laid out as a GOPROXY serves it, from a + module proxy: on a host you control, a `go-import` tag of type `mod` names it, but Go + resolves a `github.com` path by git alone, so its consumers set `GOPROXY` to the proxy. No + job publishes it: a git mirror holds no library over GitHub's 100 MiB, and a proxy is the + caller's to run. Releasing is pushing `-v`, the crate's version, which the release checks. Called without a tag, such as from a nightly, it runs everything up to publication. @@ -37,10 +43,11 @@ and the XCFramework archive carry build times, so **Re-run all jobs** fails wher registry already holds the first attempt's files. A Cargo prerelease, such as `1.0.0-rc.1`, is a GitHub prerelease too. -The builder takes the crate's `Cargo.toml` and four optional inputs the caller's tests own: +The builder takes the crate's `Cargo.toml`, the `go-module` path, which without it is the +export's own `` that no proxy serves, and optional inputs the caller's tests own: `features` to ship, an `example` whose stdout is native Rust's transcript, and the -`python-check` and `swift-check` programs that must print it through the installed -packages. Without a check, a consumer only imports the module. +`python-check`, `swift-check`, and `go-check` programs that must print it through the +installed packages. Without a check, a consumer only imports the module. ## What the builder does @@ -67,16 +74,33 @@ packages. Without a check, a consumer only imports the module. revision, since a package resolved by version cannot depend on a revision. Rust's native link libraries become linker settings, per platform where they differ; any other flag fails the build. -6. Consumers, with no checkout, Rust, or source path: each wheel installed with +6. Go: one module for every target, at the module path and `v`, the major + version's `/vN` suffix checked: the export's `go/`, whose runtime's `#cgo` lines link + `rt/lib__.a` for `linux/amd64` and `linux/arm64` (built against + glibc 2.28), `darwin/arm64` and `ios/arm64` (the Swift package's macOS and iOS device + libraries; no consumer here runs iOS), and `windows/amd64` (an + `x86_64-pc-windows-gnu` library, for Go's MinGW linker), and the license files. Each + library must need exactly the system libraries the export's line names, as rustc prints + them building it, and loses its debug info and the LLVM bitcode Go refuses. The module's + `.info`, `.mod`, and `.zip`, and a version `list` naming this release alone, which a proxy + serving earlier releases appends to its own, are laid out as a GOPROXY serves them, under Go's 500 MiB + limit on the zip and on its files, whose sizes the run's summary reports. One module + holds every platform, so a release is one version of one module. Should it near the + limit, each platform's library moves to a module of its own, whose one package's `#cgo` + line links it and which the runtime imports under that platform's build constraint, so + `go get` downloads only the platform it builds for. +7. Consumers, with no checkout, Rust, or source path: each wheel installed with `--no-index --no-deps` into a fresh environment on its oldest CPython, run with `-I` from an empty directory; and the Swift package built by its own scheme for macOS, iOS, - and the iOS Simulator, then consumed by a new executable package that runs the check. - With an `example`, each must print native Rust's stdout byte for byte. + and the iOS Simulator, then consumed by a new executable package that runs the check; + and the Go module, unpacked from its zip into a new module that requires it, vetted, + built with the newest Go and no `CGO_LDFLAGS`, and run on Linux x64 and arm64, macOS, and + Windows. With an `example`, each must print native Rust's stdout byte for byte. rust2 runs it on every manual Check and nightly through [`bindings-test.yml`](/.github/workflows/bindings-test.yml): on the api fixture, whose wheels must print its native transcript, and on moq's stand-in facade at the exporter's moq -pin, through `moq/moq.py` and `moq/moq.swift`. +pin, through `moq/moq.py`, `moq/moq.swift`, and `moq/moq.go`. ## Other registries @@ -89,7 +113,6 @@ binding. | --- | --- | --- | | Maven Central | An AAR with classes.jar, JNI libraries per Android ABI and the generated R8 rules; POM, sources and javadoc jars | The verified groupId namespace, publisher authorization, and signing key | | pub.dev | The generated Dart library with a native-assets build hook and per-platform archives, pubspec, and license | Publisher authorization and the archives' download host | -| Go | The generated cgo packages with module-relative imports, and native libraries per target | The module path, repository subdirectory, and release tags, major-version suffixes included | | npm | Needs a Node generator first: platform packages and a JS loader | Scope ownership and trusted-publisher registration | | NuGet | Needs a C# generator first: the managed assembly and RID-specific native assets | Package ID ownership, credentials, and target frameworks | diff --git a/ffi/release/package.py b/ffi/release/package.py index 9bc9b55d6..e6dcea60e 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -1,4 +1,4 @@ -"""Build and check the Python wheels and Swift package of a crate's rust2-ffi export. +"""Build and check the Python wheels, Swift package, and Go module of a crate's rust2-ffi export. bindings.yml runs each step on the host it needs; release.yml is the caller `init` writes. Every fact about the package comes from Cargo metadata or the export, so nothing here @@ -43,6 +43,22 @@ "aarch64-apple-darwin", "x86_64-pc-windows-msvc", ] +# Every target the Go module links a static library of, by its GOOS_GOARCH. Go links +# Windows through MinGW, so a -gnu library. +GO_TARGETS = { + "x86_64-unknown-linux-gnu": "linux_amd64", + "aarch64-unknown-linux-gnu": "linux_arm64", + "aarch64-apple-darwin": "darwin_arm64", + "aarch64-apple-ios": "ios_arm64", + "x86_64-pc-windows-gnu": "windows_amd64", +} +# What the go command refuses to extract: a module zip, or its files, over 500 MiB, and a +# go.mod over 16 MiB (golang.org/x/mod/zip's MaxZipFile and MaxGoMod). +GO_ZIP_MAX = 500 << 20 +GO_MOD_MAX = 16 << 20 +# The sections the go command refuses in an archive a #cgo directive names: LLVM's bitcode, +# which Rust's prebuilt compiler_builtins carries. +LTO_SECTIONS = [".llvmbc", ".llvmcmd", "__LLVM,__bitcode", "__LLVM,__cmdline"] # Cargo metadata names a library target by its crate types; a proc-macro is no library to export. LIBRARY_KINDS = {"lib", "rlib", "dylib", "cdylib", "staticlib"} COLLECTIONS = "https://github.com/apple/swift-collections" @@ -255,7 +271,8 @@ def digest(export, lib): """A hash of the generated bindings, which every build host's export must agree on.""" hasher = hashlib.sha256() export = Path(export) - paths = [export / "python" / f"{lib}.py", export / "include" / f"{lib}.h", *sorted((export / "swift").iterdir())] + paths = [export / "python" / f"{lib}.py", export / "include" / f"{lib}.h", *sorted((export / "swift").iterdir()), + *sorted(p for p in (export / "go").rglob("*") if p.is_file())] for path in paths: hasher.update(path.relative_to(export).as_posix().encode() + b"\0") # Exact bytes: each wheel ships its own host's module. @@ -470,7 +487,12 @@ def export(args): raise SystemExit(f"{args.api} lacks a staged API for one of {args.targets}") targets = [arg for target in args.targets for arg in ("--target", target)] withs = [arg for path in others for arg in ("--with", path)] - exporter(args.manifest_path, args.features, *targets, *withs, "--out", Path(args.out).resolve()) + out = Path(args.out).resolve() + if args.go_module: + # The export keeps a go.mod in go/, whose module its packages import each other by. + (out / "go").mkdir(parents=True, exist_ok=True) + (out / "go" / "go.mod").write_text(f"module {args.go_module}\n\ngo 1.24\n", newline="\n", encoding="utf-8") + exporter(args.manifest_path, args.features, *targets, *withs, "--out", out) def cargo(manifest, *args, env=None): @@ -479,14 +501,66 @@ def cargo(manifest, *args, env=None): capture_output=True, encoding="utf-8") +def staticlib(manifest, crate, lib, target, env=None): + """The C library alone, apart from the Python extension, whose CPython symbols it would + leave undefined, and the system libraries rustc says it needs beside it.""" + # Its own target directory, since cargo does not rebuild for a changed deployment target. + target_dir = crate.parent / ("target-swift" if target in APPLE_TARGETS else "target-go") + # cargo-zigbuild links Linux against the glibc floor's symbols, whichever glibc the host has. + linux = "linux" in target + # Uncolored, so the note below reads as written whatever the caller's terminal. + built = run(*(["cargo-zigbuild"] if linux else ["cargo"]), "rustc", "--color", "never", "--locked", "--release", + "--manifest-path", crate, "--target", f"{target}.{GLIBC}" if linux else target, + "--target-dir", target_dir, "--lib", "--crate-type", "staticlib", "--", "--print=native-static-libs", + cwd=Path(manifest).resolve().parent, env={**os.environ, **(env or {})}, + capture_output=True, encoding="utf-8") + sys.stderr.write(built.stderr) + notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) + if len(notes) != 1: + raise SystemExit(f"rustc reported no native libraries for {target}") + return target_dir / target / "release" / f"lib{lib}.a", notes[0].strip() + + +def go_links(export, lib, target): + """The system libraries the export's Go links beside `target`'s library, from its #cgo line.""" + platform = GO_TARGETS[target] + rt = (Path(export) / "go" / "rt" / "rt.go").read_text(encoding="utf-8") + match = re.search(rf"^#cgo \S+ LDFLAGS: \$\{{SRCDIR\}}/lib{lib}_{platform}\.a(.*)$", rt, re.M) + if match is None: + raise SystemExit(f"the export's Go links no library for {target}") + return match.group(1).strip() + + +def llvm_tool(manifest, name): + """One of LLVM's tools from the toolchain building the crate, which reads every target's objects.""" + cwd = Path(manifest).resolve().parent + run("rustup", "component", "add", "llvm-tools", cwd=cwd, capture_output=True) + sysroot = output("rustc", "--print", "sysroot", cwd=cwd).strip() + host = re.search(r"^host: (\S+)$", output("rustc", "-vV", cwd=cwd), re.M).group(1) + exe = ".exe" if sys.platform == "win32" else "" + return Path(sysroot) / "lib" / "rustlib" / host / "bin" / f"{name}{exe}" + + +def go_library(args, export_dir, lib, target, library, native, out): + """The Go module's copy of `target`'s library: checked against the export's #cgo line, + without debug info, which only bloats the module, or the bitcode the go command refuses.""" + want = go_links(export_dir, lib, target) + if native != want: + raise SystemExit(f"rustc links {target}'s library with `{native}`, but the export's Go with `{want}`") + objcopy = llvm_tool(args.manifest_path, "llvm-objcopy") + dest = out / "go" / f"lib{lib}_{GO_TARGETS[target]}.a" + dest.parent.mkdir(parents=True, exist_ok=True) + run(objcopy, "--strip-debug", *(f"--remove-section={s}" for s in LTO_SECTIONS), library, dest) + + def build(args): - """This host's wheels and Apple static libraries, the export's sources, and native output.""" + """This host's wheels, Apple and Go static libraries, the export's sources, and native output.""" info = facts(args.manifest_path) lib, export_dir, out = info["lib"], Path(args.export).resolve(), Path(args.out).resolve() crate = export_dir / "Cargo.toml" out.mkdir(parents=True, exist_ok=True) for target in args.targets: - if target not in WHEEL_TARGETS and target not in APPLE_TARGETS: + if target not in WHEEL_TARGETS and target not in APPLE_TARGETS and target not in GO_TARGETS: raise SystemExit(f"no package ships {target}") run("rustup", "target", "add", target, cwd=Path(args.manifest_path).resolve().parent) if target in WHEEL_TARGETS: @@ -503,30 +577,25 @@ def build(args): if target in APPLE_TARGETS: apple = APPLE_TARGETS[target] name = "MACOSX_DEPLOYMENT_TARGET" if apple == "macos" else "IPHONEOS_DEPLOYMENT_TARGET" - # The C library alone, apart from the Python extension, whose CPython symbols it - # would leave undefined; its own target directory, since cargo does not rebuild - # for a changed deployment target. - # Uncolored, so the note below reads as written whatever the caller's terminal. - built = cargo(args.manifest_path, "rustc", "--color", "never", "--locked", "--release", "--manifest-path", crate, - "--target", target, "--target-dir", export_dir / "target-swift", "--lib", - "--crate-type", "staticlib", "--", "--print=native-static-libs", - env={name: SWIFT_PLATFORMS[apple]}) - sys.stderr.write(built.stderr) - notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) - if len(notes) != 1: - raise SystemExit(f"rustc reported no native libraries for {target}") + library, native = staticlib(args.manifest_path, crate, lib, target, env={name: SWIFT_PLATFORMS[apple]}) static = out / "static" / target static.mkdir(parents=True) - shutil.copy(export_dir / "target-swift" / target / "release" / f"lib{lib}.a", static) - (static / "native.txt").write_text(notes[0].strip() + "\n", encoding="utf-8") - # The generated sources, for the Swift package and the hosts' agreement. + shutil.copy(library, static) + (static / "native.txt").write_text(native + "\n", encoding="utf-8") + # The Go module links the Swift package's macOS library, at its floor. + if target in GO_TARGETS: + go_library(args, export_dir, lib, target, library, native, out) + elif target in GO_TARGETS: + library, native = staticlib(args.manifest_path, crate, lib, target) + go_library(args, export_dir, lib, target, library, native, out) + # The generated sources, for the Swift package, the Go module, and the hosts' agreement. # Text this writes is LF on every host, as the jobs reading it compare bytes. - for part in ("include", "swift", "python"): + for part in ("include", "swift", "python", "go"): shutil.copytree(export_dir / part, out / "export" / part, dirs_exist_ok=True) shutil.copy(crate, out / "export") (out / "digest.txt").write_text(digest(export_dir, lib) + "\n", newline="\n", encoding="utf-8") (out / "lib.txt").write_text(lib + "\n", newline="\n", encoding="utf-8") - # Native output for a host whose wheel a consumer runs; an iOS slice runs nowhere here. + # Native output for a host whose wheel or Go module a consumer runs; an iOS slice runs nowhere here. if args.example and any(target in WHEEL_TARGETS for target in args.targets): if args.example not in info["examples"]: raise SystemExit(f"{info['name']} has no example {args.example}: it has {info['examples']}") @@ -538,13 +607,148 @@ def build(args): (out / "native.txt").write_bytes(native.stdout) -def template(info, manifest, rust2, mirror): +def go_version(version): + """Cargo's version as a Go module's: semver with a `v`, refusing build metadata, which Go has no spelling of.""" + if SEMVER.fullmatch(version) is None: + raise SystemExit(f"version {version} is no Go module version: use X.Y.Z or X.Y.Z-(alpha|beta|rc).N") + return f"v{version}" + + +def go_module_path(export): + """The module the export's go.mod declares, as the go command reads it.""" + path = Path(export) / "go" / "go.mod" + module = json.loads(output("go", "mod", "edit", "-json", path, env={**os.environ, "GOTOOLCHAIN": "local"})).get("Module") + if not module: + raise SystemExit(f"{path} declares no module") + return module["Path"] + + +def check_major(module, version): + """Fail unless the module path carries the suffix Go requires of its major version.""" + major = int(version[1:].split(".")[0]) + suffix = re.search(r"/v([0-9]+)$", module) + if major >= 2 and (suffix is None or int(suffix.group(1)) != major): + raise SystemExit(f"Go requires version {version} of {module} to be at a module path ending /v{major}") + if major < 2 and suffix is not None: + raise SystemExit(f"Go refuses version {version} of {module}, whose path names major version {suffix.group(1)}") + + +def escape(module): + """A module path as a proxy's URL spells it, each capital letter as `!` and its lower case.""" + return re.sub(r"[A-Z]", lambda m: "!" + m.group(0).lower(), module) + + +def go_files(info, export, libs): + """The module's files by path: the export's Go, each library its runtime links from the + directory `libs` beside it, and the license files, at the root where + pkg.go.dev looks for them.""" + root = Path(export) / "go" + files = {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()} + # Exactly the libraries the runtime's #cgo lines name. + want = set(re.findall(r"^#cgo \S+ LDFLAGS: \$\{SRCDIR\}/(\S+\.a)", files["rt/rt.go"].decode(), re.M)) + if want != {f"lib{info['lib']}_{platform}.a" for platform in GO_TARGETS.values()}: + raise SystemExit(f"the export's Go links {sorted(want)}, not each of {sorted(GO_TARGETS)}") + found = {p.name for p in Path(libs).iterdir()} + if found != want: + raise SystemExit(f"the Go module links {sorted(want)}, given {sorted(found)}") + for name in sorted(want): + files[f"rt/{name}"] = (Path(libs) / name).read_bytes() + for path in licenses(info): + files[path.name] = path.read_bytes() + # Two files a case-insensitive file system would make one, which the go command refuses. + folded = {} + for name in files: + if folded.setdefault(name.lower(), name) != name: + raise SystemExit(f"the Go module holds both {folded[name.lower()]} and {name}") + return files + + +def go_module(info, export, libs, out): + """The module's .info, .mod, and .zip, laid out as a GOPROXY serves them, under `out`.""" + module, version = go_module_path(export), go_version(info["version"]) + check_major(module, version) + files = go_files(info, export, libs) + if len(files["go.mod"]) > GO_MOD_MAX: + raise SystemExit(f"go.mod is {len(files['go.mod'])} bytes, over Go's {GO_MOD_MAX}") + size = sum(len(data) for data in files.values()) + if size > GO_ZIP_MAX: + raise SystemExit(f"the Go module's files are {size >> 20} MiB, over Go's {GO_ZIP_MAX >> 20} MiB") + versions = Path(out) / escape(module) / "@v" + versions.mkdir(parents=True) + (versions / f"{version}.info").write_text(json.dumps({"Version": version}) + "\n", encoding="utf-8") + (versions / f"{version}.mod").write_bytes(files["go.mod"]) + # What `go get ` lists to pick a version: this release's alone, which a proxy + # serving earlier ones appends to its own list. + (versions / "list").write_text(version + "\n", encoding="utf-8") + archive = versions / f"{version}.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z: + for name, data in files.items(): + # A fixed time, so the zip is a function of its files. + entry = zipfile.ZipInfo(f"{module}@{version}/{name}", date_time=(1980, 1, 1, 0, 0, 0)) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + z.writestr(entry, data) + if archive.stat().st_size > GO_ZIP_MAX: + raise SystemExit(f"the Go module's zip is {archive.stat().st_size >> 20} MiB, over Go's {GO_ZIP_MAX >> 20} MiB") + rows = [f"| `{name}` | {len(data) / (1 << 20):.1f} MiB |" for name, data in files.items() if name.endswith(".a")] + return "\n".join([ + f"Go module `{module}` {version}: {size / (1 << 20):.1f} MiB of files, a " + f"{archive.stat().st_size / (1 << 20):.1f} MiB zip, of Go's {GO_ZIP_MAX >> 20} MiB limit on each.", + "", "| Library | Size |", "| --- | --- |", *rows, "", + ]) + + +def zip_root(name): + """The module and version a file of a module zip, `@/`, is under.""" + match = re.match(r"([^@]+)@(v[^/]+)/", name) + if match is None: + raise SystemExit(f"{name} is in no module's zip") + return match.groups() + + +def consume_go(proxy, check, native): + """Build and run a check program, or else an import of the runtime, against the module + unpacked from its zip, in a new module of its own: no checkout, Rust, or CGO_LDFLAGS.""" + zips = list(Path(proxy).rglob("*.zip")) + if len(zips) != 1: + raise SystemExit(f"expected one module zip in {proxy}, found {zips}") + with tempfile.TemporaryDirectory() as work, zipfile.ZipFile(zips[0]) as z: + work = Path(work) + module, version = zip_root(z.namelist()[0]) + prefix = f"{module}@{version}" + z.extractall(work / "mod") + consumer = work / "consumer" + consumer.mkdir() + (consumer / "go.mod").write_text( + f"module consumer\n\ngo 1.24\n\nrequire {module} {version}\n\nreplace {module} => ../mod/{prefix}\n", + encoding="utf-8") + main = Path(check).read_text(encoding="utf-8") if check else f'package main\n\nimport _ "{module}/rt"\n\nfunc main() {{}}\n' + (consumer / "main.go").write_text(main, encoding="utf-8") + env = {k: v for k, v in os.environ.items() if not k.startswith("CGO_") or k == "CGO_ENABLED"} + env.update(CGO_ENABLED="1", GOFLAGS="-mod=mod", GOPROXY="off", GOTOOLCHAIN="local", GOWORK="off") + run("go", "vet", f"{module}/...", ".", cwd=consumer, env=env) + exe = consumer / ("check.exe" if sys.platform == "win32" else "check") + run("go", "build", "-o", exe, ".", cwd=consumer, env=env) + result = subprocess.run([exe], capture_output=True, cwd=work) + sys.stderr.buffer.write(result.stderr) + if result.returncode: + raise SystemExit(f"{check or 'the import'} failed on the Go module with status {result.returncode}") + if native and check: + compare(Path(native).read_bytes(), result.stdout, "Go module") + + +def template(info, manifest, rust2, mirror, go_module): """The caller workflow, with every identity Cargo cannot establish spelled out in it.""" if not re.fullmatch(r"[0-9a-f]{40}", rust2): raise SystemExit("pin rust2 by a full commit SHA") if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", mirror): raise SystemExit(f"the Swift mirror must be OWNER/REPO, not {mirror!r}") + # A proxy serves only a path whose first element is a host name. + if not re.fullmatch(r"[a-z0-9-]+(\.[a-z0-9-]+)+(/[A-Za-z0-9._~-]+)*", go_module): + raise SystemExit(f"the Go module must be a path a proxy serves, such as example.com/name, not {go_module!r}") + check_major(go_module, go_version(info["version"])) values = { + "GO_MODULE": go_module, "NAME": info["name"], "MANIFEST": manifest, "RUST2": rust2, @@ -570,7 +774,7 @@ def init(args): rust2 = args.rust2 or output("git", "-C", HERE, "rev-parse", "HEAD").strip() relative = manifest.resolve().relative_to(root).as_posix() path = root / ".github" / "workflows" / f"release-{info['name']}.yml" - text = template(info, relative, rust2, args.swift_mirror) + text = template(info, relative, rust2, args.swift_mirror, args.go_module) if path.exists() and path.read_text(encoding="utf-8") != text: raise SystemExit(f"{path} exists; delete it to write the release workflow") path.parent.mkdir(parents=True, exist_ok=True) @@ -593,6 +797,7 @@ def main(): p.add_argument("targets", nargs="+") if name == "export": p.add_argument("--api", required=True) + p.add_argument("--go-module", default="", help="the Go module path; the export's own, the prefix, by default") if name == "build": p.add_argument("--export", required=True) p.add_argument("--example", default="") @@ -610,6 +815,15 @@ def main(): p.add_argument("--state", required=True, help="the swift.json `swift` wrote") p.add_argument("--package", required=True) p.add_argument("--framework", required=True) + p = sub.add_parser("go", help="write the Go module as a GOPROXY serves it") + p.add_argument("--manifest-path", required=True) + p.add_argument("--export", required=True) + p.add_argument("--libs", required=True, help="a directory of every Go target's library, as build names them") + p.add_argument("--out", required=True) + p = sub.add_parser("consume-go", help="run a check program, or an import, on the unpacked Go module") + p.add_argument("--proxy", required=True, help="the directory `go` wrote") + p.add_argument("--check") + p.add_argument("--native") p = sub.add_parser("consume-python", help="run a check program, or an import, on the installed wheel") group = p.add_mutually_exclusive_group(required=True) group.add_argument("--check") @@ -622,8 +836,14 @@ def main(): p.add_argument("--manifest-path", required=True) p.add_argument("--swift-mirror", required=True, metavar="OWNER/REPO", help="the repository SwiftPM resolves the package from, whose root holds its Package.swift") + p.add_argument("--go-module", required=True, metavar="PATH", + help="the Go module's path, which a proxy you run serves, /vN suffix included") p.add_argument("--rust2", help="the rust2 commit to pin; this checkout's by default") args = parser.parse_args() + # winapi links MinGW import libraries of its own, from cargo's registry, where nothing + # linking the Go module's static library finds them: the system's serve every check and + # build, so the export's #cgo lines and rustc agree on them. + os.environ["WINAPI_NO_BUNDLED_LIBRARIES"] = "1" if args.command == "facts": info = facts(args.manifest_path) @@ -646,6 +866,10 @@ def main(): release_manifest(args.out, args.url) elif args.command == "swift-local": local_manifest(args.state, args.package, args.framework) + elif args.command == "go": + print(go_module(facts(args.manifest_path), args.export, args.libs, args.out)) + elif args.command == "consume-go": + consume_go(args.proxy, args.check, args.native) elif args.command == "consume-python": consume_python(args.check, args.native, args.module) elif args.command == "compare": diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 2c8c23bbd..a3f341bf3 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -11,6 +11,12 @@ name: Release @NAME@ bindings # The mirror's token comes from a GitHub App installed on it, whose ID and key live in # the `swift-mirror` environment. # +# The Go module @GO_MODULE@ is the build's @NAME@-go artifact, its .info, .mod, .zip, and list laid +# out as a GOPROXY serves them; no job here publishes it. Serve it from a module proxy: on a +# host you control, a go-import tag of type `mod` names it, but Go resolves a github.com path +# by git alone, so its consumers set GOPROXY to the proxy. A git mirror holds no library +# over GitHub's 100 MiB. +# # To release, push the tag @NAME@-v, naming the crate's version. Without a tag, as # from a nightly's workflow_call, every job up to publication runs. If publishing fails, use # "Re-run failed jobs": it reuses this run's build, and the publishing jobs upload only what @@ -34,8 +40,9 @@ jobs: uses: kixel-ai/rust2/.github/workflows/bindings.yml@@RUST2@ with: manifest-path: @MANIFEST@ + go-module: @GO_MODULE@ # Optional: cargo features to ship, an example of the crate whose output every check - # program must print, and those programs: `python-check` and `swift-check`. + # program must print, and those programs: `python-check`, `swift-check`, and `go-check`. tag: needs: build diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 95af04a4c..76df68b64 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -1,6 +1,7 @@ """The builder's decisions that need no toolchain: names, versions, metadata, and manifests.""" import importlib.util +import json from pathlib import Path import tempfile import unittest @@ -31,9 +32,22 @@ def export(root, floor="311"): (root / "include/moq.h").write_text("int moq_contract(void);\n") (root / "swift/moq.swift").write_text("import MoqFFI\n") (root / "Cargo.toml").write_text(f'pyo3-ffi = {{ features = ["abi3-py{floor}", "extension-module"] }}\n') + (root / "go/rt").mkdir(parents=True) + (root / "go/go.mod").write_text("module moq.dev/moq-ffi\n\ngo 1.24\n") + lines = "".join(f"#cgo {p.replace('_', ',')} LDFLAGS: ${{SRCDIR}}/libmoq_{p}.a -lgcc_s -lc\n" + for p in package.GO_TARGETS.values()) + (root / "go/rt/rt.go").write_text(f"package rt\n\n/*\n{lines}*/\nimport \"C\"\n") return root +def go_libs(root): + """A directory of every Go target's library, as `build` names them.""" + (root / "libs").mkdir() + for platform in package.GO_TARGETS.values(): + (root / "libs" / f"libmoq_{platform}.a").write_bytes(b"!\n") + return root / "libs" + + class Versions(unittest.TestCase): def test_release_and_prereleases(self): self.assertEqual(package.pep440("0.4.12"), "0.4.12") @@ -142,6 +156,84 @@ def test_any_byte_splits_the_hosts(self): # A Windows checkout once turned the embedded runtime CRLF. (second / "python/moq.py").write_bytes(b"PENDING = 0\r\n") self.assertNotEqual(package.digest(first, "moq"), package.digest(second, "moq")) + (second / "python/moq.py").write_bytes(b"PENDING = 0\n") + (second / "go/rt/rt.go").write_text("package rt\n") + self.assertNotEqual(package.digest(first, "moq"), package.digest(second, "moq")) + + +class Go(unittest.TestCase): + def test_versions_and_major_suffixes(self): + self.assertEqual(package.go_version("1.0.0-rc.1"), "v1.0.0-rc.1") + with self.assertRaises(SystemExit): + package.go_version("1.0.0+build") + package.check_major("moq.dev/moq-ffi", "v1.2.3") + package.check_major("moq.dev/moq-ffi/v2", "v2.0.0") + for module, version in [("moq.dev/moq-ffi", "v2.0.0"), ("moq.dev/moq-ffi/v3", "v2.0.0"), ("moq.dev/x/v2", "v1.0.0")]: + with self.assertRaises(SystemExit): + package.check_major(module, version) + + def test_zip_root_of_a_nested_module(self): + self.assertEqual(package.zip_root("example.com/api/v2@v2.0.0-rc.1/rt/rt.go"), ("example.com/api/v2", "v2.0.0-rc.1")) + with self.assertRaises(SystemExit): + package.zip_root("rt/rt.go") + + def test_module_path_as_go_reads_it(self): + with tempfile.TemporaryDirectory() as root: + go = Path(root) / "go" + go.mkdir() + for text, want in [("module example.com/own/v2 // a release\n", "example.com/own/v2"), + ('module "example.com/q"\n', "example.com/q"), + ("// module wrong\nmodule example.com/s\n\ngo 1.24\n", "example.com/s"), + ('module ( // the app\n\t"example.com/app"\n)\n', "example.com/app")]: + (go / "go.mod").write_text(text) + self.assertEqual(package.go_module_path(root), want) + + def test_escapes_capitals(self): + self.assertEqual(package.escape("github.com/Azure/x"), "github.com/!azure/x") + + def test_links_from_the_cgo_line(self): + with tempfile.TemporaryDirectory() as root: + root = export(Path(root)) + self.assertEqual(package.go_links(root, "moq", "x86_64-unknown-linux-gnu"), "-lgcc_s -lc") + (root / "go/rt/rt.go").write_text("package rt\n") + with self.assertRaises(SystemExit): + package.go_links(root, "moq", "x86_64-unknown-linux-gnu") + + def test_module_as_a_proxy_serves_it(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root) + built = export(root / "export") + (root / "LICENSE-MIT").write_text("MIT\n") + summary = package.go_module(info(root), built, go_libs(root), root / "proxy") + self.assertIn("`rt/libmoq_windows_amd64.a`", summary) + versions = root / "proxy/moq.dev/moq-ffi/@v" + self.assertEqual((versions / "v0.4.12.mod").read_text(), "module moq.dev/moq-ffi\n\ngo 1.24\n") + self.assertEqual(json.loads((versions / "v0.4.12.info").read_text()), {"Version": "v0.4.12"}) + self.assertEqual((versions / "list").read_text(), "v0.4.12\n") + with zipfile.ZipFile(versions / "v0.4.12.zip") as z: + self.assertEqual(sorted(z.namelist()), [f"moq.dev/moq-ffi@v0.4.12/{n}" for n in [ + "LICENSE-MIT", "go.mod", "rt/libmoq_darwin_arm64.a", "rt/libmoq_ios_arm64.a", + "rt/libmoq_linux_amd64.a", "rt/libmoq_linux_arm64.a", "rt/libmoq_windows_amd64.a", "rt/rt.go"]]) + # The module ships exactly the libraries the runtime links. + (root / "libs/libmoq_linux_arm64.a").unlink() + with self.assertRaises(SystemExit): + package.go_module(info(root), built, root / "libs", root / "other") + (root / "libs/libmoq_linux_arm64.a").write_bytes(b"!\n") + rt = built / "go/rt/rt.go" + rt.write_text(rt.read_text().replace("#cgo ios,arm64", "// none")) + with self.assertRaises(SystemExit): + package.go_module(info(root), built, root / "libs", root / "third") + + def test_refuses_names_a_case_folds_together(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root) + built = export(root / "export") + (built / "go/RT.go").write_text("package rt\n") + (built / "go/rt.go").write_text("package rt\n") + if len(list((built / "go").glob("[Rr][Tt].go"))) != 2: + self.skipTest("a case-insensitive file system holds one of the two names") + with self.assertRaises(SystemExit): + package.go_files(info(root), built, go_libs(root)) class Swift(unittest.TestCase): @@ -168,19 +260,23 @@ def test_manifest(self): class Template(unittest.TestCase): def test_fills_every_identity(self): - text = package.template(info("/src"), "rs/moq-ffi/Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi") + text = package.template(info("/src"), "rs/moq-ffi/Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi", "moq.dev/moq-ffi") self.assertIn("uses: kixel-ai/rust2/.github/workflows/bindings.yml@" + "a" * 40, text) self.assertIn('- "moq-ffi-v*"', text) self.assertIn("url: https://pypi.org/p/moq-ffi", text) self.assertIn("owner: moq-dev\n", text) self.assertIn("repositories: moq-swift-ffi\n", text) + self.assertIn("go-module: moq.dev/moq-ffi\n", text) self.assertNotRegex(text, r"@[A-Z_]+@") def test_refuses_a_movable_pin_or_bad_mirror(self): with self.assertRaises(SystemExit): - package.template(info("/src"), "Cargo.toml", "main", "moq-dev/moq-swift-ffi") + package.template(info("/src"), "Cargo.toml", "main", "moq-dev/moq-swift-ffi", "moq.dev/moq-ffi") with self.assertRaises(SystemExit): - package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-swift-ffi") + package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-swift-ffi", "moq.dev/moq-ffi") + for module in ["moq", "moq.dev/moq-ffi/", "moq.dev/moq-ffi/v2"]: + with self.assertRaises(SystemExit): + package.template(info("/src"), "Cargo.toml", "a" * 40, "moq-dev/moq-swift-ffi", module) if __name__ == "__main__": diff --git a/ffi/src/collect.rs b/ffi/src/collect.rs index b6713dc75..dd33beb6b 100644 --- a/ffi/src/collect.rs +++ b/ffi/src/collect.rs @@ -472,6 +472,7 @@ pub(crate) fn collect<'tcx>(tcx: TyCtxt<'tcx>, facade: &Facade<'_>) -> Collected crates: Vec::new(), reached: Vec::new(), targets: vec![target(tcx)], + links: [(target(tcx), links(tcx))].into(), }, }; c.walk(); @@ -6523,6 +6524,55 @@ fn target(tcx: TyCtxt<'_>) -> String { tcx.sess.opts.target_triple.tuple().to_string() } +/// The system libraries a static library of the facade links beside it on this target, as +/// rustc's `--print native-static-libs` prints them for one: what each linked crate's +/// `#[link]` and build script name that the archive does not bundle, the leaves last. +/// rustc lists only at a link, so this follows its `link_staticlib` over the checked crates. +fn links(tcx: TyCtxt<'_>) -> Vec { + use rustc_attr_parsing::eval_config_entry; + use rustc_structures::NativeLibKind; + let sess = tcx.sess; + // A dependent before what it depends on, `compiler_builtins` last; the facade, which the + // glue's build links as any other crate, depends on the rest. + let mut crates: Vec = (tcx.postorder_cnums(()).iter().rev().copied()) + .filter(|&krate| !tcx.crate_dep_kind(krate).macros_only()) + .collect(); + crates.sort_by_key(|&krate| tcx.is_compiler_builtins(krate)); + crates.insert(0, LOCAL_CRATE); + let mut out: Vec = Vec::new(); + let libs = crates.into_iter().flat_map(|krate| tcx.native_libraries(krate)); + for lib in libs.filter(|lib| (lib.cfg.as_ref()).is_none_or(|cfg| eval_config_entry(sess, cfg).as_bool())) { + let name = lib.name; + let verbatim = lib.verbatim.unwrap_or(false); + let arg = match lib.kind { + NativeLibKind::Static { + bundle: Some(false), .. + } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Unspecified => { + if sess.target.is_like_msvc { + let (prefix, suffix) = sess.staticlib_components(verbatim); + format!("{prefix}{name}{suffix}") + } else if sess.target.linker_flavor.is_gnu() { + format!("-l{}{name}", if verbatim { ":" } else { "" }) + } else { + format!("-l{name}") + } + } + NativeLibKind::Framework { .. } => format!("-framework {name}"), + NativeLibKind::Static { .. } + | NativeLibKind::LinkArg + | NativeLibKind::WasmImportModule + | NativeLibKind::RawDylib { .. } => continue, + }; + // rustc drops a library repeated right after itself, and only then. + if out.last() != Some(&arg) { + out.push(arg); + } + } + out +} + /// Whether every byte of `ty` is initialized, whatever its value and whatever layout a /// `rustc` picks at its size on this target: the bytes its fields always initialize fill /// it, so no field order leaves padding. The glue asserts the size, which is what lets diff --git a/ffi/src/driver.rs b/ffi/src/driver.rs index d452f34d4..04eb7ccd9 100644 --- a/ffi/src/driver.rs +++ b/ffi/src/driver.rs @@ -150,6 +150,28 @@ impl Export { } impl Callbacks for Export { + /// Load std, unnamed, into a `no_std` facade too: the glue links it whatever the facade + /// does, so the check that lists the system libraries a static library needs must see it. + fn after_crate_root_parsing(&mut self, compiler: &Compiler, krate: &mut ast::Crate) -> Compilation { + let source = "extern crate std as _;"; + let item = rustc_parse::new_parser_from_source_str( + &compiler.sess.psess, + FileName::anon_source_code(source), + source.to_string(), + StripTokens::Nothing, + ) + .ok() + .and_then(|mut parser| { + parser + .parse_item(ForceCollect::No, AllowConstBlockItems::DoesNotMatter) + .ok() + }) + .flatten() + .expect("an extern crate parses"); + krate.items.insert(0, item); + Compilation::Continue + } + fn after_analysis<'tcx>(&mut self, _compiler: &Compiler, tcx: TyCtxt<'tcx>) -> Compilation { if tcx.dcx().has_errors().is_some() { return Compilation::Stop; diff --git a/ffi/src/glue.rs b/ffi/src/glue.rs index 11896a261..800786caa 100644 --- a/ffi/src/glue.rs +++ b/ffi/src/glue.rs @@ -460,6 +460,8 @@ impl Crate<'_> { for (path, text) in go.files { output.file(&go_dir.join(path), &text); } + // The header the packages include, inside their module, as a module's zip holds it. + output.file(&go_dir.join("rt").join(format!("{}.h", self.abi.prefix)), &header); // Cargo names a path package's units by its name, version, and path in its // workspace, the same for every export of one facade, and judges them fresh by // mtimes, which an unchanged file keeps. So two exports in one target directory @@ -503,7 +505,7 @@ fn module_path(path: &Path) -> Result { .output() .map_err(|err| match err.kind() { std::io::ErrorKind::NotFound => format!( - "the go command is not on PATH: the export reads the module of {}, above --out, with `go mod edit -json`", + "the go command is not on PATH: the export reads the module of {} with `go mod edit -json`", path.display() ), _ => format!("running go mod edit -json {}: {err}", path.display()), @@ -544,10 +546,30 @@ fn resolve(path: &Path) -> Result { Ok(out) } -/// The import path of `/go`, from the `go.mod` of the module above it, or else the -/// module `prefix`, with the `go.mod` declaring it there. +/// The import path of `/go`, from the `go.mod` of the module it is in, as Go finds it: +/// its own, which the export keeps as it is, or one above it; or else the module `prefix`, +/// with the `go.mod` declaring it there. The `go.mod` to write in `go/` comes with it. fn go_module(out: &Path, prefix: &str) -> Result<(String, Option), String> { let out = resolve(out)?; + let default = format!("module {prefix}\n\ngo 1.24\n"); + let own = out.join("go").join("go.mod"); + if own.is_file() { + let text = std::fs::read_to_string(&own).map_err(|err| format!("reading {}: {err}", own.display()))?; + // One in the form the export writes, as the release builder writes it too, needs no + // go command to read. + let plain = (text.strip_prefix("module ")) + .and_then(|rest| rest.strip_suffix("\n\ngo 1.24\n")) + .filter(|path| { + !path.is_empty() + && !path.contains("//") + && path.chars().all(|c| c.is_ascii_alphanumeric() || "._~/-".contains(c)) + }); + let module = match plain { + Some(path) => path.to_string(), + None => module_path(&own)?, + }; + return Ok((module, Some(text))); + } for dir in out.ancestors() { let path = dir.join("go.mod"); if !path.is_file() { @@ -562,7 +584,7 @@ fn go_module(out: &Path, prefix: &str) -> Result<(String, Option), Strin .collect(); return Ok((format!("{module}/{}", rest.join("/")), None)); } - Ok((prefix.to_string(), Some(format!("module {prefix}\n\ngo 1.24\n")))) + Ok((prefix.to_string(), Some(default))) } struct Gen<'a> { diff --git a/ffi/src/go.rs b/ffi/src/go.rs index 7e3f68951..46bff3d93 100644 --- a/ffi/src/go.rs +++ b/ffi/src/go.rs @@ -1,12 +1,14 @@ //! Go over the C ABI through cgo: a package per crate whose items the facade exports, -//! `go//`, and the runtime they share, `go/rt/`, each including the C header from -//! `include/` beside `go/`. A crate's package is its name without the prefix and its `_` +//! `go//`, and the runtime they share, `go/rt/`, each including the C header the +//! runtime holds, so `go/` is a module of its own. The runtime links each exported target's +//! static library, `rt/lib__.a`, with the system libraries it needs, by +//! a `#cgo` line of that platform, and fails to compile on any other. A crate's package is its name without the prefix and its `_` //! (moq-mux's is `mux`), so callers write `mux.CodecOpusConfig_`. Cargo keeps crates from //! depending on each other in a cycle, so their packages import none in one either, as Go //! requires; Rust's modules may, and so cannot each be a package. An instance of a generic //! type, or a function naming one, belongs to the last of its own and its arguments' -//! crates, which can name them all. The import path of `go/` is the module of the `go.mod` -//! above it, or else the module `` of the `go.mod` the export writes there. +//! crates, which can name them all. The import path of `go/` is the module of its own +//! `go.mod` or one above it, or else the module `` of the `go.mod` the export writes there. //! //! Each name is its C name without the prefix and the segments outside its crate, in //! Pascal case, kept [`apart`](abi::apart) from another type's of the package that would @@ -186,7 +188,7 @@ pub(crate) fn generate(abi: &Abi<'_>, packages: &str, contract: u64, module: &st locked.insert(format!("crate {krate}"), name.to_string()); let doc = format!( "// Package {name} is the pub API of the crate {krate} in {packages}, over its C ABI\n\ - // through cgo. Link the library built with {p}.h, through CGO_LDFLAGS.\n", + // through cgo, whose runtime links the static library rt/lib{p}__.a.\n", p = abi.prefix ); files.push((format!("{name}/{name}.go"), g.file(&doc))); @@ -206,11 +208,63 @@ pub(crate) fn generate(abi: &Abi<'_>, packages: &str, contract: u64, module: &st params.join(", ") ); } - files.push(("rt/rt.go".to_string(), rt.runtime(packages, contract))); + let (links, platform) = link(abi)?; + files.push(("rt/rt.go".to_string(), rt.runtime(packages, contract, &links))); + files.push(("rt/platform.go".to_string(), platform)); files.push(("rt/types.go".to_string(), rt.file(""))); Ok(Output { files, names: locked }) } +/// Each target whose static library Go links: its `GOOS`, `GOARCH`, and the `GOOS` that +/// also satisfies its own as a build constraint, which is not it. Windows' is a `-gnu` +/// library, as Go links through MinGW; iOS's the device's, as Go tells no simulator apart. +const PLATFORMS: &[(&str, &str, &str, Option<&str>)] = &[ + ("x86_64-unknown-linux-gnu", "linux", "amd64", Some("android")), + ("aarch64-unknown-linux-gnu", "linux", "arm64", Some("android")), + ("x86_64-apple-darwin", "darwin", "amd64", Some("ios")), + ("aarch64-apple-darwin", "darwin", "arm64", Some("ios")), + ("aarch64-apple-ios", "ios", "arm64", None), + ("x86_64-linux-android", "android", "amd64", None), + ("aarch64-linux-android", "android", "arm64", None), + ("x86_64-pc-windows-gnu", "windows", "amd64", None), + ("aarch64-pc-windows-gnullvm", "windows", "arm64", None), +]; + +/// The `#cgo LDFLAGS` lines linking each exported target's library, `rt/lib__.a`, +/// and the system libraries it needs, and `rt/platform.go`, which fails to compile on any +/// other platform, so no build links without a library. +fn link(abi: &Abi<'_>) -> Result<(String, String), String> { + let p = &abi.prefix; + let mut lines = String::new(); + let mut constraints = Vec::new(); + for &(target, goos, goarch, not) in PLATFORMS { + if !abi.api.targets.iter().any(|t| t == target) { + continue; + } + let Some(libs) = abi.api.links.get(target) else { + return Err(format!( + "the staged API of {target} names no system libraries: stage it again with this rust2-ffi" + )); + }; + let not = not.map(|os| format!(",!{os}")).unwrap_or_default(); + let mut flags = vec![format!("${{SRCDIR}}/lib{p}_{goos}_{goarch}.a")]; + flags.extend(libs.iter().cloned()); + let _ = writeln!(lines, "#cgo {goos}{not},{goarch} LDFLAGS: {}", flags.join(" ")); + let not = not.replace(",!", " && !"); + constraints.push(format!("{goos}{not} && {goarch}")); + } + let only = match constraints.is_empty() { + true => String::new(), + false => format!("//go:build !({})\n\n", constraints.join(" || ")), + }; + let platform = format!( + "// Code generated by rust2-ffi. DO NOT EDIT.\n\n{only}package rt\n\n\ + // The export has no library for this platform, so nothing it declares would link.\n\ + var _ int = \"rust2-ffi exported no library of {p} for this GOOS and GOARCH\"\n" + ); + Ok((lines, platform)) +} + /// A call holding `object`'s handle as `h`, in the slot `s`, until it returns: through /// `Count`, inlined, then `Contend` once calls contend for it, running `failed` once closed. /// `slot` is the runtime's `Slot_`, as the package names it. @@ -546,7 +600,7 @@ impl<'a> Gen<'a> { let p = &self.abi.prefix; let _ = write!( out, - "/*\n#cgo CFLAGS: -I${{SRCDIR}}/../../include\n#include \n#include \"{p}.h\"\n\n" + "/*\n#cgo CFLAGS: -I${{SRCDIR}}/../rt\n#include \n#include \"{p}.h\"\n\n" ); for decl in &self.decls { out.push_str(decl); @@ -592,7 +646,7 @@ impl<'a> Gen<'a> { let mut out = format!( "// Code generated by rust2-ffi. DO NOT EDIT.\n\n\ package {name}\n\n\ - /*\n#cgo CFLAGS: -I${{SRCDIR}}/../../include\n#include \n#include \"{p}.h\"\n*/\nimport \"C\"\n\n\ + /*\n#cgo CFLAGS: -I${{SRCDIR}}/../rt\n#include \n#include \"{p}.h\"\n*/\nimport \"C\"\n\n\ import (\n\t\"context\"\n\t\"runtime\"\n\t\"runtime/cgo\"\n\t\"unicode/utf8\"\n\t\"unsafe\"\n\n\t\"{rt}\"\n)\n\n\ // The runtime's names for the export's C types and functions.\n\ type (\n\tcBuf = C.{p}_buf\n\tcWaker = C.{p}_waker\n\tcForeignFuture = C.{p}_foreign_future\n\tcForeignReader = C.{p}_foreign_reader\n\tcU128 = C.{p}_u128\n\tcDuration = C.{p}_duration\n\tcIpv4Addr = C.{p}_ipv4_addr\n\tcIpv6Addr = C.{p}_ipv6_addr\n\tcIpAddr = C.{p}_ip_addr\n\tcSocketAddrV4 = C.{p}_socket_addr_v4\n\tcSocketAddrV6 = C.{p}_socket_addr_v6\n\tcSocketAddr = C.{p}_socket_addr\n)\n\n\ @@ -606,7 +660,7 @@ impl<'a> Gen<'a> { } /// The runtime's package, which checks the library's contract once it loads. - fn runtime(&self, packages: &str, contract: u64) -> String { + fn runtime(&self, packages: &str, contract: u64, links: &str) -> String { let p = &self.abi.prefix; let upper = p.to_uppercase(); let go = format!("{p}Go"); @@ -615,7 +669,7 @@ impl<'a> Gen<'a> { // Package rt is what the Go packages of the pub API of {packages} share: the bytes\n\ // Rust returns, the error a closed object returns, and the types of no crate's.\n\ package rt\n\n\ - /*\n#cgo CFLAGS: -I${{SRCDIR}}/../../include\n#include \n#include \"{p}.h\"\n\n\ + /*\n#cgo CFLAGS: -I${{SRCDIR}}\n{links}#include \n#include \"{p}.h\"\n\n\ extern void {go}_call(void *data);\nextern void {go}_release(void *data);\nextern bool {go}_poll(void *data, {p}_waker waker);\n\ extern bool {go}_read(void *data, uint8_t *buf, size_t len, size_t *n, {p}_buf *err);\n\ extern bool {go}_seek(void *data, int whence, int64_t offset, uint64_t *position, {p}_buf *err);\n\n\ @@ -3425,6 +3479,7 @@ fn package_names_are_locked() { constants: vec![], crates: vec!["lock_foobar".into()], targets: vec!["host".into()], + links: Default::default(), reached: Vec::new(), }; let base = api.clone(); @@ -3443,3 +3498,48 @@ fn package_names_are_locked() { assert_eq!(breaks.len(), 1); assert!(breaks[0].what.contains("its Go name `foobar` is now `foobar_`")); } + +#[test] +fn links_each_target_go_builds_for() { + let targets = [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "wasm32-wasip1", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + ]; + let api = crate::ir::Api { + records: vec![], + enums: vec![], + objects: vec![], + callbacks: vec![], + functions: vec![], + constants: vec![], + crates: vec!["lock".into()], + targets: targets.map(String::from).to_vec(), + reached: Vec::new(), + links: (targets.iter()) + .filter(|t| **t != "wasm32-wasip1") + .map(|t| (t.to_string(), vec![format!("-l{}", &t[..3])])) + .collect(), + }; + let (lines, platform) = link(&Abi::new(&api, "lock")).unwrap(); + // iOS also satisfies `darwin`; MSVC's library is none Go links; wasm32 runs no cgo. + assert_eq!( + lines, + "#cgo darwin,!ios,arm64 LDFLAGS: ${SRCDIR}/liblock_darwin_arm64.a -laar\n\ + #cgo ios,arm64 LDFLAGS: ${SRCDIR}/liblock_ios_arm64.a -laar\n\ + #cgo windows,amd64 LDFLAGS: ${SRCDIR}/liblock_windows_amd64.a -lx86\n" + ); + assert!( + platform.contains("//go:build !(darwin && !ios && arm64 || ios && arm64 || windows && amd64)\n"), + "{platform}" + ); + let mut stale = api.clone(); + stale.links.clear(); + assert!( + link(&Abi::new(&stale, "lock")) + .unwrap_err() + .contains("aarch64-apple-darwin") + ); +} diff --git a/ffi/src/ir.rs b/ffi/src/ir.rs index aa822eda6..0c207e42d 100644 --- a/ffi/src/ir.rs +++ b/ffi/src/ir.rs @@ -23,6 +23,10 @@ pub(crate) struct Api { pub(crate) reached: Vec, /// The targets it serves, sorted: one for a single target's, several merged. pub(crate) targets: Vec, + /// By target, the system libraries a static library of the export links beside it, as + /// rustc's `--print native-static-libs` prints them. An API from before it holds none. + #[serde(default)] + pub(crate) links: std::collections::BTreeMap>, } /// A crate the glue depends on beside the facade, as the facade resolves it. diff --git a/ffi/src/lock.rs b/ffi/src/lock.rs index ea817a169..5c4dca59b 100644 --- a/ffi/src/lock.rs +++ b/ffi/src/lock.rs @@ -867,6 +867,7 @@ mod tests { constants: vec![], crates: vec!["lock_a".into(), "lock_b".into()], targets: vec!["host".into()], + links: Default::default(), reached: Vec::new(), } } diff --git a/ffi/src/merge.rs b/ffi/src/merge.rs index 4f084eb5f..57739ce9a 100644 --- a/ffi/src/merge.rs +++ b/ffi/src/merge.rs @@ -360,6 +360,7 @@ impl Merged { /// Add `api`'s items, of one target, to the merged API's. fn add(&mut self, mut api: Api) { + self.api.links.append(&mut api.links); let maps = Maps { records: Maps::of(&api.records, self.api.records.len(), |r| { (self.api.records.iter()).position(|m| m.shown == r.shown) @@ -821,6 +822,7 @@ mod tests { crates: ["a", "c"].map(String::from).to_vec(), reached: Vec::new(), targets: ["x", "y"].map(String::from).to_vec(), + links: Default::default(), }; let Err(cycle) = order_crates(&api) else { panic!("a cycle") @@ -853,6 +855,7 @@ mod tests { .into_iter() .collect(), targets: vec![target.into()], + links: Default::default(), }; let (_, differs) = super::merge(vec![api("x", true), api("y", false)]); assert_eq!(differs.len(), 1); diff --git a/ffi/tests/fixtures.rs b/ffi/tests/fixtures.rs index bbb48737d..32086635b 100644 --- a/ffi/tests/fixtures.rs +++ b/ffi/tests/fixtures.rs @@ -333,7 +333,7 @@ fn api_go_race() { &std::fs::read_to_string(manifest_dir().join("tests").join("api.go")).unwrap(), ); let exe = dir.join(format!("api-test{EXE_SUFFIX}")); - run(go(&module, api, &["build", "-race", "-o"]).arg(&exe)); + run(go(&module, &["build", "-race", "-o"]).arg(&exe)); // Every report fails the run. let env = [("GORACE", "halt_on_error=1")]; let output = run(Command::new(&exe).envs(env)); @@ -1794,7 +1794,7 @@ fn api_go() { reads: &[&module], writes: &[], }; - step.run(&mut go(root, api, &["vet", "./..."])); + step.run(&mut go(root, &["vet", "./..."])); } let exe = dir.join(format!("api-test{EXE_SUFFIX}")); go_build(api, &module, &exe); @@ -1858,13 +1858,10 @@ fn api_go() { } /// Lay out `module` as a Go module whose main package is `main`, requiring the export's -/// module of a package per crate, beside the header as the export lays them out. +/// module of a package per crate, with the static library its runtime links. fn go_layout(api: &Api, module: &Path, main: &str) { step::mirror(&api.out.join("go"), &module.join("export")); - step::copy( - &api.out.join("include").join("api.h"), - &module.join("include").join("api.h"), - ); + go_library(&api.staticlib(), &module.join("export")); step::write(&module.join("main.go"), main); step::write( &module.join("go.mod"), @@ -1879,7 +1876,7 @@ fn go_build(api: &Api, module: &Path, exe: &Path) { reads: &[module, &api.staticlib()], writes: &[exe], }; - step.run(go(module, api, &["build", "-o"]).arg(exe)); + step.run(go(module, &["build", "-o"]).arg(exe)); } /// Under a `go.mod`, the generated packages import each other by its module, as the go @@ -1907,6 +1904,23 @@ fn go_module() { ); let api = std::fs::read_to_string(out.join("go").join("api").join("api.go")).unwrap(); assert!(api.contains("\"example.com/app/sub/out/go/rt\""), "{api}"); + + // A go.mod in go/ itself makes it a module, as a release's is, which the export keeps + // as it is, above any other. + let own = module.join("own"); + let go_mod = "module example.com/own/v2 // a release\n\ngo 1.24\n"; + std::fs::create_dir_all(own.join("go")).unwrap(); + std::fs::write(own.join("go").join("go.mod"), go_mod).unwrap(); + let output = export("api", &own, &[], &[]); + assert!( + output.status.success(), + "exporting failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(std::fs::read_to_string(own.join("go").join("go.mod")).unwrap(), go_mod); + let api = std::fs::read_to_string(own.join("go").join("api").join("api.go")).unwrap(); + assert!(api.contains("\"example.com/own/v2/rt\""), "{api}"); + vet(&own.join("go")); } /// A crate named as the runtime, a Go keyword, or `main` is the Go package of that name and @@ -2214,18 +2228,54 @@ fn vet(go: &Path) { .env("GOTOOLCHAIN", "local")); } -/// The go command in `module`, linking cgo against the api fixture's static library. -fn go(module: &Path, api: &Api, args: &[&str]) -> Command { - let mut ldflags = api.staticlib().into_os_string(); - for lib in &api.native { - ldflags.push(" "); - ldflags.push(lib); +/// Put the static library `lib` in the Go module `export`'s runtime, as the file the +/// export's one `#cgo LDFLAGS` line links with what else it needs, without the LLVM bitcode +/// the go command refuses there. +fn go_library(lib: &Path, export: &Path) { + let rt = std::fs::read_to_string(export.join("rt").join("rt.go")).unwrap(); + let mut names = rt.lines().filter_map(|line| { + let flags = line.strip_prefix("#cgo ")?.split_once(" LDFLAGS: ${SRCDIR}/")?.1; + Some(flags.split(' ').next().unwrap().to_string()) + }); + let (Some(name), None) = (names.next(), names.next()) else { + panic!("the Go links no one library for the host: {rt}"); + }; + // Beside the module, so layouts that tests build at once strip apart. + let stripped = export.parent().unwrap().with_extension("a"); + let objcopy = llvm_objcopy(); + Step { + stamp: &step::stamp(&stripped), + reads: &[lib, objcopy], + writes: &[&stripped], } + .run( + Command::new(objcopy) + .args(["--remove-section=.llvmbc", "--remove-section=.llvmcmd"]) + .args(["--remove-section=__LLVM,__bitcode", "--remove-section=__LLVM,__cmdline"]) + .arg(lib) + .arg(&stripped), + ); + step::copy(&stripped, &export.join("rt").join(name)); +} + +/// The nightly's llvm-objcopy, which reads every target's objects. +fn llvm_objcopy() -> &'static Path { + static OBJCOPY: OnceLock = OnceLock::new(); + OBJCOPY.get_or_init(|| { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let sysroot = run(Command::new(rustc).args(["--print", "sysroot"])); + let sysroot = PathBuf::from(String::from_utf8_lossy(&sysroot.stdout).trim()); + let bin = sysroot.join("lib").join("rustlib").join(host()).join("bin"); + bin.join(format!("llvm-objcopy{EXE_SUFFIX}")) + }) +} + +/// The go command in `module`, whose runtime links the api fixture's static library. +fn go(module: &Path, args: &[&str]) -> Command { let mut cmd = Command::new("go"); cmd.args(args) .current_dir(module) .env("CGO_ENABLED", "1") - .env("CGO_LDFLAGS", ldflags) // The module is no checkout of its own, and must build with the Go on PATH. .env("GOFLAGS", "-buildvcs=false") .env("GOTOOLCHAIN", "local"); @@ -2877,6 +2927,40 @@ fn package(name: &str, files: &[(&str, &str)]) -> PathBuf { package } +/// A `no_std` facade's API still names the system libraries std links, as the glue's +/// static library links std whatever the facade does: those rustc prints for the api +/// fixture's, whose crates link nothing of their own. +#[test] +fn no_std_links() { + let package = package( + "no_std_links", + &[( + "src/lib.rs", + "#![cfg_attr(not(test), no_std)]\n\npub fn one() -> u32 {\n\t1\n}\n", + )], + ); + let out = package.join("out"); + let output = Command::new(env!("CARGO_BIN_EXE_rust2-ffi")) + .arg("--config") + .arg(package.join("ffi.toml")) + .arg("--manifest-path") + .arg(package.join("Cargo.toml")) + .arg("--out") + .arg(&out) + .env("CARGO_TARGET_DIR", package.join("build")) + .output() + .unwrap(); + assert!( + output.status.success(), + "exporting failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + let exported: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(out.join("api.json")).unwrap()).unwrap(); + let links: Vec = serde_json::from_value(exported["api"]["links"][host()].clone()).unwrap(); + assert_eq!(links, api().native); +} + /// rustc's arguments in an argfile, as cargo passes them once they overflow a command line: /// on Windows, moq-ffi's many `-L`s do. One argument past every platform's limit forces it, /// and starts with `@`, which inside an argfile rustc reads as itself, not another file. diff --git a/ffi/tests/fixtures/shapes.rs b/ffi/tests/fixtures/shapes.rs index 20be278ef..c595bc9df 100644 --- a/ffi/tests/fixtures/shapes.rs +++ b/ffi/tests/fixtures/shapes.rs @@ -1536,11 +1536,7 @@ fn round_trip(built: &Built, language: &str) { let module = dir.join("module"); let export = module.join("export"); step::mirror(&built.out.join("go"), &export); - let header = format!("{name}.h"); - step::copy( - &built.out.join("include").join(&header), - &module.join("include").join(&header), - ); + super::go_library(&built.staticlib(), &export); let source = std::fs::read_to_string(export.join(&name).join(format!("{name}.go"))).unwrap(); let rt = std::fs::read_to_string(export.join("rt").join("types.go")).unwrap_or_default(); step::write(&module.join("main.go"), go(built, calls, &source, &rt)); @@ -1548,11 +1544,6 @@ fn round_trip(built: &Built, language: &str) { &module.join("go.mod"), format!("module shapestest\n\ngo 1.24\n\nrequire {name} v0.0.0\n\nreplace {name} => ./export\n"), ); - let mut ldflags = built.staticlib().into_os_string(); - for lib in &built.native { - ldflags.push(" "); - ldflags.push(lib); - } let step = Step { stamp: &step::stamp(&exe), reads: &[&module, &built.staticlib()], @@ -1564,7 +1555,6 @@ fn round_trip(built: &Built, language: &str) { .arg(&exe) .current_dir(&module) .env("CGO_ENABLED", "1") - .env("CGO_LDFLAGS", ldflags) .env("GOFLAGS", "-buildvcs=false") .env("GOTOOLCHAIN", "local"), ); diff --git a/quest/m1/ffi-packaging/go.md b/quest/m1/ffi-packaging/go.md deleted file mode 100644 index 27b72ca89..000000000 --- a/quest/m1/ffi-packaging/go.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] A Go module that go get builds - -## Goal - -The generated Go package links its prebuilt static library with no `CGO_LDFLAGS`: the -export writes a `#cgo , LDFLAGS: ${SRCDIR}/...` line per target it exports, -with the system libraries rustc prints, and the builder releases the module with each -target's library, so `go get` and `go build` work for every target moq ships. CI builds -the Go test on Linux, macOS, and Windows (with a `-gnu` static library for Go's mingw -linker). - -## Plan - -Found by the 2026-11-18 language audit: the package comment says it links only through -`CGO_LDFLAGS` (`ffi/src/go.rs:188-189`); moq works around it with a hand-written -per-platform `cgo.go`, as [wasmtime-go](https://github.com/bytecodealliance/wasmtime-go/blob/main/ffi.go) -does. Go module zips are capped at 500 MiB, and the stand-in's `libmoq.a` is 59 to 104 MiB -stripped per target: report the module's size, and recommend a layout (one module, or one -per target) that stays under the cap with room to grow. - -Public API: the generated package's cgo directives, and the module layout.