diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index 8fba1700d..ef438281e 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -18,6 +18,8 @@ jobs: features: doubled example: transcript python-check: ffi/tests/api.py + # The small crate, so the second build costs little: moq's stand-in leaves it to its release. + reproduce: true # api.cpp rather than api.c, whose broken cases name items only wasm32 has. c-check: ffi/tests/api.cpp web-manifest-path: ffi/fixtures/api/Cargo.toml diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 09619381f..b97772eef 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -28,6 +28,10 @@ on: description: The main.swift of an executable run on the installed Swift package default: '' type: string + reproduce: + description: Build every host's packages a second time, from a new export, and fail unless the bytes match + default: false + type: boolean c-check: description: A C or C++ program built by clean CMake and pkg-config consumers of each C archive default: '' @@ -327,6 +331,20 @@ jobs: run: | read -ra targets <<< "$TARGETS" python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export export --out dist --example "$EXAMPLE" "${targets[@]}" + # The build again, as a rerun makes it: the same packages byte for byte, from a new + # export at the same path, since dlltool names the Windows GNU library's import symbols + # after the target directory. It doubles the job's time. + - name: Rebuild + if: inputs.reproduce + env: + TARGETS: ${{ matrix.targets }} + GO_MODULE: ${{ inputs.go-module }} + run: | + read -ra targets <<< "$TARGETS" + mv export first-export + python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --go-module "$GO_MODULE" --out export "${targets[@]}" + python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export export --out again/dist "${targets[@]}" + python3 -I rust2/ffi/release/package.py same dist again/dist - name: Checks env: PYTHON_CHECK: ${{ inputs.python-check }} @@ -347,7 +365,8 @@ jobs: if-no-files-found: error # The hosts' exports must be one binding: a module and a library from different builds - # refuse to load, but two wheels of different bindings would each load. + # refuse to load, but two wheels of different bindings would each load. Their SBOMs, of + # the one dependency graph, must match as well. agree: needs: [facts, build] runs-on: ubuntu-24.04 @@ -364,6 +383,10 @@ jobs: echo '::error::The build hosts exported different bindings.' exit 1 fi + if [[ $(sha256sum builds/*/sbom.cdx.json | cut -d' ' -f1 | sort -u | wc -l) != 1 ]]; then + echo '::error::The build hosts listed different dependencies.' + exit 1 + fi # The release's API, which the next release is checked against, and its changelog from # the previous release's: what the caller's release publishes beside the packages. @@ -627,6 +650,7 @@ jobs: WEB_FEATURES: ${{ inputs.web-features }} WEB_EXAMPLE: ${{ inputs.web-example }} WEB_CHECK: ${{ inputs.web-check }} + REPRODUCE: ${{ inputs.reproduce }} run: | if [[ "$(realpath -m "source/$WEB_MANIFEST")" == "$(realpath -m "source/$MANIFEST")" ]]; then if [[ -n "$WEB_FEATURES$WEB_EXAMPLE" ]]; then @@ -639,6 +663,11 @@ jobs: fi python3 -I rust2/ffi/release/package.py web --manifest-path "source/$WEB_MANIFEST" --features "$features" --scope "$SCOPE" --version "$VERSION" --example "$example" --out web if [[ -n "$WEB_CHECK" ]]; then cp "source/$WEB_CHECK" web/check.ts; fi + # The build again, from a new export, which must pack the same tarball. + if [[ "$REPRODUCE" == true ]]; then + python3 -I rust2/ffi/release/package.py web --manifest-path "source/$WEB_MANIFEST" --features "$features" --scope "$SCOPE" --version "$VERSION" --example "" --out web-again + python3 -I rust2/ffi/release/package.py same web web-again + fi - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ needs.facts.outputs.name }}-web diff --git a/ffi/release/README.md b/ffi/release/README.md index ec20054b4..b67ddf78f 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -48,16 +48,15 @@ Releasing is pushing `-v`, the crate's version, which the releas Called without a tag, such as from a nightly, it runs everything up to publication. A release that failed while publishing is recovered with **Re-run failed jobs**, which keeps the run's successful build and its artifacts: each publishing job then finishes what the -first attempt left, uploading only what is missing. A rebuild makes other bytes, as wheels -and the XCFramework archive carry build times, so **Re-run all jobs** fails wherever a -registry already holds the first attempt's files. +first attempt left, uploading only what is missing. A rebuild makes the same bytes +([Reproducible builds](#reproducible-builds)), so **Re-run all jobs** works as well. A Cargo prerelease, such as `1.0.0-rc.1`, is a GitHub prerelease too. The builder takes the crate's `Cargo.toml`, the `go-module` path, which without it is the export's own `` that no proxy serves, and optional inputs the caller's tests own: `features` to ship, an `example` whose stdout is native Rust's transcript, and the `python-check`, `swift-check`, `c-check`, and `go-check` programs that must print it through -the installed packages. Without a check, a consumer only imports the module, or links a +the installed packages, and `reproduce`, which builds everything twice. Without a check, a consumer only imports the module, or links a function of each C and C++ header. A `c-check` is C or C++ by its extension. The web package takes `web-manifest-path`, the package exported for `wasm32-wasip1`, often the crate's own, which no Cargo fact says builds for the web, and `scope`; and an optional `web-check`, a @@ -91,7 +90,7 @@ crate apart from it takes `web-features` and `web-example` instead. `win_amd64`. 6. Swift: static libraries for macOS arm64 at macOS 13, and iOS arm64 and its simulator at iOS 16, the floors of the generated Swift's `Swift.Duration`, each checked against - every object's declared minimum OS; one XCFramework of the three, zipped with `ditto`, + every object's declared minimum OS; one XCFramework of the three, zipped by the builder itself, its SHA-256 checked against `swift package compute-checksum`; and a `Package.swift` naming it at `https://github.com//releases/download/-v/FFI.xcframework.zip`. The package depends on swift-collections by the release tag at the exporter's pinned @@ -175,6 +174,44 @@ wheels, C++ consumers, and web package must print its native transcript, and on stand-in facade at the exporter's moq pin, through `moq/moq.py`, `moq/moq.swift`, `moq/moq.cpp`, and `moq/moq.go`, with its wasm32 stand-in as the web package. +## Reproducible builds + +A rebuild of a commit makes the same bytes, so **Re-run all jobs** finds each registry +holding exactly the files it would upload, and anyone with the commit can check them. + +- **Time.** Every file takes `SOURCE_DATE_EPOCH`, or else the commit's time, whatever its + archive format: the builder writes each ZIP itself, with sorted entries, fixed modes, and + no host, umask, or time of its own, and rewrites each repaired wheel the same way, its + RECORD rebuilt. Zlib's own output is the one thing a host could still change. +- **Paths.** The crate's workspace, the exported crate, and `CARGO_HOME` are remapped to + fixed names for rustc and the C compilers, so panic messages and debug info name no + checkout. This takes `RUSTFLAGS` over a `.cargo/config.toml`'s `rustflags`, so a config + that sets any fails the build: export them as `RUSTFLAGS`, which the builder keeps. Apple + builds also stamp no archive time (`ZERO_AR_DATE`), and Windows links with `/Brepro`. +- **Checked.** The Swift job builds the XCFramework twice and fails unless both ZIPs match. + With `reproduce: true`, every build host builds a second time, from a new export at the same + path, and `package.py same` fails unless each wheel, static library, and C archive, the Go + module's included, matches byte for byte, and the web job packs its npm tarball twice; rust2 turns it on + for its api fixture. It doubles the build, so a large crate may leave it to a nightly. +- **Not yet.** The exporter writes the crate's absolute path into the generated crate's + version, which cargo hashes into every symbol, so a rebuild must keep the crate at the + same path, as a rerun does + ([planned](/quest/m1/ffi-packaging/relocatable.md)). So must the export directory of a + Windows GNU library, which is the Go module's: dlltool names the import library's symbols + after the path of the target directory, which no `--remap-path-prefix` reaches. rustc's + random temporary directory in that name is replaced by zeros. + +**Attestations and SBOM.** `package.py build` writes `sbom.cdx.json`, a CycloneDX SBOM of +the crates the binding links (normal dependencies, platform-neutral, with each registry +crate's checksum from `Cargo.lock`), which every wheel also carries in +`.dist-info/sboms/` (PEP 770) and the build hosts must agree on. The caller's `attest` job +runs on a tag, before anything is published, and gives every wheel, the XCFramework, the Go +module's `.zip` and `.mod`, each C archive, and the npm tarball a build provenance +attestation and an SBOM attestation (the tarball's from its own `sbom.cdx.json`), which +`gh attestation verify --repo /` checks. Only that job holds `id-token` and `attestations` +write; the builder stays read-only. PyPI's trusted publishing uploads its own PEP 740 +attestations for the wheels as well. + ## Other registries The same boundary holds for each destination: the builder makes and checks the artifact diff --git a/ffi/release/package.py b/ffi/release/package.py index 6bd602175..ed9a7604b 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -9,11 +9,13 @@ import base64 import csv import difflib +import gzip import hashlib import io import json import os from pathlib import Path +import plistlib import re import shlex import shutil @@ -21,7 +23,10 @@ import sys import tarfile import tempfile +import time import tomllib +import urllib.parse +import uuid import zipfile HERE = Path(__file__).resolve().parent @@ -63,7 +68,10 @@ LTO_SECTIONS = [".llvmbc", ".llvmcmd", "__LLVM,__bitcode", "__LLVM,__cmdline"] # Cargo metadata names a library target by its crate types; a proc-macro is no library to export. LIBRARY_KINDS = {"lib", "rlib", "dylib", "cdylib", "staticlib"} +CRATES_IO = "registry+https://github.com/rust-lang/crates.io-index" COLLECTIONS = "https://github.com/apple/swift-collections" +# The first date a ZIP entry can hold, and so the earliest SOURCE_DATE_EPOCH a build takes. +ZIP_EPOCH = 315532800 # The target the web binding serves, whose module the npm and JSR packages hold. WEB_TARGET = "wasm32-wasip1" # JSR's cap on a version's files, unpacked. @@ -110,6 +118,14 @@ def library(package): return libs[0]["name"].replace("-", "_") +def epoch(directory): + """The build time every file of this release carries: SOURCE_DATE_EPOCH, else the commit's.""" + raw = os.environ.get("SOURCE_DATE_EPOCH") or output("git", "-C", directory, "log", "-1", "--format=%ct").strip() + if not raw.isdigit() or int(raw) < ZIP_EPOCH: + raise SystemExit(f"SOURCE_DATE_EPOCH {raw!r} is no time since 1980: set it, or build from a commit") + return int(raw) + + def facts(manifest): """The package at `manifest`, as Cargo metadata describes it.""" meta = json.loads(output("cargo", "metadata", "--format-version", "1", "--no-deps", @@ -129,6 +145,7 @@ def facts(manifest): "lib": library(package), "dir": str(path.parent), "workspace": meta["workspace_root"], + "epoch": epoch(path.parent), "examples": sorted(t["name"] for t in package["targets"] if "example" in t["kind"]), } @@ -185,14 +202,51 @@ def metadata(info, floor): return "\n".join(lines) + "\n\n" -def record(files): - """RECORD: every file's hash and size, and RECORD's own row unhashed.""" - out = io.StringIO(newline="") - writer = csv.writer(out, lineterminator="\n") - for name, data in files.items(): - digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() - writer.writerow([name, f"sha256={digest}", len(data)]) - return out +def write_zip(path, entries, stamp): + """A ZIP of `(name, data, permissions)` entries in the order given, the same bytes whoever writes it and whenever. + + Every entry takes `stamp`, a name ending in `/` is a directory, and nothing else of the + writer, such as its host or umask, is recorded. + """ + date = time.gmtime(stamp)[:6] + with zipfile.ZipFile(path, "w") as archive: + for name, data, mode in entries: + entry = zipfile.ZipInfo(name, date) + entry.create_system = 3 + if name.endswith("/"): + entry.external_attr = (0o040000 | mode) << 16 | 0x10 + else: + entry.external_attr = (0o100000 | mode) << 16 + entry.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(entry, data, compresslevel=9) + + +def seal(path, files, stamp): + """Write `files`, `{name: (data, permissions)}`, as a wheel: sorted with the dist-info last, and its RECORD hashing the rest.""" + meta = {name.split("/")[0] for name in files if name.split("/")[0].endswith(".dist-info")} + if len(meta) != 1: + raise SystemExit(f"{path} needs one .dist-info directory, has {sorted(meta)}") + record = f"{meta.pop()}/RECORD" + entries = sorted(files, key=lambda name: (name.split("/")[0].endswith(".dist-info"), name)) + rows = io.StringIO(newline="") + writer = csv.writer(rows, lineterminator="\n") + for name in entries: + digest = base64.urlsafe_b64encode(hashlib.sha256(files[name][0]).digest()).rstrip(b"=").decode() + writer.writerow([name, f"sha256={digest}", len(files[name][0])]) + writer.writerow([record, "", ""]) + write_zip(path, [(name, *files[name]) for name in entries] + [(record, rows.getvalue().encode(), 0o644)], stamp) + + +def normalize_wheel(path, stamp): + """Rewrite a repaired wheel as `seal` writes one, whatever order, times, and modes its tool left.""" + with zipfile.ZipFile(path) as archive: + files = { + entry.filename: (archive.read(entry), 0o755 if entry.external_attr >> 16 & 0o111 else 0o644) + for entry in archive.infolist() + # Repair tools add directory entries, which RECORD never lists, and rewrite RECORD itself. + if not entry.is_dir() and not entry.filename.endswith(".dist-info/RECORD") + } + seal(path, files, stamp) def wheel_platform(target): @@ -225,26 +279,22 @@ def write_wheel(info, export, library, target, out): tag = f"cp3{floor}-abi3-{wheel_platform(target)}" meta = f"{dist}-{version}.dist-info" files = { - f"{lib}/__init__.py": (Path(export) / "python" / f"{lib}.py").read_bytes(), - f"{lib}/{library_name(lib, target)}": Path(library).read_bytes(), + f"{lib}/__init__.py": ((Path(export) / "python" / f"{lib}.py").read_bytes(), 0o644), + f"{lib}/{library_name(lib, target)}": (Path(library).read_bytes(), 0o755), } for path in licenses(info): - files[f"{meta}/licenses/{path.name}"] = path.read_bytes() - files[f"{meta}/METADATA"] = metadata(info, floor).encode() + files[f"{meta}/licenses/{path.name}"] = (path.read_bytes(), 0o644) + # PEP 770: the dependencies the library was built from, where a scanner looks. + files[f"{meta}/sboms/{info['name']}.cdx.json"] = (info["sbom"], 0o644) + files[f"{meta}/METADATA"] = (metadata(info, floor).encode(), 0o644) files[f"{meta}/WHEEL"] = ( - f"Wheel-Version: 1.0\nGenerator: rust2-ffi\nRoot-Is-Purelib: false\nTag: {tag}\n" - ).encode() - rows = record(files) - csv.writer(rows, lineterminator="\n").writerow([f"{meta}/RECORD", "", ""]) - files[f"{meta}/RECORD"] = rows.getvalue().encode() + f"Wheel-Version: 1.0\nGenerator: rust2-ffi\nRoot-Is-Purelib: false\nTag: {tag}\n".encode(), 0o644) path = Path(out) / f"{dist}-{version}-{tag}.whl" - with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: - for name, data in files.items(): - archive.writestr(name, data) + seal(path, files, info["epoch"]) return path -def repair(wheel, target, out): +def repair(wheel, target, out, stamp): """Hand the wheel to its platform's repair tool, which claims the portable tag or fails.""" Path(out).mkdir(parents=True, exist_ok=True) if "linux" in target: @@ -259,6 +309,7 @@ def repair(wheel, target, out): repaired = list(Path(out).glob("*.whl")) if len(repaired) != 1: raise SystemExit(f"repair wrote {len(repaired)} wheels into {out}") + normalize_wheel(repaired[0], stamp) check_wheel(repaired[0]) return repaired[0] @@ -377,6 +428,29 @@ def min_os(archive): return max(versions) +def sort_plist(path): + """Rewrite an XCFramework's Info.plist with its libraries in name order, which xcodebuild lists in a random one.""" + with open(path, "rb") as source: + plist = plistlib.load(source) + plist["AvailableLibraries"].sort(key=lambda library: library["LibraryIdentifier"]) + with open(path, "wb") as sink: + plistlib.dump(plist, sink, sort_keys=True) + + +def zip_tree(root, path, stamp): + """The directory `root` as a ZIP holding it as its top-level directory, as `ditto --keepParent` does.""" + entries = [(f"{root.name}/", b"", 0o755)] + for found in sorted(root.rglob("*"), key=lambda p: p.relative_to(root).as_posix()): + name = f"{root.name}/{found.relative_to(root).as_posix()}" + if found.is_symlink(): + raise SystemExit(f"{found} is a symlink: the XCFramework ZIP holds only files") + if found.is_dir(): + entries.append((f"{name}/", b"", 0o755)) + else: + entries.append((name, found.read_bytes(), 0o755 if found.stat().st_mode & 0o111 else 0o644)) + write_zip(path, entries, stamp) + + def swift(info, export, libs, natives, out): """The XCFramework of every Apple slice, its checksummed ZIP, and the package around it.""" if sys.platform != "darwin": @@ -400,13 +474,23 @@ def swift(info, export, libs, natives, out): raise SystemExit("the module map no longer names ../include: update the packager") (headers / "module.modulemap").write_text(modulemap.replace(f"../include/{lib}.h", f"{lib}.h"), encoding="utf-8") framework = out / f"{module}FFI.xcframework" - args = ["xcodebuild", "-create-xcframework"] - for target in APPLE_TARGETS: - args += ["-library", libs[target], "-headers", headers] - run(*args, "-output", framework) - shutil.rmtree(headers) archive = out / f"{module}FFI.xcframework.zip" - run("/usr/bin/ditto", "-c", "-k", "--sequesterRsrc", "--keepParent", framework, archive) + again = out / "again" + # The XCFramework twice, which must zip to the same bytes: the checksum a consumer + # resolves is of this ZIP, and a rebuild must make it again. + for tree, zipped in ((framework, archive), (again / framework.name, again / archive.name)): + args = ["xcodebuild", "-create-xcframework"] + for target in APPLE_TARGETS: + args += ["-library", libs[target], "-headers", headers] + # libtool, which xcodebuild runs on each library, stamps the time unless told not to. + run(*args, "-output", tree, env={**os.environ, "ZERO_AR_DATE": "1"}) + sort_plist(tree / "Info.plist") + zip_tree(tree, zipped, info["epoch"]) + if archive.read_bytes() != (again / archive.name).read_bytes(): + raise SystemExit("xcodebuild made different XCFrameworks from the same libraries:\n" + + "\n".join(zip_diff(archive, again / archive.name))) + shutil.rmtree(headers) + shutil.rmtree(again) checksum = hashlib.sha256(archive.read_bytes()).hexdigest() # SwiftPM's own computation, as a consumer resolving the URL runs it. swift_sum = output("swift", "package", "compute-checksum", archive).strip() @@ -735,7 +819,8 @@ def hide(static, shared, lib, target, work): arch = ["-arch", "arm64", f"-mmacosx-version-min={MACOS_WHEEL}"] run("cc", "-r", "-nostdlib", *arch, *(f"-Wl,{arg}" for arg in roots), f"-Wl,-exported_symbols_list,{names}", define_commons(static, arch, work), static, "-o", hidden) - run("libtool", "-static", "-o", archive, hidden) + # Which stamps the member with the time unless told not to. + run("libtool", "-static", "-o", archive, hidden, env={**os.environ, "ZERO_AR_DATE": "1"}) # Rust's objects embed LLVM bitcode newer than Xcode's nm reads. found = output("nm", "--no-llvm-bc", "-gUj", hidden).split() else: @@ -819,7 +904,7 @@ def c_archive(info, manifest, export_dir, target, out): elif apple: # The last -install_name rustc's linker sees wins over its own unversioned one. link_args = ["-C", f"link-arg=-Wl,-install_name,@rpath/{soname}"] - env = {"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if apple else {} + env = {**reproducible(info, export_dir, target, soname), **({"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if apple else {})} # The C library alone, as the Swift package's: the glue's language features add symbols # a C program cannot resolve. Its own target directory, since its link arguments differ. # cargo-zigbuild's own `rustc`, as Cargo knows only its `zigbuild`, from the package's @@ -868,26 +953,32 @@ def c_archive(info, manifest, export_dir, target, out): (root / "lib/cps").mkdir(parents=True) (root / f"lib/cps/{lib}.cps").write_text(cps(info, files, natives), newline="\n", encoding="utf-8") (out / "c").mkdir(parents=True, exist_ok=True) - return pack(root, out / "c" / (name + (".zip" if "windows" in target else ".tar.gz"))) + return pack(root, out / "c" / (name + (".zip" if "windows" in target else ".tar.gz")), info["epoch"]) -def pack(root, path): - """`root` as a ZIP or gzipped tarball under its own name, owned by no one, readable by all.""" +def pack(root, path, stamp): + """`root` as a ZIP or gzipped tarball under its own name, owned by no one, readable by all, stamped `stamp`, and the same bytes whoever packs it.""" + found = sorted(root.rglob("*"), key=lambda p: p.relative_to(root.parent).as_posix()) if path.suffix == ".zip": - with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: - for file in sorted(p for p in root.rglob("*") if p.is_file()): - archive.write(file, file.relative_to(root.parent).as_posix()) + write_zip(path, [(f"{root.name}/", b"", 0o755)] + [ + (p.relative_to(root.parent).as_posix() + ("/" if p.is_dir() else ""), b"" if p.is_dir() else p.read_bytes(), + 0o755 if p.is_dir() or p.stat().st_mode & 0o111 else 0o644) for p in found], stamp) return path def owned(member): member.uid = member.gid = 0 member.uname = member.gname = "" + member.mtime = stamp if not member.issym(): member.mode = 0o755 if member.isdir() or member.mode & 0o100 else 0o644 return member - with tarfile.open(path, "w:gz") as archive: - archive.add(root, root.name, filter=owned) + # No name or time in gzip's header, and the members in order rather than the file system's. + with open(path, "wb") as raw, gzip.GzipFile("", "wb", 9, raw, mtime=0) as zipped, \ + tarfile.open(fileobj=zipped, mode="w", format=tarfile.PAX_FORMAT) as archive: + archive.add(root, root.name, recursive=False, filter=owned) + for member in found: + archive.add(member, member.relative_to(root.parent).as_posix(), recursive=False, filter=owned) return path @@ -1207,6 +1298,246 @@ def export(args): exporter(args.manifest_path, args.features, *targets, *withs, *baseline, "--out", out) +def rustflags_config(directory, target): + """What sets rustflags for cargo where a RUSTFLAGS override would silently replace it.""" + here = Path(directory).resolve() + home = Path(os.environ.get("CARGO_HOME") or Path.home() / ".cargo") + for root in [home, *(r / ".cargo" for r in [here, *here.parents])]: + for name in ("config.toml", "config"): + found = root / name + if not found.is_file(): + continue + config = tomllib.loads(found.read_text(encoding="utf-8")) + tables = {key: table for key, table in config.get("target", {}).items() if key == target or key.startswith("cfg(")} + if "rustflags" in config.get("build", {}) or any("rustflags" in table for table in tables.values()): + yield str(found) + for key in os.environ: + if key == "CARGO_BUILD_RUSTFLAGS" or re.fullmatch(r"CARGO_TARGET_.*_RUSTFLAGS", key): + yield key + + +def reproducible(info, export_dir, target, install=None): + """The environment that makes a build's bytes the same on any host, at any path. + + Release time is the commit's, and each directory the build reads from, which rustc and + C compilers write into panic messages and debug info, is spelled the same. A dylib is + named `install`, else by its library's own name. + """ + pairs = { + info["workspace"]: "/source", + str(export_dir): "/export", + os.environ.get("CARGO_HOME") or str(Path.home() / ".cargo"): "/cargo", + } + # The deepest last, which rustc prefers where one directory holds another. + pairs = sorted(pairs.items(), key=lambda pair: len(pair[0])) + for found in rustflags_config(info["dir"], target): + raise SystemExit(f"{found} sets rustflags, which the reproducible build's RUSTFLAGS would replace: export RUSTFLAGS instead") + encoded = os.environ.get("CARGO_ENCODED_RUSTFLAGS") + if encoded is None: + flags = os.environ.get("RUSTFLAGS", "").split() + else: + # Cargo reads an empty value as no flags, not as one empty flag. + flags = encoded.split("\x1f") if encoded else [] + flags += [f"--remap-path-prefix={old}={new}" for old, new in pairs] + env = {"SOURCE_DATE_EPOCH": str(info["epoch"])} + if "msvc" in target: + # link.exe stamps the build time into the DLL unless told to hash its content. + flags += ["-C", "link-arg=/Brepro"] + else: + # cc splits CFLAGS on whitespace, and quoting them would change how a caller's own flags read. + for old, _ in pairs: + if any(c.isspace() for c in old): + raise SystemExit(f"{old} holds whitespace, which would split its -ffile-prefix-map in CFLAGS: build from a path without it") + # cc reads CXXFLAGS, not CFLAGS, for C++. + for var in ("CFLAGS", "CXXFLAGS"): + env[var] = " ".join(filter(None, [os.environ.get(var), *(f"-ffile-prefix-map={old}={new}" for old, new in pairs)])) + if "apple" in target: + # Apple's ar and libtool stamp archive members with the time unless told not to, and + # the linker names a dylib after the path it wrote it to, in the export's build directory. + env["ZERO_AR_DATE"] = "1" + flags += ["-C", f"link-arg=-Wl,-install_name,@rpath/{install or library_name(info['lib'], target)}"] + env["CARGO_ENCODED_RUSTFLAGS"] = "\x1f".join(flags) + return env + + +def purl(package): + """The crate's package URL, qualified by its source unless that is crates.io, so two sources of one version stay two components.""" + base = f"pkg:cargo/{package['name']}@{package['version']}" + source = package.get("source") or "" + # An alternate registry may speak the git or the sparse protocol; crates.io reports the former either way. + if source.startswith(("registry+", "sparse+")) and source != CRATES_IO: + return f"{base}?repository_url={urllib.parse.quote(source.split('+', 1)[1], safe='')}" + if source.startswith("git+"): + return f"{base}?vcs_url={urllib.parse.quote(source, safe='')}" + return base + + +def sbom(info, crate, feature="python"): + """A CycloneDX SBOM of the crates the binding is built from with `feature`, the same for every host and target.""" + meta = json.loads(output("cargo", "metadata", "--locked", "--format-version", "1", "--features", feature, + "--manifest-path", crate)) + lock = tomllib.loads((Path(crate).parent / "Cargo.lock").read_text(encoding="utf-8")) + checksums = {(p["name"], p["version"], p.get("source")): p["checksum"] for p in lock["package"] if "checksum" in p} + packages = {p["id"]: p for p in meta["packages"]} + nodes = {n["id"]: n for n in meta["resolve"]["nodes"]} + root = f"pkg:cargo/{info['name']}@{info['version']}" + # The generated crate is the package it exports: its dependencies are the package's. + ref = lambda i: root if i == meta["resolve"]["root"] or purl(packages[i]) == root else purl(packages[i]) # noqa: E731 + seen, queue, depends = set(), [meta["resolve"]["root"]], {} + while queue: + node = queue.pop() + if node in seen: + continue + seen.add(node) + # What the library links is its normal dependencies; a proc-macro or build script only runs at build time. + shipped = [d["pkg"] for d in nodes[node]["deps"] + if any(k["kind"] is None for k in d["dep_kinds"]) + and not any("proc-macro" in t["kind"] for t in packages[d["pkg"]]["targets"])] + depends.setdefault(ref(node), set()).update(ref(i) for i in shipped) + queue += shipped + components = {ref(i): packages[i] for i in sorted(seen) if ref(i) != root} + def component(name, package): + out = {"type": "library", "bom-ref": name, "name": package["name"], "version": package["version"], "purl": name} + if package.get("license"): + # Cargo still takes the legacy MIT/Apache-2.0, which is no SPDX expression. + out["licenses"] = [{"license": {"name": package["license"]}} if "/" in package["license"] + else {"expression": package["license"]}] + if package.get("repository"): + out["externalReferences"] = [{"type": "vcs", "url": package["repository"]}] + checksum = checksums.get((package["name"], package["version"], package.get("source"))) + if checksum: + out["hashes"] = [{"alg": "SHA-256", "content": checksum}] + return out + document = { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "serialNumber": "", + "metadata": { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(info["epoch"])), + "component": {"type": "library", "bom-ref": root, "name": info["name"], "version": info["version"], "purl": root}, + }, + "components": [component(name, components[name]) for name in sorted(components)], + "dependencies": [{"ref": name, "dependsOn": sorted(depends[name] - {name})} for name in sorted(depends)], + } + # actions/attest takes a CycloneDX file only with a serial number: derived from the content, so every host writes the same. + document["serialNumber"] = f"urn:uuid:{uuid.uuid5(uuid.NAMESPACE_URL, json.dumps(document, sort_keys=True))}" + return (json.dumps(document, indent=2) + "\n").encode() + + +def zip_diff(one, two): + """The entries two ZIPs hold differently, each with its content hash, time, and attributes.""" + with zipfile.ZipFile(one) as a, zipfile.ZipFile(two) as b: + entries = lambda z: {i.filename: (hashlib.sha256(z.read(i)).hexdigest(), i.date_time, i.external_attr) for i in z.infolist()} # noqa: E731 + left, right = entries(a), entries(b) + return [f" {name}: {left.get(name)} against {right.get(name)}" + for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] + + +def ar_entries(data, label="the archive"): + """An archive's `(name, header fields, start, end)` per member, its symbol table and names table by their own.""" + if data[:8] != b"!\n": + raise SystemExit(f"{label} is no archive") + entries, long_names, offset = [], b"", 8 + while offset + 60 <= len(data): + name, date, uid, gid, mode, size = (data[offset + a:offset + b].decode("ascii", "replace").strip() + for a, b in ((0, 16), (16, 28), (28, 34), (34, 40), (40, 48), (48, 58))) + start, end = offset + 60, offset + 60 + int(size) + offset = end + int(size) % 2 + if name == "//": + long_names = data[start:end] + elif name.startswith("#1/") and name[3:].isdigit(): + # BSD's extended name, in the first bytes of the member's own data. + length = int(name[3:]) + name = data[start:start + length].rstrip(b"\0").decode("utf-8", "replace") + start += length + elif name.startswith("/") and name[1:].isdigit(): + first = int(name[1:]) + name = long_names[first:long_names.index(b"\n", first)].decode("utf-8", "replace").rstrip("/") + else: + name = name.rstrip("/") or "/" + entries.append((name, (date, uid, gid, mode), start, end)) + return entries + + +def ar_members(path): + """An archive's members as `{name: (content, time, owner, group, mode)}`.""" + data = Path(path).read_bytes() + members = {} + for name, fields, start, end in ar_entries(data, path): + # A name twice, as an archive may hold, is told apart by its place. + while name in members: + name += "'" + members[name] = (data[start:end], *fields) + return members + + +def ar_diff(one, two): + """The members two archives hold differently, each with where its content first differs.""" + left, right = ar_members(one), ar_members(two) + lines = [] + for name in sorted(left.keys() | right.keys()): + a, b = left.get(name), right.get(name) + if a == b: + continue + if a is None or b is None: + lines.append(f" {name}: only in the {'first' if b is None else 'second'}") + continue + at = next((i for i, (x, y) in enumerate(zip(a[0], b[0])) if x != y), min(len(a[0]), len(b[0]))) + lines.append(f" {name}: {len(a[0])} against {len(b[0])} bytes, {a[1:]} against {b[1:]}; first differing byte {at}: " + f"{a[0][max(at - 24, 0):at + 40]!r} against {b[0][max(at - 24, 0):at + 40]!r}") + return lines + + +def tar_diff(one, two): + """The members two gzipped tarballs hold differently, each with its content hash and attributes, and where a static library's own members differ.""" + def members(path): + with tarfile.open(path) as archive: + return {m.name: (archive.extractfile(m).read() if m.isfile() else m.linkname, m.mtime, m.mode, m.uid, m.gid, m.uname, m.gname) + for m in archive.getmembers()} + left, right = members(one), members(two) + lines = [] + for name in sorted(left.keys() | right.keys()): + a, b = left.get(name), right.get(name) + if a == b: + continue + digest = lambda m: (hashlib.sha256(m[0]).hexdigest(), *m[1:]) if m and isinstance(m[0], bytes) else m # noqa: E731 + lines.append(f" {name}: {digest(a)} against {digest(b)}") + if name.endswith(".a") and a and b: + with tempfile.TemporaryDirectory() as tmp: + for index, content in enumerate((a[0], b[0])): + (Path(tmp) / f"{index}.a").write_bytes(content) + lines += [" " + line for line in ar_diff(Path(tmp) / "0.a", Path(tmp) / "1.a")] + return lines + + +def same(first, second): + """Fail unless two builds' wheels, static libraries, C archives, and npm tarball, the Go module's included, are the same bytes, naming what differs.""" + found = {} + for root in (Path(first), Path(second)): + found[root] = {p.relative_to(root).as_posix(): p for pattern in ("wheels/*.whl", "static/*/*.a", "go/*.a", "c/*.tar.gz", "c/*.zip", "*.tgz") for p in root.glob(pattern)} + names = sorted(found[Path(first)]) + if not names or names != sorted(found[Path(second)]): + raise SystemExit(f"the builds hold different packages: {names} against {sorted(found[Path(second)])}") + problems = [] + for name in names: + one, two = (found[Path(root)][name] for root in (first, second)) + if one.read_bytes() == two.read_bytes(): + continue + problems.append(name) + if name.endswith(".whl"): + problems += zip_diff(one, two) + elif name.endswith(".a"): + problems += ar_diff(one, two) + elif name.endswith((".tar.gz", ".tgz")): + problems += tar_diff(one, two) + elif name.endswith(".zip"): + problems += zip_diff(one, two) + if problems: + raise SystemExit("a rebuild made other bytes:\n" + "\n".join(problems)) + print(f"{len(names)} files rebuilt byte for byte") + + def cargo(manifest, *args, env=None): """Cargo with the package's own toolchain, as rustup finds it from the package's directory.""" return run("cargo", *args, cwd=Path(manifest).resolve().parent, env={**os.environ, **(env or {})}, @@ -1253,6 +1584,21 @@ def llvm_tool(manifest, name): return Path(sysroot) / "lib" / "rustlib" / host / "bin" / f"{name}{exe}" +# The random directory rustc imports a Windows GNU library's raw-dylib symbols through, which +# dlltool names them after, as `_out_rustc<6 random characters>_kernel32_dll_imports_lib`. +RUSTC_TEMP = re.compile(rb"(?<=_out_rustc)[0-9A-Za-z]{6}(?=_[0-9a-z_]+_dll_imports_lib)") +IMPORT_MEMBER = re.compile(r".+\.dll(h|t|s[0-9]+)\.o") + + +def fix_import_names(path): + """Replace that random part, at the same length, in the import library's members and the archive's symbol table alone, so the archive is the same on every build and no other byte changes.""" + data = bytearray(Path(path).read_bytes()) + for name, _, start, end in ar_entries(bytes(data)): + if name == "/" or IMPORT_MEMBER.fullmatch(name): + data[start:end] = RUSTC_TEMP.sub(b"000000", bytes(data[start:end])) + Path(path).write_bytes(bytes(data)) + + def go_library(args, export_dir, lib, target, library, native, out): """The Go module's copy of `target`'s library: checked against the export's #cgo line, without debug info, which only bloats the module, or the bitcode the go command refuses.""" @@ -1263,6 +1609,8 @@ def go_library(args, export_dir, lib, target, library, native, out): dest = out / "go" / f"lib{lib}_{GO_TARGETS[target]}.a" dest.parent.mkdir(parents=True, exist_ok=True) run(objcopy, "--strip-debug", *(f"--remove-section={s}" for s in LTO_SECTIONS), library, dest) + if "windows-gnu" in target: + fix_import_names(dest) def build(args): @@ -1271,12 +1619,16 @@ def build(args): lib, export_dir, out = info["lib"], Path(args.export).resolve(), Path(args.out).resolve() crate = export_dir / "Cargo.toml" out.mkdir(parents=True, exist_ok=True) + info["sbom"] = sbom(info, crate) + (out / "sbom.cdx.json").write_bytes(info["sbom"]) for target in args.targets: if target not in WHEEL_TARGETS and target not in APPLE_TARGETS and target not in GO_TARGETS: raise SystemExit(f"no package ships {target}") run("rustup", "target", "add", target, cwd=Path(args.manifest_path).resolve().parent) if target in WHEEL_TARGETS: - env = {"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if "apple" in target else {} + env = reproducible(info, export_dir, target) + if "apple" in target: + env["MACOSX_DEPLOYMENT_TARGET"] = MACOS_WHEEL linux = "linux" in target # cargo-zigbuild links Linux against the glibc floor's symbols, whichever glibc the host has. built = cargo(args.manifest_path, *(["zigbuild"] if linux else ["build"]), "--locked", "--release", @@ -1285,12 +1637,13 @@ def build(args): sys.stderr.write(built.stderr) library = export_dir / "target-python" / target / "release" / library_name(lib, target) with tempfile.TemporaryDirectory() as raw: - repair(write_wheel(info, export_dir, library, target, raw), target, out / "wheels") + repair(write_wheel(info, export_dir, library, target, raw), target, out / "wheels", info["epoch"]) c_archive(info, args.manifest_path, export_dir, target, out) if target in APPLE_TARGETS: apple = APPLE_TARGETS[target] name = "MACOSX_DEPLOYMENT_TARGET" if apple == "macos" else "IPHONEOS_DEPLOYMENT_TARGET" - library, native = staticlib(args.manifest_path, crate, lib, target, env={name: SWIFT_PLATFORMS[apple]}) + library, native = staticlib(args.manifest_path, crate, lib, target, + env={**reproducible(info, export_dir, target), name: SWIFT_PLATFORMS[apple]}) static = out / "static" / target static.mkdir(parents=True) shutil.copy(library, static) @@ -1299,7 +1652,7 @@ def build(args): if target in GO_TARGETS: go_library(args, export_dir, lib, target, library, native, out) elif target in GO_TARGETS: - library, native = staticlib(args.manifest_path, crate, lib, target) + library, native = staticlib(args.manifest_path, crate, lib, target, env=reproducible(info, export_dir, target)) go_library(args, export_dir, lib, target, library, native, out) # The generated sources, for the Swift package, the Go module, and the hosts' agreement. # Text this writes is LF on every host, as the jobs reading it compare bytes. @@ -1395,13 +1748,8 @@ def go_module(info, export, libs, out): # serving earlier ones appends to its own list. (versions / "list").write_text(version + "\n", encoding="utf-8") archive = versions / f"{version}.zip" - with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z: - for name, data in files.items(): - # A fixed time, so the zip is a function of its files. - entry = zipfile.ZipInfo(f"{module}@{version}/{name}", date_time=(1980, 1, 1, 0, 0, 0)) - entry.compress_type = zipfile.ZIP_DEFLATED - entry.external_attr = 0o644 << 16 - z.writestr(entry, data) + # Stamped as every other ZIP here, so the zip is a function of its files and the commit. + write_zip(archive, [(f"{module}@{version}/{name}", data, 0o644) for name, data in files.items()], info["epoch"]) if archive.stat().st_size > GO_ZIP_MAX: raise SystemExit(f"the Go module's zip is {archive.stat().st_size >> 20} MiB, over Go's {GO_ZIP_MAX >> 20} MiB") rows = [f"| `{name}` | {len(data) / (1 << 20):.1f} MiB |" for name, data in files.items() if name.endswith(".a")] @@ -1543,10 +1891,12 @@ def web(args): run("rustup", "target", "add", WEB_TARGET, cwd=manifest.parent) built = cargo(args.manifest_path, "build", "--locked", "--release", "--manifest-path", export_dir / "Cargo.toml", "--target", WEB_TARGET, "--target-dir", export_dir / "target", "--features", "web", - env=web_profile(export_dir / "Cargo.toml")) + env={**reproducible(info, export_dir, WEB_TARGET), **web_profile(export_dir / "Cargo.toml")}) sys.stderr.write(built.stderr) wasm = export_dir / "target" / WEB_TARGET / "release" / f"{info['lib']}.wasm" npm, _ = web_package(info, export_dir / "web", wasm, args.scope, out) + # Its own, as the web crate's dependencies are not the native export's. + (out / "sbom.cdx.json").write_bytes(sbom(info, export_dir / "Cargo.toml", "web")) packed = output("npm", "pack", "--silent", "--pack-destination", out, cwd=npm).strip() if not (out / packed).is_file(): raise SystemExit(f"npm pack wrote no {packed}") @@ -1712,6 +2062,9 @@ def main(): if name == "build": p.add_argument("--export", required=True) p.add_argument("--example", default="") + p = sub.add_parser("same", help="fail unless two builds made the same wheels and libraries") + p.add_argument("first") + p.add_argument("second") p = sub.add_parser("previous", help="fetch the API of the release this version follows") p.add_argument("--manifest-path", required=True) p.add_argument("--repo", required=True, metavar="OWNER/REPO", help="the repository whose releases to read") @@ -1792,6 +2145,8 @@ def main(): export(args) elif args.command == "build": build(args) + elif args.command == "same": + same(args.first, args.second) elif args.command == "previous": previous(args) elif args.command == "changes": diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 903fdfd3b..c0e758086 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -3,8 +3,11 @@ name: Release @NAME@ bindings # Written by rust2's `ffi/release/package.py init`; yours to edit. rust2's builder, pinned # below, exports @NAME@'s pub API, builds its Python wheels, Swift package, C archives, and # web package, and checks them in clean consumers, all read-only. The jobs after it publish -# only a tag's build: +# only a tag's build, once `attest` has signed it: # +# - Attestations: a build provenance attestation and an SBOM attestation for every wheel, the +# XCFramework, the Go module's .zip and .mod, each C archive, and the npm tarball, from +# this repository's GitHub, checked with `gh attestation verify`. # - PyPI: the project @NAME@, through the trusted publisher this repository registers there # for this workflow and the `pypi` environment. # - SwiftPM: @MODULE@FFI.xcframework.zip on this repository's @NAME@-v release, which @@ -25,7 +28,7 @@ name: Release @NAME@ bindings # To release, push the tag @NAME@-v, naming the crate's version. Without a tag, as # from a nightly's workflow_call, every job up to publication runs. If publishing fails, use # "Re-run failed jobs": it reuses this run's build, and the publishing jobs upload only what -# is missing. A rebuild's files differ, so "Re-run all jobs" stops at any already published. +# is missing. A rebuild makes the same files, so "Re-run all jobs" works too. on: push: tags: @@ -83,9 +86,62 @@ jobs: exit 1 fi - pypi: + # Signs what the jobs below publish, before any of it is public, as this workflow's own + # build of this commit. The permissions are this job's alone. + attest: needs: [build, tag] runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: @NAME@-build-* + path: artifacts + merge-multiple: true + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-swift + path: artifacts/swift + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-go + path: artifacts/go + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: needs.build.outputs.web == 'true' + with: + name: @NAME@-web + path: artifacts/web + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: | + artifacts/wheels/*.whl + artifacts/swift/*FFI.xcframework.zip + artifacts/go/**/@v/*.zip + artifacts/go/**/@v/*.mod + artifacts/c/* + artifacts/web/*.tgz + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: | + artifacts/wheels/*.whl + artifacts/swift/*FFI.xcframework.zip + artifacts/go/**/@v/*.zip + artifacts/go/**/@v/*.mod + artifacts/c/* + sbom-path: artifacts/sbom.cdx.json + # The npm tarball's own, from the web export's dependencies. + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + if: needs.build.outputs.web == 'true' + with: + subject-path: artifacts/web/*.tgz + sbom-path: artifacts/web/sbom.cdx.json + + pypi: + needs: [build, tag, attest] + runs-on: ubuntu-24.04 environment: name: pypi url: https://pypi.org/p/@NAME@ @@ -136,7 +192,7 @@ jobs: # published. The release's API, api.json, is an asset too: the next release's build checks # its own against it, and its notes list what changed since the previous release's. github-release: - needs: [build, tag] + needs: [build, tag, attest] runs-on: ubuntu-24.04 permissions: contents: write @@ -249,7 +305,7 @@ jobs: # A rerun publishes only a version npm lacks, and fails on one it hosts with other bytes. npm: - needs: [build, tag] + needs: [build, tag, attest] if: needs.build.outputs.web == 'true' runs-on: ubuntu-24.04 environment: npm @@ -282,7 +338,7 @@ jobs: # JSR skips a version it already hosts. jsr: - needs: [build, tag] + needs: [build, tag, attest] if: needs.build.outputs.web == 'true' runs-on: ubuntu-24.04 environment: jsr diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index d2539e808..3edf0732a 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -2,10 +2,14 @@ import importlib.util import json +import os from pathlib import Path +import shutil +import subprocess import tarfile import tempfile import unittest +from unittest import mock import zipfile HERE = Path(__file__).resolve().parent @@ -20,6 +24,7 @@ def info(root, **overrides): "name": "moq-ffi", "version": "0.4.12", "description": "Media over QUIC,\n bindings", "license": "MIT OR Apache-2.0", "license_file": None, "repository": "https://github.com/moq-dev/moq", "homepage": None, "lib": "moq", "dir": str(root), "workspace": str(root), "examples": [], + "epoch": 1_700_000_000, "sbom": b"{}\n", } return {**base, **overrides} @@ -225,6 +230,284 @@ def test_floor_comes_from_the_glue(self): self.assertEqual(package.python_floor(export(Path(tmp), "312")), 12) +class Reproducible(unittest.TestCase): + def wheel(self, root, **touch): + """A wheel built from inputs of the given mtimes, whose repair tool put its files in any order and time.""" + built = export(root / "export") + library = root / "libmoq.so" + library.write_bytes(b"\x7fELF") + for path in (library, built / "python/moq.py"): + os.utime(path, (touch.get("time", 0), touch.get("time", 0))) + return package.write_wheel(info(root), built, library, "x86_64-unknown-linux-gnu", root) + + def test_wheel_bytes_ignore_when_and_where(self): + with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: + first = self.wheel(Path(a), time=1_000_000_000) + second = self.wheel(Path(b), time=1_600_000_000) + self.assertEqual(first.read_bytes(), second.read_bytes()) + with zipfile.ZipFile(first) as archive: + names = archive.namelist() + self.assertTrue(all(entry.date_time == (2023, 11, 14, 22, 13, 20) for entry in archive.infolist())) + self.assertTrue(names[-1].endswith(".dist-info/RECORD")) + self.assertEqual(names[:2], ["moq/__init__.py", "moq/libmoq.so"]) + + def test_a_repair_tools_wheel_is_rewritten_as_one_of_ours(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + wheel = self.wheel(root) + repaired = root / "repaired.whl" + with zipfile.ZipFile(wheel) as src, zipfile.ZipFile(repaired, "w") as dst: + # As a repair tool leaves it: directories, reversed order, its own times, and a RECORD of its own. + dst.writestr("moq/", b"") + for name in reversed(src.namelist()): + entry = zipfile.ZipInfo(name, (2031, 5, 6, 7, 8, 10)) + entry.external_attr = src.getinfo(name).external_attr + dst.writestr(entry, b"other\n" if name.endswith("/RECORD") else src.read(name)) + package.normalize_wheel(repaired, 1_700_000_000) + package.check_wheel(repaired) + self.assertEqual(repaired.read_bytes(), wheel.read_bytes()) + + def test_the_sbom_rides_in_the_wheel(self): + with tempfile.TemporaryDirectory() as tmp: + with zipfile.ZipFile(self.wheel(Path(tmp))) as archive: + self.assertEqual(archive.read("moq_ffi-0.4.12.dist-info/sboms/moq-ffi.cdx.json"), b"{}\n") + + def test_xcframework_zip_ignores_mtimes_and_umask(self): + with tempfile.TemporaryDirectory() as tmp: + zips = [] + for index, (mode, time) in enumerate(((0o644, 1), (0o664, 2_000_000_000))): + tree = Path(tmp) / str(index) / "MoqFFI.xcframework" + (tree / "macos").mkdir(parents=True) + (tree / "Info.plist").write_text("plist\n") + (tree / "macos/libmoq.a").write_bytes(b"!\n") + for path in (tree, tree / "macos", tree / "Info.plist", tree / "macos/libmoq.a"): + os.utime(path, (time, time)) + (tree / "Info.plist").chmod(mode) + package.zip_tree(tree, Path(tmp) / f"{index}.zip", 1_700_000_000) + zips.append((Path(tmp) / f"{index}.zip").read_bytes()) + self.assertEqual(zips[0], zips[1]) + with zipfile.ZipFile(Path(tmp) / "0.zip") as archive: + self.assertEqual(archive.namelist(), [ + "MoqFFI.xcframework/", "MoqFFI.xcframework/Info.plist", "MoqFFI.xcframework/macos/", + "MoqFFI.xcframework/macos/libmoq.a"]) + + def test_xcframework_plist_lists_libraries_in_name_order(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "Info.plist" + libraries = [{"LibraryIdentifier": name} for name in ("macos-arm64", "ios-arm64-simulator", "ios-arm64")] + for order in (libraries, libraries[::-1]): + path.write_bytes(package.plistlib.dumps({"AvailableLibraries": order, "CFBundlePackageType": "XFWK"})) + package.sort_plist(path) + first = path.read_bytes() if order is libraries else first + self.assertEqual(path.read_bytes(), first) + self.assertEqual([x["LibraryIdentifier"] for x in package.plistlib.loads(first)["AvailableLibraries"]], + ["ios-arm64", "ios-arm64-simulator", "macos-arm64"]) + + def test_a_symlink_in_the_xcframework_fails(self): + with tempfile.TemporaryDirectory() as tmp: + tree = Path(tmp) / "MoqFFI.xcframework" + tree.mkdir() + (tree / "link").symlink_to("target") + with self.assertRaises(SystemExit): + package.zip_tree(tree, Path(tmp) / "out.zip", 1_700_000_000) + + def test_epoch_is_the_environment_or_the_commit(self): + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "1700000000"}): + self.assertEqual(package.epoch("."), 1_700_000_000) + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "86400"}), self.assertRaises(SystemExit): + package.epoch(".") + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ): + os.environ.pop("SOURCE_DATE_EPOCH", None) + git = ["git", "-C", tmp, "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run([*git, "init", "-q"], check=True) + subprocess.run([*git, "commit", "-q", "--allow-empty", "-m", "x", "--date", "2023-11-14T22:13:20Z"], + check=True, env={**os.environ, "GIT_COMMITTER_DATE": "2023-11-14T22:13:20Z"}) + self.assertEqual(package.epoch(tmp), 1_700_000_000) + + def test_paths_are_remapped_and_rustflags_kept(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"RUSTFLAGS": "-C target-cpu=native -Lnative=C:\\deps", "CARGO_HOME": "/home/u/.cargo"}): + os.environ.pop("CARGO_ENCODED_RUSTFLAGS", None) + env = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-unknown-linux-gnu") + flags = env["CARGO_ENCODED_RUSTFLAGS"].split("\x1f") + # Cargo splits on whitespace alone, so a Windows path keeps its backslash. + self.assertEqual(flags[:3], ["-C", "target-cpu=native", "-Lnative=C:\\deps"]) + self.assertIn(f"--remap-path-prefix={tmp}/export=/export", flags) + # The deeper directory comes last, as rustc prefers the last match. + self.assertLess(flags.index(f"--remap-path-prefix={tmp}=/source"), flags.index(f"--remap-path-prefix={tmp}/export=/export")) + self.assertEqual(env["SOURCE_DATE_EPOCH"], "1700000000") + self.assertIn(f"-ffile-prefix-map={tmp}=/source", env["CFLAGS"]) + self.assertIn(f"-ffile-prefix-map={tmp}=/source", env["CXXFLAGS"]) + windows = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-pc-windows-msvc") + self.assertIn("link-arg=/Brepro", windows["CARGO_ENCODED_RUSTFLAGS"]) + self.assertNotIn("CFLAGS", windows) + self.assertNotIn("CXXFLAGS", windows) + apple = package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin") + self.assertEqual(apple["ZERO_AR_DATE"], "1") + self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.dylib", apple["CARGO_ENCODED_RUSTFLAGS"]) + named = package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin", "libmoq.0.4.12.dylib") + self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.0.4.12.dylib", named["CARGO_ENCODED_RUSTFLAGS"]) + self.assertNotIn("@rpath/libmoq.dylib", named["CARGO_ENCODED_RUSTFLAGS"]) + + def test_an_empty_encoded_rustflags_is_no_flags(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_ENCODED_RUSTFLAGS": "", "RUSTFLAGS": "-C x", "CARGO_HOME": tmp + "/home"}): + env = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-unknown-linux-gnu") + flags = env["CARGO_ENCODED_RUSTFLAGS"].split("\x1f") + self.assertTrue(flags[0].startswith("--remap-path-prefix="), flags) + self.assertNotIn("-C", flags) + + def test_a_path_cc_would_split_fails(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_HOME": tmp + "/home"}): + os.environ.pop("CARGO_ENCODED_RUSTFLAGS", None) + spaced = Path(tmp) / "a b" + with self.assertRaises(SystemExit): + package.reproducible(info(tmp), spaced, "x86_64-unknown-linux-gnu") + # MSVC's C flags are not CFLAGS, so a space there is harmless. + package.reproducible(info(tmp), spaced, "x86_64-pc-windows-msvc") + + def test_a_cargo_config_that_RUSTFLAGS_would_replace_fails(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_HOME": tmp + "/home"}): + crate = Path(tmp) / "crate" + (crate / ".cargo").mkdir(parents=True) + (crate / ".cargo/config.toml").write_text('[target.x86_64-pc-windows-msvc]\nrustflags = ["-C", "x"]\n') + self.assertEqual(list(package.rustflags_config(crate, "x86_64-unknown-linux-gnu")), []) + self.assertEqual(len(list(package.rustflags_config(crate, "x86_64-pc-windows-msvc"))), 1) + (crate / ".cargo/config.toml").write_text('[build]\nrustflags = ["-C", "x"]\n') + with self.assertRaises(SystemExit): + package.reproducible(info(crate), crate / "export", "x86_64-unknown-linux-gnu") + + def test_same_names_what_differs(self): + with tempfile.TemporaryDirectory() as tmp: + roots = [Path(tmp) / "a", Path(tmp) / "b"] + for root in roots: + (root / "static/aarch64-apple-ios").mkdir(parents=True) + (root / "static/aarch64-apple-ios/libmoq.a").write_bytes(b"same") + (root / "wheels").mkdir() + first = self.wheel(roots[0]) + second = self.wheel(roots[1]) + for root, wheel in zip(roots, (first, second)): + wheel.replace(root / "wheels" / wheel.name) + package.same(roots[0], roots[1]) + (roots[1] / "static/aarch64-apple-ios/libmoq.a").write_bytes(b"other") + with self.assertRaisesRegex(SystemExit, "static/aarch64-apple-ios/libmoq.a"): + package.same(roots[0], roots[1]) + + +class Sbom(unittest.TestCase): + def crate(self, root): + """A generated crate whose normal, proc-macro, and build dependencies are each a path crate.""" + for name, extra in (("dep", ""), ("builder", ""), ("derive", "[lib]\nproc-macro = true\n")): + (root / name / "src").mkdir(parents=True) + (root / name / "src/lib.rs").write_text("") + (root / name / "Cargo.toml").write_text(f'[package]\nname = "{name}"\nversion = "1.2.3"\nedition = "2021"\nlicense = "MIT/Apache-2.0"\n{extra}') + (root / "glue/src").mkdir(parents=True) + (root / "glue/src/lib.rs").write_text("") + (root / "glue/Cargo.toml").write_text( + '[package]\nname = "glue"\nversion = "0.0.0"\nedition = "2021"\n[features]\npython = []\n' + '[dependencies]\ndep = { path = "../dep" }\nderive = { path = "../derive" }\n' + '[build-dependencies]\nbuilder = { path = "../builder" }\n[workspace]\n') + subprocess.run(["cargo", "generate-lockfile", "--offline", "--manifest-path", root / "glue/Cargo.toml"], check=True, + capture_output=True) + return root / "glue/Cargo.toml" + + def test_what_ships_is_what_the_library_links(self): + with tempfile.TemporaryDirectory() as tmp: + crate = self.crate(Path(tmp)) + first = package.sbom(info(tmp, name="moq-ffi", version="0.4.12"), crate) + self.assertEqual(first, package.sbom(info(tmp, name="moq-ffi", version="0.4.12"), crate)) + document = json.loads(first) + self.assertEqual(document["bomFormat"], "CycloneDX") + # actions/attest rejects a CycloneDX file without one. + self.assertRegex(document["serialNumber"], r"urn:uuid:[0-9a-f-]{36}") + self.assertEqual(document["metadata"]["timestamp"], "2023-11-14T22:13:20Z") + self.assertEqual([c["name"] for c in document["components"]], ["dep"]) + self.assertEqual(document["components"][0]["licenses"], [{"license": {"name": "MIT/Apache-2.0"}}]) + self.assertEqual(document["dependencies"], [ + {"ref": "pkg:cargo/dep@1.2.3", "dependsOn": []}, + {"ref": "pkg:cargo/moq-ffi@0.4.12", "dependsOn": ["pkg:cargo/dep@1.2.3"]}, + ]) + + + def test_one_version_from_two_sources_is_two_components(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + crate = self.crate(root) + # The same name and version again, from a git repository. + fork = root / "fork" + shutil.copytree(root / "dep", fork) + git = ["git", "-C", str(fork), "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run([*git, "init", "-q"], check=True) + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "x"], check=True) + manifest = crate.read_text().replace("[build-dependencies]", f'forked = {{ package = "dep", git = "file://{fork}" }}\n[build-dependencies]') + crate.write_text(manifest) + # A local repository, which cargo fetches without a network. + subprocess.run(["cargo", "generate-lockfile", "--manifest-path", crate], check=True, capture_output=True) + document = json.loads(package.sbom(info(tmp, name="moq-ffi", version="0.4.12"), crate)) + refs = [c["bom-ref"] for c in document["components"]] + self.assertEqual(len(refs), 2) + self.assertIn("pkg:cargo/dep@1.2.3", refs) + self.assertTrue(any(ref.startswith("pkg:cargo/dep@1.2.3?vcs_url=git%2Bfile") for ref in refs), refs) + + +class Archive(unittest.TestCase): + def test_members_that_differ_are_named(self): + def ar(*members): + out = b"!\n" + for name, date, body in members: + out += f"{name + '/':<16}{date:<12}{0:<6}{0:<6}{100644:<8}{len(body):<10}`\n".encode() + body + b"\n" * (len(body) % 2) + return out + with tempfile.TemporaryDirectory() as tmp: + one, two = Path(tmp) / "one.a", Path(tmp) / "two.a" + one.write_bytes(ar(("a.o", 0, b"same"), ("b.o", 0, b"before"))) + two.write_bytes(ar(("a.o", 0, b"same"), ("b.o", 5, b"before"))) + lines = package.ar_diff(one, two) + self.assertEqual(len(lines), 1) + self.assertIn("b.o", lines[0]) + + +class ImportNames(unittest.TestCase): + @staticmethod + def archive(*members): + out = b"!\n" + for name, body in members: + out += f"{name + '/':<16}{0:<12}{0:<6}{0:<6}{100644:<8}{len(body):<10}`\n".encode() + body + b"\n" * (len(body) % 2) + return out + + def test_only_the_import_library_loses_its_random_directory(self): + imported = b"_head_D__a_x_out_rustc4TX99l_kernel32_dll_imports_lib\x00" + data = b"_out_rustcAbCdEf_kernel32_dll_imports_lib is a string the crate returns" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "lib.a" + path.write_bytes(self.archive(("", imported), ("kernel32.dllh.o", imported), ("api.o", data))) + package.fix_import_names(path) + members = package.ar_members(path) + self.assertEqual(members["kernel32.dllh.o"][0], imported.replace(b"4TX99l", b"000000")) + self.assertEqual(members["api.o"][0], data) + + +class BsdArchive(unittest.TestCase): + def test_extended_names_are_read_from_the_data(self): + body = b"a_long_object_name.o\0\0\0\0" + b"CONTENT" + header = f"{'#1/24':<16}{0:<12}{0:<6}{0:<6}{100644:<8}{len(body):<10}`\n".encode() + data = b"!\n" + header + body + b"\n" * (len(body) % 2) + entries = package.ar_entries(data) + self.assertEqual([(name, data[start:end]) for name, _, start, end in entries], [("a_long_object_name.o", b"CONTENT")]) + + +class Purl(unittest.TestCase): + def test_each_registry_is_its_own_package(self): + def purl(source): + return package.purl({"name": "dep", "version": "1.2.3", "source": source}) + crates_io = "registry+https://github.com/rust-lang/crates.io-index" + self.assertEqual(purl(crates_io), "pkg:cargo/dep@1.2.3") + self.assertEqual(purl(None), "pkg:cargo/dep@1.2.3") + git = purl("registry+https://example.com/index") + sparse = purl("sparse+https://example.com/index/") + self.assertIn("?repository_url=https%3A%2F%2Fexample.com%2Findex", git) + self.assertIn("?repository_url=https%3A%2F%2Fexample.com%2Findex%2F", sparse) + self.assertEqual(len({purl(crates_io), git, sparse}), 3) + + class Digest(unittest.TestCase): def test_any_byte_splits_the_hosts(self): with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: @@ -426,13 +709,29 @@ def test_archive_is_owned_by_no_one(self): (root / "lib/libmoq.so").symlink_to("libmoq.so.0.4.12") (root / "lib/libmoq.a").write_bytes(b"!\n") (root / "lib/libmoq.a").chmod(0o600) - path = package.pack(root, Path(tmp) / "out.tar.gz") + path = package.pack(root, Path(tmp) / "out.tar.gz", 1_700_000_000) with tarfile.open(path) as archive: members = {m.name.split("/", 1)[-1]: m for m in archive.getmembers()} self.assertEqual(members["lib/libmoq.so.0.4.12"].mode, 0o755) self.assertEqual(members["lib/libmoq.a"].mode, 0o644) self.assertEqual(members["lib/libmoq.so"].linkname, "libmoq.so.0.4.12") self.assertEqual({(m.uid, m.uname) for m in members.values()}, {(0, "")}) + self.assertEqual({m.mtime for m in members.values()}, {1_700_000_000}) + + def test_archives_are_the_same_bytes_whenever_and_wherever_packed(self): + with tempfile.TemporaryDirectory() as tmp: + made = [] + for n, (touch, suffix) in enumerate(((1, ".tar.gz"), (2_000_000_000, ".tar.gz"), (1, ".zip"), (2_000_000_000, ".zip"))): + base = Path(tmp) / f"base{n}" + root = base / "moq-ffi-0.4.12-x" + (root / "lib").mkdir(parents=True) + for name in ("lib/b.txt", "lib/a.txt", "z.txt"): + (root / name).write_text(name) + os.utime(root / name, (touch, touch)) + (root / "lib/a.txt").chmod(0o640 if n % 2 else 0o600) + made.append(package.pack(root, Path(tmp) / f"out{n}{suffix}", 1_700_000_000).read_bytes()) + self.assertEqual(made[0], made[1]) + self.assertEqual(made[2], made[3]) class Template(unittest.TestCase): diff --git a/quest/m1/ffi-packaging/relocatable.md b/quest/m1/ffi-packaging/relocatable.md new file mode 100644 index 000000000..dcfdbea42 --- /dev/null +++ b/quest/m1/ffi-packaging/relocatable.md @@ -0,0 +1,34 @@ +# [S] A release rebuilds the same from any checkout path + +## Goal + +The generated crate's version hashes its `Cargo.toml`, which names the facade by absolute +path, and cargo hashes that version into every symbol: two checkouts of one commit at +different paths make different libraries. The exporter writes the dependency relative to +its output, so [a rebuild](/ffi/release/README.md#reproducible-builds) at any path makes the +same bytes, and `reproduce` in `bindings-test.yml` builds the second copy elsewhere. + +## Plan + +Decided with the maintainer on 2026-11-19, in the pick of [#874](https://github.com/kixel-ai/rust2/pull/874): ✅ kept (rejected: dropping it). + +Found by [the reproducible-releases quest](https://github.com/kixel-ai/rust2/pull/874): +`glue.rs`'s `generate` writes `path = {facade.dir}`. Measured there: two exports of the api +fixture from copies at different paths differ in the version's hash and in every symbol +hash; the same path with another export directory is byte for byte equal on Linux and for the wheels. +The Go module's Windows GNU library is not: `dlltool` names its `kernel32` import members' +symbols (`_head_D__a_rust2_..._export_target_go_...`) after the path of the target +directory, which `--remap-path-prefix` does not reach (and a random rustc temporary directory in it, +which the builder zeroes), so `reproduce` rebuilds from a new +export at the same path. Find a target directory path that is the same everywhere (or a +rustc that remaps it) so the CI rebuild can move the export again. + +Fall back to the absolute path where no relative one exists, such as another Windows drive. +Check `cli.rs`'s other absolute manifest paths (its "each path absolute" note) for the same +dependence. + +Public API: none. + +## Related + +- [Release every language the exporter generates](/quest/m1/ffi-packaging/README.md) - the epic whose builds this makes checkable by anyone diff --git a/quest/m1/ffi-packaging/reproducible.md b/quest/m1/ffi-packaging/reproducible.md deleted file mode 100644 index c3ad376cb..000000000 --- a/quest/m1/ffi-packaging/reproducible.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] Reproducible, attested releases - -## Goal - -A rebuild of a release makes the same bytes: wheels, archives, and the XCFramework zip -take `SOURCE_DATE_EPOCH` timestamps, a deterministic file order, and remapped paths, so -**Re-run all jobs** succeeds. Every artifact gets a build provenance attestation -(`actions/attest-build-provenance`) and an SBOM. - -## Plan - -Found by the 2026-11-18 language audit: `package.py:208`'s `writestr` stamps the current -time and the XCFramework is zipped with `ditto`, which `ffi/release/README.md` admits -makes rebuilds differ. PyPI's trusted publishing already uploads PEP 740 attestations; the -other artifacts have none. - -Public API: none.