From 8ade968adbffbbd4ff69fc54f9b5aa6328df01b7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:44:25 -0800 Subject: [PATCH 01/20] chore(quest): claim quest/m1/ffi-packaging/reproducible.md Quest-Claim: bb8394a62e82ae2b From b3fa8905d24e23757f2cfb6250205d82a1db8cc5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:33:51 -0800 Subject: [PATCH 02/20] Make binding releases reproducible and attested Wheels and the XCFramework ZIP take the commit's time, sorted entries, and fixed modes; builds remap their paths and stamp no linker or archive time. Every wheel carries a CycloneDX SBOM, and the caller's tag-only attest job gives each artifact a build provenance and an SBOM attestation before anything publishes. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings-test.yml | 2 + .github/workflows/bindings.yml | 22 ++- ffi/release/README.md | 49 ++++- ffi/release/package.py | 263 ++++++++++++++++++++++--- ffi/release/release.yml | 42 +++- ffi/release/test_package.py | 164 +++++++++++++++ quest/m1/ffi-packaging/relocatable.md | 26 +++ quest/m1/ffi-packaging/reproducible.md | 17 -- 8 files changed, 526 insertions(+), 59 deletions(-) create mode 100644 quest/m1/ffi-packaging/relocatable.md delete mode 100644 quest/m1/ffi-packaging/reproducible.md diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index 545c7f8b3..7ca30c772 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -18,6 +18,8 @@ jobs: features: doubled example: transcript python-check: ffi/tests/api.py + # The small crate, so the second build costs little: moq's stand-in leaves it to its release. + reproduce: true # No swift-check: api.swift compiles against the fixture's own test export, of the host # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 677c7ea4b..146995ad3 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -27,6 +27,10 @@ on: description: The main.swift of an executable run on the installed Swift package default: '' type: string + reproduce: + description: Build every host's packages a second time, with the export elsewhere, and fail unless the bytes match + default: false + type: boolean outputs: version: description: The package's Cargo version @@ -258,6 +262,17 @@ jobs: run: | read -ra targets <<< "$TARGETS" python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export export --out dist --example "$EXAMPLE" "${targets[@]}" + # The build again, as a rerun or anyone with the commit makes it: the same packages byte + # for byte, though the export lives in another directory. It doubles the job's time. + - name: Rebuild + if: inputs.reproduce + env: + TARGETS: ${{ matrix.targets }} + run: | + read -ra targets <<< "$TARGETS" + python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --out again/export "${targets[@]}" + python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export again/export --out again/dist "${targets[@]}" + python3 -I rust2/ffi/release/package.py same dist again/dist - name: Checks env: PYTHON_CHECK: ${{ inputs.python-check }} @@ -273,7 +288,8 @@ jobs: if-no-files-found: error # The hosts' exports must be one binding: a module and a library from different builds - # refuse to load, but two wheels of different bindings would each load. + # refuse to load, but two wheels of different bindings would each load. Their SBOMs, of + # the one dependency graph, must match as well. agree: needs: [facts, build] runs-on: ubuntu-24.04 @@ -290,6 +306,10 @@ jobs: echo '::error::The build hosts exported different bindings.' exit 1 fi + if [[ $(sha256sum builds/*/sbom.cdx.json | cut -d' ' -f1 | sort -u | wc -l) != 1 ]]; then + echo '::error::The build hosts listed different dependencies.' + exit 1 + fi swift: needs: [facts, build] diff --git a/ffi/release/README.md b/ffi/release/README.md index de013ca36..cd1cc93e2 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -32,15 +32,15 @@ Releasing is pushing `-v`, the crate's version, which the releas Called without a tag, such as from a nightly, it runs everything up to publication. A release that failed while publishing is recovered with **Re-run failed jobs**, which keeps the run's successful build and its artifacts: each publishing job then finishes what the -first attempt left, uploading only what is missing. A rebuild makes other bytes, as wheels -and the XCFramework archive carry build times, so **Re-run all jobs** fails wherever a -registry already holds the first attempt's files. +first attempt left, uploading only what is missing. A rebuild makes the same bytes +([Reproducible builds](#reproducible-builds)), so **Re-run all jobs** works as well. A Cargo prerelease, such as `1.0.0-rc.1`, is a GitHub prerelease too. -The builder takes the crate's `Cargo.toml` and four optional inputs the caller's tests own: -`features` to ship, an `example` whose stdout is native Rust's transcript, and the +The builder takes the crate's `Cargo.toml` and five optional inputs the caller's tests own: +`features` to ship, an `example` whose stdout is native Rust's transcript, the `python-check` and `swift-check` programs that must print it through the installed -packages. Without a check, a consumer only imports the module. +packages, and `reproduce`, which builds everything twice. Without a check, a consumer only +imports the module. ## What the builder does @@ -60,7 +60,7 @@ packages. Without a check, a consumer only imports the module. `win_amd64`. 5. Swift: static libraries for macOS arm64 at macOS 13, and iOS arm64 and its simulator at iOS 16, the floors of the generated Swift's `Swift.Duration`, each checked against - every object's declared minimum OS; one XCFramework of the three, zipped with `ditto`, + every object's declared minimum OS; one XCFramework of the three, zipped by the builder itself, its SHA-256 checked against `swift package compute-checksum`; and a `Package.swift` naming it at `https://github.com//releases/download/-v/FFI.xcframework.zip`. The package depends on swift-collections by the release tag at the exporter's pinned @@ -78,6 +78,41 @@ rust2 runs it on every manual Check and nightly through wheels must print its native transcript, and on moq's stand-in facade at the exporter's moq pin, through `moq/moq.py` and `moq/moq.swift`. +## Reproducible builds + +A rebuild of a commit makes the same bytes, so **Re-run all jobs** finds each registry +holding exactly the files it would upload, and anyone with the commit can check them. + +- **Time.** Every file takes `SOURCE_DATE_EPOCH`, or else the commit's time, whatever its + archive format: the builder writes each ZIP itself, with sorted entries, fixed modes, and + no host, umask, or time of its own, and rewrites each repaired wheel the same way, its + RECORD rebuilt. Zlib's own output is the one thing a host could still change. +- **Paths.** The crate's workspace, the exported crate, and `CARGO_HOME` are remapped to + fixed names for rustc and the C compilers, so panic messages and debug info name no + checkout. This takes `RUSTFLAGS` over a `.cargo/config.toml`'s `rustflags`, so a config + that sets any fails the build: export them as `RUSTFLAGS`, which the builder keeps. Linux + and macOS builds also stamp no archive time (`ZERO_AR_DATE`), and Windows links with + `/Brepro`. +- **Checked.** The Swift job builds the XCFramework twice and fails unless both ZIPs match. + With `reproduce: true`, every build host builds a second time, from a new export, and + `package.py same` fails unless each wheel and static library matches byte for byte; rust2 + turns it on for its api fixture. It doubles the build, so a large crate may leave it to a + nightly. +- **Not yet.** The exporter writes the crate's absolute path into the generated crate's + version, which cargo hashes into every symbol, so a rebuild must keep the crate at the + same path, as a rerun does + ([planned](/quest/m1/ffi-packaging/relocatable.md)). + +**Attestations and SBOM.** `package.py build` writes `sbom.cdx.json`, a CycloneDX SBOM of +the crates the binding links (normal dependencies, platform-neutral, with each registry +crate's checksum from `Cargo.lock`), which every wheel also carries in +`.dist-info/sboms/` (PEP 770) and the build hosts must agree on. The caller's `attest` job +runs on a tag, before anything is published, and gives every wheel and the XCFramework a +build provenance attestation and an SBOM attestation, which `gh attestation verify + --repo /` checks. Only that job holds `id-token` and `attestations` +write; the builder stays read-only. PyPI's trusted publishing uploads its own PEP 740 +attestations for the wheels as well. + ## Other registries The same boundary holds for each destination: the builder makes and checks the artifact diff --git a/ffi/release/package.py b/ffi/release/package.py index 9bc9b55d6..3d33199f2 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -20,6 +20,8 @@ import subprocess import sys import tempfile +import time +import tomllib import zipfile HERE = Path(__file__).resolve().parent @@ -46,6 +48,8 @@ # Cargo metadata names a library target by its crate types; a proc-macro is no library to export. LIBRARY_KINDS = {"lib", "rlib", "dylib", "cdylib", "staticlib"} COLLECTIONS = "https://github.com/apple/swift-collections" +# The first date a ZIP entry can hold, and so the earliest SOURCE_DATE_EPOCH a build takes. +ZIP_EPOCH = 315532800 SEMVER = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-(alpha|beta|rc)\.(0|[1-9][0-9]*))?") @@ -77,6 +81,14 @@ def library(package): return libs[0]["name"].replace("-", "_") +def epoch(directory): + """The build time every file of this release carries: SOURCE_DATE_EPOCH, else the commit's.""" + raw = os.environ.get("SOURCE_DATE_EPOCH") or output("git", "-C", directory, "log", "-1", "--format=%ct").strip() + if not raw.isdigit() or int(raw) < ZIP_EPOCH: + raise SystemExit(f"SOURCE_DATE_EPOCH {raw!r} is no time since 1980: set it, or build from a commit") + return int(raw) + + def facts(manifest): """The package at `manifest`, as Cargo metadata describes it.""" meta = json.loads(output("cargo", "metadata", "--format-version", "1", "--no-deps", @@ -96,6 +108,7 @@ def facts(manifest): "lib": library(package), "dir": str(path.parent), "workspace": meta["workspace_root"], + "epoch": epoch(path.parent), "examples": sorted(t["name"] for t in package["targets"] if "example" in t["kind"]), } @@ -152,14 +165,51 @@ def metadata(info, floor): return "\n".join(lines) + "\n\n" -def record(files): - """RECORD: every file's hash and size, and RECORD's own row unhashed.""" - out = io.StringIO(newline="") - writer = csv.writer(out, lineterminator="\n") - for name, data in files.items(): - digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() - writer.writerow([name, f"sha256={digest}", len(data)]) - return out +def write_zip(path, entries, stamp): + """A ZIP of `(name, data, permissions)` entries in the order given, the same bytes whoever writes it and whenever. + + Every entry takes `stamp`, a name ending in `/` is a directory, and nothing else of the + writer, such as its host or umask, is recorded. + """ + date = time.gmtime(stamp)[:6] + with zipfile.ZipFile(path, "w") as archive: + for name, data, mode in entries: + entry = zipfile.ZipInfo(name, date) + entry.create_system = 3 + if name.endswith("/"): + entry.external_attr = (0o040000 | mode) << 16 | 0x10 + else: + entry.external_attr = (0o100000 | mode) << 16 + entry.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(entry, data, compresslevel=9) + + +def seal(path, files, stamp): + """Write `files`, `{name: (data, permissions)}`, as a wheel: sorted with the dist-info last, and its RECORD hashing the rest.""" + meta = {name.split("/")[0] for name in files if name.split("/")[0].endswith(".dist-info")} + if len(meta) != 1: + raise SystemExit(f"{path} needs one .dist-info directory, has {sorted(meta)}") + record = f"{meta.pop()}/RECORD" + entries = sorted(files, key=lambda name: (name.split("/")[0].endswith(".dist-info"), name)) + rows = io.StringIO(newline="") + writer = csv.writer(rows, lineterminator="\n") + for name in entries: + digest = base64.urlsafe_b64encode(hashlib.sha256(files[name][0]).digest()).rstrip(b"=").decode() + writer.writerow([name, f"sha256={digest}", len(files[name][0])]) + writer.writerow([record, "", ""]) + write_zip(path, [(name, *files[name]) for name in entries] + [(record, rows.getvalue().encode(), 0o644)], stamp) + + +def normalize_wheel(path, stamp): + """Rewrite a repaired wheel as `seal` writes one, whatever order, times, and modes its tool left.""" + with zipfile.ZipFile(path) as archive: + files = { + entry.filename: (archive.read(entry), 0o755 if entry.external_attr >> 16 & 0o111 else 0o644) + for entry in archive.infolist() + # Repair tools add directory entries, which RECORD never lists, and rewrite RECORD itself. + if not entry.is_dir() and not entry.filename.endswith(".dist-info/RECORD") + } + seal(path, files, stamp) def wheel_platform(target): @@ -192,26 +242,22 @@ def write_wheel(info, export, library, target, out): tag = f"cp3{floor}-abi3-{wheel_platform(target)}" meta = f"{dist}-{version}.dist-info" files = { - f"{lib}/__init__.py": (Path(export) / "python" / f"{lib}.py").read_bytes(), - f"{lib}/{library_name(lib, target)}": Path(library).read_bytes(), + f"{lib}/__init__.py": ((Path(export) / "python" / f"{lib}.py").read_bytes(), 0o644), + f"{lib}/{library_name(lib, target)}": (Path(library).read_bytes(), 0o755), } for path in licenses(info): - files[f"{meta}/licenses/{path.name}"] = path.read_bytes() - files[f"{meta}/METADATA"] = metadata(info, floor).encode() + files[f"{meta}/licenses/{path.name}"] = (path.read_bytes(), 0o644) + # PEP 770: the dependencies the library was built from, where a scanner looks. + files[f"{meta}/sboms/{info['name']}.cdx.json"] = (info["sbom"], 0o644) + files[f"{meta}/METADATA"] = (metadata(info, floor).encode(), 0o644) files[f"{meta}/WHEEL"] = ( - f"Wheel-Version: 1.0\nGenerator: rust2-ffi\nRoot-Is-Purelib: false\nTag: {tag}\n" - ).encode() - rows = record(files) - csv.writer(rows, lineterminator="\n").writerow([f"{meta}/RECORD", "", ""]) - files[f"{meta}/RECORD"] = rows.getvalue().encode() + f"Wheel-Version: 1.0\nGenerator: rust2-ffi\nRoot-Is-Purelib: false\nTag: {tag}\n".encode(), 0o644) path = Path(out) / f"{dist}-{version}-{tag}.whl" - with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: - for name, data in files.items(): - archive.writestr(name, data) + seal(path, files, info["epoch"]) return path -def repair(wheel, target, out): +def repair(wheel, target, out, stamp): """Hand the wheel to its platform's repair tool, which claims the portable tag or fails.""" Path(out).mkdir(parents=True, exist_ok=True) if "linux" in target: @@ -226,6 +272,7 @@ def repair(wheel, target, out): repaired = list(Path(out).glob("*.whl")) if len(repaired) != 1: raise SystemExit(f"repair wrote {len(repaired)} wheels into {out}") + normalize_wheel(repaired[0], stamp) check_wheel(repaired[0]) return repaired[0] @@ -342,6 +389,20 @@ def min_os(archive): return max(versions) +def zip_tree(root, path, stamp): + """The directory `root` as a ZIP holding it as its top-level directory, as `ditto --keepParent` does.""" + entries = [(f"{root.name}/", b"", 0o755)] + for found in sorted(root.rglob("*"), key=lambda p: p.relative_to(root).as_posix()): + name = f"{root.name}/{found.relative_to(root).as_posix()}" + if found.is_symlink(): + raise SystemExit(f"{found} is a symlink: the XCFramework ZIP holds only files") + if found.is_dir(): + entries.append((f"{name}/", b"", 0o755)) + else: + entries.append((name, found.read_bytes(), 0o755 if found.stat().st_mode & 0o111 else 0o644)) + write_zip(path, entries, stamp) + + def swift(info, export, libs, natives, out): """The XCFramework of every Apple slice, its checksummed ZIP, and the package around it.""" if sys.platform != "darwin": @@ -365,13 +426,20 @@ def swift(info, export, libs, natives, out): raise SystemExit("the module map no longer names ../include: update the packager") (headers / "module.modulemap").write_text(modulemap.replace(f"../include/{lib}.h", f"{lib}.h"), encoding="utf-8") framework = out / f"{module}FFI.xcframework" - args = ["xcodebuild", "-create-xcframework"] - for target in APPLE_TARGETS: - args += ["-library", libs[target], "-headers", headers] - run(*args, "-output", framework) - shutil.rmtree(headers) archive = out / f"{module}FFI.xcframework.zip" - run("/usr/bin/ditto", "-c", "-k", "--sequesterRsrc", "--keepParent", framework, archive) + again = out / "again" + # The XCFramework twice, which must zip to the same bytes: the checksum a consumer + # resolves is of this ZIP, and a rebuild must make it again. + for tree, zipped in ((framework, archive), (again / framework.name, again / archive.name)): + args = ["xcodebuild", "-create-xcframework"] + for target in APPLE_TARGETS: + args += ["-library", libs[target], "-headers", headers] + run(*args, "-output", tree) + zip_tree(tree, zipped, info["epoch"]) + if archive.read_bytes() != (again / archive.name).read_bytes(): + raise SystemExit("xcodebuild made different XCFrameworks from the same libraries") + shutil.rmtree(headers) + shutil.rmtree(again) checksum = hashlib.sha256(archive.read_bytes()).hexdigest() # SwiftPM's own computation, as a consumer resolving the URL runs it. swift_sum = output("swift", "package", "compute-checksum", archive).strip() @@ -473,6 +541,132 @@ def export(args): exporter(args.manifest_path, args.features, *targets, *withs, "--out", Path(args.out).resolve()) +def rustflags_config(directory, target): + """What sets rustflags for cargo where a RUSTFLAGS override would silently replace it.""" + here = Path(directory).resolve() + home = Path(os.environ.get("CARGO_HOME") or Path.home() / ".cargo") + for root in [home, *(r / ".cargo" for r in [here, *here.parents])]: + for name in ("config.toml", "config"): + found = root / name + if not found.is_file(): + continue + config = tomllib.loads(found.read_text(encoding="utf-8")) + tables = {key: table for key, table in config.get("target", {}).items() if key == target or key.startswith("cfg(")} + if "rustflags" in config.get("build", {}) or any("rustflags" in table for table in tables.values()): + yield str(found) + for key in os.environ: + if key == "CARGO_BUILD_RUSTFLAGS" or re.fullmatch(r"CARGO_TARGET_.*_RUSTFLAGS", key): + yield key + + +def reproducible(info, export_dir, target): + """The environment that makes a build's bytes the same on any host, at any path. + + Release time is the commit's, and each directory the build reads from, which rustc and + C compilers write into panic messages and debug info, is spelled the same. + """ + pairs = { + info["workspace"]: "/source", + str(export_dir): "/export", + os.environ.get("CARGO_HOME") or str(Path.home() / ".cargo"): "/cargo", + } + # The deepest last, which rustc prefers where one directory holds another. + pairs = sorted(pairs.items(), key=lambda pair: len(pair[0])) + for found in rustflags_config(info["dir"], target): + raise SystemExit(f"{found} sets rustflags, which the reproducible build's RUSTFLAGS would replace: export RUSTFLAGS instead") + flags = os.environ.get("CARGO_ENCODED_RUSTFLAGS", "").split("\x1f") if "CARGO_ENCODED_RUSTFLAGS" in os.environ \ + else shlex.split(os.environ.get("RUSTFLAGS", "")) + flags += [f"--remap-path-prefix={old}={new}" for old, new in pairs] + env = {"SOURCE_DATE_EPOCH": str(info["epoch"])} + if "msvc" in target: + # link.exe stamps the build time into the DLL unless told to hash its content. + flags += ["-C", "link-arg=/Brepro"] + else: + env["CFLAGS"] = " ".join(filter(None, [os.environ.get("CFLAGS"), *(f"-ffile-prefix-map={old}={new}" for old, new in pairs)])) + if "apple" in target: + # Apple's ar and libtool stamp archive members with the time unless told not to. + env["ZERO_AR_DATE"] = "1" + env["CARGO_ENCODED_RUSTFLAGS"] = "\x1f".join(flags) + return env + + +def sbom(info, crate): + """A CycloneDX SBOM of the crates the binding is built from, the same for every host and target.""" + meta = json.loads(output("cargo", "metadata", "--locked", "--format-version", "1", "--features", "python", + "--manifest-path", crate)) + lock = tomllib.loads((Path(crate).parent / "Cargo.lock").read_text(encoding="utf-8")) + checksums = {(p["name"], p["version"], p.get("source")): p["checksum"] for p in lock["package"] if "checksum" in p} + packages = {p["id"]: p for p in meta["packages"]} + nodes = {n["id"]: n for n in meta["resolve"]["nodes"]} + purl = lambda p: f"pkg:cargo/{p['name']}@{p['version']}" # noqa: E731 + root = f"pkg:cargo/{info['name']}@{info['version']}" + # The generated crate is the package it exports: its dependencies are the package's. + ref = lambda i: root if i == meta["resolve"]["root"] or purl(packages[i]) == root else purl(packages[i]) # noqa: E731 + seen, queue, depends = set(), [meta["resolve"]["root"]], {} + while queue: + node = queue.pop() + if node in seen: + continue + seen.add(node) + # What the library links is its normal dependencies; a proc-macro or build script only runs at build time. + shipped = [d["pkg"] for d in nodes[node]["deps"] + if any(k["kind"] is None for k in d["dep_kinds"]) + and not any("proc-macro" in t["kind"] for t in packages[d["pkg"]]["targets"])] + depends.setdefault(ref(node), set()).update(ref(i) for i in shipped) + queue += shipped + components = {ref(i): packages[i] for i in seen if ref(i) != root} + def component(name, package): + out = {"type": "library", "bom-ref": name, "name": package["name"], "version": package["version"], "purl": name} + if package.get("license"): + # Cargo still takes the legacy MIT/Apache-2.0, which is no SPDX expression. + out["licenses"] = [{"license": {"name": package["license"]}} if "/" in package["license"] + else {"expression": package["license"]}] + if package.get("repository"): + out["externalReferences"] = [{"type": "vcs", "url": package["repository"]}] + checksum = checksums.get((package["name"], package["version"], package.get("source"))) + if checksum: + out["hashes"] = [{"alg": "SHA-256", "content": checksum}] + return out + document = { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "metadata": { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(info["epoch"])), + "component": {"type": "library", "bom-ref": root, "name": info["name"], "version": info["version"], "purl": root}, + }, + "components": [component(name, components[name]) for name in sorted(components)], + "dependencies": [{"ref": name, "dependsOn": sorted(depends[name] - {name})} for name in sorted(depends)], + } + return (json.dumps(document, indent=2) + "\n").encode() + + +def same(first, second): + """Fail unless two builds' wheels and static libraries are the same bytes, naming what differs.""" + found = {} + for root in (Path(first), Path(second)): + found[root] = {p.relative_to(root).as_posix(): p for pattern in ("wheels/*.whl", "static/*/*.a") for p in root.glob(pattern)} + names = sorted(found[Path(first)]) + if not names or names != sorted(found[Path(second)]): + raise SystemExit(f"the builds hold different packages: {names} against {sorted(found[Path(second)])}") + problems = [] + for name in names: + one, two = (found[Path(root)][name] for root in (first, second)) + if one.read_bytes() == two.read_bytes(): + continue + problems.append(name) + if name.endswith(".whl"): + with zipfile.ZipFile(one) as a, zipfile.ZipFile(two) as b: + entries = lambda z: {i.filename: (hashlib.sha256(z.read(i)).hexdigest(), i.date_time, i.external_attr) for i in z.infolist()} # noqa: E731 + left, right = entries(a), entries(b) + for entry in sorted(left.keys() | right.keys()): + if left.get(entry) != right.get(entry): + problems.append(f" {entry}: {left.get(entry)} against {right.get(entry)}") + if problems: + raise SystemExit("a rebuild made other bytes:\n" + "\n".join(problems)) + print(f"{len(names)} files rebuilt byte for byte") + + def cargo(manifest, *args, env=None): """Cargo with the package's own toolchain, as rustup finds it from the package's directory.""" return run("cargo", *args, cwd=Path(manifest).resolve().parent, env={**os.environ, **(env or {})}, @@ -485,12 +679,16 @@ def build(args): lib, export_dir, out = info["lib"], Path(args.export).resolve(), Path(args.out).resolve() crate = export_dir / "Cargo.toml" out.mkdir(parents=True, exist_ok=True) + info["sbom"] = sbom(info, crate) + (out / "sbom.cdx.json").write_bytes(info["sbom"]) for target in args.targets: if target not in WHEEL_TARGETS and target not in APPLE_TARGETS: raise SystemExit(f"no package ships {target}") run("rustup", "target", "add", target, cwd=Path(args.manifest_path).resolve().parent) if target in WHEEL_TARGETS: - env = {"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if "apple" in target else {} + env = reproducible(info, export_dir, target) + if "apple" in target: + env["MACOSX_DEPLOYMENT_TARGET"] = MACOS_WHEEL linux = "linux" in target # cargo-zigbuild links Linux against the glibc floor's symbols, whichever glibc the host has. built = cargo(args.manifest_path, *(["zigbuild"] if linux else ["build"]), "--locked", "--release", @@ -499,7 +697,7 @@ def build(args): sys.stderr.write(built.stderr) library = export_dir / "target-python" / target / "release" / library_name(lib, target) with tempfile.TemporaryDirectory() as raw: - repair(write_wheel(info, export_dir, library, target, raw), target, out / "wheels") + repair(write_wheel(info, export_dir, library, target, raw), target, out / "wheels", info["epoch"]) if target in APPLE_TARGETS: apple = APPLE_TARGETS[target] name = "MACOSX_DEPLOYMENT_TARGET" if apple == "macos" else "IPHONEOS_DEPLOYMENT_TARGET" @@ -510,7 +708,7 @@ def build(args): built = cargo(args.manifest_path, "rustc", "--color", "never", "--locked", "--release", "--manifest-path", crate, "--target", target, "--target-dir", export_dir / "target-swift", "--lib", "--crate-type", "staticlib", "--", "--print=native-static-libs", - env={name: SWIFT_PLATFORMS[apple]}) + env={**reproducible(info, export_dir, target), name: SWIFT_PLATFORMS[apple]}) sys.stderr.write(built.stderr) notes = re.findall(r"note: native-static-libs: (.*)", built.stderr) if len(notes) != 1: @@ -596,6 +794,9 @@ def main(): if name == "build": p.add_argument("--export", required=True) p.add_argument("--example", default="") + p = sub.add_parser("same", help="fail unless two builds made the same wheels and libraries") + p.add_argument("first") + p.add_argument("second") p = sub.add_parser("digest", help="print the hash of the generated bindings") p.add_argument("--manifest-path", required=True) p.add_argument("--export", required=True) @@ -637,6 +838,8 @@ def main(): export(args) elif args.command == "build": build(args) + elif args.command == "same": + same(args.first, args.second) elif args.command == "digest": print(digest(args.export, facts(args.manifest_path)["lib"])) elif args.command == "swift": diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 2c8c23bbd..e0d6940c2 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -2,8 +2,11 @@ name: Release @NAME@ bindings # Written by rust2's `ffi/release/package.py init`; yours to edit. rust2's builder, pinned # below, exports @NAME@'s pub API, builds its Python wheels and Swift package, and checks -# them in clean consumers, all read-only. The jobs after it publish only a tag's build: +# them in clean consumers, all read-only. The jobs after it publish only a tag's build, +# once `attest` has signed it: # +# - Attestations: a build provenance attestation and an SBOM attestation for every wheel and +# the XCFramework, from this repository's GitHub, checked with `gh attestation verify`. # - PyPI: the project @NAME@, through the trusted publisher this repository registers there # for this workflow and the `pypi` environment. # - SwiftPM: @MODULE@FFI.xcframework.zip on this repository's @NAME@-v release, which @@ -14,7 +17,7 @@ name: Release @NAME@ bindings # To release, push the tag @NAME@-v, naming the crate's version. Without a tag, as # from a nightly's workflow_call, every job up to publication runs. If publishing fails, use # "Re-run failed jobs": it reuses this run's build, and the publishing jobs upload only what -# is missing. A rebuild's files differ, so "Re-run all jobs" stops at any already published. +# is missing. A rebuild makes the same files, so "Re-run all jobs" works too. on: push: tags: @@ -52,9 +55,40 @@ jobs: exit 1 fi - pypi: + # Signs what the jobs below publish, before any of it is public, as this workflow's own + # build of this commit. The permissions are this job's alone. + attest: needs: [build, tag] runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: @NAME@-build-* + path: artifacts + merge-multiple: true + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-swift + path: artifacts/swift + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: | + artifacts/wheels/*.whl + artifacts/swift/*FFI.xcframework.zip + - uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + with: + subject-path: | + artifacts/wheels/*.whl + artifacts/swift/*FFI.xcframework.zip + sbom-path: artifacts/sbom.cdx.json + + pypi: + needs: [build, tag, attest] + runs-on: ubuntu-24.04 environment: name: pypi url: https://pypi.org/p/@NAME@ @@ -103,7 +137,7 @@ jobs: # resolved its checksum must never see other bytes at that URL. The release stays a draft # until its asset is up, as an immutable release takes no asset once published. github-release: - needs: [build, tag] + needs: [build, tag, attest] runs-on: ubuntu-24.04 permissions: contents: write diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 95af04a4c..2576d7fac 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -1,9 +1,13 @@ """The builder's decisions that need no toolchain: names, versions, metadata, and manifests.""" import importlib.util +import json +import os from pathlib import Path +import subprocess import tempfile import unittest +from unittest import mock import zipfile HERE = Path(__file__).resolve().parent @@ -18,6 +22,7 @@ def info(root, **overrides): "name": "moq-ffi", "version": "0.4.12", "description": "Media over QUIC,\n bindings", "license": "MIT OR Apache-2.0", "license_file": None, "repository": "https://github.com/moq-dev/moq", "homepage": None, "lib": "moq", "dir": str(root), "workspace": str(root), "examples": [], + "epoch": 1_700_000_000, "sbom": b"{}\n", } return {**base, **overrides} @@ -134,6 +139,165 @@ def test_floor_comes_from_the_glue(self): self.assertEqual(package.python_floor(export(Path(tmp), "312")), 12) +class Reproducible(unittest.TestCase): + def wheel(self, root, **touch): + """A wheel built from inputs of the given mtimes, whose repair tool put its files in any order and time.""" + built = export(root / "export") + library = root / "libmoq.so" + library.write_bytes(b"\x7fELF") + for path in (library, built / "python/moq.py"): + os.utime(path, (touch.get("time", 0), touch.get("time", 0))) + return package.write_wheel(info(root), built, library, "x86_64-unknown-linux-gnu", root) + + def test_wheel_bytes_ignore_when_and_where(self): + with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: + first = self.wheel(Path(a), time=1_000_000_000) + second = self.wheel(Path(b), time=1_600_000_000) + self.assertEqual(first.read_bytes(), second.read_bytes()) + with zipfile.ZipFile(first) as archive: + names = archive.namelist() + self.assertTrue(all(entry.date_time == (2023, 11, 14, 22, 13, 20) for entry in archive.infolist())) + self.assertTrue(names[-1].endswith(".dist-info/RECORD")) + self.assertEqual(names[:2], ["moq/__init__.py", "moq/libmoq.so"]) + + def test_a_repair_tools_wheel_is_rewritten_as_one_of_ours(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + wheel = self.wheel(root) + repaired = root / "repaired.whl" + with zipfile.ZipFile(wheel) as src, zipfile.ZipFile(repaired, "w") as dst: + # As a repair tool leaves it: directories, reversed order, its own times, and a RECORD of its own. + dst.writestr("moq/", b"") + for name in reversed(src.namelist()): + entry = zipfile.ZipInfo(name, (2031, 5, 6, 7, 8, 10)) + entry.external_attr = src.getinfo(name).external_attr + dst.writestr(entry, b"other\n" if name.endswith("/RECORD") else src.read(name)) + package.normalize_wheel(repaired, 1_700_000_000) + package.check_wheel(repaired) + self.assertEqual(repaired.read_bytes(), wheel.read_bytes()) + + def test_the_sbom_rides_in_the_wheel(self): + with tempfile.TemporaryDirectory() as tmp: + with zipfile.ZipFile(self.wheel(Path(tmp))) as archive: + self.assertEqual(archive.read("moq_ffi-0.4.12.dist-info/sboms/moq-ffi.cdx.json"), b"{}\n") + + def test_xcframework_zip_ignores_mtimes_and_umask(self): + with tempfile.TemporaryDirectory() as tmp: + zips = [] + for index, (mode, time) in enumerate(((0o644, 1), (0o664, 2_000_000_000))): + tree = Path(tmp) / str(index) / "MoqFFI.xcframework" + (tree / "macos").mkdir(parents=True) + (tree / "Info.plist").write_text("plist\n") + (tree / "macos/libmoq.a").write_bytes(b"!\n") + for path in (tree, tree / "macos", tree / "Info.plist", tree / "macos/libmoq.a"): + os.utime(path, (time, time)) + (tree / "Info.plist").chmod(mode) + package.zip_tree(tree, Path(tmp) / f"{index}.zip", 1_700_000_000) + zips.append((Path(tmp) / f"{index}.zip").read_bytes()) + self.assertEqual(zips[0], zips[1]) + with zipfile.ZipFile(Path(tmp) / "0.zip") as archive: + self.assertEqual(archive.namelist(), [ + "MoqFFI.xcframework/", "MoqFFI.xcframework/Info.plist", "MoqFFI.xcframework/macos/", + "MoqFFI.xcframework/macos/libmoq.a"]) + + def test_a_symlink_in_the_xcframework_fails(self): + with tempfile.TemporaryDirectory() as tmp: + tree = Path(tmp) / "MoqFFI.xcframework" + tree.mkdir() + (tree / "link").symlink_to("target") + with self.assertRaises(SystemExit): + package.zip_tree(tree, Path(tmp) / "out.zip", 1_700_000_000) + + def test_epoch_is_the_environment_or_the_commit(self): + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "1700000000"}): + self.assertEqual(package.epoch("."), 1_700_000_000) + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "86400"}), self.assertRaises(SystemExit): + package.epoch(".") + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ): + os.environ.pop("SOURCE_DATE_EPOCH", None) + git = ["git", "-C", tmp, "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run([*git, "init", "-q"], check=True) + subprocess.run([*git, "commit", "-q", "--allow-empty", "-m", "x", "--date", "2023-11-14T22:13:20Z"], + check=True, env={**os.environ, "GIT_COMMITTER_DATE": "2023-11-14T22:13:20Z"}) + self.assertEqual(package.epoch(tmp), 1_700_000_000) + + def test_paths_are_remapped_and_rustflags_kept(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"RUSTFLAGS": "-C target-cpu=native", "CARGO_HOME": "/home/u/.cargo"}): + os.environ.pop("CARGO_ENCODED_RUSTFLAGS", None) + env = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-unknown-linux-gnu") + flags = env["CARGO_ENCODED_RUSTFLAGS"].split("\x1f") + self.assertEqual(flags[:2], ["-C", "target-cpu=native"]) + self.assertIn(f"--remap-path-prefix={tmp}/export=/export", flags) + # The deeper directory comes last, as rustc prefers the last match. + self.assertLess(flags.index(f"--remap-path-prefix={tmp}=/source"), flags.index(f"--remap-path-prefix={tmp}/export=/export")) + self.assertEqual(env["SOURCE_DATE_EPOCH"], "1700000000") + self.assertIn(f"-ffile-prefix-map={tmp}=/source", env["CFLAGS"]) + windows = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-pc-windows-msvc") + self.assertIn("link-arg=/Brepro", windows["CARGO_ENCODED_RUSTFLAGS"]) + self.assertNotIn("CFLAGS", windows) + self.assertEqual(package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin")["ZERO_AR_DATE"], "1") + + def test_a_cargo_config_that_RUSTFLAGS_would_replace_fails(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_HOME": tmp + "/home"}): + crate = Path(tmp) / "crate" + (crate / ".cargo").mkdir(parents=True) + (crate / ".cargo/config.toml").write_text('[target.x86_64-pc-windows-msvc]\nrustflags = ["-C", "x"]\n') + self.assertEqual(list(package.rustflags_config(crate, "x86_64-unknown-linux-gnu")), []) + self.assertEqual(len(list(package.rustflags_config(crate, "x86_64-pc-windows-msvc"))), 1) + (crate / ".cargo/config.toml").write_text('[build]\nrustflags = ["-C", "x"]\n') + with self.assertRaises(SystemExit): + package.reproducible(info(crate), crate / "export", "x86_64-unknown-linux-gnu") + + def test_same_names_what_differs(self): + with tempfile.TemporaryDirectory() as tmp: + roots = [Path(tmp) / "a", Path(tmp) / "b"] + for root in roots: + (root / "static/aarch64-apple-ios").mkdir(parents=True) + (root / "static/aarch64-apple-ios/libmoq.a").write_bytes(b"same") + (root / "wheels").mkdir() + first = self.wheel(roots[0]) + second = self.wheel(roots[1]) + for root, wheel in zip(roots, (first, second)): + wheel.replace(root / "wheels" / wheel.name) + package.same(roots[0], roots[1]) + (roots[1] / "static/aarch64-apple-ios/libmoq.a").write_bytes(b"other") + with self.assertRaisesRegex(SystemExit, "static/aarch64-apple-ios/libmoq.a"): + package.same(roots[0], roots[1]) + + +class Sbom(unittest.TestCase): + def crate(self, root): + """A generated crate whose normal, proc-macro, and build dependencies are each a path crate.""" + for name, extra in (("dep", ""), ("builder", ""), ("derive", "[lib]\nproc-macro = true\n")): + (root / name / "src").mkdir(parents=True) + (root / name / "src/lib.rs").write_text("") + (root / name / "Cargo.toml").write_text(f'[package]\nname = "{name}"\nversion = "1.2.3"\nedition = "2021"\nlicense = "MIT/Apache-2.0"\n{extra}') + (root / "glue/src").mkdir(parents=True) + (root / "glue/src/lib.rs").write_text("") + (root / "glue/Cargo.toml").write_text( + '[package]\nname = "glue"\nversion = "0.0.0"\nedition = "2021"\n[features]\npython = []\n' + '[dependencies]\ndep = { path = "../dep" }\nderive = { path = "../derive" }\n' + '[build-dependencies]\nbuilder = { path = "../builder" }\n[workspace]\n') + subprocess.run(["cargo", "generate-lockfile", "--offline", "--manifest-path", root / "glue/Cargo.toml"], check=True, + capture_output=True) + return root / "glue/Cargo.toml" + + def test_what_ships_is_what_the_library_links(self): + with tempfile.TemporaryDirectory() as tmp: + crate = self.crate(Path(tmp)) + first = package.sbom(info(tmp, name="moq-ffi", version="0.4.12"), crate) + self.assertEqual(first, package.sbom(info(tmp, name="moq-ffi", version="0.4.12"), crate)) + document = json.loads(first) + self.assertEqual(document["bomFormat"], "CycloneDX") + self.assertEqual(document["metadata"]["timestamp"], "2023-11-14T22:13:20Z") + self.assertEqual([c["name"] for c in document["components"]], ["dep"]) + self.assertEqual(document["components"][0]["licenses"], [{"license": {"name": "MIT/Apache-2.0"}}]) + self.assertEqual(document["dependencies"], [ + {"ref": "pkg:cargo/dep@1.2.3", "dependsOn": []}, + {"ref": "pkg:cargo/moq-ffi@0.4.12", "dependsOn": ["pkg:cargo/dep@1.2.3"]}, + ]) + + class Digest(unittest.TestCase): def test_any_byte_splits_the_hosts(self): with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: diff --git a/quest/m1/ffi-packaging/relocatable.md b/quest/m1/ffi-packaging/relocatable.md new file mode 100644 index 000000000..d366c0361 --- /dev/null +++ b/quest/m1/ffi-packaging/relocatable.md @@ -0,0 +1,26 @@ +# [S] A release rebuilds the same from any checkout path + +## Goal + +The generated crate's version hashes its `Cargo.toml`, which names the facade by absolute +path, and cargo hashes that version into every symbol: two checkouts of one commit at +different paths make different libraries. The exporter writes the dependency relative to +its output, so [a rebuild](/ffi/release/README.md#reproducible-builds) at any path makes the +same bytes, and `reproduce` in `bindings-test.yml` builds the second copy elsewhere. + +## Plan + +Found by [the reproducible-releases quest](https://github.com/kixel-ai/rust2/pull/874): +`glue.rs`'s `generate` writes `path = {facade.dir}`. Measured there: two exports of the api +fixture from copies at different paths differ in the version's hash and in every symbol +hash; the same path with another export directory is byte for byte equal. + +Fall back to the absolute path where no relative one exists, such as another Windows drive. +Check `cli.rs`'s other absolute manifest paths (its "each path absolute" note) for the same +dependence. + +Public API: none. + +## Related + +- [Release every language the exporter generates](/quest/m1/ffi-packaging/README.md) - the epic whose builds this makes checkable by anyone diff --git a/quest/m1/ffi-packaging/reproducible.md b/quest/m1/ffi-packaging/reproducible.md deleted file mode 100644 index c3ad376cb..000000000 --- a/quest/m1/ffi-packaging/reproducible.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] Reproducible, attested releases - -## Goal - -A rebuild of a release makes the same bytes: wheels, archives, and the XCFramework zip -take `SOURCE_DATE_EPOCH` timestamps, a deterministic file order, and remapped paths, so -**Re-run all jobs** succeeds. Every artifact gets a build provenance attestation -(`actions/attest-build-provenance`) and an SBOM. - -## Plan - -Found by the 2026-11-18 language audit: `package.py:208`'s `writestr` stamps the current -time and the XCFramework is zipped with `ditto`, which `ffi/release/README.md` admits -makes rebuilds differ. PyPI's trusted publishing already uploads PEP 740 attestations; the -other artifacts have none. - -Public API: none. From 815fbffe0aa26cf62cf35f9524120019f56ced76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:47:06 -0800 Subject: [PATCH 03/20] tmp: diagnose Apple rebuild differences Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings.yml | 15 +++++++++++++++ .github/workflows/tmp-api.yml | 14 ++++++++++++++ ffi/release/package.py | 19 ++++++++++++------- 3 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/tmp-api.yml diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 146995ad3..46476daf0 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -273,6 +273,21 @@ jobs: python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --out again/export "${targets[@]}" python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export again/export --out again/dist "${targets[@]}" python3 -I rust2/ffi/release/package.py same dist again/dist + - name: TMP why + if: failure() && inputs.reproduce && runner.os == 'macOS' + run: | + mkdir -p why/a why/b + unzip -q -o dist/wheels/*.whl -d why/a + unzip -q -o again/dist/wheels/*.whl -d why/b + for f in why/a/*/*.dylib; do + g=why/b/${f#why/a/} + cmp -l "$f" "$g" | wc -l + cmp -l "$f" "$g" | head -3; cmp -l "$f" "$g" | tail -3 + diff <(otool -l "$f") <(otool -l "$g") | head -60 + diff <(strings -a "$f") <(strings -a "$g") | head -40 + diff <(codesign -dvvv "$f" 2>&1) <(codesign -dvvv "$g" 2>&1) | head + diff <(nm -a "$f") <(nm -a "$g") | head -20 + done - name: Checks env: PYTHON_CHECK: ${{ inputs.python-check }} diff --git a/.github/workflows/tmp-api.yml b/.github/workflows/tmp-api.yml new file mode 100644 index 000000000..ac3aacec8 --- /dev/null +++ b/.github/workflows/tmp-api.yml @@ -0,0 +1,14 @@ +name: tmp api +on: + workflow_dispatch: +permissions: + contents: read +jobs: + api: + uses: ./.github/workflows/bindings.yml + with: + manifest-path: ffi/fixtures/api/Cargo.toml + features: doubled + example: transcript + python-check: ffi/tests/api.py + reproduce: true diff --git a/ffi/release/package.py b/ffi/release/package.py index 3d33199f2..20ca8187f 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -437,7 +437,8 @@ def swift(info, export, libs, natives, out): run(*args, "-output", tree) zip_tree(tree, zipped, info["epoch"]) if archive.read_bytes() != (again / archive.name).read_bytes(): - raise SystemExit("xcodebuild made different XCFrameworks from the same libraries") + raise SystemExit("xcodebuild made different XCFrameworks from the same libraries:\n" + + "\n".join(zip_diff(archive, again / archive.name))) shutil.rmtree(headers) shutil.rmtree(again) checksum = hashlib.sha256(archive.read_bytes()).hexdigest() @@ -641,6 +642,15 @@ def component(name, package): return (json.dumps(document, indent=2) + "\n").encode() +def zip_diff(one, two): + """The entries two ZIPs hold differently, each with its content hash, time, and attributes.""" + with zipfile.ZipFile(one) as a, zipfile.ZipFile(two) as b: + entries = lambda z: {i.filename: (hashlib.sha256(z.read(i)).hexdigest(), i.date_time, i.external_attr) for i in z.infolist()} # noqa: E731 + left, right = entries(a), entries(b) + return [f" {name}: {left.get(name)} against {right.get(name)}" + for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] + + def same(first, second): """Fail unless two builds' wheels and static libraries are the same bytes, naming what differs.""" found = {} @@ -656,12 +666,7 @@ def same(first, second): continue problems.append(name) if name.endswith(".whl"): - with zipfile.ZipFile(one) as a, zipfile.ZipFile(two) as b: - entries = lambda z: {i.filename: (hashlib.sha256(z.read(i)).hexdigest(), i.date_time, i.external_attr) for i in z.infolist()} # noqa: E731 - left, right = entries(a), entries(b) - for entry in sorted(left.keys() | right.keys()): - if left.get(entry) != right.get(entry): - problems.append(f" {entry}: {left.get(entry)} against {right.get(entry)}") + problems += zip_diff(one, two) if problems: raise SystemExit("a rebuild made other bytes:\n" + "\n".join(problems)) print(f"{len(names)} files rebuilt byte for byte") From de9f5030edd30f6bff8a41b67eb495b3e58f3800 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:47:12 -0800 Subject: [PATCH 04/20] tmp: api fixture only Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings-test.yml | 7 ------- .github/workflows/tmp-api.yml | 14 -------------- 2 files changed, 21 deletions(-) delete mode 100644 .github/workflows/tmp-api.yml diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index 7ca30c772..8d251b67b 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -23,10 +23,3 @@ jobs: # No swift-check: api.swift compiles against the fixture's own test export, of the host # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. - moq: - name: moq stand-in - uses: ./.github/workflows/bindings.yml - with: - manifest-path: ffi/target/moq-ffi/Cargo.toml - python-check: ffi/moq/moq.py - swift-check: ffi/moq/moq.swift diff --git a/.github/workflows/tmp-api.yml b/.github/workflows/tmp-api.yml deleted file mode 100644 index ac3aacec8..000000000 --- a/.github/workflows/tmp-api.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: tmp api -on: - workflow_dispatch: -permissions: - contents: read -jobs: - api: - uses: ./.github/workflows/bindings.yml - with: - manifest-path: ffi/fixtures/api/Cargo.toml - features: doubled - example: transcript - python-check: ffi/tests/api.py - reproduce: true From 448ec1ea9255c153f934d01a6ed9c641840eb1e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 02:57:14 -0800 Subject: [PATCH 05/20] tmp: upload the differing dylibs Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 46476daf0..96da6b15a 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -279,15 +279,20 @@ jobs: mkdir -p why/a why/b unzip -q -o dist/wheels/*.whl -d why/a unzip -q -o again/dist/wheels/*.whl -d why/b + set +e for f in why/a/*/*.dylib; do g=why/b/${f#why/a/} - cmp -l "$f" "$g" | wc -l - cmp -l "$f" "$g" | head -3; cmp -l "$f" "$g" | tail -3 + cmp -l "$f" "$g" | awk '{printf "%s ", $1}'; echo diff <(otool -l "$f") <(otool -l "$g") | head -60 diff <(strings -a "$f") <(strings -a "$g") | head -40 diff <(codesign -dvvv "$f" 2>&1) <(codesign -dvvv "$g" 2>&1) | head - diff <(nm -a "$f") <(nm -a "$g") | head -20 done + exit 0 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: failure() && inputs.reproduce && runner.os == 'macOS' + with: + name: why + path: why - name: Checks env: PYTHON_CHECK: ${{ inputs.python-check }} From 57d6cf81f18abca1702efde8bf72c420da423f9f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:09:46 -0800 Subject: [PATCH 06/20] Name the dylib by rpath, and stamp no time in xcodebuild's libtool Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings-test.yml | 7 +++++++ ffi/release/package.py | 7 +++++-- ffi/release/test_package.py | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bindings-test.yml b/.github/workflows/bindings-test.yml index 8d251b67b..7ca30c772 100644 --- a/.github/workflows/bindings-test.yml +++ b/.github/workflows/bindings-test.yml @@ -23,3 +23,10 @@ jobs: # No swift-check: api.swift compiles against the fixture's own test export, of the host # and wasm32-wasip1, not this one's targets. Its transcript stays the exporter's tests'. + moq: + name: moq stand-in + uses: ./.github/workflows/bindings.yml + with: + manifest-path: ffi/target/moq-ffi/Cargo.toml + python-check: ffi/moq/moq.py + swift-check: ffi/moq/moq.swift diff --git a/ffi/release/package.py b/ffi/release/package.py index 20ca8187f..cc666e7dd 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -434,7 +434,8 @@ def swift(info, export, libs, natives, out): args = ["xcodebuild", "-create-xcframework"] for target in APPLE_TARGETS: args += ["-library", libs[target], "-headers", headers] - run(*args, "-output", tree) + # libtool, which xcodebuild runs on each library, stamps the time unless told not to. + run(*args, "-output", tree, env={**os.environ, "ZERO_AR_DATE": "1"}) zip_tree(tree, zipped, info["epoch"]) if archive.read_bytes() != (again / archive.name).read_bytes(): raise SystemExit("xcodebuild made different XCFrameworks from the same libraries:\n" @@ -585,8 +586,10 @@ def reproducible(info, export_dir, target): else: env["CFLAGS"] = " ".join(filter(None, [os.environ.get("CFLAGS"), *(f"-ffile-prefix-map={old}={new}" for old, new in pairs)])) if "apple" in target: - # Apple's ar and libtool stamp archive members with the time unless told not to. + # Apple's ar and libtool stamp archive members with the time unless told not to, and + # the linker names a dylib after the path it wrote it to, in the export's build directory. env["ZERO_AR_DATE"] = "1" + flags += ["-C", f"link-arg=-Wl,-install_name,@rpath/{library_name(info['lib'], target)}"] env["CARGO_ENCODED_RUSTFLAGS"] = "\x1f".join(flags) return env diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 2576d7fac..6af04426a 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -235,7 +235,9 @@ def test_paths_are_remapped_and_rustflags_kept(self): windows = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-pc-windows-msvc") self.assertIn("link-arg=/Brepro", windows["CARGO_ENCODED_RUSTFLAGS"]) self.assertNotIn("CFLAGS", windows) - self.assertEqual(package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin")["ZERO_AR_DATE"], "1") + apple = package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin") + self.assertEqual(apple["ZERO_AR_DATE"], "1") + self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.dylib", apple["CARGO_ENCODED_RUSTFLAGS"]) def test_a_cargo_config_that_RUSTFLAGS_would_replace_fails(self): with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_HOME": tmp + "/home"}): From 50d316a92e8387d8c523f38afda040eb3089b1ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:41:15 -0800 Subject: [PATCH 07/20] Sort the XCFramework's libraries, which xcodebuild lists at random Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings.yml | 20 -------------------- ffi/release/package.py | 11 +++++++++++ ffi/release/test_package.py | 12 ++++++++++++ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 96da6b15a..146995ad3 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -273,26 +273,6 @@ jobs: python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --out again/export "${targets[@]}" python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export again/export --out again/dist "${targets[@]}" python3 -I rust2/ffi/release/package.py same dist again/dist - - name: TMP why - if: failure() && inputs.reproduce && runner.os == 'macOS' - run: | - mkdir -p why/a why/b - unzip -q -o dist/wheels/*.whl -d why/a - unzip -q -o again/dist/wheels/*.whl -d why/b - set +e - for f in why/a/*/*.dylib; do - g=why/b/${f#why/a/} - cmp -l "$f" "$g" | awk '{printf "%s ", $1}'; echo - diff <(otool -l "$f") <(otool -l "$g") | head -60 - diff <(strings -a "$f") <(strings -a "$g") | head -40 - diff <(codesign -dvvv "$f" 2>&1) <(codesign -dvvv "$g" 2>&1) | head - done - exit 0 - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - if: failure() && inputs.reproduce && runner.os == 'macOS' - with: - name: why - path: why - name: Checks env: PYTHON_CHECK: ${{ inputs.python-check }} diff --git a/ffi/release/package.py b/ffi/release/package.py index cc666e7dd..0b39ea851 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -14,6 +14,7 @@ import json import os from pathlib import Path +import plistlib import re import shlex import shutil @@ -389,6 +390,15 @@ def min_os(archive): return max(versions) +def sort_plist(path): + """Rewrite an XCFramework's Info.plist with its libraries in name order, which xcodebuild lists in a random one.""" + with open(path, "rb") as source: + plist = plistlib.load(source) + plist["AvailableLibraries"].sort(key=lambda library: library["LibraryIdentifier"]) + with open(path, "wb") as sink: + plistlib.dump(plist, sink, sort_keys=True) + + def zip_tree(root, path, stamp): """The directory `root` as a ZIP holding it as its top-level directory, as `ditto --keepParent` does.""" entries = [(f"{root.name}/", b"", 0o755)] @@ -436,6 +446,7 @@ def swift(info, export, libs, natives, out): args += ["-library", libs[target], "-headers", headers] # libtool, which xcodebuild runs on each library, stamps the time unless told not to. run(*args, "-output", tree, env={**os.environ, "ZERO_AR_DATE": "1"}) + sort_plist(tree / "Info.plist") zip_tree(tree, zipped, info["epoch"]) if archive.read_bytes() != (again / archive.name).read_bytes(): raise SystemExit("xcodebuild made different XCFrameworks from the same libraries:\n" diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 6af04426a..f6a92195e 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -200,6 +200,18 @@ def test_xcframework_zip_ignores_mtimes_and_umask(self): "MoqFFI.xcframework/", "MoqFFI.xcframework/Info.plist", "MoqFFI.xcframework/macos/", "MoqFFI.xcframework/macos/libmoq.a"]) + def test_xcframework_plist_lists_libraries_in_name_order(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "Info.plist" + libraries = [{"LibraryIdentifier": name} for name in ("macos-arm64", "ios-arm64-simulator", "ios-arm64")] + for order in (libraries, libraries[::-1]): + path.write_bytes(package.plistlib.dumps({"AvailableLibraries": order, "CFBundlePackageType": "XFWK"})) + package.sort_plist(path) + first = path.read_bytes() if order is libraries else first + self.assertEqual(path.read_bytes(), first) + self.assertEqual([x["LibraryIdentifier"] for x in package.plistlib.loads(first)["AvailableLibraries"]], + ["ios-arm64", "ios-arm64-simulator", "macos-arm64"]) + def test_a_symlink_in_the_xcframework_fails(self): with tempfile.TemporaryDirectory() as tmp: tree = Path(tmp) / "MoqFFI.xcframework" From 1c28382725c8d91cf444e61c553961b6c0a7918f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 05:16:50 -0800 Subject: [PATCH 08/20] Keep cargo's RUSTFLAGS tokenization and each dependency source apart in the SBOM Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/package.py | 17 ++++++++++++++--- ffi/release/test_package.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index 0b39ea851..6ce372e6c 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -23,6 +23,7 @@ import tempfile import time import tomllib +import urllib.parse import zipfile HERE = Path(__file__).resolve().parent @@ -48,6 +49,7 @@ ] # Cargo metadata names a library target by its crate types; a proc-macro is no library to export. LIBRARY_KINDS = {"lib", "rlib", "dylib", "cdylib", "staticlib"} +CRATES_IO = "registry+https://github.com/rust-lang/crates.io-index" COLLECTIONS = "https://github.com/apple/swift-collections" # The first date a ZIP entry can hold, and so the earliest SOURCE_DATE_EPOCH a build takes. ZIP_EPOCH = 315532800 @@ -588,7 +590,7 @@ def reproducible(info, export_dir, target): for found in rustflags_config(info["dir"], target): raise SystemExit(f"{found} sets rustflags, which the reproducible build's RUSTFLAGS would replace: export RUSTFLAGS instead") flags = os.environ.get("CARGO_ENCODED_RUSTFLAGS", "").split("\x1f") if "CARGO_ENCODED_RUSTFLAGS" in os.environ \ - else shlex.split(os.environ.get("RUSTFLAGS", "")) + else os.environ.get("RUSTFLAGS", "").split() flags += [f"--remap-path-prefix={old}={new}" for old, new in pairs] env = {"SOURCE_DATE_EPOCH": str(info["epoch"])} if "msvc" in target: @@ -613,7 +615,16 @@ def sbom(info, crate): checksums = {(p["name"], p["version"], p.get("source")): p["checksum"] for p in lock["package"] if "checksum" in p} packages = {p["id"]: p for p in meta["packages"]} nodes = {n["id"]: n for n in meta["resolve"]["nodes"]} - purl = lambda p: f"pkg:cargo/{p['name']}@{p['version']}" # noqa: E731 + def purl(package): + """The crate's package URL, qualified by its source unless that is crates.io, so two sources of one version stay two components.""" + base = f"pkg:cargo/{package['name']}@{package['version']}" + source = package.get("source") or "" + if source.startswith("registry+") and source != CRATES_IO: + return f"{base}?repository_url={urllib.parse.quote(source[len('registry+'):], safe='')}" + if source.startswith("git+"): + return f"{base}?vcs_url={urllib.parse.quote(source, safe='')}" + return base + root = f"pkg:cargo/{info['name']}@{info['version']}" # The generated crate is the package it exports: its dependencies are the package's. ref = lambda i: root if i == meta["resolve"]["root"] or purl(packages[i]) == root else purl(packages[i]) # noqa: E731 @@ -629,7 +640,7 @@ def sbom(info, crate): and not any("proc-macro" in t["kind"] for t in packages[d["pkg"]]["targets"])] depends.setdefault(ref(node), set()).update(ref(i) for i in shipped) queue += shipped - components = {ref(i): packages[i] for i in seen if ref(i) != root} + components = {ref(i): packages[i] for i in sorted(seen) if ref(i) != root} def component(name, package): out = {"type": "library", "bom-ref": name, "name": package["name"], "version": package["version"], "purl": name} if package.get("license"): diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index f6a92195e..5c8271c89 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -4,6 +4,7 @@ import json import os from pathlib import Path +import shutil import subprocess import tempfile import unittest @@ -234,11 +235,12 @@ def test_epoch_is_the_environment_or_the_commit(self): self.assertEqual(package.epoch(tmp), 1_700_000_000) def test_paths_are_remapped_and_rustflags_kept(self): - with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"RUSTFLAGS": "-C target-cpu=native", "CARGO_HOME": "/home/u/.cargo"}): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"RUSTFLAGS": "-C target-cpu=native -Lnative=C:\\deps", "CARGO_HOME": "/home/u/.cargo"}): os.environ.pop("CARGO_ENCODED_RUSTFLAGS", None) env = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-unknown-linux-gnu") flags = env["CARGO_ENCODED_RUSTFLAGS"].split("\x1f") - self.assertEqual(flags[:2], ["-C", "target-cpu=native"]) + # Cargo splits on whitespace alone, so a Windows path keeps its backslash. + self.assertEqual(flags[:3], ["-C", "target-cpu=native", "-Lnative=C:\\deps"]) self.assertIn(f"--remap-path-prefix={tmp}/export=/export", flags) # The deeper directory comes last, as rustc prefers the last match. self.assertLess(flags.index(f"--remap-path-prefix={tmp}=/source"), flags.index(f"--remap-path-prefix={tmp}/export=/export")) @@ -312,6 +314,28 @@ def test_what_ships_is_what_the_library_links(self): ]) + def test_one_version_from_two_sources_is_two_components(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + crate = self.crate(root) + # The same name and version again, from a git repository. + fork = root / "fork" + shutil.copytree(root / "dep", fork) + git = ["git", "-C", str(fork), "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run([*git, "init", "-q"], check=True) + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "x"], check=True) + manifest = crate.read_text().replace("[build-dependencies]", f'forked = {{ package = "dep", git = "file://{fork}" }}\n[build-dependencies]') + crate.write_text(manifest) + # A local repository, which cargo fetches without a network. + subprocess.run(["cargo", "generate-lockfile", "--manifest-path", crate], check=True, capture_output=True) + document = json.loads(package.sbom(info(tmp, name="moq-ffi", version="0.4.12"), crate)) + refs = [c["bom-ref"] for c in document["components"]] + self.assertEqual(len(refs), 2) + self.assertIn("pkg:cargo/dep@1.2.3", refs) + self.assertTrue(any(ref.startswith("pkg:cargo/dep@1.2.3?vcs_url=git%2Bfile") for ref in refs), refs) + + class Digest(unittest.TestCase): def test_any_byte_splits_the_hosts(self): with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: From 6612a324833c34bce781f4c9de1aff5b53cb6209 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 11:37:05 -0800 Subject: [PATCH 09/20] Settle the review's findings on the reproducible build Qualify sparse registries in the SBOM, read an empty CARGO_ENCODED_RUSTFLAGS as no flags, refuse a whitespace path that cc would split in CFLAGS, say ZERO_AR_DATE is Apple's, and attest through actions/attest. Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/README.md | 5 ++--- ffi/release/package.py | 34 ++++++++++++++++++++++------------ ffi/release/release.yml | 4 ++-- ffi/release/test_package.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index cd1cc93e2..c83434e58 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -90,9 +90,8 @@ holding exactly the files it would upload, and anyone with the commit can check - **Paths.** The crate's workspace, the exported crate, and `CARGO_HOME` are remapped to fixed names for rustc and the C compilers, so panic messages and debug info name no checkout. This takes `RUSTFLAGS` over a `.cargo/config.toml`'s `rustflags`, so a config - that sets any fails the build: export them as `RUSTFLAGS`, which the builder keeps. Linux - and macOS builds also stamp no archive time (`ZERO_AR_DATE`), and Windows links with - `/Brepro`. + that sets any fails the build: export them as `RUSTFLAGS`, which the builder keeps. Apple + builds also stamp no archive time (`ZERO_AR_DATE`), and Windows links with `/Brepro`. - **Checked.** The Swift job builds the XCFramework twice and fails unless both ZIPs match. With `reproduce: true`, every build host builds a second time, from a new export, and `package.py same` fails unless each wheel and static library matches byte for byte; rust2 diff --git a/ffi/release/package.py b/ffi/release/package.py index 6ce372e6c..d92ab7fd7 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -589,14 +589,22 @@ def reproducible(info, export_dir, target): pairs = sorted(pairs.items(), key=lambda pair: len(pair[0])) for found in rustflags_config(info["dir"], target): raise SystemExit(f"{found} sets rustflags, which the reproducible build's RUSTFLAGS would replace: export RUSTFLAGS instead") - flags = os.environ.get("CARGO_ENCODED_RUSTFLAGS", "").split("\x1f") if "CARGO_ENCODED_RUSTFLAGS" in os.environ \ - else os.environ.get("RUSTFLAGS", "").split() + encoded = os.environ.get("CARGO_ENCODED_RUSTFLAGS") + if encoded is None: + flags = os.environ.get("RUSTFLAGS", "").split() + else: + # Cargo reads an empty value as no flags, not as one empty flag. + flags = encoded.split("\x1f") if encoded else [] flags += [f"--remap-path-prefix={old}={new}" for old, new in pairs] env = {"SOURCE_DATE_EPOCH": str(info["epoch"])} if "msvc" in target: # link.exe stamps the build time into the DLL unless told to hash its content. flags += ["-C", "link-arg=/Brepro"] else: + # cc splits CFLAGS on whitespace, and quoting them would change how a caller's own flags read. + for old, _ in pairs: + if any(c.isspace() for c in old): + raise SystemExit(f"{old} holds whitespace, which would split its -ffile-prefix-map in CFLAGS: build from a path without it") env["CFLAGS"] = " ".join(filter(None, [os.environ.get("CFLAGS"), *(f"-ffile-prefix-map={old}={new}" for old, new in pairs)])) if "apple" in target: # Apple's ar and libtool stamp archive members with the time unless told not to, and @@ -607,6 +615,18 @@ def reproducible(info, export_dir, target): return env +def purl(package): + """The crate's package URL, qualified by its source unless that is crates.io, so two sources of one version stay two components.""" + base = f"pkg:cargo/{package['name']}@{package['version']}" + source = package.get("source") or "" + # An alternate registry may speak the git or the sparse protocol; crates.io reports the former either way. + if source.startswith(("registry+", "sparse+")) and source != CRATES_IO: + return f"{base}?repository_url={urllib.parse.quote(source.split('+', 1)[1], safe='')}" + if source.startswith("git+"): + return f"{base}?vcs_url={urllib.parse.quote(source, safe='')}" + return base + + def sbom(info, crate): """A CycloneDX SBOM of the crates the binding is built from, the same for every host and target.""" meta = json.loads(output("cargo", "metadata", "--locked", "--format-version", "1", "--features", "python", @@ -615,16 +635,6 @@ def sbom(info, crate): checksums = {(p["name"], p["version"], p.get("source")): p["checksum"] for p in lock["package"] if "checksum" in p} packages = {p["id"]: p for p in meta["packages"]} nodes = {n["id"]: n for n in meta["resolve"]["nodes"]} - def purl(package): - """The crate's package URL, qualified by its source unless that is crates.io, so two sources of one version stay two components.""" - base = f"pkg:cargo/{package['name']}@{package['version']}" - source = package.get("source") or "" - if source.startswith("registry+") and source != CRATES_IO: - return f"{base}?repository_url={urllib.parse.quote(source[len('registry+'):], safe='')}" - if source.startswith("git+"): - return f"{base}?vcs_url={urllib.parse.quote(source, safe='')}" - return base - root = f"pkg:cargo/{info['name']}@{info['version']}" # The generated crate is the package it exports: its dependencies are the package's. ref = lambda i: root if i == meta["resolve"]["root"] or purl(packages[i]) == root else purl(packages[i]) # noqa: E731 diff --git a/ffi/release/release.yml b/ffi/release/release.yml index e0d6940c2..f14028b3d 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -74,12 +74,12 @@ jobs: with: name: @NAME@-swift path: artifacts/swift - - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: | artifacts/wheels/*.whl artifacts/swift/*FFI.xcframework.zip - - uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: | artifacts/wheels/*.whl diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 5c8271c89..dc4c942f3 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -253,6 +253,22 @@ def test_paths_are_remapped_and_rustflags_kept(self): self.assertEqual(apple["ZERO_AR_DATE"], "1") self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.dylib", apple["CARGO_ENCODED_RUSTFLAGS"]) + def test_an_empty_encoded_rustflags_is_no_flags(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_ENCODED_RUSTFLAGS": "", "RUSTFLAGS": "-C x", "CARGO_HOME": tmp + "/home"}): + env = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-unknown-linux-gnu") + flags = env["CARGO_ENCODED_RUSTFLAGS"].split("\x1f") + self.assertTrue(flags[0].startswith("--remap-path-prefix="), flags) + self.assertNotIn("-C", flags) + + def test_a_path_cc_would_split_fails(self): + with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_HOME": tmp + "/home"}): + os.environ.pop("CARGO_ENCODED_RUSTFLAGS", None) + spaced = Path(tmp) / "a b" + with self.assertRaises(SystemExit): + package.reproducible(info(tmp), spaced, "x86_64-unknown-linux-gnu") + # MSVC's C flags are not CFLAGS, so a space there is harmless. + package.reproducible(info(tmp), spaced, "x86_64-pc-windows-msvc") + def test_a_cargo_config_that_RUSTFLAGS_would_replace_fails(self): with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_HOME": tmp + "/home"}): crate = Path(tmp) / "crate" @@ -336,6 +352,20 @@ def test_one_version_from_two_sources_is_two_components(self): self.assertTrue(any(ref.startswith("pkg:cargo/dep@1.2.3?vcs_url=git%2Bfile") for ref in refs), refs) +class Purl(unittest.TestCase): + def test_each_registry_is_its_own_package(self): + def purl(source): + return package.purl({"name": "dep", "version": "1.2.3", "source": source}) + crates_io = "registry+https://github.com/rust-lang/crates.io-index" + self.assertEqual(purl(crates_io), "pkg:cargo/dep@1.2.3") + self.assertEqual(purl(None), "pkg:cargo/dep@1.2.3") + git = purl("registry+https://example.com/index") + sparse = purl("sparse+https://example.com/index/") + self.assertIn("?repository_url=https%3A%2F%2Fexample.com%2Findex", git) + self.assertIn("?repository_url=https%3A%2F%2Fexample.com%2Findex%2F", sparse) + self.assertEqual(len({purl(crates_io), git, sparse}), 3) + + class Digest(unittest.TestCase): def test_any_byte_splits_the_hosts(self): with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: From cf063906899af90ee92ca308a40a3c5c0f6366f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 12:53:52 -0800 Subject: [PATCH 10/20] Attest the Go module, and map C++'s paths as C's Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/README.md | 4 ++-- ffi/release/package.py | 4 +++- ffi/release/release.yml | 13 +++++++++++-- ffi/release/test_package.py | 2 ++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index 4092400d5..d4a2ac280 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -130,8 +130,8 @@ holding exactly the files it would upload, and anyone with the commit can check the crates the binding links (normal dependencies, platform-neutral, with each registry crate's checksum from `Cargo.lock`), which every wheel also carries in `.dist-info/sboms/` (PEP 770) and the build hosts must agree on. The caller's `attest` job -runs on a tag, before anything is published, and gives every wheel and the XCFramework a -build provenance attestation and an SBOM attestation, which `gh attestation verify +runs on a tag, before anything is published, and gives every wheel, the XCFramework, and the Go +module's `.zip` and `.mod` a build provenance attestation and an SBOM attestation, which `gh attestation verify --repo /` checks. Only that job holds `id-token` and `attestations` write; the builder stays read-only. PyPI's trusted publishing uploads its own PEP 740 attestations for the wheels as well. diff --git a/ffi/release/package.py b/ffi/release/package.py index f2c63ad1c..ab34544f4 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -628,7 +628,9 @@ def reproducible(info, export_dir, target): for old, _ in pairs: if any(c.isspace() for c in old): raise SystemExit(f"{old} holds whitespace, which would split its -ffile-prefix-map in CFLAGS: build from a path without it") - env["CFLAGS"] = " ".join(filter(None, [os.environ.get("CFLAGS"), *(f"-ffile-prefix-map={old}={new}" for old, new in pairs)])) + # cc reads CXXFLAGS, not CFLAGS, for C++. + for var in ("CFLAGS", "CXXFLAGS"): + env[var] = " ".join(filter(None, [os.environ.get(var), *(f"-ffile-prefix-map={old}={new}" for old, new in pairs)])) if "apple" in target: # Apple's ar and libtool stamp archive members with the time unless told not to, and # the linker names a dylib after the path it wrote it to, in the export's build directory. diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 59bb6fec4..34f1b83d4 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -5,8 +5,9 @@ name: Release @NAME@ bindings # them in clean consumers, all read-only. The jobs after it publish only a tag's build, # once `attest` has signed it: # -# - Attestations: a build provenance attestation and an SBOM attestation for every wheel and -# the XCFramework, from this repository's GitHub, checked with `gh attestation verify`. +# - Attestations: a build provenance attestation and an SBOM attestation for every wheel, the +# XCFramework, and the Go module's .zip and .mod, from this repository's GitHub, checked +# with `gh attestation verify`. # - PyPI: the project @NAME@, through the trusted publisher this repository registers there # for this workflow and the `pypi` environment. # - SwiftPM: @MODULE@FFI.xcframework.zip on this repository's @NAME@-v release, which @@ -81,16 +82,24 @@ jobs: with: name: @NAME@-swift path: artifacts/swift + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: @NAME@-go + path: artifacts/go - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: | artifacts/wheels/*.whl artifacts/swift/*FFI.xcframework.zip + artifacts/go/**/@v/*.zip + artifacts/go/**/@v/*.mod - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: | artifacts/wheels/*.whl artifacts/swift/*FFI.xcframework.zip + artifacts/go/**/@v/*.zip + artifacts/go/**/@v/*.mod sbom-path: artifacts/sbom.cdx.json pypi: diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 1af803e40..18b565a42 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -259,9 +259,11 @@ def test_paths_are_remapped_and_rustflags_kept(self): self.assertLess(flags.index(f"--remap-path-prefix={tmp}=/source"), flags.index(f"--remap-path-prefix={tmp}/export=/export")) self.assertEqual(env["SOURCE_DATE_EPOCH"], "1700000000") self.assertIn(f"-ffile-prefix-map={tmp}=/source", env["CFLAGS"]) + self.assertIn(f"-ffile-prefix-map={tmp}=/source", env["CXXFLAGS"]) windows = package.reproducible(info(tmp), Path(tmp) / "export", "x86_64-pc-windows-msvc") self.assertIn("link-arg=/Brepro", windows["CARGO_ENCODED_RUSTFLAGS"]) self.assertNotIn("CFLAGS", windows) + self.assertNotIn("CXXFLAGS", windows) apple = package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin") self.assertEqual(apple["ZERO_AR_DATE"], "1") self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.dylib", apple["CARGO_ENCODED_RUSTFLAGS"]) From 7f2fc79c7d35a40d30ed45e84f713abcf1b5fd2b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 14:18:43 -0800 Subject: [PATCH 11/20] Name the members of a static library a rebuild made differently Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/package.py | 34 ++++++++++++++++++++++++++++++++++ ffi/release/test_package.py | 16 ++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/ffi/release/package.py b/ffi/release/package.py index ab34544f4..1d52f7d9e 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -714,6 +714,38 @@ def zip_diff(one, two): for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] +def ar_members(path): + """An archive's members as `{name: (content hash, time, owner, group, mode)}`, with its symbol table and names table by their own.""" + data = Path(path).read_bytes() + if data[:8] != b"!\n": + raise SystemExit(f"{path} is no archive") + members, long_names, offset = {}, b"", 8 + while offset + 60 <= len(data): + name, date, uid, gid, mode, size = (data[offset + a:offset + b].decode("ascii", "replace").strip() + for a, b in ((0, 16), (16, 28), (28, 34), (34, 40), (40, 48), (48, 58))) + body = data[offset + 60:offset + 60 + int(size)] + offset += 60 + int(size) + int(size) % 2 + if name == "//": + long_names, name = body, "//" + elif name.startswith("/") and name[1:].isdigit(): + start = int(name[1:]) + name = long_names[start:long_names.index(b"\n", start)].decode("utf-8", "replace").rstrip("/") + else: + name = name.rstrip("/") or "/" + # A name twice, as an archive may hold, is told apart by its place. + while name in members: + name += "'" + members[name] = (hashlib.sha256(body).hexdigest(), date, uid, gid, mode) + return members + + +def ar_diff(one, two): + """The members two archives hold differently.""" + left, right = ar_members(one), ar_members(two) + return [f" {name}: {left.get(name)} against {right.get(name)}" + for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] + + def same(first, second): """Fail unless two builds' wheels and static libraries, the Go module's included, are the same bytes, naming what differs.""" found = {} @@ -730,6 +762,8 @@ def same(first, second): problems.append(name) if name.endswith(".whl"): problems += zip_diff(one, two) + elif name.endswith(".a"): + problems += ar_diff(one, two) if problems: raise SystemExit("a rebuild made other bytes:\n" + "\n".join(problems)) print(f"{len(names)} files rebuilt byte for byte") diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 18b565a42..1e13ee4e2 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -369,6 +369,22 @@ def test_one_version_from_two_sources_is_two_components(self): self.assertTrue(any(ref.startswith("pkg:cargo/dep@1.2.3?vcs_url=git%2Bfile") for ref in refs), refs) +class Archive(unittest.TestCase): + def test_members_that_differ_are_named(self): + def ar(*members): + out = b"!\n" + for name, date, body in members: + out += f"{name + '/':<16}{date:<12}{0:<6}{0:<6}{100644:<8}{len(body):<10}`\n".encode() + body + b"\n" * (len(body) % 2) + return out + with tempfile.TemporaryDirectory() as tmp: + one, two = Path(tmp) / "one.a", Path(tmp) / "two.a" + one.write_bytes(ar(("a.o", 0, b"same"), ("b.o", 0, b"before"))) + two.write_bytes(ar(("a.o", 0, b"same"), ("b.o", 5, b"before"))) + lines = package.ar_diff(one, two) + self.assertEqual(len(lines), 1) + self.assertIn("b.o", lines[0]) + + class Purl(unittest.TestCase): def test_each_registry_is_its_own_package(self): def purl(source): From ee30610d936ac0dd2975a601de4566ad5916019a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 14:39:21 -0800 Subject: [PATCH 12/20] Show where a rebuilt static library's member first differs Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/package.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index 1d52f7d9e..f6fea3f52 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -715,7 +715,7 @@ def zip_diff(one, two): def ar_members(path): - """An archive's members as `{name: (content hash, time, owner, group, mode)}`, with its symbol table and names table by their own.""" + """An archive's members as `{name: (content, time, owner, group, mode)}`, with its symbol table and names table by their own.""" data = Path(path).read_bytes() if data[:8] != b"!\n": raise SystemExit(f"{path} is no archive") @@ -735,15 +735,25 @@ def ar_members(path): # A name twice, as an archive may hold, is told apart by its place. while name in members: name += "'" - members[name] = (hashlib.sha256(body).hexdigest(), date, uid, gid, mode) + members[name] = (body, date, uid, gid, mode) return members def ar_diff(one, two): - """The members two archives hold differently.""" + """The members two archives hold differently, each with where its content first differs.""" left, right = ar_members(one), ar_members(two) - return [f" {name}: {left.get(name)} against {right.get(name)}" - for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] + lines = [] + for name in sorted(left.keys() | right.keys()): + a, b = left.get(name), right.get(name) + if a == b: + continue + if a is None or b is None: + lines.append(f" {name}: only in the {'first' if b is None else 'second'}") + continue + at = next((i for i, (x, y) in enumerate(zip(a[0], b[0])) if x != y), min(len(a[0]), len(b[0]))) + lines.append(f" {name}: {len(a[0])} against {len(b[0])} bytes, {a[1:]} against {b[1:]}; first differing byte {at}: " + f"{a[0][max(at - 24, 0):at + 40]!r} against {b[0][max(at - 24, 0):at + 40]!r}") + return lines def same(first, second): From d6f8f369de2a8a61524624afd635bc8e24e24748 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 14:50:45 -0800 Subject: [PATCH 13/20] Rebuild from a new export at the same path, as dlltool names Windows GNU's import symbols by it Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/bindings.yml | 12 +++++++----- ffi/release/README.md | 10 ++++++---- quest/m1/ffi-packaging/relocatable.md | 7 ++++++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 55d9b3303..40a0cf2d5 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -29,7 +29,7 @@ on: default: '' type: string reproduce: - description: Build every host's packages a second time, with the export elsewhere, and fail unless the bytes match + description: Build every host's packages a second time, from a new export, and fail unless the bytes match default: false type: boolean go-module: @@ -273,8 +273,9 @@ jobs: run: | read -ra targets <<< "$TARGETS" python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export export --out dist --example "$EXAMPLE" "${targets[@]}" - # The build again, as a rerun or anyone with the commit makes it: the same packages byte - # for byte, though the export lives in another directory. It doubles the job's time. + # The build again, as a rerun makes it: the same packages byte for byte, from a new + # export at the same path, since dlltool names the Windows GNU library's import symbols + # after the target directory. It doubles the job's time. - name: Rebuild if: inputs.reproduce env: @@ -282,8 +283,9 @@ jobs: GO_MODULE: ${{ inputs.go-module }} run: | read -ra targets <<< "$TARGETS" - python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --go-module "$GO_MODULE" --out again/export "${targets[@]}" - python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export again/export --out again/dist "${targets[@]}" + mv export first-export + python3 -I rust2/ffi/release/package.py export --manifest-path "source/$MANIFEST" --features "$FEATURES" --api api --go-module "$GO_MODULE" --out export "${targets[@]}" + python -I rust2/ffi/release/package.py build --manifest-path "source/$MANIFEST" --features "$FEATURES" --export export --out again/dist "${targets[@]}" python3 -I rust2/ffi/release/package.py same dist again/dist - name: Checks env: diff --git a/ffi/release/README.md b/ffi/release/README.md index d4a2ac280..95592c76a 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -117,14 +117,16 @@ holding exactly the files it would upload, and anyone with the commit can check that sets any fails the build: export them as `RUSTFLAGS`, which the builder keeps. Apple builds also stamp no archive time (`ZERO_AR_DATE`), and Windows links with `/Brepro`. - **Checked.** The Swift job builds the XCFramework twice and fails unless both ZIPs match. - With `reproduce: true`, every build host builds a second time, from a new export, and - `package.py same` fails unless each wheel and static library, the Go module's included, matches byte for byte; rust2 - turns it on for its api fixture. It doubles the build, so a large crate may leave it to a + With `reproduce: true`, every build host builds a second time, from a new export at the same + path, and `package.py same` fails unless each wheel and static library, the Go module's + included, matches byte for byte; rust2 turns it on for its api fixture. It doubles the build, so a large crate may leave it to a nightly. - **Not yet.** The exporter writes the crate's absolute path into the generated crate's version, which cargo hashes into every symbol, so a rebuild must keep the crate at the same path, as a rerun does - ([planned](/quest/m1/ffi-packaging/relocatable.md)). + ([planned](/quest/m1/ffi-packaging/relocatable.md)). So must the export directory of a + Windows GNU library, which is the Go module's: dlltool names the import library's symbols + after the path of the target directory, which no `--remap-path-prefix` reaches. **Attestations and SBOM.** `package.py build` writes `sbom.cdx.json`, a CycloneDX SBOM of the crates the binding links (normal dependencies, platform-neutral, with each registry diff --git a/quest/m1/ffi-packaging/relocatable.md b/quest/m1/ffi-packaging/relocatable.md index d366c0361..31b360aeb 100644 --- a/quest/m1/ffi-packaging/relocatable.md +++ b/quest/m1/ffi-packaging/relocatable.md @@ -13,7 +13,12 @@ same bytes, and `reproduce` in `bindings-test.yml` builds the second copy elsewh Found by [the reproducible-releases quest](https://github.com/kixel-ai/rust2/pull/874): `glue.rs`'s `generate` writes `path = {facade.dir}`. Measured there: two exports of the api fixture from copies at different paths differ in the version's hash and in every symbol -hash; the same path with another export directory is byte for byte equal. +hash; the same path with another export directory is byte for byte equal on Linux and for the wheels. +The Go module's Windows GNU library is not: `dlltool` names its `kernel32` import members' +symbols (`_head_D__a_rust2_..._export_target_go_...`) after the path of the target +directory, which `--remap-path-prefix` does not reach, so `reproduce` rebuilds from a new +export at the same path. Find a target directory path that is the same everywhere (or a +rustc that remaps it) so the CI rebuild can move the export again. Fall back to the absolute path where no relative one exists, such as another Windows drive. Check `cli.rs`'s other absolute manifest paths (its "each path absolute" note) for the same From ea082712ec170939634038ecbf972bedc816592a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 15:45:08 -0800 Subject: [PATCH 14/20] Zero the random directory in a Windows GNU library's import symbols Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/README.md | 3 ++- ffi/release/package.py | 13 +++++++++++++ ffi/release/test_package.py | 9 +++++++++ quest/m1/ffi-packaging/relocatable.md | 3 ++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index 95592c76a..80f06635f 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -126,7 +126,8 @@ holding exactly the files it would upload, and anyone with the commit can check same path, as a rerun does ([planned](/quest/m1/ffi-packaging/relocatable.md)). So must the export directory of a Windows GNU library, which is the Go module's: dlltool names the import library's symbols - after the path of the target directory, which no `--remap-path-prefix` reaches. + after the path of the target directory, which no `--remap-path-prefix` reaches. rustc's + random temporary directory in that name is replaced by zeros. **Attestations and SBOM.** `package.py build` writes `sbom.cdx.json`, a CycloneDX SBOM of the crates the binding links (normal dependencies, platform-neutral, with each registry diff --git a/ffi/release/package.py b/ffi/release/package.py index f6fea3f52..219e65377 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -825,6 +825,17 @@ def llvm_tool(manifest, name): return Path(sysroot) / "lib" / "rustlib" / host / "bin" / f"{name}{exe}" +# The random directory rustc imports a Windows GNU library's raw-dylib symbols through, which +# dlltool names them after, as `_out_rustc<6 random characters>_kernel32_dll_imports_lib`. +RUSTC_TEMP = re.compile(rb"(?<=_out_rustc)[0-9A-Za-z]{6}(?=_)") + + +def fix_import_names(path): + """Replace the random part of those symbol names, in members and symbol table alike at the same length, so the archive is the same on every build.""" + data = Path(path).read_bytes() + Path(path).write_bytes(RUSTC_TEMP.sub(b"000000", data)) + + def go_library(args, export_dir, lib, target, library, native, out): """The Go module's copy of `target`'s library: checked against the export's #cgo line, without debug info, which only bloats the module, or the bitcode the go command refuses.""" @@ -835,6 +846,8 @@ def go_library(args, export_dir, lib, target, library, native, out): dest = out / "go" / f"lib{lib}_{GO_TARGETS[target]}.a" dest.parent.mkdir(parents=True, exist_ok=True) run(objcopy, "--strip-debug", *(f"--remove-section={s}" for s in LTO_SECTIONS), library, dest) + if "windows-gnu" in target: + fix_import_names(dest) def build(args): diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 1e13ee4e2..a6c9c02fd 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -385,6 +385,15 @@ def ar(*members): self.assertIn("b.o", lines[0]) +class ImportNames(unittest.TestCase): + def test_the_random_temporary_directory_is_fixed(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "lib.a" + path.write_bytes(b"_head_D__a_x_out_rustc4TX99l_kernel32_dll_imports_lib\x00_rustcAbCdEf_other") + package.fix_import_names(path) + self.assertEqual(path.read_bytes(), b"_head_D__a_x_out_rustc000000_kernel32_dll_imports_lib\x00_rustcAbCdEf_other") + + class Purl(unittest.TestCase): def test_each_registry_is_its_own_package(self): def purl(source): diff --git a/quest/m1/ffi-packaging/relocatable.md b/quest/m1/ffi-packaging/relocatable.md index 31b360aeb..ad283912c 100644 --- a/quest/m1/ffi-packaging/relocatable.md +++ b/quest/m1/ffi-packaging/relocatable.md @@ -16,7 +16,8 @@ fixture from copies at different paths differ in the version's hash and in every hash; the same path with another export directory is byte for byte equal on Linux and for the wheels. The Go module's Windows GNU library is not: `dlltool` names its `kernel32` import members' symbols (`_head_D__a_rust2_..._export_target_go_...`) after the path of the target -directory, which `--remap-path-prefix` does not reach, so `reproduce` rebuilds from a new +directory, which `--remap-path-prefix` does not reach (and a random rustc temporary directory in it, +which the builder zeroes), so `reproduce` rebuilds from a new export at the same path. Find a target directory path that is the same everywhere (or a rustc that remaps it) so the CI rebuild can move the export again. From 72c8ea1bf8eec532987c37874da18ad0b5b692df Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 16:41:08 -0800 Subject: [PATCH 15/20] Zero the random import directory only in the import library's members and symbol table Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/package.py | 42 ++++++++++++++++++++++++------------- ffi/release/test_package.py | 17 ++++++++++++--- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index 219e65377..9c49924df 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -714,28 +714,36 @@ def zip_diff(one, two): for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] -def ar_members(path): - """An archive's members as `{name: (content, time, owner, group, mode)}`, with its symbol table and names table by their own.""" - data = Path(path).read_bytes() +def ar_entries(data, label="the archive"): + """An archive's `(name, header fields, start, end)` per member, its symbol table and names table by their own.""" if data[:8] != b"!\n": - raise SystemExit(f"{path} is no archive") - members, long_names, offset = {}, b"", 8 + raise SystemExit(f"{label} is no archive") + entries, long_names, offset = [], b"", 8 while offset + 60 <= len(data): name, date, uid, gid, mode, size = (data[offset + a:offset + b].decode("ascii", "replace").strip() for a, b in ((0, 16), (16, 28), (28, 34), (34, 40), (40, 48), (48, 58))) - body = data[offset + 60:offset + 60 + int(size)] - offset += 60 + int(size) + int(size) % 2 + start, end = offset + 60, offset + 60 + int(size) + offset = end + int(size) % 2 if name == "//": - long_names, name = body, "//" + long_names = data[start:end] elif name.startswith("/") and name[1:].isdigit(): - start = int(name[1:]) - name = long_names[start:long_names.index(b"\n", start)].decode("utf-8", "replace").rstrip("/") + first = int(name[1:]) + name = long_names[first:long_names.index(b"\n", first)].decode("utf-8", "replace").rstrip("/") else: name = name.rstrip("/") or "/" + entries.append((name, (date, uid, gid, mode), start, end)) + return entries + + +def ar_members(path): + """An archive's members as `{name: (content, time, owner, group, mode)}`.""" + data = Path(path).read_bytes() + members = {} + for name, fields, start, end in ar_entries(data, path): # A name twice, as an archive may hold, is told apart by its place. while name in members: name += "'" - members[name] = (body, date, uid, gid, mode) + members[name] = (data[start:end], *fields) return members @@ -827,13 +835,17 @@ def llvm_tool(manifest, name): # The random directory rustc imports a Windows GNU library's raw-dylib symbols through, which # dlltool names them after, as `_out_rustc<6 random characters>_kernel32_dll_imports_lib`. -RUSTC_TEMP = re.compile(rb"(?<=_out_rustc)[0-9A-Za-z]{6}(?=_)") +RUSTC_TEMP = re.compile(rb"(?<=_out_rustc)[0-9A-Za-z]{6}(?=_[0-9a-z_]+_dll_imports_lib)") +IMPORT_MEMBER = re.compile(r".+\.dll(h|t|s[0-9]+)\.o") def fix_import_names(path): - """Replace the random part of those symbol names, in members and symbol table alike at the same length, so the archive is the same on every build.""" - data = Path(path).read_bytes() - Path(path).write_bytes(RUSTC_TEMP.sub(b"000000", data)) + """Replace that random part, at the same length, in the import library's members and the archive's symbol table alone, so the archive is the same on every build and no other byte changes.""" + data = bytearray(Path(path).read_bytes()) + for name, _, start, end in ar_entries(bytes(data)): + if name == "/" or IMPORT_MEMBER.fullmatch(name): + data[start:end] = RUSTC_TEMP.sub(b"000000", bytes(data[start:end])) + Path(path).write_bytes(bytes(data)) def go_library(args, export_dir, lib, target, library, native, out): diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index a6c9c02fd..d464d88d8 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -386,12 +386,23 @@ def ar(*members): class ImportNames(unittest.TestCase): - def test_the_random_temporary_directory_is_fixed(self): + @staticmethod + def archive(*members): + out = b"!\n" + for name, body in members: + out += f"{name + '/':<16}{0:<12}{0:<6}{0:<6}{100644:<8}{len(body):<10}`\n".encode() + body + b"\n" * (len(body) % 2) + return out + + def test_only_the_import_library_loses_its_random_directory(self): + imported = b"_head_D__a_x_out_rustc4TX99l_kernel32_dll_imports_lib\x00" + data = b"_out_rustcAbCdEf_kernel32_dll_imports_lib is a string the crate returns" with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "lib.a" - path.write_bytes(b"_head_D__a_x_out_rustc4TX99l_kernel32_dll_imports_lib\x00_rustcAbCdEf_other") + path.write_bytes(self.archive(("", imported), ("kernel32.dllh.o", imported), ("api.o", data))) package.fix_import_names(path) - self.assertEqual(path.read_bytes(), b"_head_D__a_x_out_rustc000000_kernel32_dll_imports_lib\x00_rustcAbCdEf_other") + members = package.ar_members(path) + self.assertEqual(members["kernel32.dllh.o"][0], imported.replace(b"4TX99l", b"000000")) + self.assertEqual(members["api.o"][0], data) class Purl(unittest.TestCase): From 90ffa3076195cee20036c4c5075e637efdfb4d19 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 18:17:53 -0800 Subject: [PATCH 16/20] Attest the npm tarball with the web export's own SBOM Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/README.md | 4 ++-- ffi/release/package.py | 8 +++++--- ffi/release/release.yml | 7 ++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ffi/release/README.md b/ffi/release/README.md index d3f349394..b67ddf78f 100644 --- a/ffi/release/README.md +++ b/ffi/release/README.md @@ -207,8 +207,8 @@ crate's checksum from `Cargo.lock`), which every wheel also carries in `.dist-info/sboms/` (PEP 770) and the build hosts must agree on. The caller's `attest` job runs on a tag, before anything is published, and gives every wheel, the XCFramework, the Go module's `.zip` and `.mod`, each C archive, and the npm tarball a build provenance -attestation and an SBOM attestation, which `gh attestation verify - --repo /` checks. Only that job holds `id-token` and `attestations` +attestation and an SBOM attestation (the tarball's from its own `sbom.cdx.json`), which +`gh attestation verify --repo /` checks. Only that job holds `id-token` and `attestations` write; the builder stays read-only. PyPI's trusted publishing uploads its own PEP 740 attestations for the wheels as well. diff --git a/ffi/release/package.py b/ffi/release/package.py index 397bbd370..aa709b1e2 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -1370,9 +1370,9 @@ def purl(package): return base -def sbom(info, crate): - """A CycloneDX SBOM of the crates the binding is built from, the same for every host and target.""" - meta = json.loads(output("cargo", "metadata", "--locked", "--format-version", "1", "--features", "python", +def sbom(info, crate, feature="python"): + """A CycloneDX SBOM of the crates the binding is built from with `feature`, the same for every host and target.""" + meta = json.loads(output("cargo", "metadata", "--locked", "--format-version", "1", "--features", feature, "--manifest-path", crate)) lock = tomllib.loads((Path(crate).parent / "Cargo.lock").read_text(encoding="utf-8")) checksums = {(p["name"], p["version"], p.get("source")): p["checksum"] for p in lock["package"] if "checksum" in p} @@ -1882,6 +1882,8 @@ def web(args): sys.stderr.write(built.stderr) wasm = export_dir / "target" / WEB_TARGET / "release" / f"{info['lib']}.wasm" npm, _ = web_package(info, export_dir / "web", wasm, args.scope, out) + # Its own, as the web crate's dependencies are not the native export's. + (out / "sbom.cdx.json").write_bytes(sbom(info, export_dir / "Cargo.toml", "web")) packed = output("npm", "pack", "--silent", "--pack-destination", out, cwd=npm).strip() if not (out / packed).is_file(): raise SystemExit(f"npm pack wrote no {packed}") diff --git a/ffi/release/release.yml b/ffi/release/release.yml index 869bff0af..a405eacf7 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -131,8 +131,13 @@ jobs: artifacts/go/**/@v/*.zip artifacts/go/**/@v/*.mod artifacts/c/* - artifacts/web/*.tgz sbom-path: artifacts/sbom.cdx.json + # The npm tarball's own, from the web export's dependencies. + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + if: needs.build.outputs.web == 'true' + with: + subject-path: artifacts/web/*.tgz + sbom-path: artifacts/web/sbom.cdx.json pypi: needs: [build, tag, attest] From 88480add193bff4f129856a182ced8531a404680 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 19:14:48 -0800 Subject: [PATCH 17/20] Name the C archive's dylib by its versioned soname under the reproducible build Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/package.py | 9 +++++---- ffi/release/test_package.py | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index aa709b1e2..9f7cd5ecb 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -903,7 +903,7 @@ def c_archive(info, manifest, export_dir, target, out): elif apple: # The last -install_name rustc's linker sees wins over its own unversioned one. link_args = ["-C", f"link-arg=-Wl,-install_name,@rpath/{soname}"] - env = {**reproducible(info, export_dir, target), **({"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if apple else {})} + env = {**reproducible(info, export_dir, target, soname), **({"MACOSX_DEPLOYMENT_TARGET": MACOS_WHEEL} if apple else {})} # The C library alone, as the Swift package's: the glue's language features add symbols # a C program cannot resolve. Its own target directory, since its link arguments differ. # cargo-zigbuild's own `rustc`, as Cargo knows only its `zigbuild`, from the package's @@ -1315,11 +1315,12 @@ def rustflags_config(directory, target): yield key -def reproducible(info, export_dir, target): +def reproducible(info, export_dir, target, install=None): """The environment that makes a build's bytes the same on any host, at any path. Release time is the commit's, and each directory the build reads from, which rustc and - C compilers write into panic messages and debug info, is spelled the same. + C compilers write into panic messages and debug info, is spelled the same. A dylib is + named `install`, else by its library's own name. """ pairs = { info["workspace"]: "/source", @@ -1353,7 +1354,7 @@ def reproducible(info, export_dir, target): # Apple's ar and libtool stamp archive members with the time unless told not to, and # the linker names a dylib after the path it wrote it to, in the export's build directory. env["ZERO_AR_DATE"] = "1" - flags += ["-C", f"link-arg=-Wl,-install_name,@rpath/{library_name(info['lib'], target)}"] + flags += ["-C", f"link-arg=-Wl,-install_name,@rpath/{install or library_name(info['lib'], target)}"] env["CARGO_ENCODED_RUSTFLAGS"] = "\x1f".join(flags) return env diff --git a/ffi/release/test_package.py b/ffi/release/test_package.py index 3e1b10f29..3edf0732a 100644 --- a/ffi/release/test_package.py +++ b/ffi/release/test_package.py @@ -344,6 +344,9 @@ def test_paths_are_remapped_and_rustflags_kept(self): apple = package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin") self.assertEqual(apple["ZERO_AR_DATE"], "1") self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.dylib", apple["CARGO_ENCODED_RUSTFLAGS"]) + named = package.reproducible(info(tmp), Path(tmp), "aarch64-apple-darwin", "libmoq.0.4.12.dylib") + self.assertIn("link-arg=-Wl,-install_name,@rpath/libmoq.0.4.12.dylib", named["CARGO_ENCODED_RUSTFLAGS"]) + self.assertNotIn("@rpath/libmoq.dylib", named["CARGO_ENCODED_RUSTFLAGS"]) def test_an_empty_encoded_rustflags_is_no_flags(self): with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ, {"CARGO_ENCODED_RUSTFLAGS": "", "RUSTFLAGS": "-C x", "CARGO_HOME": tmp + "/home"}): From 57d437cd406b84d4c520b140365e76073cccbff9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 20:29:12 -0800 Subject: [PATCH 18/20] Stamp no time in the C archive's prelinked static library on Apple, and name where a tarball's libraries differ Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/package.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ffi/release/package.py b/ffi/release/package.py index 9f7cd5ecb..ed9a7604b 100644 --- a/ffi/release/package.py +++ b/ffi/release/package.py @@ -819,7 +819,8 @@ def hide(static, shared, lib, target, work): arch = ["-arch", "arm64", f"-mmacosx-version-min={MACOS_WHEEL}"] run("cc", "-r", "-nostdlib", *arch, *(f"-Wl,{arg}" for arg in roots), f"-Wl,-exported_symbols_list,{names}", define_commons(static, arch, work), static, "-o", hidden) - run("libtool", "-static", "-o", archive, hidden) + # Which stamps the member with the time unless told not to. + run("libtool", "-static", "-o", archive, hidden, env={**os.environ, "ZERO_AR_DATE": "1"}) # Rust's objects embed LLVM bitcode newer than Xcode's nm reads. found = output("nm", "--no-llvm-bc", "-gUj", hidden).split() else: @@ -1489,14 +1490,25 @@ def ar_diff(one, two): def tar_diff(one, two): - """The members two gzipped tarballs hold differently, each with its content hash and attributes.""" + """The members two gzipped tarballs hold differently, each with its content hash and attributes, and where a static library's own members differ.""" def members(path): with tarfile.open(path) as archive: - return {m.name: (hashlib.sha256(archive.extractfile(m).read()).hexdigest() if m.isfile() else m.linkname, - m.mtime, m.mode, m.uid, m.gid, m.uname, m.gname) for m in archive.getmembers()} + return {m.name: (archive.extractfile(m).read() if m.isfile() else m.linkname, m.mtime, m.mode, m.uid, m.gid, m.uname, m.gname) + for m in archive.getmembers()} left, right = members(one), members(two) - return [f" {name}: {left.get(name)} against {right.get(name)}" - for name in sorted(left.keys() | right.keys()) if left.get(name) != right.get(name)] + lines = [] + for name in sorted(left.keys() | right.keys()): + a, b = left.get(name), right.get(name) + if a == b: + continue + digest = lambda m: (hashlib.sha256(m[0]).hexdigest(), *m[1:]) if m and isinstance(m[0], bytes) else m # noqa: E731 + lines.append(f" {name}: {digest(a)} against {digest(b)}") + if name.endswith(".a") and a and b: + with tempfile.TemporaryDirectory() as tmp: + for index, content in enumerate((a[0], b[0])): + (Path(tmp) / f"{index}.a").write_bytes(content) + lines += [" " + line for line in ar_diff(Path(tmp) / "0.a", Path(tmp) / "1.a")] + return lines def same(first, second): From ad7c3b784c8bfad67da530519016de8a17cb6f76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 21:43:03 -0800 Subject: [PATCH 19/20] Make npm and JSR wait for attest, and say what it attests Co-Authored-By: Claude Sonnet 5.5 --- ffi/release/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ffi/release/release.yml b/ffi/release/release.yml index a405eacf7..c0e758086 100644 --- a/ffi/release/release.yml +++ b/ffi/release/release.yml @@ -6,8 +6,8 @@ name: Release @NAME@ bindings # only a tag's build, once `attest` has signed it: # # - Attestations: a build provenance attestation and an SBOM attestation for every wheel, the -# XCFramework, and the Go module's .zip and .mod, from this repository's GitHub, checked -# with `gh attestation verify`. +# XCFramework, the Go module's .zip and .mod, each C archive, and the npm tarball, from +# this repository's GitHub, checked with `gh attestation verify`. # - PyPI: the project @NAME@, through the trusted publisher this repository registers there # for this workflow and the `pypi` environment. # - SwiftPM: @MODULE@FFI.xcframework.zip on this repository's @NAME@-v release, which @@ -305,7 +305,7 @@ jobs: # A rerun publishes only a version npm lacks, and fails on one it hosts with other bytes. npm: - needs: [build, tag] + needs: [build, tag, attest] if: needs.build.outputs.web == 'true' runs-on: ubuntu-24.04 environment: npm @@ -338,7 +338,7 @@ jobs: # JSR skips a version it already hosts. jsr: - needs: [build, tag] + needs: [build, tag, attest] if: needs.build.outputs.web == 'true' runs-on: ubuntu-24.04 environment: jsr From 543500175b1519c74009d39e92bb4c8296116ced Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 22:26:22 -0800 Subject: [PATCH 20/20] quest: record the maintainer's follow-up answers Co-Authored-By: Claude Opus 5.5 --- quest/m1/ffi-packaging/relocatable.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quest/m1/ffi-packaging/relocatable.md b/quest/m1/ffi-packaging/relocatable.md index ad283912c..dcfdbea42 100644 --- a/quest/m1/ffi-packaging/relocatable.md +++ b/quest/m1/ffi-packaging/relocatable.md @@ -10,6 +10,8 @@ same bytes, and `reproduce` in `bindings-test.yml` builds the second copy elsewh ## Plan +Decided with the maintainer on 2026-11-19, in the pick of [#874](https://github.com/kixel-ai/rust2/pull/874): ✅ kept (rejected: dropping it). + Found by [the reproducible-releases quest](https://github.com/kixel-ai/rust2/pull/874): `glue.rs`'s `generate` writes `path = {facade.dir}`. Measured there: two exports of the api fixture from copies at different paths differ in the version's hash and in every symbol