diff --git a/.github/workflows/flash-pgo-arena-repro-20261104.yml b/.github/workflows/flash-pgo-arena-repro-20261104.yml new file mode 100644 index 00000000..4a304e17 --- /dev/null +++ b/.github/workflows/flash-pgo-arena-repro-20261104.yml @@ -0,0 +1,96 @@ +name: Flash Arena PGO causal reproduction — exact published source + +on: + pull_request: + branches: + - flash + workflow_dispatch: + +permissions: + contents: read + +jobs: + published-pgo-comparison: + runs-on: ubuntu-24.04 + timeout-minutes: 65 + env: + CARGO_INCREMENTAL: 0 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Record exact published source and compiler provenance + run: | + set -euo pipefail + test "$(git rev-parse HEAD^{tree})" = "$(git rev-parse 78c7502bac8a5ba000057b3f083bc0595ac65750^{tree})" || echo 'Additional experiment-only files are present; checked Rust source stays published' + git diff 78c7502bac8a5ba000057b3f083bc0595ac65750 -- src/ + test -z "$(git diff --name-only 78c7502bac8a5ba000057b3f083bc0595ac65750 -- src/)" + git rev-parse HEAD + rustc --version + cargo --version + rustup component add llvm-tools-preview + mkdir -p /tmp/magma-pgo + find "$(rustc --print sysroot)/lib/rustlib" -name llvm-profdata -type f -print | head -3 + + - name: Source-pin Arena official training and magma inputs + run: | + set -euo pipefail + curl --fail --location --silent --show-error --retry 3 --max-time 120 \ + https://arena.lean-lang.org/lean-arena-tests.tar.gz \ + -o /tmp/magma-pgo/tests.tar.gz + printf '%s %s\n' 8223cb53aa5fa65f471054fbdf8cdcf50206556cfc7888f75e910f7816f4c8bb /tmp/magma-pgo/tests.tar.gz | sha256sum -c - + mkdir -p /tmp/magma-pgo/tests + tar -xzf /tmp/magma-pgo/tests.tar.gz -C /tmp/magma-pgo/tests + test "$(find /tmp/magma-pgo/tests -name init-prelude.ndjson | wc -l)" -eq 1 + test "$(find /tmp/magma-pgo/tests -name magma-list-pair-n21.ndjson | wc -l)" -eq 1 + printf '{"use_stdin":true,"nat_extension":true,"string_extension":true,"unpermitted_axiom_hard_error":false,"unsafe_permit_all_axioms":true,"num_threads":2}\n' > /tmp/magma-pgo/config.json + sha256sum "$(find /tmp/magma-pgo/tests -name init-prelude.ndjson)" /tmp/magma-pgo/config.json + + - name: Compile plain release reference from same source + run: | + set -euo pipefail + RUSTFLAGS='-C target-cpu=native' cargo build --release --locked + cp target/release/sokonanoda /tmp/magma-pgo/plain-sokonanoda + sha256sum /tmp/magma-pgo/plain-sokonanoda + + - name: Follow exact Arena PGO train-merge-use procedure + run: | + set -euo pipefail + mkdir -p "$PWD/pgo" + RUSTFLAGS="-C target-cpu=native -Cprofile-generate=$PWD/pgo" cargo build --release --locked + target/release/sokonanoda /tmp/magma-pgo/config.json < "$(find /tmp/magma-pgo/tests -name init-prelude.ndjson)" >/dev/null + LLVM_PROFDATA="$(find "$(rustc --print sysroot)/lib/rustlib" -name llvm-profdata -type f -print -quit)" + test -n "$LLVM_PROFDATA" + "$LLVM_PROFDATA" merge -o "$PWD/pgo/merged.profdata" "$PWD/pgo" + sha256sum "$PWD/pgo/merged.profdata" | tee /tmp/magma-pgo/profile.sha256 + RUSTFLAGS="-C target-cpu=native -Cprofile-use=$PWD/pgo/merged.profdata" cargo build --release --locked + cp target/release/sokonanoda /tmp/magma-pgo/pgo-sokonanoda + sha256sum /tmp/magma-pgo/plain-sokonanoda /tmp/magma-pgo/pgo-sokonanoda | tee /tmp/magma-pgo/binaries.sha256 + + - name: Compare exact magma n7/n21 verdict and resource effects of PGO + run: | + set -euo pipefail + python3 experiments/flash-pgo-arena-repro-20261104/screen.py \ + --plain /tmp/magma-pgo/plain-sokonanoda \ + --pgo /tmp/magma-pgo/pgo-sokonanoda \ + --config /tmp/magma-pgo/config.json \ + --fixtures /tmp/magma-pgo/tests \ + --archive /tmp/magma-pgo/tests.tar.gz \ + --output /tmp/magma-pgo/results + + - name: Save negative evidence, profile provenance, and resource comparisons + if: always() + uses: actions/upload-artifact@v4 + with: + name: flash-pgo-arena-repro-${{ github.sha }} + if-no-files-found: warn + path: | + /tmp/magma-pgo/config.json + /tmp/magma-pgo/profile.sha256 + /tmp/magma-pgo/binaries.sha256 + /tmp/magma-pgo/results/*.json + /tmp/magma-pgo/results/*.time + /tmp/magma-pgo/results/*.stderr + /tmp/magma-pgo/results/*.stdout + pgo/merged.profdata diff --git a/experiments/flash-pgo-arena-repro-20261104/screen.py b/experiments/flash-pgo-arena-repro-20261104/screen.py new file mode 100644 index 00000000..5c110999 --- /dev/null +++ b/experiments/flash-pgo-arena-repro-20261104/screen.py @@ -0,0 +1,92 @@ +"""A bounded exact-source PGO/no-PGO comparison for the 2026-11-04 Arena magma residual. + +Compares the plain checker against the same source with Arena-style profile-guided optimized build. This does not establish correctness on the rest of Arena. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import time + +TARGET_SHA256 = "b8279fa9189a203c6c11b04702c27c0f99a650b37f2b0165dd260b1d14e9ef84" + +def digest(path): + h = hashlib.sha256() + with path.open("rb") as source: + for b in iter(lambda: source.read(4 * 1024 * 1024), b""): + h.update(b) + return h.hexdigest() + +def trial(binary, config, fixture, output, arm, tag, seconds): + stem = output / (arm + "-" + tag) + so, se, ti = [Path(str(stem) + x) for x in (".stdout", ".stderr", ".time")] + ret = {"arm": arm, "case": tag, "fixture_sha256": digest(fixture), + "binary_sha256": digest(binary)} + start = time.monotonic() + try: + with fixture.open("rb") as f, so.open("wb") as stdout, se.open("wb") as stderr: + p = subprocess.run(["/usr/bin/time", "-f", + "wall_s=%e\ncpu_user_s=%U\ncpu_system_s=%S\nrss_kb=%M", + "-o", str(ti), str(binary), str(config)], + stdin=f, stdout=stdout, stderr=stderr, check=False, timeout=seconds) + ret["rc"] = p.returncode + except subprocess.TimeoutExpired: + ret["rc"] = "TIMEOUT" + ret["elapsed_s"] = round(time.monotonic() - start, 2) + for k, val in (line.split("=", 1) for line in ti.read_text().splitlines() + if "=" in line) if ti.exists() else []: + ret[k] = float(val) + ret["stderr_tail"] = se.read_bytes()[-800:].decode(errors="replace") if se.exists() else "" + return ret + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--plain", type=Path, required=True) + ap.add_argument("--pgo", type=Path, required=True) + ap.add_argument("--config", type=Path, required=True) + ap.add_argument("--fixtures", type=Path, required=True) + ap.add_argument("--archive", type=Path, required=True) + ap.add_argument("--output", type=Path, required=True) + args = ap.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + files = {} + for tag in ("magma-list-pair-n7", "magma-list-pair-n21"): + matches = list(args.fixtures.rglob(tag + ".ndjson")) + if len(matches) != 1: + raise SystemExit("fixture inventory incomplete for " + tag) + files[tag] = matches[0] + if digest(files["magma-list-pair-n21"]) != TARGET_SHA256: + raise SystemExit("magma n21 source SHA drift (stop, do not guess)") + report = { + "status": "INCOMPLETE", "plain": digest(args.plain), + "pgo": digest(args.pgo), "archive_sha256": digest(args.archive), + "target_sha256": TARGET_SHA256, + "hypothesis": "Arena-style profile-guided optimization may alter the n21 resource-sensitive verdict", + "method": "same runner/same source and config, plain release versus Arena-style PGO trained on exact init-prelude; not an Arena CPU replica", + "runs": [], + } + out = args.output / "magma-pgo.json" + out.write_text(json.dumps(report, indent=2) + "\n") + for index, (arm, tag) in enumerate([ + ("plain", "magma-list-pair-n7"), + ("pgo", "magma-list-pair-n7"), + ("pgo", "magma-list-pair-n21"), + ("plain", "magma-list-pair-n21"), + ]): + print("MAGMA_START", json.dumps({"index": index, "arm": arm, "case": tag}), flush=True) + result = trial(args.plain if arm == "plain" else args.pgo, + args.config, files[tag], args.output, arm, tag, 230) + report["runs"].append(result) + out.write_text(json.dumps(report, indent=2) + "\n") + print("MAGMA_RESULT", json.dumps(result), flush=True) + changed = [r for r in report["runs"] if r["case"] == "magma-list-pair-n21"] + report["status"] = "OBSERVED_BOUNDED_CAUSAL_SCREEN" if len(changed) == 2 else "INCOMPLETE" + if len(changed) == 2: + report["pgo_accepts_n21"] = changed[0]["rc"] == 0 + report["plain_accepts_n21"] = changed[1]["rc"] == 0 + out.write_text(json.dumps(report, indent=2) + "\n") + print("MAGMA_SUMMARY", json.dumps(report), flush=True) + +if __name__ == "__main__": + main()