Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions .github/workflows/flash-pgo-arena-repro-20261104.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
name: Flash Arena PGO causal reproduction — exact published source

on:
pull_request:
branches:
- flash
workflow_dispatch:

permissions:
contents: read

jobs:
published-pgo-comparison:
runs-on: ubuntu-24.04
timeout-minutes: 65
env:
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Record exact published source and compiler provenance
run: |
set -euo pipefail
test "$(git rev-parse HEAD^{tree})" = "$(git rev-parse 78c7502bac8a5ba000057b3f083bc0595ac65750^{tree})" || echo 'Additional experiment-only files are present; checked Rust source stays published'
git diff 78c7502bac8a5ba000057b3f083bc0595ac65750 -- src/
test -z "$(git diff --name-only 78c7502bac8a5ba000057b3f083bc0595ac65750 -- src/)"
git rev-parse HEAD
rustc --version
cargo --version
rustup component add llvm-tools-preview
mkdir -p /tmp/magma-pgo
find "$(rustc --print sysroot)/lib/rustlib" -name llvm-profdata -type f -print | head -3

- name: Source-pin Arena official training and magma inputs
run: |
set -euo pipefail
curl --fail --location --silent --show-error --retry 3 --max-time 120 \
https://arena.lean-lang.org/lean-arena-tests.tar.gz \
-o /tmp/magma-pgo/tests.tar.gz
printf '%s %s\n' 8223cb53aa5fa65f471054fbdf8cdcf50206556cfc7888f75e910f7816f4c8bb /tmp/magma-pgo/tests.tar.gz | sha256sum -c -
mkdir -p /tmp/magma-pgo/tests
tar -xzf /tmp/magma-pgo/tests.tar.gz -C /tmp/magma-pgo/tests
test "$(find /tmp/magma-pgo/tests -name init-prelude.ndjson | wc -l)" -eq 1
test "$(find /tmp/magma-pgo/tests -name magma-list-pair-n21.ndjson | wc -l)" -eq 1
printf '{"use_stdin":true,"nat_extension":true,"string_extension":true,"unpermitted_axiom_hard_error":false,"unsafe_permit_all_axioms":true,"num_threads":2}\n' > /tmp/magma-pgo/config.json
sha256sum "$(find /tmp/magma-pgo/tests -name init-prelude.ndjson)" /tmp/magma-pgo/config.json

- name: Compile plain release reference from same source
run: |
set -euo pipefail
RUSTFLAGS='-C target-cpu=native' cargo build --release --locked
cp target/release/sokonanoda /tmp/magma-pgo/plain-sokonanoda
sha256sum /tmp/magma-pgo/plain-sokonanoda

- name: Follow exact Arena PGO train-merge-use procedure
run: |
set -euo pipefail
mkdir -p "$PWD/pgo"
RUSTFLAGS="-C target-cpu=native -Cprofile-generate=$PWD/pgo" cargo build --release --locked
target/release/sokonanoda /tmp/magma-pgo/config.json < "$(find /tmp/magma-pgo/tests -name init-prelude.ndjson)" >/dev/null
LLVM_PROFDATA="$(find "$(rustc --print sysroot)/lib/rustlib" -name llvm-profdata -type f -print -quit)"
test -n "$LLVM_PROFDATA"
"$LLVM_PROFDATA" merge -o "$PWD/pgo/merged.profdata" "$PWD/pgo"
sha256sum "$PWD/pgo/merged.profdata" | tee /tmp/magma-pgo/profile.sha256
RUSTFLAGS="-C target-cpu=native -Cprofile-use=$PWD/pgo/merged.profdata" cargo build --release --locked
cp target/release/sokonanoda /tmp/magma-pgo/pgo-sokonanoda
sha256sum /tmp/magma-pgo/plain-sokonanoda /tmp/magma-pgo/pgo-sokonanoda | tee /tmp/magma-pgo/binaries.sha256

- name: Compare exact magma n7/n21 verdict and resource effects of PGO
run: |
set -euo pipefail
python3 experiments/flash-pgo-arena-repro-20261104/screen.py \
--plain /tmp/magma-pgo/plain-sokonanoda \
--pgo /tmp/magma-pgo/pgo-sokonanoda \
--config /tmp/magma-pgo/config.json \
--fixtures /tmp/magma-pgo/tests \
--archive /tmp/magma-pgo/tests.tar.gz \
--output /tmp/magma-pgo/results

- name: Save negative evidence, profile provenance, and resource comparisons
if: always()
uses: actions/upload-artifact@v4
with:
name: flash-pgo-arena-repro-${{ github.sha }}
if-no-files-found: warn
path: |
/tmp/magma-pgo/config.json
/tmp/magma-pgo/profile.sha256
/tmp/magma-pgo/binaries.sha256
/tmp/magma-pgo/results/*.json
/tmp/magma-pgo/results/*.time
/tmp/magma-pgo/results/*.stderr
/tmp/magma-pgo/results/*.stdout
pgo/merged.profdata
92 changes: 92 additions & 0 deletions experiments/flash-pgo-arena-repro-20261104/screen.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
"""A bounded exact-source PGO/no-PGO comparison for the 2026-11-04 Arena magma residual.

Compares the plain checker against the same source with Arena-style profile-guided optimized build. This does not establish correctness on the rest of Arena.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import time

TARGET_SHA256 = "b8279fa9189a203c6c11b04702c27c0f99a650b37f2b0165dd260b1d14e9ef84"

def digest(path):
h = hashlib.sha256()
with path.open("rb") as source:
for b in iter(lambda: source.read(4 * 1024 * 1024), b""):
h.update(b)
return h.hexdigest()

def trial(binary, config, fixture, output, arm, tag, seconds):
stem = output / (arm + "-" + tag)
so, se, ti = [Path(str(stem) + x) for x in (".stdout", ".stderr", ".time")]
ret = {"arm": arm, "case": tag, "fixture_sha256": digest(fixture),
"binary_sha256": digest(binary)}
start = time.monotonic()
try:
with fixture.open("rb") as f, so.open("wb") as stdout, se.open("wb") as stderr:
p = subprocess.run(["/usr/bin/time", "-f",
"wall_s=%e\ncpu_user_s=%U\ncpu_system_s=%S\nrss_kb=%M",
"-o", str(ti), str(binary), str(config)],
stdin=f, stdout=stdout, stderr=stderr, check=False, timeout=seconds)
ret["rc"] = p.returncode
except subprocess.TimeoutExpired:
ret["rc"] = "TIMEOUT"
ret["elapsed_s"] = round(time.monotonic() - start, 2)
for k, val in (line.split("=", 1) for line in ti.read_text().splitlines()
if "=" in line) if ti.exists() else []:
ret[k] = float(val)
ret["stderr_tail"] = se.read_bytes()[-800:].decode(errors="replace") if se.exists() else ""
return ret

def main():
ap = argparse.ArgumentParser()
ap.add_argument("--plain", type=Path, required=True)
ap.add_argument("--pgo", type=Path, required=True)
ap.add_argument("--config", type=Path, required=True)
ap.add_argument("--fixtures", type=Path, required=True)
ap.add_argument("--archive", type=Path, required=True)
ap.add_argument("--output", type=Path, required=True)
args = ap.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
files = {}
for tag in ("magma-list-pair-n7", "magma-list-pair-n21"):
matches = list(args.fixtures.rglob(tag + ".ndjson"))
if len(matches) != 1:
raise SystemExit("fixture inventory incomplete for " + tag)
files[tag] = matches[0]
if digest(files["magma-list-pair-n21"]) != TARGET_SHA256:
raise SystemExit("magma n21 source SHA drift (stop, do not guess)")
report = {
"status": "INCOMPLETE", "plain": digest(args.plain),
"pgo": digest(args.pgo), "archive_sha256": digest(args.archive),
"target_sha256": TARGET_SHA256,
"hypothesis": "Arena-style profile-guided optimization may alter the n21 resource-sensitive verdict",
"method": "same runner/same source and config, plain release versus Arena-style PGO trained on exact init-prelude; not an Arena CPU replica",
"runs": [],
}
out = args.output / "magma-pgo.json"
out.write_text(json.dumps(report, indent=2) + "\n")
for index, (arm, tag) in enumerate([
("plain", "magma-list-pair-n7"),
("pgo", "magma-list-pair-n7"),
("pgo", "magma-list-pair-n21"),
("plain", "magma-list-pair-n21"),
]):
print("MAGMA_START", json.dumps({"index": index, "arm": arm, "case": tag}), flush=True)
result = trial(args.plain if arm == "plain" else args.pgo,
args.config, files[tag], args.output, arm, tag, 230)
report["runs"].append(result)
out.write_text(json.dumps(report, indent=2) + "\n")
print("MAGMA_RESULT", json.dumps(result), flush=True)
changed = [r for r in report["runs"] if r["case"] == "magma-list-pair-n21"]
report["status"] = "OBSERVED_BOUNDED_CAUSAL_SCREEN" if len(changed) == 2 else "INCOMPLETE"
if len(changed) == 2:
report["pgo_accepts_n21"] = changed[0]["rc"] == 0
report["plain_accepts_n21"] = changed[1]["rc"] == 0
out.write_text(json.dumps(report, indent=2) + "\n")
print("MAGMA_SUMMARY", json.dumps(report), flush=True)

if __name__ == "__main__":
main()
Loading