diff --git a/.github/workflows/flash-closure-identity-20261104.yml b/.github/workflows/flash-closure-identity-20261104.yml new file mode 100644 index 00000000..c8d78163 --- /dev/null +++ b/.github/workflows/flash-closure-identity-20261104.yml @@ -0,0 +1,78 @@ +name: Flash closure identity — paired official Arena regression + +on: + push: + branches: + - research/flash-closure-identity-20261104 + pull_request: + branches: + - flash + workflow_dispatch: + +permissions: + contents: read + +jobs: + paired-regression: + runs-on: ubuntu-latest + timeout-minutes: 65 + env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + RUSTFLAGS: -C target-cpu=native + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Record exact source and official baseline + run: | + set -euo pipefail + git rev-parse HEAD + git cat-file -e 78c7502bac8a5ba000057b3f083bc0595ac65750^{commit} + rustc --version + cargo --version + mkdir -p /tmp/flash-qualification + + - name: Build published Flash source and patched candidate identically + run: | + set -euo pipefail + git worktree add --detach /tmp/flash-published 78c7502bac8a5ba000057b3f083bc0595ac65750 + cargo build --manifest-path /tmp/flash-published/Cargo.toml --release --locked + cargo build --manifest-path Cargo.toml --release --locked + sha256sum /tmp/flash-published/target/release/sokonanoda target/release/sokonanoda | tee /tmp/flash-qualification/binaries.sha256 + + - name: Fetch official Arena small-test corpus and record its SHA + run: | + set -euo pipefail + curl --fail --location --silent --show-error --retry 3 --max-time 120 \ + https://arena.lean-lang.org/lean-arena-tests.tar.gz \ + --output /tmp/flash-qualification/lean-arena-tests.tar.gz + sha256sum /tmp/flash-qualification/lean-arena-tests.tar.gz + mkdir -p /tmp/flash-qualification/tests + tar -xzf /tmp/flash-qualification/lean-arena-tests.tar.gz -C /tmp/flash-qualification/tests + find /tmp/flash-qualification/tests -name '*.ndjson' | wc -l + + - name: Qualify changed soundness case with all other bounded verdicts preserved + run: | + set -euo pipefail + printf '{"use_stdin":true,"nat_extension":true,"string_extension":true,"unpermitted_axiom_hard_error":false,"unsafe_permit_all_axioms":true,"num_threads":2}\n' > /tmp/flash-qualification/config.json + python3 experiments/flash-closure-identity-20261104/qualify.py \ + --before /tmp/flash-published/target/release/sokonanoda \ + --after "$PWD/target/release/sokonanoda" \ + --config /tmp/flash-qualification/config.json \ + --fixtures /tmp/flash-qualification/tests \ + --fixture-archive /tmp/flash-qualification/lean-arena-tests.tar.gz \ + --report /tmp/flash-qualification/report.json + + - name: Preserve source-pinned artifacts, including negative evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: flash-closure-identity-${{ github.sha }} + if-no-files-found: warn + path: | + /tmp/flash-qualification/report.json + /tmp/flash-qualification/binaries.sha256 + /tmp/flash-qualification/config.json + /tmp/flash-qualification/lean-arena-tests.tar.gz diff --git a/experiments/flash-closure-identity-20261104/qualify.py b/experiments/flash-closure-identity-20261104/qualify.py new file mode 100644 index 00000000..715059ed --- /dev/null +++ b/experiments/flash-closure-identity-20261104/qualify.py @@ -0,0 +1,122 @@ +"""Paired source-pinned Flash / Arena regression for closure identity and substitution fixes. + +The candidate is allowed two deliberate verdict improvements: reject +both official Arena closure forgery exports. All other small-test +verdicts must remain byte-for-byte equivalent at the exit-code boundary. + +This is a bounded correctness regression, not a Mathlib performance result. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + + +TARGETS = ('pi-closure-context', 'closure-level-subst') + + +def digest(path): + h = hashlib.sha256() + with path.open('rb') as source: + for block in iter(lambda: source.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() + + +def check(binary, config, fixture, timeout): + try: + with fixture.open('rb') as stream: + result = subprocess.run( + [str(binary), str(config)], stdin=stream, + capture_output=True, timeout=timeout, check=False) + return {'rc': result.returncode, + 'stderr_sha256': hashlib.sha256(result.stderr).hexdigest(), + 'stderr_tail': result.stderr[-300:].decode(errors='replace')} + except subprocess.TimeoutExpired: + return {'rc': 'TIMEOUT'} + + +def main(): + arg = argparse.ArgumentParser() + arg.add_argument('--before', type=Path, required=True) + arg.add_argument('--after', type=Path, required=True) + arg.add_argument('--config', type=Path, required=True) + arg.add_argument('--fixtures', type=Path, required=True) + arg.add_argument('--fixture-archive', type=Path, required=True) + arg.add_argument('--report', type=Path, required=True) + a = arg.parse_args() + + # Every examined test is provided by the official Arena tarball. + cases = [] + skipped = [] + for fixture in sorted(a.fixtures.rglob('*.ndjson')): + parts = fixture.relative_to(a.fixtures).parts + if not parts or parts[0] not in ('good', 'bad'): + skipped.append({'case': str(fixture.relative_to(a.fixtures)), + 'reason': 'not in good/bad directory'}) + continue + if 'perf' in parts: + skipped.append({'case': str(fixture.relative_to(a.fixtures)), + 'reason': 'performance workload excluded from bounded regression'}) + continue + cases.append((fixture, parts[0])) + assert cases and all(any(target in fixture.name for fixture, _ in cases) for target in TARGETS) + assert len({str(fixture) for fixture, _ in cases}) == len(cases) + + results = [] + for index, (fixture, expected) in enumerate(cases): + original = check(a.before, a.config, fixture, timeout=60) + candidate = check(a.after, a.config, fixture, timeout=60) + identity = str(fixture.relative_to(a.fixtures)) + is_target = any(target in fixture.name for target in TARGETS) + is_same = original['rc'] == candidate['rc'] + passes = ((original['rc'] == 0 and candidate['rc'] == 1 and expected == 'bad') + if is_target else is_same) + result = {'case': identity, 'expected': expected, + 'input_sha256': digest(fixture), + 'before': original, 'after': candidate, + 'target': is_target, 'protected': passes} + results.append(result) + print('FLASH_PAIR', json.dumps({ + 'case': identity, 'before': original['rc'], 'after': candidate['rc'], + 'expected': expected, 'protected': passes}), flush=True) + # Preserve a partial evidence report even when a case falsifies the + # expected delta. We never erase negative or UNKNOWN observations. + a.report.parent.mkdir(parents=True, exist_ok=True) + a.report.write_text(json.dumps({ + 'suite_sha256': digest(a.fixture_archive), + 'before_binary_sha256': digest(a.before), + 'after_binary_sha256': digest(a.after), + 'cases': results, 'skipped': skipped, + 'status': 'INCOMPLETE' if index + 1 < len(cases) else 'EVALUATING', + }, indent=2) + '\n') + if not passes: + break + + all_pass = len(results) == len(cases) and all(r['protected'] for r in results) + target_rows = [r for r in results if r['target']] + all_pass = all_pass and len(target_rows) == len(TARGETS) + report = { + 'status': 'WARRANTED_BOUNDED_SMALL_ARENA_NONREGRESSION' if all_pass else 'REJECTED_OR_INCOMPLETE', + 'qualified': all_pass, + 'suite_sha256': digest(a.fixture_archive), + 'before_binary_sha256': digest(a.before), + 'after_binary_sha256': digest(a.after), + 'test_count': len(cases), 'tested_count': len(results), + 'targets': list(TARGETS), + 'target_positive': all(r['protected'] for r in target_rows) and len(target_rows) == len(TARGETS), + 'cases': results, 'skipped': skipped, + 'full_mathlib': 'NOT_TESTED', 'performance': 'NOT_TESTED', + 'attribution': 'Closure-equality soundness guards motivated by intgrah/sokonanoda issues #8 and #9', + } + a.report.write_text(json.dumps(report, indent=2) + '\n') + print('FLASH_PROTECTED_SUMMARY', json.dumps({k: v for k, v in report.items() + if k not in ('cases', 'skipped')}), flush=True) + if not all_pass: + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/src/conv.rs b/src/conv.rs index 77338081..d8f9546b 100644 --- a/src/conv.rs +++ b/src/conv.rs @@ -62,7 +62,15 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> { return true; } match (a, b) { - (Env::Nil { .. }, Env::Nil { .. }) => return true, + (Env::Nil { .. }, Env::Nil { .. }) => { + // A closure with zero bound variables can still carry a + // nontrivial universe substitution. Hash equality is not proof. + return match (a.lsub(), b.lsub()) { + (None, None) => true, + (Some(x), Some(y)) => x.ks == y.ks && x.vs == y.vs, + _ => false, + }; + }, ( Env::Cons { v: va, parent: pa, hash: ha, .. }, Env::Cons { v: vb, parent: pb, hash: hb, .. }, @@ -202,6 +210,11 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> { if bx.body == by.body && std::ptr::eq(*dx, *dy) && Self::envs_ptr_equal(bx.env, by.env) + && match (bx.ctx, by.ctx) { + (None, None) => true, + (Some(x), Some(y)) => std::ptr::eq(x, y), + _ => false, + } { return true; } @@ -221,6 +234,11 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> { if bx.body == by.body && std::ptr::eq(dx, dy) && Self::envs_ptr_equal(bx.env, by.env) + && match (bx.ctx, by.ctx) { + (None, None) => true, + (Some(x), Some(y)) => std::ptr::eq(x, y), + _ => false, + } { return true; }