Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions .github/workflows/flash-closure-identity-20261104.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
name: Flash closure identity — paired official Arena regression

on:
push:
branches:
- research/flash-closure-identity-20261104
pull_request:
branches:
- flash
workflow_dispatch:

permissions:
contents: read

jobs:
paired-regression:
runs-on: ubuntu-latest
timeout-minutes: 65
env:
CARGO_INCREMENTAL: 0
CARGO_TERM_COLOR: always
RUSTFLAGS: -C target-cpu=native
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Record exact source and official baseline
run: |
set -euo pipefail
git rev-parse HEAD
git cat-file -e 78c7502bac8a5ba000057b3f083bc0595ac65750^{commit}
rustc --version
cargo --version
mkdir -p /tmp/flash-qualification

- name: Build published Flash source and patched candidate identically
run: |
set -euo pipefail
git worktree add --detach /tmp/flash-published 78c7502bac8a5ba000057b3f083bc0595ac65750
cargo build --manifest-path /tmp/flash-published/Cargo.toml --release --locked
cargo build --manifest-path Cargo.toml --release --locked
sha256sum /tmp/flash-published/target/release/sokonanoda target/release/sokonanoda | tee /tmp/flash-qualification/binaries.sha256

- name: Fetch official Arena small-test corpus and record its SHA
run: |
set -euo pipefail
curl --fail --location --silent --show-error --retry 3 --max-time 120 \
https://arena.lean-lang.org/lean-arena-tests.tar.gz \
--output /tmp/flash-qualification/lean-arena-tests.tar.gz
sha256sum /tmp/flash-qualification/lean-arena-tests.tar.gz
mkdir -p /tmp/flash-qualification/tests
tar -xzf /tmp/flash-qualification/lean-arena-tests.tar.gz -C /tmp/flash-qualification/tests
find /tmp/flash-qualification/tests -name '*.ndjson' | wc -l

- name: Qualify changed soundness case with all other bounded verdicts preserved
run: |
set -euo pipefail
printf '{"use_stdin":true,"nat_extension":true,"string_extension":true,"unpermitted_axiom_hard_error":false,"unsafe_permit_all_axioms":true,"num_threads":2}\n' > /tmp/flash-qualification/config.json
python3 experiments/flash-closure-identity-20261104/qualify.py \
--before /tmp/flash-published/target/release/sokonanoda \
--after "$PWD/target/release/sokonanoda" \
--config /tmp/flash-qualification/config.json \
--fixtures /tmp/flash-qualification/tests \
--fixture-archive /tmp/flash-qualification/lean-arena-tests.tar.gz \
--report /tmp/flash-qualification/report.json

- name: Preserve source-pinned artifacts, including negative evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: flash-closure-identity-${{ github.sha }}
if-no-files-found: warn
path: |
/tmp/flash-qualification/report.json
/tmp/flash-qualification/binaries.sha256
/tmp/flash-qualification/config.json
/tmp/flash-qualification/lean-arena-tests.tar.gz
122 changes: 122 additions & 0 deletions experiments/flash-closure-identity-20261104/qualify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
"""Paired source-pinned Flash / Arena regression for closure identity and substitution fixes.

The candidate is allowed two deliberate verdict improvements: reject
both official Arena closure forgery exports. All other small-test
verdicts must remain byte-for-byte equivalent at the exit-code boundary.

This is a bounded correctness regression, not a Mathlib performance result.
"""

import argparse
import hashlib
import json
from pathlib import Path
import subprocess


TARGETS = ('pi-closure-context', 'closure-level-subst')


def digest(path):
h = hashlib.sha256()
with path.open('rb') as source:
for block in iter(lambda: source.read(1024 * 1024), b''):
h.update(block)
return h.hexdigest()


def check(binary, config, fixture, timeout):
try:
with fixture.open('rb') as stream:
result = subprocess.run(
[str(binary), str(config)], stdin=stream,
capture_output=True, timeout=timeout, check=False)
return {'rc': result.returncode,
'stderr_sha256': hashlib.sha256(result.stderr).hexdigest(),
'stderr_tail': result.stderr[-300:].decode(errors='replace')}
except subprocess.TimeoutExpired:
return {'rc': 'TIMEOUT'}


def main():
arg = argparse.ArgumentParser()
arg.add_argument('--before', type=Path, required=True)
arg.add_argument('--after', type=Path, required=True)
arg.add_argument('--config', type=Path, required=True)
arg.add_argument('--fixtures', type=Path, required=True)
arg.add_argument('--fixture-archive', type=Path, required=True)
arg.add_argument('--report', type=Path, required=True)
a = arg.parse_args()

# Every examined test is provided by the official Arena tarball.
cases = []
skipped = []
for fixture in sorted(a.fixtures.rglob('*.ndjson')):
parts = fixture.relative_to(a.fixtures).parts
if not parts or parts[0] not in ('good', 'bad'):
skipped.append({'case': str(fixture.relative_to(a.fixtures)),
'reason': 'not in good/bad directory'})
continue
if 'perf' in parts:
skipped.append({'case': str(fixture.relative_to(a.fixtures)),
'reason': 'performance workload excluded from bounded regression'})
continue
cases.append((fixture, parts[0]))
assert cases and all(any(target in fixture.name for fixture, _ in cases) for target in TARGETS)
assert len({str(fixture) for fixture, _ in cases}) == len(cases)

results = []
for index, (fixture, expected) in enumerate(cases):
original = check(a.before, a.config, fixture, timeout=60)
candidate = check(a.after, a.config, fixture, timeout=60)
identity = str(fixture.relative_to(a.fixtures))
is_target = any(target in fixture.name for target in TARGETS)
is_same = original['rc'] == candidate['rc']
passes = ((original['rc'] == 0 and candidate['rc'] == 1 and expected == 'bad')
if is_target else is_same)
result = {'case': identity, 'expected': expected,
'input_sha256': digest(fixture),
'before': original, 'after': candidate,
'target': is_target, 'protected': passes}
results.append(result)
print('FLASH_PAIR', json.dumps({
'case': identity, 'before': original['rc'], 'after': candidate['rc'],
'expected': expected, 'protected': passes}), flush=True)
# Preserve a partial evidence report even when a case falsifies the
# expected delta. We never erase negative or UNKNOWN observations.
a.report.parent.mkdir(parents=True, exist_ok=True)
a.report.write_text(json.dumps({
'suite_sha256': digest(a.fixture_archive),
'before_binary_sha256': digest(a.before),
'after_binary_sha256': digest(a.after),
'cases': results, 'skipped': skipped,
'status': 'INCOMPLETE' if index + 1 < len(cases) else 'EVALUATING',
}, indent=2) + '\n')
if not passes:
break

all_pass = len(results) == len(cases) and all(r['protected'] for r in results)
target_rows = [r for r in results if r['target']]
all_pass = all_pass and len(target_rows) == len(TARGETS)
report = {
'status': 'WARRANTED_BOUNDED_SMALL_ARENA_NONREGRESSION' if all_pass else 'REJECTED_OR_INCOMPLETE',
'qualified': all_pass,
'suite_sha256': digest(a.fixture_archive),
'before_binary_sha256': digest(a.before),
'after_binary_sha256': digest(a.after),
'test_count': len(cases), 'tested_count': len(results),
'targets': list(TARGETS),
'target_positive': all(r['protected'] for r in target_rows) and len(target_rows) == len(TARGETS),
'cases': results, 'skipped': skipped,
'full_mathlib': 'NOT_TESTED', 'performance': 'NOT_TESTED',
'attribution': 'Closure-equality soundness guards motivated by intgrah/sokonanoda issues #8 and #9',
}
a.report.write_text(json.dumps(report, indent=2) + '\n')
print('FLASH_PROTECTED_SUMMARY', json.dumps({k: v for k, v in report.items()
if k not in ('cases', 'skipped')}), flush=True)
if not all_pass:
raise SystemExit(1)


if __name__ == '__main__':
main()
20 changes: 19 additions & 1 deletion src/conv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,15 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> {
return true;
}
match (a, b) {
(Env::Nil { .. }, Env::Nil { .. }) => return true,
(Env::Nil { .. }, Env::Nil { .. }) => {
// A closure with zero bound variables can still carry a
// nontrivial universe substitution. Hash equality is not proof.
return match (a.lsub(), b.lsub()) {
(None, None) => true,
(Some(x), Some(y)) => x.ks == y.ks && x.vs == y.vs,
_ => false,
};
},
(
Env::Cons { v: va, parent: pa, hash: ha, .. },
Env::Cons { v: vb, parent: pb, hash: hb, .. },
Expand Down Expand Up @@ -202,6 +210,11 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> {
if bx.body == by.body
&& std::ptr::eq(*dx, *dy)
&& Self::envs_ptr_equal(bx.env, by.env)
&& match (bx.ctx, by.ctx) {
(None, None) => true,
(Some(x), Some(y)) => std::ptr::eq(x, y),
_ => false,
}
{
return true;
}
Expand All @@ -221,6 +234,11 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> {
if bx.body == by.body
&& std::ptr::eq(dx, dy)
&& Self::envs_ptr_equal(bx.env, by.env)
&& match (bx.ctx, by.ctx) {
(None, None) => true,
(Some(x), Some(y)) => std::ptr::eq(x, y),
_ => false,
}
{
return true;
}
Expand Down
Loading