Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions .github/workflows/flash-soundness-stack-20261104.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
name: Flash six-guard soundness stack — official Arena and exact Mathlib

on:
pull_request:
branches:
- flash
workflow_dispatch:

permissions:
contents: read

jobs:
six-guard-regression-and-mathlib:
runs-on: ubuntu-24.04
timeout-minutes: 95
env:
CARGO_INCREMENTAL: 0
CARGO_TERM_COLOR: always
RUSTFLAGS: -C target-cpu=native
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Record exact source, compiler, and published Flash baseline
run: |
set -euo pipefail
git rev-parse HEAD
git cat-file -e 78c7502bac8a5ba000057b3f083bc0595ac65750^{commit}
rustc --version
cargo --version
mkdir -p /tmp/flash-qualification

- name: Build published Flash and stacked candidate identically
run: |
set -euo pipefail
git worktree add --detach /tmp/flash-published 78c7502bac8a5ba000057b3f083bc0595ac65750
cargo build --manifest-path /tmp/flash-published/Cargo.toml --release --locked
cargo build --manifest-path Cargo.toml --release --locked
sha256sum /tmp/flash-published/target/release/sokonanoda target/release/sokonanoda | tee /tmp/flash-qualification/binaries.sha256

- name: Fetch official Arena small corpus and hash it
run: |
set -euo pipefail
curl --fail --location --silent --show-error --retry 3 --max-time 120 \
https://arena.lean-lang.org/lean-arena-tests.tar.gz \
--output /tmp/flash-qualification/lean-arena-tests.tar.gz
printf '%s %s\n' 8223cb53aa5fa65f471054fbdf8cdcf50206556cfc7888f75e910f7816f4c8bb /tmp/flash-qualification/lean-arena-tests.tar.gz | sha256sum -c -
mkdir -p /tmp/flash-qualification/tests
tar -xzf /tmp/flash-qualification/lean-arena-tests.tar.gz -C /tmp/flash-qualification/tests

- name: Require all six false ACCEPTs fixed and all other small verdicts unchanged
run: |
set -euo pipefail
printf '{"use_stdin":true,"nat_extension":true,"string_extension":true,"unpermitted_axiom_hard_error":false,"unsafe_permit_all_axioms":true,"num_threads":2}\n' > /tmp/flash-qualification/config.json
python3 experiments/flash-soundness-stack-20261104/qualify.py \
--before /tmp/flash-published/target/release/sokonanoda \
--after "$PWD/target/release/sokonanoda" \
--config /tmp/flash-qualification/config.json \
--fixtures /tmp/flash-qualification/tests \
--fixture-archive /tmp/flash-qualification/lean-arena-tests.tar.gz \
--report /tmp/flash-qualification/report.json

- name: Restore exact independently pinned full Mathlib export
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
mkdir -p /tmp/flash-qualification/mathlib-source /tmp/flash-qualification/mathlib
gh api repos/heathsanchez/test/actions/artifacts/13231986845/zip > /tmp/flash-qualification/mathlib.zip
printf '%s %s\n' 3dd3fbb522816bdaa9d9cd2808196e075bfa27e3d6c82648731140594f87ce2b /tmp/flash-qualification/mathlib.zip | sha256sum -c -
unzip -q /tmp/flash-qualification/mathlib.zip -d /tmp/flash-qualification/mathlib-source
zstd -d /tmp/flash-qualification/mathlib-source/mathlib.ndjson.zst -o /tmp/flash-qualification/mathlib.ndjson
printf '%s %s\n' 22c5de83469408950005589a3bc5ef5157c65549b5a802f85d567c29151306a6 /tmp/flash-qualification/mathlib.ndjson | sha256sum -c -
rm /tmp/flash-qualification/mathlib.zip /tmp/flash-qualification/mathlib-source/mathlib.ndjson.zst
ls -lh /tmp/flash-qualification/mathlib.ndjson

- name: Matched published-versus-stacked full Mathlib A/B/B/A
run: |
set -euo pipefail
python3 experiments/flash-soundness-stack-20261104/profile_mathlib.py \
--before /tmp/flash-published/target/release/sokonanoda \
--after "$PWD/target/release/sokonanoda" \
--config /tmp/flash-qualification/config.json \
--input /tmp/flash-qualification/mathlib.ndjson \
--outdir /tmp/flash-qualification/mathlib \
--seconds 900 \
--expected-input-sha256 22c5de83469408950005589a3bc5ef5157c65549b5a802f85d567c29151306a6

- name: Preserve results and negative evidence even if a gate fails
if: always()
uses: actions/upload-artifact@v4
with:
name: flash-six-guard-stack-${{ github.sha }}
if-no-files-found: warn
path: |
/tmp/flash-qualification/report.json
/tmp/flash-qualification/binaries.sha256
/tmp/flash-qualification/config.json
/tmp/flash-qualification/lean-arena-tests.tar.gz
/tmp/flash-qualification/mathlib/mathlib-report.json
/tmp/flash-qualification/mathlib/*.time
/tmp/flash-qualification/mathlib/*.stderr
/tmp/flash-qualification/mathlib/*.stdout
111 changes: 111 additions & 0 deletions experiments/flash-soundness-stack-20261104/profile_mathlib.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
"""Matched full-Mathlib replay of published Flash and six-guard candidate.

This is not an Arena submission or a proof of complete soundness. It records
exact input/binary hashes, kernel verdict, wall time and peak RSS, including
failures, timeouts and disagreement. A/B/B/A reduces order confounding.
"""
import argparse
import hashlib
import json
from pathlib import Path
import statistics
import subprocess
import time


def digest(path):
h = hashlib.sha256()
with path.open('rb') as f:
for buf in iter(lambda: f.read(8 * 1024 * 1024), b''):
h.update(buf)
return h.hexdigest()


def run(binary, config, inp, outdir, arm, index, seconds):
stem = outdir / f"{index:02d}-{arm}"
stdout = stem.with_suffix(".stdout")
stderr = stem.with_suffix(".stderr")
metrics = stem.with_suffix(".time")
command = ['/usr/bin/time', '-f',
'wall_seconds=%e\nuser_seconds=%U\nsys_seconds=%S\npeak_rss_kb=%M',
'-o', str(metrics), str(binary), str(config)]
result = {'arm': arm, 'ordinal': index, 'binary_sha256': digest(binary)}
start = time.monotonic()
try:
with inp.open('rb') as source, stdout.open('wb') as so, stderr.open('wb') as se:
proc = subprocess.run(command, stdin=source, stdout=so,
stderr=se, timeout=seconds, check=False)
result['returncode'] = proc.returncode
except subprocess.TimeoutExpired:
result['returncode'] = 'TIMEOUT'
result['elapsed_monotonic_seconds'] = round(time.monotonic() - start, 3)
result['stdout_sha256'] = digest(stdout) if stdout.exists() else None
result['stderr_sha256'] = digest(stderr) if stderr.exists() else None
result['stderr_tail'] = stderr.read_bytes()[-500:].decode(errors='replace') if stderr.exists() else None
if metrics.exists():
for line in metrics.read_text(errors='replace').splitlines():
if '=' in line:
key, val = line.split('=', 1)
try:
result[key] = float(val)
except ValueError:
result[key] = val
return result


def main():
ap = argparse.ArgumentParser()
ap.add_argument('--before', required=True, type=Path)
ap.add_argument('--after', required=True, type=Path)
ap.add_argument('--config', required=True, type=Path)
ap.add_argument('--input', required=True, type=Path)
ap.add_argument('--outdir', required=True, type=Path)
ap.add_argument('--seconds', type=int, default=900)
ap.add_argument('--expected-input-sha256', required=True)
a = ap.parse_args()
a.outdir.mkdir(parents=True, exist_ok=True)
input_sha = digest(a.input)
if input_sha != a.expected_input_sha256:
raise SystemExit(f'input SHA mismatch: {input_sha}')
report = {'status': 'INCOMPLETE', 'input_sha256': input_sha,
'input_bytes': a.input.stat().st_size,
'config_sha256': digest(a.config),
'published_binary_sha256': digest(a.before),
'candidate_binary_sha256': digest(a.after),
'order': ['before', 'after', 'after', 'before'],
'method': 'sequential A/B/B/A same runner, exact input and config; wall/RSS not arena-calibrated',
'runs': []}
path = a.outdir / 'mathlib-report.json'
path.write_text(json.dumps(report, indent=2) + '\n')
for index, arm in enumerate(report['order']):
binary = a.before if arm == 'before' else a.after
print('MATHLIB_BEGIN', json.dumps({'index': index, 'arm': arm}), flush=True)
result = run(binary, a.config, a.input, a.outdir, arm, index, a.seconds)
report['runs'].append(result)
path.write_text(json.dumps(report, indent=2) + '\n')
print('MATHLIB_RUN', json.dumps(result), flush=True)
if result['returncode'] != 0:
print('MATHLIB_STOP', json.dumps({'cause': 'non-ACCEPT, timeout or resource failure',
'index': index, 'arm': arm}), flush=True)
break
both_arms = {arm: [r for r in report['runs'] if r['arm'] == arm]
for arm in ('before', 'after')}
valid = len(report['runs']) == 4 and all(r['returncode'] == 0 for r in report['runs'])
if valid:
for arm, rs in both_arms.items():
report[f'{arm}_median_wall_seconds'] = statistics.median(r['wall_seconds'] for r in rs)
report[f'{arm}_max_peak_rss_kb'] = max(r['peak_rss_kb'] for r in rs)
report['candidate_vs_published_wall_ratio'] = (
report['after_median_wall_seconds'] / report['before_median_wall_seconds'])
report['status'] = 'BOUNDED_MATCHED_MATHLIB_ACCEPTANCE'
else:
report['status'] = 'UNKNOWN_OR_REJECTED_MATCHED_MATHLIB'
path.write_text(json.dumps(report, indent=2) + '\n')
print('MATHLIB_STACK_SUMMARY', json.dumps({
k: v for k, v in report.items() if k != 'runs'}), flush=True)
if not valid:
raise SystemExit(1)


if __name__ == '__main__':
main()
128 changes: 128 additions & 0 deletions experiments/flash-soundness-stack-20261104/qualify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
"""Paired source-pinned Flash / Arena regression for the six published soundness forgeries.

The candidate must reject six official Arena forgery exports previously accepted
by the published checker. Every other case must keep its exact exit code.

This is a bounded correctness regression, not a Mathlib performance result.
"""

import argparse
import hashlib
import json
from pathlib import Path
import subprocess


TARGETS = (
'pi-closure-context',
'closure-level-subst',
'nested-rec-rule-nfields',
'nested-name-capture',
'unit-rec-rule',
'inductive-undeclared-universe',
)


def digest(path):
h = hashlib.sha256()
with path.open('rb') as source:
for block in iter(lambda: source.read(1024 * 1024), b''):
h.update(block)
return h.hexdigest()


def check(binary, config, fixture, timeout):
try:
with fixture.open('rb') as stream:
result = subprocess.run(
[str(binary), str(config)], stdin=stream,
capture_output=True, timeout=timeout, check=False)
return {'rc': result.returncode,
'stderr_sha256': hashlib.sha256(result.stderr).hexdigest(),
'stderr_tail': result.stderr[-300:].decode(errors='replace')}
except subprocess.TimeoutExpired:
return {'rc': 'TIMEOUT'}


def main():
arg = argparse.ArgumentParser()
arg.add_argument('--before', type=Path, required=True)
arg.add_argument('--after', type=Path, required=True)
arg.add_argument('--config', type=Path, required=True)
arg.add_argument('--fixtures', type=Path, required=True)
arg.add_argument('--fixture-archive', type=Path, required=True)
arg.add_argument('--report', type=Path, required=True)
a = arg.parse_args()

# Every examined test is provided by the official Arena tarball.
cases = []
skipped = []
for fixture in sorted(a.fixtures.rglob('*.ndjson')):
parts = fixture.relative_to(a.fixtures).parts
if not parts or parts[0] not in ('good', 'bad'):
skipped.append({'case': str(fixture.relative_to(a.fixtures)),
'reason': 'not in good/bad directory'})
continue
if 'perf' in parts:
skipped.append({'case': str(fixture.relative_to(a.fixtures)),
'reason': 'performance workload excluded from bounded regression'})
continue
cases.append((fixture, parts[0]))
assert cases and all(any(target in fixture.name for fixture, _ in cases) for target in TARGETS)
assert len({str(fixture) for fixture, _ in cases}) == len(cases)

results = []
for index, (fixture, expected) in enumerate(cases):
original = check(a.before, a.config, fixture, timeout=60)
candidate = check(a.after, a.config, fixture, timeout=60)
identity = str(fixture.relative_to(a.fixtures))
is_target = any(target in fixture.name for target in TARGETS)
is_same = original['rc'] == candidate['rc']
passes = ((original['rc'] == 0 and candidate['rc'] == 1 and expected == 'bad')
if is_target else is_same)
result = {'case': identity, 'expected': expected,
'input_sha256': digest(fixture),
'before': original, 'after': candidate,
'target': is_target, 'protected': passes}
results.append(result)
print('FLASH_PAIR', json.dumps({
'case': identity, 'before': original['rc'], 'after': candidate['rc'],
'expected': expected, 'protected': passes}), flush=True)
# Preserve a partial evidence report even when a case falsifies the
# expected delta. We never erase negative or UNKNOWN observations.
a.report.parent.mkdir(parents=True, exist_ok=True)
a.report.write_text(json.dumps({
'suite_sha256': digest(a.fixture_archive),
'before_binary_sha256': digest(a.before),
'after_binary_sha256': digest(a.after),
'cases': results, 'skipped': skipped,
'status': 'INCOMPLETE' if index + 1 < len(cases) else 'EVALUATING',
}, indent=2) + '\n')
if not passes:
break

all_pass = len(results) == len(cases) and all(r['protected'] for r in results)
target_rows = [r for r in results if r['target']]
all_pass = all_pass and len(target_rows) == len(TARGETS)
report = {
'status': 'WARRANTED_BOUNDED_SMALL_ARENA_NONREGRESSION' if all_pass else 'REJECTED_OR_INCOMPLETE',
'qualified': all_pass,
'suite_sha256': digest(a.fixture_archive),
'before_binary_sha256': digest(a.before),
'after_binary_sha256': digest(a.after),
'test_count': len(cases), 'tested_count': len(results),
'targets': list(TARGETS),
'target_positive': all(r['protected'] for r in target_rows) and len(target_rows) == len(TARGETS),
'cases': results, 'skipped': skipped,
'full_mathlib': 'NOT_TESTED', 'performance': 'NOT_TESTED',
'attribution': 'Upstream-motivated soundness fixes for nanoda_lib #63/#74/#79 and sokonanoda #7/#8/#9; individually qualified before stacking',
}
a.report.write_text(json.dumps(report, indent=2) + '\n')
print('FLASH_PROTECTED_SUMMARY', json.dumps({k: v for k, v in report.items()
if k not in ('cases', 'skipped')}), flush=True)
if not all_pass:
raise SystemExit(1)


if __name__ == '__main__':
main()
20 changes: 19 additions & 1 deletion src/conv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,15 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> {
return true;
}
match (a, b) {
(Env::Nil { .. }, Env::Nil { .. }) => return true,
(Env::Nil { .. }, Env::Nil { .. }) => {
// A closure with zero bound variables can still carry a
// nontrivial universe substitution. Hash equality is not proof.
return match (a.lsub(), b.lsub()) {
(None, None) => true,
(Some(x), Some(y)) => x.ks == y.ks && x.vs == y.vs,
_ => false,
};
},
(
Env::Cons { v: va, parent: pa, hash: ha, .. },
Env::Cons { v: vb, parent: pb, hash: hb, .. },
Expand Down Expand Up @@ -202,6 +210,11 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> {
if bx.body == by.body
&& std::ptr::eq(*dx, *dy)
&& Self::envs_ptr_equal(bx.env, by.env)
&& match (bx.ctx, by.ctx) {
(None, None) => true,
(Some(x), Some(y)) => std::ptr::eq(x, y),
_ => false,
}
{
return true;
}
Expand All @@ -221,6 +234,11 @@ impl<'x, 't, 'p> TypeChecker<'x, 't, 'p> {
if bx.body == by.body
&& std::ptr::eq(dx, dy)
&& Self::envs_ptr_equal(bx.env, by.env)
&& match (bx.ctx, by.ctx) {
(None, None) => true,
(Some(x), Some(y)) => std::ptr::eq(x, y),
_ => false,
}
{
return true;
}
Expand Down
Loading
Loading