From 971779aa49e0b933be3f71b80cab4ca4ccc8de6d Mon Sep 17 00:00:00 2001 From: Montana Date: Tue, 27 Oct 2026 02:14:13 -0700 Subject: [PATCH 1/4] differential: a model's checks share its cap, and a kill keeps what finished The pool kills a model's worker at its cap (900 s on the nightly), and every row of the model with it. MODEL1006230049, 0053 and 0077 take 716-751 s of that on main, so any check a pull request added to every model turned all their rows TIMEOUT: #1315, #1318 and #1319 each read 27-36 regressions, none of them a finding. check_all now takes a deadline, 0.9 of the cap. Every run has until it (run(timeout=)), no run or check starts past it, and a check that does not finish is its own TIMEOUT, said why. Each verdict is written as it is reached, so a worker the pool still kills (one compiling C past the deadline, which no run's timeout bounds) leaves the runner every row it finished; only the rest are TIMEOUT. --- parity_checks/differential/_differential.py | 103 ++++++++++++++-- .../differential/differential_run.py | 56 ++++++++- python/tests/test_differential_shapes.py | 115 ++++++++++++++++++ 3 files changed, 258 insertions(+), 16 deletions(-) diff --git a/parity_checks/differential/_differential.py b/parity_checks/differential/_differential.py index 735a1e8f8..52365a62d 100644 --- a/parity_checks/differential/_differential.py +++ b/parity_checks/differential/_differential.py @@ -69,6 +69,7 @@ import os import subprocess import sys +import time from dataclasses import dataclass, field from pathlib import Path @@ -93,6 +94,10 @@ #: Comma-decimal locales, in the order tried. COMMA_LOCALES = ("de_DE.UTF-8", "de_DE.utf8", "de_DE", "fr_FR.UTF-8", "fr_FR.utf8", "fr_FR") +#: The ``time.monotonic()`` every run of the model being checked must finish +#: by, which :func:`check_all` sets: None outside it, or with no cap. +_DEADLINE: float | None = None + CHECKS = ( "sens_runs", "sens_vs_plain", @@ -165,9 +170,22 @@ def sens_params(model) -> list[str]: return [p for p in model.primary_param_names if p not in excluded][:MAX_SENS_PARAMS] +def _left() -> float | None: + """The seconds to :data:`_DEADLINE`, a run's ``timeout``: None for none. Past + it no run starts (``SimulationTimeout``).""" + if _DEADLINE is None: + return None + left = _DEADLINE - time.monotonic() + if left <= 0: + import bngsim + + raise bngsim.SimulationTimeout("the model's cap is spent") + return left + + def run(job: Job, **kw): """A run of a fresh model (a run leaves its state on the Model it was given): - the Result and the Simulator.""" + the Result and the Simulator. It has until :data:`_DEADLINE`.""" import bngsim sim = bngsim.Simulator(load(job.path), method="ode", **{**job.sim, **kw}) @@ -177,6 +195,7 @@ def run(job: Job, **kw): rtol=job.rtol, atol=job.atol, max_steps=job.max_steps, + timeout=_left(), ) return result, sim @@ -279,7 +298,16 @@ def _declined(e: Exception) -> bool: ) +def _timed_out(e: BaseException) -> bool: + import bngsim + + return isinstance(e, bngsim.SimulationTimeout | subprocess.TimeoutExpired) + + def _verdict_of(e: Exception) -> Verdict: + if _timed_out(e): + # The model's cap, reached in this check: its own row, not every row. + return Verdict("timeout", None, f"not finished within the model's cap: {_first_line(e)}") status = "unsupported" if _declined(e) or isinstance(e, _Declined) else "exception" return Verdict(status, None, f"{type(e).__name__}: {_first_line(e)}") @@ -302,6 +330,9 @@ def check_sens(base: Base) -> dict[str, Verdict]: try: result, sim = run(base.job, sensitivity_params=params) except Exception as e: # noqa: BLE001 - that it did not complete is the verdict + if _timed_out(e): + v = _verdict_of(e) + return {"sens_runs": v, "sens_vs_plain": v} # Where the plain run completes and the sensitivity run does not: a # refusal by name, or a derivative that does not exist at the start # (a power of a species at 0), or a crash. The rule records it; the @@ -576,7 +607,11 @@ def _states(job: Job) -> list[tuple[float, np.ndarray]]: if t > job.t_start: try: bngsim.Simulator(model, method="ode", **job.sim).run( - t_span=(job.t_start, t), n_points=2, rtol=job.rtol, atol=job.atol + t_span=(job.t_start, t), + n_points=2, + rtol=job.rtol, + atol=job.atol, + timeout=_left(), ) except bngsim.SimulationError: # A state this one-interval run does not reach, where the base's @@ -651,7 +686,7 @@ def go(): [sys.executable, "-c", _LOCALE_PROBE, name, json.dumps(base.job.as_dict()), here], capture_output=True, text=True, - timeout=timeout, + timeout=timeout if _DEADLINE is None else min(timeout, _left()), ) if out.returncode != 0: last = (out.stderr.strip().splitlines() or ["?"])[-1][:300] @@ -661,19 +696,67 @@ def go(): return _guard(go) -def check_all(job: Job, locale_name: str | None, checks=CHECKS) -> dict[str, Verdict]: +def check_all( + job: Job, + locale_name: str | None, + checks=CHECKS, + deadline: float | None = None, + on_verdict=None, +) -> dict[str, Verdict]: """Every check of ``checks`` on one model. A base that does not run is no - reference: each check is then an exception on the base.""" + reference: each check is then an exception on the base. Every run has + until ``deadline`` (a ``time.monotonic()``): a check that has not finished + by then is TIMEOUT, its own row only, where the runner's cap would take + every row of the model (MODEL1006230049, 0053 and 0077 take 716-751 s of + the nightly's 900 s, so any check added to every model tipped them over). + ``on_verdict(check, verdict)`` hears each as it is reached, so a caller + killed at the cap still has those.""" + global _DEADLINE + _DEADLINE = deadline + try: + return _check_all(job, locale_name, checks, on_verdict or (lambda c, v: None)) + finally: + _DEADLINE = None + + +class _Heard(dict): + """The verdicts so far, each told to ``on_verdict`` as it is set.""" + + def __init__(self, on_verdict): + super().__init__() + self.on_verdict = on_verdict + + def __setitem__(self, check, verdict): + super().__setitem__(check, verdict) + self.on_verdict(check, verdict) + + def update(self, other): + for check, verdict in other.items(): + self[check] = verdict + + +def _check_all(job: Job, locale_name: str | None, checks, on_verdict) -> dict[str, Verdict]: try: base = Base(job) except Exception as e: # noqa: BLE001 # No base, no reference: not a finding of this suite's (the parity suites # own a model that does not load or run), and not a pass either. v = Verdict("no_base", None, f"the base run: {type(e).__name__}: {str(e)[:300]}") - return {c: v for c in checks} - out: dict[str, Verdict] = {} + return {c: _verdict_of(e) if _timed_out(e) else v for c in checks} + out: dict[str, Verdict] = _Heard(on_verdict) + + def spent() -> Verdict | None: + # No check starts past the deadline: one that builds and compiles + # Simulators without running them (backend_as_asked) has no run to bound. + try: + _left() + except Exception as e: # noqa: BLE001 - SimulationTimeout, the cap spent + return _verdict_of(e) + return None + if {"sens_runs", "sens_vs_plain"} & set(checks): - out.update(check_sens(base)) + v = spent() + out.update({"sens_runs": v, "sens_vs_plain": v} if v else check_sens(base)) one = { "ajac_vs_fdjac": check_ajac, "klu_vs_dense": check_klu, @@ -685,7 +768,7 @@ def check_all(job: Job, locale_name: str | None, checks=CHECKS) -> dict[str, Ver } for name, fn in one.items(): if name in checks: - out[name] = fn(base) + out[name] = spent() or fn(base) if "locale" in checks: - out["locale"] = check_locale(base, locale_name) + out["locale"] = spent() or check_locale(base, locale_name) return {c: out[c] for c in checks if c in out} diff --git a/parity_checks/differential/differential_run.py b/parity_checks/differential/differential_run.py index 0ac3d2e4d..d6611c665 100644 --- a/parity_checks/differential/differential_run.py +++ b/parity_checks/differential/differential_run.py @@ -39,9 +39,12 @@ from __future__ import annotations import argparse +import atexit import datetime as _dt import json +import shutil import sys +import tempfile import time from pathlib import Path @@ -60,6 +63,9 @@ RR_JOBS = HERE.parent / "rr_parity" / "ode_jobs.json" REFERENCE_ENGINE = "bngsim_paths" DEFAULT_TIMEOUT = 600.0 +#: The share of a model's cap its checks may use before the pool kills it: the +#: rest is the margin for the run that ends past the deadline, and the reply. +CAP_SHARE = 0.9 DEFAULT_MAX_SPECIES = 200 _OUTCOME = { @@ -129,6 +135,25 @@ def list_jobs(listing: Path, span: tuple[float, float]) -> list[D.Job]: return [_job_of(Path(x).resolve(), span) for x in listing.read_text().split() if x] +def _row(v) -> dict: + return {"status": v.status, "value": v.value, "note": v.note, "extra": v.extra} + + +def _keep(spec: dict, check: str, v) -> None: + """Each verdict as it is reached, to ``spec["partial"]``: a worker the pool + kills at the cap (one compiling C past the deadline) leaves the runner every + row it finished, and only the rest TIMEOUT.""" + path = spec.get("partial") + if not path: + return + path = Path(path) + rows = json.loads(path.read_text()) if path.is_file() else {} + rows[check] = _row(v) + tmp = path.with_suffix(".tmp") + tmp.write_text(json.dumps(rows, default=str)) + tmp.replace(path) + + def _worker(spec: dict, q) -> None: """One model's checks in a child process; puts one result dict on ``q``.""" t0 = time.perf_counter() @@ -150,11 +175,17 @@ def _worker(spec: dict, q) -> None: res["status"] = "over" res["why"] = f"{n} species, over the cap of {spec['max_species']}" else: - verdicts = D.check_all(job, spec["locale"], tuple(spec["checks"])) - res["rows"] = { - c: {"status": v.status, "value": v.value, "note": v.note, "extra": v.extra} - for c, v in verdicts.items() - } + # Short of the cap, at which the pool kills the worker and every row + # of the model with it: past the deadline a check is its own TIMEOUT. + deadline = time.monotonic() + spec["timeout"] * CAP_SHARE - (time.perf_counter() - t0) + verdicts = D.check_all( + job, + spec["locale"], + tuple(spec["checks"]), + deadline, + lambda c, v: _keep(spec, c, v), + ) + res["rows"] = {c: _row(v) for c, v in verdicts.items()} res["status"] = "done" except Exception as e: # noqa: BLE001 - a model that does not load is its rows' EXCEPTION res["status"] = "exception" @@ -170,6 +201,15 @@ def _value(v): def to_results(spec: dict, r: dict, stamp: dict) -> list[JobResult]: now = _dt.datetime.now().isoformat(timespec="seconds") rows = r.get("rows") + kept = spec.get("partial") and Path(spec["partial"]) + if rows is None and r.get("status") in ("timeout", "dead") and kept and kept.is_file(): + # Killed at the cap: what it finished stands, the rest timed out. + note = f"cap of {spec['timeout']:.0f} s" + done = json.loads(kept.read_text()) + rows = { + c: done.get(c) or {"status": r["status"], "value": None, "note": note, "extra": {}} + for c in spec["checks"] + } if rows is None: # the model as a whole: over the cap, raised, timed out or died status = {"over": "skip"}.get(r.get("status"), r.get("status", "exception")) note = r.get("why") or (f"cap of {spec['timeout']:.0f} s" if status == "timeout" else "") @@ -248,6 +288,9 @@ def main(argv=None) -> int: if args.models: subs = [s.strip() for s in args.models.split(",") if s.strip()] unique = [j for j in unique if any(s in j.model_id for s in subs)] + # Each model's verdicts as its worker reaches them (_keep), for one the pool kills. + kept = Path(tempfile.mkdtemp(prefix="differential-rows-")) + atexit.register(shutil.rmtree, kept, ignore_errors=True) specs = [ { "key": j.model_id, @@ -256,8 +299,9 @@ def main(argv=None) -> int: "locale": locale, "max_species": args.max_species, "timeout": args.timeout, + "partial": str(kept / f"{i}.json"), } - for j in unique + for i, j in enumerate(unique) ] print(f"differential: {len(specs)} models x {len(checks)} checks; locale {locale}") t0 = time.perf_counter() diff --git a/python/tests/test_differential_shapes.py b/python/tests/test_differential_shapes.py index b3e3d9694..5e9fa6e80 100644 --- a/python/tests/test_differential_shapes.py +++ b/python/tests/test_differential_shapes.py @@ -465,3 +465,118 @@ def fails_on_one_interval(self, *a, **kw): monkeypatch.setattr(bngsim.Simulator, "run", fails_on_one_interval) v = D.check_sparsity(base) assert v.status == "pass" and "at 1 states" in v.note, v + + +def test_a_run_past_the_models_deadline_does_not_start(monkeypatch): + import time + + monkeypatch.setattr(D, "_DEADLINE", time.monotonic() - 1.0) + with pytest.raises(bngsim.SimulationTimeout): + D.run(SHAPES["triangle-maxabs"]) + + +def test_a_cap_spent_after_the_base_times_out_each_check_alone(monkeypatch): + """The pool kills a model's worker at its cap, and every row with it: on + the nightly MODEL1006230049, 0053 and 0077 take 716-751 s of 900, so a check + added to every model turned all their rows TIMEOUT. Past the deadline each + check is its own TIMEOUT, said why, and none is an exception or a pass.""" + import time + + real = D.Base + + def base_then_spent(job): + b = real(job) + D._DEADLINE = time.monotonic() - 1.0 + return b + + monkeypatch.setattr(D, "Base", base_then_spent) + v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, deadline=time.monotonic() + 1e3) + assert {x.status for x in v.values()} <= {"timeout", "skip"}, v + assert v["grid"].status == "timeout", v["grid"] + assert "not finished within the model's cap" in v["grid"].note + assert v["sens_runs"].status == "timeout", v["sens_runs"] + assert D._DEADLINE is None + + +def test_no_check_starts_past_the_deadline(monkeypatch): + """backend_as_asked builds and compiles Simulators without running one, so no + run's timeout bounds it: on MODEL1006230049 it started past a 180 s + deadline and took the worker to 203 s, past a 200 s cap.""" + import time + + real = D.Base + started = [] + + def base_then_spent(job): + b = real(job) + D._DEADLINE = time.monotonic() - 1.0 + return b + + monkeypatch.setattr(D, "Base", base_then_spent) + monkeypatch.setattr(D, "check_backend", lambda base: started.append(1) or D.Verdict("pass")) + v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, deadline=time.monotonic() + 1e3) + assert not started and v["backend_as_asked"].status == "timeout", v["backend_as_asked"] + + +def test_a_base_past_the_deadline_is_a_timeout_on_every_row(): + import time + + v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, deadline=time.monotonic() - 1.0) + assert {x.status for x in v.values()} == {"timeout"}, v + + +def test_no_deadline_bounds_nothing(): + assert D._DEADLINE is None + v = D.check_all(SHAPES["triangle-maxabs"], None, ("grid",)) + assert v["grid"].status == "pass", v + + +def test_the_runner_hands_each_model_its_cap(tmp_path): + """The worker's deadline is the model's cap, short of it: with no time left + every row is a TIMEOUT of its own, not a model the pool killed.""" + import queue + + import differential_run as RUN + + job = SHAPES["triangle-maxabs"] + spec = { + "key": job.model_id, + "job": job.as_dict(), + "checks": ["grid"], + "locale": None, + "max_species": 200, + "timeout": 1e-6, + } + q = queue.Queue() + RUN._worker(spec, q) + res = q.get_nowait() + assert res["status"] == "done", res + assert res["rows"]["grid"]["status"] == "timeout", res + + +def test_a_model_killed_at_the_cap_keeps_the_rows_it_finished(tmp_path): + """A check compiling C past the deadline has no run to bound, and the pool + kills its worker at the cap: the verdicts it reached before stand, and only + the rest are TIMEOUT.""" + import differential_run as RUN + + job = SHAPES["triangle-maxabs"] + spec = { + "key": job.model_id, + "job": job.as_dict(), + "checks": ["grid", "backend_as_asked"], + "timeout": 900.0, + "partial": str(tmp_path / "0.json"), + } + RUN._keep(spec, "grid", D.Verdict("pass", 0.0, "agrees")) + rows = {r.method: r for r in RUN.to_results(spec, {"status": "timeout"}, {})} + assert rows["grid"].outcome == "PASS", rows["grid"] + assert rows["backend_as_asked"].outcome == "TIMEOUT" + assert rows["backend_as_asked"].exception == "cap of 900 s" + + +def test_each_verdict_is_heard_as_it_is_reached(): + heard = {} + v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, on_verdict=heard.__setitem__) + assert heard.keys() == v.keys() + assert all(heard[c] is v[c] for c in v) From 66763a6ac6ac714123a9d4d84318869cafd087ff Mon Sep 17 00:00:00 2001 From: Montana Date: Tue, 27 Oct 2026 08:12:19 -0700 Subject: [PATCH 2/4] nightly: the path-agreement job's cap at 1500 s per model With #1321, PRs stacked on this one still read 11-12 regressions: main's own checks on MODEL1006230049 and 0053 did not fit the 810 s deadline of a 900 s cap. Across five runs of the same checks those models took 689-894 s, so the cap flipped their rows with runner speed. 1500 s puts the deadline at 1350, and costs only the few models that use it. --- .github/workflows/nightly-parity.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-parity.yml b/.github/workflows/nightly-parity.yml index c4dc76571..58db0f6fe 100644 --- a/.github/workflows/nightly-parity.yml +++ b/.github/workflows/nightly-parity.yml @@ -619,10 +619,14 @@ jobs: # cap that finish in 22-90 s cold on an M3 Max (BIOMD0000000505: 48 s there), # and the slowest that finished took 248 s; a timeout that flips with runner # speed would alert as a regression. + # --timeout 1500, not 900: MODEL1006230049, 0053 and 0077 took 689-894 s + # across five runs on the same commit's checks, so a cap of 900 (its deadline + # 810, #1321) flipped their rows with runner speed. The cap costs only the + # few models that use it. - name: Run differential_run.py env: BNGSIM_CODEGEN_CACHE_DIR: ${{ runner.temp }}/codegen - run: uv run --no-sync python parity_checks/differential/differential_run.py --workers 4 --timeout 900 --out "$RUNNER_TEMP/differential.json" || true + run: uv run --no-sync python parity_checks/differential/differential_run.py --workers 4 --timeout 1500 --out "$RUNNER_TEMP/differential.json" || true - name: Compare env: GH_TOKEN: ${{ github.token }} From 78157b85cc1368ba09657788fcd31f29cadc7327 Mon Sep 17 00:00:00 2001 From: Montana Date: Tue, 27 Oct 2026 09:15:30 -0700 Subject: [PATCH 3/4] differential: a compile past bngsim's own limit is a timeout, not a crash With the 1500 s cap this PR's nightly read 2 new crashes, both BIOMD0000000385: given the time to reach its codegen compile, 4.2 MB of C ran past bngsim's 600 s compile limit, which codegen=True reports as a RuntimeError. Those rows were TIMEOUT at 900 s, and a timeout they stay: one anywhere in the cause chain, or in codegen=True's refusal, which carries the compile's timeout in its text and not as its cause. The other five rows of the model went TIMEOUT -> PASS. --- parity_checks/differential/_differential.py | 23 +++++++++++-- python/tests/test_differential_shapes.py | 37 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/parity_checks/differential/_differential.py b/parity_checks/differential/_differential.py index 52365a62d..2ec8fab1c 100644 --- a/parity_checks/differential/_differential.py +++ b/parity_checks/differential/_differential.py @@ -298,16 +298,33 @@ def _declined(e: Exception) -> bool: ) +#: The words bngsim's compile timeout is reported in (``_codegen``), where +#: ``codegen=True``'s refusal carries it in its text and not as its cause. +_COMPILE_TIMED_OUT = "Codegen compilation timed out" + + def _timed_out(e: BaseException) -> bool: + """A run stopped by the model's cap, or a compile by bngsim's own limit + (BIOMD0000000385's 4.2 MB of C past 600 s): a timeout, whatever wraps it, + and never a crash.""" import bngsim - return isinstance(e, bngsim.SimulationTimeout | subprocess.TimeoutExpired) + seen, depth = e, 0 + while seen is not None and depth < 16: + if isinstance(seen, bngsim.SimulationTimeout | subprocess.TimeoutExpired): + return True + seen, depth = seen.__cause__ or seen.__context__, depth + 1 + return _COMPILE_TIMED_OUT in str(e) def _verdict_of(e: Exception) -> Verdict: if _timed_out(e): - # The model's cap, reached in this check: its own row, not every row. - return Verdict("timeout", None, f"not finished within the model's cap: {_first_line(e)}") + # A cap reached in this check: its own row, not every row. + if _COMPILE_TIMED_OUT in str(e): + why = f"the compile timed out: {type(e).__name__}: {_first_line(e)}" + else: + why = f"not finished within the model's cap: {_first_line(e)}" + return Verdict("timeout", None, why) status = "unsupported" if _declined(e) or isinstance(e, _Declined) else "exception" return Verdict(status, None, f"{type(e).__name__}: {_first_line(e)}") diff --git a/python/tests/test_differential_shapes.py b/python/tests/test_differential_shapes.py index 5e9fa6e80..4010c290b 100644 --- a/python/tests/test_differential_shapes.py +++ b/python/tests/test_differential_shapes.py @@ -580,3 +580,40 @@ def test_each_verdict_is_heard_as_it_is_reached(): v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, on_verdict=heard.__setitem__) assert heard.keys() == v.keys() assert all(heard[c] is v[c] for c in v) + + +@pytest.mark.parametrize( + "error", + [ + # codegen=True's refusal: the compile's timeout in its text, no cause + RuntimeError( + "codegen=True requested, but the codegen build failed (RuntimeError: Codegen " + "compilation timed out after 600 s (4.2 MB C source at -O1). The RHS, ..." + ), + # the sensitivity build's: chained to the compile's TimeoutExpired + "chained", + ], + ids=["in-text", "chained"], +) +def test_a_compile_past_bngsims_limit_is_a_timeout_not_a_crash(error): + """BIOMD0000000385 compiles 4.2 MB of C, past bngsim's 600 s limit: given + the time to reach that compile, it read TIMEOUT -> EXCEPTION, a "new crash" + the nightly alerts on. A timeout, whatever wraps it.""" + import subprocess + + if error == "chained": + try: + try: + raise subprocess.TimeoutExpired(["cc"], 600) + except subprocess.TimeoutExpired as inner: + raise RuntimeError("Codegen compilation timed out after 600 s") from inner + except RuntimeError as compile_error: + try: + raise RuntimeError("Failed to build the analytical sensitivity RHS") from ( + compile_error + ) + except RuntimeError as outer: + error = outer + v = D._verdict_of(error) + assert v.status == "timeout", v + assert D._verdict_of(RuntimeError("Codegen compilation failed: syntax")).status == "exception" From e6832b4c9d5f90a964603895e0065ee013b61e77 Mon Sep 17 00:00:00 2001 From: Bill Hlavacek Date: Tue, 27 Oct 2026 12:59:59 -0600 Subject: [PATCH 4/4] differential: a locale run's own limit, and a dead worker's rows, as before the cap Three things the review of #1321 found, each pinned by a test that fails without the fix: - The locale child's own 600 s limit read as the model's cap: TIMEOUT, "not finished within the model's cap". Before the cap it read EXCEPTION, and a run a comma-decimal locale stalls, where the run under C finished, is that check's finding: from a DIFF or SKIP baseline the gate alerts on a new crash, and only files a TIMEOUT as changed. Only a limit the deadline set is the cap's now. - A worker that died after keeping rows gave the rest the note "cap of 900 s": nothing reached the cap. They now say the worker died, and its exit code. - A worker that raised past check_all dropped the rows it had kept, so a check that passed read EXCEPTION. What a worker finished stands however it ends; one block in to_results instead of two. The spent-cap test now asks for TIMEOUT on every row (it allowed SKIP, so a check skipping past the deadline went unseen), and the docs name the timeout verdict. --- parity_checks/differential/_differential.py | 28 +++++--- .../differential/differential_run.py | 24 +++---- python/tests/test_differential_shapes.py | 67 ++++++++++++++++++- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/parity_checks/differential/_differential.py b/parity_checks/differential/_differential.py index 2ec8fab1c..ea06c7079 100644 --- a/parity_checks/differential/_differential.py +++ b/parity_checks/differential/_differential.py @@ -38,9 +38,10 @@ Each returns a :class:`Verdict`: ``pass``, ``diff`` (a finding, with its size), ``unsupported`` (the path declined by type: not a wrong answer), ``skip`` (the path does not apply here, said why), ``exception`` (a path raised where the base -ran: a finding) or ``no_base`` (the model does not load or its plain run does +ran: a finding), ``no_base`` (the model does not load or its plain run does not complete, so there is nothing to compare against: the parity suites own -that). +that) or ``timeout`` (not finished within the model's cap, or a compile past +bngsim's own limit: that row only). The oracle has to be careful ---------------------------- @@ -114,7 +115,7 @@ @dataclass class Verdict: - status: str # pass | diff | unsupported | skip | exception | no_base + status: str # pass | diff | unsupported | skip | exception | no_base | timeout value: float | None = None note: str = "" extra: dict = field(default_factory=dict) @@ -699,12 +700,21 @@ def check_locale(base: Base, name: str | None, timeout: float = 600.0) -> Verdic def go(): here = str(Path(__file__).resolve().parent) - out = subprocess.run( - [sys.executable, "-c", _LOCALE_PROBE, name, json.dumps(base.job.as_dict()), here], - capture_output=True, - text=True, - timeout=timeout if _DEADLINE is None else min(timeout, _left()), - ) + # The model's cap, where less of it is left than this run's own limit. + limit = timeout if _DEADLINE is None else min(timeout, _left()) + try: + out = subprocess.run( + [sys.executable, "-c", _LOCALE_PROBE, name, json.dumps(base.job.as_dict()), here], + capture_output=True, + text=True, + timeout=limit, + ) + except subprocess.TimeoutExpired as e: + if limit < timeout: # the model's cap: TIMEOUT, this row only + raise + # Its own limit, where the run under C finished: a crash, as it read + # before the cap. A run the locale stalls is this check's finding. + return Verdict("exception", None, f"{type(e).__name__}: {_first_line(e)}") if out.returncode != 0: last = (out.stderr.strip().splitlines() or ["?"])[-1][:300] return Verdict("diff", None, f"runs under C, fails under {name}: {last}") diff --git a/parity_checks/differential/differential_run.py b/parity_checks/differential/differential_run.py index d6611c665..a3bb292e2 100644 --- a/parity_checks/differential/differential_run.py +++ b/parity_checks/differential/differential_run.py @@ -15,7 +15,8 @@ the path does not apply here (no KLU, no MIR), or the model does not load or run at all ....... SKIP it raised where the base ran .................... EXCEPTION - the model's cap was exceeded .................... TIMEOUT + it did not finish within the model's cap, or its + compile within bngsim's own limit ............. TIMEOUT The baseline (``nightly/baselines/differential.json``) holds tonight's known findings, so a night alerts on what changed: a model whose paths agreed and do @@ -201,20 +202,19 @@ def _value(v): def to_results(spec: dict, r: dict, stamp: dict) -> list[JobResult]: now = _dt.datetime.now().isoformat(timespec="seconds") rows = r.get("rows") - kept = spec.get("partial") and Path(spec["partial"]) - if rows is None and r.get("status") in ("timeout", "dead") and kept and kept.is_file(): - # Killed at the cap: what it finished stands, the rest timed out. - note = f"cap of {spec['timeout']:.0f} s" - done = json.loads(kept.read_text()) - rows = { - c: done.get(c) or {"status": r["status"], "value": None, "note": note, "extra": {}} - for c in spec["checks"] - } if rows is None: # the model as a whole: over the cap, raised, timed out or died status = {"over": "skip"}.get(r.get("status"), r.get("status", "exception")) - note = r.get("why") or (f"cap of {spec['timeout']:.0f} s" if status == "timeout" else "") + note = r.get("why") or { + "timeout": f"cap of {spec['timeout']:.0f} s", + "dead": f"the worker died (exit code {r.get('exitcode')})", + }.get(status, "") + # What a worker killed at the cap, or one that died or raised, finished + # stands (_keep): only the rest take the model's outcome. + kept = spec.get("partial") and Path(spec["partial"]) + done = json.loads(kept.read_text()) if kept and kept.is_file() else {} rows = { - c: {"status": status, "value": None, "note": note, "extra": {}} for c in spec["checks"] + c: done.get(c) or {"status": status, "value": None, "note": note, "extra": {}} + for c in spec["checks"] } out = [] for check, row in rows.items(): diff --git a/python/tests/test_differential_shapes.py b/python/tests/test_differential_shapes.py index 4010c290b..74f32c6b6 100644 --- a/python/tests/test_differential_shapes.py +++ b/python/tests/test_differential_shapes.py @@ -491,7 +491,7 @@ def base_then_spent(job): monkeypatch.setattr(D, "Base", base_then_spent) v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, deadline=time.monotonic() + 1e3) - assert {x.status for x in v.values()} <= {"timeout", "skip"}, v + assert {x.status for x in v.values()} == {"timeout"}, v assert v["grid"].status == "timeout", v["grid"] assert "not finished within the model's cap" in v["grid"].note assert v["sens_runs"].status == "timeout", v["sens_runs"] @@ -575,6 +575,71 @@ def test_a_model_killed_at_the_cap_keeps_the_rows_it_finished(tmp_path): assert rows["backend_as_asked"].exception == "cap of 900 s" +@pytest.mark.parametrize( + ("r", "note"), + [ + ({"status": "dead", "exitcode": -11}, "the worker died (exit code -11)"), + ({"status": "exception", "why": "ValueError: boom"}, "ValueError: boom"), + ], + ids=["died", "raised"], +) +def test_a_model_that_died_or_raised_keeps_the_rows_it_finished(tmp_path, r, note): + """A worker that segfaults, or raises past check_all, has finished what it + kept as surely as one killed at the cap. The rest say what happened to the + worker: one that died did not reach the cap.""" + import differential_run as RUN + + job = SHAPES["triangle-maxabs"] + spec = { + "key": job.model_id, + "job": job.as_dict(), + "checks": ["grid", "backend_as_asked"], + "timeout": 900.0, + "partial": str(tmp_path / "0.json"), + } + RUN._keep(spec, "grid", D.Verdict("pass", 0.0, "agrees")) + rows = {x.method: x for x in RUN.to_results(spec, r, {})} + assert rows["grid"].outcome == "PASS", rows["grid"] + assert rows["backend_as_asked"].outcome == "EXCEPTION" + assert rows["backend_as_asked"].exception == note + + +def _locale_runs_out(monkeypatch, left): + """``check_locale`` on a child that never finishes, with ``left`` seconds of + the model's cap (None for no cap): the timeout the child was given, and the + verdict.""" + import subprocess + import time + + base = D.Base(SHAPES["triangle-maxabs"]) + given = [] + + def never_finishes(cmd, timeout=None, **kw): + given.append(timeout) + raise subprocess.TimeoutExpired(cmd, timeout) + + monkeypatch.setattr(D.subprocess, "run", never_finishes) + monkeypatch.setattr(D, "_DEADLINE", None if left is None else time.monotonic() + left) + return given, D.check_locale(base, "de_DE.UTF-8", timeout=600.0) + + +def test_a_locale_run_past_the_models_cap_is_a_timeout(monkeypatch): + given, v = _locale_runs_out(monkeypatch, left=30.0) + assert given[0] <= 30.0, given + assert v.status == "timeout" and "the model's cap" in v.note, v + + +@pytest.mark.parametrize("left", [None, 1e4], ids=["no-cap", "cap-to-spare"]) +def test_a_locale_run_past_its_own_limit_is_not_the_caps(monkeypatch, left): + """The run under C finished; the same run under a comma-decimal locale + did not, in 600 s. That stays an EXCEPTION, as before the cap, and not a + TIMEOUT the baseline reads as a changed state: a locale that stalls a run is + the check's finding (#1235's class).""" + given, v = _locale_runs_out(monkeypatch, left=left) + assert given == [600.0], given + assert v.status == "exception" and "TimeoutExpired" in v.note, v + + def test_each_verdict_is_heard_as_it_is_reached(): heard = {} v = D.check_all(SHAPES["triangle-maxabs"], None, CHECKS, on_verdict=heard.__setitem__)