diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index d39411d3151..25a210dd4ce 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -30,7 +30,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/OfflineWhileServing.cfg ``` -The first checks every rule but the two with open findings, over four sessions: about 1.2M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 240k states. Both pass at the sync commit. The last two fail at the sync commit, each on its finding below, within seconds. `just test tla rs/moq-net/verify/reconnect` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule but the two with open findings, over four sessions: about 1.2M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 240k states. Both pass at the sync commit. The last two fail at the sync commit, each on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings diff --git a/.claude/skills/verify-resume/SKILL.md b/.claude/skills/verify-resume/SKILL.md index 91b1c1611a0..3e56ae51da6 100644 --- a/.claude/skills/verify-resume/SKILL.md +++ b/.claude/skills/verify-resume/SKILL.md @@ -28,7 +28,7 @@ quest gate just rs tla moq-net/verify/resume/ResumeReader.tla quest gate just rs tla moq-net/verify/resume/ResumeReader.tla -config rs/moq-net/verify/resume/ThreeRoutes.cfg ``` -The first checks every rule over two routes, three groups, two switches, and a reader holding each group until it finishes it: about 14.3M distinct states, in minutes. The second trades a group for a third route, so a route can come up after another failed: about 7.1M states, in minutes. Both pass at the sync commit. `just test tla rs/moq-net/verify/resume` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. +The first checks every rule over two routes, three groups, two switches, and a reader holding each group until it finishes it: about 14.3M distinct states, in minutes. The second trades a group for a third route, so a route can come up after another failed: about 7.1M states, in minutes. Both pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. ## Open findings diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index d36ed2ca136..6d60ca610dc 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -36,10 +36,9 @@ What the timer costs, as the configs show it: fired late, it costs only the wait ```sh quest gate just rs tla moq-net/verify/route/RouteHandoff.tla quest gate just rs tla moq-net/verify/route/RouteHandoff.tla -config rs/moq-net/verify/route/Ring.cfg -quest gate just test tla rs/moq-net/verify/route ``` -`just test tla rs/moq-net/verify/route` runs every config, as Nightly does, simulating each one with a `Simulate:` line and expecting `Loop.cfg` to break. Times are with `-workers 4`. `Patient` is TRUE unless the table says otherwise. +Run the configs the change touches, one at a time. Nightly runs every config (`just test tla rs/moq-net/verify/route`), simulating each one with a `Simulate:` line and expecting `Loop.cfg` to break. Times are on the recipe's 4 workers. `Patient` is TRUE unless the table says otherwise. | Config | Mesh | Bounds | Distinct states | Run | | --- | --- | --- | --- | --- | @@ -48,7 +47,7 @@ quest gate just test tla rs/moq-net/verify/route | `Twice.cfg` | The same, where two sessions close, as when relay 1 dies. Simulated: `num` counts behaviours per worker. | 1 claim change, 2 session changes | 40,000 behaviours, 4.0M states | a few minutes | | `Ring.cfg` | The publisher and three relays in a ring, with a timer that fires at any moment. | 1 claim change, 1 session change | 426,409 | a minute or two | | `Lag.cfg` | Relay 3 hears the publisher through relays 1 and 2: the lagging neighbour. | 2 claim changes, 1 session change | 1,299,405 | a few minutes | -| `LagWide.cfg` | The same, with relay 1 asking for `"a"` alone. | 2 claim changes, 1 session change | 4,194,747 | twenty minutes | +| `LagWide.cfg` | The same, with relay 1 asking for `"a"` alone. | 2 claim changes, 1 session change | 4,194,747 | ten minutes | | `Dead.cfg` | Two relays behind a relay whose only uplink asks for `"a"` alone, one with a dear session of its own to the publisher. Simulated. | 3 claim changes, 1 session change | 40,000 behaviours, 4.8M states | ten minutes | | `Blind.cfg` | A relay whose nearest neighbour asks for `"a"` alone, and a farther one knows everything. Simulated. | 2 claim changes, 1 session change | 40,000 behaviours, 4.8M states | ten minutes | | `Loop.cfg` | The publisher with a session to each of three relays in a triangle, all closing. Open finding: `RequestsLoopFree` breaks. | 3 session changes | 979,474 when it breaks | a few minutes | diff --git a/.claude/skills/verify-subscribe/SKILL.md b/.claude/skills/verify-subscribe/SKILL.md index 38d07c9f58d..f9462e7456c 100644 --- a/.claude/skills/verify-subscribe/SKILL.md +++ b/.claude/skills/verify-subscribe/SKILL.md @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla -config quest gate just rs tla moq-net/verify/subscribe/SubscribeAccounting.tla -config rs/moq-net/verify/subscribe/StreamWins.cfg ``` -The first checks every rule but `StreamWins` and `ContinuesWithinHeld` under `Pressure`, with one YIELDED reset and frames left out: about 76.5M distinct states, five minutes on 24 workers. The second gives the group one frame and two YIELDED resets, without `Pressure` and over two updates, so it also checks where each continuation starts: about 29.4M distinct states, two to three minutes. Both pass. The third checks `StreamWins` without `Pressure` or YIELDED and fails on the open finding below. `just test tla rs/moq-net/verify/subscribe` runs every config here, as Nightly does, and fails if a config with an `\* Open finding:` line passes. One group is enough: groups share only the order of the Subscribe Stream, and a second group multiplies the states (60M and more) without adding a race. +The first checks every rule but `StreamWins` and `ContinuesWithinHeld` under `Pressure`, with one YIELDED reset and frames left out: about 77.7M distinct states, fifteen minutes. The second gives the group one frame and two YIELDED resets, without `Pressure` and over two updates, so it also checks where each continuation starts: about 29.6M distinct states, five minutes. Both pass. The third checks `StreamWins` without `Pressure` or YIELDED and fails on the open finding below. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. One group is enough: groups share only the order of the Subscribe Stream, and a second group multiplies the states (60M and more) without adding a race. ## Open finding: a group read whole is named in a stale drop diff --git a/.github/justfile b/.github/justfile index 13dadeeeb60..4c85a9cf8f6 100644 --- a/.github/justfile +++ b/.github/justfile @@ -80,6 +80,7 @@ test: python3 sh/migrate.test.py python3 sh/text.test.py python3 sh/coverage-tables.test.py + python3 sh/tlc.test.py python3 sh/dispatch-query.test.py python3 sh/gh/test-inventory.test.py bun test ./sh/gh/setup.test.ts diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 27c5654660e..0c692d7a6a6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -435,17 +435,15 @@ jobs: fail-fast: false matrix: # The largest models get a runner each, and the route model's two - # largest configs (about 45 and 70 minutes on 4 workers) one per - # config; the `route` and `rest` legs take every other config, so a - # new one is checked without editing this list. sh/gh/nightly.test.py - # fails on a path that no longer exists. + # largest configs (about 30 and 15 minutes) share one; the `route` + # and `rest` legs take every other config, so a new one is checked + # without editing this list. sh/gh/nightly.test.py fails on a path + # that no longer exists. include: - name: resume paths: rs/moq-net/verify/resume - - name: route early - paths: rs/moq-net/verify/route/Early.cfg - - name: route wide - paths: rs/moq-net/verify/route/LagWide.cfg + - name: route large + paths: rs/moq-net/verify/route/Early.cfg rs/moq-net/verify/route/LagWide.cfg - name: route paths: "rs/moq-net/verify/route :!rs/moq-net/verify/route/Early.cfg :!rs/moq-net/verify/route/LagWide.cfg" - name: rest diff --git a/REVIEW.md b/REVIEW.md index eeae8f7ad39..c5d4c778b0e 100644 --- a/REVIEW.md +++ b/REVIEW.md @@ -50,7 +50,7 @@ The runtime guards fail Rust and bun tests only when a timer fires on the host c A TLA+ model that no longer describes its code keeps passing while proving nothing, and Nightly cannot tell. When a PR changes a file in a `verify-*` skill's Coverage table, run that skill on the head. Flag a covered change the model does not mirror, a config whose result changed without its `\* Open finding:` line following, and a sync commit not moved to the PR's base once the model matches (PRs squash, so the base is the latest commit it can name). -The flow models under `js/*/verify/` have no skill: give a PR that changes the code their README names the same check with `just test tla`. +The flow models under `js/*/verify/` have no skill: give a PR that changes the code their README names the same check, running the configs it touches as the README shows. # Not findings diff --git a/js/justfile b/js/justfile index fa3ab58dd38..81de35ff266 100644 --- a/js/justfile +++ b/js/justfile @@ -55,9 +55,9 @@ fix-changed list: # Model-check a TLA+ spec with TLC, e.g. `just js tla watch/verify/Watch.tla`. # Pass `-config js/.cfg` for a config not named after the spec. -# TLC runs at the lowest priority with a 4 GB heap, as `just rs tla` explains. +# sh/tlc.py says how a run shares the host. tla spec *args: - cd .. && JAVA_TOOL_OPTIONS=-Xmx4g nice -n 19 nix shell --inputs-from . nixpkgs#tlaplus18 --command tlc -workers auto -cleanup -noGenerateSpecTE -metadir "$(mktemp -d)" {{ args }} js/{{ spec }} + python3 ../sh/tlc.py js/{{ spec }} {{ args }} # Run all unit tests. test: diff --git a/js/publish/verify/README.md b/js/publish/verify/README.md index e2fba546a23..d69afbc223a 100644 --- a/js/publish/verify/README.md +++ b/js/publish/verify/README.md @@ -19,7 +19,7 @@ FAILED once the reopen budget runs out. ## Run ```sh -just test tla js/publish +quest gate just js tla publish/verify/Publish.tla ``` This must pass, as Nightly checks. diff --git a/js/room/verify/README.md b/js/room/verify/README.md index 1e8a1dac884..2f4c6b0f72f 100644 --- a/js/room/verify/README.md +++ b/js/room/verify/README.md @@ -16,7 +16,7 @@ comes back once it reconnects. ## Run ```sh -just test tla js/room +quest gate just js tla room/verify/Room.tla ``` `Room.cfg` plays only Bob's camera; add `"screen"` to diff --git a/js/watch/verify/README.md b/js/watch/verify/README.md index 7f55514f811..b1eabbb65e8 100644 --- a/js/watch/verify/README.md +++ b/js/watch/verify/README.md @@ -40,10 +40,13 @@ through `a redirected connection falls back to the configured URL once its targe ## Run ```sh -just test tla js/watch +quest gate just js tla watch/verify/Watch.tla +quest gate just js tla watch/verify/Failover.tla +quest gate just js tla watch/verify/Failover.tla -config js/watch/verify/FailoverRedirect.cfg +quest gate just js tla watch/verify/Failover.tla -config js/watch/verify/FailoverRedirectRefused.cfg ``` -This runs every config, as Nightly does. `Watch.cfg` and `Failover.cfg` check the rules that hold +Run the configs the change touches; Nightly runs every one (`just test tla js/watch`). `Watch.cfg` and `Failover.cfg` check the rules that hold and must pass; each rule that breaks has its own config, marked `\* Open finding:`, which must fail until the fix moves its rule into the passing one. A config checks the spec its name starts with. diff --git a/quest/m0/ci/tlc-cost.md b/quest/m0/ci/tlc-cost.md deleted file mode 100644 index 349a5c1b6c0..00000000000 --- a/quest/m0/ci/tlc-cost.md +++ /dev/null @@ -1,93 +0,0 @@ -# [M] TLC runs cost fewer core-seconds and share the host - -## Goal - -The `tla` recipes and their callers check every TLA+ config with the same -verdict as before for measurably fewer core-seconds, a TLC run leaves room on -the host for other gated commands, and no run, killed ones included, leaves -its working directory behind. Agents check the one config a change touches; -whole directories are Nightly's. - -## Plan - -Measured 2026-11-12 on the maintainer's agent hosts: - -- An ungated `just test tla rs/moq-net` matched one agent session's whole - usage: 21.4 CPU-hours over 1.8 hours. -- A gated `just test tla rs/moq-net/verify/route` took about 24 CPU-hours in - 53 minutes at about 31 cores, while 35 other gated commands queued behind - it, the oldest for 53 minutes. `quest gate` admits by load average, so one - run that takes every core stalls everything else. -- Another session ran TLC by hand, outside the recipes, for 2.5 hours: - 15.6 CPU-hours and 17.8 GB, on nixpkgs `tlaplus` 1.7.4 and a Java 8 JRE. -- Nightly splits these configs across five runners at about an hour each on - 4 workers. -- A killed run's `-metadir` under `/tmp` still held 26 GB. - -Decided 2026-11-12 (maintainer, /quest-plan): - -- ✅ Cap TLC's workers in `rs/justfile` and `js/justfile`, which hard-code - `-workers auto`. Recommended: 4, which the `verify-*` skills' times - already assume and Nightly's runners have; the native checker spike - ([moq-dev/moq#8600](https://github.com/moq-dev/moq/pull/8600)) measured - core-seconds near flat from 2 to 24 workers, so the cap costs wall time, - not CPU. -- ✅ TLC 1.8 on a modern JDK. The recipes moved to `nixpkgs#tlaplus18` in - [moq-dev/moq#8499](https://github.com/moq-dev/moq/pull/8499); the 1.7.4 run - above bypassed them. Leave no doc, skill, or script pointing at the 1.7.4 - package, `nixpkgs#tlaplus`. -- ✅ Replace `SumOver` in `rs/moq-net/verify/route/RouteHandoff.tla`, a - recursive function over `SUBSET S` that TLC re-evaluates on every use, with - the Community Modules' `FoldSet` (`FiniteSetsExt`), folding `f(x)` over - `S`. Not `SumSet({f(x) : x \in S})`, which sums each value once and so drops - two nodes' equal counts. TLC evaluates `FoldSet` natively only through the - Java override in `CommunityModules-deps.jar`, which nixpkgs does not - package: take it from a pinned release, put it on TLC's classpath in both - recipes, and confirm TLC's startup log reports the override loaded. - [moq-dev/moq#8564](https://github.com/moq-dev/moq/pull/8564) rewrites - `SumOver` as an Apalache fold; whichever lands second keeps one fold for the - model. -- ✅ Add `SYMMETRY` where sound. No config checks a `PROPERTY`, so every one - checks safety only, which symmetry preserves. A permutation is sound only if - it fixes every constant the config gives, such as `Links`, `Origin`, or - `Start`, and the spec never orders those values or does arithmetic on them; - so turn - a fully interchangeable set, such as `Relays` or `Endpoints`, into model - values and permute it, or the members no constant singles out. That can cut - a state space by up to n!, though `Endpoints = {"a", "b"}` with - `Start = "a"` gains nothing. Say per config whether it applies and why. -- ✅ Pass `-checkpoint 0`, and make sure no metadir outlives its run. TLC never - removes one it was killed in, and a signal handler cannot run on SIGKILL. - Recommended: put the metadir under the checkout's `.tmp/`, remove it when - the recipe exits, and at start remove the metadirs earlier runs left there - that no live run owns, so a checkout that outlives killed runs does not - collect them. Mark ownership so a concurrent run's metadir survives, such - as a lock each run holds on a file in its metadir, which the kernel drops - when the run dies. -- ✅ Agents check one config locally: the `verify-*` skills, `REVIEW.md`, and - the `js/*/verify/` READMEs name single configs, and only Nightly runs - `just test tla` over a directory. -- Rejected: Apalache, which is no cheaper here and ran out of memory on the - route model ([moq-dev/moq#8564](https://github.com/moq-dev/moq/pull/8564)); - waiting for a Rust checker, none of which ran the route model in - [moq-dev/moq#8600](https://github.com/moq-dev/moq/pull/8600). - -Not decided: the route-inductive work in progress adds a refinement check, -`RefinesClaims`, to the simulated `Blind.cfg`, where every sampled trace -re-checks it. Recommended: check a refinement on the smallest exhaustive config -that exercises it, where each distinct state is checked once, unless measuring -shows the simulated config is cheaper. - -Verification: every config keeps its verdict, and every exhaustive config -without `SYMMETRY` keeps its distinct-state count (the `verify-*` skills list them). -Measure each config's core-seconds through `quest gate` at equal worker counts -before and after, report the table in the PR, and update the times the skills -and Nightly's `tla` matrix comment quote; rebalance that matrix if its legs -change a lot. - -Public API: none. Wire: none. - -## Related - -- [A Rust TLA+ checker is rechecked once the route model drops SumOver](/quest/m2/tla-native-recheck.md) - a faster engine for the same specs, waiting on this quest's `SumOver` rewrite -- [The route rules hold from any state, not only the runs TLC reaches](/quest/m1/route-inductive.md) - rewrites `SumOver` and adds the refinement check named above diff --git a/quest/m1/route-inductive.md b/quest/m1/route-inductive.md index 51b89f9880e..a1e980dfffe 100644 --- a/quest/m1/route-inductive.md +++ b/quest/m1/route-inductive.md @@ -61,6 +61,12 @@ stay constants the config states. Say both bounds in the `verify-route` skill. `Budget` and `QuietIsSettled`. Leave it there. - Run it through the `tla` recipes with Apalache from the flake. If five relays does not fit Nightly, record the largest mesh that does and its time. +- A refinement check such as `RefinesClaims` runs on the smallest exhaustive + config that exercises it, where each distinct state is checked once, not on + a simulated config such as `Blind.cfg`, where every sampled trace re-checks + it, unless measuring shows the simulated config is cheaper. `SumOver` folds + with `FoldSet` through the Community Modules' Java override the `tla` + recipes load; keep one fold for both checkers. Public API: none. Wire: none. diff --git a/quest/m2/tla-native-recheck.md b/quest/m2/tla-native-recheck.md index 31e4e9b2ee7..422dd9f81af 100644 --- a/quest/m2/tla-native-recheck.md +++ b/quest/m2/tla-native-recheck.md @@ -21,11 +21,14 @@ model: - tlx and tla-rs exceed a 16 GB cap where TLC fits in 4 GB, tlx after finding 5% to 50% of TLC's states. -`SumOver` is a recursive function over `SUBSET S`, the likely cost for both, -and [TLC's cost quest](/quest/m0/ci/tlc-cost.md) replaces it with -`FoldSet`. Once that lands, rerun #8600's comparison at each checker's newest -revision: route and resume first, since they decide cost, then every -other exhaustive config, since adoption needs agreement on all of them: +`SumOver` was a recursive function over `SUBSET S`, the likely cost for both. +[moq-dev/moq#8628](https://github.com/moq-dev/moq/pull/8628) replaced it with +the Community Modules' `FoldSet` (`FiniteSetsExt`, through `INSTANCE`), which +`sh/tlc.py` puts on TLC's classpath, so a checker must also resolve that +module. Rerun #8600's comparison at each checker's newest revision, against +TLC remeasured after #8628: route and resume first, since they decide cost, +then every other exhaustive config, since adoption needs agreement on all of +them: - Build each from a pinned flake input inside the Nix sandbox, and run each check there as a derivation under `prlimit --as=16G`, one at a time through @@ -38,7 +41,3 @@ other exhaustive config, since adoption needs agreement on all of them: - Report upstream bugs to the maintainer rather than filing them. Public API: none. Wire: none. - -## Required - -- [TLC runs cost fewer core-seconds and share the host](/quest/m0/ci/tlc-cost.md) - replaces `SumOver` in the route model, the construct every candidate failed on, and moves TLC's baseline diff --git a/rs/justfile b/rs/justfile index 614efcb134a..01f039165b4 100644 --- a/rs/justfile +++ b/rs/justfile @@ -21,10 +21,9 @@ default: # Model-check a TLA+ spec with TLC, e.g. `just rs tla moq-net/verify/route/RouteHandoff.tla`. # Pass `-config rs/.cfg` for a config not named after the spec, and # `../.tla` for a spec outside `rs/`, such as a generated one in `.tmp/`. -# TLC runs at the lowest priority with a 4 GB heap, which no model here outgrows: -# the JVM otherwise takes a quarter of RAM per run. +# sh/tlc.py says how a run shares the host. tla spec *args: - JAVA_TOOL_OPTIONS=-Xmx4g nice -n 19 nix shell --inputs-from . nixpkgs#tlaplus18 --command tlc -workers auto -cleanup -noGenerateSpecTE -metadir "$(mktemp -d)" {{ args }} rs/{{ spec }} + python3 sh/tlc.py rs/{{ spec }} {{ args }} # Prove a Tamarin model's lemmas, e.g. `just rs tamarin moq-net/verify/route-key/RouteKey.spthy`. # Pass `-D=` for a variant the model defines. diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index 9ec5db5494a..2034b914402 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -377,10 +377,11 @@ Update(n, s0, ov, ups, gens, eb, rep, quiet) == \* queued toward it. Dropping them changes no behaviour; `Budget` still counts them. Heeded(d, m) == d[2] # Origin -\* The sum of f(x) over the finite set S. -SumOver(S, f(_)) == - LET F[T \in SUBSET S] == IF T = {} THEN 0 ELSE LET x == CHOOSE x \in T : TRUE IN f(x) + F[T \ {x}] - IN F[S] +\* The sum of f(x) for each x in the finite set S. +\* FoldSet runs natively through the Community Modules' Java override; an +\* instance keeps their `Restrict` from clashing with this module's. +LOCAL Ext == INSTANCE FiniteSetsExt +SumOver(S, f(_)) == Ext!FoldSet(LAMBDA x, acc : f(x) + acc, 0, S) \* Commit the updates `rs` (a function from nodes to `Update` results), appending \* each node's messages to `q0`. Nodes in one step send on disjoint queues. diff --git a/sh/tlc.py b/sh/tlc.py new file mode 100755 index 00000000000..813b40897b2 --- /dev/null +++ b/sh/tlc.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Model-check one TLA+ spec with TLC: `sh/tlc.py [tlc args...]`. + +The `tla` recipes run every model through here, so a run shares the host: +TLC runs at the lowest priority on 4 workers, which Nightly's runners have and +the `verify-*` skills' times assume; core-seconds stay near flat from 2 to 24 +workers, so more costs other commands their cores and saves no CPU. A 4 GB heap +fits every model here, where the JVM would otherwise take a quarter of RAM. + +TLC 1.8 runs with the Community Modules on its classpath, pinned below, so +their Java overrides (such as `FoldSet`) evaluate natively. + +TLC never removes a metadir it was killed in, and a model's can reach tens of +GB, so each run's lives under the checkout's `.tmp/tlc/`, is removed when the +run exits, and holds a lock the kernel drops when the run dies, however it +dies. A run first removes every metadir whose lock nobody holds. +""" + +import fcntl +import json +import os +import resource +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +# The newest release built for nixpkgs' TLC 1.8.0, a June 2023 build: later +# ones call `OpValue.eval`, which TLC added that November. Move both together. +COMMUNITY = ( + "https://github.com/tlaplus/CommunityModules/releases/download/202311011627/CommunityModules-deps-202311011627.jar", + "sha256-Ca3VmiMGYV/SODGw+cI8ENQOvPZj/5TdLZOtwSz5o9s=", +) + + +def nix(*args): + return subprocess.run(["nix", *args], cwd=ROOT, check=True, stdout=subprocess.PIPE, text=True).stdout + + +def classpath(): + """The JRE, and TLC's jar beside the Community Modules', from Nix.""" + jre, tla = nix( + "build", "--inputs-from", ".", "--no-link", "--print-out-paths", "nixpkgs#jre", "nixpkgs#tlaplus18" + ).split() + url, hash = COMMUNITY + community = json.loads(nix("store", "prefetch-file", "--json", "--expected-hash", hash, url))["storePath"] + return f"{jre}/bin/java", f"{tla}/share/java/tla2tools.jar:{community}" + + +def claim(base): + """Remove the metadirs no live run holds, then make and lock this run's. + + One lock over `base` keeps a sweep from taking a metadir between its + creation and its own lock. The returned descriptor holds the metadir's + lock; a child inheriting it keeps the lock while it lives. + """ + base.mkdir(parents=True, exist_ok=True) + with open(base / ".lock", "w") as guard: + fcntl.flock(guard, fcntl.LOCK_EX) + for meta in base.iterdir(): + if not meta.is_dir(): + continue + with open(meta / "lock", "a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + continue + shutil.rmtree(meta) + meta = Path(tempfile.mkdtemp(dir=base)) + fd = os.open(meta / "lock", os.O_WRONLY | os.O_CREAT, 0o644) + fcntl.flock(fd, fcntl.LOCK_EX) + return meta, fd + + +def release(base, meta): + """Remove this run's metadir under the lock a sweep holds, so neither sees the other's half-removed one.""" + with open(base / ".lock", "w") as guard: + fcntl.flock(guard, fcntl.LOCK_EX) + shutil.rmtree(meta, ignore_errors=True) + + +def main(argv): + if not argv: + sys.exit("usage: sh/tlc.py [tlc args...]") + spec, args = argv[0], argv[1:] + java, cp = classpath() + base = ROOT / ".tmp/tlc" + meta, fd = claim(base) + # Turn a SIGTERM into an ordinary exit so the metadir is removed; SIGINT + # already reaches TLC through the terminal's process group. + signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) + start, before = time.monotonic(), resource.getrusage(resource.RUSAGE_CHILDREN) + try: + command = ["nice", "-n", "19", java, "-Xmx4g", "-XX:+UseParallelGC", "-cp", cp, "tlc2.TLC"] + command += ["-workers", "4", "-checkpoint", "0", "-cleanup", "-noGenerateSpecTE", "-metadir", str(meta)] + status = subprocess.run([*command, *args, spec], cwd=ROOT, pass_fds=(fd,)).returncode + except KeyboardInterrupt: + status = 130 + finally: + release(base, meta) + after = resource.getrusage(resource.RUSAGE_CHILDREN) + cpu = after.ru_utime + after.ru_stime - before.ru_utime - before.ru_stime + print(f"tlc: {cpu:.0f} core-seconds in {time.monotonic() - start:.0f}s", file=sys.stderr) + # A child killed by a signal reports it negated; exit as a shell would. + return status if status >= 0 else 128 - status + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/sh/tlc.test.py b/sh/tlc.test.py new file mode 100755 index 00000000000..002f77e8af1 --- /dev/null +++ b/sh/tlc.test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""A TLC run's metadir sweep removes only the metadirs no live run holds.""" + +import importlib.util +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("tlc", ROOT / "sh/tlc.py") +tlc = importlib.util.module_from_spec(spec) +spec.loader.exec_module(tlc) + + +class Claim(unittest.TestCase): + def setUp(self): + scratch = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, scratch) + self.base = scratch / "tlc" + + def test_keeps_a_live_run_and_removes_a_dead_one(self): + live, fd = tlc.claim(self.base) + # A run that died, however it died, left its metadir and an unheld lock. + dead, dead_fd = tlc.claim(self.base) + os.close(dead_fd) + # A metadir from before locks, holding none. + bare = self.base / "bare" + bare.mkdir() + + mine, mine_fd = tlc.claim(self.base) + self.addCleanup(os.close, fd) + self.addCleanup(os.close, mine_fd) + self.assertTrue(live.is_dir()) + self.assertTrue(mine.is_dir()) + self.assertFalse(dead.exists()) + self.assertFalse(bare.exists()) + + def test_a_child_holding_the_lock_keeps_the_run_alive(self): + meta, fd = tlc.claim(self.base) + child = subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdin.read()"], stdin=subprocess.PIPE, pass_fds=(fd,) + ) + os.close(fd) + try: + tlc.claim(self.base) + self.assertTrue(meta.is_dir(), "the child still holds the lock") + finally: + child.communicate(b"") + tlc.claim(self.base) + self.assertFalse(meta.exists(), "the child exited, so nothing holds it") + + +if __name__ == "__main__": + unittest.main()