diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index 1f96d0fa12a..836272c7805 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -5,7 +5,9 @@ description: Model-check the lite-07 route layer (`rs/moq-net/src/route/`) with # Verify the route layer -`rs/moq-net/verify/route/RouteHandoff.tla` models one origin identity in `route::Router` as `route::tests::Sim` drives it: every router entry point is one atomic step ending in `update`, ROUTE, batches, and ROUTE_REQUEST share one FIFO per direction, and echoes travel on their own queue, free to overtake. TLC checks every ordering of those messages on the mesh and bounds each config names. +`rs/moq-net/verify/route/RouteHandoff.tla` models one origin identity in `route::Router` as `route::tests::Sim` drives it: every router entry point is one atomic step ending in `update`, ROUTE, batches, and ROUTE_REQUEST share one FIFO per direction, and echoes travel on their own queue, free to overtake. A config picks the mesh: which nodes there are, which sessions join them, which cost 2, and which nodes are clients (`Leaves`, as `Sim::leaf`) that forward no route but echo every ROUTE_END. TLC checks every ordering of those messages under each config's bounds, or samples them at random for a mesh too large to exhaust. + +One identity is enough for meshes with several origins: the router keys every piece of state by identity and recomputes one identity from that identity's state alone, so each identity's messages, read on their own, are a run of this model with that origin. Sharing one FIFO per direction only removes orderings, and a session flap is a flap for each. The configs put the origin at the end of a line, in a ring, behind a triangle, and on a client off a ring instead. ## Rules @@ -22,31 +24,42 @@ Each rule is an INVARIANT; the harm says why it matters. | `QuietIsSettled` | Once nothing is in flight, each relay reaching the origin shows its claims, available, and routes only while it has any; each cut-off relay shows nothing and keeps no route, distance, or request. | Routing never converges, or a dead origin stays shown. | | `SimplePaths` | Every route held or in flight names a loop-free path from the origin to its sender, at that path's cost. | A node took back a route that crossed it, the first step of a count to infinity. | | `NoCountToInfinity` | No relay selects a route longer than a loop-free path. | Stale routes bounce between relays with no bound. | +| `Budget` | From one quiet point to the next, the ROUTE, batch, ROUTE_REQUEST, and echo messages sent stay within 4 each way over each configured session, per route identity and per claim, for each external change made (a claim change, a session opening or closing, a collection). | One change sets off announce spam. | `SimplePaths` needs no search to believe: a node sends a route only after adding itself to hops that did not name it. It holds whatever a node forgot, which is why `NoCountToInfinity` no longer rests on the echo. +`Budget` is the one `mesh_sweep.rs` holds the implementation to, with the model's one identity. `Converges` is a PROPERTY: under fairness that something in flight is eventually delivered, routing goes quiet after the last external change and stays settled. `Budget` implies it, since the message count is part of the state: a cycle of deliveries that never goes quiet would grow the count past any budget. So every config checks `Budget`, and only `Line.cfg`, small enough for TLC's liveness checker, checks `Converges` directly. + ## Run ```sh just rs tla moq-net/verify/route/RouteHandoff.tla just rs tla moq-net/verify/route/RouteHandoff.tla -config rs/moq-net/verify/route/Behind.cfg just rs tla moq-net/verify/route/RouteHandoff.tla -config rs/moq-net/verify/route/BehindDear.cfg +just rs tla moq-net/verify/route/RouteHandoff.tla -config rs/moq-net/verify/route/Line.cfg +just rs tla moq-net/verify/route/RouteHandoff.tla -config rs/moq-net/verify/route/Ring.cfg +just rs tla moq-net/verify/route/RouteHandoff.tla -config rs/moq-net/verify/route/Chord.cfg -simulate num=1000 -depth 600 ``` -| Config | Mesh | Bounds | Distinct states | -| --- | --- | --- | --- | -| `RouteHandoff.cfg` | The origin and two relays in a triangle. | 2 claim changes, 2 session changes, 1 collection | 3,545,311 | -| `Behind.cfg` | The origin behind relay 1, which sits in a triangle with relays 2 and 3. | 1 claim change, 1 session change | 3,033,111 | -| `BehindDear.cfg` | The same, with relay 3 nearer the origin than relay 2. | 1 claim change, 1 session change | 5,377,597 | +Times are TLC 1.8.0 on Java 21 with `-workers 4` at `nice -n 19`, on a 24-core host under load from other work. + +| Config | Mesh | Bounds | Distinct states | Run | +| --- | --- | --- | --- | --- | +| `RouteHandoff.cfg` | The origin and two relays in a triangle. | 2 claim changes, 2 session changes, 1 collection | 24,192,399 | 39 min | +| `Behind.cfg` | The origin behind relay 1, which sits in a triangle with relays 2 and 3. | 1 claim change, 1 session change | 9,384,796 | 25 min | +| `BehindDear.cfg` | The same, with relay 3 nearer the origin than relay 2. | 1 claim change, 1 session change | 17,863,476 | 61 min | +| `Line.cfg` | The origin, three relays, and a watching client in a line; also checks `Converges`. | 2 claim changes, 1 session change, 1 collection | 1,543,073 | 13 min | +| `Ring.cfg` | The origin and three relays in a ring. | 2 claim changes, 1 session change | 18,437,833 | 61 min | +| `Chord.cfg` | A publishing client on a ring of five relays with a chord, and two watching clients. Simulated: `num` counts behaviours per worker. | 2 claims, 4 claim changes, 4 session changes, 2 collections | 4,000 behaviours, 4.6M states | 9 min | -All three pass at the sync commit, and `just test tla rs/moq-net/verify/route` runs them all, as Nightly does. Each takes minutes on 4 workers, so run them last. First run what answers in seconds: +All pass at the sync commit, and `just test tla rs/moq-net/verify/route` runs them all, as Nightly does, simulating `Chord.cfg` as its `Simulate:` line says. Run them last, one at a time. First run what answers in seconds: - `just rs test -p moq-net --lib -E 'test(/route::/)'`. `sweep_no_route_outgrows_a_loop_free_path` delivers every message and echo in a random order on random meshes with unequal costs, flaps, restarts, and collections, and checks the metric bound, loop freedom, and the quiet state. Raise its case count for a wider search: 40,000 cases run in 15 seconds in release. - TLC's simulation mode, which samples runs the exhaustive bounds do not reach. The config's `MaxRev`, `MaxFlaps`, and `MaxCollects` still bound a simulated run, so raise them in a copy of the config first, such as three session changes, two claim changes, and a collection, then add `-simulate num=3000 -depth 150`. Both behind-a-triangle configs fail `SimplePaths` within seconds if `Crossed` is made false, which is the count to infinity the hops fix: a relay drops its distance once its neighbours echoed, and takes back a copy of its own route that a third relay still held. -Read a counterexample with `/statechart`, passing it the spec, a config, and `RouteHandoff.view.ts`, which labels each relay's route and each step as the message that arrives. It gives the shortest path to each broken rule in those words. The view names the triangle's sessions and assumes equal costs. +Read a counterexample with `/statechart`, passing it the spec, a config, and `RouteHandoff.view.ts`, which labels each relay's route and each step as the message that arrives. It gives the shortest path to each broken rule in those words. The view labels any mesh and assumes equal costs. ## Reading a failure @@ -54,7 +67,7 @@ A counterexample is a path through the model, not yet a bug. Replay it on the ro ## Coverage -Sync commit: `c916855d163ff5641ad18cfb8e14b76dababd99a`, plus the change that added a route's hops ([moq-dev/moq#8487](https://github.com/moq-dev/moq/pull/8487)). Run `git log c916855d16.. -- rs/moq-net/src/route/` to see what changed since. After a change to a covered function, mirror it in the model, rerun every config, and move the sync commit forward only once the model describes the code at the new commit. `just check` fails when a function or type the table names no longer exists in its file (`sh/coverage-tables.py`), so each Code cell starts with a repo-relative path, then the symbols in it. +Sync commit: `caa3dd8926f16fbed15760ab22298085df73b655`. Run `git log caa3dd8926.. -- rs/moq-net/src/route/` to see what changed since. After a change to a covered function, mirror it in the model, rerun every config, and move the sync commit forward only once the model describes the code at the new commit. `just check` fails when a function or type the table names no longer exists in its file (`sh/coverage-tables.py`), so each Code cell starts with a repo-relative path, then the symbols in it. | Code | Model | | --- | --- | @@ -66,7 +79,8 @@ Sync commit: `c916855d163ff5641ad18cfb8e14b76dababd99a`, plus the change that ad | `rs/moq-net/src/route/mod.rs` `connect`, `peer_interest`, `disconnect`, `collect`, `change` | `Connect`, `Opened`, `Disconnect`, `Closed`, `Collect`, `ChangeClaim` | | `rs/moq-net/src/route/view.rs` `View::merge`, `View::restrict` | `Merge`, `Certify` | | `rs/moq-net/src/route/tests/mod.rs` `Sim` queues and `revivals`, `false_ends` | `q`, `acks`, `revivals`, `falseEnds` | +| `rs/moq-net/src/route/tests/mod.rs` `Sim::leaf`, `Stats` | `Leaves`, `burst` | ## Not covered -The model fixes every interest at the root, so it leaves out wildcards, `present`/`represent`/`retire`, interest changes, and the hidden `current` regions; prune's region bookkeeping reduces to "no certifier has a floor". It also leaves out scoped grants and qualified keys (so `advertised`, which only they read), GOAWAY, link priority, costs other than 1 and 2, the hop limit, the informational RTT, upstream links, identity rotation and restart, and `node.rs`'s per-stream keys. A batch carries the sender's whole presentation, equal to applying its diff on a FIFO stream. The origin drops the ROUTEs and batches it hears for its own identity, which change nothing there. A pass covers one claim at the bounds in the table above, and no mesh of more than three relays. `sweep_no_route_outgrows_a_loop_free_path`, `sweep_handoff`, and `sweep_scoped_handoff` still own the rest: larger meshes, more churn, restarts, and scoped grants. +The model fixes every interest at the root, so it leaves out wildcards, `present`/`represent`/`retire`, interest changes, and the hidden `current` regions; prune's region bookkeeping reduces to "no certifier has a floor". It also leaves out scoped grants and qualified keys (so `advertised`, which only they read), GOAWAY, link priority, costs other than 1 and 2, the hop limit, the informational RTT, upstream links, identity rotation and restart, `node.rs`'s per-stream keys and the seqno it advances when a route link closes, and how the origin above the router ranks two identities claiming the same broadcast. A batch carries the sender's whole presentation, equal to applying its diff on a FIFO stream. The origin drops the ROUTEs and batches it hears for its own identity, which change nothing there; `Budget` still counts them. An exhaustive pass covers one claim at the bounds in the table above, and no mesh of more than four relays. `sweep_no_route_outgrows_a_loop_free_path`, `sweep_handoff`, `sweep_scoped_handoff`, and `mesh_sweep.rs` still own the rest: larger meshes, more churn, restarts, and scoped grants. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ac5fc8d813a..796496f576d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -411,17 +411,25 @@ jobs: tla: name: TLA+ (${{ matrix.name }}) runs-on: ubuntu-24.04-arm - timeout-minutes: 120 + timeout-minutes: 180 strategy: fail-fast: false matrix: - # The two largest models get a runner each; the last leg takes every - # other config, so a new model is checked without editing this list. + # The largest models get a runner each, and the route model's three + # largest configs (about an hour each on 4 workers) one per config; the + # `route` and `rest` legs take every other config, so a new one is + # checked without editing this list. include: - name: resume paths: rs/moq-net/verify/resume + - name: route triangle + paths: rs/moq-net/verify/route/RouteHandoff.cfg + - name: route behind + paths: rs/moq-net/verify/route/BehindDear.cfg + - name: route ring + paths: rs/moq-net/verify/route/Ring.cfg - name: route - paths: rs/moq-net/verify/route + paths: "rs/moq-net/verify/route :!rs/moq-net/verify/route/RouteHandoff.cfg :!rs/moq-net/verify/route/BehindDear.cfg :!rs/moq-net/verify/route/Ring.cfg" - name: rest paths: ":!rs/moq-net/verify/resume :!rs/moq-net/verify/route" steps: diff --git a/quest/m1/route-inductive.md b/quest/m1/route-inductive.md index ee25ef8169a..d188f9ff58f 100644 --- a/quest/m1/route-inductive.md +++ b/quest/m1/route-inductive.md @@ -47,7 +47,6 @@ Public API: none. Wire: none. ## Required -- [The route model covers a mesh with clients and an announce budget](/quest/m1/route-mesh-model.md) - the spec parametrised over mesh shapes that this invariant is stated on - [The route model has a route per scope](/quest/m2/cluster-routing/routes/scope-model.md) - the rule for whose claims a relay believes, which the claim rules rest on; an invariant found before it is found for a rule about to go ## Related diff --git a/quest/m1/route-mesh-model.md b/quest/m1/route-mesh-model.md deleted file mode 100644 index a58ea119887..00000000000 --- a/quest/m1/route-mesh-model.md +++ /dev/null @@ -1,40 +0,0 @@ -# [L] The route model covers a mesh with clients and an announce budget - -## Goal - -The route model checks lite-07 routing on meshes larger than a triangle, with -more than one origin and with clients, and adds the two rules production has -broken most: a change costs a bounded number of announce messages, and -routing goes quiet after the last change. - -## Plan - -Decided 2026-11-10 (maintainer, /quest-plan), after the triangle model in -[moq-dev/moq#8447](https://github.com/moq-dev/moq/pull/8447) found count to -infinity. Routing, announce spam, and resume failover have caused many -production issues; this and the lite-07 mesh sweep -(`rs/moq-net/tests/integration/mesh_sweep.rs`) are routing's confidence gates -for finalizing lite-07. - -- Grow `RouteHandoff.tla` (or a sibling spec sharing its operators) to 4 and - 5 relays in a line, a ring, and a ring with a chord; two origin identities; - and client leaves that advertise nothing but echo every ROUTE_END. -- An invariant bounds the ROUTE, batch, and request messages sent between - external changes (an origin's claim change, a session open or close) by a - budget derived from the links, so announce spam is a counterexample. The - sweep's `budget` states 8 messages per session per route identity and per - claim; the model states the same one, or changes both. -- A liveness property, under weak fairness of delivery: after the last - external change, the queues empty and every relay shows what - `QuietIsSettled` requires. -- Keep explicit-state runs tractable with `SYMMETRY` over interchangeable - relays; run larger meshes with TLC's random simulation in Nightly, whose - `just test tla` (`sh/tla.sh`) runs each config exhaustively today and needs - a way to mark a config for simulation. -- Check `SimplePaths` and `NoCountToInfinity` on every mesh: a route names - the nodes it crossed since [moq-dev/moq#8487](https://github.com/moq-dev/moq/pull/8487). - -Replay every counterexample on `route::tests::Sim` before calling it a bug, as -the `verify-route` skill says, and pin confirmed ones as `residual_*` tests. - -Public API: none. Wire: none. diff --git a/quest/m1/route-trace-validation.md b/quest/m1/route-trace-validation.md index bd7a7742912..0aad11a4d7d 100644 --- a/quest/m1/route-trace-validation.md +++ b/quest/m1/route-trace-validation.md @@ -14,8 +14,8 @@ the route model drifting from the code ([moq-dev/moq#8447](https://github.com/mo Decided 2026-11-10 (maintainer, a later /quest-plan the same day): -- ✅ m1 (rejected: staying in m4). The [mesh model](/quest/m1/route-mesh-model.md), - the [inductive invariant](/quest/m1/route-inductive.md), and the +- ✅ m1 (rejected: staying in m4). The mesh configs beside it, the + [inductive invariant](/quest/m1/route-inductive.md), and the [cache-aware routing proof](/quest/m2/cache-routing-proof.md) all build on `RouteHandoff.tla`, so drift there costs the most. - ✅ The other three models get trace validation too, each its own m4 quest @@ -36,9 +36,6 @@ or the model grows to cover them; pick whichever costs less and say which. Run it on the sweeps' existing seeds in CI only if it stays fast; otherwise in Nightly with the sweeps' fresh seeds. -Related: [the mesh route model](/quest/m1/route-mesh-model.md), which grows -the spec these traces are checked against. - Public API: none. Wire: none. ## Required diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 3d61c5e7724..bcb2bae7a6a 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -565,10 +565,6 @@ benchmarks. subscriptions through the radio mesh and the split refuses, and after a partition each island reaches the other through its uplink. -## Required - -- [The route model covers a mesh with clients and an announce budget](/quest/m1/route-mesh-model.md) - every ordering on 4-5 relay meshes with two origins and clients, with an announce budget and convergence - ## Related - [Route wakes](/quest/m4/route-wakes.md) - index work follows the settled source-selection model diff --git a/rs/moq-net/verify/route/Behind.cfg b/rs/moq-net/verify/route/Behind.cfg index 5fcb950f9ab..07e962b6cad 100644 --- a/rs/moq-net/verify/route/Behind.cfg +++ b/rs/moq-net/verify/route/Behind.cfg @@ -24,3 +24,4 @@ INVARIANTS SimplePaths NoCountToInfinity QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/BehindDear.cfg b/rs/moq-net/verify/route/BehindDear.cfg index 4c89ba2967a..52e2ac8cbde 100644 --- a/rs/moq-net/verify/route/BehindDear.cfg +++ b/rs/moq-net/verify/route/BehindDear.cfg @@ -23,3 +23,4 @@ INVARIANTS SimplePaths NoCountToInfinity QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/Chord.cfg b/rs/moq-net/verify/route/Chord.cfg new file mode 100644 index 00000000000..e2059c4fc1f --- /dev/null +++ b/rs/moq-net/verify/route/Chord.cfg @@ -0,0 +1,29 @@ +\* Too large to check every ordering, so TLC samples it at random. +\* Simulate: -simulate num=1000 -depth 600 +\* A client (0) publishes through relay 1 of a ring of five relays (1 to 5) +\* with a chord from 2 to 4, and two clients (6 on 3, 7 on 4) watch. Enough +\* flaps cut the ring into a line. +CONSTANTS + Nodes = {0, 1, 2, 3, 4, 5, 6, 7} + Origin = 0 + Leaves = {6, 7} + Links = {{0, 1}, {1, 2}, {2, 3}, {3, 4}, {4, 5}, {5, 1}, {2, 4}, {3, 6}, {4, 7}} + Dear = {} + Claims = {"x", "y"} + MaxRev = 4 + MaxFlaps = 4 + MaxCollects = 2 + +SPECIFICATION Spec + +INVARIANTS + Truthful + AllCertified + NoRevival + NoFalseEnd + NoRegression + LoopFree + SimplePaths + NoCountToInfinity + QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/Line.cfg b/rs/moq-net/verify/route/Line.cfg new file mode 100644 index 00000000000..1c9e1999f0e --- /dev/null +++ b/rs/moq-net/verify/route/Line.cfg @@ -0,0 +1,31 @@ +\* A line: the origin (0) publishes through relays 1, 2, and 3 to a client +\* (4) that only watches. Every route has one path, so a cut leaves the far +\* side with nothing, and the client must echo every ROUTE_END it hears. +\* Converges checks under fairness what Budget implies: routing goes quiet. +CONSTANTS + Nodes = {0, 1, 2, 3, 4} + Origin = 0 + Leaves = {4} + Links = {{0, 1}, {1, 2}, {2, 3}, {3, 4}} + Dear = {} + Claims = {"x"} + MaxRev = 2 + MaxFlaps = 1 + MaxCollects = 1 + +SPECIFICATION LiveSpec + +INVARIANTS + Truthful + AllCertified + NoRevival + NoFalseEnd + NoRegression + LoopFree + SimplePaths + NoCountToInfinity + QuietIsSettled + Budget + +PROPERTIES + Converges diff --git a/rs/moq-net/verify/route/Ring.cfg b/rs/moq-net/verify/route/Ring.cfg new file mode 100644 index 00000000000..3b99e392b1b --- /dev/null +++ b/rs/moq-net/verify/route/Ring.cfg @@ -0,0 +1,26 @@ +\* A ring: the origin (0) and relays 1, 2, and 3, so relay 2 has two equal +\* paths and a cut anywhere leaves a longer way around. +CONSTANTS + Nodes = {0, 1, 2, 3} + Origin = 0 + Leaves = {} + Links = {{0, 1}, {1, 2}, {2, 3}, {3, 0}} + Dear = {} + Claims = {"x"} + MaxRev = 2 + MaxFlaps = 1 + MaxCollects = 0 + +SPECIFICATION Spec + +INVARIANTS + Truthful + AllCertified + NoRevival + NoFalseEnd + NoRegression + LoopFree + SimplePaths + NoCountToInfinity + QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/RouteHandoff.cfg b/rs/moq-net/verify/route/RouteHandoff.cfg index a549143adab..854ef63aade 100644 --- a/rs/moq-net/verify/route/RouteHandoff.cfg +++ b/rs/moq-net/verify/route/RouteHandoff.cfg @@ -23,3 +23,4 @@ INVARIANTS SimplePaths NoCountToInfinity QuietIsSettled + Budget diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index 93a75174bce..726f0d2c7bf 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -75,10 +75,12 @@ VARIABLES falseEnds, \* reader ENDs while the origin still has that item (`false_ends`) regressions,\* reader UPDATEs to an older item flaps, - collects + collects, + burst, \* ROUTE, batch, request, and echo messages sent since the last quiet point + churn \* external changes since the last quiet point vars == <> + regressions, flaps, collects, burst, churn>> \* The origin's view and seqno. OV == [rev |-> rev, items |-> own, seq |-> seqno] @@ -272,13 +274,16 @@ Update(n, s0, ov, ups, eb, prune) == Heeded(d, m) == d[2] # Origin \/ m.type = "request" \/ (m.type = "route" /\ m.reach.kind = "none") +\* The sum of f(x) over the finite set S. +SumOver(S, f(_)) == + LET F[T \in SUBSET S] == IF T = {} THEN 0 ELSE LET x == CHOOSE x \in T : TRUE IN f(x) + F[T \ {x}] + IN F[S] + \* Commit the updates `rs` (a function from nodes to `Update` results), appending \* each node's messages to `q0`. Nodes in one step send on disjoint queues. Commit(rs, q0) == LET ns == DOMAIN rs - Sum(f(_)) == LET F[S \in SUBSET ns] == - IF S = {} THEN 0 ELSE LET x == CHOOSE x \in S : TRUE IN f(x) + F[S \ {x}] - IN F[ns] + Sum(f(_)) == SumOver(ns, f) IN /\ st' = [n \in Nodes |-> IF n \in ns THEN rs[n].s ELSE st[n]] /\ q' = [d \in Dirs |-> IF d[1] \in ns THEN q0[d] \o SelectSeq(rs[d[1]].out[d[2]], LAMBDA m : Heeded(d, m)) ELSE q0[d]] @@ -287,6 +292,22 @@ Commit(rs, q0) == /\ falseEnds' = falseEnds + Sum(LAMBDA n : rs[n].ev.falseEnds) /\ regressions' = regressions + Sum(LAMBDA n : rs[n].ev.regressions) +Quiet == \A d \in Dirs : q[d] = <<>> /\ acks[d] = 0 + +\* The messages the updates `rs` send, counted before `Heeded` drops any. +Sent(rs) == SumOver(DOMAIN rs, LAMBDA n : SumOver(Nodes, LAMBDA m : Len(rs[n].out[m]))) + +\* A burst runs from one quiet point to the next. An external change (a claim +\* change, a session opening or closing, a collection) made while quiet starts +\* a fresh one; every message sent in it counts against the changes made in it. +External(sent) == + /\ burst' = (IF Quiet THEN 0 ELSE burst) + sent + /\ churn' = (IF Quiet THEN 0 ELSE churn) + 1 + +Internal(sent) == + /\ burst' = burst + sent + /\ UNCHANGED churn + ----------------------------------------------------------------------------- (* Initial state: every session open (`Sim::join`), the origin declared *) (* with no claims (`originate`), and nothing sent. *) @@ -318,6 +339,8 @@ Init == /\ regressions = 0 /\ flaps = 0 /\ collects = 0 + /\ burst = 0 + /\ churn = 0 ----------------------------------------------------------------------------- (* The origin changes a claim (`announce` / `unannounce` -> `change`). *) @@ -325,10 +348,12 @@ Init == ChangeClaim(c, item) == LET own1 == [own EXCEPT ![c] = item] ov1 == [rev |-> rev + 1, items |-> own1, seq |-> seqno] + rs == [n \in {Origin} |-> Update(Origin, st[Origin], ov1, up, ended[Origin], TRUE)] IN /\ rev' = rev + 1 /\ own' = own1 /\ hist' = Append(hist, own1) - /\ Commit([n \in {Origin} |-> Update(Origin, st[Origin], ov1, up, ended[Origin], TRUE)], q) + /\ Commit(rs, q) + /\ External(Sent(rs)) /\ UNCHANGED <> \* A new claim, or new attributes for one, under a fresh revision. @@ -401,26 +426,32 @@ Deliver(d) == ELSE CASE msg.type = "route" -> OnRoute(st[n], m, msg.reach) [] msg.type = "batch" -> OnBatch(st[n], m, msg.view) [] OTHER -> OnRequest(st[n], m, msg.seq) + rs == [x \in {n} |-> Update(n, s1, ov1, up, ended[n], ~same)] + echoes == IF isEnd THEN 1 ELSE 0 IN /\ q[d] # <<>> \* Every ROUTE_END is echoed once applied. /\ acks' = IF isEnd THEN [acks EXCEPT ![<>] = @ + 1] ELSE acks /\ seqno' = ov1.seq /\ IF skip THEN /\ q' = q0 + /\ Internal(echoes) /\ UNCHANGED <> - ELSE Commit([x \in {n} |-> Update(n, s1, ov1, up, ended[n], ~same)], q0) + ELSE /\ Commit(rs, q0) + /\ Internal(echoes + Sent(rs)) /\ UNCHANGED <> \* `echo`: an echoed ROUTE_END comes back (`Sim::deliver_ack`). DeliverAck(d) == LET m == d[1] n == d[2] + rs == [x \in {n} |-> Update(n, [st[n] EXCEPT !.unacked[m] = @ - 1], OV, up, ended[n], TRUE)] IN /\ acks[d] > 0 /\ acks' = [acks EXCEPT ![d] = @ - 1] /\ IF st[n].unacked[m] = 0 - THEN UNCHANGED <> - ELSE Commit([x \in {n} |-> - Update(n, [st[n] EXCEPT !.unacked[m] = @ - 1], OV, up, ended[n], TRUE)], q) + THEN /\ Internal(0) + /\ UNCHANGED <> + ELSE /\ Commit(rs, q) + /\ Internal(Sent(rs)) /\ UNCHANGED <> ----------------------------------------------------------------------------- @@ -441,43 +472,48 @@ Closed(s, y) == Connect(l) == LET up1 == up \cup {l} + rs == [x \in l |-> + LET y == CHOOSE y \in l : y # x + IN Update(x, Opened(st[x], y), OV, up1, ended[x], TRUE)] IN /\ l \notin up /\ flaps < MaxFlaps /\ up' = up1 /\ flaps' = flaps + 1 - /\ Commit([x \in l |-> - LET y == CHOOSE y \in l : y # x - IN Update(x, Opened(st[x], y), OV, up1, ended[x], TRUE)], q) + /\ Commit(rs, q) + /\ External(Sent(rs)) /\ UNCHANGED <> Disconnect(l) == LET up1 == up \ {l} q0 == [d \in Dirs |-> IF {d[1], d[2]} = l THEN <<>> ELSE q[d]] + rs == [x \in l |-> + LET y == CHOOSE y \in l : y # x + IN Update(x, Closed(st[x], y), OV, up1, ended[x], TRUE)] IN /\ l \in up /\ flaps < MaxFlaps /\ up' = up1 /\ flaps' = flaps + 1 /\ acks' = [d \in Dirs |-> IF {d[1], d[2]} = l THEN 0 ELSE acks[d]] - /\ Commit([x \in l |-> - LET y == CHOOSE y \in l : y # x - IN Update(x, Closed(st[x], y), OV, up1, ended[x], TRUE)], q0) + /\ Commit(rs, q0) + /\ External(Sent(rs)) /\ UNCHANGED <> \* `collect`, allowed once the node holds no route (`Sim::holds`). It \* forgets the views, so the reader's earlier ENDs no longer count. Collect(n) == + LET rs == [x \in {n} |-> + Update(n, [st[n] EXCEPT !.view = NoView, + !.heard = [m \in Nodes |-> NoHeard], + !.pending = NoPending], + OV, up, [c \in Claims |-> None], TRUE)] + IN /\ collects < MaxCollects /\ \A m \in Nodes : ~st[n].rx[m].on /\ collects' = collects + 1 - /\ Commit([x \in {n} |-> - Update(n, [st[n] EXCEPT !.view = NoView, - !.heard = [m \in Nodes |-> NoHeard], - !.pending = NoPending], - OV, up, [c \in Claims |-> None], TRUE)], q) + /\ Commit(rs, q) + /\ External(Sent(rs)) /\ UNCHANGED <> -Quiet == \A d \in Dirs : q[d] = <<>> /\ acks[d] = 0 - \* Nothing is in flight and no more churn is allowed: the run is over. Done == /\ Quiet @@ -529,12 +565,14 @@ Hop(n, k) == IF k = 0 THEN n ELSE Hop(NextHop(n), k - 1) LoopFree == \A n \in Relays : Hop(n, Cardinality(Nodes)) \notin Relays \* The loop-free paths from the origin over the configured links, each as the -\* nodes it crosses in order, and what each costs its last node. +\* nodes it crosses in order, and what each costs its last node. A path goes on +\* only from a node that forwards, so a client ends one. RECURSIVE Extend(_, _) +Last(p) == p[Len(p)] Extend(P, k) == IF k = 0 THEN P - ELSE Extend(P \cup {Append(p, n) : p \in P, n \in Relays}, k - 1) -Last(p) == p[Len(p)] + ELSE Extend(P \cup UNION {{Append(p, n) : n \in {m \in Relays : {Last(p), m} \in Links}} : + p \in {x \in P : Serves(Last(x))}}, k - 1) RECURSIVE Along(_) Along(p) == IF Len(p) = 1 THEN 0 @@ -586,4 +624,29 @@ Settled == QuietIsSettled == Quiet => Settled +\* What one external change may cost: 4 messages each way over each configured +\* session per route identity and per claim, the budget `mesh_sweep.rs` holds +\* the implementation to. Here there is one identity. +PerChange == 4 * Cardinality(Dirs) * (1 + Cardinality(Claims)) + +\* The messages sent since the last quiet point stay within the budget of the +\* external changes made since then, so a change never sets off announce spam. +\* `burst` is part of the state, so a cycle of deliveries that never goes quiet +\* would grow it past any budget: Budget also rules out routing that never +\* converges once the changes stop. +Budget == burst <= PerChange * churn + +\* While anything is in flight, something is eventually delivered. This is +\* WF_vars(Delivery) written out, since Delivery is enabled exactly when not +\* Quiet and TLC's ENABLED on these actions does not finish. It is weaker than +\* fairness per queue, so Converges fails on any cycle of deliveries, starving +\* a queue or not. +Delivery == \E d \in Dirs : Deliver(d) \/ DeliverAck(d) +Fairness == []<>Quiet \/ []<><>_vars + +LiveSpec == Spec /\ Fairness + +\* After the last external change, routing goes quiet and stays settled. +Converges == <>[](Quiet /\ Settled) + ============================================================================= diff --git a/rs/moq-net/verify/route/RouteHandoff.view.ts b/rs/moq-net/verify/route/RouteHandoff.view.ts index fe1b07be528..f9d65cf08ca 100644 --- a/rs/moq-net/verify/route/RouteHandoff.view.ts +++ b/rs/moq-net/verify/route/RouteHandoff.view.ts @@ -15,6 +15,8 @@ type Vars = { acks: Record; up: number[][]; own: Record; + burst: number; + churn: number; }; const ORIGIN = "0"; @@ -52,17 +54,22 @@ const relay = (n: string, s: Node) => { return `${how}${shows ? ", shows it" : ""}`; }; -const sessions = (v: Vars) => { - const down = ["0,1", "0,2", "1,2"].filter((l) => !v.up.some((u) => [...u].sort().join(",") === l)); - return down.length === 0 ? "all open" : down.map((l) => `${l.replace(",", "–")} closed`).join(", "); -}; +const sessions = (v: Vars) => + v.up + .map((l) => [...l].sort((a, b) => a - b).join("–")) + .sort() + .join(", ") || "none"; -export const title = "One origin identity across a triangle of relays"; +export const title = "One origin identity across a mesh of relays"; export const state = (v: Vars) => ({ - Sessions: sessions(v), - "Relay 1": relay("1", v.st["1"]), - "Relay 2": relay("2", v.st["2"]), + "Open sessions": sessions(v), + ...Object.fromEntries( + Object.keys(v.st) + .filter((n) => n !== ORIGIN) + .map((n) => [`Relay ${n}`, relay(n, v.st[n])]), + ), + "Messages this burst": `${v.burst} for ${v.churn} change${v.churn === 1 ? "" : "s"}`, }); const len = (s: Record | undefined) => (s ? Object.keys(s).length : 0); diff --git a/sh/tla.sh b/sh/tla.sh index bb817b9c3b9..742441e63e8 100755 --- a/sh/tla.sh +++ b/sh/tla.sh @@ -2,7 +2,9 @@ # Model-check every TLA+ config under the given paths (default: all of them) # through the `tla` recipes. A config with a `\* Open finding:` line must break # a rule (a deadlock, invariant, property, or assertion), so a fix that should -# promote its rule is noticed; every other config must pass. +# promote its rule is noticed; every other config must pass. A config with a +# `\* Simulate:` line is too large to exhaust: TLC samples it at random with the +# arguments that line gives. set -euo pipefail shopt -s nullglob cd "$(git rev-parse --show-toplevel)" @@ -40,11 +42,12 @@ for cfg in "${configs[@]}"; do lang=${cfg%%/*} finding=false ! grep -q '^\\\* Open finding:' "$cfg" || finding=true + read -ra simulate < <(sed -n 's/^\\\* Simulate: //p' "$cfg") || true echo "::group::$cfg" start=$SECONDS status=0 - just "$lang" tla "${specs[0]#"$lang"/}" -config "$cfg" || status=$? + just "$lang" tla "${specs[0]#"$lang"/}" -config "$cfg" "${simulate[@]}" || status=$? echo "::endgroup::" echo "$cfg: exit $status in $((SECONDS - start))s"