From 6526b0aa664d06dde14c1eb4920e2cb1975d7888 Mon Sep 17 00:00:00 2001 From: rmichaelthomas Date: Sat, 12 Sep 2026 01:46:12 -0700 Subject: [PATCH] test(corpus): gate on node kinds, because a grammar is not a bag of words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `require each` (v8a §49) had no corpus case and is not ported to TypeScript, and the reserved-word gate was green the whole time — because `require` and `each` both appear in other programs. The port's divergence list was empty and read as complete coverage. It was measuring a different thing. That is the same blind spot the Calendar Era found, one level up. v29 changed behaviour while adding no word, so a word-list gate could not see it, and the answer was a behavioural corpus. A corpus indexed by words has the identical hole: it cannot see a construct whose words are already spoken elsewhere. So the corpus now has a second gate, over the AST node kinds the parser can produce. Running it found `require each` and four more constructs the word gate had been silently satisfied about: RequireEachNode require each n in nums is above 0 EachNode the iteration verb — `each` appeared only as the pronoun inside a `where` clause, a different thing FieldAccessNode `total of o1` in a condition, a different operand path from `show total of o1`, which produces no such node CompositionCallNode a named composition, called — and called with a literal argument Eight cases added, bringing node coverage to 39 of 43. The remaining four are excused with a stated reason, on the same contract as the word list's: no entry without a reason, and a test that fails if a reason stops being true. `AboutNode` and `WhenNode` are `validate()`-level; `finish` outside a handler is refused by the analyzer; `PackVerbNode` needs a pack the generator does not load. The gate was verified by deletion: removing the `require each` cases fails it with `assert not ['RequireEachNode']`. The port now diverges on `require each`, which is correct — it does not implement it. That is the gate reporting a real gap instead of an empty list. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01E6qcDj1e1dEYvc8jLnpGZy --- tests/fixtures/conformance-0.18.1.json | 205 +++++++++++++++++++++++++ tests/fixtures/conformance_corpus.txt | 42 +++++ tests/test_conformance_corpus.py | 90 +++++++++++ 3 files changed, 337 insertions(+) diff --git a/tests/fixtures/conformance-0.18.1.json b/tests/fixtures/conformance-0.18.1.json index f85592f..0449536 100644 --- a/tests/fixtures/conformance-0.18.1.json +++ b/tests/fixtures/conformance-0.18.1.json @@ -1252,6 +1252,211 @@ ], "source": "remember a number called total with 75\nrequire total is large\ndefine large: is above 50" }, + { + "id": "require each over a list of numbers, with the condition's field elided", + "results": [ + { + "canonical": "remember a list called scores with 80 and 95", + "nodes": [ + "NumberLiteral", + "NumberLiteral", + "RememberListNode" + ], + "status": "success" + }, + { + "canonical": "require each score in scores each is above 70", + "nodes": [ + "ConditionNode", + "EachPronoun", + "NameRef", + "NumberLiteral", + "RequireEachNode" + ], + "status": "success" + } + ], + "source": "remember a list called scores with 80 and 95\nrequire each score in scores is above 70" + }, + { + "id": "the explicit binding and the elided half resolve to the same element", + "results": [ + { + "canonical": "remember a list called values with 30 and 40", + "nodes": [ + "NumberLiteral", + "NumberLiteral", + "RememberListNode" + ], + "status": "success" + }, + { + "canonical": "require each val in values each is above 0 and val is below 100", + "nodes": [ + "CompoundConditionNode", + "ConditionNode", + "ConditionNode", + "EachPronoun", + "NameRef", + "NameRef", + "NumberLiteral", + "NumberLiteral", + "RequireEachNode" + ], + "status": "success" + } + ], + "source": "remember a list called values with 30 and 40\nrequire each val in values is above 0 and val is below 100" + }, + { + "id": "where v8a and v31 meet: a predicate applied per element", + "results": [ + { + "canonical": "remember a list called nums with 60 and 90", + "nodes": [ + "NumberLiteral", + "NumberLiteral", + "RememberListNode" + ], + "status": "success" + }, + { + "canonical": "define large: each is above 50", + "nodes": [ + "ConditionNode", + "DefineNode", + "EachPronoun", + "NumberLiteral" + ], + "status": "success" + }, + { + "canonical": "require each n in nums each is large", + "nodes": [ + "EachPronoun", + "NameRef", + "PredicateApplicationNode", + "RequireEachNode" + ], + "status": "success" + } + ], + "source": "remember a list called nums with 60 and 90\ndefine large: is above 50\nrequire each n in nums is large" + }, + { + "id": "the binding name may not be the list name, because it would read ambiguously", + "results": [ + { + "canonical": "remember a list called scores with 80 and 95", + "nodes": [ + "NumberLiteral", + "NumberLiteral", + "RememberListNode" + ], + "status": "success" + }, + { + "canonical": "require each scores in scores each is above 70", + "errorKind": "semantic", + "errorMessage": "The binding name can't be the same as the list name. Try: require each in scores .", + "status": "error" + } + ], + "source": "remember a list called scores with 80 and 95\nrequire each scores in scores is above 70" + }, + { + "id": "different construct entirely", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "nodes": [ + "NumberLiteral", + "NumberLiteral", + "RememberListNode" + ], + "status": "success" + }, + { + "canonical": "each the numbers show", + "nodes": [ + "EachNode", + "NameRef", + "ShowNode" + ], + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\neach the numbers show" + }, + { + "id": "`show of ` and produces a node that path does not", + "results": [ + { + "canonical": "remember an order called o1 with total as 75", + "nodes": [ + "NumberLiteral", + "RememberRecordNode" + ], + "status": "success" + }, + { + "canonical": "require total of o1 is above 50", + "nodes": [ + "ConditionNode", + "FieldAccessNode", + "NumberLiteral", + "RequireNode" + ], + "status": "success" + } + ], + "source": "remember an order called o1 with total as 75\nrequire total of o1 is above 50" + }, + { + "id": "a named composition, called", + "results": [ + { + "canonical": "remember how to greet: show hello", + "nodes": [ + "QuotedString", + "RememberCompositionNode", + "ShowNode" + ], + "status": "success" + }, + { + "canonical": "greet", + "nodes": [ + "CompositionCallNode" + ], + "status": "success" + } + ], + "source": "remember how to greet: show \"hello\"\ngreet" + }, + { + "id": "a named composition with a parameter, called with a literal argument", + "results": [ + { + "canonical": "remember how to greet-name from name: show name", + "nodes": [ + "NameRef", + "RememberCompositionNode", + "ShowNode" + ], + "status": "success" + }, + { + "canonical": "greet-name from \"bob\"", + "nodes": [ + "CompositionCallNode", + "QuotedString" + ], + "status": "success" + } + ], + "source": "remember how to greet-name from name: show name\ngreet-name from \"bob\"" + }, { "id": "weakens, over time", "results": [ diff --git a/tests/fixtures/conformance_corpus.txt b/tests/fixtures/conformance_corpus.txt index 937c9d8..f48a097 100644 --- a/tests/fixtures/conformance_corpus.txt +++ b/tests/fixtures/conformance_corpus.txt @@ -201,6 +201,48 @@ remember a number called total with 75 require total is large define large: is above 50 +# require each (v8a §49) — iterated enforcement. Both its words appear +# elsewhere in this corpus, so the reserved-word coverage gate has always +# been satisfied while saying nothing at all about the construct. That is +# the same blind spot the Calendar Era found: a gate over words cannot see +# a grammar. +# require each over a list of numbers, with the condition's field elided + +remember a list called scores with 80 and 95 +require each score in scores is above 70 + +# the explicit binding and the elided half resolve to the same element +remember a list called values with 30 and 40 +require each val in values is above 0 and val is below 100 + +# where v8a and v31 meet: a predicate applied per element +remember a list called nums with 60 and 90 +define large: is above 50 +require each n in nums is large + +# the binding name may not be the list name, because it would read ambiguously +remember a list called scores with 80 and 95 +require each scores in scores is above 70 + +# each, the iteration verb — the word appears above inside a `where` clause, +# which is a pronoun and not this verb, so the word gate was satisfied by a +# different construct entirely +remember a list called numbers with 3 and 9 +each the numbers show + +# field access in a condition, which is a different operand path from +# `show of ` and produces a node that path does not +remember an order called o1 with total as 75 +require total of o1 is above 50 + +# a named composition, called +remember how to greet: show "hello" +greet + +# a named composition with a parameter, called with a literal argument +remember how to greet-name from name: show name +greet-name from "bob" + # weakens, over time remember a number called urgency with 10 weakens urgency over 10 diff --git a/tests/test_conformance_corpus.py b/tests/test_conformance_corpus.py index 60afe62..3622aaf 100644 --- a/tests/test_conformance_corpus.py +++ b/tests/test_conformance_corpus.py @@ -12,10 +12,22 @@ it keep up: a reserved word with no case is a piece of surface the corpus cannot speak about, and adding a word to the vocabulary fails here until a program using it exists. + +Words are not enough on their own, and `require each` is the proof. Both of +its words appear in other programs, so the reserved-word gate was satisfied +while the corpus said nothing whatsoever about the construct — and the +TypeScript port, which does not implement it, showed an empty divergence list +that read as complete coverage. The same was true of the `each` verb, of field +access inside a condition, and of a named composition being called. + +So there is a second gate below, over the AST node kinds the parser can +produce. A grammar is not a bag of words, and a gate over words cannot see +one. """ from __future__ import annotations +import inspect import json import subprocess import sys @@ -23,6 +35,7 @@ import pytest +from liminate import parser as ast_module from liminate import vocabulary as vocab ROOT = Path(__file__).resolve().parents[1] @@ -80,6 +93,83 @@ def test_no_unreachable_entry_has_stopped_being_unreachable(): assert not reached, f"{reached} are excused as unreachable and appear in the corpus" +# Node kinds that no single corpus line can produce, each with the reason. +# Same contract as UNREACHABLE above: no entry without a reason, and no reason +# allowed to go stale. +UNREACHABLE_NODES: dict[str, str] = { + "AboutNode": ( + "produced only by the `about` declaration, which `validate()` handles " + "ahead of the per-line pipeline (see UNREACHABLE['about'])" + ), + "WhenNode": ( + "produced only from a complete when-block — a header plus its " + "indented actions — which the per-line pipeline never assembles" + ), + "FinishNode": ( + "`finish` outside an event handler is refused by the analyzer " + "(\'finish\' can only be used inside an event handler), and the " + "corpus has no handler to put one inside" + ), + "PackVerbNode": ( + "produced only when a pack is loaded; the generator runs the bare " + "language, so no corpus line can reach a pack verb" + ), +} + + +def _parser_node_kinds() -> set[str]: + return { + name + for name, obj in vars(ast_module).items() + if inspect.isclass(obj) + and issubclass(obj, ast_module.ASTNode) + and obj is not ast_module.ASTNode + } + + +def _corpus_node_kinds() -> set[str]: + fixture = ROOT / "tests" / "fixtures" / f"conformance-{_version()}.json" + if not fixture.exists(): + pytest.skip("covered by the fixture-exists test") + seen: set[str] = set() + for case in json.loads(fixture.read_text())["cases"]: + for result in case["results"]: + seen.update(result.get("nodes") or []) + return seen + + +def test_every_parser_node_kind_appears_in_a_corpus_program(): + """The gate a word list cannot be. + + `require each` is why this exists: `require` and `each` both appear in + other programs, so the word gate was green while the corpus held no case + for the construct at all, and the port's empty divergence list looked like + coverage. A node kind with no case is surface the corpus cannot speak + about, however many of its words are spoken elsewhere. + """ + missing = sorted(_parser_node_kinds() - _corpus_node_kinds() - set(UNREACHABLE_NODES)) + assert not missing, ( + f"{missing} are node kinds the parser can produce and no corpus " + f"program produces, so the conformance corpus says nothing about how " + f"the port handles them" + ) + + +def test_no_unreachable_node_has_stopped_being_unreachable(): + """A stale excuse is the same defect as a stale silence.""" + reached = sorted(set(UNREACHABLE_NODES) & _corpus_node_kinds()) + assert not reached, ( + f"{reached} are excused as unreachable and the corpus produces them" + ) + + +def test_every_unreachable_node_is_still_a_node_the_parser_defines(): + """An excuse for a node kind that no longer exists is dead weight that + would silently stop guarding anything.""" + gone = sorted(set(UNREACHABLE_NODES) - _parser_node_kinds()) + assert not gone, f"{gone} are excused as unreachable but the parser defines no such node" + + def test_the_generated_fixture_matches_the_version_it_was_generated_from(): """The fixture names the version it describes, and the tree has moved since only if someone bumped the version without regenerating."""