test(corpus): gate on node kinds, because a grammar is not a bag of words

tests/fixtures/conformance-0.18.1.json

@@ -1252,6 +1252,211 @@
       ],
       "source": "remember a number called total with 75\nrequire total is large\ndefine large: is above 50"
     },
+    {
+      "id": "require each over a list of numbers, with the condition's field elided",
+      "results": [
+        {
+          "canonical": "remember a list called scores with 80 and 95",
+          "nodes": [
+            "NumberLiteral",
+            "NumberLiteral",
+            "RememberListNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "require each score in scores each is above 70",
+          "nodes": [
+            "ConditionNode",
+            "EachPronoun",
+            "NameRef",
+            "NumberLiteral",
+            "RequireEachNode"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember a list called scores with 80 and 95\nrequire each score in scores is above 70"
+    },
+    {
+      "id": "the explicit binding and the elided half resolve to the same element",
+      "results": [
+        {
+          "canonical": "remember a list called values with 30 and 40",
+          "nodes": [
+            "NumberLiteral",
+            "NumberLiteral",
+            "RememberListNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "require each val in values each is above 0 and val is below 100",
+          "nodes": [
+            "CompoundConditionNode",
+            "ConditionNode",
+            "ConditionNode",
+            "EachPronoun",
+            "NameRef",
+            "NameRef",
+            "NumberLiteral",
+            "NumberLiteral",
+            "RequireEachNode"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember a list called values with 30 and 40\nrequire each val in values is above 0 and val is below 100"
+    },
+    {
+      "id": "where v8a and v31 meet: a predicate applied per element",
+      "results": [
+        {
+          "canonical": "remember a list called nums with 60 and 90",
+          "nodes": [
+            "NumberLiteral",
+            "NumberLiteral",
+            "RememberListNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "define large: each is above 50",
+          "nodes": [
+            "ConditionNode",
+            "DefineNode",
+            "EachPronoun",
+            "NumberLiteral"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "require each n in nums each is large",
+          "nodes": [
+            "EachPronoun",
+            "NameRef",
+            "PredicateApplicationNode",
+            "RequireEachNode"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember a list called nums with 60 and 90\ndefine large: is above 50\nrequire each n in nums is large"
+    },
+    {
+      "id": "the binding name may not be the list name, because it would read ambiguously",
+      "results": [
+        {
+          "canonical": "remember a list called scores with 80 and 95",
+          "nodes": [
+            "NumberLiteral",
+            "NumberLiteral",
+            "RememberListNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "require each scores in scores each is above 70",
+          "errorKind": "semantic",
+          "errorMessage": "The binding name can't be the same as the list name. Try: require each <item> in scores <condition>.",
+          "status": "error"
+        }
+      ],
+      "source": "remember a list called scores with 80 and 95\nrequire each scores in scores is above 70"
+    },
+    {
+      "id": "different construct entirely",
+      "results": [
+        {
+          "canonical": "remember a list called numbers with 3 and 9",
+          "nodes": [
+            "NumberLiteral",
+            "NumberLiteral",
+            "RememberListNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "each the numbers show",
+          "nodes": [
+            "EachNode",
+            "NameRef",
+            "ShowNode"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember a list called numbers with 3 and 9\neach the numbers show"
+    },
+    {
+      "id": "`show <field> of <record>` and produces a node that path does not",
+      "results": [
+        {
+          "canonical": "remember an order called o1 with total as 75",
+          "nodes": [
+            "NumberLiteral",
+            "RememberRecordNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "require total of o1 is above 50",
+          "nodes": [
+            "ConditionNode",
+            "FieldAccessNode",
+            "NumberLiteral",
+            "RequireNode"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember an order called o1 with total as 75\nrequire total of o1 is above 50"
+    },
+    {
+      "id": "a named composition, called",
+      "results": [
+        {
+          "canonical": "remember how to greet: show hello",
+          "nodes": [
+            "QuotedString",
+            "RememberCompositionNode",
+            "ShowNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "greet",
+          "nodes": [
+            "CompositionCallNode"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember how to greet: show \"hello\"\ngreet"
+    },
+    {
+      "id": "a named composition with a parameter, called with a literal argument",
+      "results": [
+        {
+          "canonical": "remember how to greet-name from name: show name",
+          "nodes": [
+            "NameRef",
+            "RememberCompositionNode",
+            "ShowNode"
+          ],
+          "status": "success"
+        },
+        {
+          "canonical": "greet-name from \"bob\"",
+          "nodes": [
+            "CompositionCallNode",
+            "QuotedString"
+          ],
+          "status": "success"
+        }
+      ],
+      "source": "remember how to greet-name from name: show name\ngreet-name from \"bob\""
+    },
     {
       "id": "weakens, over time",
       "results": [

tests/fixtures/conformance_corpus.txt

@@ -201,6 +201,48 @@ remember a number called total with 75
 require total is large
 define large: is above 50
 
+# require each (v8a §49) — iterated enforcement. Both its words appear
+# elsewhere in this corpus, so the reserved-word coverage gate has always
+# been satisfied while saying nothing at all about the construct. That is
+# the same blind spot the Calendar Era found: a gate over words cannot see
+# a grammar.
+# require each over a list of numbers, with the condition's field elided
+
+remember a list called scores with 80 and 95
+require each score in scores is above 70
+
+# the explicit binding and the elided half resolve to the same element
+remember a list called values with 30 and 40
+require each val in values is above 0 and val is below 100
+
+# where v8a and v31 meet: a predicate applied per element
+remember a list called nums with 60 and 90
+define large: is above 50
+require each n in nums is large
+
+# the binding name may not be the list name, because it would read ambiguously
+remember a list called scores with 80 and 95
+require each scores in scores is above 70
+
+# each, the iteration verb — the word appears above inside a `where` clause,
+# which is a pronoun and not this verb, so the word gate was satisfied by a
+# different construct entirely
+remember a list called numbers with 3 and 9
+each the numbers show
+
+# field access in a condition, which is a different operand path from
+# `show <field> of <record>` and produces a node that path does not
+remember an order called o1 with total as 75
+require total of o1 is above 50
+
+# a named composition, called
+remember how to greet: show "hello"
+greet
+
+# a named composition with a parameter, called with a literal argument
+remember how to greet-name from name: show name
+greet-name from "bob"
+
 # weakens, over time
 remember a number called urgency with 10
 weakens urgency over 10

tests/test_conformance_corpus.py

@@ -12,17 +12,30 @@
 it keep up: a reserved word with no case is a piece of surface the corpus
 cannot speak about, and adding a word to the vocabulary fails here until a
 program using it exists.
+
+Words are not enough on their own, and `require each` is the proof. Both of
+its words appear in other programs, so the reserved-word gate was satisfied
+while the corpus said nothing whatsoever about the construct — and the
+TypeScript port, which does not implement it, showed an empty divergence list
+that read as complete coverage. The same was true of the `each` verb, of field
+access inside a condition, and of a named composition being called.
+
+So there is a second gate below, over the AST node kinds the parser can
+produce. A grammar is not a bag of words, and a gate over words cannot see
+one.
 """
 
 from __future__ import annotations
 
+import inspect
 import json
 import subprocess
 import sys
 from pathlib import Path
 
 import pytest
 
+from liminate import parser as ast_module
 from liminate import vocabulary as vocab
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -80,6 +93,83 @@ def test_no_unreachable_entry_has_stopped_being_unreachable():
     assert not reached, f"{reached} are excused as unreachable and appear in the corpus"
 
 
+# Node kinds that no single corpus line can produce, each with the reason.
+# Same contract as UNREACHABLE above: no entry without a reason, and no reason
+# allowed to go stale.
+UNREACHABLE_NODES: dict[str, str] = {
+    "AboutNode": (
+        "produced only by the `about` declaration, which `validate()` handles "
+        "ahead of the per-line pipeline (see UNREACHABLE['about'])"
+    ),
+    "WhenNode": (
+        "produced only from a complete when-block — a header plus its "
+        "indented actions — which the per-line pipeline never assembles"
+    ),
+    "FinishNode": (
+        "`finish` outside an event handler is refused by the analyzer "
+        "(\'finish\' can only be used inside an event handler), and the "
+        "corpus has no handler to put one inside"
+    ),
+    "PackVerbNode": (
+        "produced only when a pack is loaded; the generator runs the bare "
+        "language, so no corpus line can reach a pack verb"
+    ),
+}
+
+
+def _parser_node_kinds() -> set[str]:
+    return {
+        name
+        for name, obj in vars(ast_module).items()
+        if inspect.isclass(obj)
+        and issubclass(obj, ast_module.ASTNode)
+        and obj is not ast_module.ASTNode
+    }
+
+
+def _corpus_node_kinds() -> set[str]:
+    fixture = ROOT / "tests" / "fixtures" / f"conformance-{_version()}.json"
+    if not fixture.exists():
+        pytest.skip("covered by the fixture-exists test")
+    seen: set[str] = set()
+    for case in json.loads(fixture.read_text())["cases"]:
+        for result in case["results"]:
+            seen.update(result.get("nodes") or [])
+    return seen
+
+
+def test_every_parser_node_kind_appears_in_a_corpus_program():
+    """The gate a word list cannot be.
+
+    `require each` is why this exists: `require` and `each` both appear in
+    other programs, so the word gate was green while the corpus held no case
+    for the construct at all, and the port's empty divergence list looked like
+    coverage. A node kind with no case is surface the corpus cannot speak
+    about, however many of its words are spoken elsewhere.
+    """
+    missing = sorted(_parser_node_kinds() - _corpus_node_kinds() - set(UNREACHABLE_NODES))
+    assert not missing, (
+        f"{missing} are node kinds the parser can produce and no corpus "
+        f"program produces, so the conformance corpus says nothing about how "
+        f"the port handles them"
+    )
+
+
+def test_no_unreachable_node_has_stopped_being_unreachable():
+    """A stale excuse is the same defect as a stale silence."""
+    reached = sorted(set(UNREACHABLE_NODES) & _corpus_node_kinds())
+    assert not reached, (
+        f"{reached} are excused as unreachable and the corpus produces them"
+    )
+
+
+def test_every_unreachable_node_is_still_a_node_the_parser_defines():
+    """An excuse for a node kind that no longer exists is dead weight that
+    would silently stop guarding anything."""
+    gone = sorted(set(UNREACHABLE_NODES) - _parser_node_kinds())
+    assert not gone, f"{gone} are excused as unreachable but the parser defines no such node"
+
+
 def test_the_generated_fixture_matches_the_version_it_was_generated_from():
     """The fixture names the version it describes, and the tree has moved
     since only if someone bumped the version without regenerating."""