diff --git a/scripts/gen_conformance_corpus.py b/scripts/gen_conformance_corpus.py new file mode 100644 index 0000000..305b8f9 --- /dev/null +++ b/scripts/gen_conformance_corpus.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Generate the behavioural conformance corpus from this implementation. + +The Python package is the authority; `@liminate/ts-validator` is a hand-written +port of five of its stages. The existing parity gate compares the two +implementations' *reserved-word lists* against a frozen fixture — which is why +it stayed green while three minor versions of behaviour diverged. Adding `date` +to `_require_comparable` (v29, Calendar Era) changed what programs are accepted +and changed no word at all, so a word-list gate could not see it. Downstream, +`commongage` carried a code comment recording "a DATE RANGE cannot be written in +a Liminate sentence at this language version" for two months after it could. + +This emits what a word list cannot: for each program in the corpus, what the +validation pipeline actually does with it. + +The pipeline mirrored here is exactly the one the TypeScript package assembles +in `_run_line` — tokenize, reorder, parse, analyze, render — and stops where it +stops. No execution: the port has no interpreter, so runtime behaviour is not a +parity surface and is deliberately not recorded. + +Its boundary, stated rather than left to be discovered: this is the *per-line* +surface. `validate()` wraps it with two things this corpus does not reach — the +first-line handling of `about`, and when-block buffering — so a program using +either is not a case here. They are a real parity surface and an honest gap, +not a passing one: a case that silently exercised `_run_line` instead would +report agreement about a path neither side took. + +Usage: + python3 scripts/gen_conformance_corpus.py > tests/fixtures/conformance-.json +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from liminate.analyzer import analyze # noqa: E402 +from liminate.lexer import LexError, tokenize # noqa: E402 +from liminate.parser import parse # noqa: E402 +from liminate.renderer import render # noqa: E402 +from liminate.reorderer import reorder # noqa: E402 +from liminate.result import LiminateResult, ResultStatus # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +CORPUS_PATH = ROOT / "tests" / "fixtures" / "conformance_corpus.txt" + + +def language_version() -> str: + """The version of the source being read, not of whatever is installed. + + `importlib.metadata.version("liminate")` answers for the installed + distribution, which on any machine with an editable checkout and a released + wheel is a different number from the tree this script just imported. A + corpus labelled with the wrong version is worse than an unlabelled one: the + port would compare itself against a file claiming a parity it never had. + """ + for line in (ROOT / "pyproject.toml").read_text(encoding="utf-8").splitlines(): + if line.startswith("version"): + return line.split("=", 1)[1].strip().strip('"') + raise SystemExit("pyproject.toml states no version") + +_ERROR_KIND = { + ResultStatus.ERROR_PARSE: "parse", + ResultStatus.ERROR_SEMANTIC: "semantic", +} + + +def validate_line(line: str, symtab: dict) -> dict: + """One line through the ported stages, reported the way the port reports it. + + Deliberately shaped as `ValidationResult` from the TypeScript package — + `status`, `canonical`, `errorKind`, `errorMessage` — so a case can be + compared field for field without either side reshaping the other's output. + """ + try: + tokens = tokenize(line) + except LexError as e: + return {"status": "error", "errorKind": "parse", "errorMessage": str(e)} + if not tokens: + return {"status": "success", "canonical": ""} + + reordered = reorder(tokens) + if isinstance(reordered, LiminateResult): + return _from_result(reordered) + + ast = parse(reordered) + if isinstance(ast, LiminateResult): + return _from_result(ast) + + analysis = analyze(ast, symtab) + if isinstance(analysis, LiminateResult): + return {**_from_result(analysis), "canonical": render(ast)} + + return {"status": "success", "canonical": render(ast)} + + +def _from_result(result: LiminateResult) -> dict: + if result.status in (ResultStatus.AMBER_PRECEDENCE, ResultStatus.AMBER_AMBIGUITY): + return {"status": "amber", "amberMessage": result.message} + return { + "status": "error", + "errorKind": _ERROR_KIND.get(result.status, result.status.value), + "errorMessage": result.message, + } + + +def read_corpus() -> list[dict]: + """Read the corpus file. + + A case is a `#` comment block naming it, then one or more program lines. + Multi-line cases share a symbol table, because a program's later lines + depend on what its earlier lines declared — a single-line corpus could not + reach any condition over a remembered value, which is most of the surface + that drifts. + """ + cases: list[dict] = [] + name: str | None = None + lines: list[str] = [] + for raw in CORPUS_PATH.read_text(encoding="utf-8").splitlines(): + if raw.startswith("# "): + if name is not None and lines: + cases.append({"id": name, "source": "\n".join(lines)}) + name, lines = raw[2:].strip(), [] + elif raw.strip(): + lines.append(raw) + if name is not None and lines: + cases.append({"id": name, "source": "\n".join(lines)}) + return cases + + +def main() -> None: + """Validate each line, then execute it so the next line sees the state. + + The two implementations reach that state by different routes and this is + the one place the difference has to be handled. TypeScript has no + interpreter, so it simulates just enough with `update_symbol_table`; Python + has one, so it runs the line. What gets *recorded* is the validation result + either way — execution here only advances the symbol table, and its own + statuses (a fired prohibition, a runtime error) are never written to the + corpus, because the port cannot produce them and a parity file must not + contain a field one side can never match. + """ + from liminate.run import Session # noqa: PLC0415 — keeps the import local + + cases = [] + for case in read_corpus(): + session = Session() + results = [] + for line in case["source"].splitlines(): + result = validate_line(line, session.symtab) + results.append(result) + if result["status"] == "success": + session.run_line(line) + cases.append({"id": case["id"], "source": case["source"], "results": results}) + json.dump( + { + "language_version": language_version(), + "generator": "scripts/gen_conformance_corpus.py", + "surface": "tokenize -> reorder -> parse -> analyze -> render", + "cases": cases, + }, + sys.stdout, + indent=2, + sort_keys=True, + ) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/src/liminate/reorderer.py b/src/liminate/reorderer.py index 0b42404..e08dd9b 100644 --- a/src/liminate/reorderer.py +++ b/src/liminate/reorderer.py @@ -165,8 +165,22 @@ def _validate_where_head(tokens: list[Token]) -> ReorderOutput: head.type is TokenType.UNKNOWN or (head.type is TokenType.VERB and head.value == "each") ) + # `is` opens a comparison; `includes` opens a membership test, + # and `not includes` is the same test negated. All three are + # conditions the parser builds and the analyzer types — `includes` + # and `not_includes` are accepted there explicitly as + # list-membership over any operand types. Admitting only `is` here + # meant this stage reported a condition as unparseable without + # ever handing it to the stage that parses it. second_ok = ( - second.type is TokenType.OPERATOR and second.value == "is" + (second.type is TokenType.OPERATOR and second.value == "is") + or (second.type is TokenType.CONNECTIVE + and second.value == "includes") + or (second.type is TokenType.OPERATOR + and second.value == "not" + and len(rest) > 2 + and rest[2].type is TokenType.CONNECTIVE + and rest[2].value == "includes") ) if not (head_ok and second_ok): return LiminateResult( diff --git a/src/liminate/run.py b/src/liminate/run.py index d220364..85742fb 100644 --- a/src/liminate/run.py +++ b/src/liminate/run.py @@ -233,6 +233,20 @@ def record_result(self, result: LiminateResult | None) -> None: ResultStatus.ERROR_SEMANTIC, ResultStatus.ERROR_RUNTIME, ResultStatus.PACK_VERB_FAILURE, + # A fired prohibition or an unmet requirement is the deontic core + # answering, and it is the answer a caller most needs. Omitting + # them made `forbid` less consequential to a shell than `cite`, + # which is a pack verb: a failed cite exited 1 and a violated + # forbid exited 0. Every shell consumer had to string-match + # "Prohibition violated" to learn the verdict, and one that + # forgot reported a denial as an admission. + # + # This is the exit status only. Execution still continues past a + # fired rule, so a program reports every violation rather than + # the first — which is the more useful behaviour and is why the + # fix is here rather than a halt. + ResultStatus.PROHIBITION_VIOLATED, + ResultStatus.REQUIREMENT_NOT_MET, ): self.had_any_error = True @@ -565,10 +579,16 @@ def _emit( lines = source.splitlines() # Phase 2 D-4 — run contradiction detection once over the whole program - # before execution, so a warning surfaces even if a later `require`/`forbid` - # halts the run before reaching the conflicting statement. Warning-only: - # emitted as an informational SUCCESS result, never blocks execution and - # never sets had_error. + # before execution, so the warning is independent of how far execution + # gets. Warning-only: emitted as an informational SUCCESS result, never + # blocks execution and never sets had_error. + # + # This used to say "even if a later `require`/`forbid` halts the run + # before reaching the conflicting statement." Neither halts the run. A + # fired rule unwinds its own statement — `executed=False` on that result — + # and execution carries on to the next, which is what lets a program + # report every violation rather than only the first. Running the check up + # front is still right; the reason given for it was not true. contradiction_warnings = detect_contradictions(_collect_deontic_statements(lines)) if contradiction_warnings: warn_result = LiminateResult( diff --git a/tests/fixtures/conformance-0.18.1.json b/tests/fixtures/conformance-0.18.1.json new file mode 100644 index 0000000..00cdc2c --- /dev/null +++ b/tests/fixtures/conformance-0.18.1.json @@ -0,0 +1,624 @@ +{ + "cases": [ + { + "id": "remember a string", + "results": [ + { + "canonical": "remember a string called greeting with hello", + "status": "success" + }, + { + "canonical": "show greeting", + "status": "success" + } + ], + "source": "remember a string called greeting with \"hello\"\nshow greeting" + }, + { + "id": "remember a number and compare it", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "require total is above 50", + "status": "success" + } + ], + "source": "remember a number called total with 75\nrequire total is above 50" + }, + { + "id": "a date range \u2014 the Calendar Era surface (v29), invisible to a word-list gate", + "results": [ + { + "canonical": "remember a date called starts-at with 2026-03-15", + "status": "success" + }, + { + "canonical": "require starts-at is not below 2026-01-01", + "status": "success" + }, + { + "canonical": "require starts-at is not above 2026-12-31", + "status": "success" + } + ], + "source": "remember a date called starts-at with 2026-03-15\nrequire starts-at is not below 2026-01-01\nrequire starts-at is not above 2026-12-31" + }, + { + "id": "a range comparison over text is refused", + "results": [ + { + "canonical": "remember a string called title with roof", + "status": "success" + }, + { + "canonical": "require title is above 50", + "errorKind": "semantic", + "errorMessage": "'above' requires numbers or dates, but 'title' is text.", + "status": "error" + } + ], + "source": "remember a string called title with \"roof\"\nrequire title is above 50" + }, + { + "id": "membership in a condition", + "results": [ + { + "canonical": "remember a list called tags with urgent", + "status": "success" + }, + { + "canonical": "choose if tags includes urgent: show yes otherwise show no", + "status": "success" + } + ], + "source": "remember a list called tags with \"urgent\"\nchoose if tags includes \"urgent\": show \"yes\" otherwise show \"no\"" + }, + { + "id": "negated membership in a condition", + "results": [ + { + "canonical": "remember a list called tags with routine", + "status": "success" + }, + { + "canonical": "choose if tags not includes urgent: show ok otherwise show skip", + "status": "success" + } + ], + "source": "remember a list called tags with \"routine\"\nchoose if tags not includes \"urgent\": show \"ok\" otherwise show \"skip\"" + }, + { + "id": "filter a list of records by a field", + "results": [ + { + "canonical": "remember an order called order1 with total as 75 and status as active", + "status": "success" + }, + { + "canonical": "remember a list called orders with order1", + "status": "success" + }, + { + "canonical": "filter the orders where total is above 50", + "status": "success" + } + ], + "source": "remember an order called order1 with total as 75 and status as active\nremember a list called orders with order1\nfilter the orders where total is above 50" + }, + { + "id": "filter by a field that holds a list", + "results": [ + { + "canonical": "remember a list called roof-tags with urgent and roof", + "status": "success" + }, + { + "canonical": "remember an order called order1 with total as 75 and tags as roof-tags", + "status": "success" + }, + { + "canonical": "remember a list called orders with order1", + "status": "success" + }, + { + "canonical": "filter the orders where tags includes urgent", + "status": "success" + } + ], + "source": "remember a list called roof-tags with \"urgent\" and \"roof\"\nremember an order called order1 with total as 75 and tags as roof-tags\nremember a list called orders with order1\nfilter the orders where tags includes \"urgent\"" + }, + { + "id": "filter over each item", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "status": "success" + }, + { + "canonical": "filter the numbers where each is above 5", + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\nfilter the numbers where each is above 5" + }, + { + "id": "a scrambled condition is refused", + "results": [ + { + "canonical": "remember a list called orders with none", + "status": "success" + }, + { + "errorKind": "parse", + "errorMessage": "I couldn't parse the condition after 'where'. Conditions look like '[field] is [comparison] [value]', for example: total is above 50.", + "status": "error" + } + ], + "source": "remember a list called orders with \"none\"\nfilter the orders where above 50 total is" + }, + { + "id": "a verb at the end is refused", + "results": [ + { + "canonical": "remember a list called orders with none", + "status": "success" + }, + { + "errorKind": "parse", + "errorMessage": "I couldn't parse this. Try putting the verb at the front, for example: filter the orders where total is above 50.", + "status": "error" + } + ], + "source": "remember a list called orders with \"none\"\nthe orders where total is above 50 filter" + }, + { + "id": "target before verb is reordered", + "results": [ + { + "canonical": "remember an order called order1 with total as 75 and status as active", + "status": "success" + }, + { + "canonical": "remember a list called orders with order1", + "status": "success" + }, + { + "canonical": "filter the orders where total is above 50", + "status": "success" + } + ], + "source": "remember an order called order1 with total as 75 and status as active\nremember a list called orders with order1\nthe orders filter where total is above 50" + }, + { + "id": "equality and inequality", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "require total is equal to 75", + "status": "success" + }, + { + "canonical": "require total is not equal to 30", + "status": "success" + } + ], + "source": "remember a number called total with 75\nrequire total is equal to 75\nrequire total is not equal to 30" + }, + { + "id": "within a tolerance", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "require total is within 5 of 78", + "status": "success" + } + ], + "source": "remember a number called total with 75\nrequire total is within 5 of 78" + }, + { + "id": "forbid", + "results": [ + { + "canonical": "remember a value called anchor with no", + "status": "success" + }, + { + "canonical": "forbid anchor is no because \"a lifted manifest binds to no file\"", + "status": "success" + } + ], + "source": "remember a value called anchor with \"no\"\nforbid anchor is \"no\" because \"a lifted manifest binds to no file\"" + }, + { + "id": "permit", + "results": [ + { + "canonical": "remember a value called anchor with yes", + "status": "success" + }, + { + "canonical": "permit anchor is yes", + "status": "success" + } + ], + "source": "remember a value called anchor with \"yes\"\npermit anchor is \"yes\"" + }, + { + "id": "expect", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "expect total is above 50", + "status": "success" + } + ], + "source": "remember a number called total with 75\nexpect total is above 50" + }, + { + "id": "count", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "status": "success" + }, + { + "canonical": "count the numbers", + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\ncount the numbers" + }, + { + "id": "sum", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "status": "success" + }, + { + "canonical": "sum the numbers", + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\nsum the numbers" + }, + { + "id": "extrema", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "status": "success" + }, + { + "canonical": "show highest of numbers", + "status": "success" + }, + { + "canonical": "show lowest of numbers", + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\nshow the highest of numbers\nshow the lowest of numbers" + }, + { + "id": "keep", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "status": "success" + }, + { + "canonical": "keep the numbers where each is above 5", + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\nkeep the numbers where each is above 5" + }, + { + "id": "add and remove", + "results": [ + { + "canonical": "remember a list called tags with urgent", + "status": "success" + }, + { + "canonical": "add roof to tags", + "status": "success" + }, + { + "canonical": "remove roof from tags", + "status": "success" + } + ], + "source": "remember a list called tags with \"urgent\"\nadd \"roof\" to tags\nremove \"roof\" from tags" + }, + { + "id": "arithmetic", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "remember a number called doubled from total multiplied by 2", + "status": "success" + } + ], + "source": "remember a number called total with 75\nremember a number called doubled with total multiplied by 2" + }, + { + "id": "unless, which is an exception clause and not a standalone conditional", + "results": [ + { + "canonical": "remember a number called revenue with 2000000", + "status": "success" + }, + { + "canonical": "expect revenue is above 1000000 unless revenue is equal to 0", + "status": "success" + } + ], + "source": "remember a number called revenue with 2000000\nexpect revenue is above 1000000 unless revenue is equal to 0" + }, + { + "id": "a temporal prefix", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "starting \"2025-07-01\" until \"2025-12-31\" require total is above 10", + "status": "success" + } + ], + "source": "remember a number called total with 75\nstarting 2025-07-01 until 2025-12-31 require total is above 10" + }, + { + "id": "an unknown verb is refused", + "results": [ + { + "errorKind": "parse", + "errorMessage": "I don't recognize a command here. Every sentence needs a verb like 'remember', 'show', 'filter', 'count', 'gather', 'sum', 'each', or 'choose'.", + "status": "error" + } + ], + "source": "invoke \"hello\"" + }, + { + "id": "an unknown name is refused", + "results": [ + { + "canonical": "add item to nonexistent-list", + "errorKind": "semantic", + "errorMessage": "I can't find 'nonexistent-list'. You might need to 'remember' it first.", + "status": "error" + } + ], + "source": "add \"item\" to nonexistent-list" + }, + { + "id": "a named composition", + "results": [ + { + "canonical": "remember how to find-big-orders: filter the orders where total is above 50", + "status": "success" + } + ], + "source": "remember how to find-big-orders: filter the orders where total is above 50" + }, + { + "id": "assign", + "results": [ + { + "canonical": "remember a value called review-task with audit", + "status": "success" + }, + { + "canonical": "remember a value called compliance-team with legal", + "status": "success" + }, + { + "canonical": "assign review-task to compliance-team", + "status": "success" + } + ], + "source": "remember a value called review-task with \"audit\"\nremember a value called compliance-team with \"legal\"\nassign review-task to compliance-team" + }, + { + "id": "compare", + "results": [ + { + "canonical": "remember a value called original with \"a\"", + "status": "success" + }, + { + "canonical": "remember a value called copy with \"a\"", + "status": "success" + }, + { + "canonical": "compare original to copy", + "status": "success" + } + ], + "source": "remember a value called original with \"a\"\nremember a value called copy with \"a\"\ncompare original to copy" + }, + { + "id": "gather a range", + "results": [ + { + "canonical": "gather the numbers from 1 to 10", + "status": "success" + } + ], + "source": "gather the numbers from 1 to 10" + }, + { + "id": "sort by a field", + "results": [ + { + "canonical": "remember an order called order1 with total as 75 and status as active", + "status": "success" + }, + { + "canonical": "remember a list called orders with order1", + "status": "success" + }, + { + "canonical": "sort the orders by total", + "status": "success" + } + ], + "source": "remember an order called order1 with total as 75 and status as active\nremember a list called orders with order1\nsort the orders by total" + }, + { + "id": "transform a field", + "results": [ + { + "canonical": "remember an order called order1 with total as 75 and status as active", + "status": "success" + }, + { + "canonical": "remember a list called orders with order1", + "status": "success" + }, + { + "canonical": "transform total of orders by total minus 10", + "status": "success" + } + ], + "source": "remember an order called order1 with total as 75 and status as active\nremember a list called orders with order1\ntransform total of the orders by total minus 10" + }, + { + "id": "arithmetic \u2014 plus, minus, divided by, multiplied by", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "remember a number called more from total plus 5", + "status": "success" + }, + { + "canonical": "remember a number called less from total minus 5", + "status": "success" + }, + { + "canonical": "remember a number called half from total divided by 2", + "status": "success" + }, + { + "canonical": "remember a number called twice from total multiplied by 2", + "status": "success" + } + ], + "source": "remember a number called total with 75\nremember a number called more with total plus 5\nremember a number called less with total minus 5\nremember a number called half with total divided by 2\nremember a number called twice with total multiplied by 2" + }, + { + "id": "define a predicate, and inherited", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "define large: each is above 50", + "status": "success" + }, + { + "canonical": "inherited require total is above 10", + "status": "success" + } + ], + "source": "remember a number called total with 75\ndefine large: is above 50\ninherited require total is above 10" + }, + { + "id": "weakens, over time", + "results": [ + { + "canonical": "remember a number called urgency with 10", + "status": "success" + }, + { + "canonical": "weakens urgency over 10", + "status": "success" + } + ], + "source": "remember a number called urgency with 10\nweakens urgency over 10" + }, + { + "id": "or, in a condition", + "results": [ + { + "canonical": "remember a number called total with 75", + "status": "success" + }, + { + "canonical": "require total is above 50 or total is equal to 0", + "status": "success" + } + ], + "source": "remember a number called total with 75\nrequire total is above 50 or total is equal to 0" + }, + { + "id": "then, sequencing", + "results": [ + { + "canonical": "remember a list called numbers with 3 and 9", + "status": "success" + }, + { + "canonical": "count the numbers then show numbers", + "status": "success" + } + ], + "source": "remember a list called numbers with 3 and 9\ncount the numbers then show numbers" + }, + { + "id": "reverse, which is an operator on sort and not a verb", + "results": [ + { + "canonical": "remember an order called order1 with total as 75 and status as active", + "status": "success" + }, + { + "canonical": "remember a list called orders with order1", + "status": "success" + }, + { + "canonical": "sort the orders by total in reverse", + "status": "success" + } + ], + "source": "remember an order called order1 with total as 75 and status as active\nremember a list called orders with order1\nsort the orders by total in reverse" + }, + { + "id": "finish", + "results": [ + { + "canonical": "finish", + "errorKind": "semantic", + "errorMessage": "'finish' can only be used inside an event handler.", + "status": "error" + } + ], + "source": "finish" + } + ], + "generator": "scripts/gen_conformance_corpus.py", + "language_version": "0.18.1", + "surface": "tokenize -> reorder -> parse -> analyze -> render" +} diff --git a/tests/fixtures/conformance_corpus.txt b/tests/fixtures/conformance_corpus.txt new file mode 100644 index 0000000..b44aaee --- /dev/null +++ b/tests/fixtures/conformance_corpus.txt @@ -0,0 +1,171 @@ +# remember a string +remember a string called greeting with "hello" +show greeting + +# remember a number and compare it +remember a number called total with 75 +require total is above 50 + +# a date range — the Calendar Era surface (v29), invisible to a word-list gate +remember a date called starts-at with 2026-03-15 +require starts-at is not below 2026-01-01 +require starts-at is not above 2026-12-31 + +# a range comparison over text is refused +remember a string called title with "roof" +require title is above 50 + +# membership in a condition +remember a list called tags with "urgent" +choose if tags includes "urgent": show "yes" otherwise show "no" + +# negated membership in a condition +remember a list called tags with "routine" +choose if tags not includes "urgent": show "ok" otherwise show "skip" + +# filter a list of records by a field +remember an order called order1 with total as 75 and status as active +remember a list called orders with order1 +filter the orders where total is above 50 + +# filter by a field that holds a list +remember a list called roof-tags with "urgent" and "roof" +remember an order called order1 with total as 75 and tags as roof-tags +remember a list called orders with order1 +filter the orders where tags includes "urgent" + +# filter over each item +remember a list called numbers with 3 and 9 +filter the numbers where each is above 5 + +# a scrambled condition is refused +remember a list called orders with "none" +filter the orders where above 50 total is + +# a verb at the end is refused +remember a list called orders with "none" +the orders where total is above 50 filter + +# target before verb is reordered +remember an order called order1 with total as 75 and status as active +remember a list called orders with order1 +the orders filter where total is above 50 + +# equality and inequality +remember a number called total with 75 +require total is equal to 75 +require total is not equal to 30 + +# within a tolerance +remember a number called total with 75 +require total is within 5 of 78 + +# forbid +remember a value called anchor with "no" +forbid anchor is "no" because "a lifted manifest binds to no file" + +# permit +remember a value called anchor with "yes" +permit anchor is "yes" + +# expect +remember a number called total with 75 +expect total is above 50 + +# count +remember a list called numbers with 3 and 9 +count the numbers + +# sum +remember a list called numbers with 3 and 9 +sum the numbers + +# extrema +remember a list called numbers with 3 and 9 +show the highest of numbers +show the lowest of numbers + +# keep +remember a list called numbers with 3 and 9 +keep the numbers where each is above 5 + +# add and remove +remember a list called tags with "urgent" +add "roof" to tags +remove "roof" from tags + +# arithmetic +remember a number called total with 75 +remember a number called doubled with total multiplied by 2 + +# unless, which is an exception clause and not a standalone conditional +remember a number called revenue with 2000000 +expect revenue is above 1000000 unless revenue is equal to 0 + +# a temporal prefix +remember a number called total with 75 +starting 2025-07-01 until 2025-12-31 require total is above 10 + +# an unknown verb is refused +invoke "hello" + +# an unknown name is refused +add "item" to nonexistent-list + +# a named composition +remember how to find-big-orders: filter the orders where total is above 50 + +# assign +remember a value called review-task with "audit" +remember a value called compliance-team with "legal" +assign review-task to compliance-team + +# compare +remember a value called original with "a" +remember a value called copy with "a" +compare original to copy + +# gather a range +gather the numbers from 1 to 10 + +# sort by a field +remember an order called order1 with total as 75 and status as active +remember a list called orders with order1 +sort the orders by total + +# transform a field +remember an order called order1 with total as 75 and status as active +remember a list called orders with order1 +transform total of the orders by total minus 10 + +# arithmetic — plus, minus, divided by, multiplied by +remember a number called total with 75 +remember a number called more with total plus 5 +remember a number called less with total minus 5 +remember a number called half with total divided by 2 +remember a number called twice with total multiplied by 2 + +# define a predicate, and inherited +remember a number called total with 75 +define large: is above 50 +inherited require total is above 10 + +# weakens, over time +remember a number called urgency with 10 +weakens urgency over 10 + +# or, in a condition +remember a number called total with 75 +require total is above 50 or total is equal to 0 + +# then, sequencing +remember a list called numbers with 3 and 9 +count the numbers then show numbers + +# reverse, which is an operator on sort and not a verb +remember an order called order1 with total as 75 and status as active +remember a list called orders with order1 +sort the orders by total in reverse + +# finish +finish diff --git a/tests/test_cli_exit_code.py b/tests/test_cli_exit_code.py index 26acdf7..78e67db 100644 --- a/tests/test_cli_exit_code.py +++ b/tests/test_cli_exit_code.py @@ -42,6 +42,39 @@ def test_error_after_valid_lines_still_exits_nonzero(self): result = _run(source) assert result.returncode == 1 + def test_violated_prohibition_exits_nonzero(self): + """A `forbid` that fires is the deontic core saying no. It exited 0, + which made it less consequential to a shell than a pack verb: a failed + `cite` already exits 1. Every shell consumer was string-matching + "Prohibition violated" to find out, and one that forgot reported a + denial as an admission.""" + source = textwrap.dedent("""\ + remember a value called anchor with "no" + forbid anchor is "no" because "a lifted manifest binds to no file" + """) + result = _run(source) + assert result.returncode == 1 + + def test_unmet_requirement_exits_nonzero(self): + """`require` is the same verb from the other side and had the same + silence.""" + source = textwrap.dedent("""\ + remember a date called starts-at with 2025-03-15 + require starts-at is not below 2026-01-01 + """) + result = _run(source) + assert result.returncode == 1 + + def test_a_satisfied_deontic_statement_still_exits_zero(self): + """The exit code says a rule fired, not that rules exist.""" + source = textwrap.dedent("""\ + remember a value called anchor with "yes" + forbid anchor is "no" because "a lifted manifest binds to no file" + require anchor is "yes" + """) + result = _run(source) + assert result.returncode == 0 + def test_help_exits_zero(self): result = subprocess.run( LIMINATE + ["--help"], capture_output=True, text=True diff --git a/tests/test_conformance_corpus.py b/tests/test_conformance_corpus.py new file mode 100644 index 0000000..60afe62 --- /dev/null +++ b/tests/test_conformance_corpus.py @@ -0,0 +1,107 @@ +"""The corpus must cover the surface it claims to. + +The existing parity gate (`@liminate/ts-validator`, `tests/conformance.test.ts`) +compares the two implementations' reserved-word lists against a frozen fixture. +It stayed green while three minor versions of behaviour diverged, because the +thing that diverged changed no word: v29 added `date` to `_require_comparable`, +so date ranges became expressible and the word list did not move. Downstream, +`commongage` carried a comment recording that limit for two months after it was +gone. + +A behavioural corpus only fixes that if it keeps up. These tests are what make +it keep up: a reserved word with no case is a piece of surface the corpus +cannot speak about, and adding a word to the vocabulary fails here until a +program using it exists. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from liminate import vocabulary as vocab + +ROOT = Path(__file__).resolve().parents[1] +CORPUS = ROOT / "tests" / "fixtures" / "conformance_corpus.txt" +GENERATOR = ROOT / "scripts" / "gen_conformance_corpus.py" + + +def _version() -> str: + for line in (ROOT / "pyproject.toml").read_text(encoding="utf-8").splitlines(): + if line.startswith("version"): + return line.split("=", 1)[1].strip().strip('"') + raise AssertionError("pyproject.toml states no version") + + +def _corpus_words() -> set[str]: + text = CORPUS.read_text(encoding="utf-8") + programs = [ln for ln in text.splitlines() if ln.strip() and not ln.startswith("# ")] + words: set[str] = set() + for line in programs: + words.update(line.replace(":", " ").replace(",", " ").split()) + return words + + +# Words that cannot appear in a corpus program, each with the reason. A word +# with no case and no reason is a hole; a reason that stops being true is +# checked in the other direction below. +UNREACHABLE: dict[str, str] = { + "about": ( + "a declaration the wrapping `validate()` handles before the per-line " + "pipeline, and which is an error anywhere the per-line pipeline can " + "see it — a corpus case would record the refusal, not the behaviour" + ), + "when": ( + "a when-block header, which `validate()` buffers with its indented " + "action lines; the per-line pipeline never sees a complete block" + ), +} + + +def test_every_reserved_word_appears_in_a_corpus_program(): + reserved: set[str] = set() + for name in ("VERBS", "CONNECTIVES", "OPERATORS", "ARTICLES", + "MULTI_WORD_RESERVED", "DECLARATIONS"): + reserved |= set(getattr(vocab, name, set())) + missing = sorted(reserved - _corpus_words() - set(UNREACHABLE)) + assert not missing, ( + f"{missing} are reserved and no corpus program uses them, so the " + f"conformance corpus says nothing about how the port handles them" + ) + + +def test_no_unreachable_entry_has_stopped_being_unreachable(): + """A stale excuse is the same defect as a stale silence.""" + reached = sorted(set(UNREACHABLE) & _corpus_words()) + assert not reached, f"{reached} are excused as unreachable and appear in the corpus" + + +def test_the_generated_fixture_matches_the_version_it_was_generated_from(): + """The fixture names the version it describes, and the tree has moved + since only if someone bumped the version without regenerating.""" + fixture = ROOT / "tests" / "fixtures" / f"conformance-{_version()}.json" + assert fixture.exists(), ( + f"no conformance fixture for version {_version()}; regenerate with " + f"`python3 scripts/gen_conformance_corpus.py > {fixture.relative_to(ROOT)}`" + ) + assert json.loads(fixture.read_text())["language_version"] == _version() + + +def test_the_fixture_is_what_the_generator_produces_today(): + """Regenerating must be a no-op, or the committed parity file describes a + language this tree no longer is — which is the exact failure this whole + mechanism exists to stop.""" + fixture = ROOT / "tests" / "fixtures" / f"conformance-{_version()}.json" + if not fixture.exists(): + pytest.skip("covered by the fixture-exists test") + fresh = subprocess.run( + [sys.executable, str(GENERATOR)], capture_output=True, text=True, cwd=ROOT + ) + assert fresh.returncode == 0, fresh.stderr + assert json.loads(fresh.stdout) == json.loads(fixture.read_text()), ( + "the committed conformance fixture is stale; regenerate it" + ) diff --git a/tests/test_reorderer.py b/tests/test_reorderer.py index 1c52617..81dbf14 100644 --- a/tests/test_reorderer.py +++ b/tests/test_reorderer.py @@ -255,3 +255,101 @@ def test_temporal_prefix_bare_dates_with_inherited_reorders(): assert node.starting_date == "2025-07-01" assert node.until_date == "2025-12-31" assert node.inherited is True + + +# `includes` after `where` — the gate that blocked a condition every other +# layer already handled. +# +# The parser produces `op="includes"` / `op="not_includes"` for these, and the +# analyzer accepts them explicitly as list-membership. Only this two-token +# shape check stood in front, requiring the second token after `where` to be +# the operator `is`. So `title includes "ant"` was reported as an unparseable +# condition by the one stage that never tried to parse it. +# +# The gate had already been widened once, for the v25 extrema head. This is +# the same shape of change. + + +def test_where_includes_passes_through(): + src = tokenize('filter the orders where title includes "urgent"') + out = reorder(src) + assert isinstance(out, list), getattr(out, "message", out) + assert _values(out) == [ + "filter", "the", "orders", "where", "title", "includes", "urgent", + ] + + +def test_where_not_includes_passes_through(): + src = tokenize('filter the orders where title not includes "urgent"') + out = reorder(src) + assert isinstance(out, list), getattr(out, "message", out) + + +def test_keep_where_includes_passes_through(): + src = tokenize('keep the orders where title includes "urgent"') + out = reorder(src) + assert isinstance(out, list), getattr(out, "message", out) + + +def test_where_includes_reaches_the_parser_that_handles_it(): + from liminate.parser import parse + + out = reorder(tokenize('filter the orders where title includes "urgent"')) + assert isinstance(out, list), getattr(out, "message", out) + assert parse(out).condition.op == "includes" + + +def test_where_includes_filters_a_list_valued_field_correctly(): + """The reason the gate had to move: this works, and nothing could reach it. + + `includes` is a list-membership probe, so after `where` it is meaningful + exactly when the field it names holds a list. That case filters correctly + and was unreachable — not because any stage could not do it, but because + the one stage that does no parsing said the condition could not be parsed. + """ + from liminate.run import Session + from liminate.result import ResultStatus + + session = Session() + for line in [ + 'remember a list called roof-tags with "urgent" and "roof"', + 'remember a list called lawn-tags with "routine" and "lawn"', + "remember an order called order1 with total as 75 and tags as roof-tags", + "remember an order called order2 with total as 30 and tags as lawn-tags", + "remember a list called orders with order1 and order2", + ]: + assert session.run_line(line).status is ResultStatus.SUCCESS, line + + result = session.run_line('filter the orders where tags includes "urgent"') + assert result.status is ResultStatus.SUCCESS, result.message + assert len(session.symtab["orders"].value) == 1 + assert session.run_line("show orders").output == [ + "total: 75, tags: ['urgent', 'roof']" + ] + + +def test_where_includes_over_scalar_items_keeps_the_documented_answer(): + """The boundary, stated rather than discovered later. + + `each` is an item, not a list, and `includes` over a non-list operand is + false by decision — see `test_includes_with_scalar_left_operand_is_false`. + So `where each includes "x"` empties the list rather than matching text. + Widening the gate does not change that and must not be read as making + `includes` a substring test; text-contains is not in the language. + """ + from liminate.run import run + + base = ( + 'remember a list called tags with "urgent-repair"\n' + 'add "routine-check" to tags\n' + ) + result = run(base + 'filter tags where each includes "urgent"\nshow tags') + assert result.results[-1].output == [""] + + +def test_a_genuinely_scrambled_condition_is_still_rejected(): + """Widening the gate must not turn it off. `includes` is admitted in the + comparison position; a condition with nothing in that position is not.""" + src = tokenize("filter the orders where above 50 total is") + out = reorder(src) + assert not isinstance(out, list)