From a98519c6ea410428cfbb1c1e63b1cdaa3bd1fc61 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 4 May 2026 00:35:25 +0200 Subject: [PATCH 01/20] Implement SplitNTermLevels transform with tests --- src/Repr/Common/Level.fram | 32 ++++ src/Transform/SplitNTermLevels.fram | 116 ++++++++++++- test.sh | 62 +++++++ test/TestAll.fram | 1 + test/TransformTests/SplitNTermLevels.fram | 191 ++++++++++++++++++++++ test/test_suite | 5 + 6 files changed, 404 insertions(+), 3 deletions(-) create mode 100755 test.sh create mode 100644 test/TestAll.fram create mode 100644 test/TransformTests/SplitNTermLevels.fram create mode 100644 test/test_suite diff --git a/src/Repr/Common/Level.fram b/src/Repr/Common/Level.fram index 1a0b248..a530d99 100644 --- a/src/Repr/Common/Level.fram +++ b/src/Repr/Common/Level.fram @@ -14,3 +14,35 @@ pub data Level = {## The highest precedence level. ##} | LTop + +{## Equality of levels. ##} +pub method equal (l1 : Level) (l2 : Level) = + match (l1, l2) with + | (LBot, LBot) => True + | (LTop, LTop) => True + | (LNum n1, LNum n2) => n1 == n2 + | _ => False + end + +{## Strict order on levels. ##} +pub method lt (l1 : Level) (l2 : Level) = + match (l1, l2) with + | (LBot, LBot) => False + | (LBot, _) => True + | (_, LBot) => False + | (LTop, _) => False + | (_, LTop) => True + | (LNum n1, LNum n2) => n1 < n2 + end + +{## Non-strict order on levels. ##} +pub method le (l1 : Level) (l2 : Level) = + l1 == l2 || l1 < l2 + +{## Convert level to string. ##} +pub method toString (level : Level) = + match level with + | LBot => "bot" + | LTop => "top" + | LNum n => n.toString + end diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index b84c9e5..e2acb0a 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -9,8 +9,118 @@ of the original non-terminal. Such non-terminals contain only productions of the corresponding level and a single production that calls the next level. ##} import open Repr/RichGrammar +import List +import Utils/UID + +let rec insertLevel (level : Level) (levels : List Level) = + match levels with + | [] => [level] + | level' :: levels' => + if level == level' then + levels + else if level < level' then + level :: levels + else + level' :: insertLevel level levels' + end + +let collectLevels (prods : List NTermProd) = + List.foldLeft + (fn levels (prod : NTermProd) => insertLevel prod.level levels) + [] + prods + +let levelProds (level : Level) (prods : List NTermProd) = + List.filter + (fn (prod : NTermProd) => prod.level == level) + prods + +type LevelIds = List (Pair Level NTermId) +type IdTable = List (Pair NTermId LevelIds) + +let lookupLevelIds (idTable : IdTable) (id : NTermId) = + List.findMapErr + {~onError = fn _ => impossible ()} + (fn ((id', levelIds) : Pair NTermId LevelIds) => + if id == id' then Some levelIds else None) + idTable + +let rec pickLevelId (level : Level) (levelIds : LevelIds) = + match levelIds with + | [] => impossible () + | (_, id) :: [] => id + | (level', id) :: rest => + if level <= level' then id else pickLevelId level rest + end + +let lookupNTermId (idTable : IdTable) (id : NTermId) (level : Level) = + pickLevelId level (lookupLevelIds idTable id) + +let rewriteSymbol (idTable : IdTable) sym = + match sym with + | PS_Token _ => sym + | PS_NTerm {var, level} id => + PS_NTerm {var, level} (lookupNTermId idTable id level) + end + +let rewriteProd (idTable : IdTable) (NTermProd {module Prod}) = + NTermProd + { module Prod + , symbols = List.map (rewriteSymbol idTable) Prod.symbols + } + +let tagsForLevel level prods = + levelProds level prods + |> List.concatMap (fn (prod : NTermProd) => prod.tags) + +let forwardVar (nt : NTerm) = + Var {id = UID.fresh (), name = None, typ = nt.valueType} + +let forwardProd (nt : NTerm) level nextLevel nextId = + NTermProd + { symbols = + [PS_NTerm {var = forwardVar nt, level = nextLevel} nextId] + , level + , tags = tagsForLevel nextLevel nt.prods + , unless = TC_True + , action = AForward + } + +let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = + match levelIds with + | [] => [] + | (level, id) :: rest => + let prods = + levelProds level nt.prods + |> List.map (rewriteProd idTable) + let prods = + match rest with + | [] => prods + | (nextLevel, nextId) :: _ => + prods + [forwardProd nt level nextLevel nextId] + end + in + NTerm + { id + , name = nt.name + "@" + level.toString + , valueType = nt.valueType + , prods + } :: splitLevels idTable nt rest + end + +let freshLevelIds levels = + List.map (fn level => (level, NTermId.fresh ())) levels + +let ntermLevelIds (nt : NTerm) = + (nt.id, freshLevelIds (collectLevels nt.prods)) {## Split each non-terminal into multiple levels. ##} -pub let transform (g : RichGrammar) : RichGrammar = - # TODO: Implement this function. - g +pub let transform (RichGrammar {module G}) : RichGrammar = + let idTable = List.map ntermLevelIds G.nterms in + RichGrammar + { module G + , nterms = + G.nterms + |> List.concatMap + (fn nt => splitLevels idTable nt (lookupLevelIds idTable nt.id)) + } diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..5526afe --- /dev/null +++ b/test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -u + +if [ $# -ne 1 ]; then + echo "USAGE: ./test.sh TEST_SUITE" + exit 1 +fi + +binary="${DBL:-dbl}" +if ! command -v "$binary" > /dev/null; then + echo "ERROR: dbl executable not found in PATH" + exit 1 +fi + +if [ -z "${DBL_LIB:-}" ]; then + dbl_path=$(command -v "$binary") + dbl_prefix=$(dirname "$(dirname "$dbl_path")") + if [ -d "$dbl_prefix/lib/dbl/stdlib" ]; then + export DBL_LIB="$dbl_prefix/lib/dbl/stdlib" + else + echo "ERROR: DBL_LIB is not set and DBL stdlib was not found next to '$dbl_path'" + exit 1 + fi +fi + +flags="" +total_tests=0 +passed_tests=0 + +function simple_test { + total_tests=$((total_tests + 1)) + + local file="$1" + local cmd=("$binary") + if [ -n "$flags" ]; then + # shellcheck disable=SC2206 + cmd+=($flags) + fi + cmd+=("$file") + + echo "${cmd[*]}" + if "${cmd[@]}"; then + passed_tests=$((passed_tests + 1)) + else + echo "Test file failed: $file" + fi +} + +function run_with_flags { + flags="$2" + $1 +} + +source "$1" + +echo "Passed: ${passed_tests}/${total_tests}" + +if [ "$passed_tests" -eq "$total_tests" ]; then + exit 0 +else + exit 1 +fi diff --git a/test/TestAll.fram b/test/TestAll.fram new file mode 100644 index 0000000..fa057c0 --- /dev/null +++ b/test/TestAll.fram @@ -0,0 +1 @@ +import TransformTests/SplitNTermLevels diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram new file mode 100644 index 0000000..2888c81 --- /dev/null +++ b/test/TransformTests/SplitNTermLevels.fram @@ -0,0 +1,191 @@ +{# This file is part of FramYard, released under MIT license. + See LICENSE for details. + #} + +import open Testing +import /List +import open Repr/RichGrammar +import Transform/SplitNTermLevels +import Utils/UID + +let pos = + Position + { fname = "" + , line = 1 + , cnum = 0 + , bol = 0 + , length = 0 + } + +let action name = ACode {pos, code = name} + +let tokenProd level token = + NTermProd + { symbols = [PS_Token {var = None} token] + , level + , tags = [] + , unless = TC_True + , action = action token + } + +let var name = + Var {id = UID.fresh (), name = Some name, typ = TToken "Expr"} + +let exprSym id level name = + PS_NTerm {var = var name, level} id + +let infixProdWithTags level leftLevel token rightLevel tags exprId = + NTermProd + { symbols = + [ exprSym exprId leftLevel "lhs" + , PS_Token {var = None} token + , exprSym exprId rightLevel "rhs" + ] + , level + , tags + , unless = TC_True + , action = action token + } + +let parenProd exprId = + NTermProd + { symbols = + [ PS_Token {var = None} "LPAREN" + , exprSym exprId (LNum 0) "inner" + , PS_Token {var = None} "RPAREN" + ] + , level = LTop + , tags = [] + , unless = TC_True + , action = action "paren" + } + +let mkNTerm id name prods = + NTerm {id, name, valueType = TToken name, prods} + +let mkGrammar nterms = + RichGrammar + { tokens = [] + , datatypes = [] + , typeDefs = [] + , nterms + , treeSymbols = [] + , preambleCode = [] + } + +let isForwardProd (NTermProd {symbols, tags, unless, action}) = + match (symbols, unless, action) with + | ([PS_NTerm _], TC_True, AForward) => True + | _ => False + end + +let hasTrueTag tag (prod : NTermProd) = + List.exists + (fn ((prodTag, cond) : Pair Tag (TagCond Var)) => + prodTag == tag && + match cond with + | TC_True => True + | _ => False + end) + prod.tags + +let countProds (f : NTermProd ->[] Bool) (nterms : List NTerm) = + List.foldLeft + (fn acc (NTerm {prods}) => + acc + List.length (List.filter f prods)) + 0 + nterms + +let allProds (nterms : List NTerm) = + List.concatMap (fn (NTerm {prods}) => prods) nterms + +let hasProdAtLevel (level : Level) (nterm : NTerm) = + List.exists + (fn (prod : NTermProd) => + prod.level == level && not (isForwardProd prod)) + nterm.prods + +let countNTermsWithLevel (level : Level) (nterms : List NTerm) = + List.length (List.filter (hasProdAtLevel level) nterms) + +let hasTaggedForward tag (nterm : NTerm) = + List.exists + (fn (prod : NTermProd) => isForwardProd prod && hasTrueTag tag prod) + nterm.prods + +let sampleGrammar () = + let exprId = NTermId.fresh () in + let expr = + mkNTerm exprId "Expr" + [ infixProdWithTags + (LNum 0) + (LNum 0) + "PLUS" + (LNum 10) + [] + exprId + , infixProdWithTags + (LNum 10) + (LNum 10) + "STAR" + LTop + [("multiplicative", TC_True)] + exprId + , tokenProd LTop "NUMBER" + , parenProd exprId + ] + in + mkGrammar [expr] + +let sample = sampleGrammar () + +let transformedSample = + SplitNTermLevels.transform sample + +let _ = + +testSuite "SplitNTermLevels" (fn _ => + + testCase "creates one non-terminal per production level" (fn _ => + let result = transformedSample in + assertEq 3 (List.length result.nterms); + expectEq 1 (countNTermsWithLevel (LNum 0) result.nterms); + expectEq 1 (countNTermsWithLevel (LNum 10) result.nterms); + expectEq 1 (countNTermsWithLevel LTop result.nterms)); + + testCase "keeps original productions only on their own levels" (fn _ => + let result = transformedSample in + let nonForwardProds = + allProds result.nterms |> List.filter (fn prod => not (isForwardProd prod)) + in + assertEq 4 (List.length nonForwardProds); + expectEq 1 + (List.length + (List.filter + (fn (prod : NTermProd) => prod.level == LNum 0) + nonForwardProds)); + expectEq 1 + (List.length + (List.filter + (fn (prod : NTermProd) => prod.level == LNum 10) + nonForwardProds)); + expectEq 2 + (List.length + (List.filter + (fn (prod : NTermProd) => prod.level == LTop) + nonForwardProds))); + + testCase "adds forwarding productions between adjacent levels" (fn _ => + let result = transformedSample in + assertEq 2 (countProds isForwardProd result.nterms)); + + testCase "forwards semantic action and tags from the level above" (fn _ => + let result = transformedSample in + let bottomLevelWithTaggedForward = + result.nterms + |> List.filter + (fn (nterm : NTerm) => + hasProdAtLevel (LNum 0) nterm && + hasTaggedForward "multiplicative" nterm) + in + assertEq 1 (List.length bottomLevelWithTaggedForward))) diff --git a/test/test_suite b/test/test_suite new file mode 100644 index 0000000..38dc532 --- /dev/null +++ b/test/test_suite @@ -0,0 +1,5 @@ +function framyard_tests { + simple_test test/TestAll.fram +} + +run_with_flags framyard_tests "-L src -L test" From 21e3d49f39761d2f73f7915c6af1a79f7f3d1a0a Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 4 May 2026 13:34:34 +0200 Subject: [PATCH 02/20] Use map for SplitNTermLevels IdTable --- src/Transform/SplitNTermLevels.fram | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index e2acb0a..d9266c0 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -36,14 +36,12 @@ let levelProds (level : Level) (prods : List NTermProd) = prods type LevelIds = List (Pair Level NTermId) -type IdTable = List (Pair NTermId LevelIds) +type IdTable = NTermMap.T LevelIds let lookupLevelIds (idTable : IdTable) (id : NTermId) = - List.findMapErr + idTable.findErr {~onError = fn _ => impossible ()} - (fn ((id', levelIds) : Pair NTermId LevelIds) => - if id == id' then Some levelIds else None) - idTable + id let rec pickLevelId (level : Level) (levelIds : LevelIds) = match levelIds with @@ -111,12 +109,14 @@ let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = let freshLevelIds levels = List.map (fn level => (level, NTermId.fresh ())) levels -let ntermLevelIds (nt : NTerm) = - (nt.id, freshLevelIds (collectLevels nt.prods)) +let addNTermLevelIds (idTable : IdTable) (nt : NTerm) = + idTable.add nt.id (freshLevelIds (collectLevels nt.prods)) {## Split each non-terminal into multiple levels. ##} pub let transform (RichGrammar {module G}) : RichGrammar = - let idTable = List.map ntermLevelIds G.nterms in + let idTable = + List.foldLeft addNTermLevelIds NTermMap.empty G.nterms + in RichGrammar { module G , nterms = From 4eecc717868f71a8bdd381e45076a862a33ed1a4 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Sun, 31 May 2026 22:19:12 +0200 Subject: [PATCH 03/20] Use TC_False for level forwarding productions --- src/Transform/SplitNTermLevels.fram | 2 +- test/TransformTests/SplitNTermLevels.fram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index d9266c0..9fe9fbc 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -80,7 +80,7 @@ let forwardProd (nt : NTerm) level nextLevel nextId = [PS_NTerm {var = forwardVar nt, level = nextLevel} nextId] , level , tags = tagsForLevel nextLevel nt.prods - , unless = TC_True + , unless = TC_False , action = AForward } diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index 2888c81..ba80d77 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -75,7 +75,7 @@ let mkGrammar nterms = let isForwardProd (NTermProd {symbols, tags, unless, action}) = match (symbols, unless, action) with - | ([PS_NTerm _], TC_True, AForward) => True + | ([PS_NTerm _], TC_False, AForward) => True | _ => False end From 28285b9bb244eb458ecbd34990c30015d268cd14 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 02:05:24 +0200 Subject: [PATCH 04/20] Address Level matching review comments --- src/Repr/Common/Level.fram | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Repr/Common/Level.fram b/src/Repr/Common/Level.fram index a530d99..e616b6f 100644 --- a/src/Repr/Common/Level.fram +++ b/src/Repr/Common/Level.fram @@ -17,22 +17,24 @@ pub data Level = {## Equality of levels. ##} pub method equal (l1 : Level) (l2 : Level) = - match (l1, l2) with - | (LBot, LBot) => True - | (LTop, LTop) => True - | (LNum n1, LNum n2) => n1 == n2 - | _ => False + match l1, l2 with + | LBot, LBot => True + | LBot, _ => False + | LTop, LTop => True + | LTop, _ => False + | LNum n1, LNum n2 => n1 == n2 + | LNum _, _ => False end {## Strict order on levels. ##} pub method lt (l1 : Level) (l2 : Level) = - match (l1, l2) with - | (LBot, LBot) => False - | (LBot, _) => True - | (_, LBot) => False - | (LTop, _) => False - | (_, LTop) => True - | (LNum n1, LNum n2) => n1 < n2 + match l1, l2 with + | LBot, LBot => False + | LBot, _ => True + | _, LBot => False + | LTop, _ => False + | _, LTop => True + | LNum n1, LNum n2 => n1 < n2 end {## Non-strict order on levels. ##} From cf2e3c0765141dc8e96bbd084639f626504e350d Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 02:22:37 +0200 Subject: [PATCH 05/20] Use map to collect production levels --- src/Transform/SplitNTermLevels.fram | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 9fe9fbc..a3f1773 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -10,25 +10,22 @@ the corresponding level and a single production that calls the next level. ##} import open Repr/RichGrammar import List +import Map import Utils/UID -let rec insertLevel (level : Level) (levels : List Level) = - match levels with - | [] => [level] - | level' :: levels' => - if level == level' then - levels - else if level < level' then - level :: levels - else - level' :: insertLevel level levels' - end +let (Map { module LevelMap }) = Map.make { Key = Level } + +type LevelSet = LevelMap.T Unit let collectLevels (prods : List NTermProd) = - List.foldLeft - (fn levels (prod : NTermProd) => insertLevel prod.level levels) - [] - prods + let levelSet = + List.foldLeft + (fn (levelSet : LevelSet) (prod : NTermProd) => + levelSet.add prod.level ()) + LevelMap.empty + prods + in + levelSet.fold (fn {key} _ levels => levels + [key]) [] let levelProds (level : Level) (prods : List NTermProd) = List.filter From 553113709ad8245b1e611449cb5293af9003ec0c Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 02:36:43 +0200 Subject: [PATCH 06/20] Avoid append when collecting levels --- src/Transform/SplitNTermLevels.fram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index a3f1773..297077e 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -25,7 +25,7 @@ let collectLevels (prods : List NTermProd) = LevelMap.empty prods in - levelSet.fold (fn {key} _ levels => levels + [key]) [] + levelSet.fold (fn {key} _ levels => key :: levels) [] >.rev let levelProds (level : Level) (prods : List NTermProd) = List.filter From cd5cc6e156dbad12884f0542d00181f0fc50a28f Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 02:49:56 +0200 Subject: [PATCH 07/20] Fix tag forwarding conditions in SplitNTermLevels --- src/Transform/SplitNTermLevels.fram | 9 ++++--- test/TransformTests/SplitNTermLevels.fram | 33 +++++++++++++++-------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 297077e..36d20a6 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -64,19 +64,22 @@ let rewriteProd (idTable : IdTable) (NTermProd {module Prod}) = , symbols = List.map (rewriteSymbol idTable) Prod.symbols } -let tagsForLevel level prods = +let tagsForLevel var level prods = levelProds level prods |> List.concatMap (fn (prod : NTermProd) => prod.tags) + |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => + (tag, TC_Tag tag var)) let forwardVar (nt : NTerm) = Var {id = UID.fresh (), name = None, typ = nt.valueType} let forwardProd (nt : NTerm) level nextLevel nextId = + let var = forwardVar nt in NTermProd { symbols = - [PS_NTerm {var = forwardVar nt, level = nextLevel} nextId] + [PS_NTerm {var, level = nextLevel} nextId] , level - , tags = tagsForLevel nextLevel nt.prods + , tags = tagsForLevel var nextLevel nt.prods , unless = TC_False , action = AForward } diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index ba80d77..ad8cde2 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -79,15 +79,24 @@ let isForwardProd (NTermProd {symbols, tags, unless, action}) = | _ => False end -let hasTrueTag tag (prod : NTermProd) = - List.exists - (fn ((prodTag, cond) : Pair Tag (TagCond Var)) => - prodTag == tag && - match cond with - | TC_True => True - | _ => False - end) - prod.tags +let sameVar (v1 : Var) (v2 : Var) = + v1.id == v2.id + +let hasForwardedTag tag (prod : NTermProd) = + let tagUsesVar var = + List.exists + (fn ((prodTag, cond) : Pair Tag (TagCond Var)) => + prodTag == tag && + match cond with + | TC_Tag tag' var' => tag' == tag && sameVar var var' + | _ => False + end) + prod.tags + in + match prod.symbols with + | [PS_NTerm {var} _] => tagUsesVar var + | _ => False + end let countProds (f : NTermProd ->[] Bool) (nterms : List NTerm) = List.foldLeft @@ -110,9 +119,11 @@ let countNTermsWithLevel (level : Level) (nterms : List NTerm) = let hasTaggedForward tag (nterm : NTerm) = List.exists - (fn (prod : NTermProd) => isForwardProd prod && hasTrueTag tag prod) + (fn (prod : NTermProd) => isForwardProd prod && hasForwardedTag tag prod) nterm.prods +let tagSourceVar = var "tag-source" + let sampleGrammar () = let exprId = NTermId.fresh () in let expr = @@ -129,7 +140,7 @@ let sampleGrammar () = (LNum 10) "STAR" LTop - [("multiplicative", TC_True)] + [("multiplicative", TC_Tag "multiplicative" tagSourceVar)] exprId , tokenProd LTop "NUMBER" , parenProd exprId From f1c2f06f5a69bcd30ddb8375057d9114f2c3a194 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 02:59:20 +0200 Subject: [PATCH 08/20] Preserve empty non-terminals in SplitNTermLevels --- src/Transform/SplitNTermLevels.fram | 9 +++++++-- test/TransformTests/SplitNTermLevels.fram | 21 ++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 36d20a6..26cafb5 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -106,6 +106,12 @@ let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = } :: splitLevels idTable nt rest end +let splitNTerm (idTable : IdTable) (nt : NTerm) = + match lookupLevelIds idTable nt.id with + | [] => [nt] + | levelIds => splitLevels idTable nt levelIds + end + let freshLevelIds levels = List.map (fn level => (level, NTermId.fresh ())) levels @@ -121,6 +127,5 @@ pub let transform (RichGrammar {module G}) : RichGrammar = { module G , nterms = G.nterms - |> List.concatMap - (fn nt => splitLevels idTable nt (lookupLevelIds idTable nt.id)) + |> List.concatMap (splitNTerm idTable) } diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index ad8cde2..85c4844 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -153,6 +153,15 @@ let sample = sampleGrammar () let transformedSample = SplitNTermLevels.transform sample +let emptyGrammar () = + mkGrammar [mkNTerm (NTermId.fresh ()) "Empty" []] + +let emptySample = + emptyGrammar () + +let transformedEmptySample = + SplitNTermLevels.transform emptySample + let _ = testSuite "SplitNTermLevels" (fn _ => @@ -199,4 +208,14 @@ testSuite "SplitNTermLevels" (fn _ => hasProdAtLevel (LNum 0) nterm && hasTaggedForward "multiplicative" nterm) in - assertEq 1 (List.length bottomLevelWithTaggedForward))) + assertEq 1 (List.length bottomLevelWithTaggedForward)); + + testCase "preserves non-terminals without productions" (fn _ => + let result = transformedEmptySample in + assertEq 1 (List.length result.nterms); + match result.nterms with + | [nt] => + expectEq "Empty" nt.name; + expectEq 0 (List.length nt.prods) + | _ => assertTrue False + end)) From 075c68769f20aa7b3e65f4ea07034b81858e7660 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 03:10:42 +0200 Subject: [PATCH 09/20] Fail on invalid split level lookup --- src/Transform/SplitNTermLevels.fram | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 26cafb5..60e0052 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -43,7 +43,8 @@ let lookupLevelIds (idTable : IdTable) (id : NTermId) = let rec pickLevelId (level : Level) (levelIds : LevelIds) = match levelIds with | [] => impossible () - | (_, id) :: [] => id + | (level', id) :: [] => + if level <= level' then id else impossible () | (level', id) :: rest => if level <= level' then id else pickLevelId level rest end From 72d643efd440aebdfc67feab536f1e9bfa9fa6c1 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 03:16:34 +0200 Subject: [PATCH 10/20] Deduplicate forwarded tags --- src/Transform/SplitNTermLevels.fram | 12 +++++- test/TransformTests/SplitNTermLevels.fram | 52 +++++++++++++++++++++-- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 60e0052..5e9b4a7 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -65,11 +65,19 @@ let rewriteProd (idTable : IdTable) (NTermProd {module Prod}) = , symbols = List.map (rewriteSymbol idTable) Prod.symbols } +let rec dedupeTags (tags : List Tag) = + match tags with + | [] => [] + | tag :: rest => + tag :: dedupeTags (List.filter (fn tag' => not (tag' == tag)) rest) + end + let tagsForLevel var level prods = levelProds level prods |> List.concatMap (fn (prod : NTermProd) => prod.tags) - |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => - (tag, TC_Tag tag var)) + |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => tag) + |> dedupeTags + |> List.map (fn tag => (tag, TC_Tag tag var)) let forwardVar (nt : NTerm) = Var {id = UID.fresh (), name = None, typ = nt.valueType} diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index 85c4844..3e3b4a6 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -19,15 +19,18 @@ let pos = let action name = ACode {pos, code = name} -let tokenProd level token = +let tokenProdWithTags level token tags = NTermProd { symbols = [PS_Token {var = None} token] , level - , tags = [] + , tags , unless = TC_True , action = action token } +let tokenProd level token = + tokenProdWithTags level token [] + let var name = Var {id = UID.fresh (), name = Some name, typ = TToken "Expr"} @@ -122,6 +125,29 @@ let hasTaggedForward tag (nterm : NTerm) = (fn (prod : NTermProd) => isForwardProd prod && hasForwardedTag tag prod) nterm.prods +let countForwardedTag tag (prod : NTermProd) = + match prod.symbols with + | [PS_NTerm {var} _] => + List.length + (List.filter + (fn ((prodTag, cond) : Pair Tag (TagCond Var)) => + prodTag == tag && + match cond with + | TC_Tag tag' var' => tag' == tag && sameVar var var' + | _ => False + end) + prod.tags) + | _ => 0 + end + +let countForwardedTags tag nterms = + nterms + |> allProds + |> List.filter isForwardProd + |> List.foldLeft + (fn count (prod : NTermProd) => count + countForwardedTag tag prod) + 0 + let tagSourceVar = var "tag-source" let sampleGrammar () = @@ -162,6 +188,22 @@ let emptySample = let transformedEmptySample = SplitNTermLevels.transform emptySample +let duplicateTagGrammar () = + let exprId = NTermId.fresh () in + mkGrammar + [ mkNTerm exprId "Expr" + [ tokenProd (LNum 0) "LOW" + , tokenProdWithTags (LNum 10) "HIGH_A" [("dup", TC_True)] + , tokenProdWithTags (LNum 10) "HIGH_B" [("dup", TC_True)] + ] + ] + +let duplicateTagSample = + duplicateTagGrammar () + +let transformedDuplicateTagSample = + SplitNTermLevels.transform duplicateTagSample + let _ = testSuite "SplitNTermLevels" (fn _ => @@ -218,4 +260,8 @@ testSuite "SplitNTermLevels" (fn _ => expectEq "Empty" nt.name; expectEq 0 (List.length nt.prods) | _ => assertTrue False - end)) + end); + + testCase "deduplicates forwarded tags" (fn _ => + let result = transformedDuplicateTagSample in + assertEq 1 (countForwardedTags "dup" result.nterms))) From 2aaa4e096a46872078064a56990dc4ef5056f1ee Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 03:21:27 +0200 Subject: [PATCH 11/20] Use map to collect forwarded tags --- src/Transform/SplitNTermLevels.fram | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 5e9b4a7..e3f6a58 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -14,8 +14,10 @@ import Map import Utils/UID let (Map { module LevelMap }) = Map.make { Key = Level } +let (Map { module TagMap }) = Map.make { Key = String } type LevelSet = LevelMap.T Unit +type TagSet = TagMap.T Unit let collectLevels (prods : List NTermProd) = let levelSet = @@ -65,18 +67,20 @@ let rewriteProd (idTable : IdTable) (NTermProd {module Prod}) = , symbols = List.map (rewriteSymbol idTable) Prod.symbols } -let rec dedupeTags (tags : List Tag) = - match tags with - | [] => [] - | tag :: rest => - tag :: dedupeTags (List.filter (fn tag' => not (tag' == tag)) rest) - end +let collectTags (tags : List Tag) = + let tagSet = + List.foldLeft + (fn (tagSet : TagSet) tag => tagSet.add tag ()) + TagMap.empty + tags + in + tagSet.fold (fn {key} _ tags => key :: tags) [] >.rev let tagsForLevel var level prods = levelProds level prods |> List.concatMap (fn (prod : NTermProd) => prod.tags) |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => tag) - |> dedupeTags + |> collectTags |> List.map (fn tag => (tag, TC_Tag tag var)) let forwardVar (nt : NTerm) = From 44311435ccd6894cd2a49e8cab211d6a24a12fde Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 03:27:52 +0200 Subject: [PATCH 12/20] Validate test suite path --- test.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test.sh b/test.sh index 5526afe..be9768b 100755 --- a/test.sh +++ b/test.sh @@ -6,6 +6,16 @@ if [ $# -ne 1 ]; then exit 1 fi +if [ ! -f "$1" ]; then + echo "ERROR: test suite file not found: $1" + exit 1 +fi + +if [ ! -r "$1" ]; then + echo "ERROR: test suite file is not readable: $1" + exit 1 +fi + binary="${DBL:-dbl}" if ! command -v "$binary" > /dev/null; then echo "ERROR: dbl executable not found in PATH" From 08031373ae580a1b393d30b2e60ed1f959c93b14 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 03:29:55 +0200 Subject: [PATCH 13/20] Scope test runner flags locally --- test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.sh b/test.sh index be9768b..5809704 100755 --- a/test.sh +++ b/test.sh @@ -57,8 +57,8 @@ function simple_test { } function run_with_flags { - flags="$2" - $1 + local flags="$2" + "$1" } source "$1" From 2bd5c749942954783d4c05e9a6aa671fbcd4c08c Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 16:53:55 +0200 Subject: [PATCH 14/20] Keep original non-terminal id for base level --- src/Transform/SplitNTermLevels.fram | 16 ++++++++++++---- test/TransformTests/SplitNTermLevels.fram | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index e3f6a58..df6d22d 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -97,6 +97,9 @@ let forwardProd (nt : NTerm) level nextLevel nextId = , action = AForward } +let splitNTermName (nt : NTerm) (id : NTermId) (level : Level) = + if id == nt.id then nt.name else nt.name + "@" + level.toString + let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = match levelIds with | [] => [] @@ -113,7 +116,7 @@ let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = in NTerm { id - , name = nt.name + "@" + level.toString + , name = splitNTermName nt id level , valueType = nt.valueType , prods } :: splitLevels idTable nt rest @@ -125,11 +128,16 @@ let splitNTerm (idTable : IdTable) (nt : NTerm) = | levelIds => splitLevels idTable nt levelIds end -let freshLevelIds levels = - List.map (fn level => (level, NTermId.fresh ())) levels +let freshLevelIds (nt : NTerm) levels = + match levels with + | [] => [] + | level :: levels => + (level, nt.id) :: + List.map (fn level => (level, NTermId.fresh ())) levels + end let addNTermLevelIds (idTable : IdTable) (nt : NTerm) = - idTable.add nt.id (freshLevelIds (collectLevels nt.prods)) + idTable.add nt.id (freshLevelIds nt (collectLevels nt.prods)) {## Split each non-terminal into multiple levels. ##} pub let transform (RichGrammar {module G}) : RichGrammar = diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index 3e3b4a6..7bfd5dc 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -179,6 +179,12 @@ let sample = sampleGrammar () let transformedSample = SplitNTermLevels.transform sample +let sampleExprId = + match sample.nterms with + | [nt] => nt.id + | _ => impossible () + end + let emptyGrammar () = mkGrammar [mkNTerm (NTermId.fresh ()) "Empty" []] @@ -215,6 +221,19 @@ testSuite "SplitNTermLevels" (fn _ => expectEq 1 (countNTermsWithLevel (LNum 10) result.nterms); expectEq 1 (countNTermsWithLevel LTop result.nterms)); + testCase "keeps original id and name on the lowest level" (fn _ => + let result = transformedSample in + let originalNTerms = + List.filter (fn (nt : NTerm) => nt.id == sampleExprId) result.nterms + in + assertEq 1 (List.length originalNTerms); + match originalNTerms with + | [nt] => + expectEq "Expr" nt.name; + assertTrue (hasProdAtLevel (LNum 0) nt) + | _ => assertTrue False + end); + testCase "keeps original productions only on their own levels" (fn _ => let result = transformedSample in let nonForwardProds = From f88dee062b03dbc7ef6fa4df18f1029d5ada2f69 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 17:16:57 +0200 Subject: [PATCH 15/20] Use active productions in SplitNTermLevels tests --- test/TransformTests/SplitNTermLevels.fram | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index 7bfd5dc..9125174 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -24,7 +24,7 @@ let tokenProdWithTags level token tags = { symbols = [PS_Token {var = None} token] , level , tags - , unless = TC_True + , unless = TC_False , action = action token } @@ -46,7 +46,7 @@ let infixProdWithTags level leftLevel token rightLevel tags exprId = ] , level , tags - , unless = TC_True + , unless = TC_False , action = action token } @@ -59,7 +59,7 @@ let parenProd exprId = ] , level = LTop , tags = [] - , unless = TC_True + , unless = TC_False , action = action "paren" } From a662d8fa69e043f7c60edb73ed138b7c5607542a Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 17:27:06 +0200 Subject: [PATCH 16/20] Forward tags through split level chains --- src/Transform/SplitNTermLevels.fram | 15 +++++++++------ test/TransformTests/SplitNTermLevels.fram | 13 ++++++++++++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index df6d22d..112e999 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -76,9 +76,12 @@ let collectTags (tags : List Tag) = in tagSet.fold (fn {key} _ tags => key :: tags) [] >.rev -let tagsForLevel var level prods = - levelProds level prods - |> List.concatMap (fn (prod : NTermProd) => prod.tags) +let tagsForLevels var levels prods = + levels + |> List.concatMap + (fn level => + levelProds level prods + |> List.concatMap (fn (prod : NTermProd) => prod.tags)) |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => tag) |> collectTags |> List.map (fn tag => (tag, TC_Tag tag var)) @@ -86,13 +89,13 @@ let tagsForLevel var level prods = let forwardVar (nt : NTerm) = Var {id = UID.fresh (), name = None, typ = nt.valueType} -let forwardProd (nt : NTerm) level nextLevel nextId = +let forwardProd (nt : NTerm) level nextLevel nextId tagLevels = let var = forwardVar nt in NTermProd { symbols = [PS_NTerm {var, level = nextLevel} nextId] , level - , tags = tagsForLevel var nextLevel nt.prods + , tags = tagsForLevels var tagLevels nt.prods , unless = TC_False , action = AForward } @@ -111,7 +114,7 @@ let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = match rest with | [] => prods | (nextLevel, nextId) :: _ => - prods + [forwardProd nt level nextLevel nextId] + prods + [forwardProd nt level nextLevel nextId (List.map fst rest)] end in NTerm diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index 9125174..5c6825a 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -168,7 +168,7 @@ let sampleGrammar () = LTop [("multiplicative", TC_Tag "multiplicative" tagSourceVar)] exprId - , tokenProd LTop "NUMBER" + , tokenProdWithTags LTop "NUMBER" [("atomic", TC_True)] , parenProd exprId ] in @@ -271,6 +271,17 @@ testSuite "SplitNTermLevels" (fn _ => in assertEq 1 (List.length bottomLevelWithTaggedForward)); + testCase "forwards tags through multiple levels" (fn _ => + let result = transformedSample in + let bottomLevelWithTaggedForward = + result.nterms + |> List.filter + (fn (nterm : NTerm) => + hasProdAtLevel (LNum 0) nterm && + hasTaggedForward "atomic" nterm) + in + assertEq 1 (List.length bottomLevelWithTaggedForward)); + testCase "preserves non-terminals without productions" (fn _ => let result = transformedEmptySample in assertEq 1 (List.length result.nterms); From 37b3c9f334365c5f79e4c1d63df66d988fb67847 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 18:01:47 +0200 Subject: [PATCH 17/20] Use map for split level productions --- src/Transform/SplitNTermLevels.fram | 40 ++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 112e999..5fdd0bb 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -34,6 +34,20 @@ let levelProds (level : Level) (prods : List NTermProd) = (fn (prod : NTermProd) => prod.level == level) prods +type ProdsByLevel = LevelMap.T (List NTermProd) + +let prodsByLevel levels prods = + List.foldLeft + (fn (prodsByLevel : ProdsByLevel) level => + prodsByLevel.add level (levelProds level prods)) + LevelMap.empty + levels + +let lookupProds (level : Level) (prodsByLevel : ProdsByLevel) = + prodsByLevel.findErr + {~onError = fn _ => impossible ()} + level + type LevelIds = List (Pair Level NTermId) type IdTable = NTermMap.T LevelIds @@ -76,11 +90,11 @@ let collectTags (tags : List Tag) = in tagSet.fold (fn {key} _ tags => key :: tags) [] >.rev -let tagsForLevels var levels prods = +let tagsForLevels var levels prodsByLevel = levels |> List.concatMap (fn level => - levelProds level prods + lookupProds level prodsByLevel |> List.concatMap (fn (prod : NTermProd) => prod.tags)) |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => tag) |> collectTags @@ -89,13 +103,13 @@ let tagsForLevels var levels prods = let forwardVar (nt : NTerm) = Var {id = UID.fresh (), name = None, typ = nt.valueType} -let forwardProd (nt : NTerm) level nextLevel nextId tagLevels = +let forwardProd (nt : NTerm) level nextLevel nextId tagLevels prodsByLevel = let var = forwardVar nt in NTermProd { symbols = [PS_NTerm {var, level = nextLevel} nextId] , level - , tags = tagsForLevels var tagLevels nt.prods + , tags = tagsForLevels var tagLevels prodsByLevel , unless = TC_False , action = AForward } @@ -103,18 +117,24 @@ let forwardProd (nt : NTerm) level nextLevel nextId tagLevels = let splitNTermName (nt : NTerm) (id : NTermId) (level : Level) = if id == nt.id then nt.name else nt.name + "@" + level.toString -let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = +let rec splitLevels + (idTable : IdTable) + (nt : NTerm) + (prodsByLevel : ProdsByLevel) + (levelIds : LevelIds) = match levelIds with | [] => [] | (level, id) :: rest => let prods = - levelProds level nt.prods + lookupProds level prodsByLevel |> List.map (rewriteProd idTable) let prods = match rest with | [] => prods | (nextLevel, nextId) :: _ => - prods + [forwardProd nt level nextLevel nextId (List.map fst rest)] + prods + + [forwardProd + nt level nextLevel nextId (List.map fst rest) prodsByLevel] end in NTerm @@ -122,13 +142,15 @@ let rec splitLevels (idTable : IdTable) (nt : NTerm) (levelIds : LevelIds) = , name = splitNTermName nt id level , valueType = nt.valueType , prods - } :: splitLevels idTable nt rest + } :: splitLevels idTable nt prodsByLevel rest end let splitNTerm (idTable : IdTable) (nt : NTerm) = match lookupLevelIds idTable nt.id with | [] => [nt] - | levelIds => splitLevels idTable nt levelIds + | levelIds => + let prodsByLevel = prodsByLevel (List.map fst levelIds) nt.prods in + splitLevels idTable nt prodsByLevel levelIds end let freshLevelIds (nt : NTerm) levels = From a06b56105f47b5ddc2b17f5b86f30f8dd093186a Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 18:13:37 +0200 Subject: [PATCH 18/20] Index productions by level in SplitNTermLevels --- src/Transform/SplitNTermLevels.fram | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index 5fdd0bb..b48bf30 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -36,11 +36,26 @@ let levelProds (level : Level) (prods : List NTermProd) = type ProdsByLevel = LevelMap.T (List NTermProd) -let prodsByLevel levels prods = +let addProdToLevel (prodsByLevel : ProdsByLevel) (prod : NTermProd) = + let prods = + prodsByLevel.find prod.level >.unwrapOr [] + in + prodsByLevel.add prod.level (prod :: prods) + +let prodsByLevel prods = + List.foldLeft addProdToLevel LevelMap.empty prods + >.map (fn {key} prods => prods.rev) + +let addEmptyLevel (prodsByLevel : ProdsByLevel) level = + if prodsByLevel.mem level then + prodsByLevel + else + prodsByLevel.add level [] + +let ensureLevels levels prodsByLevel = List.foldLeft - (fn (prodsByLevel : ProdsByLevel) level => - prodsByLevel.add level (levelProds level prods)) - LevelMap.empty + addEmptyLevel + prodsByLevel levels let lookupProds (level : Level) (prodsByLevel : ProdsByLevel) = @@ -149,7 +164,10 @@ let splitNTerm (idTable : IdTable) (nt : NTerm) = match lookupLevelIds idTable nt.id with | [] => [nt] | levelIds => - let prodsByLevel = prodsByLevel (List.map fst levelIds) nt.prods in + let prodsByLevel = + prodsByLevel nt.prods + |> ensureLevels (List.map fst levelIds) + in splitLevels idTable nt prodsByLevel levelIds end From 5733e1e1daf481cff41c843ee5c2ac74362bb866 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Mon, 15 Jun 2026 18:37:18 +0200 Subject: [PATCH 19/20] Remove redundant split level check --- src/Transform/SplitNTermLevels.fram | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index b48bf30..ede1ea0 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -46,18 +46,6 @@ let prodsByLevel prods = List.foldLeft addProdToLevel LevelMap.empty prods >.map (fn {key} prods => prods.rev) -let addEmptyLevel (prodsByLevel : ProdsByLevel) level = - if prodsByLevel.mem level then - prodsByLevel - else - prodsByLevel.add level [] - -let ensureLevels levels prodsByLevel = - List.foldLeft - addEmptyLevel - prodsByLevel - levels - let lookupProds (level : Level) (prodsByLevel : ProdsByLevel) = prodsByLevel.findErr {~onError = fn _ => impossible ()} @@ -164,10 +152,7 @@ let splitNTerm (idTable : IdTable) (nt : NTerm) = match lookupLevelIds idTable nt.id with | [] => [nt] | levelIds => - let prodsByLevel = - prodsByLevel nt.prods - |> ensureLevels (List.map fst levelIds) - in + let prodsByLevel = prodsByLevel nt.prods in splitLevels idTable nt prodsByLevel levelIds end From 70dc91378c2f0e4f90585ab2c7385c699b686d05 Mon Sep 17 00:00:00 2001 From: AlanPietrasz Date: Fri, 19 Jun 2026 16:21:03 +0200 Subject: [PATCH 20/20] Report invalid split levels via onError --- src/Transform/SplitNTermLevels.fram | 37 +++++++++++++++++------ test/TransformTests/SplitNTermLevels.fram | 32 +++++++++++++++++++- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index ede1ea0..4e133e4 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -52,24 +52,36 @@ let lookupProds (level : Level) (prodsByLevel : ProdsByLevel) = level type LevelIds = List (Pair Level NTermId) -type IdTable = NTermMap.T LevelIds +type NTermLevelIds = Pair String LevelIds +type IdTable = NTermMap.T NTermLevelIds -let lookupLevelIds (idTable : IdTable) (id : NTermId) = +parameter ~onError + +let lookupNTermLevelIds (idTable : IdTable) (id : NTermId) = idTable.findErr {~onError = fn _ => impossible ()} id -let rec pickLevelId (level : Level) (levelIds : LevelIds) = +let lookupLevelIds idTable id = + snd (lookupNTermLevelIds idTable id) + +let invalidLevelError (ntName : String) (level : Level) = + "Requested level " + level.toString + + " for non-terminal " + ntName + + " is stronger than every production level" + +let rec pickLevelId ntName (level : Level) (levelIds : LevelIds) = match levelIds with | [] => impossible () | (level', id) :: [] => - if level <= level' then id else impossible () + if level <= level' then id else ~onError (invalidLevelError ntName level) | (level', id) :: rest => - if level <= level' then id else pickLevelId level rest + if level <= level' then id else pickLevelId ntName level rest end -let lookupNTermId (idTable : IdTable) (id : NTermId) (level : Level) = - pickLevelId level (lookupLevelIds idTable id) +let lookupNTermId (idTable : IdTable) id level = + let (ntName, levelIds) = lookupNTermLevelIds idTable id in + pickLevelId ntName level levelIds let rewriteSymbol (idTable : IdTable) sym = match sym with @@ -165,10 +177,9 @@ let freshLevelIds (nt : NTerm) levels = end let addNTermLevelIds (idTable : IdTable) (nt : NTerm) = - idTable.add nt.id (freshLevelIds nt (collectLevels nt.prods)) + idTable.add nt.id (nt.name, freshLevelIds nt (collectLevels nt.prods)) -{## Split each non-terminal into multiple levels. ##} -pub let transform (RichGrammar {module G}) : RichGrammar = +pub let transformErr (RichGrammar {module G}) : RichGrammar = let idTable = List.foldLeft addNTermLevelIds NTermMap.empty G.nterms in @@ -178,3 +189,9 @@ pub let transform (RichGrammar {module G}) : RichGrammar = G.nterms |> List.concatMap (splitNTerm idTable) } + +{## Split each non-terminal into multiple levels. ##} +pub let transform g : RichGrammar = + transformErr + {~onError = fn msg => runtimeError ("SplitNTermLevels: " + msg)} + g diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram index 5c6825a..656aeeb 100644 --- a/test/TransformTests/SplitNTermLevels.fram +++ b/test/TransformTests/SplitNTermLevels.fram @@ -8,6 +8,9 @@ import open Repr/RichGrammar import Transform/SplitNTermLevels import Utils/UID +let ~onError _ = + runtimeError "Unhandled ~onError in SplitNTermLevels test module!" + let pos = Position { fname = "" @@ -210,6 +213,30 @@ let duplicateTagSample = let transformedDuplicateTagSample = SplitNTermLevels.transform duplicateTagSample +let invalidLevelGrammar () = + let exprId = NTermId.fresh () in + mkGrammar + [ mkNTerm exprId "Expr" + [ NTermProd + { symbols = [exprSym exprId (LNum 21) "bad"] + , level = LNum 20 + , tags = [] + , unless = TC_False + , action = action "bad-level" + } + ] + ] + +let invalidLevelSample = + invalidLevelGrammar () + +let invalidLevelReportsError = + handle + ~onError = effect _ / _ => True + return _ => False + in + SplitNTermLevels.transformErr invalidLevelSample + let _ = testSuite "SplitNTermLevels" (fn _ => @@ -294,4 +321,7 @@ testSuite "SplitNTermLevels" (fn _ => testCase "deduplicates forwarded tags" (fn _ => let result = transformedDuplicateTagSample in - assertEq 1 (countForwardedTags "dup" result.nterms))) + assertEq 1 (countForwardedTags "dup" result.nterms)); + + testCase "reports references above the strongest production level" (fn _ => + assertTrue invalidLevelReportsError))