diff --git a/.github/unlaxer-source-pin b/.github/unlaxer-source-pin
index 31e72ac7..6d696722 100644
--- a/.github/unlaxer-source-pin
+++ b/.github/unlaxer-source-pin
@@ -1,3 +1,3 @@
-# unlaxer-parser commit CI builds for the declarative-module development dependency.
-# Retain until pom.xml selects a published version with the same lexical capabilities.
-commit=f825740396a6181654191de5ba040e1f83533526
+# Exact source dependency for named layout and package origins.
+# Update to a published compatible artifact when available.
+commit=fc75fe1e25e538d47908f05ea2039f585aef696e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a3a97a88..f0ebd420 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -132,12 +132,14 @@ jobs:
run: |
task_maven_repo=$(mktemp -d "$RUNNER_TEMP/ubnfc-parser.XXXXXX")
echo "MAVEN_REPO=$task_maven_repo" >> "$GITHUB_ENV"
- # Declarative lexical modules require the pinned 3.2.0-SNAPSHOT generator;
+ # Declarative lexical modules require the pinned 3.3.0-SNAPSHOT generator;
# published 3.1.1 cannot read this grammar. Build into the job-local repository.
- name: Install unlaxer from source if it is not on Maven Central yet
run: bash scripts/ci/install-unlaxer-if-unpublished.sh
# Grammar sha256 + every vendored file against UBNFC_PIN. ubnfc is private, so the full
# grammar -> IR -> Java regeneration runs where it is readable (UBNFC_DIR); see the script.
+ - name: Verify strict layout compatibility adapter
+ run: python3 scripts/test_ubnfc_layout.py
- name: Verify vendored ubnfc parser against its pin
run: bash scripts/regenerate-ubnfc-parser.sh --check
- name: Verify the generated AST converter
@@ -172,6 +174,9 @@ jobs:
with:
java-version: '21'
distribution: 'temurin'
+ - name: Set up Rust 1.85 for the shared source fixtures
+ if: matrix.generator == 'source'
+ run: bash tinyexpression/.github/scripts/setup-rust.sh 1.85.0
- name: Create an empty Maven repository on the assigned runner
run: |
task_maven_repo=$(mktemp -d "$RUNNER_TEMP/mapper-compatibility.XXXXXX")
@@ -181,7 +186,7 @@ jobs:
with:
repository: opaopa6969/unlaxer-parser
# Declarative lexical modules under a distinct development artifact version.
- ref: f825740396a6181654191de5ba040e1f83533526
+ ref: fc75fe1e25e538d47908f05ea2039f585aef696e
path: unlaxer-parser
- name: Install pinned development generator into its isolated repository
if: matrix.generator == 'source'
@@ -215,19 +220,26 @@ jobs:
-Dtinyexpression.expected.mapper.snapshot=true
-Dtinyexpression.expected.mapper.nodeAlias=true
-Dtinyexpression.expected.mapper.optionalQualifier=true
- -Dtest=P4SourceMappingTest,P4OwnedSliceSourceTest,P4TypedAstEvaluatorTest,P4JavaCodeEmitterSourceTextTest,DslGeneratedAstJavaEmitterTest,P4PreferredAstMapperPrecedenceTest,P4OptionalExternalQualifierTest,P4RustRootExpressionAcceptanceTest,P4RustNumericEvaluatorOracleTest,P4RustScalarEvaluatorOracleTest,CodeBlockSourceTest,CodeBlockNoCompilerTest
+ -Dtest=P4LayoutMigrationTest,P4SourceMappingTest,P4OwnedSliceSourceTest,P4TypedAstEvaluatorTest,P4JavaCodeEmitterSourceTextTest,DslGeneratedAstJavaEmitterTest,P4PreferredAstMapperPrecedenceTest,P4OptionalExternalQualifierTest,P4RustRootExpressionAcceptanceTest,P4RustNumericEvaluatorOracleTest,P4RustScalarEvaluatorOracleTest,CodeBlockSourceTest,CodeBlockNoCompilerTest
- name: Verify shared Java/Rust parser fixtures with the pinned source generator
if: matrix.generator == 'source'
working-directory: tinyexpression
run: >-
mvn -B test -Dgpg.skip=true -Dmaven.repo.local="$MAVEN_REPO" -Dunlaxer.version="$UNLAXER_VERSION"
-Dtinyexpression.skipRailroad=true -Dtinyexpression.rust.shared=true
- -Dtest=P4RustSharedFixtureAcceptanceTest
+ -Dtinyexpression.layout.conformance=true -Dtinyexpression.unlaxer.source=../unlaxer-parser
+ -Dtest=P4RustSharedFixtureAcceptanceTest,P4LayoutGeneratedConformanceTest
+ - name: Require the shared layout comparison evidence
+ if: matrix.generator == 'source'
+ working-directory: tinyexpression
+ run: test -s target/p4-layout-generated.tsv
- uses: actions/upload-artifact@v6
if: always()
with:
name: mapper-compatibility-${{ matrix.generator }}
- path: tinyexpression/target/surefire-reports/
+ path: |
+ tinyexpression/target/surefire-reports/
+ tinyexpression/target/p4-layout-generated.tsv
if-no-files-found: warn
# issue #173: the Java CI above resolves org.unlaxer:unlaxer-* from Maven Central, so a
diff --git a/.gitignore b/.gitignore
index 2aa42bb3..e1339a32 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,6 @@ buildNumber.properties
/node_modules/
/package.json
/package-lock.json
+
+# Python test bytecode
+/scripts/__pycache__/
diff --git a/docs/imported-layout-migration.md b/docs/imported-layout-migration.md
new file mode 100644
index 00000000..6b3728e6
--- /dev/null
+++ b/docs/imported-layout-migration.md
@@ -0,0 +1,32 @@
+# P4 の共有空白・コメント定義
+
+P4 の grammar default と `Formula`、`ImportDeclaration`、`VariableDeclaration` は、固定した `std/layout@1.0.0` の `layout.SPACES_AND_COMMENTS` を参照する。rule の明示指定は `@whitespace(layout.SPACES_AND_COMMENTS)` に置換した。従来の `@interleave(profile=javaStyle)` と同じ場所で同じ集合を使い、子 rule に独自の指定が追加された場合も rule の優先順位を保てるため、冗長に見える指定も残した。
+
+`grammar/ubnf.json` は builtin の正確な版を指定する。`ubnf.lock.json` の identity は `std/layout` / `1.0.0` / SHA-256 `2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67`。この小さな標準 artifact 自体を grammar の `.ubnf-cache/packages` に同梱する。生成は lock と検証済み bytes のみを読み、ネットワーク・認証・個人設定を必要としない。上流 generator は Maven `3.3.0-SNAPSHOT` と `.github/unlaxer-source-pin` の exact commit で固定する。CI の専用 Maven repository へその commit を install して検証する。
+
+標準定義は ASCII space と TAB..CR、CR/LF 手前までの `//`、最初の `*/` までの block comment。閉じていない block comment は読み飛ばさない。文字列と fenced Java code の token 内部へ自動 skip は入らない。AST の文字列正規化、raw CST、source-preserving model の既存の comment mask と位置契約を維持する。
+
+## backend の範囲
+
+| 経路 | 実 P4 | 名前付き空白と固定 package |
+|---|---|---|
+| unlaxer Java 生成 parser | 対応。旧 grammar と全 AST field/node span・cursor・CST 原文を比較 | 直接対応 |
+| unlaxer Java 経由 Rust 生成 / native Rust 生成 | P4 の token/layout projection で同じ入力・独立期待値・生成物 bytes を比較。実 P4 全体には従来の portability 制限がある（native は `literalBoundary: none` を拒否） | 直接対応。局所 `none` も共通 fixture で比較 |
+| TinyExpression 本番 ubnfc Java / Rust | 固定生成物を使う。共有実 P4 fixture と source positions、memo OFF/ON を検証 | **std/layout@1.0.0 の限定互換処理**。任意 package/named policy の直接対応ではない |
+
+`scripts/prepare-ubnfc-layout.py` は manifest・lock graph・artifact hash・全定義・global 1箇所/rule 3箇所を確認し、この標準定義だけを旧 javaStyle/interleave へ戻す。混在設定、追加 package/comment、未知の定義、改ざんを変換前に拒否する。導出 grammar は旧 SHA-256 と一致し、Java/Rust ubnfc の grammar→IR→生成物も byte 一致する。再生成スクリプトの両経路から同じ互換処理を呼び、文法・manifest・lock・artifact・互換処理の hash を両 pin に含めた。汎用 ubnfc 対応は [#249](https://github.com/opaopa6969/tinyexpression/issues/249) で追跡する。
+
+## 検証と定義表示
+
+`actual-p4.json` は実 P4 の空白/TAB/LF/CRLF/コメント/未閉鎖/文字列/Java fence/import/declaration の14入力。Java の旧/新 parser と Tiny 本番 Java/Rust が同じ cursor 期待値を使う。Java の旧/新 AST 全 field/node span と retained CST 原文、owned source の位置を比較する。`corpus.json` は実 P4 の lexical modules を用いる29入力で、旧/新 layout と局所 `none` を Java・Java経由Rust・native Rust へ入力し、位置・診断・AST の独立期待値を比較する。
+
+```sh
+python3 scripts/test_ubnfc_layout.py
+MAVEN_REPO=/tmp/tiny-layout-m2 scripts/ci/install-unlaxer-if-unpublished.sh
+mvn test -Dmaven.repo.local=/tmp/tiny-layout-m2 -Dtinyexpression.skipRailroad=true \
+ -Dtinyexpression.layout.conformance=true -Dtinyexpression.unlaxer.source=/path/to/pinned/unlaxer-parser \
+ -Dtest=P4LayoutMigrationTest,P4LayoutGeneratedConformanceTest
+cargo test --locked --manifest-path rust/Cargo.toml -p tinyexpression-rs shared_imported_layout
+```
+
+文法作者は UBNF authoring extension の `@whitespace` 補完から `layout.SPACES_AND_COMMENTS` を選び、hover で定義と ID/version/hash、定義への移動で読み取り専用の標準 module を確認できる。CLI `deps inspect --grammar tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf` も同じ定義・出典・code-point 位置を返す。`P4LayoutMigrationTest` は実 grammar のこの snapshot と4 policy scopeを確認する。TinyExpression の runtime LSP/DAP と文法作者向け UBNF LSP は別の役割を持つ。
diff --git a/pom.xml b/pom.xml
index ab1ad89f..15c5c722 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,7 +56,7 @@
installed into a job-local, isolated Maven repo (issue #173: unlaxer-parser master,
non-blocking) so the GAV actually resolves from that isolated repo instead of
silently falling back to the published artifact. -->
- 3.2.0-SNAPSHOT
+ 3.3.0-SNAPSHOT
diff --git a/rust/check-generated.sh b/rust/check-generated.sh
index 7d897565..0810a7d3 100755
--- a/rust/check-generated.sh
+++ b/rust/check-generated.sh
@@ -17,6 +17,12 @@ set -euo pipefail
script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
repo_dir=$(cd -- "$script_dir/.." && pwd)
+# The offline check validates the restricted layout bridge even without ubnfc.
+layout_check=$(mktemp -d)
+trap 'rm -rf "$layout_check"' EXIT
+python3 "$repo_dir/scripts/prepare-ubnfc-layout.py" \
+ "$repo_dir/tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf" \
+ "$layout_check/tinyexpression-p4.ubnf" >/dev/null
# The offline check must notice a changed imported grammar too.
(cd "$repo_dir" && sed -n 's/^grammar_file //p' rust/ubnfc-pin.txt | sha256sum -c -)
if [ -d "${UBNFC_DIR:-$repo_dir/../ubnfc}" ]; then
diff --git a/rust/scripts/regenerate-ubnfc.sh b/rust/scripts/regenerate-ubnfc.sh
index c2e0f6d1..48f993d6 100755
--- a/rust/scripts/regenerate-ubnfc.sh
+++ b/rust/scripts/regenerate-ubnfc.sh
@@ -78,7 +78,7 @@ export CARGO_TARGET_DIR="$work/target"
sha() { sha256sum "$1" | cut -d' ' -f1; }
grammar_manifest() {
- (cd "$repo_dir" && { find "$(dirname "$grammar_rel")" -type f -name '*.ubnf'; echo "$fi_grammar_rel"; } |
+ (cd "$repo_dir" && { find "$(dirname "$grammar_rel")" -type f \( -name '*.ubnf' -o -name '*.json' \); echo "$fi_grammar_rel"; echo scripts/prepare-ubnfc-layout.py; } |
LC_ALL=C sort -u | while IFS= read -r f; do echo "$(sha "$f") $f"; done)
}
@@ -100,8 +100,10 @@ if [[ "$mode" == --check ]]; then
diff <(sed -n 's/^grammar_file //p' "$pin") <(grammar_manifest)
fi
+python3 "$repo_dir/scripts/prepare-ubnfc-layout.py" "$repo_dir/$grammar_rel" "$work/compat/$grammar_rel" >/dev/null
+touch "$work/compat/.git" # preserve the original logical source root
(cd "$ubnfc_dir" && cargo run -q --offline -p ubnfc-front --bin ubnfc -- \
- ir --grammar "$repo_dir/$grammar_rel" \
+ ir --grammar "$work/compat/$grammar_rel" \
--extern-first-chars "$ubnfc_dir/scanners/first-chars.json" \
--out "$work/p4.ir.json" >/dev/null)
check_sha "$work/p4.ir.json" "$(value ir_sha256)"
@@ -177,6 +179,16 @@ case "$mode" in
rm -rf "$vendored"
cp -r "$work/generated" "$vendored"
(cd "$vendored" && find . -type f | LC_ALL=C sort | xargs sha256sum) > "$manifest"
+ python3 - "$rust_dir/tinyexpression-rs/src/api.rs" "$(sha "$repo_dir/$grammar_rel")" <<'PY'
+from pathlib import Path
+import re, sys
+path = Path(sys.argv[1])
+text, count = re.subn(r'pub const GRAMMAR_SHA256: &str = "[0-9a-f]{64}";',
+ 'pub const GRAMMAR_SHA256: &str = "' + sys.argv[2] + '";', path.read_text())
+if count != 1:
+ raise SystemExit("expected exactly one GRAMMAR_SHA256 constant")
+path.write_text(text)
+PY
echo "vendored ubnfc parser: updated"
rm -rf "$vendored_fi"
cp -r "$work/generated-formula-info" "$vendored_fi"
diff --git a/rust/tinyexpression-rs/src/api.rs b/rust/tinyexpression-rs/src/api.rs
index 6dbaeb2e..8ea11ea6 100644
--- a/rust/tinyexpression-rs/src/api.rs
+++ b/rust/tinyexpression-rs/src/api.rs
@@ -39,7 +39,7 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// fails when it differs from `ubnfc_commit` in `rust/ubnfc-pin.txt`.
pub const UBNFC_COMMIT: &str = "f54d6b90466ffde14066d32f911ed642b1e49201";
/// Exact P4 grammar source identity; checked against `rust/ubnfc-pin.txt` in CI.
-pub const GRAMMAR_SHA256: &str = "4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a";
+pub const GRAMMAR_SHA256: &str = "184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46";
/// One JSON response and the exit code that goes with it.
#[derive(Clone, Debug, PartialEq, Eq)]
diff --git a/rust/tinyexpression-rs/src/frontend/declarative_tests.rs b/rust/tinyexpression-rs/src/frontend/declarative_tests.rs
index ffdbd733..b2bfc4ba 100644
--- a/rust/tinyexpression-rs/src/frontend/declarative_tests.rs
+++ b/rust/tinyexpression-rs/src/frontend/declarative_tests.rs
@@ -4,6 +4,59 @@ use crate::generated::{
compat, ubnfc,
};
+#[test]
+fn shared_imported_layout_keeps_production_p4_positions() {
+ use crate::request::{parse_json, Json};
+ let Json::Arr(rows) = parse_json(include_str!(
+ "../../../../src/test/resources/p4-layout/actual-p4.json"
+ ))
+ .unwrap() else {
+ panic!("array")
+ };
+ for row in rows {
+ let id = row.str_field("id").unwrap();
+ let input = row.str_field("input").unwrap();
+ let Some(Json::Bool(ok)) = row.get("ok") else {
+ panic!("{id}")
+ };
+ let end = if *ok { input.chars().count() } else { 0 };
+ for memo in [false, true] {
+ let result = ubnfc::parse_entry_with_options(
+ "TinyExpressionP4",
+ Some("Formula"),
+ input,
+ ubnfc::ParseOptions {
+ require_eof: false,
+ memo,
+ ..Default::default()
+ },
+ )
+ .unwrap();
+ assert_eq!(
+ (result.ok, result.consumed_cp, result.matched_cp),
+ (*ok, end, end),
+ "{id}"
+ );
+ if *ok {
+ let converted = compat::convert(result.ast.as_ref().unwrap()).unwrap();
+ let Ast::FormulaExpr { span, .. } = converted else {
+ panic!("{id}")
+ };
+ assert_eq!(
+ (span.start, span.end),
+ (0, end),
+ "{id} owned source position"
+ );
+ assert_eq!(
+ input.chars().take(span.end).collect::(),
+ input,
+ "{id} owned source"
+ );
+ }
+ }
+ }
+}
+
#[test]
fn shared_lexical_cases_need_no_host_parser_mapping() {
use crate::request::{parse_json, Json};
diff --git a/rust/ubnfc-pin.txt b/rust/ubnfc-pin.txt
index 4d1e4c50..f33fa8d8 100644
--- a/rust/ubnfc-pin.txt
+++ b/rust/ubnfc-pin.txt
@@ -1,14 +1,18 @@
# Generated by rust/scripts/regenerate-ubnfc.sh --write; do not edit hashes by hand.
ubnfc_commit=f54d6b90466ffde14066d32f911ed642b1e49201
grammar=tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
-grammar_sha256=4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a
+grammar_sha256=184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46
grammar_file 28bcfc9f2e684b7326561f281155a1d0140dc67145d178764841bc7dc87a81cb grammar/formula-info.ubnf
+grammar_file 492cafad9f8f3278480d862d0f1fadeb5f61bf5c627b8072b32f6d90c91d3ff1 scripts/prepare-ubnfc-layout.py
+grammar_file 2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67 tools/tinyexpression-p4-lsp-vscode/grammar/.ubnf-cache/packages/2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67.json
grammar_file e33b8166d0a5b399150b583b0e466fbe2a4194325ef65c79c082297156796b76 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/characters.ubnf
grammar_file 53ad774e87bd95a498ac744a721f22bdaf877721c0da13e8045aaeaea09b8179 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/fences.ubnf
grammar_file 55f5f3b9289cb41b74d5c617e0b2ec1adbf24eb7fd8ca11aa02eda311b92bf2b tools/tinyexpression-p4-lsp-vscode/grammar/lexical/identifiers.ubnf
grammar_file 1b1fcaa633f9477ad5414ecfed64693d1d50bf0aaa02775c5d67b39a18e8244f tools/tinyexpression-p4-lsp-vscode/grammar/lexical/numbers.ubnf
grammar_file 08f209d29c84aa1addcb57eba24d7738c27bc8921cb13f1fbd5040dff6296a26 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/strings.ubnf
-grammar_file 4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+grammar_file 184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46 tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+grammar_file 2074b51b078eecd61c5a0bbd20399f423b20faada77d331b96f212692c27bdf7 tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.json
+grammar_file 25745248036e459f3b69ad202f12b52d857b6f70abc02dac23caf4e7745f4db0 tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.lock.json
ir_sha256=55d5981a9da79a887034778aa1853dffea92162cb5e9be18c840b072cafb13a0
scanners=scanners/rust/scanners.rs
scanners_sha256=593a2736f7f20fa5d61456e25e7d961cefbd427d93b05a7ced46b41506e6a412
diff --git a/scripts/ci/install-unlaxer-if-unpublished.sh b/scripts/ci/install-unlaxer-if-unpublished.sh
index 6459b344..d6fecbd3 100755
--- a/scripts/ci/install-unlaxer-if-unpublished.sh
+++ b/scripts/ci/install-unlaxer-if-unpublished.sh
@@ -4,7 +4,7 @@
#
# MAVEN_REPO=/path/to/repo scripts/ci/install-unlaxer-if-unpublished.sh
#
-# Declarative P4 lexical modules need unlaxer 3.2.0-SNAPSHOT, not the published 3.1.1.
+# Declarative P4 lexical modules need unlaxer 3.3.0-SNAPSHOT, not the published 3.1.1.
# Until a compatible version is published, CI builds the pinned unlaxer-parser commit
# (.github/unlaxer-source-pin) whose must equal pom.xml's unlaxer.version. Once the
# version is on Central this script does nothing, and the pin file can be deleted.
diff --git a/scripts/prepare-ubnfc-layout.py b/scripts/prepare-ubnfc-layout.py
new file mode 100644
index 00000000..d481edbe
--- /dev/null
+++ b/scripts/prepare-ubnfc-layout.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Lower only pinned std/layout 1.0.0 to ubnfc's existing javaStyle policy.
+
+This is a TinyExpression compatibility bridge, not a package resolver. It never
+reads credentials or downloads anything, and refuses every other named policy.
+"""
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+import sys
+
+STANDARD_HASH = "2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67"
+STANDARD_TEXT = (
+ "grammar StandardLayout {\n @ubnf: v2\n"
+ " token SPACES ::= (' ' | CHAR_RANGE('\\t', '\\r'))+;\n"
+ " token LINE_COMMENT ::= '//' NEGATION('\\r\\n')*;\n"
+ " token BLOCK_COMMENT ::= '/*' (NEGATIVE_LOOKAHEAD('*/') ANY)* '*/';\n"
+ " token SPACES_AND_COMMENTS ::= SPACES | LINE_COMMENT | BLOCK_COMMENT;\n}\n"
+)
+
+
+def strict_object(pairs):
+ result = {}
+ for name, value in pairs:
+ if name in result:
+ raise ValueError("duplicate JSON key")
+ if name == "schemaVersion" and type(value) is not int:
+ raise ValueError("schemaVersion must be an integer")
+ result[name] = value
+ return result
+
+
+def read_json(path):
+ def invalid_constant(value):
+ raise ValueError("invalid JSON numeric constant")
+ return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=strict_object,
+ parse_constant=invalid_constant)
+
+
+def prepare(source, destination):
+ base = source.parent
+ expected_dependency = {"version": "1.0.0", "source": "builtin:std/layout@1.0.0"}
+ if read_json(base / "ubnf.json") != {
+ "schemaVersion": 1, "dependencies": {"std/layout": expected_dependency}
+ }:
+ raise ValueError("only the exact builtin std/layout dependency is supported")
+ if read_json(base / "ubnf.lock.json") != {
+ "schemaVersion": 1, "roots": {"std/layout": "1.0.0"},
+ "packages": {"std/layout": {"id": "std/layout", "version": "1.0.0",
+ "sha256": STANDARD_HASH, "dependencies": {}}}
+ }:
+ raise ValueError("layout lock identity or dependency graph differs")
+ artifact_path = base / ".ubnf-cache" / "packages" / (STANDARD_HASH + ".json")
+ artifact = artifact_path.read_bytes()
+ if hashlib.sha256(artifact).hexdigest() != STANDARD_HASH:
+ raise ValueError("layout artifact SHA-256 mismatch")
+ if read_json(artifact_path) != {
+ "schemaVersion": 1, "id": "std/layout", "version": "1.0.0",
+ "entry": "layout.ubnf", "files": {"layout.ubnf": STANDARD_TEXT}, "dependencies": {}
+ }:
+ raise ValueError("layout definitions differ from the javaStyle compatibility set")
+
+ text = source.read_text(encoding="utf-8")
+ imported = " @import layout from 'pkg:std/layout'\n\n"
+ global_policy = "@whitespace: layout.SPACES_AND_COMMENTS"
+ local_policy = "@whitespace(layout.SPACES_AND_COMMENTS)"
+ if text.count(imported) != 1 or text.count(global_policy) != 1 or text.count(local_policy) != 3:
+ raise ValueError("P4 layout policy scopes differ from the verified migration")
+ lowered = text.replace(imported, "").replace(global_policy, "@whitespace: javaStyle")
+ lowered = lowered.replace(local_policy, "@interleave(profile=javaStyle)")
+ # Reject mixed/custom policies and extra package imports before invoking ubnfc.
+ if "pkg:" in lowered or "@comment" in lowered:
+ raise ValueError("additional package/comment configuration is unsupported")
+ if re.findall(r"@whitespace\s*:\s*(\S+)", lowered) != ["javaStyle"]:
+ raise ValueError("mixed or unknown global whitespace policy")
+ if re.search(r"@whitespace\s*\(", lowered):
+ raise ValueError("additional local whitespace policy is unsupported")
+ if re.findall(r"@interleave\s*\(([^)]*)\)", lowered) != ["profile=javaStyle"] * 3:
+ raise ValueError("mixed or unknown interleave policy")
+ # Keep imported module paths identical in the derived tree; no external files.
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copytree(base, destination.parent, dirs_exist_ok=True)
+ destination.write_text(lowered, encoding="utf-8")
+ return hashlib.sha256(lowered.encode("utf-8")).hexdigest()
+
+
+if __name__ == "__main__":
+ try:
+ if len(sys.argv) != 3:
+ raise ValueError("usage: prepare-ubnfc-layout.py SOURCE.ubnf DESTINATION.ubnf")
+ print(prepare(Path(sys.argv[1]), Path(sys.argv[2])))
+ except (ValueError, OSError, KeyError) as error:
+ print("E-UBNFC-LAYOUT: " + str(error), file=sys.stderr)
+ sys.exit(1)
diff --git a/scripts/regenerate-ubnfc-parser.sh b/scripts/regenerate-ubnfc-parser.sh
index d1a8fe65..afb5f5dc 100755
--- a/scripts/regenerate-ubnfc-parser.sh
+++ b/scripts/regenerate-ubnfc-parser.sh
@@ -48,7 +48,7 @@ sha() { sha256sum "$1" | cut -d' ' -f1; }
# Include imported modules, not only the entry grammar, in the reproducibility pin.
grammar_manifest() {
- (cd "$repo" && find "$(dirname "$grammar_rel")" -type f -name '*.ubnf' | LC_ALL=C sort |
+ (cd "$repo" && { find "$(dirname "$grammar_rel")" -type f \( -name '*.ubnf' -o -name '*.json' \); echo scripts/prepare-ubnfc-layout.py; } | LC_ALL=C sort |
while IFS= read -r f; do echo "$(sha "$f") $f"; done)
}
@@ -64,6 +64,10 @@ pinned_commit=$(pin_value ubnfc_commit)
fail=0
offline_check() {
local expected actual
+ local layout_check
+ layout_check=$(mktemp -d)
+ python3 "$repo/scripts/prepare-ubnfc-layout.py" "$repo/$grammar_rel" "$layout_check/$grammar_rel" >/dev/null
+ rm -rf "$layout_check"
expected=$(pin_value grammar_sha256)
actual=$(sha "$repo/$grammar_rel")
if [[ "$expected" != "$actual" ]]; then
@@ -111,8 +115,10 @@ regenerate() {
echo "building ubnfc Java backend at $rev ..."
# shellcheck disable=SC2086
mvn -q ${MVN_FLAGS:-} -f "$scratch/ubnfc/ubnfc-java/pom.xml" -Dmaven.test.skip=true package
- # Run from the repo root so the IR records the grammar by its repo-relative path.
- (cd "$repo" && "$scratch/ubnfc/target/release/ubnfc" ir --grammar "$grammar_rel" \
+ # Use the verified compatibility grammar, retaining its repo-relative path.
+ python3 "$repo/scripts/prepare-ubnfc-layout.py" "$repo/$grammar_rel" "$scratch/compat/$grammar_rel" >/dev/null
+ touch "$scratch/compat/.git" # logical source root only; no Git commands use this marker
+ (cd "$scratch/compat" && "$scratch/ubnfc/target/release/ubnfc" ir --grammar "$grammar_rel" \
--extern-first-chars "$scratch/ubnfc/$first_chars" --out "$scratch/out/grammar.ir.json" \
2> "$scratch/front-warnings.txt")
java -cp "$scratch/ubnfc/ubnfc-java/target/ubnfc-java-0.1.0-SNAPSHOT.jar" org.unlaxer.ubnfc.java.Main \
diff --git a/scripts/test_ubnfc_layout.py b/scripts/test_ubnfc_layout.py
new file mode 100644
index 00000000..05db8574
--- /dev/null
+++ b/scripts/test_ubnfc_layout.py
@@ -0,0 +1,82 @@
+import importlib.util
+import json
+from pathlib import Path
+import shutil
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parent.parent
+SPEC = importlib.util.spec_from_file_location("layout", ROOT / "scripts/prepare-ubnfc-layout.py")
+LAYOUT = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(LAYOUT)
+
+
+class LayoutCompatibilityTests(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.base = Path(self.temporary.name)
+ shutil.copytree(ROOT / "tools/tinyexpression-p4-lsp-vscode/grammar", self.base / "grammar")
+ self.source = self.base / "grammar/tinyexpression-p4.ubnf"
+ self.destination = self.base / "derived/tinyexpression-p4.ubnf"
+
+ def tearDown(self):
+ self.temporary.cleanup()
+
+ def rejects(self):
+ with self.assertRaises((ValueError, OSError)):
+ LAYOUT.prepare(self.source, self.destination)
+ self.assertFalse(self.destination.exists(), "reject before publishing the derived grammar")
+
+ def test_exact_snapshot_restores_the_pinned_old_grammar(self):
+ self.assertEqual("4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a",
+ LAYOUT.prepare(self.source, self.destination))
+ self.assertNotIn("pkg:", self.destination.read_text())
+
+ def test_custom_mixed_and_missing_policies_are_rejected(self):
+ original = self.source.read_text()
+ for changed in [
+ original.replace("layout.SPACES_AND_COMMENTS", "layout.SPACES"),
+ original.replace("@whitespace: layout.SPACES_AND_COMMENTS", "@whitespace: custom.GAP"),
+ original + "\n@whitespace: none\n",
+ original + "\n@whitespace(none)\n",
+ original + "\n@interleave(profile=commentsAndSpaces)\n",
+ original + "\n@comment: hidden\n",
+ original + "\n@import extra from 'pkg:team/layout'\n",
+ original.replace("@whitespace(layout.SPACES_AND_COMMENTS)", "", 1),
+ ]:
+ with self.subTest(changed=changed[-60:]):
+ self.source.write_text(changed)
+ self.rejects()
+
+ def test_manifest_version_and_source_are_exact(self):
+ path = self.source.parent / "ubnf.json"
+ for dependency in [{"version": "1.0.1", "source": "builtin:std/layout@1.0.0"},
+ {"version": "1.0.0", "source": "https://example.test/layout.json"}]:
+ path.write_text(json.dumps({"schemaVersion": 1, "dependencies": {"std/layout": dependency}}))
+ self.rejects()
+
+ def test_identity_hash_graph_and_artifact_tampering_are_rejected(self):
+ lock = self.source.parent / "ubnf.lock.json"
+ original = json.loads(lock.read_text())
+ for key, value in [("id", "team/layout"), ("version", "1.0.1"), ("sha256", "0" * 64),
+ ("dependencies", {"team/layout": "1.0.0"})]:
+ changed = json.loads(json.dumps(original))
+ changed["packages"]["std/layout"][key] = value
+ lock.write_text(json.dumps(changed))
+ self.rejects()
+ lock.write_text(json.dumps(original))
+ cache = self.source.parent / ".ubnf-cache/packages" / (LAYOUT.STANDARD_HASH + ".json")
+ cache.write_bytes(cache.read_bytes() + b" ")
+ self.rejects()
+
+ def test_non_integer_schema_is_rejected(self):
+ (self.source.parent / "ubnf.json").write_text('{"schemaVersion":true}')
+ self.rejects()
+
+ def test_duplicate_json_keys_are_rejected(self):
+ (self.source.parent / "ubnf.json").write_text('{"schemaVersion":1,"schemaVersion":1}')
+ self.rejects()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/src/main/java/org/unlaxer/tinyexpression/p4/ubnfc/UBNFC_PIN b/src/main/java/org/unlaxer/tinyexpression/p4/ubnfc/UBNFC_PIN
index 412e7805..ceb9cd37 100644
--- a/src/main/java/org/unlaxer/tinyexpression/p4/ubnfc/UBNFC_PIN
+++ b/src/main/java/org/unlaxer/tinyexpression/p4/ubnfc/UBNFC_PIN
@@ -4,13 +4,17 @@
ubnfc_repository=https://github.com/opaopa6969/ubnfc
ubnfc_commit=f54d6b90466ffde14066d32f911ed642b1e49201
grammar=tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
-grammar_sha256=4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a
+grammar_sha256=184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46
+grammar_file 492cafad9f8f3278480d862d0f1fadeb5f61bf5c627b8072b32f6d90c91d3ff1 scripts/prepare-ubnfc-layout.py
+grammar_file 2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67 tools/tinyexpression-p4-lsp-vscode/grammar/.ubnf-cache/packages/2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67.json
grammar_file e33b8166d0a5b399150b583b0e466fbe2a4194325ef65c79c082297156796b76 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/characters.ubnf
grammar_file 53ad774e87bd95a498ac744a721f22bdaf877721c0da13e8045aaeaea09b8179 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/fences.ubnf
grammar_file 55f5f3b9289cb41b74d5c617e0b2ec1adbf24eb7fd8ca11aa02eda311b92bf2b tools/tinyexpression-p4-lsp-vscode/grammar/lexical/identifiers.ubnf
grammar_file 1b1fcaa633f9477ad5414ecfed64693d1d50bf0aaa02775c5d67b39a18e8244f tools/tinyexpression-p4-lsp-vscode/grammar/lexical/numbers.ubnf
grammar_file 08f209d29c84aa1addcb57eba24d7738c27bc8921cb13f1fbd5040dff6296a26 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/strings.ubnf
-grammar_file 4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+grammar_file 184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46 tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+grammar_file 2074b51b078eecd61c5a0bbd20399f423b20faada77d331b96f212692c27bdf7 tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.json
+grammar_file 25745248036e459f3b69ad202f12b52d857b6f70abc02dac23caf4e7745f4db0 tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.lock.json
extern_first_chars=scanners/first-chars.json
ir_sha256=55d5981a9da79a887034778aa1853dffea92162cb5e9be18c840b072cafb13a0
java_package=org.unlaxer.tinyexpression.p4.ubnfc.generated
diff --git a/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutGeneratedConformanceTest.java b/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutGeneratedConformanceTest.java
new file mode 100644
index 00000000..4e47d38f
--- /dev/null
+++ b/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutGeneratedConformanceTest.java
@@ -0,0 +1,256 @@
+package org.unlaxer.tinyexpression.p4;
+
+import org.unlaxer.dsl.codegen.*;
+
+import static org.junit.Assert.*;
+import static org.junit.Assume.assumeTrue;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import javax.tools.DiagnosticCollector;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.ToolProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.unlaxer.context.ParseContext;
+import org.unlaxer.dsl.bootstrap.UBNFAST.GrammarDecl;
+import org.unlaxer.dsl.bootstrap.UBNFMapper;
+import org.unlaxer.dsl.codegen.CodeGenerator.GeneratedSource;
+import org.unlaxer.dsl.codegen.rust.RustBackend;
+import org.unlaxer.parser.Parser;
+
+/** Java/generated-Rust/native-frontend contract for rule-scoped trivia policy. */
+public class P4LayoutGeneratedConformanceTest extends P4LayoutTestSupport {
+ private static final String PACKAGE = "org.example.ruletrivia";
+
+ private final Path repo = Path.of(System.getProperty("tinyexpression.unlaxer.source", "../unlaxer-parser")).toAbsolutePath().normalize();
+
+ @Test public void p4LayoutPoliciesPreserveCursorsAcceptanceAstAndSpans() throws Exception {
+ assumeTrue("enable with -Dtinyexpression.layout.conformance=true (requires rustc/cargo)",
+ Boolean.getBoolean("tinyexpression.layout.conformance"));
+
+ Path runtime = temporary.getRoot().toPath().resolve("libunlaxer_runtime.rlib");
+ success(run(List.of("rustc", "--edition=2021", "--crate-type=rlib",
+ "--crate-name=unlaxer_runtime", repo.resolve("rust/unlaxer-runtime/src/lib.rs").toString(),
+ "-o", runtime.toString()), "", false));
+
+ Path nativeTarget = temporary.getRoot().toPath().resolve("native-target");
+ success(run(List.of("cargo", "build", "--locked", "--manifest-path",
+ repo.resolve("rust/Cargo.toml").toString(), "-p", "unlaxer-generator", "--target-dir",
+ nativeTarget.toString()), "", false));
+ Path nativeGenerator = nativeTarget.resolve("debug/unlaxer").toAbsolutePath();
+
+ Path fixtures = Path.of("src/test/resources/p4-layout");
+ JsonArray corpus = JsonParser.parseString(
+ Files.readString(fixtures.resolve("corpus.json"))).getAsJsonArray();
+ var report = new ArrayList<>(List.of(
+ "fixture\tcase\tinput_json\texpected_prefix\tjava_prefix\texpected_ast\tjava_ast\trust"));
+
+ int fixtureIndex = 0;
+ for (JsonElement fixtureElement : corpus) {
+ JsonObject fixture = fixtureElement.getAsJsonObject();
+ String name = fixture.get("name").getAsString();
+ String source = fixture.get("grammar").getAsString();
+ if (fixture.has("modules")) for (var module : fixture.getAsJsonObject("modules").entrySet())
+ assertEquals(name + " actual P4 module stays synchronized: " + module.getKey(),
+ Files.readString(Path.of("tools/tinyexpression-p4-lsp-vscode/grammar/lexical", module.getKey())),
+ module.getValue().getAsString());
+ Path fixtureDir = temporary.getRoot().toPath().resolve("fixture-" + fixtureIndex++);
+ Files.createDirectories(fixtureDir);
+ Path ubnf = fixtureDir.resolve("root.ubnf");
+ Files.writeString(ubnf, source);
+ if (fixture.has("modules")) for (var entry : fixture.getAsJsonObject("modules").entrySet())
+ Files.writeString(fixtureDir.resolve(entry.getKey()), entry.getValue().getAsString());
+ if (fixture.has("manifest")) {
+ Path manifest = fixtureDir.resolve("ubnf.json");
+ Files.writeString(manifest, fixture.get("manifest").toString());
+ org.unlaxer.dsl.bootstrap.UBNFPackageResolver.resolve(manifest);
+ String javaLock = Files.readString(fixtureDir.resolve("ubnf.lock.json"));
+ success(run(List.of(nativeGenerator.toString(), "deps", "resolve", "--manifest", manifest.toString()), "", true));
+ assertEquals(name + " exact package identity/hash lock parity", javaLock,
+ Files.readString(fixtureDir.resolve("ubnf.lock.json")));
+ }
+ GrammarDecl grammar = org.unlaxer.dsl.bootstrap.UBNFModuleLoader.load(ubnf).grammars().get(0);
+ GrammarValidator.validateOrThrow(grammar);
+ List javaFrontend = new RustBackend().generate(grammar);
+ assertEquals(name + " Java frontend Rust file count", 5, javaFrontend.size());
+
+ Path nativeGenerated = fixtureDir.resolve("native-generated");
+ success(run(List.of(nativeGenerator.toString(), "generate", "--grammar", ubnf.toString(),
+ "--output", nativeGenerated.toString()), "", true));
+ for (var file : javaFrontend) {
+ assertArrayEquals(name + " native frontend byte parity: " + file.relativePath(),
+ file.content().getBytes(StandardCharsets.UTF_8),
+ Files.readAllBytes(nativeGenerated.resolve(file.relativePath())));
+ }
+
+ Path generated = Files.createDirectory(fixtureDir.resolve("generated"));
+ for (var file : javaFrontend) {
+ Files.writeString(generated.resolve(file.relativePath()), file.content());
+ }
+ Files.writeString(fixtureDir.resolve("main.rs"), rustProbe());
+ success(run(List.of("rustc", "--edition=2021", "--extern",
+ "unlaxer_runtime=" + runtime, fixtureDir.resolve("main.rs").toString(), "-o",
+ fixtureDir.resolve("probe").toString()), "", false));
+
+ JsonArray cases = fixture.getAsJsonArray("cases");
+ String framed = String.join("\n", cases.asList().stream()
+ .map(row -> HexFormat.of().formatHex(row.getAsJsonObject().get("input").getAsString()
+ .getBytes(StandardCharsets.UTF_8))).toList()) + "\n";
+ ProcessResult rustResult = run(List.of(fixtureDir.resolve("probe").toString()), framed, false);
+ success(rustResult);
+ List rustLines = rustResult.output().lines().toList();
+ assertEquals(name + " Rust result count", cases.size(), rustLines.size());
+
+ try (URLClassLoader loader = compileJava(grammar)) {
+ Class parsers = loader.loadClass(PACKAGE + "." + grammar.name() + "Parsers");
+ Parser parser = (Parser) parsers.getMethod("getRootParser").invoke(null);
+ Class mapper = loader.loadClass(PACKAGE + "." + grammar.name() + "Mapper");
+ for (int i = 0; i < cases.size(); i++) {
+ JsonObject row = cases.get(i).getAsJsonObject();
+ String input = row.get("input").getAsString();
+ String context = name + "/" + row.get("id").getAsString();
+ JsonObject rust = JsonParser.parseString(rustLines.get(i)).getAsJsonObject();
+
+ JsonArray javaPrefix = prefix(parser, input);
+ assertEquals(context + " independent prefix cursor oracle", row.get("prefix"), javaPrefix);
+ assertEquals(context + " Java/Rust prefix cursor parity", javaPrefix, rust.get("prefix"));
+
+ boolean accepted = row.get("accepted").getAsBoolean();
+ Optional diagnostic =
+ (Optional) mapper.getMethod("diagnose", String.class).invoke(null, input);
+ assertEquals(context + " Java full-input acceptance", accepted, diagnostic.isEmpty());
+ assertEquals(context + " Rust full-input acceptance", accepted,
+ !rust.get("ast").isJsonNull());
+
+ if (row.has("diagnostic")) {
+ Object failure = diagnostic.orElseThrow();
+ JsonArray javaDiagnostic = new JsonArray();
+ javaDiagnostic.add((String) failure.getClass().getMethod("kind").invoke(failure));
+ javaDiagnostic.add((Integer) failure.getClass().getMethod("offset").invoke(failure));
+ assertEquals(context + " independent diagnostic category/location", row.get("diagnostic"), javaDiagnostic);
+ assertEquals(context + " Java/Rust diagnostic category/location", javaDiagnostic, rust.get("diagnostic"));
+ }
+ JsonElement expected = JsonNull.INSTANCE;
+ JsonElement javaAst = JsonNull.INSTANCE;
+ if (accepted) {
+ expected = row.get("ast");
+ Object mapped = mapper.getMethod("parseWithSourceMap", String.class).invoke(null, input);
+ Object ast = mapped.getClass().getMethod("ast").invoke(mapped);
+ javaAst = canonical(ast, mapped);
+ assertEquals(context + " Java AST all fields/node spans", expected, javaAst);
+ assertEquals(context + " Rust AST all fields/node spans", expected, rust.get("ast"));
+ }
+ report.add(name + "\t" + row.get("id").getAsString() + "\t" + row.get("input")
+ + "\t" + row.get("prefix") + "\t" + javaPrefix + "\t" + expected + "\t"
+ + javaAst + "\t" + rust);
+ }
+ }
+ }
+
+ Path target = Path.of("target");
+ Files.createDirectories(target);
+ Files.write(target.resolve("p4-layout-generated.tsv"), report, StandardCharsets.UTF_8);
+ }
+
+ private String rustProbe() {
+ return """
+ mod generated;
+ use std::io::{self, BufRead};
+ fn main() {
+ for line in io::stdin().lock().lines() {
+ let encoded = line.unwrap();
+ let bytes = (0..encoded.len()).step_by(2)
+ .map(|i| u8::from_str_radix(&encoded[i..i + 2], 16).unwrap()).collect();
+ let input = String::from_utf8(bytes).unwrap();
+ let mut context = unlaxer_runtime::ParseContext::new(&input);
+ let prefix_ok = generated::parser::parse_context(&mut context).is_ok();
+ let options = unlaxer_runtime::ParseOptions::with_memoization(unlaxer_runtime::Memoization::SafeFailures);
+ let mut memo_context = unlaxer_runtime::ParseContext::with_options(&input, options);
+ let memo_ok = generated::parser::parse_context(&mut memo_context).is_ok();
+ assert_eq!((prefix_ok, context.position(), context.matched_position()),
+ (memo_ok, memo_context.position(), memo_context.matched_position()));
+ print!(r#"{{\"prefix\":[{},{},{}],\"ast\":"#,
+ prefix_ok, context.position(), context.matched_position());
+ match generated::parser::parse_tree_detailed(&input) {
+ Ok(tree) => {
+ let ast = generated::mapper::map(&tree).unwrap();
+ drop(tree);
+ let canonical = ast.canonical_json();
+ let memo_tree = generated::parser::parse_tree_detailed_with_options(&input, options).unwrap();
+ assert_eq!(canonical, generated::mapper::map(&memo_tree).unwrap().canonical_json());
+ print!("{}", canonical);
+ }
+ Err(error) => {
+ assert_eq!(error, generated::parser::parse_tree_detailed_with_options(&input, options).unwrap_err());
+ print!(r#"null,\"diagnostic\":[\"{}\",{}]"#, error.kind, error.offset);
+ }
+ }
+ println!("}}");
+ }
+ }
+ """;
+ }
+
+ private record ProcessResult(int code, String output) {}
+
+ private ProcessResult run(List command, String input, boolean withoutJava) throws Exception {
+ Path log = temporary.newFile().toPath();
+ var builder = new ProcessBuilder(command).redirectErrorStream(true).redirectOutput(log.toFile());
+ if (withoutJava) {
+ builder.environment().put("PATH", "");
+ builder.environment().put("JAVA_HOME", "/nonexistent-unlaxer-java");
+ }
+ Process process = builder.start();
+ try {
+ try (var stdin = process.getOutputStream()) {
+ stdin.write(input.getBytes(StandardCharsets.UTF_8));
+ }
+ if (!process.waitFor(180, TimeUnit.SECONDS)) {
+ terminateProcessTree(process);
+ fail("process timed out: " + command);
+ }
+ return new ProcessResult(process.exitValue(), Files.readString(log));
+ } finally {
+ if (process.isAlive()) terminateProcessTree(process);
+ }
+ }
+
+ private void terminateProcessTree(Process process) throws InterruptedException {
+ List descendants = process.descendants().toList();
+ for (int i = descendants.size() - 1; i >= 0; i--) descendants.get(i).destroy();
+ process.destroy();
+ process.waitFor(2, TimeUnit.SECONDS);
+ for (int i = descendants.size() - 1; i >= 0; i--) {
+ ProcessHandle descendant = descendants.get(i);
+ if (descendant.isAlive()) descendant.destroyForcibly();
+ }
+ if (process.isAlive()) {
+ process.destroyForcibly();
+ process.waitFor(2, TimeUnit.SECONDS);
+ }
+ }
+
+ private void success(ProcessResult result) {
+ assertEquals(result.output(), 0, result.code());
+ }
+}
diff --git a/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutMigrationTest.java b/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutMigrationTest.java
new file mode 100644
index 00000000..4faecaeb
--- /dev/null
+++ b/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutMigrationTest.java
@@ -0,0 +1,87 @@
+package org.unlaxer.tinyexpression.p4;
+
+import static org.junit.Assert.*;
+
+import com.google.gson.JsonParser;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import org.junit.Test;
+import org.unlaxer.dsl.bootstrap.UBNFModuleLoader;
+import org.unlaxer.parser.Parser;
+import org.unlaxer.tinyexpression.generated.p4.TinyExpressionP4Mapper;
+import org.unlaxer.tinyexpression.generated.p4.TinyExpressionP4Parsers;
+import org.unlaxer.tinyexpression.p4.ubnfc.generated.TinyExpressionP4Parser;
+import org.unlaxer.tinyexpression.p4.ubnfc.generated.api.ParseOptions;
+
+/** Real P4 scopes and retained text compared with the old policy and production ubnfc. */
+public class P4LayoutMigrationTest extends P4LayoutTestSupport {
+ @Test public void authoringSnapshotShowsTheExactDefinitionAndPinnedOrigin() throws Exception {
+ var snapshot = org.unlaxer.dsl.tooling.VocabularyOrigins.inspect(Path.of(
+ "tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf"));
+ var layout = snapshot.getAsJsonArray("modules").asList().stream()
+ .map(value -> value.getAsJsonObject()).filter(value -> value.get("alias").getAsString().equals("layout"))
+ .findFirst().orElseThrow();
+ var identity = layout.getAsJsonObject("identity");
+ assertEquals("std/layout", identity.get("id").getAsString());
+ assertEquals("1.0.0", identity.get("version").getAsString());
+ assertEquals("2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67",
+ identity.get("sha256").getAsString());
+ assertEquals("layout.ubnf", identity.get("file").getAsString());
+ var definition = layout.getAsJsonArray("definitions").asList().stream()
+ .map(value -> value.getAsJsonObject())
+ .filter(value -> value.get("name").getAsString().equals("SPACES_AND_COMMENTS"))
+ .findFirst().orElseThrow();
+ String text = layout.get("text").getAsString();
+ assertEquals("token SPACES_AND_COMMENTS ::= SPACES | LINE_COMMENT | BLOCK_COMMENT;",
+ text.substring(text.offsetByCodePoints(0, definition.get("start").getAsInt()),
+ text.offsetByCodePoints(0, definition.get("end").getAsInt())));
+ for (var value : snapshot.getAsJsonArray("whitespace"))
+ assertEquals("layout.SPACES_AND_COMMENTS", value.getAsJsonObject().get("policy").getAsString());
+ }
+
+ @Test public void realP4KeepsPolicyScopesCursorsAstAndOwnedSourcePositions() throws Exception {
+ Path old = temporary.newFolder("old-layout").toPath().resolve("tinyexpression-p4.ubnf");
+ Path source = Path.of("tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf");
+ Process adapter = new ProcessBuilder("python3", "scripts/prepare-ubnfc-layout.py",
+ source.toString(), old.toString()).inheritIO().start();
+ assertEquals(0, adapter.waitFor());
+ Files.writeString(old, Files.readString(old).replace(
+ "@package: org.unlaxer.tinyexpression.generated.p4", "@package: org.example.ruletrivia"));
+ var grammar = UBNFModuleLoader.load(old).grammars().get(0);
+ var rows = JsonParser.parseString(Files.readString(Path.of(
+ "src/test/resources/p4-layout/actual-p4.json"))).getAsJsonArray();
+ try (var loader = compileJava(grammar)) {
+ Class oldParsers = loader.loadClass("org.example.ruletrivia.TinyExpressionP4Parsers");
+ Parser oldRoot = (Parser) oldParsers.getMethod("getRootParser").invoke(null);
+ Class oldMapper = loader.loadClass("org.example.ruletrivia.TinyExpressionP4Mapper");
+ for (var element : rows) {
+ var row = element.getAsJsonObject();
+ String id = row.get("id").getAsString(), input = row.get("input").getAsString();
+ var oracle = row.get("prefix");
+ assertEquals(id + " old independent cursor oracle", oracle, prefix(oldRoot, input));
+ assertEquals(id + " new independent cursor oracle", oracle,
+ prefix(TinyExpressionP4Parsers.getRootParser(), input));
+ var production = TinyExpressionP4Parser.parseEntry("TinyExpressionP4", "Formula", input,
+ new ParseOptions(true, false, false, true, Map.of()));
+ assertEquals(id + " production ubnfc acceptance", row.get("ok").getAsBoolean(), production.ok());
+ assertEquals(id + " production consumed", oracle.getAsJsonArray().get(1).getAsInt(), production.consumedCp());
+ assertEquals(id + " production matched", oracle.getAsJsonArray().get(2).getAsInt(), production.matchedCp());
+ if (row.get("ok").getAsBoolean()) {
+ Object oldMapped = oldMapper.getMethod("parseWithSourceMap", String.class).invoke(null, input);
+ Object oldAst = oldMapped.getClass().getMethod("ast").invoke(oldMapped);
+ var mapped = TinyExpressionP4Mapper.parseWithSourceMap(input);
+ assertEquals(id + " every AST field and node span", canonical(oldAst, oldMapped),
+ canonical(mapped.ast(), mapped));
+ var owned = P4PreferredAstMapper.parseByAstSimpleNamesDetailed(input,
+ java.util.List.of("FormulaExpr"), 0L);
+ assertEquals(id + " source-preserving compatibility mask",
+ org.unlaxer.tinyexpression.parser.TinyExpressionParserCapabilities.stripJavaStyleCommentsPreservingLayout(input),
+ owned.sourceText().text(owned.ast()));
+ assertArrayEquals(id + " source-preserving position", new int[]{0, input.codePointCount(0, input.length())},
+ owned.sourceText().spanOf(owned.ast()).orElseThrow());
+ }
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutTestSupport.java b/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutTestSupport.java
new file mode 100644
index 00000000..71f5d8cf
--- /dev/null
+++ b/src/test/java/org/unlaxer/tinyexpression/p4/P4LayoutTestSupport.java
@@ -0,0 +1,123 @@
+package org.unlaxer.tinyexpression.p4;
+
+import org.unlaxer.dsl.codegen.*;
+
+import static org.junit.Assert.*;
+import static org.junit.Assume.assumeTrue;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import javax.tools.DiagnosticCollector;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.ToolProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.unlaxer.context.ParseContext;
+import org.unlaxer.dsl.bootstrap.UBNFAST.GrammarDecl;
+import org.unlaxer.dsl.bootstrap.UBNFMapper;
+import org.unlaxer.dsl.codegen.CodeGenerator.GeneratedSource;
+import org.unlaxer.dsl.codegen.rust.RustBackend;
+import org.unlaxer.parser.Parser;
+
+abstract class P4LayoutTestSupport {
+ @Rule public TemporaryFolder temporary = new TemporaryFolder();
+ protected URLClassLoader compileJava(GrammarDecl grammar) throws Exception {
+ var sources = new ArrayList();
+ for (CodeGenerator generator : List.of(
+ new ParserGenerator(), new ASTGenerator(), new MapperGenerator())) {
+ sources.add(generator.generate(grammar));
+ }
+ var units = sources.stream().map(source -> new SimpleJavaFileObject(
+ URI.create("string:///" + source.packageName().replace('.', '/') + "/"
+ + source.className() + ".java"), JavaFileObject.Kind.SOURCE) {
+ @Override public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+ return source.source();
+ }
+ }).toList();
+ var compiler = ToolProvider.getSystemJavaCompiler();
+ assertNotNull("tests require a JDK, not a JRE", compiler);
+ var diagnostics = new DiagnosticCollector();
+ Path output = temporary.newFolder().toPath();
+ try (var manager = compiler.getStandardFileManager(diagnostics, Locale.ROOT,
+ StandardCharsets.UTF_8)) {
+ boolean compiled = compiler.getTask(null, manager, diagnostics,
+ List.of("--release", "17", "-sourcepath", "", "-implicit:none", "-classpath",
+ System.getProperty("java.class.path"), "-d", output.toString()),
+ null, units).call();
+ assertTrue(diagnostics.getDiagnostics().toString(), compiled);
+ }
+ return new URLClassLoader(new URL[]{output.toUri().toURL()}, getClass().getClassLoader());
+ }
+
+ protected JsonArray prefix(Parser parser, String input) throws Exception {
+ JsonArray expected = null;
+ for (boolean memo : new boolean[]{false, true}) {
+ var result = new JsonArray();
+ try (var context = new ParseContext(org.unlaxer.StringSource.createRootSource(input))) {
+ if (memo) context.enableMemoize();
+ var parsed = parser.parse(context);
+ result.add(parsed.isSucceeded());
+ if (parsed.isSucceeded()) {
+ int end = context.getConsumedPosition().value();
+ assertEquals("retained CST source at consumed code-point position",
+ input.substring(0, input.offsetByCodePoints(0, end)),
+ parsed.getConsumed().source.sourceAsString());
+ }
+ result.add(context.getConsumedPosition().value());
+ result.add(context.getMatchedPosition().value());
+ }
+ if (expected == null) expected = result;
+ else assertEquals("Java memo OFF/ON prefix parity: " + input, expected, result);
+ }
+ return expected;
+ }
+
+ protected JsonObject canonical(Object ast, Object mapped) throws Exception {
+ var result = new JsonObject();
+ result.addProperty("type", ast.getClass().getSimpleName());
+ int[] span = (int[]) ((Optional) mapped.getClass().getMethod("sourceSpanOf", Object.class)
+ .invoke(mapped, ast)).orElseThrow();
+ var position = new JsonArray();
+ position.add(span[0]);
+ position.add(span[1]);
+ result.add("span", position);
+ var fields = new JsonObject();
+ for (var component : ast.getClass().getRecordComponents()) {
+ fields.add(component.getName(), canonicalValue(component.getAccessor().invoke(ast), mapped));
+ }
+ result.add("fields", fields);
+ return result;
+ }
+
+ private JsonElement canonicalValue(Object value, Object mapped) throws Exception {
+ if (value == null) return JsonNull.INSTANCE;
+ if (value instanceof String text) return new JsonPrimitive(text);
+ if (value instanceof Optional optional) return canonicalValue(optional.orElse(null), mapped);
+ if (value instanceof List list) {
+ var result = new JsonArray();
+ for (Object item : list) result.add(canonicalValue(item, mapped));
+ return result;
+ }
+ return canonical(value, mapped);
+ }
+
+}
diff --git a/src/test/resources/p4-layout/actual-p4.json b/src/test/resources/p4-layout/actual-p4.json
new file mode 100644
index 00000000..58fa2b3a
--- /dev/null
+++ b/src/test/resources/p4-layout/actual-p4.json
@@ -0,0 +1,156 @@
+[
+ {
+ "id": "plain",
+ "input": "1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 1,
+ 1
+ ]
+ },
+ {
+ "id": "spaces",
+ "input": " 1 ",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 3,
+ 3
+ ]
+ },
+ {
+ "id": "tab",
+ "input": "\t1\t",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 3,
+ 3
+ ]
+ },
+ {
+ "id": "lf",
+ "input": "\n1\n",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 3,
+ 3
+ ]
+ },
+ {
+ "id": "crlf",
+ "input": "\r\n1\r\n",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 5,
+ 5
+ ]
+ },
+ {
+ "id": "line-lf",
+ "input": "//😀\n1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 5,
+ 5
+ ]
+ },
+ {
+ "id": "line-crlf",
+ "input": "//😀\r\n1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 6,
+ 6
+ ]
+ },
+ {
+ "id": "block",
+ "input": "/*😀*/1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 6,
+ 6
+ ]
+ },
+ {
+ "id": "string",
+ "input": "' /* // 😀 */ '",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 14,
+ 14
+ ]
+ },
+ {
+ "id": "code",
+ "input": "````java:X\n//😀\n/*x*/\n````\n1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 27,
+ 27
+ ]
+ },
+ {
+ "id": "import",
+ "input": "import a.B as b;\n1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 18,
+ 18
+ ]
+ },
+ {
+ "id": "declaration",
+ "input": "var $n as number;\n1",
+ "entry": "Formula",
+ "ok": true,
+ "prefix": [
+ true,
+ 19,
+ 19
+ ]
+ },
+ {
+ "id": "unclosed-block",
+ "input": "/*unclosed",
+ "entry": "Formula",
+ "ok": false,
+ "prefix": [
+ false,
+ 0,
+ 0
+ ]
+ },
+ {
+ "id": "unclosed-string",
+ "input": "'/*x",
+ "entry": "Formula",
+ "ok": false,
+ "prefix": [
+ false,
+ 0,
+ 0
+ ]
+ }
+]
diff --git a/src/test/resources/p4-layout/corpus.json b/src/test/resources/p4-layout/corpus.json
new file mode 100644
index 00000000..ede2792a
--- /dev/null
+++ b/src/test/resources/p4-layout/corpus.json
@@ -0,0 +1,615 @@
+[
+ {
+ "name": "p4-standard-layout",
+ "grammar": "grammar LayoutProbe { @import layout from 'pkg:std/layout' @import strings from 'strings.ubnf' @import fences from 'fences.ubnf' @ubnf: v2 @package: org.example.ruletrivia @whitespace: layout.SPACES_AND_COMMENTS token VALUE ::= '1' | strings.STRING | fences.LONG_CODE_BLOCK; @root @mapping(Root,params=[value]) @whitespace(layout.SPACES_AND_COMMENTS) Root ::= VALUE @value; }",
+ "modules": {
+ "identifiers.ubnf": "grammar Identifiers {\n @import chars from 'characters.ubnf'\n @ubnf: v2\n token IDENTIFIER ::= chars.LETTER (chars.LETTER | chars.DIGIT)*;\n token CLASS_NAME ::= IDENTIFIER ('.' IDENTIFIER)*;\n}\n",
+ "numbers.ubnf": "grammar Numbers {\n @import chars from 'characters.ubnf'\n @ubnf: v2\n // Optional sign, integer/fraction, optional complete exponent. No implicit trivia.\n token NUMBER ::= ['+' | '-']\n (chars.DIGIT+ '.' chars.DIGIT* | '.' chars.DIGIT+ | chars.DIGIT+)\n [('e' | 'E') ['+' | '-'] chars.DIGIT+];\n}\n",
+ "fences.ubnf": "grammar Fences {\n @import chars from 'characters.ubnf'\n @import names from 'identifiers.ubnf'\n @ubnf: v2\n // The legacy three-backtick form ends at its first triple-backtick sequence.\n token CODE_START ::= BOL '```' names.IDENTIFIER ':' names.CLASS_NAME (chars.NEWLINE | EOF);\n token CODE_BODY ::= { NEGATIVE_LOOKAHEAD('```') ANY } LOOKAHEAD('```');\n token CODE_END ::= BOL '```' (chars.NEWLINE | EOF);\n // Four or more backticks: closing width must equal the captured opening width.\n // A shorter/longer fence or a non-standalone line remains opaque body text.\n token LONG_CODE_BLOCK ::= BOL CAPTURE(fence, '`'{4,}) names.IDENTIFIER ':' names.CLASS_NAME chars.NEWLINE\n { NEGATIVE_LOOKAHEAD(BOL SAME_AS(fence) EOL) ANY }\n BOL SAME_AS(fence) (chars.NEWLINE | EOF);\n}\n",
+ "strings.ubnf": "grammar Strings {\n @ubnf: v2\n // Quotes and escape pairs are raw source, including newlines and non-BMP scalars.\n // AST normalization belongs to the consumer, not this recognizer.\n token STRING ::= '\"' { '\\\\' ANY | NEGATION('\"\\\\') } '\"'\n | '\\'' { '\\\\' ANY | NEGATION('\\'\\\\') } '\\'';\n}\n",
+ "characters.ubnf": "grammar Characters {\n @ubnf: v2\n token DIGIT ::= CHAR_RANGE('0', '9');\n token LETTER ::= CHAR_RANGE('a', 'z') | CHAR_RANGE('A', 'Z') | '_';\n token NEWLINE ::= '\\r\\n' | '\\r' | '\\n';\n}\n"
+ },
+ "manifest": {
+ "schemaVersion": 1,
+ "dependencies": {
+ "std/layout": {
+ "version": "1.0.0",
+ "source": "builtin:std/layout@1.0.0"
+ }
+ }
+ },
+ "cases": [
+ {
+ "id": "plain",
+ "input": "1",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 1
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "spaces",
+ "input": " 1 ",
+ "prefix": [
+ true,
+ 3,
+ 3
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 3
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "tab",
+ "input": "\t1\t",
+ "prefix": [
+ true,
+ 3,
+ 3
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 3
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "lf",
+ "input": "\n1\n",
+ "prefix": [
+ true,
+ 3,
+ 3
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 3
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "crlf",
+ "input": "\r\n1\r\n",
+ "prefix": [
+ true,
+ 5,
+ 5
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 5
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "line-lf",
+ "input": "//😀\n1",
+ "prefix": [
+ true,
+ 5,
+ 5
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 5
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "line-crlf",
+ "input": "//😀\r\n1",
+ "prefix": [
+ true,
+ 6,
+ 6
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 6
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "block",
+ "input": "/*😀*/1",
+ "prefix": [
+ true,
+ 6,
+ 6
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 6
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "first-block-end",
+ "input": "/*x*/1",
+ "prefix": [
+ true,
+ 6,
+ 6
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 6
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "string",
+ "input": "' /* // 😀 */ '",
+ "prefix": [
+ true,
+ 14,
+ 14
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 14
+ ],
+ "fields": {
+ "value": " /* // 😀 */ "
+ }
+ }
+ },
+ {
+ "id": "long-code",
+ "input": "````java:X\n//😀\n/*x*/\n````",
+ "prefix": [
+ true,
+ 25,
+ 25
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 25
+ ],
+ "fields": {
+ "value": "````java:X\n//😀\n/*x*/\n````"
+ }
+ }
+ },
+ {
+ "id": "unclosed-block",
+ "input": "1/*x",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": false,
+ "diagnostic": [
+ "trailing_input",
+ 1
+ ]
+ }
+ ]
+ },
+ {
+ "name": "p4-java-style-baseline",
+ "grammar": "grammar LayoutProbe { @import strings from 'strings.ubnf' @import fences from 'fences.ubnf' @ubnf: v2 @package: org.example.ruletrivia @whitespace: javaStyle token VALUE ::= '1' | strings.STRING | fences.LONG_CODE_BLOCK; @root @mapping(Root,params=[value]) @whitespace(javaStyle) Root ::= VALUE @value; }",
+ "modules": {
+ "identifiers.ubnf": "grammar Identifiers {\n @import chars from 'characters.ubnf'\n @ubnf: v2\n token IDENTIFIER ::= chars.LETTER (chars.LETTER | chars.DIGIT)*;\n token CLASS_NAME ::= IDENTIFIER ('.' IDENTIFIER)*;\n}\n",
+ "numbers.ubnf": "grammar Numbers {\n @import chars from 'characters.ubnf'\n @ubnf: v2\n // Optional sign, integer/fraction, optional complete exponent. No implicit trivia.\n token NUMBER ::= ['+' | '-']\n (chars.DIGIT+ '.' chars.DIGIT* | '.' chars.DIGIT+ | chars.DIGIT+)\n [('e' | 'E') ['+' | '-'] chars.DIGIT+];\n}\n",
+ "fences.ubnf": "grammar Fences {\n @import chars from 'characters.ubnf'\n @import names from 'identifiers.ubnf'\n @ubnf: v2\n // The legacy three-backtick form ends at its first triple-backtick sequence.\n token CODE_START ::= BOL '```' names.IDENTIFIER ':' names.CLASS_NAME (chars.NEWLINE | EOF);\n token CODE_BODY ::= { NEGATIVE_LOOKAHEAD('```') ANY } LOOKAHEAD('```');\n token CODE_END ::= BOL '```' (chars.NEWLINE | EOF);\n // Four or more backticks: closing width must equal the captured opening width.\n // A shorter/longer fence or a non-standalone line remains opaque body text.\n token LONG_CODE_BLOCK ::= BOL CAPTURE(fence, '`'{4,}) names.IDENTIFIER ':' names.CLASS_NAME chars.NEWLINE\n { NEGATIVE_LOOKAHEAD(BOL SAME_AS(fence) EOL) ANY }\n BOL SAME_AS(fence) (chars.NEWLINE | EOF);\n}\n",
+ "strings.ubnf": "grammar Strings {\n @ubnf: v2\n // Quotes and escape pairs are raw source, including newlines and non-BMP scalars.\n // AST normalization belongs to the consumer, not this recognizer.\n token STRING ::= '\"' { '\\\\' ANY | NEGATION('\"\\\\') } '\"'\n | '\\'' { '\\\\' ANY | NEGATION('\\'\\\\') } '\\'';\n}\n",
+ "characters.ubnf": "grammar Characters {\n @ubnf: v2\n token DIGIT ::= CHAR_RANGE('0', '9');\n token LETTER ::= CHAR_RANGE('a', 'z') | CHAR_RANGE('A', 'Z') | '_';\n token NEWLINE ::= '\\r\\n' | '\\r' | '\\n';\n}\n"
+ },
+ "cases": [
+ {
+ "id": "plain",
+ "input": "1",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 1
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "spaces",
+ "input": " 1 ",
+ "prefix": [
+ true,
+ 3,
+ 3
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 3
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "tab",
+ "input": "\t1\t",
+ "prefix": [
+ true,
+ 3,
+ 3
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 3
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "lf",
+ "input": "\n1\n",
+ "prefix": [
+ true,
+ 3,
+ 3
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 3
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "crlf",
+ "input": "\r\n1\r\n",
+ "prefix": [
+ true,
+ 5,
+ 5
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 5
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "line-lf",
+ "input": "//😀\n1",
+ "prefix": [
+ true,
+ 5,
+ 5
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 5
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "line-crlf",
+ "input": "//😀\r\n1",
+ "prefix": [
+ true,
+ 6,
+ 6
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 6
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "block",
+ "input": "/*😀*/1",
+ "prefix": [
+ true,
+ 6,
+ 6
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 6
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "first-block-end",
+ "input": "/*x*/1",
+ "prefix": [
+ true,
+ 6,
+ 6
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 6
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "string",
+ "input": "' /* // 😀 */ '",
+ "prefix": [
+ true,
+ 14,
+ 14
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 14
+ ],
+ "fields": {
+ "value": " /* // 😀 */ "
+ }
+ }
+ },
+ {
+ "id": "long-code",
+ "input": "````java:X\n//😀\n/*x*/\n````",
+ "prefix": [
+ true,
+ 25,
+ 25
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 25
+ ],
+ "fields": {
+ "value": "````java:X\n//😀\n/*x*/\n````"
+ }
+ }
+ },
+ {
+ "id": "unclosed-block",
+ "input": "1/*x",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": false,
+ "diagnostic": [
+ "trailing_input",
+ 1
+ ]
+ }
+ ]
+ },
+ {
+ "name": "p4-explicit-none",
+ "grammar": "grammar LayoutProbe { @import layout from 'pkg:std/layout' @import strings from 'strings.ubnf' @import fences from 'fences.ubnf' @ubnf: v2 @package: org.example.ruletrivia @whitespace: layout.SPACES_AND_COMMENTS token VALUE ::= '1' | strings.STRING | fences.LONG_CODE_BLOCK; @root @mapping(Root,params=[value]) @whitespace(none) Root ::= VALUE @value; }",
+ "modules": {
+ "identifiers.ubnf": "grammar Identifiers {\n @import chars from 'characters.ubnf'\n @ubnf: v2\n token IDENTIFIER ::= chars.LETTER (chars.LETTER | chars.DIGIT)*;\n token CLASS_NAME ::= IDENTIFIER ('.' IDENTIFIER)*;\n}\n",
+ "numbers.ubnf": "grammar Numbers {\n @import chars from 'characters.ubnf'\n @ubnf: v2\n // Optional sign, integer/fraction, optional complete exponent. No implicit trivia.\n token NUMBER ::= ['+' | '-']\n (chars.DIGIT+ '.' chars.DIGIT* | '.' chars.DIGIT+ | chars.DIGIT+)\n [('e' | 'E') ['+' | '-'] chars.DIGIT+];\n}\n",
+ "fences.ubnf": "grammar Fences {\n @import chars from 'characters.ubnf'\n @import names from 'identifiers.ubnf'\n @ubnf: v2\n // The legacy three-backtick form ends at its first triple-backtick sequence.\n token CODE_START ::= BOL '```' names.IDENTIFIER ':' names.CLASS_NAME (chars.NEWLINE | EOF);\n token CODE_BODY ::= { NEGATIVE_LOOKAHEAD('```') ANY } LOOKAHEAD('```');\n token CODE_END ::= BOL '```' (chars.NEWLINE | EOF);\n // Four or more backticks: closing width must equal the captured opening width.\n // A shorter/longer fence or a non-standalone line remains opaque body text.\n token LONG_CODE_BLOCK ::= BOL CAPTURE(fence, '`'{4,}) names.IDENTIFIER ':' names.CLASS_NAME chars.NEWLINE\n { NEGATIVE_LOOKAHEAD(BOL SAME_AS(fence) EOL) ANY }\n BOL SAME_AS(fence) (chars.NEWLINE | EOF);\n}\n",
+ "strings.ubnf": "grammar Strings {\n @ubnf: v2\n // Quotes and escape pairs are raw source, including newlines and non-BMP scalars.\n // AST normalization belongs to the consumer, not this recognizer.\n token STRING ::= '\"' { '\\\\' ANY | NEGATION('\"\\\\') } '\"'\n | '\\'' { '\\\\' ANY | NEGATION('\\'\\\\') } '\\'';\n}\n",
+ "characters.ubnf": "grammar Characters {\n @ubnf: v2\n token DIGIT ::= CHAR_RANGE('0', '9');\n token LETTER ::= CHAR_RANGE('a', 'z') | CHAR_RANGE('A', 'Z') | '_';\n token NEWLINE ::= '\\r\\n' | '\\r' | '\\n';\n}\n"
+ },
+ "manifest": {
+ "schemaVersion": 1,
+ "dependencies": {
+ "std/layout": {
+ "version": "1.0.0",
+ "source": "builtin:std/layout@1.0.0"
+ }
+ }
+ },
+ "cases": [
+ {
+ "id": "plain",
+ "input": "1",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 1
+ ],
+ "fields": {
+ "value": "1"
+ }
+ }
+ },
+ {
+ "id": "string",
+ "input": "' /* // 😀 */ '",
+ "prefix": [
+ true,
+ 14,
+ 14
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 14
+ ],
+ "fields": {
+ "value": " /* // 😀 */ "
+ }
+ }
+ },
+ {
+ "id": "long-code",
+ "input": "````java:X\n//😀\n/*x*/\n````",
+ "prefix": [
+ true,
+ 25,
+ 25
+ ],
+ "accepted": true,
+ "ast": {
+ "type": "Root",
+ "span": [
+ 0,
+ 25
+ ],
+ "fields": {
+ "value": "````java:X\n//😀\n/*x*/\n````"
+ }
+ }
+ },
+ {
+ "id": "none-does-not-consume-space",
+ "input": "1 ",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": false,
+ "diagnostic": [
+ "trailing_input",
+ 1
+ ]
+ },
+ {
+ "id": "none-does-not-consume-comment",
+ "input": "1/*x*/",
+ "prefix": [
+ true,
+ 1,
+ 1
+ ],
+ "accepted": false,
+ "diagnostic": [
+ "trailing_input",
+ 1
+ ]
+ }
+ ]
+ }
+]
diff --git a/tools/tinyexpression-p4-lsp-vscode/grammar/.ubnf-cache/packages/2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67.json b/tools/tinyexpression-p4-lsp-vscode/grammar/.ubnf-cache/packages/2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67.json
new file mode 100644
index 00000000..1ea437af
--- /dev/null
+++ b/tools/tinyexpression-p4-lsp-vscode/grammar/.ubnf-cache/packages/2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67.json
@@ -0,0 +1,10 @@
+{
+ "schemaVersion": 1,
+ "id": "std/layout",
+ "version": "1.0.0",
+ "entry": "layout.ubnf",
+ "files": {
+ "layout.ubnf": "grammar StandardLayout {\n @ubnf: v2\n token SPACES ::= (' ' | CHAR_RANGE('\\t', '\\r'))+;\n token LINE_COMMENT ::= '//' NEGATION('\\r\\n')*;\n token BLOCK_COMMENT ::= '/*' (NEGATIVE_LOOKAHEAD('*/') ANY)* '*/';\n token SPACES_AND_COMMENTS ::= SPACES | LINE_COMMENT | BLOCK_COMMENT;\n}\n"
+ },
+ "dependencies": {}
+}
diff --git a/tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf b/tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
index 0aa89e9e..03515833 100644
--- a/tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+++ b/tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
@@ -1,5 +1,7 @@
grammar TinyExpressionP4 {
+ @import layout from 'pkg:std/layout'
+
@import numbers from 'lexical/numbers.ubnf'
@import names from 'lexical/identifiers.ubnf'
@import strings from 'lexical/strings.ubnf'
@@ -9,7 +11,7 @@ grammar TinyExpressionP4 {
@ubnf: v2
@feature: declarativeTokensV1
@package: org.unlaxer.tinyexpression.generated.p4
- @whitespace: javaStyle
+ @whitespace: layout.SPACES_AND_COMMENTS
// Preserve P4's existing literal matching while enabling declarative token syntax.
@literalBoundary: none
@@ -23,7 +25,7 @@ grammar TinyExpressionP4 {
token EOF ::= EOF;
@root
- @interleave(profile=javaStyle)
+ @whitespace(layout.SPACES_AND_COMMENTS)
@scopeTree(mode=lexical)
@mapping(FormulaExpr, params=[imports, declarations, expression, methods, codeBlocks])
Formula ::= { CodeBlock @codeBlocks } { ImportDeclaration @imports } { VariableDeclaration @declarations }
@@ -44,7 +46,7 @@ grammar TinyExpressionP4 {
// import sample.v1.CheckAlphabets#check as checkAlphabets; (with method)
// import sample.v1.CheckAlphabets as checkAlphabets; (bare)
- @interleave(profile=javaStyle)
+ @whitespace(layout.SPACES_AND_COMMENTS)
@mapping(ImportDeclarationExpr, params=[className, method, alias])
ImportDeclaration ::= 'import' ClassName @className [ '#' IDENTIFIER @method ] 'as' IDENTIFIER @alias ';' ;
@@ -53,7 +55,7 @@ grammar TinyExpressionP4 {
// ── Variable declarations ──────────────────────────────────────────────────
- @interleave(profile=javaStyle)
+ @whitespace(layout.SPACES_AND_COMMENTS)
VariableDeclaration ::=
NumberVariableDeclaration
| StringVariableDeclaration
diff --git a/tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.json b/tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.json
new file mode 100644
index 00000000..a2e13928
--- /dev/null
+++ b/tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.json
@@ -0,0 +1,9 @@
+{
+ "schemaVersion": 1,
+ "dependencies": {
+ "std/layout": {
+ "version": "1.0.0",
+ "source": "builtin:std/layout@1.0.0"
+ }
+ }
+}
diff --git a/tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.lock.json b/tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.lock.json
new file mode 100644
index 00000000..0fba1954
--- /dev/null
+++ b/tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.lock.json
@@ -0,0 +1 @@
+{"packages":{"std/layout":{"dependencies":{},"id":"std/layout","sha256":"2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67","version":"1.0.0"}},"roots":{"std/layout":"1.0.0"},"schemaVersion":1}
diff --git a/tools/tinyexpression-p4-lsp-vscode/pom.xml b/tools/tinyexpression-p4-lsp-vscode/pom.xml
index 34ab59aa..cb834ed5 100644
--- a/tools/tinyexpression-p4-lsp-vscode/pom.xml
+++ b/tools/tinyexpression-p4-lsp-vscode/pom.xml
@@ -15,7 +15,7 @@
UTF-8
0.23.1
- 3.2.0-SNAPSHOT
+ 3.3.0-SNAPSHOT
${project.build.directory}/generated-sources/tinyexpression-p4
${project.basedir}/server-dist
${project.build.directory}/tinyexpression-p4-lsp-${project.version}.vsix