opaopa6969 · Oct 9, 2026 · Oct 8, 2026 · Oct 8, 2026 · Oct 8, 2026
diff --git a/.github/unlaxer-source-pin b/.github/unlaxer-source-pin
@@ -1,3 +1,3 @@
-# unlaxer-parser commit CI builds for the declarative-module development dependency.
-# Retain until pom.xml selects a published version with the same lexical capabilities.
-commit=f825740396a6181654191de5ba040e1f83533526
+# Exact source dependency for named layout and package origins.
+# Update to a published compatible artifact when available.
+commit=fc75fe1e25e538d47908f05ea2039f585aef696e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -132,12 +132,14 @@ jobs:
         run: |
           task_maven_repo=$(mktemp -d "$RUNNER_TEMP/ubnfc-parser.XXXXXX")
           echo "MAVEN_REPO=$task_maven_repo" >> "$GITHUB_ENV"
-      # Declarative lexical modules require the pinned 3.2.0-SNAPSHOT generator;
+      # Declarative lexical modules require the pinned 3.3.0-SNAPSHOT generator;
       # published 3.1.1 cannot read this grammar. Build into the job-local repository.
       - name: Install unlaxer from source if it is not on Maven Central yet
         run: bash scripts/ci/install-unlaxer-if-unpublished.sh
       # Grammar sha256 + every vendored file against UBNFC_PIN. ubnfc is private, so the full
       # grammar -> IR -> Java regeneration runs where it is readable (UBNFC_DIR); see the script.
+      - name: Verify strict layout compatibility adapter
+        run: python3 scripts/test_ubnfc_layout.py
       - name: Verify vendored ubnfc parser against its pin
         run: bash scripts/regenerate-ubnfc-parser.sh --check
       - name: Verify the generated AST converter
@@ -172,6 +174,9 @@ jobs:
         with:
           java-version: '21'
           distribution: 'temurin'
+      - name: Set up Rust 1.85 for the shared source fixtures
+        if: matrix.generator == 'source'
+        run: bash tinyexpression/.github/scripts/setup-rust.sh 1.85.0
       - name: Create an empty Maven repository on the assigned runner
         run: |
           task_maven_repo=$(mktemp -d "$RUNNER_TEMP/mapper-compatibility.XXXXXX")
@@ -181,7 +186,7 @@ jobs:
         with:
           repository: opaopa6969/unlaxer-parser
           # Declarative lexical modules under a distinct development artifact version.
-          ref: f825740396a6181654191de5ba040e1f83533526
+          ref: fc75fe1e25e538d47908f05ea2039f585aef696e
           path: unlaxer-parser
       - name: Install pinned development generator into its isolated repository
         if: matrix.generator == 'source'
@@ -215,19 +220,26 @@ jobs:
           -Dtinyexpression.expected.mapper.snapshot=true
           -Dtinyexpression.expected.mapper.nodeAlias=true
           -Dtinyexpression.expected.mapper.optionalQualifier=true
-          -Dtest=P4SourceMappingTest,P4OwnedSliceSourceTest,P4TypedAstEvaluatorTest,P4JavaCodeEmitterSourceTextTest,DslGeneratedAstJavaEmitterTest,P4PreferredAstMapperPrecedenceTest,P4OptionalExternalQualifierTest,P4RustRootExpressionAcceptanceTest,P4RustNumericEvaluatorOracleTest,P4RustScalarEvaluatorOracleTest,CodeBlockSourceTest,CodeBlockNoCompilerTest
+          -Dtest=P4LayoutMigrationTest,P4SourceMappingTest,P4OwnedSliceSourceTest,P4TypedAstEvaluatorTest,P4JavaCodeEmitterSourceTextTest,DslGeneratedAstJavaEmitterTest,P4PreferredAstMapperPrecedenceTest,P4OptionalExternalQualifierTest,P4RustRootExpressionAcceptanceTest,P4RustNumericEvaluatorOracleTest,P4RustScalarEvaluatorOracleTest,CodeBlockSourceTest,CodeBlockNoCompilerTest
       - name: Verify shared Java/Rust parser fixtures with the pinned source generator
         if: matrix.generator == 'source'
         working-directory: tinyexpression
         run: >-
           mvn -B test -Dgpg.skip=true -Dmaven.repo.local="$MAVEN_REPO" -Dunlaxer.version="$UNLAXER_VERSION"
           -Dtinyexpression.skipRailroad=true -Dtinyexpression.rust.shared=true
-          -Dtest=P4RustSharedFixtureAcceptanceTest
+          -Dtinyexpression.layout.conformance=true -Dtinyexpression.unlaxer.source=../unlaxer-parser
+          -Dtest=P4RustSharedFixtureAcceptanceTest,P4LayoutGeneratedConformanceTest
+      - name: Require the shared layout comparison evidence
+        if: matrix.generator == 'source'
+        working-directory: tinyexpression
+        run: test -s target/p4-layout-generated.tsv
       - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: mapper-compatibility-${{ matrix.generator }}
-          path: tinyexpression/target/surefire-reports/
+          path: |
+            tinyexpression/target/surefire-reports/
+            tinyexpression/target/p4-layout-generated.tsv
           if-no-files-found: warn
 
   # issue #173: the Java CI above resolves org.unlaxer:unlaxer-* from Maven Central, so a

diff --git a/.gitignore b/.gitignore
@@ -30,3 +30,6 @@ buildNumber.properties
 /node_modules/
 /package.json
 /package-lock.json
+
+# Python test bytecode
+/scripts/__pycache__/
diff --git a/docs/imported-layout-migration.md b/docs/imported-layout-migration.md
@@ -0,0 +1,32 @@
+# P4 の共有空白・コメント定義
+
+P4 の grammar default と `Formula`、`ImportDeclaration`、`VariableDeclaration` は、固定した `std/layout@1.0.0` の `layout.SPACES_AND_COMMENTS` を参照する。rule の明示指定は `@whitespace(layout.SPACES_AND_COMMENTS)` に置換した。従来の `@interleave(profile=javaStyle)` と同じ場所で同じ集合を使い、子 rule に独自の指定が追加された場合も rule の優先順位を保てるため、冗長に見える指定も残した。
+
+`grammar/ubnf.json` は builtin の正確な版を指定する。`ubnf.lock.json` の identity は `std/layout` / `1.0.0` / SHA-256 `2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67`。この小さな標準 artifact 自体を grammar の `.ubnf-cache/packages` に同梱する。生成は lock と検証済み bytes のみを読み、ネットワーク・認証・個人設定を必要としない。上流 generator は Maven `3.3.0-SNAPSHOT` と `.github/unlaxer-source-pin` の exact commit で固定する。CI の専用 Maven repository へその commit を install して検証する。
+
+標準定義は ASCII space と TAB..CR、CR/LF 手前までの `//`、最初の `*/` までの block comment。閉じていない block comment は読み飛ばさない。文字列と fenced Java code の token 内部へ自動 skip は入らない。AST の文字列正規化、raw CST、source-preserving model の既存の comment mask と位置契約を維持する。
+
+## backend の範囲
+
+| 経路 | 実 P4 | 名前付き空白と固定 package |
+|---|---|---|
+| unlaxer Java 生成 parser | 対応。旧 grammar と全 AST field/node span・cursor・CST 原文を比較 | 直接対応 |
+| unlaxer Java 経由 Rust 生成 / native Rust 生成 | P4 の token/layout projection で同じ入力・独立期待値・生成物 bytes を比較。実 P4 全体には従来の portability 制限がある（native は `literalBoundary: none` を拒否） | 直接対応。局所 `none` も共通 fixture で比較 |
+| TinyExpression 本番 ubnfc Java / Rust | 固定生成物を使う。共有実 P4 fixture と source positions、memo OFF/ON を検証 | **std/layout@1.0.0 の限定互換処理**。任意 package/named policy の直接対応ではない |
+
+`scripts/prepare-ubnfc-layout.py` は manifest・lock graph・artifact hash・全定義・global 1箇所/rule 3箇所を確認し、この標準定義だけを旧 javaStyle/interleave へ戻す。混在設定、追加 package/comment、未知の定義、改ざんを変換前に拒否する。導出 grammar は旧 SHA-256 と一致し、Java/Rust ubnfc の grammar→IR→生成物も byte 一致する。再生成スクリプトの両経路から同じ互換処理を呼び、文法・manifest・lock・artifact・互換処理の hash を両 pin に含めた。汎用 ubnfc 対応は [#249](https://github.com/opaopa6969/tinyexpression/issues/249) で追跡する。
+
+## 検証と定義表示
+
+`actual-p4.json` は実 P4 の空白/TAB/LF/CRLF/コメント/未閉鎖/文字列/Java fence/import/declaration の14入力。Java の旧/新 parser と Tiny 本番 Java/Rust が同じ cursor 期待値を使う。Java の旧/新 AST 全 field/node span と retained CST 原文、owned source の位置を比較する。`corpus.json` は実 P4 の lexical modules を用いる29入力で、旧/新 layout と局所 `none` を Java・Java経由Rust・native Rust へ入力し、位置・診断・AST の独立期待値を比較する。
+
+```sh
+python3 scripts/test_ubnfc_layout.py
+MAVEN_REPO=/tmp/tiny-layout-m2 scripts/ci/install-unlaxer-if-unpublished.sh
+mvn test -Dmaven.repo.local=/tmp/tiny-layout-m2 -Dtinyexpression.skipRailroad=true \
+  -Dtinyexpression.layout.conformance=true -Dtinyexpression.unlaxer.source=/path/to/pinned/unlaxer-parser \
+  -Dtest=P4LayoutMigrationTest,P4LayoutGeneratedConformanceTest
+cargo test --locked --manifest-path rust/Cargo.toml -p tinyexpression-rs shared_imported_layout
+```
+
+文法作者は UBNF authoring extension の `@whitespace` 補完から `layout.SPACES_AND_COMMENTS` を選び、hover で定義と ID/version/hash、定義への移動で読み取り専用の標準 module を確認できる。CLI `deps inspect --grammar tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf` も同じ定義・出典・code-point 位置を返す。`P4LayoutMigrationTest` は実 grammar のこの snapshot と4 policy scopeを確認する。TinyExpression の runtime LSP/DAP と文法作者向け UBNF LSP は別の役割を持つ。
diff --git a/pom.xml b/pom.xml
@@ -56,7 +56,7 @@
 		     installed into a job-local, isolated Maven repo (issue #173: unlaxer-parser master,
 		     non-blocking) so the GAV actually resolves from that isolated repo instead of
 		     silently falling back to the published artifact. -->
-		<unlaxer.version>3.2.0-SNAPSHOT</unlaxer.version>
+		<unlaxer.version>3.3.0-SNAPSHOT</unlaxer.version>
 	</properties>
 
 	<dependencies>

diff --git a/rust/check-generated.sh b/rust/check-generated.sh
@@ -17,6 +17,12 @@ set -euo pipefail
 script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
 
 repo_dir=$(cd -- "$script_dir/.." && pwd)
+# The offline check validates the restricted layout bridge even without ubnfc.
+layout_check=$(mktemp -d)
+trap 'rm -rf "$layout_check"' EXIT
+python3 "$repo_dir/scripts/prepare-ubnfc-layout.py" \
+  "$repo_dir/tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf" \
+  "$layout_check/tinyexpression-p4.ubnf" >/dev/null
 # The offline check must notice a changed imported grammar too.
 (cd "$repo_dir" && sed -n 's/^grammar_file //p' rust/ubnfc-pin.txt | sha256sum -c -)
 if [ -d "${UBNFC_DIR:-$repo_dir/../ubnfc}" ]; then

diff --git a/rust/scripts/regenerate-ubnfc.sh b/rust/scripts/regenerate-ubnfc.sh
@@ -78,7 +78,7 @@ export CARGO_TARGET_DIR="$work/target"
 
 sha() { sha256sum "$1" | cut -d' ' -f1; }
 grammar_manifest() {
-  (cd "$repo_dir" && { find "$(dirname "$grammar_rel")" -type f -name '*.ubnf'; echo "$fi_grammar_rel"; } |
+  (cd "$repo_dir" && { find "$(dirname "$grammar_rel")" -type f \( -name '*.ubnf' -o -name '*.json' \); echo "$fi_grammar_rel"; echo scripts/prepare-ubnfc-layout.py; } |
     LC_ALL=C sort -u | while IFS= read -r f; do echo "$(sha "$f") $f"; done)
 }
 
@@ -100,8 +100,10 @@ if [[ "$mode" == --check ]]; then
   diff <(sed -n 's/^grammar_file //p' "$pin") <(grammar_manifest)
 fi
 
+python3 "$repo_dir/scripts/prepare-ubnfc-layout.py" "$repo_dir/$grammar_rel" "$work/compat/$grammar_rel" >/dev/null
+touch "$work/compat/.git" # preserve the original logical source root
 (cd "$ubnfc_dir" && cargo run -q --offline -p ubnfc-front --bin ubnfc -- \
-  ir --grammar "$repo_dir/$grammar_rel" \
+  ir --grammar "$work/compat/$grammar_rel" \
   --extern-first-chars "$ubnfc_dir/scanners/first-chars.json" \
   --out "$work/p4.ir.json" >/dev/null)
 check_sha "$work/p4.ir.json" "$(value ir_sha256)"
@@ -177,6 +179,16 @@ case "$mode" in
     rm -rf "$vendored"
     cp -r "$work/generated" "$vendored"
     (cd "$vendored" && find . -type f | LC_ALL=C sort | xargs sha256sum) > "$manifest"
+    python3 - "$rust_dir/tinyexpression-rs/src/api.rs" "$(sha "$repo_dir/$grammar_rel")" <<'PY'
+from pathlib import Path
+import re, sys
+path = Path(sys.argv[1])
+text, count = re.subn(r'pub const GRAMMAR_SHA256: &str = "[0-9a-f]{64}";',
+                    'pub const GRAMMAR_SHA256: &str = "' + sys.argv[2] + '";', path.read_text())
+if count != 1:
+    raise SystemExit("expected exactly one GRAMMAR_SHA256 constant")
+path.write_text(text)
+PY
     echo "vendored ubnfc parser: updated"
     rm -rf "$vendored_fi"
     cp -r "$work/generated-formula-info" "$vendored_fi"

diff --git a/rust/tinyexpression-rs/src/api.rs b/rust/tinyexpression-rs/src/api.rs
@@ -39,7 +39,7 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 /// fails when it differs from `ubnfc_commit` in `rust/ubnfc-pin.txt`.
 pub const UBNFC_COMMIT: &str = "f54d6b90466ffde14066d32f911ed642b1e49201";
 /// Exact P4 grammar source identity; checked against `rust/ubnfc-pin.txt` in CI.
-pub const GRAMMAR_SHA256: &str = "4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a";
+pub const GRAMMAR_SHA256: &str = "184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46";
 
 /// One JSON response and the exit code that goes with it.
 #[derive(Clone, Debug, PartialEq, Eq)]

diff --git a/rust/tinyexpression-rs/src/frontend/declarative_tests.rs b/rust/tinyexpression-rs/src/frontend/declarative_tests.rs
@@ -4,6 +4,59 @@ use crate::generated::{
     compat, ubnfc,
 };
 
+#[test]
+fn shared_imported_layout_keeps_production_p4_positions() {
+    use crate::request::{parse_json, Json};
+    let Json::Arr(rows) = parse_json(include_str!(
+        "../../../../src/test/resources/p4-layout/actual-p4.json"
+    ))
+    .unwrap() else {
+        panic!("array")
+    };
+    for row in rows {
+        let id = row.str_field("id").unwrap();
+        let input = row.str_field("input").unwrap();
+        let Some(Json::Bool(ok)) = row.get("ok") else {
+            panic!("{id}")
+        };
+        let end = if *ok { input.chars().count() } else { 0 };
+        for memo in [false, true] {
+            let result = ubnfc::parse_entry_with_options(
+                "TinyExpressionP4",
+                Some("Formula"),
+                input,
+                ubnfc::ParseOptions {
+                    require_eof: false,
+                    memo,
+                    ..Default::default()
+                },
+            )
+            .unwrap();
+            assert_eq!(
+                (result.ok, result.consumed_cp, result.matched_cp),
+                (*ok, end, end),
+                "{id}"
+            );
+            if *ok {
+                let converted = compat::convert(result.ast.as_ref().unwrap()).unwrap();
+                let Ast::FormulaExpr { span, .. } = converted else {
+                    panic!("{id}")
+                };
+                assert_eq!(
+                    (span.start, span.end),
+                    (0, end),
+                    "{id} owned source position"
+                );
+                assert_eq!(
+                    input.chars().take(span.end).collect::<String>(),
+                    input,
+                    "{id} owned source"
+                );
+            }
+        }
+    }
+}
+
 #[test]
 fn shared_lexical_cases_need_no_host_parser_mapping() {
     use crate::request::{parse_json, Json};

diff --git a/rust/ubnfc-pin.txt b/rust/ubnfc-pin.txt
@@ -1,14 +1,18 @@
 # Generated by rust/scripts/regenerate-ubnfc.sh --write; do not edit hashes by hand.
 ubnfc_commit=f54d6b90466ffde14066d32f911ed642b1e49201
 grammar=tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
-grammar_sha256=4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a
+grammar_sha256=184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46
 grammar_file 28bcfc9f2e684b7326561f281155a1d0140dc67145d178764841bc7dc87a81cb grammar/formula-info.ubnf
+grammar_file 492cafad9f8f3278480d862d0f1fadeb5f61bf5c627b8072b32f6d90c91d3ff1 scripts/prepare-ubnfc-layout.py
+grammar_file 2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67 tools/tinyexpression-p4-lsp-vscode/grammar/.ubnf-cache/packages/2701f6884ea6c712d28c76ec3d340c3b27fca4575846239bd0854933205f4b67.json
 grammar_file e33b8166d0a5b399150b583b0e466fbe2a4194325ef65c79c082297156796b76 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/characters.ubnf
 grammar_file 53ad774e87bd95a498ac744a721f22bdaf877721c0da13e8045aaeaea09b8179 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/fences.ubnf
 grammar_file 55f5f3b9289cb41b74d5c617e0b2ec1adbf24eb7fd8ca11aa02eda311b92bf2b tools/tinyexpression-p4-lsp-vscode/grammar/lexical/identifiers.ubnf
 grammar_file 1b1fcaa633f9477ad5414ecfed64693d1d50bf0aaa02775c5d67b39a18e8244f tools/tinyexpression-p4-lsp-vscode/grammar/lexical/numbers.ubnf
 grammar_file 08f209d29c84aa1addcb57eba24d7738c27bc8921cb13f1fbd5040dff6296a26 tools/tinyexpression-p4-lsp-vscode/grammar/lexical/strings.ubnf
-grammar_file 4a02d31de4e646a0664fd78a4b77ff2e145e7af7dde83a209e244e38298e962a tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+grammar_file 184cce54c70b0e3ebbdedaca5a0af0068ec56b2d7e33cda39d94f861c6433f46 tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf
+grammar_file 2074b51b078eecd61c5a0bbd20399f423b20faada77d331b96f212692c27bdf7 tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.json
+grammar_file 25745248036e459f3b69ad202f12b52d857b6f70abc02dac23caf4e7745f4db0 tools/tinyexpression-p4-lsp-vscode/grammar/ubnf.lock.json
 ir_sha256=55d5981a9da79a887034778aa1853dffea92162cb5e9be18c840b072cafb13a0
 scanners=scanners/rust/scanners.rs
 scanners_sha256=593a2736f7f20fa5d61456e25e7d961cefbd427d93b05a7ced46b41506e6a412

diff --git a/scripts/ci/install-unlaxer-if-unpublished.sh b/scripts/ci/install-unlaxer-if-unpublished.sh
@@ -4,7 +4,7 @@
 #
 #   MAVEN_REPO=/path/to/repo scripts/ci/install-unlaxer-if-unpublished.sh
 #
-# Declarative P4 lexical modules need unlaxer 3.2.0-SNAPSHOT, not the published 3.1.1.
+# Declarative P4 lexical modules need unlaxer 3.3.0-SNAPSHOT, not the published 3.1.1.
 # Until a compatible version is published, CI builds the pinned unlaxer-parser commit
 # (.github/unlaxer-source-pin) whose <revision> must equal pom.xml's unlaxer.version. Once the
 # version is on Central this script does nothing, and the pin file can be deleted.