From 4c3911bfea1f4927d395f953f5965301db57c29a Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 8 Jul 2026 22:19:26 +0900 Subject: [PATCH] fix: honor UTF-16 BOM before endian heuristics An explicit big-endian BOM could be overridden by the little-endian line-ending heuristic when valid BE payload bytes resembled a little-endian carriage return. This could rewrite the content with the wrong byte order. Prioritize both explicit BOM checks before BOM-less heuristics. Add a reproducer for the ambiguous BE code unit and characterize the existing BOM-less LE and BE paths. --- .../Editor/SkillFileContentNormalizerTests.cs | 40 +++++++++++++++++++ .../SkillSetup/SkillFileContentNormalizer.cs | 14 ++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs b/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs index 10ebfd1d92..879abc414a 100644 --- a/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs +++ b/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs @@ -57,6 +57,46 @@ public void NormalizeSkillFileContent_WhenUtf16BigEndianTextUsesCrlf_PreservesEn Assert.That(actualBytes, Is.EqualTo(expectedBytes)); } + // Tests that an explicit big-endian BOM wins over byte patterns that resemble little-endian CR. + [Test] + public void NormalizeSkillFileContent_WhenUtf16BigEndianBomContainsAmbiguousCodeUnit_PreservesEncoding() + { + byte[] sourceBytes = Encoding.BigEndianUnicode.GetPreamble() + .Concat(Encoding.BigEndianUnicode.GetBytes("\u0D00\r\nline")) + .ToArray(); + byte[] expectedBytes = Encoding.BigEndianUnicode.GetPreamble() + .Concat(Encoding.BigEndianUnicode.GetBytes("\u0D00\nline")) + .ToArray(); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(expectedBytes)); + } + + // Tests that BOM-less UTF-16 little-endian text still uses line-ending heuristics. + [Test] + public void NormalizeSkillFileContent_WhenUtf16LittleEndianHasNoBom_NormalizesCrlf() + { + byte[] sourceBytes = Encoding.Unicode.GetBytes("line1\r\nline2\r\n"); + byte[] expectedBytes = Encoding.Unicode.GetBytes("line1\nline2\n"); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(expectedBytes)); + } + + // Tests that BOM-less UTF-16 big-endian text still uses line-ending heuristics. + [Test] + public void NormalizeSkillFileContent_WhenUtf16BigEndianHasNoBom_NormalizesCrlf() + { + byte[] sourceBytes = Encoding.BigEndianUnicode.GetBytes("line1\r\nline2\r\n"); + byte[] expectedBytes = Encoding.BigEndianUnicode.GetBytes("line1\nline2\n"); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(expectedBytes)); + } + // Tests that files outside the text extension allowlist keep their original bytes. [Test] public void NormalizeSkillFileContent_WhenExtensionIsNotText_ReturnsOriginalContent() diff --git a/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs b/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs index 3f088d20c4..6c1bd2fa79 100644 --- a/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs +++ b/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs @@ -39,12 +39,22 @@ internal static byte[] NormalizeSkillFileContent(string relativePath, byte[] con return content; } - if (HasUtf16LittleEndianBom(content) || HasUtf16LittleEndianLineEnding(content)) + if (HasUtf16LittleEndianBom(content)) { return NormalizeUtf16LineEndings(content, isLittleEndian: true); } - if (HasUtf16BigEndianBom(content) || HasUtf16BigEndianLineEnding(content)) + if (HasUtf16BigEndianBom(content)) + { + return NormalizeUtf16LineEndings(content, isLittleEndian: false); + } + + if (HasUtf16LittleEndianLineEnding(content)) + { + return NormalizeUtf16LineEndings(content, isLittleEndian: true); + } + + if (HasUtf16BigEndianLineEnding(content)) { return NormalizeUtf16LineEndings(content, isLittleEndian: false); }