Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions Assets/Tests/Editor/SkillFileContentNormalizerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,46 @@ public void NormalizeSkillFileContent_WhenUtf16BigEndianTextUsesCrlf_PreservesEn
Assert.That(actualBytes, Is.EqualTo(expectedBytes));
}

// Tests that an explicit big-endian BOM wins over byte patterns that resemble little-endian CR.
[Test]
public void NormalizeSkillFileContent_WhenUtf16BigEndianBomContainsAmbiguousCodeUnit_PreservesEncoding()
{
byte[] sourceBytes = Encoding.BigEndianUnicode.GetPreamble()
.Concat(Encoding.BigEndianUnicode.GetBytes("\u0D00\r\nline"))
.ToArray();
byte[] expectedBytes = Encoding.BigEndianUnicode.GetPreamble()
.Concat(Encoding.BigEndianUnicode.GetBytes("\u0D00\nline"))
.ToArray();

byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes);

Assert.That(actualBytes, Is.EqualTo(expectedBytes));
}

// Tests that BOM-less UTF-16 little-endian text still uses line-ending heuristics.
[Test]
public void NormalizeSkillFileContent_WhenUtf16LittleEndianHasNoBom_NormalizesCrlf()
{
byte[] sourceBytes = Encoding.Unicode.GetBytes("line1\r\nline2\r\n");
byte[] expectedBytes = Encoding.Unicode.GetBytes("line1\nline2\n");

byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes);

Assert.That(actualBytes, Is.EqualTo(expectedBytes));
}

// Tests that BOM-less UTF-16 big-endian text still uses line-ending heuristics.
[Test]
public void NormalizeSkillFileContent_WhenUtf16BigEndianHasNoBom_NormalizesCrlf()
{
byte[] sourceBytes = Encoding.BigEndianUnicode.GetBytes("line1\r\nline2\r\n");
byte[] expectedBytes = Encoding.BigEndianUnicode.GetBytes("line1\nline2\n");

byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes);

Assert.That(actualBytes, Is.EqualTo(expectedBytes));
}

// Tests that files outside the text extension allowlist keep their original bytes.
[Test]
public void NormalizeSkillFileContent_WhenExtensionIsNotText_ReturnsOriginalContent()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,22 @@ internal static byte[] NormalizeSkillFileContent(string relativePath, byte[] con
return content;
}

if (HasUtf16LittleEndianBom(content) || HasUtf16LittleEndianLineEnding(content))
if (HasUtf16LittleEndianBom(content))
{
return NormalizeUtf16LineEndings(content, isLittleEndian: true);
}

if (HasUtf16BigEndianBom(content) || HasUtf16BigEndianLineEnding(content))
if (HasUtf16BigEndianBom(content))
{
return NormalizeUtf16LineEndings(content, isLittleEndian: false);
}

if (HasUtf16LittleEndianLineEnding(content))
{
return NormalizeUtf16LineEndings(content, isLittleEndian: true);
}

if (HasUtf16BigEndianLineEnding(content))
{
return NormalizeUtf16LineEndings(content, isLittleEndian: false);
}
Expand Down
Loading