diff --git a/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs b/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs new file mode 100644 index 0000000000..10ebfd1d92 --- /dev/null +++ b/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs @@ -0,0 +1,93 @@ +using System.Linq; +using System.Text; + +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.Infrastructure; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies deterministic line-ending normalization for generated skill files. + /// + [TestFixture] + public class SkillFileContentNormalizerTests + { + // Tests that CRLF and lone CR line endings become LF in single-byte text files. + [Test] + public void NormalizeSkillFileContent_WhenSingleByteTextUsesCarriageReturns_ReturnsLfContent() + { + byte[] sourceBytes = Encoding.UTF8.GetBytes("line1\r\nline2\rline3\n"); + byte[] expectedBytes = Encoding.UTF8.GetBytes("line1\nline2\nline3\n"); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(expectedBytes)); + } + + // Tests that UTF-16 little-endian text keeps its BOM and encoding while line endings become LF. + [Test] + public void NormalizeSkillFileContent_WhenUtf16LittleEndianTextUsesCrlf_PreservesEncoding() + { + byte[] sourceBytes = Encoding.Unicode.GetPreamble() + .Concat(Encoding.Unicode.GetBytes("line1\r\nline2\r\n")) + .ToArray(); + byte[] expectedBytes = Encoding.Unicode.GetPreamble() + .Concat(Encoding.Unicode.GetBytes("line1\nline2\n")) + .ToArray(); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("install.ps1", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(expectedBytes)); + } + + // Tests that UTF-16 big-endian text keeps its BOM and encoding while line endings become LF. + [Test] + public void NormalizeSkillFileContent_WhenUtf16BigEndianTextUsesCrlf_PreservesEncoding() + { + byte[] sourceBytes = Encoding.BigEndianUnicode.GetPreamble() + .Concat(Encoding.BigEndianUnicode.GetBytes("line1\r\nline2\r\n")) + .ToArray(); + byte[] expectedBytes = Encoding.BigEndianUnicode.GetPreamble() + .Concat(Encoding.BigEndianUnicode.GetBytes("line1\nline2\n")) + .ToArray(); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("install.ps1", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(expectedBytes)); + } + + // Tests that files outside the text extension allowlist keep their original bytes. + [Test] + public void NormalizeSkillFileContent_WhenExtensionIsNotText_ReturnsOriginalContent() + { + byte[] sourceBytes = Encoding.UTF8.GetBytes("line1\r\nline2\r\n"); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("image.png", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(sourceBytes)); + } + + // Tests that NUL-containing binary data is not rewritten even when its extension is textual. + [Test] + public void NormalizeSkillFileContent_WhenTextExtensionContainsBinaryData_ReturnsOriginalContent() + { + byte[] sourceBytes = { 0x41, 0x42, 0x00, 0x43, 0x0D, 0x44 }; + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(sourceBytes)); + } + + // Tests that LF-only text content remains byte-identical. + [Test] + public void NormalizeSkillFileContent_WhenTextAlreadyUsesLf_ReturnsOriginalContent() + { + byte[] sourceBytes = Encoding.UTF8.GetBytes("line1\nline2\n"); + + byte[] actualBytes = SkillFileContentNormalizer.NormalizeSkillFileContent("reference.md", sourceBytes); + + Assert.That(actualBytes, Is.EqualTo(sourceBytes)); + } + } +} diff --git a/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs.meta b/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs.meta new file mode 100644 index 0000000000..a76dd2e54b --- /dev/null +++ b/Assets/Tests/Editor/SkillFileContentNormalizerTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0b97a05adf16244c2a0afe54f8d558bb +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/SkillInstallLayoutTests.cs b/Assets/Tests/Editor/SkillInstallLayoutTests.cs index a2e2188ee8..4dac816fcf 100644 --- a/Assets/Tests/Editor/SkillInstallLayoutTests.cs +++ b/Assets/Tests/Editor/SkillInstallLayoutTests.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; @@ -388,22 +387,6 @@ public void GetToolSettingsCatalogForProjectRoot_WhenSkillIsInternal_HidesToolFr Assert.That(registeredToolNames, Does.Contain("public-tool")); } - // Tests that PowerShell scripts keep their source encoding while line endings are normalized. - [Test] - public void NormalizeSkillFileContent_WhenPowerShellScriptUsesUtf16LittleEndian_PreservesEncoding() - { - byte[] sourceBytes = Encoding.Unicode.GetPreamble() - .Concat(Encoding.Unicode.GetBytes("line1\r\nline2\r\n")) - .ToArray(); - byte[] expectedBytes = Encoding.Unicode.GetPreamble() - .Concat(Encoding.Unicode.GetBytes("line1\nline2\n")) - .ToArray(); - - byte[] actualBytes = SkillInstallLayout.NormalizeSkillFileContent("install.ps1", sourceBytes); - - Assert.That(actualBytes, Is.EqualTo(expectedBytes)); - } - // Tests that skill setup file scans share the same generated-file exclusion behavior. [Test] public void IsExcludedSkillFile_WhenGeneratedSkillFileNameIsPassed_ReturnsTrue() diff --git a/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs b/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs new file mode 100644 index 0000000000..3f088d20c4 --- /dev/null +++ b/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs @@ -0,0 +1,193 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.Infrastructure +{ + /// + /// Normalizes line endings in generated text skill files without changing their encoding. + /// + internal static class SkillFileContentNormalizer + { + private const byte Utf16LittleEndianBomFirstByte = 0xFF; + private const byte Utf16LittleEndianBomSecondByte = 0xFE; + private const byte Utf16BigEndianBomFirstByte = 0xFE; + private const byte Utf16BigEndianBomSecondByte = 0xFF; + private const int Utf16CodeUnitByteCount = 2; + private const ushort CarriageReturnCodeUnit = 0x000D; + private const ushort LineFeedCodeUnit = 0x000A; + private static readonly HashSet TextSkillFileExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".json", + ".md", + ".ps1", + ".sh", + ".txt", + ".yaml", + ".yml" + }; + + internal static byte[] NormalizeSkillFileContent(string relativePath, byte[] content) + { + Debug.Assert(!string.IsNullOrEmpty(relativePath), "relativePath must not be null or empty"); + Debug.Assert(content != null, "content must not be null"); + + if (!ShouldNormalizeLineEndings(relativePath) || Array.IndexOf(content, (byte)'\r') < 0) + { + return content; + } + + if (HasUtf16LittleEndianBom(content) || HasUtf16LittleEndianLineEnding(content)) + { + return NormalizeUtf16LineEndings(content, isLittleEndian: true); + } + + if (HasUtf16BigEndianBom(content) || HasUtf16BigEndianLineEnding(content)) + { + return NormalizeUtf16LineEndings(content, isLittleEndian: false); + } + + if (Array.IndexOf(content, (byte)0) >= 0) + { + return content; + } + + return NormalizeSingleByteLineEndings(content); + } + + private static bool ShouldNormalizeLineEndings(string relativePath) + { + string extension = Path.GetExtension(relativePath); + return TextSkillFileExtensions.Contains(extension); + } + + private static byte[] NormalizeSingleByteLineEndings(byte[] content) + { + List normalized = new(content.Length); + for (int index = 0; index < content.Length; index++) + { + byte current = content[index]; + if (current != (byte)'\r') + { + normalized.Add(current); + continue; + } + + normalized.Add((byte)'\n'); + if (index + 1 < content.Length && content[index + 1] == (byte)'\n') + { + index++; + } + } + + return normalized.ToArray(); + } + + private static byte[] NormalizeUtf16LineEndings(byte[] content, bool isLittleEndian) + { + List normalized = new(content.Length); + int index = 0; + if (HasMatchingUtf16Bom(content, isLittleEndian)) + { + normalized.Add(content[0]); + normalized.Add(content[1]); + index = Utf16CodeUnitByteCount; + } + + while (index + 1 < content.Length) + { + ushort codeUnit = ReadUtf16CodeUnit(content, index, isLittleEndian); + if (codeUnit == CarriageReturnCodeUnit) + { + WriteUtf16CodeUnit(normalized, LineFeedCodeUnit, isLittleEndian); + int nextIndex = index + Utf16CodeUnitByteCount; + if (nextIndex + 1 < content.Length && + ReadUtf16CodeUnit(content, nextIndex, isLittleEndian) == LineFeedCodeUnit) + { + index += Utf16CodeUnitByteCount * 2; + continue; + } + + index += Utf16CodeUnitByteCount; + continue; + } + + WriteUtf16CodeUnit(normalized, codeUnit, isLittleEndian); + index += Utf16CodeUnitByteCount; + } + + if (index < content.Length) + { + normalized.Add(content[index]); + } + + return normalized.ToArray(); + } + + private static bool HasUtf16LittleEndianBom(byte[] content) + { + return content.Length >= Utf16CodeUnitByteCount && + content[0] == Utf16LittleEndianBomFirstByte && + content[1] == Utf16LittleEndianBomSecondByte; + } + + private static bool HasUtf16BigEndianBom(byte[] content) + { + return content.Length >= Utf16CodeUnitByteCount && + content[0] == Utf16BigEndianBomFirstByte && + content[1] == Utf16BigEndianBomSecondByte; + } + + private static bool HasMatchingUtf16Bom(byte[] content, bool isLittleEndian) + { + return isLittleEndian ? HasUtf16LittleEndianBom(content) : HasUtf16BigEndianBom(content); + } + + private static bool HasUtf16LittleEndianLineEnding(byte[] content) + { + return HasUtf16LineEnding(content, isLittleEndian: true); + } + + private static bool HasUtf16BigEndianLineEnding(byte[] content) + { + return HasUtf16LineEnding(content, isLittleEndian: false); + } + + private static bool HasUtf16LineEnding(byte[] content, bool isLittleEndian) + { + int startIndex = HasMatchingUtf16Bom(content, isLittleEndian) ? Utf16CodeUnitByteCount : 0; + for (int index = startIndex; index + 1 < content.Length; index += Utf16CodeUnitByteCount) + { + ushort codeUnit = ReadUtf16CodeUnit(content, index, isLittleEndian); + if (codeUnit == CarriageReturnCodeUnit || codeUnit == LineFeedCodeUnit) + { + return true; + } + } + + return false; + } + + private static ushort ReadUtf16CodeUnit(byte[] content, int index, bool isLittleEndian) + { + return isLittleEndian + ? (ushort)(content[index] | (content[index + 1] << 8)) + : (ushort)((content[index] << 8) | content[index + 1]); + } + + private static void WriteUtf16CodeUnit(List output, ushort codeUnit, bool isLittleEndian) + { + if (isLittleEndian) + { + output.Add((byte)(codeUnit & 0xFF)); + output.Add((byte)(codeUnit >> 8)); + return; + } + + output.Add((byte)(codeUnit >> 8)); + output.Add((byte)(codeUnit & 0xFF)); + } + } +} diff --git a/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs.meta b/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs.meta new file mode 100644 index 0000000000..903d59747c --- /dev/null +++ b/Packages/src/Editor/Infrastructure/SkillSetup/SkillFileContentNormalizer.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6c53e19fa3b9b4148821c8c9b91197df +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Infrastructure/SkillSetup/SkillInstallLayout.cs b/Packages/src/Editor/Infrastructure/SkillSetup/SkillInstallLayout.cs index f4483bc538..988fe8f429 100644 --- a/Packages/src/Editor/Infrastructure/SkillSetup/SkillInstallLayout.cs +++ b/Packages/src/Editor/Infrastructure/SkillSetup/SkillInstallLayout.cs @@ -18,23 +18,6 @@ internal static class SkillInstallLayout internal const string SkillsDirName = "skills"; internal const string ManagedSkillsDirName = "unity-cli-loop"; internal const string SkillFileName = "SKILL.md"; - private const byte Utf16LittleEndianBomFirstByte = 0xFF; - private const byte Utf16LittleEndianBomSecondByte = 0xFE; - private const byte Utf16BigEndianBomFirstByte = 0xFE; - private const byte Utf16BigEndianBomSecondByte = 0xFF; - private const int Utf16CodeUnitByteCount = 2; - private const ushort CarriageReturnCodeUnit = 0x000D; - private const ushort LineFeedCodeUnit = 0x000A; - private static readonly HashSet TextSkillFileExtensions = new(StringComparer.OrdinalIgnoreCase) - { - ".json", - ".md", - ".ps1", - ".sh", - ".txt", - ".yaml", - ".yml" - }; internal readonly struct SkillSourceInfo { @@ -353,7 +336,9 @@ private static Dictionary CollectInstalledSkillFiles(string skil } string relativePath = Path.GetRelativePath(skillDirectory, filePath); - files[relativePath] = NormalizeSkillFileContent(relativePath, File.ReadAllBytes(filePath)); + files[relativePath] = SkillFileContentNormalizer.NormalizeSkillFileContent( + relativePath, + File.ReadAllBytes(filePath)); } return files; @@ -373,171 +358,12 @@ internal static Dictionary CollectSourceSkillFiles( return new Dictionary(StringComparer.Ordinal) { - [SkillFileName] = NormalizeSkillFileContent(SkillFileName, File.ReadAllBytes(skillFilePath)) + [SkillFileName] = SkillFileContentNormalizer.NormalizeSkillFileContent( + SkillFileName, + File.ReadAllBytes(skillFilePath)) }; } - internal static byte[] NormalizeSkillFileContent(string relativePath, byte[] content) - { - Debug.Assert(!string.IsNullOrEmpty(relativePath), "relativePath must not be null or empty"); - Debug.Assert(content != null, "content must not be null"); - - if (!ShouldNormalizeLineEndings(relativePath) || Array.IndexOf(content, (byte)'\r') < 0) - { - return content; - } - - if (HasUtf16LittleEndianBom(content) || HasUtf16LittleEndianLineEnding(content)) - { - return NormalizeUtf16LineEndings(content, isLittleEndian: true); - } - - if (HasUtf16BigEndianBom(content) || HasUtf16BigEndianLineEnding(content)) - { - return NormalizeUtf16LineEndings(content, isLittleEndian: false); - } - - if (Array.IndexOf(content, (byte)0) >= 0) - { - return content; - } - - return NormalizeSingleByteLineEndings(content); - } - - private static bool ShouldNormalizeLineEndings(string relativePath) - { - string extension = Path.GetExtension(relativePath); - return TextSkillFileExtensions.Contains(extension); - } - - private static byte[] NormalizeSingleByteLineEndings(byte[] content) - { - List normalized = new(content.Length); - for (int index = 0; index < content.Length; index++) - { - byte current = content[index]; - if (current != (byte)'\r') - { - normalized.Add(current); - continue; - } - - normalized.Add((byte)'\n'); - if (index + 1 < content.Length && content[index + 1] == (byte)'\n') - { - index++; - } - } - - return normalized.ToArray(); - } - - private static byte[] NormalizeUtf16LineEndings(byte[] content, bool isLittleEndian) - { - List normalized = new(content.Length); - int index = 0; - if (HasMatchingUtf16Bom(content, isLittleEndian)) - { - normalized.Add(content[0]); - normalized.Add(content[1]); - index = Utf16CodeUnitByteCount; - } - - while (index + 1 < content.Length) - { - ushort codeUnit = ReadUtf16CodeUnit(content, index, isLittleEndian); - if (codeUnit == CarriageReturnCodeUnit) - { - WriteUtf16CodeUnit(normalized, LineFeedCodeUnit, isLittleEndian); - int nextIndex = index + Utf16CodeUnitByteCount; - if (nextIndex + 1 < content.Length && - ReadUtf16CodeUnit(content, nextIndex, isLittleEndian) == LineFeedCodeUnit) - { - index += Utf16CodeUnitByteCount * 2; - continue; - } - - index += Utf16CodeUnitByteCount; - continue; - } - - WriteUtf16CodeUnit(normalized, codeUnit, isLittleEndian); - index += Utf16CodeUnitByteCount; - } - - if (index < content.Length) - { - normalized.Add(content[index]); - } - - return normalized.ToArray(); - } - - private static bool HasUtf16LittleEndianBom(byte[] content) - { - return content.Length >= Utf16CodeUnitByteCount && - content[0] == Utf16LittleEndianBomFirstByte && - content[1] == Utf16LittleEndianBomSecondByte; - } - - private static bool HasUtf16BigEndianBom(byte[] content) - { - return content.Length >= Utf16CodeUnitByteCount && - content[0] == Utf16BigEndianBomFirstByte && - content[1] == Utf16BigEndianBomSecondByte; - } - - private static bool HasMatchingUtf16Bom(byte[] content, bool isLittleEndian) - { - return isLittleEndian ? HasUtf16LittleEndianBom(content) : HasUtf16BigEndianBom(content); - } - - private static bool HasUtf16LittleEndianLineEnding(byte[] content) - { - return HasUtf16LineEnding(content, isLittleEndian: true); - } - - private static bool HasUtf16BigEndianLineEnding(byte[] content) - { - return HasUtf16LineEnding(content, isLittleEndian: false); - } - - private static bool HasUtf16LineEnding(byte[] content, bool isLittleEndian) - { - int startIndex = HasMatchingUtf16Bom(content, isLittleEndian) ? Utf16CodeUnitByteCount : 0; - for (int index = startIndex; index + 1 < content.Length; index += Utf16CodeUnitByteCount) - { - ushort codeUnit = ReadUtf16CodeUnit(content, index, isLittleEndian); - if (codeUnit == CarriageReturnCodeUnit || codeUnit == LineFeedCodeUnit) - { - return true; - } - } - - return false; - } - - private static ushort ReadUtf16CodeUnit(byte[] content, int index, bool isLittleEndian) - { - return isLittleEndian - ? (ushort)(content[index] | (content[index + 1] << 8)) - : (ushort)((content[index] << 8) | content[index + 1]); - } - - private static void WriteUtf16CodeUnit(List output, ushort codeUnit, bool isLittleEndian) - { - if (isLittleEndian) - { - output.Add((byte)(codeUnit & 0xFF)); - output.Add((byte)(codeUnit >> 8)); - return; - } - - output.Add((byte)(codeUnit >> 8)); - output.Add((byte)(codeUnit & 0xFF)); - } - internal static bool IsSafeSkillPathComponent(string skillName) { if (string.IsNullOrEmpty(skillName))