From 7ee4fef4e893383a700355c10301c561d6fcd60d Mon Sep 17 00:00:00 2001 From: hatayama Date: Mon, 5 Oct 2026 23:58:31 +0900 Subject: [PATCH 01/10] Add the per-test-mode last-run record store for --rerun-failed --rerun-failed has to know which tests failed in the most recent completed run of a test mode, so each test mode gets one JSON record under TestResults. - Anything other than a complete record of the requested test mode reads as Unreadable, so a rerun never starts from a record it cannot trust. - A failed delete propagates, so a run never starts while the old record could still pass as the newest one. - Writes go through a uniquely named temporary file, so the real name never holds a partial record, and a write failure only logs a warning because the run's own result is still valid. --- .../Editor/RunTestsLastRunRecordStoreTests.cs | 328 ++++++++++++++++++ .../RunTestsLastRunRecordStoreTests.cs.meta | 11 + .../RunTests/RunTestsLastRunRecord.cs | 95 +++++ .../RunTests/RunTestsLastRunRecord.cs.meta | 11 + .../RunTests/RunTestsLastRunRecordStore.cs | 220 ++++++++++++ .../RunTestsLastRunRecordStore.cs.meta | 11 + 6 files changed, 676 insertions(+) create mode 100644 Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs create mode 100644 Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs.meta diff --git a/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs b/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs new file mode 100644 index 0000000000..7fd5e3eac1 --- /dev/null +++ b/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs @@ -0,0 +1,328 @@ +using System; +using System.Collections.Generic; +using System.IO; +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Tests the per-test-mode record of the last completed run that --rerun-failed reads. + /// + public sealed class RunTestsLastRunRecordStoreTests + { + private const string CompletedAt = "2026-01-02T03:04:05.0000000Z"; + + private string _recordDirectory; + private RunTestsLastRunRecordStore _store; + + [SetUp] + public void SetUp() + { + _recordDirectory = Path.Combine(Path.GetTempPath(), "uloop-last-run-" + Guid.NewGuid().ToString("N")); + _store = new RunTestsLastRunRecordStore(_recordDirectory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_recordDirectory)) + { + Directory.Delete(_recordDirectory, true); + } + } + + /// + /// What: a written record reads back as Found with the same fields and the same target order. + /// + [Test] + public void Read_AfterTryWrite_ReturnsFoundWithWrittenFields() + { + string[] targets = { "Ns.C.Second", "Ns.C.M(1,\"a\")", "Ns.FixtureWhoseOneTimeTearDownThrew" }; + + bool written = _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, targets); + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + Assert.That(written, Is.True); + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found)); + Assert.That(read.Record.FormatVersion, Is.EqualTo(1)); + Assert.That(read.Record.TestMode, Is.EqualTo("EditMode")); + Assert.That(read.Record.CompletedAt, Is.EqualTo(CompletedAt)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(targets)); + } + + /// + /// What: a second write replaces the first record instead of failing on the existing file. + /// + [Test] + public void Read_AfterSecondTryWrite_ReturnsSecondRecord() + { + _store.TryWrite(UnityCliLoopTestMode.EditMode, "2026-01-01T00:00:00.0000000Z", new[] { "Ns.C.Old" }); + + bool written = _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.New" }); + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + Assert.That(written, Is.True); + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found)); + Assert.That(read.Record.CompletedAt, Is.EqualTo(CompletedAt)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(new[] { "Ns.C.New" })); + } + + /// + /// What: an EditMode record is not visible to a PlayMode read. + /// + [Test] + public void Read_ForOtherTestMode_ReturnsMissing() + { + _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.M" }); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.PlayMode); + + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + Assert.That(read.Record, Is.Null); + } + + /// + /// What: a test mode that was never recorded reads as Missing. + /// + [Test] + public void Read_WithoutRecordFile_ReturnsMissing() + { + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + Assert.That(read.Record, Is.Null); + } + + /// + /// What: the valid fixture that the Unreadable tests break one field of reads as Found. + /// + [Test] + public void Read_WithValidFixture_ReturnsFound() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, CreateRecordJson()); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(new[] { "Ns.C.M" })); + } + + /// + /// What: a record file that is not JSON reads as Unreadable. + /// + [Test] + public void Read_WithInvalidJson_ReturnsUnreadable() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, "{ \"FormatVersion\": 1, "); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + AssertUnreadable(read, "invalid JSON"); + } + + /// + /// What: a record written by another format version reads as Unreadable. + /// + [Test] + public void Read_WithUnsupportedFormatVersion_ReturnsUnreadable() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, CreateRecordJson(formatVersion: "2")); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + AssertUnreadable(read, "FormatVersion 2"); + } + + /// + /// What: a record whose TestMode differs from the file it was read for reads as Unreadable. + /// + [Test] + public void Read_WithMismatchedTestMode_ReturnsUnreadable() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, CreateRecordJson(testMode: "\"PlayMode\"")); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + AssertUnreadable(read, "PlayMode"); + } + + /// + /// What: a record without RerunTargets reads as Unreadable instead of as nothing to rerun. + /// + [Test] + public void Read_WithoutRerunTargets_ReturnsUnreadable() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, CreateRecordJson(rerunTargets: null)); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + AssertUnreadable(read, "RerunTargets is missing"); + } + + /// + /// What: a record with an empty target name reads as Unreadable. + /// + [Test] + public void Read_WithEmptyRerunTargetName_ReturnsUnreadable() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, CreateRecordJson(rerunTargets: "[\"Ns.C.M\", \"\"]")); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + AssertUnreadable(read, "empty name"); + } + + /// + /// What: a record without CompletedAt reads as Unreadable. + /// + [Test] + public void Read_WithoutCompletedAt_ReturnsUnreadable() + { + WriteRawRecord(UnityCliLoopTestMode.EditMode, CreateRecordJson(completedAt: null)); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.EditMode); + + AssertUnreadable(read, "CompletedAt is missing"); + } + + /// + /// What: Delete removes the record, and deleting an already removed record does not throw. + /// + [Test] + public void Delete_AfterTryWrite_LeavesRecordMissingAndToleratesSecondDelete() + { + _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.M" }); + + _store.Delete(UnityCliLoopTestMode.EditMode); + + Assert.That( + _store.Read(UnityCliLoopTestMode.EditMode).Status, + Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + Assert.DoesNotThrow(() => _store.Delete(UnityCliLoopTestMode.EditMode)); + } + + /// + /// What: Delete before the record directory exists, as on a project's first run, does not throw. + /// + [Test] + public void Delete_WithoutRecordDirectory_DoesNotThrow() + { + Assert.That(Directory.Exists(_recordDirectory), Is.False); + + Assert.DoesNotThrow(() => _store.Delete(UnityCliLoopTestMode.EditMode)); + } + + /// + /// What: deleting the EditMode record keeps the PlayMode record. + /// + [Test] + public void Delete_OfOneTestMode_KeepsOtherTestModeRecord() + { + _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.EditModeTest" }); + _store.TryWrite(UnityCliLoopTestMode.PlayMode, CompletedAt, new[] { "Ns.C.PlayModeTest" }); + + _store.Delete(UnityCliLoopTestMode.EditMode); + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.PlayMode); + + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(new[] { "Ns.C.PlayModeTest" })); + } + + /// + /// What: Delete reports a record path it cannot clear as an I/O failure instead of ignoring it. + /// + [Test] + public void Delete_WhenRecordPathIsOccupiedByDirectory_ThrowsFileAccessFailure() + { + OccupyRecordPathWithDirectory(UnityCliLoopTestMode.EditMode); + + Exception thrown = null; + try + { + _store.Delete(UnityCliLoopTestMode.EditMode); + } + catch (Exception exception) + { + thrown = exception; + } + + Assert.That( + thrown is IOException || thrown is UnauthorizedAccessException, + Is.True, + "Expected IOException or UnauthorizedAccessException but got: " + thrown); + } + + /// + /// What: writing a new record and then replacing it leaves no temporary file behind. + /// + [Test] + public void TryWrite_WhenCreatingAndReplacingRecord_LeavesNoTemporaryFile() + { + _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.First" }); + _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.Second" }); + + Assert.That(Directory.GetFiles(_recordDirectory, "*.tmp"), Is.Empty); + } + + /// + /// What: a write that cannot reach the record path returns false without throwing or leaving a temporary file. + /// + [Test] + public void TryWrite_WhenRecordPathIsOccupiedByDirectory_ReturnsFalseAndLeavesNoTemporaryFile() + { + OccupyRecordPathWithDirectory(UnityCliLoopTestMode.EditMode); + + bool written = _store.TryWrite(UnityCliLoopTestMode.EditMode, CompletedAt, new[] { "Ns.C.M" }); + + Assert.That(written, Is.False); + Assert.That(Directory.GetFiles(_recordDirectory, "*.tmp"), Is.Empty); + } + + private void WriteRawRecord(UnityCliLoopTestMode testMode, string json) + { + Directory.CreateDirectory(_recordDirectory); + File.WriteAllText(_store.GetRecordPath(testMode), json); + } + + private void OccupyRecordPathWithDirectory(UnityCliLoopTestMode testMode) + { + string recordPath = _store.GetRecordPath(testMode); + Directory.CreateDirectory(recordPath); + File.WriteAllText(Path.Combine(recordPath, "keep.txt"), "occupied"); + } + + // Builds a valid record and lets each caller break or omit exactly one member; null omits it. + private static string CreateRecordJson( + string formatVersion = "1", + string testMode = "\"EditMode\"", + string completedAt = "\"" + CompletedAt + "\"", + string rerunTargets = "[\"Ns.C.M\"]") + { + List members = new(); + AddMember(members, "FormatVersion", formatVersion); + AddMember(members, "TestMode", testMode); + AddMember(members, "CompletedAt", completedAt); + AddMember(members, "RerunTargets", rerunTargets); + return "{" + string.Join(",", members) + "}"; + } + + private static void AddMember(List members, string name, string jsonValue) + { + if (jsonValue == null) + { + return; + } + + members.Add("\"" + name + "\":" + jsonValue); + } + + private static void AssertUnreadable(RunTestsLastRunRecordReadResult read, string expectedReasonFragment) + { + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Unreadable)); + Assert.That(read.Record, Is.Null); + Assert.That(read.UnreadableReason, Does.Contain(expectedReasonFragment)); + } + } +} diff --git a/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs.meta b/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs.meta new file mode 100644 index 0000000000..399d03942e --- /dev/null +++ b/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b1561a325974d4dfea32cf37a8613801 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs new file mode 100644 index 0000000000..1d5a62f9f6 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs @@ -0,0 +1,95 @@ +using System; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// JSON shape of the record that a completed run-tests execution leaves for --rerun-failed. + /// + internal sealed class RunTestsLastRunRecord + { + internal const int CurrentFormatVersion = 1; + + public int FormatVersion { get; set; } + + public string TestMode { get; set; } + + public string CompletedAt { get; set; } + + /// + /// Names to pass to the Unity testNames filter to rerun every failed or inconclusive test of the run. + /// + public string[] RerunTargets { get; set; } + } + + /// + /// Outcome of reading the last-run record of one test mode. + /// + internal enum RunTestsLastRunRecordReadStatus + { + Missing, + Unreadable, + Found + } + + /// + /// Result of reading the last-run record: Found carries the record, Unreadable carries the reason. + /// + internal readonly struct RunTestsLastRunRecordReadResult + { + public RunTestsLastRunRecordReadStatus Status { get; } + + /// + /// The record when Status is Found; null otherwise. + /// + public RunTestsLastRunRecord Record { get; } + + /// + /// Why the record was rejected when Status is Unreadable; empty otherwise. + /// + public string UnreadableReason { get; } + + private RunTestsLastRunRecordReadResult( + RunTestsLastRunRecordReadStatus status, + RunTestsLastRunRecord record, + string unreadableReason) + { + Status = status; + Record = record; + UnreadableReason = unreadableReason; + } + + public static RunTestsLastRunRecordReadResult Missing() + { + return new RunTestsLastRunRecordReadResult( + RunTestsLastRunRecordReadStatus.Missing, + null, + string.Empty); + } + + public static RunTestsLastRunRecordReadResult Unreadable(string reason) + { + if (string.IsNullOrEmpty(reason)) + { + throw new ArgumentException("An unreadable record must say why.", nameof(reason)); + } + + return new RunTestsLastRunRecordReadResult( + RunTestsLastRunRecordReadStatus.Unreadable, + null, + reason); + } + + public static RunTestsLastRunRecordReadResult Found(RunTestsLastRunRecord record) + { + if (record == null) + { + throw new ArgumentNullException(nameof(record)); + } + + return new RunTestsLastRunRecordReadResult( + RunTestsLastRunRecordReadStatus.Found, + record, + string.Empty); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs.meta b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs.meta new file mode 100644 index 0000000000..7e9f6f6ff6 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecord.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9b5ee770eef204ec9bd7f7df2a76de36 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs new file mode 100644 index 0000000000..041f2ba657 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs @@ -0,0 +1,220 @@ +using System; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +using io.github.hatayama.UnityCliLoop.ToolContracts; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Reads and writes the record of the most recent completed run of each test mode, which + /// --rerun-failed uses to pick the tests to run again. + /// + internal sealed class RunTestsLastRunRecordStore + { + private const string RecordFileNamePrefix = "last-run-"; + private const string RecordFileExtension = ".json"; + + private readonly string _testResultsDirectory; + + internal RunTestsLastRunRecordStore(string testResultsDirectory) + { + if (string.IsNullOrEmpty(testResultsDirectory) || !Path.IsPathRooted(testResultsDirectory)) + { + throw new ArgumentException( + "The record directory must be an absolute path.", + nameof(testResultsDirectory)); + } + + _testResultsDirectory = testResultsDirectory; + } + + /// + /// Creates the store for the open project's TestResults directory. Call on the main thread, + /// because the project root is read from Application.dataPath. + /// + internal static RunTestsLastRunRecordStore CreateForProject() + { + return new RunTestsLastRunRecordStore(Path.Combine( + UnityCliLoopPathResolver.GetProjectRoot(), + UnityCliLoopConstants.OUTPUT_ROOT_DIR, + UnityCliLoopConstants.TEST_RESULTS_DIR)); + } + + /// + /// Whether an exception is the file-system failure that record I/O reports instead of crashing. + /// + internal static bool IsFileAccessFailure(Exception exception) + { + return exception is IOException || exception is UnauthorizedAccessException; + } + + internal string GetRecordPath(UnityCliLoopTestMode testMode) + { + return Path.Combine(_testResultsDirectory, RecordFileNamePrefix + testMode + RecordFileExtension); + } + + /// + /// Reads the record of the test mode. Anything other than a complete record of that test mode + /// is Unreadable, so a caller never reruns from a record it cannot trust. + /// + internal RunTestsLastRunRecordReadResult Read(UnityCliLoopTestMode testMode) + { + string path = GetRecordPath(testMode); + if (!File.Exists(path)) + { + return RunTestsLastRunRecordReadResult.Missing(); + } + + string json; + try + { + json = File.ReadAllText(path); + } + catch (Exception exception) when (IsFileAccessFailure(exception)) + { + return RunTestsLastRunRecordReadResult.Unreadable("cannot read file: " + exception.Message); + } + + RunTestsLastRunRecord record; + try + { + record = JsonConvert.DeserializeObject(json); + } + catch (JsonException exception) + { + return RunTestsLastRunRecordReadResult.Unreadable("invalid JSON: " + exception.Message); + } + + return Validate(record, testMode); + } + + /// + /// Removes the record of the test mode; does nothing when there is none. File-system failures + /// propagate, because running on with the old record in place would let it pass as the newest. + /// + internal void Delete(UnityCliLoopTestMode testMode) + { + string path = GetRecordPath(testMode); + // Why not File.Delete unguarded: whether it ignores a TestResults directory that does not + // exist yet, as on a project's first run, differs between runtimes (the Windows API reports + // it as a missing path, not a missing file). Why also Directory.Exists: File.Exists is false + // for a directory, and a path we cannot clear must fail here instead of passing as deleted. + if (!File.Exists(path) && !Directory.Exists(path)) + { + return; + } + + File.Delete(path); + } + + /// + /// Writes the record of the test mode, replacing the previous one. Returns false and logs a + /// warning on a file-system failure instead of throwing, because the run's result is still valid. + /// + internal bool TryWrite(UnityCliLoopTestMode testMode, string completedAt, string[] rerunTargets) + { + if (rerunTargets == null) + { + throw new ArgumentNullException(nameof(rerunTargets)); + } + + string path = GetRecordPath(testMode); + // Why a unique temporary name: concurrent writers must not overwrite each other's partial file, + // and the real name must never hold a partially written record. + string temporaryPath = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try + { + Directory.CreateDirectory(_testResultsDirectory); + RunTestsLastRunRecord record = new() + { + FormatVersion = RunTestsLastRunRecord.CurrentFormatVersion, + TestMode = testMode.ToString(), + CompletedAt = completedAt, + RerunTargets = rerunTargets + }; + File.WriteAllText( + temporaryPath, + JsonConvert.SerializeObject(record, Formatting.Indented), + new UTF8Encoding(false)); + MoveIntoPlace(temporaryPath, path); + return true; + } + catch (Exception exception) when (IsFileAccessFailure(exception)) + { + DeleteTemporaryFileQuietly(temporaryPath); + UnityEngine.Debug.LogWarning("Failed to write the run-tests last-run record: " + exception.Message); + return false; + } + } + + private static RunTestsLastRunRecordReadResult Validate( + RunTestsLastRunRecord record, + UnityCliLoopTestMode testMode) + { + if (record == null) + { + return RunTestsLastRunRecordReadResult.Unreadable("empty file"); + } + + if (record.FormatVersion != RunTestsLastRunRecord.CurrentFormatVersion) + { + return RunTestsLastRunRecordReadResult.Unreadable( + "unsupported FormatVersion " + record.FormatVersion); + } + + if (record.TestMode != testMode.ToString()) + { + return RunTestsLastRunRecordReadResult.Unreadable( + "records " + record.TestMode + " instead of " + testMode); + } + + if (record.RerunTargets == null) + { + return RunTestsLastRunRecordReadResult.Unreadable("RerunTargets is missing"); + } + + if (Array.Exists(record.RerunTargets, string.IsNullOrWhiteSpace)) + { + return RunTestsLastRunRecordReadResult.Unreadable("RerunTargets contains an empty name"); + } + + if (string.IsNullOrWhiteSpace(record.CompletedAt)) + { + return RunTestsLastRunRecordReadResult.Unreadable("CompletedAt is missing"); + } + + return RunTestsLastRunRecordReadResult.Found(record); + } + + // Why two branches: File.Replace needs an existing destination, and the overwriting + // File.Move overload is missing from Unity's API compatibility levels. + private static void MoveIntoPlace(string temporaryPath, string path) + { + if (File.Exists(path)) + { + File.Replace(temporaryPath, path, null); + return; + } + + File.Move(temporaryPath, path); + } + + private static void DeleteTemporaryFileQuietly(string temporaryPath) + { + try + { + if (File.Exists(temporaryPath)) + { + File.Delete(temporaryPath); + } + } + catch (Exception exception) when (IsFileAccessFailure(exception)) + { + // Why swallow: the write already failed and is reported once; a leftover .tmp file + // never passes for a record because Read only opens the record's own name. + } + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs.meta b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs.meta new file mode 100644 index 0000000000..bb5b35b807 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 39c9ac74c67994392991543084d1c3f6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From f7e8d3c3874e535eee34fe6aae0ee1eb8a6ca4b3 Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:04:48 +0900 Subject: [PATCH 02/10] Collect the names that rerun every failed test of a run --rerun-failed needs, from a finished run, the names that make Unity's testNames filter select every failed or inconclusive test again. The response lists stay capped at ten, so the converter now collects a separate, uncapped list. - A suite that failed outside its tests (a thrown OneTimeSetUp or OneTimeTearDown) is rerun by its own name, using the same rule as FailedSuites; the root and a test assembly, whose names the filter never matches, are replaced by every leaf under them. - The GeneratedTestCase suffix Unity adds to duplicate names is stripped, because the filter compares the unsuffixed NUnit name and a suffixed name would silently drop the failure from the rerun. --- .../RunTestsTestFrameworkResultTests.cs | 276 +++++++++++++++++- .../SerializableTestResultConverter.cs | 111 ++++++- .../TestRunner/SerializableTestResult.cs | 7 + 3 files changed, 387 insertions(+), 7 deletions(-) diff --git a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs index e0db32750e..c5adb9f9c2 100644 --- a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs +++ b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs @@ -1103,6 +1103,250 @@ public void SaveTestResultAsXml_DoesNotCallAssetDatabaseRefresh() Assert.That(refreshCall, Is.Null, "SaveTestResultAsXml must not call AssetDatabase.Refresh."); } + /// + /// What: every failed leaf becomes a rerun target, beyond the cap that applies to FailedTests. + /// + [Test] + public void FromTestResult_WhenTwelveTestsFail_CollectsEveryFailedLeafAsRerunTarget() + { + List children = new List(); + for (int index = 1; index <= 12; index++) + { + children.Add(CreateTestCase("FailingTest" + index, TestResultStatus.Failed, 0.1)); + } + + ITestResultAdaptor resultAdaptor = CreateTestSuite("RootSuite", TestResultStatus.Failed, 0.1, children); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.failedTests.Length, Is.EqualTo(RunTestsConstants.FailedTestDetailsLimit)); + Assert.That(result.rerunTargetFullNames.Length, Is.EqualTo(12)); + Assert.That(result.rerunTargetFullNames[0], Is.EqualTo("Example.Tests.FailingTest1")); + Assert.That(result.rerunTargetFullNames[11], Is.EqualTo("Example.Tests.FailingTest12")); + } + + /// + /// What: only failed and inconclusive leaves become rerun targets, in depth-first order. + /// + [Test] + public void FromTestResult_WhenLeafStatusesAreMixed_CollectsFailedAndInconclusiveLeavesInOrder() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCase("PassingTest", TestResultStatus.Passed, 0.1), + CreateTestCase("FailingTest", TestResultStatus.Failed, 0.1), + CreateTestSuite( + "NestedFixture", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCase("NestedFailingTest", TestResultStatus.Failed, 0.1) + }, + ChildFailureMessage, + resultState: ChildFailureResultState), + CreateTestCase("SkippedTest", TestResultStatus.Skipped, 0.1), + CreateTestCase("InconclusiveTest", TestResultStatus.Inconclusive, 0.1) + }, + ChildFailureMessage, + resultState: ChildFailureResultState); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That( + result.rerunTargetFullNames, + Is.EqualTo(new[] + { + "Example.Tests.FailingTest", + "Example.Tests.NestedFailingTest", + "Example.Tests.InconclusiveTest" + })); + } + + /// + /// What: a fixture whose OneTimeTearDown threw becomes one rerun target, while its passed tests do not. + /// + [Test] + public void FromTestResult_WhenFixtureTearDownFails_CollectsFixtureAsRerunTarget() + { + SerializableTestResult result = SerializableTestResultConverter.FromTestResult( + CreateRunWithFailedTearDownFixture()); + + Assert.That(result.rerunTargetFullNames, Is.EqualTo(new[] { "Example.Tests.TearDownFixture" })); + } + + /// + /// What: a fixture whose OneTimeSetUp threw becomes one rerun target instead of its failed tests. + /// + [Test] + public void FromTestResult_WhenFixtureSetUpFails_CollectsOnlyFixtureAsRerunTarget() + { + string inheritedMessage = "OneTimeSetUp: " + SetUpFailureMessage; + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestSuite( + "SetUpFixture", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCase("FirstTest", TestResultStatus.Failed, 0.1, inheritedMessage), + CreateTestCase("SecondTest", TestResultStatus.Failed, 0.1, inheritedMessage) + }, + SetUpFailureMessage, + resultState: SetUpErrorResultState) + }, + ChildFailureMessage, + resultState: ChildFailureResultState); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.rerunTargetFullNames, Is.EqualTo(new[] { "Example.Tests.SetUpFixture" })); + } + + /// + /// What: a failed test-assembly suite, whose name no filter matches, is replaced by every leaf under it. + /// + [Test] + public void FromTestResult_WhenTestAssemblySuiteFailsOutsideItsTests_CollectsEveryLeafUnderIt() + { + ITestResultAdaptor assemblySuite = new FakeTestResultAdaptor( + new FakeTestAdaptor( + "Example.Tests.dll", + true, + fullName: "Library/ScriptAssemblies/Example.Tests.dll", + isTestAssembly: true), + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCase("FirstTest", TestResultStatus.Passed, 0.1), + CreateTestCase("SecondTest", TestResultStatus.Passed, 0.1) + }, + TearDownFailureMessage, + resultState: TearDownErrorResultState); + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List { assemblySuite }, + ChildFailureMessage, + resultState: ChildFailureResultState); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That( + result.rerunTargetFullNames, + Is.EqualTo(new[] { "Example.Tests.FirstTest", "Example.Tests.SecondTest" })); + } + + /// + /// What: a root that failed outside its tests, as a cancelled run does, is replaced by every leaf under it. + /// + [Test] + public void FromTestResult_WhenRootFailsOutsideItsTests_CollectsEveryLeafUnderIt() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestSuite( + "PassingFixture", + TestResultStatus.Passed, + 0.1, + new List + { + CreateTestCase("FirstTest", TestResultStatus.Passed, 0.1), + CreateTestCase("SecondTest", TestResultStatus.Passed, 0.1) + }) + }, + "Cancelled by user", + resultState: CancelledResultState); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That( + result.rerunTargetFullNames, + Is.EqualTo(new[] { "Example.Tests.FirstTest", "Example.Tests.SecondTest" })); + } + + /// + /// What: failed leaves sharing one name are rerun once under the unsuffixed name the filter compares. + /// + [Test] + public void FromTestResult_WhenDuplicateNamedLeavesFail_CollectsSharedFilterNameOnce() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCaseWithFullName("Ns.C.M(1)", -1, TestResultStatus.Failed), + CreateTestCaseWithFullName("Ns.C.M(1 GeneratedTestCase2)", 2, TestResultStatus.Failed) + }); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.rerunTargetFullNames, Is.EqualTo(new[] { "Ns.C.M(1)" })); + } + + /// + /// What: ToFilterName strips the GeneratedTestCase suffix only when it matches the test's own ChildIndex. + /// + [TestCase("Ns.C.M(1 GeneratedTestCase2)", 2, "Ns.C.M(1)")] + [TestCase("Ns.C.M GeneratedTestCase3", 3, "Ns.C.M")] + [TestCase("Ns.C.M(1 GeneratedTestCase2)", -1, "Ns.C.M(1 GeneratedTestCase2)")] + public void ToFilterName_StripsOnlySuffixOfOwnChildIndex(string fullName, int childIndex, string expected) + { + FakeTestAdaptor test = new FakeTestAdaptor("M", false, fullName: fullName, childIndex: childIndex); + + Assert.That(SerializableTestResultConverter.ToFilterName(test), Is.EqualTo(expected)); + } + + /// + /// What: a fully passed run has an empty, non-null rerun target list. + /// + [Test] + public void FromTestResult_WhenAllTestsPass_HasNoRerunTargets() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Passed, + 0.1, + new List + { + CreateTestCase("PassingTest", TestResultStatus.Passed, 0.1) + }); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.rerunTargetFullNames, Is.Not.Null); + Assert.That(result.rerunTargetFullNames, Is.Empty); + } + + /// + /// What: a run that produced no result tree has an empty, non-null rerun target list. + /// + [Test] + public void FromTestResult_WhenResultIsNull_HasNoRerunTargets() + { + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(null); + + Assert.That(result.rerunTargetFullNames, Is.Not.Null); + Assert.That(result.rerunTargetFullNames, Is.Empty); + } + // Resolves every method a method body calls, so a test can assert on the callee set // instead of on source text. private static IReadOnlyList ResolveCalledMethods(MethodInfo method) @@ -1294,6 +1538,16 @@ private static ITestResultAdaptor CreateTestCase( stackTrace); } + // Builds a leaf with a full name and ChildIndex as Unity reports them, for duplicate-name cases. + private static ITestResultAdaptor CreateTestCaseWithFullName( + string fullName, + int childIndex, + TestResultStatus status) + { + FakeTestAdaptor test = new FakeTestAdaptor(fullName, false, fullName: fullName, childIndex: childIndex); + return new FakeTestResultAdaptor(test, status, 0.1, new List()); + } + private sealed class FakeTestResultAdaptor : ITestResultAdaptor { private readonly ITestAdaptor _test; @@ -1370,16 +1624,28 @@ private sealed class FakeTestAdaptor : ITestAdaptor { private readonly string _name; private readonly bool _isSuite; - - public FakeTestAdaptor(string name, bool isSuite) + private readonly string _fullName; + private readonly int _childIndex; + private readonly bool _isTestAssembly; + + // Why childIndex defaults to -1: Unity reports -1 for every test without a duplicate name. + public FakeTestAdaptor( + string name, + bool isSuite, + string fullName = null, + int childIndex = -1, + bool isTestAssembly = false) { _name = name; _isSuite = isSuite; + _fullName = fullName ?? $"Example.Tests.{name}"; + _childIndex = childIndex; + _isTestAssembly = isTestAssembly; } public string Id => FullName; public string Name => _name; - public string FullName => $"Example.Tests.{_name}"; + public string FullName => _fullName; public int TestCaseCount => _isSuite ? 0 : 1; public bool HasChildren => false; public bool IsSuite => _isSuite; @@ -1390,7 +1656,7 @@ public FakeTestAdaptor(string name, bool isSuite) public NUnitMethodInfo Method => null; public object[] Arguments => Array.Empty(); public string[] Categories => Array.Empty(); - public bool IsTestAssembly => false; + public bool IsTestAssembly => _isTestAssembly; public TestRunState RunState => TestRunState.Runnable; public string Description => string.Empty; public string SkipReason => string.Empty; @@ -1398,7 +1664,7 @@ public FakeTestAdaptor(string name, bool isSuite) public string ParentFullName => string.Empty; public string UniqueName => FullName; public string ParentUniqueName => string.Empty; - public int ChildIndex => 0; + public int ChildIndex => _childIndex; public TestRunnerMode TestMode => TestRunnerMode.EditMode; } } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs index 4eb88b3d26..b9c1b331d8 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs @@ -1,6 +1,7 @@ #if ULOOP_HAS_TEST_FRAMEWORK using System; using System.Collections.Generic; +using System.Globalization; using UnityEditor.TestTools.TestRunner.Api; namespace io.github.hatayama.UnityCliLoop.FirstPartyTools @@ -10,6 +11,8 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools /// internal static class SerializableTestResultConverter { + private const string DuplicateNameSuffixPrefix = " GeneratedTestCase"; + private enum RunTestsResultClassification { NoTestsFound, @@ -38,7 +41,8 @@ public static SerializableTestResult FromTestResult(ITestResultAdaptor result) failedCount = 0, skippedCount = 0, inconclusiveCount = 0, - xmlPath = null + xmlPath = null, + rerunTargetFullNames = Array.Empty() }; } @@ -84,10 +88,34 @@ public static SerializableTestResult FromTestResult(ITestResultAdaptor result) failedTests = CollectFailedTestDetails(result), skippedTests = CollectSkippedTestFullNames(result), inconclusiveTests = CollectInconclusiveTestDetails(result), - failedSuites = failedSuites + failedSuites = failedSuites, + rerunTargetFullNames = CollectRerunTargetFullNames(result) }; } + /// + /// The name that selects the test in Unity's testNames filter: the full name without the + /// " GeneratedTestCase{ChildIndex}" suffix Unity adds to tell duplicate names apart. + /// + // Why strip it: the filter compares NUnit's unsuffixed name, so a suffixed name matches nothing + // and the failure would silently drop out of the rerun. Same-named siblings all run instead. + internal static string ToFilterName(ITestAdaptor test) + { + string fullName = test.FullName; + string suffix = DuplicateNameSuffixPrefix + test.ChildIndex.ToString(CultureInfo.InvariantCulture); + if (fullName.EndsWith(suffix + ")", StringComparison.Ordinal)) + { + return fullName.Substring(0, fullName.Length - suffix.Length - 1) + ")"; + } + + if (fullName.EndsWith(suffix, StringComparison.Ordinal)) + { + return fullName.Substring(0, fullName.Length - suffix.Length); + } + + return fullName; + } + /// /// Whether a finished run leaves anything to read in the NUnit XML: a failed or an /// inconclusive leaf, or a suite that failed outside its tests. @@ -377,6 +405,85 @@ private static bool HasFailedChildSuite(ITestResultAdaptor suite) return false; } + // Why no cap, unlike the detail lists: a rerun has to select every failure, not the first ten. + private static string[] CollectRerunTargetFullNames(ITestResultAdaptor root) + { + List names = new List(); + HashSet seen = new HashSet(StringComparer.Ordinal); + AppendRerunTargets(root, true, names, seen); + return names.ToArray(); + } + + private static void AppendRerunTargets( + ITestResultAdaptor result, + bool isRoot, + List names, + HashSet seen) + { + if (!result.Test.IsSuite) + { + if (result.TestStatus == TestStatus.Failed || result.TestStatus == TestStatus.Inconclusive) + { + AddRerunTarget(ToFilterName(result.Test), names, seen); + } + + return; + } + + // Why the AppendFailedSuiteDetails condition: the rerun must cover exactly the FailedSuites. + if (result.TestStatus == TestStatus.Failed && FailedOutsideItsTests(result)) + { + // Why every leaf instead: the filter matches the root by no name, and an assembly only by + // its file name while its full name is the dll path. + if (isRoot || result.Test.IsTestAssembly) + { + AppendEveryLeaf(result, names, seen); + return; + } + + // Why not descend: the suite's name already reruns everything beneath it. + AddRerunTarget(ToFilterName(result.Test), names, seen); + return; + } + + if (result.Children == null) + { + return; + } + + foreach (ITestResultAdaptor child in result.Children) + { + AppendRerunTargets(child, false, names, seen); + } + } + + private static void AppendEveryLeaf(ITestResultAdaptor result, List names, HashSet seen) + { + if (!result.Test.IsSuite) + { + AddRerunTarget(ToFilterName(result.Test), names, seen); + return; + } + + if (result.Children == null) + { + return; + } + + foreach (ITestResultAdaptor child in result.Children) + { + AppendEveryLeaf(child, names, seen); + } + } + + private static void AddRerunTarget(string name, List names, HashSet seen) + { + if (!string.IsNullOrEmpty(name) && seen.Add(name)) + { + names.Add(name); + } + } + private static string[] CollectSkippedTestFullNames(ITestResultAdaptor result) { List fullNames = new List(); diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs index 445eef796c..3e413244a1 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs @@ -45,6 +45,13 @@ public class SerializableTestResult /// public FailedTestDetail[] failedSuites; + /// + /// Names to pass to the Unity testNames filter to rerun every failed or inconclusive test of this run; not capped. Empty, never null. + /// + // Why an initializer unlike the arrays above: results built without the converter, such as + // test stubs, would otherwise hand null to the last-run record, which rejects it. + public string[] rerunTargetFullNames = Array.Empty(); + /// /// One inconclusive test leaf included in a run-tests response, with the message that names /// the assumption it could not meet. From bb1c4d77589ce5a8fd13660e1d10e89154c2c037 Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:08:21 +0900 Subject: [PATCH 03/10] Add a test-execution filter that runs a list of tests by full name --rerun-failed reruns the recorded failures in one Test Runner call, which the single-value filters cannot express. The new TestNames filter passes every name to Unity's testNames, which ORs exact full-name matches. It refuses an empty list because Unity reads empty testNames as no filter and would run every test. --- .../Tests/Editor/PlayModeTestExecuterTests.cs | 22 ++++++++++ .../Editor/RunTestsTestNamesFilterTests.cs | 44 +++++++++++++++++++ .../RunTestsTestNamesFilterTests.cs.meta | 11 +++++ .../TestFramework/PlayModeTestExecuter.cs | 3 ++ .../TestRunner/TestExecutionFilter.cs | 42 +++++++++++++++++- 5 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs create mode 100644 Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs.meta diff --git a/Assets/Tests/Editor/PlayModeTestExecuterTests.cs b/Assets/Tests/Editor/PlayModeTestExecuterTests.cs index e0b3027863..b029e048f3 100644 --- a/Assets/Tests/Editor/PlayModeTestExecuterTests.cs +++ b/Assets/Tests/Editor/PlayModeTestExecuterTests.cs @@ -91,5 +91,27 @@ public void CreateUnityFilter_WithRegexFilter_ShouldCreateCorrectFilter() // var expected = "^" + System.Text.RegularExpressions.Regex.Escape(regexPattern) + "(\\.|$)"; // Assert.That(unityFilter.groupNames, Is.EquivalentTo(new[] { expected })); } + + /// + /// What: a TestNames filter hands every name to Unity's testNames in order and sets no other filter. + /// + [Test] + public void CreateUnityFilter_WithTestNamesFilter_PassesEveryNameAsTestNames() + { + string[] testNames = { "Ns.C.Second", "Ns.C.M(1)", "Ns.FixtureWhoseOneTimeTearDownThrew" }; + TestExecutionFilter filter = TestExecutionFilter.ByTestNames(testNames); + System.Reflection.MethodInfo createUnityFilterMethod = typeof(PlayModeTestExecuter) + .GetMethod("CreateUnityFilter", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + Assert.That(createUnityFilterMethod, Is.Not.Null, "CreateUnityFilter method not found"); + + Filter unityFilter = (Filter)createUnityFilterMethod.Invoke( + null, + new object[] { TestMode.EditMode, filter }); + + Assert.That(unityFilter.testMode, Is.EqualTo(TestMode.EditMode)); + Assert.That(unityFilter.testNames, Is.EqualTo(testNames)); + Assert.That(unityFilter.groupNames, Is.Null.Or.Empty); + Assert.That(unityFilter.assemblyNames, Is.Null.Or.Empty); + } } } \ No newline at end of file diff --git a/Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs b/Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs new file mode 100644 index 0000000000..2d2975877c --- /dev/null +++ b/Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs @@ -0,0 +1,44 @@ +using System; +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Tests the preconditions of the filter that runs a list of tests by full name. + /// + public sealed class RunTestsTestNamesFilterTests + { + /// + /// What: ByTestNames rejects a null name list. + /// + [Test] + public void ByTestNames_WithNullNames_ThrowsArgumentException() + { + Assert.That(() => TestExecutionFilter.ByTestNames(null), Throws.InstanceOf()); + } + + /// + /// What: ByTestNames rejects an empty name list, which would otherwise run every test. + /// + [Test] + public void ByTestNames_WithNoNames_ThrowsArgumentException() + { + Assert.That( + () => TestExecutionFilter.ByTestNames(Array.Empty()), + Throws.InstanceOf()); + } + + /// + /// What: ByTestNames rejects a whitespace-only name in the list. + /// + [Test] + public void ByTestNames_WithWhitespaceName_ThrowsArgumentException() + { + Assert.That( + () => TestExecutionFilter.ByTestNames(new[] { "Ns.C.M", " " }), + Throws.InstanceOf()); + } + } +} diff --git a/Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs.meta b/Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs.meta new file mode 100644 index 0000000000..a82345e2ce --- /dev/null +++ b/Assets/Tests/Editor/RunTestsTestNamesFilterTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 1be7468fb691246f883f396d5fd0aead +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs index 3700a2bdca..19b63398fc 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs @@ -235,6 +235,9 @@ private static Filter CreateUnityFilter(TestMode testMode, TestExecutionFilter f case TestExecutionFilterType.AssemblyName: unityFilter.assemblyNames = new[] { filter.FilterValue }; break; + case TestExecutionFilterType.TestNames: + unityFilter.testNames = filter.FilterValues; + break; } return unityFilter; diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/TestExecutionFilter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/TestExecutionFilter.cs index 54d27bc601..98c8ee47c1 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/TestExecutionFilter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/TestExecutionFilter.cs @@ -1,3 +1,4 @@ +using System; using System.Text.RegularExpressions; using UnityEngine; @@ -26,7 +27,12 @@ public enum TestExecutionFilterType /// /// Filter by assembly name. /// - AssemblyName + AssemblyName, + + /// + /// Runs the tests whose full name equals any of FilterValues; used by --rerun-failed. + /// + TestNames } /// @@ -48,14 +54,20 @@ public class TestExecutionFilter /// • When FilterType is AssemblyName: "UnityCLILoop.Tests.Editor" /// public string FilterValue { get; } + + /// + /// The full names to run when FilterType is TestNames; empty for every other filter type. + /// + public string[] FilterValues { get; } /// /// Creates a test execution filter. /// - public TestExecutionFilter(TestExecutionFilterType filterType, string filterValue) + public TestExecutionFilter(TestExecutionFilterType filterType, string filterValue, string[] filterValues = null) { FilterType = filterType; FilterValue = filterValue; + FilterValues = filterValues ?? Array.Empty(); } /// @@ -103,6 +115,32 @@ public static TestExecutionFilter ByAssemblyName(string assemblyName) return new TestExecutionFilter(TestExecutionFilterType.AssemblyName, assemblyName); } + /// + /// Creates a filter that runs exactly the tests with these full names, used by --rerun-failed. + /// + public static TestExecutionFilter ByTestNames(string[] testNames) + { + if (testNames == null) + { + throw new ArgumentNullException(nameof(testNames)); + } + + // Why reject an empty list: Unity treats empty testNames as no filter and runs every test. + if (testNames.Length == 0) + { + throw new ArgumentException("At least one test name is required.", nameof(testNames)); + } + + if (Array.Exists(testNames, string.IsNullOrWhiteSpace)) + { + throw new ArgumentException("Test names must not be empty.", nameof(testNames)); + } + + // Why copy: the names were validated here, so later edits to the caller's array must not reach the filter. + string[] copiedTestNames = (string[])testNames.Clone(); + return new TestExecutionFilter(TestExecutionFilterType.TestNames, string.Empty, copiedTestNames); + } + /// /// Creates a filter to run all tests without any filter. /// From f2ec1965b59ec58c31525161dbaf83c0fc117abb Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:11:56 +0900 Subject: [PATCH 04/10] Add the RerunFailed parameter and the rerun response fields The parameter and the response shape come first so the use case can be built against them. RerunTargetCount and RerunSourceCompletedAt appear only on a --rerun-failed run, and NothingToRerun is a successful status for a rerun whose recorded run had no failures, so an agent can tell "nothing left to fix" from a failure without running anything. The embedded tool catalog does not list RerunFailed yet; it is regenerated together with the skill documentation. --- .../Editor/RunTestsResponseContractTests.cs | 63 +++++++++++++++++++ .../RunTests/RunTestsResponse.cs | 46 ++++++++++++++ .../RunTests/RunTestsSchema.cs | 5 ++ .../UnityCliLoopTestExecutionTypes.cs | 1 + 4 files changed, 115 insertions(+) diff --git a/Assets/Tests/Editor/RunTestsResponseContractTests.cs b/Assets/Tests/Editor/RunTestsResponseContractTests.cs index ea91d196e5..bf3598e3b9 100644 --- a/Assets/Tests/Editor/RunTestsResponseContractTests.cs +++ b/Assets/Tests/Editor/RunTestsResponseContractTests.cs @@ -439,5 +439,68 @@ public void RunTestsResponse_WhenWarningIsSet_SerializesExactPolicyFormSentence( Is.EqualTo( "2 active hot-reload change(s) were live during this test run. If script changes were imported during the run, the deferred domain reload that follows it discards every active hot-reload change, including introduced types - check 'uloop hot-reload --status' and re-apply, or run 'uloop compile' to bake them in.")); } + + /// + /// What: a response that is not a --rerun-failed run omits both rerun fields from JSON. + /// + [Test] + public void RunTestsResponse_WhenNotARerun_OmitsRerunFields() + { + RunTestsResponse response = new RunTestsResponse( + success: true, + message: "Test execution completed with status: Passed", + completedAt: "2026-01-01T00:00:00.0000000Z", + testCount: 1, + passedCount: 1, + failedCount: 0, + skippedCount: 0, + inconclusiveCount: 0, + xmlPath: string.Empty, + status: RunTestsExecutionStatus.Passed, + hasFailures: false, + noTestsFound: false, + noTestsFoundExplanation: string.Empty); + + JObject parsed = JObject.Parse( + JsonConvert.SerializeObject( + response, + Formatting.None, + UnityCliLoopJsonResponseSerializerSettings.Settings)); + + Assert.That(parsed.Property("RerunTargetCount"), Is.Null); + Assert.That(parsed.Property("RerunSourceCompletedAt"), Is.Null); + } + + /// + /// What: a rerun with no recorded failures serializes as a successful NothingToRerun response + /// with zero targets and the source record's completion time. + /// + [Test] + public void CreateNothingToRerun_WhenSerialized_ReportsSuccessWithZeroTargets() + { + const string sourceCompletedAt = "2026-01-02T03:04:05.0000000Z"; + RunTestsResponse response = RunTestsResponse.CreateNothingToRerun( + UnityCliLoopTestMode.EditMode, + sourceCompletedAt); + + JObject parsed = JObject.Parse( + JsonConvert.SerializeObject( + response, + Formatting.None, + UnityCliLoopJsonResponseSerializerSettings.Settings)); + + Assert.That(parsed.Value("Status"), Is.EqualTo("NothingToRerun")); + Assert.That(parsed.Value("Success"), Is.True); + Assert.That(parsed.Value("HasFailures"), Is.False); + Assert.That(parsed.Value("NoTestsFound"), Is.False); + Assert.That(parsed.Value("TestCount"), Is.EqualTo(0)); + Assert.That(parsed.Value("RerunTargetCount"), Is.EqualTo(0)); + Assert.That(parsed.Property("RerunSourceCompletedAt"), Is.Not.Null); + Assert.That(response.RerunSourceCompletedAt, Is.EqualTo(sourceCompletedAt)); + Assert.That( + response.Message, + Is.EqualTo( + "The EditMode run completed at 2026-01-02T03:04:05.0000000Z had no failed or inconclusive tests; nothing to rerun.")); + } } } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs index 40917099c5..d7ed78fcb6 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using Newtonsoft.Json; using UnityEngine; @@ -17,6 +18,10 @@ public class RunTestsResponse : UnityCliLoopToolResponse internal const string NoTestsFoundExplanationText = "No tests were discovered for this run. This is not a test failure; check TestMode, FilterType, and FilterValue. If newly added test scripts are never discovered, the most common cause is a missing test assembly: add an .asmdef with Test Assemblies enabled to the test folder (EditMode test assemblies target the Editor platform only), reference the assemblies under test, then run 'uloop compile' and rerun the tests. An .asmdef cannot reference the predefined Assembly-CSharp: if the code under test lives there, move it into its own .asmdef and add any package assemblies it uses to that .asmdef's references."; + // Format: TestMode, CompletedAt of the record the rerun read. + internal const string NothingToRerunMessageFormat = + "The {0} run completed at {1} had no failed or inconclusive tests; nothing to rerun."; + public static readonly string TestFrameworkUnavailableMessage = $"run-tests requires the Unity Test Framework package ({UnityCliLoopConstants.PACKAGE_NAME_TEST_FRAMEWORK}). Install it via Package Manager to use test execution."; @@ -143,6 +148,18 @@ public class RunTestsResponse : UnityCliLoopToolResponse /// public RunTestsTestAsmdefProposal ProposedTestAsmdef { get; set; } + /// + /// Number of tests and fixtures a --rerun-failed run asked Unity to run. Null otherwise; omitted from JSON. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public int? RerunTargetCount { get; set; } + + /// + /// CompletedAt of the recorded run whose failures --rerun-failed reran. Null otherwise; omitted from JSON. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string RerunSourceCompletedAt { get; set; } + public bool ShouldSerializeProposedTestAsmdef() { return ProposedTestAsmdef != null; @@ -230,5 +247,34 @@ public static RunTestsResponse CreateTestFrameworkUnavailable() noTestsFound: false, noTestsFoundExplanation: string.Empty); } + + /// + /// The response of a --rerun-failed run that stopped because the recorded run had no failures. + /// + internal static RunTestsResponse CreateNothingToRerun(UnityCliLoopTestMode testMode, string sourceCompletedAt) + { + return new RunTestsResponse( + success: true, + message: string.Format( + CultureInfo.InvariantCulture, + NothingToRerunMessageFormat, + testMode, + sourceCompletedAt), + completedAt: DateTime.UtcNow.ToString("o"), + testCount: 0, + passedCount: 0, + failedCount: 0, + skippedCount: 0, + inconclusiveCount: 0, + xmlPath: null, + status: RunTestsExecutionStatus.NothingToRerun, + hasFailures: false, + noTestsFound: false, + noTestsFoundExplanation: string.Empty) + { + RerunTargetCount = 0, + RerunSourceCompletedAt = sourceCompletedAt + }; + } } } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsSchema.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsSchema.cs index a6867ee4a5..23bd649395 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsSchema.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsSchema.cs @@ -45,6 +45,11 @@ public class RunTestsSchema : UnityCliLoopToolSchema /// public bool RespectEnterPlayModeSettings { get; set; } = false; + /// + /// When true, reruns only the tests that failed or were inconclusive in the most recent completed run of the same TestMode, plus whole fixtures whose OneTimeSetUp or OneTimeTearDown failed. Cannot be combined with FilterType or FilterValue. + /// + public bool RerunFailed { get; set; } = false; + /// /// Internal request identifier used for delayed result recovery across domain reload. /// diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs b/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs index 4e37951b16..25220d8d56 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs @@ -43,5 +43,6 @@ internal static class RunTestsExecutionStatus public const string Inconclusive = "Inconclusive"; public const string NoTestsFound = "NoTestsFound"; public const string ExecutionFailed = "ExecutionFailed"; + public const string NothingToRerun = "NothingToRerun"; } } From 1c689d8b327cacaf0c4b1c50746c4d695cbb3bf7 Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:24:35 +0900 Subject: [PATCH 05/10] Rerun the recorded failures and record each completed run run-tests now keeps the last-run record of each test mode current, and --rerun-failed runs the tests that record names instead of the filter. - The record is removed before a run starts and rewritten only once the run has a result, so a timed-out or cancelled run leaves no stale failures behind. Removal happens before pause points are cleared, so a record that cannot be removed stops the request with nothing changed. - A --rerun-failed request that conflicts with a filter, finds no readable record, or has nothing to rerun returns before touching the record or the pause points. - A rerun whose recorded tests are all gone says they were renamed or removed instead of giving the no-test-assembly advice. - The use case now requires the record store, so tests cannot rewrite the project's own record by forgetting it. --- Assets/Tests/Editor/RunTestsUseCaseTests.cs | 657 ++++++++++++++++++ .../RunTests/RunTestsRerunFailedResolver.cs | 161 +++++ .../RunTestsRerunFailedResolver.cs.meta | 11 + .../RunTests/RunTestsUseCase.cs | 129 +++- 4 files changed, 944 insertions(+), 14 deletions(-) create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs.meta diff --git a/Assets/Tests/Editor/RunTestsUseCaseTests.cs b/Assets/Tests/Editor/RunTestsUseCaseTests.cs index 4a65888bdd..0ff44a6a28 100644 --- a/Assets/Tests/Editor/RunTestsUseCaseTests.cs +++ b/Assets/Tests/Editor/RunTestsUseCaseTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -14,6 +15,29 @@ namespace io.github.hatayama.UnityCliLoop.Tests.Editor /// public class RunTestsUseCaseTests { + private const string RecordedCompletedAt = "2026-01-02T03:04:05.0000000Z"; + + // Why a temporary store for every use case: ExecuteAsync rewrites the last-run record, and the + // project's own record must not be replaced by stub results while uloop runs these tests. + private string _recordDirectory; + private RunTestsLastRunRecordStore _recordStore; + + [SetUp] + public void SetUp() + { + _recordDirectory = Path.Combine(Path.GetTempPath(), "uloop-last-run-" + Guid.NewGuid().ToString("N")); + _recordStore = new RunTestsLastRunRecordStore(_recordDirectory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_recordDirectory)) + { + Directory.Delete(_recordDirectory, true); + } + } + [Test] public async Task ExecuteAsync_WithInvalidExecutionState_ShouldFailFastWithoutRunningTests() { @@ -25,6 +49,7 @@ public async Task ExecuteAsync_WithInvalidExecutionState_ShouldFailFastWithoutRu new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -60,6 +85,7 @@ public async Task ExecuteAsync_WithNonPositiveTimeoutSeconds_ShouldFailFastWitho new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -86,6 +112,7 @@ public async Task ExecuteAsync_WithTimeoutSecondsAboveMax_ShouldFailFastWithoutR new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -112,6 +139,7 @@ public async Task ExecuteAsync_WithUnknownTestMode_ShouldFailFastWithoutRunningT new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -139,6 +167,7 @@ public async Task ExecuteAsync_WhenPlayModeRespectsEnterPlayModeSettings_Forward new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -167,6 +196,7 @@ public async Task ExecuteAsync_WhenEditModeRespectsEnterPlayModeSettings_DoesNot new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -193,6 +223,7 @@ public async Task ExecuteAsync_WithDefaultRequest_ShouldUseSaveUnsavedChangesMod new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new(); @@ -217,6 +248,7 @@ public async Task ExecuteAsync_WhenTestFrameworkUnavailable_ShouldFailFastWithou new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -265,6 +297,7 @@ public async Task ExecuteAsync_WhenNoTestsWereFound_ShouldExposeNoTestsFoundStat new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -301,6 +334,7 @@ public async Task ExecuteAsync_WithUnsupportedFilterType_ShouldFailFastWithoutRu new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait ); RunTestsSchema parameters = new() @@ -329,6 +363,7 @@ public async Task ExecuteAsync_AfterTestExecution_ShouldWaitForCleanup() new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: ct => { ct.ThrowIfCancellationRequested(); @@ -356,6 +391,7 @@ public async Task ExecuteAsync_WhenValidationFails_ShouldNotWaitForCleanup() new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: ct => { ct.ThrowIfCancellationRequested(); @@ -385,6 +421,7 @@ public async Task ExecuteAsync_WhenTestFrameworkUnavailable_ShouldNotWaitForClea new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: ct => { ct.ThrowIfCancellationRequested(); @@ -411,6 +448,7 @@ public async Task ExecuteAsync_AfterValidationPasses_ShouldClearActivePausePoint new TestFilterCreationService(), executionService, validationService, + _recordStore, clearActivePausePoints: () => { clearCalled = true; @@ -438,6 +476,7 @@ public async Task ExecuteAsync_WhenValidationFails_ShouldNotClearPausePoints() new TestFilterCreationService(), executionService, validationService, + _recordStore, clearActivePausePoints: () => { clearCalled = true; @@ -463,6 +502,7 @@ public async Task ExecuteAsync_WhenPausePointsCleared_ShouldReportClearedIdsInRe new TestFilterCreationService(), executionService, validationService, + _recordStore, clearActivePausePoints: () => new[] { "Assets/Scripts/Foo.cs:42", "my-custom-marker" }, waitForTestRunnerCleanupAsync: NoCleanupWait ); @@ -484,6 +524,7 @@ public async Task ExecuteAsync_WhenNoPausePointsActive_ShouldNotReportClearedIds new TestFilterCreationService(), executionService, validationService, + _recordStore, clearActivePausePoints: () => null, waitForTestRunnerCleanupAsync: NoCleanupWait ); @@ -532,6 +573,7 @@ public async Task ExecuteAsync_WhenTestsFail_CopiesFailedTestDetailsOntoResponse new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -579,6 +621,7 @@ public async Task ExecuteAsync_WhenTestsAreSkipped_CopiesSkippedTestFullNamesOnt new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -600,6 +643,7 @@ public async Task ExecuteAsync_WhenNoTestsAreSkipped_LeavesSkippedTestsNull() new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -649,6 +693,7 @@ public async Task ExecuteAsync_WhenElevenTestsFail_AppendsTruncationNoteToMessag new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -702,6 +747,7 @@ public async Task ExecuteAsync_WhenExactlyTenTestsFail_LeavesMessageWithoutTrunc new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -726,6 +772,7 @@ public async Task ExecuteAsync_WhenNoTestsFail_LeavesFailedTestsNull() new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -747,6 +794,7 @@ public async Task ExecuteAsync_WhenHotReloadChangesAreLive_AssignsExactPolicyFor new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, getActiveHotReloadChangeCount: () => 2); RunTestsSchema parameters = new RunTestsSchema(); @@ -772,6 +820,7 @@ public async Task ExecuteAsync_WhenNoHotReloadChangesAreLive_LeavesWarningEmpty( new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, getActiveHotReloadChangeCount: () => 0); RunTestsSchema parameters = new RunTestsSchema(); @@ -796,6 +845,7 @@ public async Task ExecuteAsync_WhenValidationFails_DoesNotQueryHotReloadChangeCo new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, getActiveHotReloadChangeCount: () => { @@ -828,6 +878,7 @@ public async Task ExecuteAsync_WhenDefaultCoordinationGetterReturnsThree_Assigns new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema(); @@ -861,6 +912,7 @@ public async Task ExecuteAsync_WhenRunTimesOutWithLiveHotReloadChanges_AssignsEx new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, getActiveHotReloadChangeCount: () => 2); RunTestsSchema parameters = new RunTestsSchema(); @@ -890,6 +942,7 @@ public async Task ExecuteAsync_WhenRunTimesOutWithoutLiveHotReloadChanges_Leaves new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, getActiveHotReloadChangeCount: () => 0); RunTestsSchema parameters = new RunTestsSchema(); @@ -915,6 +968,7 @@ public async Task ExecuteAsync_WhenValidationFails_LeavesWarningEmpty() new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, getActiveHotReloadChangeCount: () => 2); RunTestsSchema parameters = new RunTestsSchema(); @@ -936,6 +990,7 @@ public async Task ExecuteAsync_WithUnsupportedFilterType_ShouldNotClearPausePoin new TestFilterCreationService(), executionService, validationService, + _recordStore, clearActivePausePoints: () => { clearCalled = true; @@ -989,6 +1044,7 @@ public async Task ExecuteAsync_WhenCleanupResumesOffThread_RunsNoTestsDiagnostic new TestFilterCreationService(), executionService, validationService, + _recordStore, waitForTestRunnerCleanupAsync: cleanupResume.ResumeAsync, appendNoTestsDiagnostics: diagnosticCapture.Append); RunTestsSchema parameters = new RunTestsSchema @@ -1025,6 +1081,7 @@ public async Task ExecuteAsync_WhenFilteredNoTestsFound_EchoesUnfilteredNamesAnd new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait); RunTestsSchema parameters = new RunTestsSchema { @@ -1068,6 +1125,7 @@ public async Task ExecuteAsync_WhenFilterAllNoTestsFound_OmitsUnfilteredEchoFiel new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, appendNoTestsDiagnostics: PassThroughNoTestsDiagnostics); RunTestsSchema parameters = new RunTestsSchema @@ -1101,6 +1159,7 @@ public async Task ExecuteAsync_WhenFilterAllNoTestsFoundWithoutTestAsmdef_Attach new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, appendNoTestsDiagnostics: PassThroughNoTestsDiagnostics, proposeTestAsmdef: _ => proposal); @@ -1132,6 +1191,7 @@ public async Task ExecuteAsync_WhenFilterAllNoTestsFoundWithTestAsmdef_LeavesRes new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, appendNoTestsDiagnostics: PassThroughNoTestsDiagnostics, proposeTestAsmdef: _ => null); @@ -1164,6 +1224,7 @@ public async Task ExecuteAsync_WhenFilteredNoTestsFound_DoesNotProposeTestAsmdef new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, proposeTestAsmdef: _ => { @@ -1205,6 +1266,7 @@ public async Task ExecuteAsync_WhenFilterAllNoTestsFoundWithPredefinedAssemblyTe new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, appendNoTestsDiagnostics: PassThroughNoTestsDiagnostics); RunTestsSchema parameters = new RunTestsSchema @@ -1238,6 +1300,7 @@ public async Task ExecuteAsync_WhenFilterAllNoTestsFoundWithAsmdefHintAndPredefi new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, appendNoTestsDiagnostics: AppendPeriodTerminatedAsmdefHint); RunTestsSchema parameters = new RunTestsSchema @@ -1269,6 +1332,7 @@ public async Task ExecuteAsync_WhenFilterAllNoTestsFoundWithNoPredefinedAssembly new TestFilterCreationService(), executionService, new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, waitForTestRunnerCleanupAsync: NoCleanupWait, appendNoTestsDiagnostics: PassThroughNoTestsDiagnostics); RunTestsSchema parameters = new RunTestsSchema @@ -1319,6 +1383,589 @@ private static string PassThroughNoTestsDiagnostics( return message; } + /// + /// What: a completed run replaces the old record of its test mode with this run's failures. + /// + [Test] + public async Task ExecuteAsync_WhenRunCompletes_RecordsFailedTestsForTestMode() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateCompletedRunResult(RunTestsExecutionStatus.Failed, 3, 2, "Ns.C.FailA", "Ns.C.FailB") + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + await ExecuteToCompletionAsync(useCase, new RunTestsSchema()); + + AssertRecord(UnityCliLoopTestMode.EditMode, RunCompletedAt, "Ns.C.FailA", "Ns.C.FailB"); + } + + /// + /// What: a run that times out leaves no record, so the next --rerun-failed cannot rerun stale failures. + /// + [Test] + public async Task ExecuteAsync_WhenRunTimesOut_LeavesNoRecord() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + ThrowsExecutionTimeout = true + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync(useCase, new RunTestsSchema()); + + Assert.That(response.Success, Is.False); + AssertNoRecord(UnityCliLoopTestMode.EditMode); + } + + /// + /// What: a request cancelled before it starts leaves the existing record as it was. + /// + [Test] + public async Task ExecuteAsync_WhenCancelledBeforeStart_LeavesRecordUntouched() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + using CancellationTokenSource parent = new CancellationTokenSource(); + parent.Cancel(); + + bool canceled = await ExecuteExpectingCancellationAsync(useCase, new RunTestsSchema(), parent.Token); + + Assert.That(canceled, Is.True); + Assert.That(executionService.WasCalled, Is.False); + AssertRecord(UnityCliLoopTestMode.EditMode, RecordedCompletedAt, "Ns.Old.StaleFailure"); + } + + /// + /// What: a request cancelled while the tests run leaves no record. + /// + [Test] + public async Task ExecuteAsync_WhenParentCancelsDuringRun_LeavesNoRecord() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + using CancellationTokenSource parent = new CancellationTokenSource(); + StubTestExecutionService executionService = new StubTestExecutionService + { + OnExecuteStarted = () => parent.Cancel() + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + bool canceled = await ExecuteExpectingCancellationAsync(useCase, new RunTestsSchema(), parent.Token); + + Assert.That(canceled, Is.True); + AssertNoRecord(UnityCliLoopTestMode.EditMode); + } + + /// + /// What: a request cancelled during the cleanup wait after the run finished keeps that run's record. + /// + [Test] + public async Task ExecuteAsync_WhenParentCancelsDuringCleanupWait_KeepsRecordOfCompletedRun() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + using CancellationTokenSource parent = new CancellationTokenSource(); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateCompletedRunResult(RunTestsExecutionStatus.Failed, 2, 1, "Ns.C.FailA") + }; + RunTestsUseCase useCase = CreateRecordingUseCase( + executionService, + waitForTestRunnerCleanupAsync: ct => + { + parent.Cancel(); + ct.ThrowIfCancellationRequested(); + return Task.CompletedTask; + }); + + bool canceled = await ExecuteExpectingCancellationAsync(useCase, new RunTestsSchema(), parent.Token); + + Assert.That(canceled, Is.True); + AssertRecord(UnityCliLoopTestMode.EditMode, RunCompletedAt, "Ns.C.FailA"); + } + + /// + /// What: a record that cannot be removed stops the request before it runs tests or clears pause points. + /// + [Test] + public async Task ExecuteAsync_WhenRecordCannotBeCleared_DoesNotRun() + { + OccupyRecordPathWithDirectory(UnityCliLoopTestMode.EditMode); + bool pausePointsCleared = false; + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase( + executionService, + clearActivePausePoints: () => + { + pausePointsCleared = true; + return new[] { "pause-point-1" }; + }); + + RunTestsResponse response = await ExecuteToCompletionAsync(useCase, new RunTestsSchema()); + + Assert.That(response.Success, Is.False); + Assert.That(response.Status, Is.EqualTo(RunTestsExecutionStatus.ExecutionFailed)); + Assert.That(response.Message, Does.Contain(_recordStore.GetRecordPath(UnityCliLoopTestMode.EditMode))); + Assert.That(executionService.WasCalled, Is.False); + Assert.That(pausePointsCleared, Is.False); + Assert.That(response.ClearedPausePointIds, Is.Null); + } + + /// + /// What: a completed run writes only its own test mode's record, empty when nothing failed. + /// + [Test] + public async Task ExecuteAsync_WhenRunCompletes_LeavesOtherTestModeRecordUntouched() + { + SeedRecord(UnityCliLoopTestMode.PlayMode, new[] { "Ns.Play.Failure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateCompletedRunResult(RunTestsExecutionStatus.Passed, 1, 0) + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + RunTestsSchema parameters = new RunTestsSchema + { + TestMode = UnityCliLoopTestMode.EditMode, + FilterType = TestFilterType.@class, + FilterValue = "SomeTests" + }; + + await ExecuteToCompletionAsync(useCase, parameters); + + AssertRecord(UnityCliLoopTestMode.EditMode, RunCompletedAt); + AssertRecord(UnityCliLoopTestMode.PlayMode, RecordedCompletedAt, "Ns.Play.Failure"); + } + + /// + /// What: a run that produced no result tree fails without writing a record. + /// + [Test] + public async Task ExecuteAsync_WhenRunProducesNoResult_LeavesNoRecord() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = new SerializableTestResult + { + success = false, + status = RunTestsExecutionStatus.ExecutionFailed, + noTestsFoundExplanation = string.Empty, + message = "Test execution failed: no test result was produced", + completedAt = RunCompletedAt + } + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync(useCase, new RunTestsSchema()); + + Assert.That(response.Success, Is.False); + AssertNoRecord(UnityCliLoopTestMode.EditMode); + } + + /// + /// What: a completed run whose filter matched no tests records an empty target list. + /// + [Test] + public async Task ExecuteAsync_WhenRunFindsNoTests_RecordsEmptyTargets() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.Old.StaleFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateNoTestsFoundResult() + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + RunTestsSchema parameters = new RunTestsSchema + { + FilterType = TestFilterType.@class, + FilterValue = "NoSuchTests" + }; + + RunTestsResponse response = await ExecuteToCompletionAsync(useCase, parameters); + + Assert.That(response.NoTestsFound, Is.True); + AssertRecord(UnityCliLoopTestMode.EditMode, "2026-01-01T00:00:00.0000000Z"); + } + + /// + /// What: --rerun-failed with a filter type is rejected without running or touching the record. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithFilterType_ReturnsConflictWithoutRunning() + { + await AssertRerunFilterConflictAsync(TestFilterType.@class, "SomeTests"); + } + + /// + /// What: --rerun-failed with a filter value is rejected without running or touching the record. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithFilterValue_ReturnsConflictWithoutRunning() + { + await AssertRerunFilterConflictAsync(TestFilterType.all, "Ns.C.M"); + } + + /// + /// What: a whitespace-only filter value counts as a filter value and is rejected with --rerun-failed. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithWhitespaceFilterValue_ReturnsConflictWithoutRunning() + { + await AssertRerunFilterConflictAsync(TestFilterType.all, " "); + } + + /// + /// What: a null filter value counts as no filter value, so --rerun-failed runs the recorded tests. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithNullFilterValue_RunsRecordedTests() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.FirstFailure", "Ns.C.SecondFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateCompletedRunResult(RunTestsExecutionStatus.Failed, 2, 1, "Ns.C.SecondFailure") + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + RunTestsSchema parameters = new RunTestsSchema + { + RerunFailed = true, + FilterValue = null + }; + + RunTestsResponse response = await ExecuteToCompletionAsync(useCase, parameters); + + Assert.That(executionService.ExecuteCallCount, Is.EqualTo(1)); + Assert.That(executionService.LastFilter.FilterType, Is.EqualTo(TestExecutionFilterType.TestNames)); + Assert.That(response.RerunTargetCount, Is.EqualTo(2)); + } + + /// + /// What: --rerun-failed without a record of the test mode fails without running. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithoutRecord_ReturnsFailureWithoutRunning() + { + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(response.Success, Is.False); + Assert.That(response.Status, Is.EqualTo(RunTestsExecutionStatus.ExecutionFailed)); + Assert.That( + response.Message, + Is.EqualTo( + "No completed EditMode run is recorded for this project. Run uloop run-tests without --rerun-failed first.")); + Assert.That(executionService.WasCalled, Is.False); + } + + /// + /// What: --rerun-failed with an unreadable record fails without running and leaves the file as it was. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithUnreadableRecord_ReturnsFailureWithoutRunning() + { + const string invalidRecord = "{ \"FormatVersion\": 1, "; + Directory.CreateDirectory(_recordDirectory); + string recordPath = _recordStore.GetRecordPath(UnityCliLoopTestMode.EditMode); + File.WriteAllText(recordPath, invalidRecord); + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(response.Success, Is.False); + Assert.That(response.Status, Is.EqualTo(RunTestsExecutionStatus.ExecutionFailed)); + Assert.That(response.Message, Does.StartWith("The recorded EditMode run could not be read (invalid JSON")); + Assert.That(response.Message, Does.EndWith("Run uloop run-tests without --rerun-failed.")); + Assert.That(executionService.WasCalled, Is.False); + Assert.That(File.ReadAllText(recordPath), Is.EqualTo(invalidRecord)); + } + + /// + /// What: --rerun-failed with a record of no failures succeeds as NothingToRerun without running, + /// clearing pause points, or touching the record. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithEmptyRecord_ReturnsNothingToRerun() + { + SeedRecord(UnityCliLoopTestMode.EditMode, Array.Empty()); + bool pausePointsCleared = false; + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase( + executionService, + clearActivePausePoints: () => + { + pausePointsCleared = true; + return null; + }); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(response.Success, Is.True); + Assert.That(response.Status, Is.EqualTo(RunTestsExecutionStatus.NothingToRerun)); + Assert.That(response.HasFailures, Is.False); + Assert.That(response.NoTestsFound, Is.False); + Assert.That(response.TestCount, Is.EqualTo(0)); + Assert.That(response.RerunTargetCount, Is.EqualTo(0)); + Assert.That(response.RerunSourceCompletedAt, Is.EqualTo(RecordedCompletedAt)); + Assert.That(executionService.WasCalled, Is.False); + Assert.That(pausePointsCleared, Is.False); + AssertRecord(UnityCliLoopTestMode.EditMode, RecordedCompletedAt); + } + + /// + /// What: --rerun-failed runs the recorded tests once by name, reports the rerun, and records the new failures. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWithRecord_RunsRecordedTestsAndRewritesRecord() + { + string[] recordedTargets = { "Ns.C.FirstFailure", "Ns.C.SecondFailure" }; + SeedRecord(UnityCliLoopTestMode.EditMode, recordedTargets); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateCompletedRunResult(RunTestsExecutionStatus.Failed, 2, 1, "Ns.C.SecondFailure") + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(executionService.ExecuteCallCount, Is.EqualTo(1)); + Assert.That(executionService.LastFilter.FilterType, Is.EqualTo(TestExecutionFilterType.TestNames)); + Assert.That(executionService.LastFilter.FilterValues, Is.EqualTo(recordedTargets)); + Assert.That(response.FailedCount, Is.EqualTo(1)); + Assert.That(response.RerunTargetCount, Is.EqualTo(2)); + Assert.That(response.RerunSourceCompletedAt, Is.EqualTo(RecordedCompletedAt)); + AssertRecord(UnityCliLoopTestMode.EditMode, RunCompletedAt, "Ns.C.SecondFailure"); + } + + /// + /// What: a rerun whose recorded tests no longer exist says so instead of giving the no-test-assembly advice. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWhenRecordedTestsAreGone_ReportsRenamedOrRemoved() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.FirstFailure", "Ns.C.SecondFailure" }); + bool asmdefProposed = false; + RecordingNoTestsDiagnosticCapture diagnosticCapture = new RecordingNoTestsDiagnosticCapture(); + StubTestExecutionService executionService = new StubTestExecutionService + { + NextResult = CreateNoTestsFoundResult(), + PredefinedAssemblyTestFindings = RunTestsPredefinedAssemblyTestFindings.Create( + 1, + new[] { "Assembly-CSharp: Game.Foo.Alpha" }), + UnfilteredTestListResult = RunTestsUnfilteredTestListResult.Success(new[] { "Example.Tests.Alpha" }) + }; + RunTestsUseCase useCase = CreateRecordingUseCase( + executionService, + appendNoTestsDiagnostics: diagnosticCapture.Append, + proposeTestAsmdef: _ => + { + asmdefProposed = true; + return new RunTestsTestAsmdefProposal("Assets/Tests/Editor/Game.Tests.Editor.asmdef", "{}"); + }); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(response.NoTestsFound, Is.True); + Assert.That(response.RerunTargetCount, Is.EqualTo(2)); + Assert.That( + response.Message, + Is.EqualTo( + "None of the 2 tests recorded as failed in the EditMode run completed at 2026-01-02T03:04:05.0000000Z exist any more; they were renamed or removed. Run uloop run-tests without --rerun-failed.")); + Assert.That(response.ProposedTestAsmdef, Is.Null); + Assert.That(response.UnfilteredTestNames, Is.Null); + Assert.That(diagnosticCapture.AppendCalled, Is.False); + Assert.That(asmdefProposed, Is.False); + } + + /// + /// What: --rerun-failed in PlayMode reads only the PlayMode record, even when an EditMode record exists. + /// + [Test] + public async Task ExecuteAsync_RerunFailedUsesRecordOfRequestedTestMode() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.EditModeFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema + { + TestMode = UnityCliLoopTestMode.PlayMode, + RerunFailed = true + }); + + Assert.That(response.Success, Is.False); + Assert.That( + response.Message, + Is.EqualTo( + "No completed PlayMode run is recorded for this project. Run uloop run-tests without --rerun-failed first.")); + Assert.That(executionService.WasCalled, Is.False); + } + + /// + /// What: a rerun that times out leaves no record. + /// + [Test] + public async Task ExecuteAsync_RerunFailedTimesOut_LeavesNoRecord() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.FirstFailure", "Ns.C.SecondFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService + { + ThrowsExecutionTimeout = true + }; + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(response.Success, Is.False); + Assert.That(executionService.LastFilter.FilterType, Is.EqualTo(TestExecutionFilterType.TestNames)); + AssertNoRecord(UnityCliLoopTestMode.EditMode); + } + + private async Task AssertRerunFilterConflictAsync(TestFilterType filterType, string filterValue) + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.FirstFailure" }); + StubTestExecutionService executionService = new StubTestExecutionService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + RunTestsSchema parameters = new RunTestsSchema + { + RerunFailed = true, + FilterType = filterType, + FilterValue = filterValue + }; + + RunTestsResponse response = await ExecuteToCompletionAsync(useCase, parameters); + + Assert.That(response.Success, Is.False); + Assert.That(response.Status, Is.EqualTo(RunTestsExecutionStatus.ExecutionFailed)); + Assert.That( + response.Message, + Is.EqualTo( + "--rerun-failed cannot be combined with --filter-type or --filter-value; it reruns the failures recorded for the test mode.")); + Assert.That(executionService.WasCalled, Is.False); + AssertRecord(UnityCliLoopTestMode.EditMode, RecordedCompletedAt, "Ns.C.FirstFailure"); + } + + // Builds a use case whose Editor-touching hooks are inert, so record tests change nothing outside + // the temporary record directory. + private RunTestsUseCase CreateRecordingUseCase( + StubTestExecutionService executionService, + Func clearActivePausePoints = null, + Func waitForTestRunnerCleanupAsync = null, + Func appendNoTestsDiagnostics = null, + Func proposeTestAsmdef = null) + { + return new RunTestsUseCase( + new TestFilterCreationService(), + executionService, + new StubTestExecutionStateValidationService(ValidationResult.Success()), + _recordStore, + clearActivePausePoints: clearActivePausePoints ?? (() => null), + waitForTestRunnerCleanupAsync: waitForTestRunnerCleanupAsync ?? NoCleanupWait, + appendNoTestsDiagnostics: appendNoTestsDiagnostics ?? PassThroughNoTestsDiagnostics, + getActiveHotReloadChangeCount: () => 0, + proposeTestAsmdef: proposeTestAsmdef ?? (_ => null)); + } + + // Why catch: Unity Test Framework passes an async test that ends Canceled, so an unexpected + // cancellation must fail the test instead of skipping its assertions. + private static async Task ExecuteToCompletionAsync( + RunTestsUseCase useCase, + RunTestsSchema parameters) + { + try + { + return await useCase.ExecuteAsync(parameters, CancellationToken.None); + } + catch (OperationCanceledException exception) + { + Assert.Fail("ExecuteAsync was cancelled unexpectedly: " + exception); + return null; + } + } + + private static async Task ExecuteExpectingCancellationAsync( + RunTestsUseCase useCase, + RunTestsSchema parameters, + CancellationToken ct) + { + try + { + await useCase.ExecuteAsync(parameters, ct); + } + catch (OperationCanceledException) + { + return true; + } + + return false; + } + + private const string RunCompletedAt = "2026-01-03T04:05:06.0000000Z"; + + private static SerializableTestResult CreateCompletedRunResult( + string status, + int testCount, + int failedCount, + params string[] rerunTargets) + { + return new SerializableTestResult + { + success = status == RunTestsExecutionStatus.Passed, + status = status, + hasFailures = failedCount > 0, + noTestsFound = false, + noTestsFoundExplanation = string.Empty, + message = "Test execution completed with status: " + status, + completedAt = RunCompletedAt, + testCount = testCount, + passedCount = testCount - failedCount, + failedCount = failedCount, + rerunTargetFullNames = rerunTargets + }; + } + + private void SeedRecord(UnityCliLoopTestMode testMode, string[] rerunTargets) + { + Assert.That(_recordStore.TryWrite(testMode, RecordedCompletedAt, rerunTargets), Is.True); + } + + private void OccupyRecordPathWithDirectory(UnityCliLoopTestMode testMode) + { + string recordPath = _recordStore.GetRecordPath(testMode); + Directory.CreateDirectory(recordPath); + File.WriteAllText(Path.Combine(recordPath, "keep.txt"), "occupied"); + } + + private void AssertRecord(UnityCliLoopTestMode testMode, string completedAt, params string[] rerunTargets) + { + RunTestsLastRunRecordReadResult read = _recordStore.Read(testMode); + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found), read.UnreadableReason); + Assert.That(read.Record.CompletedAt, Is.EqualTo(completedAt)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(rerunTargets)); + } + + private void AssertNoRecord(UnityCliLoopTestMode testMode) + { + Assert.That(_recordStore.Read(testMode).Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + } + private static Task NoCleanupWait(CancellationToken ct) { ct.ThrowIfCancellationRequested(); @@ -1355,6 +2002,10 @@ private sealed class StubTestExecutionService : TestExecutionService { public bool TestFrameworkAvailable { get; set; } = true; public bool WasCalled { get; private set; } + public int ExecuteCallCount { get; private set; } + public TestExecutionFilter LastFilter { get; private set; } + // Runs first in both execute methods, before the token check, so a test can cancel mid-run. + public Action OnExecuteStarted { get; set; } public bool PlayModeWasCalled { get; private set; } public bool LastRespectEnterPlayModeSettings { get; private set; } public string LastRequestId { get; private set; } @@ -1380,8 +2031,11 @@ public override Task ExecutePlayModeTestAsync( CancellationToken ct, RunTestsPlayModeRunOptions options) { + OnExecuteStarted?.Invoke(); ct.ThrowIfCancellationRequested(); WasCalled = true; + ExecuteCallCount++; + LastFilter = filter; PlayModeWasCalled = true; LastRespectEnterPlayModeSettings = options.RespectEnterPlayModeSettings; LastRequestId = options.RequestId; @@ -1393,8 +2047,11 @@ public override Task ExecutePlayModeTestAsync( public override Task ExecuteEditModeTestAsync(TestExecutionFilter filter, CancellationToken ct) { + OnExecuteStarted?.Invoke(); ct.ThrowIfCancellationRequested(); WasCalled = true; + ExecuteCallCount++; + LastFilter = filter; if (ThrowsExecutionTimeout) { throw new OperationCanceledException(); diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs new file mode 100644 index 0000000000..719c0b12a2 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs @@ -0,0 +1,161 @@ +using System; +using System.Globalization; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides from the last-run record whether a --rerun-failed request runs, and which tests it runs. + /// + internal static class RunTestsRerunFailedResolver + { + internal const string FilterConflictMessage = + "--rerun-failed cannot be combined with --filter-type or --filter-value; it reruns the failures recorded for the test mode."; + + // Format: TestMode. + internal const string NoRecordMessageFormat = + "No completed {0} run is recorded for this project. Run uloop run-tests without --rerun-failed first."; + + // Format: TestMode, the reason the record was rejected. + internal const string UnreadableRecordMessageFormat = + "The recorded {0} run could not be read ({1}). Run uloop run-tests without --rerun-failed."; + + // Format: number of recorded targets, TestMode, CompletedAt of the recorded run. + internal const string RerunTargetsMissingMessageFormat = + "None of the {0} tests recorded as failed in the {1} run completed at {2} exist any more; they were renamed or removed. Run uloop run-tests without --rerun-failed."; + + /// + /// Resolves a --rerun-failed request: the recorded failures as a TestNames filter, or the + /// response to return without running when there is nothing it can safely rerun. + /// + internal static RunTestsExecutionTarget Resolve(RunTestsSchema parameters, RunTestsLastRunRecordStore store) + { + if (parameters == null) + { + throw new ArgumentNullException(nameof(parameters)); + } + + if (store == null) + { + throw new ArgumentNullException(nameof(store)); + } + + if (!parameters.RerunFailed) + { + throw new ArgumentException("Only a --rerun-failed request can be resolved here.", nameof(parameters)); + } + + if (parameters.FilterType != TestFilterType.all || !string.IsNullOrEmpty(parameters.FilterValue)) + { + return StopWithFailure(FilterConflictMessage); + } + + RunTestsLastRunRecordReadResult read = store.Read(parameters.TestMode); + if (read.Status == RunTestsLastRunRecordReadStatus.Missing) + { + return StopWithFailure(Format(NoRecordMessageFormat, parameters.TestMode)); + } + + if (read.Status == RunTestsLastRunRecordReadStatus.Unreadable) + { + return StopWithFailure( + Format(UnreadableRecordMessageFormat, parameters.TestMode, read.UnreadableReason)); + } + + string[] rerunTargets = read.Record.RerunTargets; + if (rerunTargets.Length == 0) + { + return RunTestsExecutionTarget.Stop( + RunTestsResponse.CreateNothingToRerun(parameters.TestMode, read.Record.CompletedAt)); + } + + return RunTestsExecutionTarget.Run( + TestExecutionFilter.ByTestNames(rerunTargets), + new RunTestsRerunSource(rerunTargets.Length, read.Record.CompletedAt)); + } + + /// + /// The message for a rerun whose recorded tests all matched nothing. + /// + internal static string FormatRerunTargetsMissingMessage( + int targetCount, + UnityCliLoopTestMode testMode, + string sourceCompletedAt) + { + return Format(RerunTargetsMissingMessageFormat, targetCount, testMode, sourceCompletedAt); + } + + private static RunTestsExecutionTarget StopWithFailure(string message) + { + return RunTestsExecutionTarget.Stop( + RunTestsUseCase.CreateFailureResponse(message, RunTestsUseCase.NoHotReloadChangesObserved)); + } + + private static string Format(string format, params object[] arguments) + { + return string.Format(CultureInfo.InvariantCulture, format, arguments); + } + } + + /// + /// What a run-tests request resolved to before anything changed: the filter to run with, or the + /// response to return without running. + /// + internal sealed class RunTestsExecutionTarget + { + /// + /// The filter to run with; null runs every test of the test mode. + /// + public TestExecutionFilter Filter { get; } + + /// + /// The recorded run a --rerun-failed run reruns; null for every other run. + /// + public RunTestsRerunSource RerunSource { get; } + + /// + /// The response to return without running; null when the tests run. + /// + public RunTestsResponse EarlyResponse { get; } + + private RunTestsExecutionTarget( + TestExecutionFilter filter, + RunTestsRerunSource rerunSource, + RunTestsResponse earlyResponse) + { + Filter = filter; + RerunSource = rerunSource; + EarlyResponse = earlyResponse; + } + + public static RunTestsExecutionTarget Stop(RunTestsResponse earlyResponse) + { + if (earlyResponse == null) + { + throw new ArgumentNullException(nameof(earlyResponse)); + } + + return new RunTestsExecutionTarget(null, null, earlyResponse); + } + + public static RunTestsExecutionTarget Run(TestExecutionFilter filter, RunTestsRerunSource rerunSource) + { + return new RunTestsExecutionTarget(filter, rerunSource, null); + } + } + + /// + /// The recorded run that a --rerun-failed run reruns, as reported back in its response. + /// + internal sealed class RunTestsRerunSource + { + public int TargetCount { get; } + + public string SourceCompletedAt { get; } + + public RunTestsRerunSource(int targetCount, string sourceCompletedAt) + { + TargetCount = targetCount; + SourceCompletedAt = sourceCompletedAt; + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs.meta b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs.meta new file mode 100644 index 0000000000..f66dea70fa --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsRerunFailedResolver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: eb04176d3a1f04cc38cb8701fca17798 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs index 212810b8b4..5128388cd8 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.Threading.Tasks; using System.Threading; using UnityEngine; @@ -17,11 +18,16 @@ public class RunTestsUseCase { // Marks a failure raised before the test run started, where no domain reload could have // discarded a hot-reload patch. - private const int NoHotReloadChangesObserved = 0; + internal const int NoHotReloadChangesObserved = 0; + + // Format: path of the record that could not be removed, the reason. + private const string ClearRecordFailedMessageFormat = + "Could not clear the last-run record at {0} ({1}), so no tests were run. Remove that path and run uloop run-tests again."; private readonly TestFilterCreationService _filterService; private readonly TestExecutionService _executionService; private readonly TestExecutionStateValidationService _validationService; + private readonly RunTestsLastRunRecordStore _lastRunRecordStore; private readonly Func _appendNoTestsDiagnostics; private readonly Func _clearActivePausePoints; private readonly Func _waitForTestRunnerCleanupAsync; @@ -33,14 +39,18 @@ public RunTestsUseCase() : this( new TestFilterCreationService(), new TestExecutionService(), - new TestExecutionStateValidationService()) + new TestExecutionStateValidationService(), + RunTestsLastRunRecordStore.CreateForProject()) { } - public RunTestsUseCase( + // Why internal: the record store is internal, and a public constructor cannot take it (CS0051). + // Why the store is required: a test that forgot it would rewrite the project's own record. + internal RunTestsUseCase( TestFilterCreationService filterService, TestExecutionService executionService, TestExecutionStateValidationService validationService, + RunTestsLastRunRecordStore lastRunRecordStore, Func clearActivePausePoints = null, Func waitForTestRunnerCleanupAsync = null, Func appendNoTestsDiagnostics = null, @@ -50,9 +60,15 @@ public RunTestsUseCase( Debug.Assert(filterService != null, "filterService must not be null"); Debug.Assert(executionService != null, "executionService must not be null"); Debug.Assert(validationService != null, "validationService must not be null"); + if (lastRunRecordStore == null) + { + throw new ArgumentNullException(nameof(lastRunRecordStore)); + } + _filterService = filterService; _executionService = executionService; _validationService = validationService; + _lastRunRecordStore = lastRunRecordStore; RunTestsNoTestsDiagnosticService noTestsDiagnosticService = new RunTestsNoTestsDiagnosticService(); _appendNoTestsDiagnostics = appendNoTestsDiagnostics ?? ((string message, bool noTestsFound, UnityCliLoopTestMode testMode, TestFilterType filterType) => @@ -102,15 +118,60 @@ public async Task ExecuteAsync(RunTestsSchema parameters, Canc } // 1. Test filter creation - TestExecutionFilter filter = null; - if (parameters.FilterType != TestFilterType.all) + RunTestsExecutionTarget target = ResolveExecutionTarget(parameters); + if (target.EarlyResponse != null) { - (TestExecutionFilter createdFilter, string filterError) = _filterService.TryCreateFilter(parameters.FilterType, parameters.FilterValue); - if (filterError != null) - { - return CreateFailureResponse(filterError, NoHotReloadChangesObserved); - } - filter = createdFilter; + return target.EarlyResponse; + } + + return await RunAndBuildResponseAsync(parameters, target, ct).ConfigureAwait(false); + } + + // Why every rejection is decided here: a request that is rejected or has nothing to rerun + // must change nothing, so it has to stop before the record or the pause points are touched. + private RunTestsExecutionTarget ResolveExecutionTarget(RunTestsSchema parameters) + { + if (parameters.RerunFailed) + { + return RunTestsRerunFailedResolver.Resolve(parameters, _lastRunRecordStore); + } + + if (parameters.FilterType == TestFilterType.all) + { + return RunTestsExecutionTarget.Run(null, null); + } + + (TestExecutionFilter createdFilter, string filterError) = _filterService.TryCreateFilter(parameters.FilterType, parameters.FilterValue); + if (filterError != null) + { + return RunTestsExecutionTarget.Stop(CreateFailureResponse(filterError, NoHotReloadChangesObserved)); + } + + return RunTestsExecutionTarget.Run(createdFilter, null); + } + + private async Task RunAndBuildResponseAsync( + RunTestsSchema parameters, + RunTestsExecutionTarget target, + CancellationToken ct) + { + // Why before the run: the record must only describe a run that finished. A run that times + // out or is cancelled must leave no record, or the next --rerun-failed reruns stale failures. + // Why before clearing pause points: deleting can fail, and failing here leaves the pause + // points in place instead of clearing them for a run that never starts. + try + { + _lastRunRecordStore.Delete(parameters.TestMode); + } + catch (Exception exception) when (RunTestsLastRunRecordStore.IsFileAccessFailure(exception)) + { + return CreateFailureResponse( + string.Format( + CultureInfo.InvariantCulture, + ClearRecordFailedMessageFormat, + _lastRunRecordStore.GetRecordPath(parameters.TestMode), + exception.Message), + NoHotReloadChangesObserved); } string[] clearedPausePointIds = _clearActivePausePoints(); @@ -131,15 +192,19 @@ public async Task ExecuteAsync(RunTestsSchema parameters, Canc if (parameters.TestMode == UnityCliLoopTestMode.PlayMode) { result = await _executionService.ExecutePlayModeTestAsync( - filter, + target.Filter, executionCt, CreatePlayModeRunOptions(parameters)).ConfigureAwait(false); } else { - result = await _executionService.ExecuteEditModeTestAsync(filter, executionCt).ConfigureAwait(false); + result = await _executionService.ExecuteEditModeTestAsync(target.Filter, executionCt).ConfigureAwait(false); } + // Why before the cleanup wait: the run has finished here, so a request cancelled while + // waiting must still leave this run's record for the next --rerun-failed. + RecordCompletedRun(parameters.TestMode, result); + // Why parent ct (not executionCt): CancelAfter only guards the RunFinished wait. // Using the linked token here would mis-report a successful run as timed out when // the fixed cleanup delay straddles the deadline after RunFinished already arrived. @@ -173,6 +238,10 @@ public async Task ExecuteAsync(RunTestsSchema parameters, Canc } response.Warning = RunTestsHotReloadDiscardWarningBuilder.Build(activeHotReloadChangeCountAtStart); + if (target.RerunSource != null) + { + return ApplyRerunFields(response, parameters.TestMode, target.RerunSource); + } // Why switch here: cleanup waits with ConfigureAwait(false), so this resume is // off-thread. No-tests diagnostics call AssetDatabase.FindAssets, and the @@ -192,6 +261,38 @@ await ApplyUnfilteredFilterEchoIfNeededAsync(response, parameters, ct) return response; } + // Why not record a run without a result tree: it has no names to rerun, and recording it as + // empty would make the next --rerun-failed report nothing to rerun after a broken run. + private void RecordCompletedRun(UnityCliLoopTestMode testMode, SerializableTestResult result) + { + if (result.status == RunTestsExecutionStatus.ExecutionFailed) + { + return; + } + + _lastRunRecordStore.TryWrite(testMode, result.completedAt, result.rerunTargetFullNames); + } + + // Why a rerun skips the no-tests diagnostics: they explain an empty project or filter, while a + // rerun that finds nothing means the recorded tests were renamed or removed since that run. + private static RunTestsResponse ApplyRerunFields( + RunTestsResponse response, + UnityCliLoopTestMode testMode, + RunTestsRerunSource source) + { + response.RerunTargetCount = source.TargetCount; + response.RerunSourceCompletedAt = source.SourceCompletedAt; + if (response.NoTestsFound) + { + response.Message = RunTestsRerunFailedResolver.FormatRerunTargetsMissingMessage( + source.TargetCount, + testMode, + source.SourceCompletedAt); + } + + return response; + } + private void AppendPredefinedAssemblyTestNoticeIfNeeded( RunTestsResponse response, RunTestsSchema parameters) @@ -281,7 +382,7 @@ private static bool IsSupportedTestMode(UnityCliLoopTestMode testMode) // Why the count is a parameter: only failures raised after the run started can have cost // an active patch, so paths that fail before it pass NoHotReloadChangesObserved and stay // warning-free instead of blaming a reload that never happened. - private static RunTestsResponse CreateFailureResponse( + internal static RunTestsResponse CreateFailureResponse( string message, int activeHotReloadChangeCountAtStart) { From d32ff17d1a535ced1e62b9ee0e0e477ebba6402a Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:29:02 +0900 Subject: [PATCH 06/10] Record PlayMode runs recovered after a domain reload A PlayMode run that respects Enter Play Mode settings can reload the domain, which cancels the use case's await before it records the run. The post-reload callback is then the run's only writer, so it now writes the PlayMode record too. The callback also receives the next run's RunFinished after a run that was abandoned with its request still pending, and that run can be EditMode, so it writes only when the result's root is a PlayMode run. --- .../Editor/RunTestsLastRunRecordStoreTests.cs | 103 ++++++++++++++++++ .../RunTests/RunTestsLastRunRecordStore.cs | 33 ++++++ .../RunTestsPendingRunCallback.cs | 11 ++ 3 files changed, 147 insertions(+) diff --git a/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs b/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs index 7fd5e3eac1..b18b376659 100644 --- a/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs +++ b/Assets/Tests/Editor/RunTestsLastRunRecordStoreTests.cs @@ -280,6 +280,109 @@ public void TryWrite_WhenRecordPathIsOccupiedByDirectory_ReturnsFalseAndLeavesNo Assert.That(Directory.GetFiles(_recordDirectory, "*.tmp"), Is.Empty); } + /// + /// What: a recovered PlayMode run with a failure becomes the PlayMode record. + /// + [Test] + public void TryRecordRecoveredPlayModeRun_WithPlayModeRun_RecordsPlayModeTargets() + { + bool recorded = RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun( + _store, + CreateRecoveredResult(RunTestsExecutionStatus.Failed, "Ns.Play.Failure"), + isPlayModeRun: true); + + Assert.That(recorded, Is.True); + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.PlayMode); + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found), read.UnreadableReason); + Assert.That(read.Record.CompletedAt, Is.EqualTo(CompletedAt)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(new[] { "Ns.Play.Failure" })); + } + + /// + /// What: a recovered result of a run that was not PlayMode writes no record of either test mode. + /// + [Test] + public void TryRecordRecoveredPlayModeRun_WithEditModeRun_WritesNoRecord() + { + bool recorded = RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun( + _store, + CreateRecoveredResult(RunTestsExecutionStatus.Failed, "Ns.Edit.Failure"), + isPlayModeRun: false); + + Assert.That(recorded, Is.False); + Assert.That(_store.Read(UnityCliLoopTestMode.PlayMode).Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + Assert.That(_store.Read(UnityCliLoopTestMode.EditMode).Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + } + + /// + /// What: a recovered PlayMode result without a result tree writes no record. + /// + [Test] + public void TryRecordRecoveredPlayModeRun_WithoutResultTree_WritesNoRecord() + { + bool recorded = RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun( + _store, + CreateRecoveredResult(RunTestsExecutionStatus.ExecutionFailed), + isPlayModeRun: true); + + Assert.That(recorded, Is.False); + Assert.That(_store.Read(UnityCliLoopTestMode.PlayMode).Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Missing)); + } + + /// + /// What: a recovered result of a run that was not PlayMode leaves an existing PlayMode record as it was. + /// + [Test] + public void TryRecordRecoveredPlayModeRun_WithEditModeRun_KeepsExistingPlayModeRecord() + { + const string existingCompletedAt = "2026-01-01T00:00:00.0000000Z"; + _store.TryWrite(UnityCliLoopTestMode.PlayMode, existingCompletedAt, new[] { "Ns.Play.Existing" }); + + RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun( + _store, + CreateRecoveredResult(RunTestsExecutionStatus.Failed, "Ns.Edit.Failure"), + isPlayModeRun: false); + + RunTestsLastRunRecordReadResult read = _store.Read(UnityCliLoopTestMode.PlayMode); + Assert.That(read.Status, Is.EqualTo(RunTestsLastRunRecordReadStatus.Found), read.UnreadableReason); + Assert.That(read.Record.CompletedAt, Is.EqualTo(existingCompletedAt)); + Assert.That(read.Record.RerunTargets, Is.EqualTo(new[] { "Ns.Play.Existing" })); + } + + /// + /// What: recording a recovered run without a store is rejected as a caller error. + /// + [Test] + public void TryRecordRecoveredPlayModeRun_WithNullStore_ThrowsArgumentNullException() + { + SerializableTestResult result = CreateRecoveredResult(RunTestsExecutionStatus.Failed, "Ns.Play.Failure"); + + Assert.That( + () => RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun(null, result, isPlayModeRun: true), + Throws.ArgumentNullException); + } + + /// + /// What: recording a recovered run without a result is rejected as a caller error. + /// + [Test] + public void TryRecordRecoveredPlayModeRun_WithNullResult_ThrowsArgumentNullException() + { + Assert.That( + () => RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun(_store, null, isPlayModeRun: true), + Throws.ArgumentNullException); + } + + private static SerializableTestResult CreateRecoveredResult(string status, params string[] rerunTargets) + { + return new SerializableTestResult + { + status = status, + completedAt = CompletedAt, + rerunTargetFullNames = rerunTargets + }; + } + private void WriteRawRecord(UnityCliLoopTestMode testMode, string json) { Directory.CreateDirectory(_recordDirectory); diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs index 041f2ba657..f1d10117e9 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsLastRunRecordStore.cs @@ -149,6 +149,39 @@ internal bool TryWrite(UnityCliLoopTestMode testMode, string completedAt, string } } + /// + /// Records a run-tests result recovered after a domain reload, but only when it came from a + /// PlayMode run. Returns whether a record was written. + /// + internal static bool TryRecordRecoveredPlayModeRun( + RunTestsLastRunRecordStore store, + SerializableTestResult result, + bool isPlayModeRun) + { + if (store == null) + { + throw new ArgumentNullException(nameof(store)); + } + + if (result == null) + { + throw new ArgumentNullException(nameof(result)); + } + + // Why not record a run without a result tree: it has no names to rerun. + if (result.status == RunTestsExecutionStatus.ExecutionFailed) + { + return false; + } + + if (!isPlayModeRun) + { + return false; + } + + return store.TryWrite(UnityCliLoopTestMode.PlayMode, result.completedAt, result.rerunTargetFullNames); + } + private static RunTestsLastRunRecordReadResult Validate( RunTestsLastRunRecord record, UnityCliLoopTestMode testMode) diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs index 0c21907519..f91eb1b6ec 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs @@ -28,6 +28,17 @@ public void RunStarted(ITestAdaptor tests) public void RunFinished(ITestResultAdaptor result) { SerializableTestResult serializableResult = SerializableTestResultConverter.FromTestResult(result); + // Why record here: this callback runs only after a domain reload during a PlayMode run that + // respects Enter Play Mode settings, and that reload cancels the use case's await before it + // can record, so this is the run's only writer. + // Why check the root's TestMode: after a run abandoned with its request still pending, this + // callback receives the next run's RunFinished, which can be an EditMode run, and an EditMode + // result must not become the PlayMode record. + bool isPlayModeRun = result != null && result.Test != null && result.Test.TestMode == TestMode.PlayMode; + RunTestsLastRunRecordStore.TryRecordRecoveredPlayModeRun( + RunTestsLastRunRecordStore.CreateForProject(), + serializableResult, + isPlayModeRun); if (SerializableTestResultConverter.ShouldSaveResultXml(serializableResult)) { serializableResult.xmlPath = PlayModeTestExecuter.TrySaveFailureXml(result); From f6deb6e1ea996a4443dec891d1b5a055422160a8 Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:33:33 +0900 Subject: [PATCH 07/10] Document --rerun-failed in the run-tests skill and tool catalog Adds the --rerun-failed row and usage note to the run-tests skill, the RerunFailed property to the embedded tool catalog, and the rerun response fields and NothingToRerun status to the field reference. The response field list and the XML section move to references/response-fields.md so the skill stays well under the 8,000-byte injection cap with room for the new option; the skill keeps a pointer to the fields agents usually need. The catalog gains a property, which is a structural change to a shared release input, so both components' shared-inputs stamps are refreshed. --- .agents/skills/uloop-run-tests/SKILL.md | 30 +++-------------- .../references/response-fields.md | 32 +++++++++++++++++++ .claude/skills/uloop-run-tests/SKILL.md | 30 +++-------------- .../references/response-fields.md | 32 +++++++++++++++++++ .../FirstPartyTools/RunTests/Skill/SKILL.md | 30 +++-------------- .../Skill/references/response-fields.md | 32 +++++++++++++++++++ .../Skill/references/response-fields.md.meta | 7 ++++ cli/common/tools/default-tools.json | 5 +++ cli/dispatcher/shared-inputs-stamp.json | 2 +- cli/project-runner/shared-inputs-stamp.json | 2 +- 10 files changed, 122 insertions(+), 80 deletions(-) create mode 100644 .agents/skills/uloop-run-tests/references/response-fields.md create mode 100644 .claude/skills/uloop-run-tests/references/response-fields.md create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md create mode 100644 Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md.meta diff --git a/.agents/skills/uloop-run-tests/SKILL.md b/.agents/skills/uloop-run-tests/SKILL.md index 8e3bdd7a72..2b20b507cd 100644 --- a/.agents/skills/uloop-run-tests/SKILL.md +++ b/.agents/skills/uloop-run-tests/SKILL.md @@ -37,36 +37,14 @@ uloop run-tests [options] | `--skip-compile` | flag | - | Skip the CLI-side compile before running tests; use only while validating active hot-reload patches. Unity still imports script edits saved since the last compile. | | `--timeout-seconds` | integer | `600` | Maximum seconds to wait for RunFinished before canceling the await (max `1500`). Increase for long suites; on timeout the Test Runner may still be running until stop handling lands | | `--respect-enter-play-mode-settings` | flag | - | PlayMode only: keep the project's Enter Play Mode settings instead of forcing Domain Reload off. A Domain Reload during the run is survived; the result is recovered after the reload. Use for projects whose libraries require a Domain Reload on Play entry. | +| `--rerun-failed` | flag | - | Rerun only the tests that failed or were inconclusive in the most recent completed run of the same --test-mode (whole fixtures for a failed OneTimeSetUp/OneTimeTearDown). Cannot be combined with --filter-type or --filter-value | By default PlayMode still forces Domain Reload off. With `--respect-enter-play-mode-settings`, a Domain Reload may run and the command takes longer; pause-point and hot-reload notes are omitted from a result recovered after reload. Canceling the CLI (Ctrl-C) does not stop the Unity-side run on this path. exact matches the full test name (Namespace.Class.Method). class runs every test of one class by bare or namespace-qualified name, e.g. --filter-type class --filter-value PlayerTests; the name is matched literally and whole, so PlayerTests does not run EnemyPlayerTests. regex matches a .NET regex against full test names, e.g. --filter-type regex --filter-value '^MyGame\.Tests\.' +`--rerun-failed` reads the record that every completed run writes for its test mode, so it reruns the failures of the most recent completed run, filtered or not. A run that timed out or was cancelled leaves no record. With nothing recorded as failed it returns `Status: NothingToRerun` without running; to tell a flaky test from a real failure, run `--rerun-failed` again after a failure. + ## Output -Returns JSON with: - -- `Success` (boolean): Whether every test passed or was skipped; `false` when a test or suite failed or a test was inconclusive -- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, or `ExecutionFailed` -- `HasFailures` (boolean): Whether any discovered test or suite failed -- `Message` (string): Summary message -- `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests -- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise -- `CompletedAt` (string): ISO timestamp when the run finished -- `TestCount` (number): Total tests executed -- `PassedCount` (number): Passed tests -- `FailedCount` (number): Failed tests -- `SkippedCount` (number): Skipped tests -- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) -- `XmlPath` (string or null): Path to NUnit XML result file. `null` when no XML was saved (typically on `Success: true`); set only when a test or suite failed or a test was inconclusive and the file exists on disk. -- `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. -- `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` -- `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. -- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. -- `FailedSuites` (array, optional): Up to 10 suites that failed outside their tests (e.g. a `OneTimeSetUp` or `OneTimeTearDown` threw), with the `FailedTests` fields. The run is `Failed` even when `FailedCount` is 0. Omitted when none. -- `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. -- `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. - -### XML Result File - -Saved to `{project_root}/.uloop/outputs/TestResults/.xml`. What it records, including failed suites: `references/xml-results.md`. +Returns JSON. `Success`, `Status`, `Message`, `FailedTests` (up to 10), and `XmlPath` (set when a test or suite failed or was inconclusive) are usually enough; every field is described in `references/response-fields.md`, and the XML file in `references/xml-results.md`. diff --git a/.agents/skills/uloop-run-tests/references/response-fields.md b/.agents/skills/uloop-run-tests/references/response-fields.md new file mode 100644 index 0000000000..fef8f8421d --- /dev/null +++ b/.agents/skills/uloop-run-tests/references/response-fields.md @@ -0,0 +1,32 @@ +# run-tests response fields + +Returns JSON with: + +- `Success` (boolean): Whether every test passed or was skipped; `false` when a test or suite failed or a test was inconclusive +- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, `ExecutionFailed`, or `NothingToRerun` (`--rerun-failed` found nothing to rerun; `Success` is `true`) +- `HasFailures` (boolean): Whether any discovered test or suite failed +- `Message` (string): Summary message +- `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests +- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise +- `CompletedAt` (string): ISO timestamp when the run finished +- `TestCount` (number): Total tests executed +- `PassedCount` (number): Passed tests +- `FailedCount` (number): Failed tests +- `SkippedCount` (number): Skipped tests +- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) +- `XmlPath` (string or null): Path to NUnit XML result file. `null` when no XML was saved (typically on `Success: true`); set only when a test or suite failed or a test was inconclusive and the file exists on disk. +- `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. +- `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` +- `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. +- `FailedSuites` (array, optional): Up to 10 suites that failed outside their tests (e.g. a `OneTimeSetUp` or `OneTimeTearDown` threw), with the `FailedTests` fields. The run is `Failed` even when `FailedCount` is 0. Omitted when none. +- `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. +- `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. +- `RerunTargetCount` (number, optional): Number of tests and fixtures a `--rerun-failed` run asked Unity to run. Omitted on other runs. +- `RerunSourceCompletedAt` (string, optional): `CompletedAt` of the recorded run whose failures `--rerun-failed` reran. Omitted on other runs. + +Both are absent when `--respect-enter-play-mode-settings` entered Play Mode with a Domain Reload: the result is then recovered after the reload (see `Warning`), and a rerun whose recorded tests are all gone reports the generic no-tests message. + +## XML Result File + +Saved to `{project_root}/.uloop/outputs/TestResults/.xml`. What it records, including failed suites: `references/xml-results.md`. diff --git a/.claude/skills/uloop-run-tests/SKILL.md b/.claude/skills/uloop-run-tests/SKILL.md index 8e3bdd7a72..2b20b507cd 100644 --- a/.claude/skills/uloop-run-tests/SKILL.md +++ b/.claude/skills/uloop-run-tests/SKILL.md @@ -37,36 +37,14 @@ uloop run-tests [options] | `--skip-compile` | flag | - | Skip the CLI-side compile before running tests; use only while validating active hot-reload patches. Unity still imports script edits saved since the last compile. | | `--timeout-seconds` | integer | `600` | Maximum seconds to wait for RunFinished before canceling the await (max `1500`). Increase for long suites; on timeout the Test Runner may still be running until stop handling lands | | `--respect-enter-play-mode-settings` | flag | - | PlayMode only: keep the project's Enter Play Mode settings instead of forcing Domain Reload off. A Domain Reload during the run is survived; the result is recovered after the reload. Use for projects whose libraries require a Domain Reload on Play entry. | +| `--rerun-failed` | flag | - | Rerun only the tests that failed or were inconclusive in the most recent completed run of the same --test-mode (whole fixtures for a failed OneTimeSetUp/OneTimeTearDown). Cannot be combined with --filter-type or --filter-value | By default PlayMode still forces Domain Reload off. With `--respect-enter-play-mode-settings`, a Domain Reload may run and the command takes longer; pause-point and hot-reload notes are omitted from a result recovered after reload. Canceling the CLI (Ctrl-C) does not stop the Unity-side run on this path. exact matches the full test name (Namespace.Class.Method). class runs every test of one class by bare or namespace-qualified name, e.g. --filter-type class --filter-value PlayerTests; the name is matched literally and whole, so PlayerTests does not run EnemyPlayerTests. regex matches a .NET regex against full test names, e.g. --filter-type regex --filter-value '^MyGame\.Tests\.' +`--rerun-failed` reads the record that every completed run writes for its test mode, so it reruns the failures of the most recent completed run, filtered or not. A run that timed out or was cancelled leaves no record. With nothing recorded as failed it returns `Status: NothingToRerun` without running; to tell a flaky test from a real failure, run `--rerun-failed` again after a failure. + ## Output -Returns JSON with: - -- `Success` (boolean): Whether every test passed or was skipped; `false` when a test or suite failed or a test was inconclusive -- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, or `ExecutionFailed` -- `HasFailures` (boolean): Whether any discovered test or suite failed -- `Message` (string): Summary message -- `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests -- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise -- `CompletedAt` (string): ISO timestamp when the run finished -- `TestCount` (number): Total tests executed -- `PassedCount` (number): Passed tests -- `FailedCount` (number): Failed tests -- `SkippedCount` (number): Skipped tests -- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) -- `XmlPath` (string or null): Path to NUnit XML result file. `null` when no XML was saved (typically on `Success: true`); set only when a test or suite failed or a test was inconclusive and the file exists on disk. -- `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. -- `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` -- `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. -- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. -- `FailedSuites` (array, optional): Up to 10 suites that failed outside their tests (e.g. a `OneTimeSetUp` or `OneTimeTearDown` threw), with the `FailedTests` fields. The run is `Failed` even when `FailedCount` is 0. Omitted when none. -- `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. -- `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. - -### XML Result File - -Saved to `{project_root}/.uloop/outputs/TestResults/.xml`. What it records, including failed suites: `references/xml-results.md`. +Returns JSON. `Success`, `Status`, `Message`, `FailedTests` (up to 10), and `XmlPath` (set when a test or suite failed or was inconclusive) are usually enough; every field is described in `references/response-fields.md`, and the XML file in `references/xml-results.md`. diff --git a/.claude/skills/uloop-run-tests/references/response-fields.md b/.claude/skills/uloop-run-tests/references/response-fields.md new file mode 100644 index 0000000000..fef8f8421d --- /dev/null +++ b/.claude/skills/uloop-run-tests/references/response-fields.md @@ -0,0 +1,32 @@ +# run-tests response fields + +Returns JSON with: + +- `Success` (boolean): Whether every test passed or was skipped; `false` when a test or suite failed or a test was inconclusive +- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, `ExecutionFailed`, or `NothingToRerun` (`--rerun-failed` found nothing to rerun; `Success` is `true`) +- `HasFailures` (boolean): Whether any discovered test or suite failed +- `Message` (string): Summary message +- `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests +- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise +- `CompletedAt` (string): ISO timestamp when the run finished +- `TestCount` (number): Total tests executed +- `PassedCount` (number): Passed tests +- `FailedCount` (number): Failed tests +- `SkippedCount` (number): Skipped tests +- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) +- `XmlPath` (string or null): Path to NUnit XML result file. `null` when no XML was saved (typically on `Success: true`); set only when a test or suite failed or a test was inconclusive and the file exists on disk. +- `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. +- `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` +- `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. +- `FailedSuites` (array, optional): Up to 10 suites that failed outside their tests (e.g. a `OneTimeSetUp` or `OneTimeTearDown` threw), with the `FailedTests` fields. The run is `Failed` even when `FailedCount` is 0. Omitted when none. +- `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. +- `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. +- `RerunTargetCount` (number, optional): Number of tests and fixtures a `--rerun-failed` run asked Unity to run. Omitted on other runs. +- `RerunSourceCompletedAt` (string, optional): `CompletedAt` of the recorded run whose failures `--rerun-failed` reran. Omitted on other runs. + +Both are absent when `--respect-enter-play-mode-settings` entered Play Mode with a Domain Reload: the result is then recovered after the reload (see `Warning`), and a rerun whose recorded tests are all gone reports the generic no-tests message. + +## XML Result File + +Saved to `{project_root}/.uloop/outputs/TestResults/.xml`. What it records, including failed suites: `references/xml-results.md`. diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md index 8e3bdd7a72..2b20b507cd 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md @@ -37,36 +37,14 @@ uloop run-tests [options] | `--skip-compile` | flag | - | Skip the CLI-side compile before running tests; use only while validating active hot-reload patches. Unity still imports script edits saved since the last compile. | | `--timeout-seconds` | integer | `600` | Maximum seconds to wait for RunFinished before canceling the await (max `1500`). Increase for long suites; on timeout the Test Runner may still be running until stop handling lands | | `--respect-enter-play-mode-settings` | flag | - | PlayMode only: keep the project's Enter Play Mode settings instead of forcing Domain Reload off. A Domain Reload during the run is survived; the result is recovered after the reload. Use for projects whose libraries require a Domain Reload on Play entry. | +| `--rerun-failed` | flag | - | Rerun only the tests that failed or were inconclusive in the most recent completed run of the same --test-mode (whole fixtures for a failed OneTimeSetUp/OneTimeTearDown). Cannot be combined with --filter-type or --filter-value | By default PlayMode still forces Domain Reload off. With `--respect-enter-play-mode-settings`, a Domain Reload may run and the command takes longer; pause-point and hot-reload notes are omitted from a result recovered after reload. Canceling the CLI (Ctrl-C) does not stop the Unity-side run on this path. exact matches the full test name (Namespace.Class.Method). class runs every test of one class by bare or namespace-qualified name, e.g. --filter-type class --filter-value PlayerTests; the name is matched literally and whole, so PlayerTests does not run EnemyPlayerTests. regex matches a .NET regex against full test names, e.g. --filter-type regex --filter-value '^MyGame\.Tests\.' +`--rerun-failed` reads the record that every completed run writes for its test mode, so it reruns the failures of the most recent completed run, filtered or not. A run that timed out or was cancelled leaves no record. With nothing recorded as failed it returns `Status: NothingToRerun` without running; to tell a flaky test from a real failure, run `--rerun-failed` again after a failure. + ## Output -Returns JSON with: - -- `Success` (boolean): Whether every test passed or was skipped; `false` when a test or suite failed or a test was inconclusive -- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, or `ExecutionFailed` -- `HasFailures` (boolean): Whether any discovered test or suite failed -- `Message` (string): Summary message -- `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests -- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise -- `CompletedAt` (string): ISO timestamp when the run finished -- `TestCount` (number): Total tests executed -- `PassedCount` (number): Passed tests -- `FailedCount` (number): Failed tests -- `SkippedCount` (number): Skipped tests -- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) -- `XmlPath` (string or null): Path to NUnit XML result file. `null` when no XML was saved (typically on `Success: true`); set only when a test or suite failed or a test was inconclusive and the file exists on disk. -- `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. -- `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` -- `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. -- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. -- `FailedSuites` (array, optional): Up to 10 suites that failed outside their tests (e.g. a `OneTimeSetUp` or `OneTimeTearDown` threw), with the `FailedTests` fields. The run is `Failed` even when `FailedCount` is 0. Omitted when none. -- `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. -- `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. - -### XML Result File - -Saved to `{project_root}/.uloop/outputs/TestResults/.xml`. What it records, including failed suites: `references/xml-results.md`. +Returns JSON. `Success`, `Status`, `Message`, `FailedTests` (up to 10), and `XmlPath` (set when a test or suite failed or was inconclusive) are usually enough; every field is described in `references/response-fields.md`, and the XML file in `references/xml-results.md`. diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md new file mode 100644 index 0000000000..fef8f8421d --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md @@ -0,0 +1,32 @@ +# run-tests response fields + +Returns JSON with: + +- `Success` (boolean): Whether every test passed or was skipped; `false` when a test or suite failed or a test was inconclusive +- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, `ExecutionFailed`, or `NothingToRerun` (`--rerun-failed` found nothing to rerun; `Success` is `true`) +- `HasFailures` (boolean): Whether any discovered test or suite failed +- `Message` (string): Summary message +- `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests +- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise +- `CompletedAt` (string): ISO timestamp when the run finished +- `TestCount` (number): Total tests executed +- `PassedCount` (number): Passed tests +- `FailedCount` (number): Failed tests +- `SkippedCount` (number): Skipped tests +- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) +- `XmlPath` (string or null): Path to NUnit XML result file. `null` when no XML was saved (typically on `Success: true`); set only when a test or suite failed or a test was inconclusive and the file exists on disk. +- `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. +- `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` +- `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. +- `FailedSuites` (array, optional): Up to 10 suites that failed outside their tests (e.g. a `OneTimeSetUp` or `OneTimeTearDown` threw), with the `FailedTests` fields. The run is `Failed` even when `FailedCount` is 0. Omitted when none. +- `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. +- `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. +- `RerunTargetCount` (number, optional): Number of tests and fixtures a `--rerun-failed` run asked Unity to run. Omitted on other runs. +- `RerunSourceCompletedAt` (string, optional): `CompletedAt` of the recorded run whose failures `--rerun-failed` reran. Omitted on other runs. + +Both are absent when `--respect-enter-play-mode-settings` entered Play Mode with a Domain Reload: the result is then recovered after the reload (see `Warning`), and a rerun whose recorded tests are all gone reports the generic no-tests message. + +## XML Result File + +Saved to `{project_root}/.uloop/outputs/TestResults/.xml`. What it records, including failed suites: `references/xml-results.md`. diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md.meta b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md.meta new file mode 100644 index 0000000000..1d65c2c10b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 522344e9576694db49152a1b0761b1ed +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/cli/common/tools/default-tools.json b/cli/common/tools/default-tools.json index ceebf72705..219eeba1ff 100644 --- a/cli/common/tools/default-tools.json +++ b/cli/common/tools/default-tools.json @@ -120,6 +120,11 @@ "type": "boolean", "description": "PlayMode only: keep the project's Enter Play Mode settings instead of forcing Domain Reload off. A Domain Reload during the run is survived; the result is recovered after the reload. Use for projects whose libraries require a Domain Reload on Play entry.", "default": false + }, + "RerunFailed": { + "type": "boolean", + "description": "Rerun only the tests that failed or were inconclusive in the most recent completed run of the same --test-mode (whole fixtures for a failed OneTimeSetUp/OneTimeTearDown). Cannot be combined with --filter-type or --filter-value", + "default": false } } } diff --git a/cli/dispatcher/shared-inputs-stamp.json b/cli/dispatcher/shared-inputs-stamp.json index aa279c7af9..19a2397921 100644 --- a/cli/dispatcher/shared-inputs-stamp.json +++ b/cli/dispatcher/shared-inputs-stamp.json @@ -1,4 +1,4 @@ { "schemaVersion": 1, - "sharedInputsHash": "57cbaca4732ff50e08d604fa20f7824741704e52" + "sharedInputsHash": "3025011c897d4017e8b390c1e33259395a0b7039" } diff --git a/cli/project-runner/shared-inputs-stamp.json b/cli/project-runner/shared-inputs-stamp.json index 30bd076238..e6c3d09599 100644 --- a/cli/project-runner/shared-inputs-stamp.json +++ b/cli/project-runner/shared-inputs-stamp.json @@ -1,4 +1,4 @@ { "schemaVersion": 1, - "sharedInputsHash": "31e0608d9f17d814152f72d21305bf0cb8ad72c6" + "sharedInputsHash": "a0269953be3a0ba5d7f70f50c807b5fcdfd26ded" } From 915bfd861fc4f0e9d075f225edc2a45b458e40f4 Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 00:33:41 +0900 Subject: [PATCH 08/10] Describe RerunFailed in the tool reference Lists the new run-tests parameter and an example call in the English and Japanese tool references, next to the other run-tests parameters. --- Packages/src/Documentation~/tools.md | 2 ++ Packages/src/Documentation~/tools_ja.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Packages/src/Documentation~/tools.md b/Packages/src/Documentation~/tools.md index 1c79e5de9b..b9f114ba97 100644 --- a/Packages/src/Documentation~/tools.md +++ b/Packages/src/Documentation~/tools.md @@ -35,11 +35,13 @@ Executes Unity Test Runner and retrieves test results. You can set conditions wi - FilterType: all (all tests), exact (individual test method name), regex (regex over full test names), assembly (assembly name), class (test class name, bare or namespace-qualified) - FilterValue: Value according to filter type (class name, namespace, etc.) - UnsavedChanges: How to handle unsaved loaded Scene and Prefab Stage changes before tests. `save` (default) writes them, `fail` stops if any remain, `discard` reloads disk state (Untitled scenes fail). +- RerunFailed: Rerun only the tests that failed or were inconclusive in the most recent completed run of the same TestMode. Cannot be combined with FilterType or FilterValue. Test results can be output as xml. The output path is returned so AI can read it. This is also a strategy to avoid consuming context. ```text → run-tests (FilterType: exact, FilterValue: "PlayerControllerTests.TestJump") → run-tests (--unsaved-changes fail, stop if editor changes are unsaved) +→ run-tests (--rerun-failed, rerun only what failed in the last completed run) → Check failed tests, fix implementation to pass tests ``` > [!WARNING] diff --git a/Packages/src/Documentation~/tools_ja.md b/Packages/src/Documentation~/tools_ja.md index b1e9f6de14..1743371c04 100644 --- a/Packages/src/Documentation~/tools_ja.md +++ b/Packages/src/Documentation~/tools_ja.md @@ -35,11 +35,13 @@ Unity Test Runnerを実行し、テスト結果を取得します。FilterType - FilterType: all（全テスト）、exact（個別テストメソッド名）、regex（テストのフルネームに対する正規表現）、assembly（アセンブリ名）、class（テストクラス名。名前空間付きでも可） - FilterValue: フィルタータイプに応じた値（クラス名、名前空間など） - UnsavedChanges: テスト前の未保存 Scene / Prefab Stage 変更の扱い。`save`（デフォルト）は保存、`fail` は残っていれば停止、`discard` はディスク状態へ戻す（Untitled シーンは破棄できず失敗）。 +- RerunFailed: 同じ TestMode で直近に完了した実行で失敗・inconclusive だったテストだけを再実行します。FilterType / FilterValue とは併用できません。 テスト結果をxmlで出力する事が可能です。出力pathを返すので、それをAIに読み取ってもらう事ができます。 これもコンテキストを圧迫しないための工夫です。 ```text → run-tests (FilterType: exact, FilterValue: "PlayerControllerTests.TestJump") → run-tests (--unsaved-changes fail、未保存のエディタ変更があれば停止) +→ run-tests (--rerun-failed、直近に完了した実行で失敗したものだけを再実行) → 失敗したテストを確認、実装を修正してテストをパス ``` > [!WARNING] From 948367786def319a43ababcd3d15aece190b4dbf Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 01:18:14 +0900 Subject: [PATCH 09/10] Explain a rerun of renamed or removed tests without asmdef advice A rerun whose recorded tests were all renamed or removed already replaced Message, but NoTestsFoundExplanation kept the default text that advises adding a test assembly. The skill tells agents to read that field for asmdef hints, so it sent them to create an .asmdef for tests that only moved. The explanation now carries the same rerun-specific message. --- .../skills/uloop-run-tests/references/response-fields.md | 2 +- .../skills/uloop-run-tests/references/response-fields.md | 2 +- Assets/Tests/Editor/RunTestsUseCaseTests.cs | 8 ++++---- .../Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs | 6 +++++- .../RunTests/Skill/references/response-fields.md | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.agents/skills/uloop-run-tests/references/response-fields.md b/.agents/skills/uloop-run-tests/references/response-fields.md index fef8f8421d..66757e5f9d 100644 --- a/.agents/skills/uloop-run-tests/references/response-fields.md +++ b/.agents/skills/uloop-run-tests/references/response-fields.md @@ -7,7 +7,7 @@ Returns JSON with: - `HasFailures` (boolean): Whether any discovered test or suite failed - `Message` (string): Summary message - `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests -- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise +- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise. When `--rerun-failed` finds that every recorded test was renamed or removed, it says that instead of the test-assembly advice. - `CompletedAt` (string): ISO timestamp when the run finished - `TestCount` (number): Total tests executed - `PassedCount` (number): Passed tests diff --git a/.claude/skills/uloop-run-tests/references/response-fields.md b/.claude/skills/uloop-run-tests/references/response-fields.md index fef8f8421d..66757e5f9d 100644 --- a/.claude/skills/uloop-run-tests/references/response-fields.md +++ b/.claude/skills/uloop-run-tests/references/response-fields.md @@ -7,7 +7,7 @@ Returns JSON with: - `HasFailures` (boolean): Whether any discovered test or suite failed - `Message` (string): Summary message - `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests -- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise +- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise. When `--rerun-failed` finds that every recorded test was renamed or removed, it says that instead of the test-assembly advice. - `CompletedAt` (string): ISO timestamp when the run finished - `TestCount` (number): Total tests executed - `PassedCount` (number): Passed tests diff --git a/Assets/Tests/Editor/RunTestsUseCaseTests.cs b/Assets/Tests/Editor/RunTestsUseCaseTests.cs index 0ff44a6a28..040919dbbd 100644 --- a/Assets/Tests/Editor/RunTestsUseCaseTests.cs +++ b/Assets/Tests/Editor/RunTestsUseCaseTests.cs @@ -1780,10 +1780,10 @@ public async Task ExecuteAsync_RerunFailedWhenRecordedTestsAreGone_ReportsRename Assert.That(response.NoTestsFound, Is.True); Assert.That(response.RerunTargetCount, Is.EqualTo(2)); - Assert.That( - response.Message, - Is.EqualTo( - "None of the 2 tests recorded as failed in the EditMode run completed at 2026-01-02T03:04:05.0000000Z exist any more; they were renamed or removed. Run uloop run-tests without --rerun-failed.")); + const string expectedMessage = + "None of the 2 tests recorded as failed in the EditMode run completed at 2026-01-02T03:04:05.0000000Z exist any more; they were renamed or removed. Run uloop run-tests without --rerun-failed."; + Assert.That(response.Message, Is.EqualTo(expectedMessage)); + Assert.That(response.NoTestsFoundExplanation, Is.EqualTo(expectedMessage)); Assert.That(response.ProposedTestAsmdef, Is.Null); Assert.That(response.UnfilteredTestNames, Is.Null); Assert.That(diagnosticCapture.AppendCalled, Is.False); diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs index 5128388cd8..f02701b741 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs @@ -284,10 +284,14 @@ private static RunTestsResponse ApplyRerunFields( response.RerunSourceCompletedAt = source.SourceCompletedAt; if (response.NoTestsFound) { - response.Message = RunTestsRerunFailedResolver.FormatRerunTargetsMissingMessage( + // Why the explanation too: its default text advises adding a test assembly, which + // would send the caller to create an .asmdef for tests that were only renamed or removed. + string rerunTargetsMissingMessage = RunTestsRerunFailedResolver.FormatRerunTargetsMissingMessage( source.TargetCount, testMode, source.SourceCompletedAt); + response.Message = rerunTargetsMissingMessage; + response.NoTestsFoundExplanation = rerunTargetsMissingMessage; } return response; diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md index fef8f8421d..66757e5f9d 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md +++ b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/references/response-fields.md @@ -7,7 +7,7 @@ Returns JSON with: - `HasFailures` (boolean): Whether any discovered test or suite failed - `Message` (string): Summary message - `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests -- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise +- `NoTestsFoundExplanation` (string): Agent-facing explanation when `NoTestsFound` is true; empty otherwise. When `--rerun-failed` finds that every recorded test was renamed or removed, it says that instead of the test-assembly advice. - `CompletedAt` (string): ISO timestamp when the run finished - `TestCount` (number): Total tests executed - `PassedCount` (number): Passed tests From 25e69c026a19dd2a0c20d0c9215d0ef8f86889ec Mon Sep 17 00:00:00 2001 From: hatayama Date: Tue, 6 Oct 2026 01:18:20 +0900 Subject: [PATCH 10/10] Decide a run-tests request before saving or discarding unsaved changes The Editor-state validation saves or discards unsaved Scene and Prefab Stage changes, and it ran before the filter and the --rerun-failed record were resolved. A request that then stopped without running, such as a rerun with nothing to rerun under --unsaved-changes discard, had already thrown edits away. Resolving reads only, so it now comes first and every rejected or empty request returns with the Editor untouched. A request that is invalid on both counts now reports the filter or record problem before the Editor-state problem. --- Assets/Tests/Editor/RunTestsUseCaseTests.cs | 56 +++++++++++++++++-- .../RunTests/RunTestsUseCase.cs | 19 ++++--- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/Assets/Tests/Editor/RunTestsUseCaseTests.cs b/Assets/Tests/Editor/RunTestsUseCaseTests.cs index 040919dbbd..765c0b588c 100644 --- a/Assets/Tests/Editor/RunTestsUseCaseTests.cs +++ b/Assets/Tests/Editor/RunTestsUseCaseTests.cs @@ -350,6 +350,7 @@ public async Task ExecuteAsync_WithUnsupportedFilterType_ShouldFailFastWithoutRu Assert.That(response.Status, Is.EqualTo(RunTestsExecutionStatus.ExecutionFailed)); Assert.That(response.Message, Does.Contain("Unsupported filter type")); Assert.That(executionService.WasCalled, Is.False); + Assert.That(validationService.WasCalled, Is.False); } [Test] @@ -1647,7 +1648,8 @@ public async Task ExecuteAsync_RerunFailedWithNullFilterValue_RunsRecordedTests( public async Task ExecuteAsync_RerunFailedWithoutRecord_ReturnsFailureWithoutRunning() { StubTestExecutionService executionService = new StubTestExecutionService(); - RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + StubTestExecutionStateValidationService validationService = CreatePassingValidationService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService, validationService: validationService); RunTestsResponse response = await ExecuteToCompletionAsync( useCase, @@ -1660,6 +1662,7 @@ public async Task ExecuteAsync_RerunFailedWithoutRecord_ReturnsFailureWithoutRun Is.EqualTo( "No completed EditMode run is recorded for this project. Run uloop run-tests without --rerun-failed first.")); Assert.That(executionService.WasCalled, Is.False); + Assert.That(validationService.WasCalled, Is.False); } /// @@ -1673,7 +1676,8 @@ public async Task ExecuteAsync_RerunFailedWithUnreadableRecord_ReturnsFailureWit string recordPath = _recordStore.GetRecordPath(UnityCliLoopTestMode.EditMode); File.WriteAllText(recordPath, invalidRecord); StubTestExecutionService executionService = new StubTestExecutionService(); - RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + StubTestExecutionStateValidationService validationService = CreatePassingValidationService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService, validationService: validationService); RunTestsResponse response = await ExecuteToCompletionAsync( useCase, @@ -1684,6 +1688,7 @@ public async Task ExecuteAsync_RerunFailedWithUnreadableRecord_ReturnsFailureWit Assert.That(response.Message, Does.StartWith("The recorded EditMode run could not be read (invalid JSON")); Assert.That(response.Message, Does.EndWith("Run uloop run-tests without --rerun-failed.")); Assert.That(executionService.WasCalled, Is.False); + Assert.That(validationService.WasCalled, Is.False); Assert.That(File.ReadAllText(recordPath), Is.EqualTo(invalidRecord)); } @@ -1697,8 +1702,10 @@ public async Task ExecuteAsync_RerunFailedWithEmptyRecord_ReturnsNothingToRerun( SeedRecord(UnityCliLoopTestMode.EditMode, Array.Empty()); bool pausePointsCleared = false; StubTestExecutionService executionService = new StubTestExecutionService(); + StubTestExecutionStateValidationService validationService = CreatePassingValidationService(); RunTestsUseCase useCase = CreateRecordingUseCase( executionService, + validationService: validationService, clearActivePausePoints: () => { pausePointsCleared = true; @@ -1717,6 +1724,7 @@ public async Task ExecuteAsync_RerunFailedWithEmptyRecord_ReturnsNothingToRerun( Assert.That(response.RerunTargetCount, Is.EqualTo(0)); Assert.That(response.RerunSourceCompletedAt, Is.EqualTo(RecordedCompletedAt)); Assert.That(executionService.WasCalled, Is.False); + Assert.That(validationService.WasCalled, Is.False); Assert.That(pausePointsCleared, Is.False); AssertRecord(UnityCliLoopTestMode.EditMode, RecordedCompletedAt); } @@ -1838,11 +1846,44 @@ public async Task ExecuteAsync_RerunFailedTimesOut_LeavesNoRecord() AssertNoRecord(UnityCliLoopTestMode.EditMode); } + /// + /// What: a rerun stopped by the Editor-state validation leaves the record and the pause points untouched. + /// + [Test] + public async Task ExecuteAsync_RerunFailedWhenStateValidationFails_LeavesRecordAndPausePointsUntouched() + { + SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.FirstFailure" }); + bool pausePointsCleared = false; + StubTestExecutionService executionService = new StubTestExecutionService(); + StubTestExecutionStateValidationService validationService = new StubTestExecutionStateValidationService( + ValidationResult.Failure("EditMode tests cannot run during play mode")); + RunTestsUseCase useCase = CreateRecordingUseCase( + executionService, + validationService: validationService, + clearActivePausePoints: () => + { + pausePointsCleared = true; + return null; + }); + + RunTestsResponse response = await ExecuteToCompletionAsync( + useCase, + new RunTestsSchema { RerunFailed = true }); + + Assert.That(response.Success, Is.False); + Assert.That(response.Message, Is.EqualTo("EditMode tests cannot run during play mode")); + Assert.That(validationService.WasCalled, Is.True); + Assert.That(executionService.WasCalled, Is.False); + Assert.That(pausePointsCleared, Is.False); + AssertRecord(UnityCliLoopTestMode.EditMode, RecordedCompletedAt, "Ns.C.FirstFailure"); + } + private async Task AssertRerunFilterConflictAsync(TestFilterType filterType, string filterValue) { SeedRecord(UnityCliLoopTestMode.EditMode, new[] { "Ns.C.FirstFailure" }); StubTestExecutionService executionService = new StubTestExecutionService(); - RunTestsUseCase useCase = CreateRecordingUseCase(executionService); + StubTestExecutionStateValidationService validationService = CreatePassingValidationService(); + RunTestsUseCase useCase = CreateRecordingUseCase(executionService, validationService: validationService); RunTestsSchema parameters = new RunTestsSchema { RerunFailed = true, @@ -1859,6 +1900,7 @@ private async Task AssertRerunFilterConflictAsync(TestFilterType filterType, str Is.EqualTo( "--rerun-failed cannot be combined with --filter-type or --filter-value; it reruns the failures recorded for the test mode.")); Assert.That(executionService.WasCalled, Is.False); + Assert.That(validationService.WasCalled, Is.False); AssertRecord(UnityCliLoopTestMode.EditMode, RecordedCompletedAt, "Ns.C.FirstFailure"); } @@ -1866,6 +1908,7 @@ private async Task AssertRerunFilterConflictAsync(TestFilterType filterType, str // the temporary record directory. private RunTestsUseCase CreateRecordingUseCase( StubTestExecutionService executionService, + StubTestExecutionStateValidationService validationService = null, Func clearActivePausePoints = null, Func waitForTestRunnerCleanupAsync = null, Func appendNoTestsDiagnostics = null, @@ -1874,7 +1917,7 @@ private RunTestsUseCase CreateRecordingUseCase( return new RunTestsUseCase( new TestFilterCreationService(), executionService, - new StubTestExecutionStateValidationService(ValidationResult.Success()), + validationService ?? CreatePassingValidationService(), _recordStore, clearActivePausePoints: clearActivePausePoints ?? (() => null), waitForTestRunnerCleanupAsync: waitForTestRunnerCleanupAsync ?? NoCleanupWait, @@ -1883,6 +1926,11 @@ private RunTestsUseCase CreateRecordingUseCase( proposeTestAsmdef: proposeTestAsmdef ?? (_ => null)); } + private static StubTestExecutionStateValidationService CreatePassingValidationService() + { + return new StubTestExecutionStateValidationService(ValidationResult.Success()); + } + // Why catch: Unity Test Framework passes an async test that ends Canceled, so an unexpected // cancellation must fail the test instead of skipping its assertions. private static async Task ExecuteToCompletionAsync( diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs index f02701b741..69e960309e 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs @@ -111,24 +111,27 @@ public async Task ExecuteAsync(RunTestsSchema parameters, Canc return CreateFailureResponse(timeoutError, NoHotReloadChangesObserved); } - ValidationResult validation = _validationService.Validate(parameters.TestMode, parameters.UnsavedChanges); - if (!validation.IsValid) - { - return CreateFailureResponse(validation.ErrorMessage, NoHotReloadChangesObserved); - } - // 1. Test filter creation + // Why before the Editor-state validation: that validation saves or discards unsaved + // changes, which a request that runs no tests must not do. RunTestsExecutionTarget target = ResolveExecutionTarget(parameters); if (target.EarlyResponse != null) { return target.EarlyResponse; } + ValidationResult validation = _validationService.Validate(parameters.TestMode, parameters.UnsavedChanges); + if (!validation.IsValid) + { + return CreateFailureResponse(validation.ErrorMessage, NoHotReloadChangesObserved); + } + return await RunAndBuildResponseAsync(parameters, target, ct).ConfigureAwait(false); } - // Why every rejection is decided here: a request that is rejected or has nothing to rerun - // must change nothing, so it has to stop before the record or the pause points are touched. + // Why every rejection is decided here, by reading only: a request that is rejected or has + // nothing to rerun must change nothing, so it has to stop before unsaved changes are saved or + // discarded and before the record or the pause points are touched. private RunTestsExecutionTarget ResolveExecutionTarget(RunTestsSchema parameters) { if (parameters.RerunFailed)