From 8bb4bb2691b2d2fbcd3b98ec9e52fb8e5855f364 Mon Sep 17 00:00:00 2001 From: hatayama Date: Mon, 28 Sep 2026 21:02:06 +0900 Subject: [PATCH 1/3] Report inconclusive tests as an unsuccessful run in run-tests NUnit can roll an inconclusive leaf up into a Passed suite, so run-tests answered Status Passed and Success true while Unity's batchmode test run exits with a failure for the same tests. A feature branch's CI failed on three Assume-based tests that run-tests had shown as passing, and the response named none of them. - Any inconclusive leaf now yields Status Inconclusive and Success false; a failed leaf still takes precedence. - The response adds InconclusiveCount and InconclusiveTests (up to 10, with the assumption's message), and the NUnit XML is saved for inconclusive runs as well as failed ones. - The run-tests skill documents the status and fields and points tests that cannot run in an environment to Assert.Ignore. --- .agents/skills/uloop-run-tests/SKILL.md | 14 +- .claude/skills/uloop-run-tests/SKILL.md | 14 +- .../Editor/RunTestsResponseContractTests.cs | 128 +++++++++++++++ .../RunTestsTestFrameworkResultTests.cs | 151 +++++++++++++++++- Assets/Tests/Editor/RunTestsToolTests.cs | 3 + .../RunTestsUnfilteredFilterEchoTests.cs | 1 + .../RunTests/RunTestsResponse.cs | 14 ++ .../RunTests/RunTestsResponseFactory.cs | 6 + .../RunTests/RunTestsUseCase.cs | 1 + .../FirstPartyTools/RunTests/Skill/SKILL.md | 14 +- .../TestFramework/PlayModeTestExecuter.cs | 2 +- .../RunTestsPendingRunCallback.cs | 2 +- .../SerializableTestResultConverter.cs | 90 ++++++++++- .../TestRunner/SerializableTestResult.cs | 19 +++ .../UnityCliLoopTestExecutionTypes.cs | 1 + 15 files changed, 437 insertions(+), 23 deletions(-) diff --git a/.agents/skills/uloop-run-tests/SKILL.md b/.agents/skills/uloop-run-tests/SKILL.md index 858855cea2..9f16c6d627 100644 --- a/.agents/skills/uloop-run-tests/SKILL.md +++ b/.agents/skills/uloop-run-tests/SKILL.md @@ -6,7 +6,7 @@ description: "Run Unity Test Runner and report detailed results. Use for EditMod # uloop run-tests -Execute Unity Test Runner. When tests fail, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. +Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. `uloop run-tests` automatically compiles pending script changes before running tests. Pass `--skip-compile` only while validating active hot-reload patches, because the compile clears those patches; otherwise let the default compile surface errors and run against current scripts. `--skip-compile` skips only the CLI-side compile: Unity still imports script edits saved since the last compile, and that import reloads the domain as soon as the run releases its assembly lock, discarding active patches and ending the request. @@ -18,6 +18,8 @@ A test run can end by discarding active hot-reload changes: script edits importe `NoTestsFound` means zero tests matched — not a test failure. Check `NoTestsFoundExplanation` and `Message` for asmdef hints. When an unfiltered run finds no tests and the project has no test assembly for the TestMode, `ProposedTestAsmdef` carries a ready-to-write `.asmdef`: save `Content` at `AssetPath`, move the test scripts under that folder, then compile and rerun. +`Status: Inconclusive` means no test failed but at least one could not meet an `Assume`; `Success` is `false`, as Unity's batchmode test run also fails for it. `InconclusiveTests` names them with the assumption's message. A test that cannot run in this environment should call `Assert.Ignore` so it reports as skipped. + ## Usage ```bash @@ -44,8 +46,8 @@ exact matches the full test name (Namespace.Class.Method). class runs every test Returns JSON with: -- `Success` (boolean): Whether all tests passed -- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `NoTestsFound`, or `ExecutionFailed` +- `Success` (boolean): Whether every test passed or was skipped; `false` when any failed or was inconclusive +- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, or `ExecutionFailed` - `HasFailures` (boolean): Whether any discovered test failed - `Message` (string): Summary message - `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests @@ -55,16 +57,18 @@ Returns JSON with: - `PassedCount` (number): Passed tests - `FailedCount` (number): Failed tests - `SkippedCount` (number): Skipped tests -- `XmlPath` (string): Path to NUnit XML result file. Empty string when no XML was saved (typically on `Success: true`); populated only when tests failed and the XML file exists on disk. +- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) +- `XmlPath` (string): Path to NUnit XML result file. Empty string when no XML was saved (typically on `Success: true`); populated only when tests failed or were inconclusive and the XML file exists on disk. - `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. - `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` - `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed. - `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. - `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. ### XML Result File -When tests fail, NUnit XML results are automatically saved to `{project_root}/.uloop/outputs/TestResults/.xml`. The XML contains per-test-case results including: +When tests fail or end inconclusive, NUnit XML results are automatically saved to `{project_root}/.uloop/outputs/TestResults/.xml`. The XML contains per-test-case results including: - Test name and full name - Pass/fail/skip status and duration diff --git a/.claude/skills/uloop-run-tests/SKILL.md b/.claude/skills/uloop-run-tests/SKILL.md index 858855cea2..9f16c6d627 100644 --- a/.claude/skills/uloop-run-tests/SKILL.md +++ b/.claude/skills/uloop-run-tests/SKILL.md @@ -6,7 +6,7 @@ description: "Run Unity Test Runner and report detailed results. Use for EditMod # uloop run-tests -Execute Unity Test Runner. When tests fail, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. +Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. `uloop run-tests` automatically compiles pending script changes before running tests. Pass `--skip-compile` only while validating active hot-reload patches, because the compile clears those patches; otherwise let the default compile surface errors and run against current scripts. `--skip-compile` skips only the CLI-side compile: Unity still imports script edits saved since the last compile, and that import reloads the domain as soon as the run releases its assembly lock, discarding active patches and ending the request. @@ -18,6 +18,8 @@ A test run can end by discarding active hot-reload changes: script edits importe `NoTestsFound` means zero tests matched — not a test failure. Check `NoTestsFoundExplanation` and `Message` for asmdef hints. When an unfiltered run finds no tests and the project has no test assembly for the TestMode, `ProposedTestAsmdef` carries a ready-to-write `.asmdef`: save `Content` at `AssetPath`, move the test scripts under that folder, then compile and rerun. +`Status: Inconclusive` means no test failed but at least one could not meet an `Assume`; `Success` is `false`, as Unity's batchmode test run also fails for it. `InconclusiveTests` names them with the assumption's message. A test that cannot run in this environment should call `Assert.Ignore` so it reports as skipped. + ## Usage ```bash @@ -44,8 +46,8 @@ exact matches the full test name (Namespace.Class.Method). class runs every test Returns JSON with: -- `Success` (boolean): Whether all tests passed -- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `NoTestsFound`, or `ExecutionFailed` +- `Success` (boolean): Whether every test passed or was skipped; `false` when any failed or was inconclusive +- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, or `ExecutionFailed` - `HasFailures` (boolean): Whether any discovered test failed - `Message` (string): Summary message - `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests @@ -55,16 +57,18 @@ Returns JSON with: - `PassedCount` (number): Passed tests - `FailedCount` (number): Failed tests - `SkippedCount` (number): Skipped tests -- `XmlPath` (string): Path to NUnit XML result file. Empty string when no XML was saved (typically on `Success: true`); populated only when tests failed and the XML file exists on disk. +- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) +- `XmlPath` (string): Path to NUnit XML result file. Empty string when no XML was saved (typically on `Success: true`); populated only when tests failed or were inconclusive and the XML file exists on disk. - `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. - `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` - `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed. - `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. - `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. ### XML Result File -When tests fail, NUnit XML results are automatically saved to `{project_root}/.uloop/outputs/TestResults/.xml`. The XML contains per-test-case results including: +When tests fail or end inconclusive, NUnit XML results are automatically saved to `{project_root}/.uloop/outputs/TestResults/.xml`. The XML contains per-test-case results including: - Test name and full name - Pass/fail/skip status and duration diff --git a/Assets/Tests/Editor/RunTestsResponseContractTests.cs b/Assets/Tests/Editor/RunTestsResponseContractTests.cs index 62d2f1a390..774fb372be 100644 --- a/Assets/Tests/Editor/RunTestsResponseContractTests.cs +++ b/Assets/Tests/Editor/RunTestsResponseContractTests.cs @@ -27,6 +27,7 @@ public void RunTestsResponse_WhenSerialized_OmitsNullFailedTestsAndNullFileLineK passedCount: 1, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: string.Empty, status: RunTestsExecutionStatus.Passed, hasFailures: false, @@ -49,6 +50,7 @@ public void RunTestsResponse_WhenSerialized_OmitsNullFailedTestsAndNullFileLineK passedCount: 0, failedCount: 1, skippedCount: 0, + inconclusiveCount: 0, xmlPath: "TestResults/example.xml", status: RunTestsExecutionStatus.Failed, hasFailures: true, @@ -96,6 +98,7 @@ public void RunTestsResponse_WhenSerialized_OmitsOrIncludesSkippedTestsByPresenc passedCount: 1, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: string.Empty, status: RunTestsExecutionStatus.Passed, hasFailures: false, @@ -118,6 +121,7 @@ public void RunTestsResponse_WhenSerialized_OmitsOrIncludesSkippedTestsByPresenc passedCount: 1, failedCount: 0, skippedCount: 1, + inconclusiveCount: 0, xmlPath: string.Empty, status: RunTestsExecutionStatus.Passed, hasFailures: false, @@ -139,6 +143,128 @@ public void RunTestsResponse_WhenSerialized_OmitsOrIncludesSkippedTestsByPresenc Assert.That(skippedTests[0].Value(), Is.EqualTo("Example.Tests.SkippedTest")); } + /// + /// What: InconclusiveCount is always written, while InconclusiveTests is omitted without + /// inconclusive leaves and serializes each name and message when present. + /// + [Test] + public void RunTestsResponse_WhenSerialized_WritesInconclusiveCountAndOmitsOrIncludesInconclusiveTests() + { + RunTestsResponse zeroInconclusive = new RunTestsResponse( + success: true, + message: "Test execution completed with status: Passed", + completedAt: "2026-01-01T00:00:00.0000000Z", + testCount: 1, + passedCount: 1, + failedCount: 0, + skippedCount: 0, + inconclusiveCount: 0, + xmlPath: string.Empty, + status: RunTestsExecutionStatus.Passed, + hasFailures: false, + noTestsFound: false, + noTestsFoundExplanation: string.Empty); + + JObject zeroInconclusiveJson = JObject.Parse( + JsonConvert.SerializeObject( + zeroInconclusive, + Formatting.None, + UnityCliLoopJsonResponseSerializerSettings.Settings)); + + Assert.That(zeroInconclusiveJson.Value("InconclusiveCount"), Is.EqualTo(0)); + Assert.That(zeroInconclusiveJson.Property("InconclusiveTests"), Is.Null); + + RunTestsResponse populated = new RunTestsResponse( + success: false, + message: "Test execution completed with status: Inconclusive", + completedAt: "2026-01-01T00:00:00.0000000Z", + testCount: 2, + passedCount: 1, + failedCount: 0, + skippedCount: 0, + inconclusiveCount: 1, + xmlPath: "TestResults/example.xml", + status: RunTestsExecutionStatus.Inconclusive, + hasFailures: false, + noTestsFound: false, + noTestsFoundExplanation: string.Empty) + { + InconclusiveTests = new[] + { + new SerializableTestResult.InconclusiveTestDetail + { + FullName = "Example.Tests.InconclusiveTest", + Message = "Release is required." + } + } + }; + + JObject populatedJson = JObject.Parse( + JsonConvert.SerializeObject( + populated, + Formatting.None, + UnityCliLoopJsonResponseSerializerSettings.Settings)); + JArray inconclusiveTests = (JArray)populatedJson["InconclusiveTests"]; + + Assert.That(populatedJson.Value("InconclusiveCount"), Is.EqualTo(1)); + Assert.That(populatedJson.Value("Status"), Is.EqualTo("Inconclusive")); + Assert.That(inconclusiveTests, Is.Not.Null); + Assert.That(inconclusiveTests.Count, Is.EqualTo(1)); + JObject first = (JObject)inconclusiveTests[0]; + Assert.That(first["FullName"]?.Value(), Is.EqualTo("Example.Tests.InconclusiveTest")); + Assert.That(first["Message"]?.Value(), Is.EqualTo("Release is required.")); + } + + /// + /// What: the response built from a stored result carries its inconclusive count and details, + /// and leaves InconclusiveTests unset when the result lists none. + /// + [Test] + public void FromResult_WhenResultHasInconclusiveLeaves_CopiesCountAndDetails() + { + SerializableTestResult withInconclusive = new SerializableTestResult + { + success = false, + status = RunTestsExecutionStatus.Inconclusive, + message = "Test execution completed with status: Inconclusive", + noTestsFoundExplanation = string.Empty, + completedAt = "2026-01-01T00:00:00.0000000Z", + testCount = 2, + passedCount = 1, + inconclusiveCount = 1, + inconclusiveTests = new[] + { + new SerializableTestResult.InconclusiveTestDetail + { + FullName = "Example.Tests.InconclusiveTest", + Message = "Release is required." + } + } + }; + SerializableTestResult withoutInconclusive = new SerializableTestResult + { + success = true, + status = RunTestsExecutionStatus.Passed, + message = "Test execution completed with status: Passed", + noTestsFoundExplanation = string.Empty, + completedAt = "2026-01-01T00:00:00.0000000Z", + testCount = 1, + passedCount = 1, + inconclusiveCount = 0, + inconclusiveTests = new SerializableTestResult.InconclusiveTestDetail[0] + }; + + RunTestsResponse copied = RunTestsResponseFactory.FromResult(withInconclusive); + RunTestsResponse empty = RunTestsResponseFactory.FromResult(withoutInconclusive); + + Assert.That(copied.InconclusiveCount, Is.EqualTo(1)); + Assert.That(copied.InconclusiveTests, Is.Not.Null); + Assert.That(copied.InconclusiveTests.Length, Is.EqualTo(1)); + Assert.That(copied.InconclusiveTests[0].FullName, Is.EqualTo("Example.Tests.InconclusiveTest")); + Assert.That(empty.InconclusiveCount, Is.EqualTo(0)); + Assert.That(empty.InconclusiveTests, Is.Null); + } + /// /// What: an empty Warning is omitted from production JSON so the key cannot reappear unnoticed. /// @@ -153,6 +279,7 @@ public void RunTestsResponse_WhenWarningIsEmpty_OmitsWarningPropertyFromJson() passedCount: 1, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: string.Empty, status: RunTestsExecutionStatus.Passed, hasFailures: false, @@ -182,6 +309,7 @@ public void RunTestsResponse_WhenWarningIsSet_SerializesExactPolicyFormSentence( passedCount: 1, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: string.Empty, status: RunTestsExecutionStatus.Passed, hasFailures: false, diff --git a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs index f81b0a4f31..ab43b602d6 100644 --- a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs +++ b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs @@ -331,10 +331,11 @@ public void FromTestResult_WhenSkippedLeavesSpanNestedSuites_CollectsAllNamesInT } /// - /// What: a root Passed aggregate containing an inconclusive leaf remains a successful run. + /// What: an inconclusive leaf makes the run non-successful even when the root aggregate is Passed, + /// as Unity's batchmode test run does with its exit code. /// [Test] - public void FromTestResult_WhenPassedAndInconclusiveLeavesHavePassedRoot_PreservesPassedSuccess() + public void FromTestResult_WhenPassedAndInconclusiveLeavesHavePassedRoot_ReturnsInconclusiveFailure() { ITestResultAdaptor resultAdaptor = CreateTestSuite( "RootSuite", @@ -348,9 +349,97 @@ public void FromTestResult_WhenPassedAndInconclusiveLeavesHavePassedRoot_Preserv SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); - Assert.That(result.success, Is.True); - Assert.That(result.status, Is.EqualTo("Passed")); - Assert.That(result.message, Is.EqualTo("Test execution completed with status: Passed")); + Assert.That(result.success, Is.False); + Assert.That(result.status, Is.EqualTo("Inconclusive")); + Assert.That(result.message, Is.EqualTo("Test execution completed with status: Inconclusive")); + } + + /// + /// What: an inconclusive leaf is counted and listed with its name and the message that + /// explains the unmet assumption. + /// + [Test] + public void FromTestResult_WhenALeafIsInconclusive_CountsItAndListsItsNameAndMessage() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Passed, + 0.1, + new List + { + CreateTestCase("PassingTest", TestResultStatus.Passed, 0.1), + CreateTestCase("InconclusiveTest", TestResultStatus.Inconclusive, 0.1, "Release is required.") + }); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.testCount, Is.EqualTo(2)); + Assert.That(result.passedCount, Is.EqualTo(1)); + Assert.That(result.inconclusiveCount, Is.EqualTo(1)); + Assert.That(result.inconclusiveTests, Is.Not.Null); + Assert.That(result.inconclusiveTests.Length, Is.EqualTo(1)); + Assert.That(result.inconclusiveTests[0].FullName, Is.EqualTo("Example.Tests.InconclusiveTest")); + Assert.That(result.inconclusiveTests[0].Message, Is.EqualTo("Release is required.")); + } + + /// + /// What: a failed leaf keeps the Failed status over an inconclusive sibling, and the + /// inconclusive leaf is still counted and listed. + /// + [Test] + public void FromTestResult_WhenFailedAndInconclusiveLeaves_ReportsFailedAndStillListsInconclusive() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCase("FailingTest", TestResultStatus.Failed, 0.1, "boom"), + CreateTestCase("InconclusiveTest", TestResultStatus.Inconclusive, 0.1, "Release is required.") + }); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.success, Is.False); + Assert.That(result.status, Is.EqualTo("Failed")); + Assert.That(result.hasFailures, Is.True); + Assert.That(result.inconclusiveCount, Is.EqualTo(1)); + Assert.That(result.inconclusiveTests, Is.Not.Null); + Assert.That(result.inconclusiveTests.Length, Is.EqualTo(1)); + } + + /// + /// What: only the first 10 inconclusive leaves are listed while the count keeps all of them. + /// + [Test] + public void FromTestResult_WhenElevenTestsAreInconclusive_ListsFirstTenInconclusiveDetails() + { + List children = new List(); + for (int index = 0; index < 11; index++) + { + string suffix = index.ToString(CultureInfo.InvariantCulture); + children.Add( + CreateTestCase( + "InconclusiveTest" + suffix, + TestResultStatus.Inconclusive, + 0.1, + "assumption " + suffix)); + } + + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Inconclusive, + 0.1, + children); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.inconclusiveCount, Is.EqualTo(11)); + Assert.That(result.inconclusiveTests, Is.Not.Null); + Assert.That(result.inconclusiveTests.Length, Is.EqualTo(10)); + Assert.That(result.inconclusiveTests[0].FullName, Is.EqualTo("Example.Tests.InconclusiveTest0")); + Assert.That(result.inconclusiveTests[9].FullName, Is.EqualTo("Example.Tests.InconclusiveTest9")); } /// @@ -510,6 +599,58 @@ public void TrySaveFailureXml_WhenExporterThrows_ReturnsNull() Assert.That(xmlPath, Is.Null); } + /// + /// What: a run whose only non-passed leaves are inconclusive still saves the result XML. + /// + [Test] + public void ShouldSaveResultXml_WhenALeafIsInconclusiveAndNoneFailed_ReturnsTrue() + { + SerializableTestResult result = new SerializableTestResult + { + testCount = 2, + passedCount = 1, + failedCount = 0, + inconclusiveCount = 1 + }; + + Assert.That(SerializableTestResultConverter.ShouldSaveResultXml(result), Is.True); + } + + /// + /// What: a run with a failed leaf saves the result XML. + /// + [Test] + public void ShouldSaveResultXml_WhenALeafFailed_ReturnsTrue() + { + SerializableTestResult result = new SerializableTestResult + { + testCount = 2, + passedCount = 1, + failedCount = 1, + inconclusiveCount = 0 + }; + + Assert.That(SerializableTestResultConverter.ShouldSaveResultXml(result), Is.True); + } + + /// + /// What: a run of only passed and skipped leaves saves no result XML. + /// + [Test] + public void ShouldSaveResultXml_WhenLeavesOnlyPassedOrWereSkipped_ReturnsFalse() + { + SerializableTestResult result = new SerializableTestResult + { + testCount = 2, + passedCount = 1, + skippedCount = 1, + failedCount = 0, + inconclusiveCount = 0 + }; + + Assert.That(SerializableTestResultConverter.ShouldSaveResultXml(result), Is.False); + } + [Test] public void SaveTestResultAsXml_DoesNotCallAssetDatabaseRefresh() { diff --git a/Assets/Tests/Editor/RunTestsToolTests.cs b/Assets/Tests/Editor/RunTestsToolTests.cs index c8af1904bb..db10d7ee35 100644 --- a/Assets/Tests/Editor/RunTestsToolTests.cs +++ b/Assets/Tests/Editor/RunTestsToolTests.cs @@ -105,6 +105,7 @@ public void CreateTestFrameworkUnavailable_ShouldReturnUnsupportedResponse() Assert.That(response.PassedCount, Is.EqualTo(0)); Assert.That(response.FailedCount, Is.EqualTo(0)); Assert.That(response.SkippedCount, Is.EqualTo(0)); + Assert.That(response.InconclusiveCount, Is.EqualTo(0)); Assert.That(response.XmlPath, Is.Null); } @@ -120,6 +121,7 @@ public void Constructor_ShouldStoreEveryArgumentVerbatim() passedCount: 3, failedCount: 2, skippedCount: 1, + inconclusiveCount: 4, xmlPath: "/tmp/results.xml", status: "CustomStatus", hasFailures: false, @@ -133,6 +135,7 @@ public void Constructor_ShouldStoreEveryArgumentVerbatim() Assert.That(response.PassedCount, Is.EqualTo(3)); Assert.That(response.FailedCount, Is.EqualTo(2)); Assert.That(response.SkippedCount, Is.EqualTo(1)); + Assert.That(response.InconclusiveCount, Is.EqualTo(4)); Assert.That(response.XmlPath, Is.EqualTo("/tmp/results.xml")); Assert.That(response.Status, Is.EqualTo("CustomStatus")); Assert.That(response.HasFailures, Is.False); diff --git a/Assets/Tests/Editor/RunTestsUnfilteredFilterEchoTests.cs b/Assets/Tests/Editor/RunTestsUnfilteredFilterEchoTests.cs index 3f02fb4fe4..e8419f3c77 100644 --- a/Assets/Tests/Editor/RunTestsUnfilteredFilterEchoTests.cs +++ b/Assets/Tests/Editor/RunTestsUnfilteredFilterEchoTests.cs @@ -143,6 +143,7 @@ private static RunTestsResponse CreateNoTestsResponse() passedCount: 0, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: null, status: RunTestsExecutionStatus.NoTestsFound, hasFailures: false, diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs index 540931c2b3..b109562091 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponse.cs @@ -70,6 +70,11 @@ public class RunTestsResponse : UnityCliLoopToolResponse /// public int SkippedCount { get; set; } + /// + /// Number of inconclusive tests, whose assumptions were not met + /// + public int InconclusiveCount { get; set; } + /// /// Path to XML result file (if saved) /// @@ -93,6 +98,12 @@ public class RunTestsResponse : UnityCliLoopToolResponse [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] public string[] SkippedTests { get; set; } + /// + /// Inconclusive leaf tests, omitted from JSON when none were inconclusive. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public SerializableTestResult.InconclusiveTestDetail[] InconclusiveTests { get; set; } + /// /// Policy warning when hot-reload changes were live at test-run start. Empty when none /// were active; omitted from JSON via ShouldSerializeWarning. @@ -171,6 +182,7 @@ public RunTestsResponse( int passedCount, int failedCount, int skippedCount, + int inconclusiveCount, string xmlPath, string status, bool hasFailures, @@ -187,6 +199,7 @@ public RunTestsResponse( PassedCount = passedCount; FailedCount = failedCount; SkippedCount = skippedCount; + InconclusiveCount = inconclusiveCount; XmlPath = xmlPath; Status = status; HasFailures = hasFailures; @@ -204,6 +217,7 @@ public static RunTestsResponse CreateTestFrameworkUnavailable() passedCount: 0, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: null, status: RunTestsExecutionStatus.ExecutionFailed, hasFailures: false, diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponseFactory.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponseFactory.cs index e967c45014..4a42a0aaf4 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponseFactory.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsResponseFactory.cs @@ -20,6 +20,7 @@ internal static RunTestsResponse FromResult(SerializableTestResult result) passedCount: result.passedCount, failedCount: result.failedCount, skippedCount: result.skippedCount, + inconclusiveCount: result.inconclusiveCount, xmlPath: result.xmlPath, status: result.status, hasFailures: result.hasFailures, @@ -51,6 +52,11 @@ private static void CopyTestDetails(SerializableTestResult result, RunTestsRespo { response.SkippedTests = result.skippedTests; } + + if (result.inconclusiveTests != null && result.inconclusiveTests.Length > 0) + { + response.InconclusiveTests = result.inconclusiveTests; + } } } } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs index e4432cf152..212810b8b4 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/RunTestsUseCase.cs @@ -293,6 +293,7 @@ private static RunTestsResponse CreateFailureResponse( passedCount: 0, failedCount: 0, skippedCount: 0, + inconclusiveCount: 0, xmlPath: null, status: RunTestsExecutionStatus.ExecutionFailed, hasFailures: false, diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md index 858855cea2..9f16c6d627 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md @@ -6,7 +6,7 @@ description: "Run Unity Test Runner and report detailed results. Use for EditMod # uloop run-tests -Execute Unity Test Runner. When tests fail, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. +Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. `uloop run-tests` automatically compiles pending script changes before running tests. Pass `--skip-compile` only while validating active hot-reload patches, because the compile clears those patches; otherwise let the default compile surface errors and run against current scripts. `--skip-compile` skips only the CLI-side compile: Unity still imports script edits saved since the last compile, and that import reloads the domain as soon as the run releases its assembly lock, discarding active patches and ending the request. @@ -18,6 +18,8 @@ A test run can end by discarding active hot-reload changes: script edits importe `NoTestsFound` means zero tests matched — not a test failure. Check `NoTestsFoundExplanation` and `Message` for asmdef hints. When an unfiltered run finds no tests and the project has no test assembly for the TestMode, `ProposedTestAsmdef` carries a ready-to-write `.asmdef`: save `Content` at `AssetPath`, move the test scripts under that folder, then compile and rerun. +`Status: Inconclusive` means no test failed but at least one could not meet an `Assume`; `Success` is `false`, as Unity's batchmode test run also fails for it. `InconclusiveTests` names them with the assumption's message. A test that cannot run in this environment should call `Assert.Ignore` so it reports as skipped. + ## Usage ```bash @@ -44,8 +46,8 @@ exact matches the full test name (Namespace.Class.Method). class runs every test Returns JSON with: -- `Success` (boolean): Whether all tests passed -- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `NoTestsFound`, or `ExecutionFailed` +- `Success` (boolean): Whether every test passed or was skipped; `false` when any failed or was inconclusive +- `Status` (string): Machine-readable execution status such as `Passed`, `Failed`, `Inconclusive`, `NoTestsFound`, or `ExecutionFailed` - `HasFailures` (boolean): Whether any discovered test failed - `Message` (string): Summary message - `NoTestsFound` (boolean): Whether Unity Test Runner discovered zero matching tests @@ -55,16 +57,18 @@ Returns JSON with: - `PassedCount` (number): Passed tests - `FailedCount` (number): Failed tests - `SkippedCount` (number): Skipped tests -- `XmlPath` (string): Path to NUnit XML result file. Empty string when no XML was saved (typically on `Success: true`); populated only when tests failed and the XML file exists on disk. +- `InconclusiveCount` (number): Inconclusive tests (an `Assume` was not met) +- `XmlPath` (string): Path to NUnit XML result file. Empty string when no XML was saved (typically on `Success: true`); populated only when tests failed or were inconclusive and the XML file exists on disk. - `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. - `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` - `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed. - `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. - `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. ### XML Result File -When tests fail, NUnit XML results are automatically saved to `{project_root}/.uloop/outputs/TestResults/.xml`. The XML contains per-test-case results including: +When tests fail or end inconclusive, NUnit XML results are automatically saved to `{project_root}/.uloop/outputs/TestResults/.xml`. The XML contains per-test-case results including: - Test name and full name - Pass/fail/skip status and duration diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs index 39daa66517..df24d75022 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/PlayModeTestExecuter.cs @@ -168,7 +168,7 @@ private static async Task ExecuteTestWithEventNotificati return; } - if (result.failedCount > 0 && rawResult != null) + if (SerializableTestResultConverter.ShouldSaveResultXml(result) && rawResult != null) { result.xmlPath = TrySaveFailureXml(rawResult); } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs index 42fc75130d..0c21907519 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/RunTestsPendingRunCallback.cs @@ -28,7 +28,7 @@ public void RunStarted(ITestAdaptor tests) public void RunFinished(ITestResultAdaptor result) { SerializableTestResult serializableResult = SerializableTestResultConverter.FromTestResult(result); - if (serializableResult.failedCount > 0) + if (SerializableTestResultConverter.ShouldSaveResultXml(serializableResult)) { serializableResult.xmlPath = PlayModeTestExecuter.TrySaveFailureXml(result); } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs index fe266251b3..053b55fbab 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs @@ -14,6 +14,7 @@ private enum RunTestsResultClassification { NoTestsFound, HasFailures, + HasInconclusive, FullyPassed, RootStatus } @@ -36,6 +37,7 @@ public static SerializableTestResult FromTestResult(ITestResultAdaptor result) passedCount = 0, failedCount = 0, skippedCount = 0, + inconclusiveCount = 0, xmlPath = null }; } @@ -44,6 +46,7 @@ public static SerializableTestResult FromTestResult(ITestResultAdaptor result) int passedTests = CountPassedTests(result); int failedTests = CountFailedTests(result); int skippedTests = CountSkippedTests(result); + int inconclusiveTests = CountInconclusiveTests(result); bool noTestsFound = totalTests == 0; bool hasFailures = failedTests > 0; RunTestsResultClassification classification = Classify( @@ -52,6 +55,7 @@ public static SerializableTestResult FromTestResult(ITestResultAdaptor result) passedTests, failedTests, skippedTests, + inconclusiveTests, noTestsFound, hasFailures); bool success = classification == RunTestsResultClassification.FullyPassed; @@ -74,18 +78,30 @@ public static SerializableTestResult FromTestResult(ITestResultAdaptor result) passedCount = passedTests, failedCount = failedTests, skippedCount = skippedTests, + inconclusiveCount = inconclusiveTests, xmlPath = null, failedTests = CollectFailedTestDetails(result), - skippedTests = CollectSkippedTestFullNames(result) + skippedTests = CollectSkippedTestFullNames(result), + inconclusiveTests = CollectInconclusiveTestDetails(result) }; } + /// + /// Whether a finished run leaves anything to read in the NUnit XML: a failed or an + /// inconclusive leaf. + /// + internal static bool ShouldSaveResultXml(SerializableTestResult result) + { + return result.failedCount > 0 || result.inconclusiveCount > 0; + } + private static RunTestsResultClassification Classify( ITestResultAdaptor result, int totalTests, int passedTests, int failedTests, int skippedTests, + int inconclusiveTests, bool noTestsFound, bool hasFailures) { @@ -99,6 +115,13 @@ private static RunTestsResultClassification Classify( return RunTestsResultClassification.HasFailures; } + // Why before the root status: NUnit can roll an inconclusive leaf up into a Passed suite, + // while Unity's batchmode run exits with a failure for it, so the leaf decides. + if (inconclusiveTests > 0) + { + return RunTestsResultClassification.HasInconclusive; + } + if (totalTests > 0 && failedTests == 0 && (result.TestStatus == TestStatus.Passed @@ -124,6 +147,11 @@ private static string CreateStatus( return RunTestsExecutionStatus.Failed; } + if (classification == RunTestsResultClassification.HasInconclusive) + { + return RunTestsExecutionStatus.Inconclusive; + } + if (classification == RunTestsResultClassification.FullyPassed) { return RunTestsExecutionStatus.Passed; @@ -172,6 +200,13 @@ private static int CountSkippedTests(ITestResultAdaptor result) return count; } + private static int CountInconclusiveTests(ITestResultAdaptor result) + { + int count = 0; + CountTestsByStatus(result, ref count, TestStatus.Inconclusive); + return count; + } + private static void CountTestsByStatus(ITestResultAdaptor result, ref int count, TestStatus? targetStatus) { if (!result.Test.IsSuite) @@ -298,6 +333,59 @@ private static void AppendSkippedTestFullNames( } } } + + private static SerializableTestResult.InconclusiveTestDetail[] CollectInconclusiveTestDetails( + ITestResultAdaptor result) + { + List details = + new List(); + AppendInconclusiveTestDetails(result, details); + if (details.Count == 0) + { + return null; + } + + return details.ToArray(); + } + + private static void AppendInconclusiveTestDetails( + ITestResultAdaptor result, + List details) + { + if (details.Count >= RunTestsConstants.FailedTestDetailsLimit) + { + return; + } + + if (!result.Test.IsSuite) + { + if (result.TestStatus == TestStatus.Inconclusive) + { + details.Add( + new SerializableTestResult.InconclusiveTestDetail + { + FullName = result.Test.FullName, + Message = result.Message ?? string.Empty + }); + } + + return; + } + + if (result.Children == null) + { + return; + } + + foreach (ITestResultAdaptor child in result.Children) + { + AppendInconclusiveTestDetails(child, details); + if (details.Count >= RunTestsConstants.FailedTestDetailsLimit) + { + return; + } + } + } } } #endif diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs index 2f0a414282..56ecba7444 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestRunner/SerializableTestResult.cs @@ -21,6 +21,7 @@ public class SerializableTestResult [SerializeField] public int passedCount; [SerializeField] public int failedCount; [SerializeField] public int skippedCount; + [SerializeField] public int inconclusiveCount; [SerializeField] public string xmlPath; /// @@ -33,6 +34,23 @@ public class SerializableTestResult /// public string[] skippedTests; + /// + /// Inconclusive leaf tests, capped for the JSON response. Null when none were inconclusive. + /// + public InconclusiveTestDetail[] inconclusiveTests; + + /// + /// One inconclusive test leaf included in a run-tests response, with the message that names + /// the assumption it could not meet. + /// + [Serializable] + public class InconclusiveTestDetail + { + public string FullName { get; set; } + + public string Message { get; set; } + } + /// /// One failed test leaf included in a run-tests response. /// @@ -65,6 +83,7 @@ public static SerializableTestResult CreateTestFrameworkUnavailable() passedCount = 0, failedCount = 0, skippedCount = 0, + inconclusiveCount = 0, xmlPath = null }; } diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs b/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs index 9460f6b71e..4e37951b16 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/UnityCliLoopTestExecutionTypes.cs @@ -40,6 +40,7 @@ internal static class RunTestsExecutionStatus { public const string Passed = "Passed"; public const string Failed = "Failed"; + public const string Inconclusive = "Inconclusive"; public const string NoTestsFound = "NoTestsFound"; public const string ExecutionFailed = "ExecutionFailed"; } From 9763016474e6e769ccf88cc98e73d7fd6db1515a Mon Sep 17 00:00:00 2001 From: hatayama Date: Mon, 28 Sep 2026 22:13:15 +0900 Subject: [PATCH 2/3] Write the reason of each inconclusive test into the saved NUnit XML The run now saves the XML when a test is inconclusive, but the exporter only wrote details for failed leaves, so an inconclusive leaf carried just its result attribute. The skill said the XML holds the messages, and beyond the ten leaves the response lists, the unmet assumptions were nowhere to be read. Inconclusive leaves now get a element, the NUnit 3 place for why a test reached no verdict, rather than , so readers of the XML do not count them as failed assertions. The skill text and its generated copies now describe what the XML contains. --- .agents/skills/uloop-run-tests/SKILL.md | 5 +-- .claude/skills/uloop-run-tests/SKILL.md | 5 +-- .../RunTestsTestFrameworkResultTests.cs | 32 +++++++++++++++++++ .../FirstPartyTools/RunTests/Skill/SKILL.md | 5 +-- .../TestFramework/NUnitXmlResultExporter.cs | 24 +++++++++++++- 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/.agents/skills/uloop-run-tests/SKILL.md b/.agents/skills/uloop-run-tests/SKILL.md index 9f16c6d627..a35cd0179d 100644 --- a/.agents/skills/uloop-run-tests/SKILL.md +++ b/.agents/skills/uloop-run-tests/SKILL.md @@ -6,7 +6,7 @@ description: "Run Unity Test Runner and report detailed results. Use for EditMod # uloop run-tests -Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. +Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with failure messages, stack traces, and inconclusive reasons are automatically saved. Read the XML file at `XmlPath` for detailed diagnosis. `uloop run-tests` automatically compiles pending script changes before running tests. Pass `--skip-compile` only while validating active hot-reload patches, because the compile clears those patches; otherwise let the default compile surface errors and run against current scripts. `--skip-compile` skips only the CLI-side compile: Unity still imports script edits saved since the last compile, and that import reloads the domain as soon as the run releases its assembly lock, discarding active patches and ending the request. @@ -62,7 +62,7 @@ Returns JSON with: - `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. - `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` - `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. -- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. - `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. - `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. @@ -73,3 +73,4 @@ When tests fail or end inconclusive, NUnit XML results are automatically saved t - Test name and full name - Pass/fail/skip status and duration - For failed tests: `` (assertion error) and `` +- For inconclusive tests: `` (the assumption that was not met) diff --git a/.claude/skills/uloop-run-tests/SKILL.md b/.claude/skills/uloop-run-tests/SKILL.md index 9f16c6d627..a35cd0179d 100644 --- a/.claude/skills/uloop-run-tests/SKILL.md +++ b/.claude/skills/uloop-run-tests/SKILL.md @@ -6,7 +6,7 @@ description: "Run Unity Test Runner and report detailed results. Use for EditMod # uloop run-tests -Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. +Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with failure messages, stack traces, and inconclusive reasons are automatically saved. Read the XML file at `XmlPath` for detailed diagnosis. `uloop run-tests` automatically compiles pending script changes before running tests. Pass `--skip-compile` only while validating active hot-reload patches, because the compile clears those patches; otherwise let the default compile surface errors and run against current scripts. `--skip-compile` skips only the CLI-side compile: Unity still imports script edits saved since the last compile, and that import reloads the domain as soon as the run releases its assembly lock, discarding active patches and ending the request. @@ -62,7 +62,7 @@ Returns JSON with: - `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. - `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` - `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. -- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. - `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. - `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. @@ -73,3 +73,4 @@ When tests fail or end inconclusive, NUnit XML results are automatically saved t - Test name and full name - Pass/fail/skip status and duration - For failed tests: `` (assertion error) and `` +- For inconclusive tests: `` (the assumption that was not met) diff --git a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs index ab43b602d6..f02c04f939 100644 --- a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs +++ b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs @@ -125,6 +125,38 @@ public void SaveTestResultAsXml_WhenResultIsInconclusive_WritesInconclusiveAggre } } + /// + /// What: the saved XML records an inconclusive leaf's unmet assumption as its reason, not as + /// a failure, so the leaves beyond the response's listing limit keep their messages. + /// + [Test] + public void SaveTestResultAsXml_WhenALeafIsInconclusive_WritesItsMessageAsReason() + { + ITestResultAdaptor result = CreateTestSuite( + "RootSuite", + TestResultStatus.Passed, + 0.5, + new List + { + CreateTestCase("InconclusiveTest", TestResultStatus.Inconclusive, 0.5, "Release is required.") + }); + + string filePath = null; + try + { + XmlDocument document = SaveResultAndLoadXml(result, out filePath); + XmlNode reasonMessage = document.SelectSingleNode("//test-case/reason/message"); + + Assert.That(reasonMessage, Is.Not.Null); + Assert.That(reasonMessage.InnerText, Is.EqualTo("Release is required.")); + Assert.That(document.SelectSingleNode("//test-case/failure"), Is.Null); + } + finally + { + DeleteIfExists(filePath); + } + } + [Test] public void FromTestResult_WhenResultIsNull_ReturnsFailureWithoutCounts() { diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md index 9f16c6d627..a35cd0179d 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/RunTests/Skill/SKILL.md @@ -6,7 +6,7 @@ description: "Run Unity Test Runner and report detailed results. Use for EditMod # uloop run-tests -Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with error messages and stack traces are automatically saved. Read the XML file at `XmlPath` for detailed failure diagnosis. +Execute Unity Test Runner. When tests fail or end inconclusive, NUnit XML results with failure messages, stack traces, and inconclusive reasons are automatically saved. Read the XML file at `XmlPath` for detailed diagnosis. `uloop run-tests` automatically compiles pending script changes before running tests. Pass `--skip-compile` only while validating active hot-reload patches, because the compile clears those patches; otherwise let the default compile surface errors and run against current scripts. `--skip-compile` skips only the CLI-side compile: Unity still imports script edits saved since the last compile, and that import reloads the domain as soon as the run releases its assembly lock, discarding active patches and ending the request. @@ -62,7 +62,7 @@ Returns JSON with: - `ClearedPausePointIds` (string[], optional): IDs of pause points that were cleared before test execution. Omitted from JSON when no pause points were active. - `FailedTests` (array, optional): Up to 10 failed leaf tests with `FullName`, `Message`, and when the stack trace contains a path:line location, `File` and `Line`. Omitted when no tests failed. When `FailedCount` is greater than 10, `Message` ends with `first 10 of N failures listed; see XmlPath for full results.` - `SkippedTests` (string[], optional): Up to 10 full names of skipped leaf tests. Omitted when no tests were skipped. When `SkippedCount` is greater than 10, only the first 10 names are listed. -- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed. +- `InconclusiveTests` (array, optional): Up to 10 inconclusive leaf tests with `FullName` and `Message`. Omitted when no test was inconclusive. When `InconclusiveCount` is greater than 10, only the first 10 are listed; the XML at `XmlPath` has every message. - `ProposedTestAsmdef` (object, optional): `AssetPath` and `Content` of a ready-to-write test `.asmdef` (test-assembly wiring plus references to the project's assemblies under test). Present only when an unfiltered run found no tests and no test assembly exists for the TestMode. - `CompileNote` (string, optional): States that the automatic compile ran and succeeded before the tests and names `--skip-compile` as the opt-out. When the compile response carried a Warning (for example active hot-reload changes dropped by the domain reload), the note repeats it. Omitted when `--skip-compile` was passed; a failed compile returns the compile error response instead. @@ -73,3 +73,4 @@ When tests fail or end inconclusive, NUnit XML results are automatically saved t - Test name and full name - Pass/fail/skip status and duration - For failed tests: `` (assertion error) and `` +- For inconclusive tests: `` (the assumption that was not met) diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/NUnitXmlResultExporter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/NUnitXmlResultExporter.cs index 97fd9d8798..db386efd33 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/NUnitXmlResultExporter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/NUnitXmlResultExporter.cs @@ -14,7 +14,7 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools { /// - /// Writes failing Unity Test Runner results as NUnit XML files for CLI diagnostics. + /// Writes failing or inconclusive Unity Test Runner results as NUnit XML files for CLI diagnostics. /// internal static class NUnitXmlResultExporter { @@ -160,6 +160,12 @@ private static XmlElement CreateTestCaseElement(XmlDocument document, ITestResul testCase.SetAttribute("end-time", FormatDateTime(result.EndTime)); testCase.SetAttribute("duration", FormatDuration(result.Duration)); + if (result.TestStatus == TestStatus.Inconclusive) + { + AppendReason(document, testCase, result); + return testCase; + } + if (result.TestStatus != TestStatus.Failed) { return testCase; @@ -171,6 +177,22 @@ private static XmlElement CreateTestCaseElement(XmlDocument document, ITestResul return testCase; } + // Why and not : NUnit 3 keeps the explanation of a test that reached no + // verdict in , so readers of the XML do not count it as a failed assertion. + private static void AppendReason(XmlDocument document, XmlElement testCase, ITestResultAdaptor result) + { + if (string.IsNullOrEmpty(result.Message)) + { + return; + } + + XmlElement reason = document.CreateElement("reason"); + XmlElement message = document.CreateElement("message"); + message.InnerText = result.Message; + reason.AppendChild(message); + testCase.AppendChild(reason); + } + private static void AppendFailureMessage(XmlDocument document, XmlElement failure, ITestResultAdaptor result) { if (!string.IsNullOrEmpty(result.Message)) From 2360f79dbc22b7b1e58d8d30096aab012f74d729 Mon Sep 17 00:00:00 2001 From: hatayama Date: Mon, 28 Sep 2026 22:30:51 +0900 Subject: [PATCH 3/3] Keep a Failed root ahead of inconclusive leaves in run-tests A OneTimeTearDown exception fails the fixture without failing any leaf. With an inconclusive leaf in that run, the new classification reported Inconclusive where the run used to report Failed, hiding the fixture failure behind the weaker status. A Failed root now falls through to the root status, as before this change, and the inconclusive leaf is still counted and listed. Raised by a review comment on the pull request. --- .../RunTestsTestFrameworkResultTests.cs | 25 +++++++++++++++++++ .../SerializableTestResultConverter.cs | 5 ++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs index f02c04f939..63014888bc 100644 --- a/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs +++ b/Assets/Tests/Editor/RunTestsTestFrameworkResultTests.cs @@ -474,6 +474,31 @@ public void FromTestResult_WhenElevenTestsAreInconclusive_ListsFirstTenInconclus Assert.That(result.inconclusiveTests[9].FullName, Is.EqualTo("Example.Tests.InconclusiveTest9")); } + /// + /// What: a Failed root outranks an inconclusive leaf when no leaf failed, as when a + /// OneTimeTearDown exception fails only the fixture, and the inconclusive leaf is still listed. + /// + [Test] + public void FromTestResult_WhenFailedRootHasInconclusiveLeafAndNoFailedLeaf_KeepsFailedStatus() + { + ITestResultAdaptor resultAdaptor = CreateTestSuite( + "RootSuite", + TestResultStatus.Failed, + 0.1, + new List + { + CreateTestCase("PassingTest", TestResultStatus.Passed, 0.1), + CreateTestCase("InconclusiveTest", TestResultStatus.Inconclusive, 0.1, "Release is required.") + }); + + SerializableTestResult result = SerializableTestResultConverter.FromTestResult(resultAdaptor); + + Assert.That(result.success, Is.False); + Assert.That(result.status, Is.EqualTo("Failed")); + Assert.That(result.inconclusiveCount, Is.EqualTo(1)); + Assert.That(result.inconclusiveTests, Has.Length.EqualTo(1)); + } + /// /// What: a non-Passed root aggregate containing an inconclusive leaf remains non-successful. /// diff --git a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs index 053b55fbab..2d24f3e62d 100644 --- a/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs +++ b/Packages/src/Editor/FirstPartyTools/RunTests/TestFramework/SerializableTestResultConverter.cs @@ -116,8 +116,9 @@ private static RunTestsResultClassification Classify( } // Why before the root status: NUnit can roll an inconclusive leaf up into a Passed suite, - // while Unity's batchmode run exits with a failure for it, so the leaf decides. - if (inconclusiveTests > 0) + // while Unity's batchmode run exits with a failure for it, so the leaf decides. A Failed + // root still outranks it: a OneTimeTearDown exception fails the fixture but no leaf. + if (inconclusiveTests > 0 && result.TestStatus != TestStatus.Failed) { return RunTestsResultClassification.HasInconclusive; }