From fe39a6861c84404a34ec7bc3e0eec814a8e127cf Mon Sep 17 00:00:00 2001 From: Masamichi Hatayama Date: Tue, 14 Jul 2026 09:45:39 +0900 Subject: [PATCH 1/3] fix: capture Device Simulator rendering via PlayModeView (#1766) Co-authored-by: Cursor --- Assets/Tests/Editor/GameViewBridgeTests.cs | 60 ++++++++++++++ .../Tests/Editor/GameViewBridgeTests.cs.meta | 11 +++ .../Editor/UnityCLILoop.Tests.Editor.asmdef | 1 + .../Screenshot/EditorWindowCaptureUtility.cs | 7 +- .../Screenshot/ScreenshotUseCase.cs | 4 +- .../Screenshot/UIElementAnnotator.cs | 2 +- .../Editor/InternalAPIBridge/AssemblyInfo.cs | 3 + .../InternalAPIBridge/AssemblyInfo.cs.meta | 11 +++ .../InternalAPIBridge/GameViewBridge.cs | 80 +++++++++++-------- 9 files changed, 139 insertions(+), 40 deletions(-) create mode 100644 Assets/Tests/Editor/GameViewBridgeTests.cs create mode 100644 Assets/Tests/Editor/GameViewBridgeTests.cs.meta create mode 100644 Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs create mode 100644 Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs.meta diff --git a/Assets/Tests/Editor/GameViewBridgeTests.cs b/Assets/Tests/Editor/GameViewBridgeTests.cs new file mode 100644 index 0000000000..ae83df5a0b --- /dev/null +++ b/Assets/Tests/Editor/GameViewBridgeTests.cs @@ -0,0 +1,60 @@ +using System.Reflection; +using NUnit.Framework; +using UnityEngine; + +using io.github.hatayama.UnityCliLoop.InternalAPIBridge; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies PlayModeView RenderTexture field resolution used by GameViewBridge. + /// + public class GameViewBridgeTests + { + private class FakePlayModeView + { +#pragma warning disable CS0414 // Field is assigned for reflection-based tests only. + private object m_TargetTexture = "texture"; +#pragma warning restore CS0414 + } + + private class FakeSimulatorWindow : FakePlayModeView + { + } + + [Test] + public void ResolveTargetTextureField_WhenCalledOnDeclaringType_FindsPrivateBaseField() + { + // Verifies GetField on the PlayModeView declaring type finds m_TargetTexture. + FieldInfo field = GameViewBridge.ResolveTargetTextureField(typeof(FakePlayModeView)); + + Assert.That(field, Is.Not.Null); + Assert.That(field.Name, Is.EqualTo("m_TargetTexture")); + Assert.That(field.DeclaringType, Is.EqualTo(typeof(FakePlayModeView))); + } + + [Test] + public void ResolveTargetTextureField_WhenDerivedTypeGetFieldMissesPrivateBaseField_DeclaringTypeStillResolves() + { + // Verifies the reflection pitfall: derived-type GetField cannot see private base fields. + FieldInfo fromDerived = typeof(FakeSimulatorWindow).GetField( + "m_TargetTexture", + BindingFlags.Instance | BindingFlags.NonPublic); + FieldInfo fromDeclaring = GameViewBridge.ResolveTargetTextureField(typeof(FakePlayModeView)); + + Assert.That(fromDerived, Is.Null); + Assert.That(fromDeclaring, Is.Not.Null); + + FakeSimulatorWindow instance = new(); + object value = fromDeclaring.GetValue(instance); + Assert.That(value, Is.EqualTo("texture")); + } + + [Test] + public void GetRenderTexture_WhenMembersResolved_DoesNotThrow() + { + // Verifies PlayModeView type/method/field resolution completes without throwing. + Assert.DoesNotThrow(() => GameViewBridge.GetRenderTexture()); + } + } +} diff --git a/Assets/Tests/Editor/GameViewBridgeTests.cs.meta b/Assets/Tests/Editor/GameViewBridgeTests.cs.meta new file mode 100644 index 0000000000..3cee14f1dc --- /dev/null +++ b/Assets/Tests/Editor/GameViewBridgeTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 72e1206d01daa484da9b23ccb96473ad +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef b/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef index 22786365d0..c95c782558 100644 --- a/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef +++ b/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef @@ -3,6 +3,7 @@ "rootNamespace": "io.github.hatayama.UnityCliLoop.Tests.Editor", "references": [ "GUID:214998e563c124e8a88199b2dd1f522d", + "GUID:5079a8d3a72924a81aa1cbc25f65ed1b", "GUID:5c4588558a3624eacbce0f50007cf1eb", "GUID:18e6dd30a99d14f32a045f79a2956f46", "GUID:2fca906c8834d45aea12a5cc143c032d", diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs index 09e10a2d8b..e62cf0bebc 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs @@ -162,7 +162,8 @@ public static string[] GetOpenWindowNames() return names.ToArray(); } - // Captures game rendering by reading GameView's composited RenderTexture (PlayMode only). + // Captures game rendering by reading the Play Mode view RenderTexture (PlayMode only). + // Works for both GameView and Device Simulator via PlayModeView.m_TargetTexture. // Contains all cameras + Screen Space Overlay Canvas, without tab bar or borders. internal static async Task<(Texture2D? texture, GameRenderingImageInfo renderingImageInfo, bool timedOut)> CaptureGameRenderingAsync( float resolutionScale, @@ -189,13 +190,13 @@ public static string[] GetOpenWindowNames() RenderTexture rt = GameViewBridge.GetRenderTexture(); if (rt == null) { - Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); + Debug.LogWarning("[EditorWindowCaptureUtility] Play Mode view RenderTexture is not available"); GameRenderingImageInfo unavailableInfo = renderingImageInfo ?? CreateUnavailableGameRenderingImageInfo(Handles.GetMainGameViewSize()); return (null, unavailableInfo, false); } - // GameView RenderTexture can be shorter than the full input area, so raw image Y needs this offset. + // Play Mode view RenderTexture can be shorter than the full input area, so raw image Y needs this offset. GameRenderingImageInfo captureInfo = renderingImageInfo ?? CreateGameRenderingImageInfo(Handles.GetMainGameViewSize(), rt.width, rt.height); diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs index 3317a49e65..42b5a2ba5c 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs @@ -168,7 +168,7 @@ private async Task CaptureRenderingAsync( if (captureTimedOut) { return CreateTimedOutResult( - "GameView rendering capture", + "Play Mode view rendering capture", correlationId, new List()); } @@ -190,7 +190,7 @@ private async Task CaptureRenderingAsync( { VibeLogger.LogError( "screenshot_rendering_unavailable", - "GameView RenderTexture is not available. Open the Game view and wait for a frame before retrying.", + "Play Mode view RenderTexture is not available. Open the Game view or Device Simulator and wait for a frame before retrying.", correlationId: correlationId ); return new ScreenshotResponse(); diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs index a78de55a98..ed182dfbfc 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs @@ -9,7 +9,7 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools { // Creates a temporary Screen Space Overlay Canvas that draws bounding boxes and labels - // over interactive UI elements. The overlay is captured by GameView's m_RenderTexture + // over interactive UI elements. The overlay is captured by PlayModeView.m_TargetTexture // (OnGUI-based overlays are NOT included in the RT). /// /// Provides UI Element Annotator behavior for Unity CLI Loop. diff --git a/Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs b/Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs new file mode 100644 index 0000000000..dfa2c7f965 --- /dev/null +++ b/Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("UnityCLILoop.Tests.Editor")] diff --git a/Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs.meta b/Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs.meta new file mode 100644 index 0000000000..4ccb36acb7 --- /dev/null +++ b/Packages/src/Editor/InternalAPIBridge/AssemblyInfo.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: df8bc76e5b3594fb98fc546217349bb1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/InternalAPIBridge/GameViewBridge.cs b/Packages/src/Editor/InternalAPIBridge/GameViewBridge.cs index 518fb86b4a..71bc674e2f 100644 --- a/Packages/src/Editor/InternalAPIBridge/GameViewBridge.cs +++ b/Packages/src/Editor/InternalAPIBridge/GameViewBridge.cs @@ -6,56 +6,59 @@ namespace io.github.hatayama.UnityCliLoop.InternalAPIBridge { /// - /// Bridge class for accessing Unity GameView internal APIs via reflection. - /// GameView is an internal class; this bridge discovers members dynamically. + /// Bridge for the active Play Mode view RenderTexture via reflection. + /// Uses PlayModeView so both GameView and Device Simulator windows work. /// public static class GameViewBridge { - private static Type _gameViewType; - private static FieldInfo _renderTextureField; + private const string PlayModeViewTypeName = "UnityEditor.PlayModeView"; + private const string GetMainPlayModeViewMethodName = "GetMainPlayModeView"; + private const string TargetTextureFieldName = "m_TargetTexture"; + + private static Type _playModeViewType; + private static MethodInfo _getMainPlayModeViewMethod; + private static FieldInfo _targetTextureField; private static bool _memberSearchDone; /// - /// Get the GameView's composited RenderTexture containing all cameras + Screen Space Overlay Canvas. + /// Get the active Play Mode view's composited RenderTexture + /// (cameras + Screen Space Overlay Canvas). /// - /// The RenderTexture, or null if GameView not found or field not accessible + /// The RenderTexture, or null if the view or field is unavailable public static RenderTexture GetRenderTexture() { EnsureMembersResolved(); - EditorWindow gameView = FindMainGameView(); - if (gameView == null || _renderTextureField == null) + EditorWindow playModeView = FindMainPlayModeView(); + if (playModeView == null || _targetTextureField == null) { return null; } - return _renderTextureField.GetValue(gameView) as RenderTexture; + return _targetTextureField.GetValue(playModeView) as RenderTexture; } - private static EditorWindow FindMainGameView() + /// + /// Resolve m_TargetTexture on the PlayModeView declaring type. + /// Must not use a derived Type: GetField does not return private fields declared on base types. + /// + internal static FieldInfo ResolveTargetTextureField(Type playModeViewType) { - if (_gameViewType == null) - { - return null; - } + Debug.Assert(playModeViewType != null, "playModeViewType must not be null"); - UnityEngine.Object[] gameViews = Resources.FindObjectsOfTypeAll(_gameViewType); - if (gameViews.Length == 0) - { - return null; - } + return playModeViewType.GetField( + TargetTextureFieldName, + BindingFlags.Instance | BindingFlags.NonPublic); + } - // Prefer focused window to match editor's active view context - foreach (UnityEngine.Object gv in gameViews) + private static EditorWindow FindMainPlayModeView() + { + if (_getMainPlayModeViewMethod == null) { - EditorWindow window = gv as EditorWindow; - if (window != null && window.hasFocus) - { - return window; - } + return null; } - return gameViews[0] as EditorWindow; + return _getMainPlayModeViewMethod.Invoke(null, null) as EditorWindow; } private static void EnsureMembersResolved() @@ -66,18 +69,27 @@ private static void EnsureMembersResolved() } _memberSearchDone = true; - _gameViewType = typeof(Editor).Assembly.GetType("UnityEditor.GameView"); - if (_gameViewType == null) + _playModeViewType = typeof(Editor).Assembly.GetType(PlayModeViewTypeName); + if (_playModeViewType == null) { - Debug.LogWarning("[GameViewBridge] GameView type not found"); + Debug.LogWarning("[GameViewBridge] PlayModeView type not found"); return; } - _renderTextureField = _gameViewType.GetField("m_RenderTexture", - BindingFlags.Instance | BindingFlags.NonPublic); - if (_renderTextureField == null) + _getMainPlayModeViewMethod = _playModeViewType.GetMethod( + GetMainPlayModeViewMethodName, + BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public); + if (_getMainPlayModeViewMethod == null) + { + Debug.LogWarning("[GameViewBridge] GetMainPlayModeView method not found"); + return; + } + + // why: private base fields are invisible to GetField on derived types (GameView / SimulatorWindow) + _targetTextureField = ResolveTargetTextureField(_playModeViewType); + if (_targetTextureField == null) { - Debug.LogWarning("[GameViewBridge] m_RenderTexture field not found"); + Debug.LogWarning("[GameViewBridge] m_TargetTexture field not found on PlayModeView"); } } } From 2454e40c344d6e1a51a4b03659f9ac6b2e0d20b4 Mon Sep 17 00:00:00 2001 From: Masamichi Hatayama Date: Tue, 14 Jul 2026 09:50:10 +0900 Subject: [PATCH 2/3] fix: fall back window screenshots to Device Simulator (#1767) Co-authored-by: Cursor --- .../ScreenshotWindowNameResolverTests.cs | 85 +++++++++++++++++++ .../ScreenshotWindowNameResolverTests.cs.meta | 11 +++ .../Screenshot/ScreenshotUseCase.cs | 37 +++++++- .../ScreenshotWindowNameResolver.cs | 51 +++++++++++ .../ScreenshotWindowNameResolver.cs.meta | 11 +++ .../Editor/ToolContracts/ScreenshotSchema.cs | 2 +- .../ToolContracts/UnityCliLoopConstants.cs | 2 + 7 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs create mode 100644 Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs.meta diff --git a/Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs b/Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs new file mode 100644 index 0000000000..d3d98d3a64 --- /dev/null +++ b/Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs @@ -0,0 +1,85 @@ +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; +using io.github.hatayama.UnityCliLoop.ToolContracts; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies window-name fallback from Game to Device Simulator for screenshot capture. + /// + public class ScreenshotWindowNameResolverTests + { + [Test] + public void ShouldFallbackToSimulator_WhenDefaultGameExactMisses_ReturnsTrue() + { + // Verifies the default Game exact lookup falls back when no window matched. + bool shouldFallback = ScreenshotWindowNameResolver.ShouldFallbackToSimulator( + UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME, + WindowMatchMode.exact, + matchCount: 0); + + Assert.That(shouldFallback, Is.True); + } + + [Test] + public void ShouldFallbackToSimulator_WhenGameAlreadyMatched_ReturnsFalse() + { + // Verifies fallback is skipped when the Game window was found. + bool shouldFallback = ScreenshotWindowNameResolver.ShouldFallbackToSimulator( + UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME, + WindowMatchMode.exact, + matchCount: 1); + + Assert.That(shouldFallback, Is.False); + } + + [Test] + public void ShouldFallbackToSimulator_WhenRequestedNameIsNotGame_ReturnsFalse() + { + // Verifies explicit non-Game names do not silently switch to Simulator. + bool shouldFallback = ScreenshotWindowNameResolver.ShouldFallbackToSimulator( + "Inspector", + WindowMatchMode.exact, + matchCount: 0); + + Assert.That(shouldFallback, Is.False); + } + + [Test] + public void ShouldFallbackToSimulator_WhenMatchModeIsNotExact_ReturnsFalse() + { + // Verifies prefix/contains Game lookups do not trigger Simulator fallback. + bool shouldFallback = ScreenshotWindowNameResolver.ShouldFallbackToSimulator( + UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME, + WindowMatchMode.contains, + matchCount: 0); + + Assert.That(shouldFallback, Is.False); + } + + [Test] + public void ResolveCaptureWindowName_WhenFallbackApplies_ReturnsSimulator() + { + // Verifies the resolved capture title becomes Simulator after a Game miss. + string resolved = ScreenshotWindowNameResolver.ResolveCaptureWindowName( + UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME, + WindowMatchMode.exact, + primaryMatchCount: 0); + + Assert.That(resolved, Is.EqualTo(UnityCliLoopConstants.SCREENSHOT_SIMULATOR_WINDOW_NAME)); + } + + [Test] + public void ResolveCaptureWindowName_WhenFallbackDoesNotApply_ReturnsRequestedName() + { + // Verifies Resolve keeps the requested title when Game already matched. + string resolved = ScreenshotWindowNameResolver.ResolveCaptureWindowName( + UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME, + WindowMatchMode.exact, + primaryMatchCount: 1); + + Assert.That(resolved, Is.EqualTo(UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME)); + } + } +} diff --git a/Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs.meta b/Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs.meta new file mode 100644 index 0000000000..c7ad0b0243 --- /dev/null +++ b/Assets/Tests/Editor/ScreenshotWindowNameResolverTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2a5f3a68bb5c448d591afcddae2e273a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs index 42b5a2ba5c..3502f2daa7 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs @@ -256,18 +256,47 @@ private async Task CaptureWindowsAsync( SynchronizationContext editorContext = CapturedEditorSynchronizationContext.RequireCurrent("window screenshot use case"); EditorWindow[] windows = EditorWindowCaptureUtility.FindWindowsByName(request.WindowName, request.MatchMode); + string captureWindowName = ScreenshotWindowNameResolver.ResolveCaptureWindowName( + request.WindowName, + request.MatchMode, + windows.Length); + bool usedSimulatorFallback = !string.Equals( + captureWindowName, + request.WindowName, + StringComparison.OrdinalIgnoreCase); + if (usedSimulatorFallback) + { + // why: Device Simulator replaces the Game tab, so the default "Game" title miss should retry Simulator + windows = EditorWindowCaptureUtility.FindWindowsByName(captureWindowName, request.MatchMode); + if (windows.Length > 0) + { + VibeLogger.LogInfo( + "screenshot_window_fallback_simulator", + $"Window '{request.WindowName}' not found; capturing '{captureWindowName}' instead", + correlationId: correlationId + ); + } + } + if (windows.Length == 0) { + string notFoundMessage = usedSimulatorFallback + ? "Neither Game nor Simulator window found; open the Game view or Device Simulator and retry" + : $"Window '{request.WindowName}' not found (MatchMode: {request.MatchMode})"; + VibeLogger.LogError( "screenshot_window_not_found", - $"Window '{request.WindowName}' not found (MatchMode: {request.MatchMode})", + notFoundMessage, correlationId: correlationId ); - return new ScreenshotResponse(); + return new ScreenshotResponse + { + Message = notFoundMessage, + }; } string outputDirectory = EnsureOutputDirectoryExists(request.OutputDirectory); - string safeWindowName = SanitizeFileName(request.WindowName); + string safeWindowName = SanitizeFileName(captureWindowName); string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss_fff"); List screenshots = new(); @@ -336,7 +365,7 @@ private async Task CaptureWindowsAsync( VibeLogger.LogInfo( "screenshot_success", $"Captured {screenshots.Count} window(s)", - new { WindowName = request.WindowName, ScreenshotCount = screenshots.Count }, + new { WindowName = captureWindowName, RequestedWindowName = request.WindowName, ScreenshotCount = screenshots.Count }, correlationId: correlationId ); diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs new file mode 100644 index 0000000000..d50cf95fc1 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs @@ -0,0 +1,51 @@ +using System; + +using io.github.hatayama.UnityCliLoop.ToolContracts; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Resolves which Editor window title to capture for screenshot --capture-mode window. + /// + internal static class ScreenshotWindowNameResolver + { + /// + /// Decide whether to retry with the Device Simulator title after a Game window miss. + /// why: CLI default window name is Game, but Device Simulator replaces that tab so the title is Simulator. + /// + internal static bool ShouldFallbackToSimulator( + string requestedWindowName, + WindowMatchMode matchMode, + int matchCount) + { + if (matchCount > 0) + { + return false; + } + + // why: only the default Game exact lookup means "capture the play view chrome" + if (matchMode != WindowMatchMode.exact) + { + return false; + } + + return string.Equals( + requestedWindowName, + UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME, + StringComparison.OrdinalIgnoreCase); + } + + internal static string ResolveCaptureWindowName( + string requestedWindowName, + WindowMatchMode matchMode, + int primaryMatchCount) + { + if (ShouldFallbackToSimulator(requestedWindowName, matchMode, primaryMatchCount)) + { + return UnityCliLoopConstants.SCREENSHOT_SIMULATOR_WINDOW_NAME; + } + + return requestedWindowName; + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs.meta b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs.meta new file mode 100644 index 0000000000..e53cc3a5b7 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotWindowNameResolver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a11098664166e4ad6a3bf77ad41fb85b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs b/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs index 4107a546b0..91068ebf16 100644 --- a/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs +++ b/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs @@ -5,7 +5,7 @@ namespace io.github.hatayama.UnityCliLoop.ToolContracts /// public class ScreenshotSchema : UnityCliLoopToolSchema { - public string WindowName { get; set; } = "Game"; + public string WindowName { get; set; } = UnityCliLoopConstants.SCREENSHOT_DEFAULT_WINDOW_NAME; public float ResolutionScale { get; set; } = 1.0f; public WindowMatchMode MatchMode { get; set; } = WindowMatchMode.exact; public string OutputDirectory { get; set; } = ""; diff --git a/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs b/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs index fbefc9b56c..1feb41fb69 100644 --- a/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs +++ b/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs @@ -103,6 +103,8 @@ public static UnityEditor.PackageManager.PackageInfo PackageInfo "simulate_mouse_x = image_x / resolutionScale; simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY"; public const string SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE = "unavailable: window screenshots include Unity Editor chrome; use capture-mode rendering for mouse input coordinates"; + public const string SCREENSHOT_DEFAULT_WINDOW_NAME = "Game"; + public const string SCREENSHOT_SIMULATOR_WINDOW_NAME = "Simulator"; public const int CORRELATION_ID_LENGTH = 8; public const string GUID_FORMAT_NO_HYPHENS = "N"; From f4f2188684a68397b00fa72f10ec1f08fb540a90 Mon Sep 17 00:00:00 2001 From: Masamichi Hatayama Date: Tue, 14 Jul 2026 09:52:29 +0900 Subject: [PATCH 3/3] docs: add Device Simulator guidance to mouse/screenshot skills (#1768) Co-authored-by: Cursor --- .agents/skills/uloop-raycast/SKILL.md | 2 ++ .agents/skills/uloop-screenshot/SKILL.md | 10 ++++++++-- .../uloop-screenshot/references/annotated-elements.md | 2 ++ .agents/skills/uloop-simulate-mouse-input/SKILL.md | 1 + .agents/skills/uloop-simulate-mouse-ui/SKILL.md | 1 + .claude/skills/uloop-raycast/SKILL.md | 2 ++ .claude/skills/uloop-screenshot/SKILL.md | 10 ++++++++-- .../uloop-screenshot/references/annotated-elements.md | 2 ++ .claude/skills/uloop-simulate-mouse-input/SKILL.md | 1 + .claude/skills/uloop-simulate-mouse-ui/SKILL.md | 1 + .../src/Editor/FirstPartyTools/Raycast/Skill/SKILL.md | 2 ++ .../Editor/FirstPartyTools/Screenshot/Skill/SKILL.md | 10 ++++++++-- .../Screenshot/Skill/references/annotated-elements.md | 2 ++ .../FirstPartyTools/SimulateMouseInput/Skill/SKILL.md | 1 + .../FirstPartyTools/SimulateMouseUi/Skill/SKILL.md | 1 + 15 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.agents/skills/uloop-raycast/SKILL.md b/.agents/skills/uloop-raycast/SKILL.md index e91412fa60..30edac8991 100644 --- a/.agents/skills/uloop-raycast/SKILL.md +++ b/.agents/skills/uloop-raycast/SKILL.md @@ -33,6 +33,8 @@ unity_x = input_x unity_y = gameViewHeight - input_y ``` +- Device Simulator play view is supported. Prefer coordinates from `uloop screenshot --capture-mode rendering` (or its annotated `SimX`/`SimY`). + ## Examples ```bash diff --git a/.agents/skills/uloop-screenshot/SKILL.md b/.agents/skills/uloop-screenshot/SKILL.md index 2717785c79..81b2ffe464 100644 --- a/.agents/skills/uloop-screenshot/SKILL.md +++ b/.agents/skills/uloop-screenshot/SKILL.md @@ -18,7 +18,7 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. | +| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. When the Game tab is Device Simulator and the title is `Simulator`, default `Game` falls back to `Simulator`. | | `--resolution-scale` | number | `1.0` | Resolution scale (0.1 to 1.0) | | `--match-mode` | enum | `exact` | Window name matching mode: `exact`, `prefix`, or `contains`. Ignored when `--capture-mode rendering`. | | `--capture-mode` | enum | `window` | `window`=capture EditorWindow including toolbar, `rendering`=capture game rendering only (PlayMode required, coordinates match simulate-mouse) | @@ -36,7 +36,13 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo ## Window Name -The window name is the text displayed in the window's title bar (tab). Common names: Game, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported. +The window name is the text displayed in the window's title bar (tab). Common names: Game, Simulator, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported. + +## Device Simulator + +- Prefer `--capture-mode rendering` for the annotate → click flow. It works with both the normal Game view and Device Simulator, and coordinates match `simulate-mouse-ui` / `simulate-mouse-input` / `raycast`. +- `--capture-mode window` captures Editor chrome (toolbar/borders). With Device Simulator as the play view, default `--window-name Game` falls back to `Simulator` when no Game tab exists; you can also pass `--window-name Simulator`. +- Simulator Fit to Screen / Scale / Safe Area overlays are chrome-only and do not change rendering-mode input coordinates. After device rotation, re-run annotate (or re-read `Screen` size) before clicking. ## Output diff --git a/.agents/skills/uloop-screenshot/references/annotated-elements.md b/.agents/skills/uloop-screenshot/references/annotated-elements.md index 79f674f193..a093729923 100644 --- a/.agents/skills/uloop-screenshot/references/annotated-elements.md +++ b/.agents/skills/uloop-screenshot/references/annotated-elements.md @@ -2,6 +2,8 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-elements true` or `--annotate-raycast-grid true` output to find coordinates for `simulate-mouse-ui`, `simulate-mouse-input`, or `raycast`. +Device Simulator is supported for this flow: prefer `--capture-mode rendering` (not `window`) so coordinates match the simulated device resolution. + ## AnnotatedElements Fields `AnnotatedElements` is empty unless `--annotate-elements true` is used, or unless `--annotate-raycast-grid true` adds clustered 3D collider candidates (with or without `--raycast-layer-mask`). UI entries are sorted by z-order, frontmost first. Each item contains: diff --git a/.agents/skills/uloop-simulate-mouse-input/SKILL.md b/.agents/skills/uloop-simulate-mouse-input/SKILL.md index 17d634f31e..01eab3b2d7 100644 --- a/.agents/skills/uloop-simulate-mouse-input/SKILL.md +++ b/.agents/skills/uloop-simulate-mouse-input/SKILL.md @@ -106,6 +106,7 @@ unity_y = gameViewHeight - input_y ``` - `Mouse.current.position` uses bottom-left Unity coordinates, so the value read inside Unity may show the converted Y. +- Device Simulator play view is supported. Prefer rendering-mode screenshots for coordinates; they match the simulated device resolution, not Simulator chrome scale. ## Prerequisites diff --git a/.agents/skills/uloop-simulate-mouse-ui/SKILL.md b/.agents/skills/uloop-simulate-mouse-ui/SKILL.md index 45aea53ee1..2fb68dc7be 100644 --- a/.agents/skills/uloop-simulate-mouse-ui/SKILL.md +++ b/.agents/skills/uloop-simulate-mouse-ui/SKILL.md @@ -67,6 +67,7 @@ uloop simulate-mouse-ui --action --x --y [options] - Dragging on empty space (no draggable UI element) returns `Success = false` - `--bypass-raycast` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path` - If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate +- Device Simulator play view is supported. Prefer `uloop screenshot --capture-mode rendering --annotate-elements` for coordinates; they use the simulated device resolution (`Handles.GetMainGameViewSize()` / `Screen`), not the Simulator chrome scale. ## Pause Point Inspection (Standard for E2E) diff --git a/.claude/skills/uloop-raycast/SKILL.md b/.claude/skills/uloop-raycast/SKILL.md index e91412fa60..30edac8991 100644 --- a/.claude/skills/uloop-raycast/SKILL.md +++ b/.claude/skills/uloop-raycast/SKILL.md @@ -33,6 +33,8 @@ unity_x = input_x unity_y = gameViewHeight - input_y ``` +- Device Simulator play view is supported. Prefer coordinates from `uloop screenshot --capture-mode rendering` (or its annotated `SimX`/`SimY`). + ## Examples ```bash diff --git a/.claude/skills/uloop-screenshot/SKILL.md b/.claude/skills/uloop-screenshot/SKILL.md index 2717785c79..81b2ffe464 100644 --- a/.claude/skills/uloop-screenshot/SKILL.md +++ b/.claude/skills/uloop-screenshot/SKILL.md @@ -18,7 +18,7 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. | +| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. When the Game tab is Device Simulator and the title is `Simulator`, default `Game` falls back to `Simulator`. | | `--resolution-scale` | number | `1.0` | Resolution scale (0.1 to 1.0) | | `--match-mode` | enum | `exact` | Window name matching mode: `exact`, `prefix`, or `contains`. Ignored when `--capture-mode rendering`. | | `--capture-mode` | enum | `window` | `window`=capture EditorWindow including toolbar, `rendering`=capture game rendering only (PlayMode required, coordinates match simulate-mouse) | @@ -36,7 +36,13 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo ## Window Name -The window name is the text displayed in the window's title bar (tab). Common names: Game, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported. +The window name is the text displayed in the window's title bar (tab). Common names: Game, Simulator, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported. + +## Device Simulator + +- Prefer `--capture-mode rendering` for the annotate → click flow. It works with both the normal Game view and Device Simulator, and coordinates match `simulate-mouse-ui` / `simulate-mouse-input` / `raycast`. +- `--capture-mode window` captures Editor chrome (toolbar/borders). With Device Simulator as the play view, default `--window-name Game` falls back to `Simulator` when no Game tab exists; you can also pass `--window-name Simulator`. +- Simulator Fit to Screen / Scale / Safe Area overlays are chrome-only and do not change rendering-mode input coordinates. After device rotation, re-run annotate (or re-read `Screen` size) before clicking. ## Output diff --git a/.claude/skills/uloop-screenshot/references/annotated-elements.md b/.claude/skills/uloop-screenshot/references/annotated-elements.md index 79f674f193..a093729923 100644 --- a/.claude/skills/uloop-screenshot/references/annotated-elements.md +++ b/.claude/skills/uloop-screenshot/references/annotated-elements.md @@ -2,6 +2,8 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-elements true` or `--annotate-raycast-grid true` output to find coordinates for `simulate-mouse-ui`, `simulate-mouse-input`, or `raycast`. +Device Simulator is supported for this flow: prefer `--capture-mode rendering` (not `window`) so coordinates match the simulated device resolution. + ## AnnotatedElements Fields `AnnotatedElements` is empty unless `--annotate-elements true` is used, or unless `--annotate-raycast-grid true` adds clustered 3D collider candidates (with or without `--raycast-layer-mask`). UI entries are sorted by z-order, frontmost first. Each item contains: diff --git a/.claude/skills/uloop-simulate-mouse-input/SKILL.md b/.claude/skills/uloop-simulate-mouse-input/SKILL.md index 17d634f31e..01eab3b2d7 100644 --- a/.claude/skills/uloop-simulate-mouse-input/SKILL.md +++ b/.claude/skills/uloop-simulate-mouse-input/SKILL.md @@ -106,6 +106,7 @@ unity_y = gameViewHeight - input_y ``` - `Mouse.current.position` uses bottom-left Unity coordinates, so the value read inside Unity may show the converted Y. +- Device Simulator play view is supported. Prefer rendering-mode screenshots for coordinates; they match the simulated device resolution, not Simulator chrome scale. ## Prerequisites diff --git a/.claude/skills/uloop-simulate-mouse-ui/SKILL.md b/.claude/skills/uloop-simulate-mouse-ui/SKILL.md index 45aea53ee1..2fb68dc7be 100644 --- a/.claude/skills/uloop-simulate-mouse-ui/SKILL.md +++ b/.claude/skills/uloop-simulate-mouse-ui/SKILL.md @@ -67,6 +67,7 @@ uloop simulate-mouse-ui --action --x --y [options] - Dragging on empty space (no draggable UI element) returns `Success = false` - `--bypass-raycast` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path` - If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate +- Device Simulator play view is supported. Prefer `uloop screenshot --capture-mode rendering --annotate-elements` for coordinates; they use the simulated device resolution (`Handles.GetMainGameViewSize()` / `Screen`), not the Simulator chrome scale. ## Pause Point Inspection (Standard for E2E) diff --git a/Packages/src/Editor/FirstPartyTools/Raycast/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/Raycast/Skill/SKILL.md index e91412fa60..30edac8991 100644 --- a/Packages/src/Editor/FirstPartyTools/Raycast/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/Raycast/Skill/SKILL.md @@ -33,6 +33,8 @@ unity_x = input_x unity_y = gameViewHeight - input_y ``` +- Device Simulator play view is supported. Prefer coordinates from `uloop screenshot --capture-mode rendering` (or its annotated `SimX`/`SimY`). + ## Examples ```bash diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md index 2717785c79..81b2ffe464 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md @@ -18,7 +18,7 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. | +| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. When the Game tab is Device Simulator and the title is `Simulator`, default `Game` falls back to `Simulator`. | | `--resolution-scale` | number | `1.0` | Resolution scale (0.1 to 1.0) | | `--match-mode` | enum | `exact` | Window name matching mode: `exact`, `prefix`, or `contains`. Ignored when `--capture-mode rendering`. | | `--capture-mode` | enum | `window` | `window`=capture EditorWindow including toolbar, `rendering`=capture game rendering only (PlayMode required, coordinates match simulate-mouse) | @@ -36,7 +36,13 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo ## Window Name -The window name is the text displayed in the window's title bar (tab). Common names: Game, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported. +The window name is the text displayed in the window's title bar (tab). Common names: Game, Simulator, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported. + +## Device Simulator + +- Prefer `--capture-mode rendering` for the annotate → click flow. It works with both the normal Game view and Device Simulator, and coordinates match `simulate-mouse-ui` / `simulate-mouse-input` / `raycast`. +- `--capture-mode window` captures Editor chrome (toolbar/borders). With Device Simulator as the play view, default `--window-name Game` falls back to `Simulator` when no Game tab exists; you can also pass `--window-name Simulator`. +- Simulator Fit to Screen / Scale / Safe Area overlays are chrome-only and do not change rendering-mode input coordinates. After device rotation, re-run annotate (or re-read `Screen` size) before clicking. ## Output diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md index 79f674f193..a093729923 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md @@ -2,6 +2,8 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-elements true` or `--annotate-raycast-grid true` output to find coordinates for `simulate-mouse-ui`, `simulate-mouse-input`, or `raycast`. +Device Simulator is supported for this flow: prefer `--capture-mode rendering` (not `window`) so coordinates match the simulated device resolution. + ## AnnotatedElements Fields `AnnotatedElements` is empty unless `--annotate-elements true` is used, or unless `--annotate-raycast-grid true` adds clustered 3D collider candidates (with or without `--raycast-layer-mask`). UI entries are sorted by z-order, frontmost first. Each item contains: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md index 17d634f31e..01eab3b2d7 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md @@ -106,6 +106,7 @@ unity_y = gameViewHeight - input_y ``` - `Mouse.current.position` uses bottom-left Unity coordinates, so the value read inside Unity may show the converted Y. +- Device Simulator play view is supported. Prefer rendering-mode screenshots for coordinates; they match the simulated device resolution, not Simulator chrome scale. ## Prerequisites diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md index 45aea53ee1..2fb68dc7be 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md @@ -67,6 +67,7 @@ uloop simulate-mouse-ui --action --x --y [options] - Dragging on empty space (no draggable UI element) returns `Success = false` - `--bypass-raycast` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path` - If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate +- Device Simulator play view is supported. Prefer `uloop screenshot --capture-mode rendering --annotate-elements` for coordinates; they use the simulated device resolution (`Handles.GetMainGameViewSize()` / `Screen`), not the Simulator chrome scale. ## Pause Point Inspection (Standard for E2E)