From 8aacb7e8d017725080dba5289018df54982bd8f3 Mon Sep 17 00:00:00 2001 From: hatayama Date: Thu, 9 Jul 2026 19:39:22 +0900 Subject: [PATCH 1/3] Stream C3 port: extend screenshot tool with raycast-grid annotation and coordinate metadata Ported main's final-state screenshot extension onto v3-beta's per-tool architecture: ScreenshotSchema/ScreenshotResponse gain AnnotateRaycastGrid, RaycastLayerMask, GameViewWidth/Height, ScreenshotToInputFormula, UnityInputFormula, RaycastGridPoints, and RaycastLayerSummaries. ScreenshotResponse's CoordinateSystem/YOffset fields were renamed to ImageCoordinateSystem/ImageToInputOffsetY to match main and support both window and rendering capture modes explicitly. EditorWindowCaptureUtility.CaptureGameRenderingAsync now shares a GameRenderingImageInfo struct with a new GetGameRenderingImageInfoAsync, so the raycast-grid path and the PNG-capture path reuse the same settled RenderTexture geometry without recomputing it twice, while preserving v3-beta's EditorFrameWaiter timeout-safety mechanism (not present in main). EditorWindowCaptureService, the public ToolContracts extension-API facade's internal bridge, now adapts to the new signature without changing its own public (texture, yOffset, timedOut) contract. UIElementAnnotator.CollectInteractiveElements threads a UiRaycastHelper.RaycastContext through element collection (replacing v3-beta's ad-hoc SharedPointerEventData reachability check), and CreateAnnotationOverlay now draws in two passes (all borders, then all labels) with a new outline-based border branch for PhysicsCollider elements, ported from main's e67dcba5. ScreenshotUseCase.CaptureRenderingAsync integrates raycast-grid collection (grid points, layer summaries, or clustered physics-collider elements depending on RaycastLayerMask) gated on AnnotateRaycastGrid, with matching parameter validation and a default-tools.json schema update needed immediately because DefaultToolsCatalogDriftTests compares the live tool schema against it. Verified: uloop compile (0 errors/0 warnings), full EditMode suite (1774 passed, 0 failed, 7 pre-existing skips), and PlayMode regression for SimulateMouseUi/InputReplayVerification to confirm no behavior change in existing UiRaycastHelper callers. --- .../Editor/EditorWindowCaptureUtilityTests.cs | 49 +++ .../EditorWindowCaptureUtilityTests.cs.meta | 11 + Assets/Tests/Editor/ScreenshotUseCaseTests.cs | 94 +++++ .../Editor/ScreenshotUseCaseTests.cs.meta | 11 + .../Tests/Editor/UIElementAnnotatorTests.cs | 43 +++ .../Screenshot/EditorWindowCaptureService.cs | 13 +- .../Screenshot/EditorWindowCaptureUtility.cs | 95 ++++- .../Screenshot/ScreenshotUseCase.cs | 190 +++++++++- .../Screenshot/UIElementAnnotator.cs | 346 ++++++++++++++---- .../ToolContracts/ScreenshotResponse.cs | 20 +- .../Editor/ToolContracts/ScreenshotSchema.cs | 5 +- .../ToolContracts/UnityCliLoopConstants.cs | 7 + .../UnityCliLoopScreenshotTypes.cs | 9 - cli/common/tools/default-tools.json | 14 +- 14 files changed, 777 insertions(+), 130 deletions(-) create mode 100644 Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs create mode 100644 Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta create mode 100644 Assets/Tests/Editor/ScreenshotUseCaseTests.cs create mode 100644 Assets/Tests/Editor/ScreenshotUseCaseTests.cs.meta diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs new file mode 100644 index 0000000000..30eeb4c4b0 --- /dev/null +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs @@ -0,0 +1,49 @@ +using NUnit.Framework; +using UnityEngine; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Test fixture that verifies Editor Window Capture Utility coordinate calculations. + /// + public class EditorWindowCaptureUtilityTests + { + [Test] + public void CalculateImageToInputOffsetY_WhenRenderTextureIsShorterThanGameView_ShouldReturnTopOffset() + { + // Tests that a RenderTexture shorter than the Game View yields a positive top offset. + Vector2 gameViewSize = new(1768f, 1383f); + + int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080); + + Assert.That(offsetY, Is.EqualTo(303)); + } + + [Test] + public void CalculateImageToInputOffsetY_WhenRenderTextureMatchesGameView_ShouldReturnZero() + { + // Tests that a RenderTexture matching the Game View height yields a zero offset. + Vector2 gameViewSize = new(1920f, 1080f); + + int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080); + + Assert.That(offsetY, Is.EqualTo(0)); + } + + [Test] + public void CreateUnavailableGameRenderingImageInfo_WhenRenderTextureIsMissing_ShouldUseGameViewSizeAndZeroOffset() + { + // Tests that the fallback info reuses the Game View size for both sizes with a zero offset. + Vector2 gameViewSize = new(1768f, 1383f); + + GameRenderingImageInfo renderingImageInfo = + EditorWindowCaptureUtility.CreateUnavailableGameRenderingImageInfo(gameViewSize); + + Assert.That(renderingImageInfo.GameViewSize, Is.EqualTo(gameViewSize)); + Assert.That(renderingImageInfo.RenderingImageSize, Is.EqualTo(gameViewSize)); + Assert.That(renderingImageInfo.ImageToInputOffsetY, Is.EqualTo(0)); + } + } +} diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta new file mode 100644 index 0000000000..1d4c1f6749 --- /dev/null +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 35f4b508a887640c298e8e5c79f9eb2b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/ScreenshotUseCaseTests.cs b/Assets/Tests/Editor/ScreenshotUseCaseTests.cs new file mode 100644 index 0000000000..5e7aaab094 --- /dev/null +++ b/Assets/Tests/Editor/ScreenshotUseCaseTests.cs @@ -0,0 +1,94 @@ +#nullable enable +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; +using io.github.hatayama.UnityCliLoop.ToolContracts; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Test fixture that verifies Screenshot Use Case parameter validation. + /// + public class ScreenshotUseCaseTests + { + [Test] + public void ExecuteAsync_WhenRaycastLayerMaskIsSetWithoutRaycastGrid_ShouldThrowValidationException() + { + // Tests that setting RaycastLayerMask without AnnotateRaycastGrid fails validation. + JObject parameters = new JObject + { + ["RaycastLayerMask"] = "Default" + }; + + UnityCliLoopToolParameterValidationException? exception = + Assert.ThrowsAsync( + async () => await ExecuteScreenshot(parameters)); + + Assert.That(exception!.Message, Does.Contain("RaycastLayerMask requires AnnotateRaycastGrid=true")); + } + + [Test] + public void ExecuteAsync_WhenElementsOnlyHasNoAnnotationMode_ShouldThrowValidationException() + { + // Tests that ElementsOnly without AnnotateElements or AnnotateRaycastGrid fails validation. + JObject parameters = new JObject + { + ["CaptureMode"] = "rendering", + ["ElementsOnly"] = true + }; + + UnityCliLoopToolParameterValidationException? exception = + Assert.ThrowsAsync( + async () => await ExecuteScreenshot(parameters)); + + Assert.That( + exception!.Message, + Does.Contain("ElementsOnly requires AnnotateElements=true or AnnotateRaycastGrid=true")); + } + + [Test] + public async Task ExecuteAsync_WhenElementsOnlyUsesRaycastGrid_ShouldPassValidation() + { + // Tests that ElementsOnly combined with AnnotateRaycastGrid passes validation + // (PlayMode is unavailable in EditMode, so capture itself no-ops after validation). + JObject parameters = new JObject + { + ["CaptureMode"] = "rendering", + ["AnnotateRaycastGrid"] = true, + ["ElementsOnly"] = true + }; + + UnityCliLoopToolResponse response = await ExecuteScreenshot(parameters); + + Assert.That(response, Is.InstanceOf()); + } + + [Test] + public void ExecuteAsync_WhenRaycastLayerMaskContainsUnknownLayer_ShouldThrowValidationException() + { + // Tests that an unrecognized layer name in RaycastLayerMask fails validation with the layer name in the message. + JObject parameters = new JObject + { + ["CaptureMode"] = "rendering", + ["AnnotateRaycastGrid"] = true, + ["RaycastLayerMask"] = "MissingLayerForTest" + }; + + UnityCliLoopToolParameterValidationException? exception = + Assert.ThrowsAsync( + async () => await ExecuteScreenshot(parameters)); + + Assert.That(exception!.Message, Does.Contain("unknown layer name")); + Assert.That(exception!.Message, Does.Contain("MissingLayerForTest")); + } + + private static async Task ExecuteScreenshot(JObject parameters) + { + ScreenshotTool tool = new(); + return await tool.ExecuteAsync(parameters, CancellationToken.None); + } + } +} diff --git a/Assets/Tests/Editor/ScreenshotUseCaseTests.cs.meta b/Assets/Tests/Editor/ScreenshotUseCaseTests.cs.meta new file mode 100644 index 0000000000..477dd0116b --- /dev/null +++ b/Assets/Tests/Editor/ScreenshotUseCaseTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 70bf64f00f65845479f85cb7f12d3d3c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/UIElementAnnotatorTests.cs b/Assets/Tests/Editor/UIElementAnnotatorTests.cs index 91acc9b760..cfd118b108 100644 --- a/Assets/Tests/Editor/UIElementAnnotatorTests.cs +++ b/Assets/Tests/Editor/UIElementAnnotatorTests.cs @@ -78,6 +78,49 @@ public void GetContrastPartnerColor_WhenColorIsDark_ShouldReturnDarkOutlineForLi Assert.That(outlineColor, Is.EqualTo(new Color(0f, 0f, 0f, 0.95f))); } + /// + /// Verifies that converting a top-left outline segment to screen space flips its Y coordinates. + /// + [Test] + public void ConvertTopLeftOutlineSegmentToScreenSegment_ShouldFlipYForCanvasSpace() + { + RaycastOutlineSegment inputSegment = new(10f, 20f, 30f, 20f); + + RaycastOutlineSegment screenSegment = + UIElementAnnotator.ConvertTopLeftOutlineSegmentToScreenSegment(inputSegment, 100f); + + Assert.That(screenSegment.StartX, Is.EqualTo(10f)); + Assert.That(screenSegment.StartY, Is.EqualTo(80f)); + Assert.That(screenSegment.EndX, Is.EqualTo(30f)); + Assert.That(screenSegment.EndY, Is.EqualTo(80f)); + } + + /// + /// Verifies that a horizontal outline segment's rect extends both endpoints by half the thickness. + /// + [Test] + public void CalculateOutlineSegmentRect_WhenHorizontalSegment_ShouldExtendBothEndpoints() + { + RaycastOutlineSegment segment = new(10f, 20f, 30f, 20f); + + Rect rect = UIElementAnnotator.CalculateOutlineSegmentRect(segment, 4f); + + Assert.That(rect, Is.EqualTo(new Rect(8f, 18f, 24f, 4f))); + } + + /// + /// Verifies that a vertical outline segment's rect extends both endpoints by half the thickness. + /// + [Test] + public void CalculateOutlineSegmentRect_WhenVerticalSegment_ShouldExtendBothEndpoints() + { + RaycastOutlineSegment segment = new(10f, 20f, 10f, 50f); + + Rect rect = UIElementAnnotator.CalculateOutlineSegmentRect(segment, 4f); + + Assert.That(rect, Is.EqualTo(new Rect(8f, 18f, 4f, 34f))); + } + [Test] public void CalculateBorderEdgeRects_WhenBoundsAreProvided_ShouldPlaceEdgesInsideTheBounds() { diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureService.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureService.cs index 180fb42002..58cb4d895e 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureService.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureService.cs @@ -37,15 +37,18 @@ public string[] GetOpenWindowNames() return EditorWindowCaptureUtility.GetOpenWindowNames(); } - public Task<(Texture2D? texture, int yOffset, bool timedOut)> CaptureGameRenderingAsync( + public async Task<(Texture2D? texture, int yOffset, bool timedOut)> CaptureGameRenderingAsync( float resolutionScale, int timeoutMilliseconds, CancellationToken ct) { - return EditorWindowCaptureUtility.CaptureGameRenderingAsync( - resolutionScale, - timeoutMilliseconds, - ct); + (Texture2D? texture, GameRenderingImageInfo renderingImageInfo, bool timedOut) = + await EditorWindowCaptureUtility.CaptureGameRenderingAsync( + resolutionScale, + null, + timeoutMilliseconds, + ct).ConfigureAwait(false); + return (texture, renderingImageInfo.ImageToInputOffsetY, timedOut); } } diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs index a27466e7f2..09e10a2d8b 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureUtility.cs @@ -6,7 +6,6 @@ using UnityEditor; using UnityEngine; -using io.github.hatayama.UnityCliLoop.Application; using io.github.hatayama.UnityCliLoop.InternalAPIBridge; using io.github.hatayama.UnityCliLoop.ToolContracts; @@ -165,11 +164,9 @@ public static string[] GetOpenWindowNames() // Captures game rendering by reading GameView's composited RenderTexture (PlayMode only). // Contains all cameras + Screen Space Overlay Canvas, without tab bar or borders. - // Coordinate mapping to simulate-mouse: - // sim_x = image_x / resolutionScale, sim_y = image_y / resolutionScale + YOffset - // where YOffset = Screen.height - RenderTexture.height (returned in the tuple). - public static async Task<(Texture2D? texture, int yOffset, bool timedOut)> CaptureGameRenderingAsync( + internal static async Task<(Texture2D? texture, GameRenderingImageInfo renderingImageInfo, bool timedOut)> CaptureGameRenderingAsync( float resolutionScale, + GameRenderingImageInfo? renderingImageInfo, int frameWaitTimeoutMilliseconds, CancellationToken ct) { @@ -184,7 +181,7 @@ public static string[] GetOpenWindowNames() ct).ConfigureAwait(false); if (!framesReady) { - return (null, 0, true); + return (null, default, true); } await CapturedEditorSynchronizationContext.SwitchTo(editorContext, ct); @@ -193,10 +190,14 @@ public static string[] GetOpenWindowNames() if (rt == null) { Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); - return (null, 0, false); + GameRenderingImageInfo unavailableInfo = renderingImageInfo ?? + CreateUnavailableGameRenderingImageInfo(Handles.GetMainGameViewSize()); + return (null, unavailableInfo, false); } - int yOffset = (int)Handles.GetMainGameViewSize().y - rt.height; + // GameView RenderTexture can be shorter than the full input area, so raw image Y needs this offset. + GameRenderingImageInfo captureInfo = renderingImageInfo ?? + CreateGameRenderingImageInfo(Handles.GetMainGameViewSize(), rt.width, rt.height); // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format RenderTextureDescriptor flipDescriptor = new(rt.width, rt.height, rt.format, 0); @@ -231,7 +232,63 @@ public static string[] GetOpenWindowNames() texture = ApplyResolutionScaling(texture, resolutionScale); } - return (texture, yOffset, false); + return (texture, captureInfo, false); + } + + // Raycast-grid annotations must use the same settled RenderTexture geometry as the PNG capture path. + internal static async Task<(GameRenderingImageInfo renderingImageInfo, bool timedOut)> GetGameRenderingImageInfoAsync( + int frameWaitTimeoutMilliseconds, + CancellationToken ct) + { + Debug.Assert(UnityEditor.EditorApplication.isPlaying, "GetGameRenderingImageInfoAsync requires PlayMode"); + + SynchronizationContext editorContext = + CapturedEditorSynchronizationContext.RequireCurrent("raycast grid rendering info capture"); + bool framesReady = await EditorFrameWaiter.WaitFramesOrTimeoutAsync( + 2, + frameWaitTimeoutMilliseconds, + ct).ConfigureAwait(false); + if (!framesReady) + { + return (default, true); + } + + await CapturedEditorSynchronizationContext.SwitchTo(editorContext, ct); + + Vector2 gameViewSize = Handles.GetMainGameViewSize(); + RenderTexture rt = GameViewBridge.GetRenderTexture(); + if (rt == null) + { + return (CreateUnavailableGameRenderingImageInfo(gameViewSize), false); + } + + return (CreateGameRenderingImageInfo(gameViewSize, rt.width, rt.height), false); + } + + internal static GameRenderingImageInfo CreateUnavailableGameRenderingImageInfo(Vector2 gameViewSize) + { + return new GameRenderingImageInfo(gameViewSize, gameViewSize, 0); + } + + private static GameRenderingImageInfo CreateGameRenderingImageInfo( + Vector2 gameViewSize, + int renderTextureWidth, + int renderTextureHeight) + { + Vector2 renderingImageSize = new(renderTextureWidth, renderTextureHeight); + int imageToInputOffsetY = CalculateImageToInputOffsetY(gameViewSize, renderTextureHeight); + return new GameRenderingImageInfo(gameViewSize, renderingImageSize, imageToInputOffsetY); + } + + /// + /// Calculates the Y offset from rendering screenshot image space to Game View input space. + /// + internal static int CalculateImageToInputOffsetY(Vector2 gameViewSize, int renderTextureHeight) + { + Debug.Assert(gameViewSize.y >= 0f, "Game View height must not be negative."); + Debug.Assert(renderTextureHeight >= 0, "RenderTexture height must not be negative."); + + return Mathf.RoundToInt(gameViewSize.y) - renderTextureHeight; } private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float scale) @@ -299,4 +356,24 @@ public static SwitchToMainThreadAwaitable SwitchTo( return MainThreadSwitcher.SwitchToMainThread(ct); } } + + /// + /// Describes the Game View geometry used to map rendering screenshots back to input coordinates. + /// + internal readonly struct GameRenderingImageInfo + { + public readonly Vector2 GameViewSize; + public readonly Vector2 RenderingImageSize; + public readonly int ImageToInputOffsetY; + + public GameRenderingImageInfo( + Vector2 gameViewSize, + Vector2 renderingImageSize, + int imageToInputOffsetY) + { + GameViewSize = gameViewSize; + RenderingImageSize = renderingImageSize; + ImageToInputOffsetY = imageToInputOffsetY; + } + } } diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs index b76723c471..2d0a2c3a4b 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs @@ -61,6 +61,12 @@ private async Task CaptureRenderingAsync( } List annotatedElements = new(); + List physicsColliderElements = new(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + List raycastGridPoints = new(); + List raycastLayerSummaries = new(); + List raycastGridOverlayElements = new(); + GameRenderingImageInfo? raycastGridRenderingInfo = null; if (request.AnnotateElements) { @@ -68,12 +74,57 @@ private async Task CaptureRenderingAsync( UIElementAnnotator.AssignLabels(annotatedElements); } + if (request.AnnotateRaycastGrid) + { + GameRenderingImageInfo renderingImageInfo; + bool gridInfoTimedOut; + (renderingImageInfo, gridInfoTimedOut) = await EditorWindowCaptureUtility.GetGameRenderingImageInfoAsync( + UnityCliLoopConstants.EDITOR_FRAME_WAIT_TIMEOUT_MS, + ct).ConfigureAwait(false); + if (gridInfoTimedOut) + { + return CreateTimedOutResult( + "raycast grid rendering info capture", + correlationId, + new List()); + } + + await CapturedEditorSynchronizationContext.SwitchTo(editorContext, ct); + + raycastGridRenderingInfo = renderingImageInfo; + gameViewSize = renderingImageInfo.GameViewSize; + RaycastLayerMaskResolution raycastLayerMaskResolution = ResolveRaycastLayerMask(request); + + if (raycastLayerMaskResolution.HasLayerNames) + { + physicsColliderElements = RaycastGridAnnotator.CollectPhysicsColliderElements( + renderingImageInfo.RenderingImageSize, + renderingImageInfo.ImageToInputOffsetY, + raycastLayerMaskResolution.Mask); + } + else + { + raycastGridPoints = RaycastGridAnnotator.CollectRaycastGridPoints( + renderingImageInfo.RenderingImageSize, + renderingImageInfo.ImageToInputOffsetY); + raycastLayerSummaries = RaycastGridAnnotator.CollectRaycastLayerSummaries( + renderingImageInfo.RenderingImageSize, + renderingImageInfo.ImageToInputOffsetY); + raycastGridOverlayElements = RaycastGridAnnotator.CreateOverlayElements(raycastGridPoints); + } + } + if (request.ElementsOnly) { - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y); - ScreenshotInfo elementsOnlyInfo = new(); - elementsOnlyInfo.CoordinateSystem = UnityCliLoopScreenshotCoordinateSystem.GameView; - elementsOnlyInfo.AnnotatedElements = annotatedElements; + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); + List elementsOnlyAnnotatedElements = + CreateResponseAnnotatedElements(annotatedElements, physicsColliderElements); + ScreenshotInfo elementsOnlyInfo = new() { ResolutionScale = request.ResolutionScale }; + int elementsOnlyImageToInputOffsetY = raycastGridRenderingInfo?.ImageToInputOffsetY ?? 0; + ApplyRenderingCoordinateMetadata(elementsOnlyInfo, gameViewSize, elementsOnlyImageToInputOffsetY); + elementsOnlyInfo.AnnotatedElements = elementsOnlyAnnotatedElements; + elementsOnlyInfo.RaycastGridPoints = raycastGridPoints; + elementsOnlyInfo.RaycastLayerSummaries = raycastLayerSummaries; return new ScreenshotResponse { Screenshots = new List { elementsOnlyInfo } @@ -82,14 +133,17 @@ private async Task CaptureRenderingAsync( GameObject annotationOverlay = null; Texture2D texture; - int yOffset; - bool timedOut; + GameRenderingImageInfo captureRenderingInfo; + bool captureTimedOut; try { - if (request.AnnotateElements) + if (request.AnnotateElements || request.AnnotateRaycastGrid) { + List overlayElements = new(annotatedElements); + overlayElements.AddRange(raycastGridOverlayElements); + overlayElements.AddRange(physicsColliderElements); annotationOverlay = UIElementAnnotator.CreateAnnotationOverlay( - annotatedElements, + overlayElements, request.ResolutionScale); Canvas.ForceUpdateCanvases(); // Chained CLI calls can read the previous GameView RT before overlay rendering catches up. @@ -108,11 +162,12 @@ private async Task CaptureRenderingAsync( await CapturedEditorSynchronizationContext.SwitchTo(editorContext, ct); } - (texture, yOffset, timedOut) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( + (texture, captureRenderingInfo, captureTimedOut) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( request.ResolutionScale, + raycastGridRenderingInfo, UnityCliLoopConstants.EDITOR_FRAME_WAIT_TIMEOUT_MS, ct).ConfigureAwait(false); - if (timedOut) + if (captureTimedOut) { return CreateTimedOutResult( "GameView rendering capture", @@ -127,7 +182,9 @@ private async Task CaptureRenderingAsync( DestroyAnnotationOverlay(annotationOverlay, editorContext); } - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y); + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); + List responseAnnotatedElements = + CreateResponseAnnotatedElements(annotatedElements, physicsColliderElements); if (texture == null) { @@ -158,11 +215,12 @@ private async Task CaptureRenderingAsync( FileSizeBytes = savedFileInfo.Length, Width = width, Height = height, - CoordinateSystem = UnityCliLoopScreenshotCoordinateSystem.GameView, ResolutionScale = request.ResolutionScale, - YOffset = yOffset, - AnnotatedElements = annotatedElements, }; + ApplyRenderingCoordinateMetadata(info, captureRenderingInfo.GameViewSize, captureRenderingInfo.ImageToInputOffsetY); + info.AnnotatedElements = responseAnnotatedElements; + info.RaycastGridPoints = raycastGridPoints; + info.RaycastLayerSummaries = raycastLayerSummaries; screenshots.Add(info); } catch (Exception ex) @@ -250,13 +308,15 @@ private async Task CaptureWindowsAsync( SaveTextureAsPng(texture, savedPath); FileInfo savedFileInfo = new(savedPath); - screenshots.Add(new ScreenshotInfo + ScreenshotInfo info = new() { ImagePath = savedPath, FileSizeBytes = savedFileInfo.Length, Width = width, Height = height, - }); + }; + ApplyWindowCoordinateMetadata(info); + screenshots.Add(info); } catch (Exception ex) { @@ -324,6 +384,35 @@ private static void DestroyAnnotationOverlay( editorContext.Post(_ => UIElementAnnotator.DestroyAnnotationOverlay(annotationOverlay), null); } + private static void ApplyRenderingCoordinateMetadata( + ScreenshotInfo info, + Vector2 gameViewSize, + int imageToInputOffsetY = 0) + { + info.ImageCoordinateSystem = UnityCliLoopConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW; + info.GameViewWidth = gameViewSize.x; + info.GameViewHeight = gameViewSize.y; + info.ImageToInputOffsetY = imageToInputOffsetY; + info.ScreenshotToInputFormula = UnityCliLoopConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA; + info.UnityInputFormula = UnityCliLoopConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY; + } + + private static void ApplyWindowCoordinateMetadata(ScreenshotInfo info) + { + info.ImageCoordinateSystem = UnityCliLoopConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; + info.ScreenshotToInputFormula = UnityCliLoopConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; + info.UnityInputFormula = ""; + } + + private static List CreateResponseAnnotatedElements( + List uiElements, + List physicsColliderElements) + { + List responseElements = new(uiElements); + responseElements.AddRange(physicsColliderElements); + return responseElements; + } + private void ValidateParameters(ScreenshotSchema request) { if (request.CaptureMode != CaptureMode.rendering && @@ -338,7 +427,7 @@ private void ValidateParameters(ScreenshotSchema request) $"ResolutionScale must be between 0.1 and 1.0, got: {request.ResolutionScale}"); } - // AnnotateElements and ElementsOnly rely on PlayMode rendering pipeline + // AnnotateElements, ElementsOnly, and AnnotateRaycastGrid rely on PlayMode rendering pipeline if (request.CaptureMode != CaptureMode.rendering) { if (request.AnnotateElements) @@ -350,12 +439,75 @@ private void ValidateParameters(ScreenshotSchema request) { throw new UnityCliLoopToolParameterValidationException("ElementsOnly is only supported when CaptureMode=rendering"); } + + if (request.AnnotateRaycastGrid) + { + throw new UnityCliLoopToolParameterValidationException("AnnotateRaycastGrid is only supported when CaptureMode=rendering"); + } + } + + if (request.ElementsOnly && + !request.AnnotateElements && + !request.AnnotateRaycastGrid) + { + throw new UnityCliLoopToolParameterValidationException( + "ElementsOnly requires AnnotateElements=true or AnnotateRaycastGrid=true"); + } + + RaycastLayerMaskResolution raycastLayerMaskResolution = ResolveRaycastLayerMask(request); + if (raycastLayerMaskResolution.HasLayerNames && !request.AnnotateRaycastGrid) + { + throw new UnityCliLoopToolParameterValidationException( + "RaycastLayerMask requires AnnotateRaycastGrid=true"); + } + + if (!raycastLayerMaskResolution.IsValid) + { + throw new UnityCliLoopToolParameterValidationException( + CreateInvalidRaycastLayerMaskMessage(raycastLayerMaskResolution)); + } + } + + private static RaycastLayerMaskResolution ResolveRaycastLayerMask(ScreenshotSchema request) + { + string raycastLayerMask = request.RaycastLayerMask ?? ""; + return RaycastLayerMaskResolver.Resolve( + raycastLayerMask, + GetAvailableLayerDefinitions()); + } + + private static List GetAvailableLayerDefinitions() + { + List layerDefinitions = new(); + for (int layerIndex = 0; layerIndex <= 31; layerIndex++) + { + string layerName = LayerMask.LayerToName(layerIndex); + if (string.IsNullOrEmpty(layerName)) + { + continue; + } + + layerDefinitions.Add(new RaycastLayerDefinition + { + Name = layerName, + Index = layerIndex + }); } - if (request.ElementsOnly && !request.AnnotateElements) + return layerDefinitions; + } + + private static string CreateInvalidRaycastLayerMaskMessage( + RaycastLayerMaskResolution raycastLayerMaskResolution) + { + string invalidLayerNames = string.Join(", ", raycastLayerMaskResolution.InvalidLayerNames); + string validLayerNames = string.Join(", ", raycastLayerMaskResolution.ValidLayerNames); + if (string.IsNullOrEmpty(validLayerNames)) { - throw new UnityCliLoopToolParameterValidationException("ElementsOnly requires AnnotateElements=true"); + validLayerNames = "(none)"; } + + return $"RaycastLayerMask contains unknown layer name(s): {invalidLayerNames}. Valid layers: {validLayerNames}"; } private string EnsureOutputDirectoryExists(string outputDirectory) diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs index 74a3c44fcb..a78de55a98 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/UIElementAnnotator.cs @@ -1,3 +1,4 @@ +#nullable enable using System.Collections.Generic; using UnityEngine; using UnityEngine.EventSystems; @@ -58,9 +59,10 @@ public static List CollectInteractiveElements() { List elements = new(); HashSet processedObjects = new(); + UiRaycastHelper.RaycastContext? raycastContext = CreateRaycastContextForCurrentEventSystem(); - CollectSelectables(elements, processedObjects); - CollectEventHandlers(elements, processedObjects); + CollectSelectables(elements, processedObjects, raycastContext); + CollectEventHandlers(elements, processedObjects, raycastContext); return elements; } @@ -117,10 +119,22 @@ public static GameObject CreateAnnotationOverlay(List elements, f canvas.sortingOrder = OVERLAY_SORT_ORDER; Font font = Resources.GetBuiltinResource("LegacyRuntime.ttf"); + List drawInfos = new(elements.Count); + float physicsAnnotationScreenHeight = CalculatePhysicsAnnotationScreenHeight(elements); foreach (UIElementInfo element in elements) { - CreateAnnotationForElement(root.transform, element, font, borderMetrics); + drawInfos.Add(CreateAnnotationDrawInfo(element, physicsAnnotationScreenHeight)); + } + + foreach (AnnotationDrawInfo drawInfo in drawInfos) + { + CreateAnnotationBorderForElement(root.transform, drawInfo, borderMetrics); + } + + foreach (AnnotationDrawInfo drawInfo in drawInfos) + { + CreateAnnotationLabelForElement(root.transform, drawInfo, font, borderMetrics); } return root; @@ -134,7 +148,10 @@ public static void DestroyAnnotationOverlay(GameObject overlay) } } - private static void CollectSelectables(List elements, HashSet processedObjects) + private static void CollectSelectables( + List elements, + HashSet processedObjects, + UiRaycastHelper.RaycastContext? raycastContext) { Selectable[] selectables = Selectable.allSelectablesArray; foreach (Selectable selectable in selectables) @@ -147,13 +164,16 @@ private static void CollectSelectables(List elements, HashSet IDropHandler > IPointerClickHandler > IPointerDownHandler - private static void CollectEventHandlers(List elements, HashSet processedObjects) + private static void CollectEventHandlers( + List elements, + HashSet processedObjects, + UiRaycastHelper.RaycastContext? raycastContext) { MonoBehaviour[] allBehaviours = Object.FindObjectsOfType(); foreach (MonoBehaviour behaviour in allBehaviours) @@ -168,18 +188,18 @@ private static void CollectEventHandlers(List elements, HashSet SharedRaycastResults = new List(); - private static PointerEventData SharedPointerEventData; - private static EventSystem SharedPointerEventSystem; - private static void AddElementInfo(List elements, GameObject go, string name, string type) + private static void AddElementInfo( + List elements, + GameObject go, + string name, + string type, + UiRaycastHelper.RaycastContext? raycastContext) { RectTransform rectTransform = go.GetComponent(); if (rectTransform == null) @@ -241,7 +263,7 @@ private static void AddElementInfo(List elements, GameObject go, float centerX = (minX + maxX) / 2f; float centerY = (minY + maxY) / 2f; - if (!IsRaycastReachable(go, centerX, centerY)) + if (!IsRaycastReachable(go, centerX, centerY, raycastContext)) { return; } @@ -266,50 +288,44 @@ private static void AddElementInfo(List elements, GameObject go, private static bool HasActiveGraphicRaycaster(Canvas canvas) { GraphicRaycaster raycaster = canvas.GetComponent(); - return raycaster != null && raycaster.isActiveAndEnabled; + return canvas.isActiveAndEnabled && raycaster != null && raycaster.isActiveAndEnabled; } - // simulate-mouse uses EventSystem.RaycastAll, so only advertise elements whose - // center point is actually hittable. Skips the check when no EventSystem exists - // (e.g. annotation-only scenes without interaction). - private static bool IsRaycastReachable(GameObject go, float centerX, float centerY) + // Uses the same raycast path as simulate-mouse so annotations match UI input behavior. + // Skips the check when no EventSystem exists, such as annotation-only scenes without interaction. + private static bool IsRaycastReachable( + GameObject go, + float centerX, + float centerY, + UiRaycastHelper.RaycastContext? raycastContext) { - EventSystem eventSystem = EventSystem.current; - if (eventSystem == null) + if (raycastContext == null) { return true; } - if (SharedPointerEventData == null || SharedPointerEventSystem != eventSystem) + RaycastResult? raycastResult = raycastContext.Raycast(new Vector2(centerX, centerY)); + if (raycastResult == null) { - SharedPointerEventData = new PointerEventData(eventSystem); - SharedPointerEventSystem = eventSystem; + return false; } - SharedPointerEventData.position = new Vector2(centerX, centerY); - - SharedRaycastResults.Clear(); - eventSystem.RaycastAll(SharedPointerEventData, SharedRaycastResults); Transform targetTransform = go.transform; - foreach (RaycastResult raycastResult in SharedRaycastResults) - { - Transform hitTransform = raycastResult.gameObject.transform; - if (hitTransform == targetTransform || hitTransform.IsChildOf(targetTransform)) - { - return true; - } - } + Transform hitTransform = raycastResult.Value.gameObject.transform; + return hitTransform == targetTransform || hitTransform.IsChildOf(targetTransform); + } - // EventSystem clips at Screen.width/height which can be smaller than the - // Canvas layout space (Game view target resolution). Check Canvas space directly. - RectTransform rectTransform = go.GetComponent(); - if (rectTransform != null) + // Reuses one raycast context while collecting annotations because a screenshot can + // test many UI elements in one frame. + private static UiRaycastHelper.RaycastContext? CreateRaycastContextForCurrentEventSystem() + { + EventSystem eventSystem = EventSystem.current; + if (eventSystem == null) { - return RectTransformUtility.RectangleContainsScreenPoint( - rectTransform, new Vector2(centerX, centerY), null); + return null; } - return false; + return new UiRaycastHelper.RaycastContext(eventSystem); } // Writes 4 corners into SharedScreenCorners in screen pixel coordinates (bottom-left origin). @@ -359,51 +375,169 @@ private static bool GetScreenCorners(RectTransform rectTransform, Canvas canvas) return true; } - private static void CreateAnnotationForElement( + private static void CreateAnnotationBorderForElement( Transform parent, - UIElementInfo element, - Font font, + AnnotationDrawInfo drawInfo, AnnotationBorderMetrics borderMetrics) { - float screenMinX = element.BoundsMinX; - float screenMaxX = element.BoundsMaxX; - float screenMinY = element.BoundsMinY; - float screenMaxY = element.BoundsMaxY; - - Color color = GetAnnotationColorForElement(element); - Color contrastColor = GetContrastingTextColor(color); - AnnotationBorderColors borderColors = GetAnnotationBorderColors(color); + if (drawInfo.OutlineSegments.Count > 0) + { + CreateAnnotationOutlineForElement(parent, drawInfo, borderMetrics); + return; + } CreateBorder( parent, "LightOuter", - screenMinX - borderMetrics.OuterOffset, - screenMinY - borderMetrics.OuterOffset, - screenMaxX + borderMetrics.OuterOffset, - screenMaxY + borderMetrics.OuterOffset, + drawInfo.ScreenMinX - borderMetrics.OuterOffset, + drawInfo.ScreenMinY - borderMetrics.OuterOffset, + drawInfo.ScreenMaxX + borderMetrics.OuterOffset, + drawInfo.ScreenMaxY + borderMetrics.OuterOffset, borderMetrics.NeutralThickness, - borderColors.Outer); + drawInfo.BorderColors.Outer); CreateBorder( parent, "ColorMiddle", - screenMinX - borderMetrics.ColorOffset, - screenMinY - borderMetrics.ColorOffset, - screenMaxX + borderMetrics.ColorOffset, - screenMaxY + borderMetrics.ColorOffset, + drawInfo.ScreenMinX - borderMetrics.ColorOffset, + drawInfo.ScreenMinY - borderMetrics.ColorOffset, + drawInfo.ScreenMaxX + borderMetrics.ColorOffset, + drawInfo.ScreenMaxY + borderMetrics.ColorOffset, borderMetrics.ColorThickness, - borderColors.Middle); - CreateBorder(parent, "DarkInner", screenMinX, screenMinY, screenMaxX, screenMaxY, borderMetrics.NeutralThickness, borderColors.Inner); + drawInfo.BorderColors.Middle); + CreateBorder( + parent, + "DarkInner", + drawInfo.ScreenMinX, + drawInfo.ScreenMinY, + drawInfo.ScreenMaxX, + drawInfo.ScreenMaxY, + borderMetrics.NeutralThickness, + drawInfo.BorderColors.Inner); + } - string labelText = CreateDisplayLabel(element); + private static void CreateAnnotationOutlineForElement( + Transform parent, + AnnotationDrawInfo drawInfo, + AnnotationBorderMetrics borderMetrics) + { + float outerThickness = borderMetrics.ColorThickness + borderMetrics.NeutralThickness * 2f; + CreateOutline( + parent, + "LightOuter", + drawInfo.OutlineSegments, + outerThickness, + drawInfo.BorderColors.Outer); + CreateOutline( + parent, + "ColorMiddle", + drawInfo.OutlineSegments, + borderMetrics.ColorThickness, + drawInfo.BorderColors.Middle); + CreateOutline( + parent, + "DarkInner", + drawInfo.OutlineSegments, + borderMetrics.NeutralThickness, + drawInfo.BorderColors.Inner); + } + + private static void CreateAnnotationLabelForElement( + Transform parent, + AnnotationDrawInfo drawInfo, + Font font, + AnnotationBorderMetrics borderMetrics) + { CreateLabel( parent, - labelText, + drawInfo.DisplayLabel, + drawInfo.ScreenMinX, + drawInfo.ScreenMaxY + borderMetrics.OuterOffset + borderMetrics.LabelOutlineDistance + borderMetrics.LabelToBorderGap, + drawInfo.Color, + drawInfo.ContrastColor, + font, + borderMetrics.LabelOutlineDistance); + } + + private static AnnotationDrawInfo CreateAnnotationDrawInfo( + UIElementInfo element, + float physicsAnnotationScreenHeight) + { + Color color = GetAnnotationColorForElement(element); + Color contrastColor = GetContrastingTextColor(color); + AnnotationBorderColors borderColors = GetAnnotationBorderColors(color); + string displayLabel = CreateDisplayLabel(element); + float screenMinX = element.BoundsMinX; + float screenMinY = element.BoundsMinY; + float screenMaxX = element.BoundsMaxX; + float screenMaxY = element.BoundsMaxY; + List outlineSegments = element.RaycastOutlineSegments; + + if (IsPhysicsColliderElement(element)) + { + Debug.Assert( + physicsAnnotationScreenHeight >= 0f, + "Physics collider annotations require a non-negative Game View height."); + screenMinY = physicsAnnotationScreenHeight - element.BoundsMaxY; + screenMaxY = physicsAnnotationScreenHeight - element.BoundsMinY; + outlineSegments = ConvertTopLeftOutlineSegmentsToScreenSegments( + element.RaycastOutlineSegments, + physicsAnnotationScreenHeight); + } + + return new AnnotationDrawInfo( screenMinX, - screenMaxY + borderMetrics.OuterOffset + borderMetrics.LabelOutlineDistance + borderMetrics.LabelToBorderGap, + screenMinY, + screenMaxX, + screenMaxY, color, contrastColor, - font, - borderMetrics.LabelOutlineDistance); + borderColors, + displayLabel, + outlineSegments); + } + + private static float CalculatePhysicsAnnotationScreenHeight(List elements) + { + foreach (UIElementInfo element in elements) + { + if (!IsPhysicsColliderElement(element)) + { + continue; + } + + return GameViewCoordinateUtility.GetMainGameViewSize().y; + } + + return 0f; + } + + private static bool IsPhysicsColliderElement(UIElementInfo element) + { + return element.Type == "PhysicsCollider"; + } + + private static List ConvertTopLeftOutlineSegmentsToScreenSegments( + List outlineSegments, + float screenHeight) + { + List screenSegments = new(outlineSegments.Count); + foreach (RaycastOutlineSegment segment in outlineSegments) + { + screenSegments.Add(ConvertTopLeftOutlineSegmentToScreenSegment(segment, screenHeight)); + } + + return screenSegments; + } + + internal static RaycastOutlineSegment ConvertTopLeftOutlineSegmentToScreenSegment( + RaycastOutlineSegment segment, + float screenHeight) + { + return new RaycastOutlineSegment( + segment.StartX, + screenHeight - segment.StartY, + segment.EndX, + screenHeight - segment.EndY); } private static void CreateBorder( @@ -419,6 +553,39 @@ private static void CreateBorder( CreateBorderEdge(parent, $"{name}_Right", borderEdgeRects.Right, color); } + private static void CreateOutline( + Transform parent, + string name, + List outlineSegments, + float thickness, + Color color) + { + for (int i = 0; i < outlineSegments.Count; i++) + { + Rect rect = CalculateOutlineSegmentRect(outlineSegments[i], thickness); + CreateBorderEdge(parent, $"{name}_Outline_{i}", rect, color); + } + } + + internal static Rect CalculateOutlineSegmentRect(RaycastOutlineSegment segment, float thickness) + { + Debug.Assert(thickness >= 0f, "Outline thickness must not be negative."); + bool horizontal = Mathf.Approximately(segment.StartY, segment.EndY); + bool vertical = Mathf.Approximately(segment.StartX, segment.EndX); + Debug.Assert(horizontal || vertical, "Raycast outline segments must be axis-aligned."); + + if (horizontal) + { + float minX = Mathf.Min(segment.StartX, segment.EndX) - thickness / 2f; + float width = Mathf.Abs(segment.EndX - segment.StartX) + thickness; + return new Rect(minX, segment.StartY - thickness / 2f, width, thickness); + } + + float minY = Mathf.Min(segment.StartY, segment.EndY) - thickness / 2f; + float height = Mathf.Abs(segment.EndY - segment.StartY) + thickness; + return new Rect(segment.StartX - thickness / 2f, minY, thickness, height); + } + internal static BorderEdgeRects CalculateBorderEdgeRects( float minX, float minY, float maxX, float maxY, float thickness) { @@ -511,7 +678,7 @@ internal static Color GetAnnotationColorForElement(UIElementInfo element) { Debug.Assert(element != null, "UIElementInfo must not be null."); - int labelIndex = GetLabelIndex(element.Label); + int labelIndex = GetLabelIndex(element!.Label); if (labelIndex < 0) { return FALLBACK_COLOR; @@ -589,7 +756,7 @@ internal static string CreateDisplayLabel(UIElementInfo element) { Debug.Assert(element != null, "UIElementInfo must not be null."); - string interaction = element.Interaction; + string interaction = element!.Interaction; if (string.IsNullOrEmpty(interaction)) { interaction = GetInteractionForType(element.Type); @@ -660,6 +827,41 @@ public AnnotationBorderColors(Color inner, Color middle, Color outer) } } + private readonly struct AnnotationDrawInfo + { + public readonly float ScreenMinX; + public readonly float ScreenMinY; + public readonly float ScreenMaxX; + public readonly float ScreenMaxY; + public readonly Color Color; + public readonly Color ContrastColor; + public readonly AnnotationBorderColors BorderColors; + public readonly string DisplayLabel; + public readonly List OutlineSegments; + + public AnnotationDrawInfo( + float screenMinX, + float screenMinY, + float screenMaxX, + float screenMaxY, + Color color, + Color contrastColor, + AnnotationBorderColors borderColors, + string displayLabel, + List outlineSegments) + { + ScreenMinX = screenMinX; + ScreenMinY = screenMinY; + ScreenMaxX = screenMaxX; + ScreenMaxY = screenMaxY; + Color = color; + ContrastColor = contrastColor; + BorderColors = borderColors; + DisplayLabel = displayLabel; + OutlineSegments = outlineSegments; + } + } + internal readonly struct AnnotationBorderMetrics { public readonly float NeutralThickness; diff --git a/Packages/src/Editor/ToolContracts/ScreenshotResponse.cs b/Packages/src/Editor/ToolContracts/ScreenshotResponse.cs index bffc107dcf..80548624d5 100644 --- a/Packages/src/Editor/ToolContracts/ScreenshotResponse.cs +++ b/Packages/src/Editor/ToolContracts/ScreenshotResponse.cs @@ -2,8 +2,6 @@ using System.Collections.Generic; -using io.github.hatayama.UnityCliLoop.ToolContracts; - namespace io.github.hatayama.UnityCliLoop.ToolContracts { /// @@ -15,20 +13,20 @@ public class ScreenshotInfo public long FileSizeBytes { get; set; } public int Width { get; set; } public int Height { get; set; } - - // "gameView": image from game RenderTexture. Convert to simulate-mouse coords: - // sim_x = image_x / ResolutionScale - // sim_y = image_y / ResolutionScale + YOffset - // "window": EditorWindow capture including toolbar - public string CoordinateSystem { get; set; } = UnityCliLoopScreenshotCoordinateSystem.Window; - + public string ImageCoordinateSystem { get; set; } = UnityCliLoopConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; public float ResolutionScale { get; set; } = 1.0f; // Y offset to add to image pixel Y to get simulate-mouse Y coordinate. - // Only meaningful when CoordinateSystem == "gameView". - public int YOffset { get; set; } + // Only meaningful when ImageCoordinateSystem is the Game View system. + public int ImageToInputOffsetY { get; set; } + public float GameViewWidth { get; set; } + public float GameViewHeight { get; set; } + public string ScreenshotToInputFormula { get; set; } = UnityCliLoopConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; + public string UnityInputFormula { get; set; } = ""; public List AnnotatedElements { get; set; } = new List(); + public List RaycastGridPoints { get; set; } = new List(); + public List RaycastLayerSummaries { get; set; } = new List(); } /// diff --git a/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs b/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs index e71d6cfe48..4107a546b0 100644 --- a/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs +++ b/Packages/src/Editor/ToolContracts/ScreenshotSchema.cs @@ -1,6 +1,3 @@ - -using io.github.hatayama.UnityCliLoop.ToolContracts; - namespace io.github.hatayama.UnityCliLoop.ToolContracts { /// @@ -15,5 +12,7 @@ public class ScreenshotSchema : UnityCliLoopToolSchema public CaptureMode CaptureMode { get; set; } = CaptureMode.window; public bool AnnotateElements { get; set; } = false; public bool ElementsOnly { get; set; } = false; + public bool AnnotateRaycastGrid { get; set; } = false; + public string RaycastLayerMask { get; set; } = ""; } } diff --git a/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs b/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs index 883429a028..b0356db22c 100644 --- a/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs +++ b/Packages/src/Editor/ToolContracts/UnityCliLoopConstants.cs @@ -93,6 +93,13 @@ public static UnityEditor.PackageManager.PackageInfo PackageInfo public const string COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY = "unity_x = input_x; unity_y = gameViewHeight - input_y"; + // Screenshot tool coordinate constants + public const string COORDINATE_SYSTEM_TOP_LEFT_WINDOW = "top-left-window"; + public const string SCREENSHOT_RENDERING_TO_INPUT_FORMULA = + "simulate_mouse_x = image_x / resolutionScale; simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY"; + public const string SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE = + "unavailable: window screenshots include Unity Editor chrome; use capture-mode rendering for mouse input coordinates"; + public const int CORRELATION_ID_LENGTH = 8; public const string GUID_FORMAT_NO_HYPHENS = "N"; diff --git a/Packages/src/Editor/ToolContracts/UnityCliLoopScreenshotTypes.cs b/Packages/src/Editor/ToolContracts/UnityCliLoopScreenshotTypes.cs index 33d3eba266..7ab4b4d2d3 100644 --- a/Packages/src/Editor/ToolContracts/UnityCliLoopScreenshotTypes.cs +++ b/Packages/src/Editor/ToolContracts/UnityCliLoopScreenshotTypes.cs @@ -12,13 +12,4 @@ public enum CaptureMode window = 0, rendering = 1 } - - /// - /// Provides Unity CLI Loop Screenshot Coordinate System behavior for Unity CLI Loop. - /// - public static class UnityCliLoopScreenshotCoordinateSystem - { - public const string Window = "window"; - public const string GameView = "gameView"; - } } diff --git a/cli/common/tools/default-tools.json b/cli/common/tools/default-tools.json index 465c71567e..78b67973a4 100644 --- a/cli/common/tools/default-tools.json +++ b/cli/common/tools/default-tools.json @@ -252,7 +252,7 @@ }, "CaptureMode": { "type": "string", - "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Response includes CoordinateSystem ('gameView' or 'window'), ResolutionScale, and YOffset. For gameView: sim_x = image_x / ResolutionScale, sim_y = image_y / ResolutionScale + YOffset.", + "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Rendering screenshots return ScreenshotToInputFormula for converting raw image pixels before calling simulate-mouse-input or raycast.", "enum": [ "window", "rendering" @@ -266,8 +266,18 @@ }, "ElementsOnly": { "type": "boolean", - "description": "Return only annotated element JSON without capturing a screenshot image. Requires AnnotateElements=true and CaptureMode=rendering in PlayMode.", + "description": "Return only annotation JSON without capturing a screenshot image. Requires AnnotateElements=true or AnnotateRaycastGrid=true, and CaptureMode=rendering in PlayMode.", "default": false + }, + "AnnotateRaycastGrid": { + "type": "boolean", + "description": "Annotate 3D physics raycast candidate points on rendering screenshots. Uses Camera.main, Camera.main.cullingMask visibility, and the same top-left Game View coordinates as simulate-mouse-input.", + "default": false + }, + "RaycastLayerMask": { + "type": "string", + "description": "Comma-separated physics layer names used by AnnotateRaycastGrid. Hits are limited to layers also visible to Camera.main.cullingMask. When set, returns clustered PhysicsCollider entries in AnnotatedElements instead of the coarse grid points.", + "default": "" } } } From a5c66ac307e0f068586c1ead4d232277ba201159 Mon Sep 17 00:00:00 2001 From: hatayama Date: Thu, 9 Jul 2026 19:55:28 +0900 Subject: [PATCH 2/3] Fix annotated element coordinates to use settled capture-time GameView size CodeRabbit flagged that ConvertToSimCoordinates used the pre-capture gameViewSize sample instead of captureRenderingInfo.GameViewSize, which is already used for the response's GameViewWidth/Height metadata a few lines later. If the Game View resizes during the overlay-render wait, the two values diverge and the returned AnnotatedElements coordinates become inconsistent with the formula the response advertises. --- .../Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs index 2d0a2c3a4b..2c7bdff4d8 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/ScreenshotUseCase.cs @@ -182,7 +182,9 @@ private async Task CaptureRenderingAsync( DestroyAnnotationOverlay(annotationOverlay, editorContext); } - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); + // Uses the settled capture-time size, not the pre-capture gameViewSize sample, so annotated + // element coordinates stay consistent with the GameViewWidth/Height this response reports. + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(captureRenderingInfo.GameViewSize.y)); List responseAnnotatedElements = CreateResponseAnnotatedElements(annotatedElements, physicsColliderElements); From 6492414020dcf5b8eb140112797ac6eb1497b43f Mon Sep 17 00:00:00 2001 From: hatayama Date: Thu, 9 Jul 2026 20:05:48 +0900 Subject: [PATCH 3/3] Fix screenshot skill docs to match renamed response fields CoordinateSystem/YOffset were renamed to ImageCoordinateSystem/ ImageToInputOffsetY earlier in this PR, but the screenshot skill docs (and their .claude/.agents generated copies) still referenced the old field names, actively guiding agents toward fields that no longer exist on the response. Regenerated the installed copies via `uloop skills install --claude --agents`. --- .agents/skills/uloop-screenshot/SKILL.md | 5 +++-- .../uloop-screenshot/references/annotated-elements.md | 10 +++++----- .claude/skills/uloop-screenshot/SKILL.md | 5 +++-- .../uloop-screenshot/references/annotated-elements.md | 10 +++++----- .../Editor/FirstPartyTools/Screenshot/Skill/SKILL.md | 5 +++-- .../Screenshot/Skill/references/annotated-elements.md | 10 +++++----- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.agents/skills/uloop-screenshot/SKILL.md b/.agents/skills/uloop-screenshot/SKILL.md index 39e81c686e..6180a4b166 100644 --- a/.agents/skills/uloop-screenshot/SKILL.md +++ b/.agents/skills/uloop-screenshot/SKILL.md @@ -79,9 +79,10 @@ Returns JSON with: - `FileSizeBytes`: Size of the saved file in bytes - `Width`: Captured image width in pixels - `Height`: Captured image height in pixels - - `CoordinateSystem`: `"gameView"` or `"window"` + - `ImageCoordinateSystem`: `"top-left-game-view"` or `"top-left-window"` - `ResolutionScale`: Resolution scale used for capture - - `YOffset`: Y offset used for gameView coordinate conversion + - `ImageToInputOffsetY`: Y offset used for top-left-game-view coordinate conversion + - `ScreenshotToInputFormula`: Formula converting raw image pixels to simulate-mouse input coordinates - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used. For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. diff --git a/.agents/skills/uloop-screenshot/references/annotated-elements.md b/.agents/skills/uloop-screenshot/references/annotated-elements.md index 283109ccc2..13b193d535 100644 --- a/.agents/skills/uloop-screenshot/references/annotated-elements.md +++ b/.agents/skills/uloop-screenshot/references/annotated-elements.md @@ -18,18 +18,18 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme ## Coordinate Conversion -When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates: +When `ImageCoordinateSystem` is `"top-left-game-view"`, convert image pixel coordinates to simulate-mouse coordinates using `ScreenshotToInputFormula`: ```text -sim_x = image_x / ResolutionScale -sim_y = image_y / ResolutionScale + YOffset +simulate_mouse_x = image_x / resolutionScale +simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY ``` When `ResolutionScale` is `1.0`, this simplifies to: ```text -sim_x = image_x -sim_y = image_y + YOffset +simulate_mouse_x = image_x +simulate_mouse_y = image_y + imageToInputOffsetY ``` ## Annotation Readability diff --git a/.claude/skills/uloop-screenshot/SKILL.md b/.claude/skills/uloop-screenshot/SKILL.md index 39e81c686e..6180a4b166 100644 --- a/.claude/skills/uloop-screenshot/SKILL.md +++ b/.claude/skills/uloop-screenshot/SKILL.md @@ -79,9 +79,10 @@ Returns JSON with: - `FileSizeBytes`: Size of the saved file in bytes - `Width`: Captured image width in pixels - `Height`: Captured image height in pixels - - `CoordinateSystem`: `"gameView"` or `"window"` + - `ImageCoordinateSystem`: `"top-left-game-view"` or `"top-left-window"` - `ResolutionScale`: Resolution scale used for capture - - `YOffset`: Y offset used for gameView coordinate conversion + - `ImageToInputOffsetY`: Y offset used for top-left-game-view coordinate conversion + - `ScreenshotToInputFormula`: Formula converting raw image pixels to simulate-mouse input coordinates - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used. For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. diff --git a/.claude/skills/uloop-screenshot/references/annotated-elements.md b/.claude/skills/uloop-screenshot/references/annotated-elements.md index 283109ccc2..13b193d535 100644 --- a/.claude/skills/uloop-screenshot/references/annotated-elements.md +++ b/.claude/skills/uloop-screenshot/references/annotated-elements.md @@ -18,18 +18,18 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme ## Coordinate Conversion -When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates: +When `ImageCoordinateSystem` is `"top-left-game-view"`, convert image pixel coordinates to simulate-mouse coordinates using `ScreenshotToInputFormula`: ```text -sim_x = image_x / ResolutionScale -sim_y = image_y / ResolutionScale + YOffset +simulate_mouse_x = image_x / resolutionScale +simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY ``` When `ResolutionScale` is `1.0`, this simplifies to: ```text -sim_x = image_x -sim_y = image_y + YOffset +simulate_mouse_x = image_x +simulate_mouse_y = image_y + imageToInputOffsetY ``` ## Annotation Readability diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md index 39e81c686e..6180a4b166 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/SKILL.md @@ -79,9 +79,10 @@ Returns JSON with: - `FileSizeBytes`: Size of the saved file in bytes - `Width`: Captured image width in pixels - `Height`: Captured image height in pixels - - `CoordinateSystem`: `"gameView"` or `"window"` + - `ImageCoordinateSystem`: `"top-left-game-view"` or `"top-left-window"` - `ResolutionScale`: Resolution scale used for capture - - `YOffset`: Y offset used for gameView coordinate conversion + - `ImageToInputOffsetY`: Y offset used for top-left-game-view coordinate conversion + - `ScreenshotToInputFormula`: Formula converting raw image pixels to simulate-mouse input coordinates - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used. For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md index 283109ccc2..13b193d535 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md @@ -18,18 +18,18 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme ## Coordinate Conversion -When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates: +When `ImageCoordinateSystem` is `"top-left-game-view"`, convert image pixel coordinates to simulate-mouse coordinates using `ScreenshotToInputFormula`: ```text -sim_x = image_x / ResolutionScale -sim_y = image_y / ResolutionScale + YOffset +simulate_mouse_x = image_x / resolutionScale +simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY ``` When `ResolutionScale` is `1.0`, this simplifies to: ```text -sim_x = image_x -sim_y = image_y + YOffset +simulate_mouse_x = image_x +simulate_mouse_y = image_y + imageToInputOffsetY ``` ## Annotation Readability