feat: Screenshot tool can now annotate 3D physics raycast candidates

.agents/skills/uloop-screenshot/SKILL.md

@@ -79,9 +79,10 @@ Returns JSON with:
   - `FileSizeBytes`: Size of the saved file in bytes
   - `Width`: Captured image width in pixels
   - `Height`: Captured image height in pixels
-  - `CoordinateSystem`: `"gameView"` or `"window"`
+  - `ImageCoordinateSystem`: `"top-left-game-view"` or `"top-left-window"`
   - `ResolutionScale`: Resolution scale used for capture
-  - `YOffset`: Y offset used for gameView coordinate conversion
+  - `ImageToInputOffsetY`: Y offset used for top-left-game-view coordinate conversion
+  - `ScreenshotToInputFormula`: Formula converting raw image pixels to simulate-mouse input coordinates
   - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used.
 
 For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools.

.agents/skills/uloop-screenshot/references/annotated-elements.md

@@ -18,18 +18,18 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme
 
 ## Coordinate Conversion
 
-When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates:
+When `ImageCoordinateSystem` is `"top-left-game-view"`, convert image pixel coordinates to simulate-mouse coordinates using `ScreenshotToInputFormula`:
 
 ```text
-sim_x = image_x / ResolutionScale
-sim_y = image_y / ResolutionScale + YOffset
+simulate_mouse_x = image_x / resolutionScale
+simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY
 ```
 
 When `ResolutionScale` is `1.0`, this simplifies to:
 
 ```text
-sim_x = image_x
-sim_y = image_y + YOffset
+simulate_mouse_x = image_x
+simulate_mouse_y = image_y + imageToInputOffsetY
 ```
 
 ## Annotation Readability

.claude/skills/uloop-screenshot/SKILL.md

@@ -79,9 +79,10 @@ Returns JSON with:
   - `FileSizeBytes`: Size of the saved file in bytes
   - `Width`: Captured image width in pixels
   - `Height`: Captured image height in pixels
-  - `CoordinateSystem`: `"gameView"` or `"window"`
+  - `ImageCoordinateSystem`: `"top-left-game-view"` or `"top-left-window"`
   - `ResolutionScale`: Resolution scale used for capture
-  - `YOffset`: Y offset used for gameView coordinate conversion
+  - `ImageToInputOffsetY`: Y offset used for top-left-game-view coordinate conversion
+  - `ScreenshotToInputFormula`: Formula converting raw image pixels to simulate-mouse input coordinates
   - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used.
 
 For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools.

.claude/skills/uloop-screenshot/references/annotated-elements.md

@@ -18,18 +18,18 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme
 
 ## Coordinate Conversion
 
-When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates:
+When `ImageCoordinateSystem` is `"top-left-game-view"`, convert image pixel coordinates to simulate-mouse coordinates using `ScreenshotToInputFormula`:
 
 ```text
-sim_x = image_x / ResolutionScale
-sim_y = image_y / ResolutionScale + YOffset
+simulate_mouse_x = image_x / resolutionScale
+simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY
 ```
 
 When `ResolutionScale` is `1.0`, this simplifies to:
 
 ```text
-sim_x = image_x
-sim_y = image_y + YOffset
+simulate_mouse_x = image_x
+simulate_mouse_y = image_y + imageToInputOffsetY
 ```
 
 ## Annotation Readability

Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs

@@ -0,0 +1,49 @@
+using NUnit.Framework;
+using UnityEngine;
+
+using io.github.hatayama.UnityCliLoop.FirstPartyTools;
+
+namespace io.github.hatayama.UnityCliLoop.Tests.Editor
+{
+    /// <summary>
+    /// Test fixture that verifies Editor Window Capture Utility coordinate calculations.
+    /// </summary>
+    public class EditorWindowCaptureUtilityTests
+    {
+        [Test]
+        public void CalculateImageToInputOffsetY_WhenRenderTextureIsShorterThanGameView_ShouldReturnTopOffset()
+        {
+            // Tests that a RenderTexture shorter than the Game View yields a positive top offset.
+            Vector2 gameViewSize = new(1768f, 1383f);
+
+            int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080);
+
+            Assert.That(offsetY, Is.EqualTo(303));
+        }
+
+        [Test]
+        public void CalculateImageToInputOffsetY_WhenRenderTextureMatchesGameView_ShouldReturnZero()
+        {
+            // Tests that a RenderTexture matching the Game View height yields a zero offset.
+            Vector2 gameViewSize = new(1920f, 1080f);
+
+            int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080);
+
+            Assert.That(offsetY, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void CreateUnavailableGameRenderingImageInfo_WhenRenderTextureIsMissing_ShouldUseGameViewSizeAndZeroOffset()
+        {
+            // Tests that the fallback info reuses the Game View size for both sizes with a zero offset.
+            Vector2 gameViewSize = new(1768f, 1383f);
+
+            GameRenderingImageInfo renderingImageInfo =
+                EditorWindowCaptureUtility.CreateUnavailableGameRenderingImageInfo(gameViewSize);
+
+            Assert.That(renderingImageInfo.GameViewSize, Is.EqualTo(gameViewSize));
+            Assert.That(renderingImageInfo.RenderingImageSize, Is.EqualTo(gameViewSize));
+            Assert.That(renderingImageInfo.ImageToInputOffsetY, Is.EqualTo(0));
+        }
+    }
+}

Assets/Tests/Editor/ScreenshotUseCaseTests.cs

@@ -0,0 +1,94 @@
+#nullable enable
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using NUnit.Framework;
+
+using io.github.hatayama.UnityCliLoop.FirstPartyTools;
+using io.github.hatayama.UnityCliLoop.ToolContracts;
+
+namespace io.github.hatayama.UnityCliLoop.Tests.Editor
+{
+    /// <summary>
+    /// Test fixture that verifies Screenshot Use Case parameter validation.
+    /// </summary>
+    public class ScreenshotUseCaseTests
+    {
+        [Test]
+        public void ExecuteAsync_WhenRaycastLayerMaskIsSetWithoutRaycastGrid_ShouldThrowValidationException()
+        {
+            // Tests that setting RaycastLayerMask without AnnotateRaycastGrid fails validation.
+            JObject parameters = new JObject
+            {
+                ["RaycastLayerMask"] = "Default"
+            };
+
+            UnityCliLoopToolParameterValidationException? exception =
+                Assert.ThrowsAsync<UnityCliLoopToolParameterValidationException>(
+                    async () => await ExecuteScreenshot(parameters));
+
+            Assert.That(exception!.Message, Does.Contain("RaycastLayerMask requires AnnotateRaycastGrid=true"));
+        }
+
+        [Test]
+        public void ExecuteAsync_WhenElementsOnlyHasNoAnnotationMode_ShouldThrowValidationException()
+        {
+            // Tests that ElementsOnly without AnnotateElements or AnnotateRaycastGrid fails validation.
+            JObject parameters = new JObject
+            {
+                ["CaptureMode"] = "rendering",
+                ["ElementsOnly"] = true
+            };
+
+            UnityCliLoopToolParameterValidationException? exception =
+                Assert.ThrowsAsync<UnityCliLoopToolParameterValidationException>(
+                    async () => await ExecuteScreenshot(parameters));
+
+            Assert.That(
+                exception!.Message,
+                Does.Contain("ElementsOnly requires AnnotateElements=true or AnnotateRaycastGrid=true"));
+        }
+
+        [Test]
+        public async Task ExecuteAsync_WhenElementsOnlyUsesRaycastGrid_ShouldPassValidation()
+        {
+            // Tests that ElementsOnly combined with AnnotateRaycastGrid passes validation
+            // (PlayMode is unavailable in EditMode, so capture itself no-ops after validation).
+            JObject parameters = new JObject
+            {
+                ["CaptureMode"] = "rendering",
+                ["AnnotateRaycastGrid"] = true,
+                ["ElementsOnly"] = true
+            };
+
+            UnityCliLoopToolResponse response = await ExecuteScreenshot(parameters);
+
+            Assert.That(response, Is.InstanceOf<ScreenshotResponse>());
+        }
+
+        [Test]
+        public void ExecuteAsync_WhenRaycastLayerMaskContainsUnknownLayer_ShouldThrowValidationException()
+        {
+            // Tests that an unrecognized layer name in RaycastLayerMask fails validation with the layer name in the message.
+            JObject parameters = new JObject
+            {
+                ["CaptureMode"] = "rendering",
+                ["AnnotateRaycastGrid"] = true,
+                ["RaycastLayerMask"] = "MissingLayerForTest"
+            };
+
+            UnityCliLoopToolParameterValidationException? exception =
+                Assert.ThrowsAsync<UnityCliLoopToolParameterValidationException>(
+                    async () => await ExecuteScreenshot(parameters));
+
+            Assert.That(exception!.Message, Does.Contain("unknown layer name"));
+            Assert.That(exception!.Message, Does.Contain("MissingLayerForTest"));
+        }
+
+        private static async Task<UnityCliLoopToolResponse> ExecuteScreenshot(JObject parameters)
+        {
+            ScreenshotTool tool = new();
+            return await tool.ExecuteAsync(parameters, CancellationToken.None);
+        }
+    }
+}

Assets/Tests/Editor/UIElementAnnotatorTests.cs

@@ -78,6 +78,49 @@ public void GetContrastPartnerColor_WhenColorIsDark_ShouldReturnDarkOutlineForLi
             Assert.That(outlineColor, Is.EqualTo(new Color(0f, 0f, 0f, 0.95f)));
         }
 
+        /// <summary>
+        /// Verifies that converting a top-left outline segment to screen space flips its Y coordinates.
+        /// </summary>
+        [Test]
+        public void ConvertTopLeftOutlineSegmentToScreenSegment_ShouldFlipYForCanvasSpace()
+        {
+            RaycastOutlineSegment inputSegment = new(10f, 20f, 30f, 20f);
+
+            RaycastOutlineSegment screenSegment =
+                UIElementAnnotator.ConvertTopLeftOutlineSegmentToScreenSegment(inputSegment, 100f);
+
+            Assert.That(screenSegment.StartX, Is.EqualTo(10f));
+            Assert.That(screenSegment.StartY, Is.EqualTo(80f));
+            Assert.That(screenSegment.EndX, Is.EqualTo(30f));
+            Assert.That(screenSegment.EndY, Is.EqualTo(80f));
+        }
+
+        /// <summary>
+        /// Verifies that a horizontal outline segment's rect extends both endpoints by half the thickness.
+        /// </summary>
+        [Test]
+        public void CalculateOutlineSegmentRect_WhenHorizontalSegment_ShouldExtendBothEndpoints()
+        {
+            RaycastOutlineSegment segment = new(10f, 20f, 30f, 20f);
+
+            Rect rect = UIElementAnnotator.CalculateOutlineSegmentRect(segment, 4f);
+
+            Assert.That(rect, Is.EqualTo(new Rect(8f, 18f, 24f, 4f)));
+        }
+
+        /// <summary>
+        /// Verifies that a vertical outline segment's rect extends both endpoints by half the thickness.
+        /// </summary>
+        [Test]
+        public void CalculateOutlineSegmentRect_WhenVerticalSegment_ShouldExtendBothEndpoints()
+        {
+            RaycastOutlineSegment segment = new(10f, 20f, 10f, 50f);
+
+            Rect rect = UIElementAnnotator.CalculateOutlineSegmentRect(segment, 4f);
+
+            Assert.That(rect, Is.EqualTo(new Rect(8f, 18f, 4f, 34f)));
+        }
+
         [Test]
         public void CalculateBorderEdgeRects_WhenBoundsAreProvided_ShouldPlaceEdgesInsideTheBounds()
         {

Packages/src/Editor/FirstPartyTools/Screenshot/EditorWindowCaptureService.cs

@@ -37,15 +37,18 @@ public string[] GetOpenWindowNames()
             return EditorWindowCaptureUtility.GetOpenWindowNames();
         }
 
-        public Task<(Texture2D? texture, int yOffset, bool timedOut)> CaptureGameRenderingAsync(
+        public async Task<(Texture2D? texture, int yOffset, bool timedOut)> CaptureGameRenderingAsync(
             float resolutionScale,
             int timeoutMilliseconds,
             CancellationToken ct)
         {
-            return EditorWindowCaptureUtility.CaptureGameRenderingAsync(
-                resolutionScale,
-                timeoutMilliseconds,
-                ct);
+            (Texture2D? texture, GameRenderingImageInfo renderingImageInfo, bool timedOut) =
+                await EditorWindowCaptureUtility.CaptureGameRenderingAsync(
+                    resolutionScale,
+                    null,
+                    timeoutMilliseconds,
+                    ct).ConfigureAwait(false);
+            return (texture, renderingImageInfo.ImageToInputOffsetY, timedOut);
         }
     }