From 089c089e72363861aadda6f40cb3e7f6d6920d7d Mon Sep 17 00:00:00 2001 From: hatayama Date: Sat, 4 Jul 2026 00:33:29 +0900 Subject: [PATCH 1/4] Clarify Game View mouse coordinates Use Game View coordinates consistently for mouse input so rendering screenshot points can be passed directly at the default scale, and expose the converted Unity input position in tool responses. - add a raycast tool and raycast-grid screenshot annotations for 3D click checks - report screenshot/input coordinate systems and formulas in CLI docs and skills - cover coordinate conversion, raycast, and mouse input metadata with tests --- .../Editor/GameViewCoordinateUtilityTests.cs | 46 +++++++ .../GameViewCoordinateUtilityTests.cs.meta | 11 ++ .../Tests/Editor/RaycastGridAnnotatorTests.cs | 44 +++++++ .../Editor/RaycastGridAnnotatorTests.cs.meta | 11 ++ Assets/Tests/Editor/RaycastToolTests.cs | 121 ++++++++++++++++++ Assets/Tests/Editor/RaycastToolTests.cs.meta | 11 ++ .../Tests/PlayMode/SimulateMouseInputTests.cs | 26 ++++ Packages/src/Cli~/src/default-tools.json | 48 ++++++- Packages/src/Editor/Api/McpTools/Raycast.meta | 8 ++ .../Api/McpTools/Raycast/RaycastResponse.cs | 35 +++++ .../McpTools/Raycast/RaycastResponse.cs.meta | 11 ++ .../Api/McpTools/Raycast/RaycastSchema.cs | 20 +++ .../McpTools/Raycast/RaycastSchema.cs.meta | 11 ++ .../Api/McpTools/Raycast/RaycastTool.cs | 88 +++++++++++++ .../Api/McpTools/Raycast/RaycastTool.cs.meta | 11 ++ .../Editor/Api/McpTools/Raycast/Skill.meta | 8 ++ .../Api/McpTools/Raycast/Skill/SKILL.md | 59 +++++++++ .../Api/McpTools/Raycast/Skill/SKILL.md.meta | 7 + .../Screenshot/RaycastGridPointInfo.cs | 17 +++ .../Screenshot/RaycastGridPointInfo.cs.meta | 11 ++ .../McpTools/Screenshot/ScreenshotResponse.cs | 25 ++-- .../McpTools/Screenshot/ScreenshotSchema.cs | 3 + .../Api/McpTools/Screenshot/ScreenshotTool.cs | 57 +++++++-- .../Api/McpTools/Screenshot/Skill/SKILL.md | 20 ++- .../Skill/references/annotated-elements.md | 18 ++- .../SimulateMouseInputResponse.cs | 9 ++ .../SimulateMouseInputSchema.cs | 4 +- .../SimulateMouseInputTool.cs | 40 ++++-- .../SimulateMouseInput/Skill/SKILL.md | 34 +++-- .../SimulateMouseUi/SimulateMouseUiSchema.cs | 8 +- .../McpTools/SimulateMouseUi/Skill/SKILL.md | 10 +- .../src/Editor/Shared/Config/McpConstants.cs | 12 ++ .../Utils/EditorWindowCaptureUtility.cs | 12 +- .../Editor/Utils/GameViewCoordinateUtility.cs | 53 ++++++++ .../Utils/GameViewCoordinateUtility.cs.meta | 11 ++ .../Editor/Utils/GameViewRaycastUtility.cs | 58 +++++++++ .../Utils/GameViewRaycastUtility.cs.meta | 11 ++ .../src/Editor/Utils/RaycastGridAnnotator.cs | 101 +++++++++++++++ .../Editor/Utils/RaycastGridAnnotator.cs.meta | 11 ++ Packages/src/README.md | 18 ++- Packages/src/TOOL_REFERENCE.md | 37 +++++- README.md | 18 ++- 42 files changed, 1082 insertions(+), 92 deletions(-) create mode 100644 Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs create mode 100644 Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta create mode 100644 Assets/Tests/Editor/RaycastGridAnnotatorTests.cs create mode 100644 Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta create mode 100644 Assets/Tests/Editor/RaycastToolTests.cs create mode 100644 Assets/Tests/Editor/RaycastToolTests.cs.meta create mode 100644 Packages/src/Editor/Api/McpTools/Raycast.meta create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/Skill.meta create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md create mode 100644 Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta create mode 100644 Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs create mode 100644 Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta create mode 100644 Packages/src/Editor/Utils/GameViewCoordinateUtility.cs create mode 100644 Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta create mode 100644 Packages/src/Editor/Utils/GameViewRaycastUtility.cs create mode 100644 Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta create mode 100644 Packages/src/Editor/Utils/RaycastGridAnnotator.cs create mode 100644 Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta diff --git a/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs new file mode 100644 index 0000000000..63e300cc32 --- /dev/null +++ b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs @@ -0,0 +1,46 @@ +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class GameViewCoordinateUtilityTests + { + [Test] + public void ConvertInputToUnity_WhenInputIsNearTop_ShouldFlipYWithinGameView() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + Vector2 inputPosition = new Vector2(0f, 100f); + + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.That(conversion.InputPosition, Is.EqualTo(inputPosition)); + Assert.That(conversion.InjectedUnityPosition, Is.EqualTo(new Vector2(0f, 980f))); + Assert.That(conversion.GameViewSize, Is.EqualTo(gameViewSize)); + } + + [Test] + public void ConvertInputToUnity_WhenInputIsAtCenter_ShouldKeepCenterY() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + Vector2 inputPosition = new Vector2(960f, 540f); + + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.That(conversion.InjectedUnityPosition, Is.EqualTo(new Vector2(960f, 540f))); + } + + [Test] + public void ConvertInputToUnity_WhenInputIsAtBottomRight_ShouldMapToBottomLeftOrigin() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + Vector2 inputPosition = new Vector2(1920f, 1080f); + + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.That(conversion.InjectedUnityPosition, Is.EqualTo(new Vector2(1920f, 0f))); + } + } +} diff --git a/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta new file mode 100644 index 0000000000..ef7653e66a --- /dev/null +++ b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9f518426086f04f21809705562fc5fce +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs new file mode 100644 index 0000000000..86a8f0e2de --- /dev/null +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using NUnit.Framework; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class RaycastGridAnnotatorTests + { + [Test] + public void CreateOverlayElements_WhenPointsIncludeMisses_ShouldAnnotateOnlyHits() + { + List points = new List + { + new RaycastGridPointInfo + { + Label = "R1", + Hit = true, + InputX = 100f, + InputY = 200f, + InjectedUnityPositionX = 100f, + InjectedUnityPositionY = 880f, + HitGameObjectName = "Cube", + HitGameObjectPath = "Cube" + }, + new RaycastGridPointInfo + { + Label = "R2", + Hit = false, + InputX = 200f, + InputY = 300f, + InjectedUnityPositionX = 200f, + InjectedUnityPositionY = 780f + } + }; + + List overlayElements = RaycastGridAnnotator.CreateOverlayElements(points); + + Assert.That(overlayElements.Count, Is.EqualTo(1)); + Assert.That(overlayElements[0].Label, Is.EqualTo("R1")); + Assert.That(overlayElements[0].Name, Is.EqualTo("Cube")); + Assert.That(overlayElements[0].Type, Is.EqualTo("RaycastHit")); + Assert.That(overlayElements[0].Interaction, Is.EqualTo("Raycast")); + } + } +} diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta new file mode 100644 index 0000000000..925444cce6 --- /dev/null +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 011b5cfbe4d944972ae894d38207e984 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/RaycastToolTests.cs b/Assets/Tests/Editor/RaycastToolTests.cs new file mode 100644 index 0000000000..f3b0c5bc26 --- /dev/null +++ b/Assets/Tests/Editor/RaycastToolTests.cs @@ -0,0 +1,121 @@ +#nullable enable +using System.Collections.Generic; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class RaycastToolTests + { + private GameObject? _cameraObject; + private GameObject? _cubeObject; + private readonly List _retaggedMainCameraObjects = new List(); + + [TearDown] + public void TearDown() + { + if (_cubeObject != null) + { + Object.DestroyImmediate(_cubeObject); + } + + if (_cameraObject != null) + { + Object.DestroyImmediate(_cameraObject); + } + + foreach (GameObject retaggedObject in _retaggedMainCameraObjects) + { + if (retaggedObject != null) + { + retaggedObject.tag = "MainCamera"; + } + } + + _retaggedMainCameraObjects.Clear(); + _cubeObject = null; + _cameraObject = null; + } + + [Test] + public async Task ExecuteAsync_WhenCoordinateIntersectsCollider_ShouldReturnHitAndConversionMetadata() + { + CreateRaycastScene(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.True); + Assert.That(response.HitGameObjectName, Is.EqualTo("RaycastToolTestsCube")); + Assert.That(response.InputCoordinateSystem, Is.EqualTo(McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW)); + Assert.That(response.UnityCoordinateSystem, Is.EqualTo(McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW)); + Assert.That(response.CoordinateConversionFormula, Is.EqualTo(McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY)); + Assert.That(response.InputPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InputPositionY, Is.EqualTo(inputPosition.y)); + Assert.That(response.InjectedUnityPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InjectedUnityPositionY, Is.EqualTo(gameViewSize.y - inputPosition.y)); + } + + [Test] + public async Task ExecuteAsync_WhenCoordinateMissesCollider_ShouldReturnNoHit() + { + CreateRaycastScene(); + Vector2 inputPosition = new Vector2(0f, 0f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.False); + Assert.That(response.HitGameObjectName, Is.Null); + } + + private void CreateRaycastScene() + { + RetagExistingMainCameras(); + + _cameraObject = new GameObject("RaycastToolTestsCamera"); + Camera camera = _cameraObject.AddComponent(); + _cameraObject.tag = "MainCamera"; + _cameraObject.transform.position = new Vector3(0f, 0f, -10f); + _cameraObject.transform.rotation = Quaternion.identity; + camera.nearClipPlane = 0.1f; + camera.farClipPlane = 100f; + + _cubeObject = GameObject.CreatePrimitive(PrimitiveType.Cube); + _cubeObject.name = "RaycastToolTestsCube"; + _cubeObject.transform.position = Vector3.zero; + } + + private void RetagExistingMainCameras() + { + Camera[] cameras = Object.FindObjectsByType(FindObjectsSortMode.None); + foreach (Camera camera in cameras) + { + if (!camera.CompareTag("MainCamera")) + { + continue; + } + + _retaggedMainCameraObjects.Add(camera.gameObject); + camera.gameObject.tag = "Untagged"; + } + } + + private static async Task ExecuteRaycast(Vector2 inputPosition) + { + RaycastTool tool = new RaycastTool(); + JObject parameters = new JObject + { + ["x"] = inputPosition.x, + ["y"] = inputPosition.y + }; + + BaseToolResponse baseResponse = await tool.ExecuteAsync(parameters); + return (RaycastResponse)baseResponse; + } + } +} diff --git a/Assets/Tests/Editor/RaycastToolTests.cs.meta b/Assets/Tests/Editor/RaycastToolTests.cs.meta new file mode 100644 index 0000000000..5cc08851f2 --- /dev/null +++ b/Assets/Tests/Editor/RaycastToolTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4ac05679db3384836aee71b74cca6777 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/PlayMode/SimulateMouseInputTests.cs b/Assets/Tests/PlayMode/SimulateMouseInputTests.cs index 7a3e94310f..60dcd1ee5b 100644 --- a/Assets/Tests/PlayMode/SimulateMouseInputTests.cs +++ b/Assets/Tests/PlayMode/SimulateMouseInputTests.cs @@ -47,6 +47,7 @@ public IEnumerator Click_Should_SetWasPressedThisFrame() Assert.IsTrue(lastResponse.Success); Assert.AreEqual("Click", lastResponse.Action); Assert.AreEqual("Left", lastResponse.Button); + AssertCoordinateMetadata(400f, 300f); // After click completes, button should be released Assert.IsFalse(mouse.leftButton.isPressed, "Left button should be released after click"); } @@ -106,6 +107,7 @@ public IEnumerator LongPress_Should_HoldButtonForDuration() Assert.IsTrue(lastResponse.Success); Assert.AreEqual("LongPress", lastResponse.Action); + AssertCoordinateMetadata(400f, 300f); // After long press completes, button should be released Assert.IsFalse(mouse.leftButton.isPressed, "Button should be released after long press"); } @@ -194,6 +196,30 @@ private IEnumerator RunTool(JObject parameters) lastResponse = (SimulateMouseInputResponse)task.Result; } + private void AssertCoordinateMetadata(float inputX, float inputY) + { + Vector2 inputPosition = new Vector2(inputX, inputY); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + GameViewCoordinateConversion expected = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.AreEqual("top-left-game-view", lastResponse.InputCoordinateSystem); + Assert.AreEqual("bottom-left-game-view", lastResponse.UnityCoordinateSystem); + Assert.AreEqual("unity_x = input_x; unity_y = gameViewHeight - input_y", lastResponse.CoordinateConversionFormula); + Assert.IsTrue(lastResponse.GameViewWidth.HasValue); + Assert.IsTrue(lastResponse.GameViewHeight.HasValue); + Assert.IsTrue(lastResponse.InputPositionX.HasValue); + Assert.IsTrue(lastResponse.InputPositionY.HasValue); + Assert.IsTrue(lastResponse.InjectedUnityPositionX.HasValue); + Assert.IsTrue(lastResponse.InjectedUnityPositionY.HasValue); + Assert.AreEqual(expected.GameViewSize.x, lastResponse.GameViewWidth!.Value); + Assert.AreEqual(expected.GameViewSize.y, lastResponse.GameViewHeight!.Value); + Assert.AreEqual(expected.InputPosition.x, lastResponse.InputPositionX!.Value); + Assert.AreEqual(expected.InputPosition.y, lastResponse.InputPositionY!.Value); + Assert.AreEqual(expected.InjectedUnityPosition.x, lastResponse.InjectedUnityPositionX!.Value); + Assert.AreEqual(expected.InjectedUnityPosition.y, lastResponse.InjectedUnityPositionY!.Value); + } + #endregion } } diff --git a/Packages/src/Cli~/src/default-tools.json b/Packages/src/Cli~/src/default-tools.json index 733efbee9b..11e0f0cc36 100644 --- a/Packages/src/Cli~/src/default-tools.json +++ b/Packages/src/Cli~/src/default-tools.json @@ -247,7 +247,7 @@ }, "CaptureMode": { "type": "string", - "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Response includes CoordinateSystem ('gameView' or 'window'), ResolutionScale, and YOffset. For gameView: sim_x = image_x / ResolutionScale, sim_y = image_y / ResolutionScale + YOffset.", + "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Rendering screenshots at ResolutionScale=1.0 use top-left Game View coordinates that can be passed directly to simulate-mouse-input and raycast.", "enum": [ "window", "rendering" @@ -263,6 +263,40 @@ "type": "boolean", "description": "Return only annotated element JSON without capturing a screenshot image. Requires AnnotateElements=true and CaptureMode=rendering in PlayMode.", "default": false + }, + "AnnotateRaycastGrid": { + "type": "boolean", + "description": "Annotate 3D physics raycast candidate points on rendering screenshots. Uses Camera.main and the same top-left Game View coordinates as simulate-mouse-input.", + "default": false + } + } + } + }, + { + "name": "raycast", + "description": "Raycast from Camera.main through a Game View coordinate. X/Y use the same top-left coordinates as screenshot --capture-mode rendering and simulate-mouse-input.", + "inputSchema": { + "type": "object", + "properties": { + "X": { + "type": "number", + "description": "Target X position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", + "default": 0 + }, + "Y": { + "type": "number", + "description": "Target Y position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", + "default": 0 + }, + "LayerMask": { + "type": "number", + "description": "Physics layer mask used by the raycast.", + "default": -5 + }, + "MaxDistance": { + "type": "number", + "description": "Maximum raycast distance in world units.", + "default": 1000 } } } @@ -348,22 +382,22 @@ }, "X": { "type": "number", - "description": "Target X position in screen pixels (origin: top-left). For Drag action, this is the destination.", + "description": "Target X position in Game View pixels (origin: top-left). For Drag action, this is the destination.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination.", + "description": "Target Y position in Game View pixels (origin: top-left). For Drag action, this is the destination.", "default": 0 }, "FromX": { "type": "number", - "description": "Start X position for Drag action (origin: top-left). Drag starts here and moves to X,Y.", + "description": "Start X position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.", "default": 0 }, "FromY": { "type": "number", - "description": "Start Y position for Drag action (origin: top-left). Drag starts here and moves to X,Y.", + "description": "Start Y position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.", "default": 0 }, "DragSpeed": { @@ -424,12 +458,12 @@ }, "X": { "type": "number", - "description": "Target X position in screen pixels (origin: top-left). Used by Click and LongPress.", + "description": "Target X position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in screen pixels (origin: top-left). Used by Click and LongPress.", + "description": "Target Y position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", "default": 0 }, "Button": { diff --git a/Packages/src/Editor/Api/McpTools/Raycast.meta b/Packages/src/Editor/Api/McpTools/Raycast.meta new file mode 100644 index 0000000000..3e9374980b --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: d7f308f6972f3483793d282829a51021 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs new file mode 100644 index 0000000000..a0ebc42ab2 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs @@ -0,0 +1,35 @@ +#nullable enable + +namespace io.github.hatayama.uLoopMCP +{ + public class RaycastResponse : BaseToolResponse + { + public bool Success { get; set; } + public string Message { get; set; } = ""; + public bool Hit { get; set; } + public string? HitGameObjectName { get; set; } + public string? HitGameObjectPath { get; set; } + public int? HitLayer { get; set; } + public string? HitLayerName { get; set; } + public float? Distance { get; set; } + public float? HitPointX { get; set; } + public float? HitPointY { get; set; } + public float? HitPointZ { get; set; } + public float? HitNormalX { get; set; } + public float? HitNormalY { get; set; } + public float? HitNormalZ { get; set; } + public string InputCoordinateSystem { get; set; } = ""; + public string UnityCoordinateSystem { get; set; } = ""; + public float GameViewWidth { get; set; } + public float GameViewHeight { get; set; } + public float InputPositionX { get; set; } + public float InputPositionY { get; set; } + public float InjectedUnityPositionX { get; set; } + public float InjectedUnityPositionY { get; set; } + public string CoordinateConversionFormula { get; set; } = ""; + + public RaycastResponse() + { + } + } +} diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta new file mode 100644 index 0000000000..83306e8c7f --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 7a2c44181637c4be8a1ca98f2d0891a4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs new file mode 100644 index 0000000000..25d7bcf0dd --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs @@ -0,0 +1,20 @@ +using System.ComponentModel; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + public class RaycastSchema : BaseToolSchema + { + [Description("Target X position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.")] + public float X { get; set; } = 0f; + + [Description("Target Y position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.")] + public float Y { get; set; } = 0f; + + [Description("Physics layer mask used by the raycast.")] + public int LayerMask { get; set; } = Physics.DefaultRaycastLayers; + + [Description("Maximum raycast distance in world units.")] + public float MaxDistance { get; set; } = McpConstants.RAYCAST_DEFAULT_MAX_DISTANCE; + } +} diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta new file mode 100644 index 0000000000..639c71a2d0 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 19cfc535ad76043ba912a44b081c9f5f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs new file mode 100644 index 0000000000..ef6eff7a04 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs @@ -0,0 +1,88 @@ +#nullable enable +using System.Threading; +using System.Threading.Tasks; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Checks what a screenshot-compatible Game View coordinate hits in 3D physics. + /// + [McpTool(Description = "Raycast from Camera.main through a Game View coordinate. X/Y use the same top-left coordinates as screenshot --capture-mode rendering and simulate-mouse-input.")] + public class RaycastTool : AbstractUnityTool + { + public override string ToolName => "raycast"; + + protected override Task ExecuteAsync(RaycastSchema parameters, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + if (parameters.MaxDistance <= 0f || float.IsNaN(parameters.MaxDistance) || float.IsInfinity(parameters.MaxDistance)) + { + return Task.FromResult(new RaycastResponse + { + Success = false, + Message = $"MaxDistance must be positive and finite, got: {parameters.MaxDistance}" + }); + } + + Vector2 inputPosition = new Vector2(parameters.X, parameters.Y); + GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( + inputPosition, + parameters.MaxDistance, + parameters.LayerMask); + + if (!raycastResult.CameraFound) + { + return Task.FromResult(new RaycastResponse + { + Success = false, + Message = "Camera.main was not found. Add an active camera tagged MainCamera before using raycast." + }); + } + + if (raycastResult.Hits.Length == 0) + { + RaycastResponse noHitResponse = CreateBaseResponse(raycastResult.Conversion); + noHitResponse.Success = true; + noHitResponse.Hit = false; + noHitResponse.Message = $"No physics hit at ({inputPosition.x:F1}, {inputPosition.y:F1})."; + return Task.FromResult(noHitResponse); + } + + RaycastHit nearestHit = raycastResult.Hits[0]; + RaycastResponse response = CreateBaseResponse(raycastResult.Conversion); + response.Success = true; + response.Hit = true; + response.Message = $"Hit {nearestHit.collider.gameObject.name} at ({inputPosition.x:F1}, {inputPosition.y:F1})."; + response.HitGameObjectName = nearestHit.collider.gameObject.name; + response.HitGameObjectPath = GameObjectPathUtility.GetFullPath(nearestHit.collider.gameObject); + response.HitLayer = nearestHit.collider.gameObject.layer; + response.HitLayerName = LayerMask.LayerToName(nearestHit.collider.gameObject.layer); + response.Distance = nearestHit.distance; + response.HitPointX = nearestHit.point.x; + response.HitPointY = nearestHit.point.y; + response.HitPointZ = nearestHit.point.z; + response.HitNormalX = nearestHit.normal.x; + response.HitNormalY = nearestHit.normal.y; + response.HitNormalZ = nearestHit.normal.z; + return Task.FromResult(response); + } + + private static RaycastResponse CreateBaseResponse(GameViewCoordinateConversion conversion) + { + return new RaycastResponse + { + InputCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, + UnityCoordinateSystem = McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW, + GameViewWidth = conversion.GameViewSize.x, + GameViewHeight = conversion.GameViewSize.y, + InputPositionX = conversion.InputPosition.x, + InputPositionY = conversion.InputPosition.y, + InjectedUnityPositionX = conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = conversion.InjectedUnityPosition.y, + CoordinateConversionFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY + }; + } + } +} diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta new file mode 100644 index 0000000000..fc957a6f53 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 74b2bf9d5a00843eda24add2c109851a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill.meta b/Packages/src/Editor/Api/McpTools/Raycast/Skill.meta new file mode 100644 index 0000000000..bb72109c41 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 8928b15eb1b67433d8bc4befaaecb9f2 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md new file mode 100644 index 0000000000..52637111ac --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md @@ -0,0 +1,59 @@ +--- +name: uloop-raycast +description: "Raycast from Camera.main through a Game View coordinate. Use when you need to check what a screenshot coordinate would hit in 3D physics before clicking or long-pressing with simulate-mouse-input." +--- + +# uloop raycast + +Check what a top-left Game View coordinate hits in 3D physics. + +## Usage + +```bash +uloop raycast --x --y [--layer-mask ] [--max-distance ] +``` + +## Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Use coordinates directly from `screenshot --capture-mode rendering`. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Use coordinates directly from `screenshot --capture-mode rendering`. | +| `--layer-mask` | number | Unity default raycast layers | Physics layer mask used by the raycast. | +| `--max-distance` | number | `1000` | Maximum raycast distance in world units. | + +## Coordinate System + +- `--x` / `--y` use the same top-left Game View coordinates as `screenshot --capture-mode rendering` and `simulate-mouse-input`. +- Do not flip Y in the caller. The tool converts internally: + +```text +unity_x = input_x +unity_y = gameViewHeight - input_y +``` + +## Examples + +```bash +# Check what is under a screenshot coordinate +uloop raycast --x 960 --y 540 + +# Check only specific layers +uloop raycast --x 960 --y 540 --layer-mask 1 +``` + +## Output + +Returns JSON with: +- `Success`: Whether the command completed +- `Message`: Status message +- `Hit`: Whether physics hit anything +- `HitGameObjectName` / `HitGameObjectPath`: Hit object identity when `Hit` is true +- `HitLayer` / `HitLayerName`: Hit object layer when `Hit` is true +- `Distance`, `HitPointX/Y/Z`, `HitNormalX/Y/Z`: Hit details when `Hit` is true +- `InputCoordinateSystem`, `UnityCoordinateSystem`, `GameViewWidth/Height`, `InputPositionX/Y`, `InjectedUnityPositionX/Y`, `CoordinateConversionFormula`: Coordinate conversion details + +## Notes + +- Requires an active `Camera.main`. +- Uses Unity Physics raycasts, not UI EventSystem raycasts. diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta new file mode 100644 index 0000000000..27cb3aa4e0 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 481bce4e5b2b344f7a0dde4afb6885d6 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs new file mode 100644 index 0000000000..f7d73ff09e --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace io.github.hatayama.uLoopMCP +{ + public class RaycastGridPointInfo + { + public string Label { get; set; } = ""; + public bool Hit { get; set; } + public float InputX { get; set; } + public float InputY { get; set; } + public float InjectedUnityPositionX { get; set; } + public float InjectedUnityPositionY { get; set; } + public string? HitGameObjectName { get; set; } + public string? HitGameObjectPath { get; set; } + public float? Distance { get; set; } + } +} diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta new file mode 100644 index 0000000000..280a6630ec --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5ebf16218669642cb9cf689cd2f68430 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs index efb709ea44..c7869c245d 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs @@ -10,32 +10,25 @@ public class ScreenshotInfo public long FileSizeBytes { get; set; } public int Width { get; set; } public int Height { get; set; } - - // "gameView": image from game RenderTexture. Convert to simulate-mouse coords: - // sim_x = image_x / ResolutionScale - // sim_y = image_y / ResolutionScale + YOffset - // "window": EditorWindow capture including toolbar - public string CoordinateSystem { get; set; } = McpConstants.COORDINATE_SYSTEM_WINDOW; - + public string ImageCoordinateSystem { get; set; } = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; public float ResolutionScale { get; set; } = 1.0f; - - // Y offset to add to image pixel Y to get simulate-mouse Y coordinate. - // Only meaningful when CoordinateSystem == "gameView". - public int YOffset { get; set; } - + public float GameViewWidth { get; set; } + public float GameViewHeight { get; set; } + public string ScreenshotToInputFormula { get; set; } = McpConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; + public string UnityInputFormula { get; set; } = ""; public List AnnotatedElements { get; set; } = new(); + public List RaycastGridPoints { get; set; } = new(); public ScreenshotInfo(string imagePath, long fileSizeBytes, int width, int height, - string coordinateSystem = McpConstants.COORDINATE_SYSTEM_WINDOW, - float resolutionScale = 1.0f, int yOffset = 0) + string imageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW, + float resolutionScale = 1.0f) { ImagePath = imagePath; FileSizeBytes = fileSizeBytes; Width = width; Height = height; - CoordinateSystem = coordinateSystem; + ImageCoordinateSystem = imageCoordinateSystem; ResolutionScale = resolutionScale; - YOffset = yOffset; } public ScreenshotInfo() diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs index a8f9deeceb..b6496a73a8 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs @@ -51,5 +51,8 @@ public class ScreenshotSchema : BaseToolSchema [Description("Return only annotated element JSON without capturing a screenshot image. Requires AnnotateElements=true and CaptureMode=rendering in PlayMode.")] public bool ElementsOnly { get; set; } = false; + + [Description("Annotate 3D physics raycast candidate points on rendering screenshots. Uses Camera.main and the same top-left Game View coordinates as simulate-mouse-input.")] + public bool AnnotateRaycastGrid { get; set; } = false; } } diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs index b6969b3b05..a9ed5ffff9 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs @@ -56,6 +56,9 @@ private async Task CaptureRenderingAsync( } List annotatedElements = new List(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + List raycastGridPoints = new List(); + List raycastGridOverlayElements = new List(); if (parameters.AnnotateElements) { @@ -63,31 +66,40 @@ private async Task CaptureRenderingAsync( UIElementAnnotator.AssignLabels(annotatedElements); } + if (parameters.AnnotateRaycastGrid) + { + raycastGridPoints = RaycastGridAnnotator.CollectRaycastGridPoints(gameViewSize); + raycastGridOverlayElements = RaycastGridAnnotator.CreateOverlayElements(raycastGridPoints); + } + if (parameters.ElementsOnly) { - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y); + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); ScreenshotInfo elementsOnlyInfo = new ScreenshotInfo(); - elementsOnlyInfo.CoordinateSystem = McpConstants.COORDINATE_SYSTEM_GAME_VIEW; + elementsOnlyInfo.ResolutionScale = parameters.ResolutionScale; + ApplyRenderingCoordinateMetadata(elementsOnlyInfo, gameViewSize); elementsOnlyInfo.AnnotatedElements = annotatedElements; + elementsOnlyInfo.RaycastGridPoints = raycastGridPoints; return new ScreenshotResponse(new List { elementsOnlyInfo }); } GameObject annotationOverlay = null; Texture2D texture; - int yOffset; try { - if (parameters.AnnotateElements) + if (parameters.AnnotateElements || parameters.AnnotateRaycastGrid) { + List overlayElements = new List(annotatedElements); + overlayElements.AddRange(raycastGridOverlayElements); annotationOverlay = UIElementAnnotator.CreateAnnotationOverlay( - annotatedElements, + overlayElements, parameters.ResolutionScale); Canvas.ForceUpdateCanvases(); // Chained CLI calls can read the previous GameView RT before overlay rendering catches up. await EditorDelay.DelayFrame(ANNOTATION_OVERLAY_RENDER_WAIT_FRAMES, ct); } - (texture, yOffset) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( + texture = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( parameters.ResolutionScale, ct); } finally @@ -95,7 +107,7 @@ private async Task CaptureRenderingAsync( UIElementAnnotator.DestroyAnnotationOverlay(annotationOverlay); } - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y); + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); if (texture == null) { @@ -122,8 +134,10 @@ private async Task CaptureRenderingAsync( FileInfo savedFileInfo = new FileInfo(savedPath); ScreenshotInfo info = new ScreenshotInfo( savedPath, savedFileInfo.Length, width, height, - McpConstants.COORDINATE_SYSTEM_GAME_VIEW, parameters.ResolutionScale, yOffset); + McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, parameters.ResolutionScale); + ApplyRenderingCoordinateMetadata(info, gameViewSize); info.AnnotatedElements = annotatedElements; + info.RaycastGridPoints = raycastGridPoints; screenshots.Add(info); } catch (Exception ex) @@ -199,7 +213,9 @@ private async Task CaptureWindowsAsync( SaveTextureAsPng(texture, savedPath); FileInfo savedFileInfo = new FileInfo(savedPath); - screenshots.Add(new ScreenshotInfo(savedPath, savedFileInfo.Length, width, height)); + ScreenshotInfo info = new ScreenshotInfo(savedPath, savedFileInfo.Length, width, height); + ApplyWindowCoordinateMetadata(info); + screenshots.Add(info); } catch (Exception ex) { @@ -226,6 +242,24 @@ private async Task CaptureWindowsAsync( return new ScreenshotResponse(screenshots); } + private static void ApplyRenderingCoordinateMetadata(ScreenshotInfo info, Vector2 gameViewSize) + { + info.ImageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW; + info.GameViewWidth = gameViewSize.x; + info.GameViewHeight = gameViewSize.y; + info.ScreenshotToInputFormula = Mathf.Approximately(info.ResolutionScale, 1.0f) + ? McpConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA + : McpConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA_SCALED; + info.UnityInputFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY; + } + + private static void ApplyWindowCoordinateMetadata(ScreenshotInfo info) + { + info.ImageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; + info.ScreenshotToInputFormula = McpConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; + info.UnityInputFormula = ""; + } + private void ValidateParameters(ScreenshotSchema parameters) { if (parameters.CaptureMode != CaptureMode.rendering && @@ -252,6 +286,11 @@ private void ValidateParameters(ScreenshotSchema parameters) { throw new ParameterValidationException("ElementsOnly is only supported when CaptureMode=rendering"); } + + if (parameters.AnnotateRaycastGrid) + { + throw new ParameterValidationException("AnnotateRaycastGrid is only supported when CaptureMode=rendering"); + } } if (parameters.ElementsOnly && !parameters.AnnotateElements) diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md index b5fa55378c..9ded5ab8b1 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md @@ -10,7 +10,7 @@ Take a screenshot of any Unity EditorWindow by name and save as PNG. ## Usage ```bash -uloop screenshot [--window-name ] [--resolution-scale ] [--match-mode ] [--capture-mode ] [--annotate-elements] [--elements-only] [--output-directory ] +uloop screenshot [--window-name ] [--resolution-scale ] [--match-mode ] [--capture-mode ] [--annotate-elements] [--annotate-raycast-grid] [--elements-only] [--output-directory ] ``` ## Parameters @@ -23,6 +23,7 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo | `--capture-mode` | enum | `window` | `window`=capture EditorWindow including toolbar, `rendering`=capture game rendering only (PlayMode required, coordinates match simulate-mouse) | | `--output-directory` | string | `""` | Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths. | | `--annotate-elements` | boolean | `false` | Annotate interactive UI elements with index labels and interaction hints (A / CLICK, B / DRAG, ...). Only works with `--capture-mode rendering` in PlayMode. | +| `--annotate-raycast-grid` | boolean | `false` | Annotate 3D physics raycast candidate points (R1, R2, ...). Only works with `--capture-mode rendering`; uses Camera.main. | | `--elements-only` | boolean | `false` | Return only annotated element JSON without capturing a screenshot image. Requires `--annotate-elements` and `--capture-mode rendering` in PlayMode. | ## Match Modes @@ -49,12 +50,15 @@ The window name is the text displayed in the window's title bar (tab). Common na # Take a screenshot of Game View (default) uloop screenshot -# Capture game rendering (coordinates match simulate-mouse, PlayMode required) +# Capture game rendering (default scale coordinates match simulate-mouse, PlayMode required) uloop screenshot --capture-mode rendering # Annotate interactive UI elements with index labels (for simulate-mouse workflow) uloop screenshot --capture-mode rendering --annotate-elements +# Annotate 3D physics raycast candidate points +uloop screenshot --capture-mode rendering --annotate-raycast-grid + # Get UI element coordinates without capturing an image (fastest) uloop screenshot --capture-mode rendering --annotate-elements --elements-only @@ -80,12 +84,15 @@ Returns JSON with: - `FileSizeBytes`: Size of the saved file in bytes - `Width`: Captured image width in pixels - `Height`: Captured image height in pixels - - `CoordinateSystem`: `"gameView"` or `"window"` + - `ImageCoordinateSystem`: `"top-left-game-view"` for `--capture-mode rendering`, or `"top-left-window"` for window captures - `ResolutionScale`: Resolution scale used for capture - - `YOffset`: Y offset used for gameView coordinate conversion + - `GameViewWidth` / `GameViewHeight`: Game View size used by mouse input tools + - `ScreenshotToInputFormula`: Formula for turning image coordinates into mouse-input coordinates + - `UnityInputFormula`: Formula used internally by mouse input tools to inject `Mouse.current.position` - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used. + - `RaycastGridPoints`: Array of 3D raycast candidate metadata. Empty unless `--annotate-raycast-grid` is used. -For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. +For `AnnotatedElements` fields and Game View coordinates, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. When multiple windows match (e.g., multiple Inspector windows or when using `contains` mode), all matching windows are captured with numbered filenames (e.g., `Inspector_1_*.png`, `Inspector_2_*.png`). @@ -94,3 +101,6 @@ When multiple windows match (e.g., multiple Inspector windows or when using `con - Use `uloop focus-window` first if needed - Target window must be open in Unity Editor - Window name matching is always case-insensitive +- Use `--capture-mode rendering` for coordinates that should be passed to `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`. +- Keep `--resolution-scale 1.0` when you want to pass image pixel coordinates directly to mouse tools. +- Do not use `window` captures as mouse-input coordinates because they include Unity Editor chrome. diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md index 283109ccc2..6c0b470ee8 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md @@ -11,27 +11,31 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. - `Type`: Element type (`Button`, `Toggle`, `Slider`, `Dropdown`, `InputField`, `Scrollbar`, `Draggable`, `DropTarget`, `Selectable`) - `Interaction`: Derived interaction category (`Click`, `Drag`, `Drop`, `Text`). Use this to choose between `simulate-mouse-ui --action Click` and drag actions. -- `SimX`, `SimY`: Center position in simulate-mouse coordinates. Use these directly with `--x` and `--y`. +- `SimX`, `SimY`: Center position in top-left Game View coordinates. Use these directly with `simulate-mouse-ui --x/--y`, `simulate-mouse-input --x/--y`, or `raycast --x/--y`. - `BoundsMinX`, `BoundsMinY`, `BoundsMaxX`, `BoundsMaxY`: Bounding box in simulate-mouse coordinates - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. ## Coordinate Conversion -When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates: +When `ImageCoordinateSystem` is `"top-left-game-view"` and `ResolutionScale` is `1.0`, image pixel coordinates from `screenshot --capture-mode rendering` are already mouse-input coordinates: ```text -sim_x = image_x / ResolutionScale -sim_y = image_y / ResolutionScale + YOffset +input_x = image_x +input_y = image_y ``` -When `ResolutionScale` is `1.0`, this simplifies to: +When `ResolutionScale` is not `1.0`, use the formula returned in `ScreenshotToInputFormula`. + +The mouse input tools convert internally to Unity Input System coordinates: ```text -sim_x = image_x -sim_y = image_y + YOffset +unity_x = input_x +unity_y = gameViewHeight - input_y ``` +Do not flip Y in the caller. + ## Annotation Readability Annotated screenshots compensate border thickness for `ResolutionScale`, so the saved PNG keeps the intended outline width after downscaling. The neutral contrast borders are 2 output pixels each, and the colored middle border is 4 output pixels. Label outlines are also compensated and are separated from element borders by a 4 output pixel gap. diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs index 86542a0303..2da50d3571 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs @@ -10,6 +10,15 @@ public class SimulateMouseInputResponse : BaseToolResponse public string? Button { get; set; } public float? PositionX { get; set; } public float? PositionY { get; set; } + public string InputCoordinateSystem { get; set; } = ""; + public string UnityCoordinateSystem { get; set; } = ""; + public float? GameViewWidth { get; set; } + public float? GameViewHeight { get; set; } + public float? InputPositionX { get; set; } + public float? InputPositionY { get; set; } + public float? InjectedUnityPositionX { get; set; } + public float? InjectedUnityPositionY { get; set; } + public string CoordinateConversionFormula { get; set; } = ""; public SimulateMouseInputResponse() { diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs index 71dc5bfcff..64a2ad7994 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs @@ -7,10 +7,10 @@ public class SimulateMouseInputSchema : BaseToolSchema [Description("Mouse input action: Click(0) - inject left/right/middle button press+release, LongPress(1) - inject button hold for Duration seconds, MoveDelta(2) - inject mouse delta for camera/look (one-shot), Scroll(3) - inject scroll wheel, SmoothDelta(4) - inject mouse delta smoothly over Duration seconds")] public MouseInputAction Action { get; set; } = MouseInputAction.Click; - [Description("Target X position in screen pixels (origin: top-left). Used by Click and LongPress to set Mouse.current.position.")] + [Description("Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from screenshot --capture-mode rendering.")] public float X { get; set; } = 0f; - [Description("Target Y position in screen pixels (origin: top-left). Used by Click and LongPress to set Mouse.current.position.")] + [Description("Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from screenshot --capture-mode rendering.")] public float Y { get; set; } = 0f; [Description("Mouse button: Left(0, default), Right(1), Middle(2). Used by Click and LongPress.")] diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs index c1b81d2080..c2b6339146 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs @@ -143,12 +143,12 @@ private static void EnsureOverlayExists() OverlayCanvasFactory.EnsureExists(); } - // Input coordinates use top-left origin; Unity Screen space uses bottom-left origin. - // Uses Screen.height (runtime resolution) because Mouse.current.position is in - // runtime screen space, not the editor Game view target resolution. - private static Vector2 InputToScreen(Vector2 inputPos) + // Public mouse-input coordinates match rendering screenshots. Unity input stays bottom-left, + // so conversion belongs inside the tool instead of in the caller's prompt or skill text. + private static GameViewCoordinateConversion ConvertInputToUnity(Vector2 inputPos) { - return new Vector2(inputPos.x, Screen.height - inputPos.y); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + return GameViewCoordinateUtility.ConvertInputToUnity(inputPos, gameViewSize); } private async Task ExecuteClick( @@ -165,13 +165,13 @@ private async Task ExecuteClick( } Vector2 inputPos = new Vector2(parameters.X, parameters.Y); - Vector2 screenPos = InputToScreen(inputPos); + GameViewCoordinateConversion conversion = ConvertInputToUnity(inputPos); MouseButton button = parameters.Button; string buttonName = button.ToString(); // Set mouse position before clicking await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( - () => MouseInputState.SetPositionState(mouse, screenPos), ct); + () => MouseInputState.SetPositionState(mouse, conversion.InjectedUnityPosition), ct); // Press button MouseInputState.SetButtonDown(button); @@ -207,7 +207,16 @@ await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( Action = MouseInputAction.Click.ToString(), Button = buttonName, PositionX = inputPos.x, - PositionY = inputPos.y + PositionY = inputPos.y, + InputCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, + UnityCoordinateSystem = McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW, + GameViewWidth = conversion.GameViewSize.x, + GameViewHeight = conversion.GameViewSize.y, + InputPositionX = conversion.InputPosition.x, + InputPositionY = conversion.InputPosition.y, + InjectedUnityPositionX = conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = conversion.InjectedUnityPosition.y, + CoordinateConversionFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY }; } @@ -225,13 +234,13 @@ private async Task ExecuteLongPress( } Vector2 inputPos = new Vector2(parameters.X, parameters.Y); - Vector2 screenPos = InputToScreen(inputPos); + GameViewCoordinateConversion conversion = ConvertInputToUnity(inputPos); MouseButton button = parameters.Button; string buttonName = button.ToString(); // Set mouse position before pressing await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( - () => MouseInputState.SetPositionState(mouse, screenPos), ct); + () => MouseInputState.SetPositionState(mouse, conversion.InjectedUnityPosition), ct); // Press button MouseInputState.SetButtonDown(button); @@ -265,7 +274,16 @@ await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( Action = MouseInputAction.LongPress.ToString(), Button = buttonName, PositionX = inputPos.x, - PositionY = inputPos.y + PositionY = inputPos.y, + InputCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, + UnityCoordinateSystem = McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW, + GameViewWidth = conversion.GameViewSize.x, + GameViewHeight = conversion.GameViewSize.y, + InputPositionX = conversion.InputPosition.x, + InputPositionY = conversion.InputPosition.y, + InjectedUnityPositionX = conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = conversion.InjectedUnityPosition.y, + CoordinateConversionFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY }; } diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md index b7ffbb20af..adb2ed2158 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md @@ -11,7 +11,7 @@ Simulate mouse input via Input System in Unity PlayMode: $ARGUMENTS ## Workflow 1. Ensure Unity is in PlayMode (use `uloop control-play-mode --action Play` if not) -2. For Click/LongPress: determine the target screen position (use `uloop screenshot` to find coordinates) +2. For Click/LongPress: determine the target Game View position with `uloop screenshot --capture-mode rendering` 3. Execute the appropriate `uloop simulate-mouse-input` command 4. Take a screenshot to verify the result: `uloop screenshot --capture-mode rendering` 5. Report what happened @@ -27,8 +27,8 @@ uloop simulate-mouse-input --action [options] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `--action` | enum | `Click` | `Click`, `LongPress`, `MoveDelta`, `SmoothDelta`, `Scroll` | -| `--x` | number | `0` | Target X position (origin: top-left). Used by Click and LongPress. | -| `--y` | number | `0` | Target Y position (origin: top-left). Used by Click and LongPress. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from `screenshot --capture-mode rendering`. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from `screenshot --capture-mode rendering`. | | `--button` | enum | `Left` | Mouse button: `Left`, `Right`, `Middle`. Used by Click and LongPress. | | `--duration` | number | `0` | Hold duration for LongPress, or interpolation duration for SmoothDelta (seconds). For Click, 0 = one-shot tap. | | `--delta-x` | number | `0` | Delta X in pixels for MoveDelta/SmoothDelta. Positive = right. | @@ -67,7 +67,7 @@ uloop simulate-mouse-input --action [options] ## Examples ```bash -# Left-click at screen center (for game logic) +# Left-click at the Game View center (for game logic) uloop simulate-mouse-input --action Click --x 400 --y 300 # Right-click at screen center (e.g. place block) @@ -89,6 +89,19 @@ uloop simulate-mouse-input --action Scroll --scroll-y -120 uloop simulate-mouse-input --action SmoothDelta --delta-x 300 --delta-y 0 --duration 0.5 ``` +## Coordinate System + +- `--x` / `--y` use **top-left Game View coordinates**. +- Coordinates read from `uloop screenshot --capture-mode rendering` can be passed directly to this tool. +- Do not flip Y in the caller. The tool converts internally for Unity Input System: + +```text +unity_x = input_x +unity_y = gameViewHeight - input_y +``` + +- `Mouse.current.position` uses bottom-left Unity coordinates, so the value read inside Unity may show the converted Y. + ## Prerequisites - Unity must be in **PlayMode** @@ -103,7 +116,12 @@ Returns JSON with: - `Message`: Status message - `Action`: Echoes which action was executed (`Click`, `LongPress`, `MoveDelta`, `SmoothDelta`, or `Scroll`) - `Button`: Which button was used (nullable string; populated for `Click` / `LongPress`, null otherwise) -- `PositionX`: Target X coordinate (nullable float; populated for `Click` / `LongPress`) -- `PositionY`: Target Y coordinate (nullable float; populated for `Click` / `LongPress`) - -These are the only six fields. There is no `DeltaX`, `DeltaY`, `ScrollX`, `ScrollY`, `Duration`, or hit-element field in the response — only the issued action, button, and target position are echoed back. Verify visual outcome with a follow-up screenshot. +- `PositionX` / `PositionY`: Target top-left Game View coordinates (nullable float; populated for `Click` / `LongPress`) +- `InputCoordinateSystem`: `"top-left-game-view"` for click/long-press coordinates +- `UnityCoordinateSystem`: `"bottom-left-game-view"` for the injected `Mouse.current.position` +- `GameViewWidth` / `GameViewHeight`: Game View size used for conversion +- `InputPositionX` / `InputPositionY`: Coordinates received from the caller +- `InjectedUnityPositionX` / `InjectedUnityPositionY`: Coordinates injected into `Mouse.current.position` +- `CoordinateConversionFormula`: Conversion formula used by the tool + +Verify visual outcome with a follow-up screenshot. diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs index f7784e35b8..02e2063e09 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs @@ -7,16 +7,16 @@ public class SimulateMouseUiSchema : BaseToolSchema [Description("Mouse action: Click(0) - click with selected Button, Drag(1) - one-shot drag, DragStart(2) - begin drag and hold, DragMove(3) - move while holding drag, DragEnd(4) - release drag, LongPress(5) - press and hold for Duration seconds")] public MouseAction Action { get; set; } = MouseAction.Click; - [Description("Target X position in screen pixels (origin: top-left). For Drag action, this is the destination.")] + [Description("Target X position in Game View pixels (origin: top-left). For Drag action, this is the destination.")] public float X { get; set; } = 0f; - [Description("Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination.")] + [Description("Target Y position in Game View pixels (origin: top-left). For Drag action, this is the destination.")] public float Y { get; set; } = 0f; - [Description("Start X position in screen pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] + [Description("Start X position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] public float FromX { get; set; } = 0f; - [Description("Start Y position in screen pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] + [Description("Start Y position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] public float FromY { get; set; } = 0f; [Description("Drag speed in pixels per second (0 for instant). Applies to Drag, DragMove, and DragEnd actions.")] diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md index 451d728303..1117f479ae 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md @@ -28,10 +28,10 @@ uloop simulate-mouse-ui --action --x --y [options] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `--action` | enum | `Click` | `Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, `LongPress` | -| `--x` | number | `0` | Target X position in screen pixels (origin: top-left). For Drag action, this is the destination. | -| `--y` | number | `0` | Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination. | -| `--from-x` | number | `0` | Start X position for Drag action. Drag starts here and moves to x,y. | -| `--from-y` | number | `0` | Start Y position for Drag action. Drag starts here and moves to x,y. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). For Drag action, this is the destination. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). For Drag action, this is the destination. | +| `--from-x` | number | `0` | Start X position in Game View pixels for Drag action. Drag starts here and moves to x,y. | +| `--from-y` | number | `0` | Start Y position in Game View pixels for Drag action. Drag starts here and moves to x,y. | | `--drag-speed` | number | `2000` | Drag speed in pixels per second (0 for instant). 2000 is fast (default), 200 is slow enough to watch. Applies to Drag, DragMove, and DragEnd actions. | | `--duration` | number | `0.5` | Hold duration in seconds for LongPress action. | | `--button` | enum | `Left` | Mouse button. `Click` and `LongPress` support `Left`, `Right`, and `Middle`. Drag actions support `Left` only; other buttons return an error. | @@ -67,7 +67,7 @@ uloop simulate-mouse-ui --action --x --y [options] ## Coordinate System - Origin is **top-left** (0, 0) -- All positions are in **screen pixels** +- All positions are in **top-left Game View pixels** - Get coordinates from `AnnotatedElements` JSON (`SimX`/`SimY`) — do NOT look up GameObject positions - Clicking or long-pressing on empty space (no UI element) still succeeds with a message indicating no element was hit - Dragging on empty space (no draggable UI element) returns `Success = false` diff --git a/Packages/src/Editor/Shared/Config/McpConstants.cs b/Packages/src/Editor/Shared/Config/McpConstants.cs index b3fd3173cc..fd48bbc538 100644 --- a/Packages/src/Editor/Shared/Config/McpConstants.cs +++ b/Packages/src/Editor/Shared/Config/McpConstants.cs @@ -168,6 +168,18 @@ public static UnityEditor.PackageManager.PackageInfo PackageInfo // Screenshot coordinate system values public const string COORDINATE_SYSTEM_GAME_VIEW = "gameView"; public const string COORDINATE_SYSTEM_WINDOW = "window"; + public const string COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW = "top-left-game-view"; + public const string COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW = "bottom-left-game-view"; + public const string COORDINATE_SYSTEM_TOP_LEFT_WINDOW = "top-left-window"; + public const string COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY = + "unity_x = input_x; unity_y = gameViewHeight - input_y"; + public const string SCREENSHOT_RENDERING_TO_INPUT_FORMULA = + "simulate_mouse_x = image_x; simulate_mouse_y = image_y"; + public const string SCREENSHOT_RENDERING_TO_INPUT_FORMULA_SCALED = + "simulate_mouse_x = image_x / resolutionScale; simulate_mouse_y = image_y / resolutionScale"; + public const string SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE = + "unavailable: window screenshots include Unity Editor chrome; use capture-mode rendering for mouse input coordinates"; + public const float RAYCAST_DEFAULT_MAX_DISTANCE = 1000f; // SimulateMouseUi constants public const float SIMULATE_MOUSE_UI_DEFAULT_DRAG_SPEED = 2000f; diff --git a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs index 999f5beade..ef48df1a49 100644 --- a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs @@ -138,10 +138,7 @@ public static string[] GetOpenWindowNames() // Captures game rendering by reading GameView's composited RenderTexture (PlayMode only). // Contains all cameras + Screen Space Overlay Canvas, without tab bar or borders. - // Coordinate mapping to simulate-mouse: - // sim_x = image_x / resolutionScale, sim_y = image_y / resolutionScale + YOffset - // where YOffset = Screen.height - RenderTexture.height (returned in the tuple). - public static async Task<(Texture2D? texture, int yOffset)> CaptureGameRenderingAsync(float resolutionScale, CancellationToken ct) + public static async Task CaptureGameRenderingAsync(float resolutionScale, CancellationToken ct) { Debug.Assert(UnityEditor.EditorApplication.isPlaying, "CaptureGameRenderingAsync requires PlayMode"); @@ -152,11 +149,9 @@ public static string[] GetOpenWindowNames() if (rt == null) { Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); - return (null, 0); + return null; } - int yOffset = (int)Handles.GetMainGameViewSize().y - rt.height; - // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format RenderTextureDescriptor flipDescriptor = new RenderTextureDescriptor(rt.width, rt.height, rt.format, 0); if (QualitySettings.activeColorSpace == ColorSpace.Linear) @@ -181,7 +176,7 @@ public static string[] GetOpenWindowNames() texture = ApplyResolutionScaling(texture, resolutionScale); } - return (texture, yOffset); + return texture; } private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float scale) @@ -206,4 +201,3 @@ private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float } } } - diff --git a/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs new file mode 100644 index 0000000000..29fe827c4a --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs @@ -0,0 +1,53 @@ +using UnityEditor; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Converts public top-left Game View coordinates into Unity's bottom-left input coordinates. + /// + internal static class GameViewCoordinateUtility + { + internal static GameViewCoordinateConversion ConvertInputToUnity( + Vector2 inputPosition, + Vector2 gameViewSize) + { + Debug.Assert(gameViewSize.x >= 0f, "Game View width must not be negative."); + Debug.Assert(gameViewSize.y >= 0f, "Game View height must not be negative."); + + Vector2 injectedUnityPosition = new Vector2( + inputPosition.x, + gameViewSize.y - inputPosition.y); + + return new GameViewCoordinateConversion( + inputPosition, + injectedUnityPosition, + gameViewSize); + } + + internal static Vector2 GetMainGameViewSize() + { + return Handles.GetMainGameViewSize(); + } + } + + /// + /// Describes one conversion from screenshot-compatible Game View input to Unity input space. + /// + internal readonly struct GameViewCoordinateConversion + { + public readonly Vector2 InputPosition; + public readonly Vector2 InjectedUnityPosition; + public readonly Vector2 GameViewSize; + + public GameViewCoordinateConversion( + Vector2 inputPosition, + Vector2 injectedUnityPosition, + Vector2 gameViewSize) + { + InputPosition = inputPosition; + InjectedUnityPosition = injectedUnityPosition; + GameViewSize = gameViewSize; + } + } +} diff --git a/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta new file mode 100644 index 0000000000..6d1c8013fb --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 357293374973742c99049bb1696cab7d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs new file mode 100644 index 0000000000..eb7e251623 --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs @@ -0,0 +1,58 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Runs physics raycasts from screenshot-compatible Game View coordinates. + /// + internal static class GameViewRaycastUtility + { + internal static GameViewRaycastResult RaycastFromInputPosition( + Vector2 inputPosition, + float maxDistance, + int layerMask) + { + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Camera mainCamera = Camera.main; + if (mainCamera == null) + { + return new GameViewRaycastResult(false, conversion, new RaycastHit[0]); + } + + Ray ray = mainCamera.ScreenPointToRay(conversion.InjectedUnityPosition); + RaycastHit[] hits = Physics.RaycastAll(ray, maxDistance, layerMask); + System.Array.Sort(hits, CompareHitsByDistance); + + return new GameViewRaycastResult(true, conversion, hits); + } + + private static int CompareHitsByDistance(RaycastHit left, RaycastHit right) + { + return left.distance.CompareTo(right.distance); + } + } + + /// + /// Contains the camera lookup result, coordinate conversion, and sorted physics hits. + /// + internal readonly struct GameViewRaycastResult + { + public readonly bool CameraFound; + public readonly GameViewCoordinateConversion Conversion; + public readonly RaycastHit[] Hits; + + public GameViewRaycastResult( + bool cameraFound, + GameViewCoordinateConversion conversion, + RaycastHit[] hits) + { + CameraFound = cameraFound; + Conversion = conversion; + Hits = hits; + } + } +} diff --git a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta new file mode 100644 index 0000000000..6a89bc821f --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 326e5d6ae2978480c94f23ad1edcdb66 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs new file mode 100644 index 0000000000..e2b8ed3ee8 --- /dev/null +++ b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs @@ -0,0 +1,101 @@ +#nullable enable +using System.Collections.Generic; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Creates 3D physics click candidates for rendering screenshots. + /// + internal static class RaycastGridAnnotator + { + private const int GRID_COLUMNS = 5; + private const int GRID_ROWS = 5; + private const float MARKER_SIZE = 18f; + + internal static List CollectRaycastGridPoints(Vector2 gameViewSize) + { + List points = new List(); + int labelIndex = 1; + + for (int row = 1; row <= GRID_ROWS; row++) + { + for (int column = 1; column <= GRID_COLUMNS; column++) + { + Vector2 inputPosition = new Vector2( + gameViewSize.x * column / (GRID_COLUMNS + 1f), + gameViewSize.y * row / (GRID_ROWS + 1f)); + GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( + inputPosition, + McpConstants.RAYCAST_DEFAULT_MAX_DISTANCE, + Physics.DefaultRaycastLayers); + + points.Add(CreatePointInfo($"R{labelIndex}", inputPosition, raycastResult)); + labelIndex++; + } + } + + return points; + } + + internal static List CreateOverlayElements(List points) + { + List overlayElements = new List(); + + foreach (RaycastGridPointInfo point in points) + { + if (!point.Hit) + { + continue; + } + + float halfSize = MARKER_SIZE / 2f; + overlayElements.Add(new UIElementInfo + { + Label = point.Label, + Name = point.HitGameObjectName ?? "", + Path = point.HitGameObjectPath ?? "", + Type = "RaycastHit", + Interaction = "Raycast", + SimX = point.InputX, + SimY = point.InputY, + BoundsMinX = point.InjectedUnityPositionX - halfSize, + BoundsMinY = point.InjectedUnityPositionY - halfSize, + BoundsMaxX = point.InjectedUnityPositionX + halfSize, + BoundsMaxY = point.InjectedUnityPositionY + halfSize, + SortingOrder = 0, + SiblingIndex = 0 + }); + } + + return overlayElements; + } + + private static RaycastGridPointInfo CreatePointInfo( + string label, + Vector2 inputPosition, + GameViewRaycastResult raycastResult) + { + RaycastGridPointInfo pointInfo = new RaycastGridPointInfo + { + Label = label, + Hit = raycastResult.Hits.Length > 0, + InputX = inputPosition.x, + InputY = inputPosition.y, + InjectedUnityPositionX = raycastResult.Conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = raycastResult.Conversion.InjectedUnityPosition.y + }; + + if (!pointInfo.Hit) + { + return pointInfo; + } + + RaycastHit hit = raycastResult.Hits[0]; + pointInfo.HitGameObjectName = hit.collider.gameObject.name; + pointInfo.HitGameObjectPath = GameObjectPathUtility.GetFullPath(hit.collider.gameObject); + pointInfo.Distance = hit.distance; + return pointInfo; + } + } +} diff --git a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta new file mode 100644 index 0000000000..ac65e30213 --- /dev/null +++ b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 23328277c0b2f44debb12cc314568f86 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/README.md b/Packages/src/README.md index 1bb5013aef..d5c06fcbba 100644 --- a/Packages/src/README.md +++ b/Packages/src/README.md @@ -154,6 +154,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi - `/uloop-get-hierarchy` - Get scene hierarchy - `/uloop-find-game-objects` - Find GameObjects - `/uloop-screenshot` - Capture EditorWindow +- `/uloop-raycast` - Check what a Game View coordinate hits in 3D physics - `/uloop-simulate-mouse-ui` - Simulate mouse click, long-press, and drag on PlayMode UI elements - `/uloop-simulate-mouse-input` - Simulate mouse input in PlayMode via Input System - `/uloop-simulate-keyboard` - Simulate keyboard input in PlayMode via Input System @@ -378,8 +379,10 @@ Great for keeping visual feedback in sync after other apps steal focus. (Linux i Take a screenshot of any EditorWindow as a PNG. Specify the window name (the text displayed in the title bar/tab) to capture. When multiple windows of the same type are open (e.g., 3 Inspector windows), all windows are saved with numbered filenames. Supports three matching modes: `exact` (default), `prefix`, and `contains` - all case-insensitive. +Use `CaptureMode: rendering` at the default resolution scale when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; those screenshots use top-left Game View coordinates that can be passed directly to those tools. ```text → screenshot (WindowName: "Console") +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) → Save Console window state as PNG → Provide visual feedback to AI ``` @@ -469,6 +472,8 @@ https://github.com/user-attachments/assets/c7ee9103-c282-4f90-8b01-64bb17400f3e ### 12. simulate-mouse-input - Simulate Mouse Input in PlayMode via Input System Simulate mouse input in PlayMode via Input System. Injects button clicks, mouse delta, and scroll wheel directly into `Mouse.current` for game logic that reads Input System. Unlike `simulate-mouse-ui` which fires EventSystem pointer events for uGUI, this tool targets game logic that reads `Mouse.current` directly. This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. +Click and LongPress coordinates use the same top-left Game View coordinates as `screenshot --capture-mode rendering`; do not flip Y before calling the tool. + Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. ```text @@ -480,7 +485,16 @@ Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. → simulate-mouse-input (Action: SmoothDelta, DeltaX: 300, DeltaY: 0, Duration: 0.5) ``` -### 13. simulate-keyboard - Simulate Keyboard Input in PlayMode +### 13. raycast - Check a 3D Physics Hit at a Game View Coordinate +Raycast from `Camera.main` through a top-left Game View coordinate. Use this before gameplay clicks when you need to confirm which 3D object a screenshot coordinate will hit. + +```text +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) +→ raycast (X: 960, Y: 540) +→ simulate-mouse-input (Action: Click, X: 960, Y: 540) +``` + +### 14. simulate-keyboard - Simulate Keyboard Input in PlayMode Simulate keyboard key input in PlayMode via Input System. Supports single key taps, sustained holds, and multi-key combinations (e.g. Shift+W for sprinting). This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. Game code must read input via Input System API (e.g. `Keyboard.current[Key.W].isPressed`), not legacy `Input.GetKey()`. Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), KeyUp (release held key). @@ -495,7 +509,7 @@ Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), → simulate-keyboard (Action: KeyUp, Key: LeftShift) ``` -### 14. record-input - Record Input During PlayMode +### 15. record-input - Record Input During PlayMode Record keyboard and mouse input during PlayMode frame-by-frame into a JSON file. Captures key presses, mouse movement, clicks, and scroll events via Input System device state diffing. This tool is available only when the Input System package is installed. ```text diff --git a/Packages/src/TOOL_REFERENCE.md b/Packages/src/TOOL_REFERENCE.md index d538577448..7abfda792f 100644 --- a/Packages/src/TOOL_REFERENCE.md +++ b/Packages/src/TOOL_REFERENCE.md @@ -184,6 +184,10 @@ All tools automatically include the following property: - `prefix`: Window name must start with the input - `contains`: Window name must contain the input anywhere - `OutputDirectory` (string): Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths (default: "") + - `CaptureMode` (enum): "window" captures Editor chrome, "rendering" captures Game View rendering with top-left Game View coordinates. With the default ResolutionScale 1.0, image pixels can be passed directly to input tools (default: "window") + - `AnnotateElements` (boolean): Annotate interactive UI elements in rendering screenshots (default: false) + - `AnnotateRaycastGrid` (boolean): Annotate 3D physics raycast candidate points in rendering screenshots (default: false) + - `ElementsOnly` (boolean): Return annotation JSON without writing an image file (default: false) - **Response**: - `ScreenshotCount` (number): Number of windows captured - `Screenshots` (array): Array of screenshot info @@ -191,6 +195,11 @@ All tools automatically include the following property: - `FileSizeBytes` (number): Size of the saved file in bytes - `Width` (number): Captured image width in pixels - `Height` (number): Captured image height in pixels + - `ImageCoordinateSystem` (string): "top-left-game-view" for rendering captures, "top-left-window" for window captures + - `GameViewWidth` / `GameViewHeight` (number): Game View size used by input tools + - `ScreenshotToInputFormula` (string): How to use screenshot coordinates with input tools + - `UnityInputFormula` (string): Internal conversion to `Mouse.current.position` + - `RaycastGridPoints` (array): 3D raycast candidate metadata when requested ### 10. control-play-mode - **Description**: Control Unity Editor play mode (play/stop/pause) @@ -216,8 +225,8 @@ All tools automatically include the following property: - `DragStart`: Begin drag at (X, Y) and hold - `DragMove`: Animate from current position to (X, Y) at the specified speed - `DragEnd`: Animate to (X, Y), then release drag - - `X` (number): Target X position in screen pixels, origin: top-left (default: 0) - - `Y` (number): Target Y position in screen pixels, origin: top-left (default: 0) + - `X` (number): Target X position in top-left Game View pixels (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels (default: 0) - `FromX` (number): Start X position for Drag action (default: 0) - `FromY` (number): Start Y position for Drag action (default: 0) - `DragSpeed` (number): Drag speed in pixels per second, 0 for instant (default: 2000) @@ -242,8 +251,8 @@ All tools automatically include the following property: - `MoveDelta`: Inject mouse delta one-shot (for FPS camera/look control) - `SmoothDelta`: Inject mouse delta smoothly over Duration seconds (human-like camera pan) - `Scroll`: Inject scroll wheel (for hotbar switching, zoom, etc.) - - `X` (number): Target X position in screen pixels, origin: top-left. Used by Click and LongPress (default: 0) - - `Y` (number): Target Y position in screen pixels, origin: top-left. Used by Click and LongPress (default: 0) + - `X` (number): Target X position in top-left Game View pixels. Use coordinates directly from `screenshot --capture-mode rendering` (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Use coordinates directly from `screenshot --capture-mode rendering` (default: 0) - `Button` (enum): Mouse button - "Left", "Right", "Middle" (default: "Left"). Used by Click and LongPress - `Duration` (number): Hold duration for LongPress, or minimum hold time for Click. 0 = one-shot tap (default: 0) - `DeltaX` (number): Delta X in pixels for MoveDelta. Positive = right (default: 0) @@ -257,8 +266,26 @@ All tools automatically include the following property: - `Button` (string): The button used (for Click/LongPress) - `PositionX` (number, nullable): X position used (null for non-positional actions) - `PositionY` (number, nullable): Y position used (null for non-positional actions) + - `InputCoordinateSystem` / `UnityCoordinateSystem` (string): Public and injected coordinate systems + - `GameViewWidth` / `GameViewHeight` (number): Game View size used for conversion + - `InputPositionX/Y` and `InjectedUnityPositionX/Y` (number, nullable): Received and injected coordinates + - `CoordinateConversionFormula` (string): `unity_x = input_x; unity_y = gameViewHeight - input_y` -### 13. simulate-keyboard +### 13. raycast +- **Description**: Raycast from `Camera.main` through a top-left Game View coordinate. Use this to check what a rendering screenshot coordinate hits in 3D physics before clicking with `simulate-mouse-input` +- **Parameters**: + - `X` (number): Target X position in top-left Game View pixels (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels (default: 0) + - `LayerMask` (number): Physics layer mask used by the raycast (default: Unity default raycast layers) + - `MaxDistance` (number): Maximum raycast distance in world units (default: 1000) +- **Response**: + - `Success` (boolean): Whether the command completed + - `Hit` (boolean): Whether physics hit anything + - `HitGameObjectName` / `HitGameObjectPath` (string): Hit object identity when `Hit` is true + - `Distance`, `HitPointX/Y/Z`, `HitNormalX/Y/Z` (number): Hit details when `Hit` is true + - `InputCoordinateSystem`, `UnityCoordinateSystem`, `GameViewWidth/Height`, `InputPositionX/Y`, `InjectedUnityPositionX/Y`, `CoordinateConversionFormula`: Coordinate conversion details + +### 14. simulate-keyboard - **Description**: Simulate keyboard key input in PlayMode via Input System. Supports single key taps, sustained holds, and multi-key combinations. Requires the Input System package, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. Game code must read input via Input System API (e.g. `Keyboard.current[Key.W].isPressed`), not legacy `Input.GetKey()` - **Parameters**: - `Action` (enum): Keyboard action - "Press", "KeyDown", "KeyUp" (default: "Press") diff --git a/README.md b/README.md index 1bb5013aef..d5c06fcbba 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi - `/uloop-get-hierarchy` - Get scene hierarchy - `/uloop-find-game-objects` - Find GameObjects - `/uloop-screenshot` - Capture EditorWindow +- `/uloop-raycast` - Check what a Game View coordinate hits in 3D physics - `/uloop-simulate-mouse-ui` - Simulate mouse click, long-press, and drag on PlayMode UI elements - `/uloop-simulate-mouse-input` - Simulate mouse input in PlayMode via Input System - `/uloop-simulate-keyboard` - Simulate keyboard input in PlayMode via Input System @@ -378,8 +379,10 @@ Great for keeping visual feedback in sync after other apps steal focus. (Linux i Take a screenshot of any EditorWindow as a PNG. Specify the window name (the text displayed in the title bar/tab) to capture. When multiple windows of the same type are open (e.g., 3 Inspector windows), all windows are saved with numbered filenames. Supports three matching modes: `exact` (default), `prefix`, and `contains` - all case-insensitive. +Use `CaptureMode: rendering` at the default resolution scale when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; those screenshots use top-left Game View coordinates that can be passed directly to those tools. ```text → screenshot (WindowName: "Console") +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) → Save Console window state as PNG → Provide visual feedback to AI ``` @@ -469,6 +472,8 @@ https://github.com/user-attachments/assets/c7ee9103-c282-4f90-8b01-64bb17400f3e ### 12. simulate-mouse-input - Simulate Mouse Input in PlayMode via Input System Simulate mouse input in PlayMode via Input System. Injects button clicks, mouse delta, and scroll wheel directly into `Mouse.current` for game logic that reads Input System. Unlike `simulate-mouse-ui` which fires EventSystem pointer events for uGUI, this tool targets game logic that reads `Mouse.current` directly. This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. +Click and LongPress coordinates use the same top-left Game View coordinates as `screenshot --capture-mode rendering`; do not flip Y before calling the tool. + Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. ```text @@ -480,7 +485,16 @@ Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. → simulate-mouse-input (Action: SmoothDelta, DeltaX: 300, DeltaY: 0, Duration: 0.5) ``` -### 13. simulate-keyboard - Simulate Keyboard Input in PlayMode +### 13. raycast - Check a 3D Physics Hit at a Game View Coordinate +Raycast from `Camera.main` through a top-left Game View coordinate. Use this before gameplay clicks when you need to confirm which 3D object a screenshot coordinate will hit. + +```text +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) +→ raycast (X: 960, Y: 540) +→ simulate-mouse-input (Action: Click, X: 960, Y: 540) +``` + +### 14. simulate-keyboard - Simulate Keyboard Input in PlayMode Simulate keyboard key input in PlayMode via Input System. Supports single key taps, sustained holds, and multi-key combinations (e.g. Shift+W for sprinting). This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. Game code must read input via Input System API (e.g. `Keyboard.current[Key.W].isPressed`), not legacy `Input.GetKey()`. Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), KeyUp (release held key). @@ -495,7 +509,7 @@ Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), → simulate-keyboard (Action: KeyUp, Key: LeftShift) ``` -### 14. record-input - Record Input During PlayMode +### 15. record-input - Record Input During PlayMode Record keyboard and mouse input during PlayMode frame-by-frame into a JSON file. Captures key presses, mouse movement, clicks, and scroll events via Input System device state diffing. This tool is available only when the Input System package is installed. ```text From dd53b12691c5e22a32a71795dce1c6edfd73a8bd Mon Sep 17 00:00:00 2001 From: hatayama Date: Sat, 4 Jul 2026 00:59:14 +0900 Subject: [PATCH 2/4] Fix rendering screenshot input offsets Restore the screenshot-to-input Y offset metadata for rendering captures so raw image pixels can be converted even when the Game View input area is taller than the RenderTexture. Align runtime tool descriptions and skills with the formula-based coordinate contract. --- .../Editor/EditorWindowCaptureUtilityTests.cs | 28 ++++++++++++ .../EditorWindowCaptureUtilityTests.cs.meta | 11 +++++ .../Tests/Editor/RaycastGridAnnotatorTests.cs | 25 +++++++++++ Assets/Tests/Editor/RaycastToolTests.cs | 30 +++++++++++++ Packages/packages-lock.json | 3 +- Packages/src/Cli~/src/default-tools.json | 12 ++--- .../Api/McpTools/Raycast/RaycastSchema.cs | 4 +- .../Api/McpTools/Raycast/RaycastTool.cs | 2 +- .../Api/McpTools/Raycast/Skill/SKILL.md | 8 ++-- .../McpTools/Screenshot/ScreenshotResponse.cs | 1 + .../McpTools/Screenshot/ScreenshotSchema.cs | 2 +- .../Api/McpTools/Screenshot/ScreenshotTool.cs | 21 ++++++--- .../Api/McpTools/Screenshot/Skill/SKILL.md | 3 +- .../Skill/references/annotated-elements.md | 8 ++-- .../SimulateMouseInputSchema.cs | 4 +- .../SimulateMouseInput/Skill/SKILL.md | 9 ++-- .../src/Editor/Shared/Config/McpConstants.cs | 4 +- .../Utils/EditorWindowCaptureUtility.cs | 45 +++++++++++++++++-- .../Editor/Utils/GameViewRaycastUtility.cs | 2 + .../src/Editor/Utils/RaycastGridAnnotator.cs | 29 ++++++++++-- Packages/src/README.md | 2 +- Packages/src/TOOL_REFERENCE.md | 13 +++--- Packages/src/package.json | 1 + README.md | 2 +- 24 files changed, 218 insertions(+), 51 deletions(-) create mode 100644 Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs create mode 100644 Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs new file mode 100644 index 0000000000..8de1586087 --- /dev/null +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs @@ -0,0 +1,28 @@ +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class EditorWindowCaptureUtilityTests + { + [Test] + public void CalculateImageToInputOffsetY_WhenRenderTextureIsShorterThanGameView_ShouldReturnTopOffset() + { + Vector2 gameViewSize = new Vector2(1768f, 1383f); + + int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080); + + Assert.That(offsetY, Is.EqualTo(303)); + } + + [Test] + public void CalculateImageToInputOffsetY_WhenRenderTextureMatchesGameView_ShouldReturnZero() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + + int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080); + + Assert.That(offsetY, Is.EqualTo(0)); + } + } +} diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta new file mode 100644 index 0000000000..3b34562bc9 --- /dev/null +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 54fcac766c4c54beab3ca57f04ef6425 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs index 86a8f0e2de..b527a0b488 100644 --- a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs @@ -1,10 +1,35 @@ using System.Collections.Generic; using NUnit.Framework; +using UnityEngine; namespace io.github.hatayama.uLoopMCP.Tests.Editor { public class RaycastGridAnnotatorTests { + [Test] + public void CalculateGridInputPosition_WhenRenderingHasTopOffset_ShouldSampleInsideCapturedImage() + { + Vector2 renderingImageSize = new Vector2(1200f, 1080f); + + Vector2 inputPosition = + RaycastGridAnnotator.CalculateGridInputPosition(renderingImageSize, 303, 1, 3); + + Assert.That(inputPosition.x, Is.EqualTo(600f)); + Assert.That(inputPosition.y, Is.EqualTo(483f)); + } + + [Test] + public void CalculateGridInputPosition_WhenRenderingHasTopOffset_ShouldKeepBottomRowVisible() + { + Vector2 renderingImageSize = new Vector2(1200f, 1080f); + + Vector2 inputPosition = + RaycastGridAnnotator.CalculateGridInputPosition(renderingImageSize, 303, 5, 3); + + Assert.That(inputPosition.x, Is.EqualTo(600f)); + Assert.That(inputPosition.y, Is.EqualTo(1203f)); + } + [Test] public void CreateOverlayElements_WhenPointsIncludeMisses_ShouldAnnotateOnlyHits() { diff --git a/Assets/Tests/Editor/RaycastToolTests.cs b/Assets/Tests/Editor/RaycastToolTests.cs index f3b0c5bc26..516580bca2 100644 --- a/Assets/Tests/Editor/RaycastToolTests.cs +++ b/Assets/Tests/Editor/RaycastToolTests.cs @@ -11,11 +11,20 @@ public class RaycastToolTests { private GameObject? _cameraObject; private GameObject? _cubeObject; + private bool _originalAutoSyncTransforms; private readonly List _retaggedMainCameraObjects = new List(); + [SetUp] + public void SetUp() + { + _originalAutoSyncTransforms = Physics.autoSyncTransforms; + } + [TearDown] public void TearDown() { + Physics.autoSyncTransforms = _originalAutoSyncTransforms; + if (_cubeObject != null) { Object.DestroyImmediate(_cubeObject); @@ -73,6 +82,27 @@ public async Task ExecuteAsync_WhenCoordinateMissesCollider_ShouldReturnNoHit() Assert.That(response.HitGameObjectName, Is.Null); } + [Test] + public async Task ExecuteAsync_WhenAutoSyncTransformsIsDisabled_ShouldRaycastAgainstLatestTransform() + { + Physics.autoSyncTransforms = false; + CreateRaycastScene(); + if (_cubeObject == null) + { + Assert.Fail("Cube should be created."); + return; + } + + _cubeObject.transform.position = new Vector3(100f, 0f, 0f); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.False); + } + private void CreateRaycastScene() { RetagExistingMainCameras(); diff --git a/Packages/packages-lock.json b/Packages/packages-lock.json index e872ac6ee7..927b5ed879 100644 --- a/Packages/packages-lock.json +++ b/Packages/packages-lock.json @@ -234,6 +234,7 @@ "depth": 0, "source": "embedded", "dependencies": { + "com.unity.modules.physics": "1.0.0", "com.unity.ugui": "1.0.0", "com.unity.nuget.newtonsoft-json": "3.2.1" } @@ -285,7 +286,7 @@ }, "com.unity.modules.physics": { "version": "1.0.0", - "depth": 2, + "depth": 1, "source": "builtin", "dependencies": {} }, diff --git a/Packages/src/Cli~/src/default-tools.json b/Packages/src/Cli~/src/default-tools.json index 11e0f0cc36..85b7f6ff57 100644 --- a/Packages/src/Cli~/src/default-tools.json +++ b/Packages/src/Cli~/src/default-tools.json @@ -247,7 +247,7 @@ }, "CaptureMode": { "type": "string", - "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Rendering screenshots at ResolutionScale=1.0 use top-left Game View coordinates that can be passed directly to simulate-mouse-input and raycast.", + "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Rendering screenshots return ScreenshotToInputFormula for converting raw image pixels before calling simulate-mouse-input or raycast.", "enum": [ "window", "rendering" @@ -274,18 +274,18 @@ }, { "name": "raycast", - "description": "Raycast from Camera.main through a Game View coordinate. X/Y use the same top-left coordinates as screenshot --capture-mode rendering and simulate-mouse-input.", + "description": "Raycast from Camera.main through a top-left Game View coordinate. Use annotated SimX/SimY, raycast-grid InputX/InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.", "inputSchema": { "type": "object", "properties": { "X": { "type": "number", - "description": "Target X position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", + "description": "Target X position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw image pixels converted with ScreenshotToInputFormula.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", + "description": "Target Y position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw image pixels converted with ScreenshotToInputFormula.", "default": 0 }, "LayerMask": { @@ -458,12 +458,12 @@ }, "X": { "type": "number", - "description": "Target X position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", + "description": "Target X position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw image pixels converted with ScreenshotToInputFormula.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.", + "description": "Target Y position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw image pixels converted with ScreenshotToInputFormula.", "default": 0 }, "Button": { diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs index 25d7bcf0dd..c26d15bc95 100644 --- a/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs @@ -5,10 +5,10 @@ namespace io.github.hatayama.uLoopMCP { public class RaycastSchema : BaseToolSchema { - [Description("Target X position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.")] + [Description("Target X position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public float X { get; set; } = 0f; - [Description("Target Y position in Game View pixels (origin: top-left). Use coordinates directly from screenshot --capture-mode rendering.")] + [Description("Target Y position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public float Y { get; set; } = 0f; [Description("Physics layer mask used by the raycast.")] diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs index ef6eff7a04..7b62ffe4a4 100644 --- a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs @@ -8,7 +8,7 @@ namespace io.github.hatayama.uLoopMCP /// /// Checks what a screenshot-compatible Game View coordinate hits in 3D physics. /// - [McpTool(Description = "Raycast from Camera.main through a Game View coordinate. X/Y use the same top-left coordinates as screenshot --capture-mode rendering and simulate-mouse-input.")] + [McpTool(Description = "Raycast from Camera.main through a top-left Game View coordinate. Use annotated SimX/SimY, raycast-grid InputX/InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public class RaycastTool : AbstractUnityTool { public override string ToolName => "raycast"; diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md index 52637111ac..b464ae09c3 100644 --- a/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md @@ -17,14 +17,16 @@ uloop raycast --x --y [--layer-mask ] [--max-distance ] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Use coordinates directly from `screenshot --capture-mode rendering`. | -| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Use coordinates directly from `screenshot --capture-mode rendering`. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula`. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula`. | | `--layer-mask` | number | Unity default raycast layers | Physics layer mask used by the raycast. | | `--max-distance` | number | `1000` | Maximum raycast distance in world units. | ## Coordinate System -- `--x` / `--y` use the same top-left Game View coordinates as `screenshot --capture-mode rendering` and `simulate-mouse-input`. +- `--x` / `--y` use the same top-left Game View input coordinates as `simulate-mouse-input`. +- Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. +- `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` can be passed directly to this tool. - Do not flip Y in the caller. The tool converts internally: ```text diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs index c7869c245d..1145d6db9c 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs @@ -12,6 +12,7 @@ public class ScreenshotInfo public int Height { get; set; } public string ImageCoordinateSystem { get; set; } = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; public float ResolutionScale { get; set; } = 1.0f; + public int ImageToInputOffsetY { get; set; } public float GameViewWidth { get; set; } public float GameViewHeight { get; set; } public string ScreenshotToInputFormula { get; set; } = McpConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs index b6496a73a8..0f969e94fc 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs @@ -43,7 +43,7 @@ public class ScreenshotSchema : BaseToolSchema [Description("Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths.")] public string OutputDirectory { get; set; } = ""; - [Description("Capture mode: window(0)=capture EditorWindow including toolbar, rendering(1)=capture game rendering only (PlayMode required, coordinates match simulate-mouse)")] + [Description("Capture mode: window(0)=capture EditorWindow including toolbar, rendering(1)=capture game rendering only (PlayMode required, returns ScreenshotToInputFormula for raw image pixels)")] public CaptureMode CaptureMode { get; set; } = CaptureMode.window; [Description("Annotate interactive UI elements with names and simulate-mouse coordinates on the screenshot. Only works with CaptureMode=rendering in PlayMode.")] diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs index a9ed5ffff9..540004271f 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs @@ -68,7 +68,11 @@ private async Task CaptureRenderingAsync( if (parameters.AnnotateRaycastGrid) { - raycastGridPoints = RaycastGridAnnotator.CollectRaycastGridPoints(gameViewSize); + (Vector2 renderingImageSize, int initialImageToInputOffsetY) = + await EditorWindowCaptureUtility.GetGameRenderingImageInfoAsync(gameViewSize, ct); + raycastGridPoints = RaycastGridAnnotator.CollectRaycastGridPoints( + renderingImageSize, + initialImageToInputOffsetY); raycastGridOverlayElements = RaycastGridAnnotator.CreateOverlayElements(raycastGridPoints); } @@ -85,6 +89,7 @@ private async Task CaptureRenderingAsync( GameObject annotationOverlay = null; Texture2D texture; + int imageToInputOffsetY = 0; try { if (parameters.AnnotateElements || parameters.AnnotateRaycastGrid) @@ -99,7 +104,7 @@ private async Task CaptureRenderingAsync( await EditorDelay.DelayFrame(ANNOTATION_OVERLAY_RENDER_WAIT_FRAMES, ct); } - texture = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( + (texture, imageToInputOffsetY) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( parameters.ResolutionScale, ct); } finally @@ -135,7 +140,7 @@ private async Task CaptureRenderingAsync( ScreenshotInfo info = new ScreenshotInfo( savedPath, savedFileInfo.Length, width, height, McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, parameters.ResolutionScale); - ApplyRenderingCoordinateMetadata(info, gameViewSize); + ApplyRenderingCoordinateMetadata(info, gameViewSize, imageToInputOffsetY); info.AnnotatedElements = annotatedElements; info.RaycastGridPoints = raycastGridPoints; screenshots.Add(info); @@ -242,14 +247,16 @@ private async Task CaptureWindowsAsync( return new ScreenshotResponse(screenshots); } - private static void ApplyRenderingCoordinateMetadata(ScreenshotInfo info, Vector2 gameViewSize) + private static void ApplyRenderingCoordinateMetadata( + ScreenshotInfo info, + Vector2 gameViewSize, + int imageToInputOffsetY = 0) { info.ImageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW; info.GameViewWidth = gameViewSize.x; info.GameViewHeight = gameViewSize.y; - info.ScreenshotToInputFormula = Mathf.Approximately(info.ResolutionScale, 1.0f) - ? McpConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA - : McpConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA_SCALED; + info.ImageToInputOffsetY = imageToInputOffsetY; + info.ScreenshotToInputFormula = McpConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA; info.UnityInputFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY; } diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md index 9ded5ab8b1..ff9f5c333b 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md @@ -86,6 +86,7 @@ Returns JSON with: - `Height`: Captured image height in pixels - `ImageCoordinateSystem`: `"top-left-game-view"` for `--capture-mode rendering`, or `"top-left-window"` for window captures - `ResolutionScale`: Resolution scale used for capture + - `ImageToInputOffsetY`: Y offset added after unscaling image pixels to get mouse-input coordinates - `GameViewWidth` / `GameViewHeight`: Game View size used by mouse input tools - `ScreenshotToInputFormula`: Formula for turning image coordinates into mouse-input coordinates - `UnityInputFormula`: Formula used internally by mouse input tools to inject `Mouse.current.position` @@ -102,5 +103,5 @@ When multiple windows match (e.g., multiple Inspector windows or when using `con - Target window must be open in Unity Editor - Window name matching is always case-insensitive - Use `--capture-mode rendering` for coordinates that should be passed to `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`. -- Keep `--resolution-scale 1.0` when you want to pass image pixel coordinates directly to mouse tools. +- Use `ScreenshotToInputFormula` before passing raw image pixels to mouse tools. `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` are already mouse-input coordinates. - Do not use `window` captures as mouse-input coordinates because they include Unity Editor chrome. diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md index 6c0b470ee8..7b47103f7f 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md @@ -18,14 +18,14 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme ## Coordinate Conversion -When `ImageCoordinateSystem` is `"top-left-game-view"` and `ResolutionScale` is `1.0`, image pixel coordinates from `screenshot --capture-mode rendering` are already mouse-input coordinates: +When `ImageCoordinateSystem` is `"top-left-game-view"`, convert raw image pixel coordinates from `screenshot --capture-mode rendering` with the formula returned in `ScreenshotToInputFormula`: ```text -input_x = image_x -input_y = image_y +input_x = image_x / resolutionScale +input_y = image_y / resolutionScale + imageToInputOffsetY ``` -When `ResolutionScale` is not `1.0`, use the formula returned in `ScreenshotToInputFormula`. +When `ResolutionScale` is `1.0` and `ImageToInputOffsetY` is `0`, raw image pixel coordinates already match mouse-input coordinates. `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` are always returned as mouse-input coordinates, so pass those values directly. The mouse input tools convert internally to Unity Input System coordinates: diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs index 64a2ad7994..6f09b11d2d 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs @@ -7,10 +7,10 @@ public class SimulateMouseInputSchema : BaseToolSchema [Description("Mouse input action: Click(0) - inject left/right/middle button press+release, LongPress(1) - inject button hold for Duration seconds, MoveDelta(2) - inject mouse delta for camera/look (one-shot), Scroll(3) - inject scroll wheel, SmoothDelta(4) - inject mouse delta smoothly over Duration seconds")] public MouseInputAction Action { get; set; } = MouseInputAction.Click; - [Description("Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from screenshot --capture-mode rendering.")] + [Description("Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public float X { get; set; } = 0f; - [Description("Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from screenshot --capture-mode rendering.")] + [Description("Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public float Y { get; set; } = 0f; [Description("Mouse button: Left(0, default), Right(1), Middle(2). Used by Click and LongPress.")] diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md index adb2ed2158..d0bca06e34 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md @@ -11,7 +11,7 @@ Simulate mouse input via Input System in Unity PlayMode: $ARGUMENTS ## Workflow 1. Ensure Unity is in PlayMode (use `uloop control-play-mode --action Play` if not) -2. For Click/LongPress: determine the target Game View position with `uloop screenshot --capture-mode rendering` +2. For Click/LongPress: determine the target Game View input position from annotated `SimX`/`SimY`, raycast-grid `InputX`/`InputY`, or raw image pixels converted with `ScreenshotToInputFormula` 3. Execute the appropriate `uloop simulate-mouse-input` command 4. Take a screenshot to verify the result: `uloop screenshot --capture-mode rendering` 5. Report what happened @@ -27,8 +27,8 @@ uloop simulate-mouse-input --action [options] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `--action` | enum | `Click` | `Click`, `LongPress`, `MoveDelta`, `SmoothDelta`, `Scroll` | -| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from `screenshot --capture-mode rendering`. | -| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use coordinates directly from `screenshot --capture-mode rendering`. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula`. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula`. | | `--button` | enum | `Left` | Mouse button: `Left`, `Right`, `Middle`. Used by Click and LongPress. | | `--duration` | number | `0` | Hold duration for LongPress, or interpolation duration for SmoothDelta (seconds). For Click, 0 = one-shot tap. | | `--delta-x` | number | `0` | Delta X in pixels for MoveDelta/SmoothDelta. Positive = right. | @@ -92,7 +92,8 @@ uloop simulate-mouse-input --action SmoothDelta --delta-x 300 --delta-y 0 --dura ## Coordinate System - `--x` / `--y` use **top-left Game View coordinates**. -- Coordinates read from `uloop screenshot --capture-mode rendering` can be passed directly to this tool. +- Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. +- `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` can be passed directly to this tool. - Do not flip Y in the caller. The tool converts internally for Unity Input System: ```text diff --git a/Packages/src/Editor/Shared/Config/McpConstants.cs b/Packages/src/Editor/Shared/Config/McpConstants.cs index fd48bbc538..3a1e4deec7 100644 --- a/Packages/src/Editor/Shared/Config/McpConstants.cs +++ b/Packages/src/Editor/Shared/Config/McpConstants.cs @@ -174,9 +174,7 @@ public static UnityEditor.PackageManager.PackageInfo PackageInfo public const string COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY = "unity_x = input_x; unity_y = gameViewHeight - input_y"; public const string SCREENSHOT_RENDERING_TO_INPUT_FORMULA = - "simulate_mouse_x = image_x; simulate_mouse_y = image_y"; - public const string SCREENSHOT_RENDERING_TO_INPUT_FORMULA_SCALED = - "simulate_mouse_x = image_x / resolutionScale; simulate_mouse_y = image_y / resolutionScale"; + "simulate_mouse_x = image_x / resolutionScale; simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY"; public const string SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE = "unavailable: window screenshots include Unity Editor chrome; use capture-mode rendering for mouse input coordinates"; public const float RAYCAST_DEFAULT_MAX_DISTANCE = 1000f; diff --git a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs index ef48df1a49..6036cceaa9 100644 --- a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs @@ -138,7 +138,7 @@ public static string[] GetOpenWindowNames() // Captures game rendering by reading GameView's composited RenderTexture (PlayMode only). // Contains all cameras + Screen Space Overlay Canvas, without tab bar or borders. - public static async Task CaptureGameRenderingAsync(float resolutionScale, CancellationToken ct) + public static async Task<(Texture2D? texture, int imageToInputOffsetY)> CaptureGameRenderingAsync(float resolutionScale, CancellationToken ct) { Debug.Assert(UnityEditor.EditorApplication.isPlaying, "CaptureGameRenderingAsync requires PlayMode"); @@ -149,10 +149,13 @@ public static string[] GetOpenWindowNames() if (rt == null) { Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); - return null; + return (null, 0); } - // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format + // GameView RenderTexture can be shorter than the full input area, so raw image Y needs this offset. + int imageToInputOffsetY = CalculateImageToInputOffsetY(Handles.GetMainGameViewSize(), rt.height); + + // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format. RenderTextureDescriptor flipDescriptor = new RenderTextureDescriptor(rt.width, rt.height, rt.format, 0); if (QualitySettings.activeColorSpace == ColorSpace.Linear) { @@ -176,7 +179,41 @@ public static string[] GetOpenWindowNames() texture = ApplyResolutionScaling(texture, resolutionScale); } - return texture; + return (texture, imageToInputOffsetY); + } + + /// + /// Gets rendering image dimensions after waiting for Game View rendering to settle. + /// + internal static async Task<(Vector2 renderingImageSize, int imageToInputOffsetY)> GetGameRenderingImageInfoAsync( + Vector2 gameViewSize, + CancellationToken ct) + { + Debug.Assert(UnityEditor.EditorApplication.isPlaying, "GetGameRenderingImageInfoAsync requires PlayMode"); + + // Raycast-grid annotations must use the same settled RenderTexture geometry as the PNG capture path. + await EditorDelay.DelayFrame(2, ct); + + RenderTexture rt = GameViewBridge.GetRenderTexture(); + if (rt == null) + { + return (gameViewSize, 0); + } + + Vector2 renderingImageSize = new Vector2(rt.width, rt.height); + int imageToInputOffsetY = CalculateImageToInputOffsetY(gameViewSize, rt.height); + return (renderingImageSize, imageToInputOffsetY); + } + + /// + /// Calculates the Y offset from rendering screenshot image space to Game View input space. + /// + internal static int CalculateImageToInputOffsetY(Vector2 gameViewSize, int renderTextureHeight) + { + Debug.Assert(gameViewSize.y >= 0f, "Game View height must not be negative."); + Debug.Assert(renderTextureHeight >= 0, "RenderTexture height must not be negative."); + + return Mathf.RoundToInt(gameViewSize.y) - renderTextureHeight; } private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float scale) diff --git a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs index eb7e251623..c2fa14557f 100644 --- a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs +++ b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs @@ -24,6 +24,8 @@ internal static GameViewRaycastResult RaycastFromInputPosition( } Ray ray = mainCamera.ScreenPointToRay(conversion.InjectedUnityPosition); + // Transform changes can be visible before the physics scene updates, so sync before screenshot-based queries. + Physics.SyncTransforms(); RaycastHit[] hits = Physics.RaycastAll(ray, maxDistance, layerMask); System.Array.Sort(hits, CompareHitsByDistance); diff --git a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs index e2b8ed3ee8..02203223a2 100644 --- a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs +++ b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs @@ -13,7 +13,9 @@ internal static class RaycastGridAnnotator private const int GRID_ROWS = 5; private const float MARKER_SIZE = 18f; - internal static List CollectRaycastGridPoints(Vector2 gameViewSize) + internal static List CollectRaycastGridPoints( + Vector2 renderingImageSize, + int imageToInputOffsetY) { List points = new List(); int labelIndex = 1; @@ -22,9 +24,11 @@ internal static List CollectRaycastGridPoints(Vector2 game { for (int column = 1; column <= GRID_COLUMNS; column++) { - Vector2 inputPosition = new Vector2( - gameViewSize.x * column / (GRID_COLUMNS + 1f), - gameViewSize.y * row / (GRID_ROWS + 1f)); + Vector2 inputPosition = CalculateGridInputPosition( + renderingImageSize, + imageToInputOffsetY, + row, + column); GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( inputPosition, McpConstants.RAYCAST_DEFAULT_MAX_DISTANCE, @@ -38,6 +42,23 @@ internal static List CollectRaycastGridPoints(Vector2 game return points; } + internal static Vector2 CalculateGridInputPosition( + Vector2 renderingImageSize, + int imageToInputOffsetY, + int row, + int column) + { + Debug.Assert(renderingImageSize.x >= 0f, "Rendering image width must not be negative."); + Debug.Assert(renderingImageSize.y >= 0f, "Rendering image height must not be negative."); + Debug.Assert(row >= 1 && row <= GRID_ROWS, "Grid row must be within the configured grid."); + Debug.Assert(column >= 1 && column <= GRID_COLUMNS, "Grid column must be within the configured grid."); + + // Grid points must be visible in the captured PNG, so sample image space before adding the input Y offset. + return new Vector2( + renderingImageSize.x * column / (GRID_COLUMNS + 1f), + imageToInputOffsetY + renderingImageSize.y * row / (GRID_ROWS + 1f)); + } + internal static List CreateOverlayElements(List points) { List overlayElements = new List(); diff --git a/Packages/src/README.md b/Packages/src/README.md index d5c06fcbba..cfb3775a78 100644 --- a/Packages/src/README.md +++ b/Packages/src/README.md @@ -379,7 +379,7 @@ Great for keeping visual feedback in sync after other apps steal focus. (Linux i Take a screenshot of any EditorWindow as a PNG. Specify the window name (the text displayed in the title bar/tab) to capture. When multiple windows of the same type are open (e.g., 3 Inspector windows), all windows are saved with numbered filenames. Supports three matching modes: `exact` (default), `prefix`, and `contains` - all case-insensitive. -Use `CaptureMode: rendering` at the default resolution scale when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; those screenshots use top-left Game View coordinates that can be passed directly to those tools. +Use `CaptureMode: rendering` when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; convert raw image pixels with `ScreenshotToInputFormula`, or pass annotated `SimX/SimY` and raycast-grid `InputX/InputY` values directly. ```text → screenshot (WindowName: "Console") → screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) diff --git a/Packages/src/TOOL_REFERENCE.md b/Packages/src/TOOL_REFERENCE.md index 7abfda792f..62c4f2e177 100644 --- a/Packages/src/TOOL_REFERENCE.md +++ b/Packages/src/TOOL_REFERENCE.md @@ -184,7 +184,7 @@ All tools automatically include the following property: - `prefix`: Window name must start with the input - `contains`: Window name must contain the input anywhere - `OutputDirectory` (string): Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths (default: "") - - `CaptureMode` (enum): "window" captures Editor chrome, "rendering" captures Game View rendering with top-left Game View coordinates. With the default ResolutionScale 1.0, image pixels can be passed directly to input tools (default: "window") + - `CaptureMode` (enum): "window" captures Editor chrome, "rendering" captures Game View rendering with top-left Game View coordinates. Use `ScreenshotToInputFormula` before passing raw image pixels to input tools (default: "window") - `AnnotateElements` (boolean): Annotate interactive UI elements in rendering screenshots (default: false) - `AnnotateRaycastGrid` (boolean): Annotate 3D physics raycast candidate points in rendering screenshots (default: false) - `ElementsOnly` (boolean): Return annotation JSON without writing an image file (default: false) @@ -196,6 +196,7 @@ All tools automatically include the following property: - `Width` (number): Captured image width in pixels - `Height` (number): Captured image height in pixels - `ImageCoordinateSystem` (string): "top-left-game-view" for rendering captures, "top-left-window" for window captures + - `ImageToInputOffsetY` (number): Y offset added after unscaling raw image pixels to get input coordinates - `GameViewWidth` / `GameViewHeight` (number): Game View size used by input tools - `ScreenshotToInputFormula` (string): How to use screenshot coordinates with input tools - `UnityInputFormula` (string): Internal conversion to `Mouse.current.position` @@ -251,8 +252,8 @@ All tools automatically include the following property: - `MoveDelta`: Inject mouse delta one-shot (for FPS camera/look control) - `SmoothDelta`: Inject mouse delta smoothly over Duration seconds (human-like camera pan) - `Scroll`: Inject scroll wheel (for hotbar switching, zoom, etc.) - - `X` (number): Target X position in top-left Game View pixels. Use coordinates directly from `screenshot --capture-mode rendering` (default: 0) - - `Y` (number): Target Y position in top-left Game View pixels. Use coordinates directly from `screenshot --capture-mode rendering` (default: 0) + - `X` (number): Target X position in top-left Game View pixels. Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) - `Button` (enum): Mouse button - "Left", "Right", "Middle" (default: "Left"). Used by Click and LongPress - `Duration` (number): Hold duration for LongPress, or minimum hold time for Click. 0 = one-shot tap (default: 0) - `DeltaX` (number): Delta X in pixels for MoveDelta. Positive = right (default: 0) @@ -272,10 +273,10 @@ All tools automatically include the following property: - `CoordinateConversionFormula` (string): `unity_x = input_x; unity_y = gameViewHeight - input_y` ### 13. raycast -- **Description**: Raycast from `Camera.main` through a top-left Game View coordinate. Use this to check what a rendering screenshot coordinate hits in 3D physics before clicking with `simulate-mouse-input` +- **Description**: Raycast from `Camera.main` through a top-left Game View coordinate. Use this to check what a converted rendering screenshot coordinate hits in 3D physics before clicking with `simulate-mouse-input` - **Parameters**: - - `X` (number): Target X position in top-left Game View pixels (default: 0) - - `Y` (number): Target Y position in top-left Game View pixels (default: 0) + - `X` (number): Target X position in top-left Game View pixels. Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) - `LayerMask` (number): Physics layer mask used by the raycast (default: Unity default raycast layers) - `MaxDistance` (number): Maximum raycast distance in world units (default: 1000) - **Response**: diff --git a/Packages/src/package.json b/Packages/src/package.json index 5c39d9fde5..33a98de8f8 100644 --- a/Packages/src/package.json +++ b/Packages/src/package.json @@ -12,6 +12,7 @@ "url": "https://github.com/hatayama/uLoopMCP" }, "dependencies": { + "com.unity.modules.physics": "1.0.0", "com.unity.ugui": "1.0.0", "com.unity.nuget.newtonsoft-json": "3.2.1" }, diff --git a/README.md b/README.md index d5c06fcbba..cfb3775a78 100644 --- a/README.md +++ b/README.md @@ -379,7 +379,7 @@ Great for keeping visual feedback in sync after other apps steal focus. (Linux i Take a screenshot of any EditorWindow as a PNG. Specify the window name (the text displayed in the title bar/tab) to capture. When multiple windows of the same type are open (e.g., 3 Inspector windows), all windows are saved with numbered filenames. Supports three matching modes: `exact` (default), `prefix`, and `contains` - all case-insensitive. -Use `CaptureMode: rendering` at the default resolution scale when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; those screenshots use top-left Game View coordinates that can be passed directly to those tools. +Use `CaptureMode: rendering` when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; convert raw image pixels with `ScreenshotToInputFormula`, or pass annotated `SimX/SimY` and raycast-grid `InputX/InputY` values directly. ```text → screenshot (WindowName: "Console") → screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) From d64065c81df668c641a05c0cc9e15fd5d94862d1 Mon Sep 17 00:00:00 2001 From: hatayama Date: Sat, 4 Jul 2026 02:30:32 +0900 Subject: [PATCH 3/4] Address mouse coordinate review feedback Keep screenshot metadata, raycast results, and UI mouse documentation consistent so review tools and agents can rely on the same Game View coordinate contract. --- Assets/Tests/Editor/RaycastToolTests.cs | 39 +++++++++++++ Packages/src/Cli~/src/default-tools.json | 6 +- .../Api/McpTools/Raycast/RaycastTool.cs | 12 ++-- .../Api/McpTools/Screenshot/ScreenshotTool.cs | 35 ++++++++---- .../SimulateMouseInputTool.cs | 43 ++++++++------- .../SimulateMouseUi/SimulateMouseUiSchema.cs | 4 +- .../SimulateMouseUi/SimulateMouseUiTool.cs | 2 +- .../McpTools/SimulateMouseUi/Skill/SKILL.md | 8 +-- .../Utils/EditorWindowCaptureUtility.cs | 55 +++++++++++++++---- .../Editor/Utils/GameViewRaycastUtility.cs | 14 +++-- .../src/Editor/Utils/RaycastGridAnnotator.cs | 5 +- Packages/src/README.md | 2 +- Packages/src/TOOL_REFERENCE.md | 10 ++-- README.md | 2 +- 14 files changed, 168 insertions(+), 69 deletions(-) diff --git a/Assets/Tests/Editor/RaycastToolTests.cs b/Assets/Tests/Editor/RaycastToolTests.cs index 516580bca2..c9f9d8c06e 100644 --- a/Assets/Tests/Editor/RaycastToolTests.cs +++ b/Assets/Tests/Editor/RaycastToolTests.cs @@ -82,6 +82,45 @@ public async Task ExecuteAsync_WhenCoordinateMissesCollider_ShouldReturnNoHit() Assert.That(response.HitGameObjectName, Is.Null); } + [Test] + public async Task ExecuteAsync_WhenCameraIsMissing_ShouldReturnConversionMetadata() + { + RetagExistingMainCameras(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.False); + Assert.That(response.Message, Does.Contain("Camera.main")); + Assert.That(response.InputPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InputPositionY, Is.EqualTo(inputPosition.y)); + Assert.That(response.InjectedUnityPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InjectedUnityPositionY, Is.EqualTo(gameViewSize.y - inputPosition.y)); + Assert.That(response.CoordinateConversionFormula, Is.EqualTo(McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY)); + } + + [Test] + public async Task ExecuteAsync_WhenColliderLayerIsHiddenByCamera_ShouldReturnNoHit() + { + CreateRaycastScene(); + if (_cameraObject == null) + { + Assert.Fail("Camera should be created."); + return; + } + + Camera camera = _cameraObject.GetComponent(); + camera.cullingMask = 0; + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.False); + } + [Test] public async Task ExecuteAsync_WhenAutoSyncTransformsIsDisabled_ShouldRaycastAgainstLatestTransform() { diff --git a/Packages/src/Cli~/src/default-tools.json b/Packages/src/Cli~/src/default-tools.json index 85b7f6ff57..c8aa0b481d 100644 --- a/Packages/src/Cli~/src/default-tools.json +++ b/Packages/src/Cli~/src/default-tools.json @@ -363,7 +363,7 @@ }, { "name": "simulate-mouse-ui", - "description": "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem screen coordinates", + "description": "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem using top-left Game View coordinates", "inputSchema": { "type": "object", "properties": { @@ -382,12 +382,12 @@ }, "X": { "type": "number", - "description": "Target X position in Game View pixels (origin: top-left). For Drag action, this is the destination.", + "description": "Target X position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in Game View pixels (origin: top-left). For Drag action, this is the destination.", + "description": "Target Y position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.", "default": 0 }, "FromX": { diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs index 7b62ffe4a4..f0647b9610 100644 --- a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs @@ -30,15 +30,15 @@ protected override Task ExecuteAsync(RaycastSchema parameters, GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( inputPosition, parameters.MaxDistance, - parameters.LayerMask); + parameters.LayerMask, + true); if (!raycastResult.CameraFound) { - return Task.FromResult(new RaycastResponse - { - Success = false, - Message = "Camera.main was not found. Add an active camera tagged MainCamera before using raycast." - }); + RaycastResponse noCameraResponse = CreateBaseResponse(raycastResult.Conversion); + noCameraResponse.Success = false; + noCameraResponse.Message = "Camera.main was not found. Add an active camera tagged MainCamera before using raycast."; + return Task.FromResult(noCameraResponse); } if (raycastResult.Hits.Length == 0) diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs index 540004271f..9804cf37ae 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs @@ -59,6 +59,7 @@ private async Task CaptureRenderingAsync( Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); List raycastGridPoints = new List(); List raycastGridOverlayElements = new List(); + GameRenderingImageInfo? raycastGridRenderingInfo = null; if (parameters.AnnotateElements) { @@ -68,11 +69,13 @@ private async Task CaptureRenderingAsync( if (parameters.AnnotateRaycastGrid) { - (Vector2 renderingImageSize, int initialImageToInputOffsetY) = - await EditorWindowCaptureUtility.GetGameRenderingImageInfoAsync(gameViewSize, ct); + GameRenderingImageInfo renderingImageInfo = + await EditorWindowCaptureUtility.GetGameRenderingImageInfoAsync(ct); + raycastGridRenderingInfo = renderingImageInfo; + gameViewSize = renderingImageInfo.GameViewSize; raycastGridPoints = RaycastGridAnnotator.CollectRaycastGridPoints( - renderingImageSize, - initialImageToInputOffsetY); + renderingImageInfo.RenderingImageSize, + renderingImageInfo.ImageToInputOffsetY); raycastGridOverlayElements = RaycastGridAnnotator.CreateOverlayElements(raycastGridPoints); } @@ -81,7 +84,8 @@ private async Task CaptureRenderingAsync( UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); ScreenshotInfo elementsOnlyInfo = new ScreenshotInfo(); elementsOnlyInfo.ResolutionScale = parameters.ResolutionScale; - ApplyRenderingCoordinateMetadata(elementsOnlyInfo, gameViewSize); + int imageToInputOffsetY = raycastGridRenderingInfo?.ImageToInputOffsetY ?? 0; + ApplyRenderingCoordinateMetadata(elementsOnlyInfo, gameViewSize, imageToInputOffsetY); elementsOnlyInfo.AnnotatedElements = annotatedElements; elementsOnlyInfo.RaycastGridPoints = raycastGridPoints; return new ScreenshotResponse(new List { elementsOnlyInfo }); @@ -89,7 +93,7 @@ private async Task CaptureRenderingAsync( GameObject annotationOverlay = null; Texture2D texture; - int imageToInputOffsetY = 0; + GameRenderingImageInfo captureRenderingInfo; try { if (parameters.AnnotateElements || parameters.AnnotateRaycastGrid) @@ -104,8 +108,10 @@ private async Task CaptureRenderingAsync( await EditorDelay.DelayFrame(ANNOTATION_OVERLAY_RENDER_WAIT_FRAMES, ct); } - (texture, imageToInputOffsetY) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( - parameters.ResolutionScale, ct); + (texture, captureRenderingInfo) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( + parameters.ResolutionScale, + raycastGridRenderingInfo, + ct); } finally { @@ -140,7 +146,10 @@ private async Task CaptureRenderingAsync( ScreenshotInfo info = new ScreenshotInfo( savedPath, savedFileInfo.Length, width, height, McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, parameters.ResolutionScale); - ApplyRenderingCoordinateMetadata(info, gameViewSize, imageToInputOffsetY); + ApplyRenderingCoordinateMetadata( + info, + captureRenderingInfo.GameViewSize, + captureRenderingInfo.ImageToInputOffsetY); info.AnnotatedElements = annotatedElements; info.RaycastGridPoints = raycastGridPoints; screenshots.Add(info); @@ -218,7 +227,13 @@ private async Task CaptureWindowsAsync( SaveTextureAsPng(texture, savedPath); FileInfo savedFileInfo = new FileInfo(savedPath); - ScreenshotInfo info = new ScreenshotInfo(savedPath, savedFileInfo.Length, width, height); + ScreenshotInfo info = new ScreenshotInfo( + savedPath, + savedFileInfo.Length, + width, + height, + McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW, + parameters.ResolutionScale); ApplyWindowCoordinateMetadata(info); screenshots.Add(info); } diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs index c2b6339146..f6b872d59e 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs @@ -200,24 +200,12 @@ await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( } string durationText = parameters.Duration > 0f ? $" for {parameters.Duration:F1}s" : ""; - return new SimulateMouseInputResponse - { - Success = true, - Message = $"Clicked {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}){durationText}", - Action = MouseInputAction.Click.ToString(), - Button = buttonName, - PositionX = inputPos.x, - PositionY = inputPos.y, - InputCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, - UnityCoordinateSystem = McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW, - GameViewWidth = conversion.GameViewSize.x, - GameViewHeight = conversion.GameViewSize.y, - InputPositionX = conversion.InputPosition.x, - InputPositionY = conversion.InputPosition.y, - InjectedUnityPositionX = conversion.InjectedUnityPosition.x, - InjectedUnityPositionY = conversion.InjectedUnityPosition.y, - CoordinateConversionFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY - }; + return CreatePositionResponse( + $"Clicked {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}){durationText}", + MouseInputAction.Click, + buttonName, + inputPos, + conversion); } private async Task ExecuteLongPress( @@ -267,11 +255,26 @@ await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( SimulateMouseInputOverlayState.SetButtonHeld(button, false); } + return CreatePositionResponse( + $"Long-pressed {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s", + MouseInputAction.LongPress, + buttonName, + inputPos, + conversion); + } + + private static SimulateMouseInputResponse CreatePositionResponse( + string message, + MouseInputAction action, + string buttonName, + Vector2 inputPos, + GameViewCoordinateConversion conversion) + { return new SimulateMouseInputResponse { Success = true, - Message = $"Long-pressed {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s", - Action = MouseInputAction.LongPress.ToString(), + Message = message, + Action = action.ToString(), Button = buttonName, PositionX = inputPos.x, PositionY = inputPos.y, diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs index 02e2063e09..cdf3f5920c 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs @@ -7,10 +7,10 @@ public class SimulateMouseUiSchema : BaseToolSchema [Description("Mouse action: Click(0) - click with selected Button, Drag(1) - one-shot drag, DragStart(2) - begin drag and hold, DragMove(3) - move while holding drag, DragEnd(4) - release drag, LongPress(5) - press and hold for Duration seconds")] public MouseAction Action { get; set; } = MouseAction.Click; - [Description("Target X position in Game View pixels (origin: top-left). For Drag action, this is the destination.")] + [Description("Target X position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.")] public float X { get; set; } = 0f; - [Description("Target Y position in Game View pixels (origin: top-left). For Drag action, this is the destination.")] + [Description("Target Y position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.")] public float Y { get; set; } = 0f; [Description("Start X position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs index be6fcf30e9..a994511a62 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs @@ -11,7 +11,7 @@ namespace io.github.hatayama.uLoopMCP { - [McpTool(Description = "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem screen coordinates")] + [McpTool(Description = "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem using top-left Game View coordinates")] public class SimulateMouseUiTool : AbstractUnityTool { public override string ToolName => "simulate-mouse-ui"; diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md index 1117f479ae..451178c995 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md @@ -1,6 +1,6 @@ --- name: uloop-simulate-mouse-ui -description: "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." +description: "Simulate PlayMode EventSystem UI mouse actions using top-left Game View coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." context: fork --- @@ -28,8 +28,8 @@ uloop simulate-mouse-ui --action --x --y [options] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `--action` | enum | `Click` | `Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, `LongPress` | -| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). For Drag action, this is the destination. | -| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). For Drag action, this is the destination. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination. | | `--from-x` | number | `0` | Start X position in Game View pixels for Drag action. Drag starts here and moves to x,y. | | `--from-y` | number | `0` | Start Y position in Game View pixels for Drag action. Drag starts here and moves to x,y. | | `--drag-speed` | number | `2000` | Drag speed in pixels per second (0 for instant). 2000 is fast (default), 200 is slow enough to watch. Applies to Drag, DragMove, and DragEnd actions. | @@ -77,7 +77,7 @@ uloop simulate-mouse-ui --action --x --y [options] ## Examples ```bash -# Click a button at screen position +# Click a button at a Game View position uloop simulate-mouse-ui --action Click --x 400 --y 300 # Force-click a button behind a raycast blocker by path diff --git a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs index 6036cceaa9..a57b65c06b 100644 --- a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs @@ -138,7 +138,10 @@ public static string[] GetOpenWindowNames() // Captures game rendering by reading GameView's composited RenderTexture (PlayMode only). // Contains all cameras + Screen Space Overlay Canvas, without tab bar or borders. - public static async Task<(Texture2D? texture, int imageToInputOffsetY)> CaptureGameRenderingAsync(float resolutionScale, CancellationToken ct) + internal static async Task<(Texture2D? texture, GameRenderingImageInfo renderingImageInfo)> CaptureGameRenderingAsync( + float resolutionScale, + GameRenderingImageInfo? renderingImageInfo, + CancellationToken ct) { Debug.Assert(UnityEditor.EditorApplication.isPlaying, "CaptureGameRenderingAsync requires PlayMode"); @@ -149,11 +152,14 @@ public static string[] GetOpenWindowNames() if (rt == null) { Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); - return (null, 0); + GameRenderingImageInfo unavailableInfo = renderingImageInfo ?? + CreateGameRenderingImageInfo(Handles.GetMainGameViewSize(), 0, 0); + return (null, unavailableInfo); } // GameView RenderTexture can be shorter than the full input area, so raw image Y needs this offset. - int imageToInputOffsetY = CalculateImageToInputOffsetY(Handles.GetMainGameViewSize(), rt.height); + GameRenderingImageInfo captureInfo = renderingImageInfo ?? + CreateGameRenderingImageInfo(Handles.GetMainGameViewSize(), rt.width, rt.height); // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format. RenderTextureDescriptor flipDescriptor = new RenderTextureDescriptor(rt.width, rt.height, rt.format, 0); @@ -179,30 +185,37 @@ public static string[] GetOpenWindowNames() texture = ApplyResolutionScaling(texture, resolutionScale); } - return (texture, imageToInputOffsetY); + return (texture, captureInfo); } /// /// Gets rendering image dimensions after waiting for Game View rendering to settle. /// - internal static async Task<(Vector2 renderingImageSize, int imageToInputOffsetY)> GetGameRenderingImageInfoAsync( - Vector2 gameViewSize, - CancellationToken ct) + internal static async Task GetGameRenderingImageInfoAsync(CancellationToken ct) { Debug.Assert(UnityEditor.EditorApplication.isPlaying, "GetGameRenderingImageInfoAsync requires PlayMode"); // Raycast-grid annotations must use the same settled RenderTexture geometry as the PNG capture path. await EditorDelay.DelayFrame(2, ct); + Vector2 gameViewSize = Handles.GetMainGameViewSize(); RenderTexture rt = GameViewBridge.GetRenderTexture(); if (rt == null) { - return (gameViewSize, 0); + return new GameRenderingImageInfo(gameViewSize, gameViewSize, 0); } - Vector2 renderingImageSize = new Vector2(rt.width, rt.height); - int imageToInputOffsetY = CalculateImageToInputOffsetY(gameViewSize, rt.height); - return (renderingImageSize, imageToInputOffsetY); + return CreateGameRenderingImageInfo(gameViewSize, rt.width, rt.height); + } + + private static GameRenderingImageInfo CreateGameRenderingImageInfo( + Vector2 gameViewSize, + int renderTextureWidth, + int renderTextureHeight) + { + Vector2 renderingImageSize = new Vector2(renderTextureWidth, renderTextureHeight); + int imageToInputOffsetY = CalculateImageToInputOffsetY(gameViewSize, renderTextureHeight); + return new GameRenderingImageInfo(gameViewSize, renderingImageSize, imageToInputOffsetY); } /// @@ -237,4 +250,24 @@ private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float return scaledTexture; } } + + /// + /// Describes the Game View geometry used to map rendering screenshots back to input coordinates. + /// + internal readonly struct GameRenderingImageInfo + { + public readonly Vector2 GameViewSize; + public readonly Vector2 RenderingImageSize; + public readonly int ImageToInputOffsetY; + + public GameRenderingImageInfo( + Vector2 gameViewSize, + Vector2 renderingImageSize, + int imageToInputOffsetY) + { + GameViewSize = gameViewSize; + RenderingImageSize = renderingImageSize; + ImageToInputOffsetY = imageToInputOffsetY; + } + } } diff --git a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs index c2fa14557f..c20ed0322d 100644 --- a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs +++ b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs @@ -11,7 +11,8 @@ internal static class GameViewRaycastUtility internal static GameViewRaycastResult RaycastFromInputPosition( Vector2 inputPosition, float maxDistance, - int layerMask) + int layerMask, + bool syncTransforms) { Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); GameViewCoordinateConversion conversion = @@ -24,9 +25,14 @@ internal static GameViewRaycastResult RaycastFromInputPosition( } Ray ray = mainCamera.ScreenPointToRay(conversion.InjectedUnityPosition); - // Transform changes can be visible before the physics scene updates, so sync before screenshot-based queries. - Physics.SyncTransforms(); - RaycastHit[] hits = Physics.RaycastAll(ray, maxDistance, layerMask); + if (syncTransforms) + { + // Transform changes can be visible before the physics scene updates, so sync before screenshot-based queries. + Physics.SyncTransforms(); + } + + int visibleLayerMask = layerMask & mainCamera.cullingMask; + RaycastHit[] hits = Physics.RaycastAll(ray, maxDistance, visibleLayerMask); System.Array.Sort(hits, CompareHitsByDistance); return new GameViewRaycastResult(true, conversion, hits); diff --git a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs index 02203223a2..b2b021ca82 100644 --- a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs +++ b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs @@ -19,6 +19,8 @@ internal static List CollectRaycastGridPoints( { List points = new List(); int labelIndex = 1; + // Sync once for the whole grid; each candidate raycast then reads the same current physics state. + Physics.SyncTransforms(); for (int row = 1; row <= GRID_ROWS; row++) { @@ -32,7 +34,8 @@ internal static List CollectRaycastGridPoints( GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( inputPosition, McpConstants.RAYCAST_DEFAULT_MAX_DISTANCE, - Physics.DefaultRaycastLayers); + Physics.DefaultRaycastLayers, + false); points.Add(CreatePointInfo($"R{labelIndex}", inputPosition, raycastResult)); labelIndex++; diff --git a/Packages/src/README.md b/Packages/src/README.md index cfb3775a78..e615fd2b20 100644 --- a/Packages/src/README.md +++ b/Packages/src/README.md @@ -143,7 +143,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi
-All 16 Bundled Skills +All 17 Bundled Skills - `/uloop-launch` - Launch Unity with correct version - `/uloop-compile` - Execute compilation diff --git a/Packages/src/TOOL_REFERENCE.md b/Packages/src/TOOL_REFERENCE.md index 62c4f2e177..df4e87b255 100644 --- a/Packages/src/TOOL_REFERENCE.md +++ b/Packages/src/TOOL_REFERENCE.md @@ -217,7 +217,7 @@ All tools automatically include the following property: --- ### 11. simulate-mouse-ui -- **Description**: Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem screen coordinates. Uses EventSystem and ExecuteEvents to dispatch pointer events directly — works independently of both old and new Input System. For game logic that reads Input System (e.g. `Mouse.current.leftButton.wasPressedThisFrame`), use `simulate-mouse-input` instead +- **Description**: Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem using top-left Game View coordinates. Uses EventSystem and ExecuteEvents to dispatch pointer events directly — works independently of both old and new Input System. For game logic that reads Input System (e.g. `Mouse.current.leftButton.wasPressedThisFrame`), use `simulate-mouse-input` instead - **Parameters**: - `Action` (enum): Mouse action - "Click", "Drag", "DragStart", "DragMove", "DragEnd", "LongPress" (default: "Click") - `Click`: Click at (X, Y). Fires PointerDown → PointerUp → PointerClick @@ -226,10 +226,10 @@ All tools automatically include the following property: - `DragStart`: Begin drag at (X, Y) and hold - `DragMove`: Animate from current position to (X, Y) at the specified speed - `DragEnd`: Animate to (X, Y), then release drag - - `X` (number): Target X position in top-left Game View pixels (default: 0) - - `Y` (number): Target Y position in top-left Game View pixels (default: 0) - - `FromX` (number): Start X position for Drag action (default: 0) - - `FromY` (number): Start Y position for Drag action (default: 0) + - `X` (number): Target X position in top-left Game View pixels. Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination (default: 0) + - `FromX` (number): Start X position in top-left Game View pixels for Drag action (default: 0) + - `FromY` (number): Start Y position in top-left Game View pixels for Drag action (default: 0) - `DragSpeed` (number): Drag speed in pixels per second, 0 for instant (default: 2000) - `Duration` (number): Hold duration in seconds for LongPress action (default: 0.5) - `Button` (enum): Mouse button - "Left", "Right", "Middle" (default: "Left") diff --git a/README.md b/README.md index cfb3775a78..e615fd2b20 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi
-All 16 Bundled Skills +All 17 Bundled Skills - `/uloop-launch` - Launch Unity with correct version - `/uloop-compile` - Execute compilation From 17ae35b5fd2bcc7a99b4728383f8207305e3a124 Mon Sep 17 00:00:00 2001 From: hatayama Date: Sat, 4 Jul 2026 02:45:23 +0900 Subject: [PATCH 4/4] Fix unavailable rendering coordinate fallback Use the same Game View-sized fallback metadata when a rendering texture is unavailable so screenshot coordinate formulas do not report a full-height Y offset for a missing image. --- .../Tests/Editor/EditorWindowCaptureUtilityTests.cs | 13 +++++++++++++ .../src/Editor/Utils/EditorWindowCaptureUtility.cs | 9 +++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs index 8de1586087..fe5818528e 100644 --- a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs @@ -24,5 +24,18 @@ public void CalculateImageToInputOffsetY_WhenRenderTextureMatchesGameView_Should Assert.That(offsetY, Is.EqualTo(0)); } + + [Test] + public void CreateUnavailableGameRenderingImageInfo_WhenRenderTextureIsMissing_ShouldUseGameViewSizeAndZeroOffset() + { + Vector2 gameViewSize = new Vector2(1768f, 1383f); + + GameRenderingImageInfo renderingImageInfo = + EditorWindowCaptureUtility.CreateUnavailableGameRenderingImageInfo(gameViewSize); + + Assert.That(renderingImageInfo.GameViewSize, Is.EqualTo(gameViewSize)); + Assert.That(renderingImageInfo.RenderingImageSize, Is.EqualTo(gameViewSize)); + Assert.That(renderingImageInfo.ImageToInputOffsetY, Is.EqualTo(0)); + } } } diff --git a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs index a57b65c06b..99661af178 100644 --- a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs @@ -153,7 +153,7 @@ public static string[] GetOpenWindowNames() { Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); GameRenderingImageInfo unavailableInfo = renderingImageInfo ?? - CreateGameRenderingImageInfo(Handles.GetMainGameViewSize(), 0, 0); + CreateUnavailableGameRenderingImageInfo(Handles.GetMainGameViewSize()); return (null, unavailableInfo); } @@ -202,12 +202,17 @@ internal static async Task GetGameRenderingImageInfoAsyn RenderTexture rt = GameViewBridge.GetRenderTexture(); if (rt == null) { - return new GameRenderingImageInfo(gameViewSize, gameViewSize, 0); + return CreateUnavailableGameRenderingImageInfo(gameViewSize); } return CreateGameRenderingImageInfo(gameViewSize, rt.width, rt.height); } + internal static GameRenderingImageInfo CreateUnavailableGameRenderingImageInfo(Vector2 gameViewSize) + { + return new GameRenderingImageInfo(gameViewSize, gameViewSize, 0); + } + private static GameRenderingImageInfo CreateGameRenderingImageInfo( Vector2 gameViewSize, int renderTextureWidth,