diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs new file mode 100644 index 0000000000..fe5818528e --- /dev/null +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs @@ -0,0 +1,41 @@ +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class EditorWindowCaptureUtilityTests + { + [Test] + public void CalculateImageToInputOffsetY_WhenRenderTextureIsShorterThanGameView_ShouldReturnTopOffset() + { + Vector2 gameViewSize = new Vector2(1768f, 1383f); + + int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080); + + Assert.That(offsetY, Is.EqualTo(303)); + } + + [Test] + public void CalculateImageToInputOffsetY_WhenRenderTextureMatchesGameView_ShouldReturnZero() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + + int offsetY = EditorWindowCaptureUtility.CalculateImageToInputOffsetY(gameViewSize, 1080); + + Assert.That(offsetY, Is.EqualTo(0)); + } + + [Test] + public void CreateUnavailableGameRenderingImageInfo_WhenRenderTextureIsMissing_ShouldUseGameViewSizeAndZeroOffset() + { + Vector2 gameViewSize = new Vector2(1768f, 1383f); + + GameRenderingImageInfo renderingImageInfo = + EditorWindowCaptureUtility.CreateUnavailableGameRenderingImageInfo(gameViewSize); + + Assert.That(renderingImageInfo.GameViewSize, Is.EqualTo(gameViewSize)); + Assert.That(renderingImageInfo.RenderingImageSize, Is.EqualTo(gameViewSize)); + Assert.That(renderingImageInfo.ImageToInputOffsetY, Is.EqualTo(0)); + } + } +} diff --git a/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta new file mode 100644 index 0000000000..3b34562bc9 --- /dev/null +++ b/Assets/Tests/Editor/EditorWindowCaptureUtilityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 54fcac766c4c54beab3ca57f04ef6425 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs new file mode 100644 index 0000000000..63e300cc32 --- /dev/null +++ b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs @@ -0,0 +1,46 @@ +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class GameViewCoordinateUtilityTests + { + [Test] + public void ConvertInputToUnity_WhenInputIsNearTop_ShouldFlipYWithinGameView() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + Vector2 inputPosition = new Vector2(0f, 100f); + + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.That(conversion.InputPosition, Is.EqualTo(inputPosition)); + Assert.That(conversion.InjectedUnityPosition, Is.EqualTo(new Vector2(0f, 980f))); + Assert.That(conversion.GameViewSize, Is.EqualTo(gameViewSize)); + } + + [Test] + public void ConvertInputToUnity_WhenInputIsAtCenter_ShouldKeepCenterY() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + Vector2 inputPosition = new Vector2(960f, 540f); + + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.That(conversion.InjectedUnityPosition, Is.EqualTo(new Vector2(960f, 540f))); + } + + [Test] + public void ConvertInputToUnity_WhenInputIsAtBottomRight_ShouldMapToBottomLeftOrigin() + { + Vector2 gameViewSize = new Vector2(1920f, 1080f); + Vector2 inputPosition = new Vector2(1920f, 1080f); + + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.That(conversion.InjectedUnityPosition, Is.EqualTo(new Vector2(1920f, 0f))); + } + } +} diff --git a/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta new file mode 100644 index 0000000000..ef7653e66a --- /dev/null +++ b/Assets/Tests/Editor/GameViewCoordinateUtilityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9f518426086f04f21809705562fc5fce +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs new file mode 100644 index 0000000000..b527a0b488 --- /dev/null +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs @@ -0,0 +1,69 @@ +using System.Collections.Generic; +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class RaycastGridAnnotatorTests + { + [Test] + public void CalculateGridInputPosition_WhenRenderingHasTopOffset_ShouldSampleInsideCapturedImage() + { + Vector2 renderingImageSize = new Vector2(1200f, 1080f); + + Vector2 inputPosition = + RaycastGridAnnotator.CalculateGridInputPosition(renderingImageSize, 303, 1, 3); + + Assert.That(inputPosition.x, Is.EqualTo(600f)); + Assert.That(inputPosition.y, Is.EqualTo(483f)); + } + + [Test] + public void CalculateGridInputPosition_WhenRenderingHasTopOffset_ShouldKeepBottomRowVisible() + { + Vector2 renderingImageSize = new Vector2(1200f, 1080f); + + Vector2 inputPosition = + RaycastGridAnnotator.CalculateGridInputPosition(renderingImageSize, 303, 5, 3); + + Assert.That(inputPosition.x, Is.EqualTo(600f)); + Assert.That(inputPosition.y, Is.EqualTo(1203f)); + } + + [Test] + public void CreateOverlayElements_WhenPointsIncludeMisses_ShouldAnnotateOnlyHits() + { + List points = new List + { + new RaycastGridPointInfo + { + Label = "R1", + Hit = true, + InputX = 100f, + InputY = 200f, + InjectedUnityPositionX = 100f, + InjectedUnityPositionY = 880f, + HitGameObjectName = "Cube", + HitGameObjectPath = "Cube" + }, + new RaycastGridPointInfo + { + Label = "R2", + Hit = false, + InputX = 200f, + InputY = 300f, + InjectedUnityPositionX = 200f, + InjectedUnityPositionY = 780f + } + }; + + List overlayElements = RaycastGridAnnotator.CreateOverlayElements(points); + + Assert.That(overlayElements.Count, Is.EqualTo(1)); + Assert.That(overlayElements[0].Label, Is.EqualTo("R1")); + Assert.That(overlayElements[0].Name, Is.EqualTo("Cube")); + Assert.That(overlayElements[0].Type, Is.EqualTo("RaycastHit")); + Assert.That(overlayElements[0].Interaction, Is.EqualTo("Raycast")); + } + } +} diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta new file mode 100644 index 0000000000..925444cce6 --- /dev/null +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 011b5cfbe4d944972ae894d38207e984 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/RaycastToolTests.cs b/Assets/Tests/Editor/RaycastToolTests.cs new file mode 100644 index 0000000000..c9f9d8c06e --- /dev/null +++ b/Assets/Tests/Editor/RaycastToolTests.cs @@ -0,0 +1,190 @@ +#nullable enable +using System.Collections.Generic; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP.Tests.Editor +{ + public class RaycastToolTests + { + private GameObject? _cameraObject; + private GameObject? _cubeObject; + private bool _originalAutoSyncTransforms; + private readonly List _retaggedMainCameraObjects = new List(); + + [SetUp] + public void SetUp() + { + _originalAutoSyncTransforms = Physics.autoSyncTransforms; + } + + [TearDown] + public void TearDown() + { + Physics.autoSyncTransforms = _originalAutoSyncTransforms; + + if (_cubeObject != null) + { + Object.DestroyImmediate(_cubeObject); + } + + if (_cameraObject != null) + { + Object.DestroyImmediate(_cameraObject); + } + + foreach (GameObject retaggedObject in _retaggedMainCameraObjects) + { + if (retaggedObject != null) + { + retaggedObject.tag = "MainCamera"; + } + } + + _retaggedMainCameraObjects.Clear(); + _cubeObject = null; + _cameraObject = null; + } + + [Test] + public async Task ExecuteAsync_WhenCoordinateIntersectsCollider_ShouldReturnHitAndConversionMetadata() + { + CreateRaycastScene(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.True); + Assert.That(response.HitGameObjectName, Is.EqualTo("RaycastToolTestsCube")); + Assert.That(response.InputCoordinateSystem, Is.EqualTo(McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW)); + Assert.That(response.UnityCoordinateSystem, Is.EqualTo(McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW)); + Assert.That(response.CoordinateConversionFormula, Is.EqualTo(McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY)); + Assert.That(response.InputPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InputPositionY, Is.EqualTo(inputPosition.y)); + Assert.That(response.InjectedUnityPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InjectedUnityPositionY, Is.EqualTo(gameViewSize.y - inputPosition.y)); + } + + [Test] + public async Task ExecuteAsync_WhenCoordinateMissesCollider_ShouldReturnNoHit() + { + CreateRaycastScene(); + Vector2 inputPosition = new Vector2(0f, 0f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.False); + Assert.That(response.HitGameObjectName, Is.Null); + } + + [Test] + public async Task ExecuteAsync_WhenCameraIsMissing_ShouldReturnConversionMetadata() + { + RetagExistingMainCameras(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.False); + Assert.That(response.Message, Does.Contain("Camera.main")); + Assert.That(response.InputPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InputPositionY, Is.EqualTo(inputPosition.y)); + Assert.That(response.InjectedUnityPositionX, Is.EqualTo(inputPosition.x)); + Assert.That(response.InjectedUnityPositionY, Is.EqualTo(gameViewSize.y - inputPosition.y)); + Assert.That(response.CoordinateConversionFormula, Is.EqualTo(McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY)); + } + + [Test] + public async Task ExecuteAsync_WhenColliderLayerIsHiddenByCamera_ShouldReturnNoHit() + { + CreateRaycastScene(); + if (_cameraObject == null) + { + Assert.Fail("Camera should be created."); + return; + } + + Camera camera = _cameraObject.GetComponent(); + camera.cullingMask = 0; + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.False); + } + + [Test] + public async Task ExecuteAsync_WhenAutoSyncTransformsIsDisabled_ShouldRaycastAgainstLatestTransform() + { + Physics.autoSyncTransforms = false; + CreateRaycastScene(); + if (_cubeObject == null) + { + Assert.Fail("Cube should be created."); + return; + } + + _cubeObject.transform.position = new Vector3(100f, 0f, 0f); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + Vector2 inputPosition = new Vector2(gameViewSize.x / 2f, gameViewSize.y / 2f); + + RaycastResponse response = await ExecuteRaycast(inputPosition); + + Assert.That(response.Success, Is.True); + Assert.That(response.Hit, Is.False); + } + + private void CreateRaycastScene() + { + RetagExistingMainCameras(); + + _cameraObject = new GameObject("RaycastToolTestsCamera"); + Camera camera = _cameraObject.AddComponent(); + _cameraObject.tag = "MainCamera"; + _cameraObject.transform.position = new Vector3(0f, 0f, -10f); + _cameraObject.transform.rotation = Quaternion.identity; + camera.nearClipPlane = 0.1f; + camera.farClipPlane = 100f; + + _cubeObject = GameObject.CreatePrimitive(PrimitiveType.Cube); + _cubeObject.name = "RaycastToolTestsCube"; + _cubeObject.transform.position = Vector3.zero; + } + + private void RetagExistingMainCameras() + { + Camera[] cameras = Object.FindObjectsByType(FindObjectsSortMode.None); + foreach (Camera camera in cameras) + { + if (!camera.CompareTag("MainCamera")) + { + continue; + } + + _retaggedMainCameraObjects.Add(camera.gameObject); + camera.gameObject.tag = "Untagged"; + } + } + + private static async Task ExecuteRaycast(Vector2 inputPosition) + { + RaycastTool tool = new RaycastTool(); + JObject parameters = new JObject + { + ["x"] = inputPosition.x, + ["y"] = inputPosition.y + }; + + BaseToolResponse baseResponse = await tool.ExecuteAsync(parameters); + return (RaycastResponse)baseResponse; + } + } +} diff --git a/Assets/Tests/Editor/RaycastToolTests.cs.meta b/Assets/Tests/Editor/RaycastToolTests.cs.meta new file mode 100644 index 0000000000..5cc08851f2 --- /dev/null +++ b/Assets/Tests/Editor/RaycastToolTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4ac05679db3384836aee71b74cca6777 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/PlayMode/SimulateMouseInputTests.cs b/Assets/Tests/PlayMode/SimulateMouseInputTests.cs index 7a3e94310f..60dcd1ee5b 100644 --- a/Assets/Tests/PlayMode/SimulateMouseInputTests.cs +++ b/Assets/Tests/PlayMode/SimulateMouseInputTests.cs @@ -47,6 +47,7 @@ public IEnumerator Click_Should_SetWasPressedThisFrame() Assert.IsTrue(lastResponse.Success); Assert.AreEqual("Click", lastResponse.Action); Assert.AreEqual("Left", lastResponse.Button); + AssertCoordinateMetadata(400f, 300f); // After click completes, button should be released Assert.IsFalse(mouse.leftButton.isPressed, "Left button should be released after click"); } @@ -106,6 +107,7 @@ public IEnumerator LongPress_Should_HoldButtonForDuration() Assert.IsTrue(lastResponse.Success); Assert.AreEqual("LongPress", lastResponse.Action); + AssertCoordinateMetadata(400f, 300f); // After long press completes, button should be released Assert.IsFalse(mouse.leftButton.isPressed, "Button should be released after long press"); } @@ -194,6 +196,30 @@ private IEnumerator RunTool(JObject parameters) lastResponse = (SimulateMouseInputResponse)task.Result; } + private void AssertCoordinateMetadata(float inputX, float inputY) + { + Vector2 inputPosition = new Vector2(inputX, inputY); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + GameViewCoordinateConversion expected = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Assert.AreEqual("top-left-game-view", lastResponse.InputCoordinateSystem); + Assert.AreEqual("bottom-left-game-view", lastResponse.UnityCoordinateSystem); + Assert.AreEqual("unity_x = input_x; unity_y = gameViewHeight - input_y", lastResponse.CoordinateConversionFormula); + Assert.IsTrue(lastResponse.GameViewWidth.HasValue); + Assert.IsTrue(lastResponse.GameViewHeight.HasValue); + Assert.IsTrue(lastResponse.InputPositionX.HasValue); + Assert.IsTrue(lastResponse.InputPositionY.HasValue); + Assert.IsTrue(lastResponse.InjectedUnityPositionX.HasValue); + Assert.IsTrue(lastResponse.InjectedUnityPositionY.HasValue); + Assert.AreEqual(expected.GameViewSize.x, lastResponse.GameViewWidth!.Value); + Assert.AreEqual(expected.GameViewSize.y, lastResponse.GameViewHeight!.Value); + Assert.AreEqual(expected.InputPosition.x, lastResponse.InputPositionX!.Value); + Assert.AreEqual(expected.InputPosition.y, lastResponse.InputPositionY!.Value); + Assert.AreEqual(expected.InjectedUnityPosition.x, lastResponse.InjectedUnityPositionX!.Value); + Assert.AreEqual(expected.InjectedUnityPosition.y, lastResponse.InjectedUnityPositionY!.Value); + } + #endregion } } diff --git a/Packages/packages-lock.json b/Packages/packages-lock.json index e872ac6ee7..927b5ed879 100644 --- a/Packages/packages-lock.json +++ b/Packages/packages-lock.json @@ -234,6 +234,7 @@ "depth": 0, "source": "embedded", "dependencies": { + "com.unity.modules.physics": "1.0.0", "com.unity.ugui": "1.0.0", "com.unity.nuget.newtonsoft-json": "3.2.1" } @@ -285,7 +286,7 @@ }, "com.unity.modules.physics": { "version": "1.0.0", - "depth": 2, + "depth": 1, "source": "builtin", "dependencies": {} }, diff --git a/Packages/src/Cli~/src/default-tools.json b/Packages/src/Cli~/src/default-tools.json index 733efbee9b..c8aa0b481d 100644 --- a/Packages/src/Cli~/src/default-tools.json +++ b/Packages/src/Cli~/src/default-tools.json @@ -247,7 +247,7 @@ }, "CaptureMode": { "type": "string", - "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Response includes CoordinateSystem ('gameView' or 'window'), ResolutionScale, and YOffset. For gameView: sim_x = image_x / ResolutionScale, sim_y = image_y / ResolutionScale + YOffset.", + "description": "Capture mode: window=capture EditorWindow including toolbar, rendering=capture game rendering only (PlayMode required). Rendering screenshots return ScreenshotToInputFormula for converting raw image pixels before calling simulate-mouse-input or raycast.", "enum": [ "window", "rendering" @@ -263,6 +263,40 @@ "type": "boolean", "description": "Return only annotated element JSON without capturing a screenshot image. Requires AnnotateElements=true and CaptureMode=rendering in PlayMode.", "default": false + }, + "AnnotateRaycastGrid": { + "type": "boolean", + "description": "Annotate 3D physics raycast candidate points on rendering screenshots. Uses Camera.main and the same top-left Game View coordinates as simulate-mouse-input.", + "default": false + } + } + } + }, + { + "name": "raycast", + "description": "Raycast from Camera.main through a top-left Game View coordinate. Use annotated SimX/SimY, raycast-grid InputX/InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.", + "inputSchema": { + "type": "object", + "properties": { + "X": { + "type": "number", + "description": "Target X position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw image pixels converted with ScreenshotToInputFormula.", + "default": 0 + }, + "Y": { + "type": "number", + "description": "Target Y position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw image pixels converted with ScreenshotToInputFormula.", + "default": 0 + }, + "LayerMask": { + "type": "number", + "description": "Physics layer mask used by the raycast.", + "default": -5 + }, + "MaxDistance": { + "type": "number", + "description": "Maximum raycast distance in world units.", + "default": 1000 } } } @@ -329,7 +363,7 @@ }, { "name": "simulate-mouse-ui", - "description": "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem screen coordinates", + "description": "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem using top-left Game View coordinates", "inputSchema": { "type": "object", "properties": { @@ -348,22 +382,22 @@ }, "X": { "type": "number", - "description": "Target X position in screen pixels (origin: top-left). For Drag action, this is the destination.", + "description": "Target X position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination.", + "description": "Target Y position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.", "default": 0 }, "FromX": { "type": "number", - "description": "Start X position for Drag action (origin: top-left). Drag starts here and moves to X,Y.", + "description": "Start X position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.", "default": 0 }, "FromY": { "type": "number", - "description": "Start Y position for Drag action (origin: top-left). Drag starts here and moves to X,Y.", + "description": "Start Y position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.", "default": 0 }, "DragSpeed": { @@ -424,12 +458,12 @@ }, "X": { "type": "number", - "description": "Target X position in screen pixels (origin: top-left). Used by Click and LongPress.", + "description": "Target X position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw image pixels converted with ScreenshotToInputFormula.", "default": 0 }, "Y": { "type": "number", - "description": "Target Y position in screen pixels (origin: top-left). Used by Click and LongPress.", + "description": "Target Y position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw image pixels converted with ScreenshotToInputFormula.", "default": 0 }, "Button": { diff --git a/Packages/src/Editor/Api/McpTools/Raycast.meta b/Packages/src/Editor/Api/McpTools/Raycast.meta new file mode 100644 index 0000000000..3e9374980b --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: d7f308f6972f3483793d282829a51021 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs new file mode 100644 index 0000000000..a0ebc42ab2 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs @@ -0,0 +1,35 @@ +#nullable enable + +namespace io.github.hatayama.uLoopMCP +{ + public class RaycastResponse : BaseToolResponse + { + public bool Success { get; set; } + public string Message { get; set; } = ""; + public bool Hit { get; set; } + public string? HitGameObjectName { get; set; } + public string? HitGameObjectPath { get; set; } + public int? HitLayer { get; set; } + public string? HitLayerName { get; set; } + public float? Distance { get; set; } + public float? HitPointX { get; set; } + public float? HitPointY { get; set; } + public float? HitPointZ { get; set; } + public float? HitNormalX { get; set; } + public float? HitNormalY { get; set; } + public float? HitNormalZ { get; set; } + public string InputCoordinateSystem { get; set; } = ""; + public string UnityCoordinateSystem { get; set; } = ""; + public float GameViewWidth { get; set; } + public float GameViewHeight { get; set; } + public float InputPositionX { get; set; } + public float InputPositionY { get; set; } + public float InjectedUnityPositionX { get; set; } + public float InjectedUnityPositionY { get; set; } + public string CoordinateConversionFormula { get; set; } = ""; + + public RaycastResponse() + { + } + } +} diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta new file mode 100644 index 0000000000..83306e8c7f --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastResponse.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 7a2c44181637c4be8a1ca98f2d0891a4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs new file mode 100644 index 0000000000..c26d15bc95 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs @@ -0,0 +1,20 @@ +using System.ComponentModel; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + public class RaycastSchema : BaseToolSchema + { + [Description("Target X position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] + public float X { get; set; } = 0f; + + [Description("Target Y position in Game View pixels (origin: top-left). Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] + public float Y { get; set; } = 0f; + + [Description("Physics layer mask used by the raycast.")] + public int LayerMask { get; set; } = Physics.DefaultRaycastLayers; + + [Description("Maximum raycast distance in world units.")] + public float MaxDistance { get; set; } = McpConstants.RAYCAST_DEFAULT_MAX_DISTANCE; + } +} diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta new file mode 100644 index 0000000000..639c71a2d0 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastSchema.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 19cfc535ad76043ba912a44b081c9f5f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs new file mode 100644 index 0000000000..f0647b9610 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs @@ -0,0 +1,88 @@ +#nullable enable +using System.Threading; +using System.Threading.Tasks; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Checks what a screenshot-compatible Game View coordinate hits in 3D physics. + /// + [McpTool(Description = "Raycast from Camera.main through a top-left Game View coordinate. Use annotated SimX/SimY, raycast-grid InputX/InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] + public class RaycastTool : AbstractUnityTool + { + public override string ToolName => "raycast"; + + protected override Task ExecuteAsync(RaycastSchema parameters, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + if (parameters.MaxDistance <= 0f || float.IsNaN(parameters.MaxDistance) || float.IsInfinity(parameters.MaxDistance)) + { + return Task.FromResult(new RaycastResponse + { + Success = false, + Message = $"MaxDistance must be positive and finite, got: {parameters.MaxDistance}" + }); + } + + Vector2 inputPosition = new Vector2(parameters.X, parameters.Y); + GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( + inputPosition, + parameters.MaxDistance, + parameters.LayerMask, + true); + + if (!raycastResult.CameraFound) + { + RaycastResponse noCameraResponse = CreateBaseResponse(raycastResult.Conversion); + noCameraResponse.Success = false; + noCameraResponse.Message = "Camera.main was not found. Add an active camera tagged MainCamera before using raycast."; + return Task.FromResult(noCameraResponse); + } + + if (raycastResult.Hits.Length == 0) + { + RaycastResponse noHitResponse = CreateBaseResponse(raycastResult.Conversion); + noHitResponse.Success = true; + noHitResponse.Hit = false; + noHitResponse.Message = $"No physics hit at ({inputPosition.x:F1}, {inputPosition.y:F1})."; + return Task.FromResult(noHitResponse); + } + + RaycastHit nearestHit = raycastResult.Hits[0]; + RaycastResponse response = CreateBaseResponse(raycastResult.Conversion); + response.Success = true; + response.Hit = true; + response.Message = $"Hit {nearestHit.collider.gameObject.name} at ({inputPosition.x:F1}, {inputPosition.y:F1})."; + response.HitGameObjectName = nearestHit.collider.gameObject.name; + response.HitGameObjectPath = GameObjectPathUtility.GetFullPath(nearestHit.collider.gameObject); + response.HitLayer = nearestHit.collider.gameObject.layer; + response.HitLayerName = LayerMask.LayerToName(nearestHit.collider.gameObject.layer); + response.Distance = nearestHit.distance; + response.HitPointX = nearestHit.point.x; + response.HitPointY = nearestHit.point.y; + response.HitPointZ = nearestHit.point.z; + response.HitNormalX = nearestHit.normal.x; + response.HitNormalY = nearestHit.normal.y; + response.HitNormalZ = nearestHit.normal.z; + return Task.FromResult(response); + } + + private static RaycastResponse CreateBaseResponse(GameViewCoordinateConversion conversion) + { + return new RaycastResponse + { + InputCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, + UnityCoordinateSystem = McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW, + GameViewWidth = conversion.GameViewSize.x, + GameViewHeight = conversion.GameViewSize.y, + InputPositionX = conversion.InputPosition.x, + InputPositionY = conversion.InputPosition.y, + InjectedUnityPositionX = conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = conversion.InjectedUnityPosition.y, + CoordinateConversionFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY + }; + } + } +} diff --git a/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta new file mode 100644 index 0000000000..fc957a6f53 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/RaycastTool.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 74b2bf9d5a00843eda24add2c109851a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill.meta b/Packages/src/Editor/Api/McpTools/Raycast/Skill.meta new file mode 100644 index 0000000000..bb72109c41 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 8928b15eb1b67433d8bc4befaaecb9f2 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md new file mode 100644 index 0000000000..b464ae09c3 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md @@ -0,0 +1,61 @@ +--- +name: uloop-raycast +description: "Raycast from Camera.main through a Game View coordinate. Use when you need to check what a screenshot coordinate would hit in 3D physics before clicking or long-pressing with simulate-mouse-input." +--- + +# uloop raycast + +Check what a top-left Game View coordinate hits in 3D physics. + +## Usage + +```bash +uloop raycast --x --y [--layer-mask ] [--max-distance ] +``` + +## Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula`. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula`. | +| `--layer-mask` | number | Unity default raycast layers | Physics layer mask used by the raycast. | +| `--max-distance` | number | `1000` | Maximum raycast distance in world units. | + +## Coordinate System + +- `--x` / `--y` use the same top-left Game View input coordinates as `simulate-mouse-input`. +- Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. +- `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` can be passed directly to this tool. +- Do not flip Y in the caller. The tool converts internally: + +```text +unity_x = input_x +unity_y = gameViewHeight - input_y +``` + +## Examples + +```bash +# Check what is under a screenshot coordinate +uloop raycast --x 960 --y 540 + +# Check only specific layers +uloop raycast --x 960 --y 540 --layer-mask 1 +``` + +## Output + +Returns JSON with: +- `Success`: Whether the command completed +- `Message`: Status message +- `Hit`: Whether physics hit anything +- `HitGameObjectName` / `HitGameObjectPath`: Hit object identity when `Hit` is true +- `HitLayer` / `HitLayerName`: Hit object layer when `Hit` is true +- `Distance`, `HitPointX/Y/Z`, `HitNormalX/Y/Z`: Hit details when `Hit` is true +- `InputCoordinateSystem`, `UnityCoordinateSystem`, `GameViewWidth/Height`, `InputPositionX/Y`, `InjectedUnityPositionX/Y`, `CoordinateConversionFormula`: Coordinate conversion details + +## Notes + +- Requires an active `Camera.main`. +- Uses Unity Physics raycasts, not UI EventSystem raycasts. diff --git a/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta new file mode 100644 index 0000000000..27cb3aa4e0 --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Raycast/Skill/SKILL.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 481bce4e5b2b344f7a0dde4afb6885d6 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs new file mode 100644 index 0000000000..f7d73ff09e --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace io.github.hatayama.uLoopMCP +{ + public class RaycastGridPointInfo + { + public string Label { get; set; } = ""; + public bool Hit { get; set; } + public float InputX { get; set; } + public float InputY { get; set; } + public float InjectedUnityPositionX { get; set; } + public float InjectedUnityPositionY { get; set; } + public string? HitGameObjectName { get; set; } + public string? HitGameObjectPath { get; set; } + public float? Distance { get; set; } + } +} diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta new file mode 100644 index 0000000000..280a6630ec --- /dev/null +++ b/Packages/src/Editor/Api/McpTools/Screenshot/RaycastGridPointInfo.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5ebf16218669642cb9cf689cd2f68430 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs index efb709ea44..1145d6db9c 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotResponse.cs @@ -10,32 +10,26 @@ public class ScreenshotInfo public long FileSizeBytes { get; set; } public int Width { get; set; } public int Height { get; set; } - - // "gameView": image from game RenderTexture. Convert to simulate-mouse coords: - // sim_x = image_x / ResolutionScale - // sim_y = image_y / ResolutionScale + YOffset - // "window": EditorWindow capture including toolbar - public string CoordinateSystem { get; set; } = McpConstants.COORDINATE_SYSTEM_WINDOW; - + public string ImageCoordinateSystem { get; set; } = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; public float ResolutionScale { get; set; } = 1.0f; - - // Y offset to add to image pixel Y to get simulate-mouse Y coordinate. - // Only meaningful when CoordinateSystem == "gameView". - public int YOffset { get; set; } - + public int ImageToInputOffsetY { get; set; } + public float GameViewWidth { get; set; } + public float GameViewHeight { get; set; } + public string ScreenshotToInputFormula { get; set; } = McpConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; + public string UnityInputFormula { get; set; } = ""; public List AnnotatedElements { get; set; } = new(); + public List RaycastGridPoints { get; set; } = new(); public ScreenshotInfo(string imagePath, long fileSizeBytes, int width, int height, - string coordinateSystem = McpConstants.COORDINATE_SYSTEM_WINDOW, - float resolutionScale = 1.0f, int yOffset = 0) + string imageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW, + float resolutionScale = 1.0f) { ImagePath = imagePath; FileSizeBytes = fileSizeBytes; Width = width; Height = height; - CoordinateSystem = coordinateSystem; + ImageCoordinateSystem = imageCoordinateSystem; ResolutionScale = resolutionScale; - YOffset = yOffset; } public ScreenshotInfo() diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs index a8f9deeceb..0f969e94fc 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotSchema.cs @@ -43,7 +43,7 @@ public class ScreenshotSchema : BaseToolSchema [Description("Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths.")] public string OutputDirectory { get; set; } = ""; - [Description("Capture mode: window(0)=capture EditorWindow including toolbar, rendering(1)=capture game rendering only (PlayMode required, coordinates match simulate-mouse)")] + [Description("Capture mode: window(0)=capture EditorWindow including toolbar, rendering(1)=capture game rendering only (PlayMode required, returns ScreenshotToInputFormula for raw image pixels)")] public CaptureMode CaptureMode { get; set; } = CaptureMode.window; [Description("Annotate interactive UI elements with names and simulate-mouse coordinates on the screenshot. Only works with CaptureMode=rendering in PlayMode.")] @@ -51,5 +51,8 @@ public class ScreenshotSchema : BaseToolSchema [Description("Return only annotated element JSON without capturing a screenshot image. Requires AnnotateElements=true and CaptureMode=rendering in PlayMode.")] public bool ElementsOnly { get; set; } = false; + + [Description("Annotate 3D physics raycast candidate points on rendering screenshots. Uses Camera.main and the same top-left Game View coordinates as simulate-mouse-input.")] + public bool AnnotateRaycastGrid { get; set; } = false; } } diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs index b6969b3b05..9804cf37ae 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs +++ b/Packages/src/Editor/Api/McpTools/Screenshot/ScreenshotTool.cs @@ -56,6 +56,10 @@ private async Task CaptureRenderingAsync( } List annotatedElements = new List(); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + List raycastGridPoints = new List(); + List raycastGridOverlayElements = new List(); + GameRenderingImageInfo? raycastGridRenderingInfo = null; if (parameters.AnnotateElements) { @@ -63,39 +67,58 @@ private async Task CaptureRenderingAsync( UIElementAnnotator.AssignLabels(annotatedElements); } + if (parameters.AnnotateRaycastGrid) + { + GameRenderingImageInfo renderingImageInfo = + await EditorWindowCaptureUtility.GetGameRenderingImageInfoAsync(ct); + raycastGridRenderingInfo = renderingImageInfo; + gameViewSize = renderingImageInfo.GameViewSize; + raycastGridPoints = RaycastGridAnnotator.CollectRaycastGridPoints( + renderingImageInfo.RenderingImageSize, + renderingImageInfo.ImageToInputOffsetY); + raycastGridOverlayElements = RaycastGridAnnotator.CreateOverlayElements(raycastGridPoints); + } + if (parameters.ElementsOnly) { - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y); + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); ScreenshotInfo elementsOnlyInfo = new ScreenshotInfo(); - elementsOnlyInfo.CoordinateSystem = McpConstants.COORDINATE_SYSTEM_GAME_VIEW; + elementsOnlyInfo.ResolutionScale = parameters.ResolutionScale; + int imageToInputOffsetY = raycastGridRenderingInfo?.ImageToInputOffsetY ?? 0; + ApplyRenderingCoordinateMetadata(elementsOnlyInfo, gameViewSize, imageToInputOffsetY); elementsOnlyInfo.AnnotatedElements = annotatedElements; + elementsOnlyInfo.RaycastGridPoints = raycastGridPoints; return new ScreenshotResponse(new List { elementsOnlyInfo }); } GameObject annotationOverlay = null; Texture2D texture; - int yOffset; + GameRenderingImageInfo captureRenderingInfo; try { - if (parameters.AnnotateElements) + if (parameters.AnnotateElements || parameters.AnnotateRaycastGrid) { + List overlayElements = new List(annotatedElements); + overlayElements.AddRange(raycastGridOverlayElements); annotationOverlay = UIElementAnnotator.CreateAnnotationOverlay( - annotatedElements, + overlayElements, parameters.ResolutionScale); Canvas.ForceUpdateCanvases(); // Chained CLI calls can read the previous GameView RT before overlay rendering catches up. await EditorDelay.DelayFrame(ANNOTATION_OVERLAY_RENDER_WAIT_FRAMES, ct); } - (texture, yOffset) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( - parameters.ResolutionScale, ct); + (texture, captureRenderingInfo) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync( + parameters.ResolutionScale, + raycastGridRenderingInfo, + ct); } finally { UIElementAnnotator.DestroyAnnotationOverlay(annotationOverlay); } - UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y); + UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, Mathf.RoundToInt(gameViewSize.y)); if (texture == null) { @@ -122,8 +145,13 @@ private async Task CaptureRenderingAsync( FileInfo savedFileInfo = new FileInfo(savedPath); ScreenshotInfo info = new ScreenshotInfo( savedPath, savedFileInfo.Length, width, height, - McpConstants.COORDINATE_SYSTEM_GAME_VIEW, parameters.ResolutionScale, yOffset); + McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, parameters.ResolutionScale); + ApplyRenderingCoordinateMetadata( + info, + captureRenderingInfo.GameViewSize, + captureRenderingInfo.ImageToInputOffsetY); info.AnnotatedElements = annotatedElements; + info.RaycastGridPoints = raycastGridPoints; screenshots.Add(info); } catch (Exception ex) @@ -199,7 +227,15 @@ private async Task CaptureWindowsAsync( SaveTextureAsPng(texture, savedPath); FileInfo savedFileInfo = new FileInfo(savedPath); - screenshots.Add(new ScreenshotInfo(savedPath, savedFileInfo.Length, width, height)); + ScreenshotInfo info = new ScreenshotInfo( + savedPath, + savedFileInfo.Length, + width, + height, + McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW, + parameters.ResolutionScale); + ApplyWindowCoordinateMetadata(info); + screenshots.Add(info); } catch (Exception ex) { @@ -226,6 +262,26 @@ private async Task CaptureWindowsAsync( return new ScreenshotResponse(screenshots); } + private static void ApplyRenderingCoordinateMetadata( + ScreenshotInfo info, + Vector2 gameViewSize, + int imageToInputOffsetY = 0) + { + info.ImageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW; + info.GameViewWidth = gameViewSize.x; + info.GameViewHeight = gameViewSize.y; + info.ImageToInputOffsetY = imageToInputOffsetY; + info.ScreenshotToInputFormula = McpConstants.SCREENSHOT_RENDERING_TO_INPUT_FORMULA; + info.UnityInputFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY; + } + + private static void ApplyWindowCoordinateMetadata(ScreenshotInfo info) + { + info.ImageCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_WINDOW; + info.ScreenshotToInputFormula = McpConstants.SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE; + info.UnityInputFormula = ""; + } + private void ValidateParameters(ScreenshotSchema parameters) { if (parameters.CaptureMode != CaptureMode.rendering && @@ -252,6 +308,11 @@ private void ValidateParameters(ScreenshotSchema parameters) { throw new ParameterValidationException("ElementsOnly is only supported when CaptureMode=rendering"); } + + if (parameters.AnnotateRaycastGrid) + { + throw new ParameterValidationException("AnnotateRaycastGrid is only supported when CaptureMode=rendering"); + } } if (parameters.ElementsOnly && !parameters.AnnotateElements) diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md index b5fa55378c..ff9f5c333b 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/SKILL.md @@ -10,7 +10,7 @@ Take a screenshot of any Unity EditorWindow by name and save as PNG. ## Usage ```bash -uloop screenshot [--window-name ] [--resolution-scale ] [--match-mode ] [--capture-mode ] [--annotate-elements] [--elements-only] [--output-directory ] +uloop screenshot [--window-name ] [--resolution-scale ] [--match-mode ] [--capture-mode ] [--annotate-elements] [--annotate-raycast-grid] [--elements-only] [--output-directory ] ``` ## Parameters @@ -23,6 +23,7 @@ uloop screenshot [--window-name ] [--resolution-scale ] [--match-mo | `--capture-mode` | enum | `window` | `window`=capture EditorWindow including toolbar, `rendering`=capture game rendering only (PlayMode required, coordinates match simulate-mouse) | | `--output-directory` | string | `""` | Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths. | | `--annotate-elements` | boolean | `false` | Annotate interactive UI elements with index labels and interaction hints (A / CLICK, B / DRAG, ...). Only works with `--capture-mode rendering` in PlayMode. | +| `--annotate-raycast-grid` | boolean | `false` | Annotate 3D physics raycast candidate points (R1, R2, ...). Only works with `--capture-mode rendering`; uses Camera.main. | | `--elements-only` | boolean | `false` | Return only annotated element JSON without capturing a screenshot image. Requires `--annotate-elements` and `--capture-mode rendering` in PlayMode. | ## Match Modes @@ -49,12 +50,15 @@ The window name is the text displayed in the window's title bar (tab). Common na # Take a screenshot of Game View (default) uloop screenshot -# Capture game rendering (coordinates match simulate-mouse, PlayMode required) +# Capture game rendering (default scale coordinates match simulate-mouse, PlayMode required) uloop screenshot --capture-mode rendering # Annotate interactive UI elements with index labels (for simulate-mouse workflow) uloop screenshot --capture-mode rendering --annotate-elements +# Annotate 3D physics raycast candidate points +uloop screenshot --capture-mode rendering --annotate-raycast-grid + # Get UI element coordinates without capturing an image (fastest) uloop screenshot --capture-mode rendering --annotate-elements --elements-only @@ -80,12 +84,16 @@ Returns JSON with: - `FileSizeBytes`: Size of the saved file in bytes - `Width`: Captured image width in pixels - `Height`: Captured image height in pixels - - `CoordinateSystem`: `"gameView"` or `"window"` + - `ImageCoordinateSystem`: `"top-left-game-view"` for `--capture-mode rendering`, or `"top-left-window"` for window captures - `ResolutionScale`: Resolution scale used for capture - - `YOffset`: Y offset used for gameView coordinate conversion + - `ImageToInputOffsetY`: Y offset added after unscaling image pixels to get mouse-input coordinates + - `GameViewWidth` / `GameViewHeight`: Game View size used by mouse input tools + - `ScreenshotToInputFormula`: Formula for turning image coordinates into mouse-input coordinates + - `UnityInputFormula`: Formula used internally by mouse input tools to inject `Mouse.current.position` - `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used. + - `RaycastGridPoints`: Array of 3D raycast candidate metadata. Empty unless `--annotate-raycast-grid` is used. -For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. +For `AnnotatedElements` fields and Game View coordinates, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools. When multiple windows match (e.g., multiple Inspector windows or when using `contains` mode), all matching windows are captured with numbered filenames (e.g., `Inspector_1_*.png`, `Inspector_2_*.png`). @@ -94,3 +102,6 @@ When multiple windows match (e.g., multiple Inspector windows or when using `con - Use `uloop focus-window` first if needed - Target window must be open in Unity Editor - Window name matching is always case-insensitive +- Use `--capture-mode rendering` for coordinates that should be passed to `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`. +- Use `ScreenshotToInputFormula` before passing raw image pixels to mouse tools. `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` are already mouse-input coordinates. +- Do not use `window` captures as mouse-input coordinates because they include Unity Editor chrome. diff --git a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md index 283109ccc2..7b47103f7f 100644 --- a/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/Api/McpTools/Screenshot/Skill/references/annotated-elements.md @@ -11,27 +11,31 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. - `Type`: Element type (`Button`, `Toggle`, `Slider`, `Dropdown`, `InputField`, `Scrollbar`, `Draggable`, `DropTarget`, `Selectable`) - `Interaction`: Derived interaction category (`Click`, `Drag`, `Drop`, `Text`). Use this to choose between `simulate-mouse-ui --action Click` and drag actions. -- `SimX`, `SimY`: Center position in simulate-mouse coordinates. Use these directly with `--x` and `--y`. +- `SimX`, `SimY`: Center position in top-left Game View coordinates. Use these directly with `simulate-mouse-ui --x/--y`, `simulate-mouse-input --x/--y`, or `raycast --x/--y`. - `BoundsMinX`, `BoundsMinY`, `BoundsMaxX`, `BoundsMaxY`: Bounding box in simulate-mouse coordinates - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. ## Coordinate Conversion -When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates: +When `ImageCoordinateSystem` is `"top-left-game-view"`, convert raw image pixel coordinates from `screenshot --capture-mode rendering` with the formula returned in `ScreenshotToInputFormula`: ```text -sim_x = image_x / ResolutionScale -sim_y = image_y / ResolutionScale + YOffset +input_x = image_x / resolutionScale +input_y = image_y / resolutionScale + imageToInputOffsetY ``` -When `ResolutionScale` is `1.0`, this simplifies to: +When `ResolutionScale` is `1.0` and `ImageToInputOffsetY` is `0`, raw image pixel coordinates already match mouse-input coordinates. `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` are always returned as mouse-input coordinates, so pass those values directly. + +The mouse input tools convert internally to Unity Input System coordinates: ```text -sim_x = image_x -sim_y = image_y + YOffset +unity_x = input_x +unity_y = gameViewHeight - input_y ``` +Do not flip Y in the caller. + ## Annotation Readability Annotated screenshots compensate border thickness for `ResolutionScale`, so the saved PNG keeps the intended outline width after downscaling. The neutral contrast borders are 2 output pixels each, and the colored middle border is 4 output pixels. Label outlines are also compensated and are separated from element borders by a 4 output pixel gap. diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs index 86542a0303..2da50d3571 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputResponse.cs @@ -10,6 +10,15 @@ public class SimulateMouseInputResponse : BaseToolResponse public string? Button { get; set; } public float? PositionX { get; set; } public float? PositionY { get; set; } + public string InputCoordinateSystem { get; set; } = ""; + public string UnityCoordinateSystem { get; set; } = ""; + public float? GameViewWidth { get; set; } + public float? GameViewHeight { get; set; } + public float? InputPositionX { get; set; } + public float? InputPositionY { get; set; } + public float? InjectedUnityPositionX { get; set; } + public float? InjectedUnityPositionY { get; set; } + public string CoordinateConversionFormula { get; set; } = ""; public SimulateMouseInputResponse() { diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs index 71dc5bfcff..6f09b11d2d 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputSchema.cs @@ -7,10 +7,10 @@ public class SimulateMouseInputSchema : BaseToolSchema [Description("Mouse input action: Click(0) - inject left/right/middle button press+release, LongPress(1) - inject button hold for Duration seconds, MoveDelta(2) - inject mouse delta for camera/look (one-shot), Scroll(3) - inject scroll wheel, SmoothDelta(4) - inject mouse delta smoothly over Duration seconds")] public MouseInputAction Action { get; set; } = MouseInputAction.Click; - [Description("Target X position in screen pixels (origin: top-left). Used by Click and LongPress to set Mouse.current.position.")] + [Description("Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use AnnotatedElements[].SimX, RaycastGridPoints[].InputX, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public float X { get; set; } = 0f; - [Description("Target Y position in screen pixels (origin: top-left). Used by Click and LongPress to set Mouse.current.position.")] + [Description("Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use AnnotatedElements[].SimY, RaycastGridPoints[].InputY, or raw screenshot image pixels converted with ScreenshotToInputFormula.")] public float Y { get; set; } = 0f; [Description("Mouse button: Left(0, default), Right(1), Middle(2). Used by Click and LongPress.")] diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs index c1b81d2080..f6b872d59e 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/SimulateMouseInputTool.cs @@ -143,12 +143,12 @@ private static void EnsureOverlayExists() OverlayCanvasFactory.EnsureExists(); } - // Input coordinates use top-left origin; Unity Screen space uses bottom-left origin. - // Uses Screen.height (runtime resolution) because Mouse.current.position is in - // runtime screen space, not the editor Game view target resolution. - private static Vector2 InputToScreen(Vector2 inputPos) + // Public mouse-input coordinates match rendering screenshots. Unity input stays bottom-left, + // so conversion belongs inside the tool instead of in the caller's prompt or skill text. + private static GameViewCoordinateConversion ConvertInputToUnity(Vector2 inputPos) { - return new Vector2(inputPos.x, Screen.height - inputPos.y); + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + return GameViewCoordinateUtility.ConvertInputToUnity(inputPos, gameViewSize); } private async Task ExecuteClick( @@ -165,13 +165,13 @@ private async Task ExecuteClick( } Vector2 inputPos = new Vector2(parameters.X, parameters.Y); - Vector2 screenPos = InputToScreen(inputPos); + GameViewCoordinateConversion conversion = ConvertInputToUnity(inputPos); MouseButton button = parameters.Button; string buttonName = button.ToString(); // Set mouse position before clicking await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( - () => MouseInputState.SetPositionState(mouse, screenPos), ct); + () => MouseInputState.SetPositionState(mouse, conversion.InjectedUnityPosition), ct); // Press button MouseInputState.SetButtonDown(button); @@ -200,15 +200,12 @@ await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( } string durationText = parameters.Duration > 0f ? $" for {parameters.Duration:F1}s" : ""; - return new SimulateMouseInputResponse - { - Success = true, - Message = $"Clicked {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}){durationText}", - Action = MouseInputAction.Click.ToString(), - Button = buttonName, - PositionX = inputPos.x, - PositionY = inputPos.y - }; + return CreatePositionResponse( + $"Clicked {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}){durationText}", + MouseInputAction.Click, + buttonName, + inputPos, + conversion); } private async Task ExecuteLongPress( @@ -225,13 +222,13 @@ private async Task ExecuteLongPress( } Vector2 inputPos = new Vector2(parameters.X, parameters.Y); - Vector2 screenPos = InputToScreen(inputPos); + GameViewCoordinateConversion conversion = ConvertInputToUnity(inputPos); MouseButton button = parameters.Button; string buttonName = button.ToString(); // Set mouse position before pressing await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( - () => MouseInputState.SetPositionState(mouse, screenPos), ct); + () => MouseInputState.SetPositionState(mouse, conversion.InjectedUnityPosition), ct); // Press button MouseInputState.SetButtonDown(button); @@ -258,14 +255,38 @@ await InputSystemUpdateHelper.ApplyOnNextConfiguredUpdate( SimulateMouseInputOverlayState.SetButtonHeld(button, false); } + return CreatePositionResponse( + $"Long-pressed {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s", + MouseInputAction.LongPress, + buttonName, + inputPos, + conversion); + } + + private static SimulateMouseInputResponse CreatePositionResponse( + string message, + MouseInputAction action, + string buttonName, + Vector2 inputPos, + GameViewCoordinateConversion conversion) + { return new SimulateMouseInputResponse { Success = true, - Message = $"Long-pressed {buttonName} at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s", - Action = MouseInputAction.LongPress.ToString(), + Message = message, + Action = action.ToString(), Button = buttonName, PositionX = inputPos.x, - PositionY = inputPos.y + PositionY = inputPos.y, + InputCoordinateSystem = McpConstants.COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW, + UnityCoordinateSystem = McpConstants.COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW, + GameViewWidth = conversion.GameViewSize.x, + GameViewHeight = conversion.GameViewSize.y, + InputPositionX = conversion.InputPosition.x, + InputPositionY = conversion.InputPosition.y, + InjectedUnityPositionX = conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = conversion.InjectedUnityPosition.y, + CoordinateConversionFormula = McpConstants.COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY }; } diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md index b7ffbb20af..d0bca06e34 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseInput/Skill/SKILL.md @@ -11,7 +11,7 @@ Simulate mouse input via Input System in Unity PlayMode: $ARGUMENTS ## Workflow 1. Ensure Unity is in PlayMode (use `uloop control-play-mode --action Play` if not) -2. For Click/LongPress: determine the target screen position (use `uloop screenshot` to find coordinates) +2. For Click/LongPress: determine the target Game View input position from annotated `SimX`/`SimY`, raycast-grid `InputX`/`InputY`, or raw image pixels converted with `ScreenshotToInputFormula` 3. Execute the appropriate `uloop simulate-mouse-input` command 4. Take a screenshot to verify the result: `uloop screenshot --capture-mode rendering` 5. Report what happened @@ -27,8 +27,8 @@ uloop simulate-mouse-input --action [options] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `--action` | enum | `Click` | `Click`, `LongPress`, `MoveDelta`, `SmoothDelta`, `Scroll` | -| `--x` | number | `0` | Target X position (origin: top-left). Used by Click and LongPress. | -| `--y` | number | `0` | Target Y position (origin: top-left). Used by Click and LongPress. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Used by Click and LongPress. Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula`. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Used by Click and LongPress. Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula`. | | `--button` | enum | `Left` | Mouse button: `Left`, `Right`, `Middle`. Used by Click and LongPress. | | `--duration` | number | `0` | Hold duration for LongPress, or interpolation duration for SmoothDelta (seconds). For Click, 0 = one-shot tap. | | `--delta-x` | number | `0` | Delta X in pixels for MoveDelta/SmoothDelta. Positive = right. | @@ -67,7 +67,7 @@ uloop simulate-mouse-input --action [options] ## Examples ```bash -# Left-click at screen center (for game logic) +# Left-click at the Game View center (for game logic) uloop simulate-mouse-input --action Click --x 400 --y 300 # Right-click at screen center (e.g. place block) @@ -89,6 +89,20 @@ uloop simulate-mouse-input --action Scroll --scroll-y -120 uloop simulate-mouse-input --action SmoothDelta --delta-x 300 --delta-y 0 --duration 0.5 ``` +## Coordinate System + +- `--x` / `--y` use **top-left Game View coordinates**. +- Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. +- `AnnotatedElements[].SimX/SimY` and `RaycastGridPoints[].InputX/InputY` can be passed directly to this tool. +- Do not flip Y in the caller. The tool converts internally for Unity Input System: + +```text +unity_x = input_x +unity_y = gameViewHeight - input_y +``` + +- `Mouse.current.position` uses bottom-left Unity coordinates, so the value read inside Unity may show the converted Y. + ## Prerequisites - Unity must be in **PlayMode** @@ -103,7 +117,12 @@ Returns JSON with: - `Message`: Status message - `Action`: Echoes which action was executed (`Click`, `LongPress`, `MoveDelta`, `SmoothDelta`, or `Scroll`) - `Button`: Which button was used (nullable string; populated for `Click` / `LongPress`, null otherwise) -- `PositionX`: Target X coordinate (nullable float; populated for `Click` / `LongPress`) -- `PositionY`: Target Y coordinate (nullable float; populated for `Click` / `LongPress`) - -These are the only six fields. There is no `DeltaX`, `DeltaY`, `ScrollX`, `ScrollY`, `Duration`, or hit-element field in the response — only the issued action, button, and target position are echoed back. Verify visual outcome with a follow-up screenshot. +- `PositionX` / `PositionY`: Target top-left Game View coordinates (nullable float; populated for `Click` / `LongPress`) +- `InputCoordinateSystem`: `"top-left-game-view"` for click/long-press coordinates +- `UnityCoordinateSystem`: `"bottom-left-game-view"` for the injected `Mouse.current.position` +- `GameViewWidth` / `GameViewHeight`: Game View size used for conversion +- `InputPositionX` / `InputPositionY`: Coordinates received from the caller +- `InjectedUnityPositionX` / `InjectedUnityPositionY`: Coordinates injected into `Mouse.current.position` +- `CoordinateConversionFormula`: Conversion formula used by the tool + +Verify visual outcome with a follow-up screenshot. diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs index f7784e35b8..cdf3f5920c 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiSchema.cs @@ -7,16 +7,16 @@ public class SimulateMouseUiSchema : BaseToolSchema [Description("Mouse action: Click(0) - click with selected Button, Drag(1) - one-shot drag, DragStart(2) - begin drag and hold, DragMove(3) - move while holding drag, DragEnd(4) - release drag, LongPress(5) - press and hold for Duration seconds")] public MouseAction Action { get; set; } = MouseAction.Click; - [Description("Target X position in screen pixels (origin: top-left). For Drag action, this is the destination.")] + [Description("Target X position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.")] public float X { get; set; } = 0f; - [Description("Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination.")] + [Description("Target Y position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination.")] public float Y { get; set; } = 0f; - [Description("Start X position in screen pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] + [Description("Start X position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] public float FromX { get; set; } = 0f; - [Description("Start Y position in screen pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] + [Description("Start Y position in Game View pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")] public float FromY { get; set; } = 0f; [Description("Drag speed in pixels per second (0 for instant). Applies to Drag, DragMove, and DragEnd actions.")] diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs index be6fcf30e9..a994511a62 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/SimulateMouseUiTool.cs @@ -11,7 +11,7 @@ namespace io.github.hatayama.uLoopMCP { - [McpTool(Description = "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem screen coordinates")] + [McpTool(Description = "Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem using top-left Game View coordinates")] public class SimulateMouseUiTool : AbstractUnityTool { public override string ToolName => "simulate-mouse-ui"; diff --git a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md index 451d728303..451178c995 100644 --- a/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md +++ b/Packages/src/Editor/Api/McpTools/SimulateMouseUi/Skill/SKILL.md @@ -1,6 +1,6 @@ --- name: uloop-simulate-mouse-ui -description: "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." +description: "Simulate PlayMode EventSystem UI mouse actions using top-left Game View coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." context: fork --- @@ -28,10 +28,10 @@ uloop simulate-mouse-ui --action --x --y [options] | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `--action` | enum | `Click` | `Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, `LongPress` | -| `--x` | number | `0` | Target X position in screen pixels (origin: top-left). For Drag action, this is the destination. | -| `--y` | number | `0` | Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination. | -| `--from-x` | number | `0` | Start X position for Drag action. Drag starts here and moves to x,y. | -| `--from-y` | number | `0` | Start Y position for Drag action. Drag starts here and moves to x,y. | +| `--x` | number | `0` | Target X position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination. | +| `--y` | number | `0` | Target Y position in Game View pixels (origin: top-left). Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination. | +| `--from-x` | number | `0` | Start X position in Game View pixels for Drag action. Drag starts here and moves to x,y. | +| `--from-y` | number | `0` | Start Y position in Game View pixels for Drag action. Drag starts here and moves to x,y. | | `--drag-speed` | number | `2000` | Drag speed in pixels per second (0 for instant). 2000 is fast (default), 200 is slow enough to watch. Applies to Drag, DragMove, and DragEnd actions. | | `--duration` | number | `0.5` | Hold duration in seconds for LongPress action. | | `--button` | enum | `Left` | Mouse button. `Click` and `LongPress` support `Left`, `Right`, and `Middle`. Drag actions support `Left` only; other buttons return an error. | @@ -67,7 +67,7 @@ uloop simulate-mouse-ui --action --x --y [options] ## Coordinate System - Origin is **top-left** (0, 0) -- All positions are in **screen pixels** +- All positions are in **top-left Game View pixels** - Get coordinates from `AnnotatedElements` JSON (`SimX`/`SimY`) — do NOT look up GameObject positions - Clicking or long-pressing on empty space (no UI element) still succeeds with a message indicating no element was hit - Dragging on empty space (no draggable UI element) returns `Success = false` @@ -77,7 +77,7 @@ uloop simulate-mouse-ui --action --x --y [options] ## Examples ```bash -# Click a button at screen position +# Click a button at a Game View position uloop simulate-mouse-ui --action Click --x 400 --y 300 # Force-click a button behind a raycast blocker by path diff --git a/Packages/src/Editor/Shared/Config/McpConstants.cs b/Packages/src/Editor/Shared/Config/McpConstants.cs index b3fd3173cc..3a1e4deec7 100644 --- a/Packages/src/Editor/Shared/Config/McpConstants.cs +++ b/Packages/src/Editor/Shared/Config/McpConstants.cs @@ -168,6 +168,16 @@ public static UnityEditor.PackageManager.PackageInfo PackageInfo // Screenshot coordinate system values public const string COORDINATE_SYSTEM_GAME_VIEW = "gameView"; public const string COORDINATE_SYSTEM_WINDOW = "window"; + public const string COORDINATE_SYSTEM_TOP_LEFT_GAME_VIEW = "top-left-game-view"; + public const string COORDINATE_SYSTEM_BOTTOM_LEFT_GAME_VIEW = "bottom-left-game-view"; + public const string COORDINATE_SYSTEM_TOP_LEFT_WINDOW = "top-left-window"; + public const string COORDINATE_CONVERSION_FORMULA_GAME_VIEW_INPUT_TO_UNITY = + "unity_x = input_x; unity_y = gameViewHeight - input_y"; + public const string SCREENSHOT_RENDERING_TO_INPUT_FORMULA = + "simulate_mouse_x = image_x / resolutionScale; simulate_mouse_y = image_y / resolutionScale + imageToInputOffsetY"; + public const string SCREENSHOT_WINDOW_TO_INPUT_FORMULA_UNAVAILABLE = + "unavailable: window screenshots include Unity Editor chrome; use capture-mode rendering for mouse input coordinates"; + public const float RAYCAST_DEFAULT_MAX_DISTANCE = 1000f; // SimulateMouseUi constants public const float SIMULATE_MOUSE_UI_DEFAULT_DRAG_SPEED = 2000f; diff --git a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs index 999f5beade..99661af178 100644 --- a/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs +++ b/Packages/src/Editor/Utils/EditorWindowCaptureUtility.cs @@ -138,10 +138,10 @@ public static string[] GetOpenWindowNames() // Captures game rendering by reading GameView's composited RenderTexture (PlayMode only). // Contains all cameras + Screen Space Overlay Canvas, without tab bar or borders. - // Coordinate mapping to simulate-mouse: - // sim_x = image_x / resolutionScale, sim_y = image_y / resolutionScale + YOffset - // where YOffset = Screen.height - RenderTexture.height (returned in the tuple). - public static async Task<(Texture2D? texture, int yOffset)> CaptureGameRenderingAsync(float resolutionScale, CancellationToken ct) + internal static async Task<(Texture2D? texture, GameRenderingImageInfo renderingImageInfo)> CaptureGameRenderingAsync( + float resolutionScale, + GameRenderingImageInfo? renderingImageInfo, + CancellationToken ct) { Debug.Assert(UnityEditor.EditorApplication.isPlaying, "CaptureGameRenderingAsync requires PlayMode"); @@ -152,12 +152,16 @@ public static string[] GetOpenWindowNames() if (rt == null) { Debug.LogWarning("[EditorWindowCaptureUtility] GameView RenderTexture is not available"); - return (null, 0); + GameRenderingImageInfo unavailableInfo = renderingImageInfo ?? + CreateUnavailableGameRenderingImageInfo(Handles.GetMainGameViewSize()); + return (null, unavailableInfo); } - int yOffset = (int)Handles.GetMainGameViewSize().y - rt.height; + // GameView RenderTexture can be shorter than the full input area, so raw image Y needs this offset. + GameRenderingImageInfo captureInfo = renderingImageInfo ?? + CreateGameRenderingImageInfo(Handles.GetMainGameViewSize(), rt.width, rt.height); - // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format + // RenderTexture uses bottom-left origin; flip vertically for standard top-left image format. RenderTextureDescriptor flipDescriptor = new RenderTextureDescriptor(rt.width, rt.height, rt.format, 0); if (QualitySettings.activeColorSpace == ColorSpace.Linear) { @@ -181,7 +185,53 @@ public static string[] GetOpenWindowNames() texture = ApplyResolutionScaling(texture, resolutionScale); } - return (texture, yOffset); + return (texture, captureInfo); + } + + /// + /// Gets rendering image dimensions after waiting for Game View rendering to settle. + /// + internal static async Task GetGameRenderingImageInfoAsync(CancellationToken ct) + { + Debug.Assert(UnityEditor.EditorApplication.isPlaying, "GetGameRenderingImageInfoAsync requires PlayMode"); + + // Raycast-grid annotations must use the same settled RenderTexture geometry as the PNG capture path. + await EditorDelay.DelayFrame(2, ct); + + Vector2 gameViewSize = Handles.GetMainGameViewSize(); + RenderTexture rt = GameViewBridge.GetRenderTexture(); + if (rt == null) + { + return CreateUnavailableGameRenderingImageInfo(gameViewSize); + } + + return CreateGameRenderingImageInfo(gameViewSize, rt.width, rt.height); + } + + internal static GameRenderingImageInfo CreateUnavailableGameRenderingImageInfo(Vector2 gameViewSize) + { + return new GameRenderingImageInfo(gameViewSize, gameViewSize, 0); + } + + private static GameRenderingImageInfo CreateGameRenderingImageInfo( + Vector2 gameViewSize, + int renderTextureWidth, + int renderTextureHeight) + { + Vector2 renderingImageSize = new Vector2(renderTextureWidth, renderTextureHeight); + int imageToInputOffsetY = CalculateImageToInputOffsetY(gameViewSize, renderTextureHeight); + return new GameRenderingImageInfo(gameViewSize, renderingImageSize, imageToInputOffsetY); + } + + /// + /// Calculates the Y offset from rendering screenshot image space to Game View input space. + /// + internal static int CalculateImageToInputOffsetY(Vector2 gameViewSize, int renderTextureHeight) + { + Debug.Assert(gameViewSize.y >= 0f, "Game View height must not be negative."); + Debug.Assert(renderTextureHeight >= 0, "RenderTexture height must not be negative."); + + return Mathf.RoundToInt(gameViewSize.y) - renderTextureHeight; } private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float scale) @@ -205,5 +255,24 @@ private static Texture2D ApplyResolutionScaling(Texture2D originalTexture, float return scaledTexture; } } -} + /// + /// Describes the Game View geometry used to map rendering screenshots back to input coordinates. + /// + internal readonly struct GameRenderingImageInfo + { + public readonly Vector2 GameViewSize; + public readonly Vector2 RenderingImageSize; + public readonly int ImageToInputOffsetY; + + public GameRenderingImageInfo( + Vector2 gameViewSize, + Vector2 renderingImageSize, + int imageToInputOffsetY) + { + GameViewSize = gameViewSize; + RenderingImageSize = renderingImageSize; + ImageToInputOffsetY = imageToInputOffsetY; + } + } +} diff --git a/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs new file mode 100644 index 0000000000..29fe827c4a --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs @@ -0,0 +1,53 @@ +using UnityEditor; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Converts public top-left Game View coordinates into Unity's bottom-left input coordinates. + /// + internal static class GameViewCoordinateUtility + { + internal static GameViewCoordinateConversion ConvertInputToUnity( + Vector2 inputPosition, + Vector2 gameViewSize) + { + Debug.Assert(gameViewSize.x >= 0f, "Game View width must not be negative."); + Debug.Assert(gameViewSize.y >= 0f, "Game View height must not be negative."); + + Vector2 injectedUnityPosition = new Vector2( + inputPosition.x, + gameViewSize.y - inputPosition.y); + + return new GameViewCoordinateConversion( + inputPosition, + injectedUnityPosition, + gameViewSize); + } + + internal static Vector2 GetMainGameViewSize() + { + return Handles.GetMainGameViewSize(); + } + } + + /// + /// Describes one conversion from screenshot-compatible Game View input to Unity input space. + /// + internal readonly struct GameViewCoordinateConversion + { + public readonly Vector2 InputPosition; + public readonly Vector2 InjectedUnityPosition; + public readonly Vector2 GameViewSize; + + public GameViewCoordinateConversion( + Vector2 inputPosition, + Vector2 injectedUnityPosition, + Vector2 gameViewSize) + { + InputPosition = inputPosition; + InjectedUnityPosition = injectedUnityPosition; + GameViewSize = gameViewSize; + } + } +} diff --git a/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta new file mode 100644 index 0000000000..6d1c8013fb --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewCoordinateUtility.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 357293374973742c99049bb1696cab7d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs new file mode 100644 index 0000000000..c20ed0322d --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs @@ -0,0 +1,66 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Runs physics raycasts from screenshot-compatible Game View coordinates. + /// + internal static class GameViewRaycastUtility + { + internal static GameViewRaycastResult RaycastFromInputPosition( + Vector2 inputPosition, + float maxDistance, + int layerMask, + bool syncTransforms) + { + Vector2 gameViewSize = GameViewCoordinateUtility.GetMainGameViewSize(); + GameViewCoordinateConversion conversion = + GameViewCoordinateUtility.ConvertInputToUnity(inputPosition, gameViewSize); + + Camera mainCamera = Camera.main; + if (mainCamera == null) + { + return new GameViewRaycastResult(false, conversion, new RaycastHit[0]); + } + + Ray ray = mainCamera.ScreenPointToRay(conversion.InjectedUnityPosition); + if (syncTransforms) + { + // Transform changes can be visible before the physics scene updates, so sync before screenshot-based queries. + Physics.SyncTransforms(); + } + + int visibleLayerMask = layerMask & mainCamera.cullingMask; + RaycastHit[] hits = Physics.RaycastAll(ray, maxDistance, visibleLayerMask); + System.Array.Sort(hits, CompareHitsByDistance); + + return new GameViewRaycastResult(true, conversion, hits); + } + + private static int CompareHitsByDistance(RaycastHit left, RaycastHit right) + { + return left.distance.CompareTo(right.distance); + } + } + + /// + /// Contains the camera lookup result, coordinate conversion, and sorted physics hits. + /// + internal readonly struct GameViewRaycastResult + { + public readonly bool CameraFound; + public readonly GameViewCoordinateConversion Conversion; + public readonly RaycastHit[] Hits; + + public GameViewRaycastResult( + bool cameraFound, + GameViewCoordinateConversion conversion, + RaycastHit[] hits) + { + CameraFound = cameraFound; + Conversion = conversion; + Hits = hits; + } + } +} diff --git a/Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta new file mode 100644 index 0000000000..6a89bc821f --- /dev/null +++ b/Packages/src/Editor/Utils/GameViewRaycastUtility.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 326e5d6ae2978480c94f23ad1edcdb66 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs new file mode 100644 index 0000000000..b2b021ca82 --- /dev/null +++ b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs @@ -0,0 +1,125 @@ +#nullable enable +using System.Collections.Generic; +using UnityEngine; + +namespace io.github.hatayama.uLoopMCP +{ + /// + /// Creates 3D physics click candidates for rendering screenshots. + /// + internal static class RaycastGridAnnotator + { + private const int GRID_COLUMNS = 5; + private const int GRID_ROWS = 5; + private const float MARKER_SIZE = 18f; + + internal static List CollectRaycastGridPoints( + Vector2 renderingImageSize, + int imageToInputOffsetY) + { + List points = new List(); + int labelIndex = 1; + // Sync once for the whole grid; each candidate raycast then reads the same current physics state. + Physics.SyncTransforms(); + + for (int row = 1; row <= GRID_ROWS; row++) + { + for (int column = 1; column <= GRID_COLUMNS; column++) + { + Vector2 inputPosition = CalculateGridInputPosition( + renderingImageSize, + imageToInputOffsetY, + row, + column); + GameViewRaycastResult raycastResult = GameViewRaycastUtility.RaycastFromInputPosition( + inputPosition, + McpConstants.RAYCAST_DEFAULT_MAX_DISTANCE, + Physics.DefaultRaycastLayers, + false); + + points.Add(CreatePointInfo($"R{labelIndex}", inputPosition, raycastResult)); + labelIndex++; + } + } + + return points; + } + + internal static Vector2 CalculateGridInputPosition( + Vector2 renderingImageSize, + int imageToInputOffsetY, + int row, + int column) + { + Debug.Assert(renderingImageSize.x >= 0f, "Rendering image width must not be negative."); + Debug.Assert(renderingImageSize.y >= 0f, "Rendering image height must not be negative."); + Debug.Assert(row >= 1 && row <= GRID_ROWS, "Grid row must be within the configured grid."); + Debug.Assert(column >= 1 && column <= GRID_COLUMNS, "Grid column must be within the configured grid."); + + // Grid points must be visible in the captured PNG, so sample image space before adding the input Y offset. + return new Vector2( + renderingImageSize.x * column / (GRID_COLUMNS + 1f), + imageToInputOffsetY + renderingImageSize.y * row / (GRID_ROWS + 1f)); + } + + internal static List CreateOverlayElements(List points) + { + List overlayElements = new List(); + + foreach (RaycastGridPointInfo point in points) + { + if (!point.Hit) + { + continue; + } + + float halfSize = MARKER_SIZE / 2f; + overlayElements.Add(new UIElementInfo + { + Label = point.Label, + Name = point.HitGameObjectName ?? "", + Path = point.HitGameObjectPath ?? "", + Type = "RaycastHit", + Interaction = "Raycast", + SimX = point.InputX, + SimY = point.InputY, + BoundsMinX = point.InjectedUnityPositionX - halfSize, + BoundsMinY = point.InjectedUnityPositionY - halfSize, + BoundsMaxX = point.InjectedUnityPositionX + halfSize, + BoundsMaxY = point.InjectedUnityPositionY + halfSize, + SortingOrder = 0, + SiblingIndex = 0 + }); + } + + return overlayElements; + } + + private static RaycastGridPointInfo CreatePointInfo( + string label, + Vector2 inputPosition, + GameViewRaycastResult raycastResult) + { + RaycastGridPointInfo pointInfo = new RaycastGridPointInfo + { + Label = label, + Hit = raycastResult.Hits.Length > 0, + InputX = inputPosition.x, + InputY = inputPosition.y, + InjectedUnityPositionX = raycastResult.Conversion.InjectedUnityPosition.x, + InjectedUnityPositionY = raycastResult.Conversion.InjectedUnityPosition.y + }; + + if (!pointInfo.Hit) + { + return pointInfo; + } + + RaycastHit hit = raycastResult.Hits[0]; + pointInfo.HitGameObjectName = hit.collider.gameObject.name; + pointInfo.HitGameObjectPath = GameObjectPathUtility.GetFullPath(hit.collider.gameObject); + pointInfo.Distance = hit.distance; + return pointInfo; + } + } +} diff --git a/Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta new file mode 100644 index 0000000000..ac65e30213 --- /dev/null +++ b/Packages/src/Editor/Utils/RaycastGridAnnotator.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 23328277c0b2f44debb12cc314568f86 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/README.md b/Packages/src/README.md index 1bb5013aef..e615fd2b20 100644 --- a/Packages/src/README.md +++ b/Packages/src/README.md @@ -143,7 +143,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi
-All 16 Bundled Skills +All 17 Bundled Skills - `/uloop-launch` - Launch Unity with correct version - `/uloop-compile` - Execute compilation @@ -154,6 +154,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi - `/uloop-get-hierarchy` - Get scene hierarchy - `/uloop-find-game-objects` - Find GameObjects - `/uloop-screenshot` - Capture EditorWindow +- `/uloop-raycast` - Check what a Game View coordinate hits in 3D physics - `/uloop-simulate-mouse-ui` - Simulate mouse click, long-press, and drag on PlayMode UI elements - `/uloop-simulate-mouse-input` - Simulate mouse input in PlayMode via Input System - `/uloop-simulate-keyboard` - Simulate keyboard input in PlayMode via Input System @@ -378,8 +379,10 @@ Great for keeping visual feedback in sync after other apps steal focus. (Linux i Take a screenshot of any EditorWindow as a PNG. Specify the window name (the text displayed in the title bar/tab) to capture. When multiple windows of the same type are open (e.g., 3 Inspector windows), all windows are saved with numbered filenames. Supports three matching modes: `exact` (default), `prefix`, and `contains` - all case-insensitive. +Use `CaptureMode: rendering` when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; convert raw image pixels with `ScreenshotToInputFormula`, or pass annotated `SimX/SimY` and raycast-grid `InputX/InputY` values directly. ```text → screenshot (WindowName: "Console") +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) → Save Console window state as PNG → Provide visual feedback to AI ``` @@ -469,6 +472,8 @@ https://github.com/user-attachments/assets/c7ee9103-c282-4f90-8b01-64bb17400f3e ### 12. simulate-mouse-input - Simulate Mouse Input in PlayMode via Input System Simulate mouse input in PlayMode via Input System. Injects button clicks, mouse delta, and scroll wheel directly into `Mouse.current` for game logic that reads Input System. Unlike `simulate-mouse-ui` which fires EventSystem pointer events for uGUI, this tool targets game logic that reads `Mouse.current` directly. This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. +Click and LongPress coordinates use the same top-left Game View coordinates as `screenshot --capture-mode rendering`; do not flip Y before calling the tool. + Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. ```text @@ -480,7 +485,16 @@ Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. → simulate-mouse-input (Action: SmoothDelta, DeltaX: 300, DeltaY: 0, Duration: 0.5) ``` -### 13. simulate-keyboard - Simulate Keyboard Input in PlayMode +### 13. raycast - Check a 3D Physics Hit at a Game View Coordinate +Raycast from `Camera.main` through a top-left Game View coordinate. Use this before gameplay clicks when you need to confirm which 3D object a screenshot coordinate will hit. + +```text +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) +→ raycast (X: 960, Y: 540) +→ simulate-mouse-input (Action: Click, X: 960, Y: 540) +``` + +### 14. simulate-keyboard - Simulate Keyboard Input in PlayMode Simulate keyboard key input in PlayMode via Input System. Supports single key taps, sustained holds, and multi-key combinations (e.g. Shift+W for sprinting). This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. Game code must read input via Input System API (e.g. `Keyboard.current[Key.W].isPressed`), not legacy `Input.GetKey()`. Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), KeyUp (release held key). @@ -495,7 +509,7 @@ Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), → simulate-keyboard (Action: KeyUp, Key: LeftShift) ``` -### 14. record-input - Record Input During PlayMode +### 15. record-input - Record Input During PlayMode Record keyboard and mouse input during PlayMode frame-by-frame into a JSON file. Captures key presses, mouse movement, clicks, and scroll events via Input System device state diffing. This tool is available only when the Input System package is installed. ```text diff --git a/Packages/src/TOOL_REFERENCE.md b/Packages/src/TOOL_REFERENCE.md index d538577448..df4e87b255 100644 --- a/Packages/src/TOOL_REFERENCE.md +++ b/Packages/src/TOOL_REFERENCE.md @@ -184,6 +184,10 @@ All tools automatically include the following property: - `prefix`: Window name must start with the input - `contains`: Window name must contain the input anywhere - `OutputDirectory` (string): Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths (default: "") + - `CaptureMode` (enum): "window" captures Editor chrome, "rendering" captures Game View rendering with top-left Game View coordinates. Use `ScreenshotToInputFormula` before passing raw image pixels to input tools (default: "window") + - `AnnotateElements` (boolean): Annotate interactive UI elements in rendering screenshots (default: false) + - `AnnotateRaycastGrid` (boolean): Annotate 3D physics raycast candidate points in rendering screenshots (default: false) + - `ElementsOnly` (boolean): Return annotation JSON without writing an image file (default: false) - **Response**: - `ScreenshotCount` (number): Number of windows captured - `Screenshots` (array): Array of screenshot info @@ -191,6 +195,12 @@ All tools automatically include the following property: - `FileSizeBytes` (number): Size of the saved file in bytes - `Width` (number): Captured image width in pixels - `Height` (number): Captured image height in pixels + - `ImageCoordinateSystem` (string): "top-left-game-view" for rendering captures, "top-left-window" for window captures + - `ImageToInputOffsetY` (number): Y offset added after unscaling raw image pixels to get input coordinates + - `GameViewWidth` / `GameViewHeight` (number): Game View size used by input tools + - `ScreenshotToInputFormula` (string): How to use screenshot coordinates with input tools + - `UnityInputFormula` (string): Internal conversion to `Mouse.current.position` + - `RaycastGridPoints` (array): 3D raycast candidate metadata when requested ### 10. control-play-mode - **Description**: Control Unity Editor play mode (play/stop/pause) @@ -207,7 +217,7 @@ All tools automatically include the following property: --- ### 11. simulate-mouse-ui -- **Description**: Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem screen coordinates. Uses EventSystem and ExecuteEvents to dispatch pointer events directly — works independently of both old and new Input System. For game logic that reads Input System (e.g. `Mouse.current.leftButton.wasPressedThisFrame`), use `simulate-mouse-input` instead +- **Description**: Simulate mouse click, long-press, and drag on PlayMode UI elements via EventSystem using top-left Game View coordinates. Uses EventSystem and ExecuteEvents to dispatch pointer events directly — works independently of both old and new Input System. For game logic that reads Input System (e.g. `Mouse.current.leftButton.wasPressedThisFrame`), use `simulate-mouse-input` instead - **Parameters**: - `Action` (enum): Mouse action - "Click", "Drag", "DragStart", "DragMove", "DragEnd", "LongPress" (default: "Click") - `Click`: Click at (X, Y). Fires PointerDown → PointerUp → PointerClick @@ -216,10 +226,10 @@ All tools automatically include the following property: - `DragStart`: Begin drag at (X, Y) and hold - `DragMove`: Animate from current position to (X, Y) at the specified speed - `DragEnd`: Animate to (X, Y), then release drag - - `X` (number): Target X position in screen pixels, origin: top-left (default: 0) - - `Y` (number): Target Y position in screen pixels, origin: top-left (default: 0) - - `FromX` (number): Start X position for Drag action (default: 0) - - `FromY` (number): Start Y position for Drag action (default: 0) + - `X` (number): Target X position in top-left Game View pixels. Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Used by Click, LongPress, DragStart, DragMove, and DragEnd; for Drag, this is the destination (default: 0) + - `FromX` (number): Start X position in top-left Game View pixels for Drag action (default: 0) + - `FromY` (number): Start Y position in top-left Game View pixels for Drag action (default: 0) - `DragSpeed` (number): Drag speed in pixels per second, 0 for instant (default: 2000) - `Duration` (number): Hold duration in seconds for LongPress action (default: 0.5) - `Button` (enum): Mouse button - "Left", "Right", "Middle" (default: "Left") @@ -242,8 +252,8 @@ All tools automatically include the following property: - `MoveDelta`: Inject mouse delta one-shot (for FPS camera/look control) - `SmoothDelta`: Inject mouse delta smoothly over Duration seconds (human-like camera pan) - `Scroll`: Inject scroll wheel (for hotbar switching, zoom, etc.) - - `X` (number): Target X position in screen pixels, origin: top-left. Used by Click and LongPress (default: 0) - - `Y` (number): Target Y position in screen pixels, origin: top-left. Used by Click and LongPress (default: 0) + - `X` (number): Target X position in top-left Game View pixels. Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) - `Button` (enum): Mouse button - "Left", "Right", "Middle" (default: "Left"). Used by Click and LongPress - `Duration` (number): Hold duration for LongPress, or minimum hold time for Click. 0 = one-shot tap (default: 0) - `DeltaX` (number): Delta X in pixels for MoveDelta. Positive = right (default: 0) @@ -257,8 +267,26 @@ All tools automatically include the following property: - `Button` (string): The button used (for Click/LongPress) - `PositionX` (number, nullable): X position used (null for non-positional actions) - `PositionY` (number, nullable): Y position used (null for non-positional actions) + - `InputCoordinateSystem` / `UnityCoordinateSystem` (string): Public and injected coordinate systems + - `GameViewWidth` / `GameViewHeight` (number): Game View size used for conversion + - `InputPositionX/Y` and `InjectedUnityPositionX/Y` (number, nullable): Received and injected coordinates + - `CoordinateConversionFormula` (string): `unity_x = input_x; unity_y = gameViewHeight - input_y` -### 13. simulate-keyboard +### 13. raycast +- **Description**: Raycast from `Camera.main` through a top-left Game View coordinate. Use this to check what a converted rendering screenshot coordinate hits in 3D physics before clicking with `simulate-mouse-input` +- **Parameters**: + - `X` (number): Target X position in top-left Game View pixels. Use `AnnotatedElements[].SimX`, `RaycastGridPoints[].InputX`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) + - `Y` (number): Target Y position in top-left Game View pixels. Use `AnnotatedElements[].SimY`, `RaycastGridPoints[].InputY`, or raw image pixels converted with `ScreenshotToInputFormula` (default: 0) + - `LayerMask` (number): Physics layer mask used by the raycast (default: Unity default raycast layers) + - `MaxDistance` (number): Maximum raycast distance in world units (default: 1000) +- **Response**: + - `Success` (boolean): Whether the command completed + - `Hit` (boolean): Whether physics hit anything + - `HitGameObjectName` / `HitGameObjectPath` (string): Hit object identity when `Hit` is true + - `Distance`, `HitPointX/Y/Z`, `HitNormalX/Y/Z` (number): Hit details when `Hit` is true + - `InputCoordinateSystem`, `UnityCoordinateSystem`, `GameViewWidth/Height`, `InputPositionX/Y`, `InjectedUnityPositionX/Y`, `CoordinateConversionFormula`: Coordinate conversion details + +### 14. simulate-keyboard - **Description**: Simulate keyboard key input in PlayMode via Input System. Supports single key taps, sustained holds, and multi-key combinations. Requires the Input System package, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. Game code must read input via Input System API (e.g. `Keyboard.current[Key.W].isPressed`), not legacy `Input.GetKey()` - **Parameters**: - `Action` (enum): Keyboard action - "Press", "KeyDown", "KeyUp" (default: "Press") diff --git a/Packages/src/package.json b/Packages/src/package.json index 5c39d9fde5..33a98de8f8 100644 --- a/Packages/src/package.json +++ b/Packages/src/package.json @@ -12,6 +12,7 @@ "url": "https://github.com/hatayama/uLoopMCP" }, "dependencies": { + "com.unity.modules.physics": "1.0.0", "com.unity.ugui": "1.0.0", "com.unity.nuget.newtonsoft-json": "3.2.1" }, diff --git a/README.md b/README.md index 1bb5013aef..e615fd2b20 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi
-All 16 Bundled Skills +All 17 Bundled Skills - `/uloop-launch` - Launch Unity with correct version - `/uloop-compile` - Execute compilation @@ -154,6 +154,7 @@ That's it! After installing Skills, LLM tools can automatically handle instructi - `/uloop-get-hierarchy` - Get scene hierarchy - `/uloop-find-game-objects` - Find GameObjects - `/uloop-screenshot` - Capture EditorWindow +- `/uloop-raycast` - Check what a Game View coordinate hits in 3D physics - `/uloop-simulate-mouse-ui` - Simulate mouse click, long-press, and drag on PlayMode UI elements - `/uloop-simulate-mouse-input` - Simulate mouse input in PlayMode via Input System - `/uloop-simulate-keyboard` - Simulate keyboard input in PlayMode via Input System @@ -378,8 +379,10 @@ Great for keeping visual feedback in sync after other apps steal focus. (Linux i Take a screenshot of any EditorWindow as a PNG. Specify the window name (the text displayed in the title bar/tab) to capture. When multiple windows of the same type are open (e.g., 3 Inspector windows), all windows are saved with numbered filenames. Supports three matching modes: `exact` (default), `prefix`, and `contains` - all case-insensitive. +Use `CaptureMode: rendering` when you need coordinates for `simulate-mouse-input`, `simulate-mouse-ui`, or `raycast`; convert raw image pixels with `ScreenshotToInputFormula`, or pass annotated `SimX/SimY` and raycast-grid `InputX/InputY` values directly. ```text → screenshot (WindowName: "Console") +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) → Save Console window state as PNG → Provide visual feedback to AI ``` @@ -469,6 +472,8 @@ https://github.com/user-attachments/assets/c7ee9103-c282-4f90-8b01-64bb17400f3e ### 12. simulate-mouse-input - Simulate Mouse Input in PlayMode via Input System Simulate mouse input in PlayMode via Input System. Injects button clicks, mouse delta, and scroll wheel directly into `Mouse.current` for game logic that reads Input System. Unlike `simulate-mouse-ui` which fires EventSystem pointer events for uGUI, this tool targets game logic that reads `Mouse.current` directly. This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. +Click and LongPress coordinates use the same top-left Game View coordinates as `screenshot --capture-mode rendering`; do not flip Y before calling the tool. + Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. ```text @@ -480,7 +485,16 @@ Supports 5 actions: Click, LongPress, MoveDelta, SmoothDelta, Scroll. → simulate-mouse-input (Action: SmoothDelta, DeltaX: 300, DeltaY: 0, Duration: 0.5) ``` -### 13. simulate-keyboard - Simulate Keyboard Input in PlayMode +### 13. raycast - Check a 3D Physics Hit at a Game View Coordinate +Raycast from `Camera.main` through a top-left Game View coordinate. Use this before gameplay clicks when you need to confirm which 3D object a screenshot coordinate will hit. + +```text +→ screenshot (CaptureMode: rendering, AnnotateRaycastGrid: true) +→ raycast (X: 960, Y: 540) +→ simulate-mouse-input (Action: Click, X: 960, Y: 540) +``` + +### 14. simulate-keyboard - Simulate Keyboard Input in PlayMode Simulate keyboard key input in PlayMode via Input System. Supports single key taps, sustained holds, and multi-key combinations (e.g. Shift+W for sprinting). This tool is available only when the Input System package is installed, and Active Input Handling must be set to `Input System Package (New)` or `Both` in Player Settings. Game code must read input via Input System API (e.g. `Keyboard.current[Key.W].isPressed`), not legacy `Input.GetKey()`. Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), KeyUp (release held key). @@ -495,7 +509,7 @@ Supports 3 actions: Press (one-shot tap or timed hold), KeyDown (hold key down), → simulate-keyboard (Action: KeyUp, Key: LeftShift) ``` -### 14. record-input - Record Input During PlayMode +### 15. record-input - Record Input During PlayMode Record keyboard and mouse input during PlayMode frame-by-frame into a JSON file. Captures key presses, mouse movement, clicks, and scroll events via Input System device state diffing. This tool is available only when the Input System package is installed. ```text