diff --git a/.agents/skills/uloop-screenshot/references/annotated-elements.md b/.agents/skills/uloop-screenshot/references/annotated-elements.md index 1ec7d61539..79f674f193 100644 --- a/.agents/skills/uloop-screenshot/references/annotated-elements.md +++ b/.agents/skills/uloop-screenshot/references/annotated-elements.md @@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. +### PhysicsCollider Entries Per Closed Region + +When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable. + +When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path ` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter. + ## RaycastLayerSummaries Fields `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`. diff --git a/.claude/skills/uloop-screenshot/references/annotated-elements.md b/.claude/skills/uloop-screenshot/references/annotated-elements.md index 1ec7d61539..79f674f193 100644 --- a/.claude/skills/uloop-screenshot/references/annotated-elements.md +++ b/.claude/skills/uloop-screenshot/references/annotated-elements.md @@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. +### PhysicsCollider Entries Per Closed Region + +When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable. + +When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path ` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter. + ## RaycastLayerSummaries Fields `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`. diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs index ee716cc466..6ac046693a 100644 --- a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs @@ -657,6 +657,199 @@ public void CreateLayerSummaries_WhenNoHits_ShouldReturnEmptyList() Assert.That(summaries, Is.Empty); } + [Test] + public void SplitIntoConnectedComponents_WhenSamplesForm4ConnectedRegion_ShouldReturnOneComponent() + { + // Tests that an L-shaped set of 4-connected samples collapses into a single connected component. + List samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 10f, 1, 2), + CreateSample(1, 10f, 20f, 2, 1) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(1)); + Assert.That(components[0].Count, Is.EqualTo(3)); + } + + [Test] + public void SplitIntoConnectedComponents_WhenSamplesFormTwoDisconnectedRegions_ShouldReturnTwoComponents() + { + // Tests that two spatially disconnected 4-connected regions come back as two separate components. + List samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 10f, 1, 2), + CreateSample(1, 50f, 50f, 5, 5), + CreateSample(1, 60f, 50f, 5, 6), + CreateSample(1, 60f, 60f, 6, 6) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(2)); + List sizes = new List { components[0].Count, components[1].Count }; + sizes.Sort(); + Assert.That(sizes, Is.EqualTo(new List { 2, 3 })); + } + + [Test] + public void SplitIntoConnectedComponents_WhenSamplesAreOnlyDiagonallyAdjacent_ShouldReturnSeparateComponents() + { + // Tests that diagonal-only adjacency is not treated as 4-connectivity, so two samples become two components. + List samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 20f, 2, 2) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(2)); + Assert.That(components[0].Count, Is.EqualTo(1)); + Assert.That(components[1].Count, Is.EqualTo(1)); + } + + [Test] + public void SplitIntoConnectedComponents_WhenSamplesHaveNoGridCell_ShouldTreatEachAsSeparateComponent() + { + // Tests that samples without a grid cell (Row/Column <= 0) each become their own component instead of collapsing. + List samples = new List + { + CreateSample(1, 10f, 10f), + CreateSample(1, 20f, 20f), + CreateSample(1, 30f, 30f) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(3)); + foreach (List component in components) + { + Assert.That(component.Count, Is.EqualTo(1)); + } + } + + [Test] + public void SplitIntoConnectedComponents_WhenInputIsEmpty_ShouldReturnEmpty() + { + // Tests that an empty sample list produces an empty component list without exceptions. + List samples = new List(); + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components, Is.Empty); + } + + [Test] + public void CreateComponentElements_WhenReachableClusterSplitsIntoTwoRegions_ShouldEmitTwoEntriesWithSharedMetadataAndDistinctBoundsAndSim() + { + // Tests that a single reachable cluster split into two 4-connected regions produces two element entries + // that share metadata (Name/Path/Layer/Components) but carry distinct Label/Bounds/SimX/SimY/RaycastOutlineSegments. + RaycastClusterInfo reachableCluster = new RaycastClusterInfo + { + Representative = CreateSample(1, 10f, 10f, 1, 1), + SampleCount = 4, + Samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 10f, 1, 2), + CreateSample(1, 80f, 80f, 8, 8), + CreateSample(1, 90f, 80f, 8, 9) + } + }; + RaycastColliderMetadata metadata = new RaycastColliderMetadata + { + Name = "SplitGrid", + Path = "Root/SplitGrid", + Layer = "Default", + Components = new List { "BoxCollider" } + }; + RaycastSampleCoverage coverage = CreateCoverage(5f, 5f, 0f, 0f, 200f, 200f); + + List elements = + RaycastGridAnnotator.CreateComponentElements(reachableCluster, metadata, coverage, 3); + + Assert.That(elements.Count, Is.EqualTo(2)); + + // Shared metadata across component entries. + Assert.That(elements[0].Name, Is.EqualTo("SplitGrid")); + Assert.That(elements[1].Name, Is.EqualTo("SplitGrid")); + Assert.That(elements[0].Path, Is.EqualTo("Root/SplitGrid")); + Assert.That(elements[1].Path, Is.EqualTo("Root/SplitGrid")); + Assert.That(elements[0].Layer, Is.EqualTo("Default")); + Assert.That(elements[1].Layer, Is.EqualTo("Default")); + Assert.That(elements[0].Components, Is.EqualTo(elements[1].Components)); + + // Continuous labels starting at startLabelNumber. + Assert.That(elements[0].Label, Is.EqualTo("R3")); + Assert.That(elements[1].Label, Is.EqualTo("R4")); + + // Top-left first: the (1,1)-(1,2) region must precede the (8,8)-(8,9) region. + Assert.That(elements[0].BoundsMinX, Is.LessThan(elements[1].BoundsMinX)); + Assert.That(elements[0].BoundsMinY, Is.LessThan(elements[1].BoundsMinY)); + + // Distinct Sim positions. + Assert.That(elements[0].SimX, Is.Not.EqualTo(elements[1].SimX)); + Assert.That(elements[0].SimY, Is.Not.EqualTo(elements[1].SimY)); + + // Both entries must have their own outline segments. + Assert.That(elements[0].RaycastOutlineSegments.Count, Is.GreaterThan(0)); + Assert.That(elements[1].RaycastOutlineSegments.Count, Is.GreaterThan(0)); + } + + [Test] + public void CreateComponentElements_WhenComponentsShareMinInputYAndMinInputX_ShouldOrderByMinRowColumnLexicographically() + { + // Tests that when two components share the same min InputY and min InputX, the tie is broken by + // the lexicographic minimum (Row, Column). This guarantees fully deterministic label assignment. + // Component A: single cell at (Row=1, Column=1) -> min InputY=10, min InputX=10, min (Row,Column)=(1,1). + // Component B: L-shape reaching down to (Row=3, Column=1) -> min InputY=10, min InputX=10, min (Row,Column)=(1,3). + RaycastClusterInfo reachableCluster = new RaycastClusterInfo + { + Representative = CreateSample(1, 10f, 10f, 1, 1), + SampleCount = 6, + Samples = new List + { + // Component A: isolated single cell (1,1). + CreateSample(1, 10f, 10f, 1, 1), + // Component B: (1,3) -> (2,3) -> (3,3) -> (3,2) -> (3,1). Its column 1 cell sits at Row=3. + CreateSample(1, 30f, 10f, 1, 3), + CreateSample(1, 30f, 20f, 2, 3), + CreateSample(1, 30f, 30f, 3, 3), + CreateSample(1, 20f, 30f, 3, 2), + CreateSample(1, 10f, 30f, 3, 1) + } + }; + RaycastColliderMetadata metadata = new RaycastColliderMetadata + { + Name = "TieBreaker", + Path = "Root/TieBreaker", + Layer = "Default", + Components = new List { "BoxCollider" } + }; + RaycastSampleCoverage coverage = CreateCoverage(5f, 5f, 0f, 0f, 200f, 200f); + + List elements = + RaycastGridAnnotator.CreateComponentElements(reachableCluster, metadata, coverage, 1); + + Assert.That(elements.Count, Is.EqualTo(2)); + // Component A wins the third key: (1,1) < (1,3) lexicographically, so it must receive R1. + Assert.That(elements[0].Label, Is.EqualTo("R1")); + Assert.That(elements[1].Label, Is.EqualTo("R2")); + // Sanity: R1 has one cell (small bounds), R2 has the L-shape (wider bounds). + float widthA = elements[0].BoundsMaxX - elements[0].BoundsMinX; + float widthB = elements[1].BoundsMaxX - elements[1].BoundsMinX; + Assert.That(widthA, Is.LessThan(widthB)); + } + private static RaycastClusterSample CreateSample( int clusterKey, float inputX, diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs index a21c5f079f..878cdd67db 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs @@ -112,16 +112,124 @@ internal static List CollectPhysicsColliderElements( RaycastColliderMetadata metadata = clusterCollection.MetadataByClusterKey[reachableCluster.Representative.ClusterKey]; - elements.Add(CreatePhysicsColliderElement( - $"R{elements.Count + 1}", + List componentElements = CreateComponentElements( reachableCluster, metadata, - sampleCoverage)); + sampleCoverage, + elements.Count + 1); + elements.AddRange(componentElements); } return elements; } + // Split the reachable cluster into 4-connected regions and materialize one UIElementInfo per region. + // Why: a single GameObject can produce multiple visually closed outline regions when UI occlusion + // splits its reachable samples, and agents need one annotation per closed region so they can address + // each one with its own Label, Bounds, SimX/SimY, and RaycastOutlineSegments. + internal static List CreateComponentElements( + RaycastClusterInfo reachableCluster, + RaycastColliderMetadata metadata, + RaycastSampleCoverage sampleCoverage, + int startLabelNumber) + { + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(reachableCluster.Samples); + components.Sort(CompareComponentsByTopLeft); + + List componentElements = new List(); + for (int j = 0; j < components.Count; j++) + { + List componentSamples = components[j]; + RaycastClusterInfo componentCluster = new RaycastClusterInfo + { + Samples = componentSamples, + SampleCount = componentSamples.Count, + Representative = RaycastHitClusterer.SelectRepresentativeSample(componentSamples) + }; + componentElements.Add(CreatePhysicsColliderElement( + $"R{startLabelNumber + j}", + componentCluster, + metadata, + sampleCoverage)); + } + return componentElements; + } + + // Order components so labels are assigned in a fully deterministic top-left-first sequence. + // Why: List.Sort is not stable, so two components sharing the same min InputY and min InputX + // would otherwise flip order between runs. The min (Row, Column) tiebreaker pins the order. + private static int CompareComponentsByTopLeft( + List left, + List right) + { + float leftMinY = MinInputY(left); + float rightMinY = MinInputY(right); + int yComparison = leftMinY.CompareTo(rightMinY); + if (yComparison != 0) + { + return yComparison; + } + + float leftMinX = MinInputX(left); + float rightMinX = MinInputX(right); + int xComparison = leftMinX.CompareTo(rightMinX); + if (xComparison != 0) + { + return xComparison; + } + + (int, int) leftMinCell = MinRowColumn(left); + (int, int) rightMinCell = MinRowColumn(right); + int rowComparison = leftMinCell.Item1.CompareTo(rightMinCell.Item1); + if (rowComparison != 0) + { + return rowComparison; + } + return leftMinCell.Item2.CompareTo(rightMinCell.Item2); + } + + private static float MinInputX(List samples) + { + float minValue = samples[0].InputX; + for (int i = 1; i < samples.Count; i++) + { + if (samples[i].InputX < minValue) + { + minValue = samples[i].InputX; + } + } + return minValue; + } + + private static float MinInputY(List samples) + { + float minValue = samples[0].InputY; + for (int i = 1; i < samples.Count; i++) + { + if (samples[i].InputY < minValue) + { + minValue = samples[i].InputY; + } + } + return minValue; + } + + private static (int, int) MinRowColumn(List samples) + { + (int, int) minCell = (samples[0].Row, samples[0].Column); + for (int i = 1; i < samples.Count; i++) + { + (int, int) candidate = (samples[i].Row, samples[i].Column); + if (candidate.Item1 < minCell.Item1 || + (candidate.Item1 == minCell.Item1 && candidate.Item2 < minCell.Item2)) + { + minCell = candidate; + } + } + return minCell; + } + private static RaycastLayerHitSample CreateLayerHitSample(GameViewRaycastResult raycastResult) { RaycastLayerHitSample sample = new RaycastLayerHitSample @@ -300,7 +408,10 @@ internal static int CreateClusterKey(Collider collider) { Debug.Assert(collider != null, "Collider is required for raycast clustering."); - // A placement area can use several colliders on one object; one annotation should describe the clickable object. + // Cluster by GameObject so multi-collider objects share metadata (name, path, layer, components). + // The 4-connected component split in CollectPhysicsColliderElements re-derives one annotation per + // visually closed region so a single GameObject can produce multiple entries when UI occlusion + // splits its reachable samples. return collider!.gameObject.GetInstanceID(); } diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs index e291731dfe..228e496f13 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs @@ -69,6 +69,80 @@ internal static RaycastClusterSample SelectRepresentativeSample(List> SplitIntoConnectedComponents( + List reachableSamples) + { + System.Diagnostics.Debug.Assert(reachableSamples != null, "Reachable samples must not be null."); + List validSamples = reachableSamples!; + + List> components = new List>(); + Dictionary<(int, int), RaycastClusterSample> cellToSample = + new Dictionary<(int, int), RaycastClusterSample>(); + List<(int, int)> gridCellOrder = new List<(int, int)>(); + + foreach (RaycastClusterSample sample in validSamples) + { + if (sample.Row > 0 && sample.Column > 0) + { + (int, int) cell = (sample.Row, sample.Column); + System.Diagnostics.Debug.Assert( + !cellToSample.ContainsKey(cell), + "Reachable samples must not contain duplicate (Row, Column) cells."); + cellToSample.Add(cell, sample); + gridCellOrder.Add(cell); + continue; + } + components.Add(new List { sample }); + } + + HashSet<(int, int)> visited = new HashSet<(int, int)>(); + foreach ((int, int) startCell in gridCellOrder) + { + if (visited.Contains(startCell)) + { + continue; + } + + List component = new List(); + Queue<(int, int)> queue = new Queue<(int, int)>(); + queue.Enqueue(startCell); + visited.Add(startCell); + while (queue.Count > 0) + { + (int row, int column) current = queue.Dequeue(); + component.Add(cellToSample[current]); + + (int, int)[] neighbors = + { + (current.row - 1, current.column), + (current.row + 1, current.column), + (current.row, current.column - 1), + (current.row, current.column + 1) + }; + foreach ((int, int) neighbor in neighbors) + { + if (visited.Contains(neighbor)) + { + continue; + } + if (!cellToSample.ContainsKey(neighbor)) + { + continue; + } + visited.Add(neighbor); + queue.Enqueue(neighbor); + } + } + components.Add(component); + } + + return components; + } + internal static RaycastClusterSample? SelectReachableRepresentativeSample( List samples, RaycastClusterSampleOcclusionCheck isOccluded) diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md index 1ec7d61539..79f674f193 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md @@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. +### PhysicsCollider Entries Per Closed Region + +When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable. + +When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path ` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter. + ## RaycastLayerSummaries Fields `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`.