feat: Raycast annotation now labels every UI-split closed region separately

.agents/skills/uloop-screenshot/references/annotated-elements.md

@@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme
 - `SortingOrder`: Canvas sorting order. Higher values are in front.
 - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies.
 
+### PhysicsCollider Entries Per Closed Region
+
+When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable.
+
+When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path <Path>` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter.
+
 ## RaycastLayerSummaries Fields
 
 `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`.

.claude/skills/uloop-screenshot/references/annotated-elements.md

@@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme
 - `SortingOrder`: Canvas sorting order. Higher values are in front.
 - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies.
 
+### PhysicsCollider Entries Per Closed Region
+
+When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable.
+
+When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path <Path>` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter.
+
 ## RaycastLayerSummaries Fields
 
 `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`.

Assets/Tests/Editor/RaycastGridAnnotatorTests.cs

@@ -657,6 +657,199 @@ public void CreateLayerSummaries_WhenNoHits_ShouldReturnEmptyList()
             Assert.That(summaries, Is.Empty);
         }
 
+        [Test]
+        public void SplitIntoConnectedComponents_WhenSamplesForm4ConnectedRegion_ShouldReturnOneComponent()
+        {
+            // Tests that an L-shaped set of 4-connected samples collapses into a single connected component.
+            List<RaycastClusterSample> samples = new List<RaycastClusterSample>
+            {
+                CreateSample(1, 10f, 10f, 1, 1),
+                CreateSample(1, 20f, 10f, 1, 2),
+                CreateSample(1, 10f, 20f, 2, 1)
+            };
+
+            List<List<RaycastClusterSample>> components =
+                RaycastHitClusterer.SplitIntoConnectedComponents(samples);
+
+            Assert.That(components.Count, Is.EqualTo(1));
+            Assert.That(components[0].Count, Is.EqualTo(3));
+        }
+
+        [Test]
+        public void SplitIntoConnectedComponents_WhenSamplesFormTwoDisconnectedRegions_ShouldReturnTwoComponents()
+        {
+            // Tests that two spatially disconnected 4-connected regions come back as two separate components.
+            List<RaycastClusterSample> samples = new List<RaycastClusterSample>
+            {
+                CreateSample(1, 10f, 10f, 1, 1),
+                CreateSample(1, 20f, 10f, 1, 2),
+                CreateSample(1, 50f, 50f, 5, 5),
+                CreateSample(1, 60f, 50f, 5, 6),
+                CreateSample(1, 60f, 60f, 6, 6)
+            };
+
+            List<List<RaycastClusterSample>> components =
+                RaycastHitClusterer.SplitIntoConnectedComponents(samples);
+
+            Assert.That(components.Count, Is.EqualTo(2));
+            List<int> sizes = new List<int> { components[0].Count, components[1].Count };
+            sizes.Sort();
+            Assert.That(sizes, Is.EqualTo(new List<int> { 2, 3 }));
+        }
+
+        [Test]
+        public void SplitIntoConnectedComponents_WhenSamplesAreOnlyDiagonallyAdjacent_ShouldReturnSeparateComponents()
+        {
+            // Tests that diagonal-only adjacency is not treated as 4-connectivity, so two samples become two components.
+            List<RaycastClusterSample> samples = new List<RaycastClusterSample>
+            {
+                CreateSample(1, 10f, 10f, 1, 1),
+                CreateSample(1, 20f, 20f, 2, 2)
+            };
+
+            List<List<RaycastClusterSample>> components =
+                RaycastHitClusterer.SplitIntoConnectedComponents(samples);
+
+            Assert.That(components.Count, Is.EqualTo(2));
+            Assert.That(components[0].Count, Is.EqualTo(1));
+            Assert.That(components[1].Count, Is.EqualTo(1));
+        }
+
+        [Test]
+        public void SplitIntoConnectedComponents_WhenSamplesHaveNoGridCell_ShouldTreatEachAsSeparateComponent()
+        {
+            // Tests that samples without a grid cell (Row/Column <= 0) each become their own component instead of collapsing.
+            List<RaycastClusterSample> samples = new List<RaycastClusterSample>
+            {
+                CreateSample(1, 10f, 10f),
+                CreateSample(1, 20f, 20f),
+                CreateSample(1, 30f, 30f)
+            };
+
+            List<List<RaycastClusterSample>> components =
+                RaycastHitClusterer.SplitIntoConnectedComponents(samples);
+
+            Assert.That(components.Count, Is.EqualTo(3));
+            foreach (List<RaycastClusterSample> component in components)
+            {
+                Assert.That(component.Count, Is.EqualTo(1));
+            }
+        }
+
+        [Test]
+        public void SplitIntoConnectedComponents_WhenInputIsEmpty_ShouldReturnEmpty()
+        {
+            // Tests that an empty sample list produces an empty component list without exceptions.
+            List<RaycastClusterSample> samples = new List<RaycastClusterSample>();
+
+            List<List<RaycastClusterSample>> components =
+                RaycastHitClusterer.SplitIntoConnectedComponents(samples);
+
+            Assert.That(components, Is.Empty);
+        }
+
+        [Test]
+        public void CreateComponentElements_WhenReachableClusterSplitsIntoTwoRegions_ShouldEmitTwoEntriesWithSharedMetadataAndDistinctBoundsAndSim()
+        {
+            // Tests that a single reachable cluster split into two 4-connected regions produces two element entries
+            // that share metadata (Name/Path/Layer/Components) but carry distinct Label/Bounds/SimX/SimY/RaycastOutlineSegments.
+            RaycastClusterInfo reachableCluster = new RaycastClusterInfo
+            {
+                Representative = CreateSample(1, 10f, 10f, 1, 1),
+                SampleCount = 4,
+                Samples = new List<RaycastClusterSample>
+                {
+                    CreateSample(1, 10f, 10f, 1, 1),
+                    CreateSample(1, 20f, 10f, 1, 2),
+                    CreateSample(1, 80f, 80f, 8, 8),
+                    CreateSample(1, 90f, 80f, 8, 9)
+                }
+            };
+            RaycastColliderMetadata metadata = new RaycastColliderMetadata
+            {
+                Name = "SplitGrid",
+                Path = "Root/SplitGrid",
+                Layer = "Default",
+                Components = new List<string> { "BoxCollider" }
+            };
+            RaycastSampleCoverage coverage = CreateCoverage(5f, 5f, 0f, 0f, 200f, 200f);
+
+            List<UIElementInfo> elements =
+                RaycastGridAnnotator.CreateComponentElements(reachableCluster, metadata, coverage, 3);
+
+            Assert.That(elements.Count, Is.EqualTo(2));
+
+            // Shared metadata across component entries.
+            Assert.That(elements[0].Name, Is.EqualTo("SplitGrid"));
+            Assert.That(elements[1].Name, Is.EqualTo("SplitGrid"));
+            Assert.That(elements[0].Path, Is.EqualTo("Root/SplitGrid"));
+            Assert.That(elements[1].Path, Is.EqualTo("Root/SplitGrid"));
+            Assert.That(elements[0].Layer, Is.EqualTo("Default"));
+            Assert.That(elements[1].Layer, Is.EqualTo("Default"));
+            Assert.That(elements[0].Components, Is.EqualTo(elements[1].Components));
+
+            // Continuous labels starting at startLabelNumber.
+            Assert.That(elements[0].Label, Is.EqualTo("R3"));
+            Assert.That(elements[1].Label, Is.EqualTo("R4"));
+
+            // Top-left first: the (1,1)-(1,2) region must precede the (8,8)-(8,9) region.
+            Assert.That(elements[0].BoundsMinX, Is.LessThan(elements[1].BoundsMinX));
+            Assert.That(elements[0].BoundsMinY, Is.LessThan(elements[1].BoundsMinY));
+
+            // Distinct Sim positions.
+            Assert.That(elements[0].SimX, Is.Not.EqualTo(elements[1].SimX));
+            Assert.That(elements[0].SimY, Is.Not.EqualTo(elements[1].SimY));
+
+            // Both entries must have their own outline segments.
+            Assert.That(elements[0].RaycastOutlineSegments.Count, Is.GreaterThan(0));
+            Assert.That(elements[1].RaycastOutlineSegments.Count, Is.GreaterThan(0));
+        }
+
+        [Test]
+        public void CreateComponentElements_WhenComponentsShareMinInputYAndMinInputX_ShouldOrderByMinRowColumnLexicographically()
+        {
+            // Tests that when two components share the same min InputY and min InputX, the tie is broken by
+            // the lexicographic minimum (Row, Column). This guarantees fully deterministic label assignment.
+            // Component A: single cell at (Row=1, Column=1) -> min InputY=10, min InputX=10, min (Row,Column)=(1,1).
+            // Component B: L-shape reaching down to (Row=3, Column=1) -> min InputY=10, min InputX=10, min (Row,Column)=(1,3).
+            RaycastClusterInfo reachableCluster = new RaycastClusterInfo
+            {
+                Representative = CreateSample(1, 10f, 10f, 1, 1),
+                SampleCount = 6,
+                Samples = new List<RaycastClusterSample>
+                {
+                    // Component A: isolated single cell (1,1).
+                    CreateSample(1, 10f, 10f, 1, 1),
+                    // Component B: (1,3) -> (2,3) -> (3,3) -> (3,2) -> (3,1). Its column 1 cell sits at Row=3.
+                    CreateSample(1, 30f, 10f, 1, 3),
+                    CreateSample(1, 30f, 20f, 2, 3),
+                    CreateSample(1, 30f, 30f, 3, 3),
+                    CreateSample(1, 20f, 30f, 3, 2),
+                    CreateSample(1, 10f, 30f, 3, 1)
+                }
+            };
+            RaycastColliderMetadata metadata = new RaycastColliderMetadata
+            {
+                Name = "TieBreaker",
+                Path = "Root/TieBreaker",
+                Layer = "Default",
+                Components = new List<string> { "BoxCollider" }
+            };
+            RaycastSampleCoverage coverage = CreateCoverage(5f, 5f, 0f, 0f, 200f, 200f);
+
+            List<UIElementInfo> elements =
+                RaycastGridAnnotator.CreateComponentElements(reachableCluster, metadata, coverage, 1);
+
+            Assert.That(elements.Count, Is.EqualTo(2));
+            // Component A wins the third key: (1,1) < (1,3) lexicographically, so it must receive R1.
+            Assert.That(elements[0].Label, Is.EqualTo("R1"));
+            Assert.That(elements[1].Label, Is.EqualTo("R2"));
+            // Sanity: R1 has one cell (small bounds), R2 has the L-shape (wider bounds).
+            float widthA = elements[0].BoundsMaxX - elements[0].BoundsMinX;
+            float widthB = elements[1].BoundsMaxX - elements[1].BoundsMinX;
+            Assert.That(widthA, Is.LessThan(widthB));
+        }
+
         private static RaycastClusterSample CreateSample(
             int clusterKey,
             float inputX,

Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs

@@ -112,16 +112,124 @@ internal static List<UIElementInfo> CollectPhysicsColliderElements(
 
                 RaycastColliderMetadata metadata =
                     clusterCollection.MetadataByClusterKey[reachableCluster.Representative.ClusterKey];
-                elements.Add(CreatePhysicsColliderElement(
-                    $"R{elements.Count + 1}",
+                List<UIElementInfo> componentElements = CreateComponentElements(
                     reachableCluster,
                     metadata,
-                    sampleCoverage));
+                    sampleCoverage,
+                    elements.Count + 1);
+                elements.AddRange(componentElements);
             }
 
             return elements;
         }
 
+        // Split the reachable cluster into 4-connected regions and materialize one UIElementInfo per region.
+        // Why: a single GameObject can produce multiple visually closed outline regions when UI occlusion
+        // splits its reachable samples, and agents need one annotation per closed region so they can address
+        // each one with its own Label, Bounds, SimX/SimY, and RaycastOutlineSegments.
+        internal static List<UIElementInfo> CreateComponentElements(
+            RaycastClusterInfo reachableCluster,
+            RaycastColliderMetadata metadata,
+            RaycastSampleCoverage sampleCoverage,
+            int startLabelNumber)
+        {
+            List<List<RaycastClusterSample>> components =
+                RaycastHitClusterer.SplitIntoConnectedComponents(reachableCluster.Samples);
+            components.Sort(CompareComponentsByTopLeft);
+
+            List<UIElementInfo> componentElements = new List<UIElementInfo>();
+            for (int j = 0; j < components.Count; j++)
+            {
+                List<RaycastClusterSample> componentSamples = components[j];
+                RaycastClusterInfo componentCluster = new RaycastClusterInfo
+                {
+                    Samples = componentSamples,
+                    SampleCount = componentSamples.Count,
+                    Representative = RaycastHitClusterer.SelectRepresentativeSample(componentSamples)
+                };
+                componentElements.Add(CreatePhysicsColliderElement(
+                    $"R{startLabelNumber + j}",
+                    componentCluster,
+                    metadata,
+                    sampleCoverage));
+            }
+            return componentElements;
+        }
+
+        // Order components so labels are assigned in a fully deterministic top-left-first sequence.
+        // Why: List<T>.Sort is not stable, so two components sharing the same min InputY and min InputX
+        // would otherwise flip order between runs. The min (Row, Column) tiebreaker pins the order.
+        private static int CompareComponentsByTopLeft(
+            List<RaycastClusterSample> left,
+            List<RaycastClusterSample> right)
+        {
+            float leftMinY = MinInputY(left);
+            float rightMinY = MinInputY(right);
+            int yComparison = leftMinY.CompareTo(rightMinY);
+            if (yComparison != 0)
+            {
+                return yComparison;
+            }
+
+            float leftMinX = MinInputX(left);
+            float rightMinX = MinInputX(right);
+            int xComparison = leftMinX.CompareTo(rightMinX);
+            if (xComparison != 0)
+            {
+                return xComparison;
+            }
+
+            (int, int) leftMinCell = MinRowColumn(left);
+            (int, int) rightMinCell = MinRowColumn(right);
+            int rowComparison = leftMinCell.Item1.CompareTo(rightMinCell.Item1);
+            if (rowComparison != 0)
+            {
+                return rowComparison;
+            }
+            return leftMinCell.Item2.CompareTo(rightMinCell.Item2);
+        }
+
+        private static float MinInputX(List<RaycastClusterSample> samples)
+        {
+            float minValue = samples[0].InputX;
+            for (int i = 1; i < samples.Count; i++)
+            {
+                if (samples[i].InputX < minValue)
+                {
+                    minValue = samples[i].InputX;
+                }
+            }
+            return minValue;
+        }
+
+        private static float MinInputY(List<RaycastClusterSample> samples)
+        {
+            float minValue = samples[0].InputY;
+            for (int i = 1; i < samples.Count; i++)
+            {
+                if (samples[i].InputY < minValue)
+                {
+                    minValue = samples[i].InputY;
+                }
+            }
+            return minValue;
+        }
+
+        private static (int, int) MinRowColumn(List<RaycastClusterSample> samples)
+        {
+            (int, int) minCell = (samples[0].Row, samples[0].Column);
+            for (int i = 1; i < samples.Count; i++)
+            {
+                (int, int) candidate = (samples[i].Row, samples[i].Column);
+                if (candidate.Item1 < minCell.Item1 ||
+                    (candidate.Item1 == minCell.Item1 && candidate.Item2 < minCell.Item2))
+                {
+                    minCell = candidate;
+                }
+            }
+            return minCell;
+        }
+
         private static RaycastLayerHitSample CreateLayerHitSample(GameViewRaycastResult raycastResult)
         {
             RaycastLayerHitSample sample = new RaycastLayerHitSample
@@ -300,7 +408,10 @@ internal static int CreateClusterKey(Collider collider)
         {
             Debug.Assert(collider != null, "Collider is required for raycast clustering.");
 
-            // A placement area can use several colliders on one object; one annotation should describe the clickable object.
+            // Cluster by GameObject so multi-collider objects share metadata (name, path, layer, components).
+            // The 4-connected component split in CollectPhysicsColliderElements re-derives one annotation per
+            // visually closed region so a single GameObject can produce multiple entries when UI occlusion
+            // splits its reachable samples.
             return collider!.gameObject.GetInstanceID();
         }