From 21d668842471b575f86ff463b9bf93f36264b0fe Mon Sep 17 00:00:00 2001 From: hatayama Date: Fri, 10 Jul 2026 01:51:17 +0900 Subject: [PATCH 1/2] Split raycast annotation entries per closed screen-space region Previously a single PhysicsCollider GameObject produced one AnnotatedElements entry even when UI occlusion carved its reachable samples into multiple visually closed regions on screen. Agents could only address one of those regions with SimX/SimY, and the screenshot overlay drew a single label even when several disjoint outlines were visible. Split the reachable cluster into 4-connected components (matching the outline builder's neighbor definition) and emit one AnnotatedElements entry per component. Path, Name, Layer, and Components stay shared across sibling entries; Label, Bounds, SimX/SimY, and RaycastOutlineSegments are independent so each closed area is separately clickable. Label numbers remain a single R1/R2/R3... sequence across all entries. Component ordering uses min InputY, then min InputX, with a min (Row, Column) lexicographic tiebreaker so the label sequence is fully deterministic across runs even when the first two keys tie. --- .../Tests/Editor/RaycastGridAnnotatorTests.cs | 193 ++++++++++++++++++ .../RaycastAnnotation/RaycastGridAnnotator.cs | 119 ++++++++++- .../RaycastAnnotation/RaycastHitClusterer.cs | 74 +++++++ 3 files changed, 382 insertions(+), 4 deletions(-) diff --git a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs index ee716cc466..6ac046693a 100644 --- a/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs +++ b/Assets/Tests/Editor/RaycastGridAnnotatorTests.cs @@ -657,6 +657,199 @@ public void CreateLayerSummaries_WhenNoHits_ShouldReturnEmptyList() Assert.That(summaries, Is.Empty); } + [Test] + public void SplitIntoConnectedComponents_WhenSamplesForm4ConnectedRegion_ShouldReturnOneComponent() + { + // Tests that an L-shaped set of 4-connected samples collapses into a single connected component. + List samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 10f, 1, 2), + CreateSample(1, 10f, 20f, 2, 1) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(1)); + Assert.That(components[0].Count, Is.EqualTo(3)); + } + + [Test] + public void SplitIntoConnectedComponents_WhenSamplesFormTwoDisconnectedRegions_ShouldReturnTwoComponents() + { + // Tests that two spatially disconnected 4-connected regions come back as two separate components. + List samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 10f, 1, 2), + CreateSample(1, 50f, 50f, 5, 5), + CreateSample(1, 60f, 50f, 5, 6), + CreateSample(1, 60f, 60f, 6, 6) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(2)); + List sizes = new List { components[0].Count, components[1].Count }; + sizes.Sort(); + Assert.That(sizes, Is.EqualTo(new List { 2, 3 })); + } + + [Test] + public void SplitIntoConnectedComponents_WhenSamplesAreOnlyDiagonallyAdjacent_ShouldReturnSeparateComponents() + { + // Tests that diagonal-only adjacency is not treated as 4-connectivity, so two samples become two components. + List samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 20f, 2, 2) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(2)); + Assert.That(components[0].Count, Is.EqualTo(1)); + Assert.That(components[1].Count, Is.EqualTo(1)); + } + + [Test] + public void SplitIntoConnectedComponents_WhenSamplesHaveNoGridCell_ShouldTreatEachAsSeparateComponent() + { + // Tests that samples without a grid cell (Row/Column <= 0) each become their own component instead of collapsing. + List samples = new List + { + CreateSample(1, 10f, 10f), + CreateSample(1, 20f, 20f), + CreateSample(1, 30f, 30f) + }; + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components.Count, Is.EqualTo(3)); + foreach (List component in components) + { + Assert.That(component.Count, Is.EqualTo(1)); + } + } + + [Test] + public void SplitIntoConnectedComponents_WhenInputIsEmpty_ShouldReturnEmpty() + { + // Tests that an empty sample list produces an empty component list without exceptions. + List samples = new List(); + + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(samples); + + Assert.That(components, Is.Empty); + } + + [Test] + public void CreateComponentElements_WhenReachableClusterSplitsIntoTwoRegions_ShouldEmitTwoEntriesWithSharedMetadataAndDistinctBoundsAndSim() + { + // Tests that a single reachable cluster split into two 4-connected regions produces two element entries + // that share metadata (Name/Path/Layer/Components) but carry distinct Label/Bounds/SimX/SimY/RaycastOutlineSegments. + RaycastClusterInfo reachableCluster = new RaycastClusterInfo + { + Representative = CreateSample(1, 10f, 10f, 1, 1), + SampleCount = 4, + Samples = new List + { + CreateSample(1, 10f, 10f, 1, 1), + CreateSample(1, 20f, 10f, 1, 2), + CreateSample(1, 80f, 80f, 8, 8), + CreateSample(1, 90f, 80f, 8, 9) + } + }; + RaycastColliderMetadata metadata = new RaycastColliderMetadata + { + Name = "SplitGrid", + Path = "Root/SplitGrid", + Layer = "Default", + Components = new List { "BoxCollider" } + }; + RaycastSampleCoverage coverage = CreateCoverage(5f, 5f, 0f, 0f, 200f, 200f); + + List elements = + RaycastGridAnnotator.CreateComponentElements(reachableCluster, metadata, coverage, 3); + + Assert.That(elements.Count, Is.EqualTo(2)); + + // Shared metadata across component entries. + Assert.That(elements[0].Name, Is.EqualTo("SplitGrid")); + Assert.That(elements[1].Name, Is.EqualTo("SplitGrid")); + Assert.That(elements[0].Path, Is.EqualTo("Root/SplitGrid")); + Assert.That(elements[1].Path, Is.EqualTo("Root/SplitGrid")); + Assert.That(elements[0].Layer, Is.EqualTo("Default")); + Assert.That(elements[1].Layer, Is.EqualTo("Default")); + Assert.That(elements[0].Components, Is.EqualTo(elements[1].Components)); + + // Continuous labels starting at startLabelNumber. + Assert.That(elements[0].Label, Is.EqualTo("R3")); + Assert.That(elements[1].Label, Is.EqualTo("R4")); + + // Top-left first: the (1,1)-(1,2) region must precede the (8,8)-(8,9) region. + Assert.That(elements[0].BoundsMinX, Is.LessThan(elements[1].BoundsMinX)); + Assert.That(elements[0].BoundsMinY, Is.LessThan(elements[1].BoundsMinY)); + + // Distinct Sim positions. + Assert.That(elements[0].SimX, Is.Not.EqualTo(elements[1].SimX)); + Assert.That(elements[0].SimY, Is.Not.EqualTo(elements[1].SimY)); + + // Both entries must have their own outline segments. + Assert.That(elements[0].RaycastOutlineSegments.Count, Is.GreaterThan(0)); + Assert.That(elements[1].RaycastOutlineSegments.Count, Is.GreaterThan(0)); + } + + [Test] + public void CreateComponentElements_WhenComponentsShareMinInputYAndMinInputX_ShouldOrderByMinRowColumnLexicographically() + { + // Tests that when two components share the same min InputY and min InputX, the tie is broken by + // the lexicographic minimum (Row, Column). This guarantees fully deterministic label assignment. + // Component A: single cell at (Row=1, Column=1) -> min InputY=10, min InputX=10, min (Row,Column)=(1,1). + // Component B: L-shape reaching down to (Row=3, Column=1) -> min InputY=10, min InputX=10, min (Row,Column)=(1,3). + RaycastClusterInfo reachableCluster = new RaycastClusterInfo + { + Representative = CreateSample(1, 10f, 10f, 1, 1), + SampleCount = 6, + Samples = new List + { + // Component A: isolated single cell (1,1). + CreateSample(1, 10f, 10f, 1, 1), + // Component B: (1,3) -> (2,3) -> (3,3) -> (3,2) -> (3,1). Its column 1 cell sits at Row=3. + CreateSample(1, 30f, 10f, 1, 3), + CreateSample(1, 30f, 20f, 2, 3), + CreateSample(1, 30f, 30f, 3, 3), + CreateSample(1, 20f, 30f, 3, 2), + CreateSample(1, 10f, 30f, 3, 1) + } + }; + RaycastColliderMetadata metadata = new RaycastColliderMetadata + { + Name = "TieBreaker", + Path = "Root/TieBreaker", + Layer = "Default", + Components = new List { "BoxCollider" } + }; + RaycastSampleCoverage coverage = CreateCoverage(5f, 5f, 0f, 0f, 200f, 200f); + + List elements = + RaycastGridAnnotator.CreateComponentElements(reachableCluster, metadata, coverage, 1); + + Assert.That(elements.Count, Is.EqualTo(2)); + // Component A wins the third key: (1,1) < (1,3) lexicographically, so it must receive R1. + Assert.That(elements[0].Label, Is.EqualTo("R1")); + Assert.That(elements[1].Label, Is.EqualTo("R2")); + // Sanity: R1 has one cell (small bounds), R2 has the L-shape (wider bounds). + float widthA = elements[0].BoundsMaxX - elements[0].BoundsMinX; + float widthB = elements[1].BoundsMaxX - elements[1].BoundsMinX; + Assert.That(widthA, Is.LessThan(widthB)); + } + private static RaycastClusterSample CreateSample( int clusterKey, float inputX, diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs index a21c5f079f..878cdd67db 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastGridAnnotator.cs @@ -112,16 +112,124 @@ internal static List CollectPhysicsColliderElements( RaycastColliderMetadata metadata = clusterCollection.MetadataByClusterKey[reachableCluster.Representative.ClusterKey]; - elements.Add(CreatePhysicsColliderElement( - $"R{elements.Count + 1}", + List componentElements = CreateComponentElements( reachableCluster, metadata, - sampleCoverage)); + sampleCoverage, + elements.Count + 1); + elements.AddRange(componentElements); } return elements; } + // Split the reachable cluster into 4-connected regions and materialize one UIElementInfo per region. + // Why: a single GameObject can produce multiple visually closed outline regions when UI occlusion + // splits its reachable samples, and agents need one annotation per closed region so they can address + // each one with its own Label, Bounds, SimX/SimY, and RaycastOutlineSegments. + internal static List CreateComponentElements( + RaycastClusterInfo reachableCluster, + RaycastColliderMetadata metadata, + RaycastSampleCoverage sampleCoverage, + int startLabelNumber) + { + List> components = + RaycastHitClusterer.SplitIntoConnectedComponents(reachableCluster.Samples); + components.Sort(CompareComponentsByTopLeft); + + List componentElements = new List(); + for (int j = 0; j < components.Count; j++) + { + List componentSamples = components[j]; + RaycastClusterInfo componentCluster = new RaycastClusterInfo + { + Samples = componentSamples, + SampleCount = componentSamples.Count, + Representative = RaycastHitClusterer.SelectRepresentativeSample(componentSamples) + }; + componentElements.Add(CreatePhysicsColliderElement( + $"R{startLabelNumber + j}", + componentCluster, + metadata, + sampleCoverage)); + } + return componentElements; + } + + // Order components so labels are assigned in a fully deterministic top-left-first sequence. + // Why: List.Sort is not stable, so two components sharing the same min InputY and min InputX + // would otherwise flip order between runs. The min (Row, Column) tiebreaker pins the order. + private static int CompareComponentsByTopLeft( + List left, + List right) + { + float leftMinY = MinInputY(left); + float rightMinY = MinInputY(right); + int yComparison = leftMinY.CompareTo(rightMinY); + if (yComparison != 0) + { + return yComparison; + } + + float leftMinX = MinInputX(left); + float rightMinX = MinInputX(right); + int xComparison = leftMinX.CompareTo(rightMinX); + if (xComparison != 0) + { + return xComparison; + } + + (int, int) leftMinCell = MinRowColumn(left); + (int, int) rightMinCell = MinRowColumn(right); + int rowComparison = leftMinCell.Item1.CompareTo(rightMinCell.Item1); + if (rowComparison != 0) + { + return rowComparison; + } + return leftMinCell.Item2.CompareTo(rightMinCell.Item2); + } + + private static float MinInputX(List samples) + { + float minValue = samples[0].InputX; + for (int i = 1; i < samples.Count; i++) + { + if (samples[i].InputX < minValue) + { + minValue = samples[i].InputX; + } + } + return minValue; + } + + private static float MinInputY(List samples) + { + float minValue = samples[0].InputY; + for (int i = 1; i < samples.Count; i++) + { + if (samples[i].InputY < minValue) + { + minValue = samples[i].InputY; + } + } + return minValue; + } + + private static (int, int) MinRowColumn(List samples) + { + (int, int) minCell = (samples[0].Row, samples[0].Column); + for (int i = 1; i < samples.Count; i++) + { + (int, int) candidate = (samples[i].Row, samples[i].Column); + if (candidate.Item1 < minCell.Item1 || + (candidate.Item1 == minCell.Item1 && candidate.Item2 < minCell.Item2)) + { + minCell = candidate; + } + } + return minCell; + } + private static RaycastLayerHitSample CreateLayerHitSample(GameViewRaycastResult raycastResult) { RaycastLayerHitSample sample = new RaycastLayerHitSample @@ -300,7 +408,10 @@ internal static int CreateClusterKey(Collider collider) { Debug.Assert(collider != null, "Collider is required for raycast clustering."); - // A placement area can use several colliders on one object; one annotation should describe the clickable object. + // Cluster by GameObject so multi-collider objects share metadata (name, path, layer, components). + // The 4-connected component split in CollectPhysicsColliderElements re-derives one annotation per + // visually closed region so a single GameObject can produce multiple entries when UI occlusion + // splits its reachable samples. return collider!.gameObject.GetInstanceID(); } diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs index e291731dfe..228e496f13 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/RaycastAnnotation/RaycastHitClusterer.cs @@ -69,6 +69,80 @@ internal static RaycastClusterSample SelectRepresentativeSample(List> SplitIntoConnectedComponents( + List reachableSamples) + { + System.Diagnostics.Debug.Assert(reachableSamples != null, "Reachable samples must not be null."); + List validSamples = reachableSamples!; + + List> components = new List>(); + Dictionary<(int, int), RaycastClusterSample> cellToSample = + new Dictionary<(int, int), RaycastClusterSample>(); + List<(int, int)> gridCellOrder = new List<(int, int)>(); + + foreach (RaycastClusterSample sample in validSamples) + { + if (sample.Row > 0 && sample.Column > 0) + { + (int, int) cell = (sample.Row, sample.Column); + System.Diagnostics.Debug.Assert( + !cellToSample.ContainsKey(cell), + "Reachable samples must not contain duplicate (Row, Column) cells."); + cellToSample.Add(cell, sample); + gridCellOrder.Add(cell); + continue; + } + components.Add(new List { sample }); + } + + HashSet<(int, int)> visited = new HashSet<(int, int)>(); + foreach ((int, int) startCell in gridCellOrder) + { + if (visited.Contains(startCell)) + { + continue; + } + + List component = new List(); + Queue<(int, int)> queue = new Queue<(int, int)>(); + queue.Enqueue(startCell); + visited.Add(startCell); + while (queue.Count > 0) + { + (int row, int column) current = queue.Dequeue(); + component.Add(cellToSample[current]); + + (int, int)[] neighbors = + { + (current.row - 1, current.column), + (current.row + 1, current.column), + (current.row, current.column - 1), + (current.row, current.column + 1) + }; + foreach ((int, int) neighbor in neighbors) + { + if (visited.Contains(neighbor)) + { + continue; + } + if (!cellToSample.ContainsKey(neighbor)) + { + continue; + } + visited.Add(neighbor); + queue.Enqueue(neighbor); + } + } + components.Add(component); + } + + return components; + } + internal static RaycastClusterSample? SelectReachableRepresentativeSample( List samples, RaycastClusterSampleOcclusionCheck isOccluded) From a58dfefa3f2f4f418385020b4fde42213ea88630 Mon Sep 17 00:00:00 2001 From: hatayama Date: Fri, 10 Jul 2026 01:51:27 +0900 Subject: [PATCH 2/2] Document per-region PhysicsCollider entries in AnnotatedElements guide Explain that a single PhysicsCollider GameObject can now appear as multiple AnnotatedElements entries when UI occlusion splits its reachable samples into disjoint 4-connected regions, and describe how simulate-mouse-ui users should reach a specific region (SimX/SimY) versus letting Unity pick one (--target-path). --- .../uloop-screenshot/references/annotated-elements.md | 6 ++++++ .../uloop-screenshot/references/annotated-elements.md | 6 ++++++ .../Screenshot/Skill/references/annotated-elements.md | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/.agents/skills/uloop-screenshot/references/annotated-elements.md b/.agents/skills/uloop-screenshot/references/annotated-elements.md index 1ec7d61539..79f674f193 100644 --- a/.agents/skills/uloop-screenshot/references/annotated-elements.md +++ b/.agents/skills/uloop-screenshot/references/annotated-elements.md @@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. +### PhysicsCollider Entries Per Closed Region + +When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable. + +When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path ` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter. + ## RaycastLayerSummaries Fields `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`. diff --git a/.claude/skills/uloop-screenshot/references/annotated-elements.md b/.claude/skills/uloop-screenshot/references/annotated-elements.md index 1ec7d61539..79f674f193 100644 --- a/.claude/skills/uloop-screenshot/references/annotated-elements.md +++ b/.claude/skills/uloop-screenshot/references/annotated-elements.md @@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. +### PhysicsCollider Entries Per Closed Region + +When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable. + +When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path ` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter. + ## RaycastLayerSummaries Fields `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`. diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md index 1ec7d61539..79f674f193 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md @@ -18,6 +18,12 @@ Read this when using `uloop screenshot --capture-mode rendering --annotate-eleme - `SortingOrder`: Canvas sorting order. Higher values are in front. - `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies. +### PhysicsCollider Entries Per Closed Region + +When a single `PhysicsCollider` GameObject's reachable raycast samples form multiple 4-connected regions on screen — typically because UI occlusion splits them apart — each region becomes its own `AnnotatedElements` entry. `Path`, `Name`, `Layer`, and `Components` are identical across those entries (they describe the same GameObject), while `Label`, `Bounds`, `SimX`, `SimY`, and `RaycastOutlineSegments` are independent per region so each closed area is separately addressable. + +When multiple entries share the same `Path`, use `SimX`/`SimY` (or the label position drawn on the PNG) to click a specific region. `simulate-mouse-ui --target-path ` still works and reaches the GameObject through whichever region is clickable, so use it when the region choice does not matter. + ## RaycastLayerSummaries Fields `RaycastLayerSummaries` is always populated when `--annotate-raycast-grid true` is used, regardless of `--raycast-layer-mask`. It is built from a dense 40x40 raycast sample pass over `Physics.DefaultRaycastLayers` (fixed, independent of `--raycast-layer-mask`), so it always tells you what else is hittable across every default-visible layer, even when you narrowed `AnnotatedElements` down to one layer with `--raycast-layer-mask`.