From 9fafef1aedbb8fb34833a2dae016ca4a9e3a52b3 Mon Sep 17 00:00:00 2001 From: Alister Trabattoni Date: Fri, 16 Oct 2026 13:08:15 +0200 Subject: [PATCH] feat(core): print heterogeneous data collections as a tree A flat table only works for a well-defined collection: every level named, and named alike across branches. Otherwise the collection prints as a tree and each homogeneous branch collapses into its dimensions and size, with its levels and leaf count hooked below; indexing into it shows its table. Empty branches force the tree so they are never dropped from the output. Leaves are now counted as arrays and tables, and an empty collection prints as . --- docs/release-notes.md | 1 + .../data-structures/datacollection.md | 41 ++++- docs/user-guide/pipeline/picking.md | 2 +- tests/test_datacollection.py | 157 ++++++++++++++++-- xdas/core/datacollection.py | 132 ++++++++++++--- xdas/core/routines.py | 6 +- 6 files changed, 290 insertions(+), 49 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 90fb7c74..aa217cc5 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -9,6 +9,7 @@ ### Improvements - **Windows and macOS are now tested and working well.** CI runs the full suite on `windows-latest` and `macos-latest` alongside Linux; fixed a `glob()` path-separator bug that broke opening files by wildcard on Windows (#87), and what the new CI then caught: an unbounded memory-limit fallback and illegal colons in temporary filenames (#88) (@ClaudioStrumia, @atrabattoni). - Opening many files is faster: coordinates now concatenate in linear rather than quadratic time, the ObsPy engine no longer reads HDF5 and netCDF files whole to refuse them during format auto-detection, and failed collection attempts leave neither queued jobs nor a progress bar behind (@atrabattoni). +- **A data collection mixing schemas prints as a tree.** Only a collection whose levels are all named, and named alike across branches, keeps its flat table; otherwise each such branch collapses into its dimensions and size, with its levels and leaf count hooked below. Leaves are now counted as arrays and tables (@atrabattoni). - `simplify` on a coordinate with a declared sampling interval keeps that interval and spends the budget it is given: a tolerance wider than the declared one thins past it and is declared on the result, so jittery clocks fold down to their real gaps; `tolerance=0` stays lossless (@atrabattoni). ## 0.2.9 diff --git a/docs/user-guide/data-structures/datacollection.md b/docs/user-guide/data-structures/datacollection.md index 1f9e63b9..5ce509ff 100644 --- a/docs/user-guide/data-structures/datacollection.md +++ b/docs/user-guide/data-structures/datacollection.md @@ -88,7 +88,7 @@ dc = xd.open(path, engine="asn") dc ``` ```text - + network cable record REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 10000, distance: 5000) 190.7 MB @@ -106,7 +106,7 @@ dc = xd.open("datacollection.nc") dc ``` ```text - + network cable record REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 10000, distance: 5000) 190.7 MB @@ -142,7 +142,7 @@ dc["REKA"]["RK2"].insert(0, da) dc ``` ```text - + network cable record REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 10000, distance: 5000) 190.7 MB @@ -151,4 +151,37 @@ REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 54000, distance: 10000) 2.0 GB ``` -You now have 2 records in your records list. \ No newline at end of file +You now have 2 records in your records list. +## Printing a collection + +A collection whose levels are all named, and named alike across branches, prints as one +table, as above. When +branches follow different schemas, say DAS acquisitions next to seismic stations, the +collection prints as a tree instead. Each homogeneous branch collapses into the +dimensions and size of its arrays, with its levels and array count hooked below: + +```python +dc = xd.DataCollection({"das": das, "seismic": seismic}, "kind") +dc +``` +```text + +kind +├── das (time, distance) 4.2 GB +│ ╰─ network › cable › record (3 arrays) +└── seismic (time) 4.1 MB + ╰─ network › station › location › channel (3 arrays) +``` + +Index into a branch to see its table: + +```python +dc["seismic"] +``` +```text + +network station location channel +FR OGSI 00 HHZ (time: 360000) 1.4 MB + HHN (time: 360000) 1.4 MB + HHE (time: 360000) 1.4 MB +``` diff --git a/docs/user-guide/pipeline/picking.md b/docs/user-guide/pipeline/picking.md index 88cefa04..80cbb08b 100644 --- a/docs/user-guide/pipeline/picking.md +++ b/docs/user-guide/pipeline/picking.md @@ -229,7 +229,7 @@ cable pair, each recorded as two consecutive records: ... "cable", ... ) >>> das - + cable record east 0 (time: 10000, distance: 3) 234.4 KB 1 (time: 10000, distance: 3) 234.4 KB diff --git a/tests/test_datacollection.py b/tests/test_datacollection.py index e4ea29fd..61c8392f 100644 --- a/tests/test_datacollection.py +++ b/tests/test_datacollection.py @@ -232,7 +232,7 @@ def test_empty_mapping_repr(self): from xdas.core.datacollection import DataMapping dm = DataMapping({}, "empty") - assert repr(dm) == "" + assert repr(dm) == "" def test_mapping_reduce(self): import pickle @@ -617,7 +617,7 @@ def test_repr_measures_a_table_by_its_columns(self): df = pd.DataFrame({"time": [1.0, 2.0], "value": [0.5, 0.9]}) dc = xd.DataCollection({"ST01": df}, "station") assert "(rows: 2, columns: 2)" in repr(dc) - assert repr(dc).splitlines()[0] == "" + assert repr(dc).splitlines()[0] == "" class TestCollectionRepr: @@ -636,7 +636,7 @@ def test_levels_head_the_columns_when_homogeneous(self): "node", ) lines = repr(dc).splitlines() - assert lines[0] == "" + assert lines[0] == "" assert lines[1] == "node cable record" assert lines[2] == "CCN N 0 (time: 4, distance: 3) 48 B" @@ -656,35 +656,162 @@ def test_a_depth_a_branch_never_reaches_is_marked_absent(self): "SER S - (time: 4) 16 B", ] - def test_an_unnamed_depth_leaves_its_column_unheaded(self): - # the named depths are still worth heading + def test_an_unnamed_depth_draws_a_tree(self): + # a column without a name is not a table: each level is spelled out dc = xd.DataCollection({"CCN": {"N": [self.array(time=4)]}}, "node") assert repr(dc).splitlines()[1:] == [ "node", - "CCN N 0 (time: 4) 16 B", + "└── CCN 1 array, 16 B", + " └── N 1 array, 16 B", + " └── 0 (time: 4) 16 B", ] - def test_a_disputed_depth_leaves_its_column_unheaded(self): - # one branch calls it a cable, the other a record: neither speaks for - # the column, but the root still does + def test_a_disputed_depth_draws_a_tree(self): + # one branch calls it a cable, the other a record: no column could be + # headed, so each branch collapses into its own summary line dc = xd.DataCollection( { "CCN": ("cable", [self.array(time=4)]), - "SER": ("record", [self.array(time=4)]), + "SER": ("record", [self.array(time=4), self.array(time=4)]), }, "node", ) + assert repr(dc).splitlines() == [ + "", + "node", + "├── CCN (time) 16 B", + "│ ╰─ cable (1 array)", + "└── SER (time) 32 B", + " ╰─ record (2 arrays)", + ] + + def test_branches_with_different_schemas_collapse(self): + das = xd.DataCollection( + {"N01": ("cable", {"CAB1": ("record", [self.array(time=4, distance=3)])})}, + "node", + ) + seismic = xd.DataCollection( + {"FR": ("station", {"OGSI": ("channel", {"HHZ": self.array(time=4)})})}, + "network", + ) + dc = xd.DataCollection({"das": das, "seismic": seismic}, "kind") + assert repr(dc).splitlines() == [ + "", + "kind", + "├── das (time, distance) 48 B", + "│ ╰─ node › cable › record (1 array)", + "└── seismic (time) 16 B", + " ╰─ network › station › channel (1 array)", + ] + # indexing into a collapsed branch shows its table + assert repr(dc["das"]).splitlines()[1] == "node cable record" + + def test_leaves_stand_next_to_collapsed_branches(self): + df = pd.DataFrame({"time": [1.0, 2.0], "value": [0.5, 0.9]}) + dc = xd.DataCollection( + { + "das": ("cable", {"N": [self.array(time=4)]}), + "seismic": ("station", {"SSB": self.array(time=4)}), + "picks": df, + "stack": self.array(time=2, distance=2), + }, + "kind", + ) + assert repr(dc).splitlines() == [ + "", + "kind", + "├── das 1 array, 16 B", + "│ cable", + "│ └── N 1 array, 16 B", + "│ └── 0 (time: 4) 16 B", + "├── seismic (time) 16 B", + "│ ╰─ station (1 array)", + "├── picks (rows: 2, columns: 2) 32 B", + "└── stack (time: 2, distance: 2) 16 B", + ] + + def test_heterogeneous_branches_open_a_subtree(self): + mixed = { + "das": ("cable", [self.array(time=4)]), + "seismic": ("channel", {"HHZ": self.array(time=4)}), + } + dc = xd.DataCollection( + { + "OBS-A": ("kind", mixed), + "OBS-B": ("kind", {"das": ("cable", [self.array(time=4)])}), + }, + "site", + ) + assert repr(dc).splitlines() == [ + "", + "site", + "├── OBS-A 2 arrays, 32 B", + "│ kind", + "│ ├── das (time) 16 B", + "│ │ ╰─ cable (1 array)", + "│ └── seismic (time) 16 B", + "│ ╰─ channel (1 array)", + "└── OBS-B (time) 16 B", + " ╰─ kind › cable (1 array)", + ] + + def test_an_unnamed_tree_level_has_no_field_line(self): + dc = xd.DataCollection( + [ + { + "das": ("cable", [self.array(time=4)]), + "seismic": [self.array(time=4)], + }, + self.array(time=4), + ] + ) + assert repr(dc).splitlines() == [ + "", + "├── 0 2 arrays, 32 B", + "│ ├── das (time) 16 B", + "│ │ ╰─ cable (1 array)", + "│ └── seismic 1 array, 16 B", + "│ └── 0 (time: 4) 16 B", + "└── 1 (time: 4) 16 B", + ] + + def test_an_empty_branch_is_marked_empty(self): + dc = xd.DataCollection( + { + "das": ("cable", [self.array(time=4)]), + "seismic": ("station", {}), + "other": ("channel", [self.array(time=4)]), + }, + "kind", + ) + assert repr(dc).splitlines()[2:] == [ + "├── das (time) 16 B", + "│ ╰─ cable (1 array)", + "├── seismic (empty)", + "└── other (time) 16 B", + " ╰─ channel (1 array)", + ] + + def test_a_deep_empty_branch_is_not_hidden_by_the_table(self): + # an empty branch has no leaf to give it a row: a table would drop it + dc = xd.DataCollection( + {"N01": ("cable", {"CAB1": ("record", [self.array(time=4)]), "CAB2": []})}, + "node", + ) assert repr(dc).splitlines()[1:] == [ "node", - "CCN 0 (time: 4) 16 B", - "SER 0 (time: 4) 16 B", + "└── N01 1 array, 16 B", + " cable", + " ├── CAB1 (time) 16 B", + " │ ╰─ record (1 array)", + " └── CAB2 (empty)", ] - def test_no_header_when_nothing_is_named(self): + def test_a_nameless_collection_draws_a_tree(self): dc = xd.DataCollection([self.array(time=4), self.array(time=4)]) assert repr(dc).splitlines()[1:] == [ - "0 (time: 4) 16 B", - "1 (time: 4) 16 B", + "├── 0 (time: 4) 16 B", + "└── 1 (time: 4) 16 B", ] def test_repeated_keys_are_blanked(self): diff --git a/xdas/core/datacollection.py b/xdas/core/datacollection.py index 47efb975..00cd93ed 100644 --- a/xdas/core/datacollection.py +++ b/xdas/core/datacollection.py @@ -54,7 +54,7 @@ class DataCollection: ... "instrument", ... ) >>> dc - + instrument record das1 0 (time: 300, distance: 401) 939.8 KB 1 (time: 300, distance: 401) 939.8 KB @@ -143,7 +143,7 @@ def query(self, indexers=None, **indexers_kwargs): ... "instrument", ... ) >>> dc.query(instrument="das1", record=0) - + instrument record das1 0 (time: 300, distance: 401) 939.8 KB @@ -671,11 +671,18 @@ def copy(self, deep=True): def format_collection(dc): """ - Render a data collection as a flat table, one row per leaf. + Render a data collection, one row per leaf or per homogeneous subtree. - Each row spells out the keys that address a leaf, then that leaf's shape - and the memory it takes once loaded. Repeated keys are blanked so that a - row shows only what changed from the one above. + A homogeneous collection, whose every depth is named and named alike by + the branches that reach it, is a flat table: each row spells out the keys + that address a leaf, then that leaf's shape and the memory it takes once + loaded. Repeated keys are blanked so that a row shows only what changed + from the one above. + + Where branches disagree or leave a depth unnamed, the collection is + drawn as a tree instead, and + each homogeneous subtree under it is collapsed into a single summary + line. Indexing into that subtree shows its table. Parameters ---------- @@ -689,13 +696,15 @@ def format_collection(dc): """ rows = get_leaves(dc) - header = ( - f"" - ) + header = f"" if not rows: return header + body = format_table(rows) if is_homogeneous(dc, rows) else format_tree(dc) + return "\n".join([header] + body) + +def format_table(rows): + """Render the leaves *rows* of a homogeneous collection as flat lines.""" keys = [[key for _, key in path] for path, _ in rows] ncolumns = max(len(row) for row in keys) # a column is a depth, so a branch that stops short has no key to put in @@ -717,8 +726,7 @@ def format_collection(dc): for index, (row, (_, leaf)) in enumerate(zip(grid, rows)) ] - fields = get_fields(rows) - heads = (fields if fields else [""] * ncolumns) + ["", ""] + heads = get_names(rows) + ["", ""] # positional keys and the memory column are right aligned so that they # can be compared down the column columns = zip(*[row + cells[-2:] for row, cells in zip(grid, body)]) @@ -731,11 +739,38 @@ def format_collection(dc): max(len(heads[d]), *(len(row[d]) for row in body)) for d in range(len(heads)) ] - lines = [header] - if any(heads): - lines.append(format_row(heads, widths, right)) - lines.extend(format_row(row, widths, right) for row in body) - return "\n".join(lines) + return [format_row(row, widths, right) for row in [heads] + body] + + +def format_tree(node): + """Render a heterogeneous *node* as tree lines, one branch per child. + + A leaf child shows its shape and size. A homogeneous child collapses into + the dimensions and size of its leaves, as a leaf would, with its chain of + field names and leaf count hooked below. A heterogeneous child opens a + subtree of its own. + """ + items = list(enumerate(node) if node.issequence() else node.items()) + lines = [node.name] if node.name is not None else [] + for index, (key, value) in enumerate(items): + last = index == len(items) - 1 + branch, indent = ("└── ", " ") if last else ("├── ", "│ ") + if not isinstance(value, DataCollection): + lines.append( + f"{branch}{key} {get_shape(value)} {to_human(get_nbytes(value))}" + ) + continue + rows = get_leaves(value) + if not rows: + lines.append(f"{branch}{key} (empty)") + elif is_homogeneous(value, rows): + nbytes = sum(get_nbytes(leaf) for _, leaf in rows) + lines.append(f"{branch}{key} {get_dims(rows)} {to_human(nbytes)}") + lines.append(f"{indent}╰─ {' › '.join(get_names(rows))} ({count(rows)})") + else: + lines.append(f"{branch}{key} {describe(rows)}") + lines.extend((indent + line).rstrip() for line in format_tree(value)) + return lines def format_row(cells, widths, right): @@ -763,22 +798,67 @@ def walk(node, path): return leaves -def get_fields(rows): - """Return the name of each depth, empty where the branches disagree. +def get_names(rows): + """Return the field name of each depth, None where it has no single name. A field name belongs to a node, not to a depth: two branches can name the same depth differently or not at all, and only some of them reach - the deepest ones. Such a depth is left unheaded rather than headed with - a name that speaks for one branch only, but the depths that do agree are - still worth naming. + the deepest ones. """ ncolumns = max(len(path) for path, _ in rows) - heads = [] + names = [] for depth in range(ncolumns): found = {path[depth][0] for path, _ in rows if depth < len(path)} - name = next(iter(found)) if len(found) == 1 else None - heads.append(name if name is not None else "") - return heads + names.append(next(iter(found)) if len(found) == 1 else None) + return names + + +def is_homogeneous(node, rows): + """Tell whether every depth of *node*, of leaves *rows*, bears one name. + + Only such a collection is well defined enough for a flat table: its + columns can all be headed. Branches may stop at different depths, as + long as those they share are named alike. An empty branch has no leaf + to give it a row, so it calls for a tree, where it can be shown. + """ + return None not in get_names(rows) and not has_empty(node) + + +def has_empty(node): + """Tell whether *node* holds an empty collection at any depth.""" + values = node if node.issequence() else node.values() + return any( + isinstance(value, DataCollection) and (not len(value) or has_empty(value)) + for value in values + ) + + +def get_dims(rows): + """Return the dimensions met across the leaves of *rows*, in order.""" + dims = [] + for _, leaf in rows: + names = ("rows", "columns") if isinstance(leaf, pd.DataFrame) else leaf.dims + dims.extend(dim for dim in names if dim not in dims) + return f"({', '.join(dims)})" + + +def count(rows): + """Count the leaves of *rows* by kind, as ``"4 arrays, 1 table"``.""" + ntables = sum(isinstance(leaf, pd.DataFrame) for _, leaf in rows) + narrays = len(rows) - ntables + parts = [ + f"{number} {noun}{'' if number == 1 else 's'}" + for number, noun in ((narrays, "array"), (ntables, "table")) + if number + ] + return ", ".join(parts) + + +def describe(rows): + """Summarize the leaves of *rows*: their count by kind and their size.""" + if not rows: + return "empty" + return f"{count(rows)}, {to_human(sum(get_nbytes(leaf) for _, leaf in rows))}" def get_shape(leaf): diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 2e818d77..f7a7847c 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -434,7 +434,7 @@ def open_mfdatatree( >>> import xdas as xd >>> paths = "/data/{node}/{cable}/[record]/proc/[record].h5" >>> xd.open_mfdatatree(paths, engine="asn") # doctest: +SKIP - + node cable record CCN N 0 (time: ..., distance: ...) ... 1 (time: ..., distance: ...) ... @@ -1505,7 +1505,7 @@ def stack(dc, level, dim=None, join=None, tolerance=None): ... "station", ... ) >>> dc - + station channel SX01 SHZ (time: 4) 32 B SHN (time: 4) 32 B @@ -1514,7 +1514,7 @@ def stack(dc, level, dim=None, join=None, tolerance=None): >>> stacked = xd.stack(dc, "channel") >>> stacked - + station SX01 (channel: 2, time: 4) 64 B SX02 (channel: 2, time: 4) 64 B