diff --git a/docs/release-notes.md b/docs/release-notes.md index c38ba6dd..eb6a4e0e 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -10,6 +10,7 @@ ### Improvements - **Windows and macOS are now tested and working well.** CI runs the full suite on `windows-latest` and `macos-latest` alongside Linux; fixed a `glob()` path-separator bug that broke opening files by wildcard on Windows (#87), and what the new CI then caught: an unbounded memory-limit fallback and illegal colons in temporary filenames (#88) (@ClaudioStrumia, @atrabattoni). - Opening many files is faster: coordinates now concatenate in linear rather than quadratic time, the ObsPy engine no longer reads HDF5 and netCDF files whole to refuse them during format auto-detection, and failed collection attempts leave neither queued jobs nor a progress bar behind (@atrabattoni). +- **A data collection mixing schemas prints as a tree.** Only a collection whose levels are all named, and named alike across branches, keeps its flat table; otherwise each such branch collapses into its dimensions and size, with its levels and leaf count hooked below. Leaves are now counted as arrays and tables (@atrabattoni). - `simplify` on a coordinate with a declared sampling interval keeps that interval and spends the budget it is given: a tolerance wider than the declared one thins past it and is declared on the result, so jittery clocks fold down to their real gaps; `tolerance=0` stays lossless (@atrabattoni). ## 0.2.9 diff --git a/docs/user-guide/data-structures/datacollection.md b/docs/user-guide/data-structures/datacollection.md index 1f9e63b9..5ce509ff 100644 --- a/docs/user-guide/data-structures/datacollection.md +++ b/docs/user-guide/data-structures/datacollection.md @@ -88,7 +88,7 @@ dc = xd.open(path, engine="asn") dc ``` ```text - + network cable record REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 10000, distance: 5000) 190.7 MB @@ -106,7 +106,7 @@ dc = xd.open("datacollection.nc") dc ``` ```text - + network cable record REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 10000, distance: 5000) 190.7 MB @@ -142,7 +142,7 @@ dc["REKA"]["RK2"].insert(0, da) dc ``` ```text - + network cable record REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 10000, distance: 5000) 190.7 MB @@ -151,4 +151,37 @@ REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB 1 (time: 54000, distance: 10000) 2.0 GB ``` -You now have 2 records in your records list. \ No newline at end of file +You now have 2 records in your records list. +## Printing a collection + +A collection whose levels are all named, and named alike across branches, prints as one +table, as above. When +branches follow different schemas, say DAS acquisitions next to seismic stations, the +collection prints as a tree instead. Each homogeneous branch collapses into the +dimensions and size of its arrays, with its levels and array count hooked below: + +```python +dc = xd.DataCollection({"das": das, "seismic": seismic}, "kind") +dc +``` +```text + +kind +├── das (time, distance) 4.2 GB +│ ╰─ network › cable › record (3 arrays) +└── seismic (time) 4.1 MB + ╰─ network › station › location › channel (3 arrays) +``` + +Index into a branch to see its table: + +```python +dc["seismic"] +``` +```text + +network station location channel +FR OGSI 00 HHZ (time: 360000) 1.4 MB + HHN (time: 360000) 1.4 MB + HHE (time: 360000) 1.4 MB +``` diff --git a/docs/user-guide/pipeline/picking.md b/docs/user-guide/pipeline/picking.md index 50e6314e..93854382 100644 --- a/docs/user-guide/pipeline/picking.md +++ b/docs/user-guide/pipeline/picking.md @@ -229,7 +229,7 @@ cable pair, each recorded as two consecutive records: ... "cable", ... ) >>> das - + cable record east 0 (time: 10000, distance: 3) 234.4 KB 1 (time: 10000, distance: 3) 234.4 KB diff --git a/tests/test_datacollection.py b/tests/test_datacollection.py index e4ea29fd..61c8392f 100644 --- a/tests/test_datacollection.py +++ b/tests/test_datacollection.py @@ -232,7 +232,7 @@ def test_empty_mapping_repr(self): from xdas.core.datacollection import DataMapping dm = DataMapping({}, "empty") - assert repr(dm) == "" + assert repr(dm) == "" def test_mapping_reduce(self): import pickle @@ -617,7 +617,7 @@ def test_repr_measures_a_table_by_its_columns(self): df = pd.DataFrame({"time": [1.0, 2.0], "value": [0.5, 0.9]}) dc = xd.DataCollection({"ST01": df}, "station") assert "(rows: 2, columns: 2)" in repr(dc) - assert repr(dc).splitlines()[0] == "" + assert repr(dc).splitlines()[0] == "" class TestCollectionRepr: @@ -636,7 +636,7 @@ def test_levels_head_the_columns_when_homogeneous(self): "node", ) lines = repr(dc).splitlines() - assert lines[0] == "" + assert lines[0] == "" assert lines[1] == "node cable record" assert lines[2] == "CCN N 0 (time: 4, distance: 3) 48 B" @@ -656,35 +656,162 @@ def test_a_depth_a_branch_never_reaches_is_marked_absent(self): "SER S - (time: 4) 16 B", ] - def test_an_unnamed_depth_leaves_its_column_unheaded(self): - # the named depths are still worth heading + def test_an_unnamed_depth_draws_a_tree(self): + # a column without a name is not a table: each level is spelled out dc = xd.DataCollection({"CCN": {"N": [self.array(time=4)]}}, "node") assert repr(dc).splitlines()[1:] == [ "node", - "CCN N 0 (time: 4) 16 B", + "└── CCN 1 array, 16 B", + " └── N 1 array, 16 B", + " └── 0 (time: 4) 16 B", ] - def test_a_disputed_depth_leaves_its_column_unheaded(self): - # one branch calls it a cable, the other a record: neither speaks for - # the column, but the root still does + def test_a_disputed_depth_draws_a_tree(self): + # one branch calls it a cable, the other a record: no column could be + # headed, so each branch collapses into its own summary line dc = xd.DataCollection( { "CCN": ("cable", [self.array(time=4)]), - "SER": ("record", [self.array(time=4)]), + "SER": ("record", [self.array(time=4), self.array(time=4)]), }, "node", ) + assert repr(dc).splitlines() == [ + "", + "node", + "├── CCN (time) 16 B", + "│ ╰─ cable (1 array)", + "└── SER (time) 32 B", + " ╰─ record (2 arrays)", + ] + + def test_branches_with_different_schemas_collapse(self): + das = xd.DataCollection( + {"N01": ("cable", {"CAB1": ("record", [self.array(time=4, distance=3)])})}, + "node", + ) + seismic = xd.DataCollection( + {"FR": ("station", {"OGSI": ("channel", {"HHZ": self.array(time=4)})})}, + "network", + ) + dc = xd.DataCollection({"das": das, "seismic": seismic}, "kind") + assert repr(dc).splitlines() == [ + "", + "kind", + "├── das (time, distance) 48 B", + "│ ╰─ node › cable › record (1 array)", + "└── seismic (time) 16 B", + " ╰─ network › station › channel (1 array)", + ] + # indexing into a collapsed branch shows its table + assert repr(dc["das"]).splitlines()[1] == "node cable record" + + def test_leaves_stand_next_to_collapsed_branches(self): + df = pd.DataFrame({"time": [1.0, 2.0], "value": [0.5, 0.9]}) + dc = xd.DataCollection( + { + "das": ("cable", {"N": [self.array(time=4)]}), + "seismic": ("station", {"SSB": self.array(time=4)}), + "picks": df, + "stack": self.array(time=2, distance=2), + }, + "kind", + ) + assert repr(dc).splitlines() == [ + "", + "kind", + "├── das 1 array, 16 B", + "│ cable", + "│ └── N 1 array, 16 B", + "│ └── 0 (time: 4) 16 B", + "├── seismic (time) 16 B", + "│ ╰─ station (1 array)", + "├── picks (rows: 2, columns: 2) 32 B", + "└── stack (time: 2, distance: 2) 16 B", + ] + + def test_heterogeneous_branches_open_a_subtree(self): + mixed = { + "das": ("cable", [self.array(time=4)]), + "seismic": ("channel", {"HHZ": self.array(time=4)}), + } + dc = xd.DataCollection( + { + "OBS-A": ("kind", mixed), + "OBS-B": ("kind", {"das": ("cable", [self.array(time=4)])}), + }, + "site", + ) + assert repr(dc).splitlines() == [ + "", + "site", + "├── OBS-A 2 arrays, 32 B", + "│ kind", + "│ ├── das (time) 16 B", + "│ │ ╰─ cable (1 array)", + "│ └── seismic (time) 16 B", + "│ ╰─ channel (1 array)", + "└── OBS-B (time) 16 B", + " ╰─ kind › cable (1 array)", + ] + + def test_an_unnamed_tree_level_has_no_field_line(self): + dc = xd.DataCollection( + [ + { + "das": ("cable", [self.array(time=4)]), + "seismic": [self.array(time=4)], + }, + self.array(time=4), + ] + ) + assert repr(dc).splitlines() == [ + "", + "├── 0 2 arrays, 32 B", + "│ ├── das (time) 16 B", + "│ │ ╰─ cable (1 array)", + "│ └── seismic 1 array, 16 B", + "│ └── 0 (time: 4) 16 B", + "└── 1 (time: 4) 16 B", + ] + + def test_an_empty_branch_is_marked_empty(self): + dc = xd.DataCollection( + { + "das": ("cable", [self.array(time=4)]), + "seismic": ("station", {}), + "other": ("channel", [self.array(time=4)]), + }, + "kind", + ) + assert repr(dc).splitlines()[2:] == [ + "├── das (time) 16 B", + "│ ╰─ cable (1 array)", + "├── seismic (empty)", + "└── other (time) 16 B", + " ╰─ channel (1 array)", + ] + + def test_a_deep_empty_branch_is_not_hidden_by_the_table(self): + # an empty branch has no leaf to give it a row: a table would drop it + dc = xd.DataCollection( + {"N01": ("cable", {"CAB1": ("record", [self.array(time=4)]), "CAB2": []})}, + "node", + ) assert repr(dc).splitlines()[1:] == [ "node", - "CCN 0 (time: 4) 16 B", - "SER 0 (time: 4) 16 B", + "└── N01 1 array, 16 B", + " cable", + " ├── CAB1 (time) 16 B", + " │ ╰─ record (1 array)", + " └── CAB2 (empty)", ] - def test_no_header_when_nothing_is_named(self): + def test_a_nameless_collection_draws_a_tree(self): dc = xd.DataCollection([self.array(time=4), self.array(time=4)]) assert repr(dc).splitlines()[1:] == [ - "0 (time: 4) 16 B", - "1 (time: 4) 16 B", + "├── 0 (time: 4) 16 B", + "└── 1 (time: 4) 16 B", ] def test_repeated_keys_are_blanked(self): diff --git a/xdas/core/datacollection.py b/xdas/core/datacollection.py index 47efb975..00cd93ed 100644 --- a/xdas/core/datacollection.py +++ b/xdas/core/datacollection.py @@ -54,7 +54,7 @@ class DataCollection: ... "instrument", ... ) >>> dc - + instrument record das1 0 (time: 300, distance: 401) 939.8 KB 1 (time: 300, distance: 401) 939.8 KB @@ -143,7 +143,7 @@ def query(self, indexers=None, **indexers_kwargs): ... "instrument", ... ) >>> dc.query(instrument="das1", record=0) - + instrument record das1 0 (time: 300, distance: 401) 939.8 KB @@ -671,11 +671,18 @@ def copy(self, deep=True): def format_collection(dc): """ - Render a data collection as a flat table, one row per leaf. + Render a data collection, one row per leaf or per homogeneous subtree. - Each row spells out the keys that address a leaf, then that leaf's shape - and the memory it takes once loaded. Repeated keys are blanked so that a - row shows only what changed from the one above. + A homogeneous collection, whose every depth is named and named alike by + the branches that reach it, is a flat table: each row spells out the keys + that address a leaf, then that leaf's shape and the memory it takes once + loaded. Repeated keys are blanked so that a row shows only what changed + from the one above. + + Where branches disagree or leave a depth unnamed, the collection is + drawn as a tree instead, and + each homogeneous subtree under it is collapsed into a single summary + line. Indexing into that subtree shows its table. Parameters ---------- @@ -689,13 +696,15 @@ def format_collection(dc): """ rows = get_leaves(dc) - header = ( - f"" - ) + header = f"" if not rows: return header + body = format_table(rows) if is_homogeneous(dc, rows) else format_tree(dc) + return "\n".join([header] + body) + +def format_table(rows): + """Render the leaves *rows* of a homogeneous collection as flat lines.""" keys = [[key for _, key in path] for path, _ in rows] ncolumns = max(len(row) for row in keys) # a column is a depth, so a branch that stops short has no key to put in @@ -717,8 +726,7 @@ def format_collection(dc): for index, (row, (_, leaf)) in enumerate(zip(grid, rows)) ] - fields = get_fields(rows) - heads = (fields if fields else [""] * ncolumns) + ["", ""] + heads = get_names(rows) + ["", ""] # positional keys and the memory column are right aligned so that they # can be compared down the column columns = zip(*[row + cells[-2:] for row, cells in zip(grid, body)]) @@ -731,11 +739,38 @@ def format_collection(dc): max(len(heads[d]), *(len(row[d]) for row in body)) for d in range(len(heads)) ] - lines = [header] - if any(heads): - lines.append(format_row(heads, widths, right)) - lines.extend(format_row(row, widths, right) for row in body) - return "\n".join(lines) + return [format_row(row, widths, right) for row in [heads] + body] + + +def format_tree(node): + """Render a heterogeneous *node* as tree lines, one branch per child. + + A leaf child shows its shape and size. A homogeneous child collapses into + the dimensions and size of its leaves, as a leaf would, with its chain of + field names and leaf count hooked below. A heterogeneous child opens a + subtree of its own. + """ + items = list(enumerate(node) if node.issequence() else node.items()) + lines = [node.name] if node.name is not None else [] + for index, (key, value) in enumerate(items): + last = index == len(items) - 1 + branch, indent = ("└── ", " ") if last else ("├── ", "│ ") + if not isinstance(value, DataCollection): + lines.append( + f"{branch}{key} {get_shape(value)} {to_human(get_nbytes(value))}" + ) + continue + rows = get_leaves(value) + if not rows: + lines.append(f"{branch}{key} (empty)") + elif is_homogeneous(value, rows): + nbytes = sum(get_nbytes(leaf) for _, leaf in rows) + lines.append(f"{branch}{key} {get_dims(rows)} {to_human(nbytes)}") + lines.append(f"{indent}╰─ {' › '.join(get_names(rows))} ({count(rows)})") + else: + lines.append(f"{branch}{key} {describe(rows)}") + lines.extend((indent + line).rstrip() for line in format_tree(value)) + return lines def format_row(cells, widths, right): @@ -763,22 +798,67 @@ def walk(node, path): return leaves -def get_fields(rows): - """Return the name of each depth, empty where the branches disagree. +def get_names(rows): + """Return the field name of each depth, None where it has no single name. A field name belongs to a node, not to a depth: two branches can name the same depth differently or not at all, and only some of them reach - the deepest ones. Such a depth is left unheaded rather than headed with - a name that speaks for one branch only, but the depths that do agree are - still worth naming. + the deepest ones. """ ncolumns = max(len(path) for path, _ in rows) - heads = [] + names = [] for depth in range(ncolumns): found = {path[depth][0] for path, _ in rows if depth < len(path)} - name = next(iter(found)) if len(found) == 1 else None - heads.append(name if name is not None else "") - return heads + names.append(next(iter(found)) if len(found) == 1 else None) + return names + + +def is_homogeneous(node, rows): + """Tell whether every depth of *node*, of leaves *rows*, bears one name. + + Only such a collection is well defined enough for a flat table: its + columns can all be headed. Branches may stop at different depths, as + long as those they share are named alike. An empty branch has no leaf + to give it a row, so it calls for a tree, where it can be shown. + """ + return None not in get_names(rows) and not has_empty(node) + + +def has_empty(node): + """Tell whether *node* holds an empty collection at any depth.""" + values = node if node.issequence() else node.values() + return any( + isinstance(value, DataCollection) and (not len(value) or has_empty(value)) + for value in values + ) + + +def get_dims(rows): + """Return the dimensions met across the leaves of *rows*, in order.""" + dims = [] + for _, leaf in rows: + names = ("rows", "columns") if isinstance(leaf, pd.DataFrame) else leaf.dims + dims.extend(dim for dim in names if dim not in dims) + return f"({', '.join(dims)})" + + +def count(rows): + """Count the leaves of *rows* by kind, as ``"4 arrays, 1 table"``.""" + ntables = sum(isinstance(leaf, pd.DataFrame) for _, leaf in rows) + narrays = len(rows) - ntables + parts = [ + f"{number} {noun}{'' if number == 1 else 's'}" + for number, noun in ((narrays, "array"), (ntables, "table")) + if number + ] + return ", ".join(parts) + + +def describe(rows): + """Summarize the leaves of *rows*: their count by kind and their size.""" + if not rows: + return "empty" + return f"{count(rows)}, {to_human(sum(get_nbytes(leaf) for _, leaf in rows))}" def get_shape(leaf): diff --git a/xdas/core/routines.py b/xdas/core/routines.py index 2e818d77..f7a7847c 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -434,7 +434,7 @@ def open_mfdatatree( >>> import xdas as xd >>> paths = "/data/{node}/{cable}/[record]/proc/[record].h5" >>> xd.open_mfdatatree(paths, engine="asn") # doctest: +SKIP - + node cable record CCN N 0 (time: ..., distance: ...) ... 1 (time: ..., distance: ...) ... @@ -1505,7 +1505,7 @@ def stack(dc, level, dim=None, join=None, tolerance=None): ... "station", ... ) >>> dc - + station channel SX01 SHZ (time: 4) 32 B SHN (time: 4) 32 B @@ -1514,7 +1514,7 @@ def stack(dc, level, dim=None, join=None, tolerance=None): >>> stacked = xd.stack(dc, "channel") >>> stacked - + station SX01 (channel: 2, time: 4) 64 B SX02 (channel: 2, time: 4) 64 B