Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/release-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
### Improvements
- **Windows and macOS are now tested and working well.** CI runs the full suite on `windows-latest` and `macos-latest` alongside Linux; fixed a `glob()` path-separator bug that broke opening files by wildcard on Windows (#87), and what the new CI then caught: an unbounded memory-limit fallback and illegal colons in temporary filenames (#88) (@ClaudioStrumia, @atrabattoni).
- Opening many files is faster: coordinates now concatenate in linear rather than quadratic time, the ObsPy engine no longer reads HDF5 and netCDF files whole to refuse them during format auto-detection, and failed collection attempts leave neither queued jobs nor a progress bar behind (@atrabattoni).
- **A data collection mixing schemas prints as a tree.** Only a collection whose levels are all named, and named alike across branches, keeps its flat table; otherwise each such branch collapses into its dimensions and size, with its levels and leaf count hooked below. Leaves are now counted as arrays and tables (@atrabattoni).
- `simplify` on a coordinate with a declared sampling interval keeps that interval and spends the budget it is given: a tolerance wider than the declared one thins past it and is declared on the result, so jittery clocks fold down to their real gaps; `tolerance=0` stays lossless (@atrabattoni).

## 0.2.9
Expand Down
41 changes: 37 additions & 4 deletions docs/user-guide/data-structures/datacollection.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ dc = xd.open(path, engine="asn")
dc
```
```text
<xdas.DataCollection: 4 leaves, 4.5 GB>
<xdas.DataCollection: 4 arrays, 4.5 GB>
network cable record
REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB
1 (time: 10000, distance: 5000) 190.7 MB
Expand All @@ -106,7 +106,7 @@ dc = xd.open("datacollection.nc")
dc
```
```text
<xdas.DataCollection: 4 leaves, 4.5 GB>
<xdas.DataCollection: 4 arrays, 4.5 GB>
network cable record
REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB
1 (time: 10000, distance: 5000) 190.7 MB
Expand Down Expand Up @@ -142,7 +142,7 @@ dc["REKA"]["RK2"].insert(0, da)
dc
```
```text
<xdas.DataCollection: 5 leaves, 17.3 GB>
<xdas.DataCollection: 5 arrays, 17.3 GB>
network cable record
REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB
1 (time: 10000, distance: 5000) 190.7 MB
Expand All @@ -151,4 +151,37 @@ REKA RK1 0 (time: 54000, distance: 10000) 2.0 GB
1 (time: 54000, distance: 10000) 2.0 GB
```

You now have 2 records in your records list.
You now have 2 records in your records list.
## Printing a collection

A collection whose levels are all named, and named alike across branches, prints as one
table, as above. When
branches follow different schemas, say DAS acquisitions next to seismic stations, the
collection prints as a tree instead. Each homogeneous branch collapses into the
dimensions and size of its arrays, with its levels and array count hooked below:

```python
dc = xd.DataCollection({"das": das, "seismic": seismic}, "kind")
dc
```
```text
<xdas.DataCollection: 6 arrays, 4.2 GB>
kind
├── das (time, distance) 4.2 GB
│ ╰─ network › cable › record (3 arrays)
└── seismic (time) 4.1 MB
╰─ network › station › location › channel (3 arrays)
```

Index into a branch to see its table:

```python
dc["seismic"]
```
```text
<xdas.DataCollection: 3 arrays, 4.1 MB>
network station location channel
FR OGSI 00 HHZ (time: 360000) 1.4 MB
HHN (time: 360000) 1.4 MB
HHE (time: 360000) 1.4 MB
```
2 changes: 1 addition & 1 deletion docs/user-guide/pipeline/picking.md
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ cable pair, each recorded as two consecutive records:
... "cable",
... )
>>> das
<xdas.DataCollection: 4 leaves, 937.5 KB>
<xdas.DataCollection: 4 arrays, 937.5 KB>
cable record
east 0 (time: 10000, distance: 3) 234.4 KB
1 (time: 10000, distance: 3) 234.4 KB
Expand Down
157 changes: 142 additions & 15 deletions tests/test_datacollection.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ def test_empty_mapping_repr(self):
from xdas.core.datacollection import DataMapping

dm = DataMapping({}, "empty")
assert repr(dm) == "<xdas.DataCollection: 0 leaves, 0 B>"
assert repr(dm) == "<xdas.DataCollection: empty>"

def test_mapping_reduce(self):
import pickle
Expand Down Expand Up @@ -617,7 +617,7 @@ def test_repr_measures_a_table_by_its_columns(self):
df = pd.DataFrame({"time": [1.0, 2.0], "value": [0.5, 0.9]})
dc = xd.DataCollection({"ST01": df}, "station")
assert "(rows: 2, columns: 2)" in repr(dc)
assert repr(dc).splitlines()[0] == "<xdas.DataCollection: 1 leaf, 32 B>"
assert repr(dc).splitlines()[0] == "<xdas.DataCollection: 1 table, 32 B>"


class TestCollectionRepr:
Expand All @@ -636,7 +636,7 @@ def test_levels_head_the_columns_when_homogeneous(self):
"node",
)
lines = repr(dc).splitlines()
assert lines[0] == "<xdas.DataCollection: 1 leaf, 48 B>"
assert lines[0] == "<xdas.DataCollection: 1 array, 48 B>"
assert lines[1] == "node cable record"
assert lines[2] == "CCN N 0 (time: 4, distance: 3) 48 B"

Expand All @@ -656,35 +656,162 @@ def test_a_depth_a_branch_never_reaches_is_marked_absent(self):
"SER S - (time: 4) 16 B",
]

def test_an_unnamed_depth_leaves_its_column_unheaded(self):
# the named depths are still worth heading
def test_an_unnamed_depth_draws_a_tree(self):
# a column without a name is not a table: each level is spelled out
dc = xd.DataCollection({"CCN": {"N": [self.array(time=4)]}}, "node")
assert repr(dc).splitlines()[1:] == [
"node",
"CCN N 0 (time: 4) 16 B",
"└── CCN 1 array, 16 B",
" └── N 1 array, 16 B",
" └── 0 (time: 4) 16 B",
]

def test_a_disputed_depth_leaves_its_column_unheaded(self):
# one branch calls it a cable, the other a record: neither speaks for
# the column, but the root still does
def test_a_disputed_depth_draws_a_tree(self):
# one branch calls it a cable, the other a record: no column could be
# headed, so each branch collapses into its own summary line
dc = xd.DataCollection(
{
"CCN": ("cable", [self.array(time=4)]),
"SER": ("record", [self.array(time=4)]),
"SER": ("record", [self.array(time=4), self.array(time=4)]),
},
"node",
)
assert repr(dc).splitlines() == [
"<xdas.DataCollection: 3 arrays, 48 B>",
"node",
"├── CCN (time) 16 B",
"│ ╰─ cable (1 array)",
"└── SER (time) 32 B",
" ╰─ record (2 arrays)",
]

def test_branches_with_different_schemas_collapse(self):
das = xd.DataCollection(
{"N01": ("cable", {"CAB1": ("record", [self.array(time=4, distance=3)])})},
"node",
)
seismic = xd.DataCollection(
{"FR": ("station", {"OGSI": ("channel", {"HHZ": self.array(time=4)})})},
"network",
)
dc = xd.DataCollection({"das": das, "seismic": seismic}, "kind")
assert repr(dc).splitlines() == [
"<xdas.DataCollection: 2 arrays, 64 B>",
"kind",
"├── das (time, distance) 48 B",
"│ ╰─ node › cable › record (1 array)",
"└── seismic (time) 16 B",
" ╰─ network › station › channel (1 array)",
]
# indexing into a collapsed branch shows its table
assert repr(dc["das"]).splitlines()[1] == "node cable record"

def test_leaves_stand_next_to_collapsed_branches(self):
df = pd.DataFrame({"time": [1.0, 2.0], "value": [0.5, 0.9]})
dc = xd.DataCollection(
{
"das": ("cable", {"N": [self.array(time=4)]}),
"seismic": ("station", {"SSB": self.array(time=4)}),
"picks": df,
"stack": self.array(time=2, distance=2),
},
"kind",
)
assert repr(dc).splitlines() == [
"<xdas.DataCollection: 3 arrays, 1 table, 80 B>",
"kind",
"├── das 1 array, 16 B",
"│ cable",
"│ └── N 1 array, 16 B",
"│ └── 0 (time: 4) 16 B",
"├── seismic (time) 16 B",
"│ ╰─ station (1 array)",
"├── picks (rows: 2, columns: 2) 32 B",
"└── stack (time: 2, distance: 2) 16 B",
]

def test_heterogeneous_branches_open_a_subtree(self):
mixed = {
"das": ("cable", [self.array(time=4)]),
"seismic": ("channel", {"HHZ": self.array(time=4)}),
}
dc = xd.DataCollection(
{
"OBS-A": ("kind", mixed),
"OBS-B": ("kind", {"das": ("cable", [self.array(time=4)])}),
},
"site",
)
assert repr(dc).splitlines() == [
"<xdas.DataCollection: 3 arrays, 48 B>",
"site",
"├── OBS-A 2 arrays, 32 B",
"│ kind",
"│ ├── das (time) 16 B",
"│ │ ╰─ cable (1 array)",
"│ └── seismic (time) 16 B",
"│ ╰─ channel (1 array)",
"└── OBS-B (time) 16 B",
" ╰─ kind › cable (1 array)",
]

def test_an_unnamed_tree_level_has_no_field_line(self):
dc = xd.DataCollection(
[
{
"das": ("cable", [self.array(time=4)]),
"seismic": [self.array(time=4)],
},
self.array(time=4),
]
)
assert repr(dc).splitlines() == [
"<xdas.DataCollection: 3 arrays, 48 B>",
"├── 0 2 arrays, 32 B",
"│ ├── das (time) 16 B",
"│ │ ╰─ cable (1 array)",
"│ └── seismic 1 array, 16 B",
"│ └── 0 (time: 4) 16 B",
"└── 1 (time: 4) 16 B",
]

def test_an_empty_branch_is_marked_empty(self):
dc = xd.DataCollection(
{
"das": ("cable", [self.array(time=4)]),
"seismic": ("station", {}),
"other": ("channel", [self.array(time=4)]),
},
"kind",
)
assert repr(dc).splitlines()[2:] == [
"├── das (time) 16 B",
"│ ╰─ cable (1 array)",
"├── seismic (empty)",
"└── other (time) 16 B",
" ╰─ channel (1 array)",
]

def test_a_deep_empty_branch_is_not_hidden_by_the_table(self):
# an empty branch has no leaf to give it a row: a table would drop it
dc = xd.DataCollection(
{"N01": ("cable", {"CAB1": ("record", [self.array(time=4)]), "CAB2": []})},
"node",
)
assert repr(dc).splitlines()[1:] == [
"node",
"CCN 0 (time: 4) 16 B",
"SER 0 (time: 4) 16 B",
"└── N01 1 array, 16 B",
" cable",
" ├── CAB1 (time) 16 B",
" │ ╰─ record (1 array)",
" └── CAB2 (empty)",
]

def test_no_header_when_nothing_is_named(self):
def test_a_nameless_collection_draws_a_tree(self):
dc = xd.DataCollection([self.array(time=4), self.array(time=4)])
assert repr(dc).splitlines()[1:] == [
"0 (time: 4) 16 B",
"1 (time: 4) 16 B",
"├── 0 (time: 4) 16 B",
"└── 1 (time: 4) 16 B",
]

def test_repeated_keys_are_blanked(self):
Expand Down
Loading
Loading