From 6b8e31bbd1b7cd8293325a7624826c89c67dc2c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Falco?= Date: Mon, 11 May 2026 17:23:20 +0200 Subject: [PATCH 1/7] Return DataArray object even if object is empty --- xdas/core/routines.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xdas/core/routines.py b/xdas/core/routines.py index f7a7847..1d22fbc 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -1319,6 +1319,12 @@ def concat( return objs[0] if objs else DataArray([]) objs = non_empty + if virtual is None: + virtual = all(isinstance(da.data, (VirtualSource, VirtualStack)) for da in objs) + + if len(objs) == 0: + return DataArray() # return empty dataarray + if dim in objs[0].dims + ("first", "last"): axis = objs[0].get_axis_num(dim) dim = objs[0].dims[axis] # ensure not "first" or "last" From c7019a1b257b341043637a4cbc93cef164d8cd41 Mon Sep 17 00:00:00 2001 From: afalco Date: Wed, 15 Jul 2026 10:59:11 +0200 Subject: [PATCH 2/7] to_zarr() method --- xdas/core/dataarray.py | 87 ++++++++++++++++++ xdas/io/zarr.py | 196 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 xdas/io/zarr.py diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index c58fa0b..8e5f24f 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -1027,6 +1027,93 @@ def from_netcdf(cls, fname, group=None): return open_dataarray(fname, group) + + def to_zarr( + self, + fname, + mode="w", + group=None, + virtual=None, + encoding=None, + create_dirs=False, + ): + """ + Write DataArray contents to a zarr file. + + Parameters + ---------- + fname : str + Path to which to save this dataset. + mode : {'w', 'a'}, optional + Write ('w') or append ('a') mode. If mode='a', the file must already exist. + group : str, optional + Path to the zarr4 group in the given file to open. + virtual : bool, optional + Weather to write a virtual dataset. The DataArray data must be a VirtualSource + or a VirtualLayout. Default (None) is to try to write a virtual dataset if + possible. + encoding : dict, optional + Dictionary of encoding attributes. Because a DataArray contains a unique + data variable, the encoding dictionary should not contain the variable name. + For more information on encoding, see the `xarray documentation + `_. + create_dirs : bool, optional + Whether to create parent directories if they do not exist. Default is False. + + Examples + -------- + >>> import os + >>> import tempfile + + >>> import numpy as np + >>> import xdas as xd + >>> import hdf5plugin + + Create some sample data array: + + >>> da = xd.DataArray(np.random.rand(100, 100)) + + Save the dataset with ZFP compression: + + >>> encoding = {"chunks": (10, 10), **hdf5plugin.Zfp(accuracy=0.001)} + >>> with tempfile.TemporaryDirectory() as tmpdir: + ... tmpfile = os.path.join(tmpdir, "path.nc") + ... da.to_zarr(tmpfile, encoding=encoding) + + """ + from ..io.zarr import save_dataarray_zarr + + save_dataarray_zarr(self, fname, mode, group, virtual, encoding, create_dirs) + + + def to_dict(self): + """Convert the DataArray to a dictionary.""" + if isinstance(self.data, VirtualArray): + raise NotImplementedError("cannot convert a virtual array to a dictionary") + elif isinstance(self.data, np.ndarray): + data = self.data.tolist() + elif isinstance(self.data, DaskArray): + data = to_dict(self.data) + return { + "data": data, + "coords": self.coords.to_dict()["coords"], + "dims": self.dims, + "name": self.name, + "attrs": self.attrs, + } + + @classmethod + def from_dict(cls, dct): + """Create a DataArray from a dictionary.""" + if isinstance(dct["data"], list): + data = np.array(dct["data"]) + elif isinstance(dct["data"], dict): + data = from_dict(dct["data"]) + else: + raise ValueError("data must be a list or a dictionary") + coords = Coordinates.from_dict({key: dct[key] for key in ["coords", "dims"]}) + return cls(data, coords, dct["dims"], dct["name"], dct["attrs"]) + def plot(self, *args, **kwargs): """ Plot a DataArray. diff --git a/xdas/io/zarr.py b/xdas/io/zarr.py new file mode 100644 index 0000000..d4340f3 --- /dev/null +++ b/xdas/io/zarr.py @@ -0,0 +1,196 @@ +import os +from pathlib import Path + +import zarr +import h5py +import hdf5plugin +import xarray as xr +from dask.array import Array as DaskArray + +from ..coordinates import Coordinates +from ..core.dataarray import DataArray +from ..core.datacollection import DataCollection, DataMapping, DataSequence +from ..dask.core import create_variable, loads +from ..virtual import VirtualArray, VirtualSource +from .core import Engine + + +class XdasZarrEngine(Engine, name="zarr"): + + def open_dataarray(self, fname, **kwargs): + return xr.open_zarr(fname, **kwargs) + + def save_dataarray_zarr(self, da, fname, **kwargs): + return save_dataarray_zarr(da, fname, **kwargs) + + def open_datacollection(self, fname, **kwargs): + return open_datamapping(fname, **kwargs) + + def save_datacollection(self, dc, fname, **kwargs): + return save_datamapping(dc, fname, **kwargs) + + +def open_zarr(fname, group=None): + if isinstance(fname, Path): + fname = str(fname) + + # read metadata + with xr.open_dataset( + fname, group=group, engine="zarr", decode_timedelta=False + ) as dataset: + # check file format + if not ( + "Conventions" in dataset.attrs and "CF" in dataset.attrs["Conventions"] + ): + raise TypeError( + "file format not recognized. please provide the file format " + "with the `engine` keyword argument" + ) + + # identify the "main" data array + if len(dataset) == 1: + name = next(iter(dataset.keys())) + else: + data_vars = { + key: var + for key, var in dataset.items() + if any("coordinate" in attr for attr in var.attrs) + } + if len(data_vars) == 1: + name = next(iter(data_vars.keys())) + else: + raise ValueError("several possible data arrays detected") + + # read coordinates + coords = Coordinates.from_dataset(dataset, name) + + # read data + if "__dask_array__" in dataset[name].attrs: + data = loads(dataset[name].attrs.pop("__dask_array__")) + else: + with h5py.File(fname) as file: + if group: + file = file[group] + variable = file["__values__" if name is None else name] + data = VirtualSource(variable) + + # pack everything + return DataArray( + data, + coords, + dataset[name].dims, + name, + None if dataset[name].attrs == {} else dataset[name].attrs, + ) + + +def save_dataarray_zarr( + da, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + dx = da.to_xarray() + dx.name = "data" + + ### TODO: da coords != dx coords. Write without xarray? + # chunks=(1000,1000) + # codec = codecs.ZFPY(mode=4, tolerance=tolerance) + # z_write = zarr.array(da.data, chunks=chunks, codecs=[codec], store=fname, overwrite=True) + + xr.DataArray.to_zarr(dx,fname, mode=mode, group=group, encoding=encoding) + + +def open_datacollection(fname, group=None): + if isinstance(fname, Path): + fname = str(fname) + dc = open_datamapping(fname, group) + try: + keys = [int(key) for key in dc.keys()] + if keys == list(range(len(keys))): + return DataSequence.from_mapping(dc) + else: + return dc + except ValueError: + return dc + + +def save_datacollection( + dc, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + if isinstance(fname, Path): + fname = str(fname) + + if isinstance(dc, DataSequence): + save_datasequence(dc, fname, mode, group, virtual, encoding, create_dirs) + elif isinstance(dc, DataCollection): + save_datamapping(dc, fname, mode, group, virtual, encoding, create_dirs) + else: + raise ValueError("can only save a DataCollection or a DataSequence") + + +def open_datamapping(fname, group=None): + if isinstance(fname, Path): + fname = str(fname) + + with h5py.File(fname, "r") as file: + if group is None: + group = file[list(file.keys())[0]] + else: + group = file[group] + name = group.name.split("/")[-1] + if isinstance(group, h5py.Dataset): + raise ValueError( + "it looks like you are trying to open a data array as a data collection." + ) + else: + if not isinstance(group, h5py.Group): + raise RuntimeError( + "something went wrong while opening the data collection." + ) + keys = list(group.keys()) + dm = DataMapping({}, name=None if name == "collection" else name) + for key in keys: + subgroup = group[key] + if _get_depth(subgroup) == 0: + dm[key] = DataArray.from_netcdf(fname, subgroup.name) + else: + subgroup = subgroup[list(subgroup.keys())[0]] + dm[key] = DataCollection.from_netcdf(fname, subgroup.name) + return dm + + +def save_datamapping( + dm, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + if mode == "w" and group is None and os.path.exists(fname): + os.remove(fname) + for key in dm: + name = dm.name if dm.name is not None else "collection" + location = "/".join([name, str(key)]) + if group is not None: + location = "/".join([group, location]) + if create_dirs: + dirname = os.path.dirname(fname) + if dirname: + os.makedirs(dirname, exist_ok=True) + dm[key].to_zarr( + fname, mode="a", group=location, virtual=virtual, encoding=encoding + ) + + +def open_datasequence(fname, group=None): + dm = open_datamapping(fname, group) + return DataSequence.from_mapping(dm) + + +def save_datasequence( + ds, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + dm = ds.to_mapping() + save_datamapping(dm, fname, mode, group, virtual, encoding, create_dirs) + + +def _get_depth(group): + if not isinstance(group, h5py.Group): + raise ValueError("not a group") + depths = [] + group.visit(lambda name: depths.append(name.count("/"))) + return max(depths) From 3b3efa31a1d5f07afaecf05b126d157aea96a77b Mon Sep 17 00:00:00 2001 From: afalco Date: Wed, 15 Jul 2026 11:03:10 +0200 Subject: [PATCH 3/7] Set chunk size to minimum to not crash --- xdas/processing/core.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/xdas/processing/core.py b/xdas/processing/core.py index 1d2640e..1faf4f6 100644 --- a/xdas/processing/core.py +++ b/xdas/processing/core.py @@ -1001,10 +1001,7 @@ def __init__(self, da, chunks, max_buffers=1, max_workers=1, pool="threads"): f"dimensions {da.dims}" ) if chunk_size > da.sizes[chunk_dim]: - raise ValueError( - f"chunking size {chunk_size} is greater than `da` " - f"size {da.sizes[chunk_dim]} along dim {chunk_dim}" - ) + chunk_size = min(chunk_size, da.sizes[chunk_dim]) size = da.sizes[chunk_dim] divs = list(range(0, size, chunk_size)) + [size] else: From cc577ee7e867dbeb4b9640cdad6ae86db7e3c7b0 Mon Sep 17 00:00:00 2001 From: afalco Date: Wed, 22 Jul 2026 16:12:57 +0200 Subject: [PATCH 4/7] open_dataarray_zarr functional --- xdas/core/dataarray.py | 21 +++++++++++++ xdas/io/__init__.py | 3 +- xdas/io/zarr.py | 67 ++++++------------------------------------ 3 files changed, 32 insertions(+), 59 deletions(-) diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index 8e5f24f..6f77d05 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -1085,6 +1085,27 @@ def to_zarr( save_dataarray_zarr(self, fname, mode, group, virtual, encoding, create_dirs) + @classmethod + def from_zarr(cls, fname, group=None): + """ + Read a data array from a zarr file. + + Parameters + ---------- + fname: str + The path of the file to open. + group: str, optional + The location of the data array within the file. Root by default + + Returns + ------- + DataArray + The openend data array. + """ + from ..io.zarr import open_dataarray_zarr + + return open_dataarray_zarr(fname, group) + def to_dict(self): """Convert the DataArray to a dictionary.""" diff --git a/xdas/io/__init__.py b/xdas/io/__init__.py index 5b5c90e..13c9084 100644 --- a/xdas/io/__init__.py +++ b/xdas/io/__init__.py @@ -21,9 +21,10 @@ "silixa", "terra15", "xdas", + "zarr", ] -from . import apsensing, aragon, asn, febus, prodml, silixa, terra15, xdas +from . import apsensing, aragon, asn, febus, prodml, silixa, terra15, xdas, zarr from .core import AutoEngine, Engine, get_free_port # isort: split diff --git a/xdas/io/zarr.py b/xdas/io/zarr.py index d4340f3..3b9f440 100644 --- a/xdas/io/zarr.py +++ b/xdas/io/zarr.py @@ -18,9 +18,9 @@ class XdasZarrEngine(Engine, name="zarr"): def open_dataarray(self, fname, **kwargs): - return xr.open_zarr(fname, **kwargs) + return open_dataarray_zarr(fname, **kwargs) - def save_dataarray_zarr(self, da, fname, **kwargs): + def save_dataarray(self, da, fname, **kwargs): return save_dataarray_zarr(da, fname, **kwargs) def open_datacollection(self, fname, **kwargs): @@ -29,60 +29,10 @@ def open_datacollection(self, fname, **kwargs): def save_datacollection(self, dc, fname, **kwargs): return save_datamapping(dc, fname, **kwargs) - -def open_zarr(fname, group=None): - if isinstance(fname, Path): - fname = str(fname) - - # read metadata - with xr.open_dataset( - fname, group=group, engine="zarr", decode_timedelta=False - ) as dataset: - # check file format - if not ( - "Conventions" in dataset.attrs and "CF" in dataset.attrs["Conventions"] - ): - raise TypeError( - "file format not recognized. please provide the file format " - "with the `engine` keyword argument" - ) - - # identify the "main" data array - if len(dataset) == 1: - name = next(iter(dataset.keys())) - else: - data_vars = { - key: var - for key, var in dataset.items() - if any("coordinate" in attr for attr in var.attrs) - } - if len(data_vars) == 1: - name = next(iter(data_vars.keys())) - else: - raise ValueError("several possible data arrays detected") - - # read coordinates - coords = Coordinates.from_dataset(dataset, name) - - # read data - if "__dask_array__" in dataset[name].attrs: - data = loads(dataset[name].attrs.pop("__dask_array__")) - else: - with h5py.File(fname) as file: - if group: - file = file[group] - variable = file["__values__" if name is None else name] - data = VirtualSource(variable) - - # pack everything - return DataArray( - data, - coords, - dataset[name].dims, - name, - None if dataset[name].attrs == {} else dataset[name].attrs, - ) - +def open_dataarray_zarr(fname, **kwargs): + xr_da = xr.open_zarr(fname, **kwargs) + xr_da["name"] = fname + return DataArray.from_xarray(xr_da) def save_dataarray_zarr( da, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False @@ -150,10 +100,11 @@ def open_datamapping(fname, group=None): for key in keys: subgroup = group[key] if _get_depth(subgroup) == 0: - dm[key] = DataArray.from_netcdf(fname, subgroup.name) + dm[key] = DataArray.from_zarr(fname, subgroup.name) else: + raise NotImplementedError("zarr datacollection TBD") subgroup = subgroup[list(subgroup.keys())[0]] - dm[key] = DataCollection.from_netcdf(fname, subgroup.name) + dm[key] = DataCollection.from_zarr(fname, subgroup.name) return dm From a8916a0525051691bff8db1adbfbec7e91e5b08f Mon Sep 17 00:00:00 2001 From: afalco Date: Wed, 22 Jul 2026 16:13:15 +0200 Subject: [PATCH 5/7] dependency to arro3.core --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e49820f..4ba02e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ authors = [ { name = "Alister Trabattoni", email = "alister.trabattoni@gmail.com" }, ] dependencies = [ + "arro3.core", + "dask<2025.4.0", "h5netcdf", "h5py", "hdf5plugin", From 936abd109baef094bc4845427fedbede9b5d331f Mon Sep 17 00:00:00 2001 From: afalco Date: Tue, 10 Nov 2026 15:30:22 +0100 Subject: [PATCH 6/7] Write engine (zarr & xdas for now) --- xdas/io/zarr.py | 5 ----- xdas/processing/core.py | 14 ++++++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/xdas/io/zarr.py b/xdas/io/zarr.py index 3b9f440..17843d6 100644 --- a/xdas/io/zarr.py +++ b/xdas/io/zarr.py @@ -3,15 +3,10 @@ import zarr import h5py -import hdf5plugin import xarray as xr -from dask.array import Array as DaskArray -from ..coordinates import Coordinates from ..core.dataarray import DataArray from ..core.datacollection import DataCollection, DataMapping, DataSequence -from ..dask.core import create_variable, loads -from ..virtual import VirtualArray, VirtualSource from .core import Engine diff --git a/xdas/processing/core.py b/xdas/processing/core.py index 1faf4f6..231d9e8 100644 --- a/xdas/processing/core.py +++ b/xdas/processing/core.py @@ -122,10 +122,14 @@ def _load(da): return da.load() -def _dump(chunk, path, encoding): +def _dump(chunk, path, encoding, engine): """Write *chunk* to *path* and return it virtually. Egress unit of work.""" - chunk.to_netcdf(path, encoding=encoding) - return open_dataarray(path) + if engine == "zarr": + chunk.to_zarr(path, encoding=encoding) + else: + chunk.to_netcdf(path, encoding=encoding) + return open_dataarray(path, engine=engine) + def process(atom, source, out=None, chunks=None, until=None, merge=True): @@ -1174,6 +1178,7 @@ def __init__( self, dirpath, encoding=None, + engine="xdas", max_buffers=1, max_workers=1, create_dirs=False, @@ -1200,6 +1205,7 @@ def __init__( self.dirpath = dirpath self.dim = dim self.encoding = encoding + self.engine = engine self.max_buffers = max_buffers self.max_workers = max_workers self.pool = pool @@ -1228,7 +1234,7 @@ def submit(self, chunk): result = future.result() self._results.append(result) path = os.path.join(self.dirpath, f"{self._count:09d}") - self._futures.append(self._executor.submit(_dump, chunk, path, self.encoding)) + self._futures.append(self._executor.submit(_dump, chunk, path, self.encoding, self.engine)) self._count += 1 def write(self, chunk): From 911b833648251c140b694ae3dfdd3605fe9fd97b Mon Sep 17 00:00:00 2001 From: afalco Date: Tue, 10 Nov 2026 15:29:58 +0100 Subject: [PATCH 7/7] Allow to assign parent when creating Coordinates --- xdas/coordinates/core.py | 23 ++++++++++++++--------- xdas/core/dataarray.py | 5 ++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/xdas/coordinates/core.py b/xdas/coordinates/core.py index f741f20..31f556b 100644 --- a/xdas/coordinates/core.py +++ b/xdas/coordinates/core.py @@ -114,13 +114,15 @@ class Coordinates(dict): interrogator: 'SRN' """ - def __init__(self, coords=None, dims=None): + def __init__(self, coords=None, dims=None, parent=None): super().__init__() if isinstance(coords, Coordinates): if dims is None: dims = coords.dims coords = dict(coords) self._dims = () if dims is None else tuple(dims) + if parent is not None: + self._parent = weakref.ref(parent) if coords is not None: for name in coords: self[name] = coords[name] @@ -144,14 +146,17 @@ def __setitem__(self, key, value): else: if coord.dim is not None: if coord.dim not in self.dims: - raise KeyError( - f"cannot add new dimension {coord.dim} to an existing DataArray" - ) - size = self.parent.sizes[coord.dim] - if not len(coord) == size: - raise ValueError( - f"conflicting sizes for dimension {coord.dim}: size {len(coord)} " - f"in `coords` and size {size} in `data`" + pass + # warnings.warn( + # # raise KeyError( + # f"cannot add new dimension {coord.dim} to an existing DataArray" + # ) + elif hasattr(self.parent, "sizes"): + size = self.parent.sizes[coord.dim] + if not len(coord) == size: + raise ValueError( + f"conflicting sizes for dimension {coord.dim}: size {len(coord)} " + f"in `coords` and size {size} in `data`" ) coord._assign_parent(self) return super().__setitem__(key, coord) diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index 6f77d05..ae2ea68 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -66,9 +66,8 @@ def __init__(self, data=None, coords=None, dims=None, name=None, attrs=None): dims = coords.dims if dims is not None and len(dims) != data.ndim: raise ValueError("different number of dimensions on `data` and `dims`") - coords = Coordinates(coords, dims) - coords._assign_parent(self) - self._coords = coords + new_coords = Coordinates(coords, dims, self) + self._coords = new_coords # metadata self.name = name