diff --git a/pyproject.toml b/pyproject.toml index e49820f1..4ba02e67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ authors = [ { name = "Alister Trabattoni", email = "alister.trabattoni@gmail.com" }, ] dependencies = [ + "arro3.core", + "dask<2025.4.0", "h5netcdf", "h5py", "hdf5plugin", diff --git a/xdas/coordinates/core.py b/xdas/coordinates/core.py index f741f204..31f556b8 100644 --- a/xdas/coordinates/core.py +++ b/xdas/coordinates/core.py @@ -114,13 +114,15 @@ class Coordinates(dict): interrogator: 'SRN' """ - def __init__(self, coords=None, dims=None): + def __init__(self, coords=None, dims=None, parent=None): super().__init__() if isinstance(coords, Coordinates): if dims is None: dims = coords.dims coords = dict(coords) self._dims = () if dims is None else tuple(dims) + if parent is not None: + self._parent = weakref.ref(parent) if coords is not None: for name in coords: self[name] = coords[name] @@ -144,14 +146,17 @@ def __setitem__(self, key, value): else: if coord.dim is not None: if coord.dim not in self.dims: - raise KeyError( - f"cannot add new dimension {coord.dim} to an existing DataArray" - ) - size = self.parent.sizes[coord.dim] - if not len(coord) == size: - raise ValueError( - f"conflicting sizes for dimension {coord.dim}: size {len(coord)} " - f"in `coords` and size {size} in `data`" + pass + # warnings.warn( + # # raise KeyError( + # f"cannot add new dimension {coord.dim} to an existing DataArray" + # ) + elif hasattr(self.parent, "sizes"): + size = self.parent.sizes[coord.dim] + if not len(coord) == size: + raise ValueError( + f"conflicting sizes for dimension {coord.dim}: size {len(coord)} " + f"in `coords` and size {size} in `data`" ) coord._assign_parent(self) return super().__setitem__(key, coord) diff --git a/xdas/core/dataarray.py b/xdas/core/dataarray.py index c58fa0ba..ae2ea689 100644 --- a/xdas/core/dataarray.py +++ b/xdas/core/dataarray.py @@ -66,9 +66,8 @@ def __init__(self, data=None, coords=None, dims=None, name=None, attrs=None): dims = coords.dims if dims is not None and len(dims) != data.ndim: raise ValueError("different number of dimensions on `data` and `dims`") - coords = Coordinates(coords, dims) - coords._assign_parent(self) - self._coords = coords + new_coords = Coordinates(coords, dims, self) + self._coords = new_coords # metadata self.name = name @@ -1027,6 +1026,114 @@ def from_netcdf(cls, fname, group=None): return open_dataarray(fname, group) + + def to_zarr( + self, + fname, + mode="w", + group=None, + virtual=None, + encoding=None, + create_dirs=False, + ): + """ + Write DataArray contents to a zarr file. + + Parameters + ---------- + fname : str + Path to which to save this dataset. + mode : {'w', 'a'}, optional + Write ('w') or append ('a') mode. If mode='a', the file must already exist. + group : str, optional + Path to the zarr4 group in the given file to open. + virtual : bool, optional + Weather to write a virtual dataset. The DataArray data must be a VirtualSource + or a VirtualLayout. Default (None) is to try to write a virtual dataset if + possible. + encoding : dict, optional + Dictionary of encoding attributes. Because a DataArray contains a unique + data variable, the encoding dictionary should not contain the variable name. + For more information on encoding, see the `xarray documentation + `_. + create_dirs : bool, optional + Whether to create parent directories if they do not exist. Default is False. + + Examples + -------- + >>> import os + >>> import tempfile + + >>> import numpy as np + >>> import xdas as xd + >>> import hdf5plugin + + Create some sample data array: + + >>> da = xd.DataArray(np.random.rand(100, 100)) + + Save the dataset with ZFP compression: + + >>> encoding = {"chunks": (10, 10), **hdf5plugin.Zfp(accuracy=0.001)} + >>> with tempfile.TemporaryDirectory() as tmpdir: + ... tmpfile = os.path.join(tmpdir, "path.nc") + ... da.to_zarr(tmpfile, encoding=encoding) + + """ + from ..io.zarr import save_dataarray_zarr + + save_dataarray_zarr(self, fname, mode, group, virtual, encoding, create_dirs) + + @classmethod + def from_zarr(cls, fname, group=None): + """ + Read a data array from a zarr file. + + Parameters + ---------- + fname: str + The path of the file to open. + group: str, optional + The location of the data array within the file. Root by default + + Returns + ------- + DataArray + The openend data array. + """ + from ..io.zarr import open_dataarray_zarr + + return open_dataarray_zarr(fname, group) + + + def to_dict(self): + """Convert the DataArray to a dictionary.""" + if isinstance(self.data, VirtualArray): + raise NotImplementedError("cannot convert a virtual array to a dictionary") + elif isinstance(self.data, np.ndarray): + data = self.data.tolist() + elif isinstance(self.data, DaskArray): + data = to_dict(self.data) + return { + "data": data, + "coords": self.coords.to_dict()["coords"], + "dims": self.dims, + "name": self.name, + "attrs": self.attrs, + } + + @classmethod + def from_dict(cls, dct): + """Create a DataArray from a dictionary.""" + if isinstance(dct["data"], list): + data = np.array(dct["data"]) + elif isinstance(dct["data"], dict): + data = from_dict(dct["data"]) + else: + raise ValueError("data must be a list or a dictionary") + coords = Coordinates.from_dict({key: dct[key] for key in ["coords", "dims"]}) + return cls(data, coords, dct["dims"], dct["name"], dct["attrs"]) + def plot(self, *args, **kwargs): """ Plot a DataArray. diff --git a/xdas/core/routines.py b/xdas/core/routines.py index f7a7847c..1d22fbc2 100644 --- a/xdas/core/routines.py +++ b/xdas/core/routines.py @@ -1319,6 +1319,12 @@ def concat( return objs[0] if objs else DataArray([]) objs = non_empty + if virtual is None: + virtual = all(isinstance(da.data, (VirtualSource, VirtualStack)) for da in objs) + + if len(objs) == 0: + return DataArray() # return empty dataarray + if dim in objs[0].dims + ("first", "last"): axis = objs[0].get_axis_num(dim) dim = objs[0].dims[axis] # ensure not "first" or "last" diff --git a/xdas/io/__init__.py b/xdas/io/__init__.py index 5b5c90ea..13c9084f 100644 --- a/xdas/io/__init__.py +++ b/xdas/io/__init__.py @@ -21,9 +21,10 @@ "silixa", "terra15", "xdas", + "zarr", ] -from . import apsensing, aragon, asn, febus, prodml, silixa, terra15, xdas +from . import apsensing, aragon, asn, febus, prodml, silixa, terra15, xdas, zarr from .core import AutoEngine, Engine, get_free_port # isort: split diff --git a/xdas/io/zarr.py b/xdas/io/zarr.py new file mode 100644 index 00000000..17843d68 --- /dev/null +++ b/xdas/io/zarr.py @@ -0,0 +1,142 @@ +import os +from pathlib import Path + +import zarr +import h5py +import xarray as xr + +from ..core.dataarray import DataArray +from ..core.datacollection import DataCollection, DataMapping, DataSequence +from .core import Engine + + +class XdasZarrEngine(Engine, name="zarr"): + + def open_dataarray(self, fname, **kwargs): + return open_dataarray_zarr(fname, **kwargs) + + def save_dataarray(self, da, fname, **kwargs): + return save_dataarray_zarr(da, fname, **kwargs) + + def open_datacollection(self, fname, **kwargs): + return open_datamapping(fname, **kwargs) + + def save_datacollection(self, dc, fname, **kwargs): + return save_datamapping(dc, fname, **kwargs) + +def open_dataarray_zarr(fname, **kwargs): + xr_da = xr.open_zarr(fname, **kwargs) + xr_da["name"] = fname + return DataArray.from_xarray(xr_da) + +def save_dataarray_zarr( + da, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + dx = da.to_xarray() + dx.name = "data" + + ### TODO: da coords != dx coords. Write without xarray? + # chunks=(1000,1000) + # codec = codecs.ZFPY(mode=4, tolerance=tolerance) + # z_write = zarr.array(da.data, chunks=chunks, codecs=[codec], store=fname, overwrite=True) + + xr.DataArray.to_zarr(dx,fname, mode=mode, group=group, encoding=encoding) + + +def open_datacollection(fname, group=None): + if isinstance(fname, Path): + fname = str(fname) + dc = open_datamapping(fname, group) + try: + keys = [int(key) for key in dc.keys()] + if keys == list(range(len(keys))): + return DataSequence.from_mapping(dc) + else: + return dc + except ValueError: + return dc + + +def save_datacollection( + dc, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + if isinstance(fname, Path): + fname = str(fname) + + if isinstance(dc, DataSequence): + save_datasequence(dc, fname, mode, group, virtual, encoding, create_dirs) + elif isinstance(dc, DataCollection): + save_datamapping(dc, fname, mode, group, virtual, encoding, create_dirs) + else: + raise ValueError("can only save a DataCollection or a DataSequence") + + +def open_datamapping(fname, group=None): + if isinstance(fname, Path): + fname = str(fname) + + with h5py.File(fname, "r") as file: + if group is None: + group = file[list(file.keys())[0]] + else: + group = file[group] + name = group.name.split("/")[-1] + if isinstance(group, h5py.Dataset): + raise ValueError( + "it looks like you are trying to open a data array as a data collection." + ) + else: + if not isinstance(group, h5py.Group): + raise RuntimeError( + "something went wrong while opening the data collection." + ) + keys = list(group.keys()) + dm = DataMapping({}, name=None if name == "collection" else name) + for key in keys: + subgroup = group[key] + if _get_depth(subgroup) == 0: + dm[key] = DataArray.from_zarr(fname, subgroup.name) + else: + raise NotImplementedError("zarr datacollection TBD") + subgroup = subgroup[list(subgroup.keys())[0]] + dm[key] = DataCollection.from_zarr(fname, subgroup.name) + return dm + + +def save_datamapping( + dm, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + if mode == "w" and group is None and os.path.exists(fname): + os.remove(fname) + for key in dm: + name = dm.name if dm.name is not None else "collection" + location = "/".join([name, str(key)]) + if group is not None: + location = "/".join([group, location]) + if create_dirs: + dirname = os.path.dirname(fname) + if dirname: + os.makedirs(dirname, exist_ok=True) + dm[key].to_zarr( + fname, mode="a", group=location, virtual=virtual, encoding=encoding + ) + + +def open_datasequence(fname, group=None): + dm = open_datamapping(fname, group) + return DataSequence.from_mapping(dm) + + +def save_datasequence( + ds, fname, mode="w", group=None, virtual=None, encoding=None, create_dirs=False +): + dm = ds.to_mapping() + save_datamapping(dm, fname, mode, group, virtual, encoding, create_dirs) + + +def _get_depth(group): + if not isinstance(group, h5py.Group): + raise ValueError("not a group") + depths = [] + group.visit(lambda name: depths.append(name.count("/"))) + return max(depths) diff --git a/xdas/processing/core.py b/xdas/processing/core.py index 1d2640e7..231d9e85 100644 --- a/xdas/processing/core.py +++ b/xdas/processing/core.py @@ -122,10 +122,14 @@ def _load(da): return da.load() -def _dump(chunk, path, encoding): +def _dump(chunk, path, encoding, engine): """Write *chunk* to *path* and return it virtually. Egress unit of work.""" - chunk.to_netcdf(path, encoding=encoding) - return open_dataarray(path) + if engine == "zarr": + chunk.to_zarr(path, encoding=encoding) + else: + chunk.to_netcdf(path, encoding=encoding) + return open_dataarray(path, engine=engine) + def process(atom, source, out=None, chunks=None, until=None, merge=True): @@ -1001,10 +1005,7 @@ def __init__(self, da, chunks, max_buffers=1, max_workers=1, pool="threads"): f"dimensions {da.dims}" ) if chunk_size > da.sizes[chunk_dim]: - raise ValueError( - f"chunking size {chunk_size} is greater than `da` " - f"size {da.sizes[chunk_dim]} along dim {chunk_dim}" - ) + chunk_size = min(chunk_size, da.sizes[chunk_dim]) size = da.sizes[chunk_dim] divs = list(range(0, size, chunk_size)) + [size] else: @@ -1177,6 +1178,7 @@ def __init__( self, dirpath, encoding=None, + engine="xdas", max_buffers=1, max_workers=1, create_dirs=False, @@ -1203,6 +1205,7 @@ def __init__( self.dirpath = dirpath self.dim = dim self.encoding = encoding + self.engine = engine self.max_buffers = max_buffers self.max_workers = max_workers self.pool = pool @@ -1231,7 +1234,7 @@ def submit(self, chunk): result = future.result() self._results.append(result) path = os.path.join(self.dirpath, f"{self._count:09d}") - self._futures.append(self._executor.submit(_dump, chunk, path, self.encoding)) + self._futures.append(self._executor.submit(_dump, chunk, path, self.encoding, self.engine)) self._count += 1 def write(self, chunk):