diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 4f47b0cae3..d1b7cbb869 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -100,13 +100,6 @@ jobs: cd ${{ github.workspace }}/main pip install --no-deps -e . - # Make UMRead library - - name: Make UMRead - shell: bash -l {0} - run: | - cd ${{ github.workspace }}/main/cf/umread_lib/c-lib - make - # Install the coverage library # We do so with conda which was setup in a previous step. - name: Install coverage diff --git a/.gitignore b/.gitignore index 7a465fe460..fc6090568c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,11 +21,6 @@ cf/test/*.txt cf/test/.coverage cf/test/cf_coverage_report/ -# C object files generated by tests under cf/umread_lib/ and its sub-dirs: -cf/umread_lib/c-lib/*.o -cf/umread_lib/c-lib/*.so -cf/umread_lib/c-lib/type-dep/*.o - # packaging-related files changing with environment adjustments: *.egg-info/ diff --git a/Changelog.rst b/Changelog.rst index cfc1137c65..6f7b43b844 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -3,6 +3,24 @@ Version NEXTVERSION **2026-08-??** +* New keywords to `cf.read`: ``backend``, ``backend_options``, + ``cfa_filesystem``, ``cfa_backend``, ``cfa_backend_options`` + (https://github.com/NCAS-CMS/cf-python/issues/961) +* Deprecated keyword to `cfdm.read`: ``netcdf_backend`` + (https://github.com/NCAS-CMS/cf-python/issues/961) +* Dataset reads are now entirely managed by `xnetcdf` and its backend + libraries (via `cfdm.read`) + (https://github.com/NCAS-CMS/cf-python/issues/961) +* Read with `cf.read` anything that can be read by `xarray` + (https://github.com/NCAS-CMS/cf-python/issues/961) +* Convert `xarray.Dataset` and `xarray.DataTree` to `cf.Field` via + `cf.read` (https://github.com/NCAS-CMS/cf-python/issues/961) +* Convert `pyfive.File`, `zarr.Group`, `h5py.File`, `umfile.File`, and + `xnetcdf.Dataset` to `cf.Field` via `cf.read` + (https://github.com/NCAS-CMS/cf-python/issues/961) +* Extend `cf.Field.create_latlon_coordinates` to allow the creation of + 2-d latitudes/longitudes from plane projection and rotated pole + coordinates (https://github.com/NCAS-CMS/cf-python/issues/962) * Automatically create consolidated HDF5 metadata with `cf.write`. New keywords to `cf.write`: ``hdf5_consolidated_metadata`` and ``hdf5_expansion_factor`` diff --git a/MANIFEST.in b/MANIFEST.in index 6b545ff796..675685f6a0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,7 +9,7 @@ recursive-exclude cf/data *.rst prune cf/test recursive-include cf/test __init__.py test_*.py recursive-include cf/test cfa_test.sh run_tests.py setup_create_field.py create_test_files*.py individual_tests.sh -recursive-include cf/test test_file.nc test_file[2-4].nc file.nc file[1-9].nc ugrid_global_1.nc ugrid_global_2.nc create_test_files.npz create_test_files_2.npz wgdos_packed.pp extra_data.pp file1.pp *.cdl create_test_files.npz create_test_files_2.npz example_field_0.nc cell_measures.nc +recursive-include cf/test test_file.nc test_file[2-4].nc file.nc file[1-9].nc ugrid_global_1.nc ugrid_global_2.nc create_test_files.npz create_test_files_2.npz wgdos_packed.pp extra_data.pp file1.pp rotated_pole.pp *.cdl create_test_files.npz create_test_files_2.npz example_field_0.nc cell_measures.nc recursive-include cf/test/example_field_0.zarr[23] * include cf/test/example_field_0.kerchunk prune cf/test/dir diff --git a/cf/__init__.py b/cf/__init__.py index c782adf6f2..35b6f81332 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -104,8 +104,8 @@ # Check the version of cfdm (this is worth doing because of the very # tight coupling between cf and cfdm, and the risk of bad things # happening at run time if the versions are mismatched). -_minimum_vn = "1.13.2.1" -_maximum_vn = "1.13.3.0" +_minimum_vn = "1.13.3.0" +_maximum_vn = "1.13.4.0" _cfdm_vn = Version(cfdm.__version__) if _cfdm_vn < Version(_minimum_vn) or _cfdm_vn >= Version(_maximum_vn): raise RuntimeError( @@ -172,6 +172,7 @@ ScipyNetcdfFileArray, SubsampledArray, UMArray, + XnetcdfArray, ZarrArray, ) diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index cfa4ce156c..a178f444b0 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -32,16 +32,12 @@ BoundsFromNodesArray, CellConnectivityArray, GatheredArray, - H5netcdfArray, - NetCDF4Array, PointTopologyArray, - PyfiveArray, RaggedContiguousArray, RaggedIndexedArray, RaggedIndexedContiguousArray, - ScipyNetcdfFileArray, SubsampledArray, - ZarrArray, + XnetcdfArray, ) from .functions import CF @@ -147,18 +143,14 @@ def set_construct(self, parent, construct, axes=None, copy=True, **kwargs): BoundsFromNodesArray=BoundsFromNodesArray, CellConnectivityArray=CellConnectivityArray, GatheredArray=GatheredArray, - H5netcdfArray=H5netcdfArray, - NetCDF4Array=NetCDF4Array, - ScipyNetcdfFileArray=ScipyNetcdfFileArray, PointTopologyArray=PointTopologyArray, - PyfiveArray=PyfiveArray, Quantization=Quantization, RaggedContiguousArray=RaggedContiguousArray, RaggedIndexedArray=RaggedIndexedArray, RaggedIndexedContiguousArray=RaggedIndexedContiguousArray, SubsampledArray=SubsampledArray, TiePointIndex=TiePointIndex, - ZarrArray=ZarrArray, + XnetcdfArray=XnetcdfArray, ) @@ -174,51 +166,5 @@ def implementation(): `CFImplementation` A container for the CF data model implementation. - **Examples** - - >>> i = cf.implementation() - >>> i - - >>> i.classes() - {'AuxiliaryCoordinate': cf.auxiliarycoordinate.AuxiliaryCoordinate, - 'BoundsFromNodesArray': cf.data.array.boundsfromnodesarray.BoundsFromNodesArray, - 'CellConnectivity': cf.cellconnectivity.CellConnectivity, - 'CellConnectivityArray': cf.data.array.cellconnectivityarray.CellConnectivityArray, - 'CellMeasure': cf.cellmeasure.CellMeasure, - 'CellMethod': cf.cellmethod.CellMethod, - 'CoordinateReference': cf.coordinatereference.CoordinateReference, - 'DimensionCoordinate': cf.dimensioncoordinate.DimensionCoordinate, - 'Domain': cf.domain.Domain, - 'DomainAncillary': cf.domainancillary.DomainAncillary, - 'DomainAxis': cf.domainaxis.DomainAxis, - 'DomainTopology': cf.domaintopology.DomainTopology, - 'Field': cf.field.Field, - 'FieldAncillary': cf.fieldancillary.FieldAncillary, - 'Bounds': cf.bounds.Bounds, - 'InteriorRing': cf.interiorring.InteriorRing, - 'InterpolationParameter': cf.interpolationparameter.InterpolationParameter, - 'CoordinateConversion': cf.coordinateconversion.CoordinateConversion, - 'Datum': cf.datum.Datum, - 'List': cf.list.List, - 'Index': cf.index.Index, - 'Count': cf.count.Count, - 'NodeCountProperties': cf.nodecountproperties.NodeCountProperties, - 'PartNodeCountProperties': cf.partnodecountproperties.PartNodeCountProperties, - 'Data': cf.data.data.Data, - 'GatheredArray': cf.data.array.gatheredarray.GatheredArray, - 'H5netcdfArray': cf.data.array.h5netcdfarray.H5netcdfArray, - 'NetCDF4Array': cf.data.array.netcdf4array.NetCDF4Array, - 'ScipyNetcdfFileArray': cf.data.array.scipynetcdffilearray.ScipyNetcdfFileArray, - 'PointTopologyArray': , - 'PyfiveArray': cf.data.array.pyfivearray.PyfiveArray, - 'Quantization': cf.quantization.Quantization, - 'RaggedContiguousArray': cf.data.array.raggedcontiguousarray.RaggedContiguousArray, - 'RaggedIndexedArray': cf.data.array.raggedindexedarray.RaggedIndexedArray, - 'RaggedIndexedContiguousArray': cf.data.array.raggedindexedcontiguousarray.RaggedIndexedContiguousArray, - 'SubsampledArray': cf.data.array.subsampledarray.SubsampledArray, - 'TiePointIndex': cf.tiepointindex.TiePointIndex, - 'ZarrArray': cf.data.array.zarrarray.ZarrArray, - } - """ return _implementation.copy() diff --git a/cf/data/array/__init__.py b/cf/data/array/__init__.py index 2d0c03b3a2..fb06a4ef7b 100644 --- a/cf/data/array/__init__.py +++ b/cf/data/array/__init__.py @@ -14,4 +14,5 @@ from .raggedindexedcontiguousarray import RaggedIndexedContiguousArray from .subsampledarray import SubsampledArray from .umarray import UMArray +from .xnetcdfarray import XnetcdfArray from .zarrarray import ZarrArray diff --git a/cf/data/array/aggregatedarray.py b/cf/data/array/aggregatedarray.py index 1c451bb2fb..f159e1efaf 100644 --- a/cf/data/array/aggregatedarray.py +++ b/cf/data/array/aggregatedarray.py @@ -19,5 +19,5 @@ def __new__(cls, *args, **kwargs): """ # Override the inherited FragmentFileArray class instance = super().__new__(cls) - instance._FragmentArray["uri"] = FragmentFileArray + instance._AggregatedArray__FragmentArray["uri"] = FragmentFileArray return instance diff --git a/cf/data/array/h5netcdfarray.py b/cf/data/array/h5netcdfarray.py index e54ceb4e13..fc97a8d1c5 100644 --- a/cf/data/array/h5netcdfarray.py +++ b/cf/data/array/h5netcdfarray.py @@ -1,16 +1,15 @@ -import cfdm - -from ...mixin_container import Container -from .mixin import ActiveStorageMixin - - -class H5netcdfArray( - ActiveStorageMixin, - Container, - cfdm.H5netcdfArray, -): +class H5netcdfArray: """A netCDF array accessed with `h5netcdf` using the `h5py` backend. .. versionadded:: 3.16.3 """ + + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" + + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use XnetcdfArray instead." + ) diff --git a/cf/data/array/netcdf4array.py b/cf/data/array/netcdf4array.py index 49c2c05c50..8ec9f864f6 100644 --- a/cf/data/array/netcdf4array.py +++ b/cf/data/array/netcdf4array.py @@ -1,12 +1,11 @@ -import cfdm - -from ...mixin_container import Container -from .mixin import ActiveStorageMixin +class NetCDF4Array: + """A netCDF array accessed with `netCDF4`.""" + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" -class NetCDF4Array( - ActiveStorageMixin, - Container, - cfdm.NetCDF4Array, -): - """A netCDF array accessed with `netCDF4`.""" + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use XnetcdfArray instead." + ) diff --git a/cf/data/array/pyfivearray.py b/cf/data/array/pyfivearray.py index 3d6972cbd8..f9835bde74 100644 --- a/cf/data/array/pyfivearray.py +++ b/cf/data/array/pyfivearray.py @@ -1,16 +1,15 @@ -import cfdm - -from ...mixin_container import Container -from .mixin import ActiveStorageMixin - - -class PyfiveArray( - ActiveStorageMixin, - Container, - cfdm.PyfiveArray, -): +class PyfiveArray: """A netCDF array accessed with `pyfive`. .. versionadded:: 3.20.0 """ + + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" + + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use XnetcdfArray instead." + ) diff --git a/cf/data/array/scipynetcdffilearray.py b/cf/data/array/scipynetcdffilearray.py index 8643ced143..54cb21eabb 100644 --- a/cf/data/array/scipynetcdffilearray.py +++ b/cf/data/array/scipynetcdffilearray.py @@ -1,14 +1,15 @@ -import cfdm - -from ...mixin_container import Container - - -class ScipyNetcdfFileArray( - Container, - cfdm.ScipyNetcdfFileArray, -): +class ScipyNetcdfFileArray: """A netCDF-3 array accessed with `scipy.io.netcdf_file`. .. versionadded:: 3.20.0 """ + + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" + + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use XnetcdfArray instead." + ) diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 32944ab1d4..d363e9ac18 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -1,734 +1,11 @@ -import cfdm +class UMArray: + """A UM array.""" -from cf.constants import _stash2standard_name -from cf.functions import _DEPRECATION_ERROR_ATTRIBUTE, load_stash2standard_name -from cf.umread_lib.umfile import File, Rec + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" -from .abstract import Array - - -class UMArray( - cfdm.data.mixin.IndexMixin, - cfdm.data.abstract.FileArray, - Array, -): - """A sub-array stored in a PP or UM fields file.""" - - def __init__( - self, - filename=None, - address=None, - dtype=None, - shape=None, - fmt=None, - word_size=None, - byte_ordering=None, - mask=True, - unpack=True, - attributes=None, - storage_protocol=None, - storage_options=None, - source=None, - copy=True, - ): - """**Initialisation** - - :Parameters: - - filename: (sequence of) `str`, optional - The file name(s). - - address: (sequence of) `int`, optional - The start position in the file(s) of the header(s). - - .. versionadded:: 3.15.0 - - dtype: `numpy.dtype` - The data type of the data array on disk. - - shape: `tuple` - The shape of the unpacked data array. Note that this - is the shape as required by the object containing the - `UMArray` object, and so may contain extra size one - dimensions. When read, the data on disk is reshaped to - *shape*. - - fmt: `str`, optional - ``'PP'`` or ``'FF'`` - - word_size: `int`, optional - ``4`` or ``8`` - - byte_ordering: `str`, optional - ``'little_endian'`` or ``'big_endian'`` - - {{init attributes: `dict` or `None`, optional}} - - During the first `__getitem__` call, any of the - ``_FillValue``, ``add_offset``, ``scale_factor``, - ``units``, and ``calendar`` attributes which haven't - already been set will be inferred from the lookup - header and cached for future use. - - .. versionadded:: 3.16.3 - - {{init storage_protocol: `None` or `str`, optional}} - - .. versionadded:: 3.20.0 - - {{init storage_options: `dict` or `None`, optional}} - - .. versionadded:: 3.20.0 - - {{init source: optional}} - - {{init copy: `bool`, optional}} - - size: `int` - Deprecated at version 3.14.0. - - ndim: `int` - Deprecated at version 3.14.0. - - header_offset: `int` - Deprecated at version 3.15.0. Use the *address* - parameter instead. - - data_offset: `int`, optional - Deprecated at version 3.15.0. - - disk_length: `int`, optional - Deprecated at version 3.15.0. - - units: `str` or `None`, optional - Deprecated at version 3.16.3. Use the - *attributes* parameter instead. - - calendar: `str` or `None`, optional - Deprecated at version 3.16.3. Use the - *attributes* parameter instead. - - """ - super().__init__( - filename=filename, - address=address, - dtype=dtype, - shape=shape, - mask=mask, - unpack=unpack, - attributes=attributes, - storage_protocol=storage_protocol, - storage_options=storage_options, - source=source, - copy=copy, - ) - - if source is not None: - try: - fmt = source._get_component("fmt", None) - except AttributeError: - fmt = None - - try: - word_size = source._get_component("word_size", None) - except AttributeError: - word_size = None - - try: - byte_ordering = source._get_component("byte_ordering", None) - except AttributeError: - byte_ordering = None - - if fmt is not None: - self._set_component("fmt", fmt, copy=False) - - if byte_ordering is not None: - self._set_component("byte_ordering", byte_ordering, copy=False) - - if word_size is not None: - self._set_component("word_size", word_size, copy=False) - - # By default, close the UM file after data array access - self._set_component("close", True, copy=False) - - def _get_array(self, index=None): - """Returns a subspace of the dataset variable. - - .. versionadded:: 3.16.3 - - .. seealso:: `__array__`, `index` - - :Parameters: - - {{index: `tuple` or `None`, optional}} - - :Returns: - - `numpy.ndarray` - The subspace. - - """ - # Note: No need to lock the UM file - concurrent reads are OK. - - if index is None: - index = self.index() - - f, header_offset = self.open() - rec = self._get_rec(f, header_offset) - - int_hdr = rec.int_hdr - real_hdr = rec.real_hdr - array = rec.get_data().reshape(self.original_shape) - - self.close(f) - del f, rec - - # Set the netCDF attributes for the data - attributes = self.get_attributes({}) - self._set_units(int_hdr, attributes) - self._set_FillValue(int_hdr, real_hdr, attributes) - self._set_unpack(int_hdr, real_hdr, attributes) - self._set_component("attributes", attributes, copy=False) - - # Get the data subspace, applying any masking and unpacking - array = cfdm.netcdf_indexer( - array, - mask=self.get_mask(), - unpack=self.get_unpack(), - always_masked_array=False, - orthogonal_indexing=True, - attributes=attributes, - copy=False, - ) - array = array[index] - - if int_hdr.item(38) == 3: - # Convert the data to a boolean array - array = array.astype(bool) - - # Set the data type - self._set_component("dtype", array.dtype, copy=False) - - # Return the numpy array - return array - - def _get_rec(self, f, header_offset): - """Get a container for a record. - - This includes the lookup header and file offsets. - - .. versionadded:: 3.14.0 - - .. seealso:: `close`, `open` - - :Parameters: - - f: `umread_lib.umfile.File` - The open PP or FF file. - - header_offset: `int` - - :Returns: - - `umread_lib.umfile.Rec` - The record container. - - """ - return Rec.from_file_and_offsets(f, header_offset) - - # ------------------------------------------------------------ - # Leave the following commented code here for debugging - # purposes. Replacing the above line with this code moves the - # calculation of the data offset and disk length from pure - # Python to the C library, at the expense of completely - # parsing the file. Note: If you do replace the above line - # with the commented code, then you *must* also set - # 'parse=True' in the `open` method. - # ------------------------------------------------------------ - - # for v in f.vars: - # for r in v.recs: - # if r.hdr_offset == header_offset: - # return r - - def _set_FillValue(self, int_hdr, real_hdr, attributes): - """Set the ``_FillValue`` attribute. - - .. versionadded:: 3.16.3 - - :Parameters: - - int_hdr: `numpy.ndarray` - The integer header of the data. - - real_header: `numpy.ndarray` - The real header of the data. - - attributes: `dict` - The dictionary in which to store the new - attributes. If a new attribute exists then - *attributes* is updated in-place. - - :Returns: - - `None - - """ - if "FillValue" in attributes: - return - - # Set the fill_value from BMDI - _FillValue = real_hdr.item(17) - if _FillValue != -1.0e30: - # -1.0e30 is the flag for no missing data - if int_hdr.item(38) == 2: - # Must have an integer _FillValue for integer data - _FillValue = int(_FillValue) - - attributes["_FillValue"] = _FillValue - - def _set_units(self, int_hdr, attributes): - """Set the ``units`` attribute. - - .. versionadded:: 3.14.0 - - :Parameters: - - int_hdr: `numpy.ndarray` - The integer header of the data. - - real_header: `numpy.ndarray` - The real header of the data. - - attributes: `dict` - The dictionary in which to store the new - attributes. If a new attribute exists then - *attributes* is updated in-place. - - :Returns: - - `None` - - """ - if "units" in attributes: - return - - units = None - if not _stash2standard_name: - load_stash2standard_name() - - submodel = int_hdr.item(44) - stash = int_hdr.item(41) - records = _stash2standard_name.get((submodel, stash)) - if records: - LBSRCE = int_hdr.item(37) - version, source = divmod(LBSRCE, 10000) - if version <= 0: - version = 405.0 - - for ( - long_name, - units0, - valid_from, - valid_to, - standard_name, - cf_info, - condition, - ) in records: - if not self._test_version( - valid_from, valid_to, version - ) or not self._test_condition(condition, int_hdr): - continue - - units = units0 - break - - attributes["units"] = units - - def _set_unpack(self, int_hdr, real_hdr, attributes): - """Set the ``add_offset`` and ``scale_factor`` attributes. - - .. versionadded:: 3.16.3 - - :Parameters: - - int_hdr: `numpy.ndarray` - The integer header of the data. - - real_header: `numpy.ndarray` - The real header of the data. - - attributes: `dict` - The dictionary in which to store the new - attributes. If any new attributes exist then - *attributes* is updated in-place. - - :Returns: - - `None - - """ - if "scale_factor" not in attributes: - # Treat BMKS as a scale_factor if it is neither 0 nor 1 - scale_factor = real_hdr.item(18) - if scale_factor != 1.0 and scale_factor != 0.0: - if int_hdr.item(38) == 2: - # Must have an integer scale_factor for integer data - scale_factor = int(scale_factor) - - attributes["scale_factor"] = scale_factor - - if "add_offset" not in attributes: - # Treat BDATUM as an add_offset if it is not 0 - add_offset = real_hdr.item(4) - if add_offset != 0.0: - if int_hdr.item(38) == 2: - # Must have an integer add_offset for integer data - add_offset = int(add_offset) - - attributes["add_offset"] = add_offset - - def _test_condition(self, condition, int_hdr): - """Return `True` if a field satisfies a condition for a STASH - code to standard name conversion. - - .. versionadded:: 3.14.0 - - :Parameters: - - condition: `str` - The condition. If False then the condition is always - passed, otherwise the condition is specified as - ``'true_latitude_longitude'`` or - ``'rotated_latitude_longitude'``. - - int_hdr: `numpy.ndarray` - The integer lookup header used to evaluate the - condition. - - :Returns: - - `bool` - `True` if the data satisfies the condition specified, - `False` otherwise. - - """ - if not condition: - return True - - if condition == "true_latitude_longitude": - LBCODE = int_hdr.item(15) - # LBCODE 1: Unrotated regular lat/long grid - # LBCODE 2 = Regular lat/lon grid boxes (grid points are - # box centres) - if LBCODE in (1, 2): - return True - elif condition == "rotated_latitude_longitude": - LBCODE = int_hdr.item(15) - # LBCODE 101: Rotated regular lat/long grid - # LBCODE 102: Rotated regular lat/lon grid boxes (grid - # points are box centres) - # LBCODE 111: ? - if LBCODE in (101, 102, 111): - return True - else: - return False - - def _test_version(self, valid_from, valid_to, version): - """Return `True` if the UM version applicable to this field is - within the given range. - - If possible, the UM version is derived from the PP header and - stored in the metadata object. Otherwise it is taken from the - *version* parameter. - - .. versionadded:: 3.14.0 - - :Parameters: - - valid_from: number or `None` - The lower bound of the version range, e.g. ``4.5``, - ``606.1``, etc. - - valid_to: number or `None` - The upper bound of the version range, e.g. ``4.5``, - ``606.1``, etc. - - version: number - The version of field, e.g. ``4.5``, ``606.1``, etc. - - :Returns: - - `bool` - `True` if the UM version applicable to this data is - within the given range, `False` otherwise. - - """ - if valid_to is None: - if valid_from is None: - return True - - if valid_from <= version: - return True - elif valid_from is None: - if version <= valid_to: - return True - elif valid_from <= version <= valid_to: - return True - - return False - - @property - def file_address(self): - """The file name and address. - - Deprecated at version 3.14.0. Use methods `get_filename` - and `get_address` instead. - - :Returns: - - `tuple` - The file name and file address. - - **Examples** - - >>> a.file_address() - ('file.pp', 234835) - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "file_address", - "Use methods 'get_filename' and 'get_address' instead.", - version="3.14.0", - removed_at="5.0.0", - ) # pragma: no cover - - @property - def header_offset(self): - """The start position in the file of the header. - - :Returns: - - `int` or `None` - The address, or `None` if there isn't one. - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "header_offset", - "Use method 'get_address' instead.", - version="3.15.0", - removed_at="5.0.0", - ) # pragma: no cover - - @property - def data_offset(self): - """The start position in the file of the data array. - - :Returns: - - `int` - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "data_offset", - version="3.15.0", - removed_at="5.0.0", - ) # pragma: no cover - - @property - def disk_length(self): - """The number of words on disk for the data array. - - :Returns: - - `int` - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "disk_length", - version="3.15.0", - removed_at="5.0.0", - ) # pragma: no cover - - @property - def fmt(self): - """The file format of the UM file containing the array. - - Deprecated at version 3.14.0. Use method `get_fmt` - instead. - - :Returns: - - `str` - 'FF' or 'PP' - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "fmt", - "Use method 'get_fmt' instead.", - version="3.14.0", - removed_at="5.0.0", - ) # pragma: no cover - - @property - def byte_ordering(self): - """The endianness of the data. - - Deprecated at version 3.14.0. Use method - `get_byte_ordering` instead. - - :Returns: - - `str` - 'little_endian' or 'big_endian' - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "byte_ordering", - "Use method 'get_byte_ordering' instead.", - version="3.14.0", - removed_at="5.0.0", - ) # pragma: no cover - - @property - def word_size(self): - """Word size in bytes. - - Deprecated at version 3.14.0. Use method `get_word_size` - instead. - - :Returns: - - `int` - 4 or 8 - - """ - _DEPRECATION_ERROR_ATTRIBUTE( - self, - "word_size", - "Use method 'get_word_size' instead.", - version="3.14.0", - removed_at="5.0.0", - ) # pragma: no cover - - def close(self, f): - """Close the dataset containing the data. - - :Parameters: - - f: `umfile_lib.File` - The UM or PP dataset to be be closed. - - .. versionadded:: 3.14.0 - - :Returns: - - `None` - - """ - if self._get_component("close"): - f.close_fd() - - def get_byte_ordering(self): - """The endianness of the data. - - .. versionadded:: 3.14.0 - - .. seealso:: `open` - - :Returns: - - `str` or `None` - ``'little_endian'`` or ``'big_endian'``. If the byte - ordering has not been set then `None` is returned, in - which case byte ordering will be detected - automatically (if possible) when the file is opened - with `open`. - - """ - return self._get_component("byte_ordering", None) - - def get_fmt(self): - """The file format of the UM file containing the array. - - .. versionadded:: 3.14.0 - - .. seealso:: `open` - - :Returns: - - `str` or `None` - ``'FF'`` or ``'PP'``. If the word size has not been - set then `None` is returned, in which case file format - will be detected automatically (if possible) when the - file is opened with `open`. - - """ - return self._get_component("fmt", None) - - def get_format(self): - """The format of the files. - - .. versionadded:: 3.15.0 - - .. seealso:: `get_address`, `get_filename`, `get_formats` - - :Returns: - - `str` - The file format. Always ``'um'``, signifying PP/UM. - - **Examples** - - >>> a.get_format() - 'um' - - """ - return "um" - - def get_word_size(self): - """Word size in bytes. - - .. versionadded:: 3.14.0 - - .. seealso:: `open` - - :Returns: - - `int` or `None` - ``4`` or ``8``. If the word size has not been set then - `None` is returned, in which case word size will be - detected automatically (if possible) when the file is - opened with `open`. - - """ - return self._get_component("word_size", None) - - def open(self): - """Returns an open dataset and the address of the data. - - :Returns: - - `umfile_lib.umfile.File`, `int` - The open file object, and the start address in bytes - of the lookup header. - - **Examples** - - >>> f.open() - (, 4) - - """ - return super().open( - File, - byte_ordering=self.get_byte_ordering(), - word_size=self.get_word_size(), - fmt=self.get_fmt(), - parse=False, + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use XnetcdfArray instead." ) diff --git a/cf/data/array/xnetcdfarray.py b/cf/data/array/xnetcdfarray.py new file mode 100644 index 0000000000..ee4b40019d --- /dev/null +++ b/cf/data/array/xnetcdfarray.py @@ -0,0 +1,16 @@ +import cfdm + +from ...mixin_container import Container +from .mixin import ActiveStorageMixin + + +class XnetcdfArray( + ActiveStorageMixin, + Container, + cfdm.XnetcdfArray, +): + """A netCDF array accessed with `xnetcdf`. + + .. versionadded:: NEXTVERSION + + """ diff --git a/cf/data/array/zarrarray.py b/cf/data/array/zarrarray.py index 2d3d8c784f..813839a725 100644 --- a/cf/data/array/zarrarray.py +++ b/cf/data/array/zarrarray.py @@ -1,15 +1,11 @@ -import cfdm - -from ...mixin_container import Container - -# Uncomment when we can use active storage on Zarr datasets: -# from .mixin import ActiveStorageMixin +class ZarrArray: + """A Zarr array accessed with `zarr`.""" + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" -class ZarrArray( - # Uncomment when we can use active storage on Zarr datasets: - # ActiveStorageMixin, - Container, - cfdm.ZarrArray, -): - """A Zarr array accessed with `zarr`.""" + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use XnetcdfArray instead." + ) diff --git a/cf/data/collapse/collapse_active.py b/cf/data/collapse/collapse_active.py index 79fae066f6..8cd09b135d 100644 --- a/cf/data/collapse/collapse_active.py +++ b/cf/data/collapse/collapse_active.py @@ -163,14 +163,8 @@ def active_chunk_function(method, *args, **kwargs): return axis = kwargs.get("axis") - if axis is not None: - if isinstance(axis, Integral): - axis = (axis,) - - if len(axis) < x.ndim: - # Active storage is not (yet) allowed for reductions over - # a subset of the axes - return + if isinstance(axis, Integral): + axis = (axis,) # ---------------------------------------------------------------- # Still here? Set up an Active instance that will carry out the @@ -193,10 +187,18 @@ def active_chunk_function(method, *args, **kwargs): address = None dataset = x.get_variable(None) if dataset is None: - # Dateaset is a string, not a variable object. + # Dataset is a string, not a variable object. storage_options = x.get_storage_options() address = x.get_address() dataset = x.get_filename() + else: + # For datasets that follow the pyfive API (e.g. `pyfive`, + # `umfive`), we can pass the dataset variable directly to + # `Active`. + if dataset.backend_api not in ("pyfive",): + return + + dataset = dataset.backend_accessor active_kwargs = { "dataset": dataset, diff --git a/cf/data/fragment/fragmentfilearray.py b/cf/data/fragment/fragmentfilearray.py index 72d685ba8d..53078fb03d 100644 --- a/cf/data/fragment/fragmentfilearray.py +++ b/cf/data/fragment/fragmentfilearray.py @@ -12,19 +12,3 @@ class FragmentFileArray( .. versionadded:: 3.17.0 """ - - def __new__(cls, *args, **kwargs): - """Store fragment classes. - - .. versionadded:: 3.17.0 - - """ - # Import fragment classes. Do this here (as opposed to outside - # the class) to aid subclassing. - from .fragmentumarray import FragmentUMArray - - instance = super().__new__(cls) - instance._FragmentArrays = instance._FragmentArrays + ( - FragmentUMArray, - ) - return instance diff --git a/cf/data/fragment/fragmentumarray.py b/cf/data/fragment/fragmentumarray.py index 6cfa8bbde2..ec1190a117 100644 --- a/cf/data/fragment/fragmentumarray.py +++ b/cf/data/fragment/fragmentumarray.py @@ -1,13 +1,15 @@ -import cfdm - -from ..array.umarray import UMArray - - -class FragmentUMArray( - cfdm.data.fragment.mixin.FragmentFileArrayMixin, UMArray -): +class FragmentUMArray: """A fragment of aggregated data in a PP or UM file. .. versionadded:: 3.14.0 """ + + def __init__(self, *args, **kwargs): + class DeprecationError(Exception): + """Deprecation error.""" + + raise DeprecationError( + f"{self.__class__.__name__} was deprecated at version NEXTVERSION " + "and is no longer available. Use FragmentFileArray instead." + ) diff --git a/cf/functions.py b/cf/functions.py index ebbc9f4feb..04d7555438 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -1,5 +1,4 @@ import atexit -import os import platform import warnings from collections.abc import Iterable @@ -19,7 +18,7 @@ import numpy as np from . import __file__, __version__ -from .constants import OperandBoundsCombination, _stash2standard_name +from .constants import OperandBoundsCombination from .docstring import _docstring_substitution_definitions @@ -2469,7 +2468,9 @@ def equivalent(x, y, rtol=None, atol=None, traceback=False): ) -def load_stash2standard_name(table=None, delimiter="!", merge=True): +def load_stash2standard_name( + table=None, delimiter="!", merge=True, reset=False +): """Load a STASH to standard name conversion table from a file. This used when reading PP and UM fields files. @@ -2502,13 +2503,9 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): :Parameters: - table: `str`, optional + table: `str` or `None`, optional Use the conversion table at this file location. By default - the table will be looked for at - ``os.path.join(os.path.dirname(cf.__file__),'etc/STASH_to_CF.txt')`` - - Setting *table* to `None` will reset the table, removing - any modifications that have previously been made. + the default table of the `umfive` library will be loaded. delimiter: `str`, optional The delimiter of the table columns. By default, ``!`` is @@ -2521,8 +2518,9 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): into the existing table, overwriting any entries which already exist. - If *table* is `None` then *merge* is taken as False, - regardless of its given value. + reset: `bool`, optional + If True then clear all entries and re-load the default + table. :Returns: @@ -2538,118 +2536,17 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): >>> cf.load_stash2standard_name('my_table4.txt', merge=False) """ - import csv - import re - - # 0 Model - # 1 STASH code - # 2 STASH name - # 3 units - # 4 valid from UM vn - # 5 valid to UM vn - # 6 standard_name - # 7 CF extra info - # 8 PP extra info - # Number matching regular expression - number_regex = r"([-+]?\d*\.?\d+(e[-+]?\d+)?)" - - if table is None: - # Use default conversion table - merge = False - package_path = os.path.dirname(__file__) - table = os.path.join(package_path, "etc/STASH_to_CF.txt") - else: - # User supplied table - table = abspath(os.path.expanduser(os.path.expandvars(table))) - - with open(table, "r") as open_table: - lines = csv.reader( - open_table, delimiter=delimiter, skipinitialspace=True + try: + import umfive + except Exception: + raise ImportError( + "Must install 'umfive' to load a STASH to standard name " + "conversion table." ) - lines = list(lines) - - raw_list = [] - [raw_list.append(line) for line in lines] - - # Get rid of comments - for line in raw_list[:]: - if line[0].startswith("#"): - raw_list.pop(0) - continue - - break - - # Convert to a dictionary which is keyed by (submodel, STASHcode) - # tuples - ( - model, - stash, - name, - units, - valid_from, - valid_to, - standard_name, - cf, - pp, - ) = list(range(9)) - - stash2sn = {} - for x in raw_list: - key = (int(x[model]), int(x[stash])) - - if not x[units]: - x[units] = None - - try: - cf_info = {} - if x[cf]: - for d in x[7].split(): - if d.startswith("height="): - cf_info["height"] = re.split( - number_regex, d, re.IGNORECASE - )[1:4:2] - if cf_info["height"] == "": - cf_info["height"][1] = "1" - - if d.startswith("below_"): - cf_info["below"] = re.split( - number_regex, d, re.IGNORECASE - )[1:4:2] - if cf_info["below"] == "": - cf_info["below"][1] = "1" - - if d.startswith("where_"): - cf_info["where"] = d.replace("where_", "where ", 1) - if d.startswith("over_"): - cf_info["over"] = d.replace("over_", "over ", 1) - - x[cf] = cf_info - except IndexError: - pass - try: - x[valid_from] = float(x[valid_from]) - except ValueError: - x[valid_from] = None - - try: - x[valid_to] = float(x[valid_to]) - except ValueError: - x[valid_to] = None - - x[pp] = x[pp].rstrip() - - line = (x[name:],) - - if key in stash2sn: - stash2sn[key] += line - else: - stash2sn[key] = line - - if not merge: - _stash2standard_name.clear() - - _stash2standard_name.update(stash2sn) + umfive.load_stash_table( + table=table, delimiter=delimiter, merge=merge, reset=reset + ) def stash2standard_name(): @@ -2661,7 +2558,15 @@ def stash2standard_name(): .. seealso:: `load_stash2standard_name` """ - return _stash2standard_name.copy() + try: + import umfive + except Exception: + raise ImportError( + "Must install 'umfive' to get the STASH to standard name " + "conversion table." + ) + + return umfive.stash_table() def flat(x): diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index b00e3db987..2bd226ef47 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -482,7 +482,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): arg0, arg1 = value.value if arg0 > arg1: # Query has swapped operands (i.e. arg0 > - # arg1) => Create a new equivalant Query + # arg1) => Create a new equivalent Query # that has arg0 < arg1, for a new # arg1. E.g. for a period of 360, # cf.wi(355, 5) is transformed to @@ -560,7 +560,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): raise ValueError( "Error: Can't specify multiple " "conditions for a single axis when " - f"one of those condtions ({value!r}) " + f"one of those conditions ({value!r}) " "is effectively a cyclic slice: " f"{index}. Consider applying the " "conditions separately." @@ -705,7 +705,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): for value, construct in zip(points, transposed_constructs) ] - # Find loctions that are True in all of the + # Find locations that are True in all of the # constructs' matches item_match = item_matches.pop() for m in item_matches: @@ -2358,11 +2358,11 @@ def healpix_to_ugrid(self, cache=True, inplace=False): # If 1-d lat/lon coordinates do not exist, then derive them # from the HEALPix indices. Setting the pole_longitude to - # something other than None - it doesn't matter what - ensures - # that the north (south) polar vertex comes out as a single - # node in the domain topology. + # something other than `None` - it doesn't matter what - + # ensures that the north (south) polar vertex comes out as a + # single node in the domain topology. f.create_latlon_coordinates( - two_d=False, pole_longitude=0, cache=cache, inplace=True + two_d=False, longitude_at_pole=0, cache=cache, inplace=True ) # Get the lat/lon coordinates @@ -2428,7 +2428,7 @@ def healpix_to_ugrid(self, cache=True, inplace=False): del _ # We are guaranteed unique node values when - # nodes=y_indices*y_indice.size+x_indices + # nodes=y_indices*y_indices.size+x_indices nodes = y_indices del y_indices nodes *= nodes.size @@ -2456,11 +2456,12 @@ def create_latlon_coordinates( self, one_d=True, two_d=True, - pole_longitude=None, + longitude_at_pole=None, overwrite=False, cache=True, inplace=False, verbose=None, + pole_longitude=None, ): """Create latitude and longitude coordinates. @@ -2470,11 +2471,23 @@ def create_latlon_coordinates( new coordinates are only created if the {{class}} doesn't already include any latitude or longitude coordinates. + .. note:: Latitude and longitude coordinates can only be + created if each relevant coordinate reference + construct has a ``grid_mapping_name`` parameter set + to a valid CF grid mapping name, and this is also + the case when there is a ``crs_wkt`` parameter. See + CF 5.6.1: Use of the CRS Well-known Text Format + (https://doi.org/10.5281/zenodo.14274886). + When it is not possible to create latitude and longitude coordinates, the reason why will be reported if the log level is at ``2``/``'INFO'`` or higher (as set by `cf.log_level` or the *verbose* parameter). + If the log level is at ``3``/``'DEBUG'``/``-1`` then + information on how the latitude and longitude coordinates were + created is also reported. + .. versionadded:: 3.20.0 .. seealso:: `healpix_to_ugrid` @@ -2491,16 +2504,16 @@ def create_latlon_coordinates( latitude and longitude coordinates. If False then 2-d coordinates will not be created. - pole_longitude: `None` or number - Define the longitudes of coordinates or coordinate - bounds that lie exactly on the north or south pole. If - `None` (the default) then the longitudes of such - points are determined by whichever algorithm was used - to create the coordinates, which could result in - different points on a pole having different - longitudes. If set to a number, then the longitudes of - all points on the north or south pole will be given - that value. + longitude_at_pole: `None` or number + Define the treatment of longitudes of coordinates or + coordinate bounds that lie exactly on the north or + south pole. If `None` (the default) then the + longitudes of such points are determined by whichever + algorithm was used to create the coordinates, which + could result in different grid points on a pole having + different longitudes. If set to a number, then the + longitudes of all grid points on the north or south + pole will be given that value. overwrite: `bool`, optional If True then remove any existing latitude and @@ -2527,11 +2540,14 @@ def create_latlon_coordinates( {{verbose: `int` or `str` or `None`, optional}} + pole_longitude: Deprecated at version NEXTVERSION + Use *longitude_at_pole* instead. + :Returns: `{{class}}` or `None` - A new {{class}}, with new latitude and longitude - constructs if any could be created. If the operation + The {{class}} with new latitude and longitude + constructs, if any could be created. If the operation was in-place then `None` is returned. **Examples** @@ -2560,6 +2576,16 @@ def create_latlon_coordinates( Coord references: grid_mapping_name:healpix """ + if pole_longitude is not None: + _DEPRECATION_ERROR_KWARGS( + self, + "create_latlon_coordinates", + {"pole_longitude": pole_longitude}, + message="Use 'longitude_at_pole' instead.", + version="NEXTVERSION", + removed_at="4.0.0", + ) # pragma: no cover + f = _inplace_enabled_define_and_cleanup(self) # ------------------------------------------------------------ @@ -2615,9 +2641,10 @@ def create_latlon_coordinates( # Remove a 'latitude_longitude' grid mapping (if there is one) # from the dictionary, saving it for later. - latlon_cr = coordinate_references.pop( + cr_latlon = coordinate_references.pop( "grid_mapping_name:latitude_longitude", None ) + if not coordinate_references: if is_log_level_info(logger): logger.info( @@ -2655,30 +2682,44 @@ def create_latlon_coordinates( # -------------------------------------------------------- # 1-d lat/lon coordinates # -------------------------------------------------------- - if identity == "grid_mapping_name:healpix": - # ---------------------------------------------------- - # HEALPix - # ---------------------------------------------------- - from ..healpix_utils import _healpix_create_latlon_coordinates + match identity: + case "grid_mapping_name:healpix": + # ------------------------------------------------ + # HEALPix + # ------------------------------------------------ + from ..healpix_utils import ( + _healpix_create_latlon_coordinates, + ) - lat_key, lon_key = _healpix_create_latlon_coordinates( - f, pole_longitude, cache - ) - coords_created = lat_key is not None + lat_key, lon_key = _healpix_create_latlon_coordinates( + f, longitude_at_pole, cache + ) + + coords_created = lat_key is not None + + # Paving the way for reduced_gaussian ... if two_d and not coords_created: # -------------------------------------------------------- - # 2-d lat/lon coordinates + # 2-d lat/lon coordinates from 1-d projection coordinates # -------------------------------------------------------- - pass # For now ... + from .utils import create_2d_latlon_coordinates + + lat_key, lon_key = create_2d_latlon_coordinates( + f, + cr, + cr_latlon, + longitude_at_pole=longitude_at_pole, + ) + coords_created = lat_key is not None # ------------------------------------------------------------ # Update the appropriate coordinate reference with any new # coordinate keys # ------------------------------------------------------------ if coords_created: - if latlon_cr is not None: - latlon_cr.set_coordinates((lat_key, lon_key)) + if cr_latlon is not None: + cr_latlon.set_coordinates((lat_key, lon_key)) else: cr.set_coordinates((lat_key, lon_key)) @@ -2761,7 +2802,7 @@ def cyclic( # Note: We have to do a "dry run" on the 'autocyclic' call # in the if test in order to prevent corrupting # self._cyclic in the case that an axis tested by - # autocyclic is already marked as cylcic, but + # autocyclic is already marked as cyclic, but # nonetheless autocyclic returns False (sounds # niche, but this really happens!). if len(cyclic) < len( @@ -3890,13 +3931,17 @@ def to_xarray(self, group=True): {{cf_xarray description}} Note that ``ds = f.to_xarray()`` is identical to ``ds = - cf.write(f, fmt='XARRAY')``; and multiple {{class_lower}}s may - be written to the same `xarray` dataset with - `cf.{{class}}List.to_xarray`, or with `cf.write` (e.g. ``ds = - cf.write([f, g], fmt='XARRAY')``). Also, `cf.write` allows a + {{package}}.write(f, fmt='XARRAY')``; and multiple + {{class_lower}}s may be written to the same `xarray` dataset + with `{{package}}.write` (e.g. ``ds = {{package}}.write([f, + g], fmt='XARRAY')``). Also, `{{package}}.write` allows a mixture of fields and domains to be written to the same `xarray` dataset. + An `xarray` dataset can be converted to one or more fields + with ``f = {{package}}.read(ds)``, or domains with ``f = + {{package}}.read(ds, domain=True)``. + .. versionadded:: NEXTVERSION .. seealso:: `cf.{{class}}List.to_xarray`, `cf.write` @@ -3904,7 +3949,6 @@ def to_xarray(self, group=True): :Parameter: group: `bool`, optional - If False then create a "flat" dataset, i.e. one with only the root group, regardless of any group structure specified by the netCDF interfaces of the diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index 1c98d038cb..bbfd687a74 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -4587,8 +4587,8 @@ def identity( By default the identity is the first found of the following: - * The "standard_name" property. * The "id" attribute, preceded by ``'id%'``. + * The "standard_name" property. * The "cf_role" property, preceded by ``'cf_role='``. * The "axis" property, preceded by ``'axis='``. * The "long_name" property, preceded by ``'long_name='``. @@ -4671,14 +4671,14 @@ def identity( return default - n = self.get_property("standard_name", None) - if n is not None: - return str(n) - n = getattr(self, "id", None) if n is not None: return f"id%{n}" + n = self.get_property("standard_name", None) + if n is not None: + return str(n) + if relaxed: if strict: raise ValueError( diff --git a/cf/mixin/utils/__init__.py b/cf/mixin/utils/__init__.py new file mode 100644 index 0000000000..8b7e916868 --- /dev/null +++ b/cf/mixin/utils/__init__.py @@ -0,0 +1 @@ +from .latlon_utils import create_2d_latlon_coordinates diff --git a/cf/mixin/utils/grid_mapping.py b/cf/mixin/utils/grid_mapping.py new file mode 100644 index 0000000000..0e5735e8fa --- /dev/null +++ b/cf/mixin/utils/grid_mapping.py @@ -0,0 +1,993 @@ +"""Utilities for creating `pyproj.CRS` instances. + +:Glossary: + +Definitions of `pyproj.CRS` parameters that map to CF grid mapping +parameters. + +For further details see +https://proj.org/en/stable/operations/projections and +https://github.com/cf-convention/cf-conventions/wiki/Mapping-from-CF-Grid-Mapping-Attributes-to-CRS-WKT-Elements + +* a: Semi-major axis of the ellipsoid. + +* alpha: Azimuth of centerline clockwise from north at the centre + point of the line. If gamma is not given then alpha + determines the value of gamma. + +* b: Semi-minor axis of the ellipsoid. + +* ellps: The name of a built-in ellipsoid definition. + +* f: Flattening of the ellipsoid. + +* h: Height of the view point above the Earth and must be in the same + units as the radius of the sphere or semimajor axis of the + ellipsoid. + +* k_0: Scale factor. Determines scale factor used in the projection. + +* lat_0: Latitude of natural origin, latitude of false origin or + latitude of projection centre (naming and meaning depend on + the projection method). + +* lat_1: First standard parallel. + +* lat_2: Second standard parallel. + +* lat_ts: Latitude of true scale. Defines the latitude where scale + is not distorted. Takes precedence over +k_0 if both + options are used together. + +* lon_0: Central meridian/longitude of natural origin, longitude of + origin or longitude of false origin (naming and meaning + depend on the projection method). + +* o_lat_p: Latitude of the North pole of the unrotated source CRS, + expressed in the rotated geographic CRS. + +* o_lon_p: Longitude of the North pole of the unrotated source CRS, + expressed in the rotated geographic CRS. + +* o_proj: Oblique projection. + +* pm: Prime meridian. + +* R: Radius of the sphere, given in meters. If used in conjunction + with ellps, R takes precedence. + +* rf: Reverse flattening of the ellipsoid, 1/f + +* sweep: Sweep angle axis of the viewing instrument. Valid options are + "x" and "y". + +* y_0: False northing, northing at false origin or northing at + projection centre (naming and meaning depend on the projection + method). Always in meters. + +* x_0: False easting, easting at false origin or easting at projection + centre (naming and meaning depend on the projection + method). Always in meters. + +""" + +import logging +import warnings + +from cfdm import is_log_level_debug, is_log_level_info + +# Suppress warning about lossy WKT-to-PROJ conversion,it only refers +# to lost information that doesn't affect the transformation. +warnings.filterwarnings( + "ignore", + category=UserWarning, + message=".*lose important projection information.*", +) + +logger = logging.getLogger(__name__) + + +def _ellipsoid_parameters(cr): + """Get ellipsoid parameters from a coordinate reference construct. + + https://proj.org/en/stable/usage/ellipsoids.html + + https://proj.org/en/stable/usage/projections.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct, or `None`, in which + case the CF default ellipsoid is assumed. + + :Returns: + + `dict` + The `pyproj.CRS` ellipsoid parameters. + + """ + kwargs = {} + + p = cr.datum.parameters() + crs_wkt = cr.coordinate_conversion.get_parameter("crs_wkt", None) + + inverse_flattening = p.get("inverse_flattening") + semi_major_axis = p.get("semi_major_axis") + semi_minor_axis = p.get("semi_minor_axis") + earth_radius = p.get("earth_radius") + reference_ellipsoid_name = p.get("reference_ellipsoid_name") + + if inverse_flattening == 0: + # Sphere + if semi_major_axis is not None: + kwargs["R"] = semi_major_axis + elif earth_radius is not None: + kwargs["R"] = earth_radius + elif not crs_wkt and reference_ellipsoid_name is None: + reference_ellipsoid_name = "sphere" + else: + # Ellipsoid + if earth_radius is not None: + kwargs["R"] = earth_radius + else: + if semi_major_axis is not None: + kwargs["a"] = semi_major_axis + + if semi_minor_axis is not None: + kwargs["b"] = semi_minor_axis + + if inverse_flattening is not None: + kwargs["rf"] = inverse_flattening + + if reference_ellipsoid_name is not None: + kwargs["ellps"] = reference_ellipsoid_name + + if not crs_wkt and not kwargs: + # Default to a sphere, in the absence of other information. + kwargs = {"ellps": "sphere"} + + prime_meridian_name = p.get("prime_meridian_name") + if prime_meridian_name is not None: + kwargs["pm"] = prime_meridian_name + elif not crs_wkt: + kwargs["pm"] = p.get("longitude_of_prime_meridian", 0) + + return kwargs + + +def _crs_wkt_parameters(cr): + """Get `pyproj.CRS` parameters from a crs_wkt parameter. + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct. + + :Returns: + + `dict` + The `pyproj.CRS` parameters derived from coordinate + reference construct crs_wkt parameters, if any. + + """ + + crs_wkt = cr.coordinate_conversion.get_parameter("crs_wkt", None) + if crs_wkt is None: + return {} + + import pyproj + + return pyproj.CRS.from_wkt(crs_wkt).to_dict() + + +def _create_pyproj_CRS(kwargs, cr, ellipsoid_only=False): + """Create a `pyproj.CRS` instance. + + .. versionadded:: NEXTVERSION + + :Parameters: + + kwargs: `dict` + A dictionary of keyword arguments for initialising the + `pyproj.CRS` instance. + + The keyword arguments should not include a description of + the ellipsoid, as this is automatically derived from *cr*. + + If the *cr* contains a ``crs_wkt`` parameter then it is + converted to `pyproj.CRS` keyword arguments that are + automatically included. + + cr: `CoordinateReference` + The coordinate reference construct from which *kwargs* was + derived. + + ellipsoid_only: `bool`, optional + Return the CRS defined only by the ellipsoid and prime + meridian. + + :Returns: + + `pyproj.CRS` or `None` + The created CRS, or `None` if one couldn't be created. + + """ + import pyproj + + # Remove `None` values + kwargs = {k: v for k, v in kwargs.items() if v is not None} + + # Specify the units of the 1-d coordinates + kwargs["units"] = "m" + + kwargs = _crs_wkt_parameters(cr) | _ellipsoid_parameters(cr) | kwargs + + try: + proj = pyproj.CRS(**kwargs) + except Exception as error: + if is_log_level_info(logger): + logger.info( + f"Can't create a pyproj.CRS for {cr!r}: {error}" + ) # pragma: no cover + + return + + if ( + ellipsoid_only + and cr.coordinate_conversion.get_parameter("grid_mapping_name", None) + != "latitude_longitude" + ): + # Return the CRS defined by the ellipsoid and prime meridian + # of a non-latitude_longitude coordinate reference + proj = proj.geodetic_crs + + if is_log_level_debug(logger): + logger.debug(f"pyproj.CRS: {proj}") + + return proj + + +def _cc_parameter(p, parameter, default=None): + """Get a parameter from a coordinate reference construct. + + If there is a ``crs_wkt`` parameter then: + + * `None` will be returned if the *parameter* does not exist. + + If there is not a ``crs_wkt`` parameter then: + + * If *default* is not `None`, then that *default* will be returned + if the *parameter* does not exist. + + * If *default* is `None`, then a `KeyError` will be raised if the + *parameter* does not exist. + + This behaviour allows a ``crs_wkt`` parameter to provide a value + for a missing CF grid mapping parameter (which happens later on in + `_create_pyproj_CRS`). + + .. versionadded:: NEXTVERSION + + :Parameters: + + p: `dict` + A dictionary of the coordinate reference construct + parameters. + + parameter: `str` + The name of the parameter. + + default: optional + What to do if the parameter doesn't not exist (see above). + + :Returns: + + The parameter value. + + """ + if "crs_wkt" in p: + return p.get(parameter) + + if default is not None: + return p.get(parameter, default) + + return p[parameter] + + +# ==================================================================== +# Functions for creating a `pyproj.CRS` instance for each CF grid +# mapping type. +# ==================================================================== + + +def albers_equal_area(cr): + """Create an albers_equal_area CRS. + + https://proj.org/en/stable/operations/projections/aea.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from which the CRS is + derived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "aea", + "lon_0": _cc_parameter(p, "longitude_of_central_meridian"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + standard_parallel = _cc_parameter(p, "standard_parallel") + if standard_parallel is not None: + try: + lat_1 = standard_parallel[0] + except Exception: + lat_1 = standard_parallel + else: + try: + kwargs["lat_2"] = standard_parallel[1] + except Exception: + pass + + kwargs["lat_1"] = lat_1 + + return _create_pyproj_CRS(kwargs, cr) + + +def azimuthal_equidistant(cr): + """Create an azimuthal_equidistant CRS. + + https://proj.org/en/stable/operations/projections/aeqd.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "aeqd", + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + return _create_pyproj_CRS(kwargs, cr) + + +def geostationary(cr): + """Create a geostationary CRS. + + https://proj.org/en/stable/operations/projections/geos.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + crs_wkt = "crs_wkt" in p + + kwargs = { + "proj": "geos", + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "h": _cc_parameter(p, "perspective_point_height"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + sweep_angle_axis = p.get("sweep_angle_axis") + fixed_angle_axis = p.get("fixed_angle_axis") + match sweep_angle_axis: + case "x": + ok = fixed_angle_axis in (None, "y") + case "y": + ok = fixed_angle_axis in (None, "x") + case None: + ok = True + if fixed_angle_axis == "x": + sweep_angle_axis = "y" + elif fixed_angle_axis == "y": + sweep_angle_axis = "x" + else: + ok = False + case _: + ok = False + + if not crs_wkt and not ok: + if is_log_level_info(logger): + logger.info( + f"Can't create coordinates for {cr!r}: " + f"Bad 'sweep_angle_axis' parameter: {sweep_angle_axis!r}, " + f"or bad 'fixed_angle_axis' parameter: {fixed_angle_axis!r}" + ) # pragma: no cover + + return + + kwargs["sweep"] = sweep_angle_axis + + if p.get("latitude_of_projection_origin", 0) != 0: + if is_log_level_info(logger): + logger.info( + f"Can't create coordinates for {cr!r}: " + "Bad 'latitude_of_projection_origin' parameter: " + f"{p['latitude_of_projection_origin']!r}" + ) # pragma: no cover + + return + + return _create_pyproj_CRS(kwargs, cr) + + +def lambert_azimuthal_equal_area(cr): + """Create a lambert_azimuthal_equal_area CRS. + + https://proj.org/en/stable/operations/projections/laea.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "laea", + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + return _create_pyproj_CRS(kwargs, cr) + + +def lambert_conformal_conic(cr): + """Create a lambert_conformal_conic CRS. + + https://proj.org/en/stable/operations/projections/lcc.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "lcc", + "lon_0": _cc_parameter(p, "longitude_of_central_meridian"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + standard_parallel = _cc_parameter(p, "standard_parallel") + if standard_parallel is not None: + try: + lat_1 = standard_parallel[0] + except Exception: + lat_1 = standard_parallel + else: + try: + kwargs["lat_2"] = standard_parallel[1] + except Exception: + pass + + kwargs["lat_1"] = lat_1 + + return _create_pyproj_CRS(kwargs, cr) + + +def lambert_cylindrical_equal_area(cr): + """Create a lambert_cylindrical_equal_area CRS. + + https://proj.org/en/stable/operations/projections/cea.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + crs_wkt = "crs_wkt" in p + + kwargs = { + "proj": "cea", + "lon_0": _cc_parameter(p, "longitude_of_central_meridian"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + standard_parallel = _cc_parameter(p, "standard_parallel") + if standard_parallel is not None: + kwargs["lat_ts"] = standard_parallel + elif not crs_wkt: + kwargs["k_0"] = p["scale_factor_at_projection_origin"] + + return _create_pyproj_CRS(kwargs, cr) + + +def latitude_longitude(cr): + """Create a latitude_longitude CRS. + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from which to create + the CRS, or `None` if there isn't one (in which case a + spherical CRS is created). + + .. note:: Only the datum parameters of *cr* are used, so + the coordinate reference construct does not need + to be a latitude_longitude grid mapping. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + kwargs = {"proj": "longlat"} + + return _create_pyproj_CRS(kwargs, cr, ellipsoid_only=True) + + +def mercator(cr): + """Create a mercator CRS. + + https://proj.org/en/stable/operations/projections/merc.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + crs_wkt = "crs_wkt" in p + + kwargs = { + "proj": "merc", + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + standard_parallel = _cc_parameter(p, "standard_parallel") + if standard_parallel is not None: + kwargs["lat_ts"] = standard_parallel + elif not crs_wkt: + kwargs["k_0"] = p["scale_factor_at_projection_origin"] + + return _create_pyproj_CRS(kwargs, cr) + + +def oblique_mercator(cr): + """Create an oblique_mercator CRS. + + https://proj.org/en/stable/operations/projections/omerc.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "omerc", + "alpha": _cc_parameter(p, "azimuth_of_central_line"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "lonc": _cc_parameter(p, "longitude_of_projection_origin"), + "k_0": _cc_parameter(p, "scale_factor_at_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + return _create_pyproj_CRS(kwargs, cr) + + +def orthographic(cr): + """Create an orthographic CRS. + + https://proj.org/en/stable/operations/projections/ortho.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "ortho", + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + return _create_pyproj_CRS(kwargs, cr) + + +def polar_stereographic(cr): + """Create a polar_stereographic CRS. + + https://proj.org/en/stable/operations/projections/stere.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + crs_wkt = "crs_wkt" in p + + kwargs = { + "proj": "stere", + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + longitude_of_projection_origin = _cc_parameter( + p, "longitude_of_projection_origin" + ) + if longitude_of_projection_origin is not None: + kwargs["lon_0"] = longitude_of_projection_origin + elif not crs_wkt: + kwargs["lon_0"] = p["straight_vertical_longitude_from_pole"] + + standard_parallel = _cc_parameter(p, "standard_parallel") + if standard_parallel is not None: + kwargs["lat_ts"] = standard_parallel + elif not crs_wkt: + kwargs["k_0"] = p["scale_factor_at_projection_origin"] + + latitude_of_projection_origin = _cc_parameter( + p, "latitude_of_projection_origin" + ) + if latitude_of_projection_origin is not None: + try: + ok = ( + latitude_of_projection_origin == -90 + or latitude_of_projection_origin == 90 + ) + except Exception: + ok = False + + if not ok: + if is_log_level_info(logger): + logger.info( + f"Can't create coordinates for {cr!r}: " + "Bad 'latitude_of_projection_origin' parameter: " + f"{latitude_of_projection_origin!r}" + ) # pragma: no cover + + return + + kwargs["lat_0"] = latitude_of_projection_origin + + return _create_pyproj_CRS(kwargs, cr) + + +def rotated_latitude_longitude(cr): + """Create a rotated_latitude_longitude CRS. + + https://proj.org/en/stable/operations/projections/ob_tran.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "ob_tran", + "o_proj": "longlat", + "o_lat_p": _cc_parameter(p, "grid_north_pole_latitude"), + "o_lon_p": _cc_parameter(p, "north_pole_grid_longitude", 0), + } + + grid_north_pole_longitude = _cc_parameter(p, "grid_north_pole_longitude") + if grid_north_pole_longitude is not None: + try: + kwargs["lon_0"] = float(grid_north_pole_longitude) + 180 + except Exception: + if is_log_level_info(logger): + logger.info( + f"Can't create coordinates for {cr!r}: " + "Bad 'grid_north_pole_longitude' parameter: " + f"{grid_north_pole_longitude!r}" + ) # pragma: no cover + + return + + return _create_pyproj_CRS(kwargs, cr) + + +def sinusoidal(cr): + """Create a sinusoidal CRS. + + https://proj.org/en/stable/operations/projections/sinu.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "sinu", + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + return _create_pyproj_CRS(kwargs, cr) + + +def stereographic(cr): + """Create a stereographic CRS. + + https://proj.org/en/stable/operations/projections/stere.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "stere", + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "k_0": _cc_parameter(p, "scale_factor_at_projection_origin"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + return _create_pyproj_CRS(kwargs, cr) + + +def transverse_mercator(cr): + """Create a transverse_mercator CRS. + + https://proj.org/en/stable/operations/projections/tmerc.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "tmerc", + "lon_0": _cc_parameter(p, "longitude_of_central_meridian"), + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "k_0": _cc_parameter(p, "scale_factor_at_central_meridian"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + + return _create_pyproj_CRS(kwargs, cr) + + +def vertical_perspective(cr): + """Create a vertical_perspective CRS. + + https://proj.org/en/stable/operations/projections/nsper.html + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct from the CRS is deived. + + :Returns: + + `pyproj.CRS` + The created CRS, or `None` if one couldn't be created. + + """ + p = cr.coordinate_conversion.parameters() + + kwargs = { + "proj": "nsper", + "lat_0": _cc_parameter(p, "latitude_of_projection_origin"), + "lon_0": _cc_parameter(p, "longitude_of_projection_origin"), + "h": _cc_parameter(p, "perspective_point_height"), + "x_0": _cc_parameter(p, "false_easting", 0), + "y_0": _cc_parameter(p, "false_northing", 0), + } + return _create_pyproj_CRS(kwargs, cr) + + +def create_projection_CRS(cr, grid_mapping_name): + """Create a projection CRS. + + .. versionadded:: NEXTVERSION + + :Parameters: + + cr: `CoordinateReference` + The coordinate reference construct that defines the + coordinate system. + + grid_mapping_name: `str` + The grid mapping name. + + :Returns: + + `pyproj.CRS` or `None` + The projection CRS, or `None` if it couldn't be created. + + """ + proj = None + try: + match grid_mapping_name: + case "albers_equal_area": + proj = albers_equal_area(cr) + case "azimuthal_equidistant": + proj = azimuthal_equidistant(cr) + case "geostationary": + proj = geostationary(cr) + case "lambert_azimuthal_equal_area": + proj = lambert_azimuthal_equal_area(cr) + case "lambert_conformal_conic": + proj = lambert_conformal_conic(cr) + case "lambert_cylindrical_equal_area": + proj = lambert_cylindrical_equal_area(cr) + case "latitude_longitude": + proj = latitude_longitude(cr) + case "mercator": + proj = mercator(cr) + case "oblique_mercator": + proj = oblique_mercator(cr) + case "orthographic": + proj = orthographic(cr) + case "polar_stereographic": + proj = polar_stereographic(cr) + case "rotated_latitude_longitude": + proj = rotated_latitude_longitude(cr) + case "sinusoidal": + proj = sinusoidal(cr) + case "stereographic": + proj = stereographic(cr) + case "transverse_mercator": + proj = transverse_mercator(cr) + case "vertical_perspective": + proj = vertical_perspective(cr) + except KeyError as error: + # Trap a KeyError arising from a missing mandatory coordinate + # reference parameter + if is_log_level_info(logger): + logger.info( + f"{cr!r} has missing coordinate conversion property: {error}" + ) # pragma: no cover + + return proj diff --git a/cf/mixin/utils/latlon_utils.py b/cf/mixin/utils/latlon_utils.py new file mode 100644 index 0000000000..a8725e895e --- /dev/null +++ b/cf/mixin/utils/latlon_utils.py @@ -0,0 +1,369 @@ +"""Utilities for creating 2-d latitude/longitude coordinates.""" + +import logging + +import numpy as np +from cfdm import is_log_level_info + +from cf import Units + +from .grid_mapping import create_projection_CRS + +logger = logging.getLogger(__name__) + + +def create_2d_latlon_coordinates(f, cr, cr_latlon, longitude_at_pole=None): + """Create 2-d latitude and longitude coordinates and bounds. + + Creates the 2-d latitude and longitude coordinate constructs that + are implied by the coordinate reference constructs. + + When it is not possible to create latitude and longitude + coordinates, the reason why will be reported if the log level is + at ``2``/``'INFO'`` or higher. + + If the log level is at ``3``/``'DEBUG'``/``-1`` then a description + of the `pyproj.CRS` instances used to create 2-d latitude and + longitude coordinates will also be shown. + + See CF Appendix F: Grid Mappings + (https://doi.org/10.5281/zenodo.14274886). + + .. versionadded:: NEXTVERSION + + :Parameters: + + f: `Field` or `Domain` + The Field or Domain, which will be updated in-place, + containing non-latitude_longitude grid. + + cr: `CoordinateReference` + The coordinate reference construct for the + non-latitude_longitude grid mapping. + + cr_latlon: `CoordinateReference` or `None` + The coordinate reference construct for the + latitude_longitude grid mapping, or `None` if there isn't + one, in which case a spherical latitude_longitude grid + mapping is assumed. + + longitude_at_pole: `None` or number + Define the treatment of longitudes of coordinates or + coordinate bounds that lie exactly on the north or south + pole. If `None` (the default) then the longitudes of such + points are determined by whichever algorithm was used to + create the coordinates, which could result in different + grid points on a pole having different longitudes. If set + to a number, then the longitudes of all grid points on the + north or south pole will be given that value. + + :Returns: + + (`str`, `str`) or (`None`, `None`) + The keys of the new 2-d latitude and longitude coordinate + constructs, in that order; or two `None`s if the 2-d + coordinates could not be created. + + """ + try: + import pyproj + except Exception: + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + "Must install the 'pyproj' library" + ) # pragma: no cover + + return (None, None) + + grid_mapping_name = cr.coordinate_conversion.get_parameter( + "grid_mapping_name", None + ) + if grid_mapping_name is None: + # Invalid non-latitude_longitude coordinate reference + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + f"Unable to create a {grid_mapping_name} pyproj.CRS object" + ) # pragma: no cover + + return (None, None) + + # ---------------------------------------------------------------- + # Get the source 1-d grid coordinates and axes + # ---------------------------------------------------------------- + one_d = _get_1d_coordinates(f, cr) + if one_d is None: + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + "Can't find all 1-d dimension coordinates" + ) # pragma: no cover + + return (None, None) + + # ---------------------------------------------------------------- + # Create the source projection CRS + # ---------------------------------------------------------------- + proj_src = create_projection_CRS(cr, grid_mapping_name) + if proj_src is None: + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + f"Unable to create a {grid_mapping_name} pyproj.CRS object" + ) # pragma: no cover + + return (None, None) + + # ---------------------------------------------------------------- + # Create the destination latitude_longitude CRS + # ---------------------------------------------------------------- + if cr_latlon is None: + # When a specific latitude_longitude coordinate reference has + # not been provided, get the shape of the ellipsoid from the + # source projection coordinate reference. + cr_latlon = cr + + proj_latlon = create_projection_CRS(cr_latlon, "latitude_longitude") + if proj_latlon is None: + # Invalid latitude_longitude coordinate reference + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + "Unable to create a latitude_longitude pyproj.CRS object " + f"from {cr_latlon!r}" + ) # pragma: no cover + + return (None, None) + + # ---------------------------------------------------------------- + # Create the transform function that converts source coordinates + # to destination coordinates + # ---------------------------------------------------------------- + try: + transformer = pyproj.Transformer.from_crs( + proj_src, proj_latlon, always_xy=True + ) + except Exception as error: + # Invalid latitude_longitude coordinate reference + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + f"Error during pyproj.Transformer.from_crs: {error}" + ) # pragma: no cover + + return (None, None) + + # ---------------------------------------------------------------- + # Create 2-d lat/lon coordinates from 1-d grid coordinate centres + # ---------------------------------------------------------------- + x = one_d["x"] + y = one_d["y"] + + # The transform function requires that the projection coordinates + # have units of metres + metres = Units("m") + if x.Units.equivalent(metres): + x = x.to_units(metres) + + if y.Units.equivalent(metres): + y = y.to_units(metres) + + # Create meshes of x and y cell centres + x_mesh, y_mesh = np.meshgrid(x.array, y.array) + + try: + lon, lat = transformer.transform( + x_mesh, y_mesh, errcheck=True, radians=False + ) + except Exception as error: + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinates for {cr!r}: " + f"Error during pyproj coordinate transformation: {error}" + ) # pragma: no cover + + return (None, None) + + del x_mesh, y_mesh + + if longitude_at_pole is not None: + # Set the longitude at the poles + lon = np.where((lat == -90) | (lat == 90), longitude_at_pole, lon) + + lat = f._Data(lat, "degrees_north") + lon = f._Data(lon, "degrees_east") + + # ---------------------------------------------------------------- + # Create the 2-d lat/lon bounds from 1-d grid coordinate bounds + # ---------------------------------------------------------------- + xb = x.get_bounds_data(None) + yb = y.get_bounds_data(None) + if xb is None or yb is None: + lat_bounds = None + lon_bounds = None + else: + xb = xb.array + yb = yb.array + + # Create meshes of x and y vertices. + shape = (y.size, x.size) + xb = np.broadcast_to(xb[np.newaxis, :, :], shape + (2,)) + yb = np.broadcast_to(yb[:, np.newaxis, :], shape + (2,)) + + x_mesh = np.empty(shape + (4,), dtype=xb.dtype) + y_mesh = np.empty(shape + (4,), dtype=yb.dtype) + + x_mesh[..., 0] = xb[..., 0] + y_mesh[..., 0] = yb[..., 0] + + x_mesh[..., 1] = xb[..., 0] + y_mesh[..., 1] = yb[..., 1] + + x_mesh[..., 2] = xb[..., 1] + y_mesh[..., 2] = yb[..., 1] + + x_mesh[..., 3] = xb[..., 1] + y_mesh[..., 3] = yb[..., 0] + del xb, yb + + try: + lon_bounds, lat_bounds = transformer.transform(x_mesh, y_mesh) + except Exception as error: + if is_log_level_info(logger): + logger.info( + f"Can't create 2-d lat/lon coordinate bounds for {cr!r}: " + f"Error during pyproj transformation: {error}" + ) # pragma: no cover + + return (None, None) + + del x_mesh, y_mesh + + if longitude_at_pole is not None: + # Set the longitude at the poles + lon_bounds = np.where( + (lat_bounds == -90) | (lat_bounds == 90), + longitude_at_pole, + lon_bounds, + ) + + lat_bounds = f._Bounds(data=f._Data(lat_bounds)) + lon_bounds = f._Bounds(data=f._Data(lon_bounds)) + + # ---------------------------------------------------------------- + # Add the 2-d lat/lon coordinates to the domain + # ---------------------------------------------------------------- + aux_lat = f._AuxiliaryCoordinate( + data=lat, + bounds=lat_bounds, + properties={"standard_name": "latitude"}, + ) + aux_lon = f._AuxiliaryCoordinate( + data=lon, + bounds=lon_bounds, + properties={"standard_name": "longitude"}, + ) + + axes = (one_d["axis_y"], one_d["axis_x"]) + + lat_key = f.set_construct(aux_lat, axes=axes, copy=False) + lon_key = f.set_construct(aux_lon, axes=axes, copy=False) + + return (lat_key, lon_key) + + +def _get_1d_coordinates(f, cr): + """Get 1-d dimension coordinates and axes. + + .. versionadded:: NEXTVERSION + + :Parameters: + + f: `Field` or `Domain` + The Field or Domain containing the 1-d dimension + coordinates. + + cr: `CoordinateReference` + The coordinate reference construct that defines or implies + the 1-d dimension coordinates. + + :Returns: + + `dict` or `None` + The 1-d coordinates and axes in the following dictionary + keys: + + * ``'x'``: The X dimension coordinate construct. + * ``'y'``: The Y dimension coordinate construct. + * ``'axis_x'``: The X domain axis construct key. + * ``'axis_y'``: The Y domain axis construct key. + + If both 1-d dimension coordinates could not be found then + `None` is returned. + + **Examples:** + + >>> _get_1d_coordinates(f, cr) + {'x': , + 'y': , + 'axis_x': 'domainaxis1', + 'axis_y': 'domainaxis0'} + + >>> _get_1d_coordinates(f, cr) + {'x': , + 'y': , + 'axis_x': 'domainaxis1', + 'axis_y': 'domainaxis0'} + + """ + x = None + y = None + + # Look for dimension coordinates named by the coordinate reference + for key in cr.coordinates(): + dc = f.dimension_coordinate(f"key%{key}", default=None) + if dc is None: + continue + + if dc.X: + key_x = key + x = dc + elif dc.Y: + key_y = key + y = dc + + if x is None and y is None: + # Look for 1-d coordinates by identity + grid_mapping_name = cr.coordinate_conversion.get_parameter( + "grid_mapping_name", None + ) + match grid_mapping_name: + case "rotated_latitude_longitude": + identity_x = "grid_longitude" + identity_y = "grid_latitude" + case _: + identity_x = "projection_x_coordinate" + identity_y = "projection_y_coordinate" + + key_x, x = f.dimension_coordinate( + identity_x, item=True, default=(None, None) + ) + key_y, y = f.dimension_coordinate( + identity_y, item=True, default=(None, None) + ) + + if x is None or y is None: + # Can't find both 1-d dimension coordinates + return + + # Make sure that the 1-d coordinates are referenced from the + # coordinate reference + cr.set_coordinates((key_x, key_y)) + + return { + "x": x, + "y": y, + "axis_x": f.get_data_axes(key_x)[0], + "axis_y": f.get_data_axes(key_y)[0], + } diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 6cf6411dbe..4ec846acf3 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -1,9 +1,7 @@ import logging -from functools import partial from re import Pattern import cfdm -from cfdm.read_write.exceptions import DatasetTypeError from ..aggregate import aggregate as cf_aggregate from ..cfimplementation import implementation @@ -12,7 +10,6 @@ from ..fieldlist import FieldList from ..functions import _DEPRECATION_ERROR_FUNCTION_KWARGS from ..query import Query -from .um import UMRead logger = logging.getLogger(__name__) @@ -94,16 +91,12 @@ class read(cfdm.read): **PP and UM fields files** - 32-bit and 64-bit PP and UM fields files of any endian-ness can be - read. In nearly all cases the file format is auto-detected from - the first 64 bits in the file, but for the few occasions when this - is not possible, the *um* keyword allows the format to be - specified, as well as the UM version (if the latter is not - inferrable from the PP or lookup header information). + 32-bit and 64-bit Met Office (UK) PP files and Met Office (UK) + fields files of any endian-ness can be read. - 2-d "slices" within a single file are always combined, where - possible, into field constructs with 3-d, 4-d or 5-d data. This is - done prior to any field construct aggregation (see the *aggregate* + 2-d "slices" within a single file are combined, where possible, + into field constructs with 3-d, 4-d or 5-d data. This is done + prior to any field construct aggregation (see the *aggregate* parameter). When reading PP and UM fields files, the *relaxed_units* aggregate @@ -176,76 +169,7 @@ class read(cfdm.read): {{read warnings: `bool`, optional}} - um: `dict`, optional - For Met Office (UK) PP files and Met Office (UK) fields - files only, provide extra decoding instructions. This - option is ignored for input files which are not PP or - fields files. In most cases, how to decode a file is - inferrable from the file's contents, but if not then each - key/value pair in the dictionary sets a decoding option as - follows: - - * ``'fmt'``: `str` - - The file format (``'PP'`` or ``'FF'``) - - * ``'word_size'``: `int` - - The word size in bytes (``4`` or ``8``). - - * ``'endian'``: `str` - - The byte order (``'big'`` or ``'little'``). - - * ``'version'``: `int` or `str` - - The UM version to be used when decoding the - header. Valid versions are, for example, ``4.2``, - ``'6.6.3'`` and ``'8.2'``. In general, a given version - is ignored if it can be inferred from the header (which - is usually the case for files created by the UM at - versions 5.3 and later). The exception to this is when - the given version has a third element (such as the 3 in - 6.6.3), in which case any version in the header is - ignored. The default version is ``4.5``. - - * ``'height_at_top_of_model'``: `float` - - The height in metres of the upper bound of the top model - level. By default the height at top model is taken from - the top level's upper bound defined by BRSVD1 in the - lookup header. If the height can't be determined from - the header, or the given height is less than or equal to - 0, then a coordinate reference system will still be - created that contains the 'a' and 'b' formula term - values, but without an atmosphere hybrid height - dimension coordinate construct. - - .. note:: A current limitation is that if pseudolevels - and atmosphere hybrid height coordinates are - defined by same the lookup headers then the - height **can't be determined - automatically**. In this case the height may - be found after reading as the maximum value of - the bounds of the domain ancillary construct - containing the 'a' formula term. The file can - then be re-read with this height as a *um* - parameter. - - If format is specified as ``'PP'`` then the word size and - byte order default to ``4`` and ``'big'`` respectively. - - This parameter replaces the deprecated *umversion* and - *height_at_top_of_model* parameters. - - *Parameter example:* - To specify that the input files are 32-bit, big-endian - PP files: ``um={'fmt': 'PP'}`` - - *Parameter example:* - To specify that the input files are 32-bit, - little-endian PP files from version 5.1 of the UM: - ``um={'fmt': 'PP', 'endian': 'little', 'version': 5.1}`` + {{read um: `dict` or `None`, optional}} .. versionadded:: 1.5 @@ -297,9 +221,13 @@ class read(cfdm.read): .. versionadded:: 3.11.0 - {{read netcdf_backend: `None` or (sequence of) `str`, optional}} + {{read backend: `None` or (sequence of) `str`, optional}} - .. versionadded:: 3.17.0 + .. versionadded:: NEXTVERSION + + {{read backend_options: `None` or `dict`, optional}} + + .. versionadded:: NEXTVERSION {{read storage_options: `dict` or `None`, optional}} @@ -329,6 +257,18 @@ class read(cfdm.read): .. versionadded:: 3.17.0 + {{read cfa_filesystem: `None` or filesystem, optional}} + + .. versionadded:: NEXTVERSION + + {{read cfa_backend: `None` or (sequence of) `str`, optional}} + + .. versionadded:: NEXTVERSION + + {{cfa_backend_options: `None` or `dict`, optional}} + + .. versionadded:: NEXTVERSION + {{read to_memory: (sequence of) `str`, optional}} .. versionadded:: 3.17.0 @@ -341,6 +281,10 @@ class read(cfdm.read): .. versionadded:: 3.20.0 + {{read _noncompliance_report: `bool`, optional}} + + ..versionadded:: NEXTVERSION + umversion: deprecated at version 3.0.0 Use the *um* parameter instead. @@ -371,11 +315,15 @@ class read(cfdm.read): file_type: deprecated at version 3.18.0 Use the *dataset_type* parameter instead. + netcdf_backend: Deprecated at version NEXTVERSION + Use *backend* instead. + :Returns: `FieldList` or `DomainList` The field or domain constructs found in the input dataset(s). The list may be empty. + **Examples** >>> x = cf.read('file.nc') @@ -452,7 +400,8 @@ def __new__( cfa=None, cfa_write=None, to_memory=None, - netcdf_backend=None, + backend=None, + backend_options=None, storage_options=None, cache=True, chunks="auto", @@ -461,6 +410,8 @@ def __new__( file_type=None, group_dimension_search="closest_ancestor", filesystem=None, + legacy_um_backend=False, + netcdf_backend=None, ): """Read field or domain constructs from a dataset.""" kwargs = locals() @@ -555,50 +506,71 @@ def _finalise(self): `None` """ - # Whether or not there were only netCDF datasets - only_netCDF = self.unique_dataset_categories == set(("netCDF",)) - - # Whether or not there were any UM datasets - some_UM = "UM" in self.unique_dataset_categories - # ---------------------------------------------------------------- # Select matching constructs from netCDF datasets (before # aggregation) # ---------------------------------------------------------------- select = self.select - if select and only_netCDF: + if select: self.constructs = self.constructs.select_by_identity(*select) # ---------------------------------------------------------------- # Aggregate the output fields or domains # ---------------------------------------------------------------- - if self.aggregate and len(self.constructs) > 1: + self.aggregate = self.aggregate and len(self.constructs) > 1 + if self.aggregate: + UM = False # True if there is at least one UM field + non_UM = False # True if there is at least one non-UM field + for f in self.constructs: + try: + if f.get_property("um_identity", "").startswith("UM_"): + UM = True + else: + non_UM = True + except Exception: + non_UM = True + + if UM and non_UM: + break + + if UM and non_UM: + self.aggregate = False + logger.warning( + "Won't aggregate fields from a mixture of UM and " + "non-UM sources (a field from a UM source is defined as " + "having a string-valued um_identity property that starts " + "with 'UM_')." + "\n" + "Aggregation may still be possible with cf.aggregate." + ) + # This is because the aggregation of UM fields + # requires + # `aggregate_options["field_identity"]="um_identity"` + # in order to overcome the many-to-one relationship between + # STASH codes and standard names. + + if self.aggregate: aggregate_options = self.aggregate_options - # Set defaults specific to UM fields - if some_UM and "strict_units" not in aggregate_options: - aggregate_options["relaxed_units"] = True + if UM: + # Set extra aggregate options for fields created from + # UM data. + # + # We can't trust the the standard_name to provide the + # identity for UM fields (multiple STASH codes can + # have the same standard name), so instead we have to + # the um_identity property (which encapsulates the + # submodel, stash/field code and UM version). + aggregate_options["field_identity"] = "um_identity" + + if "strict_units" not in aggregate_options: + aggregate_options["relaxed_units"] = True + + # Do the aggregation self.constructs = cf_aggregate( self.constructs, **aggregate_options ) - # ---------------------------------------------------------------- - # Add standard names to non-netCDF fields (after aggregation) - # ---------------------------------------------------------------- - if not only_netCDF: - for f in self.constructs: - standard_name = f._custom.get("standard_name", None) - if standard_name is not None: - f.set_property("standard_name", standard_name, copy=False) - del f._custom["standard_name"] - - # ---------------------------------------------------------------- - # Select matching constructs from non-netCDF files (after - # setting their standard names) - # ---------------------------------------------------------------- - if select and not only_netCDF: - self.constructs = self.constructs.select_by_identity(*select) - super()._finalise() def _initialise(self): @@ -660,115 +632,3 @@ def _initialise(self): self.aggregate = aggregate self.aggregate_options = aggregate_options - - def _read(self, dataset): - """Read a given dataset into field or domain constructs. - - The constructs are stored in the `dataset_contents` attribute. - - Called by `__new__`. - - .. versionadded:: 3.18.0 - - :Parameters: - - dataset: `str` - The pathname of the dataset to be read. - - :Returns: - - `None` - - """ - dataset_type = self.dataset_type - - # ------------------------------------------------------------ - # Try to read as a netCDF dataset - # ------------------------------------------------------------ - super()._read(dataset) - - if self.dataset_contents is not None: - # Successfully read the dataset - return - - # ------------------------------------------------------------ - # Try to read as a PP/UM dataset - # ------------------------------------------------------------ - if dataset_type is None or dataset_type.intersection( - self.UM_dataset_types - ): - if not hasattr(self, "um_read"): - # Initialise the UM read function - kwargs = self.kwargs - um_kwargs = { - key: kwargs[key] - for key in ( - "height_at_top_of_model", - "squeeze", - "unsqueeze", - "domain", - "dataset_type", - "unpack", - "verbose", - "filesystem", - "storage_options", - ) - } - um_kwargs["set_standard_name"] = False - um_kwargs["select"] = self.select - um = self.um - um_kwargs["um_version"] = um.get("version") - um_kwargs["fmt"] = um.get("fmt") - um_kwargs["word_size"] = um.get("word_size") - um_kwargs["endian"] = um.get("endian") - - self.um_read = partial( - UMRead(self.implementation).read, **um_kwargs - ) - - try: - # Try to read the dataset - self.dataset_contents = self.um_read(dataset) - except DatasetTypeError as error: - if dataset_type is None: - self.dataset_format_errors.append(error) - else: - # Successfully read the dataset - self.unique_dataset_categories.add("UM") - - if self.dataset_contents is not None: - # Successfully read the dataset - return - - # ------------------------------------------------------------ - # Try to read as a GRIB dataset - # - # Not yet available. When (if!) the time comes, the framework - # will be: - # ------------------------------------------------------------ - # - # if dataset_type is None or dataset_type.intersection( - # self.GRIB_dataset_types - # ): - # if not hasattr(self, "grib_read"): - # # Initialise the GRIB read function - # kwargs = self.kwargs - # grib_kwargs = ... # - # - # self.grib_read = partial( - # GRIBRead(self.implementation).read, **grib_kwargs - # ) - # - # try: - # # Try to read the dataset - # self.dataset_contents = self.grib_read(dataset) - # except DatasetTypeError as error: - # if dataset_type is None: - # self.dataset_format_errors.append(error) - # else: - # # Successfully read the dataset - # self.unique_dataset_categories.add("GRIB") - # - # if self.dataset_contents is not None: - # # Successfully read the dataset - # return diff --git a/cf/read_write/um/__init__.py b/cf/read_write/um/__init__.py deleted file mode 100644 index 2e8a395343..0000000000 --- a/cf/read_write/um/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .umread import UMRead diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py deleted file mode 100644 index 0f0fa56d48..0000000000 --- a/cf/read_write/um/umread.py +++ /dev/null @@ -1,3872 +0,0 @@ -import itertools -import logging -import textwrap -from datetime import datetime -from uuid import uuid4 - -import cfdm -import numpy as np -from cfdm import Constructs, is_log_level_info -from cfdm.read_write.exceptions import DatasetTypeError - -from cf import __Conventions__, __version__ -from cf.constants import _stash2standard_name -from cf.data import Data -from cf.data.array import UMArray -from cf.decorators import ( - _manage_log_level_via_verbose_attr, - _manage_log_level_via_verbosity, -) -from cf.functions import abspath -from cf.functions import atol as cf_atol -from cf.functions import load_stash2standard_name -from cf.functions import rtol as cf_rtol -from cf.umread_lib.umfile import File -from cf.units import Units - -logger = logging.getLogger(__name__) - -_cached_runid = {} -_cached_latlon = {} -_cached_ctime = {} -_cached_size_1_height_coordinate = {} -_cached_date2num = {} -_cached_model_level_number_coordinate = {} -_cached_regular_array = {} -_cached_regular_bounds = {} - -# -------------------------------------------------------------------- -# Constants -# -------------------------------------------------------------------- -_pi_over_180 = np.pi / 180.0 - -# PP missing data indicator -_pp_rmdi = -1.0e30 - -# No no-missing-data value of BMDI (as described in UMDP F3 v805) -_BMDI_no_missing_data_value = -1.0e30 - -# Reference surface pressure in Pascals -_pstar = 1.0e5 - -# -------------------------------------------------------------------- -# Characters used in decoding LBEXP into a runid -# -------------------------------------------------------------------- -_characters = ( - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", -) - -_n_characters = len(_characters) - -# # -------------------------------------------------------------------- -# # Number matching regular expression -# # -------------------------------------------------------------------- -# _number_regex = '([-+]?\d*\.?\d+(e[-+]?\d+)?)' - -_Units = { - None: Units(), - "": Units(""), - "1": Units("1"), - "Pa": Units("Pa"), - "m": Units("m"), - "hPa": Units("hPa"), - "K": Units("K"), - "degrees": Units("degrees"), - "degrees_east": Units("degrees_east"), - "degrees_north": Units("degrees_north"), - "days": Units("days"), - "gregorian 1752-09-13": Units("days since 1752-09-13", "gregorian"), - "365_day 1752-09-13": Units("days since 1752-09-13", "365_day"), - "360_day 0-1-1": Units("days since 0-1-1", "360_day"), -} - -# -------------------------------------------------------------------- -# Names of PP integer and real header items -# -------------------------------------------------------------------- -_header_names = ( - "LBYR", - "LBMON", - "LBDAT", - "LBHR", - "LBMIN", - "LBDAY", - "LBYRD", - "LBMOND", - "LBDATD", - "LBHRD", - "LBMIND", - "LBDAYD", - "LBTIM", - "LBFT", - "LBLREC", - "LBCODE", - "LBHEM", - "LBROW", - "LBNPT", - "LBEXT", - "LBPACK", - "LBREL", - "LBFC", - "LBCFC", - "LBPROC", - "LBVC", - "LBRVC", - "LBEXP", - "LBEGIN", - "LBNREC", - "LBPROJ", - "LBTYP", - "LBLEV", - "LBRSVD1", - "LBRSVD2", - "LBRSVD3", - "LBRSVD4", - "LBSRCE", - "LBUSER1", - "LBUSER2", - "LBUSER3", - "LBUSER4", - "LBUSER5", - "LBUSER6", - "LBUSER7", - "BRSVD1", - "BRSVD2", - "BRSVD3", - "BRSVD4", - "BDATUM", - "BACC", - "BLEV", - "BRLEV", - "BHLEV", - "BHRLEV", - "BPLAT", - "BPLON", - "BGOR", - "BZY", - "BDY", - "BZX", - "BDX", - "BMDI", - "BMKS", -) - -# -------------------------------------------------------------------- -# Positions of PP header items in their arrays -# -------------------------------------------------------------------- -( - lbyr, - lbmon, - lbdat, - lbhr, - lbmin, - lbday, - lbyrd, - lbmond, - lbdatd, - lbhrd, - lbmind, - lbdayd, - lbtim, - lbft, - lblrec, - lbcode, - lbhem, - lbrow, - lbnpt, - lbext, - lbpack, - lbrel, - lbfc, - lbcfc, - lbproc, - lbvc, - lbrvc, - lbexp, - lbegin, - lbnrec, - lbproj, - lbtyp, - lblev, - lbrsvd1, - lbrsvd2, - lbrsvd3, - lbrsvd4, - lbsrce, - lbuser1, - lbuser2, - lbuser3, - lbuser4, - lbuser5, - lbuser6, - lbuser7, -) = tuple(range(45)) - -( - brsvd1, - brsvd2, - brsvd3, - brsvd4, - bdatum, - bacc, - blev, - brlev, - bhlev, - bhrlev, - bplat, - bplon, - bgor, - bzy, - bdy, - bzx, - bdx, - bmdi, - bmks, -) = tuple(range(19)) - -# -------------------------------------------------------------------- -# Map PP axis codes to CF standard names (The full list of field code -# keys may be found at -# http://cms.ncas.ac.uk/html_umdocs/wave/@header.) -# -------------------------------------------------------------------- -_coord_standard_name = { - 0: None, # Sigma (or eta, for hybrid coordinate data). - 1: "air_pressure", # Pressure (mb). - 2: "height", # Height above sea level (km) - # Eta (U.M. hybrid coordinates) only: - 3: "atmosphere_hybrid_sigma_pressure_coordinate", - 4: "depth", # Depth below sea level (m) - 5: "model_level_number", # Model level. - 6: "air_potential_temperature", # Theta - 7: "atmosphere_sigma_coordinate", # Sigma only. - 8: None, # Sigma-theta - 10: "latitude", # Latitude (degrees N). - 11: "longitude", # Longitude (degrees E). - # Site number (set of parallel rows or columns e.g.Time series): - 13: None, # "region", - 14: "atmosphere_hybrid_height_coordinate", - 15: "height", - 20: "time", # Time (days) (Gregorian calendar (not 360 day year)) - 21: "time", # Time (months) - 22: "time", # Time (years) - 23: "time", # Time (model days with 360 day model calendar) - 40: None, # pseudolevel - 99: None, # Other - -10: "grid_latitude", # Rotated latitude (degrees). - -11: "grid_longitude", # Rotated longitude (degrees). - -20: "radiation_wavelength", -} - -# -------------------------------------------------------------------- -# Map PP axis codes to CF long names -# -------------------------------------------------------------------- -_coord_long_name = {13: "site"} - -# -------------------------------------------------------------------- -# Map PP axis codes to UDUNITS strings -# -------------------------------------------------------------------- -# _coord_units = { -_axiscode_to_units = { - 0: "1", # Sigma (or eta, for hybrid coordinate data) - 1: "hPa", # air_pressure - 2: "m", # altitude - 3: "1", # atmosphere_hybrid_sigma_pressure_coordinate - 4: "m", # depth - 5: "1", # model_level_number - 6: "K", # air_potential_temperature - 7: "1", # atmosphere_sigma_coordinate - 10: "degrees_north", # latitude - 11: "degrees_east", # longitude - 13: "", # region - 14: "1", # atmosphere_hybrid_height_coordinate - 15: "m", # height - 20: "days", # time (gregorian) - 23: "days", # time (360_day) - 40: "1", # pseudolevel - -10: "degrees", # rotated latitude (not an official axis code) - -11: "degrees", # rotated longitude (not an official axis code) -} - -# -------------------------------------------------------------------- -# Map PP axis codes to Units objects -# -------------------------------------------------------------------- -_axiscode_to_Units = { - 0: _Units["1"], # Sigma (or eta, for hybrid coordinate data) - 1: _Units["hPa"], # air_pressure - 2: _Units["m"], # altitude - 3: _Units["1"], # atmosphere_hybrid_sigma_pressure_coordinate - 4: _Units["m"], # depth - 5: _Units["1"], # model_level_number - 6: _Units["K"], # air_potential_temperature - 7: _Units["1"], # atmosphere_sigma_coordinate - 10: _Units["degrees_north"], # latitude - 11: _Units["degrees_east"], # longitude - 13: _Units[""], # region - 14: _Units["1"], # atmosphere_hybrid_height_coordinate - 15: _Units["m"], # height - 20: _Units["days"], # time (gregorian) - 23: _Units["days"], # time (360_day) - 40: _Units["1"], # pseudolevel - -10: _Units["degrees"], # rotated latitude (not an official axis code) - -11: _Units["degrees"], # rotated longitude (not an official axis code) -} - -# -------------------------------------------------------------------- -# Map PP axis codes to CF axis attributes -# -------------------------------------------------------------------- -_coord_axis = { - 1: "Z", # air_pressure - 2: "Z", # altitude - 3: "Z", # atmosphere_hybrid_sigma_pressure_coordinate - 4: "Z", # depth - 5: "Z", # model_level_number - 6: "Z", # air_potential_temperature - 7: "Z", # atmosphere_sigma_coordinate - 10: "Y", # latitude - 11: "X", # longitude - 13: None, # region - 14: "Z", # atmosphere_hybrid_height_coordinate - 15: "Z", # height - 20: "T", # time (gregorian) - 23: "T", # time (360_day) - 40: None, # pseudolevel - -10: "Y", # rotated latitude (not an official axis code) - -11: "X", # rotated longitude (not an official axis code) -} - -# -------------------------------------------------------------------- -# Map PP axis codes to CF positive attributes -# -------------------------------------------------------------------- -_coord_positive = { - 1: "down", # air_pressure - 2: "up", # altitude - 3: "down", # atmosphere_hybrid_sigma_pressure_coordinate - 4: "down", # depth - 5: None, # model_level_number - 6: "up", # air_potential_temperature - 7: "down", # atmosphere_sigma_coordinate - 10: None, # latitude - 11: None, # longitude - 13: None, # region - 14: "up", # atmosphere_hybrid_height_coordinate - 15: "up", # height - 20: None, # time (gregorian) - 23: None, # time (360_day) - 40: None, # pseudolevel - -10: None, # rotated latitude (not an official axis code) - -11: None, # rotated longitude (not an official axis code) -} - -# -------------------------------------------------------------------- -# Map LBVC codes to PP axis codes. The full list of field code keys -# may be found at http://cms.ncas.ac.uk/html_umdocs/wave/@fcodes -# -------------------------------------------------------------------- -_lbvc_to_axiscode = { - 1: 2, # altitude (Height) - 2: 4, # depth (Depth) - 3: None, # (Geopotential (= g*height)) - 4: None, # (ICAO height) - 6: 4, # model_level_number # Changed from 5 !!! - 7: None, # (Exner pressure) - 8: 1, # air_pressure (Pressure) - 9: 3, # atmosphere_hybrid_sigma_pressure_coordinate (Hybrid pressure) - # dch check: - 10: 7, # atmosphere_sigma_coordinate (Sigma (= p/surface p)) - 16: None, # (Temperature T) - 19: 6, # air_potential_temperature (Potential temperature) - 27: None, # (Atmospheric) density - 28: None, # (d(p*)/dt . p* = surface pressure) - 44: None, # (Time in seconds) - 65: 14, # atmosphere_hybrid_height_coordinate (Hybrid height) - 129: None, # Surface - 176: 10, # latitude (Latitude) - 177: 11, # longitude (Longitude) -} - -# -------------------------------------------------------------------- -# Map model identifier codes to model names. The model identifier code -# is the last four digits of LBSRCE. -# -------------------------------------------------------------------- -_lbsrce_model_codes = {1111: "UM"} - -# -------------------------------------------------------------------- -# Names of PP extra data codes -# -------------------------------------------------------------------- -_extra_data_name = { - 1: "x", - 2: "y", - 3: "y_domain_lower_bound", - 4: "x_domain_lower_bound", - 5: "y_domain_upper_bound", - 6: "x_domain_upper_bound", - 7: "z_domain_lower_bound", - 8: "x_domain_upper_bound", - 9: "title", - 10: "domain_title", - 11: "x_lower_bound", - 12: "x_upper_bound", - 13: "y_lower_bound", - 14: "y_upper_bound", -} - -# -------------------------------------------------------------------- -# LBCODE values for unrotated latitude longitude grids -# -------------------------------------------------------------------- -_true_latitude_longitude_lbcodes = set((1, 2)) - -# -------------------------------------------------------------------- -# LBCODE values for rotated latitude longitude grids -# -------------------------------------------------------------------- -_rotated_latitude_longitude_lbcodes = set((101, 102, 111)) - -# _axis = {'t' : 'dim0', -# 'z' : 'dim1', -# 'y' : 'dim2', -# 'x' : 'dim3', -# 'r' : 'dim4', -# 'p' : 'dim5', -# 'area': None, -# } - -_axis = {"area": "area"} - -_autocyclic_false = {"no-op": True, "X": False, "cyclic": False} - - -class UMField: - """Represents Fields derived from a UM fields file.""" - - def __init__( - self, - filename, - var, - fmt, - byte_ordering, - word_size, - um_version, - set_standard_name, - height_at_top_of_model, - verbose=None, - implementation=None, - select=None, - info=False, - squeeze=False, - unsqueeze=False, - unpack=True, - storage_protocol=None, - storage_options=None, - **kwargs, - ): - """**Initialisation** - - :Parameters: - - filename: `str` - The name of the PP/UM file. - - .. versionadded:: 3.20.0 - - var: `umfile.Var` - - byte_ordering: `str` - ``'little_endian'` or ``'big_endian'``. - - word_size: `int` - Word size in bytes (4 or 8). - - fmt: `str` - ``'PP'` or ``'FF'`` - - um_version: number - - set_standard_name: `bool` - If True then set the standard_name CF property. - - height_at_top_of_model: `float` - - verbose: `int` or `str` or `None`, optional - If an integer from ``-1`` to ``3``, or an equivalent string - equal ignoring case to one of: - - * ``'DISABLE'`` (``0``) - * ``'WARNING'`` (``1``) - * ``'INFO'`` (``2``) - * ``'DETAIL'`` (``3``) - * ``'DEBUG'`` (``-1``) - - set for the duration of the method call only as the minimum - cut-off for the verboseness level of displayed output (log) - messages, regardless of the globally-configured `cf.log_level`. - Note that increasing numerical value corresponds to increasing - verbosity, with the exception of ``-1`` as a special case of - maximal and extreme verbosity. - - Otherwise, if `None` (the default value), output messages will - be shown according to the value of the `cf.log_level` setting. - - Overall, the higher a non-negative integer or equivalent string - that is set (up to a maximum of ``3``/``'DETAIL'``) for - increasing verbosity, the more description that is printed - about the read process. - - squeeze: `bool`, optional - If True then remove all size 1 dimensions from field - construct data arrays, regardless of how the data are - stored in the dataset. If False (the default) then the - presence or not of size 1 dimensions is determined by - how the data are stored in its dataset. - - .. versionadded:: 3.17.0 - - unsqueeze: `bool`, optional - If True then ensure that field construct data arrays - span all of the size 1 dimensions, regardless of how - the data are stored in the dataset. If False (the - default) then the presence or not of size 1 dimensions - is determined by how the data are stored in its - dataset. - - .. versionadded:: 3.17.0 - - unpack: `bool`, optional - If True, the default, then unpack arrays by convention - when the data is read from disk. - - Unpacking is determined by netCDF conventions for the - following variable attributes ``add_offset`` and - ``scale_factor``, as applied to lookup header entries - BDATUM and BMKS respectively. - - .. versionadded:: 3.17.0 - - storage_protocol: `None` or `str`, optional - The `fsspec` file system protocol (e.g, ``'file'``, - ``'s3'``, ``'http'``). If `None` (the default) then a - local file system is assumed. - - .. versionadded:: 3.20.0 - - storage_options: `dict` or `None`, optional - Key/value pairs to be passed on to the creation of - `s3fs.S3FileSystem` file systems to control the - opening of files in S3 object stores. Ignored for - files not in an S3 object store, i.e. those whose - names do not start with ``s3:``. - - By default, or if `None`, then *storage_options* is - taken as ``{}``. - - If the ``'endpoint_url'`` key is not in - *storage_options* or is not in a dictionary defined by - the ``'client_kwargs`` key (which is always the case - when *storage_options* is `None`), then one will be - automatically inserted for accessing an S3 file. For - example, for a file name of - ``'s3://store/data/file.nc'``, an ``'endpoint_url'`` - key with value ``'https://store'`` would be created. - - *Parameter example:* - For a file name of ``'s3://store/data/file.nc'``, - the following are equivalent: ``None``, ``{}``, and - ``{'endpoint_url': 'https://store'}``, - ``{'client_kwargs': {'endpoint_url': - 'https://store'}}`` - - *Parameter example:* - ``{'key': 'scaleway-api-key...', 'secret': - 'scaleway-secretkey...', 'endpoint_url': - 'https://s3.fr-par.scw.cloud', 'client_kwargs': - {'region_name': 'fr-par'}}`` - - .. versionadded:: 3.20.0 - - kwargs: *optional* - Keyword arguments providing extra CF properties for each - return field construct. - - """ - if squeeze and unsqueeze: - raise ValueError("'squeeze' and 'unsqueeze' can not both be True") - - self._bool = False - - self.info = info - - self.implementation = implementation - - self.verbose = verbose - - self.fmt = fmt - self.height_at_top_of_model = height_at_top_of_model - self.byte_ordering = byte_ordering - self.word_size = word_size - self.unpack = unpack - - self.atol = cf_atol() - - self.field = self.implementation.initialise_Field() - - cf_properties = {} - attributes = {} - - self.fields = [] - - self.filename = filename - self.storage_protocol = storage_protocol - self.storage_options = storage_options - - groups = var.group_records_by_extra_data() - - n_groups = len(groups) - - if n_groups == 1: - # There is one group of records - groups_nz = [var.nz] - groups_nt = [var.nt] - elif n_groups > 1: - # There are multiple groups of records, distinguished by - # different extra data. - groups_nz = [] - groups_nt = [] - groups2 = [] - for group in groups: - group_size = len(group) - if group_size == 1: - # There is only one record in this group - split_group = False - nz = 1 - elif group_size > 1: - # There are multiple records in this group - # Find the lengths of runs of identical times - times = [ - (self.header_vtime(rec), self.header_dtime(rec)) - for rec in group - ] - lengths = [ - len(tuple(g)) for k, g in itertools.groupby(times) - ] - if len(set(lengths)) == 1: - # Each run of identical times has the same - # length, so it is possible that this group - # forms a variable of nz x nt records. - split_group = False - nz = lengths.pop() - z0 = [self.z for rec in group[:nz]] - for i in range(nz, group_size, nz): - z1 = [ - self.header_z(rec) for rec in group[i : i + nz] - ] - if z1 != z0: - split_group = True - break - else: - # Different runs of identical times have - # different lengths, so it is not possible for - # this group to form a variable of nz x nt - # records. - split_group = True - nz = 1 - - if split_group: - # This group doesn't form a complete nz x nt - # matrix, so split it up into 1 x 1 groups. - groups2.extend([[rec] for rec in group]) - groups_nz.extend([1] * group_size) - groups_nt.extend([1] * group_size) - else: - # This group forms a complete nz x nt matrix, so - # it may be considered as a variable in its own - # right and doesn't need to be split up. - groups2.append(group) - groups_nz.append(nz) - groups_nt.append(group_size / nz) - - groups = groups2 - - rec0 = groups[0][0] - - int_hdr = rec0.int_hdr - self.int_hdr_dtype = int_hdr.dtype - self.real_hdr_dtype = rec0.real_hdr.dtype - int_hdr = int_hdr.tolist() - - real_hdr = rec0.real_hdr.tolist() - self.int_hdr = int_hdr - self.real_hdr = real_hdr - - # ------------------------------------------------------------ - # Set some metadata quantities which are guaranteed to be the - # same for all records in a variable - # ------------------------------------------------------------ - LBNPT = int_hdr[lbnpt] - LBROW = int_hdr[lbrow] - LBTIM = int_hdr[lbtim] - LBCODE = int_hdr[lbcode] - LBPROC = int_hdr[lbproc] - LBVC = int_hdr[lbvc] - stash = int_hdr[lbuser4] - LBUSER5 = int_hdr[lbuser5] - submodel = int_hdr[lbuser7] - BPLAT = real_hdr[bplat] - BPLON = real_hdr[bplon] - BDX = real_hdr[bdx] - BDY = real_hdr[bdy] - - if not LBROW or not LBNPT: - logger.warn( - f"WARNING: Skipping STASH code {stash} with LBROW={LBROW}, " - f"LBNPT={LBNPT}, LBPACK={int_hdr[lbpack]} " - "(possibly runlength encoded)" - ) # pragma: no cover - self.field = (None,) - return - - if stash: - section, item = divmod(stash, 1000) - um_stash_source = "m%02ds%02di%03d" % (submodel, section, item) - else: - um_stash_source = None - - header_um_version, source = divmod(int_hdr[lbsrce], 10000) - - if header_um_version > 0 and int(um_version) == um_version: - model_um_version = header_um_version - self.um_version = header_um_version - else: - model_um_version = None - self.um_version = um_version - - # Set source - source = _lbsrce_model_codes.setdefault(source, None) - if source is not None and model_um_version is not None: - source += f" vn{model_um_version}" - - # Only process the requested fields - ok = True - if select: - values1 = ( - f"stash_code={stash}", - f"lbproc={LBPROC}", - f"lbtim={LBTIM}", - f"runid={self.decode_lbexp()}", - f"submodel={submodel}", - ) - if um_stash_source is not None: - values1 += (f"um_stash_source={um_stash_source}",) - if source: - values1 += (f"source={source}",) - - ok = False - for value0 in select: - for value1 in values1: - ok = Constructs._matching_values( - value0, None, value1, basic=True - ) - if ok: - break - - if ok: - break - - if not ok: - # This PP/UM field does not match the requested selection - self.field = (None,) - return - - # Still here? - self.lbnpt = LBNPT - self.lbrow = LBROW - self.lbtim = LBTIM - self.lbproc = LBPROC - self.lbvc = LBVC - self.bplat = BPLAT - self.bplon = BPLON - self.bdx = BDX - self.bdy = BDY - - # ------------------------------------------------------------ - # Set some derived metadata quantities which are (as good as) - # guaranteed to be the same for all records in a variable - # ------------------------------------------------------------ - self.lbtim_ia, ib = divmod(LBTIM, 100) - self.lbtim_ib, ic = divmod(ib, 10) - - if ic == 1: - calendar = "gregorian" - elif ic == 4: - calendar = "365_day" - else: - calendar = "360_day" - - self.calendar = calendar - self.reference_time_Units() - - if source: - cf_properties["source"] = source - - # ------------------------------------------------------------ - # Set the T, Z, Y and X axis codes. These are guaranteed to be - # the same for all records in a variable. - # ------------------------------------------------------------ - if LBCODE == 1 or LBCODE == 2: - # 1 = Unrotated regular lat/long grid - # 2 = Regular lat/lon grid boxes (grid points are box - # centres) - ix = 11 - iy = 10 - elif LBCODE == 101 or LBCODE == 102: - # 101 = Rotated regular lat/long grid - # 102 = Rotated regular lat/lon grid boxes (grid points - # are box centres) - ix = -11 # rotated longitude (not an official axis code) - iy = -10 # rotated latitude (not an official axis code) - elif LBCODE >= 10000: - # Cross section - ix, iy = divmod(divmod(LBCODE, 10000)[1], 100) - else: - ix = None - iy = None - - iz = _lbvc_to_axiscode.setdefault(LBVC, None) - - # Set it from the calendar type - if iy in (20, 23) or ix in (20, 23): - # Time is dealt with by x or y - it = None - elif calendar == "gregorian": - it = 20 - else: - it = 23 - - self.ix = ix - self.iy = iy - self.iz = iz - self.it = it - - self.cf_info = {} - - # Set a identifying name based on the submodel and STASHcode - # (or field code). - # stash = int_hdr[lbuser4]# - self.stash = stash - - # The STASH code has been set in the PP header, so try to find - # its standard_name from the conversion table - stash_records = _stash2standard_name.get((submodel, stash), None) - - um_Units = None - um_condition = None - - long_name = None - standard_name = None - - if stash_records: - um_version = self.um_version - for ( - long_name, - units, - valid_from, - valid_to, - standard_name, - cf_info, - um_condition, - ) in stash_records: - # Check that conditions are met - if not self.test_um_version(valid_from, valid_to, um_version): - continue - - if um_condition: - if not self.test_um_condition( - um_condition, LBCODE, BPLAT, BPLON - ): - continue - - # Still here? Then we have our standard_name, etc. - # if standard_name: - # if set_standard_name: - # cf_properties['standard_name'] = standard_name - # else: - # attributes['_standard_name'] = standard_name - if standard_name and set_standard_name: - cf_properties["standard_name"] = standard_name - - cf_properties["long_name"] = long_name.rstrip() - - um_Units = _Units.get(units, None) - if um_Units is None: - um_Units = Units(units) - _Units[units] = um_Units - - self.um_Units = um_Units - self.cf_info = cf_info - - break - - if um_stash_source is not None: - cf_properties["um_stash_source"] = um_stash_source - identity = f"UM_{um_stash_source}_vn{self.um_version}" - else: - identity = f"UM_{submodel}_fc{int_hdr[lbfc]}_vn{self.um_version}" - - if um_Units is None: - self.um_Units = _Units[None] - - if um_condition: - identity += f"_{um_condition}" - - if long_name is None: - cf_properties["long_name"] = identity - - for recs, nz, nt in zip(groups, groups_nz, groups_nt): - self.recs = recs - self.nz = nz - self.nt = nt - self.z_recs = recs[:nz] - self.t_recs = recs[::nz] - - LBUSER5 = recs[0].int_hdr.item(lbuser5) - - # self.cell_method_axis_name = {'area': 'area'} - - self.down_axes = set() - self.z_axis = "z" - - # -------------------------------------------------------- - # Get the extra data for this group - # -------------------------------------------------------- - extra = recs[0].get_extra_data() - self.extra = extra - - # -------------------------------------------------------- - # Create the 'T' dimension coordinate - # -------------------------------------------------------- - axiscode = it - if axiscode is not None: - c = self.time_coordinate(axiscode) - - # -------------------------------------------------------- - # Create the 'Z' dimension coordinate - # -------------------------------------------------------- - axiscode = iz - if axiscode is not None: - # Get 'Z' coordinate from LBVC - if axiscode == 3: - c = self.atmosphere_hybrid_sigma_pressure_coordinate( - axiscode - ) - elif axiscode == 2 and "height" in self.cf_info: - # Create the height coordinate from the information - # given in the STASH to standard_name conversion table - height, units = self.cf_info["height"] - c = self.size_1_height_coordinate(axiscode, height, units) - elif axiscode == 14: - c = self.atmosphere_hybrid_height_coordinate(axiscode) - else: - c = self.z_coordinate(axiscode) - - # Create a model_level_number auxiliary coordinate - LBLEV = int_hdr[lblev] - if LBVC in (2, 9, 65) or LBLEV in (7777, 8888): # CHECK! - self.LBLEV = LBLEV - c = self.model_level_number_coordinate(aux=bool(c)) - - # -------------------------------------------------------- - # Create the 'Y' dimension coordinate - # -------------------------------------------------------- - axiscode = iy - yc = None - if axiscode is not None: - if axiscode in (20, 23): - # 'Y' axis is time-since-reference-date - if extra.get("y", None) is not None: - c = self.time_coordinate_from_extra_data(axiscode, "y") - else: - LBUSER3 = int_hdr[lbuser3] - if LBUSER3 == LBROW: - self.lbuser3 = LBUSER3 - c = self.time_coordinate_from_um_timeseries( - axiscode, "y" - ) - else: - ykey, yc, yaxis = self.xy_coordinate(axiscode, "y") - if axiscode == 13: - _axis["site_axis"] = yaxis - self.site_coordinates_from_extra_data() - - # -------------------------------------------------------- - # Create the 'X' dimension coordinate - # -------------------------------------------------------- - axiscode = ix - xc = None - xkey = None - if axiscode is not None: - if axiscode in (20, 23): - # X axis is time since reference date - if extra.get("x", None) is not None: - c = self.time_coordinate_from_extra_data(axiscode, "x") - else: - LBUSER3 = int_hdr[lbuser3] - if LBUSER3 == LBNPT: - self.lbuser3 = LBUSER3 - c = self.time_coordinate_from_um_timeseries( - axiscode, "x" - ) - else: - xkey, xc, xaxis = self.xy_coordinate(axiscode, "x") - if axiscode == 13: - _axis["site_axis"] = xaxis - self.site_coordinates_from_extra_data() - - # -10: rotated latitude (not an official axis code) - # -11: rotated longitude (not an official axis code) - - if (iy, ix) == (-10, -11) or (iy, ix) == (-11, -10): - # ---------------------------------------------------- - # Create a ROTATED_LATITUDE_LONGITUDE coordinate - # reference - # ---------------------------------------------------- - ref = self.implementation.initialise_CoordinateReference() - - cc = self.implementation.initialise_CoordinateConversion( - parameters={ - "grid_mapping_name": "rotated_latitude_longitude", - "grid_north_pole_latitude": BPLAT, - "grid_north_pole_longitude": BPLON, - } - ) - - self.implementation.set_coordinate_conversion(ref, cc) - - self.implementation.set_coordinate_reference( - self.field, ref, copy=False - ) - - # ---------------------------------------------------- - # Create UNROTATED, 2-D LATITUDE and LONGITUDE - # auxiliary coordinates - # ---------------------------------------------------- - aux_keys = self.latitude_longitude_2d_aux_coordinates(yc, xc) - - self.implementation.set_coordinate_reference_coordinates( - ref, [ykey, xkey] + aux_keys - ) - - # -------------------------------------------------------- - # Create a RADIATION WAVELENGTH dimension coordinate - # -------------------------------------------------------- - try: - rwl, rwl_units = self.cf_info["below"] - except (KeyError, TypeError): - pass - else: - c = self.radiation_wavelength_coordinate(rwl, rwl_units) - - # Set LBUSER5 to zero so that it is not confused for a - # pseudolevel - LBUSER5 = 0 - - # -------------------------------------------------------- - # Create a PSEUDOLEVEL dimension coordinate. This must be - # done *after* the possible creation of a radiation - # wavelength dimension coordinate. - # -------------------------------------------------------- - if LBUSER5 != 0: - self.pseudolevel_coordinate(LBUSER5) - - attributes["int_hdr"] = int_hdr[:] - attributes["real_hdr"] = real_hdr[:] - attributes["file"] = filename - attributes["id"] = identity - - cf_properties["Conventions"] = __Conventions__ - cf_properties["runid"] = self.decode_lbexp() - cf_properties["lbproc"] = str(LBPROC) - cf_properties["lbtim"] = str(LBTIM) - cf_properties["stash_code"] = str(stash) - cf_properties["submodel"] = str(submodel) - - # Convert the UM version to a string and provide it as a - # CF property. E.g. 405 -> '4.5', 606.3 -> '6.6.3', 1002 - # -> '10.2' - # - # Note: We don't just do `divmod(self.um_version, 100)` - # because if self.um_version has a fractional part - # then it would likely get altered in the divmod - # calculation. - a, b = divmod(int(self.um_version), 100) - fraction = str(self.um_version).split(".")[-1] - um = f"{a}.{b}" - if fraction != "0" and fraction != str(self.um_version): - um += f".{fraction}" - - cf_properties["um_version"] = um - - # -------------------------------------------------------- - # Set the data and extra data - # -------------------------------------------------------- - data = self.create_data() - - # -------------------------------------------------------- - # Insert data into the field - # -------------------------------------------------------- - field = self.field - - self.implementation.set_data( - field, self.data, axes=self.data_axes, copy=False - ) - - # -------------------------------------------------------- - # Insert attributes and CF properties into the field - # -------------------------------------------------------- - fill_value = data.fill_value - if fill_value is not None: - cf_properties["_FillValue"] = data.fill_value - - # Add kwargs to the CF properties - cf_properties.update(kwargs) - - self.implementation.set_properties( - field, cf_properties, copy=False - ) - - field.id = identity - - if standard_name and not set_standard_name: - field._custom["standard_name"] = standard_name - - self.implementation.nc_set_variable(field, identity) - - # -------------------------------------------------------- - # Create and insert cell methods - # -------------------------------------------------------- - cell_methods = self.create_cell_methods() - for cm in cell_methods: - self.implementation.set_cell_method(field, cm) - - logger.info(f"down_axes = {self.down_axes}") # pragma: no cover - - # Force cyclic X axis for particular values of LBHEM - if xkey is not None and int_hdr[lbhem] in (0, 1, 2, 4): - field.cyclic( - xkey, - iscyclic=True, - config={ - "axis": xaxis, - "coord": xc, - "period": Data(360.0, xc.Units), - }, - ) - - self.fields.append(field) - - # ------------------------------------------------------------ - # Squeeze/unsqueeze size 1 axes in field constructs - # ------------------------------------------------------------ - if unsqueeze: - for f in self.fields: - f.unsqueeze(inplace=True) - elif squeeze: - for f in self.fields: - f.squeeze(inplace=True) - - self._bool = True - - def __bool__(self): - """x.__bool__() <==> bool(x)""" - return self._bool - - def __repr__(self): - """x.__repr__() <==> repr(x)""" - return self.fdr() - - def __str__(self): - """x.__str__() <==> str(x)""" - out = [self.fdr()] - - attrs = ( - "endian", - "reftime", - "vtime", - "dtime", - "um_version", - "source", - "it", - "iz", - "ix", - "iy", - "site_time_cross_section", - "timeseries", - "file", - ) - - for attr in attrs: - out.append(f"{attr}={getattr(self, attr, None)}") - - out.append("") - - return "\n".join(out) - - def _reorder_z_axis(self, indices, z_axis, pmaxes): - """Reorder the Z axis `Rec` instances. - - :Parameters: - - indices: `list` - Aggregation axis indices. See `create_data` for - details. - - z_axis: `int` - The identifier of the Z axis. - - pmaxes: sequence of `int` - The aggregation axes, which include the Z axis. - - :Returns: - - `list` - - **Examples** - - >>> _reorder_z_axis([(0, ), (1, )], 0, [0]) - [(0, ), (1, )] - - >>> _reorder_z_axis( - ... [(0, 0, ), - ... (0, 1, ), - ... (1, 0, ), - ... (1, 1, )], - ... 1, [0, 1] - ... ) - [(0, 0, ), (0, 1, ), (1, 0, ), (1, 1, )] - - """ - indices_new = [] - zpos = pmaxes.index(z_axis) - aaa0 = indices[0] - indices2 = [aaa0] - for aaa in indices[1:]: - if aaa[zpos] > aaa0[zpos]: - indices2.append(aaa) - else: - indices_new.extend(indices2[::-1]) - aaa0 = aaa - indices2 = [aaa0] - - indices_new.extend(indices2[::-1]) - - indices = [a[:-1] + b[-1:] for a, b in zip(indices, indices_new)] - return indices - - def atmosphere_hybrid_height_coordinate(self, axiscode): - """`atmosphere_hybrid_height_coordinate` when not an array axis. - - **From appendix A of UMDP F3** - - From UM Version 5.2, the method of defining the model levels in PP - headers was revised. At vn5.0 and 5.1, eta values were used in the - PP headers to specify the levels of model data, which was of - limited use when plotting data on model levels. From 5.2, the PP - headers were redefined to give information on the height of the - level. Given a 2D orography field, the height field for a given - level can then be derived. The height coordinates for PP-output - are defined as: - - Z(i,j,k)=Zsea(k)+C(k)*orography(i,j) - - where Zsea(k) and C(k) are height based hybrid coefficients. - - Zsea(k) = eta_value(k)*Height_at_top_of_model - - C(k)=[1-eta_value(k)/eta_value(first_constant_rho_level)]**2 for - levels less than or equal to first_constant_rho_level - C(k)=0.0 for levels greater than first_constant_rho_level - - where eta_value(k) is the eta_value for theta or rho level k. The - eta_value is a terrain-following height coordinate; full details - are given in UMDP15, Appendix B. - - The PP headers store Zsea and C as follows :- - - * 46 = bulev = brsvd1 = Zsea of upper layer boundary - * 47 = bhulev = brsvd2 = C of upper layer boundary - * 52 = blev = Zsea of level - * 53 = brlev = Zsea of lower layer boundary - * 54 = bhlev = C of level - * 55 = bhrlev = C of lower layer boundary - - :Parameters: - - axiscode: `int` - - :Returns: - - `DimensionCoordinate` or `None` - - """ - field = self.field - - # "a" domain ancillary - array = np.array( - [rec.real_hdr[blev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, # Zsea - ) - bounds0 = np.array( - [rec.real_hdr[brlev] for rec in self.z_recs], # Zsea lower - dtype=self.real_hdr_dtype, - ) - bounds1 = np.array( - [rec.real_hdr[brsvd1] for rec in self.z_recs], # Zsea upper - dtype=self.real_hdr_dtype, - ) - bounds = self.create_bounds_array(bounds0, bounds1) - - # Insert new Z axis - da = self.implementation.initialise_DomainAxis(size=array.size) - axis_key = self.implementation.set_domain_axis( - self.field, da, copy=False - ) - _axis["z"] = axis_key - - ac = self.implementation.initialise_DomainAncillary() - ac = self.coord_data(ac, array, bounds, units=_Units["m"]) - ac.id = "UM_atmosphere_hybrid_height_coordinate_a" - self.implementation.set_properties( - ac, {"long_name": "height based hybrid coeffient a"}, copy=False - ) - key_a = self.implementation.set_domain_ancillary( - field, ac, axes=[_axis["z"]], copy=False - ) - - # Height at top of atmosphere - toa_height = self.height_at_top_of_model - if toa_height is None: - pseudolevels = any( - [ - rec.int_hdr.item( - lbuser5, - ) - for rec in self.z_recs - ] - ) - if pseudolevels: - # Pseudolevels and atmosphere hybrid height - # coordinates are both present => can't reliably infer - # height. This is due to a current limitation in the C - # library that means it can only create Z-T - # aggregations, rather than the required Z-T-P - # aggregations. - toa_height = -1 - - if toa_height is None: - toa_height = bounds1.max() - if toa_height <= 0: - toa_height = None - elif toa_height <= 0: - toa_height = None - else: - toa_height = float(toa_height) - - # atmosphere_hybrid_height_coordinate dimension coordinate - if toa_height is None: - dc = None - else: - array = array / toa_height - bounds = bounds / toa_height - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, bounds, units=_Units["1"]) - self.implementation.set_properties( - dc, - {"standard_name": "atmosphere_hybrid_height_coordinate"}, - copy=False, - ) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_positive(dc, axiscode, _axis["z"]) - key_dc = self.implementation.set_dimension_coordinate( - field, - dc, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - # "b" domain ancillary - array = np.array( - [rec.real_hdr[bhlev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds0 = np.array( - [rec.real_hdr[bhrlev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds1 = np.array( - [rec.real_hdr[brsvd2] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds = self.create_bounds_array(bounds0, bounds1) - - ac = self.implementation.initialise_DomainAncillary() - ac = self.coord_data(ac, array, bounds, units=_Units["1"]) - ac.id = "UM_atmosphere_hybrid_height_coordinate_b" - self.implementation.set_properties( - ac, {"long_name": "height based hybrid coeffient b"}, copy=False - ) - key_b = self.implementation.set_domain_ancillary( - field, ac, axes=[_axis["z"]], copy=False - ) - - # atmosphere_hybrid_height_coordinate coordinate reference - ref = self.implementation.initialise_CoordinateReference() - cc = self.implementation.initialise_CoordinateConversion( - parameters={ - "standard_name": "atmosphere_hybrid_height_coordinate" - }, - domain_ancillaries={"a": key_a, "b": key_b, "orog": None}, - ) - self.implementation.set_coordinate_conversion(ref, cc) - if dc is not None: - self.implementation.set_coordinate_reference_coordinates( - ref, (key_dc,) - ) - - self.implementation.set_coordinate_reference(field, ref, copy=False) - - return dc - - def depth_coordinate(self, axiscode): - """`atmosphere_hybrid_height_coordinate_*k` depth coordinate. - - Only applicable when not an array axis. - - :Parameters: - - axiscode: `int` - - :Returns: - - `DimensionCoordinate` or `None` - - """ - dc = self.model_level_number_coordinate(aux=False) - - field = self.field - - array = np.array( - [rec.real_hdr[blev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds0 = np.array( - [rec.real_hdr[brlev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds1 = np.array( - [rec.real_hdr[brsvd1] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds = self.create_bounds_array(bounds0, bounds1) - - # Create Z domain axis construct - da = self.implementation.initialise_DomainAxis(size=array.size) - axisZ = self.implementation.set_domain_axis(field, da, copy=False) - _axis["z"] = axisZ - - # ac = AuxiliaryCoordinate() - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, array, bounds, units=_Units["m"]) - ac.id = "UM_atmosphere_hybrid_height_coordinate_ak" - ac.long_name = "atmosphere_hybrid_height_coordinate_ak" - # field.insert_aux(ac, axes=[zdim], copy=False) - self.implementation.set_auxiliary_coordinate( - field, - ac, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - array = np.array( - [rec.real_hdr[bhlev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds0 = np.array( - [rec.real_hdr[bhrlev] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds1 = np.array( - [rec.real_hdr[brsvd2] for rec in self.z_recs], - dtype=self.real_hdr_dtype, - ) - bounds = self.create_bounds_array(bounds0, bounds1) - - # ac = AuxiliaryCoordinate() - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, array, bounds, units=_Units["1"]) - ac.id = "UM_atmosphere_hybrid_height_coordinate_bk" - ac.long_name = "atmosphere_hybrid_height_coordinate_bk" - self.implementation.set_auxiliary_coordinate( - field, - ac, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - return dc - - def atmosphere_hybrid_sigma_pressure_coordinate(self, axiscode): - """`atmosphere_hybrid_sigma_pressure_coordinate` - - Only applicable when not an array axis. - - 46 BULEV Upper layer boundary or BRSVD(1) - - 47 BHULEV Upper layer boundary or BRSVD(2) - - For hybrid levels: - - BULEV is B-value at half-level above. - - BHULEV is A-value at half-level above. - - For hybrid height levels (vn5.2-, Smooth heights) - - BULEV is Zsea of upper layer boundary - * If rho level: Zsea for theta level above - * If theta level: Zsea for rho level above - - BHLEV is C of upper layer boundary - * If rho level: C for theta level above - * If theta level: C for rho level above - - :Parameters: - - axiscode: `int` - - :Returns: - - `DimensionCoordinate` - - """ - array = [] - bounds = [] - ak_array = [] - ak_bounds = [] - bk_array = [] - bk_bounds = [] - - for rec in self.z_recs: - BLEV, BRLEV, BHLEV, BHRLEV, BULEV, BHULEV = self.header_bz(rec) - - array.append(BLEV + BHLEV / _pstar) - bounds.append([BRLEV + BHRLEV / _pstar, BULEV + BHULEV / _pstar]) - - ak_array.append(BHLEV) - ak_bounds.append((BHRLEV, BHULEV)) - - bk_array.append(BLEV) - bk_bounds.append((BRLEV, BULEV)) - - array = np.array(array, dtype=float) - bounds = np.array(bounds, dtype=float) - ak_array = np.array(ak_array, dtype=float) - ak_bounds = np.array(ak_bounds, dtype=float) - bk_array = np.array(bk_array, dtype=float) - bk_bounds = np.array(bk_bounds, dtype=float) - - field = self.field - - # Insert new Z axis - da = self.implementation.initialise_DomainAxis(size=array.size) - axis_key = self.implementation.set_domain_axis(field, da, copy=False) - _axis["z"] = axis_key - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data( - dc, - array, - bounds, - units=_axiscode_to_Units.setdefault(axiscode, None), - ) - dc = self.coord_positive(dc, axiscode, _axis["z"]) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - self.implementation.set_dimension_coordinate( - field, - dc, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, ak_array, ak_bounds, units=_Units["Pa"]) - ac.id = "UM_atmosphere_hybrid_sigma_pressure_coordinate_ak" - ac.long_name = "atmosphere_hybrid_sigma_pressure_coordinate_ak" - - self.implementation.set_auxiliary_coordinate( - field, - ac, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, bk_array, bk_bounds, units=_Units["1"]) - - self.implementation.set_auxiliary_coordinate( - field, - ac, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - ac.id = "UM_atmosphere_hybrid_sigma_pressure_coordinate_bk" - ac.long_name = "atmosphere_hybrid_sigma_pressure_coordinate_bk" - - return dc - - def create_bounds_array(self, bounds0, bounds1): - """Stack two 1-d arrays to create a bounds array. - - The returned array will have a trailing dimension of size 2. - - The leading dimension size and data type are taken from - *bounds0*. - - :Parameters: - - bounds0: `numpy.ndarray` - The bounds which are to occupy ``[:, 0]`` in the - returned bounds array. - - bounds1: `numpy.ndarray` - The bounds which are to occupy ``[:, 1]`` in the - returned bounds array. - - :Returns: - - `numpy.ndarray` - - """ - bounds = np.empty((bounds0.size, 2), dtype=bounds0.dtype) - bounds[:, 0] = bounds0 - bounds[:, 1] = bounds1 - return bounds - - def create_cell_methods(self): - """Create the cell methods. - - **UMDP F3** - - LBPROC Processing code. This indicates what processing has - been done to the basic field. It should be 0 if no processing - has been done, otherwise add together the relevant numbers - from the list below: - - 1 Difference from another experiment. - 2 Difference from zonal (or other spatial) mean. - 4 Difference from time mean. - 8 X-derivative (d/dx) - 16 Y-derivative (d/dy) - 32 Time derivative (d/dt) - 64 Zonal mean field - 128 Time mean field - 256 Product of two fields - 512 Square root of a field - 1024 Difference between fields at levels BLEV and BRLEV - 2048 Mean over layer between levels BLEV and BRLEV - 4096 Minimum value of field during time period - 8192 Maximum value of field during time period - 16384 Magnitude of a vector, not specifically wind speed - 32768 Log10 of a field - 65536 Variance of a field - 131072 Mean over an ensemble of parallel runs - - :Returns: - - `list` of `str` - The cell methods. - - """ - cell_methods = [] - - LBPROC = self.lbproc - LBTIM_IB = self.lbtim_ib - tmean_proc = 0 - - # ------------------------------------------------------------ - # Ensemble mean cell method - # ------------------------------------------------------------ - if 131072 <= LBPROC < 262144: - cell_methods.append("realization: mean") - LBPROC -= 131072 - - if LBTIM_IB in (2, 3) and LBPROC in (128, 192, 2176, 4224, 8320): - tmean_proc = 128 - LBPROC -= 128 - - # ------------------------------------------------------------ - # Area cell methods - # ------------------------------------------------------------ - # -10: rotated latitude (not an official axis code) - # -11: rotated longitude (not an official axis code) - if self.ix in (10, 11, 12, -10, -11) and self.iy in ( - 10, - 11, - 12, - -10, - -11, - ): - cf_info = self.cf_info - - if "where" in cf_info: - cell_methods.append("area: mean") - - cell_methods.append(cf_info["where"]) - if "over" in cf_info: - cell_methods.append(cf_info["over"]) - - if LBPROC == 64: - cell_methods.append("x: mean") - - # dch : do special zonal mean as as in pp_cfwrite - - # ------------------------------------------------------------ - # Vertical cell methods - # ------------------------------------------------------------ - if LBPROC == 2048: - cell_methods.append("z: mean") - - # ------------------------------------------------------------ - # Time cell methods - # ------------------------------------------------------------ - if "t" in _axis: - axis = "t" - else: - axis = "time" - - if LBTIM_IB == 0 or LBTIM_IB == 1: - if axis == "t": - cell_methods.append(axis + ": point") - elif LBPROC == 4096: - cell_methods.append(axis + ": minimum") - elif LBPROC == 8192: - cell_methods.append(axis + ": maximum") - if tmean_proc == 128: - if LBTIM_IB == 2: - cell_methods.append(axis + ": mean") - elif LBTIM_IB == 3: - cell_methods.append(axis + ": mean within years") - cell_methods.append(axis + ": mean over years") - - if not cell_methods: - return [] - - cell_methods = self.implementation.initialise_CellMethod().create( - " ".join(cell_methods) - ) - - for cm in cell_methods: - cm.change_axes(_axis, inplace=True) - - return cell_methods - - def coord_axis(self, c, axiscode): - """Map axis codes to CF axis attributes for the coordinate.""" - axis = _coord_axis.setdefault(axiscode, None) - if axis is not None: - c.axis = axis - - return c - - def coord_data( - self, - c, - array=None, - bounds=None, - units=None, - fill_value=None, - climatology=False, - ): - """Set the data array of a coordinate construct. - - :Parameters: - - c: Coordinate construct - - data: array-like, optional - The data array. - - bounds: array-like, optional - The Cell bounds for the data array. - - units: `Units`, optional - The units of the data array. - - fill_value: optional - - climatology: `bool`, optional - Whether or not the coordinate construct is a time - climatology. By default it is not. - - :Returns: - - Coordinate construct - - """ - if array is not None: - data = Data(array, units, fill_value=fill_value) - self.implementation.set_data(c, data, copy=False) - - if bounds is not None: - data = Data(bounds, units, fill_value=fill_value) - bounds = self.implementation.initialise_Bounds() - self.implementation.set_data(bounds, data, copy=False) - self.implementation.set_bounds(c, bounds, copy=False) - - return c - - def coord_names(self, coord, axiscode): - """Map axis codes to CF standard names for the coordinate. - - :Parameters: - - coord: Coordinate construct - - axiscode: `int` - - :Returns: - - Coordinate construct - - """ - standard_name = _coord_standard_name.setdefault(axiscode, None) - - if standard_name is not None: - coord.set_property("standard_name", standard_name, copy=False) - coord.ncvar = standard_name - else: - long_name = _coord_long_name.setdefault(axiscode, None) - if long_name is not None: - coord.set_property("long_name", long_name, copy=False) - - return coord - - def coord_positive(self, c, axiscode, domain_axis_key): - """Map axis codes to CF positive attributes for the coordinate. - - :Parameters: - - c: `Coordinate` - A 1-d coordinate construct - - axiscode: `int` - - domain_axis_key: `str` - - :Returns: - - Coordinate construct - - """ - positive = _coord_positive.setdefault(axiscode, None) - if positive is not None: - c.positive = positive - if positive == "down" and axiscode != 4: - self.down_axes.add(domain_axis_key) - c.flip(inplace=True) - - return c - - def ctime(self, rec): - """Return elapsed time since the clock time of the given - record.""" - import cftime - - reftime = self.refUnits - LBVTIME = tuple(self.header_vtime(rec)) - LBDTIME = tuple(self.header_dtime(rec)) - - key = (LBVTIME, LBDTIME, self.refunits, self.calendar) - ctime = _cached_ctime.get(key, None) - if ctime is None: - LBDTIME = list(LBDTIME) - LBDTIME[0] = LBVTIME[0] - - ctime = cftime.datetime(*LBDTIME, calendar=self.calendar) - - if ctime < cftime.datetime(*LBVTIME, calendar=self.calendar): - LBDTIME[0] += 1 - ctime = cftime.datetime(*LBDTIME, calendar=self.calendar) - - ctime = Data(ctime, reftime).array.item() - _cached_ctime[key] = ctime - - return ctime - - def header_vtime(self, rec): - """Return the list [LBYR, LBMON, LBDAT, LBHR, LBMIN] for the - given record. - - :Parameters: - - rec: - - :Returns: - - `list` - - **Examples** - - >>> u.header_vtime(rec) - [1991, 1, 1, 0, 0] - - """ - return rec.int_hdr[lbyr : lbmin + 1] - - def header_dtime(self, rec): - """Return the list [LBYRD, LBMOND, LBDATD, LBHRD, LBMIND] for - the given record. - - :Parameters: - - rec: - - :Returns: - - `list` - - **Examples** - - >>> u.header_dtime(rec) - [1991, 2, 1, 0, 0] - - """ - return rec.int_hdr[lbyrd : lbmind + 1] - - def header_bz(self, rec): - """Return the list [BLEV, BRLEV, BHLEV, BHRLEV, BULEV, BHULEV] - for the given record. - - :Parameters: - - rec: - - :Returns: - - `list` - - **Examples** - - >>> u.header_bz(rec) - - """ - real_hdr = rec.real_hdr - return ( - real_hdr[blev : bhrlev + 1].tolist() - + real_hdr[ # BLEV, BRLEV, BHLEV, BHRLEV - brsvd1 : brsvd2 + 1 - ].tolist() # BULEV, BHULEV - ) - - def header_lz(self, rec): - """Return the list [LBLEV, LBUSER5] for the given record. - - :Parameters: - - rec: - - :Returns: - - `list` - - **Examples** - - >>> u.header_lz(rec) - - """ - int_hdr = rec.int_hdr - return [int_hdr.item(lblev), int_hdr.item(lbuser5)] - - def header_z(self, rec): - """Return the list [LBLEV, LBUSER5, BLEV, BRLEV, BHLEV, BHRLEV, - BULEV, BHULEV] for the given record. - - These header items are used by the compare_levels function in - compare.c - - :Parameters: - - rec: - - :Returns: - - `list` - - **Examples** - - >>> u.header_z(rec) - - """ - return self.header_lz + self.header_bz - - @_manage_log_level_via_verbose_attr - def create_data(self): - """Sets the data and data axes. - - :Returns: - - `Data` - - """ - import dask.array as da - from dask.array.core import getter, normalize_chunks - from dask.base import tokenize - - if self.info: - logger.info("Creating data:") # pragma: no cover - - LBROW = self.lbrow - LBNPT = self.lbnpt - - yx_shape = (LBROW, LBNPT) - - nz = self.nz - nt = self.nt - recs = self.recs - - um_Units = self.um_Units - attributes = { - "units": getattr(um_Units, "units", None), - "calendar": getattr(um_Units, "calendar", None), - } - - data_type_in_file = self.data_type_in_file - - data_axes = [_axis["y"], _axis["x"]] - - # Initialise a dask graph for the uncompressed array, and some - # dask.array.core.getter arguments - token = tokenize((nt, nz) + yx_shape, uuid4()) - name = (UMArray().__class__.__name__ + "-" + token,) - dsk = {} - full_slice = Ellipsis - klass_name = UMArray().__class__.__name__ - - umarray_kwargs = { - "filename": self.filename, - "fmt": self.fmt, - "word_size": self.word_size, - "byte_ordering": self.byte_ordering, - "attributes": attributes, - "unpack": self.unpack, - "storage_protocol": self.storage_protocol, - "storage_options": self.storage_options, - } - - if len(recs) == 1: - # -------------------------------------------------------- - # 0-d partition matrix - # -------------------------------------------------------- - pmaxes = [] - file_data_types = set() - - rec = recs[0] - - fill_value = rec.real_hdr.item(bmdi) - if fill_value == _BMDI_no_missing_data_value: - fill_value = None - - data_shape = yx_shape - - subarray = UMArray( - address=rec.hdr_offset, - shape=yx_shape, - dtype=data_type_in_file(rec), - **umarray_kwargs, - ) - - key = f"{klass_name}-{tokenize(subarray)}" - dsk[key] = subarray - dsk[name + (0, 0)] = (getter, key, full_slice, False, False) - - dtype = data_type_in_file(rec) - chunks = normalize_chunks((-1, -1), shape=data_shape, dtype=dtype) - else: - # -------------------------------------------------------- - # 1-d or 2-d partition matrix - # -------------------------------------------------------- - file_data_types = set() - - # Find the partition matrix shape - pmshape = [n for n in (nt, nz) if n > 1] - - if len(pmshape) == 1: - # ---------------------------------------------------- - # 1-d partition matrix - # ---------------------------------------------------- - z_axis = _axis.get(self.z_axis) - if nz > 1: - pmaxes = [z_axis] - data_shape = (nz, LBROW, LBNPT) - else: - pmaxes = [_axis["t"]] - data_shape = (nt, LBROW, LBNPT) - - indices = [(i, rec) for i, rec in enumerate(recs)] - - if nz > 1 and z_axis in self.down_axes: - indices = self._reorder_z_axis(indices, z_axis, pmaxes) - - for i, rec in indices: - # Find the data type of the array in the file - file_data_type = data_type_in_file(rec) - file_data_types.add(file_data_type) - - shape = (1,) + yx_shape - - subarray = UMArray( - address=rec.hdr_offset, - shape=shape, - dtype=file_data_type, - **umarray_kwargs, - ) - - key = f"{klass_name}-{tokenize(subarray)}" - dsk[key] = subarray - dsk[name + (i, 0, 0)] = ( - getter, - key, - full_slice, - False, - False, - ) - - dtype = np.result_type(*file_data_types) - chunks = normalize_chunks( - (1, -1, -1), shape=data_shape, dtype=dtype - ) - else: - # ---------------------------------------------------- - # 2-d partition matrix - # ---------------------------------------------------- - z_axis = _axis[self.z_axis] - pmaxes = [_axis["t"], z_axis] - - data_shape = (nt, nz, LBROW, LBNPT) - - indices = [ - divmod(i, nz) + (rec,) for i, rec in enumerate(recs) - ] - if z_axis in self.down_axes: - indices = self._reorder_z_axis(indices, z_axis, pmaxes) - - for t, z, rec in indices: - # Find the data type of the array in the file - file_data_type = data_type_in_file(rec) - file_data_types.add(file_data_type) - - shape = (1, 1) + yx_shape - - subarray = UMArray( - address=rec.hdr_offset, - shape=shape, - dtype=file_data_type, - **umarray_kwargs, - ) - - key = f"{klass_name}-{tokenize(subarray)}" - dsk[key] = subarray - dsk[name + (t, z, 0, 0)] = ( - getter, - key, - full_slice, - False, - False, - ) - - dtype = np.result_type(*file_data_types) - chunks = normalize_chunks( - (1, 1, -1, -1), shape=data_shape, dtype=dtype - ) - - data_axes = pmaxes + data_axes - - # Set the data array - fill_value = recs[0].real_hdr.item(bmdi) - if fill_value == _BMDI_no_missing_data_value: - fill_value = None - - # Create the dask array - dx = da.Array(dsk, name[0], chunks=chunks, dtype=dtype) - - # Create the Data object - data = Data(dx, units=um_Units, fill_value=fill_value) - data._nc_set_aggregation_write_status(True) - - self.data = data - self.data_axes = data_axes - - return data - - def decode_lbexp(self): - """Decode the integer value of LBEXP in the PP header into a - runid. - - If this value has already been decoded, then it will be returned - from the cache, otherwise the value will be decoded and then added - to the cache. - - :Returns: - - `str` - A string derived from LBEXP. If LBEXP is a negative integer - then that number is returned as a string. - - **Examples** - - >>> self.decode_lbexp() - 'aaa5u' - >>> self.decode_lbexp() - '-34' - - """ - LBEXP = self.int_hdr[lbexp] - - runid = _cached_runid.get(LBEXP, None) - if runid is not None: - # Return a cached decoding of this LBEXP - return runid - - if LBEXP < 0: - runid = str(LBEXP) - else: - # Convert LBEXP to a binary string, filled out to 30 bits with - # zeros - bits = bin(LBEXP) - bits = bits.lstrip("0b").zfill(30) - - # Step through 6 bits at a time, converting each 6 bit chunk into - # a decimal integer, which is used as an index to the characters - # lookup list. - runid = [] - for i in range(0, 30, 6): - index = int(bits[i : i + 6], 2) - if index < _n_characters: - runid.append(_characters[index]) - - runid = "".join(runid) - - # Enter this runid into the cache - _cached_runid[LBEXP] = runid - - # Return the runid - return runid - - def dtime(self, rec): - """Return the elapsed time since the data time of the given - record. - - :Parameters: - - rec: - - :Returns: - - `float` - - **Examples** - - >>> u.dtime(rec) - 31.5 - - """ - units = self.refunits - calendar = self.calendar - - LBDTIME = tuple(self.header_dtime(rec)) - - key = (LBDTIME, units, calendar) - time = _cached_date2num.get(key, None) - if time is None: - from netCDF4 import date2num as netCDF4_date2num - - # It is important to use the same time_units as vtime - try: - if self.calendar == "gregorian": - time = netCDF4_date2num( - datetime(*LBDTIME), units, calendar - ) - else: - import cftime - - time = netCDF4_date2num( - cftime.datetime(*LBDTIME, calendar=self.calendar), - units, - calendar, - ) - - _cached_date2num[key] = time - except ValueError: - time = np.nan # ppp - - return time - - def fdr(self): - """Return a the contents of PP field headers as strings. - - This is a bit like printfdr in the UKMO IDL PP library. - - :Returns: - - `list` - - """ - out2 = [] - for i, rec in enumerate(self.recs): - out = [f"Field {i}:"] - - x = [ - f"{name}::{value}" - for name, value in zip( - _header_names, self.int_hdr + self.real_hdr - ) - ] - - x = textwrap.fill(" ".join(x), width=79) - out.append(x.replace("::", ": ")) - - if self.extra: - out.append("EXTRA DATA:") - for key in sorted(self.extra): - out.append(f"{key}: {str(self.extra[key])}") - - out.append("file: " + self.filename) - out.append( - f"fmt, byte order, word size: {self.fmt}, " - f"{self.byte_ordering}, {self.word_size}" - ) - - out.append("") - - out2.append("\n".join(out)) - - return out2 - - def latitude_longitude_2d_aux_coordinates(self, yc, xc): - """Set the latitude and longitude auxiliary coordinates. - - :Parameters: - - yc: `DimensionCoordinate` - - xc: `DimensionCoordinate` - - :Returns: - - `list` - The keys of the auxiliary coordinates. - - """ - BDX = self.bdx - BDY = self.bdy - LBNPT = self.lbnpt - LBROW = self.lbrow - BPLAT = self.bplat - BPLON = self.bplon - - # Create the unrotated latitude and longitude arrays if we - # couldn't find them in the cache - cache_key = (LBNPT, LBROW, BDX, BDY, BPLAT, BPLON) - lat, lon = _cached_latlon.get(cache_key, (None, None)) - - if lat is None: - lat, lon = self.unrotated_latlon(yc.array, xc.array, BPLAT, BPLON) - - atol = self.atol - if abs(BDX) >= atol and abs(BDY) >= atol: - _cached_latlon[cache_key] = (lat, lon) - - if xc.has_bounds() and yc.has_bounds(): # TODO push to implementation - cache_key = ("bounds",) + cache_key - lat_bounds, lon_bounds = _cached_latlon.get( - cache_key, (None, None) - ) - if lat_bounds is None: - xb = np.empty(xc.size + 1) - xb[:-1] = xc.bounds.subspace[:, 0].squeeze(1).array - xb[-1] = xc.bounds.datum(-1, 1) - - yb = np.empty(yc.size + 1) - yb[:-1] = yc.bounds.subspace[:, 0].squeeze(1).array - yb[-1] = yc.bounds.datum(-1, 1) - - temp_lat_bounds, temp_lon_bounds = self.unrotated_latlon( - yb, xb, BPLAT, BPLON - ) - - lat_bounds = np.empty(lat.shape + (4,)) - lon_bounds = np.empty(lon.shape + (4,)) - - lat_bounds[..., 0] = temp_lat_bounds[0:-1, 0:-1] - lon_bounds[..., 0] = temp_lon_bounds[0:-1, 0:-1] - - lat_bounds[..., 1] = temp_lat_bounds[1:, 0:-1] - lon_bounds[..., 1] = temp_lon_bounds[1:, 0:-1] - - lat_bounds[..., 2] = temp_lat_bounds[1:, 1:] - lon_bounds[..., 2] = temp_lon_bounds[1:, 1:] - - lat_bounds[..., 3] = temp_lat_bounds[0:-1, 1:] - lon_bounds[..., 3] = temp_lon_bounds[0:-1, 1:] - - atol = self.atol - if abs(BDX) >= atol and abs(BDY) >= atol: - _cached_latlon[cache_key] = (lat_bounds, lon_bounds) - else: - lat_bounds = None - lon_bounds = None - - axes = [_axis["y"], _axis["x"]] - - keys = [] - for axiscode, array, bounds in zip( - (10, 11), (lat, lon), (lat_bounds, lon_bounds) - ): - # ac = AuxiliaryCoordinate() - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data( - ac, - array, - bounds=bounds, - units=_axiscode_to_Units.setdefault(axiscode, None), - ) - ac = self.coord_names(ac, axiscode) - - key = self.implementation.set_auxiliary_coordinate( - self.field, ac, axes=axes, copy=False - ) - keys.append(key) - - return keys - - def model_level_number_coordinate(self, aux=False): - """model_level_number dimension or auxiliary coordinate. - - :Parameters: - - aux: `bool` - - :Returns: - - out : `AuxiliaryCoordinate` or `DimensionCoordinate` or `None` - - """ - array = tuple([rec.int_hdr.item(lblev) for rec in self.z_recs]) - - key = array - c = _cached_model_level_number_coordinate.get(key, None) - - if c is not None: - if aux: - self.field.insert_aux(c, axes=[_axis["z"]], copy=True) - self.implementation.set_auxiliary_coordinate( - self.field, - c, - axes=[_axis["z"]], - copy=True, - autocyclic=_autocyclic_false, - ) - else: - self.implementation.set_dimension_coordinate( - self.field, - c, - axes=[_axis["z"]], - copy=True, - autocyclic=_autocyclic_false, - ) - else: - array = np.array(array, dtype=self.int_hdr_dtype) - - if array.min() < 0: - return - - array = np.where(array == 9999, 0, array) - - axiscode = 5 - - if aux: - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, array, units=Units("1")) - ac = self.coord_names(ac, axiscode) - self.implementation.set_auxiliary_coordinate( - self.field, - ac, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - else: - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, units=Units("1")) - dc = self.coord_names(dc, axiscode) - dc = self.coord_axis(dc, axiscode) - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - _cached_model_level_number_coordinate[key] = c - - return c - - def data_type_in_file(self, rec): - """Return the data type of the data array. - - :Parameters: - - rec: `umfile.Rec` - - :Returns: - - `numpy.dtype` - - """ - # Find the data type - if rec.int_hdr.item(lbuser2) == 3: - # Boolean - return np.dtype(bool) - - # Int or float - return rec.get_type_and_num_words()[0] - - def printfdr(self, display=False): - """Print out the contents of PP field headers. - - This is a bit like printfdr in the UKMO IDL PP library. - - **Examples** - - >>> u.printfdr() - - """ - if display: - for header in self.fdr(): - print(header) - else: - for header in self.fdr(): - logger.info(header) - - def pseudolevel_coordinate(self, LBUSER5): - """Create and return the pseudolevel coordinate.""" - if self.nz == 1: - array = np.array((LBUSER5,), dtype=self.int_hdr_dtype) - else: - # 'Z' aggregation has been done along the pseudolevel axis - array = np.array( - [rec.int_hdr.item(lbuser5) for rec in self.z_recs], - dtype=self.int_hdr_dtype, - ) - self.z_axis = "p" - - axiscode = 40 - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data( - dc, array, units=_axiscode_to_Units.setdefault(axiscode, None) - ) - self.implementation.set_properties( - dc, {"long_name": "pseudolevel"}, copy=False - ) - dc.id = "UM_pseudolevel" - - da = self.implementation.initialise_DomainAxis(size=array.size) - axisP = self.implementation.set_domain_axis(self.field, da, copy=False) - _axis["p"] = axisP - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["p"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - return dc - - def radiation_wavelength_coordinate(self, rwl, rwl_units): - """Creata and return the radiation wavelength coordinate.""" - array = np.array((rwl,), dtype=float) - bounds = np.array(((0.0, rwl)), dtype=float) - - units = _Units.get(rwl_units, None) - if units is None: - units = Units(rwl_units) - _Units[rwl_units] = units - - axiscode = -20 - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, bounds, units=units) - dc = self.coords_names(dc, axiscode) - - da = self.implementation.initialise_DomainAxis(size=array.size) - axisR = self.implementation.set_domain_axis(self.field, da) - _axis["r"] = axisR - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["r"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - return dc - - def reference_time_Units(self): - """Return the units of the `reference_time`.""" - LBYR = self.int_hdr[lbyr] - time_units = f"days since {LBYR}-1-1" - calendar = self.calendar - - key = time_units + " calendar=" + calendar - units = _Units.get(key, None) - if units is None: - units = Units(time_units, calendar) - _Units[key] = units - - self.refUnits = units - self.refunits = time_units - - return units - - def size_1_height_coordinate(self, axiscode, height, units): - """Create and return the size-one height coordinate.""" - # Create the height coordinate from the information given in the - # STASH to standard_name conversion table - - key = (axiscode, height, units) - dc = _cached_size_1_height_coordinate.get(key, None) - - da = self.implementation.initialise_DomainAxis(size=1) - axisZ = self.implementation.set_domain_axis(self.field, da, copy=False) - _axis["z"] = axisZ - - if dc is not None: - copy = True - else: - height_units = _Units.get(units, None) - if height_units is None: - height_units = Units(units) - _Units[units] = height_units - - array = np.array((height,), dtype=float) - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, units=height_units) - dc = self.coord_positive(dc, axiscode, _axis["z"]) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - _cached_size_1_height_coordinate[key] = dc - copy = False - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["z"]], - copy=copy, - autocyclic=_autocyclic_false, - ) - return dc - - def test_um_condition(self, um_condition, LBCODE, BPLAT, BPLON): - """Return `True` if a field satisfies the condition specified - for a STASH code to standard name conversion. - - :Parameters: - - um_condition: `str` - - LBCODE: `int` - - BPLAT: `float` - - BPLON: `float` - - :Returns: - - `bool` - `True` if a field satisfies the condition specified, - `False` otherwise. - - **Examples** - - >>> ok = u.test_um_condition('true_latitude_longitude', ...) - - """ - if um_condition == "true_latitude_longitude": - if LBCODE in _true_latitude_longitude_lbcodes: - return True - - # Check pole location in case of incorrect LBCODE - atol = self.atol - if ( - abs(BPLAT - 90.0) <= atol + cf_rtol() * 90.0 - and abs(BPLON) <= atol - ): - return True - - elif um_condition == "rotated_latitude_longitude": - if LBCODE in _rotated_latitude_longitude_lbcodes: - return True - - # Check pole location in case of incorrect LBCODE - atol = self.atol - if not ( - abs(BPLAT - 90.0) <= atol + cf_rtol() * 90.0 - and abs(BPLON) <= atol - ): - return True - - else: - raise ValueError( - "Unknown UM condition in STASH code conversion table: " - f"{um_condition!r}" - ) - - # Still here? Then the condition has not been satisfied. - return - - def test_um_version(self, valid_from, valid_to, um_version): - """Return `True` if the UM version applicable to this field is - within the given range. - - If possible, the UM version is derived from the PP header and - stored in the metadata object. Otherwise it is taken from the - *um_version* parameter. - - :Parameters: - - valid_from: `int`, `float` or `None` - - valid_to: `int`, `float` or `None` - - um_version: `int` or `float` - - :Returns: - - `bool` - `True` if the UM version applicable to this field - construct is within the range, `False` otherwise. - - **Examples** - - >>> ok = u.test_um_version(401, 505, 1001) - >>> ok = u.test_um_version(401, None, 606.3) - >>> ok = u.test_um_version(None, 405, 401) - - """ - if valid_to is None: - if valid_from is None: - return True - - if valid_from <= um_version: - return True - elif valid_from is None: - if um_version <= valid_to: - return True - elif valid_from <= um_version <= valid_to: - return True - - return False - - def time_coordinate(self, axiscode): - """Return the T dimension coordinate. - - :Parameters: - - axiscode: `int` - - :Returns: - - `DimensionCoordinate` - - """ - recs = self.t_recs - - vtimes = np.array([self.vtime(rec) for rec in recs], dtype=float) - dtimes = np.array([self.dtime(rec) for rec in recs], dtype=float) - - if np.isnan(vtimes.sum()) or np.isnan(dtimes.sum()): - return # ppp - - IB = self.lbtim_ib - - if IB <= 1 or vtimes.item(0) >= dtimes.item(0): - array = vtimes - bounds = None - climatology = False - elif IB == 3: - # The field is a time mean from T1 to T2 for each year - # from LBYR to LBYRD - ctimes = np.array([self.ctime(rec) for rec in recs]) - array = 0.5 * (vtimes + ctimes) - bounds = self.create_bounds_array(vtimes, dtimes) - - climatology = True - else: - array = 0.5 * (vtimes + dtimes) - bounds = self.create_bounds_array(vtimes, dtimes) - - climatology = False - - da = self.implementation.initialise_DomainAxis(size=array.size) - axisT = self.implementation.set_domain_axis(self.field, da, copy=False) - _axis["t"] = axisT - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data( - dc, array, bounds, units=self.refUnits, climatology=climatology - ) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["t"]], - copy=False, - autocyclic=_autocyclic_false, - ) - return dc - - def time_coordinate_from_extra_data(self, axiscode, axis): - """Create the time coordinate from extra data and return it. - - :Returns: - - `DimensionCoordinate` - - """ - extra = self.extra - - array = extra[axis] - bounds = extra.get(axis + "_bounds", None) - - calendar = self.calendar - if calendar == "360_day": - units = _Units["360_day 0-1-1"] - elif calendar == "gregorian": - units = _Units["gregorian 1752-09-13"] - elif calendar == "365_day": - units = _Units["365_day 1752-09-13"] - else: - units = None - - # Create time domain axis. - # - # Note that `axis` might not be "t". For instance, it could be - # "y" if the time coordinates are coming from extra data. - da = self.implementation.initialise_DomainAxis(size=array.size) - axisT = self.implementation.set_domain_axis(self.field, da, copy=False) - _axis[axis] = axisT - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, bounds, units=units) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=(axisT,), - copy=False, - autocyclic=_autocyclic_false, - ) - - return dc - - def time_coordinate_from_um_timeseries(self, axiscode, axis): - """Create the time coordinate from a timeseries field.""" - # This PP/FF field is a timeseries. The validity time is - # taken to be the time for the first sample, the data time - # for the last sample, with the others evenly between. - rec = self.recs[0] - vtime = self.vtime(rec) - dtime = self.dtime(rec) - - size = self.lbuser3 - 1.0 - delta = (dtime - vtime) / size - - calendar = self.calendar - if calendar == "360_day": - units = _Units["360_day 0-1-1"] - elif calendar == "gregorian": - units = _Units["gregorian 1752-09-13"] - elif calendar == "365_day": - units = _Units["365_day 1752-09-13"] - else: - units = None - - array = np.arange(vtime, vtime + delta * size, size, dtype=float) - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, units=units) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis[axis]], - copy=False, - autocyclic=_autocyclic_false, - ) - return dc - - def vtime(self, rec): - """Return the elapsed time since the validity time of the given - record. - - :Parameters: - - rec: - - :Returns: - - `float` - - **Examples** - - >>> u.vtime(rec) - 31.5 - - """ - units = self.refunits - calendar = self.calendar - - LBVTIME = tuple(self.header_vtime(rec)) - - key = (LBVTIME, units, calendar) - - time = _cached_date2num.get(key, None) - if time is None: - import cftime - - # It is important to use the same time_units as dtime - try: - time = cftime.date2num( - cftime.datetime(*LBVTIME, calendar=self.calendar), - units, - calendar, - ) - - _cached_date2num[key] = time - except ValueError: - time = np.nan # ppp - - return time - - # def dddd(self): - # """TODO.""" - # for axis_code, extra_type in zip((11, 10), ("x", "y")): - # coord_type = extra_type + "_domain_bounds" - # - # if coord_type in p.extra: - # p.extra[coord_type] - # # Create, from extra data, an auxiliary coordinate - # # with 1) data and bounds, if the upper and lower - # # bounds have no missing values; or 2) data but no - # # bounds, if the upper bound has missing values - # # but the lower bound does not. - # - # # Should be the axis which has axis_code 13 - # file_position = ppfile.tell() - # bounds = p.extra[coord_type][...] - # - # # Reset the file pointer after reading the extra - # # data into a numpy array - # ppfile.seek(file_position, os.SEEK_SET) - # data = None - # # dch also test in bmdi?: - # if np.any(bounds[..., 1] == _pp_rmdi): - # # dch also test in bmdi?: - # if not np.any(bounds[..., 0] == _pp_rmdi): - # data = bounds[..., 0] - # bounds = None - # else: - # data = np.mean(bounds, axis=1) - # - # if (data, bounds) != (None, None): - # aux = "aux%(auxN)d" % locals() - # auxN += 1 # Increment auxiliary number - # - # coord = _create_Coordinate( - # domain, - # aux, - # axis_code, - # p=p, - # array=data, - # aux=True, - # bounds_array=bounds, - # pubattr={"axis": None}, - # # DCH xdim? should be the axis which has axis_code 13: - # dimensions=[xdim], - # ) - # else: - # coord_type = "{0}_domain_lower_bound".format(extra_type) - # if coord_type in p.extra: - # # Create, from extra data, an auxiliary - # # coordinate with data but no bounds, if the - # # data noes not contain any missing values - # file_position = ppfile.tell() - # data = p.extra[coord_type][...] - # # Reset the file pointer after reading the - # # extra data into a numpy array - # ppfile.seek(file_position, os.SEEK_SET) - # if not np.any(data == _pp_rmdi): # dch + test in bmdi - # aux = "aux%(auxN)d" % locals() - # auxN += 1 # Increment auxiliary number - # coord = _create_Coordinate( - # domain, - # aux, - # axis_code, - # p=p, - # aux=True, - # array=np.array(data), - # pubattr={"axis": None}, - # dimensions=[xdim], - # ) # DCH xdim? - - def unrotated_latlon(self, rotated_lat, rotated_lon, pole_lat, pole_lon): - """Create 2-d arrays of unrotated latitudes and longitudes. - - :Parameters: - - rotated_lat: `numpy.ndarray` - - rotated_lon: `numpy.ndarray` - - pole_lat: `float` - - pole_lon: `float` - - :Returns: - - lat, lon: `numpy.ndarray`, `numpy.ndarray` - - """ - # Make sure rotated_lon and pole_lon is in [0, 360) - pole_lon = pole_lon % 360.0 - - # Convert everything to radians - pole_lon *= _pi_over_180 - pole_lat *= _pi_over_180 - - cos_pole_lat = np.cos(pole_lat) - sin_pole_lat = np.sin(pole_lat) - - # Create appropriate copies of the input rotated arrays - rot_lon = rotated_lon.copy() - rot_lat = rotated_lat.view() - - # Make sure rotated longitudes are between -180 and 180 - rot_lon %= 360.0 - rot_lon = np.where(rot_lon < 180.0, rot_lon, rot_lon - 360) - - # Create 2-d arrays of rotated latitudes and longitudes in radians - nlat = rot_lat.size - nlon = rot_lon.size - rot_lon = np.resize(np.deg2rad(rot_lon), (nlat, nlon)) - rot_lat = np.resize(np.deg2rad(rot_lat), (nlon, nlat)) - rot_lat = np.transpose(rot_lat, axes=(1, 0)) - - # Find unrotated latitudes - CPART = np.cos(rot_lon) * np.cos(rot_lat) - sin_rot_lat = np.sin(rot_lat) - x = cos_pole_lat * CPART + sin_pole_lat * sin_rot_lat - x = np.clip(x, -1.0, 1.0) - unrotated_lat = np.arcsin(x) - - # Find unrotated longitudes - x = -cos_pole_lat * sin_rot_lat + sin_pole_lat * CPART - x /= np.cos(unrotated_lat) - # dch /0 or overflow here? surely lat could be ~+-pi/2? if so, - # does x ~ cos(lat)? - x = np.clip(x, -1.0, 1.0) - unrotated_lon = -np.arccos(x) - - unrotated_lon = np.where(rot_lon > 0.0, -unrotated_lon, unrotated_lon) - if pole_lon >= self.atol: - SOCK = pole_lon - np.pi - else: - SOCK = 0 - unrotated_lon += SOCK - - # Convert unrotated latitudes and longitudes to degrees - unrotated_lat = np.rad2deg(unrotated_lat) - unrotated_lon = np.rad2deg(unrotated_lon) - - # Return unrotated latitudes and longitudes - return (unrotated_lat, unrotated_lon) - - def xy_coordinate(self, axiscode, axis): - """Create an X or Y dimension coordinate from header entries or - extra data. - - :Parameters: - - axiscode: `int` - - axis: `str` - Which type of coordinate to create: ``'x'`` or - ``'y'``. - - :Returns: - - (`str`, `DimensionCoordinate`) - - """ - X = axiscode in (11, -11) - - if X: - autocyclic = {"X": True} - else: - autocyclic = _autocyclic_false - - if axis == "x": - delta = self.bdx - origin = self.real_hdr[bzx] - size = self.lbnpt - - da = self.implementation.initialise_DomainAxis(size=size) - axis_key = self.implementation.set_domain_axis( - self.field, da, copy=False - ) - _axis["x"] = axis_key - else: - delta = self.bdy - origin = self.real_hdr[bzy] - size = self.lbrow - - da = self.implementation.initialise_DomainAxis(size=size) - axis_key = self.implementation.set_domain_axis( - self.field, da, copy=False - ) - _axis["y"] = axis_key - - autocyclic = _autocyclic_false - - if abs(delta) > self.atol: - # Create regular coordinates from header items - if axiscode == 11 or axiscode == -11: - origin -= divmod(origin + delta * size, 360.0)[0] * 360 - while origin + delta * size > 360.0: - origin -= 360.0 - while origin + delta * size < -360.0: - origin += 360.0 - - array = _cached_regular_array.get((origin, delta, size)) - if array is None: - array = np.arange( - origin + delta, - origin + delta * (size + 0.5), - delta, - dtype=float, - ) - _cached_regular_array[(origin, delta, size)] = array - - # Create the coordinate bounds - if axiscode in (13, 31, 40, 99): - # The following axiscodes do not have bounds: - # 13 = Site number (set of parallel rows or columns - # e.g.Time series) - # 31 = Logarithm to base 10 of pressure in mb - # 40 = Pseudolevel - # 99 = Other - bounds = None - else: - bounds = _cached_regular_bounds.get((origin, delta, size)) - if bounds is None: - delta_by_2 = 0.5 * delta - bounds = self.create_bounds_array( - array - delta_by_2, array + delta_by_2 - ) - _cached_regular_bounds[(origin, delta, size)] = bounds - else: - # Create coordinate from extra data - array = self.extra.get(axis, None) - lower_bounds = self.extra.get(axis + "_lower_bound", None) - upper_bounds = self.extra.get(axis + "_upper_bound", None) - if lower_bounds is not None and upper_bounds is not None: - bounds = self.create_bounds_array(lower_bounds, upper_bounds) - else: - bounds = None - - units = _axiscode_to_Units.setdefault(axiscode, None) - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, bounds, units=units) - dc = self.coord_positive(dc, axiscode, axis_key) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - if X and bounds is not None: - autocyclic["cyclic"] = abs(bounds[0, 0] - bounds[-1, -1]) == 360.0 - autocyclic["period"] = Data(360.0, units) - autocyclic["axis"] = axis_key - autocyclic["coord"] = dc - - key = self.implementation.set_dimension_coordinate( - self.field, dc, axes=[axis_key], copy=False, autocyclic=autocyclic - ) - - return key, dc, axis_key - - def site_coordinates_from_extra_data(self): - """Create site-related coordinates from extra data. - - :Returns: - - `None` - - """ - # Create coordinate from extra data - for axis, standard_name, units in zip( - ("x", "y"), - ("longitude", "latitude"), - (_Units["degrees_east"], _Units["degrees_north"]), - ): - lower_bounds = self.extra.get(axis + "_domain_lower_bound", None) - upper_bounds = self.extra.get(axis + "_domain_upper_bound", None) - if lower_bounds is None or upper_bounds is None: - continue - - # Still here? - bounds = self.create_bounds_array(lower_bounds, upper_bounds) - array = np.average(bounds, axis=1) - - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, array, bounds, units=units) - - ac.standard_name = standard_name - ac.long_name = "region limit" - self.implementation.set_auxiliary_coordinate( - self.field, - ac, - axes=[_axis["site_axis"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - array = self.extra.get("domain_title", None) - if array is not None: - ac = self.implementation.initialise_AuxiliaryCoordinate() - ac = self.coord_data(ac, array, None, units=None) - - ac.standard_name = "region" - self.implementation.set_auxiliary_coordinate( - self.field, - ac, - axes=[_axis["site_axis"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - @_manage_log_level_via_verbose_attr - def z_coordinate(self, axiscode): - """Create a Z dimension coordinate from BLEV. - - :Parameters: - - axiscode: `int` - - :Returns: - - `DimensionCoordinate` - - """ - if self.info: - logger.info( - "Creating Z coordinates and bounds from BLEV, BRLEV and " - "BRSVD1:" - ) # pragma: no cover - - z_recs = self.z_recs - array = tuple([rec.real_hdr.item(blev) for rec in z_recs]) - bounds0 = tuple( - [rec.real_hdr[brlev] for rec in z_recs] - ) # lower level boundary - bounds1 = tuple([rec.real_hdr[brsvd1] for rec in z_recs]) # bulev - if _coord_positive.get(axiscode, None) == "down": - bounds0, bounds1 = bounds1, bounds0 - - array = np.array(array, dtype=self.real_hdr_dtype) - bounds0 = np.array(bounds0, dtype=self.real_hdr_dtype) - bounds1 = np.array(bounds1, dtype=self.real_hdr_dtype) - bounds = self.create_bounds_array(bounds0, bounds1) - - if (bounds0 == bounds1).all() or np.allclose(bounds.min(), _pp_rmdi): - bounds = None - else: - bounds = self.create_bounds_array(bounds0, bounds1) - - da = self.implementation.initialise_DomainAxis(size=array.size) - axisZ = self.implementation.set_domain_axis(self.field, da, copy=False) - _axis["z"] = axisZ - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data( - dc, - array, - bounds=bounds, - units=_axiscode_to_Units.setdefault(axiscode, None), - ) - dc = self.coord_positive(dc, axiscode, _axis["z"]) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["z"]], - copy=False, - autocyclic=_autocyclic_false, - ) - - return dc - - -class UMRead(cfdm.read_write.IORead): - """A container for instantiating Fields from a UM fields file.""" - - @_manage_log_level_via_verbosity - def read( - self, - dataset, - um_version=None, - aggregate=True, - endian=None, - word_size=None, - set_standard_name=True, - height_at_top_of_model=None, - fmt=None, - chunk=True, - verbose=None, - select=None, - squeeze=False, - unsqueeze=False, - domain=False, - dataset_type=None, - ignore_unknown_type=False, - unpack=True, - filesystem=None, - storage_options=None, - ): - """Read fields from a PP file or UM fields file. - - The file may be big or little endian, 32 or 64 bit - - :Parameters: - - dataset: `file` or `str` - A string giving the file name, or an open file object, - from which to read fields. - - um_version: number, optional - The Unified Model (UM) version to be used when decoding - the PP header. Valid versions are, for example, ``402`` - (v4.2), ``606.3`` (v6.6.3) and ``1001`` (v10.1). The - default version is ``405`` (v4.5). The version is ignored - if it can be inferred from the PP headers, which will - generally be the case for files created at versions 5.3 - and later. Note that the PP header can not encode tertiary - version elements (such as the ``3`` in ``606.3``), so it - may be necessary to provide a UM version in such cases. - - verbose: `int` or `str` or `None`, optional - If an integer from ``-1`` to ``3``, or an equivalent string - equal ignoring case to one of: - - * ``'DISABLE'`` (``0``) - * ``'WARNING'`` (``1``) - * ``'INFO'`` (``2``) - * ``'DETAIL'`` (``3``) - * ``'DEBUG'`` (``-1``) - - set for the duration of the method call only as the minimum - cut-off for the verboseness level of displayed output (log) - messages, regardless of the globally-configured `cf.log_level`. - Note that increasing numerical value corresponds to increasing - verbosity, with the exception of ``-1`` as a special case of - maximal and extreme verbosity. - - Otherwise, if `None` (the default value), output messages will - be shown according to the value of the `cf.log_level` setting. - - Overall, the higher a non-negative integer or equivalent string - that is set (up to a maximum of ``3``/``'DETAIL'``) for - increasing verbosity, the more description that is printed - about the read process. - - set_standard_name: `bool`, optional - - select: (sequence of) `str` or `Query` or `re.Pattern`, optional - Only return field constructs whose identities match - the given values(s), i.e. those fields ``f`` for which - ``f.match_by_identity(*select)`` is `True`. See - `cf.Field.match_by_identity` for details. - - This is equivalent to, but faster than, not using the - *select* parameter but applying its value to the - returned field list with its - `cf.FieldList.select_by_identity` method. For example, - ``fl = cf.read(file, select='stash_code=3236')`` is - equivalent to ``fl = - cf.read(file).select_by_identity('stash_code=3236')``. - - squeeze: `bool`, optional - If True then remove all size 1 dimensions from field - construct data arrays, regardless of how the data are - stored in the dataset. If False (the default) then the - presence or not of size 1 dimensions is determined by - how the data are stored in its dataset. - - .. versionadded:: 3.17.0 - - unsqueeze: `bool`, optional - If True then ensure that field construct data arrays - span all of the size 1 dimensions, regardless of how - the data are stored in the dataset. If False (the - default) then the presence or not of size 1 dimensions - is determined by how the data are stored in its - dataset. - - .. versionadded:: 3.17.0 - - unpack: `bool`, optional - If True, the default, then unpack arrays by convention - when the data is read from disk. - - Unpacking is determined by netCDF conventions for the - following variable attributes ``add_offset`` and - ``scale_factor``, as applied to lookup header entries - BDATUM and BMKS respectively. - - .. versionadded:: 3.17. - - storage_protocol: `None` or `str`, optional - The `fsspec` file system protocol (e.g, ``'file'``, - ``'s3'``, ``'http'``). If `None` (the default) then a - local file system is assumed. - - .. versionadded:: 3.20.0 - - storage_options: `dict` or `None`, optional - Key/value pairs to be passed on to the creation of - `s3fs.S3FileSystem` file systems to control the - opening of files in S3 object stores. Ignored for - files not in an S3 object store, i.e. those whose - names do not start with ``s3:``. - - By default, or if `None`, then *storage_options* is - taken as ``{}``. - - If the ``'endpoint_url'`` key is not in - *storage_options* or is not in a dictionary defined by - the ``'client_kwargs`` key (which is always the case - when *storage_options* is `None`), then one will be - automatically inserted for accessing an S3 file. For - example, for a file name of - ``'s3://store/data/file.nc'``, an ``'endpoint_url'`` - key with value ``'https://store'`` would be created. - - *Parameter example:* - For a file name of ``'s3://store/data/file.nc'``, - the following are equivalent: ``None``, ``{}``, and - ``{'endpoint_url': 'https://store'}``, - ``{'client_kwargs': {'endpoint_url': - 'https://store'}}`` - - *Parameter example:* - ``{'key': 'scaleway-api-key...', 'secret': - 'scaleway-secretkey...', 'endpoint_url': - 'https://s3.fr-par.scw.cloud', 'client_kwargs': - {'region_name': 'fr-par'}}`` - - .. versionadded:: 3.20.0 - - :Returns: - - `list` - The fields in the file. - - **Examples** - - >>> f = read('file.pp') - >>> f = read('*/file[0-9].pp', um_version=708) - - """ - if domain: - raise ValueError( - "Can't read Domain constructs from UM or PP datasets " - "(only Field constructs)" - ) - - representation = self.dataset_representation(dataset) - if representation != "path": - raise NotImplementedError( - "Can't yet read Field constructs from a UM or PP " - f"{representation!r} dataset: {dataset!r}" - ) - - if not _stash2standard_name: - # -------------------------------------------------------- - # Create the STASH code to standard_name conversion - # dictionary - # -------------------------------------------------------- - load_stash2standard_name() - - if endian: - byte_ordering = endian + "_endian" - else: - byte_ordering = None - - if fmt is not None: - fmt = fmt.upper() - - if um_version is None: - um_version = 405 - else: - um_version = float(str(um_version).replace(".", "0", 1)) - - # ------------------------------------------------------------ - # Parse the 'dataset' keyword parameter - # ------------------------------------------------------------ - if filesystem is None: - try: - dataset = abspath(dataset, uri=False) - except ValueError: - dataset = abspath(dataset) - - filename = dataset - self.read_vars = { - "filename": filename, - "byte_ordering": byte_ordering, - "word_size": word_size, - "fmt": fmt, - } - - history = f"Converted from UM/PP by cf-python v{__version__}" - - if endian: - byte_ordering = endian + "_endian" - else: - byte_ordering = None - - # ------------------------------------------------------------ - # Parse the 'dataset_type' keyword parameter - # ------------------------------------------------------------ - if dataset_type is not None: - if isinstance(dataset_type, str): - dataset_type = (dataset_type,) - - dataset_type = set(dataset_type) - if not dataset_type.intersection(("UM",)): - # Return now if there are valid file types - return [] - - if storage_options is not None: - raise NotImplementedError( - "Can't yet open PP/UM files with file system storage options" - ) - - if filesystem is not None: - raise NotImplementedError( - "Can't yet open PP/UM files from a pre-defined file system" - ) - - storage_protocol = None - - f = self.dataset_open(dataset, parse=True) - - info = is_log_level_info(logger) - - um = [ - UMField( - filename, - var, - f.fmt, - f.byte_ordering, - f.word_size, - um_version, - set_standard_name, - history=history, - height_at_top_of_model=height_at_top_of_model, - verbose=verbose, - implementation=self.implementation, - select=select, - info=info, - unpack=unpack, - storage_protocol=storage_protocol, - storage_options=storage_options, - ) - for var in f.vars - ] - - self.dataset_close() - - return [field for x in um for field in x.fields if field] - - def _open_um_file( - self, - filename, - aggregate=True, - fmt=None, - word_size=None, - byte_ordering=None, - parse=True, - ): - """Open a UM fields file or PP file. - - :Parameters: - - filename: `str` - The file to be opened. - - parse: `bool`, optional - If True, the default, then parse the contents. If - False then the contents are not parsed, which can be - considerably faster in cases when the contents are not - required. - - .. versionadded:: 3.16.2 - - :Returns: - - `umread_lib.umfile.File` - The open PP or FF file object. - - """ - self.dataset_close() - try: - f = File( - filename, - byte_ordering=byte_ordering, - word_size=word_size, - fmt=fmt, - parse=parse, - ) - except Exception: - try: - f.close_fd() - except Exception: - pass - - raise DatasetTypeError( - f"\nCan't interpret {filename} as a PP or UM dataset" - ) - - self._um_file = f - return f - - def is_um_file(self, filename): - """Whether or not a file is a PP file or UM fields file. - - Note that the file type is determined by inspecting the file's - content and any file suffix is not considered. - - :Parameters: - - filename: `str` - The file. - - :Returns: - - `bool` - - **Examples** - - >>> r.is_um_file('ppfile') - True - - """ - try: - # Note: No need to completely parse the file to ascertain - # if it's PP or FF. - self.dataset_open(filename, parse=False) - except Exception: - self.dataset_close() - return False - else: - self.dataset_close() - return True - - def dataset_close(self): - """Close the file that has been read. - - :Returns: - - `None` - - """ - f = getattr(self, "_um_file", None) - if f is not None: - f.close_fd() - - self._um_file = None - - def dataset_open(self, filename, parse=True): - """Open the file for reading. - - :Paramters: - - filename: `str` - The file to be read. - - parse: `bool`, optional - If True, the default, then parse the contents. If - False then the contents are not parsed, which can be - considerably faster in cases when the contents are not - required. - - .. versionadded:: 3.16.2 - - :Returns: - - `umread_lib.umfile.File` - The open PP or FF file object. - - """ - g = getattr(self, "read_vars", {}) - - return self._open_um_file( - filename, - byte_ordering=g.get("byte_ordering"), - word_size=g.get("word_size"), - fmt=g.get("fmt"), - parse=parse, - ) - - @classmethod - def dataset_representation(cls, dataset): - """Return the logical representation type of the input dataset. - - .. versionadded:: 3.20.0 - - :Parameters: - - dataset: - The dataset. May be a string-valued path or a - file-like object. - - :Returns: - - `str` - The dataset representation: - - * ``'path'``: A string-valued path. - - * ``'file_handle'``: An open file handle (such as - returned by `fsspec.filesystem.open`) - - * ``'unknown'``: Anything else. - - """ - # Strings (Paths) - if isinstance(dataset, str): - return "path" - - # Check for a "binary stream" (file handle) - if hasattr(dataset, "read") and hasattr(dataset, "seek"): - return "file_handle" - - return "unknown" - - -""" -Problems: - -Z and P coordinates -/home/david/data/pp/aaaao/aaaaoa.pmh8dec.03328.pp - -/net/jasmin/chestnut/data-24/david/testpp/026000000000c.fc0607.000128.0000.00.04.0260.0020.1491.12.01.00.00.pp -skipping variable stash code=0, 0, 0 because: grid code not supported -umfile: error condition detected in routine list_copy_to_ptr_array -umfile: error condition detected in routine process_vars -umfile: error condition detected in routine file_parse -OK 2015-04-01 - -/net/jasmin/chestnut/data-24/david/testpp/026000000000c.fc0619.000128.0000.00.04.0260.0020.1491.12.01.00.00.pp -skipping variable stash code=0, 0, 0 because: grid code not supported -umfile: error condition detected in routine list_copy_to_ptr_array -umfile: error condition detected in routine process_vars -umfile: error condition detected in routine file_parse -OK 2015-04-01 - -/net/jasmin/chestnut/data-24/david/testpp/lbcode_10423.pp -skipping variable stash code=0, 0, 0 because: grid code not supported -umfile: error condition detected in routine list_copy_to_ptr_array -umfile: error condition detected in routine process_vars -umfile: error condition detected in routine file_parse -OK 2015-04-01 - -/net/jasmin/chestnut/data-24/david/testpp/lbcode_11323.pp -skipping variable stash code=0, 0, 0 because: grid code not supported -umfile: error condition detected in routine list_copy_to_ptr_array -umfile: error condition detected in routine process_vars -umfile: error condition detected in routine file_parse -OK 2015-04-01 - -EXTRA_DATA: -/net/jasmin/chestnut/data-24/david/testpp/ajnjgo.pmm1feb.pp - -SLOW: (Not any more! 2015-04-01) -/net/jasmin/chestnut/data-24/david/testpp/xgdria.pdk949a.pp -/net/jasmin/chestnut/data-24/david/testpp/xhbmaa.pm27sep.pp - -RUN LENGTH ENCODED dump (not fields file) -/home/david/data/um/xhlska.dak69h0 -Field 115 (stash code 9) - -dch@eslogin008:/nerc/n02/n02/dch> ff2pp xgvwko.piw96b0 xgvwko.piw96b0.pp - -file xgvwko.piw96b0 is a byte swapped 64 bit ieee um file - -""" diff --git a/cf/test/rotated_pole.pp b/cf/test/rotated_pole.pp new file mode 100644 index 0000000000..7088c9ae99 Binary files /dev/null and b/cf/test/rotated_pole.pp differ diff --git a/cf/test/test_2d_latlon.py b/cf/test/test_2d_latlon.py new file mode 100644 index 0000000000..297177d1bf --- /dev/null +++ b/cf/test/test_2d_latlon.py @@ -0,0 +1,789 @@ +import datetime +import unittest + +import numpy as np +import pyproj + +import cf + +ellps = "WGS84" +km = "km" + +f0 = cf.example_field(0)[0, 0] + +key_x, x = f0.dimension_coordinate("X", item=True) +x.del_bounds() +x.standard_name = "projection_x_coordinate" +x.override_units(km, inplace=True) + +key_y, y = f0.dimension_coordinate("Y", item=True) +y.del_bounds() +y.standard_name = "projection_y_coordinate" +y.override_units(km, inplace=True) + +cr = cf.CoordinateReference() +cr.datum.set_parameters({"reference_ellipsoid_name": ellps}) +cr.set_coordinates((key_x, key_y)) +f0.set_construct(cr) + +paris_lon = 2.2945 # La Tour Eiffel, WGS84 +paris_lat = 48.8584 # La Tour Eiffel, WGS84 + +longlat = pyproj.CRS.from_string("+proj=longlat +ellps=WGS84") + + +def check_paris(g, atol=1e13, verbose=False): + """Check if field `g` has Paris coordinates.""" + if verbose: + print( + [ + g.auxiliary_coordinate("X").array, + g.auxiliary_coordinate("Y").array, + ], + [paris_lon, paris_lat], + ) + + ok = np.allclose(g.auxiliary_coordinate("X"), paris_lon, rtol=0, atol=atol) + ok = ok & np.allclose( + g.auxiliary_coordinate("Y"), paris_lat, rtol=0, atol=atol + ) + return ok + + +def set_coordinate_conversion(f, parameters): + """Replace the coordinate reference of field `f` with new parameters.""" + cr = f.coordinate_reference() + + cr.coordinate_conversion.clear_parameters() + cr.coordinate_conversion.set_parameters(parameters) + + +def field_paris(proj): + """Return a field for Paris with projection grid `proj`.""" + t = pyproj.Transformer.from_crs(longlat, proj, always_xy=1) + x_coords, y_coords = t.transform(paris_lon, paris_lat) + + f = f0.copy() + x = f.dimension_coordinate("X") + x[...] = x_coords + + y = f.dimension_coordinate("Y") + y[...] = y_coords + + rotated_latitude_longitude = "+proj=ob_tran" in proj.srs + if rotated_latitude_longitude: + x.standard_name = "grid_longitude" + y.standard_name = "grid_latitude" + x.override_units("degrees", inplace=True) + y.override_units("degrees", inplace=True) + + return f + + +class LatLon2dTest(unittest.TestCase): + """Test the creation of 2-d lat/lon coordinates.""" + + def test_Field_2d_create_latlon_coordinates_albers_equal_area(self): + """Test albers_equal_area.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_1 = 43 + lat_2 = 62 + lat_0 = 30 + lon_0 = 10 + proj = pyproj.CRS( + proj="aea", + lat_1=lat_1, + lat_2=lat_2, + lat_0=lat_0, + lon_0=lon_0, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "albers_equal_area", + "longitude_of_central_meridian": lon_0, + "latitude_of_projection_origin": lat_0, + "standard_parallel": [lat_1, lat_2], + }, + ) + + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "albers_equal_area", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_azimuthal_equidistant(self): + """Test azimuthal_equidistant.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_0 = 48.8584 + lon_0 = 2.2945 + proj = pyproj.CRS( + proj="aeqd", + lat_0=lat_0, + lon_0=lon_0, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "azimuthal_equidistant", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": lat_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "azimuthal_equidistant", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_geostationary(self): + """Test geostationary.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + h = 35785831 + lon_0 = 0 + sweep = "y" + proj = pyproj.CRS( + proj="geos", + lon_0=lon_0, + h=h, + x_0=0, + y_0=0, + sweep=sweep, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "geostationary", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": 0, + "perspective_point_height": h, + "sweep_angle_axis": sweep, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g, atol=1e-12)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, {"grid_mapping_name": "geostationary", "crs_wkt": proj.to_wkt()} + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_lambert_azimuthal_equal_area( + self, + ): + """Test lambert_azimuthal_equal_area.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_0 = 52 + lon_0 = 10 + x_0 = 4321000 + y_0 = 3210000 + proj = pyproj.CRS( + proj="laea", + lon_0=lon_0, + lat_0=lat_0, + x_0=x_0, + y_0=y_0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "lambert_azimuthal_equal_area", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": lat_0, + "false_easting": x_0, + "false_northing": y_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g, atol=1e-8)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "lambert_azimuthal_equal_area", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_lambert_conformal_conic(self): + """Test lambert_conformal_conic.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_1 = 33 + lat_2 = 45 + lat_0 = 39 + lon_0 = -96 + proj = pyproj.CRS( + proj="lcc", + lon_0=lon_0, + lat_0=lat_0, + lat_1=lat_1, + lat_2=lat_2, + ellps=ellps, + x_0=0, + y_0=0, + units=km, + ) + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "lambert_conformal_conic", + "longitude_of_central_meridian": lon_0, + "latitude_of_projection_origin": lat_0, + "standard_parallel": [lat_1, lat_2], + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "lambert_conformal_conic", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_lambert_cylindrical_equal_area( + self, + ): + """Test lambert_cylindrical_equal_area.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lon_0 = 0 + lat_ts = 30 + proj = pyproj.CRS( + proj="cea", + lon_0=lon_0, + lat_ts=lat_ts, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "lambert_cylindrical_equal_area", + "longitude_of_central_meridian": lon_0, + "standard_parallel": lat_ts, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "lambert_cylindrical_equal_area", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_mercator(self): + """Test mercator.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lon_0 = 0 + lat_ts = 0 + proj = pyproj.CRS( + proj="merc", + lon_0=lon_0, + lat_ts=lat_ts, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "mercator", + "longitude_of_projection_origin": lon_0, + "standard_parallel": lat_ts, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, {"grid_mapping_name": "mercator", "crs_wkt": proj.to_wkt()} + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_oblique_mercator(self): + """Test oblique_mercator.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_0 = 45 + lonc = 10 + alpha = 45 + k_0 = 1 + proj = pyproj.CRS( + proj="omerc", + lonc=lonc, + lat_0=lat_0, + alpha=alpha, + k_0=k_0, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "oblique_mercator", + "azimuth_of_central_line": alpha, + "latitude_of_projection_origin": lat_0, + "longitude_of_projection_origin": lonc, + "scale_factor_at_projection_origin": k_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "oblique_mercator", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_orthographic(self): + """Test orthographic.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_0 = 48.8584 + lon_0 = 2.2945 + proj = pyproj.CRS( + proj="ortho", + lon_0=lon_0, + lat_0=lat_0, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "orthographic", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": lat_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, {"grid_mapping_name": "orthographic", "crs_wkt": proj.to_wkt()} + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_polar_stereographic(self): + """Test polar_stereographic.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_ts = 90 + lat_0 = 90 + lon_0 = 0 + proj = pyproj.CRS( + proj="stere", + lon_0=lon_0, + lat_0=lat_0, + lat_ts=lat_ts, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "polar_stereographic", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": lat_0, + "standard_parallel": lat_ts, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "polar_stereographic", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_rotated_latitude_longitude( + self, + ): + """Test rotated_latitude_longitude.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lon_0 = 190 + o_lat_p = 38 + o_lon_p = 0 + proj = pyproj.CRS( + proj="ob_tran", + o_proj="longlat", + o_lon_p=o_lon_p, + o_lat_p=o_lat_p, + lon_0=lon_0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "rotated_latitude_longitude", + "grid_north_pole_latitude": o_lat_p, + "grid_north_pole_longitude": lon_0, + "north_pole_grid_longitude": o_lon_p, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "rotated_latitude_longitude", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_sinusoidal(self): + """Test sinusoidal.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lon_0 = 0 + proj = pyproj.CRS( + proj="sinu", + lon_0=lon_0, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "sinusoidal", + "longitude_of_projection_origin": lon_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, {"grid_mapping_name": "sinusoidal", "crs_wkt": proj.to_wkt()} + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_stereographic(self): + """Test stereographic.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_0 = 90 + lon_0 = 0 + k_0 = 0.994 + proj = pyproj.CRS( + proj="stere", + lon_0=lon_0, + lat_0=lat_0, + k_0=k_0, + x_0=0, + y_0=0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "stereographic", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": lat_0, + "scale_factor_at_projection_origin": k_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, {"grid_mapping_name": "stereographic", "crs_wkt": proj.to_wkt()} + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_transverse_mercator(self): + """Test transverse_mercator.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + lat_0 = 0 + lon_0 = 3 + k_0 = 0.9996012717 + x_0 = 500000 + y_0 = 0 + proj = pyproj.CRS( + proj="tmerc", + lon_0=lon_0, + lat_0=lat_0, + k_0=k_0, + x_0=x_0, + y_0=y_0, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "transverse_mercator", + "longitude_of_central_meridian": lon_0, + "latitude_of_projection_origin": lat_0, + "scale_factor_at_central_meridian": k_0, + "false_easting": x_0, + "false_northing": y_0, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "transverse_mercator", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_vertical_perspective(self): + """Test vertical_perspective.""" + # Get the correct projected coordinates for Paris, convert + # these projected coordinates to lat/lon using + # `cf.Field.create_latlon_coordinates`, and check that we end + # up in Paris. + h = 3000000 + lat_0 = 48.8584 + lon_0 = 2.2945 + proj = pyproj.CRS( + proj="nsper", + lon_0=lon_0, + lat_0=lat_0, + h=h, + ellps=ellps, + units=km, + ) + + f = field_paris(proj) + set_coordinate_conversion( + f, + { + "grid_mapping_name": "vertical_perspective", + "longitude_of_projection_origin": lon_0, + "latitude_of_projection_origin": lat_0, + "perspective_point_height": h, + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + # Do the same, but with a WKT-defined coordinate reference + # construct. + set_coordinate_conversion( + f, + { + "grid_mapping_name": "vertical_perspective", + "crs_wkt": proj.to_wkt(), + }, + ) + g = f.create_latlon_coordinates() + self.assertTrue(check_paris(g)) + + def test_Field_2d_create_latlon_coordinates_bounds(self): + """Test lat/lon bounds.""" + # Check that lat/lon coordinate bounds are correctly created. + f = cf.read("rotated_pole.pp")[0] + + cr = f.coordinate_reference() + self.assertEqual( + cr.coordinate_conversion.parameters(), + { + "grid_mapping_name": "rotated_latitude_longitude", + "grid_north_pole_latitude": 38, + "grid_north_pole_longitude": 190, + }, + ) + + self.assertFalse(f.auxiliary_coordinates()) + + self.assertIsNone(f.create_latlon_coordinates(inplace=True)) + + # Compare the 2-d lat/lon coordinates against + # known-to-be-correct values + lat = f.auxiliary_coordinate("latitude") + self.assertEqual(lat.shape, (110, 106)) + self.assertTrue(np.allclose(lat[0, 0].array, 67.1246604)) + + self.assertEqual(lat.bounds.units, lat.units) + self.assertTrue( + np.allclose( + lat[0, 0].bounds.array, + [67.13411912, 66.82618815, 67.11220769, 67.42286415], + ) + ) + + lon = f.auxiliary_coordinate("longitude") + self.assertEqual(lon.shape, (110, 106)) + self.assertTrue(np.allclose(lon[0, 0].array, -45.98136153)) + + self.assertEqual(lon.bounds.units, lon.units) + self.assertTrue( + np.allclose( + lon[0, 0].bounds.array, + [-46.7492162, -45.94548426, -45.21355527, -46.01992883], + ) + ) + + +if __name__ == "__main__": + print("Run date:", datetime.datetime.now()) + cf.environment() + print("") + unittest.main(verbosity=2) diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index 4745b55c0c..324fe38bf8 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -287,8 +287,8 @@ def test_CFA_unique_value(self): self.assertEqual( fa.data.nc_get_aggregated_data(), { - "map": "fragment_map_uid", - "unique_values": "fragment_value_uid", + "map": "/fragment_map_uid", + "unique_values": "/fragment_value_uid", }, ) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index d399f3e18f..798709337c 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -145,7 +145,7 @@ def test_Field_compress_uncompress(self): u = f.uncompress() self.assertFalse(bool(u.data.get_compression_type()), message) - self.assertTrue(f.equals(u, verbose=2), message) + self.assertTrue(f.equals(u), message) for method1 in methods: message += ", method1=" + method1 @@ -160,16 +160,16 @@ def test_Field_compress_uncompress(self): bool(c.data.get_compression_type()), message ) - self.assertTrue(u.equals(c, verbose=2), message) - self.assertTrue(f.equals(c, verbose=2), message) + self.assertTrue(u.equals(c), message) + self.assertTrue(f.equals(c), message) c = f.compress(method1) self.assertTrue( bool(c.data.get_compression_type()), message ) - self.assertTrue(u.equals(c, verbose=2), message) - self.assertTrue(f.equals(c, verbose=2), message) + self.assertTrue(u.equals(c), message) + self.assertTrue(f.equals(c), message) cf.write(c, tmpfile) c = cf.read(tmpfile)[0] @@ -177,7 +177,7 @@ def test_Field_compress_uncompress(self): self.assertTrue( bool(c.data.get_compression_type()), message ) - self.assertTrue(f.equals(c, verbose=2), message) + self.assertTrue(f.equals(c), message) def test_Field_apply_masking(self): f = self.f0.copy() @@ -194,7 +194,7 @@ def test_Field_apply_masking(self): d = f.data.copy() g = f.copy() self.assertIsNone(f.apply_masking(inplace=True)) - self.assertTrue(f.equals(g, verbose=1)) + self.assertTrue(f.equals(g)) x = 0.11 y = 0.1 @@ -205,31 +205,31 @@ def test_Field_apply_masking(self): g = f.apply_masking() e = d.apply_masking(fill_values=[x]) - self.assertTrue(e.equals(g.data, verbose=1)) + self.assertTrue(e.equals(g.data)) self.assertEqual(g.data.array.count(), g.data.size - 1) f.set_property("valid_range", [y, z]) d = f.data.copy() g = f.apply_masking() e = d.apply_masking(fill_values=[x], valid_range=[y, z]) - self.assertTrue(e.equals(g.data, verbose=1)) + self.assertTrue(e.equals(g.data)) f.del_property("valid_range") f.set_property("valid_min", y) g = f.apply_masking() e = d.apply_masking(fill_values=[x], valid_min=y) - self.assertTrue(e.equals(g.data, verbose=1)) + self.assertTrue(e.equals(g.data)) f.del_property("valid_min") f.set_property("valid_max", z) g = f.apply_masking() e = d.apply_masking(fill_values=[x], valid_max=z) - self.assertTrue(e.equals(g.data, verbose=1)) + self.assertTrue(e.equals(g.data)) f.set_property("valid_min", y) g = f.apply_masking() e = d.apply_masking(fill_values=[x], valid_min=y, valid_max=z) - self.assertTrue(e.equals(g.data, verbose=1)) + self.assertTrue(e.equals(g.data)) def test_Field_flatten(self): f = self.f.copy() @@ -242,16 +242,16 @@ def test_Field_flatten(self): g = f.flatten() h = f.flatten(list(range(f.ndim))) - self.assertTrue(h.equals(g, verbose=2)) + self.assertTrue(h.equals(g)) g = f.flatten("time") - self.assertTrue(g.equals(f, verbose=2)) + self.assertTrue(g.equals(f)) for i in (0, 1, 2): g = f.flatten(i) - self.assertTrue(g.equals(f, verbose=2)) + self.assertTrue(g.equals(f)) g = f.flatten([i, "time"]) - self.assertTrue(g.equals(f, verbose=2)) + self.assertTrue(g.equals(f)) for axes in axes_combinations(f): g = f.flatten(axes) @@ -271,7 +271,7 @@ def test_Field_flatten(self): self.assertEqual(g.ndim, f.ndim - len(axes) + 1) self.assertEqual(g.size, f.size) - self.assertTrue(f.equals(f.flatten([]), verbose=2)) + self.assertTrue(f.equals(f.flatten([]))) self.assertIsNone(f.flatten(inplace=True)) def test_Field_bin(self): @@ -344,14 +344,14 @@ def test_Field_weights(self): w = f.weights(None) self.assertIsInstance(w, cf.Field) - self.assertTrue(w.data.equals(cf.Data(1.0, "1"), verbose=2)) + self.assertTrue(w.data.equals(cf.Data(1.0, "1"))) w = f.weights(data=True) self.assertIsInstance(w, cf.Data) w = f.weights(None, data=True) self.assertIsInstance(w, cf.Data) - self.assertTrue(w.equals(cf.Data(1.0, "1"), verbose=2)) + self.assertTrue(w.equals(cf.Data(1.0, "1"))) w = f.weights(components=True) self.assertIsInstance(w, dict) @@ -369,7 +369,7 @@ def test_Field_weights(self): w = f.weights() x = f.weights(w) - self.assertTrue(x.equals(w, verbose=2)) + self.assertTrue(x.equals(w)) for components in (False, True): for m in (False, True): @@ -460,7 +460,7 @@ def test_Field_collapse(self): a = f.collapse(method, axes=axes, weights=weights).data b = getattr(f.data, method)(axes=axes) self.assertTrue( - a.equals(b, rtol=1e-05, atol=1e-08, verbose=2), + a.equals(b, rtol=1e-05, atol=1e-08), ) for method in ( @@ -480,7 +480,7 @@ def test_Field_collapse(self): a = f.collapse(method, axes=axes, weights=weights).data b = getattr(f.data, method)(axes=axes, weights=d_weights) self.assertTrue( - a.equals(b, rtol=1e-05, atol=1e-08, verbose=2), + a.equals(b, rtol=1e-05, atol=1e-08), ) for method in ("integral",): @@ -491,7 +491,7 @@ def test_Field_collapse(self): ).data b = getattr(f.data, method)(axes=axes, weights=d_weights) self.assertTrue( - a.equals(b, rtol=1e-05, atol=1e-08, verbose=2), + a.equals(b, rtol=1e-05, atol=1e-08), ) for axes in axes_combinations(f): @@ -510,7 +510,7 @@ def test_Field_collapse(self): axes=axes, ddof=1, weights=d_weights ) self.assertTrue( - a.equals(b, rtol=1e-05, atol=1e-08, verbose=2), + a.equals(b, rtol=1e-05, atol=1e-08), ) for method in ("mean_of_upper_decile",): @@ -523,7 +523,7 @@ def test_Field_collapse(self): a = f.collapse(method, axes=axes, weights=weights).data b = getattr(f.data, method)(axes=axes, weights=d_weights) self.assertTrue( - a.equals(b, rtol=1e-05, atol=1e-08, verbose=2), + a.equals(b, rtol=1e-05, atol=1e-08), ) # Test the remove_vertical_crs keyword @@ -577,23 +577,23 @@ def test_Field_atol_rtol(self): f = self.f g = f.copy() - self.assertTrue(f.equals(g, verbose=2)) + self.assertTrue(f.equals(g)) g[0, 0, 0] += 0.001 self.assertFalse(f.equals(g)) - self.assertTrue(f.equals(g, atol=0.1, verbose=2)) + self.assertTrue(f.equals(g, atol=0.1)) self.assertFalse(f.equals(g)) self.assertEqual(cf.atol(), cf.ATOL()) atol = cf.atol(0.1) - self.assertTrue(f.equals(g, verbose=2)) + self.assertTrue(f.equals(g)) cf.atol(atol) self.assertFalse(f.equals(g)) - self.assertTrue(f.equals(g, rtol=10, verbose=2)) + self.assertTrue(f.equals(g, rtol=10)) self.assertFalse(f.equals(g)) self.assertEqual(cf.rtol(), cf.RTOL()) rtol = cf.rtol(10) - self.assertTrue(f.equals(g, verbose=2)) + self.assertTrue(f.equals(g)) cf.rtol(rtol) self.assertFalse(f.equals(g)) @@ -730,15 +730,15 @@ def test_Field__add__(self): f = self.f.copy() g = f * 0 - self.assertTrue((f + g).equals(f, verbose=2)) - self.assertTrue((g + f).equals(f, verbose=2)) + self.assertTrue((f + g).equals(f)) + self.assertTrue((g + f).equals(f)) g.transpose(inplace=True) - self.assertTrue((f + g).equals(f, verbose=2)) + self.assertTrue((f + g).equals(f)) for g in (f, f.copy(), f * 0): - self.assertTrue((f + g).equals(g + f, verbose=2)) - self.assertTrue((g + f).equals(f + g, verbose=2)) + self.assertTrue((f + g).equals(g + f)) + self.assertTrue((g + f).equals(f + g)) g = f.subspace(grid_longitude=[0]) * 0 @@ -752,8 +752,8 @@ def test_Field__add__(self): for key in a.cell_measures(filter_by_axis=(axis,), axis_mode="or"): a.del_construct(key) - self.assertTrue(a.equals(b, verbose=2)) - self.assertTrue(b.equals(a, verbose=2)) + self.assertTrue(a.equals(b)) + self.assertTrue(b.equals(a)) with self.assertRaises(TypeError): f + ("a string",) @@ -792,7 +792,7 @@ def test_Field_cumsum(self): g = f.copy() h = g.cumsum(1) self.assertIsNone(g.cumsum(1, inplace=True)) - self.assertTrue(g.equals(h, verbose=2)) + self.assertTrue(g.equals(h)) # Check that a new cell method that has been added cell_methods = h.cell_methods(todict=True) @@ -838,23 +838,23 @@ def test_Field_flip(self): g = f.subspace(**kwargs) h = f.flip() - self.assertTrue(h.equals(g, verbose=1)) + self.assertTrue(h.equals(g)) h = f.flip(f.get_data_axes()) - self.assertTrue(h.equals(g, verbose=1)) + self.assertTrue(h.equals(g)) h = f.flip(list(range(f.ndim))) - self.assertTrue(h.equals(g, verbose=1)) + self.assertTrue(h.equals(g)) h = f.flip(["X", "Z", "Y"]) - self.assertTrue(h.equals(g, verbose=1)) + self.assertTrue(h.equals(g)) h = f.flip((re.compile("^atmos"), "grid_latitude", "grid_longitude")) - self.assertTrue(h.equals(g, verbose=1)) + self.assertTrue(h.equals(g)) g = f.subspace(grid_longitude=slice(None, None, -1)) self.assertIsNone(f.flip("X", inplace=True)) - self.assertTrue(f.equals(g, verbose=1)) + self.assertTrue(f.equals(g)) def test_Field_anchor(self): f = self.f.copy() @@ -1127,8 +1127,8 @@ def test_Field_get_data_axes(self): def test_Field_equals(self): f = self.f.copy() g = f.copy() - self.assertTrue(f.equals(f, verbose=2)) - self.assertTrue(f.equals(g, verbose=2)) + self.assertTrue(f.equals(f)) + self.assertTrue(f.equals(g)) g.set_property("foo", "bar") self.assertFalse(f.equals(g)) g = f.copy() @@ -1137,12 +1137,12 @@ def test_Field_equals(self): # Symmetry f = cf.example_field(2) g = f.copy() - self.assertTrue(f.equals(g, verbose=2)) - self.assertTrue(g.equals(f, verbose=2)) + self.assertTrue(f.equals(g)) + self.assertTrue(g.equals(f)) g.del_construct("dimensioncoordinate0") - self.assertFalse(f.equals(g, verbose=2)) - self.assertFalse(g.equals(f, verbose=2)) + self.assertFalse(f.equals(g)) + self.assertFalse(g.equals(f)) def test_Field_insert_dimension(self): f = self.f.copy() @@ -2269,9 +2269,7 @@ def test_Field_auxiliary_coordinate(self): for identity in ("auxiliarycoordinate1", "latitude"): key, c = f.construct_item(identity) - self.assertTrue( - f.auxiliary_coordinate(identity).equals(c, verbose=2) - ) + self.assertTrue(f.auxiliary_coordinate(identity).equals(c)) self.assertEqual(f.auxiliary_coordinate(identity, key=True), key) with self.assertRaises(ValueError): @@ -2303,9 +2301,7 @@ def test_Field_coordinate_reference(self): key = f.construct_key(identity) c = f.construct(identity) - self.assertTrue( - f.coordinate_reference(identity).equals(c, verbose=2) - ) + self.assertTrue(f.coordinate_reference(identity).equals(c)) self.assertEqual(f.coordinate_reference(identity, key=True), key) key = f.construct_key( @@ -2336,9 +2332,7 @@ def test_Field_coordinate_reference(self): key = f.construct_key(identity) c = f.construct(identity) - self.assertTrue( - f.get_coordinate_reference(identity).equals(c, verbose=2) - ) + self.assertTrue(f.get_coordinate_reference(identity).equals(c)) self.assertEqual( f.get_coordinate_reference(identity, key=True), key ) @@ -2360,7 +2354,7 @@ def test_Field_coordinate_reference(self): cr = f.del_coordinate_reference( "standard_name:atmosphere_hybrid_height_coordinate" ) - self.assertTrue(cr.equals(c, verbose=2)) + self.assertTrue(cr.equals(c)) self.assertEqual(len(f.coordinate_references()), 1) self.assertEqual(len(f.domain_ancillaries()), 0) @@ -2414,9 +2408,7 @@ def test_Field_dimension_coordinate(self): else: key, c = f.construct(identity, item=True) - self.assertTrue( - f.dimension_coordinate(identity).equals(c, verbose=2) - ) + self.assertTrue(f.dimension_coordinate(identity).equals(c)) self.assertEqual(f.dimension_coordinate(identity, key=True), key) k, v = f.dimension_coordinate(identity, item=True) @@ -2439,10 +2431,10 @@ def test_Field_cell_measure(self): for identity in ("measure:area", "cellmeasure0"): key, c = f.construct_item(identity) - self.assertTrue(f.cell_measure(identity).equals(c, verbose=2)) + self.assertTrue(f.cell_measure(identity).equals(c)) self.assertEqual(f.cell_measure(identity, key=True), key) - self.assertTrue(f.cell_measure(identity).equals(c, verbose=2)) + self.assertTrue(f.cell_measure(identity).equals(c)) self.assertEqual(f.cell_measure(identity, key=True), key) self.assertEqual(len(f.cell_measures()), 1) @@ -2460,7 +2452,7 @@ def test_Field_cell_method(self): for identity in ("method:mean", "cellmethod0"): key, c = f.construct_item(identity) - self.assertTrue(f.cell_method(identity).equals(c, verbose=2)) + self.assertTrue(f.cell_method(identity).equals(c)) self.assertEqual(f.cell_method(identity, key=True), key) def test_Field_domain_ancillary(self): @@ -2468,7 +2460,7 @@ def test_Field_domain_ancillary(self): for identity in ("surface_altitude", "domainancillary0"): key, c = f.construct_item(identity) - self.assertTrue(f.domain_ancillary(identity).equals(c, verbose=2)) + self.assertTrue(f.domain_ancillary(identity).equals(c)) self.assertEqual(f.domain_ancillary(identity, key=True), key) with self.assertRaises(ValueError): @@ -2479,7 +2471,7 @@ def test_Field_field_ancillary(self): for identity in ("ancillary0", "fieldancillary0"): key, c = f.construct_item(identity) - self.assertTrue(f.field_ancillary(identity).equals(c, verbose=2)) + self.assertTrue(f.field_ancillary(identity).equals(c)) self.assertEqual(f.field_ancillary(identity, key=True), key) with self.assertRaises(ValueError): @@ -2492,7 +2484,7 @@ def test_Field_transpose(self): # Null transpose g = f.transpose([0, 1, 2]) - self.assertTrue(f0.equals(g, verbose=2)) + self.assertTrue(f0.equals(g)) self.assertIsNone(f.transpose([0, 1, 2], inplace=True)) self.assertTrue(f0.equals(f)) @@ -2510,7 +2502,7 @@ def test_Field_transpose(self): inplace=True, ) - self.assertTrue(h.equals(h0, verbose=2)) + self.assertTrue(h.equals(h0)) self.assertTrue((h.array == f.array).all()) with self.assertRaises(Exception): @@ -2538,9 +2530,9 @@ def test_Field_where(self): g = f.where(landfrac >= 54, cf.masked) self.assertTrue(g.data.count() == 9 * 6, g.data.count()) - self.assertTrue(f.equals(f.where(None), verbose=2)) + self.assertTrue(f.equals(f.where(None))) self.assertIsNone(f.where(None, inplace=True)) - self.assertTrue(f.equals(f0, verbose=2)) + self.assertTrue(f.equals(f0)) g = f.where(cf.wi(25, 31), -99, 11, construct="grid_longitude") g = f.where(cf.wi(25, 31), f * 9, f * -7, construct="grid_longitude") @@ -3285,8 +3277,8 @@ def test_Field_healpix_to_ugrid(self): self.f0.healpix_to_ugrid() @unittest.skipUnless(healpix_available, "Requires 'healpix' package.") - def test_Field_create_latlon_coordinates(self): - """Test Field.create_latlon_coordinates.""" + def test_Field_create_latlon_coordinates_healpix(self): + """Test Field.create_latlon_coordinates with HEALPix.""" # ------------------------------------------------------------ # HEALPix field # ------------------------------------------------------------ @@ -3306,7 +3298,7 @@ def test_Field_create_latlon_coordinates(self): g.auxiliary_coordinate(c).equals(f.auxiliary_coordinate(c)) ) - # pole_longitude. Note that bounds index 0 is the + # longitude_at_pole. Note that bounds index 0 is the # northern-most vertex, and bounds index 2 is the # southern-most vertex. f = self.f12 @@ -3321,7 +3313,7 @@ def test_Field_create_latlon_coordinates(self): np.allclose(longitude[32:48:4, 2], longitude[32:48:4, 0]) ) - g = f.create_latlon_coordinates(pole_longitude=3.14) + g = f.create_latlon_coordinates(longitude_at_pole=3.14) longitude = g.auxiliary_coordinate("X").bounds.array # North pole self.assertTrue(np.allclose(longitude[3:16:4, 0], 3.14)) diff --git a/cf/test/test_dsg.py b/cf/test/test_dsg.py index e5066b7aa6..5c9d82ae5d 100644 --- a/cf/test/test_dsg.py +++ b/cf/test/test_dsg.py @@ -184,7 +184,7 @@ def test_DSG_contiguous(self): ragged_array = cf.Data( np.array([280, 282.5, 281, 279, 278, 279.5], dtype="float32") ) - + # Define the count array values count_array = [2, 4] diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index b686a98b40..23f0306ada 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -311,6 +311,7 @@ def test_environment(self): "HDF5 library", "netcdf library", "netCDF4", + "xnetcdf", "h5netcdf", "h5py", "pyfive", @@ -329,6 +330,8 @@ def test_environment(self): "cartopy", "cfplot", "cf", + "xarray", + "umfive", ] # Ensure all expected components are present diff --git a/cf/test/test_kerchunk.py b/cf/test/test_kerchunk.py index b37d09718d..f1b8b38aa4 100644 --- a/cf/test/test_kerchunk.py +++ b/cf/test/test_kerchunk.py @@ -66,11 +66,11 @@ def test_kerchunk_original_filenames(self): self.assertEqual(k.get_original_filenames(), set()) def test_read_dict(self): - """Test cf.read with an Kerchunk dictionary.""" + """Test cf.read with a Kerchunk dictionary.""" with open(kerchunk_file, "r") as fh: d = json.load(fh) - with self.assertRaises(ValueError): + with self.assertRaises(Exception): cf.read(d) fs = fsspec.filesystem("reference", fo=d) diff --git a/cf/test/test_pp.py b/cf/test/test_pp.py index dfe5d259ef..5d48ccdac8 100644 --- a/cf/test/test_pp.py +++ b/cf/test/test_pp.py @@ -54,26 +54,19 @@ class ppTest(unittest.TestCase): def test_PP_read_um(self): f = cf.read(self.ppextradata)[0] - g = cf.read(self.ppextradata, um={"fmt": "pp"})[0] + g = cf.read(self.ppextradata)[0] self.assertTrue(f.equals(g)) - for vn in (4.5, 405, "4.5"): - g = cf.read(self.ppextradata, um={"fmt": "pp", "version": vn})[0] - self.assertTrue(f.equals(g)) + g = cf.read(self.ppextradata, um={"um_version": "4.5"})[0] + self.assertTrue(f.equals(g)) p = cf.read("wgdos_packed.pp")[0] p0 = cf.read( "wgdos_packed.pp", - um={ - "fmt": "PP", - "endian": "little", - "word_size": 4, - "version": 4.5, - "height_at_top_of_model": 23423.65, - }, + um={"um_version": "4.5", "height_at_top_of_model": 23423.65}, )[0] - self.assertTrue(p.equals(p0, verbose=2)) + self.assertTrue(p.equals(p0)) def test_load_stash2standard_name(self): f = cf.read(self.ppfile)[0] @@ -85,7 +78,7 @@ def test_load_stash2standard_name(self): f = cf.read(self.ppfile)[0] self.assertEqual(f.identity(), "NEW_NAME") self.assertEqual(f.Units, cf.Units("Pa")) - cf.load_stash2standard_name() + cf.load_stash2standard_name(reset=True) f = cf.read(self.ppfile)[0] self.assertEqual(f.identity(), "eastward_wind") self.assertEqual(f.Units, cf.Units("m s-1")) @@ -117,7 +110,7 @@ def test_PP_WGDOS_UNPACKING(self): g = cf.read(tmpfile)[0] self.assertTrue((f.array == array).all()) - self.assertTrue(f.equals(g, verbose=2)) + self.assertTrue(f.equals(g)) def test_PP_extra_data(self): f = cf.read(self.ppextradata)[0] @@ -128,9 +121,9 @@ def test_PP_extra_data(self): sites = f.dimension_coordinate("long_name=site") self.assertTrue(np.allclose(sites, [1, 2, 3])) - regions = f.auxiliary_coordinate("region").array + regions = f.auxiliary_coordinate("region") self.assertEqual( - regions.tolist(), + regions.array.tolist(), ["Northern Hemisphere", "Southern Hemisphere", "Global"], ) @@ -142,17 +135,16 @@ def test_PP_um_version(self): f = cf.read(self.ppfile)[0] self.assertEqual(f.get_property("um_version"), "11.0") - f = cf.read(self.ppfile, um={"version": "6.6.3"})[0] + f = cf.read(self.ppfile, um={"um_version": "6.6.3"})[0] self.assertEqual(f.get_property("um_version"), "6.6.3") - # def test_PP_file_object(self): - # # Can't yet read PP/UM from file-like objects - # with open(self.ppfile, "rb") as fh: - # with self.assertRaises(NotImplementedError): - # cf.read(fh) - # - # # Check that the file has been rewound - # self.assertEqual(fh.tell(), 0) + def test_PP_file_object(self): + # Can't yet read PP/UM from file-like objects + with open(self.ppfile, "rb") as fh: + cf.read(fh) + + # Check that the file has been rewound + self.assertEqual(fh.tell(), 0) if __name__ == "__main__": diff --git a/cf/test/test_quantization.py b/cf/test/test_quantization.py index f5a08326d1..0f3f543197 100644 --- a/cf/test/test_quantization.py +++ b/cf/test/test_quantization.py @@ -78,7 +78,7 @@ def test_quantization_read_write(self): f.set_quantize_on_write(q0) # Write the field and read it back in - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") g = cf.read(tmpfile1)[0] # Check that f and g have different data (i.e. that @@ -189,7 +189,7 @@ def test_quantization_write_exceptions(self): # digit_round f.set_quantize_on_write(algorithm="digitround", quantization_nsd=2) with self.assertRaises(ValueError): - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") # NetCDF3 formats for fmt in self.netcdf3_fmts: @@ -199,29 +199,29 @@ def test_quantization_write_exceptions(self): # Integer data type f.data.dtype = int with self.assertRaises(ValueError): - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") # Out-of-range quantization_nsd f.data.dtype = "float32" f.set_quantize_on_write(algorithm="bitgroom", quantization_nsd=8) with self.assertRaises(ValueError): - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") f.data.dtype = "float64" f.set_quantize_on_write(algorithm="bitgroom", quantization_nsd=16) with self.assertRaises(ValueError): - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") # Out-of-range quantization_nsb f.data.dtype = "float32" f.set_quantize_on_write(algorithm="bitround", quantization_nsb=24) with self.assertRaises(ValueError): - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") f.data.dtype = "float64" f.set_quantize_on_write(algorithm="bitround", quantization_nsb=53) with self.assertRaises(ValueError): - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile1, backend="netCDF4") def test_quantization_copy(self): """Test that quantization information gets copied.""" @@ -246,14 +246,14 @@ def test_quantization_backends(self): # Backends that allow quantisation-on-write for backend in ("netCDF4",): - cf.write(f, tmpfile1, netcdf_backend=backend) + cf.write(f, tmpfile1, backend=backend) # Backends that do not allow quantisation-on-write with self.assertRaises(NotImplementedError): - cf.write(f, tmpdir, fmt="ZARR3", netcdf_backend="zarr") + cf.write(f, tmpdir, fmt="ZARR3", backend="zarr") with self.assertRaises(NotImplementedError): - cf.write(f, tmpfile1, netcdf_backend="h5netcdf-h5py") + cf.write(f, tmpfile1, backend="h5netcdf-h5py") if __name__ == "__main__": diff --git a/cf/test/test_read_write.py b/cf/test/test_read_write.py index 1f3b60a122..5e4e537783 100644 --- a/cf/test/test_read_write.py +++ b/cf/test/test_read_write.py @@ -357,7 +357,7 @@ def test_write_netcdf_mode(self): tmpfile, fmt=fmt, mode="a", - netcdf_backend="netCDF4", + backend="netCDF4", ) f = cf.read(tmpfile) @@ -443,21 +443,21 @@ def test_write_netcdf_mode(self): tmpfile, fmt=fmt, mode="a", - netcdf_backend="netCDF4", + backend="netCDF4", ) # 2. now append f = cf.read(tmpfile) self.assertEqual(len(f), overall_length) # Also test the mode="r+" alias for mode="a". cf.write( - g, tmpfile, fmt=fmt, mode="w", netcdf_backend="netCDF4" + g, tmpfile, fmt=fmt, mode="w", backend="netCDF4" ) # 1. overwrite to wipe cf.write( append_ex_fields, tmpfile, fmt=fmt, mode="r+", - netcdf_backend="netCDF4", + backend="netCDF4", ) # 2. now append f = cf.read(tmpfile) self.assertEqual(len(f), overall_length) @@ -560,10 +560,10 @@ def test_write_netcdf_mode(self): # Check behaviour when append identical fields, as an edge case: cf.write( - g, tmpfile, fmt=fmt, mode="w", netcdf_backend="netCDF4" + g, tmpfile, fmt=fmt, mode="w", backend="netCDF4" ) # 1. overwrite to wipe cf.write( - g_copy, tmpfile, fmt=fmt, mode="a", netcdf_backend="netCDF4" + g_copy, tmpfile, fmt=fmt, mode="a", backend="netCDF4" ) # 2. now append f = cf.read(tmpfile) self.assertEqual(len(f), 2) @@ -653,7 +653,7 @@ def test_read_write_unlimited(self): else: backend = None - g = cf.read(tmpfile, netcdf_backend=backend)[0] + g = cf.read(tmpfile, backend=backend)[0] domain_axes = g.domain_axes() self.assertTrue(domain_axes["domainaxis0"].nc_is_unlimited(), fmt) @@ -869,7 +869,7 @@ def test_write_omit_data(self): f = self.f1 cf.write(f, tmpfile) - cf.write(f, tmpfile, omit_data="all", netcdf_backend="netCDF4") + cf.write(f, tmpfile, omit_data="all", backend="netCDF4") g = cf.read(tmpfile) self.assertEqual(len(g), 1) g = g[0] @@ -885,7 +885,7 @@ def test_write_omit_data(self): f, tmpfile, omit_data=("field", "dimension_coordinate"), - netcdf_backend="netCDF4", + backend="netCDF4", ) g = cf.read(tmpfile)[0] @@ -895,7 +895,7 @@ def test_write_omit_data(self): self.assertFalse(np.ma.count(g.construct("grid_latitude").array)) self.assertTrue(np.ma.count(g.construct("latitude").array)) - cf.write(f, tmpfile, omit_data="field", netcdf_backend="netCDF4") + cf.write(f, tmpfile, omit_data="field", backend="netCDF4") g = cf.read(tmpfile)[0] # Check that only the field data are missing @@ -909,7 +909,7 @@ def test_read_url(self): """Test reading remote url.""" for scheme in ("http", "https"): remote = f"{scheme}:///psl.noaa.gov/thredds/dodsC/Datasets/cru/crutem5/Monthlies/air.mon.anom.nobs.nc" - f = cf.read(remote, netcdf_backend="netCDF4") + f = cf.read(remote, backend="netCDF4") self.assertEqual(len(f), 1) @unittest.skipUnless( @@ -981,19 +981,19 @@ def test_read_zarr(self): z = cf.read(zarr_dataset, dataset_type="Zarr") self.assertEqual(len(z), 1) - def test_write_netcdf_backend(self): - """Test cf.write with different netCDF backends.""" + def test_write_backend(self): + """Test cf.write with different backends.""" f = self.f0 - cf.write(f, tmpfile0, netcdf_backend="h5netcdf-h5py") - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile0, backend="h5netcdf-h5py") + cf.write(f, tmpfile1, backend="netCDF4") f0 = cf.read(tmpfile0)[0] f1 = cf.read(tmpfile1)[0] self.assertTrue(f1.equals(f0)) f = cf.read(filename) - cf.write(f, tmpfile0, netcdf_backend="h5netcdf-h5py") - cf.write(f, tmpfile1, netcdf_backend="netCDF4") + cf.write(f, tmpfile0, backend="h5netcdf-h5py") + cf.write(f, tmpfile1, backend="netCDF4") f0 = cf.read(tmpfile0)[0] f1 = cf.read(tmpfile1)[0] self.assertTrue(f1.equals(f0)) @@ -1001,17 +1001,15 @@ def test_write_netcdf_backend(self): # Bad fmt/backend combinations for backend in ("netCDF4", "h5netcdf-h5py"): with self.assertRaises(ValueError): - cf.write(f, tmpfile, fmt="ZARR3", netcdf_backend=backend) + cf.write(f, tmpfile, fmt="ZARR3", backend=backend) for backend in ("zarr", "h5netcdf-h5py"): with self.assertRaises(ValueError): - cf.write( - f, tmpfile, fmt="NETCDF3_CLASSIC", netcdf_backend=backend - ) + cf.write(f, tmpfile, fmt="NETCDF3_CLASSIC", backend=backend) for backend in ("zarr",): with self.assertRaises(ValueError): - cf.write(f, tmpfile, fmt="NETCDF4", netcdf_backend=backend) + cf.write(f, tmpfile, fmt="NETCDF4", backend=backend) def test_write_h5py_options(self): """Test cf.write with h5py_options.""" @@ -1026,7 +1024,7 @@ def test_write_h5py_options(self): cf.write( f, tmpfile1, - netcdf_backend="h5netcdf-h5py", + backend="h5netcdf-h5py", h5py_options=h5py_options, ) self.assertTrue(os.path.getsize(tmpfile1) > size) @@ -1039,7 +1037,7 @@ def test_write_h5py_options(self): cf.write( f, tmpfile0, - netcdf_backend="netCDF4", + backend="netCDF4", h5py_options=h5py_options, ) @@ -1053,7 +1051,7 @@ def test_read_netcdf_file(self): f = self.f0 cf.write(f, tmpfile, fmt="NETCDF3_CLASSIC") - g = cf.read(tmpfile, netcdf_backend="netcdf_file")[0] + g = cf.read(tmpfile, backend="netcdf_file")[0] self.assertTrue(g.equals(f)) diff --git a/cf/test/test_zarr.py b/cf/test/test_zarr.py index 2310a83953..d9b164126f 100644 --- a/cf/test/test_zarr.py +++ b/cf/test/test_zarr.py @@ -248,13 +248,6 @@ def test_zarr_groups_dimension(self): self.assertTrue(z.equals(n)) self.assertTrue(z.equals(f)) - # Check that grouped netCDF datasets can only be read with - # 'closest_ancestor' - cf.read(grouped_file, group_dimension_search="closest_ancestor") - for gsn in ("furthest_ancestor", "local", "BAD VALUE"): - with self.assertRaises(ValueError): - cf.read(grouped_file, group_dimension_search=gsn) - def test_zarr_groups_DSG(self): """Test Zarr groups containing DSGs.""" f = cf.example_field(4) @@ -289,6 +282,9 @@ def test_zarr_groups_DSG(self): self.assertTrue(z.equals(n)) self.assertTrue(z.equals(f)) + # Check 'closest_ancestor' + cf.read(grouped_file, group_dimension_search="closest_ancestor") + def test_zarr_groups_geometry(self): """Test Zarr groups containing cell geometries.""" f = cf.example_field(6) diff --git a/cf/umread_lib/__init__.py b/cf/umread_lib/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cf/umread_lib/c-lib/Makefile b/cf/umread_lib/c-lib/Makefile deleted file mode 100644 index 8111177976..0000000000 --- a/cf/umread_lib/c-lib/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -HEADERS=umfile.h umfileint.h bits/*.h - -LIBRARY=umfile.so -CFLAGS=-Wall -fPIC - -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) - CC=gcc - LD=ld - LDFLAGS=-shared --build-id - CFLAGS += -DLINUX - LD_ARCHIVE_FLAGS=--whole-archive -endif -ifeq ($(UNAME_S),Darwin) - CC=clang - LD=clang - LDFLAGS=-dynamiclib - CFLAGS += -DOSX - LD_ARCHIVE_FLAGS=-force_load -endif - -CPP=gcc -E -P -OBJS = umfile.o error.o filetype.o \ - malloc.o linklist.o new_structs.o swap.o - -TYPE_DEP_LIBRARY = umfile_typedep.a -TYPE_DEP_DIR = type-dep -TYPE_DEP_LIBRARY_PATH = $(TYPE_DEP_DIR)/$(TYPE_DEP_LIBRARY) - -export CC CFLAGS CPP TYPE_DEP_LIBRARY - -.PHONY: clean all type-dep - -all: $(LIBRARY) - -clean: - rm -f $(OBJS) - $(MAKE) -C $(TYPE_DEP_DIR) clean - -type-dep: - $(MAKE) -C $(TYPE_DEP_DIR) - -$(LIBRARY): $(OBJS) type-dep - $(LD) $(LDFLAGS) -o $@ $(OBJS) $(LD_ARCHIVE_FLAGS) $(TYPE_DEP_LIBRARY_PATH) - -%.o: %.c $(HEADERS) - $(CC) $(CFLAGS) -c $< - -#bits/protos_sgl.h: bits/type_dep_protos.h -# $(CPP) -DBUILD_HDR -DSINGLE $< > $@ -# -#bits/protos_dbl.h: bits/type_dep_protos.h -# $(CPP) -DBUILD_HDR -DDOUBLE $< > $@ diff --git a/cf/umread_lib/c-lib/README b/cf/umread_lib/c-lib/README deleted file mode 100644 index 898bdc7e89..0000000000 --- a/cf/umread_lib/c-lib/README +++ /dev/null @@ -1,25 +0,0 @@ - -For source files in the subdirectory type-dep, two versions of each object -file will be built, one compiled with -DSINGLE, the other compiled with --DDOUBLE, and this will cause INTEGER and REAL and other dependent data types -to be typedef-ed accordingly, meaning that they are suitable for data read in -from 32- or 64-bit UM files without doing any casting. Functions should be -prototyped in bits/type_dep_protos.h. The symbol names in these -type-dependent functions will be renamed from 'foo' to 'foo_sgl' and 'foo_dbl' -as appropriate for the two compiled versions. This happens automatically by -the build process. This also includes renaming of external symbols where the -functions called are provided by other source files also in the type-dep -directory. - -Files in this (the parent) directory will be built without either -DSINGLE or --DDOUBLE, and no renaming of objects will take place, and the symbol names are -unmodified. They should be protyped in bits/type_indep_protos.h. - -In the few places where a type-independent function calls a type-dependent -function, the "_sgl" or "_dbl" part of the symbol name must be included -explicitly (this will usually be in a 'switch' statement in a despatch -function). These entry points into the type dependent code should be -prototyped in bits/type_dep_entry_protos.h, using the processor macro -WITH_LEN(foo), which expands to foo_sgl or foo_dbl; the header file is -included twice with different defines, so a single prototype statement using -this macro will suffice to prototype both versions of the function. diff --git a/cf/umread_lib/c-lib/bits/constants.h b/cf/umread_lib/c-lib/bits/constants.h deleted file mode 100644 index 608765c9c4..0000000000 --- a/cf/umread_lib/c-lib/bits/constants.h +++ /dev/null @@ -1,9 +0,0 @@ -/* int_missing_data is convention in input file */ -#define INT_MISSING_DATA -32768 - -/* for float comparisons */ -#if defined(SINGLE) -#define REAL_TOLERANCE 1e-5 -#elif defined(DOUBLE) -#define REAL_TOLERANCE 1e-13 -#endif diff --git a/cf/umread_lib/c-lib/bits/datatype.h b/cf/umread_lib/c-lib/bits/datatype.h deleted file mode 100644 index f51437f903..0000000000 --- a/cf/umread_lib/c-lib/bits/datatype.h +++ /dev/null @@ -1,15 +0,0 @@ -#if defined(DOUBLE) - -#define REAL float64_t -#define INTEGER int64_t -#define COMPILED_TYPE (double_precision) -#define WORD_SIZE 8 - -#elif defined(SINGLE) - -#define REAL float32_t -#define INTEGER int32_t -#define COMPILED_TYPE (single_precision) -#define WORD_SIZE 4 - -#endif diff --git a/cf/umread_lib/c-lib/bits/err_macros.h b/cf/umread_lib/c-lib/bits/err_macros.h deleted file mode 100644 index 1e84a37bee..0000000000 --- a/cf/umread_lib/c-lib/bits/err_macros.h +++ /dev/null @@ -1,36 +0,0 @@ - -/* error-checking macros */ - -/* these are to allow a compact way of incorporating error-checking of - * the return value of a function call, without obfuscating the basic purpose - * of the line of code, which is executing the function call. - * - * CKI used for integer functions which return negative value on failure - * CKP used for pointer functions which return NULL on failure - * CKF for floats for good measure (probably not used) - * - * put the ERRBLK (or ERRBLKI or ERRBLKP) at the end of the subroutine - */ - -#define FLT_ERR -1e38 - -#ifdef DEBUG -#define ERR abort(); -#else -/* ERR: unconditional branch */ -#define ERR goto err; -#endif - -#define CKI(i) if ((i) < 0){ ERR } -#define CKP(p) if ((p) == NULL){ ERR } -#define CKF(f) if ((f) == FLT_ERR){ ERR } - -/* ERRIF: conditional branch */ -#define ERRIF(i) if (i){ ERR } - -#define GRIPE gripe(__func__); -#define SWITCH_BUG switch_bug(__func__); ERR; -#define ERRBLK(rtn) err: GRIPE; return (rtn); -#define ERRBLKI ERRBLK(-1); -#define ERRBLKP ERRBLK(NULL); -#define ERRBLKF ERRBLK(FLT_ERR); diff --git a/cf/umread_lib/c-lib/bits/ordering.h b/cf/umread_lib/c-lib/bits/ordering.h deleted file mode 100644 index 1fb013f876..0000000000 --- a/cf/umread_lib/c-lib/bits/ordering.h +++ /dev/null @@ -1,16 +0,0 @@ -#if defined(LINUX) -#include -#elif defined(OSX) -#include -#else -#error build type not specified -#endif -#if BYTE_ORDER == LITTLE_ENDIAN -#define NATIVE_ORDERING little_endian -#define REVERSE_ORDERING big_endian -#elif BYTE_ORDER == BIG_ENDIAN -#define NATIVE_ORDERING big_endian -#define REVERSE_ORDERING little_endian -#else -#error BYTE_ORDER not defined properly -#endif diff --git a/cf/umread_lib/c-lib/bits/pp_header.h b/cf/umread_lib/c-lib/bits/pp_header.h deleted file mode 100644 index a442be6267..0000000000 --- a/cf/umread_lib/c-lib/bits/pp_header.h +++ /dev/null @@ -1,76 +0,0 @@ -/* ----------------------------------------------------------- */ -/* PP header interpretation */ - -#define N_INT_HDR 45 -#define N_REAL_HDR 19 -#define N_HDR (N_INT_HDR + N_REAL_HDR) - -#define INDEX_LBYR 0 -#define INDEX_LBMON 1 -#define INDEX_LBDAT 2 -#define INDEX_LBHR 3 -#define INDEX_LBMIN 4 -#define INDEX_LBDAY 5 -#define INDEX_LBYRD 6 -#define INDEX_LBMOND 7 -#define INDEX_LBDATD 8 -#define INDEX_LBHRD 9 -#define INDEX_LBMIND 10 -#define INDEX_LBDAYD 11 -#define INDEX_LBTIM 12 -#define INDEX_LBFT 13 -#define INDEX_LBLREC 14 -#define INDEX_LBCODE 15 -#define INDEX_LBHEM 16 -#define INDEX_LBROW 17 -#define INDEX_LBNPT 18 -#define INDEX_LBEXT 19 -#define INDEX_LBPACK 20 -#define INDEX_LBREL 21 -#define INDEX_LBFC 22 -#define INDEX_LBCFC 23 -#define INDEX_LBPROC 24 -#define INDEX_LBVC 25 -#define INDEX_LBRVC 26 -#define INDEX_LBEXP 27 -#define INDEX_LBBEGIN 28 -#define INDEX_LBNREC 29 -#define INDEX_LBPROJ 30 -#define INDEX_LBTYP 31 -#define INDEX_LBLEV 32 -#define INDEX_LBRSVD1 33 -#define INDEX_LBRSVD2 34 -#define INDEX_LBRSVD3 35 -#define INDEX_LBRSVD4 36 -#define INDEX_LBSRCE 37 -#define INDEX_LBUSER1 38 -#define INDEX_LBUSER2 39 -#define INDEX_LBUSER3 40 -#define INDEX_LBUSER4 41 -#define INDEX_LBUSER5 42 -#define INDEX_LBUSER6 43 -#define INDEX_LBUSER7 44 -#define INDEX_BULEV 0 -#define INDEX_BHULEV 1 -#define INDEX_BRSVD3 2 -#define INDEX_BRSVD4 3 -#define INDEX_BDATUM 4 -#define INDEX_BACC 5 -#define INDEX_BLEV 6 -#define INDEX_BRLEV 7 -#define INDEX_BHLEV 8 -#define INDEX_BHRLEV 9 -#define INDEX_BPLAT 10 -#define INDEX_BPLON 11 -#define INDEX_BGOR 12 -#define INDEX_BZY 13 -#define INDEX_BDY 14 -#define INDEX_BZX 15 -#define INDEX_BDX 16 -#define INDEX_BMDI 17 -#define INDEX_BMKS 18 - -#define LOOKUP(rec, index) (((INTEGER *)((rec)->int_hdr))[index]) -#define RLOOKUP(rec, index) (((REAL *)((rec)->real_hdr))[index]) - -/* ----------------------------------------------------------- */ diff --git a/cf/umread_lib/c-lib/bits/type_dep_entry_protos.h b/cf/umread_lib/c-lib/bits/type_dep_entry_protos.h deleted file mode 100644 index 9f10ef0756..0000000000 --- a/cf/umread_lib/c-lib/bits/type_dep_entry_protos.h +++ /dev/null @@ -1,42 +0,0 @@ -/* prototypes for entry points to the type-dependent code; - * need to be usable without INTEGER, etc, being defined, - * so void* used for some pointer types that have specific - * pointer types defined in the function declarations themselves - */ - -void WITH_LEN(swap_bytes)(void *ptr, size_t num_words); - -int WITH_LEN(get_type_and_num_words_core)(const void *int_hdr, - Data_type *type_rtn, - size_t *num_words_rtn); - -int WITH_LEN(read_hdr_at_offset)(int fd, - size_t header_offset, - Byte_ordering byte_ordering, - void *int_hdr_rtn, - void *real_hdr_rtn); - -int WITH_LEN(read_record_data_core)(int fd, - size_t data_offset, - size_t disk_length, - Byte_ordering byte_ordering, - const void *int_hdr, - const void *real_hdr, - size_t nwords, - void *data_return); - -File *WITH_LEN(file_parse_core)(int fd, - File_type file_type); - -size_t WITH_LEN(get_extra_data_offset_and_length_core)(const void *int_hdr, - size_t data_offset, - size_t disk_length, - size_t *extra_data_offset_rtn, - size_t *extra_data_length_rtn); - -int WITH_LEN(read_extra_data_core)(int fd, - size_t extra_data_offset, - size_t extra_data_length, - Byte_ordering byte_ordering, - void *extra_data_rtn); - diff --git a/cf/umread_lib/c-lib/bits/type_dep_protos.h b/cf/umread_lib/c-lib/bits/type_dep_protos.h deleted file mode 100644 index 00e9a76c9e..0000000000 --- a/cf/umread_lib/c-lib/bits/type_dep_protos.h +++ /dev/null @@ -1,105 +0,0 @@ -/* interpret_header.c */ -Data_type get_type(const INTEGER *int_hdr); -size_t get_num_data_words (const INTEGER *int_hdr); -size_t get_extra_data_length(const INTEGER *int_hdr); - -int var_is_missing(const INTEGER *int_hdr); -int get_var_stash_model(const INTEGER *int_hdr); -int get_var_stash_section(const INTEGER *int_hdr); -int get_var_stash_item(const INTEGER *int_hdr); -int get_var_compression(const INTEGER *int_hdr); -int get_var_gridcode(const INTEGER *int_hdr); -int get_var_packing(const INTEGER *int_hdr); -REAL get_var_real_fill_value(const REAL *int_hdr); - -/* read.c */ - -size_t read_words(int fd, - void *ptr, - size_t num_words, - Byte_ordering byte_ordering); - -int read_extra_data_at_offset(int fd, - size_t extra_data_offset, - size_t extra_data_length, - Byte_ordering byte_ordering, - void *extra_data_rtn); - -int read_hdr(int fd, - Byte_ordering byte_ordering, - INTEGER *int_hdr_rtn, - REAL *real_hdr_rtn); - -Rec *get_record(File *file, List *heaplist); -int read_all_headers(File *file, List *heaplist); -size_t skip_fortran_record(File *file); -int skip_word(File *file); -int read_all_headers_pp(File *file, List *heaplist); -int read_all_headers_ff(File *file, List *heaplist); -size_t get_ff_disk_length(INTEGER *ihdr); -int get_valid_records_ff(int fd, - Byte_ordering byte_ordering, - size_t hdr_start, size_t hdr_size, int nrec, - int valid[], int *n_valid_rec_return); -int unpack_run_length_encoded(REAL *datain, INTEGER nin, REAL *dataout, INTEGER nout, REAL mdi); - -/* process_vars.c */ -int process_vars(File *file, List *heaplist); -int test_skip_var(const Rec *rec); -int initialise_records(Rec **recs, int nrec, List *heaplist); -int get_vars(int nrec, Rec **recs, - List *vars, - List *heaplist); -int set_disambig_index(Z_axis *z_axis, T_axis *t_axis, - Rec **recs, int nvrec, int svindex); -int add_axes_to_var(Var *var, - Z_axis *z_axis, T_axis *t_axis, - List *z_axes, List *t_axes, - List *heaplist); -int grid_supported(INTEGER *int_hdr); -int var_has_regular_z_t(Z_axis *z_axis, T_axis *t_axis, Rec **recs, int nvrec); - -/* level.c */ -int lev_set(Level *lev, const Rec *rec); -Lev_type level_type(const Rec *rec); - -/* date_and_time.c */ -REAL get_mean_period(const Time *time); -int is_time_mean(INTEGER LBTIM); -REAL time_diff(INTEGER lbtim, const Date *date, const Date *orig_date); -REAL sec_to_day(int64_t seconds); -Calendar_type calendar_type(INTEGER type); -int64_t gregorian_to_secs(const Date *date); -int time_set(Time *time, const Rec *rec); - -/* axes.c */ -Z_axis *new_z_axis(List *heaplist); -int free_z_axis(Z_axis *z_axis, List *heaplist); -T_axis *new_t_axis(List *heaplist); -int free_t_axis(T_axis *t_axis, List *heaplist); -int t_axis_add(T_axis *t_axis, const Time *time, - int *index_return, List *heaplist); -int z_axis_add(Z_axis *z_axis, const Level *lev, - int *index_return, List *heaplist); - -/* compare.c */ -int compare_records_between_vars(const Rec *a, const Rec *b); -int compare_mean_periods(const Rec *a, const Rec *b); -int compare_records_within_var(const Rec *a, const Rec *b); -int compare_records(const void *p1, const void *p2); -int records_from_different_vars(const Rec *a, const Rec *b); -int compare_lists(const List *l1, const List *l2, int (*compfunc)(const void*, const void*)); -int compare_levels(const void *p1, const void *p2); -int compare_times(const void *p1, const void *p2); -int compare_dates(const Date *a, const Date *b); - -/* unwgdos.c */ -int unwgdos(void *datain, int nbytes, REAL *dataout, int nout, REAL mdi); - - -/* Debug_dump.c */ -void debug_dump_all_headers(File *file); - -#ifdef MAIN -int main(); -#endif diff --git a/cf/umread_lib/c-lib/bits/type_dep_redefs.h b/cf/umread_lib/c-lib/bits/type_dep_redefs.h deleted file mode 100644 index 93c81a24d0..0000000000 --- a/cf/umread_lib/c-lib/bits/type_dep_redefs.h +++ /dev/null @@ -1,137 +0,0 @@ -#if defined(SINGLE) - -#define swap_bytes swap_bytes_sgl - -#define calendar_type calendar_type_sgl -#define compare_dates compare_dates_sgl -#define compare_levels compare_levels_sgl -#define compare_lists compare_lists_sgl -#define compare_mean_periods compare_mean_periods_sgl -#define compare_records compare_records_sgl -#define compare_records_between_vars compare_records_between_vars_sgl -#define compare_records_within_var compare_records_within_var_sgl -#define compare_times compare_times_sgl -#define debug_dump_all_headers debug_dump_all_headers_sgl -#define file_parse_core file_parse_core_sgl -#define free_t_axis free_t_axis_sgl -#define free_z_axis free_z_axis_sgl -#define get_extra_data_length get_extra_data_length_sgl -#define get_extra_data_offset_and_length_core get_extra_data_offset_and_length_core_sgl -#define get_ff_disk_length get_ff_disk_length_sgl -#define get_mean_period get_mean_period_sgl -#define get_num_data_words get_num_data_words_sgl -#define get_record get_record_sgl -#define get_type get_type_sgl -#define get_type_and_num_words_core get_type_and_num_words_core_sgl -#define get_valid_records_ff get_valid_records_ff_sgl -#define get_var_compression get_var_compression_sgl -#define get_var_gridcode get_var_gridcode_sgl -#define get_var_packing get_var_packing_sgl -#define get_var_real_fill_value get_var_real_fill_value_sgl -#define get_var_stash_item get_var_stash_item_sgl -#define get_var_stash_model get_var_stash_model_sgl -#define get_var_stash_section get_var_stash_section_sgl -#define get_vars get_vars_sgl -#define gregorian_to_secs gregorian_to_secs_sgl -#define grid_supported grid_supported_sgl -#define initialise_records initialise_records_sgl -#define is_time_mean is_time_mean_sgl -#define lev_set lev_set_sgl -#define level_type level_type_sgl -#define new_t_axis new_t_axis_sgl -#define new_z_axis new_z_axis_sgl -#define process_vars process_vars_sgl -#define read_all_headers read_all_headers_sgl -#define read_all_headers_ff read_all_headers_ff_sgl -#define read_all_headers_pp read_all_headers_pp_sgl -#define read_extra_data_core read_extra_data_core_sgl -#define read_hdr read_hdr_sgl -#define read_hdr_at_offset read_hdr_at_offset_sgl -#define read_record_data_core read_record_data_core_sgl -#define read_record_data_dummy read_record_data_dummy_sgl -#define read_words read_words_sgl -#define records_from_different_vars records_from_different_vars_sgl -#define sec_to_day sec_to_day_sgl -#define set_disambig_index set_disambig_index_sgl -#define skip_fortran_record skip_fortran_record_sgl -#define skip_word skip_word_sgl -#define t_axis_add t_axis_add_sgl -#define test_skip_var test_skip_var_sgl -#define time_diff time_diff_sgl -#define time_set time_set_sgl -#define unpack_run_length_encoded unpack_run_length_encoded_sgl -#define unwgdos unwgdos_sgl -#define var_has_regular_z_t var_has_regular_z_t_sgl -#define var_is_missing var_is_missing_sgl -#define z_axis_add z_axis_add_sgl - -#elif defined(DOUBLE) - -#define swap_bytes swap_bytes_dbl - -#define calendar_type calendar_type_dbl -#define compare_dates compare_dates_dbl -#define compare_levels compare_levels_dbl -#define compare_lists compare_lists_dbl -#define compare_mean_periods compare_mean_periods_dbl -#define compare_records compare_records_dbl -#define compare_records_between_vars compare_records_between_vars_dbl -#define compare_records_within_var compare_records_within_var_dbl -#define compare_times compare_times_dbl -#define debug_dump_all_headers debug_dump_all_headers_dbl -#define file_parse_core file_parse_core_dbl -#define free_t_axis free_t_axis_dbl -#define free_z_axis free_z_axis_dbl -#define get_extra_data_length get_extra_data_length_dbl -#define get_extra_data_offset_and_length_core get_extra_data_offset_and_length_core_dbl -#define get_ff_disk_length get_ff_disk_length_dbl -#define get_mean_period get_mean_period_dbl -#define get_num_data_words get_num_data_words_dbl -#define get_record get_record_dbl -#define get_type get_type_dbl -#define get_type_and_num_words_core get_type_and_num_words_core_dbl -#define get_valid_records_ff get_valid_records_ff_dbl -#define get_var_compression get_var_compression_dbl -#define get_var_gridcode get_var_gridcode_dbl -#define get_var_packing get_var_packing_dbl -#define get_var_real_fill_value get_var_real_fill_value_dbl -#define get_var_stash_item get_var_stash_item_dbl -#define get_var_stash_model get_var_stash_model_dbl -#define get_var_stash_section get_var_stash_section_dbl -#define get_vars get_vars_dbl -#define gregorian_to_secs gregorian_to_secs_dbl -#define grid_supported grid_supported_dbl -#define initialise_records initialise_records_dbl -#define is_time_mean is_time_mean_dbl -#define lev_set lev_set_dbl -#define level_type level_type_dbl -#define new_t_axis new_t_axis_dbl -#define new_z_axis new_z_axis_dbl -#define process_vars process_vars_dbl -#define read_all_headers read_all_headers_dbl -#define read_all_headers_ff read_all_headers_ff_dbl -#define read_all_headers_pp read_all_headers_pp_dbl -#define read_extra_data_core read_extra_data_core_dbl -#define read_hdr read_hdr_dbl -#define read_hdr_at_offset read_hdr_at_offset_dbl -#define read_record_data_core read_record_data_core_dbl -#define read_record_data_dummy read_record_data_dummy_dbl -#define read_words read_words_dbl -#define records_from_different_vars records_from_different_vars_dbl -#define sec_to_day sec_to_day_dbl -#define set_disambig_index set_disambig_index_dbl -#define skip_fortran_record skip_fortran_record_dbl -#define skip_word skip_word_dbl -#define t_axis_add t_axis_add_dbl -#define test_skip_var test_skip_var_dbl -#define time_diff time_diff_dbl -#define time_set time_set_dbl -#define unpack_run_length_encoded unpack_run_length_encoded_dbl -#define unwgdos unwgdos_dbl -#define var_has_regular_z_t var_has_regular_z_t_dbl -#define var_is_missing var_is_missing_dbl -#define z_axis_add z_axis_add_dbl - -#else -#error Need to compile this file with -DSINGLE or -DDOUBLE -#endif diff --git a/cf/umread_lib/c-lib/bits/type_indep_protos.h b/cf/umread_lib/c-lib/bits/type_indep_protos.h deleted file mode 100644 index 2bd249fc2f..0000000000 --- a/cf/umread_lib/c-lib/bits/type_indep_protos.h +++ /dev/null @@ -1,61 +0,0 @@ -/* PROTOTYPES */ - -/* error.c */ -void switch_bug(const char *routine); -void gripe(const char *routine); -void error_mesg(const char *fmt, ...); -void debug(const char *fmt, ...); -void errorhandle_init(); - -/* malloc.c */ -void *malloc_(size_t size, List *heaplist); -void *dup_(const void *inptr, size_t size, List *heaplist); -int free_(void *ptr, List *heaplist); -int free_all(List *heaplist); - -/* swap.c */ -/* NB the _sgl and _dbl functions are explicitly coded, hence not handled by type_dep - * (reason: even in the DOUBLE case, we need swap_bytes_sgl available for 32-bit packed data) - */ -void swap_bytes_sgl(void *ptr, size_t num_words); -void swap_bytes_dbl(void *ptr, size_t num_words); - -/* linklist.c */ -typedef int(*free_func) (void *, List *); - -void *list_new(List *heaplist); -int list_free(List *list, int free_ptrs, List *heaplist); -int list_size(const List *list); -int list_add(List *list, void *ptr, List *heaplist); -int list_add_or_find(List *list, - void *item_in, - int (*compar)(const void *, const void *), - int matchval, - free_func free_function, - int *index_return, - List *heaplist); -int list_del(List *list, void *ptr, List *heaplist); -int list_del_by_listel(List *list, List_element *p, List *heaplist); -int list_startwalk(const List *list, List_handle *handle); -void *list_walk(List_handle *handle, int return_listel); -void *list_find(List *list, - const void *item, - int (*compar)(const void *, const void *), - int matchval, - int *index_return); -int list_copy_to_ptr_array(const List *list, - int *n_return, - void *ptr_array_return, - List *heaplist); - -/* filetype.c */ -int detect_file_type_(int fd, File_type *file_type); - -/* new_structs.c */ -Rec *new_rec(int word_size, List *heaplist); -int free_rec(Rec *rec, List *heaplist); -Var *new_var(List *heaplist); -int free_var(Var *var, List *heaplist); -File *new_file(); -int free_file(File *file); - diff --git a/cf/umread_lib/c-lib/bits/typedefs.h b/cf/umread_lib/c-lib/bits/typedefs.h deleted file mode 100644 index 04db2a204c..0000000000 --- a/cf/umread_lib/c-lib/bits/typedefs.h +++ /dev/null @@ -1,208 +0,0 @@ -#include - -typedef float float32_t; -typedef double float64_t; - -enum { single_precision, double_precision }; - -/*---------------------------*/ -/* for linked list */ - -struct _list_element -{ - void *ptr; - struct _list_element *prev; - struct _list_element *next; -}; -typedef struct _list_element List_element; - -typedef struct -{ - int n; - List_element *first; - List_element *last; -} - List; - -typedef struct -{ - /* This is a little structure which stores the information needed for - * pp_list_walk. Its main purpose is to store the position outside the list - * structure itself, so that for read-only scanning of the list, the PPlist* - * can be declared as const. - */ - List_element *current; - const List *list; -} - List_handle; - -/*---------------------------*/ - -typedef enum -{ - pseudo_lev_type, - height_lev_type, - depth_lev_type, - hybrid_sigmap_lev_type, - hybrid_height_lev_type, - pressure_lev_type, - soil_lev_type, - boundary_layer_top_lev_type, - top_of_atmos_lev_type, - mean_sea_lev_type, - surface_lev_type, - tropopause_lev_type, - other_lev_type -} - Lev_type; - -typedef enum -{ - gregorian, - cal360day, - model -} - Calendar_type; - -typedef enum -{ - lev_type, - hybrid_sigmap_a_type, - hybrid_sigmap_b_type, - hybrid_height_a_type, - hybrid_height_b_type -} - Lev_val_type; - - -#if defined(INTEGER) - -typedef struct -{ - INTEGER year; - INTEGER month; - INTEGER day; - INTEGER hour; - INTEGER minute; - INTEGER second; -} - Date; - -typedef struct -{ - /* this is a value on time axis */ - INTEGER type; - Date time1; - Date time2; -} - Time; - -typedef struct -{ - List *values; -} - T_axis; - -typedef struct -{ - Lev_type type; - - union - { - struct - { - REAL level; -#ifdef BDY_LEVS - REAL ubdy_level; - REAL lbdy_level; -#endif - } - misc; - - struct - { - REAL a; - REAL b; -#ifdef BDY_LEVS - REAL ubdy_a; - REAL ubdy_b; - REAL lbdy_a; - REAL lbdy_b; -#endif - } - hybrid_sigmap; - - struct - { - REAL a; - REAL b; -#ifdef BDY_LEVS - REAL ubdy_a; - REAL ubdy_b; - REAL lbdy_a; - REAL lbdy_b; -#endif - } - hybrid_height; - - struct - { - INTEGER index; - } - pseudo; - } - values; -} - Level; - -typedef struct -{ - List *values; -} - Z_axis; - -#else - -typedef void Z_axis; -typedef void T_axis; -typedef void Time; -typedef void Level; - -#endif - -/*---------------------------*/ - -struct _File -{ - List *heaplist; - int nrec; - Rec **recs; -}; - -struct _Var -{ - Z_axis *z_axis; - T_axis *t_axis; - Rec *first_rec; - int first_rec_no; - int last_rec_no; -}; - -struct _Rec -{ - Level *lev; - Time *time; - int zindex; /* index on z axis within a variable - used for detecting vars with irreg z,t */ - int tindex; /* index on t axis within a variable - used for detecting vars with irreg z,t */ - int disambig_index; /* index used for splitting variables with irreg z,t into - * sets of variables with regular z,t */ - int supervar_index; /* when a variable is split, this is set to an index which is common - * across the set, but different from sets generated from other - * super-variables - */ - float64_t mean_period; /* period (in days) of time mean - (store here so as to calculate once only) */ -}; - -/*---------------------------*/ - diff --git a/cf/umread_lib/c-lib/error.c b/cf/umread_lib/c-lib/error.c deleted file mode 100644 index 5734318bbc..0000000000 --- a/cf/umread_lib/c-lib/error.c +++ /dev/null @@ -1,68 +0,0 @@ -#include -#include - -#include "umfileint.h" - -static FILE *output = NULL; - -static int verbose; - -static const int do_debug = 1; - -void switch_bug(const char *routine) -{ - gripe("no match in switch statement in routine; " - "may indicate coding bug in umfile or unexpected header value"); -} - -void gripe(const char *routine) -{ - if (verbose || do_debug) - { - fprintf(output, "umfile: error condition detected in routine %s\n", - routine); - fflush(output); - } - verbose = 0; -} - -void error_mesg(const char *fmt, ...) -{ - va_list args; - - if (verbose || do_debug) - { - va_start(args, fmt); - vfprintf(output, fmt, args); - fprintf(output, "\n"); - va_end(args); - fflush(output); - } - verbose = 0; -} - -void debug(const char *fmt, ...) -{ - va_list args; - - if (do_debug) - { - va_start(args, fmt); - fprintf(output, "DEBUG: "); - vfprintf(output, fmt, args); - fprintf(output, "\n"); - va_end(args); - fflush(output); - } -} - -void errorhandle_init() -{ - /* init sets verbose -- called at start of each of the interface routines -- - * then first call to error will cause a diagnostic to be printed, - * but then unsets verbose to avoid series of knock-on messages - */ - verbose = 1; - if (output == NULL) - output = stderr; -} diff --git a/cf/umread_lib/c-lib/filetype.c b/cf/umread_lib/c-lib/filetype.c deleted file mode 100644 index 4ea24a4f2a..0000000000 --- a/cf/umread_lib/c-lib/filetype.c +++ /dev/null @@ -1,200 +0,0 @@ - /* Routines for auto-determining file type from the start of the file - * contents. - * - * ================= - * Fields file tests - * ================= - * - * These are done first. We test the second word, which should be the - * submodel ID - is this 1, 2 or 4? - * - * ==> Could the fields file test give a false +ve with PP file? - * - * Test for fields file only true if the first 16 bytes, when viewed - * as 4 32-bit integers, are one of the following: - * - * bytes 1-4 bytes 5-8 bytes 9-12 bytes 13-16 - * --------------------------------------------------- - * any any 0 1/2/4(BE) <-- 64-bit BE FF - * any any 1/2/4(LE) 0 <-- 64-bit LE FF - * any 1/2/4(BE) any any <-- 32-bit BE FF - * any 1/2/4(LE) any any <-- 32-bit LE FF - * - * For PP files, we in fact have: - * - * 0 512/1024(BE) 0 lbyr(BE) <--- 64-bit BE PP - * 512/1024(LE) 0 lbyr(LE) 0 <--- 64-bit LE PP - * 256/512(BE) lbyr(BE) lbmon(BE) lbdat(BE) <--- 32-bit BE PP - * 256/512(LE) lbyr(LE) lbmon(LE) lbdat(LE) <--- 32-bit LE PP - * - * Possible false positives: - * - * - any PP with lbyr=1/2/4 looks like FF of same length and endianness - * - 32-bit BE PP with lbmon=0, lbdat=1/2/4 looks like 64-bit BE FF - * - 32-bit LE PP with lbmon=1/2/4, lbdat=0 looks like 64-bit LE FF - * - * Do we care about these cases? - * lbyr=1/2/4: probably NO - * lbmon=0, lbdat non-zero: probably NO - * lbmon=1/2/4, lbdat=0 - possible monthly climatology? <== YES - * - * Always option for user to force file type, but: - * **FIXME**: additional test could help. - * - * ======== - * PP tests - * ======== - * - * If the fields-file test is false, then test for types of plain PP file. - * Here we test the first word, which should be record length (put there by - * fortran). - * - * Check first for a 64-bit PP file, but in addition to the first word being - * a valid possibility, this must also pass the stringent test of every - * other 32-bit value being zero throughout the first 14 64-bit words, - * although because of endianness issues we accept the sequence of - * alternating zeros to start either at the first or second 32-bit value. - * The point is that for a true 64-bit file, these should all be small - * integers, so the most significant bytes will be 0. (The first possibly - * large integer is the 15th: LBLREC.) However, for a 32-bit file this test - * will span the first 28 elements. Even if the date elements (first 12 - * words) are all 0, LBROW (18) and LBNPT (19) should both be non-zero, so - * both the set of even-positioned integers and the set of odd-positioned - * integers will each contain at least one non-zero value, and the test will - * fail. - * - * If the 64-bit tests fail, try 32-bit. - */ - -#include -#include -#include - -#include "umfileint.h" - -/* values passed to valid_um_word2 and valid_pp_word1 could be 32 or - * 64-bit. Declare as longer of these two (int64_t), and shorter will be - * accommodated also. - */ - -static int valid_um_word2(int64_t val) -{ - /* second word should be 1,2 or 4, reflecting model ID in fixed length - header */ - return (val == 1 || val == 2 || val == 4); -} - -static int valid_pp_word1(int64_t val, int wsize) -{ - /* first word should be integer from Fortan representing length of header - record */ - return (val == 64 * wsize || val == 128 * wsize); -} - -/* tests whether sequence of integers has every other value = 0, but - * only when starting at first value - */ -static int is_alternating_zeros_without_offset(int32_t *vals, int num_pairs) -{ - int i; - int32_t *p; - p = vals; - for (i = 0; i < num_pairs; i++) - { - if (*p != 0) return 0; - p += 2; - } - return 1; -} - -/* tests whether sequence of integers has every other value = 0, but - * can either be when starting at first or second value - */ -static int is_alternating_zeros(int32_t *vals, int num_pairs) -{ - return (is_alternating_zeros_without_offset(vals, num_pairs) || - is_alternating_zeros_without_offset(vals + 1, num_pairs)); -} - -#define N_PAIRS 14 -int detect_file_type_(int fd, File_type *file_type) -{ - int32_t data4[2 * N_PAIRS], data4s[2]; - int64_t data8[2], data8s[2]; - - /* read and store first 24 4-byte words - * and store first two integers of this according to possible suppositions - * of 4- or 8- byte, and of native or swapped byte ordering - */ - lseek(fd, 0, SEEK_SET); - if(read(fd, data4, 8 * N_PAIRS) != 8 * N_PAIRS) return 1; - - memcpy(data8, data4, 16); - - memcpy(data4s, data4, 8); - swap_bytes_sgl(data4s, 2); - - memcpy(data8s, data4, 16); - swap_bytes_dbl(data8s, 2); - - - /* --- Fields file cases -- */ - - if (valid_um_word2(data4[1])) - { - file_type->fmt = fields_file; - file_type->byte_ordering = NATIVE_ORDERING; - file_type->word_size = 4; - } - else if (valid_um_word2(data8[1])) - { - file_type->fmt = fields_file; - file_type->byte_ordering = NATIVE_ORDERING; - file_type->word_size = 8; - } - else if (valid_um_word2(data4s[1])) - { - file_type->fmt = fields_file; - file_type->byte_ordering = REVERSE_ORDERING; - file_type->word_size = 4; - } - else if (valid_um_word2(data8s[1])) - { - file_type->fmt = fields_file; - file_type->byte_ordering = REVERSE_ORDERING; - file_type->word_size = 8; - } - - /* --- Plain PP cases -- */ - - else if (valid_pp_word1(data8[0], 8) && is_alternating_zeros(data4, N_PAIRS)) - { - file_type->fmt = plain_pp; - file_type->byte_ordering = NATIVE_ORDERING; - file_type->word_size = 8; - } - else if (valid_pp_word1(data8s[0], 8) && is_alternating_zeros(data4, N_PAIRS)) - { - file_type->fmt = plain_pp; - file_type->byte_ordering = REVERSE_ORDERING; - file_type->word_size = 8; - } - else if (valid_pp_word1(data4[0], 4)) - { - file_type->fmt = plain_pp; - file_type->byte_ordering = NATIVE_ORDERING; - file_type->word_size = 4; - } - else if (valid_pp_word1(data4s[0], 4)) - { - file_type->fmt = plain_pp; - file_type->byte_ordering = REVERSE_ORDERING; - file_type->word_size = 4; - } - else - { - /* type not identified */ - return 1; - } - return 0; -} diff --git a/cf/umread_lib/c-lib/linklist.c b/cf/umread_lib/c-lib/linklist.c deleted file mode 100644 index 536ff828ad..0000000000 --- a/cf/umread_lib/c-lib/linklist.c +++ /dev/null @@ -1,261 +0,0 @@ -#include "umfileint.h" - -/* LINKED LIST FUNCTIONS */ - -void *list_new(List *heaplist) -{ - List *list; - CKP( list = malloc_(sizeof(List), heaplist) ); - list->first = NULL; - list->last = NULL; - list->n = 0; - return list; - ERRBLKP; -} - -/* This function frees a list; - * Set free_ptrs if the pointers which have been explicitly stored on the - * list (2nd argument to list_add) are to be freed, not just the pointers - * which are implicit in the linked list structure. NB there is no further - * recursion, in the sense that if the stored pointers are to datatypes which - * contain further pointers then these may have to be freed explicitly. - */ -int list_free(List *list, int free_ptrs, List *heaplist) -{ - List_element *p, *next; - CKP(list); - for (p = list->first ; p != NULL ; p = next) - { - next = p->next; - if (free_ptrs) - CKI( free_(p->ptr, heaplist) ); - CKI( free_(p,heaplist) ); - } - CKI( free_(list, heaplist) ); - return 0; - ERRBLKI; -} - - -int list_size(const List *list) -{ - CKP(list); - return list->n; - ERRBLKI; -} - - -int list_add(List *list, void *ptr, List *heaplist) -{ - List_element *el; - CKP(list); - CKP( el = malloc_(sizeof(List_element), heaplist) ); - list->n ++; - el->ptr = ptr; - el->next = NULL; - if (list->first == NULL) - { - el->prev = NULL; - list->first = list->last = el; - } - else - { - list->last->next = el; - el->prev = list->last; - list->last = el; - } - return 0; - ERRBLKI; -} - -/* list_add_or_find takes a pointer to an item and tries to find it on the - * list, using the comparision function as in list_find. - * - * If it already exists, it changes the item to point to the old value, and - * calls the supplied function (if non-null) to free the item. If it does - * not exist, it adds the item to the list. - * - * Return values: - * 0 time already existed in axis - * 1 time has been added to axis - * -1 an error occurred (probably in memory allocation) - * - * NOTE: the return value of this function may be tested with the CKI() macro. - * Do not add non-error cases with negative return values. - * - * NOTE 2: The item is formally declared as a void* but the - * thing pointed to should itself be a pointer (to heap memory), - * so you should pass in a foo** of some sort. The only reason for - * not declaring as void** is that void* has the special property of - * being treated by the compiler as a generic pointer hence no - * warnings about incompatible pointer type - */ -int list_add_or_find(List *list, - void *item_in, - int (*compar)(const void *, const void *), - int matchval, - free_func free_function, - int *index_return, - List *heaplist) -{ - void *oldptr; - void **item = (void**) item_in; - - if ((oldptr = list_find(list, *item, compar, - matchval, index_return)) != NULL) - { - if (free_function != NULL) - CKI( free_function(*item, heaplist) ); - *item = oldptr; - return 0; - } - else - { - CKI( list_add(list, *item, heaplist) ); - if (index_return != NULL) - *index_return = list_size(list) - 1; - return 1; - } - ERRBLKI; -} - - -/* call list_del to find a pointer ("ptr" element contained within the - * listel structure) on the list, and then delete that element from the list, - * or call list_del_by_listel directly (more efficient) if you already - * have the listel structure pointer for what you want to delete. - */ - -int list_del(List *list, void *ptr, List *heaplist) -{ - List_element *p; - CKP(list); - for (p = list->first; p != NULL; p = p->next) - if (p->ptr == ptr) - return list_del_by_listel(list, p, heaplist); - - /* if what we're trying to remove is not found, fall through - * to error exit - */ - ERRBLKI; -} - - -int list_del_by_listel(List *list, List_element *p, List *heaplist) -{ - List_element *prev, *next; - next = p->next; - prev = p->prev; - if (next != NULL) next->prev = prev; - if (prev != NULL) prev->next = next; - if (p == list->first) list->first = next; - if (p==list->last) list->last = prev; - CKI( free_(p, heaplist) ); - list->n --; - return 0; - ERRBLKI; -} - -/* call list_startwalk before a sequence of calls to list_walk */ -int list_startwalk(const List *list, List_handle *handle) -{ - CKP(list); - CKP(handle); - handle->current = list->first; - handle->list = list; - return 0; - ERRBLKI; -} - - -/* list_walk: - * designed to be called repeatedly, and returns the next element of the - * list each time (but must not call either add or del between calls) - * - * (Set return_listel to nonzero to return the list element structure rather - * than the pointer it contains. This is just so that if you put null - * pointers on the list you can tell the difference from end of list.) - */ -void *list_walk(List_handle *handle, int return_listel) -{ - void *ptr; - CKP(handle); - if (handle->current == NULL) - return NULL; - else - { - ptr = (return_listel) ? (void *) handle->current : handle->current->ptr; - handle->current = handle->current->next; - return ptr; - } - ERRBLKP; -} - -/*------------------------------------------------------------------------------*/ -/* list_find: find first item on the list matching specified item, where - * "compar" is the matching function, and "matchval" is return value from - * compar in the event of a match - * - * The pointer index_return, if non-NULL, is used to return the index number - * on the list (set to -1 if not found). - */ -void *list_find(List *list, - const void *item, - int (*compar)(const void *, const void *), - int matchval, - int *index_return) -{ - int index; - List_element *listel; - List_handle handle; - void *ptr; - - list_startwalk(list, &handle); - index = 0; - while ((listel = list_walk(&handle, 1)) != NULL) - { - ptr = listel->ptr; - if (compar(&item, &ptr) == matchval) - { - if (index_return != NULL) - *index_return = index; - return ptr; - } - index++; - } - if (index_return != NULL) - *index_return = -1; - return NULL; -} - - -int list_copy_to_ptr_array(const List *list, - int *n_return, - void *ptr_array_return, - List *heaplist) -{ - int n; - List_handle handle; - List_element *listel; - void **ptr_array, **p; - - n = list_size(list); - if (n == 0) - ptr_array = NULL; - else - { - CKP( ptr_array = malloc_(n * sizeof(void *), heaplist) ); - p = ptr_array; - - list_startwalk(list, &handle); - while ((listel = list_walk(&handle, 1)) != NULL) - { - *p = listel->ptr; - p++; - } - } - *n_return = n; - * (void ***) ptr_array_return = ptr_array; - return 0; - ERRBLKI; -} diff --git a/cf/umread_lib/c-lib/malloc.c b/cf/umread_lib/c-lib/malloc.c deleted file mode 100644 index 2023d54b8a..0000000000 --- a/cf/umread_lib/c-lib/malloc.c +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include - -#include "umfileint.h" - -/* Malloc functions - * - * These routines are closely integrated with the link list functions; they - * are called with a linked list "heaplist"; the malloc_ function adds the - * newly allocated pointer to this list, and the free_ function removes it - * from the list. (They can also be called with NULL in which case they - * ignore the heaplist; this is necessary when allocating or freeing memory - * for the heaplist itself.) - * - * The idea is that all the dynamically memory allocation associated with - * a given file should be through these functions. Then whenever the file - * is closed properly or because an error condition gave an abort, the - * memory can be freed without needing complicated tests to work out what - * has been allocated: just go through the linked list freeing pointers. - * - * NOTE: this routine now allocates a little more memory than requested, - * and saves the pointer to the list element on the heaplist at the start, - * before returning to the calling routine the pointer to the actual block - * of memory that the caller is interested in. This ensures that when freeing - * the memory, list_del_by_listel can be used instead of list_del, giving - * efficiency gains. - */ - -static const int extrasize = sizeof(List_element*); - -void *malloc_(size_t size, List *heaplist){ - - void *ptr; - List_element* *elp; - - if (size == 0) - return NULL; - - /* The only call to malloc in umfile c-lib (except in unwgdos.c and packed_data in read.c) */ - ptr = malloc(size + extrasize); - - if (ptr == NULL) - { - error_mesg("unable to allocate of %d bytes of memory", - size); - } - else - { - /* copy the pointer so we can use the start of the address to store - * the List_element* - */ - elp = (List_element**) ptr; - - /* Now increment the pointer (to after our stored List_element*) to give - * what the calling routine calling routine sees the start of memory - * (cast to char* for ptr arithmetic. Do this *before* storing it - * on the heaplist, because pointers on will be freed with free - */ - ptr = (void*) ((char*)ptr + extrasize); - - if (heaplist != NULL) - { - CKI( list_add(heaplist, ptr, NULL) ); - - /* we just added to the list, so that heaplist->last will - * contain pointer to the relevant List_element* - */ - *elp = heaplist->last; - } - else - *elp = NULL; - } - - return ptr; - ERRBLKP; -} - - -void *dup_(const void *inptr, size_t size, List *heaplist) -{ - void *outptr; - - CKP( outptr = malloc_(size, heaplist) ); - memcpy(outptr, inptr, size); - return outptr; - ERRBLKP; -} - - -int free_(void *ptr, List *heaplist) -{ - List_element *el; - - CKP(ptr); - /* first subtract off the extra size we added (see malloc_) */ - ptr = (void*) ((char*) ptr - extrasize); - - /* this is our list element */ - el = * (List_element**) ptr; - - /* The only call to free in umfile c-lib - * (except in unwgdos.c and packed_data in read.c) - */ - free(ptr); - - /* printf ("free: %p\n",ptr); */ - if (heaplist != NULL) - CKI( list_del_by_listel(heaplist, el, NULL) ); - - return 0; - ERRBLKI; -} - - -int free_all(List *heaplist) -{ - return list_free(heaplist, 1, NULL); -} diff --git a/cf/umread_lib/c-lib/new_structs.c b/cf/umread_lib/c-lib/new_structs.c deleted file mode 100644 index 15d70f2f37..0000000000 --- a/cf/umread_lib/c-lib/new_structs.c +++ /dev/null @@ -1,106 +0,0 @@ -#include "umfileint.h" - -/* functions to create pointers to new structures (such as Rec, Var and File) - * from heap memory - * - * Also for good measure functions to free these, although most are not - * crucial if they are not called, because of the garbage collection procedure - * employed in free_file() - */ - -Rec *new_rec(int word_size, List *heaplist) -{ - Rec *rec; - - CKP( rec = malloc_(sizeof(Rec), heaplist) ); - CKP( rec->internp = malloc_(sizeof(struct _Rec), heaplist) ); - CKP( rec->int_hdr = malloc_(N_INT_HDR * word_size, heaplist) ); - CKP( rec->real_hdr = malloc_(N_REAL_HDR * word_size, heaplist) ); - - rec->header_offset = -1; - rec->data_offset = -1; - rec->disk_length = -1; - return rec; - ERRBLKP; -} - -int free_rec(Rec *rec, List *heaplist) -{ - CKI( free_(rec->internp, heaplist) ); - CKI( free_(rec->int_hdr, heaplist) ); - CKI( free_(rec->real_hdr, heaplist) ); - CKI( free_(rec, heaplist) ); - return 0; - ERRBLKI; -} - -Var *new_var(List *heaplist) -{ - Var *var; - CKP( - var = malloc_(sizeof(Var), heaplist) - ); - CKP( - var->internp = malloc_(sizeof(struct _Var), heaplist) - ); - var->nz = 0; - var->nt = 0; - var->supervar_index = -1; - var->recs = NULL; - return var; - ERRBLKP; -} - -int free_var(Var *var, List *heaplist) -{ - CKI( free_(var->internp, heaplist) ); - if (var->recs) - CKI( free_(var->recs, heaplist) ); - CKI( free_(var, heaplist) ); - return 0; - ERRBLKI; -} - - -/* new_file is a rather special case, because the heaplist is initialised as - * part of the structure rather than supplied externally. Also free_file will - * free everything on the heaplist. - */ - -File *new_file() -{ - File *file; - - if ( (file = malloc_(sizeof(File), NULL)) == NULL) goto err1; - if ( (file->internp = malloc_(sizeof(struct _File), NULL)) == NULL) goto err2; - if ( (file->internp->heaplist = list_new(NULL)) == NULL) goto err3; - - file->nvars = 0; - file->vars = NULL; - file->fd = -1; - file->file_type.fmt = -1; - file->file_type.byte_ordering = -1; - file->file_type.word_size = -1; - file->internp->nrec = 0; - file->internp->recs = NULL; - - return file; - - err3: - free_(file->internp, NULL); - err2: - free_(file, NULL); - err1: - GRIPE; - return NULL; -} - - -int free_file(File *file) -{ - CKI( free_all(file->internp->heaplist) ); - CKI( free_(file->internp, NULL) ); - CKI( free_(file, NULL) ); - return 0; - ERRBLKI; -} diff --git a/cf/umread_lib/c-lib/swap.c b/cf/umread_lib/c-lib/swap.c deleted file mode 100644 index 5e68197457..0000000000 --- a/cf/umread_lib/c-lib/swap.c +++ /dev/null @@ -1,37 +0,0 @@ -#include "umfileint.h" - -#define DO_SWAP(x, y) {t = p[x]; p[x] = p[y]; p[y] = t;} - -void swap_bytes_sgl(void *ptr, size_t num_words) -{ - int i; - char *p; - char t; - - p = (char*) ptr; - for (i = 0; i < num_words; i++) - { - DO_SWAP(3, 0); - DO_SWAP(2, 1); - p += 4; - } -} - -void swap_bytes_dbl(void *ptr, size_t num_words) -{ - int i; - char *p; - char t; - - p = (char*) ptr; - for (i = 0; i < num_words; i++) - { - DO_SWAP(7, 0); - DO_SWAP(6, 1); - DO_SWAP(5, 2); - DO_SWAP(4, 3); - p += 8; - } -} - - diff --git a/cf/umread_lib/c-lib/type-dep/Makefile b/cf/umread_lib/c-lib/type-dep/Makefile deleted file mode 100644 index a3bda54740..0000000000 --- a/cf/umread_lib/c-lib/type-dep/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -STEMS = umfile_test_typedep interpret_header read process_vars \ - debug_dump date_and_time compare levels axes unwgdos - -CFLAGS += -I.. - -SGL_OBJS=$(foreach stem, $(STEMS), $(stem)_sgl.o) -DBL_OBJS=$(foreach stem, $(STEMS), $(stem)_dbl.o) - -OBJS = $(SGL_OBJS) $(DBL_OBJS) - -LIB = $(TYPE_DEP_LIBRARY) - -.PHONY: all clean - -all: $(LIB) - -clean: - rm -f $(OBJS) $(LIB) - -%_dbl.o: %.c $(HEADERS) - $(CC) $(CFLAGS) -c -DDOUBLE -o $@ $< - -%_sgl.o: %.c $(HEADERS) - $(CC) $(CFLAGS) -c -DSINGLE -o $@ $< - -$(LIB): $(OBJS) - rm -f $@ - ar r $@ $(OBJS) diff --git a/cf/umread_lib/c-lib/type-dep/axes.c b/cf/umread_lib/c-lib/type-dep/axes.c deleted file mode 100644 index 1dd90f8a6e..0000000000 --- a/cf/umread_lib/c-lib/type-dep/axes.c +++ /dev/null @@ -1,75 +0,0 @@ -#include - -#include "umfileint.h" - -/* - * Functions relating to time and z axes. - * - * These functions have very similar content for Z and T axes, because in fact - * the Z_axis and T_axis struct each only contains a list of values. In cdunifpp, - * there are other elements (e.g. a time axis origin) that are used when comparing - * axes (maybe unnecessarily). In umread, we do not even bother to compare axes - * because axes are not returned to the caller: they are just used to evaluate the - * shape of the variable (nz, nt) and ensure that it is regular. - */ - -T_axis *new_t_axis(List *heaplist) -{ - T_axis *t_axis; - CKP( t_axis = malloc_(sizeof(T_axis), heaplist) ); - t_axis->values = list_new(heaplist); - return t_axis; - ERRBLKP; -} - -int free_t_axis(T_axis *t_axis, List *heaplist) -{ - CKI( list_free(t_axis->values, 1, heaplist) ); - CKI( free_(t_axis, heaplist) ); - return 0; - ERRBLKI; -} - - -Z_axis *new_z_axis(List *heaplist) -{ - Z_axis *z_axis; - CKP( z_axis = malloc_(sizeof(Z_axis), heaplist) ); - z_axis->values = list_new(heaplist); - return z_axis; - ERRBLKP; -} - -int free_z_axis(Z_axis *z_axis, List *heaplist) -{ - CKI( list_free(z_axis->values, 1, heaplist) ); - CKI( free_(z_axis, heaplist) ); - return 0; - ERRBLKI; -} - - -int t_axis_add(T_axis *t_axis, const Time *time, - int *index_return, List *heaplist) -{ - Time *timecopy; - - CKP( timecopy = dup_(time, sizeof(Time), heaplist) ); - return list_add_or_find(t_axis->values, &timecopy, compare_times, 0, - free_, index_return, heaplist); - ERRBLKI; -} - - - -int z_axis_add(Z_axis *z_axis, const Level *lev, - int *index_return, List *heaplist) -{ - Level *levcopy; - - CKP( levcopy = dup_(lev, sizeof(Level), heaplist) ); - return list_add_or_find(z_axis->values, &levcopy, compare_levels, 0, - free_, index_return, heaplist); - ERRBLKI; -} - diff --git a/cf/umread_lib/c-lib/type-dep/compare.c b/cf/umread_lib/c-lib/type-dep/compare.c deleted file mode 100644 index 71e010f3aa..0000000000 --- a/cf/umread_lib/c-lib/type-dep/compare.c +++ /dev/null @@ -1,345 +0,0 @@ -/* - * COMPARISON FUNCTIONS. - * - * NOTE: functions which take arguments of type void* (except for - * compare_ptrs) are designed to be used with generic routines: - * compare_records is envisaged for use with qsort; several other functions - * are envisaged for use with compare_lists (below). - * - * In these cases if supplying pointers directly to the relevant structures, - * need to generate an extra level of pointer with "&" syntax. - * - * But not all functions below are like that. Don't assume functions can be - * used analogously without first examining the argument lists. - */ - -/* The code profiler suggests that compare_ints and compare_reals are - * candidates for inlining; however, unfortunately this sometimes gets - * compiled with c89 which doesn't support inline functions. Use a #define - * for compare_ints. compare_reals, which is more awkward to #define, is just - * going to have to stay as it is for now (it's called less often). - */ - -#include - -#include "umfileint.h" - -#define compare_ints(a, b) ((a) < (b) ? (-1) : (a) > (b) ? 1 : 0) - -/* - * static int compare_ints(INTEGER a, INTEGER b) - * { - * if (ab) return 1; - * return 0; - * } - */ - -static int compare_reals(REAL a, REAL b) -{ - REAL delta; - - /* first test for special case (unnecessary, but code profiler shows - * slightly more efficient) - */ - if (a == b) - return 0; - - delta = fabs(b * REAL_TOLERANCE); - if (a < b - delta) return -1; - if (a > b + delta) return 1; - - return 0; -} - -#define COMPARE_INTS(tag) {int cmp = compare_ints(LOOKUP(a, tag), LOOKUP(b, tag)); if (cmp != 0) return cmp;} -#define COMPARE_REALS(tag) {int cmp = compare_reals(RLOOKUP(a, tag), RLOOKUP(b, tag)); if (cmp != 0) return cmp;} - - -/* routine to compare two PP records, to see if they are in the same - * variable - * - * returns: - * - * -1 or 1 headers are from different variables; - * sign of return value gives consistent ordering - * - * 0 headers are from same variable - */ -int compare_records_between_vars(const Rec *a, const Rec *b) -{ - int cmp; - COMPARE_INTS(INDEX_LBUSER4); - COMPARE_INTS(INDEX_LBUSER7); - COMPARE_INTS(INDEX_LBCODE); - COMPARE_INTS(INDEX_LBVC); - COMPARE_INTS(INDEX_LBTIM); - COMPARE_INTS(INDEX_LBPROC); - COMPARE_REALS(INDEX_BPLAT); - COMPARE_REALS(INDEX_BPLON); - COMPARE_INTS(INDEX_LBHEM); - COMPARE_INTS(INDEX_LBROW); - COMPARE_INTS(INDEX_LBNPT); - - COMPARE_REALS(INDEX_BGOR); - COMPARE_REALS(INDEX_BZY); - COMPARE_REALS(INDEX_BDY); - COMPARE_REALS(INDEX_BZX); - COMPARE_REALS(INDEX_BDX); - - cmp = compare_mean_periods(a, b); - if (cmp != 0) return cmp; - - /* Disambig index is used to force distinction between variables for records - * whose headers are the same. It is initialised to the same value for all - * records (in fact -1), but may later be set to different values according - * to some heuristic. - */ - - cmp = compare_ints(a->internp->disambig_index, b->internp->disambig_index); - if (cmp != 0) return cmp; - - return 0; -} - - -/* helper routine for compare_mean_periods - test if both periods are in specified range - * note - assumes that low, high are positive - */ -static int both_values_in_range(REAL low, REAL high, REAL a, REAL b) -{ - REAL low1 = low * (1. - REAL_TOLERANCE); - REAL high1 = high * (1. + REAL_TOLERANCE); - return (a >= low1) && (a <= high1) && (b >= low1) & (b <= high1); -} - - -/* Routine to compare if two PP records have different meaning periods, such - * that they should be considered to be part of different variables. Normally - * this will be true if the mean periods differ by more than "delta", but in - * the case of Gregorian calendar, we allow some tolerance relating to - * climatology data. - * - * This should only get called if both records have already been checked for - * having the same LBTIM and LBPROC. - */ - -int compare_mean_periods(const Rec *a, const Rec *b) -{ - int cmp; - - cmp = compare_reals(a->internp->mean_period, b->internp->mean_period); - if (cmp == 0) return 0; - - /* if we get here, times differ - but for gregorian cut some slack */ - if (calendar_type(LOOKUP(a, INDEX_LBTIM)) == gregorian) - { - if (both_values_in_range(28., 31., - a->internp->mean_period, b->internp->mean_period) /* monthly */ - || both_values_in_range(90., 92., - a->internp->mean_period, b->internp->mean_period) /* seasonal */ - || both_values_in_range(365., 366., - a->internp->mean_period, b->internp->mean_period)) /* annual */ - return 0; - } - return cmp; -} - - -/* routine to compare two PP records that the calling routine - * has already established are in the same variable. - * - * returns: - * - * -1 or 1 times or levels differ; - * sign of return value gives consistent ordering - * of times and levels - * - * 0 records do not differ within values tested - */ -int compare_records_within_var(const Rec *a, const Rec *b) -{ - int a_surface, b_surface; - - COMPARE_INTS(INDEX_LBFT); - - COMPARE_INTS(INDEX_LBYR); - COMPARE_INTS(INDEX_LBMON); - COMPARE_INTS(INDEX_LBDAT); - COMPARE_INTS(INDEX_LBDAY); - COMPARE_INTS(INDEX_LBHR); - COMPARE_INTS(INDEX_LBMIN); - - COMPARE_INTS(INDEX_LBYRD); - COMPARE_INTS(INDEX_LBMOND); - COMPARE_INTS(INDEX_LBDATD); - COMPARE_INTS(INDEX_LBDAYD); - COMPARE_INTS(INDEX_LBHRD); - COMPARE_INTS(INDEX_LBMIND); - - /* - * Ordering of levels: - * - * Generally we want to sort on LBLEV before sorting on BLEV. - * - * This is because in the case of hybrid levels, BLEV contains the B values - * (in p = A + B p_s), which won't do for sorting, and fortunately in this - * case LBLEV contains the model level index which is fine. - * - * But there is a nasty special case: surface and boundary layer heat flux - * has LBLEV = 9999, 2, 3, 4, ... where 9999 is the surface layer. In this - * case we *could* in fact sort on BLEV, but then we need to know when it's - * okay to do this (STASH code?). - * - * Maybe safer, treat 9999 lower than any level if comparing it with - * another level. (9999 should always be a special value and it is rare - * for it to be mixed with non-special values in the same variable.) - */ - a_surface = (LOOKUP(a, INDEX_LBLEV) == 9999); - b_surface = (LOOKUP(b, INDEX_LBLEV) == 9999); - if (a_surface && !b_surface) - return -1; - else if (b_surface && !a_surface) - return 1; - - COMPARE_INTS(INDEX_LBLEV); - COMPARE_REALS(INDEX_BLEV); - COMPARE_REALS(INDEX_BHLEV); - - return 0; -} - - -/* routine to compare two PP records. - * returns: - * -2 or 2 headers are from different variable - * -1 or 1 headers are from same variable - * 0 difference not found in elements inspected - * - */ -int compare_records(const void *p1, const void *p2) -{ - const Rec *a = * (Rec **) p1; - const Rec *b = * (Rec **) p2; - - int cmp; - - cmp = compare_records_between_vars(a, b); - if (cmp != 0) { - // debug("compare_records - variables differ %d %d", LOOKUP(a,INDEX_LBUSER4), - // LOOKUP(b,INDEX_LBUSER4)); - - return cmp * 2; - } - cmp = compare_records_within_var(a, b); - if (cmp != 0){ - // debug("compare_records - variables same %d %d", LOOKUP(a,INDEX_LBUSER4), - //LOOKUP(b,INDEX_LBUSER4)); - - return cmp; - } -//debug("compare_records - records same"); - return 0; -} - - -int records_from_different_vars(const Rec *a, const Rec *b) -{ - return (compare_records_between_vars(a, b) != 0); -} - - -int compare_lists(const List *l1, const List *l2, int (*compfunc)(const void*, const void*)) -{ - int i, n, cmp; - const void *item1, *item2; - List_handle handle1, handle2; - - /* differ if number of items differs */ - n = list_size(l1); - if ((cmp = compare_ints(n, list_size(l2))) != 0) return cmp; - - /* differ if any individual item differs */ - list_startwalk(l1, &handle1); - list_startwalk(l2, &handle2); - for (i = 0; i < n; i++) { - item1 = list_walk(&handle1, 0); - item2 = list_walk(&handle2, 0); - if ((cmp = compfunc(&item1, &item2)) != 0) return cmp; - } - return 0; -} - -int compare_levels(const void *p1, const void *p2) -{ - const Level *a = *(Level **)p1; - const Level *b = *(Level **)p2; - - /* macros called LCOMPARE_INTS and LCOMPARE_REALS to emphasise difference from those in compare_records */ - -#define LCOMPARE_INTS(tag) {int cmp = compare_ints(a->tag, b->tag); if (cmp != 0) return cmp;} -#define LCOMPARE_REALS(tag) {int cmp = compare_reals(a->tag, b->tag); if (cmp != 0) return cmp;} - - LCOMPARE_INTS(type); - - switch (a->type) { - case hybrid_height_lev_type: - LCOMPARE_REALS(values.hybrid_height.a); - LCOMPARE_REALS(values.hybrid_height.b); -#ifdef BDY_LEVS - LCOMPARE_REALS(values.hybrid_height.ubdy_a); - LCOMPARE_REALS(values.hybrid_height.ubdy_b); - LCOMPARE_REALS(values.hybrid_height.lbdy_a); - LCOMPARE_REALS(values.hybrid_height.lbdy_b); -#endif - break; - case hybrid_sigmap_lev_type: - LCOMPARE_REALS(values.hybrid_sigmap.a); - LCOMPARE_REALS(values.hybrid_sigmap.b); -#ifdef BDY_LEVS - LCOMPARE_REALS(values.hybrid_sigmap.ubdy_a); - LCOMPARE_REALS(values.hybrid_sigmap.ubdy_b); - LCOMPARE_REALS(values.hybrid_sigmap.lbdy_a); - LCOMPARE_REALS(values.hybrid_sigmap.lbdy_b); -#endif - break; - case pseudo_lev_type: - LCOMPARE_INTS(values.pseudo.index); - break; - default: - LCOMPARE_REALS(values.misc.level); -#ifdef BDY_LEVS - LCOMPARE_REALS(values.misc.ubdy_level); - LCOMPARE_REALS(values.misc.lbdy_level); -#endif - break; - } - return 0; -} - -int compare_times(const void *p1, const void *p2) -{ - const Time *a = * (Time **) p1; - const Time *b = * (Time **) p2; - int cmp; - - /* LBTYP: ignore 100s digit = sampling frequency, as we don't use it for anything */ - if ((cmp = compare_ints(a->type % 100, b->type % 100)) != 0) return cmp; - - if ((cmp = compare_dates(&a->time1, &b->time1)) != 0) return cmp; - if ((cmp = compare_dates(&a->time2, &b->time2)) != 0) return cmp; - return 0; -} - -int compare_dates(const Date *a, const Date *b) -{ - int cmp; - if ((cmp = compare_ints(a->year ,b->year )) != 0) return cmp; - if ((cmp = compare_ints(a->month ,b->month )) != 0) return cmp; - if ((cmp = compare_ints(a->day ,b->day )) != 0) return cmp; - if ((cmp = compare_ints(a->hour ,b->hour )) != 0) return cmp; - if ((cmp = compare_ints(a->minute,b->minute)) != 0) return cmp; - if ((cmp = compare_ints(a->second,b->second)) != 0) return cmp; - return 0; -} - diff --git a/cf/umread_lib/c-lib/type-dep/date_and_time.c b/cf/umread_lib/c-lib/type-dep/date_and_time.c deleted file mode 100644 index 2e1c87ce96..0000000000 --- a/cf/umread_lib/c-lib/type-dep/date_and_time.c +++ /dev/null @@ -1,169 +0,0 @@ -#include "umfileint.h" - -/* - * Aside from the time_set() function, most of the infrastructure in this file - * is for the purpose of calculating the length of the time mean period, just so - * that e.g. monthly and daily means in the same file can be separated out into - * different variables. - */ - -REAL get_mean_period(const Time *time) -{ - /* returns the averaging period in days, or 0. if it is not a mean field */ - INTEGER lbtim = time->type; - if (!is_time_mean(lbtim)) - return 0.; - return time_diff(lbtim, &time->time2, &time->time1); -} - -int is_time_mean(INTEGER LBTIM) -{ - int ib; - ib = (LBTIM / 10) % 10; - return (ib == 2) || (ib == 3); -} - -REAL time_diff(INTEGER lbtim, const Date *date, const Date *orig_date) -{ - int64_t secs; - - switch(calendar_type(lbtim)) - { - case gregorian: - return sec_to_day(gregorian_to_secs(date) - gregorian_to_secs(orig_date)); - break; /* notreached */ - case cal360day: - secs = - date->second - orig_date->second + - 60 * (date->minute - orig_date->minute + - 60 * (date->hour - orig_date->hour + - 24 * (date->day - orig_date->day + - 30 * (date->month - orig_date->month + - 12 * (int64_t) (date->year - orig_date->year) )))); - - return sec_to_day(secs); - break; /* notreached */ - case model: - secs = - date->second - orig_date->second + - 60 * (date->minute - orig_date->minute + - 60 * (date->hour - orig_date->hour + - 24 * (int64_t) (date->day - orig_date->day))); - - return sec_to_day(secs); - break; /* notreached */ - default: - SWITCH_BUG; - } - ERRBLKF; -} - -REAL sec_to_day(int64_t seconds) -{ - /* convert seconds to days, avoiding rounding where possible - * by using integer arithmetic for the whole days - */ - const int secs_per_day = 86400; - - int64_t days, remainder; - days = seconds / secs_per_day; - remainder = seconds % secs_per_day; - - return days + remainder / (REAL) secs_per_day; -} - -Calendar_type calendar_type(INTEGER type) -{ - switch(type % 10) - { - case 0: - /* fallthrough */ - case 3: - return model; - break; /* notreached */ - case 1: - return gregorian; - break; /* notreached */ - case 2: - return cal360day; - break; /* notreached */ - default: - SWITCH_BUG; - } - - /* on error return -1 (though only useful to calling routine if stored in an int - * not a Calendar_type) - */ - ERRBLKI; -} - -int64_t gregorian_to_secs(const Date *date) -{ - /* Convert from Gregorian calendar to seconds since a fixed origin - * - * Can be with respect to any arbitary origin, because return values from this are - * differenced. - * - * Arbitrary origin is what would be 1st Jan in the year 0 if hypothetically the - * system was completely consistent going back this far. This simplifies the - * calculation. - * - * Strictly, this is proleptic_gregorian rather than gregorian (see CF docs) - * as this is more likely to match the code actually in the model. The UM - * docs call it "gregorian" but I'm speculating (without checking model code) - * that the model really doesn't have all that jazz with Julian calendar - * before fifteen-something. - */ - const int sid = 86400; /* seconds in day */ - const int sih = 3600; - const int sim = 60; - - int64_t nsec; - int year,nleap,nday,isleap; - - /* offsets from 1st Jan to 1st of each month in non-leap year */ - int dayno[12] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }; - - year = date->year; - - /* is the year leap? */ - if (year % 400 == 0) isleap = 1; - else if (year % 100 == 0) isleap = 0; - else if (year % 4 == 0) isleap = 1; - else isleap=0; - - /* nleap is number of 29th Febs passed between origin date and supplied date. */ - nleap = year / 4 - year / 100 + year / 400; - if (isleap && date->month <= 2) - nleap--; - - nday = (year * 365) + dayno[date->month - 1] + (date->day - 1) + nleap; - - nsec = (int64_t) nday * sid + date->hour * sih + date->minute * sim + date->second; - - return nsec; -} - - -int time_set(Time *time, const Rec *rec) -{ - time->type = LOOKUP(rec, INDEX_LBTIM); - - time->time1.year = LOOKUP(rec, INDEX_LBYR); - time->time1.month = LOOKUP(rec, INDEX_LBMON); - time->time1.day = LOOKUP(rec, INDEX_LBDAT); - time->time1.hour = LOOKUP(rec, INDEX_LBHR); - time->time1.minute = LOOKUP(rec, INDEX_LBMIN); - time->time1.second = 0; - - time->time2.year = LOOKUP(rec, INDEX_LBYRD); - time->time2.month = LOOKUP(rec, INDEX_LBMOND); - time->time2.day = LOOKUP(rec, INDEX_LBDATD); - time->time2.hour = LOOKUP(rec, INDEX_LBHRD); - time->time2.minute = LOOKUP(rec, INDEX_LBMIND); - time->time2.second = 0; - - return 0; -} - - diff --git a/cf/umread_lib/c-lib/type-dep/debug_dump.c b/cf/umread_lib/c-lib/type-dep/debug_dump.c deleted file mode 100644 index 996fcb852d..0000000000 --- a/cf/umread_lib/c-lib/type-dep/debug_dump.c +++ /dev/null @@ -1,88 +0,0 @@ -#include "umfileint.h" - -void debug_dump_all_headers(File *file) -{ - int irec; - Rec *rec; - - debug("fd = %d", file->fd); - debug("fmt = %d", file->file_type.fmt); - debug("byte_ordering = %d", file->file_type.byte_ordering); - debug("word_size = %d", file->file_type.word_size); - debug("nrec = %d", file->internp->nrec); - debug(""); - - for (irec = 0; irec < file->internp->nrec; irec++) - { - rec = file->internp->recs[irec]; - debug("rec %d", irec); - debug("header_offset = %d", rec->header_offset); - debug("data_offset = %d", rec->data_offset); - debug("disk_length = %d", rec->disk_length); - - debug("LBYR = %d", LOOKUP(rec, INDEX_LBYR)); - debug("LBMON = %d", LOOKUP(rec, INDEX_LBMON)); - debug("LBDAT = %d", LOOKUP(rec, INDEX_LBDAT)); - debug("LBHR = %d", LOOKUP(rec, INDEX_LBHR)); - debug("LBMIN = %d", LOOKUP(rec, INDEX_LBMIN)); - debug("LBDAY = %d", LOOKUP(rec, INDEX_LBDAY)); - debug("LBYRD = %d", LOOKUP(rec, INDEX_LBYRD)); - debug("LBMOND = %d", LOOKUP(rec, INDEX_LBMOND)); - debug("LBDATD = %d", LOOKUP(rec, INDEX_LBDATD)); - debug("LBHRD = %d", LOOKUP(rec, INDEX_LBHRD)); - debug("LBMIND = %d", LOOKUP(rec, INDEX_LBMIND)); - debug("LBDAYD = %d", LOOKUP(rec, INDEX_LBDAYD)); - debug("LBTIM = %d", LOOKUP(rec, INDEX_LBTIM)); - debug("LBFT = %d", LOOKUP(rec, INDEX_LBFT)); - debug("LBLREC = %d", LOOKUP(rec, INDEX_LBLREC)); - debug("LBCODE = %d", LOOKUP(rec, INDEX_LBCODE)); - debug("LBHEM = %d", LOOKUP(rec, INDEX_LBHEM)); - debug("LBROW = %d", LOOKUP(rec, INDEX_LBROW)); - debug("LBNPT = %d", LOOKUP(rec, INDEX_LBNPT)); - debug("LBEXT = %d", LOOKUP(rec, INDEX_LBEXT)); - debug("LBPACK = %d", LOOKUP(rec, INDEX_LBPACK)); - debug("LBREL = %d", LOOKUP(rec, INDEX_LBREL)); - debug("LBFC = %d", LOOKUP(rec, INDEX_LBFC)); - debug("LBCFC = %d", LOOKUP(rec, INDEX_LBCFC)); - debug("LBPROC = %d", LOOKUP(rec, INDEX_LBPROC)); - debug("LBVC = %d", LOOKUP(rec, INDEX_LBVC)); - debug("LBRVC = %d", LOOKUP(rec, INDEX_LBRVC)); - debug("LBEXP = %d", LOOKUP(rec, INDEX_LBEXP)); - debug("LBBEGIN = %d", LOOKUP(rec, INDEX_LBBEGIN)); - debug("LBNREC = %d", LOOKUP(rec, INDEX_LBNREC)); - debug("LBPROJ = %d", LOOKUP(rec, INDEX_LBPROJ)); - debug("LBTYP = %d", LOOKUP(rec, INDEX_LBTYP)); - debug("LBLEV = %d", LOOKUP(rec, INDEX_LBLEV)); - debug("LBRSVD1 = %d", LOOKUP(rec, INDEX_LBRSVD1)); - debug("LBRSVD2 = %d", LOOKUP(rec, INDEX_LBRSVD2)); - debug("LBRSVD3 = %d", LOOKUP(rec, INDEX_LBRSVD3)); - debug("LBRSVD4 = %d", LOOKUP(rec, INDEX_LBRSVD4)); - debug("LBSRCE = %d", LOOKUP(rec, INDEX_LBSRCE)); - debug("LBUSER1 = %d", LOOKUP(rec, INDEX_LBUSER1)); - debug("LBUSER2 = %d", LOOKUP(rec, INDEX_LBUSER2)); - debug("LBUSER3 = %d", LOOKUP(rec, INDEX_LBUSER3)); - debug("LBUSER4 = %d", LOOKUP(rec, INDEX_LBUSER4)); - debug("LBUSER5 = %d", LOOKUP(rec, INDEX_LBUSER5)); - debug("LBUSER6 = %d", LOOKUP(rec, INDEX_LBUSER6)); - debug("LBUSER7 = %d", LOOKUP(rec, INDEX_LBUSER7)); - debug("BULEV = %f", RLOOKUP(rec, INDEX_BULEV)); - debug("BHULEV = %f", RLOOKUP(rec, INDEX_BHULEV)); - debug("BRSVD3 = %f", RLOOKUP(rec, INDEX_BRSVD3)); - debug("BRSVD4 = %f", RLOOKUP(rec, INDEX_BRSVD4)); - debug("BDATUM = %f", RLOOKUP(rec, INDEX_BDATUM)); - debug("BACC = %f", RLOOKUP(rec, INDEX_BACC)); - debug("BLEV = %f", RLOOKUP(rec, INDEX_BLEV)); - debug("BRLEV = %f", RLOOKUP(rec, INDEX_BRLEV)); - debug("BHLEV = %f", RLOOKUP(rec, INDEX_BHLEV)); - debug("BHRLEV = %f", RLOOKUP(rec, INDEX_BHRLEV)); - debug("BPLAT = %f", RLOOKUP(rec, INDEX_BPLAT)); - debug("BPLON = %f", RLOOKUP(rec, INDEX_BPLON)); - debug("BGOR = %f", RLOOKUP(rec, INDEX_BGOR)); - debug("BZY = %f", RLOOKUP(rec, INDEX_BZY)); - debug("BDY = %f", RLOOKUP(rec, INDEX_BDY)); - debug("BZX = %f", RLOOKUP(rec, INDEX_BZX)); - debug("BDX = %f", RLOOKUP(rec, INDEX_BDX)); - debug("BMDI = %f", RLOOKUP(rec, INDEX_BMDI)); - debug("BMKS = %f", RLOOKUP(rec, INDEX_BMKS)); - } -} diff --git a/cf/umread_lib/c-lib/type-dep/interpret_header.c b/cf/umread_lib/c-lib/type-dep/interpret_header.c deleted file mode 100644 index 0d5bd9a090..0000000000 --- a/cf/umread_lib/c-lib/type-dep/interpret_header.c +++ /dev/null @@ -1,141 +0,0 @@ -#include "umfileint.h" - -Data_type get_type(const INTEGER *int_hdr) -{ - switch (int_hdr[INDEX_LBUSER1]) - { - case(2): - case(-2): - case(3): - case(-3): - return int_type; - /* break; */ - case(1): - case(-1): - return real_type; - /* break; */ - default: - error_mesg("Warning: datatype %d not recognised, assuming real", - int_hdr[INDEX_LBUSER1]); - return real_type; - } -} - -/* Get number of data words. Does not include extra data. */ -size_t get_num_data_words(const INTEGER *int_hdr) -{ - if (int_hdr[INDEX_LBPACK] != 0 - && int_hdr[INDEX_LBROW] > 0 - && int_hdr[INDEX_LBNPT] > 0) - /* if packed and horizontal grid sizes set, use them */ - return int_hdr[INDEX_LBROW] * int_hdr[INDEX_LBNPT]; - else - /* otherwise use LBLREC */ - return int_hdr[INDEX_LBLREC] - get_extra_data_length(int_hdr) / WORD_SIZE; -} - -/* get length of (any) extra data in bytes */ -size_t get_extra_data_length(const INTEGER *int_hdr) -{ - if (int_hdr[INDEX_LBEXT] > 0) - return int_hdr[INDEX_LBEXT] * WORD_SIZE; - return 0; -} - -size_t get_extra_data_offset_and_length_core(const INTEGER *int_hdr, - size_t data_offset, - size_t disk_length, - size_t *extra_data_offset_rtn, - size_t *extra_data_length_rtn) -{ - size_t extra_data_length; - - extra_data_length = get_extra_data_length(int_hdr); - *extra_data_length_rtn = extra_data_length; - - /* If data is packed, the only way of telling where the extra data is - * is to assume that it is at the very end of data_length. - * - * If data is not packed, then can use data_length to work out where the - * extra data starts, allowing resilience against the possibility that - * disk_length might include some possible padding - */ - if (int_hdr[INDEX_LBPACK] != 0) - *extra_data_offset_rtn = data_offset + disk_length - extra_data_length; - else - *extra_data_offset_rtn = data_offset + get_num_data_words(int_hdr) * WORD_SIZE; - - return 0; -} - -int get_type_and_num_words_core(const INTEGER *int_hdr, - Data_type *type_rtn, - size_t *num_words_rtn) -{ - *type_rtn = get_type(int_hdr); - *num_words_rtn = get_num_data_words(int_hdr); - return 0; -} - - -/* sometimes a variable is included but which has some - * really essential header elements to missing data flag, - * so the variable is essentially missing in that any - * attempt to process the variable is only going to - * lead to errors - * - * pp_var_missing() tests for this. - * - * FIXME: expand to test other header elements - */ -int var_is_missing(const INTEGER *int_hdr) -{ - if (int_hdr[INDEX_LBNPT] == INT_MISSING_DATA) - return 1; - - if (int_hdr[INDEX_LBROW] == INT_MISSING_DATA) - return 1; - - return 0; -} - -int get_var_stash_model(const INTEGER *int_hdr) -{ - return int_hdr[INDEX_LBUSER7]; -} - -int get_var_stash_section(const INTEGER *int_hdr) -{ - return int_hdr[INDEX_LBUSER4] / 1000; -} - -int get_var_stash_item(const INTEGER *int_hdr) -{ - return int_hdr[INDEX_LBUSER4] % 1000; -} - -int get_var_compression(const INTEGER *int_hdr) -{ - return (int_hdr[INDEX_LBPACK] / 10) % 10; -} - - -int get_var_gridcode(const INTEGER *int_hdr) -{ - return int_hdr[INDEX_LBCODE]; -} - - -int get_var_packing(const INTEGER *int_hdr) -{ - return int_hdr[INDEX_LBPACK] % 10; -} - - -/* get the fill value from the floating point header. - * caller needs to check that it is actually floating point data! - */ -REAL get_var_real_fill_value(const REAL *real_hdr) -{ - return real_hdr[INDEX_BMDI]; -} diff --git a/cf/umread_lib/c-lib/type-dep/levels.c b/cf/umread_lib/c-lib/type-dep/levels.c deleted file mode 100644 index 251fcdf800..0000000000 --- a/cf/umread_lib/c-lib/type-dep/levels.c +++ /dev/null @@ -1,97 +0,0 @@ -#include "umfileint.h" - - -int lev_set(Level *lev, const Rec *rec) -{ - lev->type = level_type(rec); - - switch (lev->type) - { - case hybrid_height_lev_type: - lev->values.hybrid_height.a = RLOOKUP(rec, INDEX_BLEV); - lev->values.hybrid_height.b = RLOOKUP(rec, INDEX_BHLEV); -#ifdef BDY_LEVS - lev->values.hybrid_height.ubdy_a = RLOOKUP(rec, INDEX_BULEV); - lev->values.hybrid_height.ubdy_b = RLOOKUP(rec, INDEX_BHULEV); - lev->values.hybrid_height.lbdy_a = RLOOKUP(rec, INDEX_BRLEV); - lev->values.hybrid_height.lbdy_b = RLOOKUP(rec, INDEX_BHRLEV); -#endif - break; - - case hybrid_sigmap_lev_type: - lev->values.hybrid_sigmap.a = RLOOKUP(rec, INDEX_BHLEV); - lev->values.hybrid_sigmap.b = RLOOKUP(rec, INDEX_BLEV); -#ifdef BDY_LEVS - lev->values.hybrid_sigmap.ubdy_a = RLOOKUP(rec, INDEX_BHULEV); - lev->values.hybrid_sigmap.ubdy_b = RLOOKUP(rec, INDEX_BULEV); - lev->values.hybrid_sigmap.lbdy_a = RLOOKUP(rec, INDEX_BHRLEV); - lev->values.hybrid_sigmap.lbdy_b = RLOOKUP(rec, INDEX_BRLEV); -#endif - break; - - case pseudo_lev_type: - lev->values.pseudo.index = LOOKUP(rec, INDEX_LBUSER5); - break; - - default: - if (RLOOKUP(rec, INDEX_BLEV) == 0 - && LOOKUP(rec, INDEX_LBLEV) != 9999 - && LOOKUP(rec, INDEX_LBLEV) != 8888) - lev->values.misc.level = LOOKUP(rec, INDEX_LBLEV); - else - lev->values.misc.level = RLOOKUP(rec, INDEX_BLEV); -#ifdef BDY_LEVS - lev->values.misc.ubdy_level = RLOOKUP(rec, INDEX_BULEV); - lev->values.misc.lbdy_level = RLOOKUP(rec, INDEX_BRLEV); -#endif - break; - } - return 0; -} - - -Lev_type level_type(const Rec *rec) -{ - if (LOOKUP(rec, INDEX_LBUSER5) != 0 - && LOOKUP(rec, INDEX_LBUSER5) != INT_MISSING_DATA) - return pseudo_lev_type; - - switch (LOOKUP(rec, INDEX_LBVC)) - { - /* - * 1 Height (m) 8 Pressure (mb) - * 9 Hybrid co-ordinates 10 Sigma (=p/p*) - * 128 Mean sea level 129 Surface - * 130 Tropopause level 131 Maximum wind level - * 132 Freezing level 142 Upper hybrid level - * 143 Lower hybrid level 176 Latitude (deg) - * 177 Longitude (deg) - */ - /* also new dynamics: 65 hybrid height */ - - case 1: - return height_lev_type; - case 2: - return depth_lev_type; - case 5: - return boundary_layer_top_lev_type; - case 6: - return soil_lev_type; - case 8: - return pressure_lev_type; - case 9: - return hybrid_sigmap_lev_type; - case 65: - return hybrid_height_lev_type; - case 128: - return mean_sea_lev_type; - case 129: - return surface_lev_type; - case 130: - return tropopause_lev_type; - case 133: - return top_of_atmos_lev_type; - default: - return other_lev_type; - } -} diff --git a/cf/umread_lib/c-lib/type-dep/process_vars.c b/cf/umread_lib/c-lib/type-dep/process_vars.c deleted file mode 100644 index d41f9c60b1..0000000000 --- a/cf/umread_lib/c-lib/type-dep/process_vars.c +++ /dev/null @@ -1,350 +0,0 @@ -#include - -#include "umfileint.h" - - -File *file_parse_core(int fd, - File_type file_type) -{ - File *file; - List *heaplist; - - CKP( file = new_file() ); - file->fd = fd; - file->file_type = file_type; - - heaplist = file->internp->heaplist; - - CKI( read_all_headers(file, heaplist) ); - CKI( process_vars(file, heaplist) ); - - return file; - - err: - if (file) - free_file(file); - return NULL; -} - - -int process_vars(File *file, List *heaplist) -{ - int nrec; - Rec **recs; - List *vars; - - nrec = file->internp->nrec; - recs = file->internp->recs; - - /* initialise elements in the records before sorting */ - CKI( initialise_records(recs, nrec, heaplist) ); - - /* sort the records */ - qsort(recs, nrec, sizeof(Rec*), compare_records); - - /* now sort out the list of variables and dimensions */ - CKP( vars = list_new(heaplist) ); - CKI( get_vars(nrec, recs, vars, heaplist) ); - /* move the variables from the linked list to the array */ - CKI( list_copy_to_ptr_array(vars, &file->nvars, &file->vars, heaplist) ); - CKI( list_free(vars, 0, heaplist) ); - return 0; - ERRBLKI; -} - - -/* - * scan records for vars and add all to list - */ - -int get_vars(int nrec, Rec **recs, - List *vars, - List *heaplist) -{ - int recno; - int at_start_rec, at_end_rec = 0; - Rec *rec, **vrecs; - Z_axis *z_axis; - T_axis *t_axis; - Var *var; - int zindex, tindex; - int nvrec; - int svindex = 1; - - var = NULL; - z_axis = NULL; - t_axis = NULL; - - for (recno=0; recno < nrec ; recno++) - { - rec = recs[recno]; - - /* Some fieldsfiles have header fields with missing values */ - if (var_is_missing(rec->int_hdr)) - { - error_mesg("skipping variable stash code=%d, %d, %d " - "because of missing header data", - get_var_stash_model(rec->int_hdr), - get_var_stash_section(rec->int_hdr), - get_var_stash_item(rec->int_hdr)); - continue; - } - - /* we are at start record of a variable at the very start, or if at we - * were at the end record last time - */ - at_start_rec = ( recno == 0 || at_end_rec ); - - /* we are at end record of a variable at the very end, or if the header - * shows a difference from the next record which constitutes a different - * variable - * - * We also force end record of a variable if the grid type is not - * supported; any such records are passed back as single-record - * variables for the caller to deal with. - */ - at_end_rec = ( recno == nrec - 1 || - records_from_different_vars(recs[recno + 1], rec) || - !grid_supported(rec->int_hdr)); - - /* allow for variables which are unsupported for some reason */ - if (at_start_rec && test_skip_var(rec)) - continue; - - /* initialise new variable and axes if at first record of a variable */ - if (at_start_rec) - { - CKP( var = new_var(heaplist) ); - CKP( z_axis = new_z_axis(heaplist) ); - CKP( t_axis = new_t_axis(heaplist) ); - - var->internp->first_rec_no = recno; - var->internp->last_rec_no = -1; - var->internp->first_rec = rec; - } - - /* for every record, add the z, t values to the axes */ - - CKI( z_axis_add(z_axis, rec->internp->lev, &zindex, heaplist) ); - rec->internp->zindex = zindex; - - CKI( t_axis_add(t_axis, rec->internp->time, &tindex, heaplist) ); - rec->internp->tindex = tindex; - - if (at_end_rec) - { - var->internp->last_rec_no = recno; - nvrec = var->internp->last_rec_no - var->internp->first_rec_no + 1; - vrecs = recs + var->internp->first_rec_no; - - /* now if the axes are not regular, free the axes, split the variable - * into a number of variables and try again... - */ - if (set_disambig_index(z_axis, t_axis, vrecs, nvrec, svindex)) - { - /* increment the supervar index, used later to show the connection - * between the separate variables into which this one will be split - */ - svindex++; - - /* now re-sort this part of the record list, - * now that we have set the disambig index */ - qsort(vrecs, nvrec, sizeof(Rec *), compare_records); - - /* now go back to the start record of the variable; set to one less - * because it will get incremented in the "for" loop reinitialisation - */ - recno = var->internp->first_rec_no - 1; - - /* and free the stuff associated with the var we won't be using */ - CKI( free_z_axis(z_axis, heaplist) ); - CKI( free_t_axis(t_axis, heaplist) ); - CKI( free_var(var, heaplist) ); - - continue; - } - - /* add the metadata the caller needs */ - - var->nz = list_size(z_axis->values); - var->nt = list_size(t_axis->values); - var->recs = &recs[var->internp->first_rec_no]; - svindex = var->internp->first_rec->internp->supervar_index; - if (svindex >= 0) - var->supervar_index = svindex; - - /* add the variable */ - CKI( list_add(vars, var, heaplist) ); - - /* don't need the axes any more */ - CKI( free_z_axis(z_axis, heaplist) ); - CKI( free_t_axis(t_axis, heaplist) ); - } - } - return 0; - ERRBLKI; -} - - -int test_skip_var(const Rec *rec) -{ - char *skip_reason; - INTEGER *int_hdr; - - int_hdr = rec->int_hdr; - skip_reason = NULL; - - if (var_is_missing(int_hdr)) - skip_reason = "PP record has essential header data set to missing data value"; - - /* Compressed field index */ - if (get_var_compression(int_hdr) == 1) - skip_reason = "compressed field index not supported"; - - /* remove grid_supported test - now used to split up variables, not to - * exclude them. - * - * if (grid_supported(int_hdr) == 0) - * skip_reason = "grid code not supported"; - */ - - /* ADD ANY MORE VARIABLE SKIPPING CASES HERE. */ - - if (skip_reason != NULL) - { - error_mesg("skipping variable stash code=%d, %d, %d because: %s", - get_var_stash_model(int_hdr), - get_var_stash_section(int_hdr), - get_var_stash_item(int_hdr), - skip_reason); - return 1; - } - return 0; -} - - -int initialise_records(Rec **recs, int nrec, List *heaplist) -{ - int recno; - Rec *rec; - - for (recno = 0; recno < nrec ; recno++) - { - rec = recs[recno]; - rec->internp = rec->internp; - - rec->internp->disambig_index = -1; - rec->internp->supervar_index = -1; - - /* store level info */ - CKP( rec->internp->lev = malloc_(sizeof(Level), heaplist) ); - CKI( lev_set(rec->internp->lev, rec) ); - - /* store time info */ - CKP( rec->internp->time = malloc_(sizeof(Time), heaplist) ); - CKI( time_set(rec->internp->time, rec) ); - rec->internp->mean_period = get_mean_period(rec->internp->time); - } - return 0; - ERRBLKI; -} - - -/* - * set the disambig index on all records within a super-variable - */ -int set_disambig_index(Z_axis *z_axis, T_axis *t_axis, - Rec **recs, int nvrec, int svindex) -{ - int var_rec_no; - Rec *vrec; - int zindex, tindex, dindex; - int prev_zindex, prev_tindex, prev_dindex; - - prev_zindex = prev_tindex = prev_dindex = 0; - - /* do nothing if axes are regular */ - if (var_has_regular_z_t(z_axis, t_axis, recs, nvrec)) - return 0; - - for (var_rec_no=0; var_rec_no < nvrec; var_rec_no++) - { - vrec = recs[var_rec_no]; - - zindex = vrec->internp->zindex; - tindex = vrec->internp->tindex; - - /* check for dups coord pairs */ - /* the exact expressions for dindex are fairly arbitrary -- just need to - * ensure that indices for dup coordinate pairs will be different from - * indices for non-dups on other levels - */ - if (var_rec_no > 0 - && zindex == prev_zindex - && tindex == prev_tindex) - dindex = prev_dindex + 1; - else - dindex = zindex * nvrec; - - vrec->internp->disambig_index = dindex; - - if (vrec->internp->supervar_index < 0) - vrec->internp->supervar_index = svindex; - - /* save vals for next iter */ - prev_zindex = zindex; - prev_tindex = tindex; - prev_dindex = dindex; - } - return 1; -} - - -int grid_supported(INTEGER *int_hdr) -{ - int gridcode; - gridcode = get_var_gridcode(int_hdr); - - switch(gridcode) { - - case 1: - case 101: - case 11110: - return 1; - - default: - return 0; - } -} - - -/* routine to test t and z indices to check whether the variable is on regular - * array of times and levels (NB "regular" here refers to the ordering, not to - * whether the spacing is uniform) - */ - -int var_has_regular_z_t(Z_axis *z_axis, T_axis *t_axis, Rec **recs, int nvrec) -{ - int var_rec_no, nz, nt; /* needed for check on variables */ - Rec *rec; - - nz = list_size(z_axis->values); - nt = list_size(t_axis->values); - - /*------------------------------------------------------------*/ - - /* first test the most obvious case of irregular (for possible speed) */ - if (nvrec != nz * nt) - return 0; - - /* z indices (faster varying) should loop be vrec % nz */ - /* t indices (slower varying) should loop be vrec / nz */ - for (var_rec_no=0; var_rec_no < nvrec; var_rec_no++) - { - rec = recs[var_rec_no]; - if (rec->internp->zindex != var_rec_no % nz - || rec->internp->tindex != var_rec_no / nz) - return 0; - } - return 1; -} diff --git a/cf/umread_lib/c-lib/type-dep/read.c b/cf/umread_lib/c-lib/type-dep/read.c deleted file mode 100644 index 59b79d899f..0000000000 --- a/cf/umread_lib/c-lib/type-dep/read.c +++ /dev/null @@ -1,501 +0,0 @@ -#include -#include - -#include "umfileint.h" - -#define file_pos(f) (lseek(f, 0, SEEK_CUR)) - - -/* - * reads n words from file, storing them at ptr, with byte swapping as required - * returns number of words read (i.e. n, unless there's a short read) - */ -size_t read_words(int fd, - void *ptr, - size_t num_words, - Byte_ordering byte_ordering) -{ - size_t nread; - - CKP(ptr); - nread = read(fd, ptr, num_words * WORD_SIZE) / WORD_SIZE; - if (byte_ordering == REVERSE_ORDERING) - swap_bytes(ptr, nread); - return nread; - ERRBLKI; -} - - -int read_extra_data_core(int fd, - size_t extra_data_offset, - size_t extra_data_length, - Byte_ordering byte_ordering, - void *extra_data_rtn) -{ - /* reads extra data into storage provided by the caller - * The caller must provide the offset and length, obtained - * by a previous call to get_extra_data_offset_and_length() - */ - size_t extra_data_words; - - extra_data_words = extra_data_length / WORD_SIZE; - - CKI( lseek(fd, extra_data_offset, SEEK_SET) ); - ERRIF( extra_data_length % WORD_SIZE != 0 ); - ERRIF( read_words(fd, extra_data_rtn, - extra_data_words, byte_ordering) != extra_data_words); - return 0; - ERRBLKI; -} - - -int read_hdr_at_offset(int fd, - size_t header_offset, - Byte_ordering byte_ordering, - INTEGER *int_hdr_rtn, - REAL *real_hdr_rtn) -{ - /* as read_hdr below, but also specifying the file offset in bytes */ - - CKI( lseek(fd, header_offset, SEEK_SET) ); - return read_hdr(fd, byte_ordering, int_hdr_rtn, real_hdr_rtn); - ERRBLKI; -} - - -int read_hdr(int fd, - Byte_ordering byte_ordering, - INTEGER *int_hdr_rtn, - REAL *real_hdr_rtn) -{ - /* reads a PP header at specified word offset into storage - provided by the caller */ - - ERRIF( read_words(fd, int_hdr_rtn, - N_INT_HDR, byte_ordering) != N_INT_HDR); - - ERRIF( read_words(fd, real_hdr_rtn, - N_REAL_HDR, byte_ordering) != N_REAL_HDR); - - return 0; - ERRBLKI; -} - - -Rec *get_record(File *file, List *heaplist) -{ - /* reads PP headers and returns a Rec structure -- - * - * file must be positioned at start of header (after any fortran record length integer) on entry, - * and will be positioned at end of header on return - * - * the Rec structure will contain the headers in elements int_hdr and real_hdr, - * but other elements will be left as initialised by new_rec() - */ - - Rec *rec; - - CKP( rec = new_rec(WORD_SIZE, heaplist) ); - - CKI( - read_hdr(file->fd, - file->file_type.byte_ordering, - rec->int_hdr, - rec->real_hdr) - ); - - return rec; /* success */ - ERRBLKP; -} - -int read_all_headers(File *file, List *heaplist) -{ - switch (file->file_type.fmt) - { - case plain_pp: - return read_all_headers_pp(file, heaplist); - case fields_file: - return read_all_headers_ff(file, heaplist); - default: - switch_bug("read_all_headers"); - ERR; - } - return 0; - ERRBLKI; -} - -/* skip_fortran_record: skips a fortran record, and returns how big it was (in bytes), - * or -1 for end of file, or -2 for any error which may imply corrupt file - * (return value of 0 is a legitimate empty record). - */ -size_t skip_fortran_record(File *file) -{ - INTEGER rec_bytes, rec_bytes_2; - - if( read_words(file->fd, &rec_bytes, 1, - file->file_type.byte_ordering) != 1) return -1; - CKI( lseek(file->fd, rec_bytes, SEEK_CUR) ); - ERRIF( read_words(file->fd, &rec_bytes_2, 1, - file->file_type.byte_ordering) != 1); - ERRIF(rec_bytes != rec_bytes_2); - return rec_bytes; - ERRBLK(-2); -} - -int skip_word(File *file) -{ - CKI( lseek(file->fd, WORD_SIZE, SEEK_CUR) ); - return 0; - - ERRBLKI; -} - -int read_all_headers_pp(File *file, List *heaplist) -{ - int fd; - size_t nrec, rec_bytes, recno, header_offset; - Rec **recs, *rec; - - fd = file->fd; - - /* count the PP records in the file */ - lseek(fd, 0, SEEK_SET); - for (nrec = 0; (rec_bytes = skip_fortran_record(file)) != -1; nrec++) - { - ERRIF(rec_bytes == -2); - if (rec_bytes != N_HDR * WORD_SIZE) { - error_mesg("unsupported header length in PP file: %d words", - rec_bytes / WORD_SIZE); - ERR; - } - ERRIF( skip_fortran_record(file) < 0); /* skip the data record */ - } - - /* now rewind, and read in all the PP header data */ - CKP( recs = malloc_(nrec * sizeof(Rec *), heaplist) ); - file->internp->nrec = nrec; - file->internp->recs = recs; - - lseek(fd, 0, SEEK_SET); - for (recno = 0; recno < nrec; recno++) - { - CKI( skip_word(file) ); - header_offset = file_pos(fd); - CKP( rec = get_record(file, heaplist) ); - CKI( skip_word(file) ); - recs[recno] = rec; - - /* skip data record but store length */ - rec->header_offset = header_offset; - rec->data_offset = file_pos(fd) + WORD_SIZE; - rec->disk_length = skip_fortran_record(file); - } - return 0; - ERRBLKI; -} - - -#define READ_ITEM(x) \ - ERRIF( read_words(fd, &x, 1, byte_ordering) != 1); - -int read_all_headers_ff(File *file, List *heaplist) -{ - int fd; - size_t hdr_start, hdr_size, header_offset, data_offset_calculated, data_offset_specified; - int *valid, n_valid_rec, n_raw_rec, i_valid_rec, i_raw_rec; - Byte_ordering byte_ordering; - Rec *rec, **recs; - INTEGER start_lookup, nlookup1, nlookup2, dataset_type, start_data; - - fd = file->fd; - byte_ordering = file->file_type.byte_ordering; - - /* pick out certain information from the fixed length header */ - CKI( lseek(fd, 4 * WORD_SIZE, SEEK_SET) ); - READ_ITEM(dataset_type); - - ERRIF( read_words(fd, &dataset_type, 1, byte_ordering) != 1); - - CKI( lseek(fd, 149 * WORD_SIZE, SEEK_SET) ); - READ_ITEM(start_lookup); - READ_ITEM(nlookup1); - READ_ITEM(nlookup2); - - CKI( lseek(fd, 159 * WORD_SIZE, SEEK_SET) ); - READ_ITEM(start_data); - - /* (first dim of lookup documented as being 64 or 128, so - * allow header longer than n_hdr (64) -- discarding excess -- but not shorter) - */ - - if (nlookup1 < N_HDR) - { - error_mesg("unsupported header length: %d words", nlookup1); - ERR; - } - - CKP( valid = malloc_(nlookup2 * sizeof(int), heaplist) ); - - hdr_start = (start_lookup - 1) * WORD_SIZE; - hdr_size = nlookup1 * WORD_SIZE; - n_raw_rec = nlookup2; - CKI( get_valid_records_ff(fd, byte_ordering, hdr_start, hdr_size, n_raw_rec, - valid, &n_valid_rec) ); - - /* now read in all the PP header data */ - - CKP( recs = malloc_(n_valid_rec * sizeof(Rec *), heaplist) ); - /* debug("n_raw_rec=%d n_valid_rec=%d", n_raw_rec, n_valid_rec); */ - file->internp->nrec = n_valid_rec; - file->internp->recs = recs; - - i_valid_rec = 0; - data_offset_calculated = (start_data - 1) * WORD_SIZE; - for (i_raw_rec = 0; i_raw_rec < n_raw_rec; i_raw_rec++) - { - if (valid[i_raw_rec]) - { - header_offset = hdr_start + i_raw_rec * hdr_size; - CKI( lseek(fd, header_offset, SEEK_SET) ); - CKP( rec = get_record(file, heaplist) ); - recs[i_valid_rec] = rec; - - rec->header_offset = header_offset; - rec->disk_length = get_ff_disk_length(rec->int_hdr); - - data_offset_specified = (size_t) LOOKUP(rec, INDEX_LBBEGIN) * WORD_SIZE; - /* use LBBEGIN if available */ - rec->data_offset = - (data_offset_specified != 0) ? data_offset_specified : data_offset_calculated; - - data_offset_calculated += rec->disk_length; - - i_valid_rec++; - } - } - - CKI( free_(valid, heaplist) ); - return 0; - ERRBLKI; -} - - -size_t get_ff_disk_length(INTEGER *ihdr) -{ - /* work out disk length in bytes */ - /* Input array size (packed field): - * First try LBNREC - * then if Cray 32-bit packing, know ratio of packed to unpacked lengths; - * else use LBLREC - */ - if (ihdr[INDEX_LBPACK] != 0 && ihdr[INDEX_LBNREC] != 0) - return ihdr[INDEX_LBNREC] * WORD_SIZE; - if (ihdr[INDEX_LBPACK] % 10 == 2) - return get_num_data_words(ihdr) * 4; - return ihdr[INDEX_LBLREC] * WORD_SIZE; -} - - -/* - * check which PP records are valid; populate an array provided by the caller with 1s and 0s - * and also provide the total count - */ -int get_valid_records_ff(int fd, - Byte_ordering byte_ordering, - size_t hdr_start, size_t hdr_size, int n_raw_rec, - int valid[], int *n_valid_rec_return) -{ - int n_valid_rec, irec; - INTEGER lbbegin; - n_valid_rec = 0; - - for (irec = 0; irec < n_raw_rec; irec++) - { - valid[irec] = 0; - CKI( lseek(fd, hdr_start + irec * hdr_size + INDEX_LBBEGIN * WORD_SIZE, SEEK_SET) ); - READ_ITEM(lbbegin); - if (lbbegin != -99) - { - /* valid record */ - valid[irec] = 1; - n_valid_rec++; - } - else - valid[irec] = 0; - } - *n_valid_rec_return = n_valid_rec; - return 0; - ERRBLKI; -} - - -int read_record_data_core(int fd, - size_t data_offset, - size_t disk_length, - Byte_ordering byte_ordering, - const void *int_hdr, - const void *real_hdr, - size_t nwords, - void *data_return) -{ - int pack; - size_t packed_bytes, ipt, packed_words; - void *packed_data; - REAL mdi; - - packed_data = NULL; - - CKI( lseek(fd, data_offset, SEEK_SET) ); - pack = get_var_packing(int_hdr); - - if (pack == 0) - { - /* unpacked data -- read, and byte swap if necessary */ - ERRIF( read_words(fd, data_return, nwords, byte_ordering) != nwords); - } - else - { - /* PACKING IN USE */ - - /* Complain if not REAL data. In cdunifpp, this test was applied only to Cray 32-bit packing, - * but in fact also unwgdos assumes real, so apply to both packing types. - */ - if (get_type(int_hdr) != real_type) - { - error_mesg("Unpacking supported only for REAL type data"); - ERR; - } - - /* first allocate array and read in packed data */ - - /* disk_length includes extra data, so subtract off */ - packed_bytes = disk_length - get_extra_data_length(int_hdr); - - /* ---------------------------- - * Possible alternative envisaged, that reads in slightly more data to - * give tolerance against any situation where LBNREC does not include - * the extra data. However, this is now of dubious gain because reading in - * the extra data requires LBNREC to be used consistently so that the extra - * data can be found at the end of the record where the packed data that precedes - * it is of variable length. - * - * packed_bytes = disk_length; - *----------------------------- - */ - - - /* An exception to the usual strategy heap memory management: no heaplist available, so use plain malloc - * and be careful to free even if an error arose. - * - * (This is fairly much unavoidable: heaplist is attached to a File struct, but these exist only while - * parsing the file metadata, not while the read callback is executed. A Python File object will exist - * in the calling code, but that's not the same thing: it may have been instantiated with parse=False; - * see the Python code.) - */ - CKP( packed_data = malloc(packed_bytes) ); - ERRIF( read(fd, packed_data, packed_bytes) != packed_bytes ); - - /* NOW UNPACK ACCORDING TO PACKING TYPE (including byte swapping where necessary). */ - - switch(pack) - { - case 1: - /* WGDOS */ - - /* unwgdos routine wants to know number of native integers in input. - * input type might not be native int, so calculate: - */ - mdi = get_var_real_fill_value(real_hdr); - - /* Note - even though we read in raw (unswapped) data from the file, we do not - * byte swap prior to calling unwgdos, as the packed data contains a mixture - * of types of different lengths, so leave it to unwgdos() that knows about - * this and has appropriate byte swapping code. - */ - CKI( unwgdos(packed_data, packed_bytes, data_return, nwords, mdi) ); - - break; - - case 2: - if (byte_ordering == REVERSE_ORDERING) - swap_bytes_sgl(packed_data, packed_bytes / 4); - - for (ipt = 0; ipt < nwords ; ipt++) - ((REAL*) data_return)[ipt] = ((float32_t *) packed_data)[ipt]; - - break; - - case 3: - error_mesg("GRIB unpacking not supported"); - ERR; - - /* break; */ - - case 4: - packed_words = packed_bytes / WORD_SIZE; - if (byte_ordering == REVERSE_ORDERING) - swap_bytes(packed_data, packed_words); - mdi = get_var_real_fill_value(real_hdr); - CKI( unpack_run_length_encoded(packed_data, packed_words, data_return, nwords, mdi) ); - break; - - default: - SWITCH_BUG; - } - free(packed_data); - } - return 0; - err: - GRIPE; - if (packed_data != NULL) - free(packed_data); - return -1; -} - - -int unpack_run_length_encoded(REAL *datain, INTEGER nin, REAL *dataout, INTEGER nout, REAL mdi) -{ - REAL *src, *dest, *end_src, *end_dest, data; - INTEGER repeat; - - /* some pointers: - * src and dest are current positions; - * end_src and end_dest are the first position off the end of each array - */ - src = datain; - dest = dataout; - end_src = src + nin; - end_dest = dest + nout; - - /* syntax reminder: *p++ means first dereference p and then increment p - */ - while (src < end_src && dest < end_dest) - { - data = *src++; - if (data != mdi) - *dest++ = data; - else - { - /* check we didn't read the MDI as the last item in the input */ - ERRIF(src == end_src); - - /* read in next word, round to nearest integer, and output MDI that many times - * while checking we don't go beyond end of output data array - */ - for (repeat = (INTEGER)(0.5 + *src++); repeat > 0 && dest < end_dest; repeat--) - *dest++ = mdi; - - /* check we didn't reach end of output data array with copies of the MDI still to write - * (or read in a negative repeat count) - */ - ERRIF(repeat != 0); - } - } - /* check we reached end of output data, - * (not necessarily end of input data - it could be padded) - */ - ERRIF (dest != end_dest); - - return 0; - ERRBLKI; -} diff --git a/cf/umread_lib/c-lib/type-dep/redefs_dbl b/cf/umread_lib/c-lib/type-dep/redefs_dbl deleted file mode 100644 index 3991c45292..0000000000 --- a/cf/umread_lib/c-lib/type-dep/redefs_dbl +++ /dev/null @@ -1,62 +0,0 @@ -read_record_data_dummy read_record_data_dummy_sgl -get_extra_data_length get_extra_data_length_sgl -get_extra_data_offset_and_length_core get_extra_data_offset_and_length_core_sgl -get_num_data_words get_num_data_words_sgl -get_type get_type_sgl -get_type_and_num_words_core get_type_and_num_words_core_sgl -get_var_compression get_var_compression_sgl -get_var_gridcode get_var_gridcode_sgl -get_var_packing get_var_packing_sgl -get_var_real_fill_value get_var_real_fill_value_sgl -get_var_stash_item get_var_stash_item_sgl -get_var_stash_model get_var_stash_model_sgl -get_var_stash_section get_var_stash_section_sgl -var_is_missing var_is_missing_sgl -get_ff_disk_length get_ff_disk_length_sgl -get_record get_record_sgl -get_valid_records_ff get_valid_records_ff_sgl -read_all_headers read_all_headers_sgl -read_all_headers_ff read_all_headers_ff_sgl -read_all_headers_pp read_all_headers_pp_sgl -read_extra_data_core read_extra_data_core_sgl -read_hdr read_hdr_sgl -read_hdr_at_offset read_hdr_at_offset_sgl -read_record_data_core read_record_data_core_sgl -read_words read_words_sgl -skip_fortran_record skip_fortran_record_sgl -skip_word skip_word_sgl -unpack_run_length_encoded unpack_run_length_encoded_sgl -file_parse_core file_parse_core_sgl -get_vars get_vars_sgl -grid_supported grid_supported_sgl -initialise_records initialise_records_sgl -process_vars process_vars_sgl -set_disambig_index set_disambig_index_sgl -test_skip_var test_skip_var_sgl -var_has_regular_z_t var_has_regular_z_t_sgl -debug_dump_all_headers debug_dump_all_headers_sgl -calendar_type calendar_type_sgl -gregorian_to_secs gregorian_to_secs_sgl -is_time_mean is_time_mean_sgl -mean_period mean_period_sgl -sec_to_day sec_to_day_sgl -time_diff time_diff_sgl -time_set time_set_sgl -compare_dates compare_dates_sgl -compare_levels compare_levels_sgl -compare_lists compare_lists_sgl -compare_mean_periods compare_mean_periods_sgl -compare_records compare_records_sgl -compare_records_between_vars compare_records_between_vars_sgl -compare_records_within_var compare_records_within_var_sgl -compare_times compare_times_sgl -records_from_different_vars records_from_different_vars_sgl -level_type level_type_sgl -lev_set lev_set_sgl -free_t_axis free_t_axis_sgl -free_z_axis free_z_axis_sgl -new_t_axis new_t_axis_sgl -new_z_axis new_z_axis_sgl -t_axis_add t_axis_add_sgl -z_axis_add z_axis_add_sgl -unwgdos unwgdos_sgl diff --git a/cf/umread_lib/c-lib/type-dep/redefs_sgl b/cf/umread_lib/c-lib/type-dep/redefs_sgl deleted file mode 100644 index 7d5b976471..0000000000 --- a/cf/umread_lib/c-lib/type-dep/redefs_sgl +++ /dev/null @@ -1,62 +0,0 @@ -read_record_data_dummy read_record_data_dummy_dbl -get_extra_data_length get_extra_data_length_dbl -get_extra_data_offset_and_length_core get_extra_data_offset_and_length_core_dbl -get_num_data_words get_num_data_words_dbl -get_type get_type_dbl -get_type_and_num_words_core get_type_and_num_words_core_dbl -get_var_compression get_var_compression_dbl -get_var_gridcode get_var_gridcode_dbl -get_var_packing get_var_packing_dbl -get_var_real_fill_value get_var_real_fill_value_dbl -get_var_stash_item get_var_stash_item_dbl -get_var_stash_model get_var_stash_model_dbl -get_var_stash_section get_var_stash_section_dbl -var_is_missing var_is_missing_dbl -get_ff_disk_length get_ff_disk_length_dbl -get_record get_record_dbl -get_valid_records_ff get_valid_records_ff_dbl -read_all_headers read_all_headers_dbl -read_all_headers_ff read_all_headers_ff_dbl -read_all_headers_pp read_all_headers_pp_dbl -read_extra_data_core read_extra_data_core_dbl -read_hdr read_hdr_dbl -read_hdr_at_offset read_hdr_at_offset_dbl -read_record_data_core read_record_data_core_dbl -read_words read_words_dbl -skip_fortran_record skip_fortran_record_dbl -skip_word skip_word_dbl -unpack_run_length_encoded unpack_run_length_encoded_dbl -file_parse_core file_parse_core_dbl -get_vars get_vars_dbl -grid_supported grid_supported_dbl -initialise_records initialise_records_dbl -process_vars process_vars_dbl -set_disambig_index set_disambig_index_dbl -test_skip_var test_skip_var_dbl -var_has_regular_z_t var_has_regular_z_t_dbl -debug_dump_all_headers debug_dump_all_headers_dbl -calendar_type calendar_type_dbl -gregorian_to_secs gregorian_to_secs_dbl -is_time_mean is_time_mean_dbl -mean_period mean_period_dbl -sec_to_day sec_to_day_dbl -time_diff time_diff_dbl -time_set time_set_dbl -compare_dates compare_dates_dbl -compare_levels compare_levels_dbl -compare_lists compare_lists_dbl -compare_mean_periods compare_mean_periods_dbl -compare_records compare_records_dbl -compare_records_between_vars compare_records_between_vars_dbl -compare_records_within_var compare_records_within_var_dbl -compare_times compare_times_dbl -records_from_different_vars records_from_different_vars_dbl -level_type level_type_dbl -lev_set lev_set_dbl -free_t_axis free_t_axis_dbl -free_z_axis free_z_axis_dbl -new_t_axis new_t_axis_dbl -new_z_axis new_z_axis_dbl -t_axis_add t_axis_add_dbl -z_axis_add z_axis_add_dbl -unwgdos unwgdos_dbl diff --git a/cf/umread_lib/c-lib/type-dep/umfile_test_typedep.c b/cf/umread_lib/c-lib/type-dep/umfile_test_typedep.c deleted file mode 100644 index 4daad26106..0000000000 --- a/cf/umread_lib/c-lib/type-dep/umfile_test_typedep.c +++ /dev/null @@ -1,154 +0,0 @@ -/* datatype dependent parts of the test code */ - -#include -#include -#include -#include - -#include "umfileint.h" - -//Rec *rec_alloc() -//{ -// Rec *rec; -// rec = xmalloc(sizeof(Rec)); -// rec->internp = xmalloc(sizeof(struct _Rec)); -// rec->int_hdr = xmalloc(45 * sizeof(INTEGER)); -// rec->real_hdr = xmalloc(19 * sizeof(REAL)); -// return rec; -//} -// -//void rec_free(Rec *rec) -//{ -// _rec_internals_free(rec); -// { -// INTEGER *ihdr = rec->int_hdr; -// REAL *rhdr = rec->real_hdr; -// printf("freeing rec with ihdr=%d %d... rhdr=%f %f...\n", -// ihdr[0], ihdr[1], rhdr[0], rhdr[1]); -// } -// xfree(rec->int_hdr); -// xfree(rec->real_hdr); -// xfree(rec); -//} -// -// -//Rec *rec_create_dummy(int k) -//{ -// int i; -// Rec *rec; -// rec = rec_alloc(); -// for (i = 0; i < 45 ; i++) -// ((INTEGER *)rec->int_hdr)[i] = k * 100 + i; -// for (i = 0; i < 19 ; i++) -// ((REAL *)rec->real_hdr)[i] = k + i / 100.; -// rec->header_offset = k; -// rec->data_offset = 500 + k; -// rec->internp->blahblah = 200 + k; -// return rec; -//} -// -//int get_type_and_length_dummy(const void *int_hdr, Data_type *type_rtn, size_t *num_words_rtn) -//{ -// const INTEGER *int_hdr_4 = int_hdr; -// *num_words_rtn = int_hdr_4[0]; -// *type_rtn = real_type; -// return 0; -//} -// -void read_record_data_dummy(size_t nwords, - void *data_return) -{ - int i; - REAL *data_return_4 = data_return; - for (i = 0; i < nwords; i++) - { - data_return_4[i] = i / 100.; - } -} - - -// int read_record_data_core(int fd, -// size_t data_offset, -// size_t disk_length, -// Byte_ordering byte_ordering, -// int word_size, -// const void *int_hdr, -// const void *real_hdr, -// size_t nwords, -// void *data_return) -// { -// int i; -// assert(byte_ordering == little_endian); -// assert(word_size == 4); -// -// printf("start of int header seen in read_record_data_dummy():"); -// for (i = 0; i < 5; i++) -// printf(" %d", ((INTEGER *) int_hdr)[i]); -// printf("\n"); -// printf("start of real header seen in read_record_data_dummy():"); -// for (i = 0; i < 5; i++) -// printf(" %f", ((REAL *) real_hdr)[i]); -// printf("\n"); -// -// read_record_data_dummy(nwords, data_return); -// return 0; -// } -// - -#ifdef MAIN -int main() -{ - int i, j, k, nrec; - int fd; - File *file; - File_type file_type; - Var *var; - Rec *rec; - REAL *data; - int word_size; - Data_type data_type; - size_t nwords, nbytes; - - fd = 3; - detect_file_type(fd, &file_type); - printf("word size = %d\n", file_type.word_size); - - file = file_parse(fd, file_type); - for (i = 0; i < file->nvars; i++) - { - printf("var %d\n", i); - var = file->vars[i]; - printf("nz = %d, nt = %d\n", var->nz, var->nt); - nrec = var->nz * var->nt; - for (j = 0; j < nrec; j++) - { - rec = var->recs[j]; - printf("var %d rec %d\n", i, j); - printf("int header:\n"); - for (k = 0; k < 45; k++) - printf(" ihdr[%d] = %d\n", k, ((INTEGER *)rec->int_hdr)[k]); - printf("real header:\n"); - for (k = 0; k < 19; k++) - printf(" rhdr[%d] = %f\n", k, ((REAL *)rec->real_hdr)[k]); - - word_size = file_type.word_size; - get_type_and_length(word_size, rec->int_hdr, &data_type, &nwords); - nbytes = word_size * nwords;; - printf("data (%ld items)\n", nwords); - data = xmalloc(nbytes * sizeof(float)); - read_record_data(fd, - rec->data_offset, - file_type.byte_ordering, - file_type.word_size, - rec->int_hdr, - rec->real_hdr, - nwords, - data); - for (k = 0; k < nwords; k++) - printf(" data[%d] = %f\n", k, data[k]); - xfree(data); - } - } - return 0; -} -#endif diff --git a/cf/umread_lib/c-lib/type-dep/umfile_typedep.a b/cf/umread_lib/c-lib/type-dep/umfile_typedep.a deleted file mode 100644 index d1a02b38cb..0000000000 Binary files a/cf/umread_lib/c-lib/type-dep/umfile_typedep.a and /dev/null differ diff --git a/cf/umread_lib/c-lib/type-dep/unwgdos.c b/cf/umread_lib/c-lib/type-dep/unwgdos.c deleted file mode 100644 index 1644311a2c..0000000000 --- a/cf/umread_lib/c-lib/type-dep/unwgdos.c +++ /dev/null @@ -1,508 +0,0 @@ - -/* - * Modified version of unwgdos to remove the dependence on all the Cray - * utilities, most of which were irrelevant (although some needed functions - * have been added below and it also calls byte-swapping code in swap.c). - * It is for use on machines with native IEEE integer types. - * - * Note API change compared to version taken from xconv: datain is void* and - * second argument is number of bytes (not ints) - */ - -/* unwgdos.c is unpack.c file from xconv but with GRIB stuff stripped out */ - -#include -#include -#include -#include -#include - -#include "umfileint.h" - - -#define TRUE 1 -#define FALSE 0 - -static int xpnd(int, int32_t *, REAL *, REAL, int, REAL, int, REAL); -static int extrin(int32_t *, int, int, int, int *, int); -static int bit_test(void *, int); -static void move_bits(void *, int, int, void *); -static float32_t get_float32(void *); -static int16_t get_int16(void *, Byte_ordering); -static int32_t get_int32(void *, Byte_ordering); - - -int unwgdos(void *datain, int nbytes, REAL *dataout, int nout, REAL mdi) -{ - int /* len, */ isc, ix, iy; - REAL prec, base; - int icx, j; - int ibit, nop; - int swap; - char *p, *p1; - - /* Determine if data needs byte swapping */ - - p = datain; - - swap = -1; - - ix = get_int16(p + 8, big_endian); - iy = get_int16(p + 10, big_endian); - if (ix*iy == nout) swap = 0; - - if (swap == -1) - { - /* see if data is byte swapped with 4 byte words */ - ix = get_int16(p + 10, little_endian); - iy = get_int16(p + 8, little_endian); - if (ix*iy == nout) swap = 4; - } - - if (swap == -1) - { - /* see if data is byte swapped with 8 byte words */ - ix = get_int16(p + 14, little_endian); - iy = get_int16(p + 12, little_endian); - if (ix*iy == nout) swap = 8; - } - - if (swap == -1) - { - error_mesg("WGDOS data header record mismatch "); - return 1; - } - else if (swap == 4) - { - swap_bytes_sgl(datain, nbytes / 4); - } - else if (swap == 8) - { - swap_bytes_dbl(datain, nbytes / 8); - } - - - /* Below only works for 32 bit integers, therefore there must be a - 32 bit integer type */ - - /* Extract scale factor and number of columns and rows from header */ - isc = get_int32(p + 4, big_endian); - ix = get_int16(p + 8, big_endian); - iy = get_int16(p + 10, big_endian); - - /* Expand compressed data */ - - prec = pow(2.0, (double) isc); - icx = 3; - - for (j=0; j= 128) - { - btzer = TRUE; - btmap = TRUE; - ibit -= 128; - } - - /* check if bitmap used for minimum values */ - - if (ibit >= 64) - { - btmin = TRUE; - btmap = TRUE; - ibit -= 64; - } - - /* check if bitmap used for missing data values */ - - if (ibit >= 32) - { - btmis = TRUE; - btmap = TRUE; - ibit -= 32; - } - - if (ibit > 32) - { - error_mesg("Number of bits used to pack wgdos data = %d must be <= 32 ", - ibit); - return 1; - } - - if (btmap) - { - if ( (imap = malloc (ix*sizeof(int))) == NULL ) - { - error_mesg("Error unable to allocate memory for imap in xpnd ix = %d ", - ix); - return 1; - } - - for (j=0; j 0) - jbit--; - else - { - jbit = 31; - jword++; - } - } - } - - /* Extract minimum value bitmap */ - - if (btmin) - { - if ( (imin = malloc (ix*sizeof(int))) == NULL ) - { - error_mesg("Error unable to allocate memory for imin in xpnd ix = %d ", - ix); - return 1; - } - - for (j=0; j 0) - jbit--; - else - { - jbit = 31; - jword++; - } - } - } - - /* Extract zero value bitmap */ - - if (btzer) - { - if ( (izer = malloc (ix*sizeof(int))) == NULL ) - { - error_mesg("Error unable to allocate memory for izer in xpnd ix = %d ", - ix); - return 1; - } - - for (j=0; j 0) - jbit--; - else - { - jbit = 31; - jword++; - } - } - } - - - /* If bitmap used reset pointers to beginning of 32 bit boundary */ - - if (btmap && jbit != 31) - { - jbit = 31; - jword++; - } - - if (ibit > 0) - { - /* Unpack scaled values */ - - for (j=0; j> ibit) & ~(~0 << 1); - - if (i == 1) - return TRUE; - else - return FALSE; -} - -/* - * Move nbits from 32 bit word1 starting at start1 into 32 bit word2. - * 0 =< nbits <= 32, bits can cross into word1+1. - */ - -static void move_bits(void *word1, int start1, int nbits, void *word2) -{ - uint32_t *ui1, *ui2, temp1, temp2; - - ui1 = (uint32_t *) word1; - ui2 = (uint32_t *) word2; - - if (start1+1-nbits >= 0) - { - /* move bits within one word */ - - ui2[0] = (ui1[0] >> (start1+1-nbits)) & ~(~0 << nbits); - } - else - { - /* move bits within two words */ - - temp1 = (ui1[0] << (nbits-start1-1)) & ~(~0 << nbits); - temp2 = (ui1[1] >> (32+start1+1-nbits)) & ~(~0 << (nbits-start1-1)); - ui2[0] = temp1 | temp2; - } -} - - -#define I32_INFP 0x7f800000 -#define I32_INFN 0xff800000 -#define I32_ZEROP 0x00000000 -#define I32_ZERON 0x80000000 - -/* based on ibmr4_to_r4 */ -static float32_t get_float32(void *in) -{ - unsigned char *pin; - unsigned long man; - int exp, sign; - double d; - uint32_t i32; - - pin = (unsigned char *) in; - - sign = pin[0] & 0x80; - exp = pin[0] & 0x7f; - man = ((unsigned long) pin[1] << 16) | - ((unsigned long) pin[2] << 8) | (unsigned long) pin[3]; - - d = ldexp((double) man ,4*(exp-64-6)); - - if (d > (double) FLT_MAX || errno == ERANGE) - { - i32 = (sign ? I32_INFN : I32_INFP); - return *(float32_t *) &i32; - } - else if (d < (double) FLT_MIN) - { - i32 = (sign ? I32_ZERON : I32_ZEROP); - return *(float32_t *) &i32; - } - else - return (sign ? -d : d); -} - - -/* functions to get data values that are stored starting at a specified - * pointer with the specified byte ordering; if they are in native byte - * ordering then just copy them, otherwise byte-swap them - */ -static int16_t get_int16(void *start, Byte_ordering byte_ordering) -{ - if (byte_ordering == NATIVE_ORDERING) - { - return *(int16_t *) start; - } - else - { - char *in, out[2]; - in = (char *) start; - out[0] = in[1]; - out[1] = in[0]; - return *(int16_t *) out; - } -} - -static int32_t get_int32(void *start, Byte_ordering byte_ordering) -{ - if (byte_ordering == NATIVE_ORDERING) - { - return *(int32_t *) start; - } - else - { - char *in, out[4]; - in = (char *) start; - out[0] = in[3]; - out[1] = in[2]; - out[2] = in[1]; - out[3] = in[0]; - return *(int32_t *) out; - } -} - diff --git a/cf/umread_lib/c-lib/umfile.c b/cf/umread_lib/c-lib/umfile.c deleted file mode 100644 index bff514ba81..0000000000 --- a/cf/umread_lib/c-lib/umfile.c +++ /dev/null @@ -1,165 +0,0 @@ -#include -#include - -#include "umfileint.h" - -int get_type_and_num_words(int word_size, - const void *int_hdr, - Data_type *type_rtn, - size_t *num_words_rtn) -{ - errorhandle_init(); - switch (word_size) - { - case 4: - return get_type_and_num_words_core_sgl(int_hdr, type_rtn, num_words_rtn); - case 8: - return get_type_and_num_words_core_dbl(int_hdr, type_rtn, num_words_rtn); - default: - return -1; - } -} - -int get_extra_data_offset_and_length(int word_size, - const void *int_hdr, - size_t data_offset, - size_t disk_length, - size_t *extra_data_offset_rtn, - size_t *extra_data_length_rtn) -{ - errorhandle_init(); - switch (word_size) - { - case 4: - return get_extra_data_offset_and_length_core_sgl(int_hdr, - data_offset, - disk_length, - extra_data_offset_rtn, - extra_data_length_rtn); - case 8: - return get_extra_data_offset_and_length_core_dbl(int_hdr, - data_offset, - disk_length, - extra_data_offset_rtn, - extra_data_length_rtn); - default: - return -1; - } -} - -int detect_file_type(int fd, File_type *file_type) -{ - errorhandle_init(); - return detect_file_type_(fd, file_type); -} - -int read_extra_data(int fd, - size_t extra_data_offset, - size_t extra_data_length, - Byte_ordering byte_ordering, - int word_size, - void *extra_data_return) -{ - errorhandle_init(); - switch (word_size) - { - case 4: - return read_extra_data_core_sgl(fd, - extra_data_offset, - extra_data_length, - byte_ordering, - extra_data_return); - case 8: - return read_extra_data_core_dbl(fd, - extra_data_offset, - extra_data_length, - byte_ordering, - extra_data_return); - default: - return -1; - } -} - -int read_header(int fd, - size_t header_offset, - Byte_ordering byte_ordering, - int word_size, - void *int_hdr_rtn, - void *real_hdr_rtn) -{ - errorhandle_init(); - switch (word_size) - { - case 4: - return read_hdr_at_offset_sgl(fd, header_offset, byte_ordering, - int_hdr_rtn, real_hdr_rtn); - case 8: - return read_hdr_at_offset_dbl(fd, header_offset, byte_ordering, - int_hdr_rtn, real_hdr_rtn); - default: - return -1; - } -} - - -File *file_parse(int fd, - File_type file_type) -{ - File *file; - - errorhandle_init(); - - switch (file_type.word_size) - { - case 4: - CKP( file = file_parse_core_sgl(fd, file_type) ); - break; - case 8: - CKP( file = file_parse_core_dbl(fd, file_type) ); - break; - default: - ERR; - } - return file; - ERRBLKP; -} - - -void file_free(File *file) -{ - errorhandle_init(); - - CKI( free_file(file) ); - return; - - err: - GRIPE; -} - - -int read_record_data(int fd, - size_t data_offset, - size_t disk_length, - Byte_ordering byte_ordering, - int word_size, - const void *int_hdr, - const void *real_hdr, - size_t nwords, - void *data_return) -{ - errorhandle_init(); - - switch(word_size) - { - case 4: - CKI( read_record_data_core_sgl(fd, data_offset, disk_length, byte_ordering, - int_hdr, real_hdr, nwords, data_return) ); - return 0; - case 8: - CKI( read_record_data_core_dbl(fd, data_offset, disk_length, byte_ordering, - int_hdr, real_hdr, nwords, data_return) ); - return 0; - } - /* invalid word size falls through to error return */ - ERRBLKI; -} diff --git a/cf/umread_lib/c-lib/umfile.h b/cf/umread_lib/c-lib/umfile.h deleted file mode 100644 index fdccde8e91..0000000000 --- a/cf/umread_lib/c-lib/umfile.h +++ /dev/null @@ -1,204 +0,0 @@ -/* - * ============================================================== - * Header file only for stuff intended to be called directly from - * python. Everything else should be in umfileint.h. - * ============================================================== - */ - -#include - -typedef enum -{ - plain_pp, - fields_file -} - File_format; - -typedef enum -{ - little_endian, - big_endian -} - Byte_ordering; - -typedef enum -{ - int_type, - real_type -} - Data_type; - - -/* ---------------- - * Placeholders for internal structures to be defined in umfileint.h - * Here just need to predeclare them so that we can have pointers to them. - */ -struct _File; -struct _Var; -struct _Rec; -/* ---------------- */ - -typedef struct -{ - File_format fmt; - Byte_ordering byte_ordering; - int word_size; -} - File_type ; - -typedef struct -{ - void *int_hdr; - void *real_hdr; - size_t header_offset; /* in bytes */ - size_t data_offset; /* in bytes */ - size_t disk_length; /* in bytes */ - struct _Rec *internp; -} - Rec; - -typedef struct -{ - Rec **recs; - int nz; - int nt; - int supervar_index; - struct _Var *internp; -} - Var; - -typedef struct -{ - int fd; - File_type file_type; - int nvars; - Var **vars; - struct _File *internp; -} - File; - -/* ------------------------------------------------------------------- */ - -int detect_file_type(int fd, File_type *file_type_rtn); -/* - Given an open file, detect type of file (caller provides storage for info - returned). if detection was successful, returns 0 and populates the - File_type structure provided by the caller, otherwise returns 1. -*/ - -File *file_parse(int fd, - File_type file_type); -/* - Given an open file handle, parse a file into a File structure, with - embedded Var and Rec structures, relating the PP records to variables - within the file. - - Caller should pass a File_type structure as either returned from - detect_file_type() or populated by the caller. -*/ - -/* commented out - to handle in Python */ -/* void close_fd(File *file); */ -/* int reopen_fd(File *file); */ - -void file_free(File *file); -/* - * Free memory associated with a File structure (including anything hung off - * the internal pointer) and all variables and records underneath it - */ - -/* ------------------------------------------------------------------- */ -/* functions for reading the actual data, not dependent on the above objects - * (although may call common code in the implementation) - */ - -int read_header(int fd, - size_t header_offset, - Byte_ordering byte_ordering, - int word_size, - void *int_hdr_rtn, - void *real_hdr_rtn); -/* - reads a PP header at specified offset; function will do byte-swapping - as necessary, but returned header data will match word size, and - caller must provide storage of appropriate length to contain these - (45 and 19 words respectively) -*/ - - -int get_type_and_length(int word_size, - const void *int_hdr, - Data_type *type_rtn, - size_t *num_words_rtn); -/* - Parses integer PP header, works out number of words and data type. - - Caller provides integer header as array of 4 or 8 byte ints - as appropriate to passed word_size, and provides storage for - returned info. - - Return value is 0 for success, otherwise 1. - */ - -int read_record_data(int fd, - size_t data_offset, - size_t disk_length, - Byte_ordering byte_ordering, - int word_size, - const void *int_hdr, - const void *real_hdr, - size_t nwords, - void *data_return); -/* - Reads record data at specified offset; function will do byte-swapping and - unpacking as necessary. Caller provides PP headers as arrays of 4 or 8 - byte ints and floats/doubles as appropriate to passed word_size (real - header needed for missing data value). This must match actual file word - size, and there will be no casting of data except as appropriate when - unpacking packed fields. - - Returns in data_return an array of int or float at this word size; caller - must provide storage of size nwords words, and nwords must have been - obtained by calling get_nwords(). - - Return value is 0 for success, 1 for failure. -*/ -/* ------------------------------------------------------------------- */ - - -int get_extra_data_offset_and_length(int word_size, - const void *int_hdr, - size_t data_offset, - size_t disk_length, - size_t *extra_data_offset_rtn, - size_t *extra_data_length_rtn); -/* - Parses integer PP header in conjunction with data offset and length - (where the length includes the extra data), to works out the offset - and length of extra data - - Caller provides integer header as array of 4 or 8 byte ints - as appropriate to passed word_size, and provides storage for - returned info. - - Return value is length, or -1 on failure - */ - -int read_extra_data(int fd, - size_t extra_data_offset, - size_t extra_data_length, - Byte_ordering byte_ordering, - int word_size, - void *extra_data_return); - -/* - Reads extra data at specified offset; function will do byte-swapping - as necessary. - - Returns raw data (aside from the byte swapping) in extra_data_return; - caller must provide storage of size extra_data_length * word_size bytes, - and extra_data_length must have been obtained by calling - get_extra_data_length(). - - Return value is 0 for success, 1 for failure. -*/ diff --git a/cf/umread_lib/c-lib/umfileint.h b/cf/umread_lib/c-lib/umfileint.h deleted file mode 100644 index 7b0502d8ca..0000000000 --- a/cf/umread_lib/c-lib/umfileint.h +++ /dev/null @@ -1,25 +0,0 @@ -#include "umfile.h" - -/*---------------------------*/ - -#include "bits/constants.h" -#include "bits/datatype.h" -#include "bits/ordering.h" -#include "bits/typedefs.h" -#include "bits/type_indep_protos.h" -#include "bits/pp_header.h" -#include "bits/err_macros.h" - -/* ----------------------------------------------------------- */ - -#if defined(SINGLE) || defined(DOUBLE) -#include "bits/type_dep_redefs.h" -#include "bits/type_dep_protos.h" -#else -#define WITH_LEN(x) x ## _sgl -#include "bits/type_dep_entry_protos.h" -#undef WITH_LEN -#define WITH_LEN(x) x ## _dbl -#include "bits/type_dep_entry_protos.h" -#undef WITH_LEN -#endif diff --git a/cf/umread_lib/cInterface.py b/cf/umread_lib/cInterface.py deleted file mode 100644 index cbe4a2409d..0000000000 --- a/cf/umread_lib/cInterface.py +++ /dev/null @@ -1,656 +0,0 @@ -import ctypes as CT -import os - -import numpy -import numpy.ctypeslib - -from . import umfile - -_len_real_hdr = 19 -_len_int_hdr = 45 - - -class File_type(CT.Structure): - _fields_ = [ - ("fmt", CT.c_int), - ("byte_ordering", CT.c_int), - ("word_size", CT.c_int), - ] - - -def _get_ctypes_array(dtype, size=None): - """Get ctypes corresponding to a numpy array of a given type. - - The size should not be necessary unless the storage for the array is - allocated in the C code. - - """ - kwargs = { - "dtype": dtype, - "ndim": 1, - "flags": ("C_CONTIGUOUS", "WRITEABLE"), - } - if size: - kwargs["shape"] = (size,) - - return numpy.ctypeslib.ndpointer(**kwargs) - - -def _gen_rec_class(int_type, float_type): - class Rec(CT.Structure): - """ctypes object corresponding to the `Rec` object in the C - code.""" - - _fields_ = [ - ("int_hdr", _get_ctypes_array(int_type, _len_int_hdr)), - ("real_hdr", _get_ctypes_array(float_type, _len_real_hdr)), - ("header_offset", CT.c_size_t), - ("data_offset", CT.c_size_t), - ("disk_length", CT.c_size_t), - ("_internp", CT.c_void_p), - ] - - return Rec - - -Rec32 = _gen_rec_class(numpy.int32, numpy.float32) -Rec64 = _gen_rec_class(numpy.int64, numpy.float64) - - -def _gen_var_class(rec_class): - class Var(CT.Structure): - """ctypes object corresponding to the `Var` object in the C - code.""" - - _fields_ = [ - ("recs", CT.POINTER(CT.POINTER(rec_class))), - ("nz", CT.c_int), - ("nt", CT.c_int), - ("supervar_index", CT.c_int), - ("_internp", CT.c_void_p), - ] - - return Var - - -Var32 = _gen_var_class(Rec32) -Var64 = _gen_var_class(Rec64) - - -def _gen_file_class(var_class): - class File(CT.Structure): - """ctypes object corresponding to the `File` object in the C - code.""" - - _fields_ = [ - ("fd", CT.c_int), - ("file_type", File_type), - ("nvars", CT.c_int), - ("vars", CT.POINTER(CT.POINTER(var_class))), - ("_internp", CT.c_void_p), - ] - - return File - - -File32 = _gen_file_class(Var32) -File64 = _gen_file_class(Var64) - - -class Enum: - def __init__(self, *names): - self.names = names - - def as_name(self, val): - if isinstance(val, str): - return val - else: - return self.names[val] - - def as_index(self, val): - if isinstance(val, int): - return val - - return self.names.index(val) - - -enum_file_format = Enum("PP", "FF") -enum_byte_ordering = Enum("little_endian", "big_endian") -enum_data_type = Enum("integer", "real") - - -class CInterface: - """Interface to the C shared library functions.""" - - def __init__(self, lib_name="umfile.so"): - """**Initialisation** - - :Parameters: - - lib_name: `str` - The name of the C library binary. - - """ - lib_dir = os.path.join(os.path.dirname(__file__) or ".", "c-lib") - lib_path = os.path.join(lib_dir, lib_name) - self.lib = CT.CDLL(lib_path) - - def _is_null_pointer(self, ptr): - """True if the pointer is a null pointer. - - :Returns: - - `bool` - - """ - try: - ptr.contents - return False - except ValueError: - return True - - def detect_file_type(self, fd): - """Auto-detect file type. - - :Parameters: - - fd: `int` - The file descriptor of the open file. - - :Returns: - - `File_type` - A `File_type` ctypes object that can be passed to - `file_parse`, or raises an exception if file type cannot - be detected. - - """ - file_type = File_type() - rv = self.lib.detect_file_type(fd, CT.pointer(file_type)) - if rv != 0: - raise umfile.UMFileException("File type could not be detected") - - return file_type - - def file_type_obj_to_dict(self, file_type): - """Converts a `File_type` object returned by `detect_file_type` into a - dictionary that includes meaningful string values in place of the - integers that derive from the C enum statments, specifically: - 'fmt': 'PP' or 'FF' 'byte_ordering': 'little_endian' or - 'big_endian' and also 'word_size': 4 or 8 - - :Returns: - - `dict` - - """ - fmt = enum_file_format.as_name(file_type.fmt) - byte_ordering = enum_byte_ordering.as_name(file_type.byte_ordering) - word_size = file_type.word_size - return { - "fmt": fmt, - "byte_ordering": byte_ordering, - "word_size": word_size, - } - - def create_file_type(self, fmt, byte_ordering, word_size): - """Creates a `File_type` object for passing to `file_parse`. - - :Parameters: - - fmt: `str` - 'PP' or 'FF' - - byte_ordering: `str` - 'little_endian' or 'big_endian' - - word_size: `str` - 4 or 8 - - :Returns: - - `File_type` - A `File_type` object (ctypes structure containing integer - values) that can be passed to `file_parse`. - - """ - return File_type( - fmt=enum_file_format.as_index(fmt), - byte_ordering=enum_byte_ordering.as_index(byte_ordering), - word_size=word_size, - ) - - def set_word_size(self, val): - """Sets the word size used to interpret returned pointers from - subsequent calls, in particular the pointers to PP headers - embedded in the tree of objects returned by `file_parse` and the - data array that is populated by `read_record_data`. - - :Parameters: - - val: `int` or `File_type` - Either just the word_size value to use or a `File_type` - object from which it is to be extracted. - - """ - if isinstance(val, File_type): - word_size = val.word_size - else: - word_size = val - - if word_size == 4: - self.file_class = File32 - self.file_data_int_type = numpy.int32 - self.file_data_real_type = numpy.float32 - self._int_ptr = CT.POINTER(CT.c_int32) - self._real_ptr = CT.POINTER(CT.c_float) - elif word_size == 8: - self.file_class = File64 - self.file_data_int_type = numpy.int64 - self.file_data_real_type = numpy.float64 - self._int_ptr = CT.POINTER(CT.c_int64) - self._real_ptr = CT.POINTER(CT.c_double) - else: - raise ValueError(f"Word size must be 4 or 8 (not {word_size!r})") - - def _get_ctypes_int_array(self, size=None): - """Get ctypes corresponding to the `numpy` integer array.""" - return _get_ctypes_array(self.file_data_int_type, size) - - def _get_ctypes_real_array(self, size=None): - """Get ctypes corresponding to the `numpy` real array.""" - return _get_ctypes_array(self.file_data_real_type, size) - - def _get_empty_real_array(self, size): - """Get empty `numpy` real array according to word size - previously set with `set_word_size`.""" - return numpy.empty(size, dtype=self.file_data_real_type) - - def _get_empty_int_array(self, size): - """Get empty `numpy` integer array according to word size - previously set with `set_word_size`.""" - return numpy.empty(size, dtype=self.file_data_int_type) - - def parse_file(self, fh, file_type): - """Given an open file handle, work out information from the - file. - - :Parameters: - - fh: `int` - Low-level file handle. - - file_type: `File_type` - `File_type` object as returned by `detect_file_type` or - `create_file_type`. - - :Returns: - - `dict` - The information from the file. Currently the only key - actually implemented is ``'vars'``, containing a list of - variables, as that is all that the caller requires. - - """ - func = self.lib.file_parse - file_p_type = CT.POINTER(self.file_class) - func.restype = file_p_type - - file_p = func(fh, file_type) - if self._is_null_pointer(file_p): - raise umfile.UMFileException("File parsing failed") - - file = file_p.contents - c_vars = file.vars[: file.nvars] - rv = {"vars": list(map(self.c_var_to_py_var, c_vars))} - - # Now that we have copied all the data into python objects for - # the caller, free any memory allocated in the C code before - # returning - free_func = self.lib.file_free - free_func._fields_ = file_p_type - free_func(file_p) - - return rv - - def c_var_to_py_var(self, c_var_p): - """Create a `umfile.Var` object from a ctypes object - corresponding to 'Var*' in the C code. - - :Returns: - - `umfile.Var` - - """ - c_var = c_var_p.contents - nz = c_var.nz - nt = c_var.nt - svi = c_var.supervar_index - if svi < 0: - svi = None - - c_recs = c_var.recs - recs = [ - self.c_rec_to_py_rec(c_recs[recid]) for recid in range(nz * nt) - ] - - return umfile.Var(recs, nz, nt, svi) - - def c_rec_to_py_rec(self, c_rec_p): - """Create a `umfile.Rec` object from a ctypes object - corresponding to 'Rec*' in the C code. - - :Returns: - - `umfile.Rec` - - """ - c_rec = c_rec_p.contents - - # ============================================================ - # Previous code - causing memory leaks per - # https://github.com/numpy/numpy/issues/6511 - # - # # numpy.copy used here so we can go back and free the memory - # # allocated by C without affecting the python object - # int_hdr = numpy.copy(numpy.ctypeslib.as_array(c_rec.int_hdr)) - # real_hdr = numpy.copy(numpy.ctypeslib.as_array(c_rec.real_hdr)) - # ============================================================ - - # - # ------------------------------- - # Workaround: instead cast to a pointer, obtain the values knowing - # the length of the header arrays, and copy into an appropriate - # numpy array - # - ptr = CT.cast(c_rec.int_hdr, self._int_ptr) - int_hdr = numpy.array( - ptr[:_len_int_hdr], dtype=self.file_data_int_type - ) - - ptr = CT.cast(c_rec.real_hdr, self._real_ptr) - real_hdr = numpy.array( - ptr[:_len_real_hdr], dtype=self.file_data_real_type - ) - # ============================================================ - - header_offset = c_rec.header_offset - data_offset = c_rec.data_offset - disk_length = c_rec.disk_length - return umfile.Rec( - int_hdr, real_hdr, header_offset, data_offset, disk_length - ) - - def get_type_and_num_words(self, int_hdr): - """From the integer header, work out data type and number of - words to read (`read_record_data` requires this). - - :Returns: - - `str`, `int` - The datatype ('integer' or 'real') and the number of words. - - """ - word_size = int_hdr.itemsize - self.lib.get_type_and_num_words.argtypes = [ - CT.c_int, - self._get_ctypes_int_array(), - CT.POINTER(CT.c_int), - CT.POINTER(CT.c_size_t), - ] - data_type = CT.c_int() - num_words = CT.c_size_t() - rv = self.lib.get_type_and_num_words( - word_size, int_hdr, CT.pointer(data_type), CT.pointer(num_words) - ) - if rv != 0: - raise umfile.UMFileException( - "Error determining data type and size from integer header" - ) - - return enum_data_type.as_name(data_type.value), num_words.value - - def get_extra_data_offset_and_length( - self, int_hdr, data_offset, disk_length - ): - """From the integer header, gets offset and length of extra - data. - - :Returns: - - `int`, `int` - The offset and length, both in units of BYTES. - - """ - word_size = int_hdr.itemsize - func = self.lib.get_extra_data_offset_and_length - func.argtypes = [ - CT.c_int, - self._get_ctypes_int_array(), - CT.c_size_t, - CT.c_size_t, - CT.POINTER(CT.c_size_t), - CT.POINTER(CT.c_size_t), - ] - extra_data_offset = CT.c_size_t() - extra_data_length = CT.c_size_t() - rv = func( - word_size, - int_hdr, - data_offset, - disk_length, - CT.pointer(extra_data_offset), - CT.pointer(extra_data_length), - ) - if rv != 0: - raise umfile.UMFileException( - "Error determining extra data length from integer header" - ) - - return extra_data_offset.value, extra_data_length.value - - def read_header(self, fd, header_offset, byte_ordering, word_size): - """Reads the header from open file. - - :Returns: - - `numpy.ndarray`, `numpy.ndarray` - The integer and real parts of the header. - - """ - self.lib.read_header.argtypes = [ - CT.c_int, - CT.c_size_t, - CT.c_int, - CT.c_int, - self._get_ctypes_int_array(), - self._get_ctypes_real_array(), - ] - - int_hdr = self._get_empty_int_array(_len_int_hdr) - real_hdr = self._get_empty_real_array(_len_real_hdr) - rv = self.lib.read_header( - fd, - header_offset, - enum_byte_ordering.as_index(byte_ordering), - word_size, - int_hdr, - real_hdr, - ) - if rv != 0: - raise umfile.UMFileException("Error reading header data") - - return int_hdr, real_hdr - - def read_extra_data( - self, - fd, - extra_data_offset, - extra_data_length, - byte_ordering, - word_size, - ): - """Reads record data from open file. - - inputs: - fd - integer low-level file descriptor - extra_data_offset - offset in bytes - extra_disk_length - disk length of extra data in bytes - byte_ordering - 'little_endian' or 'big_endian' - word_size - 4 or 8 - - returns: extra data as string - - """ - extra_data = b"\0" * extra_data_length - - self.lib.read_extra_data.argtypes = [ - CT.c_int, - CT.c_size_t, - CT.c_size_t, - CT.c_int, - CT.c_int, - CT.c_char_p, - ] - - rv = self.lib.read_extra_data( - fd, - extra_data_offset, - extra_data_length, - enum_byte_ordering.as_index(byte_ordering), - word_size, - extra_data, - ) - if rv != 0: - raise umfile.UMFileException("Error reading extra data") - - return extra_data - - def read_record_data( - self, - fd, - data_offset, - disk_length, - byte_ordering, - word_size, - int_hdr, - real_hdr, - data_type, - nwords, - ): - """Reads record data from open file. - - inputs: - fd - integer low-level file descriptor - data_offset - offset in words - disk_length - disk length of data record in words - byte_ordering - 'little_endian' or 'big_endian' - word_size - 4 or 8 - int_hdr - integer PP headers (numpy array) - real_hdr - real PP headers (numpy array) - data_type - 'integer' or 'real' - nwords - number of words to read - type and nwords should have been returned by - get_type_and_num_words() - - """ - if data_type == "integer": - data = self._get_empty_int_array(nwords) - ctypes_data = self._get_ctypes_int_array() - elif data_type == "real": - data = self._get_empty_real_array(nwords) - ctypes_data = self._get_ctypes_real_array() - else: - raise ValueError("data_type must be 'integer' or 'real'") - - self.lib.read_record_data.argtypes = [ - CT.c_int, - CT.c_size_t, - CT.c_size_t, - CT.c_int, - CT.c_int, - self._get_ctypes_int_array(), - self._get_ctypes_real_array(), - CT.c_size_t, - ctypes_data, - ] - - rv = self.lib.read_record_data( - fd, - data_offset, - disk_length, - enum_byte_ordering.as_index(byte_ordering), - word_size, - int_hdr, - real_hdr, - nwords, - data, - ) - - if rv != 0: - raise umfile.UMFileException("Error reading record data") - - return data - - -if __name__ == "__main__": - import sys - - c = CInterface() - fd = os.open(sys.argv[1], os.O_RDONLY) - file_type = c.detect_file_type(fd) - print(c.file_type_obj_to_dict(file_type)) - c.set_word_size(file_type) - info = c.parse_file(fd, file_type) - - for var in info["vars"]: - print("nz = %s, nt = %s" % (var.nz, var.nt)) - for rec in var.recs: - print(rec.hdr_offset) - print("data offset", rec.data_offset) - print("disk length", rec.disk_length) - print("int hdr", rec.int_hdr) - print("real hdr", rec.real_hdr) - data_type, nwords = c.get_type_and_num_words(rec.int_hdr) - print("data_type = %s nwords = %s" % (data_type, nwords)) - - data = c.read_record_data( - fd, - rec.data_offset, - rec.disk_length, - file_type.byte_ordering, - file_type.word_size, - rec.int_hdr, - rec.real_hdr, - data_type, - nwords, - ) - print( - "data (%s values): %s ... %s" - % (len(data), data[:10], data[-10:]) - ) - ( - extra_data_offset, - extra_data_length, - ) = c.get_extra_data_offset_and_length( - rec.int_hdr, rec.data_offset, rec.disk_length - ) - print("extra data offset: %s" % extra_data_offset) - print("extra data length: %s" % extra_data_length) - extra_data = c.read_extra_data( - fd, - extra_data_offset, - extra_data_length, - file_type.byte_ordering, - file_type.word_size, - ) - print("extra data (%s bytes) read" % (len(extra_data))) - - print( - c.read_header( - fd, - info["vars"][0].recs[0].hdr_offset, - file_type.byte_ordering, - file_type.word_size, - ) - ) diff --git a/cf/umread_lib/extraData.py b/cf/umread_lib/extraData.py deleted file mode 100644 index 6559bde36c..0000000000 --- a/cf/umread_lib/extraData.py +++ /dev/null @@ -1,171 +0,0 @@ -import sys - -import numpy as np - - -def cmp(a, b): - """Workaround to get a Python-2-like `cmp` function in Python 3.""" - return (a > b) - (a < b) - - -_codes = { - 1: ("x", float), - 2: ("y", float), - 3: ("y_domain_lower_bound", float), - 4: ("x_domain_lower_bound", float), - 5: ("y_domain_upper_bound", float), - 6: ("x_domain_upper_bound", float), - 7: ("z_domain_lower_bound", float), - 8: ("z_domain_upper_bound", float), - 10: ("title", str), - 11: ("domain_title", str), - 12: ("x_lower_bound", float), - 13: ("x_upper_bound", float), - 14: ("y_lower_bound", float), - 15: ("y_upper_bound", float), -} - - -class ExtraData(dict): - """Extends dictionary class with a comparison method between extra - data for different records.""" - - _key_to_type = dict([(key, typ) for key, typ in _codes.values()]) - - def sorted_keys(self): - k = self.keys() - k.sort() - return k - - _tolerances = {np.dtype(np.float32): 1e-5, np.dtype(np.float64): 1e-13} - - def _cmp_floats(self, a, b, tolerance): - if a == b: - return 0 - - delta = abs(b * tolerance) - if a < b - delta: - return -1 - - if a > b + delta: - return 1 - - return 0 - - def _cmp_float_arrays(self, avals, bvals): - n = len(avals) - c = cmp(n, len(bvals)) - if c != 0: - return c - - tolerance = self._tolerances[avals.dtype] - for i in range(n): - c = self._cmp_floats(avals[i], bvals[i], tolerance) - if c != 0: - return c - - return 0 - - def __cmp__(self, other): - """Compare two extra data dictionaries returned by unpacker.""" - if other is None: - return 1 - ka = self.sorted_keys() - kb = other.sorted_keys() - c = cmp(ka, kb) - if c != 0: - return c - - for key in ka: - valsa = self[key] - valsb = other[key] - typ = self._key_to_type[key] - if typ == float: - c = self._cmp_float_arrays(valsa, valsb) - elif type == str: - c = cmp(valsa, valsb) - else: - assert False - - if c != 0: - return c - - return 0 - - -class ExtraDataUnpacker: - _int_types = {4: np.int32, 8: np.int64} - _float_types = {4: np.float32, 8: np.float64} - - def __init__(self, raw_extra_data, word_size, byte_ordering): - self.rdata = raw_extra_data - self.ws = word_size - self.itype = self._int_types[word_size] - self.ftype = self._float_types[word_size] - # byte_ordering is 'little_endian' or 'big_endian' - # sys.byteorder is 'little' or 'big' - self.is_swapped = not byte_ordering.startswith(sys.byteorder) - - def next_words(self, n): - """return next n words as raw data string, and pop them off the - front of the string.""" - pos = n * self.ws - rv = self.rdata[:pos] - assert len(rv) == pos - self.rdata = self.rdata[pos:] - return rv - - def convert_bytes_to_string(self, st): - """Convert bytes to string. - - :Returns: - - `str` - - """ - if self.is_swapped: - indices = slice(None, None, -1) - else: - indices = slice(None) - - st = "".join( - [ - st[pos : pos + self.ws][indices].decode("utf-8") - for pos in range(0, len(st), self.ws) - ] - ) - - while st.endswith("\x00"): - st = st[:-1] - - return st - - def get_data(self): - """Get the (key, value) pairs for extra data. - - :Returns: - - `ExtraData` - - """ - d = {} - while self.rdata: - i = np.frombuffer(self.next_words(1), self.itype)[0] - if i == 0: - break - - ia, ib = divmod(i, 1000) - key, etype = _codes[ib] - - rawvals = self.next_words(ia) - if etype == float: - vals = np.frombuffer(rawvals, self.ftype) - elif etype == str: - vals = np.array([self.convert_bytes_to_string(rawvals)]) - - if key not in d: - d[key] = vals - else: - d[key] = np.append(d[key], vals) - - return ExtraData(d) diff --git a/cf/umread_lib/umfile.py b/cf/umread_lib/umfile.py deleted file mode 100644 index 7183653e64..0000000000 --- a/cf/umread_lib/umfile.py +++ /dev/null @@ -1,517 +0,0 @@ -import os -from functools import cmp_to_key - -import numpy -from cfdm.read_write.exceptions import DatasetTypeError - -from . import cInterface -from .extraData import ExtraDataUnpacker - - -class UMFileException(Exception): - pass - - -# Lookup header pointers -LBLREC = 14 # Length of data record (including any extra data) -LBPACK = 20 # Packing method indicator -LBEGIN = 28 # Disk address/Start Record - - -class File: - """A class for a UM file that gives a view of the file including - sets of PP records combined into variables.""" - - def __init__( - self, path, byte_ordering=None, word_size=None, fmt=None, parse=True - ): - """Open and parse a UM file. - - The optional *byte_ordering*, *word_size* and *fmt* arguments - specify the file type. If all three are set, then this forces the - file type; otherwise, the file type is autodetected and any of - them that are set are ignored. - - :Parameters: - - path: `str` - The name of the UM file. - - byte_ordering: `str`, optional - 'little_endian' or 'big_endian' - - word_size: `int`, optional - The size in bytes of one word. Either ``4`` or ``8``. - - fmt: `str`, optional - 'FF' or 'PP' - - parse: `bool`, optional - The default action is to open the file, store the file - type from the arguments or autodetection as described - above, and then parse the contents, giving a tree of - variables and records under the `File` object. However, if - *parse* is False, then an object is returned in which the - last step is omitted, so only the file type is stored, and - there are no variables under it. Such an object can be - passed when instantiating Rec objects, and contains - sufficient info about the file type to ensure that the - `get_data` method of those `Rec` objects will work. - - """ - c = cInterface.CInterface() - self._c_interface = c - - self.path = path - self.fd = None - self.open_fd() - - if byte_ordering and word_size and fmt: - self.fmt = fmt - self.byte_ordering = byte_ordering - self.word_size = word_size - else: - self._detect_file_type() - - self.path = path - file_type_obj = c.create_file_type( - self.fmt, self.byte_ordering, self.word_size - ) - - # Set the word size used to interpret file pointers - c.set_word_size(file_type_obj) - - if parse: - # -------------------------------------------------------- - # Work out information from the file and store it in the - # `vars` attribute. - # - # Note that the word size used to interpret file pointers - # needs to have been previously set. - # -------------------------------------------------------- - info = c.parse_file(self.fd, file_type_obj) - self.vars = info["vars"] - self._add_back_refs() - - def open_fd(self): - """(Re)open the low-level file descriptor. - - :Returns: - - `int` - The file descriptor. - - """ - if self.fd is None: - self.fd = os.open(self.path, os.O_RDONLY) - - return self.fd - - def close_fd(self): - """Close the low-level file descriptor. - - :Returns: - - `None` - - """ - if self.fd: - os.close(self.fd) - - self.fd = None - - def _detect_file_type(self): - """Store string values describing the auto-detected file type. - - :Returns: - - `None` - - """ - c = self._c_interface - try: - file_type_obj = c.detect_file_type(self.fd) - except Exception: - self.close_fd() - raise DatasetTypeError( - f"Can't open {self.path} as a PP or UM dataset" - ) - - d = c.file_type_obj_to_dict(file_type_obj) - self.fmt = d["fmt"] - self.byte_ordering = d["byte_ordering"] - self.word_size = d["word_size"] - - def _add_back_refs(self): - """Add file attribute to `Var` objects, and both `!file` and - `!var` attributes to `Rec` objects. - - The important one is the file attribute in the `Rec` object, as - this is used when reading data. The others are provided for extra - convenience. - - :Returns: - - `None` - - """ - for var in self.vars: - var.file = self - for rec in var.recs: - rec.var = var - rec.file = self - - -class Var: - """Container for some information about variables.""" - - def __init__(self, recs, nz, nt, supervar_index=None): - self.recs = recs - self.nz = nz - self.nt = nt - self.supervar_index = supervar_index - - @staticmethod - def _compare(x, y): - """Method equivalent to the Python 2 'cmp'. - - Note that (x > y) - (x < y) is equivalent but not as performant - since it would not short-circuit. - - :Returns: - - `int` - - """ - if x == y: - return 0 - elif x > y: - return 1 - else: - return -1 - - def _compare_recs_by_extra_data(self, a, b): - """Compare records with respect to their extra data. - - :Returns: - - `int` - - """ - return self._compare(a.get_extra_data(), b.get_extra_data()) - - def _compare_recs_by_orig_order(self, a, b): - """Compare records with respect to their original order. - - :Returns: - - `int` - - """ - return self._compare(self.recs.index(a), self.recs.index(b)) - - def group_records_by_extra_data(self): - """Group records by matching extra data. - - Returns a list of (sub)lists of records where each record - within each sublist has matching extra data (if - any). Therefore, if the whole variable has consistent extra - data then the returned value will be a list of length 1. - - Within each group, the ordering of returned records is the - same as in the `!recs` attribute. - - :Returns: - - `list` - - """ - compare = self._compare_recs_by_extra_data - recs = self.recs[:] - n = len(recs) - if n == 0: - # shouldn't have a var without records, but... - return [] - - # recs.sort(compare) #python2 - recs.sort(key=cmp_to_key(compare)) - - # optimise simple case - if two ends of a sorted list match, - # the whole list matches - if not compare(recs[0], recs[-1]): - return [self.recs[:]] - - groups = [] - this_grp = [] - for i, rec in enumerate(recs): - this_grp.append(rec) - if i == n - 1 or compare(rec, recs[i + 1]): - this_grp.sort(key=self._compare_recs_by_orig_order) - groups.append(this_grp) - this_grp = [] - - return groups - - -class Rec: - """Container for some information about records.""" - - def __init__( - self, - int_hdr, - real_hdr, - hdr_offset, - data_offset, - disk_length, - file=None, - ): - """Default instantiation, which stores the supplied headers and - offsets. - - :Parameters: - - file: `File`, optional - Used to set the `!file` attribute. Does not need to be - supplied, but if it is not then it will have to be set on - the returned `Rec` object before calling `get_data` will - work. If set it should be set to the `File` object that - contains the returned `Rec` object. Normally this would be - done by the calling code instantiating via `File` rather - than directly. - - """ - self.int_hdr = int_hdr - self.real_hdr = real_hdr - self.hdr_offset = hdr_offset - self.data_offset = data_offset - self.disk_length = disk_length - self._extra_data = None - if file: - self.file = file - - @classmethod - def from_file_and_offsets( - cls, file, hdr_offset, data_offset=None, disk_length=None - ): - """Instantiate a `Rec` object from the `File` object and the - header and data offsets. - - The lookup header is read from disk immediately, and the - returned record object is ready for calling `get_data`. - - :Parameters: - - file: `File` - A view of a file including sets of PP records combined - into variables. - - hdr_offset: `int` - The file start address of the header, in bytes. - - data_offset: `int`, optional - The file start address of the data, in bytes. If - `None`, the default, then the data offset will be - calculated from the integer header. - - disk_length: `int` - The length in bytes of the data in the file. If - `None`, the default, then the disk length will be - calculated from the integer header. - - :Returns: - - `Rec` - - """ - c = file._c_interface - word_size = file.word_size - int_hdr, real_hdr = c.read_header( - file.fd, hdr_offset, file.byte_ordering, word_size - ) - - if data_offset is None: - # Calculate the data offset from the integer header - if file.fmt == "PP": - # We only support 64-word headers, so the data starts - # 66 words after the header_offset, i.e. after 64 - # words of the header, plus 2 block control words. - data_offset = hdr_offset + 66 * word_size - else: - # Fields file - data_offset = int_hdr[LBEGIN] * word_size - - if disk_length is None: - # Calculate the disk length from the integer header - disk_length = int_hdr[LBLREC] - if int_hdr[LBPACK] % 10 == 2: - # Cray 32-bit packing - disk_length = disk_length * 4 - else: - disk_length = disk_length * word_size - - return cls( - int_hdr, - real_hdr, - hdr_offset, - data_offset, - disk_length, - file=file, - ) - - def read_extra_data(self): - """Read the extra data associated with the record. - - :Returns: - - `numpy.ndarray` - - """ - file = self.file - c = file._c_interface - - ( - extra_data_offset, - extra_data_length, - ) = c.get_extra_data_offset_and_length( - self.int_hdr, self.data_offset, self.disk_length - ) - - raw_extra_data = c.read_extra_data( - file.fd, - extra_data_offset, - extra_data_length, - file.byte_ordering, - file.word_size, - ) - - edu = ExtraDataUnpacker( - raw_extra_data, file.word_size, file.byte_ordering - ) - - return edu.get_data() - - def get_extra_data(self): - """Get extra data associated with the record. - - This is done either by reading or using cached read. - - :Returns: - - `numpy.ndarray` - - """ - if self._extra_data is None: - self._extra_data = self.read_extra_data() - - return self._extra_data - - def get_type_and_num_words(self): - """Get the data type and number of words. - - :Returns: - - `numpy.dtype`, `int` - - """ - c = self.file._c_interface - ntype, num_words = c.get_type_and_num_words(self.int_hdr) - if ntype == "integer": - dtype = numpy.dtype(c.file_data_int_type) - elif ntype == "real": - dtype = numpy.dtype(c.file_data_real_type) - - return dtype, num_words - - def get_data(self): - """Get the data array associated with the record. - - :Returns: - - `numpy.ndarray` - - """ - file = self.file - c = file._c_interface - int_hdr = self.int_hdr - data_type, nwords = c.get_type_and_num_words(int_hdr) - - return c.read_record_data( - file.fd, - self.data_offset, - self.disk_length, - file.byte_ordering, - file.word_size, - int_hdr, - self.real_hdr, - data_type, - nwords, - ) - - -if __name__ == "__main__": - import sys - - path = sys.argv[1] - f = File(path) - print(f.fmt, f.byte_ordering, f.word_size) - print("num variables: %s" % len(f.vars)) - for varno, var in enumerate(f.vars): - print() - print("var %s: nz = %s, nt = %s" % (varno, var.nz, var.nt)) - for recno, rec in enumerate(var.recs): - print("var %s record %s" % (varno, recno)) - print("hdr offset: %s" % rec.hdr_offset) - print("data offset: %s" % rec.data_offset) - print("disk length: %s" % rec.disk_length) - print("int hdr: %s" % rec.int_hdr) - print("real hdr: %s" % rec.real_hdr) - print("data: %s" % rec.get_data()) - print("extra_data: %s" % rec.get_extra_data()) - print("type %s, num words: %s" % rec.get_type_and_num_words()) - # if recno == 1: - # rec._extra_data['y'] += .01 - # print("massaged_extra_data: %s" % rec.get_extra_data()) - print("-----------------------") - - print("all records", var.recs) - print( - "records grouped by extra data ", var.group_records_by_extra_data() - ) - print("===============================") - - f.close_fd() - - # also read a record using saved metadata - if f.vars: - fmt = f.fmt - byte_ordering = f.byte_ordering - word_size = f.word_size - myrec = f.vars[0].recs[0] - hdr_offset = myrec.hdr_offset - data_offset = myrec.data_offset - disk_length = myrec.disk_length - - del f - - fnew = File( - path, - fmt=fmt, - byte_ordering=byte_ordering, - word_size=word_size, - parse=False, - ) - - rnew = Rec.from_file_and_offsets( - fnew, hdr_offset, data_offset, disk_length - ) - print("record read using saved file type and offsets:") - print("int hdr: %s" % rnew.int_hdr) - print("real hdr: %s" % rnew.real_hdr) - print("data: %s" % rnew.get_data()) - print("extra data: %s" % rnew.get_extra_data()) - print("nx = %s" % rnew.int_hdr[18]) - print("ny = %s" % rnew.int_hdr[17]) - rdata = open("recdata0.txt", "w") - for value in rnew.get_data(): - rdata.write("%s\n" % value) - rdata.close() diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 9143c914bf..2ef7e90b8f 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -227,8 +227,8 @@ Required * `scipy `_, version 1.10.0 or newer. -* `cfdm `_, version 1.13.2.1 or up to, - but not including, 1.13.3.0. +* `cfdm `_, version 1.13.3.0 or up to, + but not including, 1.13.4.0. * `cfunits `_, version 3.3.7 or newer. @@ -312,6 +312,13 @@ environments for which these features are not required. regridding, some changes to the refinement level, and some collapses. +.. rubric:: Grid mapping manipulations + +* `pyproj `_, version 3.7.2 or + newer. This package is required to create 2-d latitude and longitude + coordinates from grid mappings. + + ---- .. _Tests: diff --git a/generate_stub_files.py b/generate_stub_files.py index afa1632fc5..df6a114641 100644 --- a/generate_stub_files.py +++ b/generate_stub_files.py @@ -9,7 +9,6 @@ import re import sys - from pathlib import Path if len(sys.argv) > 1: @@ -19,6 +18,7 @@ OUT_DIR = SOURCE_DIR.parent + def generate_stub_files(rst_path: Path): content = rst_path.read_text(encoding="utf-8") @@ -54,7 +54,7 @@ def generate_stub_files(rst_path: Path): out_dir = OUT_DIR / "method" else: out_dir = OUT_DIR / "attribute" - + out_dir.mkdir(parents=True, exist_ok=True) out_file = out_dir / f"{entry}.rst" diff --git a/requirements.txt b/requirements.txt index 26133c246d..5341bb8b79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ netCDF4>=1.7.2 cftime>=1.6.4 numpy>=2.0.0 -cfdm>=1.13.2.1, <1.13.3.0 +cfdm>=1.13.3.0, <1.13.4.0 psutil>=0.6.0 cfunits>=3.3.7 dask>=2025.5.1 diff --git a/setup.py b/setup.py index 4848db62e2..66e790cb82 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ import fnmatch import os import re -import subprocess -from distutils.command.build import build from setuptools import find_packages, setup @@ -48,64 +46,9 @@ def _get_version(): version = _get_version() packages = ["cf"] etc_files = [f for f in find_package_data_files("cf/etc")] -umread_files = [f for f in find_package_data_files("cf/umread_lib/c-lib")] test_files = [f for f in find_test_files()] -package_data = etc_files + umread_files + test_files - - -class build_umread(build): - """Adpated from https://github.com/Turbo87/py- - xcsoar/blob/master/setup.py.""" - - def run(self): - # Run original build code - build.run(self) - - # Build umread - print("Running build_umread") - - build_dir = os.path.join( - os.path.abspath(self.build_lib), "cf/umread_lib/c-lib" - ) - - cmd = ["make", "-C", build_dir] - - def compile(): - print("*" * 80) - print("Running:", " ".join(cmd), "\n") - - try: - rc = subprocess.call(cmd) - except Exception as error: - print(error) - rc = 40 - - print("\n", "-" * 80) - if not rc: - print("SUCCESSFULLY built UM read C library") - else: - print("WARNING: Failed to build the UM read C library.") - print( - " Attempting to read UKMO PP and UM format files " - "will result in failure." - ) - print( - " This will not affect any other cf functionality." - ) - print( - " In particular, netCDF file processing is " - "unaffected." - ) - - print("-" * 80) - print("\n", "*" * 80) - print() - print("cf build successful") - print() - - self.execute(compile, [], "compiling umread") - +package_data = etc_files + test_files long_description = """ CF Python @@ -324,6 +267,4 @@ def compile(): # 'udunits2==2.2.25', # ], # - # https://docs.python.org/2/distutils/apiref.html: - cmdclass={"build": build_umread}, )